text:
'''
The code is used to train BC imitator, or pretrained GAIL imitator
'''
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data
from torch import optim

# from baselines.common.running_mean_std import RunningMeanStd

import argparse
import tempfile
import os.path as osp
import gym
from tqdm.auto import tqdm

from utilities import normal_log_density, set_random_seed, to_tensor, save_checkpoint, load_pickle, onehot, get_logger
from abc import ABC, abstractmethod
from typing import List
# import tensorflow as tf
from inference import get_start_state, model_infer_vis, model_inference_env, visualize_trajs_new

# from baselines.gail import mlp_policy
from baselines import bench
# TODO: migrate ffjord's logger to here (and suppress the logging of full source code (or at least suppress the output of that part))
# from baselines import logger
import logging
# from baselines.common import set_global_seeds, tf_util as U
from baselines.common.misc_util import boolean_flag
# from baselines.common.mpi_adam import MpiAdam
# from baselines.gail.run_mujoco import runner
# from baselines.gail.dataset.mujoco_dset import Mujoco_Dset

from torch.utils.data import Dataset, TensorDataset, DataLoader

class ExpertTrajectory:
    def __init__(self, path):
        # path = "/mnt/SSD3/Qiujing_exp/Imitation_learning/data/three_modes_traj_train_everywhere_static.pkl"
        #exp_data = load_pickle("three_modes_traj.pkl")
        exp_data = load_pickle(path)
        #self.exp_states = np.concatenate([val['state'].reshape(-1,2,5) for val in exp_data.values()])
        self.expert_states = np.concatenate([val['state'].transpose(
            (0, 1, 3, 2)).reshape(-1, 10) for val in exp_data.values()])
        self.expert_actions = np.concatenate(
            [val['action'].reshape(-1, 2) for val in exp_data.values()])
        self.n_transitions = self.expert_actions.shape[0]
        self.mode_names = list(exp_data.keys())
        self.num_step = exp_data[self.mode_names[0]]['state'].shape[1]
        print("mode names", self.mode_names)
        print("number of transitions:", self.n_transitions)
        print("number of steps in each traj:", self.num_step)
        # np.random.seed(2)

    def sample(self, batch_size):
        """
        Option1: generate latent code for each sample action
        Option2: fix latent code for each traj. Sample states together with latent code
        FIXME: why can the latent code be inconsistent across batches?
        """
        indexes = np.sort(np.random.randint(
            0, self.n_transitions, size=batch_size))
        traj_index = indexes//self.num_step
        unique_traj, traj_fre = np.unique(traj_index, return_counts=True)
        num_unique_traj = len(unique_traj)
        fake_z0 = np.random.randint(3, size=num_unique_traj)
        #new_z = np.zeros(batch_size, dtype=int)
        #print("begin t before latent code sampling:", time() -  start_t)

        new_z = np.repeat(fake_z0, traj_fre)
        #print("end t after latent code sampling:", time() -  start_t)
        fake_z = onehot(new_z, 3)

        state = self.expert_states[indexes]
        action = self.expert_actions[indexes]
        #print("sampled data shape", np.array(state).shape, np.array(action).shape)
        #print("end t after sampling:", time() -  start_t)

        return np.array(state), np.array(action), fake_z, indexes

class TrajectoryDataset(TensorDataset):
    def __init__(self, X, y, c, transform=None, device="cpu"):
        # assert X.shape[0] == y.shape[0] == c.shape[0],\
        #     f"Data length is not aligned: X.shape[0] == {X.shape[0]}, "\
        #     f"y.shape[0] == {y.shape[0]}, c.shape[0] == {c.shape[0]}"
        self.transform = transform if transform is not None else lambda x: x
        X = self.transform(X)
        self.X = torch.as_tensor(X, device=device)  # state
        self.y = torch.as_tensor(y, device=device)  # action
        self.c = torch.as_tensor(c, device=device)
        super(TrajectoryDataset, self).__init__(self.X, self.y, self.c)

def _train(epoch, net, dataloader, optimizer, criterion, device, writer):
    net.train()
    train_loss = 0
    # dataloader
    num_batch = len(dataloader)
    for batch_idx, (state, action, latent_code) in enumerate(dataloader):

        optimizer.zero_grad()
        state, action, latent_code = to_tensor(state, device),\
            to_tensor(action, device),\
            to_tensor(latent_code, device)

        outputs = net(state, latent_code)
        #print(outputs, state, latent_code)
        loss = criterion(outputs, action)
        loss.backward()
        optimizer.step()
        #print("loss data", loss.data)
        train_loss += loss.item()

        if batch_idx % 100 == 0:
            print('Loss: %.3f ' % (train_loss/((batch_idx+1)*3)))
            if writer is not None:
                writer.add_scalars(
                    "Loss/BC", {"train_loss": train_loss/((batch_idx+1)*3)}, batch_idx + num_batch * (epoch-1))

class BC():
    def __init__(self, epochs=300, lr=0.0001, eps=1e-5, device="cpu", policy_activation=F.relu,
                 tb_writer=None, validate_freq=1, checkpoint_dir=".", code_dim=None):
        self.epochs = epochs
        self.device = device
        self.policy = MlpPolicyNet(
            state_dim=10, code_dim=code_dim, ft_dim=128, activation=policy_activation
        ).to(device)
        self.optimizer = optim.Adam(self.policy.parameters(), lr=lr, eps=eps)
        self.criterion = nn.MSELoss()
        self.writer = tb_writer
        self.validate_freq = validate_freq
        self.checkpoint_dir = checkpoint_dir

    def train(self, expert_loader, val_loader):
        best_loss = float("inf")
        for epoch in tqdm(range(self.epochs)):
            print('\nEpoch: %d' % epoch)
            _train(epoch, self.policy, expert_loader, self.optimizer,
                   self.criterion, self.device, self.writer)
            if epoch % self.validate_freq == 0:
                best_loss, checkpoint_path = _validate(epoch, self.policy, val_loader,
                                                       self.criterion, self.device, best_loss,
                                                       self.writer, self.checkpoint_dir)
        best_loss, checkpoint_path = _validate(epoch, self.policy, val_loader,
                                               self.criterion, self.device, best_loss,
                                               self.writer, self.checkpoint_dir)
        self.load_best_checkpoint(checkpoint_path)

    def load_best_checkpoint(self, checkpoint_path):
        self.policy.load_state_dict(torch.load(checkpoint_path)['state_dict'])

def _validate(epoch, net, val_loader, criterion, device, best_loss, writer, checkpoint_dir):
    net.eval()
    valid_loss = 0
    number_batches = len(val_loader)
    avg_valid_loss = None
    for batch_idx, (state, action, latent_code) in enumerate(val_loader):
        state, action, latent_code = to_tensor(state, device),\
            to_tensor(action, device),\
            to_tensor(latent_code, device)
        outputs = net(state, latent_code)
        loss = criterion(outputs, action)

        valid_loss += loss.item()

        avg_valid_loss = valid_loss/(batch_idx+1)
        if batch_idx % 100 == 0:
            print('Valid Loss: %.3f ' % (valid_loss/(batch_idx+1)))

            if writer is not None:
                writer.add_scalars("Loss/BC_val", {"val_loss": valid_loss/(
                    (batch_idx+1))}, batch_idx + number_batches * (epoch-1))
    checkpoint_path = osp.join(
        checkpoint_dir, 'checkpoints/bestbc_model_new_everywhere.pth')
    assert avg_valid_loss is not None, "Empty avg_valid_loss. Possibly empty dataloader?"
    if avg_valid_loss <= best_loss:
        best_loss = avg_valid_loss
        print('Best epoch: ' + str(epoch))
        save_checkpoint({'epoch': epoch,
                         'avg_loss': avg_valid_loss,
                         'state_dict': net.state_dict(),
                         }, save_path=checkpoint_path)
    return best_loss, checkpoint_path

def load_data(data_file, one_hot: bool = True, one_hot_dim: int = None, code_map=None):
    # TODO: remove the radii selection lines
    # TODO: Currently the one-hot encoding is done in memory all at once. Potentially it needs to be moved to a custom DataLoader like ExpertTrajectory above
    """For Arash's format
    """
    data_dict = torch.load(data_file)

    lengths = data_dict["lengths"]
    # lengths = lengths[data_dict["radii"] == 10]

    X_all = data_dict["states"]
    # X_all = X_all[data_dict["radii"] == 10]
    num_traj, max_traj_len_x, dim_state = X_all.shape
    # X_all = X_all.reshape(-1, dim_state)
    X_all = torch.cat([X[:l] for X, l in zip(X_all, lengths)], dim=0)

    y_all = data_dict["actions"]
    # y_all = y_all[data_dict["radii"] == 10]
    num_traj, max_traj_len_y, dim_action = y_all.shape
    # y_all = y_all.reshape(-1, dim_action)
    y_all = torch.cat([y[:l] for y, l in zip(y_all, lengths)], dim=0)

    c = data_dict["radii"]
    # c = c[data_dict["radii"] == 10]

    # change to scalar encoding here in case it's useful
    unique_c, inv = np.unique(c, return_inverse=True)
    dim = len(unique_c)

    # make sure that one_hot_dim >= the inferred dim
    if one_hot_dim is None:
        one_hot_dim = dim
    elif one_hot_dim < dim:
        raise ValueError(f"one_hot_dim ({one_hot_dim}) is smaller than the"
                         f" number of unique values in c ({dim})")

    if code_map is None:
        codes = np.argsort(np.argsort(unique_c))
        code_map = dict(zip(unique_c, codes))
        # c, fake_c = codes[inv], np.random.choice(codes, size=len(c))
        # c_all, fake_c_all = np.repeat(c, lengths), np.repeat(fake_c, lengths)
        c = codes[inv]
    else:
        c = np.array([code_map[c_] for c_ in unique_c])[inv]

    try:
        # for torch.tensor lengths
        c_all, fake_c_all = (np.repeat(c, lengths),
                             np.random.randint(dim, size=lengths.sum().item()))
    except:
        # for np.array lengths
        c_all, fake_c_all = (np.repeat(c, lengths),
                             np.random.randint(dim, size=lengths.sum()))

    if one_hot:
        c_all, fake_c_all = (onehot(c_all, dim=one_hot_dim),
                             onehot(fake_c_all, dim=one_hot_dim))

    # c_all = torch.zeros(num_traj, traj_len, dtype=torch.int64)
    # c_all[c == 10, :] = 1
    # c_all[c == 20, :] = 2
    # c_all[c == -10, :] = 3
    # c_all = c_all.flatten()

    return X_all, y_all, c_all, fake_c_all, code_map

def create_dataset(train_data_path, val_data_path=None, fake=True, one_hot=True, one_hot_dim=None):
    from sklearn.model_selection import train_test_split
    X, y, c, fake_c, code_map = load_data(
        train_data_path, one_hot=one_hot, one_hot_dim=one_hot_dim
    )
    if fake:
        c = fake_c
    if val_data_path is None:
        X_train, X_val, y_train, y_val, c_train, c_val = train_test_split(
            X, y, c, test_size=0.2)
    else:
        X_train, y_train, c_train = X, y, c
        X_val, y_val, c_val, fake_c_val, code_map = load_data(
            val_data_path, one_hot=one_hot, code_map=code_map
        )
        if fake:
            c_val = fake_c_val

    train_dataset = TrajectoryDataset(X_train, y_train, c_train)
    val_dataset = TrajectoryDataset(X_val, y_val, c_val)
    return train_dataset, val_dataset

def create_dataloader(train_dataset, val_dataset, batch_size=4):
    train_dataloader = DataLoader(train_dataset, batch_size=batch_size,
                                  shuffle=True, num_workers=0)
    val_dataloader = DataLoader(val_dataset, batch_size=batch_size,
                                shuffle=True, num_workers=0)
    return train_dataloader, val_dataloader

def get_task_name(args):
    task_name = 'BC'
    task_name += '.{}'.format(args.env_id.split("-")[0])
    task_name += '.traj_limitation_{}'.format(args.traj_limitation)
    task_name += ".seed_{}".format(args.seed)
    return task_name

class PolicyNet(ABC, nn.Module):
    def __init__(self):
        super(PolicyNet, self).__init__()

    @abstractmethod
    def get_log_prob(self, state, actions):
        pass

    @abstractmethod
    def select_action(self, state, stochastic):
        pass

class MlpPolicyNet(PolicyNet):
    def __init__(self, state_dim=10, code_dim=3, ft_dim=128, activation=F.relu):
        super(MlpPolicyNet, self).__init__()
        self.activation = activation
        self.fc_s1 = nn.Linear(state_dim, ft_dim)
        self.fc_s2 = nn.Linear(ft_dim, ft_dim)
        self.code_dim = code_dim
        if code_dim is not None:
            self.fc_c1 = nn.Linear(code_dim, ft_dim)
        self.fc_sum = nn.Linear(ft_dim, 2)
        self.action_logstds = torch.log(
            torch.from_numpy(np.array([2, 2])).clone().float())
        self.action_std = torch.from_numpy(np.array([2, 2])).clone().float()

    def forward(self, state, latent_code):
        output = self.fc_s2(self.activation(self.fc_s1(state), inplace=True))
        if self.code_dim is not None:
            output += self.fc_c1(latent_code)
        final_out = self.fc_sum(self.activation(output, inplace=True))
        return final_out

    def get_log_prob(self, state, latent_code, actions):
        """
        For continuous action space. fixed action log std
        """
        action_mu = self.forward(state, latent_code)
        device = state.device
        return normal_log_density(actions, action_mu, self.action_logstds.to(device), self.action_std.to(device))

    def select_action(self, state, latent_code, stochastic=True):
        action_mu = self.forward(state, latent_code)
        #normal_log_density_fixedstd(x, action_mu)
        device = state.device
        if stochastic:
            action = torch.normal(action_mu, self.action_std.to(device))
        else:
            action = action_mu.to(device)
        return action

# def argsparser():
#     parser = argparse.ArgumentParser(
#         "PyTorch Adaption of `baselines` Behavior Cloning")
#     parser.add_argument('--env_id', help='environment ID', default='Hopper-v2')
#     parser.add_argument('--seed', help='RNG seed', type=int, default=0)
#     parser.add_argument('--expert_path', type=str,
#                         default='data/deterministic.trpo.Hopper.0.00.npz')
#     parser.add_argument(
#         '--checkpoint_dir', help='the directory to save model', default='checkpoint')
#     parser.add_argument(
#         '--log_dir', help='the directory to save log file', default='log')
#     #  Mujoco Dataset Configuration
#     parser.add_argument('--traj_limitation', type=int, default=-1)
#     # Network Configuration (Using MLP Policy)
#     parser.add_argument('--policy_hidden_size', type=int, default=100)
#     # for evaluation
#     boolean_flag(parser, 'stochastic_policy', default=False,
#                  help='use stochastic/deterministic policy to evaluate')
#     boolean_flag(parser, 'save_sample', default=False,
#                  help='save the trajectories or not')
#     parser.add_argument(
#         '--BC_max_iter', help='Max iteration for training BC', type=int, default=1e5)
#     return parser.parse_args()
def parse_args(*args):
    parser = argparse.ArgumentParser("Behavior Cloning for Circle env")
    parser.add_argument('--seed', help='RNG seed', type=int, default=3)
    parser.add_argument(
        '--train_data', help='Training dataset', type=str, default='trajs_circles_mix',
        choices=['trajs_circles', 'trajs_circles_flip',
                 'trajs_cicles_mix', 'trajs_circles_four']
    )
    parser.add_argument(
        '--code_dim', type=int, default=None,
        help='Latent code dimension, None for disabling'
    )
    parser.add_argument(
        '--consistent', type=bool, default=True,
        help='During inference use consistent code for each trajectory.'
             ' Otherwise the code is random for each state-action.'
    )
    parser.add_argument(
        '--noise_level', help='The noise level in env.', type=float, default=0.1
    )
    parser.add_argument('--name', help='The name of the inference run.',
                        type=str, default="inference_codeless_0.1")
    parser.add_argument(
        '--device', default="cuda:1", help='The device to use.'
    )
    boolean_flag(
        parser, 'fake', default=True,
        help='Train with fake codes. Otherwise true codes will be provided.'
    )
    boolean_flag(parser, 'train', default=True, help='Train the model')
    boolean_flag(parser, 'inference', default=True, help='Inference the model')
    boolean_flag(parser, 'render', default=False,
                 help='Render during the inference')
    return parser.parse_args(*args)

if __name__ == '__main__':
    args = parse_args()
    code_dim = args.code_dim
    train_with_fake_code = args.fake
    inference_noise = args.noise_level
    inference_name = args.name
    device = args.device
    train = args.train
    inference = args.inference
    consistent_inference_code = args.consistent
    render = args.render
    train_data_path = f"/home/shared/datasets/gail_experts/{args.train_data}.pt"
    set_random_seed(args.seed, using_cuda=True)

    ############### Train ###############
    if train:
        # train_data_path = "three_modes_traj_train_everywhere.pkl"
        # val_data_path = "three_modes_traj_val.pkl"
        bc = BC(epochs=30, lr=1e-4, eps=1e-5, device=device, code_dim=code_dim)
        # bc = BC(epochs=30, lr=1e-4, eps=1e-5, device="cuda:0", code_dim=None)
        # train_data_path = "/home/shared/datasets/gail_experts/trajs_circles.pt"
        train_dataset, val_dataset = create_dataset(
            train_data_path, fake=train_with_fake_code, one_hot=True, one_hot_dim=code_dim)
        train_loader, val_loader = create_dataloader(
            train_dataset, val_dataset, batch_size=400)
        bc.train(train_loader, val_loader)
        model = bc.policy
    ############### Load Checkpoint ###############
    else:
        # train_data_path = "/home/shared/datasets/gail_experts/trajs_circles.pt"
        train_dataset, val_dataset = create_dataset(
            train_data_path, fake=False, one_hot=True, one_hot_dim=code_dim)
        model = MlpPolicyNet(code_dim=code_dim)
        # model = MlpPolicyNet(code_dim=None)
        checkpoint = torch.load(
            "checkpoints/bestbc_model_new_everywhere.pth")["state_dict"]
        model.load_state_dict(checkpoint)

    ############### Inference ###############
    if not inference:
        if not train:
            raise ValueError("Why bother if you don't train nor inference")
    else:
        num_trajs = 20
        start_state = get_start_state(
            num_trajs, mode="sample_data", dataset=val_dataset)
        # print(start_state.shape)
        if code_dim is not None and consistent_inference_code:
            fake_code = onehot(np.random.randint(
                code_dim, size=num_trajs), dim=code_dim)
        else:
            fake_code = None
        # fake_code = torch.zeros(num_trajs, code_dim)
        # fake_code[:,0] = 1
        traj_len = 1000
        # model_infer_vis(model, start_state, fake_code, traj_len, save_fig_name=f"{inference_name}")

        ############### use env for inference ###############
        flat_state_arr, action_arr = model_inference_env(
            model, num_trajs, traj_len, state_len=5, radii=[-10, 10, 20],
            codes=fake_code, noise_level=inference_noise, render=render
        )
        visualize_trajs_new(flat_state_arr, action_arr,
                            f"./imgs/circle/env_{inference_name}.png")


text:
# If using the csv dataset

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import csv
import cv2

from tensorflow.keras.preprocessing.image import ImageDataGenerator

def getdata(path):
    labels = []
    pixels = []
    with open(path) as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        next(reader)
        for row in csvfile:
            row = row.split(',')
            labels.append(row[0])

            pix = row[1:]
            pix = np.array(pix)
            pix = pix.astype('uint8')
            pix = pix.reshape(28, 28, 1)
            # Here frame is of type numpy.ndarray
            blur = cv2.GaussianBlur(pix, (3, 3), 0)
            ret, pix = cv2.threshold(blur, 100, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
            pix = list(pix.flatten())
            pixels.append(pix)

    labels = np.array(labels)
    pixels = np.array(pixels)
    labels = labels.astype(int)
    pixels = pixels.astype(float)
    pixels = pixels.reshape((-1, 28, 28, 1))

    return pixels, labels

train_path = '.\sign_mnist_train.csv'
test_path = '.\sign_mnist_test.csv'

train_images, train_labels = getdata(train_path)
test_images, test_labels = getdata(test_path)

print(train_images.shape)
print(train_labels.shape)
print(test_images.shape)
print(test_labels.shape)
'''
train_datagen = ImageDataGenerator(
    rescale=1 / 255.,
    rotation_range=30,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=0.1,
    zoom_range=0.4,
    horizontal_flip=True,
    brightness_range=(0.6, 1.0)
)
'''

train_datagen = ImageDataGenerator(rescale=1/255., zoom_range=0.4, rotation_range=30, horizontal_flip=True, brightness_range=(0.8, 1.0))
test_datagen = ImageDataGenerator(rescale=1/255.)

train_gen = train_datagen.flow(train_images, train_labels, batch_size=200)
test_gen = test_datagen.flow(test_images, test_labels, batch_size=100)

model = tf.keras.Sequential([
    tf.keras.layers.Conv2D(32, (3, 3), input_shape=(28, 28, 1), activation='relu')])
model.add(tf.keras.layers.MaxPooling2D(2, 2))
model.add(tf.keras.layers.Conv2D(32, (3, 3), activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(2, 2))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dense(len(np.unique(train_labels)) + 1, activation='softmax'))

NUM_EPOCHS = 25

class myCallback(tf.keras.callbacks.Callback):
    def on_epoch_end(self, epochs, logs={}):
        if (logs.get('accuracy') > 0.99):
            print("Stopping training. Accuracy:", logs.get('accuracy') * 100, "%")
            self.model.stop_training=True

callback1 = myCallback()

log_dir = "logs/fit/"
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)

model.compile(loss='sparse_categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(), metrics=['accuracy'])
history = model.fit(train_gen, epochs=NUM_EPOCHS, validation_data=test_gen, verbose=1, callbacks=[callback1, tensorboard_callback])
print(model.evaluate(test_gen, verbose=0))

def plot_metrics(metric):
    plt.plot(history.history[metric])
    plt.plot(history.history['val_' + metric])
    plt.xlabel('NUM_EPOCHS')
    plt.ylabel(metric)
    plt.legend([metric, 'val_' + metric])
    plt.show()

plot_metrics('accuracy')
plot_metrics('loss')

model.save('cv_model.h5')
print("\nNow loading new model: ")
new_model = tf.keras.models.load_model('cv_model.h5')
print(model.summary())
print(new_model.evaluate(test_gen))

ext: py

reason: import numpy


text:
import cv2
import numpy as np

def crop_rectangle_from_image(image: np.ndarray, corners: np.ndarray) -> np.ndarray:
    """``crop_rectangle_from_image`` takes an image and 4 corners in pixel coordinates, it returns
    the sub-image inside that cropped area, rotated upright.

    Args:
        image (np.ndarray): image to crop from
        corners (np.ndarray): corners, array of shape (4,2)

    Returns:
        np.ndarray: crop from input containing the corners
    """
    rect = cv2.minAreaRect(np.array(corners[:, ::-1], dtype=np.float32))
    center, size, theta = rect
    center, size = tuple(center), tuple(np.int0(size))

    M = cv2.getRotationMatrix2D(center, theta, 1)
    dest = cv2.warpAffine(image, M, tuple((np.array(image.shape[:2]) * 2).astype(np.int64)))

    out = cv2.getRectSubPix(dest, size, center)
    return out


text:
# TODO merge naive and weighted loss.
import numpy as np
import torch
import torch.nn.functional as F

from ..bbox import bbox_overlaps
from ...ops import sigmoid_focal_loss

def weighted_nll_loss(pred, label, weight, avg_factor=None):
    if avg_factor is None:
        avg_factor = max(torch.sum(weight > 0).float().item(), 1.)
    raw = F.nll_loss(pred, label, reduction='none')
    return torch.sum(raw * weight)[None] / avg_factor

def weighted_cross_entropy(pred, label, weight, avg_factor=None, reduce=True):
    if avg_factor is None:
        avg_factor = max(torch.sum(weight > 0).float().item(), 1.)
    raw = F.cross_entropy(pred, label, reduction='none')
    if reduce:
        return torch.sum(raw * weight)[None] / avg_factor
    else:
        return raw * weight / avg_factor

def weighted_binary_cross_entropy(pred, label, weight, avg_factor=None):
    if pred.dim() != label.dim():
        label, weight = _expand_binary_labels(label, weight, pred.size(-1))
    if avg_factor is None:
        avg_factor = max(torch.sum(weight > 0).float().item(), 1.)
    return F.binary_cross_entropy_with_logits(
        pred, label.float(), weight.float(),
        reduction='sum')[None] / avg_factor

def py_sigmoid_focal_loss(pred,
                          target,
                          weight,
                          gamma=2.0,
                          alpha=0.25,
                          reduction='mean'):
    pred_sigmoid = pred.sigmoid()
    target = target.type_as(pred)
    pt = (1 - pred_sigmoid) * target + pred_sigmoid * (1 - target)
    weight = (alpha * target + (1 - alpha) * (1 - target)) * weight
    weight = weight * pt.pow(gamma)
    loss = F.binary_cross_entropy_with_logits(
        pred, target, reduction='none') * weight
    reduction_enum = F._Reduction.get_enum(reduction)
    # none: 0, mean:1, sum: 2
    if reduction_enum == 0:
        return loss
    elif reduction_enum == 1:
        return loss.mean()
    elif reduction_enum == 2:
        return loss.sum()

def weighted_sigmoid_focal_loss(pred,
                                target,
                                weight,
                                gamma=2.0,
                                alpha=0.25,
                                avg_factor=None,
                                num_classes=80):
    if avg_factor is None:
        avg_factor = torch.sum(weight > 0).float().item() / num_classes + 1e-6
    return torch.sum(
        sigmoid_focal_loss(pred, target, gamma, alpha, 'none') *
        weight.view(-1, 1))[None] / avg_factor

def mask_cross_entropy(pred, target, label):
    num_rois = pred.size()[0]
    inds = torch.arange(0, num_rois, dtype=torch.long, device=pred.device)
    pred_slice = pred[inds, label].squeeze(1)
    return F.binary_cross_entropy_with_logits(
        pred_slice, target, reduction='mean')[None]

def smooth_l1_loss(pred, target, beta=1.0, reduction='mean'):
    assert beta > 0
    assert pred.size() == target.size() and target.numel() > 0
    diff = torch.abs(pred - target)
    loss = torch.where(diff < beta, 0.5 * diff * diff / beta,
                       diff - 0.5 * beta)
    reduction_enum = F._Reduction.get_enum(reduction)
    # none: 0, mean:1, sum: 2
    if reduction_enum == 0:
        return loss
    elif reduction_enum == 1:
        return loss.sum() / pred.numel()
    elif reduction_enum == 2:
        return loss.sum()

def weighted_smoothl1(pred, target, weight, beta=1.0, avg_factor=None):
    if avg_factor is None:
        avg_factor = torch.sum(weight > 0).float().item() / 4 + 1e-6
    loss = smooth_l1_loss(pred, target, beta, reduction='none')
    return torch.sum(loss * weight)[None] / avg_factor

def balanced_l1_loss(pred,
                     target,
                     beta=1.0,
                     alpha=0.5,
                     gamma=1.5,
                     reduction='none'):
    assert beta > 0
    assert pred.size() == target.size() and target.numel() > 0

    diff = torch.abs(pred - target)
    b = np.e**(gamma / alpha) - 1
    loss = torch.where(
        diff < beta, alpha / b *
        (b * diff + 1) * torch.log(b * diff / beta + 1) - alpha * diff,
        gamma * diff + gamma / b - alpha * beta)

    reduction_enum = F._Reduction.get_enum(reduction)
    # none: 0, elementwise_mean:1, sum: 2
    if reduction_enum == 0:
        return loss
    elif reduction_enum == 1:
        return loss.sum() / pred.numel()
    elif reduction_enum == 2:
        return loss.sum()

    return loss

def weighted_balanced_l1_loss(pred,
                              target,
                              weight,
                              beta=1.0,
                              alpha=0.5,
                              gamma=1.5,
                              avg_factor=None):
    if avg_factor is None:
        avg_factor = torch.sum(weight > 0).float().item() / 4 + 1e-6
    loss = balanced_l1_loss(pred, target, beta, alpha, gamma, reduction='none')
    return torch.sum(loss * weight)[None] / avg_factor

def bounded_iou_loss(pred, target, beta=0.2, eps=1e-3, reduction='mean'):
    """Improving Object Localization with Fitness NMS and Bounded IoU Loss,
    https://arxiv.org/abs/1711.00164.

    Args:
        pred (tensor): Predicted bboxes.
        target (tensor): Target bboxes.
        beta (float): beta parameter in smoothl1.
        eps (float): eps to avoid NaN.
        reduction (str): Reduction type.
    """
    pred_ctrx = (pred[:, 0] + pred[:, 2]) * 0.5
    pred_ctry = (pred[:, 1] + pred[:, 3]) * 0.5
    pred_w = pred[:, 2] - pred[:, 0] + 1
    pred_h = pred[:, 3] - pred[:, 1] + 1
    with torch.no_grad():
        target_ctrx = (target[:, 0] + target[:, 2]) * 0.5
        target_ctry = (target[:, 1] + target[:, 3]) * 0.5
        target_w = target[:, 2] - target[:, 0] + 1
        target_h = target[:, 3] - target[:, 1] + 1

    dx = target_ctrx - pred_ctrx
    dy = target_ctry - pred_ctry

    loss_dx = 1 - torch.max(
        (target_w - 2 * dx.abs()) /
        (target_w + 2 * dx.abs() + eps), torch.zeros_like(dx))
    loss_dy = 1 - torch.max(
        (target_h - 2 * dy.abs()) /
        (target_h + 2 * dy.abs() + eps), torch.zeros_like(dy))
    loss_dw = 1 - torch.min(target_w / (pred_w + eps), pred_w /
                            (target_w + eps))
    loss_dh = 1 - torch.min(target_h / (pred_h + eps), pred_h /
                            (target_h + eps))
    loss_comb = torch.stack([loss_dx, loss_dy, loss_dw, loss_dh],
                            dim=-1).view(loss_dx.size(0), -1)

    loss = torch.where(loss_comb < beta, 0.5 * loss_comb * loss_comb / beta,
                       loss_comb - 0.5 * beta)
    reduction_enum = F._Reduction.get_enum(reduction)
    # none: 0, mean:1, sum: 2
    if reduction_enum == 0:
        return loss
    elif reduction_enum == 1:
        return loss.sum() / pred.numel()
    elif reduction_enum == 2:
        return loss.sum()

def weighted_iou_loss(pred,
                      target,
                      weight,
                      style='naive',
                      beta=0.2,
                      eps=1e-3,
                      avg_factor=None):
    if style not in ['bounded', 'naive']:
        raise ValueError('Only support bounded iou loss and naive iou loss.')
    inds = torch.nonzero(weight[:, 0] > 0)
    if avg_factor is None:
        avg_factor = inds.numel() + 1e-6

    if inds.numel() > 0:
        inds = inds.squeeze(1)
    else:
        return (pred * weight).sum()[None] / avg_factor

    if style == 'bounded':
        loss = bounded_iou_loss(
            pred[inds], target[inds], beta=beta, eps=eps, reduction='sum')
    else:
        loss = iou_loss(pred[inds], target[inds], eps=eps, reduction='sum')
    loss = loss[None] / avg_factor
    return loss

def accuracy(pred, target, topk=1):
    if isinstance(topk, int):
        topk = (topk, )
        return_single = True
    else:
        return_single = False

    maxk = max(topk)
    _, pred_label = pred.topk(maxk, 1, True, True)
    pred_label = pred_label.t()
    correct = pred_label.eq(target.view(1, -1).expand_as(pred_label))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
        res.append(correct_k.mul_(100.0 / pred.size(0)))
    return res[0] if return_single else res

def _expand_binary_labels(labels, label_weights, label_channels):
    bin_labels = labels.new_full((labels.size(0), label_channels), 0)
    inds = torch.nonzero(labels >= 1).squeeze()
    if inds.numel() > 0:
        bin_labels[inds, labels[inds] - 1] = 1
    bin_label_weights = label_weights.view(-1, 1).expand(
        label_weights.size(0), label_channels)
    return bin_labels, bin_label_weights

def iou_loss(pred_bboxes, target_bboxes, eps=1e-6, reduction='mean'):
    ious = bbox_overlaps(pred_bboxes, target_bboxes, is_aligned=True) + eps
    loss = -ious.log()

    reduction_enum = F._Reduction.get_enum(reduction)
    if reduction_enum == 0:
        return loss
    elif reduction_enum == 1:
        return loss.mean()
    elif reduction_enum == 2:
        return loss.sum()


text:
"""
Using Pandas, CSV and ARFF files
=====================================

In this example we will show you how to use Pandas, CSV and ARFF in PyMFE.
"""

# Necessary imports
import pandas as pd
import numpy as np
from numpy import genfromtxt
from pymfe.mfe import MFE
import csv
import arff

###############################################################################
# Pandas
# ------
# Generating synthetic dataset
np.random.seed(42)

sample_size = 150
numeric = pd.DataFrame({
    'num1': np.random.randint(0, 100, size=sample_size),
    'num2': np.random.randint(0, 100, size=sample_size)
})
categoric = pd.DataFrame({
    'cat1': np.repeat(('cat1-1', 'cat1-2'), sample_size/2),
    'cat2': np.repeat(('cat2-1', 'cat2-2', 'cat2-3'), sample_size/3)
})
X = numeric.join(categoric)
y = pd.Series(np.repeat(['C1', 'C2'], sample_size/2))

###############################################################################
# Exploring characteristics of the data
print("X shape --> ", X.shape)
print("y shape --> ", y.shape)
print("classes --> ", np.unique(y.values))
print("X dtypes --> \n", X.dtypes)
print("y dtypes --> ", y.dtypes)

###############################################################################
# For extracting meta-features, you should send ``X`` and ``y`` as a sequence,
# like numpy array or Python list.
# It is easy to make this using pandas:
mfe = MFE(
    groups=["general", "statistical", "info-theory"],
    random_state=42
)
mfe.fit(X.values, y.values)
ft = mfe.extract(cat_cols='auto', suppress_warnings=True)
print("\n".join("{:50} {:30}".format(x, y) for x, y in zip(ft[0], ft[1])))

###############################################################################
# Pandas CSV
# ----------
# Getting data from CSV format
df = pd.read_csv('../data/data.csv')
X, y = df.drop('class', axis=1), df['class']

###############################################################################
# Exploring characteristics of the data
print("X shape --> ", X.shape)
print("y shape --> ", y.shape)
print("classes --> ", np.unique(y))
print("X dtypes --> \n", X.dtypes)
print("y dtypes --> ", y.dtypes)

###############################################################################
# For extracting meta-features, you should send ``X`` and ``y`` as a sequence,
# like numpy array or Python list.
# It is easy to make this using pandas:
mfe = MFE(
    groups=["general", "statistical", "info-theory"],
    random_state=42
)
mfe.fit(X.values, y.values)
ft = mfe.extract(cat_cols='auto', suppress_warnings=True)
print("\n".join("{:50} {:30}".format(x, y) for x, y in zip(ft[0], ft[1])))

###############################################################################
# ARFF
# ----
# Getting data from ARFF format:
data = arff.load(open('../data/data.arff', 'r'))['data']
X = [i[:4] for i in data]
y = [i[-1] for i in data]

###############################################################################
# Exploring characteristics of the data
print("X shape --> ", len(X))
print("y shape --> ", len(y))
print("classes --> ", np.unique(y))
print("X dtypes --> ", type(X))
print("y dtypes --> ", type(y))

###############################################################################
# For extracting meta-features, you should send ``X`` and ``y`` as a sequence,
# like numpy array or Python list.
# You can do this directly:
mfe = MFE(
    groups=["general", "statistical", "info-theory"],
    random_state=42
)
mfe.fit(X, y)
ft = mfe.extract(cat_cols='auto', suppress_warnings=True)
print("\n".join("{:50} {:30}".format(x, y) for x, y in zip(ft[0], ft[1])))

###############################################################################
# As a final example, we do not use the automatic detection of feature type
# here. We use the ids provided by the liac-arff package.
classid = 4
data = arff.load(open('../data/data.arff', 'r'), encode_nominal=True)
cat_cols = [n for n, i in enumerate(data['attributes'][:classid])
            if isinstance(i[1], list)]
data = np.array(data['data'])
X = data[:, :classid]
y = data[:, classid]

###############################################################################
# Exploring characteristics of the data
print("X shape --> ", len(X))
print("y shape --> ", len(y))
print("classes --> ", np.unique(y))
print("X dtypes --> ", type(X))
print("y dtypes --> ", type(y))

###############################################################################
# For extracting meta-features, you should send ``X`` and ``y`` as a sequence,
# like numpy array or python list.
mfe = MFE(
    groups=["general", "statistical", "info-theory"],
    random_state=42
)
mfe.fit(X, y, cat_cols=cat_cols)
ft = mfe.extract(suppress_warnings=True)
print("\n".join("{:50} {:30}".format(x, y) for x, y in zip(ft[0], ft[1])))

max_stars_repo_name: jhosoume/pymfe

max_issues_repo_name: jhosoume/pymfe

max_forks_repo_name: jhosoume/pymfe

reason: import numpy,from numpy


text:
import numpy as np
import sys

class Atom:
    def __init__(self, number, atomid, resid, segid, resnum, absres, x, y, z, q=0.0, elem = "", tempfactor = 0.0,occupancy=0.0,radius=0.0):
        self.number = number
        self.atomid = atomid
        self.resid = resid
        self.segid = segid
        self.resnum = resnum
        self.absres = absres
        self.xyz = [float(x), float(y), float(z)]
        self.elem = elem
        self.q   = float(q)
        self.radius     = 0.0
        self.occupancy  = occupancy
        self.tempfactor = tempfactor
        self.radius = radius

    def change_resid(self, newresid):
        self.resid = newresid

    def set_number(self, newnumber):
        self.number = newnumber

    def set_radius(self, radii, patches):
        successfully_set_radius = False

        # check if resid is in radii directly
        if self.resid in radii.data:
            if self.atomid in radii.data[self.resid]:
                self.radius = radii.data[self.resid][self.atomid]
                successfully_set_radius = True

        # note that in the above clause, we don't just return because we might have a patch that adjusts the radius!!
        print("segid = " + self.segid + " and resnum = " + str(self.resnum))
        patch = patches.get_patch(segid = self.segid, resnum = self.resnum)
        if patch is not None:
            print("Checking patch " + self.segid + " : " + str(self.resnum))
            if self.atomid in radii.data["global"]:
                self.radius = radii.data["global"][self.atomid]
                successfully_set_radius = True

#        if successfully_set_radius == False:
#            print 'Error in atom.set_radius!  Atom atomnum does not have a radius entry anywhere!\n'
#            sys.exit(1)

    def set_charge(self, q):
        self.q = q

    def set_q_from_psf(self, psf):
        for atom in psf['ATOMS']:
#            print 'in set_q_from_psf with'
#            print '\tatomid='+str(atom.atomid)+' resnum='+str(atom.resnum)+' resid='+str(atom.resid)+'  ==> q='+str(atom.q)
#            print '\tatomid='+str(self.atomid)+' resnum='+str(self.resnum)+' resid='+str(self.resid)

            if self.atomid == atom.atomid and self.resnum == atom.resnum and self.resid == atom.resid:
#                print 'atomid='+str(atom.atomid)+' resnum='+str(atom.resnum)+' resid='+str(atom.resid)+'  ==> q='+str(atom.q)
                self.q = atom.q

    def set_q_from_topology(self, topology, patches):
        successfully_set_charge = False

        if self.resid in topology.residues:
            if self.atomid in topology.residues[self.resid]:
                self.q = topology.residues[self.resid][self.atomid]['charge']
                successfully_set_charge = True

        # Note that we don't "return" from the above clause because we might have a patch that we need to apply!

        # ... check if this atom is part of a patch
        patch = patches.get_patch(segid = self.segid, resnum = self.resnum)
        if patch in topology.patches:
            if self.atomid in topology.patches[patch]:
                self.q = topology.patches[patch][self.atomid]['charge']
                successfully_set_charge = True

#        if successfully_set_charge == False:
#            print 'Error in atom.set_q!  Atom atomnum does not have a charge entry anywhere!\n'
#            sys.exit(1)

    def crg_entry(self):
        entry_string = "%-6s%3s%4d%1s%8.5f\n"%(self.atomid,self.resid,self.resnum,self.segid,self.q)
        return entry_string

    def crd_entry(self):
        entry_string = "%5d%5d %-4s %-4s%10.5f%10.5f%10.5f %-4.4s %-4.4s%10.5f\n"%(self.number, self.absres,self.resid,self.atomid, self.xyz[0], self.xyz[1], self.xyz[2], self.segid, self.resnum, self.q)
        return entry_string

    def pdb_entry(self):
        entry_string = "ATOM  %5d %-4.4s %3.3s  %4d    %8.3f%8.3f%8.3f  1.00%6.2f\n" % (self.number, self.atomid, self.resid, self.resnum, self.xyz[0], self.xyz[1], self.xyz[2], self.q)

        return entry_string

    # needs better checking against Atom.pm (line 390)
    def pdb2_entry(self):
        atom_string = self.atomid
        if len(atom_string) != 4:
            atom_string = " " + atom_string

        formalcharge = 0

        entry_string = "ATOM  %5d %-4.4s %-4.4s%1s%4d    %8.3f%8.3f%8.3f%6.2f%6.2f      %-4.4s%+2.2s%+2.2s\n" % (self.number, atom_string, self.resid, self.segid[0], self.resnum,self.xyz[0],self.xyz[1],self.xyz[2],self.occupancy,self.tempfactor, self.segid,self.elem,formalcharge)

        return entry_string

    def mead_pqr_entry(self):
        atom_string = self.atomid
        if len(atom_string) != 4:
            atom_string = " " + atom_string

        entry_string = "ATOM  %5d %-4.4s %-4.4s %4d    %8.3f%8.3f%8.3f %7.4f %7.4f\n" % (self.number, atom_string, self.resid, self.resnum,self.xyz[0],self.xyz[1],self.xyz[2],self.q,self.radius)
        return entry_string

    def apbs_pqr_entry(self):
        atom_string = self.atomid
        if len(atom_string) != 4:
            atom_string = " " + atom_string

        entry_string = "ATOM  %5d %-4.4s %-4.4s %-2.2s %4d    %8.3f%8.3f%8.3f %6.3f %6.3f\n" % (self.number, atom_string, self.resid, self.segid, self.resnum,self.xyz[0],self.xyz[1],self.xyz[2],self.q,self.radius)
        return entry_string

    def accurate_pqr_entry(self):
        atom_string = self.atomid
        if len(atom_string) != 4:
            atom_string = " " + atom_string

        entry_string = "ATOM  %5d %-4.4s %-4.4s %-2.2s %4d    %10.6f%10.6f%10.6f %10.6f %10.6f\n" % (self.number, atom_string, self.resid, self.segid, self.resnum,self.xyz[0],self.xyz[1],self.xyz[2],self.q,self.radius)
        return entry_string

    def xyzr_entry(self):
        entry_string = "%8.3f %8.3f %8.3f %6.3f\n" %(self.xyz[0],self.xyz[1],self.xyz[2],self.radius)
        return entry_string

    def rotateZ(self, angle = 0.0):
        # note: angle is in degrees, not radians
        angle = angle * np.pi / 180.0
        sinTheta = np.sin(angle)
        cosTheta = np.cos(angle)
        matrix  = np.array([
            [ cosTheta, -sinTheta, 0.0],
            [ sinTheta, cosTheta, 0.0],
            [ 0.0, 0.0, 1.0]])
        xyz = np.array([
            [self.xyz[0]],
            [self.xyz[1]],
            [self.xyz[2]]])
        newxyz = np.dot(matrix,xyz)
#        print("Newxyz is " + str(newxyz))
        self.xyz[0] = float(newxyz[0])
        self.xyz[1] = float(newxyz[1])
        self.xyz[2] = float(newxyz[2])

    def calculate_dipole_moment(self):
        return [self.xyz[0]*self.q,self.xyz[1]*self.q,self.xyz[2]*self.q]

    def translate(self, vec = None, x = None, y = None, z = None):
        if (vec == None) and (x==None or y==None or z==None):
            print("Error: translate needs either a vector or all x,y,z!\n")
            sys.exit(1)

        if (vec != None and x != None and y != None and z != None):
            print("Error: translate needs _either_ vec or x,y,z, but not both!\n")
            sys.exit(1)

        if vec != None:
            x = vec[0]
            y = vec[1]
            z = vec[2]

        self.xyz[0] = self.xyz[0] + x
        self.xyz[1] = self.xyz[1] + y
        self.xyz[2] = self.xyz[2] + z

max_stars_repo_name: jbardhan/molman

max_issues_repo_name: jbardhan/molman

max_forks_repo_name: jbardhan/molman


text:
import numpy as np
import tensorflow as tf

import shap

# tf-estimator-nightly   2.4.0.dev2020092901
# tf-nightly             2.4.0.dev20200929

def test_dense():
    x = np.random.random((200, 10))
    w_ = np.random.random((10,))
    y = x.dot(w_)

    m = tf.keras.models.Sequential()
    m.add(
        tf.keras.layers.Dense(1, activation="linear", use_bias=None, input_shape=(10,))
    )
    m.compile(loss="MSE", optimizer=tf.keras.optimizers.Adam(lr=2))
    m.fit(x, y, epochs=20)
    pred = m.predict(x)[:, 0]

    w = m.get_weights()[0][:, 0]
    assert np.allclose(w, w_, atol=0.01, rtol=0.05)

    exp = shap.DeepExplainer(m, data=x[15:30])
    vals = exp.shap_values(x)
    rec = vals[0].sum(axis=1) + exp.expected_value.numpy()

    assert np.allclose(rec, pred)

def test_lstm():
    x = np.random.random((200, 10, 5))
    w_ = np.random.random((10,))
    y = x.sum(axis=-1).dot(w_)

    m = tf.keras.models.Sequential()
    m.add(
        tf.keras.layers.LSTM(
            1, input_shape=(10, 5), return_sequences=True, activation="linear"
        )
    )
    m.add(tf.keras.layers.Flatten())
    m.add(tf.keras.layers.Dense(1))
    m.summary()
    m.compile(loss="MSE", optimizer=tf.keras.optimizers.Adam(lr=0.1))
    m.fit(x, y, epochs=50)
    pred = m.predict(x)[:, 0]

    exp = shap.DeepExplainer(m, data=x[15:30])
    vals = exp.shap_values(x, check_additivity=False)
    rec = vals[0].sum(axis=(1, 2)) + exp.expected_value.numpy()
    # print(pred)
    # print(rec)

    print(np.abs((pred - rec)/pred).mean())
    assert not np.allclose(vals, 0)
    assert np.allclose(rec, pred)

def test_cnn():
    x = np.random.random((200, 28, 28, 3))
    y = x.sum(axis=(1, 2, 3))

    m = tf.keras.models.Sequential()
    m.add(tf.keras.layers.Conv2D(3, (3, 3), input_shape=(28, 28, 3)))
    m.add(tf.keras.layers.Conv2D(3, (3, 3)))
    m.add(tf.keras.layers.MaxPool2D(2))
    m.add(tf.keras.layers.Conv2D(3, (3, 3)))
    m.add(tf.keras.layers.Flatten())
    m.add(tf.keras.layers.Dense(1))
    m.summary()
    m.compile(loss="MSE", optimizer=tf.keras.optimizers.Adam(lr=0.1))
    m.fit(x, y, epochs=50)
    pred = m.predict(x)[:, 0]

    exp = shap.DeepExplainer(m, data=x[15:30])
    vals = exp.shap_values(x, check_additivity=False)
    rec = vals[0].sum(axis=(1, 2, 3)) + exp.expected_value.numpy()
    print(pred)
    print(rec)
    print(vals)
    assert not np.allclose(vals, 0)
    assert np.allclose(rec, pred)

test_lstm()

max_stars_repo_name: Zahlii/shap

max_issues_repo_name: Zahlii/shap

max_forks_repo_name: Zahlii/shap


text:
import copy
import numpy as np
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier

from adapt.parameter_based import TransferTreeClassifier, TransferForestClassifier

methods = [
    'relab',
    'ser',
    'strut',
    'ser_nr',
    'ser_no_ext',
    'ser_nr_lambda',
    'strut_nd',
    'strut_lambda',
    'strut_np'
    'strut_lambda_np',
    'strut_lambda_np2'
#    'strut_hi'
]

def test_transfer_tree():

    np.random.seed(0)

    # Generate training source data
    ns = 200
    ns_perclass = ns // 2
    mean_1 = (1, 1)
    var_1 = np.diag([1, 1])
    mean_2 = (3, 3)
    var_2 = np.diag([2, 2])
    Xs = np.r_[np.random.multivariate_normal(mean_1, var_1, size=ns_perclass),
               np.random.multivariate_normal(mean_2, var_2, size=ns_perclass)]
    ys = np.zeros(ns)
    ys[ns_perclass:] = 1
    # Generate training target data
    nt = 50
    # imbalanced
    nt_0 = nt // 10
    mean_1 = (6, 3)
    var_1 = np.diag([4, 1])
    mean_2 = (5, 5)
    var_2 = np.diag([1, 3])
    Xt = np.r_[np.random.multivariate_normal(mean_1, var_1, size=nt_0),
               np.random.multivariate_normal(mean_2, var_2, size=nt - nt_0)]
    yt = np.zeros(nt)
    yt[nt_0:] = 1
    # Generate testing target data
    nt_test = 1000
    nt_test_perclass = nt_test // 2
    Xt_test = np.r_[np.random.multivariate_normal(mean_1, var_1, size=nt_test_perclass),
                    np.random.multivariate_normal(mean_2, var_2, size=nt_test_perclass)]
    yt_test = np.zeros(nt_test)
    yt_test[nt_test_perclass:] = 1

    # Source classifier
    RF_SIZE = 10
    clf_source_dt = DecisionTreeClassifier(max_depth=None)
    clf_source_rf = RandomForestClassifier(n_estimators=RF_SIZE)
    clf_source_dt.fit(Xs, ys)
    clf_source_rf.fit(Xs, ys)
    #score_src_src = clf_source.score(Xs, ys)
    #score_src_trgt = clf_source.score(Xt_test, yt_test)
    #print('Training score Source model: {:.3f}'.format(score_src_src))
    #print('Testing score Source model: {:.3f}'.format(score_src_trgt))
    clfs = []
    scores = []
    # Transfer with SER
    #clf_transfer = copy.deepcopy(clf_source)
    #transferred_dt = TransferTreeClassifier(estimator=clf_transfer,Xt=Xt,yt=yt)

    for method in methods:
        Nkmin = sum(yt == 0 )
        root_source_values = clf_source_dt.tree_.value[0].reshape(-1)
        props_s = root_source_values
        props_s = props_s / sum(props_s)
        props_t = np.zeros(props_s.size)
        for k in range(props_s.size):
            props_t[k] = np.sum(yt == k) / yt.size

        coeffs = np.divide(props_t, props_s)

        clf_transfer_dt = copy.deepcopy(clf_source_dt)
        clf_transfer_rf = copy.deepcopy(clf_source_rf)

        if method == 'relab':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="")
            transferred_dt.fit(Xt,yt)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="",bootstrap=True)
            transferred_rf.fit(Xt,yt)
        if method == 'ser':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt.set_params(max_depth=10),algo="ser")
            transferred_dt.fit(Xt,yt)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="ser")
            transferred_rf.fit(Xt,yt)
        if method == 'ser_nr':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="ser")
            transferred_dt._ser(Xt, yt,node=0,original_ser=False,no_red_on_cl=True,cl_no_red=[0])
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="ser")
            transferred_rf._ser_rf(Xt, yt,original_ser=False,no_red_on_cl=True,cl_no_red=[0])

        if method == 'ser_no_ext':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="ser")
            transferred_dt._ser(Xt, yt,node=0,original_ser=False,no_ext_on_cl=True,cl_no_ext=[0],ext_cond=True)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="ser")
            transferred_rf._ser_rf(Xt, yt,original_ser=False,no_ext_on_cl=True,cl_no_ext=[0],ext_cond=True)
        if method == 'ser_nr_lambda':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="ser")
            transferred_dt._ser(Xt, yt,node=0,original_ser=False,no_red_on_cl=True,cl_no_red=[0],
                                leaf_loss_quantify=True,leaf_loss_threshold=0.5,
                                root_source_values=root_source_values,Nkmin=Nkmin,coeffs=coeffs)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="ser")
            transferred_rf._ser_rf(Xt, yt,original_ser=False,no_red_on_cl=True,cl_no_red=[0],
                                leaf_loss_quantify=True,leaf_loss_threshold=0.5,
                                root_source_values=root_source_values,Nkmin=Nkmin,coeffs=coeffs)
        if method == 'strut':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="strut")
            transferred_dt.fit(Xt,yt)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="strut")
            transferred_rf.fit(Xt,yt)
        if method == 'strut_nd':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="strut")
            transferred_dt._strut(Xt, yt,node=0,use_divergence=False)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="strut")
            transferred_rf._strut_rf(Xt, yt,use_divergence=False)
        if method == 'strut_lambda':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="strut")
            transferred_dt._strut(Xt, yt,node=0,adapt_prop=True,root_source_values=root_source_values,
                                  Nkmin=Nkmin,coeffs=coeffs)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="strut")
            transferred_rf._strut_rf(Xt, yt,adapt_prop=True,root_source_values=root_source_values,
                                  Nkmin=Nkmin,coeffs=coeffs)
        if method == 'strut_np':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="strut")
            transferred_dt._strut(Xt, yt,node=0,adapt_prop=False,no_prune_on_cl=True,cl_no_prune=[0],
                                leaf_loss_quantify=False,leaf_loss_threshold=0.5,no_prune_with_translation=False,
                                root_source_values=root_source_values,Nkmin=Nkmin,coeffs=coeffs)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="strut")
            transferred_rf._strut_rf(Xt, yt,adapt_prop=False,no_prune_on_cl=True,cl_no_prune=[0],
                                leaf_loss_quantify=False,leaf_loss_threshold=0.5,no_prune_with_translation=False,
                                root_source_values=root_source_values,Nkmin=Nkmin,coeffs=coeffs)
        if method == 'strut_lambda_np':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="strut")
            transferred_dt._strut(Xt, yt,node=0,adapt_prop=False,no_prune_on_cl=True,cl_no_prune=[0],
                                leaf_loss_quantify=False,leaf_loss_threshold=0.5,no_prune_with_translation=False,
                                root_source_values=root_source_values,Nkmin=Nkmin,coeffs=coeffs)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="strut")
            transferred_rf._strut_rf(Xt, yt,adapt_prop=True,no_prune_on_cl=True,cl_no_prune=[0],
                                leaf_loss_quantify=True,leaf_loss_threshold=0.5,no_prune_with_translation=False,
                                root_source_values=root_source_values,Nkmin=Nkmin,coeffs=coeffs)
        if method == 'strut_lambda_np2':
            #decision tree
            transferred_dt = TransferTreeClassifier(estimator=clf_transfer_dt,algo="strut")
            transferred_dt._strut(Xt, yt,node=0,adapt_prop=False,no_prune_on_cl=True,cl_no_prune=[0],
                                leaf_loss_quantify=False,leaf_loss_threshold=0.5,no_prune_with_translation=False,
                                root_source_values=root_source_values,Nkmin=Nkmin,coeffs=coeffs)
            #random forest
            transferred_rf = TransferForestClassifier(estimator=clf_transfer_rf,algo="strut")
            transferred_rf._strut_rf(Xt, yt,adapt_prop=True,no_prune_on_cl=True,cl_no_prune=[0],
                                leaf_loss_quantify=True,leaf_loss_threshold=0.5,no_prune_with_translation=True,
                                root_source_values=root_source_values,Nkmin=Nkmin,coeffs=coeffs)

        score = transferred_dt.estimator.score(Xt_test, yt_test)
        #score = clf_transfer.score(Xt_test, yt_test)
        print('Testing score transferred model ({}) : {:.3f}'.format(method, score))
        clfs.append(transferred_dt.estimator)
        #clfs.append(clf_transfer)
        scores.append(score)

max_stars_repo_name: atiqm/adapt

max_issues_repo_name: atiqm/adapt

max_forks_repo_name: atiqm/adapt


text:
import numpy as np
from collections import defaultdict
from gensim.models.keyedvectors import KeyedVectors

def load_word_embedding(filepath):
    """Given a filepath to embeddings file, return a word to vec dictionary.
    E.g. {'word': array([0.1, 0.2, ...])}
    Parameters
    ----------
    filepath: str
        The path to the embeddings file

    Notes
    -----
    -   On the following websites embeddings are found that can be used as input:
            https://nlp.stanford.edu/projects/glove/
            https://cogcomp.org/page/resource_view/106

    Returns
    -------
    The word to vec dictionary
    """
    def _get_vec(word, *arr):
        return word, np.asarray(arr, dtype='float32')

    print('load word embedding ......')
    try:
        word_embedding = dict(_get_vec(*w.split(' ')) for w in open(filepath))
    except UnicodeDecodeError:
        word_embedding = dict(_get_vec(*w.split(' ')) for w in open(
            filepath, encoding="utf8", errors='ignore'))

    # Sanity check for word vector length
    words_to_del = []
    for word, vec in word_embedding.items():
        if len(vec) != 300:
            words_to_del.append(word)
    for word in words_to_del:
        del word_embedding[word]

    return word_embedding

def load_word2vec(filepath, return_as_dict=True):
    """ Given a filepath to embeddings file, return a word to vec dictionary or KeyedVector.
    E.g. {'word': array([0.1, 0.2, ...])}

    Parameters
    ----------
    filepath: str,
        The path to the embeddings file.
    return_as_dict: boolean, optional (default: True),
        Whether to return a normal dictionary (True) or a gensim KeyedVectors object (False).

    Notes
    -----
    -   On the following websites embeddings are found that can be used as input:
            https://code.google.com/archive/p/word2vec/

    Returns
    -------
    Dict or KeyedVectors object containing the word embeddings.
    """
    word2vec = KeyedVectors.load_word2vec_format(filepath, binary=True)
    word_vectors = word2vec.wv

    if return_as_dict:
        embedding_dict = defaultdict(lambda: None)
        for word in word_vectors.vocab:
            embedding_dict[word] = word_vectors[word]
        return embedding_dict
    else:
        return word_vectors

def create_embedding_matrix(word_index, embedding):
    """ Create a matrix of weights to use in an embedding layer.

    Parameters
    ----------
    word_index: dict like {word -> index (unique int)},
        The tokenizer that was fitted on the relevant data.
    embedding: dict like {word -> vector},
        The pretrained embedding to use.

    Returns
    -------
    embedding_matrix: np.ndarray,
        The matrix of word vectors to use as weights in an Embedding layer.
    """

    # Initialize
    mu, sigma = np.mean(list(embedding.values())), np.std(list(embedding.values()))
    embed_size = embedding[list(embedding.keys())[0]].shape[0]
    matrix = np.random.normal(mu, sigma, (len(word_index) + 1, embed_size))

    counter = 0
    for word, i in word_index.items():
        vec = embedding.get(word)
        if vec is not None:
            matrix[i] = vec
            counter += 1

    print("Found {} embeddings of {} total words in vocabulary.".format(counter, len(word_index)))
    return matrix


text:
import numpy as np
import math
import sys, copy
sys.path.insert(0,'../Robots')
import robot_toy_example as robot_moel
import uvs as uvss
import time

robot = robot_moel.toy_blocks_robot()
estimate_jacobian_random_motion_range = [2, 5]
step_normalize_range = [2, 3]
uvs = uvss.UVS(robot, 0.5, 0.1, 2, step_normalize_range,estimate_jacobian_random_motion_range)
trajectory = []

def set_intermediate_target(dx, dy):
    current_joints = robot.current_joints()
    robot.set_intermediate_target_fake(current_joints[0] + dx, current_joints[1] + dy)

def estimate_jacobian(trials=3):
    current_joints = robot.current_joints()
    delta_joints = []
    delta_errors = []

    prev_error = robot.current_joints()
    for i in range(trials):

        input("Press Enter to continue...")
        r_motion = uvs.estimate_jacobian_random_motion(i)

        r_error = robot.current_joints()
        delta_error = r_error - prev_error

        if uvs.estimate_jacobian_motion_quality(delta_error):
            delta_joints.append(r_motion)
            delta_errors.append(delta_error)
        # move back to current joints
        input("Press Enter to continue...")
        robot.move_to(current_joints)
        print('back to origin')
    delta_joints = np.asarray(delta_joints)
    delta_errors = np.asarray(delta_errors)
    uvs.estimate_jacobian(delta_joints, delta_errors)

max_it = 100
def go_loop():
    it_count = 0
    trajectory.append(robot.get_recording_state())
    while it_count < max_it and robot.task_done_fake() is not True:
        prev_error = robot.current_joints()
        r_motion = uvs.move_step(robot.get_fake_error())
        print('step:')
        print(r_motion)
        current_joints = robot.current_joints()
        trajectory.append(robot.get_recording_state())
        print('current state x: ' + str(current_joints[0]) + '  y: ' + str(current_joints[1]))
        r_error = robot.current_joints()
        delta_error = r_error - prev_error
        print('delta error')
        print(delta_error)
        print('delta_joints')
        print(r_motion)
        # broyden update
        uvs.broyden_update(delta_error, r_motion)
        input("Press Enter to continue...")
        it_count += 1
    print('task done')

# current_error = robot.get_fake_error()

max_stars_repo_name: atlas-jj/state-representation-robotics

max_issues_repo_name: atlas-jj/state-representation-robotics

max_forks_repo_name: atlas-jj/state-representation-robotics


text:
import os
from glob import glob
import numpy as np
import matplotlib.pyplot as plt
import astropy.units as u
from astropy.stats import mad_std

from toolkit import (photometry, transit_model_b,
                     PhotometryResults, PCA_light_curve, params_b)
from astropy.io import fits

image_paths = sorted(glob('/Users/brettmorris/data/rem/20190829/IMG*UR*.fits'))[:-3]

images = []

for p in image_paths:
    images.append(fits.getdata(p))

fits.writeto('outputs/masterdark_ur.fits', np.nanmedian(images, axis=0), overwrite=True)

reason: import numpy,import astropy,from astropy


text:
from base.base_model import BaseModel
import tensorflow as tf
import numpy as np
from models.utils_model import *

# STATUS: At least appears to train but needs verification, regression loading and loss not
# implemented.

class FasterRcnnModel(BaseModel):
    '''
    Model for the entire (adapted) Faster RCNN we use here
    '''

    def __init__(self, config):
        super(FasterRcnnModel, self).__init__(config)
        self.build_model()
        self.init_saver()

    def build_model(self):
        self.is_training = tf.placeholder(tf.bool)

        with tf.name_scope('data'):

            self.x = tf.placeholder(tf.float32, shape=[None, 768, 768, self.config.num_channels])
            self.y_map = tf.placeholder(tf.float32, shape=[None, 768, 768, None])
            self.y_reg = tf.placeholder(tf.float32, shape=[None, 768, 768, None, 4])

            with tf.name_scope('expand_anchor_shapes_for_reg'):
                anchor_shapes = [(21,21), (21, 41), (41,21), (41, 81), (81, 41), (51,51), (151,81), (81, 151), (101,101), (201,201)]
                n_anchors = len(anchor_shapes)
                y_anchors = np.zeros((1, 768, 768, n_anchors, 4))
                x = np.arange(768)
                X, Y = np.meshgrid(x, x)
                y_anchors[:, :, :, :, 0] = np.reshape(X, (1, 768, 768, 1))
                y_anchors[:, :, :, :, 1] = np.reshape(Y, (1, 768, 768, 1))
                i = 0
                for anchor_shape in anchor_shapes:
                    y_anchors[:, :, :, i, 2] = np.ones((1, 768, 768)) * anchor_shape[0]  # width
                    y_anchors[:, :, :, i, 3] = np.ones((1, 768, 768)) * anchor_shape[1]  # length
                    i += 1
                # this last part needs to stay here with this code construction: we change the first dimension of
                # the reg_anchors to the batch size implicitly (unsure if it works if done explicitly)
                reg_anchors = tf.tile(y_anchors, multiples=[tf.shape(self.y_reg)[0], 1, 1, 1, 1])
                reg_anchors = tf.cast(reg_anchors, tf.float32)

                if self.config.debug == 1:
                    print("reg_anchors", reg_anchors.shape)
                # self.handle = tf.placeholder(tf.string, shape=[])

            with tf.name_scope('expand_y'):
                # Here is where we unwrap the y masks to be IOU maps and then maps that match the loss.
                y_class = []
                selected_boat_index = []
                iou_mask = []
                iou_average_for_summary = []
                pos_mask = []
                neg_mask = []
                n_box = tf.shape(self.y_map)[-1]

                for i, anchor_shape in enumerate(anchor_shapes):
                    # Loop over number of anchors ~ of order 9
                    # The y_map is of shape [batch_size, h, w, max_n_boats]
                    # The kenerl we convolve with is of shape [anchor_shape[0], anchor_shape[1], max_n_boats, max_n_boats]
                    # where every entry is 0 except for [:,:, i, i] for all i.
                    # (I think) this is equivalent to running a seperate "all ones" [anchor_shape[0], anchor_shape[1], 1]
                    # kernel over each of the max_n_boats inputs.

                    anchor = tf.zeros((anchor_shape[0], anchor_shape[1], n_box, n_box))
                    diagonal = tf.ones((anchor_shape[0], anchor_shape[1], n_box))
                    anchor_area = anchor_shape[0]*anchor_shape[1]

                    # Assigns ones to anchor[:,:,i,i] (https://www.tensorflow.org/api_docs/python/tf/linalg/set_diag)
                    anchor = tf.linalg.set_diag(anchor, diagonal)

                    # Calculates the intersection of anchor with each gt map in y_map simulatenously (as above)
                    intersection = tf.nn.conv2d(self.y_map, anchor, strides=[1, 1, 1, 1], padding='SAME')

                    # union is the area of the map (per map layer, and per batch entry) + the anchor area (in 2d)

                    union = tf.reduce_sum(self.y_map, [1, 2], keepdims=True) + anchor_area - intersection
                    ious = tf.divide(intersection, union)

                    max_iou_over_ground_truth = tf.reduce_max(ious, -1)
                    # for the regression ,we need to know which boat we want to look at
                    argmax_iou_over_ground_truth = tf.argmax(ious, -1)

                    iou_average_for_summary.append(max_iou_over_ground_truth)

                    labels = tf.greater(max_iou_over_ground_truth, self.config.iou_positive_threshold)
                    labels = tf.cast(labels, tf.float32)

                    # We only deal in both losses with boxes with IOU above a upper threshold and
                    # below a lower threshold and so here we create a mask which will be 1 for
                    # all such iou scores and 0 for those inside the threshold so we can
                    # use it as a weighting for the losses
                    pos_labels = tf.cast(tf.greater(max_iou_over_ground_truth,
                                                    self.config.iou_positive_threshold), tf.float32)
                    neg_labels = tf.cast(tf.less(max_iou_over_ground_truth,
                                                 self.config.iou_negative_threshold), tf.float32)
                    if self.config.debug:
                        print("pos_labels", pos_labels.shape)

                    pos_mask.append(pos_labels)
                    neg_mask.append(neg_labels)

                    #iou_mask_anchor = pos_labels + neg_labels
                    # iou_mask shape: [batch, 768, 768, n_proposal_boxes]
                    #iou_mask.append(tf.cast(iou_mask_anchor, tf.float32))

                    y_class.append(labels)
                    selected_boat_index.append(argmax_iou_over_ground_truth)

                # Stack all the anchors together in the end this is then of shape [batch, 768, 768, n_anchor]
                self.y_class = tf.stack(y_class, axis=-1)
                selected_boat_index = tf.stack(selected_boat_index, axis=-1)

                # Stack IOU masks
                temp_pos_mask = tf.stack(pos_mask, -1)
                temp_neg_mask = tf.stack(neg_mask, -1)
                #iou_mask = tf.stack(iou_mask, -1)

                if self.config.debug == 1:
                    print('self.y_class.shape', self.y_class.shape)
                    print('selected_boat_index', selected_boat_index.shape)
                    print('temp_pos_mask', temp_pos_mask.shape)

                # Stack iou_average_for_summary
                iou_average_for_summary = tf.stack(iou_average_for_summary, -1)
                iou_average_for_summary = tf.reduce_mean(iou_average_for_summary, -1)
                tf.summary.scalar(name='max_over_pixels_average_groundtruth_iou_over_anchors',
                                  tensor=tf.reduce_max(iou_average_for_summary))

                summarise_map(name='average_groundtruth_iou_over_anchors', tensor=iou_average_for_summary)

                # Filter y_reg by which boxes are positive
                y_reg_gt = []
                with tf.name_scope('filter_groundtruth_regression'):
                    for k in range(n_anchors):
                        # For each anchor run select_with_matrix_tf which filters the map of regression
                        # coordinates y_reg to the (per pixel) particular boat indicated by selected_boat_index
                        y_reg_gt_anchor = select_with_matrix_tf(tensor=self.y_reg,
                                                                indexer=selected_boat_index[:,:,:, k],
                                                                batch_size=self.config.batch_size)
                        y_reg_gt.append(y_reg_gt_anchor)

                    self.y_reg_gt = tf.stack(y_reg_gt, -2)

            # Some summaries
            #tf.summary.image(name='input_images', tensor=self.x, max_outputs=3)
            #tf.summary.image(name='y_map', tensor=tf.reduce_sum(self.y_map, -1, keepdims=True), max_outputs=1)

            with tf.name_scope('sample'):
                # counting the number of positive samples.
                # if there are zero, then  we are in a "no_boats" batch image and sample consequently
                # note: instead of tf.reduce_sum(self.y_map) > 0, we want tf.reduce_sum(self.y_map, [1,2,3])
                # the latter gives us the sum of the ground truth per image
                # if the sum is positive, then there are boats in the image and we want to sample some
                class_mask = []
                pos_mask = []

                for i in range(self.config.batch_size):
                    sliced_temp_pos_mask = tf.slice(temp_pos_mask,begin=[i,0,0,0],size=[1,-1,-1,-1])
                    summed_pos_mask = tf.reduce_sum(sliced_temp_pos_mask)
                    n_positive_samples = tf.cond(summed_pos_mask > 0,
                                             lambda: self.config.n_positive_samples,
                                             lambda: 0)
                    n_negative_samples = tf.cond(summed_pos_mask > 0,
                                             lambda: self.config.n_negative_samples,
                                             lambda: self.config.n_negative_samples_when_no_boats)
                    # sampling
                    # note that atm pos_sample applies the same sampling over the whole batch
                    # refer to utils for the function
                    pos_sample = tf.py_func(np_sample, [temp_pos_mask[i], 1, n_positive_samples], tf.float64)
                    pos_sample = tf.cast(pos_sample, tf.float32)
                    #pos_mask = temp_pos_mask * pos_sample
                    neg_sample = tf.py_func(np_sample, [temp_neg_mask[i], 1, n_negative_samples], tf.float64)
                    neg_sample = tf.cast(neg_sample, tf.float32)
                    #neg_mask = temp_neg_mask * neg_sample

                    if self.config.debug:
                       print("pos_sample.shape", pos_sample.shape)
                       print("temp_pos_mask.shape", temp_pos_mask.shape)
                    temp_class_mask = pos_sample + neg_sample
                    class_mask.append(temp_class_mask)
                    pos_mask.append(pos_sample)

                class_mask = tf.stack(class_mask, 0)
                pos_mask = tf.stack(pos_mask, 0)

        with tf.name_scope('model'):
            with tf.name_scope('feature_maps'):

                normalized_x = tf.cond(tf.math.greater(self.config.normalization, tf.constant(0)),
                                             lambda: tf.map_fn(lambda frame: tf.image.per_image_standardization(frame), self.x),
                                             lambda: self.x)

                layer_conv1 = create_convolutional_layer(input=normalized_x,
                                                         num_input_channels=self.config.num_channels,
                                                         conv_filter_size=self.config.filter_size_conv1,
                                                         num_filters=self.config.num_filters_conv1,
                                                         maxpool=0,
                                                         name="conv_layer_1")

                layer_conv2 = create_convolutional_layer(input=layer_conv1,
                                                         num_input_channels=self.config.num_filters_conv1,
                                                         conv_filter_size=self.config.filter_size_conv2,
                                                         num_filters=self.config.num_filters_conv2,
                                                         maxpool=0,
                                                         name='conv_layer_2')

                layer_conv3 = create_convolutional_layer(input=layer_conv2,
                                                         num_input_channels=self.config.num_filters_conv2,
                                                         conv_filter_size=self.config.filter_size_conv3,
                                                         num_filters=self.config.num_filters_conv3,
                                                         maxpool=0,
                                                         name='conv_layer_3')

                layer_conv4 = create_convolutional_layer(input=layer_conv3,
                                                         num_input_channels=self.config.num_filters_conv3,
                                                         conv_filter_size=self.config.filter_size_conv4,
                                                         num_filters=self.config.num_filters_conv4,
                                                         maxpool=0,
                                                         name='conv_layer_4')

                layer_conv5 = create_convolutional_layer(input=layer_conv4,
                                                         num_input_channels=self.config.num_filters_conv4,
                                                         conv_filter_size=self.config.filter_size_conv5,
                                                         num_filters=self.config.num_filters_conv5,
                                                         maxpool=0,
                                                         name='conv_layer_5')

                pool = tf.nn.max_pool(value=layer_conv5,
                                      ksize=[1, 2, 2, 1],
                                      strides=[1, 2, 2, 1],
                                      padding='SAME')

                self.feature_maps = create_deconvolutional_layer(input=pool,
                                                            num_filters=self.config.num_filters_conv5,
                                                            name='deconvolution',
                                                            upscale_factor=2)

                # self.feature_maps = create_convolutional_layer(input=layer_deconv,
                #                                                num_input_channels=self.config.num_filters_conv2,
                #                                                conv_filter_size=self.config.initial_im_size,
                #                                                num_filters=self.config.num_features,
                #                                                maxpool=0)

                #tf.summary.image(name='feature_maps', tensor=self.feature_maps[:,:,:,0:3], max_outputs=3)

            with tf.name_scope('region_proposal_network'):

                with tf.name_scope('cnn'):
                    # Realise that the sliding window can be implemented as a convolution
                    with tf.name_scope('sliding_window'):
                        window_outputs = create_convolution(input=self.feature_maps,
                                                            num_input_channels=self.config.num_filters_conv5,
                                                            conv_filter_size=self.config.window_size,
                                                            num_filters=self.config.sliding_hidden_layer_size,
                                                            stride=1,
                                                            data_format="NHWC")

                        window_outputs = tf.nn.relu(window_outputs)

                    with tf.name_scope('classification_layer'):
                        self.class_scores = create_convolution(input=window_outputs,
                                                               num_input_channels=self.config.sliding_hidden_layer_size,
                                                               conv_filter_size=1,
                                                               #num_filters=self.config.n_proposal_boxes,
                                                               num_filters=self.config.n_proposal_boxes*2,
                                                               stride=1,
                                                               data_format="NHWC")

                        # tf.summary.image(name='classification_scores',
                        #                  tensor=tf.reduce_sum(self.class_scores, -1, keepdims=True),
                        #                  max_outputs=1)

                    with tf.name_scope('regression_layer'):
                        # reg_outputs: [batch_size, 768, 768, 4*self.config.n_proposal_boxes]
                        reg_outputs = create_convolution(input=window_outputs,
                                                             num_input_channels=self.config.sliding_hidden_layer_size,
                                                             conv_filter_size=1,
                                                             num_filters=4*self.config.n_proposal_boxes,
                                                             stride=1,
                                                             data_format="NHWC")

                        # self.reg_scores: [batch_size, 768, 768, self.config.n_proposal_boxes, 4]
                        self.reg_scores = tf.reshape(tensor=reg_outputs,
                                                     shape=[-1, 768, 768, self.config.n_proposal_boxes, 4],
                                                     name='reshape_regression_outputs')

            with tf.name_scope('loss'):

                epsilon = 0.001

                #sigmoid_ce = tf.nn.sigmoid_cross_entropy_with_logits(labels=self.y_class,
                #                                                     logits=self.class_scores + epsilon)
                self.class_scores = tf.reshape(self.class_scores,
                                               [self.config.batch_size, 768, 768, self.config.n_proposal_boxes, 2])

                y_classes = tf.cast(self.y_class, tf.int32)

                sigmoid_ce = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_classes,
                                                                     logits=self.class_scores)

                # Remember that we only look at positive (> upper iou thresh) and negative (< iou thresh) boxes
                masked_signoid_ce = tf.multiply(class_mask, sigmoid_ce)

                classification_loss_per_sample = tf.reduce_sum(masked_signoid_ce, axis = [1,2,3])
                classification_loss = tf.reduce_mean(classification_loss_per_sample)

                if self.config.debug == 1:
                    print('sigmoid_ce  ', sigmoid_ce.shape)
                    print('classification_loss  ', classification_loss.shape)

                with tf.name_scope('create_adjusted_coordinates'):
                    # t_x = (x_predict - x_anchor)/ w_anchor
                    epsilon = 0.001
                    t_x = (self.reg_scores[:, :, :, :, 0] - reg_anchors[:, :, :, :, 0]) / (reg_anchors[:, :, :, :, 2] + epsilon)
                    t_x_star = (self.y_reg_gt[:, :, :, :, 0] - reg_anchors[:, :, :, :, 0]) / (reg_anchors[:, :, :, :, 2] + epsilon)

                    t_y = (self.reg_scores[:, :, :, :, 1] - reg_anchors[:, :, :, :, 1]) / (reg_anchors[:, :, :, :, 3] + epsilon)
                    t_y_star = (self.y_reg_gt[:, :, :, :, 1] - reg_anchors[:, :, :, :, 1]) / (reg_anchors[:, :, :, :, 3] + epsilon)

                    reg_adj_1 = tf.maximum(self.reg_scores[:, :, :, :, 2], epsilon)
                    t_w = tf.log(reg_adj_1 / (reg_anchors[:, :, :, :, 2] + epsilon))
                    t_w_star = tf.log(self.y_reg_gt[:, :, :, :, 2] / (reg_anchors[:, :, :, :, 2] + epsilon))

                    reg_adj_2 = tf.maximum(self.reg_scores[:, :, :, :, 3], epsilon)
                    t_h = tf.log(reg_adj_2 / (reg_anchors[:, :, :, :, 3] + epsilon))
                    t_h_star = tf.log(self.y_reg_gt[:, :, :, :, 3] / (reg_anchors[:, :, :, :, 3] + epsilon))

                    y_reg_loss_pred = tf.stack([t_x, t_y, t_w, t_h], axis=-1)
                    y_reg_loss_gt = tf.stack([t_x_star, t_y_star, t_w_star, t_h_star], axis=-1)

                regression_loss_per_pixel = tf.losses.huber_loss(labels=y_reg_loss_gt,
                                                                 predictions=y_reg_loss_pred,
                                                                 delta=self.config.delta,
                                                                 reduction=tf.losses.Reduction.NONE)

                # Remember that we only look at positive (> upper iou thresh) boxes
                # Expand mask to have dimension for 4 coordiantes. Now of shape [batch, 768, 768, n_proposal_boxes, 4]
                iou_mask_regression = tf.tile(tf.expand_dims(pos_mask, -1), [1, 1, 1, 1, 4])

                masked_regression_loss_per_pixel = tf.multiply(regression_loss_per_pixel, iou_mask_regression)

                # Replace nans with 0s
                masked_regression_loss_per_pixel = tf.where(tf.is_nan(masked_regression_loss_per_pixel),
                                                            tf.zeros_like(masked_regression_loss_per_pixel),
                                                            masked_regression_loss_per_pixel)

                # For summary, summarise the regression loss for each of the 4 coordinates seperately to see
                # if any particular one is under peforming.
                seperate_coordinate_losses = tf.reduce_mean(
                    tf.reduce_sum(masked_regression_loss_per_pixel, axis=[1,2,3]),
                    axis=0
                )

                tf.summary.scalar(name='x_center_reg_loss', tensor=seperate_coordinate_losses[0])
                tf.summary.scalar(name='y_center_reg_loss', tensor=seperate_coordinate_losses[1])
                tf.summary.scalar(name='width_reg_loss', tensor=seperate_coordinate_losses[2])
                tf.summary.scalar(name='height_reg_loss', tensor=seperate_coordinate_losses[3])

                regression_loss_per_sample = tf.reduce_sum(masked_regression_loss_per_pixel, axis=[1,2,3,4])
                regression_loss = tf.reduce_mean(regression_loss_per_sample)

                tf.summary.scalar(name='classification_loss', tensor=classification_loss)
                tf.summary.scalar(name='regression_loss', tensor=regression_loss)

                self.loss = classification_loss + (self.config.alpha_loss)*regression_loss

                tf.summary.scalar(name='total_loss', tensor=self.loss)

        with tf.name_scope('optimizer'):
            self.train_step = tf.train.AdamOptimizer(self.config.learning_rate).minimize(self.loss,
                                                                                global_step=self.global_step_tensor)

        with tf.name_scope('summaries'):
            self.summaries = tf.summary.merge_all()

    def init_saver(self):
        # here you initialize the tensorflow saver that will be used in saving the checkpoints.
        self.saver = tf.train.Saver(max_to_keep=self.config.max_to_keep)


text:
import numpy as np

# We create a rank 1 ndarray
x = np.array([1,2,3])

# We create a 3 x 3 ndarray
Y = np.array([[1,2,3],[4,5,6],[7,8,9]])

# We create a 3 x 1 ndarray
Z = np.array([1,2,3]).reshape(3,1)

# We print x
print()
print('x = ', x)
print()

# We print Y
print()
print('Y = \n', Y)
print()

# We print Z
print()
print('Z = \n', Z)
print()

print('x + Y = \n', x + Y)
print()
print('Z + Y = \n',Z + Y)


text:
import pandas as pd
import numpy as np
import os
import glob
import time
import sys
from argparse import ArgumentParser
from utilities import create_directory, get_file_content, extract_bitext, detect_non_eng

BEARER_TOKEN =

def preprocess_dataframe(df_data):
    """
    Add index and PRID(press release id) columns to dataframe
    Remove Null Posting_Datetime entry
    """
    df_data["Posting_Date"] = pd.to_datetime(df_data["Posting_Date"], errors="coerce")
    df_data["Posting_Datetime"] = pd.to_datetime(
        df_data["Posting_Datetime"], errors="coerce"
    )
    if "index" not in df_data.columns.tolist():
        df_data.reset_index(inplace=True)

    if not "PRID" in df_data.columns.tolist():
        df_data["PRID"] = df_data["Link"].apply(lambda x: x.split("=")[-1])

    df_data["Posting_Datetime"] = df_data.apply(
        lambda x: x["Posting_Date"]
        if pd.isnull(x["Posting_Datetime"])
        else x["Posting_Datetime"],
        axis=1,
    )
    return df_data

def main():
    parser = ArgumentParser()
    parser.add_argument(
        "--output-dir", help="output directory", type=str, required=True
    )
    parser.add_argument("--month", help="month", type=str, required=True)
    parser.add_argument("--year", help="year", type=str, required=True)
    args = parser.parse_args()

    n_month, n_year = str(args.month).lower(), str(args.year)
    work_dir = args.output_dir + "//" + n_month + "_" + n_year
    align_loc = work_dir + "//" + "align_" + n_month + "_" + n_year
    tokenize_loc_en = work_dir + "//" + "tokenize_en_" + n_month + "_" + n_year
    tokenize_loc_hi = work_dir + "//" + "tokenize_hi_" + n_month + "_" + n_year
    submit_aligner = work_dir + "//" + "submit_aligner_" + n_month + "_" + n_year
    en_data_file = "_".join(["English", "data", n_month, n_year]) + ".csv"
    hi_data_file = "_".join(["Hindi", "data", n_month, n_year]) + ".csv"

    create_directory(align_loc)
    create_directory(submit_aligner)

    df_en = pd.read_csv(work_dir + "//" + en_data_file, encoding="utf-16")
    df_hi = pd.read_csv(work_dir + "//" + hi_data_file, encoding="utf-16")
    df_en = preprocess_dataframe(df_en)
    df_hi = preprocess_dataframe(df_hi)
    df_en.to_csv(work_dir + "//" + en_data_file, index=False, encoding="utf-16")
    df_hi.to_csv(work_dir + "//" + hi_data_file, index=False, encoding="utf-16")

    # Crete files which are parallel based on Ministry Name and Posting Date
    k_hi = pd.DataFrame(
        df_hi[["English_Ministry_Name", "Posting_Date", "index"]]
        .groupby(["English_Ministry_Name", "Posting_Date"])["index"]
        .apply(lambda x: x.tolist())
    )
    k_en = pd.DataFrame(
        df_en[["English_Ministry_Name", "Posting_Date", "index"]]
        .groupby(["English_Ministry_Name", "Posting_Date"])["index"]
        .apply(lambda x: x.tolist())
    )
    k_merge = pd.merge(
        k_en,
        k_hi,
        left_index=True,
        right_index=True,
        how="inner",
        suffixes=("_en", "_hi"),
    )
    k_merge.to_csv(
        work_dir + "//" + "submit_aligner_" + n_month + "_" + n_year + ".csv",
        index=True,
        encoding="utf-16",
    )

    fl_tok_en = sorted(glob.glob(tokenize_loc_en + "//" + "*.txt"))
    fl_tok_hi = sorted(glob.glob(tokenize_loc_hi + "//" + "*.txt"))
    no_sen_df = pd.DataFrame(
        columns=[
            "Filename_en",
            "Total_sentences_en",
            "Filename_hi",
            "Total_sentences_hi",
        ]
    )
    for count, i in enumerate(k_merge.iterrows()):
        en_align_file = (
            submit_aligner
            + "//subalign_"
            + str(count).zfill(4)
            + "_en_"
            + "_".join(i[0][0].split())
            + "_"
            + i[0][1].strftime("%Y-%m-%d")
            + ".txt"
        )
        hi_align_file = (
            submit_aligner
            + "//subalign_"
            + str(count).zfill(4)
            + "_hi_"
            + "_".join(i[0][0].split())
            + "_"
            + i[0][1].strftime("%Y-%m-%d")
            + ".txt"
        )
        with open(en_align_file, encoding="utf-16", mode="w") as flw_en:
            count_en = 0
            for ind in i[1]["index_en"]:
                with open(fl_tok_en[ind], encoding="utf-16", mode="r") as flr_en:
                    k_en = flr_en.read()
                    count_en += k_en.count("\n")
                    flw_en.write(k_en)
        with open(hi_align_file, encoding="utf-16", mode="w") as flw_hi:
            count_hi = 0
            for ind in i[1]["index_hi"]:
                with open(fl_tok_hi[ind], encoding="utf-16", mode="r") as flr_hi:
                    k_hi = flr_hi.read()
                    count_hi += k_hi.count("\n")
                    flw_hi.write(k_hi)
        no_sen_df = no_sen_df.append(
            {
                "Filename_en": os.path.basename(en_align_file),
                "Total_sentences_en": count_en,
                "Filename_hi": os.path.basename(hi_align_file),
                "Total_sentences_hi": count_hi,
            },
            ignore_index=True,
        )
        print(
            f"Writing {os.path.basename(en_align_file)} and {os.path.basename(hi_align_file)} done"
        )
    no_sen_df.to_csv(
        work_dir + "//" + "tok_sen_count_" + n_month + "_" + n_year + ".csv",
        index=False,
        encoding="utf-16",
    )
    fl_list = glob.glob(submit_aligner + "//" + "*.txt")
    en_fl = sorted([i for i in fl_list if os.path.basename(i).split("_")[2] == "en"])
    hi_fl = sorted([i for i in fl_list if os.path.basename(i).split("_")[2] == "hi"])
    c_fl = list(zip(en_fl, hi_fl))
    for i in c_fl:
        extract_bitext(BEARER_TOKEN, align_loc, i[0], i[1])

if __name__ == "__main__":
    main()

max_stars_repo_name: PromodhPinto/anuvaad-corpus-tools

max_issues_repo_name: PromodhPinto/anuvaad-corpus-tools

max_forks_repo_name: PromodhPinto/anuvaad-corpus-tools