text:
"""Encoder

OnehotToLabel, LabeltoOnehot are implemented.

"""

import numpy as np

class OnehotToLabel():
    """Transform onehot-encoded data to label-encoded data

    Attributes:
        K (int) : number of class

    """
    def __init__(self):
        pass

    def fit(self,X):
        """fit

        Args:
            X (2-D array) : onehot-encoded data, shape = (N_samples,N_class)

        """
        self.K = X.shape[1]

    def transform(self,X):
        """transform

        Args:
            X (2-D array) : onehot-encoded data, shape = (N_samples,N_class)

        Returns:
            1-D array: label-encoded data, shape = (N_samples)

        """
        y = np.zeros(X.shape[0])
        for k in range(self.K):
            y[X[:,k] == 1] = k
        return y

    def fit_transform(self,X):
        """fit and transform

        Args:
            X (2-D array) : onehot-encoded data, shape = (N_samples,N_class)

        Returns:
            1-D array: label-encoded data, shape = (N_samples)

        """
        self.fit(X)
        return self.transform(X)

    def inverse(self,y):
        """inverse transform

        Args:
            y (1-D array) : label-encoded data, shape = (N_samples)

        Returns:
            2-D array: onehot-encoded data, shape = (N_samples,N_class)

        """
        X = np.zeros((y.shape[0],self.K))
        for k in range(self.K):
            X[y == k,k] = 1
        return X

class LabelToOnehot():
    """Transform label-encoded data to onehot-encoded data

    Attributes:
        K (int) : number of class
        data_group (1-D array) : data which is labeled

    """
    def __init__(self):
        pass

    def fit(self,y):
        """fit

        Args:
            y (1-D array) : label-encoded data, shape = (N_samples)

        """
        self.data_group = np.unique(y)
        self.K = len(self.data_group)

    def transform(self,y):
        """transform

        Args:
            y (1-D array) : label-encoded data, shape = (N_samples)

        Returns:
            2-D array: onehot-encoded data, shape = (N_samples,N_class)

        """
        X = np.zeros((y.shape[0],self.K))
        for k,g in enumerate(self.data_group):
            X[y == g,k] = 1
        return X

    def fit_transform(self,y):
        """fit and transform

        Args:
            y (1-D array) : label-encoded data, shape = (N_samples)

        Returns:
            2-D array: onehot-encoded data, shape = (N_samples,N_class)

        """
        self.fit(y)
        return self.transform(y)

    def inverse(self,X):
        """inverse transform

        Args:
            X (2-D array) : onehot-encoded data, shape = (N_samples,N_class)

        Returns:
            1-D array: label-encoded data, shape = (N_samples)

        """
        y = np.zeros(X.shape[0])
        for k,g in enumerate(self.data_group):
            y[X[:,k] == 1] = g
        return y


text:
from code.pytorch.LAMPO.core.config import config
import argparse
from code.pytorch.LAMPO.core.collector import RunModel, RunModelMujoco
from code.pytorch.LAMPO.core.colome_torras import CT_ImitationLearning, CT_ReinforcementLearning
from code.pytorch.LAMPO.core.task_interface import TaskInterface
from code.pytorch.LAMPO.core.plot import LampoMonitor
import numpy as np
import json
import os
import commentjson

from mppca.mixture_ppca import MPPCA
from code.pytorch.LAMPO.core.lampo import Lampo
from code.pytorch.LAMPO.core.rl_bench_box import *
from code.pytorch.LAMPO.core.model import RLModel

from envs.gym_kuka_mujoco.envs import *

def get_arguments_dict():
    parser = argparse.ArgumentParser()

    parser.add_argument("-t", "--task_name",
                        help="Task name.",
                        default="reach_target")
    parser.add_argument("-i", "--id",
                        help="Identifier of the process.",
                        type=int, default=10)
    parser.add_argument(
                        "--batch_size",
                        help="How many episodes before improvement.",
                        type=int,
                        default=30)
    parser.add_argument(
                        "--imitation_learning",
                        help="How many episodes before improvement.",
                        type=int, default=50)

    parser.add_argument("--il_noise",
                        help="Add noise on the context",
                        type=float,
                        default=0.03)
    parser.add_argument("--dense_reward",
                        help="Use dense reward",
                        default=False)
    parser.add_argument("-k", "--kl_bound",
                        help="Bound the improvement kl.",
                        type=float,
                        default=0.2)
    parser.add_argument("-c", "--context_kl_bound",
                        help="Bound the context kl.",
                        type=float,
                        default=50.)
    parser.add_argument("--context_reg",
                        help="Bound the improvement kl.",
                        type=float,
                        default=1E-4)
    parser.add_argument("-z", "--normalize",
                        help="Normalized Importance Sampling",
                        default=True)
    parser.add_argument("-m",
                        "--max_iter",
                        help="Maximum number of iterations.",
                        type=int,
                        default=1)
    parser.add_argument("--n_evaluations",
                        help="Number of the evaluation batch.",
                        type=int,
                        default=20)
    parser.add_argument("--not_dr",
                        help="Don't do dimensionality reduction.",
                        default=False)
    parser.add_argument("--plot",
                        help="Don't do dimensionality reduction.",
                        default=True)
    parser.add_argument("--forgetting_rate",
                        help="The forgetting rate of the IRWR-GMM.",
                        type=float,
                        default=1.)
    parser.add_argument('--param_file',
                        type=str, default='h-cps/assisitive/IMOGICPegInsertion.json',
                        help='the parameter file to use')

    parser.add_argument('--filter_warning',
                        choices=['error', 'ignore', 'always', 'default', 'module', 'once'],
                        default='default',
                        help='the treatment of warnings')
    parser.add_argument('--profile',
                        action='store_true',
                        help='runs in a profiler')
    parser.add_argument('--final',
                        action='store_true',
                        help='puts the data in the final directory for easy tracking/plotting')
    parser.add_argument('--num_restarts',
                        type=int,
                        default=1,
                        help='The number of trials to run.')

    parser.add_argument("--policy_name",
                        default='SAC')
    parser.add_argument("--env_name",
                        default="insertion")  # OpenAI gym environment name
    parser.add_argument("--log_path",
                        default='com_cps/vic')
    parser.add_argument("--eval_only",
                        default=False)
    parser.add_argument("--eval_max_timesteps",
                        default=1e3,
                        type=int)

    # ============================= Policy Search : GPREPS ===================================
    parser.add_argument("--num_policy_update",
                        default=1000,
                        type=int)
    parser.add_argument("--num_real_episodes",
                        default=5,
                        type=int)
    parser.add_argument("--num_simulated_episodes",
                        default=15,
                        type=int)
    parser.add_argument("--num_average_episodes",
                        default=10,
                        type=int)
    parser.add_argument("--max_episode_steps",
                        default=400,
                        type=int)
    parser.add_argument("--start_policy_update_idx",
                        default=25,
                        type=int)
    parser.add_argument("--eval_policy_update",
                        default=10,
                        type=int)
    parser.add_argument("--eval_max_context_pairs",
                        default=5,
                        type=int)
    parser.add_argument("--max_eval_episode",
                        default=5,
                        type=int)
    parser.add_argument("--render",
                        default=False)
    parser.add_argument("--eps",
                        default=[0.2, 0.65])
    parser.add_argument("-r", "--slurm",
                        help="Don't look for CPU usage.",
                        action="store_true")
    parser.add_argument("--forward",
                        help="Bound the improvement kl.",
                        default=False)

    parser.add_argument("--alg",
                        default='MPPCA')

    parser.add_argument("--load_data",
                        default=True)

    args = parser.parse_args()
    return args

class Objectview(object):

    def __init__(self, d):
        self.__dict__ = d

def process_parameters(parameters, n_samples, n_context, noise=0.03):
    parameters = parameters[:n_samples].copy()
    parameters[:, :n_context] += noise * np.random.normal(size=parameters[:, :n_context].shape)
    return parameters

if __name__ == "__main__":
    args = get_arguments_dict()

    param_dir = 'params/'
    param_file = 'IMOGICPegInsertion.json'

    param_file = os.path.join(param_dir, param_file)
    with open(param_file) as f:
        set_params = commentjson.load(f)

    # make env
    env_cls = globals()[set_params['env']]
    env = env_cls(**set_params['env_options'])

    n_clusters = set_params["alg_options"]["n_cluster"]

    task = globals()[set_params["alg_options"]["task_class"]](env, set_params)
    print("context_dim :", task.get_context_dim())
    print("latent_dim :", task.get_impedance_dim())
    # print("context :", task.read_context())
    print("+" * 100)

    state_dim = task.get_context_dim()
    # print("state_dim :", state_dim)
    #
    # # reward, info = task.send_movement(params)
    # # print("reward :", reward, "info", info)

    print('+' * 100)
    if args.load_data:
        print('Load datasets !')
        # parameters = np.load('./demo/insertion_demonstration.npy')
        env_name = 'FeedingSawyerHuman-v1'
        parameters = np.load('./demo/' + env_name + '_demonstration.npy')
    else:
        print('Collect datasets !')
        parameters = task.get_demonstrations(num_traj=50)[:args.imitation_learning]
        # print("parameters :::", parameters[:, state_dim:])
        np.save('./demo/insertion_demonstration.npy', parameters)

    print("parameters :", parameters.shape)
    parameters = process_parameters(parameters,
                                    args.imitation_learning,
                                    state_dim,
                                    args.il_noise)
    # print("parameters :::", parameters.shape)

    n_evaluation_samples = args.n_evaluations
    n_batch = args.batch_size
    kl_bound = args.kl_bound
    kl_context_bound = args.context_kl_bound
    if args.forward:
        kl_type = "forward"
    else:
        kl_type = "reverse"

    normalize = args.normalize

    print('+' * 100)
    print('parameters fitting !')

    if args.alg=='REPS':
        " imitation learning "
        imitation = CT_ImitationLearning(state_dim,
                                         parameters.shape[1] - set_params["alg_options"]["latent_dim"],
                                         set_params["alg_options"]["latent_dim"],
                                         n_clusters, use_dr=not args.not_dr)
        state_list = parameters[:, :state_dim]
        print("state_list :", state_list)
        context_list = parameters[:, state_dim:]
        print("params_list :", context_list)
        imitation.fit(parameters[:, :state_dim],
                      parameters[:, state_dim:],
                      forgetting_rate=args.forgetting_rate)

        rl_model = CT_ReinforcementLearning(imitation, kl_bound=kl_bound)
    elif args.alg=='MPPCA':
        " MPPCA "
        mppca = MPPCA(n_clusters, int(set_params["alg_options"]["latent_dim"]), n_init=5)
        mppca.fit(parameters)

        rl_model = RLModel(mppca,
                           context_dim=state_dim, kl_bound=kl_bound,
                           kl_bound_context=kl_context_bound, kl_reg=args.context_reg,
                           normalize=normalize,
                           kl_type=kl_type
                           )

        sr = Lampo(rl_model, wait=not args.slurm)
    else:
        print("Please give parameter fit model !")

    myplot = LampoMonitor(kl_bound, kl_context_bound=kl_context_bound,
                          title="class_log kl=%.2f, %d samples" %
                          (kl_bound, n_batch))

    print("rl_model :", rl_model)
    collector = RunModelMujoco(task, rl_model, args.dense_reward)

    reward_list = []
    for i in range(args.max_iter):
        s, r, actions, latent, cluster, observation = \
            collector.collect_samples(n_evaluation_samples, isomorphic_noise=False)

        if args.alg == 'REPS':
            rl_model.add_dataset(observation[:n_batch], actions[:n_batch], r[:n_batch])
            print("ITERATION", i)
            print("SUCCESS:", np.mean(s))

        if args.alg == 'MPPCA':
            sr.add_dataset(actions[:n_batch], latent[:n_batch], cluster[:n_batch], observation[:n_batch], r[:n_batch])
            print("ITERATION", i)
            print("SUCCESS:", np.mean(s))

        reward_list.append(np.mean(s))

        myplot.notify_outer_loop(np.mean(s), np.mean(r))

        if args.alg == "MPPCA":
            sr.improve()
            # print("Optimization %f" % sr.rlmodel._f)
            # print("KL %f <= %f" % (sr.rlmodel._g, kl_bound))
            # if kl_context_bound> 0:
            #     print("KL context %f <= %f" % (sr.rlmodel._h, kl_context_bound))
        myplot.notify_inner_loop(0., 0., 0., 0.)

        if args.plot:
            myplot.visualize()

    s, r, actions, latent, cluster, observation = collector.collect_samples(n_evaluation_samples, isomorphic_noise=False)

    print("ITERATION", args.max_iter)
    print("SUCCESS:", np.mean(s))
    myplot.notify_outer_loop(np.mean(s), np.mean(r))

    if args.plot:
        myplot.visualize(last=True)

    print("reward_list :", np.array(reward_list))
    np.save(args.alg + "_" + args.env_name + "_" + "reward_list.npy", np.array(reward_list))

    # if args.save:
    #     myplot.save(experiment_path + "result_%d.npz" % args.id)

    # import time
    # for i in range(100):
    #     print("+"* 50, i)
    #     task.reset()
    #     task._env.render()
    #     time.sleep(0.5)
    #     # while True:
    #     #     task._env.render()

    # context, target_pos, target_quat, target_euler = task.read_context()
    #
    # # set parameters :::
    # scale = np.array([2.0, 0.0, 0.0])
    # num_waypoints = 3
    # scale_list = scale.repeat([6, 6, 6], axis=0).reshape(num_waypoints, 6)
    # # print("scale_list :::", scale_list)
    #
    # stiffness_list = np.array([[10., 10., 6., 6., 6., 6.],
    #                            [0., 0., 0., 0., 0., 0.],
    #                            [0., 0., 0., 0., 0., 0.]])
    #
    # control_scale = np.ones_like(stiffness_list) * 10
    # stiffness_list = control_scale * stiffness_list
    # damping_list = scale_list * np.sqrt(stiffness_list)
    # weight_list = np.array([1.0, 0.0, 0.0])
    #
    # # pos_set_list = np.array([[-0.80349109, 0.09318907, 0.07348721],
    # #                          [-0.78349109, 0.09318907, 0.07348721],
    # #                          [-0.80349109, 0.09318907, -0.03348721]])
    #
    # # quat_set_list = np.array([[1.9265446606661554, -0.40240192959667226, -1.541555812071902],
    # #                           [1.9265446606661554, -0.40240192959667226, -1.541555812071902],
    # #                           [1.9265446606661554, -0.40240192959667226, -1.541555812071902]])
    #
    # pos_set_list = np.zeros((3, 3))
    # quat_set_list = np.zeros((3, 3))
    # for i in range(num_waypoints):
    #     pos_set_list[i, :] = target_pos
    #     quat_set_list[i, :] = target_euler
    #
    # way_points_list = np.concatenate((pos_set_list, quat_set_list), axis=1)
    # # print("way_point_list :::", way_points_list)
    #
    # task.set_waypoints(way_points_list)
    #
    # print("params :", stiffness_list.reshape(1, 18))
    # print("weight_list :", weight_list)
    # params = np.hstack((stiffness_list.reshape(1, 18)[0], weight_list))
    #

ext: py

reason: import numpy


text:
import json
import sys
# import matplotlib.pyplot as plt
import copy
import numpy as np
import tensorflow as tf
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.utils import class_weight
from collections import Counter
import random
from tensorflow.keras.callbacks import Callback
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
from nltk.tokenize import sent_tokenize
import os

# read entire json file
# if loading the original dataset ignore the reviews with score 0
# return a list of json entries
def readJson(file_path, original=False):
    data = []
    with open(file_path, encoding="utf8") as json_file:
        for line in json_file:
            entry = json.loads(line)
            if original == True:
                if entry['_source']['Review']['ReviewRating'] == 0:
                    continue
            data.append(entry)
    return data

# compute histogram of review scores
# input -> list of jsons
# output -> dict score -> #reviews
def computeScoreHistogram(data, normalize = False):
    histo = {}

    for entry in data:
        score = entry['_source']['Review']['ReviewRating']
        if score in histo:
            histo[score] += 1
        else:
            histo[score] = 1

    if normalize == True:
        for key, value in histo.items():
            histo[key] = 1.0 * value / len(data)
    print(histo)
    return histo

def computeTextStatistics(data, superior_threshold=None, inferior_threshold=None):
    histo_char = {}
    histo_word = {}
    histo_category = {}

    sup_threshold = 0
    inf_threshold = 0

    for entry in data:
        text = entry['_source']['Review']['ReviewBody']
        category = entry['_source']['Product']['ProductCategory']
        chars = len(text)
        words = len(text.split(" "))

        if superior_threshold != None and words > superior_threshold:
            sup_threshold += 1

        if inferior_threshold != None and words < inferior_threshold:
            inf_threshold += 1

        if chars in histo_char:
            histo_char[chars] += 1
        else:
            histo_char[chars] = 1

        if words in histo_word:
            histo_word[words] += 1
        else:
            histo_word[words] = 1

        if category in histo_category:
            histo_category[category] += 1
        else:
            histo_category[category] = 1

    return histo_char, histo_word, histo_category, sup_threshold, inf_threshold

def computeDatasetStatistics(data, superior_threshold=None, inferior_threshold=None):

    histo_scores = computeScoreHistogram(data)
    histo_chars, histo_words, histo_category, sup_threshold, inf_threshold = computeTextStatistics(data, superior_threshold, inferior_threshold)

    print("Reviews with number of words over", superior_threshold, "=", sup_threshold, "percentage =", 100.0*sup_threshold/len(data))
    print("Reviews with number of words under", inferior_threshold, "=", inf_threshold, "percentage =", 100.0*inf_threshold/len(data))
    print(histo_category)
    plt.bar(histo_scores.keys(), histo_scores.values(), 1.0, color='g')
    plt.title("Scores")
    plt.show()

    plt.bar(histo_chars.keys(), histo_chars.values(), 1.0, color='g')
    plt.title("Chars")
    plt.show()

    plt.bar(histo_words.keys(), histo_words.values(), 1.0, color='g')
    plt.title("Words")
    plt.show()

# split the dataset in 5 vs ALL -> 1,2,3,4 -> label 0
#                                        5 -> label 1
# input -> dataset list of jsons
# output -> dataset list of jsons
def splitData5vAll(data):
    new_data = copy.deepcopy(data)
    for entry in new_data:
        if entry['_source']['Review']['ReviewRating'] == 5:
            entry['_source']['Review']['ReviewRating'] = 1
        else:
            entry['_source']['Review']['ReviewRating'] = 0
    return new_data

# save the dataset
# input -> dataset list of jsons, filename to save
def saveData(data, filename):

    with open(filename, 'w') as outfile:
        for entry in data:
            json.dump(entry, outfile)
            outfile.write("\n")

# get features from data
# input -> data list of json
# sample_majority -> sample or not from majority class
# sample_count -> how many entries to sample from majority class
# set seed -> random seed value
# output -> list of dicts | one entry is a dict with features and labels
def getFeatures(data, use_review_text=True, sample_majority=False, sample_count=0, seed=None, majority_class=3):

    if sample_majority == False:
        train_list = []
        for data_entry in data:
            train_entry = {}
            if use_review_text == True:
                train_entry['features:review_text'] = data_entry['_source']['Review']['ReviewBody']
            train_entry['label'] = data_entry['_source']['Review']['ReviewRating']
            train_list.append(train_entry)
        return train_list

    elif sample_majority == True:

        majority_list = []
        for data_entry in data:
            majority_entry = {}
            if data_entry['_source']['Review']['ReviewRating'] == majority_class:
                if use_review_text == True:
                    majority_entry['features:review_text'] = data_entry['_source']['Review']['ReviewBody']
                majority_entry['label'] = data_entry['_source']['Review']['ReviewRating']
                majority_list.append(majority_entry)

        random.seed(seed)
        sampled_majority_list = random.sample(majority_list, sample_count)
        random.seed()

        train_list = []
        for data_entry in data:
            train_entry = {}
            if data_entry['_source']['Review']['ReviewRating'] != majority_class:
                if use_review_text == True:
                    train_entry['features:review_text'] = data_entry['_source']['Review']['ReviewBody']
                train_entry['label'] = data_entry['_source']['Review']['ReviewRating']
                # train_list.append(train_entry)
                sampled_majority_list.append(train_entry)

        # train_list.extend(sampled_majority_list)
        train_list = sampled_majority_list
        return train_list

# get processed features and labels
# input -> features
# output -> list of processed features, list of labels, dict of class_weights
def processFeatures(data, bert_proc):
    features = []
    labels = []

    iids = []
    sids = []
    i = 0
    for entry in data:
        review_text = entry["features:review_text"]
        input_ids, segment_ids = bert_proc.process_text(review_text)
        iids.append(input_ids)
        sids.append(segment_ids)
        labels.append(entry['label'])

    features = [np.array(iids), np.array(sids)]
    class_weights = class_weight.compute_class_weight('balanced', np.unique(labels), labels)
    class_weights = class_weights.astype(np.float32)
    return features, labels, class_weights

# get processed features and labels from texst
# input -> features
# output -> list of processed features, list of labels, dict of class_weights
def processFeaturesRawText(data, bert_proc):
    features = []
    iids = []
    sids = []
    i = 0
    for entry in data:
        review_text = entry
        input_ids, segment_ids = bert_proc.process_text(review_text)
        iids.append(input_ids)
        sids.append(segment_ids)

    features = [np.array(iids), np.array(sids)]
    return features

# split data in train dev test split using stratified
# input -> data
# output -> train, dev, test data
def splitTrainDevTest(data):

    train_data = []
    dev_data = []
    test_data = []

    full_indices = np.array(range(len(data)))
    full_classes = np.array(list(map(lambda x: x['_source']['Review']['ReviewRating'], data)))

    sss = StratifiedShuffleSplit(n_splits=1, test_size=0.1)

    for tr, te in sss.split(full_indices, full_classes):
        aux_train_indexes = tr
        test_indexes = te

    aux_train_data = []
    for i in test_indexes:
        test_data.append(data[i])
    for i in aux_train_indexes:
        aux_train_data.append(data[i])

    indices = np.array(range(len(aux_train_data)))
    classes = np.array(list(map(lambda x: x['_source']['Review']['ReviewRating'], aux_train_data)))

    sss_ = StratifiedShuffleSplit(n_splits=1, test_size=0.111111)

    for tr, de in sss_.split(indices, classes):
        train_indexes = tr
        dev_indexes = de

    for i in dev_indexes:
        dev_data.append(aux_train_data[i])
    for i in train_indexes:
        train_data.append(aux_train_data[i])

    print(len(train_data), len(dev_data), len(test_data), len(train_data) + len(dev_data) + len(test_data), len(data))
    print(len(list(set(train_indexes) & set(dev_indexes) & set(test_indexes))))

    return train_data, dev_data, test_data

# split the dataset in 4 classes -> 1 -> label 0
#                                   2,3 -> label 1
#                                   4 -> label 2
#                                   5 -> label 3
# input -> dataset list of jsons
# output -> dataset list of jsons
def splitData4Classes(data):
    new_data = copy.deepcopy(data)
    for entry in new_data:
        if entry['_source']['Review']['ReviewRating'] == 1:
            entry['_source']['Review']['ReviewRating'] = 0
        elif entry['_source']['Review']['ReviewRating'] == 2 or entry['_source']['Review']['ReviewRating'] == 3:
            entry['_source']['Review']['ReviewRating'] = 1
        elif entry['_source']['Review']['ReviewRating'] == 4:
            entry['_source']['Review']['ReviewRating'] = 2
        elif entry['_source']['Review']['ReviewRating'] == 5:
            entry['_source']['Review']['ReviewRating'] = 3
    return new_data

class FScoreCallback(Callback):
    def __init__(self, dataset, steps, labels):

        super().__init__()
        self.steps = steps

        self.dataset = dataset
        self.labels_int = []
        for x in labels:
            self.labels_int.append(np.argmax(x))

    def on_test_end(self, epoch, logs={}):
        y_pred = []
        y_true = self.labels_int
        predict_results = self.model.predict(self.dataset, steps=self.steps)
        for prediction in predict_results:
            y_pred.append(np.argmax(prediction))

        print()
        print(classification_report(y_true, y_pred, digits=4))

def compute_parameters(model_folder_path):

    # define input
    input_ids = tf.keras.layers.Input(shape=(64), dtype=tf.int32, name="input_ids")
    segment_ids = tf.keras.layers.Input(shape=(64), dtype=tf.int32, name="segment_ids")

    import BertModel
    import tensorflow.keras as keras
    import bert

    # define model
    bert_model = BertModel.BertModel(model_folder_path, 64)
    bert_output = bert_model.bert_layer([input_ids, segment_ids])
    cls_output = keras.layers.Lambda(lambda seq: seq[:, 0, :])(bert_output)
    cls_drop = keras.layers.Dropout(0.1)(cls_output)
    fc1 = keras.layers.Dense(units=100, activation="relu")(cls_drop)
    prediction = keras.layers.Dense(units=10, activation="softmax")(fc1)

    # build model
    model = keras.Model(inputs=[input_ids, segment_ids], outputs=prediction)
    model.build(input_shape=[(None, 64), (None, 64)])
    # load pretrained
    bert.load_bert_weights(bert_model.bert_layer, model_folder_path+"bert_model.ckpt")

    model.compile(optimizer=keras.optimizers.Adam(lr=0.1), loss = 'categorical_crossentropy', metrics = [tf.keras.metrics.categorical_accuracy])
    model.summary()

    from tensorflow.python.keras.utils.layer_utils import count_params
    trainable_count = count_params(model.trainable_weights)
    non_trainable_count = count_params(model.non_trainable_weights)

    print(trainable_count/1e6)
    print(non_trainable_count)

    # return model, bert_model

def build_reallife_corpus(model_folder_path):

    new_model_folder_path = "/".join(model_folder_path.split("/")[:-2])
    new_model_folder_path = os.path.join(new_model_folder_path, "reallife")

    train_data = readJson(model_folder_path+"train.json")
    train_data = clean_dict(train_data)
    new_train_data = add_last_sentence_to_data(train_data)
    new_train_data_over = perform_oversampling(new_train_data)
    print(len(train_data), len(new_train_data), len(new_train_data_over))
    saveData(new_train_data_over, os.path.join(new_model_folder_path, "train.json"))

    dev_data = readJson(model_folder_path+"dev.json")
    dev_data = clean_dict(dev_data)
    new_dev_data = add_last_sentence_to_data(dev_data)
    new_dev_data_over = perform_oversampling(new_dev_data)
    print(len(dev_data), len(new_dev_data), len(new_dev_data_over))
    saveData(new_dev_data_over, os.path.join(new_model_folder_path, "dev.json"))

    test_data = readJson(model_folder_path+"test.json")
    test_data = clean_dict(test_data)
    new_test_data = add_last_sentence_to_data(test_data)
    new_test_data_over = perform_oversampling(new_test_data)
    print(len(test_data), len(new_test_data), len(new_test_data_over))
    saveData(new_test_data_over, os.path.join(new_model_folder_path, "test.json"))

def add_last_sentence_to_data(data):
    new_data = copy.deepcopy(data)
    new_entries = []
    count = 0
    for entry in new_data:
        review_text = entry['_source']['Review']['ReviewBody']
        sentences = sent_tokenize(review_text)
        if len(sentences) > 1:
            # add new entry to dataset
            new_entry = copy.deepcopy(entry)
            new_entry['_source']['Review']['ReviewBody'] = sentences[-1]
            new_entry['_score'] = 2
            new_entries.append(new_entry)
            if entry == new_entry:
                print(entry)
                print(new_entry)
                sys.exit()
            count += 1
    # print(new_entries)
    new_data.extend(new_entries)
    return new_data

def perform_oversampling(data):
    new_data = copy.deepcopy(data)
    new_entries = []
    counter = [0,0,0,0,0]
    for entry in new_data:
        label = entry['_source']['Review']['ReviewRating']
        counter[label-1] += 1

    while True:

        random_entry = random.choice(data)
        random_label = random_entry['_source']['Review']['ReviewRating']

        if counter[random_label-1] == counter[-1]:
            continue

        else:
            new_entries.append(random_entry)
            counter[random_label-1] += 1

            if counter[0] == counter[1] and counter[1] == counter[2] and counter[2] == counter[3] and counter[3] == counter[4]:
                break

    print(counter)
    new_data.extend(new_entries)
    return new_data

def clean_dict(data):
    new_data = copy.deepcopy(data)
    for entry in new_data:
        del entry["_index"]
        del entry["_type"]
        del entry["_id"]
        del entry["_score"]
        del entry["_source"]["Review"]["ReviewTitle"]
        del entry["_source"]["Review"]["ReviewDate"]
        del entry["_source"]["Review"]["ReviewProductVerified"]
        del entry["_source"]["Product"]
    return new_data

if __name__ == "__main__":

    # data = readJson("../Dataset/Reviews/4Classes/train.json")
    # computeDatasetStatistics(data, 32, 32)

    # print("--------------------------DEV--------------------------")
    # data = readJson("../Dataset/Reviews/4Classes/dev.json")
    # computeDatasetStatistics(data, 32, 32)

    # print("--------------------------TEST--------------------------")
    # data = readJson("../Dataset/Reviews/4Classes/test.json")
    # computeDatasetStatistics(data, 32, 32)

    # compute_parameters("../Models/raw/small/clean/trained_512/ro2/")
    # sys.exit()

    # # split data
    # raw = readJson("../Dataset/Reviews/all_reviews.json", original=True)
    # # computeDatasetStatistics(raw, 256, 256)
    # train_data, dev_data, test_data = splitTrainDevTest(raw)
    # saveData(train_data, "../Dataset/Reviews/emag_train.json")
    # saveData(dev_data, "../Dataset/Reviews/emag_dev.json")
    # saveData(test_data, "../Dataset/Reviews/emag_test.json")

    # raw = readJson("../Dataset/Reviews/all_reviews.json", original=True)
    train_data = readJson("../Dataset/Reviews/emag_train.json")
    # computeDatasetStatistics(train_data, 256, 256)
    dev_data = readJson("../Dataset/Reviews/emag_dev.json")
    test_data = readJson("../Dataset/Reviews/emag_test.json")

    computeScoreHistogram(train_data, normalize=True)
    split_train = splitData4Classes(train_data)
    computeScoreHistogram(split_train, normalize=True)
    saveData(split_train, "../Dataset/Reviews/4Classes/train.json")

    computeScoreHistogram(dev_data, normalize=True)
    split_dev = splitData4Classes(dev_data)
    computeScoreHistogram(split_dev, normalize=True)
    saveData(split_dev, "../Dataset/Reviews/4Classes/dev.json")

    computeScoreHistogram(test_data, normalize=True)
    split_test = splitData4Classes(test_data)
    computeScoreHistogram(split_test, normalize=True)
    saveData(split_test, "../Dataset/Reviews/4Classes/test.json")

max_stars_repo_name: readerbench/ReaderBench

max_issues_repo_name: rwth-acis/readerbenchpy

max_forks_repo_name: rwth-acis/readerbenchpy


text:
#!/usr/bin/env python
import click
import os
import numpy as np
import matplotlib.pyplot as plt
from deepdiva.utils.visualisation_utils import *

@click.command()
@click.option('--data-path', 'data_path', default="./data/dataset", required=False,
              type=click.Path(exists=True), show_default=True, help='Path to data folder')
@click.option('--show-audio/--no-show-audio', 'show_audio', default=False,
              show_default=True, help='Whether to visualize the audio')
@click.option('--audio-file', 'audio_file', default="audio.npy",
              required=False, type=str, show_default=True, help='File name for audio dataset (.npy)')
@click.option('--show-spectrogram/--no-show-spectrogram', 'show_spectrogram', default=False,
              show_default=True, help='Whether to visualize the mel spectrogram')
@click.option('--spectrogram-file', 'spectrogram_file', default="melspectrogram.npy",
              required=False, type=str, show_default=True, help='File name for mel spectrogram dataset (.npy)')

def click_main(data_path, audio_file, spectrogram_file, show_spectrogram, show_audio):
    """
    Interface for Click CLI.
    """

    main(data_path=data_path, audio_file=audio_file, spectrogram_file=spectrogram_file,
         show_spectrogram=show_spectrogram, show_audio=show_audio)

def main(data_path, audio_file, spectrogram_file, show_spectrogram, show_audio):
    """Runs data visualisation scripts to inspect data"""

    if not show_audio and not show_spectrogram:
        print("Please select at least one type of plot: audio or mel spectrogram")

    if show_audio:
        # Load data file
        audio = np.load(os.path.join(data_path, audio_file))
        print(f"The shape of the audio file: {audio.shape}")

        # Plot 6 audio files
        fig1 = plt.figure(num="Audio signal", figsize=(12, 6))

        for sp in range(6):
            plt.subplot(2, 3, sp+1)
            sample = np.random.choice(audio.shape[0])
            plot_audio(audio[sample])
            plt.title(f"sample: {sample}")

        plt.show()
        plt.close()

    if show_spectrogram:
        # Load data file
        spectrogram = np.load(os.path.join(data_path, spectrogram_file))
        print(f"The shape of the mel spectrogram file: {spectrogram.shape}")

        # Plot 6 mel spectrograms
        fig2 = plt.figure(num="Mel spectrogram", figsize=(12, 6))

        for sp in range(6):
            plt.subplot(2, 3, sp+1)
            sample = np.random.choice(spectrogram.shape[0])
            plot_mel(spectrogram[sample])
            plt.title(f"sample: {sample}")

        plt.show()
        plt.close()

if __name__ == '__main__':

    click_main()

max_stars_repo_name: maudgrol/deepdiva

max_issues_repo_name: maudgrol/deepdiva

max_forks_repo_name: maudgrol/deepdiva


text:
import numpy as np
import scipy.optimize as spo
import scipy.integrate as spi
from scipy.integrate import odeint
import matplotlib.pyplot as plt

class property_plot:
    """
    This class must be inherited from a viscosity model class.
      It is not well written and cannot stand alone, which is a problem.
    """

    def __init__(self):
        self.rate_min=.001
        self.rate_max=10000.

    def visc_plot(self):
        """
        Method to create log-log plot of viscosity versus shear rate.
        50 pts are evenly spaced on log axis betwen rate_min and rate_max.
        This class expects to be inherited by a viscosity function class.
        """
        x = np.logspace(np.log10(.001),np.log10(10000.),51)
        y = list( map( lambda rate: self.calc_visc(rate), x))
        plt.loglog(x,y,'-')
        plt.xlabel('Shear rate')
        plt.ylabel('Viscosity')
        plt.title(self.name)

    def stress_plot(self):
        """
        Method to create log-log plot of shear stress versus shear rate.
        50 pts are evenly spaced on log axis betwen rate_min and rate_max.
        This class expects to be inherited by a viscosity function class.
        """
        x = np.logspace(np.log10(.001),np.log10(10000.),51)
        y = list( map( lambda rate: self.calc_visc(rate)*rate, x))
        plt.loglog(x,y,'-')
        plt.xlabel('Shear rate')
        plt.ylabel('Stress')
        plt.title(self.name)

class newtonian(property_plot):
    def __init__(self,name='Default',mu=1.):
        self.name = name
        self.mu = mu

    def __str__(self):
        return str(self.name+'\n'+
            'mu ='+str(self.mu)+'\n')

    def calc_visc(self,rate):
        return self.mu

class power_law(property_plot):
    def __init__(self,name='Default',k=1.,n=.5):
        self.name = name
        self.k = k
        self.n = n

    def __str__(self):
        return str(self.name+'\n'+
            'k ='+str(self.k)+'\n'+
            'n='+str(self.n)+'\n')

    def calc_visc(self,rate):
        return self.k*(rate+1.e-9)**(self.n-1.)

class carreau(property_plot):
    def __init__(self,name='Default',eta0=10.,etainf=.1,reltime=1.,a=2.,n=.5):
        self.name = name
        self.eta0 = eta0
        self.etainf = etainf
        self.reltime = reltime
        self.a = a
        self.n = n

    def __str__(self):
        return str(self.name+'\n'+
            'eta0 ='+str(self.eta0)+'\n'+
            'etainf ='+str(self.etainf)+'\n'+
            'reltime ='+str(self.reltime)+'\n'+
            'a ='+str(self.a)+'\n'+
            'n='+str(self.n)+'\n')

    def calc_visc(self,rate):
        return self.etainf + (self.eta0-self.etainf)/(1.0+(self.reltime*rate)**self.a)**((1.-self.n)/self.a)

class herschel_bulkley(property_plot):
    """
    Herschel-Bulkley viscosity model using Papanastasiou modification with m=1000.
    Also has eps=1.e-9 with shear rate in denominator of viscosity equation.
    """
    def __init__(self,name='Default',tauy=1.,k=1.,n=1.,m=1000.,m_flag=1):
        self.name = name
        self.tauy = tauy
        self.k = k
        self.n = n
        self.m=m
        self.m_flag=m_flag

    def __str__(self):
        return str(self.name+'\n'+
            'tauy ='+str(self.tauy)+'\n'+
            'k ='+str(self.k)+'\n'+
            'n='+str(self.n)+'\n'+
            'm=',str(self.m)+'\n'  )

    def calc_visc(self,rate):
        if self.m_flag==0.:
            return self.tauy/(rate+1.e-9) + self.k*(rate+1.e-9)**(self.n-1.)
        if self.m_flag==1:
            return (1.-np.exp(-self.m*rate))*self.tauy/(rate+1.e-9) + self.k*(rate+1.e-9)**(self.n-1.)

class three_component(property_plot):
    """
    Marco's 3-component viscosity model using Papanastasiou modification with m=1000.
    Also has eps=1.e-9 with shear rate in denominator of viscosity equation.
    """
    def __init__(self,name='Default',tauy=1.,gamma_crit=1.,eta_bg=1.,m=1000.,m_flag=1):
        self.name = name
        self.tauy = tauy
        self.gamma_crit = gamma_crit
        self.eta_bg = eta_bg
        self.m = m
        self.m_flag=m_flag

    def __str__(self):
        return str(self.name+'\n'+
            'tauy ='+str(self.tauy)+'\n'+
            'gamma_crit ='+str(self.gamma_crit)+'\n'+
            'eta_bg='+str(self.eta_bg)+'\n'+
            'm=',str(self.m)+'\n')

    def calc_visc(self,rate):
        if self.m_flag==0:
            return self.tauy/(rate+1.e-9) + self.tauy/(rate+1.e-9)*(rate/self.gamma_crit)**0.5 + (self.eta_bg)
        if self.m_flag==1:
            return (1.-np.exp(-self.m*rate))*self.tauy/(rate+1.e-9) + \
                self.tauy/(rate+1.e-9)*(rate/self.gamma_crit)**0.5 + \
                (self.eta_bg)

class bi_power_law(property_plot):
    def __init__(self,name='Default',k_low=1.,n_low=.9,k_high=1.,n_high=.5):
        self.name = name
        self.k_low = k_low
        self.k_high = k_high
        self.n_low = n_low
        self.n_high = n_high
        self.rate_switch = 10.**(np.log10(self.k_high/self.k_low)/(self.n_low-self.n_high))

    def __str__(self):
        return str(self.name+'\n'+
                  'k_low ='+str(self.k_low)+'\n'+
                   'k_high ='+str(self.k_high)+'\n'+
                   'n_low ='+str(self.n_low)+'\n'+
                   'n_high ='+str(self.n_high)+'\n'+'\n')

    def calc_visc(self,rate):
        eps = 1.e-9
        self.rate_switch = 10.**(np.log10(self.k_high/self.k_low)/(self.n_low-self.n_high))
        if (rate >= self.rate_switch):
            return self.k_high*(rate+eps)**(self.n_high-1.)
        else:
            return self.k_low*(rate+eps)**(self.n_low-1.)

max_stars_repo_name: shahsavari/rheoflow

max_issues_repo_name: shahsavari/rheoflow

max_forks_repo_name: shahsavari/rheoflow

reason: import numpy,import scipy,from scipy


text:
import pandas as pd
import sys, re, math, gzip
import numpy as np

cellLine = sys.argv[1]
doseResponse = sys.argv[2]
screenedComponents = sys.argv[3]
RACS = sys.argv[4]
variants = sys.argv[5]
expressionIn = sys.argv[6]
clinicalOut = sys.argv[7]
tmpExpression = sys.argv[8]
finalExpression = sys.argv[9]

def readSheetToMultipleDfs(xl,sheetname, indecisBetweenTables) :
    line = []
    previous = -1
    tmpDfsList = []
    for i in indecisBetweenTables :
        if i - previous > 2 :
            nrows = xl.book.sheet_by_name(sheetname).nrows
            df = xl.parse(sheetname, skiprows=previous+1, skip_footer = nrows-i-1).dropna(axis=1, how='all')
            tmpDfsList.append(df)
        previous = i
    return tmpDfsList

def readXlToListDf(file) :
    xl = pd.ExcelFile(file)
    dfsList = []
    for sheetname in xl.sheet_names :
        df = xl.parse(sheetname)
        if df.empty == True :
            continue
        elif any(math.isnan(s) for s in list(df[df.columns[0]]) if type(s) is float) :
            indecisBetweenTables = list(s for s in range(len(list(df[df.columns[0]]))) if type(list(df[df.columns[0]])[s]) is float)
            indecisBetweenTables.append(len(list(df[df.columns[0]])))
            tmpDfsList = readSheetToMultipleDfs(xl,sheetname,indecisBetweenTables)
            for df in tmpDfsList :
                dfsList.append(df)
        else :
            dfsList.append(df)
    return dfsList

def checkIfNan(value) :
    if type(value) == float:
        if math.isnan(value) == True:
            return True
    return False

print("making dataframes from excel")
cellLineDfs = readXlToListDf(cellLine) # 0.Cell line details 1.COSMIC tissue classification 2.TCGA tissue classification 3.Microsatillite instability data 4.Growth media
doseResponseDfs = readXlToListDf(doseResponse) # 0.fitted_dose_response
screenedComponentsDfs = readXlToListDf(screenedComponents) # 0.Screened_Compounds
RACSDfs = readXlToListDf(RACS) # 0.RACS
variantsDfs = readXlToListDf(variants) # 0.WES_variants 1.Legend

##The first three dictionaries are used to decode the MSI, Cancer type, and Screen Medium in the Cell Line file
##The fourth dictionary stores information from the DrugID in the screen compounds file to be used when making the dose response.
print("Making dictionaries")

TCGADict = {}
for i in range(len(cellLineDfs[2][cellLineDfs[2].columns[0]])) :
    TCGADict[cellLineDfs[2][cellLineDfs[2].columns[0]][i]] = cellLineDfs[2][cellLineDfs[2].columns[1]][i]

MIDDict = {}
for i in range(len(cellLineDfs[3][cellLineDfs[3].columns[0]])) :
    MIDDict[cellLineDfs[3][cellLineDfs[3].columns[0]][i]] = cellLineDfs[3][cellLineDfs[3].columns[1]][i]

GMDict = {}
for i in range(len(cellLineDfs[4][cellLineDfs[4].columns[0]])) :
    GMDict[cellLineDfs[4][cellLineDfs[4].columns[0]][i]] = cellLineDfs[4][cellLineDfs[4].columns[1]][i]

DrugIDDict = {}
for i in range(len(screenedComponentsDfs[0][screenedComponentsDfs[0].columns[0]])) :
    drugList = []
    if (type(screenedComponentsDfs[0][screenedComponentsDfs[0].columns[2]][i]) != float) :
        if screenedComponentsDfs[0][screenedComponentsDfs[0].columns[2]][i] == "-":
            if (len(screenedComponentsDfs[0][screenedComponentsDfs[0].columns[2]][i].split(", ")) > 1) :
                drugList = screenedComponentsDfs[0][screenedComponentsDfs[0].columns[2]][i].split(", ")
        else :
            drugList = screenedComponentsDfs[0][screenedComponentsDfs[0].columns[2]][i].split(", ")
    drugList.append(screenedComponentsDfs[0][screenedComponentsDfs[0].columns[1]][i])
    DrugIDDict[screenedComponentsDfs[0][screenedComponentsDfs[0].columns[0]][i]] = drugList

EnsembliIDDict = {}
numberOfCellLineRows = 0
for i in range(len(cellLineDfs[1][cellLineDfs[1].columns[1]])) :
    numberOfCellLineRows = numberOfCellLineRows + 1
    EnsembliIDDict[str(cellLineDfs[1][cellLineDfs[1].columns[1]][i])] = cellLineDfs[1][cellLineDfs[1].columns[0]][i]

##Write out Expression data to data.tsv.gz
headersNotConverted = []
headerValues = 0
indecisOfInterest = []
headersList = []
print("Writing Expression data to data.tsv.gz")
with gzip.open(tmpExpression, 'w') as oF :
    with gzip.open(expressionIn, 'r') as iF :
        headersList = iF.readline().decode().strip('\n').split('\t')

        first = True
        for i in range(len(headersList)) :
            if first == True:
                first = False
                indecisOfInterest.append(i)
            else :
                headerValues = headerValues + 1
                try :
                    headersList[i] = str(EnsembliIDDict[headersList[i]])
                    indecisOfInterest.append(i)
                except KeyError :
                    headersNotConverted.append(headersList[i])
        first = True
        firstOACp4C = True
        firstSKMEL28 = True
        firstKMH2 = True
        firstOCIAML5 = True
        for i in indecisOfInterest :
            if first == True :
                first = False
                oF.write(("Sample").encode())
            elif headersList[i] == "OACp4C" : #The original dataset has two variables titled OACp4C, this will label only the second in the header so we can average the values on each line and leave the value in this indeci
                if firstOACp4C == True :
                    firstOACp4C = False
                    continue
                else :
                    oF.write(("\t" + headersList[i]).encode())
            elif headersList[i] == "SK-MEL-28" : #The original dataset has two variables titled OACp4C, this will label only the second in the header so we can average the values on each line and leave the value in this indeci
                if firstSKMEL28 == True :
                    firstSKMEL28 = False
                    continue
                else :
                    oF.write(("\t" + headersList[i]).encode())
            elif headersList[i] == "KM-H2" : #The original dataset has two variables titled OACp4C, this will label only the second in the header so we can average the values on each line and leave the value in this indeci
                if firstKMH2 == True :
                    firstKMH2 = False
                    continue
                else :
                    oF.write(("\t" + headersList[i]).encode())
            elif headersList[i] == "OCI-AML5" : #The original dataset has two variables titled OACp4C, this will label only the second in the header so we can average the values on each line and leave the value in this indeci
                if firstOCIAML5 == True :
                    firstOCIAML5 = False
                    continue
                else :
                    oF.write(("\t" + headersList[i]).encode())
            else :
                oF.write(("\t" + headersList[i]).encode())
        oF.write(("\n").encode())

        j = 0
        for line in iF :
            j = j + 1
            lineList = line.decode().strip('\n').split('\t')
            first = True
            firstOACp4C = True
            firstValueOACp4C = 0
            firstSKMEL28 = True
            firstValueSKMEL28 = 0
            firstKMH2 = True
            firstValueKMH2 = 0
            firstOCIAML5 = True
            firstValueOCIAML5 = 0
            for i in indecisOfInterest :
                if first == True :
                    first = False
                    oF.write((lineList[i]).encode())
                elif headersList[i] == "OACp4C" : #The original dataset has two variables titled OACp4C, this will label only the second in the header so we can average the values on each line and leave the value in this indeci
                    if firstOACp4C == True :
                        firstOACp4C = False
                        firstValueOACp4C = float(lineList[i])
                        continue
                    else :
                        averagedValue = (firstValueOACp4C + float(lineList[i])) / 2
                        oF.write(("\t" + str(averagedValue)).encode())
                elif headersList[i] == "SK-MEL-28" : #The original dataset has two variables titled OACp4C, this will label only the second in the header so we can average the values on each line and leave the value in this indeci
                    if firstSKMEL28 == True :
                        firstSKMEL28 = False
                        firstValueSKMEL28 = float(lineList[i])
                        continue
                    else :
                        averagedValue = (firstValueSKMEL28 + float(lineList[i])) / 2
                        oF.write(("\t" + str(averagedValue)).encode())
                elif headersList[i] == "KM-H2" : #The original dataset has two variables titled OACp4C, this will label only the second in the header so we can average the values on each line and leave the value in this indeci
                    if firstKMH2 == True :
                        firstKMH2 = False
                        firstValueKMH2 = float(lineList[i])
                        continue
                    else :
                        averagedValue = (firstValueKMH2 + float(lineList[i])) / 2
                        oF.write(("\t" + str(averagedValue)).encode())
                elif headersList[i] == "OCI-AML5" : #The original dataset has two variables titled OACp4C, this will label only the second in the header so we can average the values on each line and leave the value in this indeci
                    if firstOCIAML5 == True :
                        firstOCIAML5 = False
                        firstValueOCIAML5 = float(lineList[i])
                        continue
                    else :
                        averagedValue = (firstValueOCIAML5 + float(lineList[i])) / 2
                        oF.write(("\t" + str(averagedValue)).encode())
                else :
                    oF.write(("\t" + str(lineList[i])).encode())
            oF.write(("\n").encode())

print("number of values in header: " + str(headerValues))
print("number of cellLineRows: " + str(numberOfCellLineRows))
print("Headers not converted: " + str(headersNotConverted))

noDetailsList = []
noDoseResposeList = []
noRACSList = []
noVariants = []
keyErrorSet = set()

print("Writing Metadata")
with gzip.open(clinicalOut, 'w') as oFile :
    oFile.write(("Sample\tVariable\tValue\n").encode())

    first = True
    for headerIndeci in indecisOfInterest :
        if first == True :
            first = False
            continue
        i = list(i for i in range(len(cellLineDfs[1][cellLineDfs[1].columns[0]])) if headersList[headerIndeci] == str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]))
        i = i[0]

        ##writing CellLineDetails sheet Cosmic tissue classification info
        oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + str(list(cellLineDfs[1].columns.values)[2]) + "\t" + str(cellLineDfs[1][cellLineDfs[1].columns[2]][i]) + "\n").encode())
        oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + str(list(cellLineDfs[1].columns.values)[3]) + "\t" + str(cellLineDfs[1][cellLineDfs[1].columns[3]][i]) + "\n").encode())

        ##writing CellLineDetails sheet Cell line details - 7 through 12 - map 9 - 11 on to tables in sheet Decode
        #Since they are in a different order, we need to find the index
        j = list(j for j in range(len(cellLineDfs[0][cellLineDfs[0].columns[1]]) - 1) if int(cellLineDfs[0][cellLineDfs[0].columns[1]][j]) == cellLineDfs[1][cellLineDfs[1].columns[1]][i])
        if len(j) > 0 :
            j = j[0]

            if(checkIfNan(cellLineDfs[0][cellLineDfs[0].columns[7]][j]) != True) :
                oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + str(list(cellLineDfs[0].columns.values)[7]).replace("\n"," ") + "\t" + str(cellLineDfs[0][cellLineDfs[0].columns[7]][j]) + "\n").encode())
            if(checkIfNan(cellLineDfs[0][cellLineDfs[0].columns[8]][j]) != True) :
                oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + str(list(cellLineDfs[0].columns.values)[8]).replace("\n"," ") + "\t" + str(cellLineDfs[0][cellLineDfs[0].columns[8]][j]) + "\n").encode())
            if(checkIfNan(cellLineDfs[0][cellLineDfs[0].columns[9]][j]) != True) :
                try :
                    oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + str(list(cellLineDfs[0].columns.values)[9]).replace("\n"," ") + "\t" + str(TCGADict[cellLineDfs[0][cellLineDfs[0].columns[9]][j]]) + "\n").encode())
                except KeyError :
                    if (cellLineDfs[0][cellLineDfs[0].columns[9]][j] == "COAD/READ") :
                        oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + str(list(cellLineDfs[0].columns.values)[9]).replace("\n"," ") + "\t" + "Colon adenocarcinoma and Rectum adenocarcinoma" + "\n").encode())
                    else :
                        keyErrorSet.add(cellLineDfs[0][cellLineDfs[0].columns[9]][j])
            if(checkIfNan(cellLineDfs[0][cellLineDfs[0].columns[10]][j]) != True) :
                tmpMIS = cellLineDfs[0][cellLineDfs[0].columns[10]][j].split('/')
                tmpMIS = list(MIDDict[i] for i in tmpMIS)
                tmpMIS = "/".join(tmpMIS)
                oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + str(list(cellLineDfs[0].columns.values)[10]).replace("\n"," ") + "\t" + str(tmpMIS) + "\n").encode())
            if(checkIfNan(cellLineDfs[0][cellLineDfs[0].columns[11]][j]) != True) :
                oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + str(list(cellLineDfs[0].columns.values)[11]).replace("\n"," ") + "\t" + str(GMDict[cellLineDfs[0][cellLineDfs[0].columns[11]][j]]) + "\n").encode())
            if(checkIfNan(cellLineDfs[0][cellLineDfs[0].columns[12]][j]) != True) :
                oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + str(list(cellLineDfs[0].columns.values)[12]).replace("\n"," ") + "\t" + str(cellLineDfs[0][cellLineDfs[0].columns[12]][j]) + "\n").encode())
        else :
            noDetailsList.append(i)

        ##Writing dose response
        j = list(j for j in range(len(doseResponseDfs[0][doseResponseDfs[0].columns[2]])) if doseResponseDfs[0][doseResponseDfs[0].columns[2]][j] == cellLineDfs[1][cellLineDfs[1].columns[1]][i])

        if(len(j) > 0) :
            for k in j :
                drugList = DrugIDDict[doseResponseDfs[0][doseResponseDfs[0].columns[3]][k]]

                for drug in drugList :
                    oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + "Drug_" + str(drug) + "_" + str(list(doseResponseDfs[0].columns.values)[4]) + "\t" + str(doseResponseDfs[0][doseResponseDfs[0].columns[4]][k]) + "\n").encode())
                    oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + "Drug_" + str(drug) + "_" + str(list(doseResponseDfs[0].columns.values)[5]) + "\t" + str(doseResponseDfs[0][doseResponseDfs[0].columns[5]][k]) + "\n").encode())
                    oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + "Drug_" + str(drug) + "_" + str(list(doseResponseDfs[0].columns.values)[6]) + "\t" + str(doseResponseDfs[0][doseResponseDfs[0].columns[6]][k]) + "\n").encode())
                    oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + "Drug_" + str(drug) + "_" + str(list(doseResponseDfs[0].columns.values)[7]) + "\t" + str(doseResponseDfs[0][doseResponseDfs[0].columns[7]][k]) + "\n").encode())
        else :
            noDoseResposeList.append(i)

        ##Writing Racs
        j = list(j for j in range(len(RACSDfs[0][RACSDfs[0].columns[1]])) if RACSDfs[0][RACSDfs[0].columns[1]][j] == cellLineDfs[1][cellLineDfs[1].columns[1]][i])

        if (len(j)) > 0 :
            for k in j :
                oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + "RACS_" + str(RACSDfs[0][RACSDfs[0].columns[5]][k]) + "\t" + str(RACSDfs[0][RACSDfs[0].columns[6]][k]) + "\n").encode())
        else :
            noRACSList.append(i)

        ##Writing WES_Variants
        j = list(j for j in range(len(variantsDfs[0][variantsDfs[0].columns[1]])) if variantsDfs[0][variantsDfs[0].columns[1]][j] == cellLineDfs[1][cellLineDfs[1].columns[1]][i])

        if len(j) > 0 :
            for k in j :
                oFile.write((str(cellLineDfs[1][cellLineDfs[1].columns[0]][i]) + "\t" + "Variant_" + str(variantsDfs[0][variantsDfs[0].columns[7]][k]) + "\t" +  str(variantsDfs[0][variantsDfs[0].columns[3]][k]) + "\n").encode())
        else :
            noVariants.append(i)

print("KeyError result: " + str(keyErrorSet))
print("Iterations no details: " + str(noDetailsList))
print("Iterations no racs info: " + str(noRACSList))
print("Iterations no variants info: " + str(noVariants))
print("Iterations no Dose Response: " + str(noDoseResposeList))

print("transpose the expressionFile")
with gzip.open(tmpExpression, 'r') as f :
    with gzip.open(finalExpression, 'w') as oF :
        data = np.genfromtxt(f,delimiter='\t',dtype=str)
        for line in data.T :
            stringLine = ("\t").join([str(element) for element in line]) + "\n"
            oF.write((stringLine).encode())

max_stars_repo_name: glenrs/WishbuilderData

max_issues_repo_name: glenrs/WishbuilderData

max_forks_repo_name: glenrs/WishbuilderData


text:
# Add the parent folder (outside of `examples`) to the path, so we can find the `arch` module
import sys, os
sys.path.append(os.path.dirname(sys.path[0]))

try:
    import colored_traceback.auto
except ImportError:
    pass

import abc
import arch.port as port
from arch.port import var
from arch.block import Block
from arch.connectivity import Connectivity
from arch.models import Model, SymbolicModel, NumericModel, SourceModel
from arch.blocks.optics import Beamsplitter, PhaseShifter, MachZehnder
from arch.blocks.sources import LaserCW
from arch.architecture import Architecture

if __name__ == '__main__':

	"""
	print ("Testing NumericModel composition")

	port_a  = var("a",  kind=port.kind.real, direction=port.direction.inp, default=0)
	port_bo = var("bo", kind=port.kind.real, direction=port.direction.out, default=0)
	port_bi = var("bi", kind=port.kind.real, direction=port.direction.inp, default=0)
	port_c  = var("c",  kind=port.kind.real, direction=port.direction.inp, default=0)
	port_d  = var("d",  kind=port.kind.real, direction=port.direction.out, default=0)

	# Funcs need to take a dict as input, keyed by port
	#  return a dict as output, keyed by port

	ports_bo = {port_a}
	def func_bo(port_dict):
		a = port_dict[port_a]
		return {port_bo: 2*a}

	mod_b = NumericModel("mod_b", ports={port_a, port_bo}, out_func=func_bo)

	ports_d = {port_bi, port_c}
	def func_d(port_dict):
		b = port_dict[port_bi]
		c = port_dict[port_c]
		return {port_d: b + c}

	mod_d = NumericModel("mod_d", ports={port_bi, port_c, port_d}, out_func=func_d)

	con = Connectivity( [(port_bo, port_bi)] )

	mod_comp = NumericModel.compound(name="comp", models={mod_b, mod_d}, connectivity=con)

	state = {port_a:1, port_c:0}
	print(mod_comp.out_func(state))
	print(state)
	"""

	print ("Welcome to the new arch")

	laser = LaserCW()
	mz0 = MachZehnder()
	mz1 = MachZehnder()#mz0.copy()
	ps0 = PhaseShifter()
	ps1 = PhaseShifter()
	bs = Beamsplitter()
	bs1 = Beamsplitter()

	laser.delay = 1
	mz0.delay = 1
	mz1.delay = 1
	ps0.delay = 1
	ps1.delay = 1
	bs.delay = 1

	config = 'complex'

	if config == 'simple':
		connections = Connectivity( [
						(ps0.out, ps1.inp)
						] )

	elif config == 'complex':

		connections = Connectivity( [
						(laser.out, mz0.in0),
						(mz0.out0, ps0.inp),
						(mz0.out1, ps1.inp),
						(ps0.out, bs.in0),
						(ps1.out, bs.in1),
						(bs.out0, mz1.in0),
						(bs.out1, mz1.in1),
						] )

	elif config == 'loops':

		connections = Connectivity( [
						(bs.out1, ps0.inp),
						(ps0.out, bs.in1),
						] )

	elif config == 'unbalanced delay':

		connections = Connectivity( [
						(bs.out0, ps0.inp),
						(ps0.out, bs1.in0),
						(bs.out1, bs1.in1),
						] )

	else:
		raise RuntimeError("Bad config")

	cm = bs.model.compound("compound name", connections.models, connections)

	print('\ncm is',cm)

	print('cm is',cm)
	print('\n out exprs',cm.out_exprs)
	print('out func', cm.out_func)
	print('ports',cm.ports)
	print('in ports',cm.in_ports)
	print('out ports',cm.out_ports)
	print('properties',cm.properties)
	if 'optical' in cm.properties:
		print('U',cm.U)
	if 'symbolic' in cm.properties:
		print('out exprs',cm.out_exprs)

	import time

	state = cm.default_input_state
	state[ps0.phi] = 3.14
	state[bs.R] = 0.01
	state[bs.in0] = 1.0
	print("default input state",state)

	eval_time = time.time()
	state = cm.out_func(state)
	eval_time = time.time() - eval_time

	print("default output state",state)

	print(f"Took {eval_time} s")

# 	oe = cm.out_exprs[mz1.out0]
# 	import IPython
# 	IPython.embed()
# 	quit()

	import networkx as nx

	def get_delay_map(connectivity, default_delay=0, block_delay_func=lambda b:b.delay):
		"""
		Build map between the external output ports of `connectivity` and its input ports
		mapping the delay along each path between those ports. Checks that all paths have
		matching delays.

		connectivity: `Connectivity` object describing configuration
		default_delay: value to be taken as default when no delay is present
		block_delay_func: function to compute delay from block value; exceptions use default
		"""

		def _integrate_path_delay(path):
			"""Add up delay attributes of blocks in list `path`"""

			total_delay = 0
			for block in path:
				try:
					delay = block_delay_func(block)
				except:
					delay = default_delay
				total_delay += delay

			return total_delay

		delay_map = dict()
		for op in connectivity.external_out_ports:
			delay_map |= {op:dict()}
			for ip in connectivity.external_in_ports:

				if ip.block is op.block:
					# If input port and output port are from the same block, no delay
					paths = [[ip.block]]
				else:
					# Find all paths between input and output blocks, integrate delays
					paths = list(nx.algorithms.simple_paths.all_simple_paths(
										connectivity.block_graph, ip.block, op.block))

				delays = {_integrate_path_delay(p) for p in paths}

				# Do checks
				if not len(delays):
					# No path exists
					delays = [default_delay]

				assert len(delays) == 1, f"Varying delays found between {ip} and {op}"

				# Update map
				delay_map[op] |= {ip:next(iter(delays))}

		return delay_map

	"""
	NOTE TO SELF:

	Need a simulator that simulates each internal variable at each time step.
	Simulator might gobble up subgraphs with synchronised time:
		1. examine when delay paths diverge
		2. chop back subgraph until that point
		3. compound
		4. repeat
	Compile all outputs as a function of all inputs
	Each frame:
		1. Compute all internal and external port values
		2. Handle delays
	"""

	connections.draw(draw_ports=True)

	dm = get_delay_map(connections)
	print(dm)
	print(cm)

	print(cm.in_ports)
	print(cm.out_ports)

# 	connections.draw()

	# Functions for producing time series
	def constant(v):
		return lambda t : v

	def step(v0, v1, t_step):
		return lambda t : v0 if t < t_step else v1

	def sinusoid(amp, offset, t_period, phase):
		from math import sin, pi
		return lambda t : (amp/2)*sin(2*pi*t/t_period + phase) + offset

	def ramp(v0, v1, t_period, phase):
		from math import pi
		return lambda t : (v1-v0)*(t%t_period)/t_period + v0

	from math import pi

	in_time_funcs = {
				laser.P: step(0,1.1,50),
				mz0.phi: constant(pi/2),
				mz1.phi: constant(pi/2),
				ps0.phi: ramp(0, pi, 50, 0),
				ps1.phi: constant(0)}

	states_ts = []

	state = cm.default_input_state

	import time

	t_setup = 0
	t_out_func = 0
	n_out_func_calls = 0
	t_close_copy = 0

	t_start = time.time()
	for t in range(200):

		for op in cm.out_ports:

			# Get delayed inputs
			t0 = time.time()
			for ip in cm.in_ports:
				if ip in in_time_funcs:
					state |= {ip:in_time_funcs[ip](t - dm[op][ip])}
			t_setup += time.time()-t0

			# Update output using delayed inputs
			t0 = time.time()
			state |= cm.out_func(state)
			t_out_func += time.time()-t0
			n_out_func_calls += 1

			# Update inputs to current time
			t0 = time.time()
			for ip in cm.in_ports:
				if ip in in_time_funcs:
					state |= {ip:in_time_funcs[ip](t)}

			states_ts.append((t,state.copy()))
			t_close_copy += time.time()-t0

	t_total = time.time() - t_start

	print("Spent time:")
	print(" t_setup =",t_setup)
	print(" t_out_func =",t_out_func,f" in {n_out_func_calls} calls")
	print(" t_close_copy =",t_close_copy)
	print(" t_total =",t_total)

	from matplotlib import pyplot

	pyplot.plot([s[0] for s in states_ts], [s[1][laser.P] for s in states_ts])
	pyplot.plot([s[0] for s in states_ts], [s[1][ps0.phi] for s in states_ts])
	pyplot.plot([s[0] for s in states_ts], [abs(s[1][mz1.out0]) for s in states_ts])
	pyplot.plot([s[0] for s in states_ts], [abs(s[1][mz1.out1]) for s in states_ts])
	pyplot.show()

	quit()

# 	from arch.block import AutoCompoundBlock
# 	acb = AutoCompoundBlock(blocks=[laser,mz0], connectivity=connections)
# 	print("ACB outs:",acb.model.out_exprs)

# 	acb = AutoCompoundBlock(blocks=[laser,mz0,ps0,ps1,mz1,bs], connectivity=connections)
# 	print("Can use favourite port labels:", mz0.in1 is acb.MZ0_in1)

# 	from math import pi
# 	out_dict = acb.model.out_func(in_port_values={mz0.in0:1, mz0.in1:0, mz1.phi:pi/2})

# 	acb.connectivity.draw(draw_ports=True)
# 	acb.connectivity.draw(draw_ports=False)

# 	arch = Architecture(connections, None)

# 	print([(sym,sym.data) for m in ps.models for sym in m.out_exprs[0].free_symbols])

max_stars_repo_name: bigphoton/arch

max_issues_repo_name: bigphoton/arch

max_forks_repo_name: bigphoton/arch

reason: import networkx


text:
from __future__ import print_function
import os
from datetime import datetime
import warnings
import numpy as np
from matplotlib.figure import Figure
import matplotlib.animation as animation
from pywt import wavedec, waverec, dwt_max_level
from tqdm import tqdm, trange
from .denoise import clean_frames, clean_frames_quick, clean_frames_quickest

# ------- Headline method -------

def clean_blocks(data_source, block_size, block_overlap=0, multiresolution=False, wavelet='db2', wave_levels=None,
                 skip_lowpass=False, **cleaner_kwargs):
    """
    Take blocks from a DataSource object and clean the frames by removing a subspace
    learned to represent the structured noise.
    """

    chan_map = data_source.channel_map
    data_source.write_parameters(block_size=block_size, multiresolution=multiresolution, wavelet=wavelet,
                                 wave_levels=wave_levels, **cleaner_kwargs)
    L = int(data_source.samp_rate * block_size)
    P = int(data_source.samp_rate * block_overlap)
    prev_sl = None
    prev_clean_block = None
    for block, sl in tqdm(data_source.iter_blocks(L, overlap=P, return_slice=True)):
        if multiresolution:
            b_coefs = wavedec(block, wavelet, axis=1, level=wave_levels)
            start_coefs = int(skip_lowpass)
            clean_coefs = [clean_frames_quickest(c, chan_map, **cleaner_kwargs) for c in b_coefs[start_coefs:]]
            if skip_lowpass:
                clean_coefs.insert(0, b_coefs[0])
            clean_block = waverec(clean_coefs, wavelet, axis=1)
        else:
            clean_block = clean_frames_quickest(block, chan_map, **cleaner_kwargs)
        # write current block
        data_source[sl] = clean_block
        # if overlapping, stich overlap region with the previous block
        if P > 0 and prev_sl:
            stitch_frames(data_source, prev_sl, sl, prev_clean_block, clean_block)
        prev_sl = sl
        prev_clean_block = clean_block

def stitch_frames(data_source, left_slice, right_slice, left_clean, right_clean):
    """Linearly interpolate between residual templates for the overlapped duration between slices."""
    P = left_slice.stop - right_slice.start
    # compute overlapping slices for the data source and the left/right blocks
    data_slice = slice(right_slice.start, left_slice.stop)
    L = left_slice.stop - left_slice.start
    left_overlap = (slice(None), slice(L - P, L))
    right_overlap = (slice(None), slice(0, P))

    raw_overlap = data_source[data_slice]
    left_resid = raw_overlap - left_clean[left_overlap]
    right_resid = raw_overlap - right_clean[right_overlap]

    # the left-block and right-block residuals will be crossed over linearly from
    # fully left to fully right
    right_mix = np.linspace(0, 1, P)
    data_source[data_slice] = raw_overlap - (1 - right_mix) * left_resid - right_mix * right_resid
    return

# ------- Diagnostics -------

def new_save_dir():
    """Create a new plots directory based on the current time"""
    save_dir = datetime.now().strftime('%y%m%d-%H%M%S')
    save_dir = os.path.join('diagnostic_plots', save_dir)
    n = 0
    while os.path.exists(save_dir + '_{}'.format(n)):
        print('adding to', save_dir)
        n += 1
    if n > 0:
        save_dir = save_dir + '_{}'.format(n)
        print('renamed', save_dir)
    os.makedirs(save_dir)
    return save_dir

def all_wavelets_diagnostics(data_source, block_size=0.5, wavelet='db2', **kwargs):
    L = int(block_size * data_source.samp_rate)
    max_level = dwt_max_level(L, wavelet)
    kwargs['multiresolution'] = True
    if kwargs.get('block', None) is None:
        start = np.random.randint(0, data_source.series_length - L)
        kwargs['block'] = start // L
    coefs = []
    save_dir = new_save_dir()
    for level in range(0, max_level + 1):
        cf = diagnostics(data_source,
                         block_size=block_size,
                         wavelet=wavelet,
                         wave_level=level,
                         return_cleaned=True,
                         save_dir=save_dir,
                         **kwargs)[2]
        coefs.append(cf)
    if kwargs.get('video', True):
        cleaned = waverec(coefs, wavelet, axis=1)
        start = kwargs['block'] * L
        raw = data_source[start:start + L]
        video_file = os.path.join(save_dir, 'error_image_frames_fullband.mp4')
        make_video(raw, cleaned, data_source.channel_map, video_file)

def diagnostics(data_source,
                block_size=0.5,
                block=None,
                multiresolution=False,
                wavelet='db2',
                wave_level=2,
                video=True,
                save_figs=True,
                return_cleaned=False,
                save_dir='',
                **cleaner_kwargs):
    if (save_figs or video) and not save_dir:
        print('need a savedir')
        save_dir = new_save_dir()
    chan_map = data_source.channel_map
    L = int(block_size * data_source.samp_rate)
    if block is None:
        start = np.random.randint(0, data_source.series_length - L)
    else:
        start = block * L
    block = data_source[start:start + L]
    if multiresolution:
        b_coefs = wavedec(block, wavelet, axis=1)
        clean_coefs = []
        # clean_coefs = [clean_frames(c, chan_map, **cleaner_kwargs) for c in b_coefs]
        # clean_block = waverec(clean_coefs, wavelet, axis=1)
        # hold out the 2nd level for plots?
        if wave_level >= len(b_coefs):
            print('wave level too high -- using highest available')
            wave_level = len(b_coefs) - 1
        for n in range(len(b_coefs)):
            if n == wave_level:
                # c, params = clean_frames(b_coefs[n], chan_map, return_diagnostics=True, **cleaner_kwargs)
                c, params = clean_frames_quickest(b_coefs[n], chan_map, return_diagnostics=True, **cleaner_kwargs)
                clean_coefs.append(c)
                continue
            # clean_coefs[n][:] = 0
            b_coefs[n][:] = 0
            clean_coefs.append(b_coefs[n].copy())
        clean_block = waverec(clean_coefs, wavelet, axis=1)
        block = waverec(b_coefs, wavelet, axis=1)
        ## f1, f2 = make_process_figures(raw_holdout, clean_holdout, chan_map, params)
        ## f1.savefig(os.path.join(save_dir, 'denoising_figure.pdf'))
        ## f2.savefig(os.path.join(save_dir, 'subspaces_figure.pdf'))
        ## if video:
        ##     make_video(raw_holdout, clean_holdout, chan_map, os.path.join(save_dir, 'error_image_frames.mp4'))
        img_tag = 'WL_{}'.format(wave_level)
        print('Model order:', params['model_order'], 'resid order:', params['resid_basis'].shape[1])
        # print 'Resid order:', params['resid_basis'].shape[1]
    else:
        # clean_block, params = clean_frames(block, chan_map, return_diagnostics=True, **cleaner_kwargs)
        clean_block, params = clean_frames_quickest(block, chan_map, return_diagnostics=True, **cleaner_kwargs)
        img_tag = 'full_band'
    cleaner_kwargs.pop('max_order', None)
    cleaner_kwargs.pop('use_local_regression', None)
    f1, f2 = make_process_figures(block, clean_block, chan_map, params, **cleaner_kwargs)
    if save_figs:
        f1.savefig(os.path.join(save_dir, 'denoising_figure_{}.pdf'.format(img_tag)))
        f2.savefig(os.path.join(save_dir, 'subspaces_figure_{}.pdf'.format(img_tag)))
    if video:
        make_video(block, clean_block, chan_map, os.path.join(save_dir, 'error_image_frames_{}.mp4'.format(img_tag)))
    if return_cleaned:
        cleaned = clean_coefs[wave_level] if multiresolution else clean_blocks
        return f1, f2, cleaned
    return f1, f2

def make_process_figures(raw, clean, channel_map, params, **kwargs):
    from .denoise import covar_model, error_image
    import matplotlib.pyplot as pp
    error = raw - clean
    c_model = {'theta': params['theta']}
    lam, Vm = covar_model(c_model, channel_map)

    # figure of low, median, and high error frames

    frame_var = np.argsort(np.var(raw, axis=0))
    n_frames = len(frame_var)

    f_frames, axs = pp.subplots(3, 3, figsize=(10, 10))
    row_titles = ['Raw frame', 'Clean frame', 'Error frame']
    clim = np.percentile(raw, [5, 95])
    for r, pt in enumerate([25, 50, 75]):
        i = frame_var[int(pt * n_frames / 100.0)]
        row_images = [raw[:, i], clean[:, i], error[:, i]]

        for n in range(3):
            channel_map.image(row_images[n], ax=axs[r, n], cbar=False, clim=clim)
            if r == 0:
                axs[0, n].set_title(row_titles[n])
    f_frames.tight_layout()

    # figure of model AR
    n_row = 3 if 'resid_basis' in params else 2
    f_AR, axs = pp.subplots(n_row, 3, figsize=(10, 10 / 3.0 * n_row))
    # row 1: variogram, eigenvec 1, eigenvec 2
    axs[0, 0].plot(params['xb'], params['yb'], marker='s', ls='--')
    axs[0, 0].set_xlabel('Site-site distance (mm)')
    axs[0, 0].set_ylabel('Semivariane (uV^2)')
    axs[0, 0].axhline(params['y_half'], color='gray')
    axs[0, 0].axvline(params['x_half'], color='gray')
    axs[0, 0].set_title('Length scale: {:0.1f} mm'.format(params['theta']))
    channel_map.image(Vm[:, -1], cbar=False, ax=axs[0, 1])
    axs[0, 1].set_title('Eigenvec 1')
    channel_map.image(Vm[:, -2], cbar=False, ax=axs[0, 2])
    axs[0, 2].set_title('Eigenvec 2')

    # row 2: local filter, AR image, error image
    i = frame_var[n_frames // 2]
    ## # AR_resid = error_image(raw[:, i:i+1], c_model, params['model_order'], channel_map, **kwargs).squeeze()
    ## AR_resid = error_image(raw[:, i:i+1], c_model, len(channel_map) - 1, channel_map, **kwargs).squeeze()
    ## AR_image = raw[:, i] - AR_resid
    ## # make a prediction filter for channel 3, 3
    ## i_pred = channel_map.lookup(3, 3)
    ## i_samp = np.setdiff1d(np.arange(len(channel_map)), i_pred)
    ## from denoise import _kriging_predictor
    ## # W = _kriging_predictor(Vm, lam, i_pred, i_samp, params['model_order'])
    ## W = _kriging_predictor(Vm, lam, i_pred, i_samp, len(channel_map) - 1)
    ## ## Vr = Vm[:, -params['model_order']:]
    ## ## C_inv = np.dot(Vr[i_samp], Vr[i_samp].T)
    ## ## C_xn = np.dot(Vr[i_samp], Vr[i_pred])
    ## ## W = np.dot(C_inv, C_xn)
    ## W = channel_map.as_channels(channel_map.subset(i_samp).embed(W))
    ## row_images = [W, AR_image, AR_resid]
    ## row_titles = ['AR filter', 'AR image', 'AR error']
    ## for n in range(3):
    ##     channel_map.image(row_images[n], ax=axs[1, n], cbar=False)
    ##     axs[1, n].set_title(row_titles[n])

    # row 2: raw_image, projected image, residual image
    Vr = Vm[:, -params['model_order']:]
    raw_frame = raw[:, i]
    lowpass = np.dot(Vr, np.dot(Vr.T, raw[:, i]))
    resid = raw_frame - lowpass
    row_images = [raw_frame, lowpass, resid]
    clim = np.percentile(raw_frame, [2, 98])
    row_titles = ['Raw', 'Lowpass {} modes'.format(params['model_order']), 'Resid']
    for n in range(3):
        channel_map.image(row_images[n], ax=axs[1, n], cbar=False, clim=clim)
        axs[1, n].set_title(row_titles[n])

    # row 3: eigenimage 1, 2, 3
    if n_row == 3:
        Vn = params['resid_basis']
        # row_images = [Vn[:, -1], Vn[:, -2], Vn[:, -3]]
        row_images = Vn[:, ::-1].T
        row_titles = ['Eigenvec 1 ({})', 'Eigenvec 2 ({})', 'Eigenvec 3 ({})']
        resid_order = Vn.shape[1]
        row_titles = [t.format(resid_order) for t in row_titles]
        for n in range(3):
            if n >= len(row_images):
                break
            channel_map.image(row_images[n], ax=axs[2, n], cbar=False)
            axs[2, n].set_title(row_titles[n])

    f_AR.tight_layout()

    return f_frames, f_AR

def plot_projections(frames, channel_map, model_var=0.95, deviation=0.5, bias=0,
                     multiresolution=False, wavelet='db2', wave_level=4, compress_range=False,
                     projected=True, f_idx=None):
    from .denoise import fast_semivariogram, covar_model, error_image, range_compression
    import matplotlib.pyplot as pp
    # 3x3 panels:
    # 1st row: raw, "correct" projection (# of eigenvectors), residual
    # 2nd-3rd row: raw, "misjudged" projection, residual

    if multiresolution:
        coefs = wavelet_bandpass(frames, wavelet, wave_level, return_coefs=True)
        if compress_range:
            cx_scale = range_compression(coefs[wave_level])
            coefs[wave_level] *= cx_scale[:, None]
        xb, yb = fast_semivariogram(coefs[wave_level], channel_map.site_combinations, xbin=0.5, trimmed=True)
        frames = waverec(coefs, wavelet, axis=1)
    else:
        if compress_range:
            cx_scale = range_compression(frames)
            frames *= cx_scale[:, None]
        xb, yb = fast_semivariogram(frames, channel_map.site_combinations, xbin=0.5, trimmed=True)

    yb = yb[xb < 0.7 * xb.max()]
    xb = xb[xb < 0.7 * xb.max()]
    y_half = 0.5 * np.percentile(yb, [10, 90]).sum()
    x_half = xb[np.where(yb > y_half)[0][0]]
    theta = x_half / np.log(2) + bias

    if not f_idx:
        f_idx = np.random.randint(0, frames.shape[1])
    F = frames[:, f_idx]
    clim = F.min(), F.max()
    n_rows = 4 if deviation > 0 else 2
    f, axs = pp.subplots(n_rows, 2, figsize=(7, 3 * n_rows))

    channel_map.image(F, cbar=False, clim=clim, ax=axs[0, 0])
    axs[0, 0].set_title('Raw frame')
    axs[0, 1].plot(xb, yb, marker='s', ls='--')
    axs[0, 1].axhline(y_half)
    axs[0, 1].axvline(x_half)
    axs[0, 1].set_title('Scale: {:.1f}'.format(theta))

    for row in range(1, n_rows):
        if row == 1:
            scale = 1.0
            theta_ = scale * theta
        elif row == 2:
            scale = 1 + deviation
            theta_ = scale * theta
        elif row == 3:
            scale = 1 - deviation
            theta_ = scale * theta
        lam, V = covar_model({'theta': theta_}, chan_map=channel_map)
        pct_var = np.cumsum(lam[::-1]) / np.sum(lam)
        if model_var >= pct_var.max():
            order = len(lam)
        else:
            order = np.where(pct_var > model_var)[0][0]
        if projected:
            Vr = V[:, -order:]
            smoothed = np.dot(Vr, np.dot(Vr.T, frames))
        else:
            resid = error_image(frames, {'theta': theta_}, order, chan_map=channel_map, radius=3.5, dist='inf')
            smoothed = frames - resid
        p_var = smoothed.var() / frames.var()
        images = [smoothed[:, f_idx], F - smoothed[:, f_idx]]
        for ax, vec, label in zip(axs[row], images, ('projected', 'residual')):
            if label == 'residual':
                channel_map.image(vec, cbar=False, ax=ax)
                ax.set_title(label + ' ({:.2f}x scale)'.format(scale))
            else:
                channel_map.image(vec, cbar=False, clim=clim, ax=ax)
                ax.set_title('{} dim '.format(order) + label + ' ({:.2f} var)'.format(p_var))
    f.tight_layout()
    return f

def wavelet_bandpass(frames, wavelet, wave_level, level=None, return_coefs=False):
    coefs = wavedec(frames, wavelet, level=level, axis=1)
    for n in range(len(coefs)):
        if n == wave_level:
            continue
        coefs[n][:] = 0
    if return_coefs:
        return coefs
    return waverec(coefs, wavelet, axis=1)

def make_video(raw, clean, channel_map, fname):
    error = raw - clean
    frames = np.concatenate([channel_map.embed(raw.T, axis=1),
                             channel_map.embed(clean.T, axis=1),
                             channel_map.embed(error.T, axis=1)], axis=2)
    clim = np.percentile(raw, [2, 98])
    write_frames(frames, fname, quicktime=True, origin='upper', clim=clim, figsize=(10, 3.5))

def setup_animated_frames(
        frames, timer='ms', time=(), static_title='', axis_toggle='on',
        figsize=None, colorbar=False, cbar_label='', cbar_orientation='vertical',
        figure_canvas=True,
        **imshow_kw
        ):
    if figure_canvas:
        from matplotlib.pyplot import figure
        f = figure(figsize=figsize)
    else:
        f = Figure(figsize=figsize)
    ax = f.add_subplot(111)
    im = ax.imshow(frames[0], **imshow_kw)
    ax.axis('image')
    ax.axis(axis_toggle)
    if isinstance(time, bool) and time:
        time = np.arange(len(frames))
        timer = 'samp'
    if len(time):
        ttl = ax.set_title('{0:.2f} {1}'.format(time[0], timer))
    elif static_title:
        ax.set_title(static_title)
        ttl = None
    else:
        ttl = None
    def _step_time(num, frames, frame_im):
        frame_im.set_data(frames[num])
        if ttl:
            ttl.set_text('{0:.2f} {1}'.format(time[num], timer))
            return (frame_im, ttl)
        return (frame_im,)
    func = lambda x: _step_time(x, frames, im)
    if colorbar:
        cb = f.colorbar(im, ax=ax, use_gridspec=True, orientation=cbar_orientation)
        cb.set_label(cbar_label)
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        f.tight_layout(pad=0.2)

    return f, func

def write_anim(
        fname, fig, func, n_frame,
        title='Array Movie', fps=5, quicktime=False, qtdpi=300
        ):

    FFMpegWriter = animation.writers['ffmpeg']
    metadata = dict(title=title, artist='ecoglib')
    writer = FFMpegWriter(
        fps=fps, metadata=metadata, codec='h264'
        )
    if quicktime:
        # do arguments that are quicktime compatible
        extra_args = ['-pix_fmt', 'yuv420p', '-qp', '1']
        # yuv420p looks a bit crappy, but upping the res helps
        dpi = qtdpi
    else:
        # yuv422p seems pretty good
        extra_args = ['-pix_fmt', 'yuv422p', '-qp', '0']
        dpi = fig.dpi
    writer.extra_args = extra_args
    fname = fname.split('.mp4')[0]
    with writer.saving(fig, fname+'.mp4', dpi):
        print('Writing {0} frames'.format(n_frame))
        for n in trange(n_frame):
            func(n)
            writer.grab_frame()

def write_frames(
        frames, fname, fps=5, quicktime=False, qtdpi=300,
        title='Array movie', **anim_kwargs
        ):

    f, func = setup_animated_frames(frames, figure_canvas=False, **anim_kwargs)
    write_anim(
        fname, f, func, frames.shape[0], fps=fps, title=title,
        quicktime=quicktime, qtdpi=qtdpi
        )

def animate_frames(frames, fps=5, blit=False, **anim_kwargs):
    f, func = setup_animated_frames(frames, figure_canvas=True, **anim_kwargs)
    anim = animation.FuncAnimation(
        f, func, frames=len(frames), interval=1000.0 / fps, blit=blit
        )
    return anim


text:
from __future__ import print_function

import keras
from keras.datasets import mnist
from keras.layers import *
from keras.models import Sequential, load_model
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.callbacks import ModelCheckpoint

import numpy as np
import os
import sys
nnscript = os.path.abspath('../../scripts')
sys.path.append(nnscript)

from nnom_utils import *

model_name = 'mnist_simple_trained_model.h5'
save_dir = os.path.join(os.getcwd(), 'saved_models')

def image_to_cfile(data, label, size, file='image.h'):
    # test
    with open(file, 'w') as f:
        num_of_image = size
        for i in range(num_of_image):
            selected = np.random.randint(0, 1000) # select 10 out of 1000.
            f.write('#define IMG%d {'% (i))
            np.round(data[selected]).flatten().tofile(f, sep=", ", format="%d") # convert 0~1 to 0~127
            f.write('} \n')
            f.write('#define IMG%d_LABLE'% (i))
            f.write(' %d \n \n' % label[selected])
        f.write('#define TOTAL_IMAGE %d \n \n'%(num_of_image))

        f.write('static const int8_t img[%d][%d] = {' % (num_of_image, data[0].flatten().shape[0]))
        f.write('IMG0')
        for i in range(num_of_image -1):
            f.write(',IMG%d'%(i+1))
        f.write('};\n\n')

        f.write('static const int8_t label[%d] = {' % (num_of_image))
        f.write('IMG0_LABLE')
        for i in range(num_of_image -1):
            f.write(',IMG%d_LABLE'%(i+1))
        f.write('};\n\n')

def octave_conv2d(xh, xl, ch=12):
    # one octave convolution is consist of the 2 equations
    # YH=f(XH;WH→H)+upsample(f(XL;WL→H),2)
    # YL=f(XL;WL→L)+f(pool(XH,2);WH→L))

    # f(XL;WL→L)
    xhh = Conv2D(ch, kernel_size=(3, 3), strides=(1, 1), padding='same')(xh)

    # f(XH;WH→H)
    xll = Conv2D(ch, kernel_size=(3, 3), strides=(1, 1), padding='same')(xl)

    # upsample(f(XL;WL→H),2)
    xlh = Conv2D(ch, kernel_size=(3, 3), strides=(1, 1), padding='same')(xl)
    xlh = UpSampling2D(size=(2, 2))(xlh)

    # f(pool(XH,2);WH→L))
    xhl = Conv2D(ch, kernel_size=(3, 3), strides=(1, 1), padding='same')(xh)
    xhl = MaxPool2D(pool_size=(2, 2), padding='same')(xhl)
    #xhl = AvgPool2D(pool_size=(2, 2), padding='same')(xhl)

    # yh = xhh + xlh
    # yl = xll + xhl
    yh = add([xhh, xlh])
    yl = add([xll, xhl])

    return yh, yl

def train(x_train, y_train, x_test, y_test, batch_size= 64, epochs = 100):

    inputs = Input(shape=x_train.shape[1:])
    x = Conv2D(12, kernel_size=(3, 3), strides=(1, 1), padding='same')(inputs)
    xh = ReLU()(x)
    xl = MaxPool2D((2,2),strides=(2,2), padding="same")(x)

    # octa 1
    xh, xl = octave_conv2d(xh, xl, 12)

    # max pool
    xh = MaxPool2D()(xh)
    xl = MaxPool2D()(xl)

    # octa 2
    xh, xl = octave_conv2d(xh, xl, 12)

    # reduce xh dimention to fit xl
    xh = MaxPool2D()(xh)

    x = concatenate([xh, xl], axis=-1)

    # reduce size
    x = Conv2D(12, kernel_size=(3, 3), strides=(1, 1), padding='valid')(x)

    x = Flatten()(x)
    x = Dense(96)(x)
    x = Dropout(0.2)(x)

    x = ReLU()(x)
    x = Dense(10)(x)
    predictions = Softmax()(x)

    model = Model(inputs=inputs, outputs=predictions)

    model.compile(loss='categorical_crossentropy',
                  optimizer='adadelta',
                  metrics=['accuracy'])

    model.summary()

    # Save model and weights
    if not os.path.isdir(save_dir):
        os.makedirs(save_dir)
    model_path = os.path.join(save_dir, model_name)

    # save best
    checkpoint = ModelCheckpoint(filepath=model_path,
            monitor='val_acc',
            verbose=0,
            save_best_only='True',
            mode='auto',
            period=1)
    callback_lists = [checkpoint]

    history =  model.fit(x_train, y_train,
              batch_size=batch_size,
              epochs=epochs,
              verbose=2,
              validation_data=(x_test, y_test),
              shuffle=True, callbacks=callback_lists)

    del model
    K.clear_session()

    return history

if __name__ == "__main__":

    # fixed the gpu error
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    session = tf.Session(config=config)

    epochs = 5
    num_classes = 10

    # The data, split between train and test sets:
    (x_train, y_train_num), (x_test, y_test_num) = mnist.load_data()

    print(x_train.shape[0], 'train samples')
    print(x_test.shape[0], 'test samples')

    # Convert class vectors to binary class matrices.
    y_train = keras.utils.to_categorical(y_train_num, num_classes)
    y_test = keras.utils.to_categorical(y_test_num, num_classes)

    # reshape to 4 d becaue we build for 4d?
    x_train = x_train.reshape(x_train.shape[0], x_train.shape[1], x_train.shape[2], 1)
    x_test = x_test.reshape(x_test.shape[0], x_test.shape[1], x_test.shape[2], 1)
    print('x_train shape:', x_train.shape)

    # quantize the range to 0~255 -> 0~1
    x_test = x_test/255
    x_train = x_train/255
    print("data range", x_test.min(), x_test.max())

    # select a few image and write them to image.h
    image_to_cfile(x_test*127, y_test_num, 10, file='image.h')

    # train model, save the best accuracy model
    history = train(x_train, y_train, x_test, y_test, batch_size=64, epochs=epochs)

    # reload best model
    model_path = os.path.join(save_dir, model_name)
    model = load_model(model_path)

    # evaluate
    evaluate_model(model, x_test, y_test)

    # save weight
    generate_model(model, np.vstack((x_train, x_test)), name="weights.h")

    # plot
    acc = history.history['acc']
    val_acc = history.history['val_acc']

    plt.plot(range(0, epochs), acc, color='red', label='Training acc')
    plt.plot(range(0, epochs), val_acc, color='green', label='Validation acc')
    plt.title('Training and validation accuracy')
    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.legend()
    plt.show()

max_stars_repo_name: YuehChuan/nnom

max_issues_repo_name: YuehChuan/nnom

max_forks_repo_name: YuehChuan/nnom


text:
import tensorflow as tf
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, GaussianNoise

kdBP = [0., 16., 35.]
kdV = [0.05, 0.935, 1.65]

kpV = [1.2, 0.8, 0.5]
kiV = [0.18, 0.12]

x = np.array([[1.2, 0.8, 0.5] for _ in range(5000)])
y = np.array([[0.05, 0.935, 1.65] for _ in range(5000)])

model = Sequential()
model.add(GaussianNoise(0.03, input_shape=(3,)))
model.add(Dense(12, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(3))

model.compile('adam', loss='mae', metrics='mse')
model.fit(x, y, epochs=1000, batch_size=16)

max_stars_repo_name: iamlukewang/openpilot

max_issues_repo_name: iamlukewang/openpilot

max_forks_repo_name: iamlukewang/openpilot