text:
import os, sys, re, json, random, copy, argparse, pickle, importlib
import numpy as np
import pandas as pd
from collections import OrderedDict
from tqdm import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
import logomaker as lm
from util import *
import warnings
warnings.filterwarnings('ignore')

class Predictor():
    def __init__(self, mhc_encode_dict, model_file, model_state_files, encoding_method):
        # MHC binding domain encoding
        self.mhc_encode_dict = mhc_encode_dict

        # device: gpu or cpu
        if torch.cuda.is_available():
            self.device = torch.device('cuda')
            self.batch_size = 4096
        else:
            self.device = torch.device('cpu')
            self.batch_size = 64

        # model
        if encoding_method == 'onehot':
            dim = 21
        elif encoding_method == 'blosum':
            dim = 24
        else:
            print("Wrong encoding method")
            raise ValueError
        model_file = '.'.join(model_file.split('.')[0].split('/'))
        module = importlib.import_module(model_file)
        self.model = module.CombineModel(module.MHCModel(dim), module.EpitopeModel(dim))

        # model states
        self.models = OrderedDict()
        for i in range(len(model_state_files)):
            basename = re.split(r'[\/\.]', model_state_files[i])[-2]
            model_state_dict = torch.load(model_state_files[i], map_location=self.device)
            self.models[basename] = copy.deepcopy(self.model)
            self.models[basename].load_state_dict(model_state_dict['model_state_dict'])
            self.models[basename].to(self.device)

    def __call__(self, df, dataset, allele=None):
        result_df = pd.DataFrame(index=df.index, columns=list(self.models.keys()))
        result_df['sequence'] = df['sequence']

        # general mode
        if allele:
            dataloader = torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, shuffle=False)
            preds = self._predict(allele, dataloader)
            result_df.loc[:, list(self.models.keys())] = preds

        # specific mode
        else:
            result_df['mhc'] = df['mhc']
            for allele, sub_df in tqdm(df.groupby('mhc'), desc='alleles', leave=False, position=0):
                idx = sub_df.index
                sub_dataset = torch.utils.data.Subset(dataset, idx)
                sub_dataloader = torch.utils.data.DataLoader(sub_dataset, batch_size=self.batch_size, shuffle=False)
                preds = self._predict(allele, sub_dataloader)
                result_df.loc[idx, list(self.models.keys())] = preds

        return result_df

    def _predict(self, allele, dataloader):
        mhc_encode = self.mhc_encode_dict[allele]
        df = pd.DataFrame()
        for key, model in tqdm(self.models.items(), desc='models', leave=False, position=1):
            for j, (x,y) in enumerate(tqdm(dataloader, desc='batches', leave=False, position=2)):
                with torch.no_grad():
                    model.eval()
                    num = x.shape[0]
                    epitope_encode = x.to(self.device).float()
                    mhc_encode_tile = torch.FloatTensor(np.tile(mhc_encode, (num, 1, 1))).to(self.device)
                    pred = model(mhc_encode_tile, epitope_encode).to('cpu')
                    pred = pred.view(-1,).numpy()
                    if j==0:
                        preds = pred
                    else:
                        preds = np.append(preds, pred, axis=0)
            df[key] = preds
        return df.values

class Interpretation():
    def __init__(self, interpretation_db, output_dir):
        self.aa_str = 'ACDEFGHIKLMNPQRSTVWY'
        self.sub_motif_len = 4
        self.dpi = 200
        self.fontsize = 8

        self.interp_dict = interpretation_db
        self.positions = self.interp_dict['important_positions']
        self.mhc_dict = self.interp_dict['seq']
        self.motif_dict = self.interp_dict['motif']
        self.output_dir = output_dir
        if not os.path.isdir(output_dir):
            os.mkdir(output_dir)

    def __call__(self, allele):
        hla = allele.split('*')[0]
        motif_df = pd.DataFrame(self.motif_dict[allele], columns=list(self.aa_str))
        allele_df = self._get_allele_seqlogo(allele)

        fig, ax = plt.subplots(2, 4, figsize=(10, 4), dpi=self.dpi, gridspec_kw={'width_ratios': [1, 4, 1, 4]})
        ax_y = 0

        for side in ['N', 'C']:
            # cluster
            cluster = self.interp_dict['cluster']['%s_%s'%(hla, side)][allele]

            # sub motif
            if side == 'N':
                sub_motif_df = motif_df.iloc[:4]
            else:
                sub_motif_df = motif_df.iloc[-4:].reset_index(drop=True)

            # sub-motif plot
            self._motif_plot(sub_motif_df, side, ax[1][ax_y])

            # check cluster
            if cluster not in self.interp_dict['hyper_motif']['%s_%s'%(hla, side)].keys():
                _ = ax[0][ax_y+1].set_title('%s-terminus: not well classified'%side, loc='left')
                _ = ax[1][ax_y+1].set_title('%s-terminus: allele information'%side, loc='left')
                print('%s-terminal of %s is not well classified'%(side, allele))
                continue

            # hyper-motif plot
            hyper_motif = self.interp_dict['hyper_motif']['%s_%s'%(hla, side)][cluster]
            hyper_motif = pd.DataFrame(hyper_motif, columns=list(self.aa_str))
            self._motif_plot(hyper_motif, side, ax[0][ax_y])

            # allele signature plot
            allele_signature = self.interp_dict['allele_signature']['%s_%s'%(hla, side)][cluster]
            allele_signature = pd.DataFrame(allele_signature, columns=list(self.aa_str))
            self._mhcseq_plot(allele_signature, ax[0][ax_y+1], title='%s-terminus: cluster information'%side)

            # highlighted allele signature plot
            allele_df[allele_df > 0] = 1
            allele_signature[allele_signature < 0] = 0
            self._mhcseq_plot(allele_df * allele_signature, ax[1][ax_y+1], title='%s-terminus: allele information'%side)

            ax_y += 2

        fig.tight_layout()
        fig.savefig('%s/%s%s%s.png'%(self.output_dir, hla, allele[2:4], allele[5:]))

    def _get_allele_seqlogo(self, allele):
        seq = self.mhc_dict[allele]
        seq = ''.join([seq[i] for i in self.positions])
        seqlogo_df = lm.alignment_to_matrix(sequences=[seq], to_type='counts')
        df = pd.DataFrame(columns=list(self.aa_str))
        return pd.concat([df, seqlogo_df], axis=0)[list(self.aa_str)].fillna(0)

    def _motif_plot(self, seqlogo_df, side, ax, ylim=4, title=None, turn_off_label=False):
        if side == 'N':
            xticklabels = list(range(1, self.sub_motif_len+1))
        else:
            xticklabels = list(range(-self.sub_motif_len, 0))
        logo = lm.Logo(seqlogo_df, color_scheme='skylign_protein', ax=ax)
        _ = ax.set_xticks(list(range(len(xticklabels))))
        _ = ax.set_xticklabels(xticklabels)
        _ = ax.set_ylim(0,ylim)
        _ = ax.set_title(title)
        for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] + ax.get_xticklabels() + ax.get_yticklabels()):
            item.set_fontsize(self.fontsize-2)

        _ = ax.set_yticks([])
        _ = ax.set_yticklabels([])

        if turn_off_label:
            _ = ax.set_xticks([])
            _ = ax.set_xticklabels([])
            _ = ax.set_title(None)

    def _mhcseq_plot(self, seqlogo_df, ax, ylim=1, title=None, turn_off_label=False):
        logo = lm.Logo(seqlogo_df, color_scheme='skylign_protein', ax=ax)
        _ = ax.set_ylim(0, ylim)
        _ = ax.set_xticks(range(len(self.positions)))
        _ = ax.set_xticklabels([i+1 for i in self.positions], rotation=90)
        _ = ax.set_title(title, loc='left')
        for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] + ax.get_xticklabels() + ax.get_yticklabels()):
            item.set_fontsize(self.fontsize-2)

        _ = ax.set_yticks([])
        _ = ax.set_yticklabels([])

        if turn_off_label:
            _ = ax.set_xticks([])
            _ = ax.set_xticklabels([])
            _ = ax.set_title(None)

def ArgumentParser():
    description = '''
    MHCfovea, an MHCI-peptide binding predictor. In this prediction process, GPU is recommended.

    Having two modes:
    1. specific mode: each peptide has its corresponding MHC-I allele in the input file; column "mhc" or "allele" is required
    2. general mode: all peptides are predicted with all alleles in the "alleles" argument

    Input file:
    only .csv file is acceptable
    column "sequence" or "peptide" is required as peptide sequences
    column "mhc" or "allele" is optional as MHC-I alleles

    Output directory contains:
    1. prediction.csv: with new column "score" for specific mode or [allele] for general mode
    2. interpretation: a directory contains interpretation figures of each allele
    3. metrics.json: all and allele-specific metrics (AUC, AUC0.1, AP, PPV); column "bind" as benchmark is required
    '''

    parser = argparse.ArgumentParser(prog='predictor', description=description,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('input', help='The input file')
    parser.add_argument('output_dir', help='The output directory')
    parser.add_argument('--alleles', required=False, default=None, help='alleles for general mode')
    parser.add_argument('--get_metrics', required=False, default=False, action='store_true', help='calculate the metrics between prediction and benchmark')

    return parser

def main(args=None):
    """""""""""""""""""""""""""""""""""""""""
    # Arguments
    """""""""""""""""""""""""""""""""""""""""
    args = ArgumentParser().parse_args(args)
    current_dir = os.path.abspath(os.getcwd())
    os.chdir(os.path.dirname(os.path.abspath(__file__)))

    # input
    if args.input.startswith('/'):
        peptide_dataframe = args.input
    else:
        peptide_dataframe = '%s/%s'%(current_dir, args.input)

    # output
    if args.output_dir.startswith('/'):
        output_dir = args.output_dir
    else:
        output_dir = '%s/%s'%(current_dir, args.output_dir)
    if not os.path.isdir(output_dir):
        os.mkdir(output_dir)

    # data
    rank_file = '../data/score_rank.csv'
    rank_df = pd.read_csv(rank_file, index_col=0)
    db_file = '../data/interpretation.pkl'
    db = pickle.load(open(db_file, 'rb'))
    mhc_dict = db['seq']
    alleles = args.alleles
    encoding_method = 'onehot'

    # model
    model_file = 'model.py'
    model_state_dir = 'model_state'

    # others
    get_metrics = args.get_metrics

    """""""""""""""""""""""""""""""""""""""""
    # Loading Data & Model
    """""""""""""""""""""""""""""""""""""""""
    print("Loading data and model...")

    # model state files
    model_state_files = list()
    for file in os.listdir(model_state_dir):
        model_state_files.append('%s/%s'%(model_state_dir, file))
    model_state_files.sort()

    # peptide dataframe
    df = pd.read_csv(peptide_dataframe)
    df = df.rename(columns={'peptide':'sequence', 'peptides':'sequence', 'sequences':'sequence',
                            'allele':'mhc', 'alleles':'mhc', 'binds':'bind'})
    ## check column "sequence"
    if 'sequence' not in df.columns:
        print('input file has no peptide sequence column; column name has to be "sequence" or "peptide"')
        raise ValueError
    cols = ['sequence']
    ## check column "mhc"
    if not args.alleles:
        if 'mhc' not in df.columns:
            print('input file has no MHC-I allele column; column name has to be "mhc" or "allele"')
            raise ValueError
        df['mhc'] = df['mhc'].apply(lambda x: ConvertAllele(x))
        cols.append('mhc')
        alleles = list(df['mhc'].unique())
    else:
        alleles = [ConvertAllele(x) for x in args.alleles.split(',')]
    ## check column "bind"
    if get_metrics:
        if 'bind' not in df.columns:
            print('input file has no benchmark column; column name has to be "bind"')
            raise ValueError
        cols.append('bind')
    df = df[cols]

    # peptide dataset
    dataset = BuildDataset(df, 'onehot', 15, with_label=get_metrics)

    # mhc encoding dict
    mhc_encode_dict = dict()
    for allele in alleles:
        mhc_encode_dict[allele] = OneHotEncoder(mhc_dict[allele], 182, True)

    """""""""""""""""""""""""""""""""""""""""
    # Prediction & Interpretation
    """""""""""""""""""""""""""""""""""""""""
    print("Predicting...")

    # predictor
    Pred = Predictor(mhc_encode_dict, model_file, model_state_files, encoding_method)

    # interpretation
    Interp = Interpretation(db, '%s/interpretation'%output_dir)

    # seqlogo dict
    seqlogo_dict = dict()

    # general mode
    if args.alleles:
        metrics_dict = dict()
        for allele in tqdm(alleles, desc='alleles', leave=False, position=0):
            pred_df = Pred(df, dataset, allele=allele)
            df[allele] = pred_df[list(Pred.models.keys())].mean(axis=1).round(3)

            # %rank
            df['{}_%rank'.format(allele)] = AssignRank(rank_df, allele, df[allele])

            # interpretation
            Interp(allele)

            # metrics
            if get_metrics:
                metrics_dict[allele] = CalculateMetrics(df['bind'], df[allele])

    # specific mode
    else:
        pred_df = Pred(df, dataset)
        df['score'] = pred_df[list(Pred.models.keys())].mean(axis=1).round(3)

        # %rank & interpretation
        df['%rank'] = 0
        for allele, sub_df in df.groupby('mhc'):
            df.loc[sub_df.index, '%rank'] = AssignRank(rank_df, allele, sub_df['score'])
            Interp(allele)

        # metrics
        if get_metrics:
            all_metrics = CalculateMetrics(df['bind'], df['score'])
            allele_metrics = CalculateAlleleMetrics(df['mhc'], df['bind'], df['score'])
            metrics_dict = {'all': all_metrics, **allele_metrics}

    """""""""""""""""""""""""""""""""""""""""
    # Save result and record
    """""""""""""""""""""""""""""""""""""""""
    # result
    df.to_csv('%s/prediction.csv'%output_dir, index=False)
    ##np.save('%s/motif.npy'%output_dir, seqlogo_dict)
    if get_metrics:
        json.dump(metrics_dict, open('%s/metrics.json'%output_dir, 'w'))

    print('Done')

if __name__=="__main__":
    main()


text:
import numpy as np
import recurrentshop
from recurrentshop.cells import *
from recurrentshop import LSTMCell, RecurrentSequential
from keras.models import Model
from keras.layers import Input, Dense, Lambda, Activation
from keras.layers import add, multiply
from keras.layers import Dense, Dropout, TimeDistributed, Bidirectional, Input

class LSTMDecoderCell(ExtendedRNNCell):

    def __init__(self, hidden_dim=None, **kwargs):
        if hidden_dim:
            self.hidden_dim = hidden_dim
        else:
            self.hidden_dim = self.output_dim
        super(LSTMDecoderCell, self).__init__(**kwargs)

    def build_model(self, input_shape):
        hidden_dim = self.hidden_dim
        output_dim = self.output_dim

        x = Input(batch_shape=input_shape)
        h_tm1 = Input(batch_shape=(input_shape[0], hidden_dim))
        c_tm1 = Input(batch_shape=(input_shape[0], hidden_dim))

        W1 = Dense(hidden_dim * 4,
                   kernel_initializer=self.kernel_initializer,
                   kernel_regularizer=self.kernel_regularizer,
                   use_bias=False)
        W2 = Dense(output_dim,
                   kernel_initializer=self.kernel_initializer,
                   kernel_regularizer=self.kernel_regularizer,)
        U = Dense(hidden_dim * 4,
                  kernel_initializer=self.kernel_initializer,
                  kernel_regularizer=self.kernel_regularizer,)

        z = add([W1(x), U(h_tm1)])

        z0, z1, z2, z3 = get_slices(z, 4)
        i = Activation(self.recurrent_activation)(z0)
        f = Activation(self.recurrent_activation)(z1)
        c = add([multiply([f, c_tm1]), multiply([i, Activation(self.activation)(z2)])])
        o = Activation(self.recurrent_activation)(z3)
        h = multiply([o, Activation(self.activation)(c)])
        y = Activation(self.activation)(W2(h))

        return Model([x, h_tm1, c_tm1], [y, h, c])

def Seq2Seq(output_dim, output_length, batch_input_shape=None,
            input_shape=None, batch_size=None, input_dim=None, input_length=None,
            hidden_dim=None, depth=1, broadcast_state=True, unroll=False,
            stateful=False, inner_broadcast_state=True, teacher_force=False,
            peek=False, dropout=0.):

    '''
    Seq2seq model based on [1] and [2].
    This model has the ability to transfer the encoder hidden state to the decoder's
    hidden state(specified by the broadcast_state argument). Also, in deep models
    (depth > 1), the hidden state is propogated throughout the LSTM stack(specified by
    the inner_broadcast_state argument. You can switch between [1] based model and [2]
    based model using the peek argument.(peek = True for [2], peek = False for [1]).
    When peek = True, the decoder gets a 'peek' at the context vector at every timestep.
    [1] based model:
            Encoder:
            X = Input sequence
            C = LSTM(X); The context vector
            Decoder:
    y(t) = LSTM(s(t-1), y(t-1)); Where s is the hidden state of the LSTM (h and c)
    y(0) = LSTM(s0, C); C is the context vector from the encoder.
    [2] based model:
            Encoder:
            X = Input sequence
            C = LSTM(X); The context vector
            Decoder:
    y(t) = LSTM(s(t-1), y(t-1), C)
    y(0) = LSTM(s0, C, C)
    Where s is the hidden state of the LSTM (h and c), and C is the context vector
    from the encoder.
    Arguments:
    output_dim : Required output dimension.
    hidden_dim : The dimension of the internal representations of the model.
    output_length : Length of the required output sequence.
    depth : Used to create a deep Seq2seq model. For example, if depth = 3,
                    there will be 3 LSTMs on the enoding side and 3 LSTMs on the
                    decoding side. You can also specify depth as a tuple. For example,
                    if depth = (4, 5), 4 LSTMs will be added to the encoding side and
                    5 LSTMs will be added to the decoding side.
    broadcast_state : Specifies whether the hidden state from encoder should be
                                      transfered to the deocder.
    inner_broadcast_state : Specifies whether hidden states should be propogated
                                                    throughout the LSTM stack in deep models.
    peek : Specifies if the decoder should be able to peek at the context vector
               at every timestep.
    dropout : Dropout probability in between layers.
    '''

    if isinstance(depth, int):
        depth = (depth, depth)
    if batch_input_shape:
        shape = batch_input_shape
    elif input_shape:
        shape = (batch_size,) + input_shape
    elif input_dim:
        if input_length:
            shape = (batch_size,) + (input_length,) + (input_dim,)
        else:
            shape = (batch_size,) + (None,) + (input_dim,)
    else:
        # TODO Proper error message
        raise TypeError
    if hidden_dim is None:
        hidden_dim = output_dim

    encoder = RecurrentSequential(readout=True, state_sync=inner_broadcast_state,
                                  unroll=unroll, stateful=stateful,
                                  return_states=broadcast_state)
    for _ in range(depth[0]):
        encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], hidden_dim)))
        encoder.add(Dropout(dropout))

    dense1 = TimeDistributed(Dense(hidden_dim))
    dense1.supports_masking = True
    dense2 = Dense(output_dim)

    decoder = RecurrentSequential(readout='add' if peek else 'readout_only',
                                  state_sync=inner_broadcast_state, decode=True,
                                  output_length=output_length, unroll=unroll,
                                  stateful=stateful, teacher_force=teacher_force)

    for _ in range(depth[1]):
        decoder.add(Dropout(dropout, batch_input_shape=(shape[0], output_dim)))
        decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim,
                                    batch_input_shape=(shape[0], output_dim)))

    _input = Input(batch_shape=shape)
    _input._keras_history[0].supports_masking = True
    encoded_seq = dense1(_input)
    encoded_seq = encoder(encoded_seq)
    if broadcast_state:
        assert type(encoded_seq) is list
        states = encoded_seq[-2:]
        encoded_seq = encoded_seq[0]
    else:
        states = None
    encoded_seq = dense2(encoded_seq)
    inputs = [_input]
    if teacher_force:
        truth_tensor = Input(batch_shape=(shape[0], output_length, output_dim))
        truth_tensor._keras_history[0].supports_masking = True
        inputs += [truth_tensor]

    decoded_seq = decoder(encoded_seq,
                          ground_truth=inputs[1] if teacher_force else None,
                          initial_readout=encoded_seq, initial_state=states)

    model = Model(inputs, decoded_seq)
    model.encoder = encoder
    model.decoder = decoder
    return model

# input_length = 5
# input_dim = 3

# output_length = 3
# output_dim = 4

# samples = 100
# hidden_dim = 24

# x = np.random.random((samples, input_length, input_dim))
# y = np.random.random((samples, output_length, output_dim))
# print(x.shape)
# print(y.shape)
# print([x,y])
# print(y)
# model = Seq2Seq(output_dim=output_dim, hidden_dim=hidden_dim, output_length=output_length, input_shape=(input_length, input_dim), peek=True, depth=2, teacher_force=True)
# model.compile(loss='mse', optimizer='rmsprop')
# model.fit([x, y], y, epochs=1)

ext: py

max_stars_repo_name: changkun/MasterThesisHCI

max_issues_repo_name: changkun/MasterThesisHCI

max_forks_repo_name: changkun/MasterThesisHCI

reason: import numpy


text:
import numpy as np
import torch
import os
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils
import pandas as pd
import ast
import json

from PIL import Image
from munch import munchify

import time
import random

def TransformImage(img):

    transform_list = []
    mean = [0.43216, 0.394666, 0.37645]
    std = [0.22803, 0.22145, 0.216989]

    transform_list.append(transforms.Resize([224,224]))
    transform_list.append(transforms.ToTensor())
    transform_list.append(transforms.Normalize(mean, std))
    trans = transforms.Compose(transform_list)
    frame_tensor = trans(img)

    return frame_tensor

def load_frame_info(img_path):

    # img_info = os.path.join(img_path, img_file)
    # img = Image.open(img_info).convert('RGB')
    img = Image.open(img_path).convert('RGB')
    frame_tensor = TransformImage(img)

    return frame_tensor

def image_info(video_name):

    path = "./data/frames/"
    img_path = os.path.join(path, video_name)

    img_list = os.listdir(img_path)
    img_list.sort()

    select_img = []
    for frame_idx in range(0,len(img_list),1):
        if frame_idx < 60:
            video_frames_path = os.path.join(img_path, str(frame_idx+1).zfill(6)+".jpg")

            frame_tensor_info = load_frame_info(video_frames_path)
            select_img.append(frame_tensor_info.cpu().numpy())

    select_img = np.array(select_img)

    return select_img

def ids_to_multinomial(id, categories):
    """ label encoding
    Returns:
      1d array, multimonial representation, e.g. [1,0,1,0,0,...]
    """
    id_to_idx = {id: index for index, id in enumerate(categories)}

    return id_to_idx[id]

class AVQA_dataset(Dataset):

    def __init__(self, label, audio_dir, video_dir, st_dir, transform=None):

        samples = json.load(open('../data/json/avqa-train.json', 'r'))

        # nax =  nne
        ques_vocab = ['<pad>']
        ans_vocab = []
        i = 0
        for sample in samples:
            i += 1
            # print(sample, sample['video_id'], sample['question_content'], len(ast.literal_eval(sample['templ_values'])))
            question = sample['question_content'].rstrip().split(' ')
            question[-1] = question[-1][:-1]

            p = 0
            for pos in range(len(question)):
                if '<' in question[pos]:
                    question[pos] = ast.literal_eval(sample['templ_values'])[p]
                    p += 1

            for wd in question:
                if wd not in ques_vocab:
                    ques_vocab.append(wd)
            if sample['anser'] not in ans_vocab:
                ans_vocab.append(sample['anser'])

        self.ques_vocab = ques_vocab
        self.ans_vocab = ans_vocab
        self.word_to_ix = {word: i for i, word in enumerate(self.ques_vocab)}

        self.samples = json.load(open(label, 'r'))
        self.max_len = 14

        self.audio_dir = audio_dir
        self.video_dir = video_dir
        self.st_dir = st_dir
        self.transform = transform

        video_list = []
        for sample in self.samples:
            video_name = sample['video_id']
            if video_name not in video_list:
                video_list.append(video_name)

        self.video_list = video_list
        self.video_len = 60 * len(video_list)

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, idx):
        sample = self.samples[idx]

        name = sample['video_id']
        audio = np.load(os.path.join(self.audio_dir, name + '.npy'))
        audio=audio[::6,:]

        visual_out_res18_path='./data/feats/res18_14x14'
        visual_posi=np.load(os.path.join(visual_out_res18_path, name + '.npy'))

        # visual_posi = torch.from_numpy(visual_posi)
        visual_posi=visual_posi[::6,:]
        # visual_posi 60 512 14 14

        video_idx=self.video_list.index(name)

        # visual_nega=None
        # print("visual_posi: ", visual_posi.shape)   # (10, 512, 14, 14)
        T,C,H,W = torch.from_numpy(visual_posi).size()
        visual_nega=torch.zeros(T,C,H,W)

        # for i in range(visual_posi.shape[0]):
        #     while(1):
        #         neg_frame_id = random.randint(0, self.video_len - 1)
        #         if (int(neg_frame_id/60) != video_idx):
        #             # print(" Successfully! ")
        #             break
        #     neg_video_id = int(neg_frame_id / 60)
        #     neg_frame_flag = neg_frame_id % 60
        #     neg_video_name = self.video_list[neg_video_id]
        #     visual_nega_out_res18=np.load(os.path.join(visual_out_res18_path, neg_video_name + '.npy'))

        #     visual_nega_out_res18 = torch.from_numpy(visual_nega_out_res18)
        #     visual_nega_clip=visual_nega_out_res18[neg_frame_flag,:,:,:].unsqueeze(0)

        #     if(i==0):
        #         visual_nega=visual_nega_clip
        #     else:
        #         visual_nega=torch.cat((visual_nega,visual_nega_clip),dim=0)

        ### visual nega 60,512,14,14

        # question_
        question_id = sample['question_id']
        question = sample['question_content'].rstrip().split(' ')
        question[-1] = question[-1][:-1]

        p = 0
        for pos in range(len(question)):
            if '<' in question[pos]:
                question[pos] = ast.literal_eval(sample['templ_values'])[p]
                p += 1
        if len(question) < self.max_len:
            n = self.max_len - len(question)
            for i in range(n):
                question.append('<pad>')
        idxs = [self.word_to_ix[w] for w in question]
        ques = torch.tensor(idxs, dtype=torch.long)

        # answer
        answer = sample['anser']
        label = ids_to_multinomial(answer, self.ans_vocab)
        label = torch.from_numpy(np.array(label)).long()

        sample = {'audio': audio, 'visual_posi': visual_posi, 'visual_nega': visual_nega, 'question': ques, 'label': label, 'question_id':question_id}

        if self.transform:
            sample = self.transform(sample)

        return sample

class ToTensor(object):

    def __call__(self, sample):

        audio = sample['audio']
        visual_posi = sample['visual_posi']
        visual_nega = sample['visual_nega']
        label = sample['label']
        question_id=sample['question_id']

        return {
                'audio': torch.from_numpy(audio),
                'visual_posi': sample['visual_posi'],
                'visual_nega': sample['visual_nega'],
                'question': sample['question'],
                'label': label,
                'question_id':question_id}

max_issues_repo_name: GeWu-Lab/MUSIC-AVQA

max_forks_repo_name: GeWu-Lab/MUSIC-AVQA


text:
from numpy import ndarray
from typing import List, Tuple, Union

GroupPairs = List[Tuple[List[str], List[str]]]
Bins = Union[int, ndarray, range, list]
StringOrStringTuple = Union[str, Tuple[str]]

max_stars_repo_name: vahndi/quant-survey

max_issues_repo_name: vahndi/quant-survey

max_forks_repo_name: vahndi/quant-survey

reason: from numpy


text:
def areaPL(mirror, blkFlag=True, printFigs=False):
    """Plot area Powergen and Load"""
    import matplotlib.pyplot as plt
    import numpy as np
    # custom color cycler
    from cycler import cycler
    ltdColors=[ [0,0,0], # black
            [.7,.7,.7], # grey
            [0,1,0], # green
            [1,0,1], # magenta
            "#17becf", # light blue
            [1,.647,0],# orange
        ]
    default_cycler = (cycler(color=ltdColors))
    plt.rc('axes', prop_cycle=default_cycler)

    mir = mirror
    caseName = mir.simParams['fileName'][:-1]
    mins = np.array(mir.r_t)/60.0;
    xend = max(mins)
    mini = 1

    fig, ax = plt.subplots()

    ax.set_title('Area Generation and Demand\n Case: ' + caseName)

    for area in mir.Area:
        ax.plot(mins, area.r_Pe,
                    linewidth = 2,
                    label = r'$P_e$ Area '+ str(area.Area)
                    )
        ax.plot(mins, area.r_P,
                    linewidth = 2,
                    linestyle = '--',
                    label = r'$P_{load}$ Area '+ str(area.Area)
                    )

    ax.set_ylabel('MW')
    ax.set_xlabel('Time [minutes]')
    ax.legend(loc=1)
    ax.set_xlim(0,xend)
    ax.grid(True)
    fig.set_dpi(150)
    fig.set_size_inches(9, 2.5)
    fig.tight_layout()
    plt.show(block=blkFlag)
    if printFigs: plt.savefig(caseName+'areaPL'+'.pdf', dpi=300)
    plt.pause(0.00001) # required for true non-blocking print...

max_stars_repo_name: thadhaines/PSLTDSim

max_issues_repo_name: thadhaines/PSLTDSim

max_forks_repo_name: thadhaines/PSLTDSim


text:
# -*- coding: utf-8 -*-
"""
Created on 19-4-26 下午4:56
IDE PyCharm

@author: Meng Dong
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import sys
import argparse
import os
import numpy as np
from skimage import io
from libtiff import TIFF
from glob import glob
import pprint

def add_path(path):
    if path not in sys.path:
        sys.path.insert(0, path)

this_dir = os.path.dirname(__file__)

lib_path = os.path.join(this_dir, 'lib')
add_path(lib_path)

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
from torchvision import transforms as standard_transforms
from torch.utils.data import DataLoader

from tools import transforms as own_transforms
from prm.peak_response_mapping_3d import PeakResponseMapping_3d
from models.CrowdCounter import CrowdCounter
from utils.modelsummary import get_model_summary
from config import config
from config import update_config
from utils.utils import AverageMeter, create_logger
from datasets import macaque_h5

def parse_args():
    parser = argparse.ArgumentParser(description='Train segmentation network')

    parser.add_argument('--cfg',
                        help='experiment configure file name',
                        required=True,
                        type=str)
    parser.add_argument('opts',
                        help="Modify config options using the command-line",
                        default=None,
                        nargs=argparse.REMAINDER)

    args = parser.parse_args()
    update_config(config, args)

    return args

torch.cuda.set_device(0)
torch.backends.cudnn.benchmark = True

def main():
    args = parse_args()

    logger, final_output_dir = create_logger(config, args.cfg, 'test')

    logger.info(pprint.pformat(args))
    logger.info(pprint.pformat(config))

    # cudnn related setting
    cudnn.benchmark = config.CUDNN.BENCHMARK
    cudnn.deterministic = config.CUDNN.DETERMINISTIC
    cudnn.enabled = config.CUDNN.ENABLED

    gpus = list(config.GPUS)
    if len(gpus) == 1:
        torch.cuda.set_device(config.GPUS[0])

    CC = CrowdCounter(config).cuda()
    net = CC
    net = net.eval()
    for i in range(1):
        out_path = '/media/dongmeng/Data/Code/cell-count-pt/instance_segmentation_macaque_bithub_ours/exp/03101605/best'
        #
        model_file = '/media/dongmeng/Data/Code/cell-count-pt/instance_segmentation_macaque_bithub_ours/exp/03101605/best.pth'
            #
        if not os.path.exists(out_path):
            os.makedirs(out_path)
        #if not os.path.exists(config.TEST.FINAL_OUTPUT_PATH):
        #    os.makedirs(config.TEST.FINAL_OUTPUT_PATH)

        if config.TEST.MODEL_FILE:
            model_state_file = config.TEST.MODEL_FILE
        else:
            #model_state_file = os.path.join(final_output_dir,
            #                                'final_state.pth')
            model_state_file = model_file
        logger.info('=> loading model from {}'.format(model_state_file))

        check_point = torch.load(model_state_file)
        net_dict = net.state_dict()
        pretrained_dict = {k[7:11]+k[18:]: v for k, v in check_point.items()}# if k[7:] in net_dict}
        for k, _ in pretrained_dict.items():
            logger.info(
                '=> loading {} from pretrained model'.format(k))
        net_dict.update(pretrained_dict)
        net.load_state_dict(net_dict)

        test_imgs = glob('/media/dongmeng/Hulk/dataset/cell-count-macaque/test/*.tif')
        for _, fpath in enumerate(test_imgs):
            img = io.imread(fpath)
            if len(img.shape) == 3:
                img = img[np.newaxis, :]
            elif len(img.shape) == 4:
                img = np.transpose(img, (1, 0, 2, 3))

            img = img_transform(config, img)
            img = img[np.newaxis, :]

            img = torch.from_numpy(img)
            img = img.float().cuda()

            # forward
            pred_map = net.test_forward(img)

            pred = pred_map.cpu().data.numpy()[0, 1, :, :, :]
            pred_img = (pred - np.min(pred)) / np.max(pred) * 255
            pred_img = pred_img.astype(np.uint8)
            io.imsave(os.path.join(out_path, fpath.split('/')[-1][:-4] + '_pred.tif'), pred_img)

            del pred_map

            print('img_name {} '.format(fpath.split('/')[-1][:-4]))

def img_transform(config, img):
    img = img.astype(np.float32)
    if img.shape[0] == 1:
        avg = config.DATA.MEAN
        var = config.DATA.VAR
        img = (img-avg[0])/var[0]
    elif img.shape[0] == 2:
        avg = config.DATA.MEAN
        var = config.DATA.VAR
        img[0,:] = (img[0,:] - avg[0]) / var[0]
        img[1, :] = (img[1, :] - avg[1]) / var[1]

    return img

if __name__ == '__main__':
    main()


text:
import pandas as pd
import numpy as np
from utils import *

state_list = ["current_price", "rolling_mean", "rolling_std", "cross_upper_band", "cross_lower_band", "upper_band",
             "lower_band", "price_over_sma"]

class Environment:

    # load pricing data
    # initialize the environment variables
    def __init__(self, data, states=state_list, recent_k = 0, bollinger_threshold=0):
        self.series = data
        self.states = states
        self.recent_k = recent_k
        self.bollinger_threshold=bollinger_threshold
        self.length = len(self.series)

        if self.recent_k == 0:
            self.current_index = 0
        else:
            self.current_index = self.length - self.recent_k

        self.__init()

    # deriving the features used for the state definition
    def __init(self):
        self.isDone = np.zeros(self.series["Adj Close"].shape, dtype=bool)
        self.isDone[-1] = True

        ### States
        self.rm = self.series["Adj Close"].rolling(window=20).mean()
        self.rstd = self.series["Adj Close"].rolling(window=20).std()
        self.upper_band, self.lower_band = self.rm + 2 * self.rstd, self.rm - 2 * self.rstd

        ### Mapping states to their names
        self.state_dict = {}
        self.state_dict["current_price"] = self.series["Adj Close"]
        self.state_dict["rolling_mean"] = self.rm
        self.state_dict["rolling_std"] = self.rstd
        self.state_dict["cross_upper_band"] = self.__crossUpperBand()
        self.state_dict["cross_lower_band"] = self.__crossLowerBand()
        self.state_dict["upper_band"] = self.upper_band
        self.state_dict["lower_band"] = self.lower_band
        self.state_dict["price_over_sma"] = self.series["Adj Close"]/self.rm

    def __crossUpperBand(self):
        crossUpperBand = [0]
        for i in range(1, len(self.series)):
            crossUpperBand.append(self.__checkCrossUpperBand(i)*1)
        return crossUpperBand

    def __crossLowerBand(self):
        crossLowerBand = [0]
        for i in range(1, len(self.series)):
            crossLowerBand.append(self.__checkCrossLowerBand(i)*1)
        return crossLowerBand

    def __checkCrossUpperBand(self, curr_index):
        return (
            curr_index - 1 >= 0
            and self.upper_band[curr_index]-self.bollinger_threshold >= self.state_dict["current_price"][curr_index]
            and self.upper_band[curr_index-1]-self.bollinger_threshold < self.state_dict["current_price"][curr_index-1]
        )

    def __checkCrossLowerBand(self, curr_index):
        return (
            curr_index - 1 >= 0
            and self.lower_band[curr_index]+self.bollinger_threshold <= self.state_dict["current_price"][curr_index]
            and self.lower_band[curr_index-1]+self.bollinger_threshold > self.state_dict["current_price"][curr_index-1]
        )

    ## PUBLIC METHODS ##

    # simulate a forward step in the environment, i.e.: moving one day
    def step(self):
        isDone = self.isDone[self.current_index]
        observation = []
        for state in self.states:
            observation.append(self.state_dict[state][self.current_index])
        if not isDone:
            self.current_index += 1
        return isDone, observation

    def getCurrentStates(self, states=None):
        if not states:
            states = self.states
        return [self.state_dict[state][self.current_index] for state in states]

    ## Add method to get current price as it is commonly used
    def getCurrentPrice(self):
        return self.state_dict["current_price"][self.current_index]

    def getFinalPrice(self):
        return self.state_dict["current_price"][self.length-1]

    def getPriceAt(self, index):
        if index < 0:
            return self.state_dict["current_price"][0]
        if index >= self.length:
            return self.getFinalPrice()
        return self.state_dict["current_price"][index]

    def plot(self, states_to_plot=None):
        import matplotlib.pyplot as plt
        if not states_to_plot:
            states_to_plot = self.states

        plt.figure(figsize=(10,6))
        for state in states_to_plot:
            ax = self.state_dict[state][self.current_index:].plot()
        ax.legend(states_to_plot)
        plt.show()

    def reset(self):
        if self.recent_k == 0:
            self.current_index = 0
        else:
            self.current_index = self.length - self.recent_k

    def getReward(self, action):
        a = 0
        if action == Action.BUY:
            a = 1
        elif action == Action.SELL:
            a = -1

        price_t = self.getCurrentPrice()
        price_t_minus_1 = self.getPriceAt(self.current_index - 1)
        price_t_minus_n = self.getPriceAt(self.current_index - 10)

        r = (1 + a*(price_t - price_t_minus_1)/price_t_minus_1)*price_t_minus_1/price_t_minus_n
        return r

max_stars_repo_name: guangzhixie/stock-analysis

max_issues_repo_name: guangzhixie/stock-analysis

max_forks_repo_name: guangzhixie/stock-analysis


text:
import gym
import numpy as np
import time
from pathlib import Path
from collections import deque

from sweet.interface.tf.tf_platform import TFPlatform
from sweet.interface.torch.torch_platform import TorchPlatform
from sweet.common.logging import Logger
from sweet.agents.agent_runner import Runner
from sweet.agents.runner.stop_condition import NstepsStopCond
from sweet.common.math import explained_variance
from sweet.agents.a2c.a2c_agent import A2CAgent
from sweet.common.time import dt_to_str
from sweet.common.utils import now_str

def learn(
    ml_platform=TorchPlatform,
    env_name='CartPole-v1',
    model='pi_actor_critic',
    total_timesteps=1e7,
    nenvs=1,
    nsteps=32,
    lr=0.001,
    coeff_critic=0.5,
    gamma: float = 0.95,
    model_checkpoint_freq: int = 1e5,
    log_interval: int = 10,
    targets: dict = {
        'output_dir': Path('./target/'),
        'models_dir': 'models_checkpoints',
        'logs_dir': 'logs',
        'tb_dir': 'tb_events'
    }
):
    """
    Train agent with A2C algorithm

    Parameters
    ----------
        ml_platform: sweet.interface.MLPlatform
            Machine Learning platform (either TF2 or Torch)
        env_name: str
            Name of OpenAI Gym environment
        model: str or model from ML platform
            Neural network or string describing neural network
        total_timesteps: int
            Number of training steps
        nenvs: int
            Number of parallel environment to collect experience
        nsteps: int
            Number of steps executed by each environment to collect experience.
            So batch size is nenvs * nsteps for each update
        lr_actor: float or sweet.common.schedule.Schedule
            Learning rate for actor
        lr_actor: float or sweet.common.schedule.Schedule
            Learning rate for critic
        gamma: float
            Discount factor
        model_checkpoint_freq: int
            Save model each "model_checkpoint_freq" update
            (so each nenvs*nsteps)
        log_interval: int
            Network update frequency on which logs are printed out
            (console + tensorboard)
        targets: dict
            Output directories:
                output_dir: Main target directory
                run_format_dir: Current run target directory
                models_dir: Models checkpoint directory
                logs_dir: Logs directory
                tb_dir: TensorBoard event files directory
    """
    # Target paths and save conf
    run_target_dir = targets['output_dir'] / f"run_{now_str()}"
    models_dir = run_target_dir / targets['models_dir']

    logger = Logger(
        "a2c-train",
        target_dir=run_target_dir,
        logs_dir=targets['logs_dir'],
        tb_dir=targets['tb_dir'])
    logger.save(run_target_dir / Path('configuration.json'), locals())

    # Load OpenAI Gym env
    env = gym.make(env_name)

    # Load DQN agent
    agent = A2CAgent(
        ml_platform=ml_platform,
        state_shape=env.observation_space.shape,
        action_size=env.action_space.n,
        model=model,
        lr=lr,
        coeff_critic=coeff_critic,
        gamma=gamma,
    )

    nbatch = nenvs * nsteps
    nudpates = int(total_timesteps // nbatch + 1)
    model_checkpoint = 0

    runner = Runner(env, agent, stop_cond=NstepsStopCond(nsteps))
    tstart = time.time()

    # Collect infos on last 10 batch runs
    u_steps, u_rewards = deque(maxlen=10), deque(maxlen=10)

    for nupdate in range(1, nudpates):
        # Collect mini-batch of experience
        obs, _, rewards, actions, dones, values, infos = runner.run()

        # Optimize both actor and critic with gradient descent
        loss_pi = agent.update(
            obs, rewards, actions, dones, values)

        # Post-processing (logging, ..)
        nseconds = time.time() - tstart
        fps = int((nupdate * nbatch) / nseconds)
        expl_variance = explained_variance(np.squeeze(values), rewards)
        expected_remaining_dt = (
            total_timesteps - (nupdate * nbatch)
            ) / (fps + 1e-8)

        steps = [x['steps'] for x in infos]
        rewards = [x['rewards'] for x in infos]

        if len(steps):
            u_steps.append(np.mean(steps))
            u_rewards.append(np.mean(rewards))

        mean_episode_length = np.mean(u_steps)
        mean_episode_rew = np.mean(u_rewards)

        # Save model
        if (nupdate - model_checkpoint) > model_checkpoint_freq:
            agent.save_model(models_dir / f'model-{nupdate}')
            model_checkpoint = model_checkpoint_freq

        # Logging
        if nupdate % log_interval == 0 or nupdate == 1:
            logger.info(f"Update #{nupdate}")

            logger.record_tabular("total_timesteps", nbatch*nupdate)
            logger.record_tabular("FPS", fps)
            logger.record_tabular("explained_varaince", expl_variance)
            logger.record_tabular("Loss_pi", loss_pi)
            logger.record_tabular("Mean rewards", mean_episode_rew)
            logger.record_tabular("Mean episode length", mean_episode_length)
            logger.record_tabular("Time elapsed", dt_to_str(nseconds))
            logger.record_tabular("ETA", dt_to_str(expected_remaining_dt))

            logger.dump_tabular()

if __name__ == "__main__":
    learn()

max_stars_repo_name: Hadjubuntu/sweet-rl

max_issues_repo_name: Hadjubuntu/sweet-rl

max_forks_repo_name: Hadjubuntu/sweet-rl


text:
#!/usr/bin/env
"""kernels.py: kernel functions which can take Pyomo variables as arguments."""
import pyomo.environ as p
import numpy as np

class Kernel():
    """ Abstract kernel. """
    def __init__(self):
        raise NotImplementedError

    def calc(self, x1, x2):
        raise NotImplementedError

    def __call__(self, x, y):
        return self.calc(x, y)

    def euclidian_squared(self, x, y):
        """ Vectorized euclidian squared distance between X and Y.

            :param x: numpy.ndarray (n,k)
            :param y: numpy.ndarray (m,k)

            returns numpy.ndarray (n, m)
        """
        assert x.shape[1] == x.shape[1]
        n = x.shape[0]
        m = y.shape[0]
        X = x[:, None, :].repeat(m, axis=1)
        Y = y[None, :, :].repeat(n, axis=0)
        return np.sum((X - Y)**2, axis=2)

class RBF(Kernel):
    """ Squared exponential kernel function. """
    def __init__(self, lengthscale=1.0, variance=1.0):
        assert lengthscale > 0
        assert variance > 0
        self.lengthscale = lengthscale
        self.variance = variance

    def calc(self, x, y):
        """ Calculate k(x1, x2)

        :param x:
        :param y:
        :type x: int, float, pyomo.environ.Var, ...
        :type y: int, float, pyomo.environ.Var, ...
        :returns: squared exponential distance
        """
        r_sqr = self.euclidian_squared(x, y)/(self.lengthscale**2)
        exp = np.vectorize(p.exp, otypes=[object])
        return self.variance * exp(-0.5 * r_sqr)

max_stars_repo_name: cog-imperial/rogp

max_issues_repo_name: johwiebe/rogp

max_forks_repo_name: johwiebe/rogp

reason: import numpy,import pyomo


text:
import sys

# from features import *
# from rdkit import Chem

import numpy as np

class Molecule:

    def __init__(self, all_atom_types, all_bond_types):
        self.all_bond_types = all_bond_types
        self.all_atom_types = all_atom_types
        self.atom_idx = {}
        self.bond_idx = {}

    def load_from_lists(self, atom_types, bond_types, bonds, target):
        self.atom_types = {k: v for k, v in atom_types}
        self.bond_types = {k: v for k, v in bond_types}
        self.bonds = {k: (v, w) for v, w, k in bonds}
        self.target = target

    def to_graph(self):
        graph = {}

        atom_feats = []
        bond_feats = []

        for i, (atom, type) in enumerate(self.atom_types.items()):
            self.atom_idx[atom] = i
            atom_feats.append(to_onehot(self.all_atom_types[type], len(self.all_atom_types)))

        for i, (bond, type) in enumerate(self.bond_types.items()):
            self.bond_idx[bond] = i
            bond_feats.append(to_onehot(self.all_bond_types[type], len(self.all_bond_types)))

        edges_list = [None] * len(self.bonds)
        for i, (bond, (a1, a2)) in enumerate(self.bonds.items()):
            edges_list[self.bond_idx[bond]] = (self.atom_idx[a1], self.atom_idx[a2])

        graph["edge_index"] = np.array(edges_list, dtype=np.int64).T
        graph['edge_feat'] = np.array(bond_feats, dtype=np.float)
        graph['node_feat'] = np.array(atom_feats, dtype=np.float)

        graph['num_nodes'] = len(atom_feats)

        graph["target"] = self.target

        return graph

def to_onehot(ind, size):
    listofzeros = [0] * size
    listofzeros[ind] = 1
    return listofzeros

def graph2prolog(graph):
    bonds = get_bonds(graph["edge_index"].transpose().tolist())
    bond_feats, b_embeds = get_bond_features(graph["edge_feat"].tolist())  # ,graph['bond_types'])
    atom_feats, a_embeds = get_atom_features(graph["node_feat"].tolist())  # , graph['atom_types'])
    all_feats = []
    all_feats.extend(bonds)
    all_feats.extend(bond_feats)
    all_feats.extend(atom_feats)
    string = ",".join(all_feats)
    string += "."
    all_embeds = set()
    all_embeds.update(a_embeds)
    all_embeds.update(b_embeds)

    if graph['target']:
        string = graph['target'].replace("\n", "").replace("-1", "0").replace("+1", "1") + " predict :- " + string

    return string, all_embeds

def get_bonds(pairList):
    bond = 0
    bonds = []
    for ind, (i, j) in enumerate(pairList):

        if ind % 2 == 0:
            suff = "l"
            bond += 1
        else:
            suff = "r"

        next = f' bond(a{i}, a{j}, b{bond}{suff})'
        bonds.append(next)
    return bonds

def get_bond_features(triplesList, bond_types=None):
    bond = 0
    bonds = set()

    embeds = set()
    i = 0

    for ind, featrs in enumerate(triplesList):
        if ind % 2 == 0:
            suff = "l"
            bond += 1
        else:
            suff = "r"

        next = f' <[{", ".join(str(x) for x in featrs)}]> b_feats(b{bond}{suff})'
        bonds.add(next)

        if bond_types:
            type = "bondic" + bond_types[i]
            bonds.add(type + f'(b{bond}{suff})')
            embeds.add(type)

    return bonds, embeds

def get_atom_features(featureList, atom_types=None):
    atom = 0
    atoms = set()

    embeds = set()

    if atom_types:
        for i, featrs in enumerate(featureList):
            type = "atomic" + atom_types[i]
            embeds.add(type)
            next = type + "(a" + str(i) + ")"
            atoms.add(next)

    for i, featrs in enumerate(featureList):
        next = " <[" + ', '.join(str(x) for x in featrs) + "]> a_feats(a" + str(i) + ")"
        atoms.add(next)
    return atoms, embeds

max_stars_repo_name: GustikS/GNNwLRNNs

max_issues_repo_name: GustikS/GNNwLRNNs

max_forks_repo_name: GustikS/GNNwLRNNs


text:
import pandas as pd
from keras.models import Model
from keras.layers import Dense, Input
from keras.models import model_from_json
from keras import backend as K
from User import User
import matplotlib.pyplot as plt
import numpy as np

class GamePlayerNetwork:

    def __init__(self, screen_width, screen_height):
        self.update_learning_graph_count = 0

        inputs = Input(shape=(screen_width*screen_height,))
        x = Dense(400, activation=K.sigmoid)(inputs)
        #x = Dense(400, activation=K.sigmoid)(x)
        predictions = Dense(1, activation=K.tanh)(x)
        model = Model(inputs=inputs, outputs=predictions)
        model.summary()
        model.compile(optimizer='rmsprop',
                      loss='mean_squared_error',
                      metrics=['accuracy'])
        self.model = model
        self.plot = None
        self.results = None

    def train(self, filename):
        df = pd.read_csv(filename)
        training_set = df.values[:, 0:400]
        target_set = df.values[:,-1]
        self.results = self.model.fit(training_set, target_set, batch_size=100, epochs=2000)

    def save_model(self, filename):
        # serialize model to JSON
        model_json = self.model.to_json()
        json_name = "{0}.json".format(filename)
        h5_name = "{0}.h5".format(filename)
        with open(json_name, "w") as json_file:
            json_file.write(model_json)
        # serialize weights to HDF5
        self.model.save_weights(h5_name)
        print("Saved model to disk")

    def load_model(self,filename):
        # load json and create model
        json_file = open('{0}.json'.format(filename), 'r')
        loaded_model_json = json_file.read()
        json_file.close()
        self.model = model_from_json(loaded_model_json)
        # load weights into new model
        self.model.load_weights("{0}.h5".format(filename))
        print("Loaded model from disk")

    def display_training_results(self):
        plt.plot(self.results.history['loss'])
        #plt.plot(self.results.history['val_loss'])
        plt.title('model loss')
        plt.ylabel('loss')
        plt.xlabel('epoch')
        plt.show(block=True)

    def get_player_action(self, screen_array):

        o = self.model.predict(screen_array.reshape(1, 400))[0][0]
        result = None
        if o <= -0.5:
            result = User.LEFT
        elif o >= 0.5:
            result = User.RIGHT
        return result

max_stars_repo_name: fractalbass/neuralnest

max_issues_repo_name: fractalbass/neuralnest

max_forks_repo_name: fractalbass/neuralnest


text:
from typing import Union, List, Optional
import numpy as np
from causaldag.utils.core_utils import to_list
from ._utils import combined_mat
from causaldag.utils.ci_tests.hsic import hsic_test

def hsic_invariance_test(
        suffstat,
        context,
        i: int,
        cond_set: Optional[Union[List[int], int]]=None,
        alpha: float=0.05
):
    """
    TODO

    Parameters
    ----------
    TODO

    Examples
    --------
    TODO
    """
    cond_set = to_list(cond_set)
    obs_samples = suffstat['obs_samples']
    iv_samples = suffstat[context]

    mat = combined_mat(obs_samples, iv_samples, i, cond_set)
    return hsic_test(mat, 0, 1, list(range(2, 2+len(cond_set))), alpha=alpha)

max_stars_repo_name: kpj/causaldag

max_issues_repo_name: anastasiyabel/causaldag

max_forks_repo_name: anastasiyabel/causaldag


text:
import re
import operator
from collections import defaultdict
from uuid import uuid4
from itertools import chain
import matplotlib.pyplot as plt
import matplotlib.transforms as mtransforms

from six import string_types as basestring

import numpy as np
from glypy import monosaccharides
from glypy.structure import constants
from glypy.utils import root
from glypy.io.nomenclature import identity
from glypy.algorithms.similarity import is_derivatized
from glypy.composition.composition_transform import strip_derivatization

from .buchheim import BalancedTreeLayout
from .topological_layout import TopologicalTreeLayout
from . import cfg_symbols, iupac_symbols, snfg_symbols
from .geometry import centroid, interpolate
from .fragment_annotation import BondCleavageArtist

line_to = cfg_symbols.CFGNomenclature().line_to

nomenclature_map = {
    "cfg": cfg_symbols.CFGNomenclature(),
    'iupac': iupac_symbols.IUPACTextSymbolicNomenclature(),
    "snfg": snfg_symbols.SNFGNomenclature()
}

layout_map = {
    "balanced": BalancedTreeLayout,
    "topological": TopologicalTreeLayout
}

DEFAULT_SYMBOL_SCALE_FACTOR = 0.17

#: :data:`special_cases` contains a list of names for
#: special case monosaccharides
_special_case_fucose = monosaccharides.Fuc
_special_case_fucose.configuration = None
_special_case_fucose.anomer = None
_special_case_xylose = monosaccharides.Xyl
_special_case_xylose.configuration = None
_special_case_xylose.anomer = None

special_cases = [_special_case_fucose, _special_case_xylose]

anomer_symbol_map = {
    constants.Anomer.alpha: r'\alpha',
    constants.Anomer.beta: r'\beta',
    constants.Anomer.uncyclized: r'o',
    constants.Anomer.x: r'?'
}

def sign(x):
    if x > 0:
        return 1
    else:
        return -1

def flatten(iterable):
    acc = []
    for item in iterable:
        try:
            acc.extend(flatten(item))
        except Exception:
            acc.append(item)
    return acc

def is_special_case(node):
    '''
    Check to see if `node` is a special case which requires a different layout
    scheme. See `special_cases`
    '''
    if is_derivatized(node):
        node = strip_derivatization(node.clone())
    for case in special_cases:
        if identity.is_a(node, case) and len(list(node.children())) == 0:
            return True
    return False

def box(node, ax=None):  # pragma: no cover
    if ax is None:
        ax = node.axes
    xmin, xmax, ymin, ymax = node.extrema()
    ax.plot((xmin - 0.2, xmax + 0.2), (ymin - 0.2, ymin - 0.2), c='b')
    ax.plot((xmin - 0.2, xmax + 0.2), (ymax + 0.2, ymax + 0.2), c='b')
    ax.plot((xmin - 0.2, xmin - 0.2), (ymin - 0.2, ymax + 0.2), c='b')
    ax.plot((xmax + 0.2, xmax + 0.2), (ymin - 0.2, ymax + 0.2), c='b')

def make_gid(s):  # pragma: no cover
    '''Formats a string to use ``kebab-case`` and sanitize illegal characters'''
    s = str(s)
    return re.sub(r'[\s\|\(\):]', '-', s)

def breadth_first_traversal(tree, visited=None):
    if visited is None:
        visited = set()
    if tree.id not in visited:
        visited.add(tree.id)
        yield tree
        for child in tree:
            for descend in breadth_first_traversal(child, visited=visited):
                yield descend

def _bounding_box_to_limits(dtree):
    xlim, ylim = dtree._compute_bounding_box()
    xlim = list(xlim)
    xlim[0] -= 2
    xlim[1] += 2
    ylim = list(ylim)
    ylim[0] -= 2
    ylim[1] += 2
    return np.array(xlim), np.array(ylim)

def enumerate_tree(tree, ax, labeler=None):
    """Label each node of `tree` on `ax` with the node's :attr:`id`

    Parameters
    ----------
    tree: :class:`DrawTreeNode`
        A drawn :class:`DrawTreeNode` object
    ax: matplotlib.Axes
        The axes on which `tree` has been drawn

    Returns
    -------
    tree, ax
    """
    if labeler is None:
        labeler = operator.attrgetter('id')
    nodes = list(breadth_first_traversal(tree))
    for node in nodes:
        x, y = node.coords()
        ax.text(x, y + 0.2, labeler(node.tree), color='red')
    # Update the plot
    ax.get_figure().canvas.draw()
    return tree, ax

def find_link(parent, child):  # pragma: no cover
    for p, link in parent.links.items():
        if child.parent_linkage == p:
            if link.child == child:
                return link

def get(tree, node_id):
    for node in breadth_first_traversal(tree):
        if node.tree.id == node_id:
            return node

def get_link_pair(tree, link_id):
    link = None
    for node in breadth_first_traversal(tree):
        for p, t_link in node.tree.links.items():
            if t_link.id == link_id:
                link = t_link
                break
    if link is None:
        raise Exception("Link not found")
    parent = get(tree, link.parent.id)
    child = get(tree, link.child.id)
    return parent, child

INF = float("inf")

class DrawTreeNode(object):
    def __init__(self, tree, parent=None, depth=0, number=1, parent_linkage=None, visited=None):
        if visited is None:
            visited = dict()
        self.id = tree.id
        if self.id in visited:
            raise Exception("Cycle detected")
        visited[self.id] = self
        self.x = -1.
        self.y = depth
        self.depth = depth
        self.tree = tree
        self.parent = parent
        self.parent_linkage = parent_linkage

        self._axes = None

        self.mask_special_cases = True
        # Unused. Artefact of Buccheim
        self.thread = None
        self.mod = 0
        self.ancestor = self
        self.change = self.shift = 0
        self._lmost_sibling = None

        # Number of the node in its group of siblings 1..n
        self.number = number

        if self.parent is not None:
            self.data = self.parent.data
            self.uuid = self.parent.uuid
        else:
            self.data = defaultdict(lambda: defaultdict(dict))
            self.uuid = uuid4().hex

        self.children = [self.resolve_creation_cycle(c[1], depth + 1, i + 1, c[0], visited)
                         for i, c in enumerate(sorted(tree.children(), key=lambda x: x[0]))]

        # A node owns all lines originating from it.
        self.lines = []

    def resolve_creation_cycle(self, node, depth, i, parent_linkage, visited):
        if node.id in visited:
            return visited[node.id]
        else:
            return self.__class__(node, self, depth + 1, i + 1, parent_linkage, visited)

    def __iter__(self):
        return iter(self.children)

    def traverse(self):
        return breadth_first_traversal(self)

    @property
    def children(self):
        if self.mask_special_cases:
            return [child for child in self._children if not (
                is_special_case(child.tree) and len(child.children) == 0)]
        else:
            return self._children

    @children.setter
    def children(self, value):
        self._children = value

    @property
    def axes(self):
        return self._axes

    @axes.setter
    def axes(self, value):
        self._axes = value
        for child in self:
            if child.axes != value:
                child.axes = value

    def fix_special_cases(self, offset=0.5, visited=None):
        if visited is None:
            visited = set()
        if self.id in visited:
            return
        visited.add(self.id)
        self.mask_special_cases = False
        for child in self.children:
            if is_special_case(child.tree):
                if child.parent_linkage > 3 or child.parent_linkage == -1:
                    child.y = self.y
                    child.x = self.x + offset
                elif child.parent_linkage > 1:
                    child.y = self.y
                    child.x = self.x - offset
                else:
                    raise Exception(
                        "Don't know how to handle special case child node %r, %r." % (
                            child, child.parent_linkage))
            child.fix_special_cases(offset, visited)

    def draw(self, at=(0, 0), ax=None, symbol_nomenclature="snfg", label=True, **kwargs):
        if isinstance(symbol_nomenclature, basestring):
            symbol_nomenclature = nomenclature_map[symbol_nomenclature]
        elif symbol_nomenclature is None:
            symbol_nomenclature = snfg_symbols.SNFGNomenclature()
        if ax is None:  # pragma: no cover
            ax = self.axes
            if ax is None:
                fig, ax = plt.subplots()
        self.draw_branches(at=at, ax=ax, symbol_nomenclature=symbol_nomenclature,
                           label=label, visited=set(), **kwargs)
        self.draw_nodes(at=at, ax=ax, symbol_nomenclature=symbol_nomenclature,
                        label=label, visited=set(), **kwargs)

    def coords(self, at=None):
        if at is None:
            at = (0, 0)
        if self.transform:
            at = self.transform.transform(at)
            x, y = self.transform.transform((self.x, self.y))
        else:
            x, y = self.x, self.y
        return x + at[0], y + at[1]

    def draw_branches(self, at=(0, 0), ax=None, symbol_nomenclature=None,
                      label=True, visited=None, **kwargs):
        '''
        Draw the edges linking parent nodes to child nodes. Also draws the parent outlink position
        and the child anomer symbol
        Parameters
        ----------
        at: :class:`tuple`
            The x, y coordinates at which to draw the glycan's reducing_end. Defaults to `(0, 0)`
        ax: :class:`matplotlib.axes.Axis`
            A matplotlib axis on which to draw. Useful for adding glycans to existing figures. If
            an axis is not provided, a new figure will be created and used.
        symbol_nomenclature: |str|
            A string mapping to the symbol nomenclature to use. Defaults to |None|. When `symbol_nomenclature`
            is |None|, `snfg` will be used.
        visited: |set|
            A |set| instance containing the node ids already visited while drawing. If the current
            node was already visited, its id will already be present. Used to avoid infinite cycles.
        '''
        if visited is None:
            visited = set()
        x, y = self.coords(at)
        if self.id in visited:
            return x, y
        visited.add(self.id)
        edge_weight = kwargs.get("edge_weight", 0.5)
        for child in self:
            cx, cy = child.draw_branches(at=at, ax=ax,
                                         symbol_nomenclature=symbol_nomenclature,
                                         label=label, visited=visited, **kwargs)
            patch = symbol_nomenclature.line_to(
                ax, x, y, cx, cy,
                color=kwargs.get('link_color', symbol_nomenclature.default_line_color),
                zorder=1, lw=edge_weight)
            self.data['lines'][self.id, child.id] = [patch]
            self.lines.append(patch)
        return x, y

    def draw_nodes(self, at=(0, 0), ax=None, symbol_nomenclature=None, label=True, visited=None, **kwargs):
        '''
        Draw the symbol representing the individual monosaccharide and its substituents.

        Parameters
        ----------
        at: :class:`tuple`
            The x, y coordinates at which to draw the glycan's reducing_end. Defaults to `(0, 0)`
        ax: :class:`matplotlib.axes.Axis`
            A matplotlib axis on which to draw. Useful for adding glycans to existing figures. If
            an axis is not provided, a new figure will be created and used.
        symbol_nomenclature: |str|
            A string mapping to the symbol nomenclature to use. Defaults to |None|. When `symbol_nomenclature`
            is |None|, `cfg` will be used.
        visited: |set|
            A |set| instance containing the node ids already visited while drawing. If the current
            node was already visited, its id will already be present. Used to avoid infinite cycles.
        '''
        if visited is None:
            visited = set()
        if self.id in visited:
            return
        visited.add(self.id)
        x, y = self.coords(at)
        patch_node = symbol_nomenclature.draw(self.tree, x, y, ax, tree_node=self,
                                              **kwargs)
        self.patch_node = patch_node
        residue_elements, substituent_elements, saccharide_label = patch_node
        self.data["patches"][self.tree.id] = [residue_elements]
        self.substituents_text = self.data["text"]["substituents"][self.id] = [substituent_elements]
        self.saccharide_label = self.data['text']['saccharide_label'][self.id] = [saccharide_label]
        self.data["position"][self.tree.id] = x, y

        patch_node.set_gid(self.uuid, self.id)

        for child in self:
            child.draw_nodes(at=at, ax=ax, symbol_nomenclature=symbol_nomenclature,
                             visited=visited, **kwargs)

    def _draw_linkage_label(self, child, at=(0, 0), ax=None, symbol_nomenclature=None, **kwargs):
        sx, sy = self.coords(at)
        cx, cy = child.coords(at)

        fontsize = kwargs.get("fontsize", 6)
        style = kwargs.get('label_style', 'split')

        position_x = (sx * 0.65 + cx * 0.35)
        if abs(sx - cx) < 1e-2:
            position_y = (sy + cy) / 2
        else:
            position_y = interpolate(sx, sy, cx, cy, position_x)

        position_num = -1
        for pos, link in self.tree.links.items():
            if link.is_child(child.tree):
                position_num = pos
                break

        anomer_x = (sx * 0.35 + cx * 0.65)
        if abs(sx - cx) < 1e-2:
            anomer_y = (sy + cy) / 2
        else:
            anomer_y = interpolate(sx, sy, cx, cy, anomer_x)

        if abs(anomer_x - position_x) < 1e-2 and abs(anomer_y - position_y) < 1e-2:
            anomer_x -= 0.07
            position_x += 0.05
            anomer_y -= 0.03
            position_y -= 0.03

        if style == 'split':
            position_text = symbol_nomenclature.draw_text(
                ax=ax, x=position_x, y=position_y + 0.03,
                text=str(position_num) if position_num != -1 else "?",
                fontsize=fontsize, ha='center', va='bottom')
            anomer_text = symbol_nomenclature.draw_text(
                ax, anomer_x, anomer_y + 0.03,
                r'${}$'.format(anomer_symbol_map.get(
                    child.tree.anomer, child.tree.anomer.name)),
                fontsize=fontsize, ha='center', va='bottom')
            self.data['text'][self.id, child.id]['linkage'] = [position_text, anomer_text]
        elif style == 'centered':
            symbol_nomenclature.draw_text(
                ax=ax, x=(position_x + anomer_x) / 2.0, y=(position_y + anomer_y) / 2.0,
                text=(str(position_num) if position_num != -1 else "?") + r'${}$'.format(anomer_symbol_map.get(
                    child.tree.anomer, child.tree.anomer.name)),
                fontsize=fontsize, ha='center', va='bottom')
        else:
            raise ValueError(
                "Unrecognized label_style position style %r" % (style, ))

    def draw_linkage_annotations(self, at=(0, 0), ax=None, symbol_nomenclature=None, child=None, **kwargs):
        '''
        Draw the parent outlink position and the child anomer symbol

        Parameters
        ----------
        at: :class:`tuple`
            The x, y coordinates at which to draw the glycan's reducing_end. Defaults to `(0, 0)`
        ax: :class:`matplotlib.axes.Axis`
            A matplotlib axis on which to draw. Useful for adding glycans to existing figures. If
            an axis is not provided, a new figure will be created and used.
        symbol_nomenclature: |str|
            A string mapping to the symbol nomenclature to use. Defaults to |None|. When `symbol_nomenclature`
            is |None|, `cfg` will be used.
        child: :class:`DrawTreeNode`
            The child node to draw relative to.
        '''
        if child is None:
            for child in self:
                self.draw_linkage_annotations(
                    at=at, ax=ax, symbol_nomenclature=symbol_nomenclature,
                    child=child, **kwargs)
                child.draw_linkage_annotations(
                    at=at, ax=ax, symbol_nomenclature=symbol_nomenclature, **kwargs)
            return
        self._draw_linkage_label(child, at, ax, symbol_nomenclature, **kwargs)

    # Tree Layout Helpers
    def left(self):
        return self.thread or len(self.children) and self.children[0]

    def right(self):
        return self.thread or len(self.children) and self.children[-1]

    def lbrother(self):
        n = None
        if self.parent:
            for node in self.parent.children:
                if node == self:
                    return n
                else:
                    n = node
        return n

    @property
    def lmost_sibling(self):
        if not self._lmost_sibling and self.parent and self != self.parent.children[0]:
            self._lmost_sibling = self.parent.children[0]
        return self._lmost_sibling

    def __str__(self):  # pragma: no cover
        return "%s: id=%s  x=%s mod=%s" % (self.tree, self.id, self.x, self.mod)

    def __repr__(self):  # pragma: no cover
        return self.__str__()

    def extrema(self, at=(0, 0), xmin=INF, xmax=-INF, ymin=INF, ymax=-INF, visited=None):
        '''
        Finds the most extreme points describing the area this node and its children occupy.

        Parameters
        ----------
        at: tuple of int
            The position to orient relative to in the xy-plane. A tuple of integers corresponding
            to the center x and y respectively.
        xmin: float
        xmax: float
        ymin: float
        ymax: float
            Used internally to pass along current extreme values recursively to child nodes.

        Returns
        -------
        xmin: float
        xmax: float
        ymin: float
        ymax: float

        '''
        if visited is None:
            visited = set()
        if self.id in visited:
            return xmin, xmax, ymin, ymax
        visited.add(self.id)
        x, y = self.coords(at)
        if x < xmin:
            xmin = x
        if x > xmax:
            xmax = x
        if y < ymin:
            ymin = y
        if y > ymax:
            ymax = y
        for child in sorted(self.children, key=lambda x: len(x.children)):
            n_xmin, n_xmax, n_ymin, n_ymax = child.extrema(at, xmin, xmax, ymin, ymax, visited=visited)
            xmin = min(xmin, n_xmin)
            ymin = min(ymin, n_ymin)
            xmax = max(xmax, n_xmax)
            ymax = max(ymax, n_ymax)
        return xmin, xmax, ymin, ymax

    def get(self, node_id):
        return get(self, node_id)

    def get_link_pair(self, link_id):
        return get_link_pair(self, link_id)

    def annotate_fragment(self, fragment, ax=None, color='red', label=True, **kwargs):
        if ax is None:
            ax = self.axes
        return BondCleavageArtist(self, fragment, ax, color=color, label=label, **kwargs)

    def set_transform(self, transform):
        """Apply the passed Affine2D to each graphical unit. The transform is additive
        with any previous affine transformation.

        where `M` is the value of `transform.get_matrix()`

        Parameters
        ----------
        transform : matplotlib.transforms.Affine2D
        """
        current_transform = self.transform
        if current_transform is not None:
            transform = transform + current_transform
        self.data["transform"] = transform

        # NOTE: Merge the aggregated transform of the graph
        # structure with the transformation induced by the Axes object
        # which draws each component. If this is skipped, all of the
        # geometries are wrong and the drawing will fail (or not even appear)
        transform = transform + self.axes.transData
        for i, patch_set in self.data['patches'].items():
            for entity in patch_set:
                for p in entity:
                    if isinstance(p, tuple):
                        [_.set_transform(transform) for _ in p]
                    else:
                        p.set_transform(transform)
        for i, patch_set in self.data['lines'].items():
            for p in patch_set:
                if isinstance(p, tuple):
                    [_.set_transform(transform) for _ in p]
                else:
                    p.set_transform(transform)
        for i, groups in self.data['text'].items():
            text = flatten(groups.values())
            [t.set_transform(transform) for t in text]

    def get_transform(self):
        if "transform" in self.data:
            trans = self.data["transform"]
            return trans
        else:
            return None

    transform = property(get_transform, set_transform)

    def _compute_bounding_box(self, pad_factor=0.2):
        xmin, xmax, ymin, ymax = self.extrema()
        dx = (max(xmin, xmax) - min(xmin, xmax)) * pad_factor
        dy = (max(ymin, ymax) - min(ymin, ymax)) * pad_factor
        return (xmin - dx, xmax + dx), (ymin - dy, ymax + dy)

    def update_text_position(self, degrees=0, preserve_orientation=True):
        '''
        This is a hack needed to orient text appropriately when the tree is rotated after labels
        are inscribed, as any new transform will override the correction.
        '''
        current_transform = self.transform
        if current_transform is None:
            current_transform = mtransforms.Affine2D()
        for i, groups in self.data['text'].items():
            for t in flatten(groups.values()):
                cent = (centroid(t.get_path().transformed(current_transform)))
                trans = current_transform.frozen().get_affine().translate(0, 0).rotate_deg(degrees=degrees)
                new_cent = centroid(t.get_path().transformed(trans))
                shift = cent - new_cent
                trans.translate(*shift)
                trans += self.axes.transData
                t.set_transform(trans)

class DrawTree(object):  # pragma: no cover
    def __init__(self, structure, figure=None, ax=None, layout=BalancedTreeLayout,
                 symbol_nomenclature=snfg_symbols.SNFGNomenclature(), **kwargs):
        self.structure = structure
        self.root = DrawTreeNode(root(structure))
        self.figure = figure
        self.ax = ax
        self.layout_scheme = layout
        self.symbol_nomenclature = symbol_nomenclature

    def layout(self, **kwargs):
        self.layout_scheme(self.root).layout()
        self.root.fix_special_cases()

    def draw(self, ax=None, center=True, **kwargs):
        if ax is not None:
            self.ax = ax
            self.figure = ax.get_figure()
        else:
            self.figure, self.ax = plt.subplots()

        at = (0, 0)

        self.root.draw(ax=self.ax, at=at, symbol_nomenclature=self.symbol_nomenclature)
        if center:
            xmin, xmax, ymin, ymax = self.root.extrema(at=at)
            ax = self.ax
            ax.set_xlim(sign(xmin) * (abs(xmin) + 2), (1 * abs(xmax) + 2))
            ax.set_ylim(sign(ymin) * (abs(ymin) + 2), (1 * abs(ymax) + 2))
            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(False)

    def get(self, node_id):
        return self.root.get(node_id)

    def get_link_pair(self, link_id):
        return self.root.get_link_pair(link_id)

    @property
    def transform(self):
        return self.root.transform

    def run(self):
        self.layout()
        self.draw(ax=self.ax)
        self.figure.canvas.draw()

    def redraw(self, **kwargs):
        self.draw(**kwargs)
        self.figure.canvas.draw()

def plot(tree, at=(0, 0), ax=None, orientation='h', center=False, label=False,
         symbol_nomenclature='snfg', layout='balanced', layout_args=None, **kwargs):
    '''
    Draw the parent outlink position and the child anomer symbol

    Parameters
    ----------
    tree: Glycan or Monosaccharide
        The saccharide structure to draw.
    orientation: str
        A string corresponding to `h` or `horizontal` will draw the glycan horizontally
        from right to left. `v` or `vertical` will draw the glycan from bottom to top.
        Defaults to `h`
    at: tuple
        The x, y coordinates at which to draw the glycan's reducing_end. Defaults to `(0, 0)`
    ax: :class:`matplotlib.axes.Axis`
        A matplotlib axis on which to draw. Useful for adding glycans to existing figures. If
        an axis is not provided, a new figure will be created and used.
    center: bool
        Should the plot limits be centered around this glycan? Defaults to |False| but will be
        be set to |True| if `ax` is |None|
    label: bool
        Should the bond annotations for `tree` be drawn? Defaults to |False|
    symbol_nomenclature: str or :class:`~.SymbolicNomenclatureBase`
        Either the name of a symbol nomenclature or an instance of :class:`~.SymbolicNomenclatureBase`.
        Recognized names are "cfg", "snfg", and "iupac".
    layout: str or :class:`~.TreeLayoutBase`
        Either the name of a tree layout strategy or an instance of :class:`~.TreeLayoutBase`.
        Recognized names are "balanced" and "topological".
    layout_args: dict
        Extra parameters to pass to :class:`~.TreeLayoutBase`. It may take its own "transform" parameter
        passed here.
    scale: (float, float) or float
        Scale coefficients. Pass a pair to scale x and y dimensions respectively.
    symbol_scale: float, optional
        A scale coefficient for the size of the monosaccharide symbols, without changing the overall size
        of the layout.
    edge_weight: float, optional
        The line weight of the edges connecting monosaccharide symbols.
    label_style: str, optional
        The layout style used for drawing edge labels. May be one of "split" or "centered", where
        "split" will place the position close to the parent symbol and the anomer close to the
        child residue and "centered" places the symbols together at the centroid of the edge.
        Defaults to 'split'.
    '''

    scale = DEFAULT_SYMBOL_SCALE_FACTOR * kwargs.pop('symbol_scale', 1.0)
    transform_scale = kwargs.pop("scale", (1,))
    try:
        transform_scale = tuple(transform_scale)
    except Exception:
        transform_scale = (transform_scale, transform_scale)

    if isinstance(symbol_nomenclature, basestring):
        symbol_nomenclature = nomenclature_map.get(symbol_nomenclature)
    elif symbol_nomenclature is None:
        symbol_nomenclature = snfg_symbols.SNFGNomenclature()

    if isinstance(layout, basestring):
        layout = layout_map.get(layout)

    tree_root = root(tree)
    dtree = DrawTreeNode(tree_root)

    layout_algorithm = layout(dtree)
    layout_algorithm.layout(**(layout_args or {}))

    fig = None
    # Create a figure if no axes are provided
    if ax is None:
        fig, ax = plt.subplots()
        # at = (0, 0)
        center = True
        ax.axis('off')
    (layout_transform,
     substituent_transform,
     _) = symbol_nomenclature.get_layout_transform(orientation)
    layout_algorithm.transform(layout_transform)
    dtree.draw(at=at, ax=ax, scale=scale, label=False,
               symbol_nomenclature=symbol_nomenclature,
               annotation_transform=substituent_transform,
               **kwargs)
    dtree.axes = ax
    dtree.data['orientation'] = orientation
    dtree.set_transform(mtransforms.Affine2D())
    if label:
        dtree.draw_linkage_annotations(
            at=at, ax=ax, scale=scale,
            symbol_nomenclature=symbol_nomenclature,
            **kwargs)
    # If the figure is stand-alone, center it
    if fig is not None or center:
        xlim, ylim = _bounding_box_to_limits(dtree)
        ax.set_xlim(*xlim)
        ax.set_ylim(*ylim)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
    return (dtree, ax)

max_stars_repo_name: dcambie/glypy

max_issues_repo_name: dcambie/glypy

max_forks_repo_name: dcambie/glypy


text:
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for the key functions in pruning library."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# import g3
import numpy as np
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util as tf_test_util
from tensorflow.python.keras import backend as K
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import partitioned_variables
from tensorflow.python.ops import state_ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import variables
from tensorflow.python.platform import test
from tensorflow_model_optimization.python.core.sparsity.keras import pruning_impl
from tensorflow_model_optimization.python.core.sparsity.keras import pruning_schedule
from tensorflow_model_optimization.python.core.sparsity.keras import pruning_utils

@tf_test_util.run_all_in_graph_and_eager_modes
class PruningTest(test.TestCase):

  def setUp(self):
    super(PruningTest, self).setUp()
    self.global_step = K.zeros([], dtype=dtypes.int32)
    self.block_size = (1, 1)
    self.block_pooling_type = "AVG"

    def training_step_fn():
      return self.global_step

    self.constant_sparsity = pruning_schedule.ConstantSparsity(0.5, 0, 100, 1)
    self.training_step_fn = training_step_fn

  def testUpdateSingleMask(self):
    weight = K.variable(np.linspace(1.0, 100.0, 100), name="weights")
    mask = K.ones(weight.get_shape())
    threshold = K.zeros([])

    p = pruning_impl.Pruning(
        pruning_vars=[(weight, mask, threshold)],
        training_step_fn=self.training_step_fn,
        pruning_schedule=self.constant_sparsity,
        block_size=self.block_size,
        block_pooling_type=self.block_pooling_type)

    mask_before_pruning = K.get_value(mask)
    self.assertAllEqual(np.count_nonzero(mask_before_pruning), 100)

    if context.executing_eagerly():
      p.conditional_mask_update()
    else:
      K.get_session().run(p.conditional_mask_update())

    mask_after_pruning = K.get_value(mask)
    self.assertAllEqual(np.count_nonzero(mask_after_pruning), 50)

  def testConstructsMaskAndThresholdCorrectly(self):
    p = pruning_impl.Pruning(
        lambda: 0, None,
        # Sparsity math often returns values with small tolerances.
        lambda x: (True, 0.200000018),
        (1, 1), None)

    # input matrix is [ 1.0, 2.0, ..., 8.0, 9.0, 10.0 ]
    threshold, mask = p._update_mask(np.arange(1, 11))

    self.assertEqual(3, K.get_value(threshold))
    self.assertAllEqual(
        # expected matrix is [ 0.0, 0.0, 1.0, 1.0 ... 1.0 ]
        np.concatenate((np.zeros(2), np.ones(8))), K.get_value(mask))

  def _blockMasking(self, block_size, block_pooling_type, weight,
                    expected_mask):
    mask = K.ones(weight.get_shape())
    threshold = K.zeros([])

    # Set up pruning
    p = pruning_impl.Pruning(
        pruning_vars=[(weight, mask, threshold)],
        training_step_fn=self.training_step_fn,
        pruning_schedule=self.constant_sparsity,
        block_size=block_size,
        block_pooling_type=block_pooling_type)

    _, new_mask = p._maybe_update_block_mask(weight)
    # Check if the mask is the same size as the weights
    self.assertAllEqual(new_mask.get_shape(), weight.get_shape())
    mask_after_pruning = K.get_value(new_mask)
    self.assertAllEqual(mask_after_pruning, expected_mask)

  def testBlockMaskingAvg(self):
    block_size = (2, 2)
    block_pooling_type = "AVG"
    weight = K.variable([[0.1, 0.1, 0.2, 0.2], [0.1, 0.1, 0.2, 0.2],
                         [0.3, 0.3, 0.4, 0.4], [0.3, 0.3, 0.4, 0.4]])
    expected_mask = [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0],
                     [1., 1., 1., 1.], [1., 1., 1., 1.]]

    self._blockMasking(block_size, block_pooling_type, weight, expected_mask)

  def testBlockMaskingMax(self):
    block_size = (2, 2)
    block_pooling_type = "MAX"
    weight = constant_op.constant([[0.1, 0.0, 0.2, 0.0], [0.0, -0.1, 0.0, -0.2],
                                   [0.3, 0.0, 0.4, 0.0], [0.0, -0.3, 0.0,
                                                          -0.4]])
    expected_mask = [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0],
                     [1., 1., 1., 1.], [1., 1., 1., 1.]]

    self._blockMasking(block_size, block_pooling_type, weight, expected_mask)

  def testBlockMaskingWithHigherDimensionsRaisesError(self):
    block_size = (2, 2)
    block_pooling_type = "AVG"
    # Weights as in testBlockMasking, but with one extra dimension.
    weight = constant_op.constant([[[0.1, 0.1, 0.2, 0.2], [0.1, 0.1, 0.2, 0.2],
                                    [0.3, 0.3, 0.4, 0.4], [0.3, 0.3, 0.4,
                                                           0.4]]])
    expected_mask = [[[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0],
                      [1., 1., 1., 1.], [1., 1., 1., 1.]]]

    # Block masking should only be used with 2 Dimensional weights.
    with self.assertRaises(ValueError):
      self._blockMasking(block_size, block_pooling_type, weight, expected_mask)

  def testConditionalMaskUpdate(self):
    weight = K.variable(np.linspace(1.0, 100.0, 100), name="weights")
    mask = K.ones(weight.get_shape())
    threshold = K.zeros([])

    def linear_sparsity(step):
      sparsity_val = ops.convert_to_tensor(
          [0.0, 0.1, 0.1, 0.3, 0.3, 0.5, 0.5, 0.5, 0.5, 0.5])
      return ops.convert_to_tensor(True), sparsity_val[step]

    # Set up pruning
    p = pruning_impl.Pruning(
        pruning_vars=[(weight, mask, threshold)],
        training_step_fn=self.training_step_fn,
        pruning_schedule=linear_sparsity,
        block_size=self.block_size,
        block_pooling_type=self.block_pooling_type)

    non_zero_count = []
    for _ in range(10):
      if context.executing_eagerly():
        p.conditional_mask_update()
        p.weight_mask_op()
        state_ops.assign_add(self.global_step, 1)
      else:
        K.get_session().run(p.conditional_mask_update())
        K.get_session().run(p.weight_mask_op())
        K.get_session().run(state_ops.assign_add(self.global_step, 1))

      non_zero_count.append(np.count_nonzero(K.get_value(weight)))

    # Weights pruned at steps 1,3,5
    expected_non_zero_count = [100, 90, 90, 70, 70, 50, 50, 50, 50, 50]
    self.assertAllEqual(expected_non_zero_count, non_zero_count)

if __name__ == "__main__":
  test.main()


text:
from scipy import interpolate
from itertools import islice
import numpy as np
import platform
import csv
import os

class TwoDimLookup_motor:

    def __init__(self, gearRatio, tireRadius, CoG_X, m, CoP_X, C_la, rho, fr, Lift2Drag, DriveType):

        self.gearRatio = gearRatio
        self.tireRadius = tireRadius
        self.CoG_X = CoG_X  #Center of gravity (0 at front axis, 1 at rear axis)
        self.m = m      #Weight [kg]
        self.g = 9.81   #Gravity [m/s²]
        self.CoP_X = CoP_X  #Center of pressure
        self.C_la = C_la    #Coefficient (lift coefficient*front area)
        self.rho = rho
        self.fr = fr
        self.Lift2Drag = Lift2Drag
        self.DriveType = DriveType  #2WD or 4WD

        cwd = os.getcwd()
        if('Windows' in platform.platform()):
            file = cwd + '\CSV\motor_map.csv'
        else:
            cwd = cwd.replace('\\', '/')
            file = cwd + '/CSV/motor_map.csv'

        ## Import csv motor map
        Revolutions, Torque = [], [] # define variables

        data = csv.reader(open(file,'r'))     # open and read CSV file

        for row in islice(data, 1, None):     # Put data into list; start with row 18 (because of header)
            try:
                Revolutions.append(row[0])
                Torque.append(row[1])
            except IndexError:
                pass

        for i in range(len(Revolutions)):                                 # e.g. "1,12" string into 1.12 float
            Revolutions[i] = Revolutions[i].replace(",",".")
            try:
                Revolutions[i] = float(Revolutions[i])
            except ValueError:
                Revolutions[i] = 0
        Revolutions = np.asarray(Revolutions)

        for i in range(len(Torque)):                             # e.g. "1,12" string into 1.12 float
            Torque[i] = Torque[i].replace(",",".")
            try:
                Torque[i] = float(Torque[i])
            except ValueError:
                Torque[i] = 0
        Torque = np.asarray(Torque)

        #Covert engine speed into vehicle speed (m/s)
        vehicle_speed = (np.pi * Revolutions)/(30 * self.gearRatio) * self.tireRadius
        # Convert Torque into longitudinal force
        x_force = (Torque * self.gearRatio)/self.tireRadius

        # Create function for F over v
        self.motorMap = interpolate.interp1d(vehicle_speed, x_force, bounds_error=False, fill_value=0)

    def ax_motor(self, speed_values):
        # Calculate motor force for each speed value
        motor_force = []
        resistance_force = []
        speed = [i[0] for i in speed_values]

        for i in range(len(speed)):
            motor_force.append(self.motorMap(speed[i]))
            resistance_force.append(self.Driving_Resistances(speed[i]))

        # 2WD
        if self.DriveType == '2WD':
            ax_motor = 2 * (np.asarray(motor_force) - np.asarray(resistance_force))/ self.m

        # 4WD
        if self.DriveType == '4WD':
            ax_motor = 4 * (np.asarray(motor_force) - np.asarray(resistance_force))/ self.m

        for i in range(len(ax_motor)):
            if ax_motor[i] < 0:
                ax_motor[i] = 0

#         # Plots
#         a, axarr = plt.subplots(2, sharex=True)
#         axarr[0].plot(speed, resistance_force, 'g', label = 'Driving resistance')
#         axarr[0].plot(speed, motor_force, 'r', label = 'Motor force at tire')
#         axarr[0].set_title('Speed vs. Force')
#         axarr[0].set_xlabel('Speed [m/s]')
#         axarr[0].set_ylabel('Force [N]')
#         axarr[0].legend(shadow=True, fancybox=True)
#         axarr[0].grid(True)
#
#         axarr[1].plot(speed, ax_motor, label = 'x - acceleration')
#         axarr[1].set_title('Speed vs. Acceleration')
#         axarr[1].set_xlabel('Speed [m/s]')
#         axarr[1].set_ylabel('Acceleration [m/s^2]')
#         axarr[1].legend(shadow=True, fancybox=True)
#         axarr[1].grid(True)
#         plt.show()

        return ax_motor

    def Driving_Resistances(self, speed):

        # Mass
        FZ_m_F = (1-self.CoG_X) * self.m * self.g
        FZ_m_R = self.CoG_X * self.m * self.g

        #Downforce
        FZ_d_F = (1-self.CoP_X) * self.C_la * 0.5*self.rho * speed**2
        FZ_d_R = (self.CoP_X) * self.C_la * 0.5*self.rho * speed**2

        # FZ ovr
        FZ_ovr_F = FZ_m_F + FZ_d_F
        FZ_ovr_R = FZ_m_R + FZ_d_R

        FZ_ovr = FZ_ovr_F + FZ_ovr_R

        # Friction of Tires
        FR_tire = self.fr * FZ_ovr

        #Drag
        FR_aero = self.C_la/self.Lift2Drag * 0.5*self.rho * speed**2

        # Sum of driving resistances
        FR_ovr = FR_tire + FR_aero

        return FR_ovr

reason: import numpy,from scipy


text:
#!/usr/bin/env python3

import numpy as np
import random
import os

counter = 0

def random_matrix(size):
    lists = []
    list = []
    for j in range(size):
        for i in range(size):
            list.append(random.randint(1, 100))
        lists.append(list)
        list = []
    return lists

def new_matrix(r, c):
    """Create a new matrix filled with zeros."""
    matrix = [[0 for row in range(r)] for col in range(c)]
    global counter
    counter = counter + r*c
    return matrix

def split(matrix):
    """Split matrix into quarters."""
    a = b = c = d = matrix
    global counter

    while len(a) > len(matrix)/2:
        a = a[:len(a)//2]
        b = b[:len(b)//2]
        c = c[len(c)//2:]
        d = d[len(d)//2:]
        counter = counter + 1

    while len(a[0]) > len(matrix[0])//2:
        for i in range(len(a[0])//2):
            counter = counter + 1
            a[i] = a[i][:len(a[i])//2]
            b[i] = b[i][len(b[i])//2:]
            c[i] = c[i][:len(c[i])//2]
            d[i] = d[i][len(d[i])//2:]

    return a, b, c, d
    # matrix = np.array(matrix)
    # # print(matrix.shape)
    # row, col = matrix.shape
    # row2, col2 = row//2, col//2
    # return matrix[:row2, :col2], matrix[:row2, col2:], matrix[row2:, :col2], matrix[row2:, col2:]

def add_matrix(a, b):
    global counter

    if type(a) == int:
        d = a + b
    else:
        d = []
        for i in range(len(a)):
            c = []
            for j in range(len(a[0])):
                c.append(a[i][j] + b[i][j])
                counter = counter + 1
            d.append(c)
    return d

def subtract_matrix(a, b):
    global counter

    if type(a) == int:
        d = a - b
    else:
        d = []
        for i in range(len(a)):
            c = []
            for j in range(len(a[0])):
                c.append(a[i][j] - b[i][j])
                counter = counter + 1
            d.append(c)
    return d

def strassen(x, y, n):
    # base case: 1x1 matrix
    global counter

    if n == 1:
        z = [[0]]
        z[0][0] = x[0][0] * y[0][0]
        counter = counter + 1
        return z
    else:
        # split matrices into quarters
        a, b, c, d = split(x)
        e, f, g, h = split(y)

        # p1 = a*(f-h)
        p1 = strassen(a, subtract_matrix(f, h), n/2)

        # p2 = (a+b)*h
        p2 = strassen(add_matrix(a, b), h, n/2)

        # p3 = (c+d)*e
        p3 = strassen(add_matrix(c, d), e, n/2)

        # p4 = d*(g-e)
        p4 = strassen(d, subtract_matrix(g, e), n/2)

        # p5 = (a+d)*(e+h)
        p5 = strassen(add_matrix(a, d), add_matrix(e, h), n/2)

        # p6 = (b-d)*(g+h)
        p6 = strassen(subtract_matrix(b, d), add_matrix(g, h), n/2)

        # p7 = (a-c)*(e+f)
        p7 = strassen(subtract_matrix(a, c), add_matrix(e, f), n/2)

        z11 = add_matrix(subtract_matrix(add_matrix(p5, p4), p2), p6)

        z12 = add_matrix(p1, p2)

        z21 = add_matrix(p3, p4)

        z22 = add_matrix(subtract_matrix(subtract_matrix(p5, p3), p7), p1)

        z = new_matrix(len(z11)*2, len(z11)*2)
        for i in range(len(z11)):
            for j in range(len(z11)):
                z[i][j] = z11[i][j]
                z[i][j+len(z11)] = z12[i][j]
                z[i+len(z11)][j] = z21[i][j]
                z[i+len(z11)][j+len(z11)] = z22[i][j]
                counter = counter + 1

        return z

if __name__ == '__main__':
    for i in range(100):
        size = 2 ** random.randint(1, 5)
        depth = 0
        x = random_matrix(size)
        y = random_matrix(size)

        path = "./strassen"
        try:
            os.mkdir(path)
        except OSError as error:
            pass
        strassen(x, y, size)

        with open("./strassen/traces", 'a') as f:
            print("{};{}".format(size, counter), file=f)
        counter = 0

max_stars_repo_name: JIexa/complexity-recurrence

max_issues_repo_name: JIexa/complexity-recurrence

max_forks_repo_name: JIexa/complexity-recurrence