text:
from scipy import signal
from pylab import rcParams
import pylab
from pdb import set_trace as bp
from matplotlib.pyplot import figure
import argparse
import datetime
import math
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from pandas.plotting import register_matplotlib_converters
import random
import statsmodels as sm
from tools import Tools as tl
register_matplotlib_converters()

def args():
	parser = argparse.ArgumentParser(description='LSTM RNN for Time Series Forecasting')
	parser.add_argument('ticker', type=str, help='Provide a ticker name from Yahoo!Finance')
	parser.add_argument('date', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d').date(), help='Fetch data from: yyyy,m,d')
	parser.add_argument('smooth', type=int, help='Number of times for centered moving average')
	args = parser.parse_args()
	return args

import numpy
import math
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
# train_size - range (0.0 - 1.0)

def create_dataset(dataset, look_back=5):
    dataX, dataY = [], []
    for i in range(len(dataset)-look_back-1):
        a = dataset[i:(i+look_back), 0]
        dataX.append(a)
        dataY.append(dataset[i+look_back, 0])
    return numpy.array(dataX), numpy.array(dataY)

def lstm(train_size, past_values, epochs, units, data, batchsize, name):
	dataset = data.values
	dataset = dataset.astype('float32')
	scaler = MinMaxScaler(feature_range=(0, 1))
	dataset = scaler.fit_transform(dataset)

	train_size = int(len(dataset) * train_size)
	test_size = len(dataset) - train_size

	train, test = dataset[0:train_size,:], dataset[train_size:len(dataset),:]
	print("Using",len(train),"samples for training and" , len(test) , "for testing.")

	#trainX, trainY = tl.split_window(train, past_values)
	#testX, testY = tl.split_window(test, past_values)
	trainX, trainY = create_dataset(train, past_values)
	testX, testY = create_dataset(test, past_values)

	trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))
	testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1]))

	# defining model
	model = Sequential()
	model.add(LSTM(units, input_shape=(1, past_values)))
	model.add(Dense(1))
	model.compile(loss='mean_squared_error', optimizer='adam')
	model.fit(trainX, trainY, epochs, batchsize, verbose=3)

	trainPredict = model.predict(trainX)
	testPredict = model.predict(testX)

	model.summary()

	trainPredict = scaler.inverse_transform(trainPredict)
	trainY = scaler.inverse_transform([trainY])
	testPredict = scaler.inverse_transform(testPredict)
	testY = scaler.inverse_transform([testY])

	rmse = math.sqrt(mean_squared_error(testY[0], testPredict[:,0]))
	nmse = tl.NMSE(testY[0], testPredict[:, 0])
	print("RMSE = ", rmse)
	print("NMSE = ", nmse)
	# shift train predictions for plotting
	trainPredictPlot = np.empty_like(dataset)
	trainPredictPlot[:, :] = np.nan
	trainPredictPlot[past_values:len(trainPredict)+past_values, :] = trainPredict

	# shift test predictions for plotting
	testPredictPlot = np.empty_like(dataset)
	testPredictPlot[:, :] = np.nan
	testPredictPlot[len(trainPredict)+(past_values*2)+1  :len(dataset)-1:, :] = testPredict
	figure(num=None, figsize=(16, 9), dpi=60, facecolor='w', edgecolor='k')

	# plot baseline and predictions
	plt.title("LSTM: " + name)
	plt.plot(scaler.inverse_transform(dataset), label='true', lw=0.8, color='green', ls='dashdot')
	plt.plot(trainPredictPlot, label='training', lw=0.3, color='blue')
	plt.plot(testPredictPlot, label='testing', lw=0.3, color='red')
	plt.legend()
	plt.show()

def main(args):
	numpy.random.seed(1)
	ticker = args.ticker
	date = args.date
	smooth = args.smooth
	prices, name = tl.get_prices(date, ticker)
	# prices to dataframe;
	pnorm = tl.normalize(prices)
	ptrend = signal.detrend(pnorm)
	for _ in range(smooth):
		ptrend = tl.moving_average(ptrend)
	prices = pd.DataFrame(ptrend)

	#plt.title(name + " from: " + str(date))
	#plt.plot(prices)
	#plt.show()

	#lstm(train_size, past_values, epochs, units, data, batchsize, name):
	lstm(0.67, 3, 1, 50, prices, 20, name)
if __name__ == '__main__':
	main()

reason: import numpy,from scipy,import statsmodels


text:
import numpy as np
import random
import configs
from colorama import Fore, Back, Style

class LargeGrid:
    def __init__(self):
        self.currentRightColumn = configs.GRIDWIDTH
        self.currentLeftColumn = 0
        self.grid = []
        self.numericGrid = []
        # stores the location of magnet in grid
        self.startMagnetColumn = -1
        self.magnetX = -1
        self.magnetY = -1

    '''
    create the grid randomly
    '''

    def createGrid(self, N):

        # Make the basic objects of the game
        coins = "$$ $$ $$ $$$$ $$ $$ $$$$ $$ $$ $$"
        horizontalBeam = "           zzzzzzzzzzz           "
        verticalBeam = "zzzzzz"
        mainAngledBeam = "z      z      z      z      z      z"
        offAngledBeam = "     z    z    z    z    z    z     "
        dragon = "DDDD"
        magnet = "MMMM"

        coins = np.array(list(coins))
        horizontalBeam = np.array(list(horizontalBeam))
        verticalBeam = np.array(list(verticalBeam))
        mainAngledBeam = np.array(list(mainAngledBeam))
        offAngledBeam = np.array(list(offAngledBeam))
        dragon = np.array(list(dragon))
        magnet = np.array(list(dragon))

        coins = coins.reshape(3, 11)
        horizontalBeam = horizontalBeam.reshape(3, 11)
        verticalBeam = verticalBeam.reshape(6, 1)
        mainAngledBeam = mainAngledBeam.reshape(6, 6)
        offAngledBeam = offAngledBeam.reshape(6, 6)
        dragon = dragon.reshape(2, 2)
        magnet = magnet.reshape(2, 2)

        obstaclesSizes = \
        [[3,11],[3,11],[6,1],[6,6],[6,6],[2,2],[2,2]]
        obstacles = [
            coins, horizontalBeam, verticalBeam, mainAngledBeam, offAngledBeam,
            dragon, magnet
        ]
        '''
        we need to make a H*W*N grid which means N repetition of H*W grid
        '''
        H = configs.GRIDHEIGHT
        W = configs.GRIDWIDTH

        self.grid = [' ' for i in range(H * W * N)]
        self.grid = np.array(self.grid)
        self.grid = self.grid.reshape(H, W * N)

        self.numericGrid = [0 for i in range(H * W * N)]
        self.numericGrid = np.array(self.numericGrid)
        self.numericGrid = self.numericGrid.reshape(H, W * N)

        # create the ground
        for i in range(W * N):
            self.grid[H - 1][i] = self.grid[H - 2][i] = 'X'
            self.numericGrid[H-1][i] = self.numericGrid[H-2][i] =\
            configs.GROUNDID

        # create the sky
        for i in range(W * N):
            self.grid[0][i] = self.grid[1][i] = 'X'
            self.numericGrid[0] = self.numericGrid[1] = \
            configs.SKYID

        obstacleInterval = 30

        # After every 30 character a obstacle/coins will appear
        loops = int((W * N) / obstacleInterval)
        # randomly generate the starting location of the obstacle/coins
        currentStartCol = 0
        dragonFound = False
        magnetFound = False

        for loop in range(loops):
            if loop == 0:
                currentStartCol += obstacleInterval
                continue
            x_start1 = random.randrange(5, 12, 1)
            y_start1 = random.randrange(5, 8, 1)
            x_start2 = random.randrange(20, 28, 1)
            y_start2 = random.randrange(9, 12, 1)
            obj_type1 = random.randrange(0, configs.NUMBEROFOBSTACLES, 1)
            obj_type2 = random.randrange(0, configs.NUMBEROFOBSTACLES, 1)

            if obj_type1 + 1 == configs.DRAGONID:
                if dragonFound:
                    obj_type1 = configs.COINID
                else:
                    dragonFound = True
            if obj_type1 + 1 == configs.MAGNETID:
                if magnetFound:
                    obj_type1 = configs.COINID
                else:
                    magnetFound = True
                    self.startMagnetColumn = currentStartCol + y_start1
            if obj_type2 + 1 == configs.DRAGONID:
                if dragonFound:
                    obj_type2 = configs.COINID
                else:
                    dragonFound = True
            if obj_type2 + 1 == configs.MAGNETID:
                if magnetFound:
                    obj_type2 = configs.COINID
                else:
                    magnetFound = True
                    self.startMagnetColumn = currentStartCol + y_start2

            for i in range(obstaclesSizes[obj_type1][0]):
                for j in range(obstaclesSizes[obj_type1][1]):
                    char = obstacles[obj_type1][i][j]
                    self.grid[x_start1 + i][currentStartCol + y_start1 +
                                            j] = char
                    if char in ['z', '$', 'M', 'D']:
                        self.numericGrid[x_start1+i][currentStartCol+y_start1+j]\
                         =  obj_type1+1
            for i in range(obstaclesSizes[obj_type2][0]):
                for j in range(obstaclesSizes[obj_type2][1]):
                    char = obstacles[obj_type2][i][j]
                    self.grid[x_start2 + i][currentStartCol + y_start2 +
                                            j] = char
                    if char in ['z', '$', 'M', 'D']:
                        self.numericGrid[x_start2+i][currentStartCol+y_start2+j]\
                         =  obj_type2+1

            currentStartCol += obstacleInterval

'''
This will be the class which will interface with the rest of the game.
It will also manage the large Grid.
'''

class SmallGrid:
    def __init__(self):
        self.largeGrid = LargeGrid()
        self.grid = [['0' for j in range(configs.GRIDWIDTH)] for i in\
            range(configs.GRIDHEIGHT)]
        self.numericGrid = [[0 for j in range(configs.GRIDWIDTH)] for i in \
            range(configs.GRIDHEIGHT)]
        self.grid = np.array(list(self.grid))
        self.grid = self.grid.reshape(configs.GRIDHEIGHT, configs.GRIDWIDTH)
        self.numericGrid = np.array(list(self.numericGrid))
        self.numericGrid =\
        self.numericGrid.reshape(configs.GRIDHEIGHT,configs.GRIDWIDTH)
        self.N = 0

    # creates the large grid. This needs to be called only once.
    def initialiseLargeGrid(self, N):
        self.largeGrid.createGrid(N)
        self.N = N

    # moves the grid leftwards
    def progressGame(self, step):
        W = configs.GRIDWIDTH

        self.largeGrid.currentLeftColumn = \
            (self.largeGrid.currentLeftColumn+step)%(W*self.N)

        self.largeGrid.currentRightColumn = \
            (self.largeGrid.currentRightColumn+step)%(W*self.N)

    # moves the grid rightwards
    def regressGame(self, step):
        W = configs.GRIDWIDTH

        self.largeGrid.currentLeftColumn = \
            (self.largeGrid.currentLeftColumn -step + W*self.N)%(W*self.N)

        self.largeGrid.currentRightColumn = \
            (self.largeGrid.currentRightColumn -step + W*self.N)%(W*self.N)

    def loadSmallGrid(self):
        l = self.largeGrid.currentLeftColumn
        r = self.largeGrid.currentRightColumn
        W = configs.GRIDWIDTH
        '''
        There is a flaw in the game the enemy dragon fires the ice Ball they get
        repeated when the large grid hits end and start repeating from the
        beginning, so we have more ice balls on the screen then there should be
        '''
        iceBall = ['o', '\\', '/', '-']
        assert l >= 0 and l < W * self.N, "l correct"
        assert r >= 0 and r < W * self.N, "r correct"
        for i in range(configs.GRIDHEIGHT):
            if r > l:
                for j in range(self.largeGrid.currentLeftColumn,
                               self.largeGrid.currentRightColumn):
                    self.grid[i][j - l] = self.largeGrid.grid[i][j]
                    self.numericGrid[i][j -
                                        l] = self.largeGrid.numericGrid[i][j]
            else:
                cnt = 0
                for j in range(l, W * self.N):
                    cnt += 1
                    self.grid[i][j - l] = self.largeGrid.grid[i][j]
                    self.numericGrid[i][j -
                                        l] = self.largeGrid.numericGrid[i][j]
                for j in range(r):
                    self.grid[i][cnt + j] = self.largeGrid.grid[i][j]
                    self.numericGrid[i][cnt +
                                        j] = self.largeGrid.numericGrid[i][j]

    # Renders the small screen using colors(colorama)
    def printSmallGrid(self):

        coinsColor = Back.CYAN
        groundColor = Back.GREEN
        beamColor1 = Back.YELLOW
        beamColor2 = Back.RED
        bonusColor = Back.MAGENTA
        dragonColor = Back.GREEN
        magnetColor = Back.YELLOW

        for i in range(configs.GRIDHEIGHT):
            for j in range(configs.GRIDWIDTH):

                char = ''

                if self.numericGrid[i][j] == configs.COINID:
                    char = coinsColor + Fore.BLACK + '$'
                elif self.numericGrid[i][j] in [
                        configs.HORIZONTALBEAMID, configs.VERTICALBEAMID
                ]:
                    char = beamColor1 + Fore.RED + 'z'
                elif self.numericGrid[i][j] in\
                [configs.MAINANGLEDBEAMID, configs.OFFANGLEDBEAMID]:
                    char = beamColor2 + Fore.YELLOW + 'z'
                elif self.numericGrid[i][j] == configs.GROUNDID:
                    char = groundColor + Fore.RED + 'X'
                elif self.numericGrid[i][j] == configs.SKYID:
                    char = configs.BACKGROUNDCOLOR + Fore.RED + 'X'
                elif self.numericGrid[i][j] == configs.MAGNETID:
                    char = magnetColor + Fore.BLACK + 'M'
                elif self.numericGrid[i][j] == configs.DRAGONID:
                    char = dragonColor + Fore.MAGENTA + 'D'
                elif self.numericGrid[i][j] == configs.ICEBALLID:
                    char = configs.BACKGROUNDCOLOR + configs.CHARACTERFORECOLOR + self.grid[i][j]
                else:
                    char = configs.BACKGROUNDCOLOR + ' '

                print(char, end='')

            print(Style.RESET_ALL + '')

ext: py

reason: import numpy


text:
#!/usr/bin/env python
# coding: utf-8
from __future__ import absolute_import, division, print_function

import os
import torch
import logging
import argparse
import numpy as np
import pandas as pd
import warnings
from ood_main import load_dataset
warnings.filterwarnings('ignore')

from ood_main import load_extra_dataset

from simpletransformers.classification import  ClassificationModel
from simpletransformers.language_modeling import LanguageModelingModel
logging.basicConfig(level=logging.INFO)
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.WARNING)
seed=42

def set_binary_label(dataframe, indomain, col='labels'):
    if indomain:
        dataframe[col].values[:] = 1
    else:
        dataframe[col].values[:] = 0

def train_process(args):
    # load data
    data_type = args.data_type.strip()
    print("data type %s"%( data_type))
    train_df, num_classes = load_dataset('clinc150_train', data_type=data_type)
    test_df, _ = load_dataset('clinc150_test', data_type=data_type)
    eval_df, _ = load_dataset('clinc150_val', data_type=data_type)

    if args.model_class == 'bert':
        model = ClassificationModel(
        'bert',
        'bert-base-uncased',
        num_labels=num_classes,
        use_cuda=True,
        cuda_device=int(args.gpu_id),
        args={'num_train_epochs': 5,
              'fp16':False,
              'n_gpu':int(args.n_gpu),
              'learning_rate': 4e-5,
              'warmup_ratio': 0.10,
              'train_batch_size': 32,
              'eval_batch_size': 32,
              'evaluate_during_training': False,
              'evaluate_during_training_steps': 2000,
              'do_lower_case': True,
              'reprocess_input_data': True,
              'overwrite_output_dir': False,
              'output_dir': './models/%s_outputs/'%(data_type),
              'best_model_dir': "./models/%s_outputs/best_model"%(data_type),
              'cache_dir': "./models/%s_cache_dir/"%(data_type)})
    else:
        raise NotImplementedError

    model.train_model(train_df, eval_df=eval_df)
    result, model_outputs, wrong_predictions = model.eval_model(test_df)
    # pdb.set_trace()
    print(result)
    print("---------------")

def finetune_bcad(args):
    # load data
    sed = 42
    data_type = args.data_type.strip()
    print("data type %s"%( data_type))
    if args.load_path:
        output_dir = './models/{}_{}_ft_MLM_binary_intent_outputs_{}/'.format(args.model_class, data_type, str(args.neg_sample))
    else:
        output_dir = './models/{}_{}_ft_binary_intent_outputs_{}/'.format(args.model_class, data_type, str(args.neg_sample))
    print (output_dir)
    os.makedirs(output_dir, exist_ok=True)
    if data_type == "sst":
        train_df = load_extra_dataset("./dataset/sst/sst-train.txt")
        eval_df = load_extra_dataset("./dataset/sst/sst-dev.txt")
        test_df = load_extra_dataset("./dataset/sst/sst-test.txt")
        for df in [train_df, test_df, eval_df]:
            set_binary_label(df, indomain=True) # set label to one (in domain)
    else:
        train_df = load_dataset('clinc150_train', data_type=data_type)
        test_df = load_dataset('clinc150_test', data_type=data_type)
        eval_df = load_dataset('clinc150_val', data_type=data_type)
        for df in [train_df, test_df, eval_df]:
            set_binary_label(df, indomain=True) # set label to one (in domain)
        # neg_df = load_extra_dataset("./dataset/sst-train.txt")
        # neg_val_df = load_extra_dataset("./dataset/sst-dev.txt")
        # neg_test_df = load_extra_dataset("./dataset/sst-test.txt")
    print ("training df size", len(train_df))
    book_df = load_extra_dataset("./dataset/bookcorpus/subset_books.txt")
    set_binary_label(book_df, indomain=False)
    wiki_df = load_extra_dataset("./dataset/wikipedia/squad_train_wiki.txt")
    set_binary_label(wiki_df, indomain=False)
    # pdb.set_trace()
    # train_df = train_df.sample(n=200, random_state=seed)
    neg_book_df = book_df.sample(n=args.neg_sample, random_state=seed)
    neg_book_val_df = book_df.sample(n=1000, random_state=seed)
    neg_book_test_df = book_df.sample(n=2000, random_state=seed)
    neg_wiki_df = wiki_df.sample(n=args.neg_sample, random_state=seed)
    neg_wiki_val_df = wiki_df.sample(n=1000, random_state=seed)
    neg_wiki_test_df = wiki_df.sample(n=2000, random_state=seed)

    if data_type != "sst":
        neg_book_df['text'] = neg_book_df['text'].apply(lambda x: x.strip(".? "))
        neg_book_val_df['text'] = neg_book_val_df['text'].apply(lambda x: x.strip(".? "))
        neg_book_test_df['text'] = neg_book_test_df['text'].apply(lambda x: x.strip(".? "))
        neg_wiki_df['text'] = neg_wiki_df['text'].apply(lambda x: x.strip(".? "))
        neg_wiki_val_df['text'] = neg_wiki_val_df['text'].apply(lambda x: x.strip(".? "))
        neg_wiki_test_df['text'] = neg_wiki_test_df['text'].apply(lambda x: x.strip(".? "))

    train_df = pd.concat([train_df, neg_book_df, neg_wiki_df])
    val_df = pd.concat([eval_df, neg_book_val_df, neg_wiki_val_df])

    print ("train_df", len(train_df['labels']), train_df['labels'][10:20], train_df['text'][0:10])
    if args.load_path:
        load_path = args.load_path
    else:
        load_path = 'bert-base-uncased' if args.model_class == 'bert' else 'roberta-base'
    if args.model_class == 'bert' or args.model_class == 'roberta':
        model = ClassificationModel(
        args.model_class,
        load_path,
        num_labels=2,
        use_cuda=True,
        cuda_device=int(args.gpu_id),
        args={'num_train_epochs': 2,
              'fp16':False,
              'n_gpu':int(args.n_gpu),
              'learning_rate': 4e-5,
              'warmup_ratio': 0.10,
              'do_lower_case': True,
              "max_seq_length": 256,
              "train_batch_size": 16,
              'reprocess_input_data': True,
              'overwrite_output_dir': True,
              'save_model_every_epoch': False,
              'evaluate_during_training': True,
              'evaluate_during_training_verbose': True,
              'evaluate_during_training_steps': 2000,
              'output_dir': output_dir,
              'best_model_dir': "{}/best_model".format(output_dir),
              'cache_dir': output_dir.replace("outputs", "cache_dir")})
    else:
        raise NotImplementedError

    # model.train_model(pd.concat([train_df, neg_df]), eval_df=pd.concat([eval_df, neg_val_df]))
    model.train_model(train_df, eval_df=eval_df)
    # result, model_outputs, wrong_predictions = model.eval_model(pd.concat([test_df, neg_test_df]))
    result, model_outputs, wrong_predictions = model.eval_model(eval_df)
    # pdb.set_trace()
    print(result)
    print (output_dir)
    print("---------------")

def finetune_imlm(args):
    # load data
    data_type = args.data_type.strip()
    print("data type %s"%( data_type))
    output_dir = './models/{}_{}_mlm_ft_outputs/'.format(args.model_class, data_type)
    os.makedirs(output_dir, exist_ok=True)
    train_file = os.path.join(output_dir, 'train.txt')
    eval_file = os.path.join(output_dir, 'eval.txt')
    test_file = os.path.join(output_dir, 'test.txt')
    if not (os.path.exists(train_file) and os.path.exists(eval_file)):
        if data_type == "sst":
            train_df = load_extra_dataset("./dataset/sst/sst-train.txt", label=1)
            eval_df = load_extra_dataset("./dataset/sst/sst-dev.txt", label=1)
            # test_df = load_extra_dataset("./dataset/sst-test.txt", label=1)
        else:
            train_df = load_dataset('clinc150_train', data_type=data_type)
            # test_df = load_dataset('clinc150_test', data_type=data_type)
            eval_df  = load_dataset('clinc150_val', data_type=data_type)
        # pdb.set_trace()
        print ("number of training instances", len(train_df['text']))
        with open(train_file, "w") as f:
            for line in train_df['text']:
                f.write(line + "\n")

        print ("number of eval instances", len(eval_df['text']))
        with open(eval_file, "w") as f:
            for line in eval_df['text']:
                f.write(line + "\n")

    if args.model_class == 'bert' or args.model_class == 'roberta':
        model = LanguageModelingModel(
        args.model_class,
        'bert-base-uncased' if args.model_class == 'bert' else 'roberta-base',
        use_cuda=True,
        cuda_device=int(args.gpu_id),
        args={'num_train_epochs': 10,
              'fp16':False,
              'n_gpu':int(args.n_gpu),
              'do_lower_case': True,
              "max_seq_length": 128,
              "train_batch_size": 4,
              'reprocess_input_data': True,
              'overwrite_output_dir': True,
              'save_model_every_epoch': False,
              'evaluate_during_training': True,
              'output_dir': output_dir,
              'best_model_dir': "{}/best_model".format(output_dir),
              'cache_dir': output_dir.replace("outputs", "cache_dir")})
    else:
        raise NotImplementedError
    model.train_model(train_file, eval_file=eval_file)
    model.eval_model(eval_file)
    # print(result)
    print("---------------")

def main(args):
    run_type = args.type
    # set seed
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(seed)
        torch.cuda.set_device(args.gpu_id)
    if run_type == 'train_classifier':
        train_process(args)
    elif run_type == 'finetune_bcad':
        finetune_bcad(args)
    elif run_type == 'finetune_imlm':
        finetune_imlm(args)
    else:
        raise NotImplementedError

if __name__ == '__main__':
    parser = argparse.ArgumentParser("Bert Model OOD Fine-tuning")
    parser.add_argument('--type', default='finetune_bcad', choices=['train_classifier', 'finetune_bcad', 'finetune_imlm'])
    parser.add_argument('--model_class', default='bert', type=str)
    parser.add_argument('--gpu_id', default=0, type=int)
    parser.add_argument('--n_rep', default=1, type=int)
    parser.add_argument('--n_gpu', default=1, type=int)
    parser.add_argument('--load_path', default=None, type=str)
    parser.add_argument('--data_type', default='full', choices = ['clinic', 'sst'])
    parser.add_argument('--neg_sample',  default=7500, type=int)

    args = parser.parse_args()
    main(args)

max_stars_repo_name: mxnno/BERT-unsupervised-OOD

max_issues_repo_name: mxnno/BERT-unsupervised-OOD

max_forks_repo_name: mxnno/BERT-unsupervised-OOD


text:
import torch
import torch.nn as nn
import torch.nn.functional as F

import numpy as np

class CvConfidence(nn.Module):
    def __init__(self, device):
        super(CvConfidence, self).__init__()
        self._device = device

    def forward(self, prob_volume, disps):
        N, _, H, W = prob_volume.shape

        # generate coordinates
        n_coords = torch.arange(N, device=prob_volume.device, dtype=torch.long)
        n_coords = n_coords.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)

        x_coords = torch.arange(W, device=prob_volume.device, dtype=torch.long)
        y_coords = torch.arange(H, device=prob_volume.device, dtype=torch.long)
        y_coords = y_coords.unsqueeze(-1)
        # xl, yl = torch.meshgrid((x_coords, y_coords)) # with torch >= 0.4.1

        nl = n_coords.repeat((1, 1, H, W)).long()
        xl = x_coords.repeat((N, 1, H, 1)).long()
        yl = y_coords.repeat((N, 1, 1, W)).long()

        cv_confidence = prob_volume[nl, torch.round(disps).long(), yl, xl]
        return cv_confidence

class LrDistance(nn.Module):
    def __init__(self, device):
        super(LrDistance, self).__init__()
        self._device = device

    def forward(self, disps_lr, disps_rl):
        dispsLR = disps_lr.float()
        dispsRL = disps_rl.float()

        S, _, M, N = dispsLR.shape

        # generate coordinates
        x_coords = torch.arange(N, device=self._device, dtype=torch.float)
        y_coords = torch.arange(M, device=self._device, dtype=torch.float)

        xl = x_coords.repeat((S, M, 1)).unsqueeze(1)
        yl = y_coords.repeat((S, N, 1)).transpose(2, 1).unsqueeze(1)

        xr = xl - dispsLR

        # normalize coordinates for sampling between [-1, 1]
        xr_normed = (2 * xr / (N - 1)) - 1.0
        yl_normed = (2 * yl / (M - 1)) - 1.0

        # coords must have sahpe N x OH x OW x 2
        sample_coords = torch.stack((xr_normed[:, 0], yl_normed[:, 0]), dim=-1)
        dispsRL_warped = nn.functional.grid_sample(dispsRL, sample_coords)

        lr_distance = torch.abs(dispsLR + dispsRL_warped)
        lr_distance[(xr >= N) | (xr < 0)] = 100.0

        # import matplotlib.pyplot as plt
        # plt.ion()
        # plt.figure(), plt.imshow(disps_lr[0,0].detach().cpu()), plt.title('LR')
        # plt.figure(), plt.imshow(disps_rl[0,0].detach().cpu()), plt.title('RL')
        # plt.figure(), plt.imshow(dispsRL_warped[0,0].detach().cpu()), plt.title('RL WARPED')
        # plt.figure(), plt.imshow(lr_distance[0,0].detach().cpu()), plt.title('DIST')

        # import pdb
        # pdb.set_trace()

        return lr_distance

class LrCheck(nn.Module):
    def __init__(self, device, eps):
        super(LrCheck, self).__init__()
        self._device = device
        self._eps = eps

    def forward(self, lr_dists):
        lr_mask = torch.ones_like(lr_dists)
        lr_mask[lr_dists > self._eps] = 0.0

        zero = torch.tensor([0], dtype=torch.float).to(self._device)
        confidence = torch.max(self._eps - lr_dists, zero) / self._eps

        # import matplotlib.pyplot as plt
        # plt.ion()
        # plt.imshow(confidence.detach()[0,0]), plt.title('conf')
        # plt.figure(), plt.imshow(lr_mask.detach()[0,0]), plt.title('mask')

        # import pdb
        # pdb.set_trace()

        return lr_mask, confidence

class TemperatureSoftmax(nn.Module):
    def __init__(self, dim, init_temp=1.0):
        nn.Module.__init__(self)
        self.T = nn.Parameter(torch.ones(1).float().cuda() * init_temp, requires_grad=True)
        self.dim = dim

    def forward(self, x):
        return F.softmax(x / self.T, dim=self.dim)

class TemperatureSoftmin(nn.Module):
    def __init__(self, dim, init_temp=1.0):
        nn.Module.__init__(self)
        self.T = nn.Parameter(torch.ones(1).float().cuda() * init_temp, requires_grad=True)
        self.dim = dim

    def forward(self, x):
        return F.softmin(x / self.T, dim=self.dim)

class Pad(nn.Module):
    def __init__(self, divisor, extra_pad=(0, 0)):
        nn.Module.__init__(self)
        self.divisor = divisor
        self.extra_pad_h = extra_pad[0] # pad at top and bottom with specified value
        self.extra_pad_w = extra_pad[1] # pad at left and right with specified value

        self.l = 0
        self.r = 0
        self.t = 0
        self.b = 0

    def pad(self, x):
        N, C, H, W = x.shape

        w_add = 0
        while W % self.divisor != 0:
            W += 1
            w_add += 1

        h_add = 0
        while H % self.divisor != 0:
            H += 1
            h_add += 1

        # additionally pad kitti imgs
        self.l = self.extra_pad_w + np.ceil(w_add / 2.0).astype('int')
        self.r = self.extra_pad_w + np.floor(w_add / 2.0).astype('int')
        self.t = self.extra_pad_h + np.ceil(h_add / 2.0).astype('int')
        self.b = self.extra_pad_h + np.floor(h_add / 2.0).astype('int')

        padded = F.pad(x, (self.l, self.r, self.t, self.b), mode='reflect')
        return padded

    def forward(self, x):
        return self.pad(x)

class Unpad(nn.Module):
    def __init__(self):
        nn.Module.__init__(self)

    def unpad_NCHW(self, x, l, r, t, b):
        x = x[:, :, t:, l:]
        if b > 0:
            x = x[:, :, :-b, :]
        if r > 0:
            x = x[:, :, :, :-r]
        return x.contiguous()

    def unpad_NHWC(self, x, l, r, t, b):
        x = x[:, t:, l:, :]
        if b > 0:
            x = x[:, :-b, :, :]
        if r > 0:
            x = x[:, :, :-r, :]
        return x.contiguous()

    def unpad_flow_N2HWC(self, x, l, r, t, b):
        x = x[:, :, t:, l:, :]
        if b > 0:
            x = x[:, :,  :-b, :, :]
        if r > 0:
            x = x[:, :, :, :-r, :]
        return x.contiguous()

    def unpad_flow_N2CHW(self, x, l, r, t, b):
        x = x[:, :, :, t:, l:]
        if b > 0:
            x = x[:, :, :, :-b, :]
        if r > 0:
            x = x[:, :, :, :, :-r]
        return x.contiguous()

    def forward(self, x, l, r, t, b, NCHW=True):
        if NCHW:
            if len(x.shape) == 4:
                return self.unpad_NCHW(x, l, r, t, b)
            else:
                return self.unpad_flow_N2CHW(x, l, r, t, b)
        else:
            if len(x.shape) == 4:
                return self.unpad_NHWC(x, l, r, t, b)
            else:
                return self.unpad_flow_N2HWC(x, l, r, t, b)

class PadUnpad(Pad, Unpad):
    def __init__(self, net, divisor=1, extra_pad=(0, 0)):
        Pad.__init__(self, divisor, extra_pad)
        Unpad.__init__(self)

        self.net = net

    def forward(self, ipt):
        out = self.net.forward(self.pad(ipt))
        res = []
        for o in out:
            res.append(self.unpad_NCHW(o, self.l, self.r, self.t, self.b))
        return res

class PaddedConv2d(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, bias=False):
        super(PaddedConv2d, self).__init__()

        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size,
                              stride=stride, padding=0, dilation=dilation, bias=bias)

        self.dilation = dilation

    def forward(self, x):
        pd = (self.conv.kernel_size[0] // 2) * self.dilation
        x = F.pad(x, (pd, pd, pd, pd), mode='reflect')
        x = self.conv(x)
        return x

class ResidualBlock(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, padding=0,
                 activation='ReLU', transposed=False, with_bn=True, leaky_alpha=1e-2):
        super(ResidualBlock, self).__init__()

        self.convbnact1 = ConvBatchNormAct(in_channels, out_channels, kernel_size, stride, dilation,
                                      padding, activation, transposed, with_bn)

        bias = False # because the parameter is already contained in group-norm!!
        self.conv2 = PaddedConv2d(out_channels, out_channels, kernel_size, stride, dilation, bias)
        num_groups = out_channels
        self.bn2 = nn.GroupNorm(num_groups, out_channels, affine=True)

        if activation.lower() == 'relu':
            self.act2 = nn.ReLU(inplace=True)
        elif activation.lower() == 'leakyrelu':
            self.act2 = nn.LeakyReLU(negative_slope=leaky_alpha, inplace=True)
        elif activation.lower() == 'elu':
            self.act2 = nn.ELU(inplace=True)
        else:
            raise NotImplementedError("Activation " + activation + " is currently not implemented!")

    def forward(self, x):
        residual = x
        x = self.convbnact1(x)

        x = self.conv2(x)
        x = self.bn2(x)

        x += residual
        x = self.act2(x)
        return x

class ConvBatchNormAct(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, padding=0,
                 activation='ReLU', transposed=False, with_bn=True):
        super(ConvBatchNormAct, self).__init__()

        if with_bn:
            bias = False
        else:
            bias = True

        if transposed:
            self.conv = nn.ConvTranspose2d(in_channels, out_channels, kernel_size,
                                           stride=stride, padding=padding, dilation=dilation)
        else:
            self.conv = PaddedConv2d(in_channels, out_channels, kernel_size, stride=stride,
                                     dilation=dilation, bias=bias)

        self.bn = None
        if with_bn:
            #self.bn = nn.BatchNorm2d(out_channels, affine=True, track_running_stats=False)
            num_groups = out_channels
            self.bn = nn.GroupNorm(num_groups, out_channels, affine=True)

        if activation.lower() == 'relu':
            self.act = nn.ReLU(inplace=True)
        elif activation.lower() == 'leakyrelu':
            self.act = nn.LeakyReLU(inplace=True)
        elif activation.lower() == 'elu':
            self.act = nn.ELU(inplace=True)
        else:
            raise NotImplementedError("Activation " + activation + " is currently not implemented!")

    def forward(self, x):
        x = self.conv(x)
        if self.bn:
            x = self.bn(x)
        x = self.act(x)
        return x

class BaseNet(nn.Module):
    def __init__(self):
        super(BaseNet, self).__init__()

        self._divisor = -1
        self._out_channels = -1

    @property
    def divisor(self):
        return self._divisor

    @property
    def out_channels(self):
        return self._out_channels

    @property
    def num_output_levels(self):
        return self._num_output_levels

    def forward(self, *input):
        raise NotImplementedError

class StereoUnaryUnetDyn(BaseNet):
    def __init__(self, multi_level_output=False, activation='relu',
                 with_bn=True, with_upconv=False, with_output_bn=True):
        super(StereoUnaryUnetDyn, self).__init__()

        self.multi_level_output = multi_level_output
        self.with_upconv = with_upconv
        self.with_output_bn = with_output_bn
        self._divisor = 8
        self._out_channels = [64]
        if multi_level_output:
            self._out_channels = [64, 64, 64]

        self._num_output_levels = 1
        if multi_level_output:
            self._num_output_levels = 3

        self._out_channels = [32]
        if multi_level_output:
            self._out_channels = [32, 32, 32]

        self.conv1 = ConvBatchNormAct(3, 16, 3, padding=1, activation=activation, with_bn=with_bn)
        self.conv2 = ConvBatchNormAct(16, 16, 3, padding=1, activation=activation, with_bn=with_bn)
        self.pool1 = nn.MaxPool2d(2)

        self.conv3 = ConvBatchNormAct(16, 32, 3, padding=1, activation=activation, with_bn=with_bn)
        self.conv4 = ConvBatchNormAct(32, 32, 3, padding=1, activation=activation, with_bn=with_bn)
        self.pool2 = nn.MaxPool2d(2)

        self.conv5 = ConvBatchNormAct(32, 64, 3, padding=1, activation=activation, with_bn=with_bn)
        self.conv6 = ConvBatchNormAct(64, 64, 3, padding=1, activation=activation, with_bn=with_bn)

        if self.with_upconv:
            self.upconv7 = ConvBatchNormAct(64, 32, 3, stride=2, padding=0, activation=activation,
                                            transposed=True, with_bn=with_bn)
            self.conv8 = ConvBatchNormAct(64, 32, 3, padding=1, activation=activation,
                                          with_bn=with_bn)
        else:
            self.conv8 = ConvBatchNormAct(96, 32, 3, padding=1, activation=activation,
                                          with_bn=with_bn)
        self.conv9 = ConvBatchNormAct(32, 32, 3, padding=1, activation=activation, with_bn=with_bn)

        if self.with_upconv:
            self.upconv10 = ConvBatchNormAct(32, 16, 3, stride=2, padding=0, activation=activation,
                                             transposed=True, with_bn=with_bn)
            self.conv11 = ConvBatchNormAct(32, 32, 3, padding=1, activation=activation,
                                           with_bn=with_bn)
        else:
            self.conv11 = ConvBatchNormAct(48, 32, 3, padding=1, activation=activation,
                                           with_bn=with_bn)

        self.conv12 = PaddedConv2d(32, 32, 3)
        self.bn12 = None
        if with_bn and with_output_bn:
#            self.bn12 = nn.BatchNorm2d(32, affine=True, track_running_stats=False)
            self.bn12 = nn.GroupNorm(32, 32, affine=True)

        if self.multi_level_output:
            self.conv_lvl1 = PaddedConv2d(32, 32, 3)
            self.conv_lvl2 = PaddedConv2d(64, 32, 3)

    def forward(self, x_in):
        x = x_in
        x = self.conv1(x)
        lvl0 = self.conv2(x)
        x = self.pool1(lvl0)

        x = self.conv3(x)
        lvl1 = self.conv4(x)
        x = self.pool2(lvl1)

        x = self.conv5(x)
        x = self.conv6(x)

        if self.multi_level_output:
            lvl2_out = self.conv_lvl2(x)

        if self.with_upconv:
            x = self.upconv7(x)[:, :, 1:, 1:]
        else:
            x = F.interpolate(x, lvl1.shape[2:], mode='bilinear')
        x = torch.cat([lvl1, x], dim=1)

        x = self.conv8(x)
        x = self.conv9(x)

        if self.multi_level_output:
            lvl1_out = self.conv_lvl1(x)

        if self.with_upconv:
            x = self.upconv10(x)[:, :, :-1, :-1]
        else:
            x = F.interpolate(x, lvl0.shape[2:], mode='bilinear')
        x = torch.cat([lvl0, x], dim=1)

        x = self.conv11(x)
        x = self.conv12(x)
        if self.bn12:
            x = self.bn12(x)

        if self.multi_level_output:
            return x, lvl1_out, lvl2_out

        return [x]

max_stars_repo_name: VLOGroup/bp-layers

max_issues_repo_name: VLOGroup/bp-layers

max_forks_repo_name: VLOGroup/bp-layers


text:
#!/usr/bin/python3
# -*- coding: utf-8 -*-

import numpy as np
from matplotlib import pyplot as plt
import GA_utils

## Initializing Values:
N = 50
Pc = 0.9
Pm = 0.1
ITER = 100
m = 2
BS = np.array([10,10])
L = sum(BS)
Lo = np.array([-4,-1.5])
Hi = np.array([2,1])

best_so_far = [0]
best_ans_r = None  # just init value!
Average_fitness = []

## First Generation:
Population = np.round(np.random.rand(N,L)).astype(int)

### show everythings! ###
### if you don't want to draw, comment this section:----------------------
plt.ion()
plt.figure()
#####  -------------------------------------------------------------------
## ## ## ##
for it in range(ITER):
    real_val=GA_utils.chrome_decode(Population, BS, m, Lo, Hi)
    ## Evaluation:
    fit = GA_utils.cost_function(real_val)

    max_fit = fit.max()
    best_ans_r = real_val[fit.argmax(), :] if best_so_far[-1] < max_fit else best_ans_r ## opt_sol in your code!
    best_so_far.append(max(max_fit, best_so_far[-1]))
    Average_fitness.append(fit.mean())

    ## Show everythings: --------------------------
    ### if you want speed so comment this section:
    plt.cla()
    plt.plot(real_val[:,0], real_val[:,1], '.')
    plt.plot(best_ans_r[0], best_ans_r[1], 'X')
    plt.grid()
    plt.axis([Lo[0],Hi[0], Lo[1], Hi[1]])
    plt.title('iteration = ' + str(it))
    plt.pause(0.01)
    ### -----------------------------------------
    ## Selection new Parents
    parents = GA_utils.selection(fit, Population)

    ## Crossover:
    children = GA_utils.crossingover(parents, Pc)

    ## Mutation:
    Population = GA_utils.mutation(children, Pm)

plt.ioff()
plt.cla()
plt.plot(real_val[:,0], real_val[:,1], '.')
plt.plot(best_ans_r[0], best_ans_r[1], 'X')
plt.grid()
plt.axis([Lo[0],Hi[0], Lo[1], Hi[1]])
plt.title("Last Population")
plt.legend(["Population", "best answer"])
plt.figure()
plt.plot(best_so_far[1::], 'g')
plt.plot(Average_fitness, 'r')
plt.title("fitness over iterations")
plt.legend(["best so far", "Average of fitness"])
plt.show()
print("best answer: ", best_ans_r, '\n', "best fitness: ", best_so_far[-1])
print (GA_utils.cost_function(np.array([best_ans_r])))

max_stars_repo_name: Bahar-nkr/Genetic-Algorithm

max_issues_repo_name: Bahar-nkr/Genetic-Algorithm

max_forks_repo_name: Bahar-nkr/Genetic-Algorithm


text:
from keras.datasets import mnist
from keras.layers import Input, Dense
from keras import regularizers, models, optimizers
import numpy as np
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

# Analytical PCA of the training set
def AnalyticalPCA(y, dimension):
    pca = PCA(n_components=dimension)
    pca.fit(y)
    loadings = pca.components_
    return loadings

# Linear Autoencoder
def LinearAE(y, dimension, learning_rate = 1e-4, regularization = 5e-4, epochs=3):
    input = Input(shape=(y.shape[1],))
    encoded = Dense(dimension, activation='linear',
                    kernel_regularizer=regularizers.l2(regularization))(input)
    decoded = Dense(y.shape[1], activation='linear',
                    kernel_regularizer=regularizers.l2(regularization))(encoded)
    autoencoder = models.Model(input, decoded)
    autoencoder.compile(optimizer=optimizers.adam(lr=learning_rate), loss='mean_squared_error')
    autoencoder.fit(y, y, epochs=epochs, batch_size=4, shuffle=True)
    (w1,b1,w2,b2)=autoencoder.get_weights()
    return (w1,b1,w2,b2)

def PlotResults(p,dimension,name):
    sqrt_dimension = int(np.ceil(np.sqrt(dimension)))
    plt.figure()
    for i in range(p.shape[0]):
        plt.subplot(sqrt_dimension, sqrt_dimension, i + 1)
        plt.imshow(p[i, :, :],cmap='gray')
        plt.axis('off')
    plt.savefig(name + '.png')

dimension = 16                                                                  # feel free to change this, but you may have to tune hyperparameters
(y, _), (_, _) = mnist.load_data()                                              # load MNIST training images
shape_y = y.shape                                                               # store shape of y before reshaping it
y = np.reshape(y,[shape_y[0],shape_y[1]*shape_y[2]]).astype('float32')/255      # reshape y to be a 2D matrix of the dataset
p_analytical = AnalyticalPCA(y,dimension)                                       # PCA by applying SVD to y
(_, _, w2, _) = LinearAE(y, dimension)                                          # train a linear autoencoder
(p_linear_ae, _, _) = np.linalg.svd(w2.T, full_matrices=False)                    # PCA by applying SVD to linear autoencoder weights
p_analytical = np.reshape(p_analytical,[dimension,shape_y[1],shape_y[2]])       # reshape loading vectors before plotting
w2 = np.reshape(w2,[dimension,shape_y[1],shape_y[2]])                         # reshape autoencoder weights before plotting
p_linear_ae = np.reshape(p_linear_ae.T, [dimension, shape_y[1], shape_y[2]])    # reshape loading vectors before plotting
PlotResults(p_analytical,dimension,'AnalyticalPCA')
PlotResults(w2,dimension,'W2')
PlotResults(p_linear_ae,dimension,'LinearAE_PCA')

max_stars_repo_name: plaut/linear-ae-pca

max_issues_repo_name: plaut/linear-ae-pca

max_forks_repo_name: plaut/linear-ae-pca


text:
"""
Focal
Unet inference runner

poLicensed under the MIT License (see LICENSE for details)
"""

import cv2
import os
import logging
import numpy as np

from keras.models import load_model

from utils.mrcnn_helper import resize_image
from pipeline.unet_model import _mean_iou_unet_wrapper

from pipeline.model_interface import ModelInterface

class UNetInference(ModelInterface):
    """Class to load UNet model and run inference."""

    def __init__(self, model_path, image_resize=256):
        self.logger = logging.getLogger('UNetInference')

        self._model_path = os.path.abspath(model_path)
        if os.path.isfile(self._model_path) is False:
            err_str = 'Invalid model path provided: {}'.format(
                self._model_path)
            self.logger.error(err_str)
            raise Exception(err_str)

        self._unet_infer = load_model(
            self._model_path,
            custom_objects={'_mean_iou_unet_wrapper': _mean_iou_unet_wrapper })

        self._resize = image_resize

    def _normalize_image(self, image_orig):
        return image_orig/255

    def _process_input_image(self, image_orig):
        """
        Normalize the image, convert to grayscale and resize.
        Return the processed image
        """
        image = image_orig
        if len(image.shape) > 2 and image.shape[2] > 1:
            image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)

        image = self._normalize_image(image)

        image_resized, window, scale, padding, crop = resize_image(
            image, min_dim=self._resize, max_dim=self._resize, mode="square")

        if len(image_resized.shape) < 3:
            image_resized = image_resized.reshape(
                (image_resized.shape[0],image_resized.shape[1], 1))

        assert image_resized.shape == (256, 256, 1)

        return image_resized, window

    def run(self, input_image, model_config=None):
        # Prepare the input image, get the window into it.
        image_processed, window = self._process_input_image(input_image)

        image_mask = self._unet_infer.predict(
            np.array([image_processed]))[0]

        # Convert to binary mask
        image_mask = (image_mask > 0.5).astype('uint8')

        # Upscale the mask output
        w = window
        mask_window = image_mask[w[0]:w[2], w[1]:w[3], :]
        mask_window_orig = cv2.resize(mask_window,
                                      (input_image.shape[1],
                                       input_image.shape[0]))
        return mask_window_orig


text:
import logging
import time
from abc import abstractmethod
from typing import Dict, Any, Iterable, List, Set
import numpy as np

from guacamol.utils.math import cos_similarity
from guacamol.utils.chemistry import canonicalize_list, is_valid, calculate_pc_descriptors, continuous_kldiv, \
    discrete_kldiv, calculate_internal_pairwise_similarities, tokenizer, fragment_list, scaffold_list, \
    get_fingerprints_from_smileslist, average_agg_tanimoto, pass_through_filters
from guacamol.utils.data import get_random_subset
from guacamol.utils.sampling_helpers import sample_valid_molecules, sample_unique_molecules

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class DistributionLearningBenchmarkResult:
    """
    Contains the results of a distribution learning benchmark.

    NB: timing does not make sense since training happens outside of DistributionLearningBenchmark.
    """

    def __init__(self, benchmark_name: str, score: float, sampling_time: float, metadata: Dict[str, Any]) -> None:
        """
        Args:
            benchmark_name: name of the distribution-learning benchmark
            score: benchmark score
            sampling_time: time for sampling the molecules in seconds
            metadata: benchmark-specific information
        """
        self.benchmark_name = benchmark_name
        self.score = score
        self.sampling_time = sampling_time
        self.metadata = metadata

class DistributionLearningBenchmark:
    """
    Base class for assessing how well a model is able to generate molecules matching a molecule distribution.

    Derived class should implement the assess_molecules function
    """

    def __init__(self, name: str, number_samples: int) -> None:
        self.name = name
        self.number_samples = number_samples

    @abstractmethod
    def assess_model(self, model) -> DistributionLearningBenchmarkResult:
        """
        Assess a distribution-matching generator model.

        Args:
            model: model to assess
        """

class ValidityBenchmark(DistributionLearningBenchmark):
    """
    Assesses what percentage of molecules generated by a model are valid molecules (i.e. have a valid SMILES string)
    """

    def __init__(self, number_samples, return_valid=True, use_filters=False) -> None:
        super().__init__(name='Validity', number_samples=number_samples)
        self.return_valid = return_valid
        self.use_filters = use_filters

    def assess_model(self, model, prior_gen=[]):
        start_time = time.time()
        n_gen = self.number_samples - len(prior_gen)
        molecules = model.generate(number_samples=n_gen)
        molecules += prior_gen
        end_time = time.time()

        if len(molecules) != self.number_samples:
            raise Exception('The model did not generate the correct number of molecules, {} / {}'.format(len(molecules), self.number_samples))

        valid = []
        for smi in molecules:
            if is_valid(smi):
                valid.append(smi)
        if self.use_filters:
            valid = pass_through_filters(valid)
            self.name = self.name + '_filters'
        number_valid = len(valid)
        validity_ratio = number_valid / self.number_samples
        metadata = {
            'number_samples': self.number_samples,
            'number_valid': number_valid,
            'use_filters': self.use_filters
        }

        result = DistributionLearningBenchmarkResult(benchmark_name=self.name,
                                                     score=validity_ratio,
                                                     sampling_time=end_time - start_time,
                                                     metadata=metadata)
        if self.return_valid:
            return result, valid
        else:
            return result

class UniquenessBenchmark(DistributionLearningBenchmark):
    """
    Assesses what percentage of molecules generated by a model are unique.
    """

    def __init__(self, number_samples, return_unique=True, use_filters=False) -> None:
        super().__init__(name='Uniqueness', number_samples=number_samples)
        self.return_unique = return_unique
        self.use_filters = use_filters

    def assess_model(self, model, prior_gen=[]):
        start_time = time.time()
        molecules = sample_valid_molecules(model=model, number_molecules=self.number_samples,
                                           prior_gen=prior_gen, use_filters=self.use_filters)
        end_time = time.time()

        if len(molecules) != self.number_samples:
            logger.warning('The model could not generate enough valid molecules. The score will be penalized.')

        # canonicalize_list removes duplicates (and invalid molecules, but there shouldn't be any)
        unique_molecules = canonicalize_list(molecules)

        unique_ratio = len(unique_molecules) / self.number_samples
        metadata = {
            'number_samples': self.number_samples,
            'number_unique': len(unique_molecules),
            'use_filters': self.use_filters
        }

        result =  DistributionLearningBenchmarkResult(benchmark_name=self.name,
                                                      score=unique_ratio,
                                                      sampling_time=end_time - start_time,
                                                      metadata=metadata)
        if self.return_unique:
            return result, unique_molecules,
        else:
            return result

class NoveltyBenchmark(DistributionLearningBenchmark):
    def __init__(self, number_samples: int, training_set: Iterable[str], return_novel=True,
                 use_filters=False, return_train_mols=False, canonicalize=False) -> None:
        """
        Args:
            number_samples: number of samples to generate from the model
            training_set: molecules from the training set
        """
        super().__init__(name='Novelty', number_samples=number_samples)
        self.training_set_molecules = set(training_set)
        self.return_novel = return_novel
        self.use_filters = use_filters
        self.return_train_mols = return_train_mols

    def assess_model(self, model, prior_gen=[]):
        """
        Assess a distribution-matching generator model.

        Args:
            model: model to assess
        """
        start_time = time.time()
        molecules = sample_unique_molecules(model=model, number_molecules=self.number_samples,
                                            prior_gen=prior_gen, use_filters=self.use_filters,
                                            max_tries=2)
        end_time = time.time()

        if len(molecules) != self.number_samples:
            logger.warning('The model could not generate enough unique molecules. The score will be penalized.')

        # canonicalize_list in order to remove stereo information (also removes duplicates and invalid molecules, but there shouldn't be any)
        unique_molecules = set(canonicalize_list(molecules))

        novel_molecules = unique_molecules.difference(self.training_set_molecules)

        novel_ratio = len(novel_molecules) / self.number_samples

        metadata = {
            'number_samples': self.number_samples,
            'number_novel': len(novel_molecules)
        }

        result =  DistributionLearningBenchmarkResult(benchmark_name=self.name,
                                                      score=novel_ratio,
                                                      sampling_time=end_time - start_time,
                                                      metadata=metadata)
        if self.return_novel and self.return_train_mols:
            return result, novel_molecules, list(self.training_set_molecules)
        elif self.return_novel:
            return result, novel_molecules
        elif self.return_train_mols:
            return result, list(self.training_set_molecules)
        else:
            return result

class KLDivBenchmark(DistributionLearningBenchmark):
    """
    Computes the KL divergence between a number of samples and the training set for physchem descriptors
    """

    def __init__(self, number_samples: int, training_set: List[str], use_filters=False) -> None:
        """
        Args:
            number_samples: number of samples to generate from the model
            training_set: molecules from the training set
        """
        super().__init__(name='KL divergence', number_samples=number_samples)
        self.training_set_molecules = get_random_subset(training_set, self.number_samples, seed=42)
        self.pc_descriptor_subset = [
            'BertzCT',
            'MolLogP',
            'MolWt',
            'TPSA',
            'NumHAcceptors',
            'NumHDonors',
            'NumRotatableBonds',
            'NumAliphaticRings',
            'NumAromaticRings'
        ]
        self.use_filters = use_filters

    def assess_model(self, model, prior_gen=[]):
        """
        Assess a distribution-matching generator model.

        Args:
            model: model to assess
        """
        start_time = time.time()
        if len(prior_gen) > 0:
            molecules = prior_gen
        else:
            molecules = sample_unique_molecules(model=model, number_molecules=n_gen, max_tries=2)
        end_time = time.time()

        if len(molecules) != self.number_samples:
            logger.warning('The model could not generate enough unique molecules. The score will be penalized.')

        # canonicalize_list in order to remove stereo information (also removes duplicates and invalid molecules, but there shouldn't be any)
        unique_molecules = set(canonicalize_list(molecules))

        # first we calculate the descriptors, which are np.arrays of size n_samples x n_descriptors
        d_sampled = calculate_pc_descriptors(unique_molecules, self.pc_descriptor_subset)
        d_chembl = calculate_pc_descriptors(self.training_set_molecules, self.pc_descriptor_subset)

        kldivs = {}

        # now we calculate the kl divergence for the float valued descriptors ...
        for i in range(4):
            kldiv = continuous_kldiv(X_baseline=d_chembl[:, i], X_sampled=d_sampled[:, i])
            kldivs[self.pc_descriptor_subset[i]] = kldiv

        # ... and for the int valued ones.
        for i in range(4, 9):
            kldiv = discrete_kldiv(X_baseline=d_chembl[:, i], X_sampled=d_sampled[:, i])
            kldivs[self.pc_descriptor_subset[i]] = kldiv

        # pairwise similarity

        chembl_sim = calculate_internal_pairwise_similarities(self.training_set_molecules)
        chembl_sim = chembl_sim.max(axis=1)

        sampled_sim = calculate_internal_pairwise_similarities(unique_molecules)
        sampled_sim = sampled_sim.max(axis=1)

        kldiv_int_int = continuous_kldiv(X_baseline=chembl_sim, X_sampled=sampled_sim)
        kldivs['internal_similarity'] = kldiv_int_int

        # for some reason, this runs into problems when both sets are identical.
        # cross_set_sim = calculate_pairwise_similarities(self.training_set_molecules, unique_molecules)
        # cross_set_sim = cross_set_sim.max(axis=1)
        #
        # kldiv_ext = discrete_kldiv(chembl_sim, cross_set_sim)
        # kldivs['external_similarity'] = kldiv_ext
        # kldiv_sum += kldiv_ext

        metadata = {
            'number_samples': self.number_samples,
            'kl_divs': kldivs
        }

        # Each KL divergence value is transformed to be in [0, 1].
        # Then their average delivers the final score.
        partial_scores = [np.exp(-score) for score in kldivs.values()]
        score = sum(partial_scores) / len(partial_scores)

        result =  DistributionLearningBenchmarkResult(benchmark_name=self.name,
                                                      score=score,
                                                      sampling_time=end_time - start_time,
                                                      metadata=metadata)
        return result

class ReconstructionBenchmark(DistributionLearningBenchmark):
    """
    Computes the reconstruction accuracy for a set of holdout molecules
    """
    def __init__(self, test_set: List[str], sample_size: int, use_filters=False,
                 canonicalize=False) -> None:
        """
        Args:
            test_set: list of smiles to reconstruct
            sample_size: number of smiles to reconstruct
        """
        super().__init__(name='Reconstruction', number_samples=sample_size)
        if canonicalize:
            self.test_set_molecules = canonicalize_list(get_random_subset(test_set, self.number_samples, seed=42))
        else:
            self.test_set_molecules = get_random_subset(test_set, self.number_samples, seed=42)
        self.test_set_tokenized = [tokenizer(smi) for smi in self.test_set_molecules]
        self.use_filters = use_filters

    def assess_model(self, model, prior_gen=[]):
        """
        Assess a distribution-matching generator model

        Args:
            model: model to assess
        """
        start_time = time.time()
        reconstructed_smiles = model.reconstruct(self.test_set_molecules)
        reconstructed_smiles = canonicalize_list(reconstructed_smiles)
        end_time = time.time()
        reconstructed_tokenized = [tokenizer(smi) for smi in reconstructed_smiles]
        smile_accs = []
        hits = 0
        misses = 0
        position_accs = np.zeros((2, model.args.max_length+1))

        for in_smi, out_smi in zip(self.test_set_tokenized, reconstructed_tokenized):
            if in_smi == out_smi:
                smile_accs.append(1)
            else:
                smile_accs.append(0)

            misses += abs(len(in_smi) - len(out_smi))
            for j, (token_in, token_out) in enumerate(zip(in_smi, out_smi)):
                if token_in == token_out:
                    hits += 1
                    position_accs[0,j] += 1
                else:
                    misses += 1
                position_accs[1,j] += 1

        smile_acc = np.mean(smile_accs)
        token_acc = hits / (hits + misses)
        position_acc = []
        for i in range(model.args.max_length+1):
            position_acc.append(position_accs[0,i] / position_accs[1,i])
        score = (smile_acc, token_acc, position_acc)

        metadata = {
            'number_samples': self.number_samples,
            'accuracy_types': ['smiles', 'tokens', 'positional']
        }

        result = DistributionLearningBenchmarkResult(benchmark_name=self.name,
                                                     score=score,
                                                     sampling_time=end_time-start_time,
                                                     metadata=metadata)
        return result

class FragBenchmark(DistributionLearningBenchmark):
    """
    Computes the cosine similarity between generated fragments and holdout set fragments
    """
    def __init__(self, test_set: List[str], sample_size: int, type: str, use_filters=False) -> None:
        """
        Args:
            test_set: list of smiles to use for fragment comparison
            sample_size: number of smiles to sample
        """
        super().__init__(name='Frag', number_samples=sample_size)
        self.type = type
        self.ref_frags = fragment_list(test_set)
        self.use_filters = use_filters

    def assess_model(self, model, prior_gen=[]):
        """
        Assess a distribution-matching generator model

        Args:
            model: model to assess
        """
        start_time = time.time()
        if len(prior_gen) > 0:
            molecules = prior_gen
        else:
            molecules = sample_valid_molecules(model=model, number_molecules=self.number_samples)
        mol_frags = fragment_list(molecules)
        end_time = time.time()

        metadata = {
            'number_samples': self.number_samples,
            'number_valid': len(molecules)
        }

        score = cos_similarity(self.ref_frags, mol_frags)

        result = DistributionLearningBenchmarkResult(benchmark_name=self.name+'_{}'.format(self.type),
                                                     score=score,
                                                     sampling_time=end_time-start_time,
                                                     metadata=metadata)
        return result

class ScafBenchmark(DistributionLearningBenchmark):
    """
    Computes the cosine similarity between generated scaffolds and holdout set scaffolds
    """
    def __init__(self, test_set: List[str], sample_size: int, type: str, use_filters=False) -> None:
        """
        Args:
            test_set: list of smiles to use for scaffold comparison
            sample_size: number of smiles to sample
        """
        super().__init__(name='Scaf', number_samples=sample_size)
        self.type = type
        self.ref_scafs = scaffold_list(test_set)
        self.use_filters = use_filters

    def assess_model(self, model, prior_gen=[]):
        """
        Assess a distribution-matching generator model

        Args:
            model: model to assess
        """
        start_time = time.time()
        if len(prior_gen) > 0:
            molecules = prior_gen
        else:
            molecules = sample_valid_molecules(model=model, number_molecules=self.number_samples)
        mol_scafs = scaffold_list(molecules)
        end_time = time.time()

        metadata = {
            'number_samples': self.number_samples,
            'number_valid': len(molecules)
        }

        score = cos_similarity(self.ref_scafs, mol_scafs)

        result = DistributionLearningBenchmarkResult(benchmark_name=self.name+'_{}'.format(self.type),
                                                     score=score,
                                                     sampling_time=end_time-start_time,
                                                     metadata=metadata)
        return result

class SNNBenchmark(DistributionLearningBenchmark):
    """
    Computes the average max similarities of generated smiles to holdout smiles
    """
    def __init__(self, test_set: List[str], sample_size: int) -> None:
        """
        Args:
            test_set: list of smiles to compare to generated set
            sample_size: number of smiles to sample
        """
        super().__init__(name='SNN', number_samples=sample_size)
        self.test_fps = np.vstack(get_fingerprints_from_smileslist(test_set))

    def assess_model(self, model, prior_gen=[]) -> DistributionLearningBenchmarkResult:
        """
        Assess a distribution-matching generator model

        Args:
            model: model to assess
        """
        start_time = time.time()
        if len(prior_gen) > 0:
            molecules = prior_gen
        else:
            molecules = sample_valid_molecules(model=model, number_molecules=self.number_samples)
        gen_fps = np.vstack(get_fingerprints_from_smileslist(molecules))
        end_time = time.time()

        metadata = {
            'number_samples': self.number_samples,
            'number_valid': len(molecules)
        }

        score = average_agg_tanimoto(self.test_fps, gen_fps)

        return DistributionLearningBenchmarkResult(benchmark_name=self.name,
                                                   score=score,
                                                   sampling_time=end_time-start_time,
                                                   metadata=metadata)

class IntDivBenchmark(DistributionLearningBenchmark):
    """
    Computes the average max similarities of generated smiles to themselves
    """
    def __init__(self, number_samples: int, p: int) -> None:
        """
        Args:
            sample_size: number of smiles to sample
        """
        super().__init__(name='IntDiv', number_samples=number_samples)
        self.p = p

    def assess_model(self, model, prior_gen=[]) -> DistributionLearningBenchmarkResult:
        """
        Assess a distribution-matching generator model

        Args:
            model: model to assess
        """
        start_time = time.time()
        if len(prior_gen) > 0:
            molecules = prior_gen
        else:
            molecules = sample_valid_molecules(model=model, number_molecules=self.number_samples)
        gen_fps = np.vstack(get_fingerprints_from_smileslist(molecules))
        end_time = time.time()

        metadata = {
            'number_samples': self.number_samples,
            'number_valid': len(molecules),
            'p': self.p
        }

        score = average_agg_tanimoto(gen_fps, gen_fps, p=self.p)

        return DistributionLearningBenchmarkResult(benchmark_name=self.name+'{}'.format(self.p),
                                                   score=score,
                                                   sampling_time=end_time-start_time,
                                                   metadata=metadata)

max_stars_repo_name: oriondollar/guacamol

max_issues_repo_name: oriondollar/guacamol

max_forks_repo_name: oriondollar/guacamol


text:
import tensorflow as tf
import numpy as np
import cv2
from model import get_segmentation_model, get_detection_model
import json
import time

IMG_WIDTH = 512
IMG_HEIGHT = 512
IMG_CHANNELS = 3

s_model = get_segmentation_model()
d_model = get_detection_model()

labels = ['BrownSpot', 'Healthy', 'Hispa', 'LeafBlast']
# cap_live = ""

def get_prediction_video(filename):
    cap = cv2.VideoCapture(filename)
    while (cap.isOpened()):
        ret, frame = cap.read()
        frame = cv2.resize(frame, (IMG_HEIGHT, IMG_WIDTH))
        inp = tf.keras.preprocessing.image.img_to_array(frame)
        res = s_model.predict(np.expand_dims(inp, axis=0))
        res = res * 255
        redImg = np.zeros(inp.shape, inp.astype('uint8').dtype)
        redImg[:,:] = (0, 0, 255)
        redMask = cv2.bitwise_and(redImg, redImg, mask=np.squeeze(res).astype('uint8'))
        mask_t = cv2.addWeighted(redMask, 0.5, inp.astype('uint8'), 0.5, 0, inp.astype('uint8'))
        ret, buffer = cv2.imencode('.jpg', mask_t)
        out = buffer.tobytes()
        yield (b'--frame\r\n'b'Content-Type: image/jpeg\r\n\r\n' + out + b'\r\n'b'Test: test \r\n')

def get_live_video():
    # global cap_live
    cap_live = cv2.VideoCapture(0)
    while (cap_live.isOpened()):
        ret, frame = cap_live.read()
        frame = cv2.resize(frame, (IMG_HEIGHT, IMG_WIDTH))
        inp = tf.keras.preprocessing.image.img_to_array(frame)
        res = s_model.predict(np.expand_dims(inp, axis=0))
        res = res * 255
        redImg = np.zeros(inp.shape, inp.astype('uint8').dtype)
        redImg[:,:] = (0, 0, 255)
        redMask = cv2.bitwise_and(redImg, redImg, mask=np.squeeze(res).astype('uint8'))
        mask_t = cv2.addWeighted(redMask, 0.5, inp.astype('uint8'), 0.5, 0, inp.astype('uint8'))
        ret, buffer = cv2.imencode('.jpg', mask_t)
        out = buffer.tobytes()
        yield (b'--frame\r\n'b'Content-Type: image/jpeg\r\n\r\n' + out + b'\r\n'b'Test: test \r\n')

def get_prediction_label(filename):
    cap = cv2.VideoCapture(filename)
    while (cap.isOpened()):
        ret, frame = cap.read()
        frame = cv2.resize(frame, (256, 256))
        inp = tf.keras.preprocessing.image.img_to_array(frame) / 255.0
        res = d_model.predict(np.expand_dims(inp, axis=0))
        label = labels[np.argmax(res)]
        time.sleep(.1)
        yield "data: %s \n\n" % (json.dumps({"pred": res.tolist()}))

def get_live_label():
    # global cap_live
    cap_live = cv2.VideoCapture(0)
    while (cap_live.isOpened()):
        ret, frame = cap_live.read()
        frame = cv2.resize(frame, (256, 256))
        inp = tf.keras.preprocessing.image.img_to_array(frame) / 255.0
        res = d_model.predict(np.expand_dims(inp, axis=0))
        label = labels[np.argmax(res)]
        time.sleep(.1)
        yield "data: %s \n\n" % (json.dumps({"pred": res.tolist()}))


text:
import pandas as pd
import numpy as np
from datetime import date
import sys
import os

# print('Number of arguments passed: ',len(sys.argv))
currentDir = os.getcwd()
if len(sys.argv) == 4:
  thisdate=str(sys.argv[1])
  filenameSrc=str(sys.argv[2])
  filenameDest=str(sys.argv[3])
else:
  if len(sys.argv) == 2:
    thisdate=sys.argv[1]
  else:
    thisdate=str(date.today())
  filenameDest=currentDir+'/data/interim/'+thisdate+'.pkl'
  filenameSrc=currentDir+'/data/interim/'+thisdate+'.tsv'

#I imagine this should impact the size of resulting pickled Df
resampleFreq = '30S'

#If .TSV file exists, pickle it & save
if os.path.isfile(filenameSrc):
  df = pd.read_csv(filenameSrc, sep='\t', header=0).set_index('record_id')
  #print('Imported: ' + filenameSrc)

  #Preprocess: remove un-used column
  df.drop('when_stored', axis=1, inplace=True)

  #Preprocess: change datatypes
  df['when_day_time'] = pd.to_datetime(df['when_day_time'])
  df[['tempF','humidity']] = df[['tempF', 'humidity']].astype('float64')

  #Preprocess: pivoting dataframe
  resamp = df.pivot(index='when_day_time', columns='sensor_id', values=['tempF', 'humidity']).resample(resampleFreq)
  newDf = resamp.agg(np.nanmean)

  #Preprocess: PICKLE!
  newDf.to_pickle(filenameDest)
  print(newDf.tail(10))

  #Output whether successful
  if os.path.isfile(filenameDest):
    print('Exported: ' + filenameDest)
  else:
    print('Tried to pickle but file didnt create: ' + filenameDest)
#Otherwise, no such TSV file exists yet
else:
  print('No such file: ' + filenameSrc)

max_stars_repo_licenses: FTL RSA-MD

max_issues_repo_licenses: FTL RSA-MD

max_forks_repo_licenses: FTL RSA-MD


text:
# References:
#
# http://software.intel.com/en-us/intel-mkl
# https://code.google.com/p/numexpr/wiki/NumexprVML

from __future__ import print_function
import datetime
import sys
from scipy import stats
import numpy as np
import numexpr as ne
import time
import gc
import os.path
import cPickle as pickle

data_dir = './'

def time_dgemm(N=100, trials=3, dtype=np.double):
    A = np.asarray(np.random.rand(N, N), dtype=dtype)
    B = np.asarray(np.random.rand(N, N), dtype=dtype)
    gcold = gc.isenabled()
    gc.disable()
    tic = time.time()
    for i in xrange(trials):
        C = A.dot(B)
    toc = time.time()-tic
    if gcold:
        gc.enable()
    return toc/trials, 2*N*N*N*1e-9

def time_cholesky(N=100, trials=3, dtype=np.double):
    A = np.asarray(np.random.rand(N, N), dtype=dtype)
    A = A*A.transpose() + N*np.eye(N)
    gcold = gc.isenabled()
    gc.disable()
    tic = time.time()
    for i in xrange(trials):
        L = np.linalg.cholesky(A)
    toc = time.time()-tic
    if gcold:
        gc.enable()
    return toc/trials, N*N*N/3.0*1e-9

def time_numexpr(N=100, trials=3, dtype=np.double):
    x = np.asarray(np.linspace(-1, 1, N), dtype=dtype)
    y = np.asarray(np.linspace(-1, 1, N), dtype=dtype)
    z = np.empty_like(x)
    gcold = gc.isenabled()
    gc.disable()
    tic = time.time()
    for i in xrange(trials):
        ne.evaluate('2*y+4*x', out = z)
    toc = time.time()-tic
    if gcold:
        gc.enable()
    return (toc/trials, dtype().itemsize*3*N*1e-9)

def test_timers():
    N = 500
    trials = 3
    dtype = np.double
    s, gflop = time_dgemm(N, trials, dtype)
    print("DGEMM   : N: %d s: %e GFLOP/s: %e" % (N, s, gflop/s))
    s, gflop = time_cholesky(N, trials, dtype)
    print("Cholesky: N: %d s: %e GFLOP/s: %e" % (N, s, gflop/s))
    s, gbyte = time_numexpr(50000, trials, dtype)
    print("NumExpr : N: %d s: %e GBytes/s: %e" % (N, s, gbyte/s))

def bench(test_fun, Ns, trials, dtype=None):
    data = np.empty((len(Ns),2))
    print("%d tests" % len(Ns))
    tic = time.time()
    for i in xrange(len(Ns)):
        sys.stdout.write('.')
        sys.stdout.flush()
        if dtype is not None:
            out_tuple= test_fun(Ns[i],trials,dtype)
        else:
            out_tuple= test_fun(Ns[i],trials)

        if len(out_tuple) > 1:
            data[i,:] = (Ns[i], out_tuple[1]/out_tuple[0])
        else:
            data[i,:] = (Ns[i], out_tuple[0])
    print('done')
    toc = time.time() - tic
    print('tests took: %e seconds' % toc)
    return data

def dump_data(data, data_dir, backend, algo):
    filename = backend + '-' + algo + '.pkl'
    out_pickle = os.path.join(data_dir, filename)
    with open(out_pickle,'w') as data_file:
        pickle.dump(data, data_file)

if __name__ == '__main__':
    try:
        import mkl
        have_mkl = True
        backend = 'anaconda+mkl'
        print("Running with MKL Acceleration")
    except ImportError:
        have_mkl = False
        backend = 'anaconda'
        print("Running with normal backends")

    print("checking timers...")
    test_timers()
    logNs = np.arange(6,13.5,0.5) # uncomment to run the big stuff
#    logNs = np.arange(3,7,0.5) # uncomment to run quick tests
    Ns = np.exp2(logNs)
    trials = 5
    dtype = np.double

    print('benchmarking DGEMM')
    dgemm_data = bench(time_dgemm, Ns, trials, dtype)
    dump_data(dgemm_data, data_dir, backend, 'DGEMM')

    print('benchmarking Cholesky')
    cholesky_data = bench(time_cholesky, Ns, trials, dtype)
    dump_data(cholesky_data, data_dir, backend, 'Cholesky')

    print('benchmarking NumExpr')
    logNs = np.arange(12, 18.5, 0.5) # uncomment to run big tests
#    logNs = np.arange(6,13.5,0.5) # uncomment to run quick tests
    Ns = np.exp2(logNs)
    numexpr_data = bench(time_numexpr, Ns, trials, dtype)
    dump_data(numexpr_data, data_dir, backend, 'NumExpr')

max_stars_repo_name: ahmadia/mkl-optimizations-benchmarks

max_issues_repo_name: ahmadia/mkl-optimizations-benchmarks

max_forks_repo_name: ahmadia/mkl-optimizations-benchmarks

reason: import numpy,from scipy,import numexpr


text:
"""
Inference mode for DeeProtein.
"""
import tensorflow as tf
import numpy as np
import json
import wget
import helpers
import argparse
from goatools.obo_parser import GODag
from DeeProtein import DeeProtein

def main():
    if FLAGS.obo_file:
        obo_file = FLAGS.obo_file
    else: # no GOdag file was specified -> download it.
        url = 'http://purl.obolibrary.org/obo/go.obo'
        obo_file = wget.download(url)
    GOdag = GODag(obo_file, optional_attrs=['relationship'])

    with open(FLAGS.config_json) as config_fobj:
        config_dict = json.load(config_fobj)

    # set the gpu context
    if not FLAGS.gpu:
        if config_dict["gpu"] == 'True':
            config_dict["gpu"] = "False"

    opts = helpers.OptionHandler(config_dict)
    deeprotein = DeeProtein(opts, inference=True)

    with tf.Graph().as_default():
        deeprotein.initialize_helpers()

        # graph for inference:
        deeprotein.is_train = False
        deeprotein._opts._batchsize = 1
        input_seq_node = tf.placeholder(tf.float32, [deeprotein._opts._batchsize, deeprotein._opts._depth,
                                                     deeprotein._opts._windowlength, 1])
        inference_net, _ = deeprotein.model(input_seq_node, valid_mode=False)
        double_sigmoid_logits = tf.nn.sigmoid(inference_net.outputs)

        # as its a double outlayer take a mean and get the var
        mean_inf_sigmoid_logits, var_inf_sigmoid_logits = tf.nn.moments(double_sigmoid_logits,
                                                                        axes=[2])

        deeprotein.session.run(tf.global_variables_initializer())
        deeprotein.saver = tf.train.Saver()
        # restore the session from ckpt
        if FLAGS.restore_from_explicit_ckpt:
            deeprotein.restore_model_from_checkpoint(FLAGS.restore_from_explicit_ckpt, deeprotein.session)
        else:
        # restore all the weights from dir in checkpointfile:
            _ = deeprotein.load_model_weights(inference_net, session=deeprotein.session,
                                              name='Classifier') #THIS RUNS THE SESSION INTERNALLY

        encoded_seq = deeprotein.batchgen._encode_single_seq(FLAGS.sequence)
        # expand it to batch dim and channels dim:
        assert len(encoded_seq.shape) == 2
        encoded_seq = np.reshape(encoded_seq, [1, encoded_seq.shape[0], encoded_seq.shape[1], 1])

        # run the session and retrieve the predicted classes
        class_logits, class_variance = deeprotein.session.run([mean_inf_sigmoid_logits, var_inf_sigmoid_logits],
                                                              feed_dict={input_seq_node: encoded_seq})
        # logits is a 2D array of shape [1, nclasses]
        thresholded_logits = np.where(class_logits >= 0.5)

        # get the IDs of the classes:
        predicted_IDs = thresholded_logits[1].tolist()

        # get the corresponding classes from the EC_file:
        predicted_classes = [deeprotein.batchgen.id_to_class[_id] for _id in predicted_IDs]
        deeprotein.log_file.write('Predicted classes:\n')

        for i, go in enumerate(predicted_classes):
            try:
                go = go.split('_')[1]
            except:
                pass
            output = '%s\t%s:\t%f, %f\n' % (go, GOdag[go].name,
                                                             class_logits[0, predicted_IDs[i]],
                                                             class_variance[0, predicted_IDs[i]])
            deeprotein.log_file.write(output)
            print(output)
        if not predicted_classes:
            output = 'The sequence could not be classified with enough fidelity.\n\n'
            deeprotein.log_file.write(output)
            print(output)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--sequence',
        type=str,
        required=True,
        help='The amino_acid sequence to classify. Should at least 175AA long and contain only canonical AAs.')
    parser.add_argument(
        '--config_json',
        type=str,
        required=True,
        help='Path to the config.JSON')
    parser.add_argument(
        '--obo_file',
        type=str,
        default=False,
        help='The path to the download of the gene ontology file (go.obo). '
             'If None is spefified, the file will be downloaded prior to inference.')
    parser.add_argument(
        '--restore_from_explicit_ckpt',
        type=str,
        default=False,
        help='Restore the model from this explicit ckpt and not form the dir/ specified in config_json.')
    parser.add_argument(
        '--gpu',
        type=str,
        default=True,
        help='Wheter to train in gpu context or not '
             '(optional). Defaults to True.')
    FLAGS, unparsed = parser.parse_known_args()

    if unparsed:
        print('Error, unrecognized flags:', unparsed)
        exit(-1)

    main()