text:
#-------------------------------------------------------------------------------
# General Libraries
#-------------------------------------------------------------------------------

import inspect
import json
import pickle

import random
import numpy as np
import pandas as pd
import numexpr as ne
import sklearn
from sklearn import preprocessing
from sklearn import cluster
from sklearn.preprocessing import StandardScaler
import copy

from matplotlib import pyplot as plt

from joblib import dump, load

import os,sys,inspect

#-------------------------------------------------------------------------------
# CRBM libraries and tools
#-------------------------------------------------------------------------------

from . import timeseries
from . import crbm_tools
from . crbm_tools import build_slices_from_list_of_arrays as build_slices

#-------------------------------------------------------------------------------
# Classes
#-------------------------------------------------------------------------------

class TimeseriesPipeline():

	def __init__(self, steps, memory = None, verbose = False):
		self.steps = steps
		self.memory = memory
		self.verbose = verbose

	def predict(self, list_of_timeseries):
		list_of_outputs = []
		for x in list_of_timeseries:
			step_output = x
			for step in self.steps:
				step_output = step.predict(step_output)
			list_of_outputs.append(step_output)
		return list_of_outputs

	def getStep (self, idx):
		return self.steps[idx]

class AI4DL():

	def __init__(self):

		# Configuration vars
		self.exec_id = None
		self.features = None

		# Data for Training
		self.list_of_timeseries = None

		# Trained Objects
		self.scaler = None
		self.pipeline = None

	'''
	Reads, Scales and Transforms a dataset (e.g. reading a Training-Set)
	'''
	def LoadTrainingDataset (self, data_file, features, exec_id):

		## Save configuration info
		self.exec_id = exec_id
		self.features = features
		self.n_features = len(features)

		list_of_timeseries = []

		## Load Dataset
		dataset = pd.read_csv(data_file)

		## Create Scaler from df_train
		scaler = StandardScaler()
		scaler.fit(dataset[self.features].values)
		dataset_scaled = copy.deepcopy(dataset)
		dataset_scaled[self.features] = scaler.transform(dataset[self.features].values.astype(np.float))

		## Prepare Timeseries
		unique_ids = dataset_scaled[self.exec_id].unique()
		for identifier in unique_ids:
			indices = dataset_scaled[self.exec_id]==identifier
			data = dataset_scaled[indices][self.features].values
			list_of_timeseries.append(data)

		self.scaler = scaler
		self.list_of_timeseries = list_of_timeseries

	'''
	Reads and Transforms a dataset, scaled by the Training Data (e.g. reading a Test-Set)
	'''
	def TransformData (self, data_file):

		list_of_timeseries = []
		dataset  = pd.read_csv(data_file)

		dataset_scaled = copy.deepcopy(dataset)
		dataset_scaled[self.features] = self.scaler.transform(dataset[self.features].values.astype(np.float))

		unique_ids = dataset_scaled[self.exec_id].unique()
		for identifier in unique_ids:
			indices = dataset_scaled[self.exec_id]==identifier
			data = dataset_scaled[indices][self.features].values
			list_of_timeseries.append(data)

		return list_of_timeseries

	'''
	Returns the "List of Timeseries" from the Training-Set
	'''
	def GetTrainingDataset (self):
		return self.list_of_timeseries

	'''
	Returns the Scaler
	'''
	def GetScaler (self):
		return self.scaler

	'''
	Trains the full model CRBM + k-Means using the Training Data
	'''
	def TrainModel (self, n_clusters, n_hidden, n_history, learning_rate = 0.001, n_epochs = 100, crbm_save = None, kmeans_save = None, scaler_save = None, seed = 123):

		## Save configuration info
		self.n_clusters = n_clusters
		self.n_hidden = n_hidden
		self.n_history = n_history
		self.learning_rate = learning_rate
		self.n_epochs = n_epochs

		## Train CRBM
		X_slices = build_slices(self.list_of_timeseries, self.n_history, self.n_features)
		crbm = crbm_tools.CRBM(n_vis = self.n_features, n_hid = self.n_hidden, n_his = self.n_history, learning_rate = self.learning_rate, n_epochs = self.n_epochs)
		crbm.fit(X_slices)

		## Train K-Means
		list_of_activations = []
		for l in self.list_of_timeseries:
			list_of_activations.append(crbm.predict(l))

		X_activations = np.vstack(list_of_activations)
		kmeans = sklearn.cluster.KMeans(n_clusters = n_clusters, random_state = seed)
		kmeans.fit(X_activations)

		## Save Sub_Models in files
		if not crbm_save is None:
			crbm.save("./", crbm_save)

		if not kmeans_save is None:
			dump(kmeans, kmeans_save)

		if not scaler_save is None:
			dump(self.scaler, scaler_save)

		## Storing the models and Pipeline
		steps = (crbm, kmeans)
		self.pipeline = TimeseriesPipeline(steps)

	'''
	Loads the model CRBM + k-Means (or parts of it) from files
	'''
	def LoadModel (self, crbm_save, kmeans_save, scaler_save, crbm_sub = None, kmeans_sub = None, scaler_sub = None):

		crbm = crbm_tools.CRBM(1,1,1)
		if not crbm_save is None:
			crbm = crbm.load(crbm_save)
		else:
			crbm = crbm_sub

		if not kmeans_save is None:
			kmeans = load(kmeans_save)
		else:
			kmeans = kmeans_sub

		if not scaler_save is None:
			self.scaler = load(scaler_save)
		else:
			self.scaler = scaler_sub

		## Storing the models and Pipeline
		steps = (crbm, kmeans)
		self.pipeline = TimeseriesPipeline(steps)

	'''
	Predicts a "List of Timeseries"
	'''
	def Predict (self, list_of_timeseries):
		return self.pipeline.predict(list_of_timeseries)

	'''
	Predicts the Training-Set "List of Timeseries"
	'''
	def Evaluate (self):
		return self.Predict(self.list_of_timeseries)

	'''
	Loads a file and transforms into a "List of Timeseries", then predicts it
	'''
	def LoadAndPredict (self, data_file):
		list_of_timeseries = self.TransformData(data_file)
		return self.Predict(list_of_timeseries)

	'''
	Printingt Functions - Print the trace from a TimeSeries
	'''
	def PrintTrace (self, list_of_timeseries, predicted_seq_phases, n_exec, cpu_idx, mem_idx, palette, col_names, f_name):

		## Print into a File
		timeseries.plot_timeseries_with_phases(list_of_timeseries[n_exec][:,(cpu_idx, mem_idx)], phases = predicted_seq_phases[n_exec], num_to_color = palette, column_names = col_names, fig_size = (20, 10), y_lim = (-3, 3 + 0.1), y_ticks_delta = 2, file_to_save = f_name)

	'''
	Printingt Functions - Print the variance analysis plots of a "List of TimeSeries"
	'''
	def PrintVarAnalysis (self, list_of_timeseries, pred_seq_phases, cpu_idx, mem_idx, palette, col_names, f_name):

		## Get number of phases
		n_phases = max([item for sublist in pred_seq_phases for item in sublist]) + 1
		palette = {k: palette[k] for k in list(palette)[:n_phases]}

		## Get Phase Information
		phase_sets_cpu = []
		phase_sets_mem = []
		for k in range(0, n_phases):
			phase_sets_cpu.append([])
			phase_sets_mem.append([])

		for x in range(0, len(pred_seq_phases)):
			for y in range(0, len(pred_seq_phases[x])):
				curr_phase = pred_seq_phases[x][y]
				phase_sets_cpu[curr_phase].append(list_of_timeseries[x][y, cpu_idx])
				phase_sets_mem[curr_phase].append(list_of_timeseries[x][y, mem_idx])

		## Print into a File
		fig, axs = plt.subplots(nrows = 1, ncols = 3, figsize = (20, 6))
		axs[0].set_title(col_names[0])
		axs[0].boxplot(phase_sets_cpu, labels=palette.values())
		axs[1].set_title(col_names[1])
		axs[1].boxplot(phase_sets_mem, labels=palette.values())
		axs[2].set_title("Freq. phases")
		N, bins, patches = axs[2].hist(np.concatenate(pred_seq_phases), bins = n_phases, range=[0,n_phases])
		for x in range(0, len(patches)):
			patches[x].set_facecolor(palette[x])
		fig.tight_layout(pad = 2.0, w_pad = 0.0, h_pad = 3.0)
		fig.savefig(f_name)

reason: import numpy,import numexpr


text:
import numpy as np

from caffe2.python import workspace, cnn, core
from caffe2.python import timeout_guard
from caffe2.proto import caffe2_pb2

def init_model(self):
    # if cudnn needs to be turned off, several other places
    # need to be modified:
    # 1. operators need to be constructed with engine option, like below:
    #     conv_blob = model.Conv(...engine=engine)
    # 2. when launch model, opts['model_param']['engine'] = "" instead of "CUDNN"
    # 2. caffe2_disable_implicit_engine_preference in operator.cc set to true
    train_model = cnn.CNNModelHelper(
        order="NCHW",
        name="resnet",
        use_cudnn=False,
        cudnn_exhaustive_search=False,
    )
    self.train_model = train_model

    # test_model = cnn.CNNModelHelper(
    #     order="NCHW",
    #     name="resnet_test",
    #     use_cudnn=False,
    #     cudnn_exhaustive_search=False,
    #     init_params=False,
    # )
    self.test_model = None

    self.log.info("Model creation completed")

def fun_per_epoch_b4RunNet(self, epoch):
    pass

def fun_per_iter_b4RunNet(self, epoch, epoch_iter):
    learning_rate = 0.05
    for idx in range(self.opts['distributed']['first_xpu_id'],
                     self.opts['distributed']['first_xpu_id'] +
                     self.opts['distributed']['num_xpus']):
        caffe2_pb2_device = caffe2_pb2.CUDA if \
            self.opts['distributed']['device'] == 'gpu' else \
            caffe2_pb2.CPU
        with core.DeviceScope(core.DeviceOption(caffe2_pb2_device, idx)):
            workspace.FeedBlob(
                '{}_{}/lr'.format(self.opts['distributed']['device'], idx),
                np.array(learning_rate, dtype=np.float32)
            )

def run_training_net(self):
    timeout = 2000.0
    with timeout_guard.CompleteInTimeOrDie(timeout):
        workspace.RunNet(self.train_model.net.Proto().name)

ext: py

max_stars_repo_name: Hacky-DH/pytorch

max_issues_repo_name: Hacky-DH/pytorch

max_forks_repo_name: Hacky-DH/pytorch

reason: import numpy


text:
import numpy as np
import pandas as pd

from plot import plot_bin, plot_curve, plot_int, plot_net

def load_prob(args, ind, plot_set, params):
    plot_data = {}
    for clf in args.clf:
        if plot_set == 'valid':
            file_path = args.ssc_path[clf] + 'valid_prob.csv' if not ind else args.ssc_path[clf] + 'ind_prob.csv'
        else:
            file_path = args.ssc_path[clf] + 'test_prob.csv' if not ind else args.ssc_path[clf] + 'ind_prob.csv'
        clf_prob = pd.read_csv(file_path, dtype=np.float).to_dict('list')
        top_n = params['top_n'][clf]
        for n in range(1, top_n + 1):
            plot_data[clf + '_' + str(n)] = clf_prob['top_' + str(n)]
    if args.rank == 'ltr' and plot_set == 'test':
        int_file_path = args.int_path + 'prob.npy' if not ind else args.int_path + 'ind_prob.npy'
        plot_data[args.rank] = list(np.load(int_file_path))

    return plot_data

def load_results(args, ind, plot_set, params):
    labels = []
    aupr_list = []
    auc_list = []
    ndcg_list = []
    for clf in args.clf:
        top_n = params['top_n'][clf]
        for n in range(1, top_n + 1):
            if plot_set == 'valid':
                file_name = 'top_' + str(n) + '_eval_results.csv' if not ind else 'top_' + str(n) + '_eval_results_ind.csv'
            else:
                file_name = 'top_' + str(n) + '_test_results.csv' if not ind else 'top_' + str(n) + '_test_results_ind.csv'
            file_path = args.ssc_path[clf] + file_name
            aupr_list.append(np.mean(pd.read_csv(file_path, dtype=np.float).to_dict('list')['aupr']))
            auc_list.append(np.mean(pd.read_csv(file_path, dtype=np.float).to_dict('list')['auc']))
            ndcg_list.append(np.mean(pd.read_csv(file_path, dtype=np.float).to_dict('list')['ndcg']))
            labels.append(clf + '_' + str(n))
    if args.rank != 'ltr' and plot_set == 'test':
        rank_file_path = args.int_path + 'eval_results.csv' if not ind else args.int_path + 'eval_results_ind.csv'
        aupr_list.append(np.mean(pd.read_csv(rank_file_path, dtype=np.float).to_dict('list')['aupr']))
        auc_list.append(np.mean(pd.read_csv(rank_file_path, dtype=np.float).to_dict('list')['auc']))
        ndcg_list.append(np.mean(pd.read_csv(rank_file_path, dtype=np.float).to_dict('list')['ndcg']))
        labels.append('ltr')

    return aupr_list, auc_list, ndcg_list, labels

def load_metric(args, ind, plot_set, plot_metric, params):
    results = {}
    for clf in args.clf:
        top_n = params['top_n'][clf]
        for n in range(1, top_n + 1):
            if plot_set == 'valid':
                file_name = 'top_' + str(n) + '_eval_results.csv' if not ind else 'top_' + str(n) + '_test_results_ind.csv'
            else:
                file_name = 'top_' + str(n) + '_test_results.csv' if not ind else 'top_' + str(n) + '_test_results_ind.csv'
            eval_data = pd.read_csv(args.ssc_path[clf] + file_name, dtype=np.float).to_dict('list')[plot_metric]
            results[clf + '_' + str(n)] = np.mean(eval_data)
    if args.rank == 'ltr' and plot_set == 'test':
        int_file_path = args.int_path + 'eval_results.csv' if not ind else args.int_path + 'eval_results_ind.csv'
        results[args.rank] = np.mean(pd.read_csv(int_file_path, dtype=np.float).to_dict('list')[plot_metric])

    return results

def load_eval(args, ind, plot_set, params):
    results = {}
    for clf in args.clf:
        top_n = params['top_n'][clf]
        for n in range(1, top_n + 1):
            if plot_set == 'valid':
                file_name = 'top_' + str(n) + '_eval_results.csv' if not ind else 'top_' + str(n) + '_eval_results_ind.csv'
            else:
                file_name = 'top_' + str(n) + '_test_results.csv' if not ind else 'top_' + str(n) + '_test_results_ind.csv'
            eval_data = pd.read_csv(args.ssc_path[clf] + file_name, dtype=np.float).mean().tolist()
            results[clf + '_' + str(n)] = eval_data[1:]
    if args.rank == 'ltr' and plot_set == 'test':
        int_file_path = args.int_path + 'eval_results.csv' if not ind else args.int_path + 'eval_results_ind.csv'
        int_eval_data = pd.read_csv(int_file_path, dtype=np.float).mean().tolist()
        results[args.rank] = int_eval_data[1:]

    return results

def data_rus(vec_arr, prob_arr, label_arr):
    from collections import Counter
    from imblearn.under_sampling import RandomUnderSampler
    print('Original dataset shape %s' % Counter(label_arr))

    rus = RandomUnderSampler(random_state=1025)
    all_fea = np.hstack((vec_arr, prob_arr))
    X_res, y_res = rus.fit_resample(all_fea, label_arr)
    print('Resampled dataset shape %s' % Counter(y_res))
    dim = vec_arr.shape[1]
    return X_res[:, :dim], X_res[:, dim:], y_res

def prob_rus(prob_dict, label_arr):
    from collections import Counter
    from imblearn.under_sampling import RandomUnderSampler
    print('Original dataset shape %s' % Counter(label_arr))
    prob_list = list(prob_dict.values())
    rus = RandomUnderSampler(random_state=1025)
    prob_arr = np.array(prob_list).transpose()
    # print(prob_arr.shape)
    X_res, y_res = rus.fit_resample(prob_arr, label_arr)
    # print(X_res[0])
    # print('Resampled dataset shape %s' % Counter(y_res))
    return X_res

def check_args4plot(args):
    if 'box' in args.plot:
        args.plot_metric = ['aupr', 'auc', 'ndcg']
    return args

def plot_fig(args, ind, plot_set, params):
    args = check_args4plot(args)
    # First step: load data
    # prc, roc, box, hp, 3d, dist, pie, bar
    print(args.plot)
    for pl in args.plot:
        if pl == 'net':
            if plot_set == 'valid':
                net_arr = np.load(args.data_dir + 'valid_net.npy') if not ind else np.load(args.data_dir + 'ind_net.npy')
            else:
                net_arr = np.load(args.data_dir + 'test_net.npy') if not ind else np.load(args.data_dir + 'ind_net.npy')
            args.rank = 'none'
            prob_dict = load_prob(args, ind, plot_set, params)
            args.rank = 'ltr'
            txt_path = args.fig_dir + 'net.txt' if not ind else args.fig_dir + 'net_ind.txt'
            fig_path = args.fig_dir + 'net.png' if not ind else args.fig_dir + 'net_ind.png'
            plot_net.save_txt(args.data_type, net_arr, prob_dict, txt_path)
            plot_net.net_fig(txt_path, fig_path)
        elif pl in ['prc', 'roc']:
            metric = 'aupr' if pl == 'prc' else 'auc'
            au_dict = load_metric(args, ind, plot_set, metric, params)
            print('au_dict: ', au_dict)
            if plot_set == 'valid':
                true_y = np.load(args.data_dir + 'valid_y.npy') if not ind else np.load(args.data_dir + 'ind_y.npy')
            else:
                true_y = np.load(args.data_dir + 'test_y.npy') if not ind else np.load(args.data_dir + 'ind_y.npy')
            prob_dict = load_prob(args, ind, plot_set, params)
            if pl == 'prc':
                fig_path = args.fig_dir + 'prc.png' if not ind else args.fig_dir + 'ind_prc.png'
                plot_curve.plot_prc(true_y, prob_dict, au_dict, fig_path)
            else:
                fig_path = args.fig_dir + 'roc.png' if not ind else args.fig_dir + 'ind_roc.png'
                plot_curve.plot_roc(true_y, prob_dict, au_dict, fig_path)
        elif pl == 'box':
            aupr_list, auc_list, ndcg_list, labels = load_results(args, ind, plot_set, params)
            fig_path = args.fig_dir + 'box.png' if not ind else args.fig_dir + 'ind_box.png'
            plot_bin.box_fig(aupr_list, auc_list, ndcg_list, labels, fig_path)
        elif pl == 'polar':
            result_dict = load_eval(args, ind, plot_set, params)
            print(result_dict)
            fig_path = args.fig_dir + 'polar.png' if not ind else args.fig_dir + 'ind_polar.png'
            plot_bin.polar_fig(list(result_dict.keys()), list(result_dict.values()), params['metrics'], fig_path)
        elif pl == 'dr':
            prob_dict = load_prob(args, ind, plot_set, params)
            prob_arr = np.array(list(prob_dict.values()), dtype=np.float).transpose()
            if plot_set == 'valid':
                true_x = np.load(args.data_dir + 'valid_x.npy') if not ind else np.load(args.data_dir + 'ind_x.npy')
                true_y = np.load(args.data_dir + 'valid_y.npy') if not ind else np.load(args.data_dir + 'ind_y.npy')
            else:
                true_x = np.load(args.data_dir + 'test_x.npy') if not ind else np.load(args.data_dir + 'ind_x.npy')
                true_y = np.load(args.data_dir + 'test_y.npy') if not ind else np.load(args.data_dir + 'ind_y.npy')
            new_x, new_prob, new_y = data_rus(true_x, prob_arr, true_y)
            old_fig_path = args.fig_dir + 'old_dr_3d.png' if not ind else args.fig_dir + 'old_dr_3d_ind.png'
            plot_bin.plot_3d(new_x, new_y, old_fig_path, old=True)
            new_fig_path = args.fig_dir + 'new_dr_3d.png' if not ind else args.fig_dir + 'new_dr_3d_ind.png'
            plot_bin.plot_3d(new_prob, new_y, new_fig_path, old=False)
        elif pl == 'dist':
            args.rank = 'none'
            if plot_set == 'valid':
                true_y = np.load(args.data_dir + 'valid_y.npy') if not ind else np.load(args.data_dir + 'ind_y.npy')
            else:
                true_y = np.load(args.data_dir + 'test_y.npy') if not ind else np.load(args.data_dir + 'ind_y.npy')
            prob_dict = load_prob(args, ind, plot_set, params)
            new_prob = prob_rus(prob_dict, true_y)
            fig_path = args.fig_dir + 'dist.png' if not ind else args.fig_dir + 'dist_ind.png'
            plot_bin.dist_fig(new_prob, list(prob_dict.keys()), fig_path)
            args.rank = 'ltr'
        elif pl == 'hp':
            args.rank = 'none'
            prob_dict = load_prob(args, ind, plot_set, params)
            fig_path = args.fig_dir + 'hp.png' if not ind else args.fig_dir + 'hp_ind.png'
            plot_bin.hp_fig(pd.DataFrame(prob_dict), fig_path)
            args.rank = 'ltr'
        elif pl == 'pie':
            if args.rank == 'ltr':
                args.rank = 'none'
                prob_dict = load_prob(args, ind, plot_set, params)
                fig_path = args.fig_dir + 'pie.png' if not ind else args.fig_dir + 'pie_ind.png'
                plot_int.pie_fig(args.int_path + 'ltr_model.pkl', list(prob_dict.keys()), fig_path)
                args.rank = 'ltr'
            else:
                print('If you want to plot pie fig, the rank method ga or de should be selected!\n')
        elif pl == 'bar':
            if args.rank == 'ltr':
                args.rank = 'none'
                prob_dict = load_prob(args, ind, plot_set, params)
                fig_path = args.fig_dir + 'bar.png' if not ind else args.fig_dir + 'bar_ind.png'
                plot_int.bar_fig(args.int_path + 'ltr_model.pkl', list(prob_dict.keys()), fig_path)
                args.rank = 'ltr'
            else:
                print('If you want to plot bar fig, the rank method ltr should be selected!\n')


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 23:59:24 2019

@author: orteg
"""
import click
import pandas as pd
import networkx as nx
from preprop_utils import splitgraph
from datetime import datetime

###################################################################

@click.command()
@click.argument('input_file',
                type=click.Path(exists=True, readable=True, dir_okay=False))
@click.argument('output_directory',
                type=click.Path(exists=True, file_okay=False, dir_okay=True))
@click.option('--generate_dynamic', is_flag=True, required = True)
@click.option('--training_size', default=9, required = True)
@click.option('--interm_directory',
              type=click.Path(exists=True, file_okay=False, dir_okay=True))

def main(input_file, output_directory, interm_directory, generate_dynamic, training_size):

  print('Preprocessing Facebook Graph Data')

  Gdf_fb = pd.read_csv(input_file, header = None)
  Gdf_fb.columns = ['node1','node2','timestamp']

  ## Replace node ids for a simpler sequence
  G_fb = nx.from_pandas_edgelist(Gdf_fb, 'node1', 'node2')
  dict_nodes_fb = dict(zip(G_fb.nodes(),list(range(len(G_fb.nodes())))))
  G_fb = nx.relabel_nodes(G_fb, dict_nodes_fb)

  ## Replace nodes for a pandas index
  Gdf_fb['node1'] = Gdf_fb['node1'].map(dict_nodes_fb)
  Gdf_fb['node2'] = Gdf_fb['node2'].map(dict_nodes_fb)

  # Create a time variables
  f_utc = '%Y-%m-%d %H:%M:%S'
  Gdf_aux = [datetime.utcfromtimestamp(Gdf_fb.timestamp[i]).strftime(f_utc) \
             for i in range(0,len(Gdf_fb))]

  df_time = pd.DataFrame({
                        'year': [Gdf_aux[i][0:4] for i in range(0,len(Gdf_aux))],
                        'month_day': [Gdf_aux[i][5:10] for i in range(0,len(Gdf_aux))],
                        'month': [Gdf_aux[i][5:7] for i in range(0,len(Gdf_aux))],
                        'iso_week': [datetime.isocalendar(datetime.utcfromtimestamp(Gdf_fb.timestamp[i]))[1] for i in range(0,len(Gdf_aux))],
                        'day': [Gdf_aux[i][8:10] for i in range(0,len(Gdf_aux))]
                        })
  Gdf_fb = pd.concat([df_time, Gdf_fb], axis = 1)

  #### CREATION OF DYNAMIC GRAPHS TIME SERIES ####

  ### STATIC NETWORKS ###
  train_limit = training_size

  if generate_dynamic == True:

  ### DYNAMIC NETWORKS ###

    # Create For Dynamic Setting
    G_dy_train_fb, Gsub_dy_test_fb = splitgraph(Gdf_fb,
                                                ('month', train_limit),
                                                dynamic = True,
                                                unseen_nodes_test = True)

    ## Export Dynamic Networks ##

    ## Export Graphs as Pickle
    nx.write_gpickle(G_dy_train_fb,output_directory+'G_dy_train_fb')
    nx.write_gpickle(Gsub_dy_test_fb,output_directory+'Gsub_dy_test_fb')

  else:
    # Create for Static Setting
    G_st_train_fb, Gsub_st_test_fb = splitgraph(Gdf_fb,
                                                ('month', train_limit),
                                                dynamic = False,
                                                unseen_nodes_test = True)
    ## Export Static Networks ##
    # Static Graphs
    nx.write_gpickle(G_st_train_fb,output_directory+'G_st_train_fb')
    nx.write_gpickle(Gsub_st_test_fb,output_directory+'Gsub_st_test_fb')

  ### EXPORTING CSV Files ###

  ## Export CSV
  fb_train = Gdf_fb[Gdf_fb['month']<=train_limit]
  fb_test = Gdf_fb[Gdf_fb['month']>train_limit]

  fb_train.to_csv(interm_directory+'train.csv', index = False)
  fb_test.to_csv(interm_directory+'test.csv', index = False)

if __name__ == '__main__':
  main()

max_stars_repo_name: CarlosOrtegaV/dynamite

max_issues_repo_name: CarlosOrtegaV/dynamite

max_forks_repo_name: CarlosOrtegaV/dynamite

reason: import networkx


text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

#importing the necessary libraries
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn import svm
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler

# # Data collection and analysis

# In[2]:

#lloading dataset
df = pd.read_csv('C:/Users/Hemant/jupyter_codes/ML Project 1/Diabetes detection/diabetes.csv')

# In[3]:

df.head()

# In[4]:

#shape of the dataset
df.shape

# In[5]:

#getting the statistical measure of the dataset
df.describe()

# In[6]:

df['Outcome'].value_counts() #give the total count of different values in the column
'''
0 ----> non-diabetic
1 ----> diabetic
'''

# In[7]:

#let us check the mean of the data across the outcome column
df.groupby('Outcome').mean()

# In[8]:

#sepearting data and labels
X = df.drop(columns = 'Outcome', axis = 1)
Y = df['Outcome']

# Standardize the data

# In[9]:

scaler = StandardScaler()
standardize_data = scaler.fit(X)
standardize_data = scaler.transform(X)

# In[10]:

standardize_data

# In[11]:

X = standardize_data
y = df['Outcome']

# split the data in training and testing data

# In[12]:

x_train , x_test, y_train, y_test = train_test_split(X, Y, test_size = 0.2, stratify = y, random_state = 2)

# # TRAINING THE MODEL

# In[13]:

classifier = svm.SVC(kernel = 'linear')

# In[14]:

#training the SUPPORT VECTOR MACHINE CLASSIFIER
classifier.fit(x_train, y_train)

# evaluating the model

# In[15]:

#accuracy score on the training data
x_train_predict = classifier.predict(x_train)
training_data_accuracy = accuracy_score(x_train_predict, y_train)

# In[16]:

print('ACCURAY SCORE OF THE TRAINING DATA : ', training_data_accuracy)

# In[17]:

#accuracy score on the testing data
x_test_predict = classifier.predict(x_test)
test_data_accuracy = accuracy_score(x_test_predict, y_test)

# In[18]:

print('ACCURAY SCORE OF THE TESTING DATA : ', test_data_accuracy)

# # MAKING A PREDICTING SYSTEM

# In[24]:

input_data = input()
input_array = [float(i) for i in input_data.split(',')]

#convert the input into numpy array
input_array = np.asarray(input_array)

#reshape the array as we are predicting only on one instance
reshaped_array = input_array.reshape(1, -1)

#standardize the input data
std_data = scaler.transform(reshaped_array)
print(std_data)

#prediction
prediction = classifier.predict(std_data)
print(prediction)
if prediction == 0:
    print('NON-DIABETIC')
else:
    print('DIABETIC')

# In[ ]:


text:
from __future__ import print_function

import os
import json
import logging
import numpy as np
from datetime import datetime

def prepare_dirs_and_logger(config):
    formatter = logging.Formatter("%(asctime)s:%(levelname)s::%(message)s")
    logger = logging.getLogger()

    for hdlr in logger.handlers:
        logger.removeHandler(hdlr)

    handler = logging.StreamHandler()
    handler.setFormatter(formatter)

    logger.addHandler(handler)

    if config.load_path:
        if config.load_path.startswith(config.log_dir):
            config.model_dir = config.load_path
        else:
            if config.load_path.startswith(config.dataset):
                config.model_name = config.load_path
            else:
                config.model_name = "{}_{}".format(config.dataset, config.load_path)
    else:
        config.model_name = "{}_{}".format(config.dataset, get_time())

    if not hasattr(config, 'model_dir'):
        config.model_dir = os.path.join(config.log_dir, config.model_name)
    config.data_path = os.path.join(config.data_dir, config.dataset)

    for path in [config.log_dir, config.data_dir, config.model_dir]:
        if not os.path.exists(path):
            os.makedirs(path)

def get_time():
    return datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

def save_config(config):
    param_path = os.path.join(config.model_dir, "params.json")

    print("[*] MODEL dir: %s" % config.model_dir)
    print("[*] PARAM path: %s" % param_path)

    with open(param_path, 'w') as fp:
        json.dump(config.__dict__, fp, indent=4, sort_keys=True)


text:
import abc

import numpy as np

__all__ = ["ReLU", "Sigmoid", "Identity"]

class Activation:
    """An activation function.

    Each activation function is represented by a class and implements two methods. The first method
    is `apply`, which evaluates the activation on an array. The second method is `gradient`, which
    computes the gradient with respect to the input array. Both methods are intended to be pure
    with no side-effects. In other words they do not modify their inputs.

    """

    @abc.abstractstaticmethod
    def apply(self, z):
        """Apply the activation function to a layer output z."""

    @abc.abstractstaticmethod
    def gradient(self, z):
        """Return the gradient with respect to a layer output z."""

class ReLU(Activation):
    """Rectified Linear Unit (ReLU) activation function."""

    @staticmethod
    def apply(z):
        a = np.copy(z)
        a[a < 0] = 0
        return a

    @staticmethod
    def gradient(z):
        a = np.zeros_like(z, dtype=z.dtype)
        a[z > 0] = 1
        return a

class Sigmoid(Activation):
    """Sigmoid activation function."""

    @staticmethod
    def apply(z):
        return 1 / (1 + np.exp(-z))

    @staticmethod
    def gradient(z):
        s = Sigmoid.apply(z)
        return s * (1 - s)

class Identity(Activation):
    """Identity activation function."""

    @staticmethod
    def apply(z):
        return np.copy(z)

    @staticmethod
    def gradient(z):
        return np.ones_like(z, dtype=z.dtype)

max_stars_repo_name: fox-ds/river

max_issues_repo_name: raphaelsty/river

max_forks_repo_name: raphaelsty/river


text:
import tensorflow as tf
import os
from tensorflow.python.keras.models import Model
from tensorflow.keras.layers import Layer, InputSpec, Conv2D, DepthwiseConv2D, UpSampling2D, ZeroPadding2D, Lambda, AveragePooling2D, MaxPooling2D, Conv2DTranspose, Input, Activation, Concatenate, Add, Reshape, BatchNormalization, Dropout, concatenate
from tensorflow.keras import backend as K

import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import classification_report

class DLModel :
    def __init__(self,training,num_features,num_classes,batch_size,optimiser,loss,metrics,checkpoint_file):
        self.num_features = num_features
        self.num_classes  = num_classes
        self.optimiser    = optimiser
        self.loss         = loss
        self.metrics      = metrics
        self.checkpoint_file = checkpoint_file

    def unet(self,pretrained_weights = None):
        def conv_block(input_tensor, num_filters):
            encoder = Conv2D(num_filters, (3, 3), padding='same',kernel_initializer = 'he_normal')(input_tensor)
            encoder = BatchNormalization()(encoder)
            encoder = Activation('relu')(encoder)
            encoder = Conv2D(num_filters, (3, 3), padding='same',kernel_initializer = 'he_normal')(encoder)
            encoder = BatchNormalization()(encoder)
            encoder = Activation('relu')(encoder)
            return encoder

        def encoder_block(input_tensor, num_filters):
            encoder = conv_block(input_tensor, num_filters)
            encoder_pool = MaxPooling2D((2, 2), strides=(2, 2))(encoder)
            return encoder_pool, encoder

        def decoder_block(input_tensor, concat_tensor, num_filters):
            decoder = Conv2DTranspose(num_filters, (2, 2), strides=(2, 2), padding='same',kernel_initializer = 'he_normal')(input_tensor)
            decoder = concatenate([concat_tensor, decoder], axis=-1)
            decoder = BatchNormalization()(decoder)
            decoder = Activation('relu')(decoder)
            decoder = Conv2D(num_filters, (3, 3), padding='same',kernel_initializer = 'he_normal')(decoder)
            decoder = BatchNormalization()(decoder)
            decoder = Activation('relu')(decoder)
            decoder = Conv2D(num_filters, (3, 3), padding='same',kernel_initializer = 'he_normal')(decoder)
            decoder = BatchNormalization()(decoder)
            decoder = Activation('relu')(decoder)
            return decoder

        inputs = Input(shape=[None, None, self.num_features]) # 256
        encoder_pool, encoder0 = encoder_block(inputs, 32) # 128
        encoder_pool, encoder1 = encoder_block(encoder_pool, 64) # 64
        encoder_pool, encoder2 = encoder_block(encoder_pool, 128) # 32
        encoder_pool, encoder3 = encoder_block(encoder_pool, 256) # 16
        encoder_pool, encoder4 = encoder_block(encoder_pool, 512) # 8
        center = conv_block(encoder_pool, 1024) # center
        decoder = decoder_block(center, encoder4, 512) # 16
        del center,encoder4
        decoder = decoder_block(decoder, encoder3, 256) # 32
        del encoder3
        decoder = decoder_block(decoder, encoder2, 128) # 64
        del encoder2
        decoder = decoder_block(decoder, encoder1, 64) # 128
        del encoder1
        decoder = decoder_block(decoder, encoder0, 32) # 256
        del encoder0
        outputs = Conv2D(self.num_classes, (1, 1), activation='softmax')(decoder)
        del decoder
        model = tf.python.keras.models.Model(inputs=[inputs], outputs=[outputs])

        model.compile(
            optimizer=tf.python.keras.optimizers.get(self.optimiser),
            loss=tf.python.keras.losses.get(self.loss),
            metrics=[tf.python.keras.metrics.get(metric) for metric in self.metrics]
            )
        if pretrained_weights:
            pretrained = tf.keras.models.load_model(pretrained_weights).get_weights()
            model.set_weights(pretrained)
        return model

    def init_model(self,from_checkpoint=False):
        if from_checkpoint :
            if not os.path.isdir(self.checkpoint_file) :
                if self.num_features == 12 :
                    m = self.unet('models/unet_12_bands_sgd_wscc')#pretrained model
                if self.num_features == 20 :
                    m = self.unet('models/unet_20_bands_sgd_wscc')#pretrained model
                else :
                    m = self.unet()
            else :
                m = self.unet(self.checkpoint_file)
        else :
            m = self.unet()
        return m

class ModelEvaluation():
    def __init__(self,model_name,eval_size,target_names,label_names):
        self.model_name   = model_name
        self.eval_size    = eval_size
        self.target_names = target_names
        self.label_names  = label_names

    def get_labels(self,model,evaluation) :
        y_true = []
        y_pred = []
        i=0
        for feature,label in evaluation.take(self.eval_size) :
            if i%500 == 0 :
                print('processing evaluation sample number ',i)
            i+=1
            y_true.append(label.numpy())
            y_pred.append(tf.argmax(model.predict(
                x=feature,
                batch_size=1,
                steps=1,
                verbose=0),axis=3).numpy())
        y_pred = np.stack(y_pred).reshape([-1])
        y_true = np.stack(y_true).reshape([-1])
        return y_true, y_pred

    def get_performance(self,y_true,y_pred,label_names,target_names):
        results = classification_report(y_true,y_pred,labels=label_names,target_names=target_names,output_dict=True)
        df = pd.DataFrame.from_dict(results).transpose()
        df['precision'] *= 100
        df['recall']    *= 100
        df['f1-score']  *= 100
        df = df.round(2)
        print(df)

    def visualize_confusion_matrix(self,y_true,y_pred_argmax,name):
        """
        param y_pred_arg
        param y_true
        return:
        """
        cm = tf.math.confusion_matrix(y_true,y_pred_argmax).numpy()
        con_mat_df = pd.DataFrame(cm)
        sns.heatmap(con_mat_df, annot=True, fmt='g', cmap=plt.cm.Blues)
        plt.tight_layout()
        plt.ylabel('True label')
        plt.xlabel('Predicted label')
        plt.savefig('results/'+name+'_confusion_matrix.png')
        plt.show()
        return con_mat_df

    def evaluate(self,model,evaluation):
        y_true,y_pred = self.get_labels(model,evaluation)
        self.get_performance(y_true,y_pred,self.label_names,self.target_names)
        self.visualize_confusion_matrix(y_true,y_pred,self.model_name)


text:
import numpy as np

from sklearn.metrics import pairwise_distances, silhouette_score

from scipy.cluster import hierarchy
from scipy.cluster.hierarchy import dendrogram, linkage, fcluster, ward
from scipy.spatial.distance import pdist

import matplotlib
matplotlib.use('TkAgg')
#plt = matplotlib.pyplot
from matplotlib import pyplot as plt
import seaborn as sns

from .wit3langs import wit3_map
from .load_datasets import get_list_languages, PATH_FEATS
PATH_FIGS = "./figs_obj.v3/"

# computing hierarchical stuf....

def inertia_score(X, a, metric="euclidean"):
    '''
    a: assignments (predictions)
    X: dataset
    '''
    W = [np.mean(pairwise_distances(X[a == c, :], metric=metric)) for c in np.unique(a)]
    return np.mean(W)

def linkage_matrix(X, method="ward", metric="euclidean"):
    y = pdist(X, metric = metric)
    if method == "ward":
        #y = pdist(X, metric = metric)
        Z = ward(y)
    else:
        #Z = linkage(X, method = method, metric = metric)
        Z = linkage(y, method = method, metric = metric)
    return Z, y

# transformation ...
from sklearn.decomposition import TruncatedSVD, PCA

def svd_transform(X, mode="truncated"):
    n = X.shape[1] - 1
    if mode == "truncated":
        #svd = TruncatedSVD(random_state=42, n_components=n, algorithm="arpack")
        svd = TruncatedSVD(random_state=42, n_iter=15, n_components=n)
    elif mode == "full":
        svd = PCA(random_state=42, n_components=n, svd_solver="full")
    X_svd = svd.fit_transform(X)
    '''
    if th > 0:
        p = 0
        sum_variance = 0
        for i in range(n):
            sum_variance += svd.explained_variance_ratio_[i]
            p += 1
            if sum_variance >= th:
                break
        X_svd = X_svd[:, :p]
    '''
    return X_svd, svd

def get_X_svd_filter(X, X_svd, svd_model, svd_th=1.):
    if svd_th == 1.:
        return X
    elif svd_th < 1.:
        dimensions = trunc_svd_by_th(svd_model, svd_th)
        X_filter = X_svd[:, :dimensions]
        return X_filter

def trunc_svd_by_th(svd_model, th):
    p = 0
    sum_variance = 0
    n = svd_model.n_components
    for i in range(n):
        sum_variance += svd_model.explained_variance_ratio_[i]
        p += 1
        if sum_variance >= th:
            break
    return p

# features & plotting ....
wit3_bib24_ids = ['arb', 'bul', 'ces', 'deu', 'ell', 'spa', 'pes', 'fra', 'hun', 'ind', 'ita', 'jpn',
                  'kor', 'nld', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'tha', 'tur', 'vie', 'zho']
wit3_bib22_ids = ['arb', 'bul', 'ces', 'deu', 'ell', 'spa', 'pes', 'fra', 'hun', 'ita', 'jpn',
                  'nld', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'tha', 'tur', 'vie', 'zho']

def load_families(list_langs):
    import lang2vec.lang2vec as l2v
    # extracting the one-hot-vectors of families
    X_families = l2v.get_features(list_langs, "fam", header=True)
    # family names
    list_fam_names = X_families['CODE']
    # one-hot-codes
    X_fam_data = []
    for lang in list_langs:
        X_fam_data.append(X_families[lang])
    X_fam_data = np.array(X_fam_data)
    # extracting the families with at least one entry
    X_dict_fam = {}
    for idx_fam, fam in enumerate(X_fam_data.T):
        # if it has at least one entry with 1 ...
        if max(fam) == 1:
            fam_name = list_fam_names[idx_fam]
            X_dict_fam[fam_name] = []
            # identify the languages that belongs to that family group
            for idx_lang, lang_value in enumerate(fam):
                if lang_value == 1:
                    X_dict_fam[fam_name].append(list_langs[idx_lang])

    # reducing the "duplicated" family groups: with the same members
    family_values = [" ".join(sorted(group)) for group in X_dict_fam.values()]
    # we sort the groups by LEN for further processing
    lang_groups = sorted(list(set(family_values)), key=len)
    # converting back to a list, for easier processing
    lang_groups_lists = [group.split() for group in lang_groups]
    # if the last entry has more than half of the entries, we truncate it: (Indo-European)
    if len(lang_groups_lists[-1]) > len(list_langs) / 2:
        lang_groups_lists = lang_groups_lists[:-1]
    '''
    reverse_idx = -1
    while len(lang_groups_lists[reverse_idx]) > 50:
        reverse_idx -= 1
    reverse_idx += 1
    if reverse_idx < 0:
        lang_groups_lists = lang_groups_lists[:reverse_idx]
    '''
    # obtaining the final list of family groups that are not included in bigger groups
    final_list = []
    for idx, group in enumerate(lang_groups_lists):
        # flag to identify if a group is included in a posterior one (pre-sorting is a must)
        flag_included = False
        for other_group in lang_groups_lists[idx+1:]:
            # if all elements of other_group includes elements from group
            if all(elem in other_group for elem in group):
                flag_included = True
                break
        # if the group is not included, we should keep it
        if not flag_included:
            final_list.append(group)
    # obtaining the final names
    final_dict = {}
    for g in final_list:
        # we asume that the family names are sorted in priority/inclusivity (e.g. Italic before Romance)
        for key, val in X_dict_fam.items():
            # we compare at string level...
            if " ".join(g) == " ".join(sorted(val)):
                final_dict[key] = g
                break
    #for key, val in final_dict.items():
    #    print(key, len(val))

    return final_dict

def load_features(space_name, source_name):

    # source_name : ["Bib22", "Bib24", "Tan23", "Own23", "Own25", "Fact23", "Fact25"]
    label_total = None
    X_total = []
    #print(space_name, source_name)
    if space_name == "uriel":
        feat_set_uriel = source_name + "<" + space_name # syntax_knn<uriel
        X_npdict = np.load(PATH_FEATS + feat_set_uriel + ".npy")
        label_total = list(get_list_languages(source = "uriel"))
        #print(len(label_total))

    elif space_name == "learn":
        if source_name in ["Bib22", "Bib24", "Bible"]:
            feat_set_learn = "learned<learned"
            X_npdict = np.load(PATH_FEATS + feat_set_learn + ".npy")
            if source_name == "Bible":
                label_total = list(get_list_languages(source = "learned"))
            elif source_name == "Bib22":
                label_total = wit3_bib22_ids
            elif source_name == "Bib24":
                label_total = wit3_bib24_ids
            #label_total[label_total.index("zho")] = "cmn" # unifying Chinese (zho, cmn)

        #elif source_name in ["Tan23", "Own23", "Fact23", "Own25", "Fact25", "Fact23-A", "Fact23-B"] or source_name.startswith("ted"):
        else:
            learn_emb_file = "data/embeddings/%s.npy" % source_name.split("-")[0]
            X_npdict = np.load(learn_emb_file)
            label_total = sorted(list(X_npdict.item().keys()))

    for l in label_total:
    #    if source_name == "Fact23-A":
    #        X_total.append(X_npdict.item().get(l)[:256])
    #    elif source_name == "Fact23-B":
    #       X_total.append(X_npdict.item().get(l)[256:])
    #    else:
        X_total.append(X_npdict.item().get(l))

    # tag 2 iso codes
    #if source_name in ["Own23", "Fact23", "Own25", "Fact25", "Fact23-A", "Fact23-B"]:
    #    label_total = [wit3_map[l] for l in label_total]

    if "zho" in label_total:
        label_total[label_total.index("zho")] = "cmn"
    if "hbo" in label_total:
        label_total[label_total.index("hbo")] = "heb"

    # adding extra iso codes in URIEL (they are there!!!!!)
    if space_name == "uriel":
        list_extra_uriel = ["aze", "mon", "sqi"]
        import lang2vec.lang2vec as l2v
        X_extra_dict = l2v.get_features(list_extra_uriel, source_name)
        X_extra = []
        for l in list_extra_uriel:
            label_total.append(l)
            X_extra.append(X_extra_dict[l])
        X_total = np.concatenate((X_total, np.array(X_extra)), axis = 0)

    return np.array(X_total), label_total

new_colors = ['#1f77b4', '#17becf', '#2ca02c', '#d62728',
              '#9467bd', '#8c564b', '#e377c2', '#7f7f7f',
              '#bcbd22', '#ff7f0e']

def save_numc_and_dend_plot(Z, langs_names, maxclusters, X_inertia, X_silhouette, name, path, color_th=0, ang_rot=90, orient="top"):
    if path is None:
        path = PATH_FIGS
    f, (a0, a1, a2) = plt.subplots(1, 3, figsize=(15,2), gridspec_kw={'width_ratios': [1, 1, 3]})

    x_items = min(20,maxclusters+1)
    if x_items < len(X_inertia):
        X_inertia = X_inertia[:x_items-2]
        X_silhouette = X_silhouette[:x_items-2]
    #First subplot: Elbow method (Inertia)
    a0.title.set_text("Elbow method")
    a0.grid(True, axis='x', linewidth=0.5)
    a0.plot(range(2, x_items), X_inertia,'-') #, label='data')
    a0.set_xlabel('# clusters')
    #a0.set_ylabel('Inertia score')
    a0.set_xticks(np.arange(2, x_items, step=2))
    a0.tick_params(axis='x', which='minor', labelsize=2)

    #Second subplot: Silhouette analysis
    a1.title.set_text("Silhouette analysis")
    #a1.yaxis.tick_right()
    #a1.yaxis.set_label_position("right")
    a1.grid(True, axis='x', linewidth=0.5)
    a1.plot(range(2, x_items), X_silhouette, '-')
    #a1.set_ylabel('Silhouetthe score')
    a1.set_xlabel('# clusters')
    a1.set_xticks(np.arange(2, x_items, step=2))

    #Third subplot: Dendrogram
    # analyse color_th:
    if color_th == 0: color_th = np.median(Z[:,2])

    a2.title.set_text("Dendrogram: " + name)
    a2.yaxis.tick_right()
    a2.yaxis.set_label_position("right")
    a2.grid(False)
    hierarchy.set_link_color_palette(new_colors[0:7] + new_colors[8:])
    hierarchy.dendrogram( Z = Z,
                orientation=orient,
                labels=langs_names,
                distance_sort='descending',
                color_threshold=color_th,
                show_leaf_counts=True,
                leaf_rotation=ang_rot,
                above_threshold_color=new_colors[7]
            )
    f.savefig(path + name + ".pdf", bbox_inches='tight', transparent=True)
    plt.close()

syntax_feat_areas = [0,12,13,14,16,18,20,22,24,26,30,32,35,37,49,52,54,56,58,60,61,65,66,67,68,69,70,71,72,74,76,81,84,87,89,90,92,101,103]
def get_related_features(idx, feat_class="syntax"):
    #looking for the interval...
    if feat_class == "syntax":
        feat_areas = syntax_feat_areas
        for i, boundary in enumerate(feat_areas):
            if idx >= boundary:
                ini_idx = boundary
                fin_idx = feat_areas[i+1]
            break
        array_to_delete = []
        for i in range(ini_idx, fin_idx):
            array_to_delete.append(i)
        return np.array(array_to_delete)
    else:
        return np.array([idx])

max_stars_repo_name: aoncevay/multiview-langrep

max_issues_repo_name: aoncevay/multiview-langrep

max_forks_repo_name: aoncevay/multiview-langrep

reason: import numpy,from scipy


text:
import matplotlib
from matplotlib import pyplot as plt
import matplotlib.colors as mcolors
from matplotlib.patches import Patch
import numpy as np
import os
from matplotlib.pyplot import bar, figure
import json

matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42

if os.path.isdir("./results/cache-effects/"):
    RESULTS_PATH = "./results/cache-effects/"
else:
    RESULTS_PATH = "./results-paper/cache-effects/"

def bootstrap_breakdown():

    figure(figsize=(8,3.6))
    lines=2
    matplotlib.rcParams['hatch.linewidth'] = lines
    new_kernels = [ "bzImage-lupine4-nokaslr-lz4-cache.txt", "bzImage-aws-nokaslr-lz4-cache.txt", "bzImage-ubuntu-nokaslr-lz4-cache.txt"]

    ax = [1, 1.5, 2]
    labels = ["lupine", "aws", "ubuntu"]
    bar_width = 0.15
    colors= mcolors.CSS4_COLORS
    bootstrap_color=colors["darkcyan"]
    parse_elf_color=colors["deepskyblue"]
    decompress_color=colors["mediumseagreen"]
    bar_color="white"

    legend = [
        Patch(edgecolor=parse_elf_color, facecolor=bar_color, label="parse_elf", hatch="\\\\\\"),
        Patch(edgecolor=decompress_color, facecolor=bar_color, label="decompression", hatch="..."),
        Patch(edgecolor=bootstrap_color, facecolor=bar_color, label="bootstrap setup", hatch="///"),
    ]

    decomp_percents = []

    for k, x in zip(new_kernels, ax):
        filename = RESULTS_PATH + k

        bootstrap_times = []
        decomp_times = []
        parse_elf_times = []
        handle_relocations_times = []
        post_relocations_times = []

        with open(filename) as f:
            all_json = f.readlines()

            for i in range(len(all_json)):
                try:
                    all_json[i]=json.loads(all_json[i])
                    bootstrap_times.append(all_json[i]["bootstrap"]/1000.0)
                    decomp_times.append(all_json[i]["decomp"]/1000.0)
                    parse_elf_times.append(all_json[i]["parse_elf"]/1000.0)
                except:
                    print(filename)

        bootstrap_time = np.average(bootstrap_times)
        bootstrap_min = np.min(bootstrap_times)
        bootstrap_max = np.max(bootstrap_times)
        decomp_time = np.average(decomp_times)
        decomp_min = np.min(decomp_times)
        decomp_max = np.max(decomp_times)
        parse_elf = np.average(parse_elf_times)
        parse_elf_min = np.min(parse_elf_times)
        parse_elf_max = np.max(parse_elf_times)

        decomp_percents.append(decomp_time/(bootstrap_time + decomp_time + parse_elf))

        plt.bar(x, parse_elf, width=bar_width, bottom=decomp_time + bootstrap_time, edgecolor=parse_elf_color, hatch="\\\\", color=bar_color, linewidth=lines, yerr=parse_elf_max-parse_elf_min, capsize=lines*2)
        plt.bar(x, decomp_time, width=bar_width, bottom=bootstrap_time, edgecolor=decompress_color, hatch="..", color=bar_color, linewidth=lines, yerr=decomp_max-decomp_min, capsize=lines*2)
        plt.bar(x, bootstrap_time, width=bar_width, bottom=None, edgecolor=bootstrap_color, hatch="//", color=bar_color, linewidth=lines, yerr=bootstrap_max-bootstrap_min, capsize=lines*2)

    plt.ylabel("Time (ms)")
    plt.xticks(ax, labels)
    plt.legend(handles=legend)
    plt.savefig("./graphs/bootstrap-loader-breakdown.pdf", format="pdf")
    plt.cla()

if __name__ == "__main__":
    bootstrap_breakdown()


text:
'''
Import data from ERNIE postgres and generate a Chord Diagram
Author: VJ Davey
'''
import psycopg2
import argparse
import pandas as pd
import numpy as np
import chord
from math import ceil

def fibonacci(n,memo):
    if (n < 2):
        return 1
    if n in memo:
        return memo[n]
    else:
        memo[n] = fibonacci(n-1,memo) + fibonacci(n-2,memo)
        return memo[n]

def fibonacci_color(h,s,l,a=0.9,num_colors=3):
    color_list=[]
    memo={}
    for i in range(num_colors):
        h = ((h+fibonacci(i+1,memo))%200) +100
        s = s#((s+fibonacci(i+1,memo))% 20) + 30
        l = l#((l-fibonacci(i+1,memo))% 20) + 30
        color_list+=["hsla({},{},{},{})".format(h,s,l,a)]
    return color_list

# Collect user input and possibly override defaults based on that input
parser = argparse.ArgumentParser(description='''
 This script interfaces with the PostgreSQL database and then generates a Sankey diagram
''', formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-ph','--postgres_host',help='the server hosting the PostgreSQL server',default='localhost',type=str)
parser.add_argument('-pd','--postgres_dbname',help='the database to query in the PostgreSQL server',type=str,required=True)
parser.add_argument('-pp','--postgres_port',help='the port hosting the PostgreSQL service on the server', default='5432',type=int)
parser.add_argument('-U','--postgres_user',help='the PostgreSQL user to log in as',required=True)
parser.add_argument('-W','--postgres_password',help='the password of the PostgreSQL user',required=True)
parser.add_argument('-A','--award_numbers',help='a string containing a comma seperated list of award_numbers (e.g "1038028,1305427")',required=True)
parser.add_argument('-S','--start_year',help='the lower end of the publication year range to consider',default='1930')
parser.add_argument('-E','--end_year',help='the upper end of the publication year range to consider',default='2018')
parser.add_argument('-f','--output_file_prepend',help='name to prepend to the output file')
args = parser.parse_args()
postgres_dsn={'host':args.postgres_host,'dbname':args.postgres_dbname,'port':args.postgres_port,'user':args.postgres_user,'password':args.postgres_password}
postgres_conn=psycopg2.connect(" ".join("{}={}".format(k,postgres_dsn[k]) for k in postgres_dsn))
award_numbers=tuple(args.award_numbers.split(','))
start_year=args.start_year
end_year=args.end_year
print(award_numbers)
# select coauthor pairs to form ideogram edges and interconnected ribbons
coauthor_df=pd.read_sql_query('''
                    WITH
                    award_author_documents AS
                    (
                      SELECT DISTINCT
                       concat(foo.given_name,' ', foo.surname) as name,
                       a.scopus_id,
                       (SELECT b.publication_year FROM cci_s_documents_jeroen_updated b WHERE a.scopus_id=b.scopus_id) as publication_year
                      FROM cci_s_author_document_mappings a
                      INNER JOIN
                      (SELECT author_id,surname,given_name
                            FROM cci_s_author_search_results
                            WHERE award_number IN %(award_numbers)s
                            AND author_id IN (SELECT author_id FROM sl_sr_all_personel_and_comp)
                      ) foo
                      ON a.author_id=foo.author_id
                    ),
                    co_author_links AS
                    (
                        SELECT a.name as auth1,b.name as auth2, a.scopus_id
                        FROM award_author_documents a
                        JOIN award_author_documents b
                        ON a.scopus_id=b.scopus_id
                        WHERE
                        a.publication_year BETWEEN %(start_year)s AND %(end_year)s
                        AND a.name > b.name -- NOTE: This is done to avoid double counts
                    )

                    SELECT auth1,auth2,n_links
                    FROM
                    (
                      -- Collect intranetwork links between authors
                     SELECT auth1,auth2,count(scopus_id) as n_links
                     FROM co_author_links
                     GROUP BY auth1,auth2
                    ) foo
                    ORDER BY auth1;
''',params={'award_numbers':award_numbers,'start_year':start_year,'end_year':end_year}, con=postgres_conn)
# Build a sorted list of author_ids
print(coauthor_df)
print(sum(coauthor_df['n_links']))
unique_authors_series=sorted(coauthor_df['auth1'].append(coauthor_df['auth2']).unique())
unique_authors=sorted(coauthor_df['auth1'].append(coauthor_df['auth2']).unique())
unique_authors={unique_authors[i]:i for i in range(len(unique_authors))}
print(unique_authors)
# Create and populate an NxN Matrix for the unique authors in the list: M[i][j] = COUNT(coauthor pairs)
data_matrix=np.zeros((len(unique_authors),len(unique_authors)),dtype=int)
for row in coauthor_df.itertuples():
    #print("{} and {} have {} documents together".format(row.auth1,row.auth2,row.n_links))
    i = unique_authors[row.auth1]
    j = unique_authors[row.auth2]
    data_matrix[i,j]+=row.n_links
    data_matrix[j,i]+=row.n_links #NOTE: sum of matrix will exceed expected value due to double count here
print(data_matrix.shape)
data_df=pd.DataFrame(data_matrix)
for axis in ['columns', 'index']:
    data_df=data_df.rename({i:unique_authors_series[i] for i in range(data_matrix.shape[0])}, axis=axis)
print(data_df)
data_df.to_csv('{}_author_collaboration_matrix.csv'.format(args.output_file_prepend))
three_col_df=data_df.stack().reset_index()
three_col_df=three_col_df.rename({'level_0':'Author1','level_1':'Author2',0:'Number Of Collaborations'}, axis='columns')
print(three_col_df)
three_col_df.to_csv('{}_author_collaboration_three_cols.csv'.format(args.output_file_prepend),index=False)

labels=list(unique_authors)
seed_color=[180, 65, 35, .85]
colors=fibonacci_color(seed_color[0],seed_color[1],seed_color[2],seed_color[3],num_colors=data_matrix.shape[0])
print('length_colors:{}'.format(len(colors)))
title='{} Chord Diagram'.format(args.output_file_prepend)
dimensions=1200
hover_text={
            'inside':'',
            'outside':'{} has appeared on a document {} times with {} ',
            'ideogram':'{} <br> Has {} links to other authors in this network'
    }
chord.buildDiagram(data_matrix,dimensions,labels,colors,title,hover_text)

max_issues_repo_name: NETESOLUTIONS/ERNIE

max_forks_repo_name: NETESOLUTIONS/ERNIE


text:
from matplotlib import pyplot as plt
import numpy as np

__all__ = ["variations"]

# object for all physics variations with file name, short label, long label
# and the colour used in plotting with matplotlib
variations = np.array(
    [
        {
            "file": "fiducial",
            "short": "A",
            "med": "fiducial",
            "long": "Fiducial",
            "colour": "grey"
        },
        {
            "file": "massTransferEfficiencyFixed_0_25",
            "short": "B",
            "med": r"$\beta=0.25$",
            "long": r"Fixed mass transfer efficiency of $\beta=0.25$",
            "colour": plt.get_cmap("tab20b")(15)
        },
        {
            "file": "massTransferEfficiencyFixed_0_5",
            "short": "C",
            "med": r"$\beta=0.5$",
            "long": r"Fixed mass transfer efficiency of $\beta=0.5$",
            "colour": plt.get_cmap("tab20b")(14)
        },
        {
            "file": "massTransferEfficiencyFixed_0_75",
            "short": "D",
            "med": r"$\beta=0.75$",
            "long": r"Fixed mass transfer efficiency of $\beta=0.75$",
            "colour": plt.get_cmap("tab20b")(13)
        },
        {
            "file": "unstableCaseBB",
            "short": "E'",
            "med": "Case BB\nunstable",
            "long": "Case BB mass transfer is always unstable",
            "colour": plt.get_cmap("tab20")(3)
        },
        {
            "file": "unstableCaseBB_opt",
            "short": "F",
            "med": "E' + K",
            "long": "Case BB Unstable + Optimistic CE",
            "colour": plt.get_cmap("tab20")(2)
        },
        {
            "file": "alpha0_1",
            "short": "G",
            "med": r"$\alpha_{\rm CE}=0.1$",
            "long": r"CE efficiency $\alpha = 0.1$",
            "colour": plt.get_cmap("tab20b")(11)
        },
        {
            "file": "alpha0_5",
            "short": "H",
            "med": r"$\alpha_{\rm CE}=0.5$",
            "long": r"CE efficiency $\alpha = 0.5$",
            "colour": plt.get_cmap("tab20b")(10)
        },
        {
            "file": "alpha2_0",
            "short": "I",
            "med": r"$\alpha_{\rm CE}=2.0$",
            "long": r"CE efficiency $\alpha = 2.0$",
            "colour": plt.get_cmap("tab20b")(9)
        },
        {
            "file": "alpha10",
            "short": "J",
            "med": r"$\alpha_{\rm CE}=10.0$",
            "long": r"CE efficiency $\alpha = 10.0$",
            "colour": plt.get_cmap("tab20b")(8)
        },
        {
            "file": "optimistic",
            "short": "K",
            "med": "optimistic\nCE",
            "long": "HG donor stars initiating a CE survive CE",
            "colour": plt.get_cmap("tab20")(11)
        },
        {
            "file": "rapid",
            "short": "L",
            "med": "rapid SN",
            "long": "Fryer rapid SN remnant mass prescription",
            "colour": "lightseagreen"
        },
        {
            "file": "maxNSmass2_0",
            "short": "M",
            "med": r"max $m_{\rm NS}$" + "\n" + r"$2.0 \, {\rm M_{\odot}}$",
            "long": r"Maximum NS mass = 2.0 ${\rm M_{\odot}}$",
            "colour": plt.get_cmap("tab20")(1/20)
        },
        {
            "file": "maxNSmass3_0",
            "short": "N",
            "med": r"max $m_{\rm NS}$" + "\n" + r"$3.0 \, {\rm M_{\odot}}$",
            "long": r"Maximum NS mass = 3.0 ${\rm M_{\odot}}$",
            "colour": plt.get_cmap("tab20")(0/20)
        },
        {
            "file": "noPISN",
            "short": "O",
            "med": "no PISN",
            "long": "PISN and pulsational-PISN not implemented",
            "colour": "deepskyblue"
        },
        {
            "file": "ccSNkick_100km_s",
            "short": "P",
            "med": r"$\sigma_{\rm cc}$" + "\n" + r"$100 \, {\rm km s^{-1}}$",
            "long": r"$\sigma_{\rm RMS}^{\rm 1D} = 100 \ {\rm km\ s^{-1}}$ for core-collapse supernova",
            "colour": plt.get_cmap("tab20c")(11/20)
        },
        {
            "file": "ccSNkick_30km_s",
            "short": "Q",
            "med": r"$\sigma_{\rm cc}$" + "\n" + r"$30 \, {\rm km s^{-1}}$",
            "long": r"$\sigma_{\rm RMS}^{\rm 1D} = 30 \ {\rm km\ s^{-1}}$ for core-collapse supernova",
            "colour": plt.get_cmap("tab20c")(8/20)
        },
        {
            "file": "noBHkick",
            "short": "R",
            "med": "no BH\nkicks",
            "long": "Black holes receive not natal kick",
            "colour": "darkgreen"
        },
        {
            "file": "wolf_rayet_multiplier_0_1",
            "short": "S",
            "med": r"$f_{\rm WR} = 0.1$",
            "long": r"Wolf-Rayet wind factor $f_{\rm WR} = 0.1$",
            "colour": "thistle"
        },
        {
            "file": "wolf_rayet_multiplier_5",
            "short": "T",
            "med": r"$f_{\rm WR} = 5$",
            "long": r"Wolf-Rayet wind factor $f_{\rm WR} = 5.0$",
            "colour": "purple"
        }
    ]
)

max_stars_repo_name: TomWagg/detecting-DCOs-in-LISA

max_issues_repo_name: TomWagg/detecting-DCOs-in-LISA

max_forks_repo_name: TomWagg/detecting-DCOs-in-LISA


text:
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import os
import theano
from theano import tensor as T
from theano.tensor.nnet.conv import conv2d
from theano.tensor.signal.pool import pool_2d
import numpy as np
import _pickle as cPickle
from math import sqrt

from PIL import Image
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from django.conf import settings

class CNNTest:
    def floatX(self, X):
        return np.asarray(X, dtype=theano.config.floatX)  # @UndefinedVariable

    def rectify(self, X):
        return T.maximum(X, 0.)

    def softmax(self, X):
        e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))
        return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')

    def generateHeWeights(self, count, n):
        return np.random.randn(count) * sqrt(2.0/n)

    def init_conv_weights(self, shape):
        weights = np.zeros(shape)
        for i in range(shape[0]):
            for j in range(shape[1]):
                w = self.generateHeWeights(shape[2]*shape[3], shape[1])
                w = w.reshape(shape[2], shape[3])
                weights[i][j] = w
        return theano.shared(self.floatX(weights))

    def init_xavier_weights(self, shape):
        weights = np.random.uniform((-np.sqrt(6.0 / (shape[0] + shape[1]))), (np.sqrt(6.0 / (shape[0] + shape[1]))), size=(shape[0], shape[1]))
        return theano.shared(self.floatX(weights))

    def init_weights(self, shape):
        return theano.shared(self.floatX(np.zeros(shape)))

    def init_bias(self, shape):
        return theano.shared(self.floatX(np.zeros(shape)))

    def load_weights(self, w, b, w2, b2, w3, b3, w4, b4, w_o, b_o):
        bobot = os.path.join(settings.STORAGE_ROOT, 'bobot2_CNN.save')
        save_file = open(bobot, 'rb')
        w.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)
        b.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)
        w2.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)
        b2.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)
        w3.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)
        b3.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)
        w4.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)
        b4.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)
        w_o.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)
        b_o.set_value(cPickle.load(save_file, encoding='latin1'), borrow=True)

    def model(self, X, w, b, w2, b2, w3, b3, w4, b4, w_o, b_o):
        l1 = self.rectify(conv2d(X, w) + b.dimshuffle('x', 0, 'x', 'x'))
        l1 = pool_2d(l1, (2, 2), ignore_border=True)
        l2 = self.rectify(conv2d(l1, w2) + b2.dimshuffle('x', 0, 'x', 'x'))
        l2 = pool_2d(l2, (2, 2), ignore_border=True)
        l3 = self.rectify(conv2d(l2, w3) + b3.dimshuffle('x', 0, 'x', 'x'))
        l3 = pool_2d(l3, (2, 2), ignore_border=True)
        l3 = T.flatten(l3, outdim=2)
        l4 = T.tanh(T.dot(l3, w4) + b4)
        pyx = self.softmax(T.dot(l4, w_o) + b_o)
        return pyx

    def run(self, imagePath):
        X = T.ftensor4()

        w = self.init_conv_weights((32, 1, 3, 3))
        b = self.init_bias((32,))
        w2 = self.init_conv_weights((64, 32, 2, 2))
        b2 = self.init_bias((64,))
        w3 = self.init_conv_weights((128, 64, 3, 3))
        b3 = self.init_bias((128,))
        w4 = self.init_xavier_weights((512, 1000))
        b4 = self.init_bias((1000,))
        w_o = self.init_xavier_weights((1000, 20))
        b_o = self.init_bias((20,))

        py_x = self.model(X, w, b, w2, b2, w3, b3, w4, b4, w_o, b_o)
        y_x = T.argmax(py_x, axis=1)

        predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)

        self.load_weights(w, b, w2, b2, w3, b3, w4, b4, w_o, b_o)

        img = Image.open(imagePath).convert('L')
        img = img.resize((28, 28), Image.ANTIALIAS)
        img = np.array(img)
        img = 255 - img
        img = img / 255.

        img = img.reshape(1, 1, 28, 28)
        hasil = (predict(img)+1)

        return hasil

max_stars_repo_name: nmfzone/django-upload

max_issues_repo_name: nmfzone/django-upload

max_forks_repo_name: nmfzone/django-upload

reason: import numpy,import theano,from theano


text:
import pytest

import numpy as np
from gym_pathfinding.games.gridworld import path_exists

def test_path_exists():
    grid = np.array([
        [1,1,1,1,1,1,1],
        [1,0,0,0,0,0,1],
        [1,1,1,1,1,1,1],
        [1,0,0,0,0,0,1],
        [1,1,1,1,1,1,1],
    ])

    assert path_exists(grid, (1, 1), (1, 5))
    assert not path_exists(grid, (1, 1), (3, 5))

from gym_pathfinding.games.gridworld import add_rectangle

def test_add_rectangle():
    grid = np.array([
        [0,0,0,0,0,0,0],
        [0,0,0,0,0,0,0],
        [0,0,0,0,0,0,0],
        [0,0,0,0,0,0,0],
        [0,0,0,0,0,0,0],
    ])

    expected = np.array([
        [0,0,0,0,0,0,0],
        [0,1,1,1,0,0,0],
        [0,1,1,1,0,0,0],
        [0,0,0,1,1,0,0],
        [0,0,0,1,1,0,0],
    ])

    add_rectangle(grid, (1, 1, 3, 2))
    add_rectangle(grid, (3, 3, 2, 2))

    assert np.all(grid == expected)


text:
# XXX: Put a license here

"""Class for a single septin simulation"""

import gc
import os
import re
import sys

import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl

sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'lib'))
from septin_seed import SeptinSeed
from sim_base import SimulationBase

from seed_graph_funcs import *
from sim_graph_funcs import *

class SeptinSim(SimulationBase):
    def __init__(self, path, opts, seedType=SeptinSeed):
        print(f"SeptinSimulation::__init__")
        SimulationBase.__init__(self, path, opts, seedType=seedType)

        # Define functions to graph we want on a per-seed basis
        self.graph_perseed_functions = [ graph_seed_temperature,
                                         graph_seed_pressure,
                                         graph_seed_area ]

        # Now for anything that can be averaged
        self.graph_distribution_functions = [ (graph_sim_membranemodes, 1, 1, (6, 4)) ]

        print(f"SeptinSimulation::__init__ return")

    def Analyze(self):
        r""" Analyze all of the underlying seeds
        """
        for sd in self.seeds: sd.Analyze()
        print(f"- Sim {self.name} analyzed -")

    def GraphSimulation(self):
        r""" Graph simulation parameters and variables
        """
        # A couple of quick checks to make sure data directories are setup correctly
        if not self.opts.datadir: self.opts.datadir = create_datadir(self.opts.workdir)
        if self.opts.workdir != self.sim_path:
            self.sim_datadir = create_datadir(self.opts.datadir, datadir_name = "{}_data".format(self.name))
        else:
            self.sim_datadir = self.opts.datadir

        self.GraphDynamicData()
        self.GraphDistributions()

    def GraphDynamicData(self):
        r""" Graph dynamic data (similar to seed methods) for simulation
        """
        fig, axarr = plt.subplots(3, 2, figsize=(15,10))
        colors = mpl.cm.rainbow(np.linspace(0,1,len(self.seeds)))

        # Zip together the list like we had before to be clever
        # This first loop zips together the functions we want to graph and the axarr we generated
        for graph, axr in zip(self.graph_perseed_functions, axarr):
            # The second loop creates different colors for each seed
            for sd, col in zip(self.seeds, colors):
                yarr = graph(sd, axr[0], color=col, xlabel = False) # Actual magic of the graphing call

        #axarr[1,0].legend(loc = 'center left', bbox_to_anchor=(2.2,-0.19))
        fig.tight_layout()
        plt.savefig("{}_{}.pdf".format(os.path.join(self.sim_datadir, 'dynamicdata'), self.name), dpi=fig.dpi)

        # Clean up
        fig.clf()
        plt.close()
        gc.collect()
        mpl.rcdefaults()

    def GraphDistributions(self):
        r""" Graph distribution data for simulation
        """
        # This is a 'clever' loop over the correct distributions as defined earlier

        for gf, rn, cn, fs in self.graph_distribution_functions:
            fig, axarr = plt.subplots(rn, cn, figsize = fs)
            gf(self, axarr)
            plt.figure(fig.number)
            fig.tight_layout()
            pruned_name = self.name[0:127]
            fig.savefig("{}_{}.pdf".format(os.path.join(self.sim_datadir, gf.__name__), pruned_name), dpi=fig.dpi)

            # Clean up
            fig.clf()
            plt.close()
            gc.collect()

        mpl.rcdefaults()

max_stars_repo_name: cedelmaier/dragonfruit

max_issues_repo_name: cedelmaier/dragonfruit

max_forks_repo_name: cedelmaier/dragonfruit


text:
import numpy as np

class Grid():

	def __init__(self,n):
		self.grid = np.zeros((n,n))


text:
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
#   Licensed under the Apache License, Version 2.0 (the "License").
#   You may not use this file except in compliance with the License.
#   A copy of the License is located at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   or in the "license" file accompanying this file. This file is distributed
#   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
#   express or implied. See the License for the specific language governing
#   permissions and limitations under the License.
# ==============================================================================

import io
import json
import mxfusion as mf
import mxnet as mx
import numpy as np
import zipfile
from ..common.exceptions import SerializationError
from ..common.config import get_default_device

__GRAPH_JSON_VERSION__ = '1.0'
SERIALIZATION_VERSION = '2.0'
DEFAULT_ZIP = 'inference.zip'
FILENAMES = {
    'graphs' : 'graphs.json',
    'mxnet_params' : 'mxnet_parameters.npz',
    'mxnet_constants' : 'mxnet_constants.npz',
    'variable_constants' : 'variable_constants.json',
    'configuration' : 'configuration.json',
    'version_file' : 'version.json'
}
ENCODINGS = {
    'json' : 'json',
    'numpy' : 'numpy'
}

class ModelComponentEncoder(json.JSONEncoder):

    def default(self, obj):
        """
        Serializes a ModelComponent object. Note: does not serialize the successor attribute as it isn't  necessary for serialization.
        """
        if isinstance(obj, mf.components.ModelComponent):
            object_dict = obj.as_json()
            object_dict["version"] = __GRAPH_JSON_VERSION__
            object_dict["type"] = obj.__class__.__name__
            return object_dict
        return super(ModelComponentEncoder, self).default(obj)

class ModelComponentDecoder(json.JSONDecoder):
    def __init__(self, *args, **kwargs):
        json.JSONDecoder.__init__(
            self, object_hook=self.object_hook, *args, **kwargs)

    def object_hook(self, obj):
        """
        Reloads a ModelComponent object. Note: does not reload the successor attribute as it isn't necessary for serialization.
        """
        if not isinstance(obj, type({})) or 'uuid' not in obj:
            return obj
        if obj['version'] != __GRAPH_JSON_VERSION__:
            raise SerializationError('The format of the stored model component '+str(obj['name'])+' is from an old version '+str(obj['version'])+'. The current version is '+__GRAPH_JSON_VERSION__+'. Backward compatibility is not supported yet.')
        if 'graphs' in obj:
            v = mf.modules.Module(None, None, None, None)
            v.load_module(obj)
        else:
            v = mf.components.ModelComponent()
            v.inherited_name = obj['inherited_name'] if 'inherited_name' in obj else None
        v.name = obj['name']
        v._uuid = obj['uuid']
        v.attributes = obj['attributes']
        v.type = obj['type']
        return v

def load_json_file(target_file, decoder=None):
    with open(target_file) as f:
        return json.load(f, cls=decoder)

def load_json_from_zip(zip_filename, target_file, decoder=None):
    """
    Utility function that loads a json file from inside a zip file without unzipping the zip file
    and returns the loaded json as a dictionary.
    :param encoder: optional. a JSONDecoder class to pass to the json.load function for loading back in the dict.
    """
    with zipfile.ZipFile(zip_filename, 'r') as zip_file:
        json_file = zip_file.open(target_file)
        # json.load only takes str in 3.4/3.5 so we read, decode to UTF-8, and convert to a StringIO
        loaded = json.load(io.StringIO(json_file.read().decode()), cls=decoder)
    return loaded

def make_numpy(obj):
    """
    Utility function that takes a dictionary of numpy or MXNet arrays and
    returns a dictionary of numpy arrays. Used to standardize serialization.
    """
    ERR_MSG = "This function shouldn't be called on anything except " + \
             " dictionaries of numpy and MXNet arrays."
    if not isinstance(obj, type({})):
        raise SerializationError(ERR_MSG)

    np_obj = {}
    for k,v in obj.items():
        if isinstance(v, np.ndarray):
            np_obj[k] = v
        elif isinstance(v, mx.ndarray.ndarray.NDArray):
            np_obj[k] = v.asnumpy()
        else:
            raise SerializationError(ERR_MSG)
    return np_obj

def load_parameters(npz_filename, zip_file, context=None):
    """
    Helper function to load the parameters from a npz file directly into a dictionary as mxnet arrays.
    """
    context = context if context is not None else get_default_device()
    params_file = zip_file.read(npz_filename)
    try:
        loaded = np.load(io.BytesIO(params_file))
    except OSError as e:
        """
        Numpy load doesn't handle reloading an empty .npz directory after savez so just continue with an empty
        dict if it throws an OSError here when loading back.
        See https://github.com/chainer/chainer/issues/4542
        """
        return {}
    parameters = {}
    for k,v in loaded.items():
        parameters[k] = mx.nd.array(v, dtype=v.dtype, ctx=context)
    return parameters

max_stars_repo_name: JeremiasKnoblauch/MXFusion

max_issues_repo_name: JeremiasKnoblauch/MXFusion

max_forks_repo_name: JeremiasKnoblauch/MXFusion


text:
# -*- coding: utf-8 -*-
"""
Created on Sun Apr 21 15:55:52 2019

@author: Colin
"""

import numpy as np
import sys
from os import makedirs, listdir
from os.path import isdir, join, exists
sys.path.insert(0, '/home/colin/Documents/GitHub/autodisc/vabacon/')
sys.path.insert(0, '/home/colin/Documents/GitHub/autodisc/AutomatedDiscovery/DADS2018/')
import approximate_kinds
import scipy.io
import pandas

def load_data(directory):
    # loads the data into a list
    set_names = listdir(directory)
    sets = []

    for set_name in set_names:
        set_path = join(directory, set_name)
        mat_data = scipy.io.loadmat(set_path)
        data_name = list(mat_data.keys())[0]
        data = np.asarray(mat_data[data_name])
        sets.append(data.T)
    min_size = np.min([np.shape(x)[-1] for x in sets])
    for i, data in enumerate(sets):
        cut_data = np.take(data, range(min_size), -1)
        sets[i] = cut_data
    return sets

def test_data():
    data_list = []
    # make somewhere between 3 and 10 sets of data
    num_sets = np.random.randint(3, 10)
    for i in range(num_sets):
        # with a variable number of samples in each dataset
        num_samples = np.random.randint(100, 2000)
        data = np.random.random((3, num_samples))
        data_list.append(data)
    return data_list

def choose_pairs(sysA_dir, sysB_dir, sysC_dir):
    #TODO: make this generalized for any number of systems
    dataA = load_data(sysA_dir)
    dataB = load_data(sysB_dir)
    dataC = load_data(sysC_dir)

    ica1 = np.take(dataA[0], [0], -1).flatten()
    ica2 = np.take(dataA[1], [0], -1).flatten()
    icb1 = np.take(dataB[0], [0], -1).flatten()
    icb2 = np.take(dataB[1], [0], -1).flatten()
    icc1 = np.take(dataC[0], [0], -1).flatten()
    icc2 = np.take(dataC[1], [0], -1).flatten()
#    ica1 = dataA[0][0]
#    ica2 = dataA[1][0]
#    icb1 = dataB[0][0]
#    icb2 = dataB[1][0]
#    icc1 = dataC[0][0]
#    icc2 = dataC[1][0]
#
    a1_b1_dist = np.linalg.norm(ica1-icb1)
    a1_b2_dist = np.linalg.norm(ica1-icb2)
    a1_b_min_index = np.argmin([a1_b1_dist, a1_b2_dist])

    a1_c1_dist = np.linalg.norm(ica1-icc1)
    a1_c2_dist = np.linalg.norm(ica1-icc2)
    a1_c_min_index = np.argmin([a1_c1_dist, a1_c2_dist])

    data_dict = {"untransA":dataA[0], "transA":dataA[1],
                 "untransB":dataB[a1_b_min_index],
                 "transB":dataB[not a1_b_min_index],
                 "untransC":dataC[a1_c_min_index],
                 "transC":dataC[not a1_c_min_index]}
    return data_dict

def choose_data(sysA_dir, sysB_dir):
    # load dataA
    dataA = test_data()
#    dataA = load_data(sysA_dir)
    # load dataB
    dataB = test_data()
#    dataB = load_data(sysB_dir)
    # choose data from sysA using choose_trans_untrans
    #TODO: how to deal with data of different sample size?
    #   currently do this by cutting each dataset to smallest size
    untransA, transA = approximate_kinds.choose_untrans_trans([dataA], 1)
    # then choose data from sysB that is closest to untrans sysA
    # first, find the initial condition of the untransformed data
    # i.e. take the first entries on the last axis
    ic_untrans = np.take(untransA, [0], -1).flatten()
    # next, find the initial condition of the transformed data
    ic_trans = np.take(transA, [0], -1).flatten()

    untrans_dists = []
    trans_dists = []

    untransB_index = None
    transB_index = None
    for i, bset in enumerate(dataB):
        cur_ic = np.take(bset, [0], -1).flatten()
        untrans_dist = np.linalg.norm(ic_untrans-cur_ic)
        untrans_dists.append(untrans_dist)
        trans_dist = np.linalg.norm(ic_trans-cur_ic)
        trans_dists.append(trans_dist)

    untransB_index = np.argmin(untrans_dists)
    transB_index = np.argmin(trans_dists)
    # in the case that the two are the same, choose the second best for transB
    if untransB_index == transB_index:
        trans_dists[transB_index] = np.inf
        transB_index = np.argmin(trans_dists)

    untransB = dataB[untransB_index]
    transB = dataB[transB_index]

    if len(np.shape(untransA)) > 2:
        untransA = np.squeeze(untransA)
    if len(np.shape(transA)) > 2:
        transA = np.squeeze(transA)
    return {"untransA":untransA, "transA":transA, "untransB":untransB,
            "transB":transB}

if __name__ == '__main__':
    # systemsA systemsB output

    # Get systemsA folder
    sysA_dir = sys.argv[1]
    print(sysA_dir)
    assert isdir(sysA_dir), "Can't find Systems A directory."
    if not sysA_dir[-1] == '/':
        sysA_dir = sysA_dir + '/'
    sysA_names = listdir(sysA_dir)
    assert len(sysA_names) > 0, "Systems A directory is empty."

    # Get systemsB folder
    sysB_dir = sys.argv[2]
    assert isdir(sysB_dir), "Can't find Systems B directory."
    if not sysB_dir[-1] == '/':
        sysB_dir = sysB_dir + '/'
    sysB_names = listdir(sysB_dir)
    assert len(sysB_names) > 0, "Systems B directory is empty."

    # Get destination folder
    out_dir = sys.argv[3]
    if not exists(out_dir):
        makedirs(out_dir)

    sysC_dir = None
    if len(sys.argv) > 4:
        sysC_dir = sys.argv[3]
        assert isdir(sysC_dir), "Can't find Systems B directory."
        if not sysC_dir[-1] == '/':
            sysC_dir = sysC_dir + '/'
        sysC_names = listdir(sysC_dir)
        assert len(sysC_names) > 0, "Systems C directory is empty."
        out_dir = sys.argv[4]
        if not exists(out_dir):
            makedirs(out_dir)

    #selected = choose_data(sysA_dir, sysB_dir)
    selected = choose_pairs(sysA_dir, sysB_dir, sysC_dir)
    # each selected is a dataset of dim x samples
    # put each dataset in a csv
    for name, data in selected.items():
        out_path = join(out_dir, name + ".csv")
        df = pandas.DataFrame(data)
        df.to_csv(out_path, index=False, header=False)

max_stars_repo_name: jantzen/eugene

max_issues_repo_name: jantzen/eugene

max_forks_repo_name: jantzen/eugene

reason: import numpy,import scipy


text:
import os
import json
import time
import requests
import datetime
import numpy as np
from PIL import Image
from io import BytesIO
import onnxruntime as rt
from torchvision import transforms

session, transform, classes, input_name = None, None, None, None

def init():
    global session, transform, classes, input_name

    model_path = '../outputs'

    meta_file = os.path.join(model_path, 'model.meta')
    with open(meta_file, 'r') as f:
        model_meta = json.load(f)
    classes = model_meta['classes']
    session = rt.InferenceSession(os.path.join(model_path, 'model.onnx'))
    input_name = session.get_inputs()[0].name
    transform = transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

def run(raw_data):
    prev_time = time.time()

    post = json.loads(raw_data)
    image_url = post['image']
    response = requests.get(image_url)
    img = Image.open(BytesIO(response.content))
    v = transform(img)
    pred_onnx = session.run(None, {input_name: v.unsqueeze(0).numpy()})[0][0]

    current_time = time.time()
    inference_time = datetime.timedelta(seconds=current_time - prev_time)

    predictions = {}
    for i in range(len(classes)):
        predictions[classes[i]] = str(pred_onnx[i])

    payload = {
        'time': str(inference_time.total_seconds()),
        'prediction': classes[int(np.argmax(pred_onnx))],
        'scores': predictions
    }

    #print('Input ({}), Prediction ({})'.format(post['image'], payload))

    return payload

if __name__ == '__main__':
    init()
    rock = 'http://localhost:8000/rock.png'

    print('---------------------Inference with rock:')
    print(json.dumps(run(json.dumps({'image': rock})), indent=4))

    paper = 'http://localhost:8000/paper.png'

    print('---------------------Inference with paper:')
    print(json.dumps(run(json.dumps({'image': paper})), indent=4))

    paper = 'http://localhost:8000/scissors.png'

    print('---------------------Inference with scissors:')
    print(json.dumps(run(json.dumps({'image': paper})), indent=4))

max_stars_repo_name: OldDirtyRob/Roshambo

max_issues_repo_name: OldDirtyRob/Roshambo

max_forks_repo_name: OldDirtyRob/Roshambo


text:
from flask import Flask, render_template, Response, redirect, request, session
import cv2
import time
import requests
from flask_socketio import SocketIO, emit
import os
import threading
import tts_stt
import base64

import numpy as np
import glob
import argparse
import dlib
import os
from utils.aux_functions import *
import math
from RetinaFace.retinaface_cov import RetinaFaceCoV
from PIL import Image, ImageFile
import tensorflow as tf
from parameters import *
import keras

from numpy import load
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import LabelBinarizer
from sklearn.preprocessing import Normalizer
from sklearn.svm import SVC
import pickle

from numpy import dot
from numpy.linalg import norm

import loadmodel

import re
import sys
import time

import subprocess
import inspect

import requests
from bs4 import BeautifulSoup
import pandas as pd
import datetime
import json

import smtplib
from email.mime.text import MIMEText

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel

## function define ##
def send_mail(text):
    smtp = smtplib.SMTP('smtp.gmail.com', 587)

    smtp.ehlo()
    smtp.starttls()

    my_email = # Fill your Email
    my_email_pw = # Fill your PW
    your_email = text['email']

    smtp.login(my_email, my_email_pw)

    results = ""
    for i in range(len(text['title'])):
        results += '\n'
        results += '제목 : ' + text['title'][i] + '\n'
        results += '-' * (3 * len(text['title'][i]) + 5)  + '\n'
        results += '내용 : ' + text['content'][i] + '\n'
        results += '\n'

    results += '\n'
    message = MIMEText(results)
    message['Subject'] = str(datetime.datetime.now()) + '_기사 모음'
    message['From'] = my_email
    message['To'] = your_email

    smtp.sendmail(my_email, your_email, message.as_string())

    smtp.quit()

def img_to_encoding2(image, model, path=True):
    img = image
    img = tf.keras.applications.inception_resnet_v2.preprocess_input(img)
    x_train = np.array([img])
    embedding = model.predict_on_batch(x_train)
    return embedding

def cos_sim(A, B):
    return dot(A,B) / (norm(A) * norm(B))

## function define ##

print('Ready to Starting POSVATOR SERVER......')
app = Flask(__name__)
app.config['SECRET_KEY'] = 'poscointernational'
socketio = SocketIO(app)
thread = None
detector = None
model = None

known_face_encodings = None
known_face_names = None

def gen():
    global detector, model, known_face_encodings, known_face_names
    """Video streaming generator function."""

    detector, model = loadmodel.load_models()

    ## extract embeddings from databases ##
    thresh = 0.8
    mask_thresh = 0.2

    known_face_encodings = dict()

    users = dict()
    files = glob.glob('./users/*.jpg')

    total_news = dict()
    news = dict()

    tfidf = TfidfVectorizer()

    korname2engname = dict()

    for file in files:
        sep = file.split('/')[2]
        r_sep = sep.split('_')
        if r_sep[1] == 'augmentation':
            pass
        else:
            total_news[r_sep[3]] = { 'title' : [] , 'contents' : [], 'summary' : [] }
            known_face_encodings[r_sep[0]] = []
            korname2engname[r_sep[3]] = r_sep[0]

    print(known_face_encodings)
    print(korname2engname)

    crawled_news = pd.read_csv('reco2.csv')

    for i in range(len(crawled_news)):
        user = crawled_news.iloc[i]['user']
        title = crawled_news.iloc[i]['title']
        content = crawled_news.iloc[i]['contents']
        summary = crawled_news.iloc[i]['summary']
        total_news[user]['title'].append(title)
        total_news[user]['contents'].append(content)
        total_news[user]['summary'].append(summary)

    for file in files:
        scales = [640, 1080]
        img = cv2.imread(file)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

        img = cv2.resize(img, (640, 480), interpolation=cv2.INTER_CUBIC)

        im_shape = img.shape
        target_size = scales[0]
        max_size = scales[1]
        im_size_min = np.min(im_shape[0:2])
        im_size_max = np.max(im_shape[0:2])
        im_scale = float(target_size) / float(im_size_min)

        if np.round(im_scale * im_size_max) > max_size:
            im_scale = float(max_size) / float(im_size_max)

        scales = [im_scale]
        flip = False

        faces, landmarks = detector.detect(img, thresh, scales=scales, do_flip=flip)

        if len(faces) != 0:
            box = faces[0][0:4].astype(np.int)
            cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), (0,0,255), 2)
            croped_img = img[box[1]:box[3], box[0]:box[2]]
            croped_img = cv2.resize(croped_img, (160, 160), cv2.INTER_CUBIC)

            lab = cv2.cvtColor(croped_img, cv2.COLOR_BGR2LAB)
            lab_planes = cv2.split(lab)
            clahe = cv2.createCLAHE(clipLimit=2.0,tileGridSize=(8 ,8))
            lab_planes[0] = clahe.apply(lab_planes[0])
            lab = cv2.merge(lab_planes)
            croped_img = cv2.cvtColor(lab, cv2.COLOR_LAB2BGR)

            emb = img_to_encoding2(croped_img, model)[0]
            emb = emb.reshape(-1)

            infos = file.split('/')[2]
            info = infos.split('_')

            if info[1] == 'augmentation':
                known_face_encodings[korname2engname[info[0]]].append(emb)
                continue
            else:
                known_face_encodings[info[0]].append(emb)

            # floor, company, count, real_name, email
            users[info[0]] = [info[1], info[2], 0, info[3], info[4][:-4]]

    cctv_url = 'http://posproject201013.iptime.org:8787/video_feed'
    video = cv2.VideoCapture(cctv_url)
    mask_count = 0
    mask_count_thresh = 15

    while True:
        try:
            scales = [640, 1080]
            _, img = video.read()

            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

            im_shape = img.shape

            target_size = scales[0]
            max_size = scales[1]
            im_size_min = np.min(im_shape[0:2])
            im_size_max = np.max(im_shape[0:2])
            #im_scale = 1.0
            #if im_size_min>target_size or im_size_max>max_size:
            im_scale = float(target_size) / float(im_size_min)
            # prevent bigger axis from being more than max_size:
            if np.round(im_scale * im_size_max) > max_size:
                im_scale = float(max_size) / float(im_size_max)

            scales = [im_scale]
            flip = False

            faces, landmarks = detector.detect(img, thresh, scales=scales, do_flip=flip)

            if faces is None:
                continue

            if len(faces) != 0:
                for i in range(faces.shape[0]):
                    face = faces[i]
                    box = face[0:4].astype(np.int)
                    mask = face[5]

                    name = "Unknown"

                    if mask>=mask_thresh:
                        name = 'Mask Detected. Please take off your mask.'
                        ###      R G B
                        color = (0,0,255)
                    else:
                        color = (0,255,0)
                        croped_img = img[box[1]:box[3], box[0]:box[2]]
                        croped_img = cv2.resize(croped_img, (160, 160), cv2.INTER_CUBIC)

                        lab = cv2.cvtColor(croped_img, cv2.COLOR_BGR2LAB)
                        lab_planes = cv2.split(lab)
                        clahe = cv2.createCLAHE(clipLimit=2.0,tileGridSize=(8 ,8))
                        lab_planes[0] = clahe.apply(lab_planes[0])
                        lab = cv2.merge(lab_planes)
                        croped_img = cv2.cvtColor(lab, cv2.COLOR_LAB2BGR)
                        emb = img_to_encoding2(croped_img, model)[0]
                        emb = emb.reshape(-1)
                        sim = 0.5
                        idx = 0

                        for names in known_face_encodings.keys():
                            for user_emb in known_face_encodings[names]:
                                cur_sim = cos_sim(user_emb, emb)
                                if sim < cur_sim:
                                    sim = cur_sim
                                    name = names

                        if sim < 0.80:
                            name = 'Unknown'

                    cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), color, 2)
                    font = cv2.FONT_HERSHEY_DUPLEX
                    cv2.putText(img, name, (box[0] + 6, box[1] - 6), font, 1.0, (255, 255, 255), 1)

                    if name != 'Mask Detected. Please take off your mask.' and name != 'Unknown':
                        print(name, 'detected !!!')
                        users[name][2] += 1
                        if users[name][2] >= 15:
                            print('데이터전송')
                            socketio.emit('message', {'name': users[name][3], 'floor' : users[name][0], 'company': users[name][1], 'title':total_news[users[name][3]]['title'], 'content':total_news[users[name][3]]['summary'],'send':total_news[users[name][3]]['contents'], 'email' : users[name][4]})
                            users[name][2] = 0
                    else:
                        if name == 'Mask Detected. Please take off your mask.':
                            mask_count += 1
                            if mask_count >= mask_count_thresh:
                                mask_count_thresh += mask_count * 2
                                mask_count = 0
                                print('음성 재생 필요!')
                                requests.get(url = 'http://posproject201013.iptime.org:8787/take_mask')

                        print('no one detected.')
            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
            ret, jpeg = cv2.imencode('.jpg', img)
            # print("after get_frame")
            if img is not None:
                yield (b'--frame\r\n'
                       b'Content-Type: image/jpeg\r\n\r\n' + jpeg.tobytes() + b'\r\n')
        except Exception as e:
            print('error occured', e)
        time.sleep(0.01)

@socketio.on('email')
def handle_my_custom_event(json):
    print('received json: ' + str(json))
    send_mail(json)

@app.route('/video_feed')
def video_feed():
    """Video streaming route. Put this in the src attribute of an img tag."""
    return Response(gen(), mimetype='multipart/x-mixed-replace; boundary=frame')

@app.route("/")
def index():
    return render_template("index.html", data=None  )

@socketio.on('connect')
def connect():
    print('client connected!')

if __name__ == '__main__':
    socketio.run(app, host='0.0.0.0', port=8999, debug=True)

reason: import numpy,from numpy