text:
import tensorflow as tf
# from keras.utils import multi_gpu_model
from keras.utils import to_categorical
from keras import backend as K
import csv
import sys
import time
import timeit
import numpy as np
import math
import matplotlib.pyplot as plt
import seaborn as sns
# from sklearn.utils import class_weight
import utils
from rgb_model import rgb_create_model, compile_model
from rgb_data import load_split, get_AVA_set, get_AVA_labels
from itertools import islice

def labels_to_numpy(labels):
    nplabels = []
    for key in labels:
        nplabels.append(labels[key]['pose'])
        for a in labels[key]['human-object']:
            nplabels.append(a)
        for a in labels[key]['human-human']:
            nplabels.append(a)
    return np.array(nplabels)

def main():

    # root_dir = '../../../AVA2.1/' # root_dir for the files
    root_dir = '../../data/AVA/files/'

    # Erase previous models from GPU memory
    K.clear_session()

    # Load list of action classes and separate them
    classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')

    # Parameters for training
    params = {'dim': (224, 224), 'batch_size': 32,
              'n_classes': len(classes['label_id']), 'n_channels': 3,
              'shuffle': False, 'nb_epochs': 200, 'model': 'resnet50', 'email': True,
              'freeze_all': True, 'conv_fusion': False, 'train_chunk_size': 2**12,
              'validation_chunk_size': 2**12}
    soft_sigmoid = True
    minValLoss = 9999990.0

    # Get ID's and labels from the actual dataset
    partition = {}
    partition['train'] = get_AVA_set(classes=classes, filename=root_dir + "AVA_Train_Custom_Corrected.csv", soft_sigmoid=soft_sigmoid)  # IDs for training
    partition['validation'] = get_AVA_set(classes=classes, filename=root_dir + "AVA_Val_Custom_Corrected.csv", soft_sigmoid=soft_sigmoid)  # IDs for validation

    # Labels
    labels_train = get_AVA_labels(classes, partition, "train", filename=root_dir + "AVA_Train_Custom_Corrected.csv", soft_sigmoid=soft_sigmoid)
    labels_val = get_AVA_labels(classes, partition, "validation", filename=root_dir + "AVA_Val_Custom_Corrected.csv", soft_sigmoid=soft_sigmoid)

    # http://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_class_weight.html
    # Logistic Regression in Rare Events Data, Gary King, Langche Zeng
    # Keras needs a dict though
    # From documentation: class_weight: Optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function (during training only).
    # This can be useful to tell the model to "pay more attention" to samples from an under-represented class.
    y = labels_to_numpy(labels_train)

    penalizing_method = 'balanced'
    mu = 0.7
    # penalizing_method = 'weighted_log'
    class_weights = np.zeros(30)
    for i in y:
        class_weights[i] += 1
    max_class = max(class_weights)
    for i in range(len(class_weights)):
        if class_weights[i] != 0.0:
            if penalizing_method == 'balanced':
                print(str(i) + " " + str(class_weights[i]) + " " + str(len(y) / (class_weights[i])))
                #class_weights[i] = len(y) / (class_weights[i])
                class_weights[i] = max_class / (class_weights[i])
            elif penalizing_method == 'weighted_log':
                print(str(i) + " " + str(class_weights[i]) + " " + str(math.log(mu * len(y) / (class_weights[i]))))
                class_weights[i] = math.log(mu * len(y) / (class_weights[i]))
        else:
            print(str(i) + " " + str(class_weights[i]) + " inf ")
            class_weights[i] = 0.0
    g = sns.barplot(x=[str(i) for i in range(len(class_weights))], y=class_weights)
    plt.xticks(rotation=-90)
    plt.title("Class weights " + penalizing_method)
    plt.grid(True)
    plt.show()

    class_dictionary = {}
    print(len(class_weights))
    for i in range(len(class_weights)):
        class_dictionary[i] = class_weights[i]
    print(class_dictionary)

    it = iter(class_weights)
    seclist = [utils.POSE_CLASSES, utils.OBJ_HUMAN_CLASSES, utils.HUMAN_HUMAN_CLASSES]
    class_lists = [list(islice(it, 0, i)) for i in seclist]
    print(class_lists)

    # Create + compile model, load saved weights if they exist
    saved_weights = None
    # saved_weights = "../models/rgbextra_gauss_resnet50_1807250030.hdf5"
    model, keras_layer_names = rgb_create_model(classes=classes['label_id'], soft_sigmoid=soft_sigmoid, model_name=params['model'], freeze_all=params['freeze_all'], conv_fusion=params['conv_fusion'])
    model = compile_model(model, soft_sigmoid=soft_sigmoid)

    # TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
    initialization = True  # Set to True to use initialization
    kinetics_weights = ""
    ucf_weights = "../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"

    if saved_weights is not None:
        model.load_weights(saved_weights)
    else:
        if initialization is True:
            if ucf_weights is None:
                print("Loading MConvNet weights: ")
                if params['model'] == "resnet50":
                    ucf_weights = utils.loadmat("../models/ucf_matconvnet/ucf101-img-resnet-50-split1.mat")
                    utils.convert_resnet(model, ucf_weights)
                    model.save("../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5")
            if kinetics_weights is None:
                if params['model'] == "inceptionv3":
                    print("Loading Keras weights: ")
                    keras_weights = ["../models/kinetics_keras/tsn_rgb_params_names.pkl", "../models/kinetics_keras/tsn_rgb_params.pkl"]
                    utils.convert_inceptionv3(model, keras_weights, keras_layer_names)
                    model.save("../models/kinetics_keras/keras-kinetics-rgb-inceptionv3.hdf5")
    # Try to train on more than 1 GPU if    possible
    # try:
    #    print("Trying MULTI-GPU")
    #    model = multi_gpu_model(model)

    print("Training set size: " + str(len(partition['train'])))

    # Make spltis
    train_splits = utils.make_chunks(original_list=partition['train'], size=len(
        partition['train']), chunk_size=params['train_chunk_size'])
    val_splits = utils.make_chunks(original_list=partition['validation'], size=len(
        partition['validation']), chunk_size=params['validation_chunk_size'])

    time_str = time.strftime("%y%m%d%H%M", time.localtime())

    # TODO Don't forget to change your names :)
    filter_type = "gauss"
    bestModelPath = "../models/rgb_weightsfinal_" + filter_type + "_" + params['model'] + "_" + time_str + ".hdf5"
    traincsvPath = "../loss_acc_plots/rgb_weightsfinal_train_" + filter_type + "_plot_" + params['model'] + "_" + time_str + ".csv"
    valcsvPath = "../loss_acc_plots/rgb_weightsfinal_val_" + filter_type + "_plot_" + params['model'] + "_" + time_str + ".csv"

    for epoch in range(params['nb_epochs']):
        epoch_chunks_count = 0
        for trainIDS in train_splits:
            # Load and train
            start_time = timeit.default_timer()
            # -----------------------------------------------------------
            x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
            x_train, y_train_pose, y_train_object, y_train_human = load_split(trainIDS, labels_train, params['dim'], params['n_channels'], "train", filter_type, soft_sigmoid=soft_sigmoid)

            y_t = []
            y_t.append(to_categorical(y_train_pose, num_classes=utils.POSE_CLASSES))
            y_t.append(utils.to_binary_vector(y_train_object, size=utils.OBJ_HUMAN_CLASSES, labeltype='object-human'))
            y_t.append(utils.to_binary_vector(y_train_human, size=utils.HUMAN_HUMAN_CLASSES, labeltype='human-human'))

            history = model.fit(x_train, y_t, class_weight=class_lists, batch_size=params['batch_size'], epochs=1, verbose=0)
            utils.learning_rate_schedule(model, epoch, params['nb_epochs'])

            # ------------------------------------------------------------
            elapsed = timeit.default_timer() - start_time

            print("Epoch " + str(epoch) + " chunk " + str(epoch_chunks_count) + " (" + str(elapsed) + ") acc[pose,obj,human] = [" + str(history.history['pred_pose_categorical_accuracy']) + "," +
                  str(history.history['pred_obj_human_categorical_accuracy']) + "," + str(history.history['pred_human_human_categorical_accuracy']) + "] loss: " + str(history.history['loss']))
            with open(traincsvPath, 'a') as f:
                writer = csv.writer(f)
                avg_acc = (history.history['pred_pose_categorical_accuracy'][0] + history.history['pred_obj_human_categorical_accuracy']
                           [0] + history.history['pred_human_human_categorical_accuracy'][0]) / 3
                writer.writerow([str(avg_acc), history.history['pred_pose_categorical_accuracy'], history.history['pred_obj_human_categorical_accuracy'],
                                 history.history['pred_human_human_categorical_accuracy'], history.history['loss']])
            epoch_chunks_count += 1
        # Load val_data
        print("Validating data: ")
        # global_loss, pose_loss, object_loss, human_loss, pose_acc, object_acc, human_acc
        loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
        for valIDS in val_splits:
            x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
            x_val, y_val_pose, y_val_object, y_val_human = load_split(valIDS, labels_val, params[
                'dim'], params['n_channels'], "val", filter_type, soft_sigmoid=soft_sigmoid)
            y_v = []
            y_v.append(to_categorical(
                y_val_pose, num_classes=utils.POSE_CLASSES))
            y_v.append(utils.to_binary_vector(
                y_val_object, size=utils.OBJ_HUMAN_CLASSES, labeltype='object-human'))
            y_v.append(utils.to_binary_vector(
                y_val_human, size=utils.HUMAN_HUMAN_CLASSES, labeltype='human-human'))

            # NOTE Can't have weights on validation
            vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
                x_val, y_v, batch_size=params['batch_size'])
            loss_acc_list[0] += vglobal_loss
            loss_acc_list[1] += vpose_loss
            loss_acc_list[2] += vobject_loss
            loss_acc_list[3] += vhuman_loss
            loss_acc_list[4] += vpose_acc
            loss_acc_list[5] += vobject_acc
            loss_acc_list[6] += vhuman_acc
        # Average over all validation chunks
        loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
        with open(valcsvPath, 'a') as f:
            writer = csv.writer(f)
            # We consider accuracy as the average accuracy over the three
            # types of accuracy
            acc = (loss_acc_list[4] +
                   loss_acc_list[5] + loss_acc_list[6]) / 3
            writer.writerow([str(acc), loss_acc_list[4], loss_acc_list[5], loss_acc_list[6],
                             loss_acc_list[0], loss_acc_list[1], loss_acc_list[2], loss_acc_list[3]])
        if loss_acc_list[0] < minValLoss:
            print("New best loss " + str(loss_acc_list[0]))
            model.save(bestModelPath)
            minValLoss = loss_acc_list[0]

    if params['email']:
        utils.sendemail(from_addr='pythonscriptsisr@gmail.com',
                        to_addr_list=['pedro_abreu95@hotmail.com'],
                        subject='Finished training RGB-stream with class_weights ' + penalizing_method,
                        message='Training RGB with following params: ' +
                        str(params),
                        login='pythonscriptsisr@gmail.com',
                        password='1!qwerty')

if __name__ == '__main__':
    main()

max_stars_repo_name: pedro-abreu/twostream-attention

max_issues_repo_name: pedro-abreu/twostream-attention

max_forks_repo_name: pedro-abreu/twostream-attention


text:
"""Work In Progress."""

from svtk.vtk_classes.vtk_animation_timer_callback import VTKAnimationTimerCallback

import pnums.int_to_xcoords
from pnums.neuralfloat import NeuralFloat
from svtk.vtk_classes.vtk_displayer import VTKDisplayer
import time

yellow_dark = [152, 122, 38]
yellow_bright = [255, 238, 188]

red_dark = [152, 57, 38]
red_bright = [255, 200, 188]

brown_dark = [152, 95, 37]
brown_bright = [255, 222, 188]

class NeuralBinaryAnimator(VTKAnimationTimerCallback):
    def __init__(self):
        super(NeuralBinaryAnimator, self).__init__()
        self.nb = NeuralFloat(0, min_val=0.01, max_val=2 ** 100)

    def loop(self, obj, event):
        super(NeuralBinaryAnimator, self).loop(obj, event)

class NBCallbackClass(NeuralBinaryAnimator):
    def __init__(self):
        super(NBCallbackClass, self).__init__()
        self.i = 1.1
        self.ento_callback = None

    def at_start(self):
        self.add_point_field(
            widths=[self.nb._array.shape[1], self.nb._array.shape[0], 1],
            normal=[0, 1, 0],
            center=[0, 1, 0],
            color=[brown_dark],
        )

    def set_nb_callback(self, callback):
        self.nb_callback = callback

    def loop(self, obj, event):
        super(NBCallbackClass, self).loop(obj, event)
        if self.nb_callback:
            self.i = self.nb_callback(self.i)
        self.nb = NeuralFloat(self.i, min_val=0.01, max_val=2 ** 100)
        self.set_all_point_colors(brown_dark)
        self.set_point_colors(brown_bright, list(self.nb.indexes()))

import numpy as np

class RobustBinaryAnimator(VTKAnimationTimerCallback):
    def __init__(self):
        super(RobustBinaryAnimator, self).__init__()
        self.nb = pnums.int_to_xcoords.ints_to_2d(0, 32, 8, 0, 32, 8)

    def loop(self, obj, event):
        super(RobustBinaryAnimator, self).loop(obj, event)

class RobustCallbackClass(RobustBinaryAnimator):
    def __init__(self):
        super(RobustCallbackClass, self).__init__()
        self.i = 1.1
        self.robust_callback = None

    def at_start(self):
        """Setup the point field at the start."""
        self.add_point_field(
            widths=[self.nb.shape[2], self.nb.shape[1], self.nb.shape[0]],
            normal=[0, 1, 0],
            center=[0, 1, 0],
            color=[brown_dark],
        )

    def set_nb_callback(self, callback):
        """Set the callback."""
        self.robust_callback = callback

    def loop(self, obj, event):
        """Run the main loop."""
        super(RobustCallbackClass, self).loop(obj, event)
        if self.robust_callback:
            self.i = self.robust_callback(self.i)
        self.nb = pnums.int_to_xcoords.ints_to_2d(self.i, 32, 8, self.i / 3, 32, 8)
        self.set_all_point_colors(brown_dark)
        indexes = np.nonzero(np.ravel(self.nb))
        self.set_point_colors(brown_bright, list(indexes))

if __name__ == "__main__":
    ela = VTKDisplayer(RobustCallbackClass)
    ela.point_size = 10

    ento_callback = lambda i: int((i + 1) % 2 ** 32)

    ela.callback_instance.set_nb_callback(ento_callback)
    ela.visualize()

ext: py

max_stars_repo_name: SimLeek/coordencode

max_issues_repo_name: SimLeek/coordencode

max_forks_repo_name: SimLeek/coordencode

reason: import numpy


text:
"""
 Background:
 ===========
 SOAP2ArchiveCSV.py

 Purpose:
 ========
  Format results from soap call to clean "traditional" csv archival format.  This format is
  known as the .y or 'year files' and was previously created by DGK and Benny's routines

 File Format:
 ============
  Currently only processes csv returned files from SOAP (not xml).

 (Very Long) Example Usage:
 ==========================

 History:
 ========

 Compatibility:
 ==============
 python >=3.6 **tested**
 python 2.7 **tested** but may break in the future

"""

import argparse
import pandas as pd
import numpy as np
import datetime

# parse incoming command line options
parser = argparse.ArgumentParser(description='Format ARGOS Soap retrieved files to archive csv files')
parser.add_argument('infile',
    metavar='infile',
    type=str,
    help='full path to infile')
parser.add_argument('-year','--archive_year',
    type=str,
    help='year of data being input, will default to current year if omitted')
parser.add_argument('-getactiveids', '--getactiveids',
    action="store_true",
    help='get active listing of platformIds')
parser.add_argument('-buoyyearfiles', '--buoyyearfiles',
    action="store_true",
    help='create buoy year files - 28882 or 28883')
parser.add_argument('-buoyid', '--buoyid',
    type=str,
    default='28882',
    help='default - 28882')
parser.add_argument('-drifteryearfiles', '--drifteryearfiles',
    action="store_true",
    help='create all drifter year files from activeid listing')

args = parser.parse_args()

df = pd.read_csv(args.infile,sep=';',index_col=False,dtype=object,error_bad_lines=False)

if not args.archive_year:
    year = str(datetime.datetime.now().year)
else:
    year = args.archive_year

if args.getactiveids:
    gb = df.groupby('platformId')
    print(gb.groups.keys())

if args.buoyyearfiles:
    gb = df.groupby('platformId')

    keep_columns=['platformId','latitude','longitude','bestDate','value'] + ['value.'+str(i) for i in range(1,32)] + ['locationClass']
    try:
      bd = gb.get_group(args.buoyid)

      bd_thinned = bd[keep_columns].copy()
      bd_thinned['year'] = bd_thinned.apply(lambda row: str(pd.to_datetime(row['bestDate']).year), axis=1)
      bd_thinned['doy'] = bd_thinned.apply(lambda row: str(pd.to_datetime(row['bestDate']).dayofyear), axis=1)
      bd_thinned['hhmm'] = bd_thinned.apply(lambda row: str(pd.to_datetime(row['bestDate']).hour).zfill(2)+str(pd.to_datetime(row['bestDate']).minute).zfill(2), axis=1)

      out_columns=['platformId','latitude','longitude','year','doy','hhmm','value'] + ['value.'+str(i) for i in range(1,32)] + ['locationClass']
      bd_thinned[out_columns].to_csv('0'+args.buoyid+'.y' + year,' ',header=False,index=False,na_rep=np.nan,mode='a')
    except:
      print("no 28882 data in this file")

if args.drifteryearfiles:
    gb = df.groupby('platformId')

    keep_columns=['platformId','latitude','longitude','locationDate','value'] + ['value.'+str(i) for i in range(1,7)] + ['locationClass']

    for k in gb.groups.keys():
      if k not in ['28882','28883']:
        print(k)
        bd = gb.get_group(k)
        bd_thinned = bd[keep_columns].copy()
        bd_thinned['year'] = bd_thinned.apply(lambda row: str(pd.to_datetime(row['locationDate']).year), axis=1)
        bd_thinned['doy'] = bd_thinned.apply(lambda row: str(pd.to_datetime(row['locationDate']).dayofyear), axis=1)
        bd_thinned['hhmm'] = bd_thinned.apply(lambda row: str(pd.to_datetime(row['locationDate']).hour).zfill(2)+str(pd.to_datetime(row['locationDate']).minute).zfill(2), axis=1)
        #make special case for 122531, the peggy backup locator buoy
        if k=='122531':
          out_columns=['platformId','latitude','longitude','year','doy','hhmm','value'] + ['locationClass']
          bd_thinned[out_columns].dropna().to_csv(k + '.y' + year,' ',header=False,index=False,na_rep=np.nan,mode='a')
        else:
          out_columns=['platformId','latitude','longitude','year','doy','hhmm','value'] + ['value.'+str(i) for i in range(1,7)] + ['locationClass']
          bd_thinned[out_columns].dropna().to_csv(k + '.y' + year,' ',header=False,index=False,na_rep=np.nan,mode='a')

max_forks_repo_name: shaunwbell/ArgosDataOps


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from dtwalign import dtw
def Magnitude(tmp):
    return np.sqrt((np.square(tmp[0]))+(np.square(tmp[1]))+(np.square(tmp[2])))

def CalMagnitude(X,Y,Z):
    arrtmp = []
    for i in range(len(X)):
        a = X[i]
        b = Y[i]
        c = Z[i]
        tmp = [a,b,c]
        arrtmp.append(Magnitude(tmp))

    return arrtmp

def processdata(p, RMSAcc,value,i):
    valuearr = []
    IDarr =[]
    parr = []
    tmprmsraw = []
    while i < len(p):
        parr.append(p[i])
        tmprmsraw.append(RMSAcc[i])
        valuearr.append(value[i])
        i+=1
    j=0
    sums = 0
    while j < len(parr):
        IDarr.append(sums)
        sums = round(sums+0.04,2)
        j+=1
    return  IDarr, parr, tmprmsraw, valuearr

def calLPF(alpha, value, tmparr, info):
    if len(tmparr) == 0:
        va = alpha * value;
        tmparr.append(va)
    else:
        va = alpha * value + (1-alpha) * tmparr[info-1]
        tmparr.append(va)
    return va

def calalpha(cutoff, fs):
    dt = 1/fs
    T  = 1/cutoff
    return round(dt/(T+dt),2)

def LPF(alpha,rawarr):
    i = 0
    tmparr = []
    lpfarr = []
    while i < len(rawarr):
        va = calLPF(alpha,rawarr[i],tmparr, i)
        i+=1
        lpfarr.append(va)
    return lpfarr

def process(total):
    Ax = total.Ax
    Ay = total.Ay
    Az = total.Az
    MagnitudeAcc = CalMagnitude(Ax, Ay, Az)
    P = total.P
    Type = total.Type

    IDaa, paa, rmsaccraw, Type = processdata(P, MagnitudeAcc, Type, 0)
    fs = 25
    cutoff = 1.5
    alpha = calalpha(cutoff, fs)
    plpf = LPF(alpha, paa)
    plpf = LPF(alpha, plpf)
    rmsacclpf = LPF(alpha, rmsaccraw)
    rmsacclpf = LPF(alpha, rmsacclpf)

    IDaa, pa, acc, typearr = processdata(plpf, rmsacclpf, Type, 300)

    return acc

if __name__ == '__main__':
    totaldung  = pd.read_csv('Downstair2.csv') #dung
    totalsai = pd.read_csv('Downstair4.csv') #sai

    magdung = np.array(process(totaldung))
    magsai = np.array(process(totalsai))

    res = dtw(magdung, magsai)

    print("dtw distance: {}".format(res.distance))
    print("dtw normalized distance: {}".format(res.normalized_distance))

    plt.plot(magdung, label="query")
    plt.plot(magsai, label="reference")
    plt.legend()
    plt.show()


text:
"""binary_classifier: script to make a framework to spot check binary
machine learning classification models
"""
import warnings
import matplotlib.pyplot as plt
from numpy import mean
from numpy import std
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import RidgeClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import ExtraTreeClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier

# create a dict of standard models to evaluate {name:object}
def define_models(models=dict()):
    # linear models
    models["logistic"] = LogisticRegression()
    alpha = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
    for value in alpha:
        models["ridge-" + str(value)] = RidgeClassifier(alpha=value)
    models["sgd"] = SGDClassifier(max_iter=1000, tol=1e-3)
    models["pa"] = PassiveAggressiveClassifier(max_iter=1000, tol=1e-3)
    # non-linear models
    n_neighbors = range(1, 21)
    for k in n_neighbors:
        models["knn-" + str(k)] = KNeighborsClassifier(n_neighbors=k)
    models["cart"] = DecisionTreeClassifier()
    models["extra"] = ExtraTreeClassifier()
    models["svml"] = SVC(kernel="linear")
    models["svmp"] = SVC(kernel="poly")
    c_values = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
    for c in c_values:
        models["svmr" + str(c)] = SVC(C=c)
    models["bayes"] = GaussianNB()
    # ensemble models
    n_trees = 100
    models["ada"] = AdaBoostClassifier(n_estimators=n_trees)
    models["bag"] = BaggingClassifier(n_estimators=n_trees)
    models["rf"] = RandomForestClassifier(n_estimators=n_trees)
    models["et"] = ExtraTreesClassifier(n_estimators=n_trees)
    models["gbm"] = GradientBoostingClassifier(n_estimators=n_trees)
    print("Defined %d models" % len(models))
    return models

# create a feature preparation pipeline for a model
# Below exist 4 options: none, standardize, normalize, or stand.+norm.
def pipeline_none(model):
    """No transforms pipeline"""
    return model

def pipeline_standardize(model):
    """Standardize transform pipeline"""
    steps = list()
    steps.append(("standardize", StandardScaler()))
    steps.append(("model", model))
    pipeline = Pipeline(steps=steps)
    return pipeline

def pipeline_normalize(model):
    """Normalize transform pipeline"""
    steps = list()
    steps.append(("normalize", MinMaxScaler()))
    steps.append(("model", model))
    pipeline = Pipeline(steps=steps)
    return pipeline

def pipeline_std_norm(model):
    """ Standardize and normalize pipeline"""
    steps = list()
    steps.append(("standardize", StandardScaler()))
    steps.append(("normalize", MinMaxScaler()))
    steps.append(("model", model))
    pipeline = Pipeline(steps=steps)
    return pipeline

def evaluate_model(X, y, model, folds, metric, pipe_func):
    """ Evaluate a signle model with a pipeline"""
    pipeline = pipe_func(model)
    scores = cross_val_score(pipeline, X, y, scoring=metric, cv=folds, n_jobs=-1)
    return scores

def robust_evaluate_model(X, y, model, folds, metric, pipe_func):
    """ Evaluate a model and try to trap errors and and hide warnings"""
    scores = None
    try:
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore")
            scores = evaluate_model(X, y, model, folds, metric, pipe_func)
    except:
        scores = None
    return scores

def evaluate_models(X, y, models, pipe_funcs, folds=10, metric="accuracy"):
    """ Evaluate a dict of models {name:object}, returns {name:score}"""
    results = dict()
    for name, model in models.items():
        # evaluate model under each preparation function
        for i in range(len(pipe_funcs)):
            # evaluate the model
            scores = robust_evaluate_model(X, y, model, folds, metric, pipe_funcs[i])
            # update name
            run_name = str(i) + name
            # show process
            if scores is not None:
                # store a result
                results[run_name] = scores
                mean_score, std_score = mean(scores), std(scores)
                print(">%s: %.3f (+/-%.3f)" % (run_name, mean_score, std_score))
            else:
                print(">%s: error" % run_name)
    return results

def summarize_results(results, maximize=True, top_n=10):
    """ Print and plot the top n results"""
    # check for no results
    if len(results) == 0:
        print("no results")
        return
    # determine how many results to summarize
    n = min(top_n, len(results))
    # create a list of (name, mean(scores)) tuples
    mean_scores = [(k, mean(v)) for k, v in results.items()]
    # sort tuples by mean score
    mean_scores = sorted(mean_scores, key=lambda x: x[1])
    # reverse for descending order (e.g. for accuracy)
    if maximize:
        mean_scores = list(reversed(mean_scores))
    # retrieve the top n for summarization
    names = [x[0] for x in mean_scores[:n]]
    scores = [results[x[0]] for x in mean_scores[:n]]
    # print the top n
    print()
    for i in range(n):
        name = names[i]
        mean_score, std_score = mean(results[name]), std(results[name])
        print(
            "Rank=%d, Name=%s, Score=%.3f (+/- %.3f)"
            % (i + 1, name, mean_score, std_score)
        )
    # boxplot for the top n
    plt.boxplot(scores, labels=names)
    _, labels = plt.xticks()
    plt.setp(labels, rotation=90)
    plt.savefig("spotcheck.png")

max_stars_repo_name: kwsebesta/tools-and-scripts

max_issues_repo_name: kwsebesta/tools-and-scripts

max_forks_repo_name: kwsebesta/tools-and-scripts

reason: from numpy


text:
import sys
sys.path.append('/home/holiestcow/Documents/2017_fall/ne697_hayward')

# import ROOT
# import numpy as np
from radsynth.fileio.root_io import GeantImport
import matplotlib.pyplot as plt

def main():
    datgeant4path = '/home/holiestcow/Documents/2017_fall/ne697_hayward/ddli-code/geant4/'
    desired_path = '{}lib/libddli-sim'.format(datgeant4path)

    rootio = GeantImport(desired_path)
    x, y, y_norm = rootio.get_spectra('../../ddli-code/geant4/output/trailer_test.root',
                                      bits=10, detectorID=None)
    fig = plt.figure()
    plt.plot(x, y)
    plt.xlabel('Energy')
    plt.ylabel('Frequency')
    fig.savefig('rootioclass_test.png')
    plt.close(fig)

    fig = plt.figure()
    plt.plot(x, y_norm)
    plt.xlabel('Energy')
    plt.ylabel('Normalized Frequency')
    fig.savefig('rootioclass_normalized_test.png')
    plt.close(fig)
    return

main()

max_stars_repo_name: HoliestCow/utk-radsynth

max_issues_repo_name: HoliestCow/utk-radsynth

max_forks_repo_name: HoliestCow/utk-radsynth


text:
import matplotlib.pyplot as plt
import networkx as nx
from networkx.drawing.nx_agraph import graphviz_layout

def display_resolution_tree(resolution_tree: nx.classes.DiGraph):
    _draw_resolution_tree_(resolution_tree)
    plt.show()

def _draw_resolution_tree_(tree: nx.classes.DiGraph, enable_edge_labels: bool = True, rotate_edge_labels: bool = False):
    plt.figure()

    # graph
    nodes_pos = graphviz_layout(tree, prog='dot')
    nx.draw(tree, nodes_pos,
            node_size=150, edge_color='#7d7d7d')

    # nodes labels
    pos_attrs = {}
    for node, coords in nodes_pos.items():
        pos_attrs[node] = (coords[0], coords[1] - 10)

    custom_node_attrs = {}
    for node, attr in tree.nodes.items():
        custom_node_attrs[node] = str(node)

    nodes_bbox = dict(facecolor="w", edgecolor="#d3d3d3", pad=6, lw=0.1)
    nx.draw_networkx_labels(
        tree, pos_attrs, labels=custom_node_attrs, font_size=13, bbox=nodes_bbox)

    # edge labels
    if enable_edge_labels:
        edges_pos = graphviz_layout(tree, prog='dot')
        edge_labels = nx.get_edge_attributes(tree, 'transformation')
        nx.draw_networkx_edge_labels(
            tree, pos=edges_pos, edge_labels=edge_labels, font_size=13, rotate=rotate_edge_labels)

max_stars_repo_name: szymanskir/msi

max_issues_repo_name: szymanskir/msi

max_forks_repo_name: szymanskir/msi

reason: import networkx,from networkx


text:
"""
Stein Variational Gradient Descent for HM-DenseED.
"""

import torch
import numpy as np
from time import time
from args import args, device
from models.model_det import DenseED
from models.Bayesian_model_NN import Bayesian_model_NN
from models.model_train import Bayesian_model_train
from utils.train_data_upload import train_load_data
import json
import scipy.io as io
import sys
import os

# deterministic NN
dense_ed = DenseED(in_channels=1,
                    out_channels=1,
                    blocks=args.blocks,
                    growth_rate=args.growth_rate,
                    init_features=args.init_features,
                    drop_rate=args.drop_rate,
                    bn_size=args.bn_size,
                    bottleneck=args.bottleneck,
                    out_activation='Sigmoid')

n_out_pixels_train = args.ntrain*128*128

# Bayesian NN
Bayesian_model = Bayesian_model_NN(dense_ed, n_samples=args.n_samples).to(device)
def mkdir(path):
    if not os.path.exists(path):
        os.makedirs(path)

# load data
train_loader = train_load_data()
print('Loaded data!')

# Initialize SVGD
Bayesian_train = Bayesian_model_train(Bayesian_model, train_loader)

def main():
    print('Start training.........................................................')
    tic = time()
    train_loss = []
    for epoch in range(1, args.epochs + 1):
        print ('epoch number .......................................',epoch)
        ticc = time()
        mse_tot2 = Bayesian_train.train(epoch)
        print('Training RMSE 2:.......',mse_tot2)
        train_loss.append(mse_tot2)
        tocc = time()
        print('.............training done....................', tocc-ticc)
    train_loss = np.array(train_loss)
    io.savemat('train_loss.mat', dict([('train_loss',np.array(train_loss))]))
    training_time = time() - tic
    print('Finished training:\n{} epochs\n{} data\n{} samples (SVGD)\n{} seconds'
        .format(args.epochs, args.ntrain, args.n_samples, training_time))

    torch.save(Bayesian_model, 'model_%d.pt'%args.ntrain)

if __name__ == '__main__':
    main()

max_stars_repo_name: zabaras/bayesmultiscale

max_issues_repo_name: zabaras/bayesmultiscale

max_forks_repo_name: zabaras/bayesmultiscale

reason: import numpy,import scipy


text:
import dummy
import numpy
import redis
import struct
import numpy as np

def adapt_array(array):
    """
    Adapt numpy array for saving into redisDB

    :param numpy.array array: NumPy array to encode
    :return: encoded NumPy array
    """
    array = array.astype(np.float32)
    h = array.shape[0]

    shape = struct.pack('>I',h)
    encoded = shape + array.tobytes()

    return encoded

def convert_array(encoded):
    """

    :param BLOG encoded: encoded NumPy array
    :return: One steaming hot NumPy array
    :rtype: numpy.array
    """

    h = struct.unpack('>I',encoded[:4])
    array = np.frombuffer(encoded, dtype=np.float32, offset=4).reshape(h)

    return array

db = redis.Redis(host='localhost', port=6379, db=0)

# #########
# # Write #
# #########
for key, emb in dummy.embeddings():
    arr = adapt_array(emb)
    db.set(key, arr)

# ########
# # Read #
# ########
for key, _ in dummy.embeddings():
    obj = db.get(key)
    emb = convert_array(obj)
    assert(type(emb) is numpy.ndarray)

# del couchserver[dbname]


text:
import numpy as np
import pandas as pd
import os
import time
from pythonosc import udp_client
from pythonosc.udp_client import SimpleUDPClient
from PyQt5 import QtCore, QtGui, QtWidgets, QtTest
from PyQt5.QtWidgets import QFileDialog

'''
Setup ip and port
'''

ip = "127.0.0.1"
port = 5510 #'faust default'
client = SimpleUDPClient(ip, port)  # Create client

'''
PyQt Gui
'''

class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(350, 200)

        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.progressBar = QtWidgets.QProgressBar(self.centralwidget)
        self.progressBar.setGeometry(QtCore.QRect(20, 100, 300, 21))
        self.progressBar.setProperty("value", 0)
        self.progressBar.setObjectName("progressBar")
        self.label_title = QtWidgets.QLabel(self.centralwidget)
        self.label_title.setGeometry(QtCore.QRect(10, 0, 121, 21))
        self.label_title.setObjectName("label_title")
        self.label_parameter = QtWidgets.QLabel(self.centralwidget)
        self.label_parameter.setGeometry(QtCore.QRect(20, 130, 141, 20))
        self.label_parameter.setObjectName("label_parameter")
        self.pushButton_start = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_start.setGeometry(QtCore.QRect(125, 70, 113, 32))
        self.pushButton_start.setCheckable(True)
        self.pushButton_start.setChecked(False)
        self.pushButton_start.setAutoDefault(False)
        self.pushButton_start.setObjectName("pushButton_start")
        self.layoutWidget = QtWidgets.QWidget(self.centralwidget)
        self.layoutWidget.setGeometry(QtCore.QRect(20, 32, 311, 33))
        self.layoutWidget.setObjectName("layoutWidget")
        self.horizontalLayout = QtWidgets.QHBoxLayout(self.layoutWidget)
        self.horizontalLayout.setContentsMargins(0, 0, 0, 0)
        self.horizontalLayout.setObjectName("horizontalLayout")
        self.label_dataset = QtWidgets.QLabel(self.layoutWidget)
        self.label_dataset.setObjectName("label_dataset")
        self.horizontalLayout.addWidget(self.label_dataset)
        self.lineEdit_dataset = QtWidgets.QLineEdit(self.layoutWidget)
        self.lineEdit_dataset.setObjectName("lineEdit_dataset")
        self.horizontalLayout.addWidget(self.lineEdit_dataset)
        self.pushButton_openfile = QtWidgets.QPushButton(self.layoutWidget)
        self.pushButton_openfile.setObjectName("pushButton_openfile")
        self.horizontalLayout.addWidget(self.pushButton_openfile)
        self.layoutWidget4 = QtWidgets.QWidget(self.centralwidget)
        self.layoutWidget4.setGeometry(QtCore.QRect(250, 70, 76, 47))
        self.layoutWidget4.setObjectName("layoutWidget4")
        self.verticalLayout_2 = QtWidgets.QVBoxLayout(self.layoutWidget4)
        self.verticalLayout_2.setContentsMargins(0, 0, 0, 0)
        self.verticalLayout_2.setObjectName("verticalLayout_2")
        self.label_speed = QtWidgets.QLabel(self.layoutWidget4)

        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 350, 22))
        self.menubar.setObjectName("menubar")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

        self.pushButton_start.clicked.connect(self.stream_osc_msg)
        self.pushButton_openfile.clicked.connect(self.browse_file)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "MainWindow"))
        self.label_title.setText(_translate("MainWindow", "OSC Data Streamer"))
        self.pushButton_start.setText(_translate("MainWindow", "Start"))
        self.label_dataset.setText(_translate("MainWindow", "Dataset"))
        self.pushButton_openfile.setText(_translate("MainWindow", "Browse"))

    '''
    Read Data
    '''

    def browse_file(self):
        print ("pressed button")
        self.open_dialog_box()

    def open_dialog_box(self):
        filename = QFileDialog.getOpenFileName()
        print('your file %s is loaded' %(os.path.basename(filename[0])))
        self.path = filename[0]
        print (self.path)

        self.eeg_data = pd.read_csv(self.path, delimiter=";", decimal=",")
        self.eeg_data = self.eeg_data.astype(float)
        self.eeg_data_clean = self.eeg_data.drop(self.eeg_data.index[0:4])
        print (self.eeg_data_clean.shape[0])

        self.lineEdit_dataset.setText(os.path.basename(filename[0]))

    '''
    Create OSC messages and stream data
    '''
    def stream_osc_msg(self):
        if self.pushButton_start.isChecked() == True:
            for i in range(self.eeg_data_clean.shape[0]):
                row = self.eeg_data_clean.iloc[i,:].values #prints row as array
                QtTest.QTest.qWait(1000) #in ms
                self.progressBar.setValue ((i * 100) / self.eeg_data_clean.shape[0])

                for j in range(8):
                    msg_freq = "/Brainwave_Virtual_Instrument/Synth/Channel_%d/modFreq" % (j+1) #reads each value of a row and creates osc message with that value
                    value_freq = (20 + ((40*int(((j+1) * 0.5)+0.5)))) + row[j]
                    client.send_message(msg_freq, value_freq)
                    print (value_freq)

                if self.pushButton_start.isChecked() == False:
                    break

if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = Ui_MainWindow()
    ui.setupUi(MainWindow)
    MainWindow.show()
    sys.exit(app.exec_())


text:
# -*- coding: utf-8 -*-

"""Feature view."""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------

import logging
import re

import numpy as np

from phylib.utils import Bunch, emit
from phy.utils.color import selected_cluster_color
from phy.plot.transform import Range
from phy.plot.visuals import ScatterVisual, TextVisual, LineVisual
from .base import ManualClusteringView, MarkerSizeMixin, ScalingMixin

logger = logging.getLogger(__name__)

# -----------------------------------------------------------------------------
# Feature view
# -----------------------------------------------------------------------------

def _get_default_grid():
    """In the grid specification, 0 corresponds to the best channel, 1
    to the second best, and so on. A, B, C refer to the PC components."""
    s = """
    time,0A 1A,0A   0B,0A   1B,0A
    0A,1A   time,1A 0B,1A   1B,1A
    0A,0B   1A,0B   time,0B 1B,0B
    0A,1B   1A,1B   0B,1B   time,1B
    """.strip()
    return [[_ for _ in re.split(' +', line.strip())] for line in s.splitlines()]

def _get_point_color(clu_idx=None):
    if clu_idx is not None:
        color = selected_cluster_color(clu_idx, .5)
    else:
        color = (.5,) * 4
    assert len(color) == 4
    return color

def _get_point_masks(masks=None, clu_idx=None):
    masks = masks if masks is not None else 1.
    # NOTE: we add the cluster relative index for the computation of the depth on the GPU.
    return masks * .99999 + (clu_idx or 0)

def _get_masks_max(px, py):
    mx = px.get('masks', None)
    my = py.get('masks', None)
    if mx is None or my is None:
        return None
    return np.maximum(mx, my)

def _uniq(seq):
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]

class FeatureView(MarkerSizeMixin, ScalingMixin, ManualClusteringView):
    """This view displays a 4x4 subplot matrix with different projections of the principal
    component features. This view keeps track of which channels are currently shown.

    Constructor
    -----------

    features : function
        Maps `(cluster_id, channel_ids=None, load_all=False)` to
        `Bunch(data, channel_ids, channel_labels, spike_ids , masks)`.
        * `data` is an `(n_spikes, n_channels, n_features)` array
        * `channel_ids` contains the channel ids of every row in `data`
        * `channel_labels` contains the channel labels of every row in `data`
        * `spike_ids` is a `(n_spikes,)` array
        * `masks` is an `(n_spikes, n_channels)` array

        This allows for a sparse format.

    attributes : dict
        Maps an attribute name to a 1D array with `n_spikes` numbers (for example, spike times).

    """

    _default_position = 'right'
    cluster_ids = ()

    # Whether to disable automatic selection of channels.
    fixed_channels = False
    feature_scaling = 1.

    default_shortcuts = {
        'change_marker_size': 'ctrl+wheel',
        'increase': 'ctrl++',
        'decrease': 'ctrl+-',
        'add_lasso_point': 'ctrl+click',
        'stop_lasso': 'ctrl+right click',
        'toggle_automatic_channel_selection': 'c',
    }

    def __init__(self, features=None, attributes=None, **kwargs):
        super(FeatureView, self).__init__(**kwargs)
        self.state_attrs += ('fixed_channels', 'feature_scaling')

        assert features
        self.features = features
        self._lim = 1

        self.grid_dim = _get_default_grid()  # 2D array where every item a string like `0A,1B`
        self.n_rows, self.n_cols = np.array(self.grid_dim).shape
        self.canvas.set_layout('grid', shape=(self.n_rows, self.n_cols))
        self.canvas.enable_lasso()

        # Channels being shown.
        self.channel_ids = None

        # Attributes: extra features. This is a dictionary
        # {name: array}
        # where each array is a `(n_spikes,)` array.
        self.attributes = attributes or {}

        self.visual = ScatterVisual()
        self.canvas.add_visual(self.visual)

        self.text_visual = TextVisual()
        self.canvas.add_visual(self.text_visual)

        self.line_visual = LineVisual()
        self.canvas.add_visual(self.line_visual)

    def set_grid_dim(self, grid_dim):
        """Change the grid dim dynamically.

        Parameters
        ----------
        grid_dim : array-like (2D)
            `grid_dim[row, col]` is a string with two values separated by a comma. Each value
            is the relative channel id (0, 1, 2...) followed by the PC (A, B, C...). For example,
            `grid_dim[row, col] = 0B,1A`. Each value can also be an attribute name, for example
            `time`. For example, `grid_dim[row, col] = time,2C`.

        """
        self.grid_dim = grid_dim
        self.n_rows, self.n_cols = np.array(grid_dim).shape
        self.canvas.grid.shape = (self.n_rows, self.n_cols)

    # Internal methods
    # -------------------------------------------------------------------------

    def _iter_subplots(self):
        """Yield (i, j, dim)."""
        for i in range(self.n_rows):
            for j in range(self.n_cols):
                dim = self.grid_dim[i][j]
                dim_x, dim_y = dim.split(',')
                yield i, j, dim_x, dim_y

    def _get_axis_label(self, dim):
        """Return the channel label from a dimension, if applicable."""
        if str(dim[:-1]).isdecimal():
            n = len(self.channel_ids)
            channel_id = self.channel_ids[int(dim[:-1]) % n]
            return self.channel_labels[channel_id] + dim[-1]
        else:
            return dim

    def _get_channel_and_pc(self, dim):
        """Return the channel_id and PC of a dim."""
        if self.channel_ids is None:
            return
        assert dim not in self.attributes  # This is called only on PC data.
        s = 'ABCDEFGHIJ'
        # Channel relative index, typically just 0 or 1.
        c_rel = int(dim[:-1])
        # Get the channel_id from the currently-selected channels.
        channel_id = self.channel_ids[c_rel % len(self.channel_ids)]
        pc = s.index(dim[-1])
        return channel_id, pc

    def _get_axis_data(self, bunch, dim, cluster_id=None, load_all=None):
        """Extract the points from the data on a given dimension.

        bunch is returned by the features() function.
        dim is the string specifying the dimensions to extract for the data.

        """
        if dim in self.attributes:
            return self.attributes[dim](cluster_id, load_all=load_all)
        masks = bunch.get('masks', None)
        channel_id, pc = self._get_channel_and_pc(dim)
        # Skip the plot if the channel id is not displayed.
        if channel_id not in bunch.channel_ids:  # pragma: no cover
            return Bunch(data=np.zeros((bunch.data.shape[0],)))
        # Get the column index of the current channel in data.
        c = list(bunch.channel_ids).index(channel_id)
        if masks is not None:
            masks = masks[:, c]
        return Bunch(data=self.feature_scaling * bunch.data[:, c, pc], masks=masks)

    def _get_axis_bounds(self, dim, bunch):
        """Return the min/max of an axis."""
        if dim in self.attributes:
            # Attribute: specified lim, or compute the min/max.
            vmin, vmax = bunch.get('lim', (0, 0))
            assert vmin is not None
            assert vmax is not None
            return vmin, vmax
        return (-self._lim, +self._lim)

    def _plot_points(self, bunch, clu_idx=None):
        cluster_id = self.cluster_ids[clu_idx] if clu_idx is not None else None
        for i, j, dim_x, dim_y in self._iter_subplots():
            px = self._get_axis_data(bunch, dim_x, cluster_id=cluster_id)
            py = self._get_axis_data(bunch, dim_y, cluster_id=cluster_id)
            # Skip empty data.
            if px is None or py is None:  # pragma: no cover
                logger.warning("Skipping empty data for cluster %d.", cluster_id)
                return
            assert px.data.shape == py.data.shape
            xmin, xmax = self._get_axis_bounds(dim_x, px)
            ymin, ymax = self._get_axis_bounds(dim_y, py)
            assert xmin <= xmax
            assert ymin <= ymax
            data_bounds = (xmin, ymin, xmax, ymax)
            masks = _get_masks_max(px, py)
            # Prepare the batch visual with all subplots
            # for the selected cluster.
            self.visual.add_batch_data(
                x=px.data, y=py.data,
                color=_get_point_color(clu_idx),
                # Reduced marker size for background features
                size=self._marker_size,
                masks=_get_point_masks(clu_idx=clu_idx, masks=masks),
                data_bounds=data_bounds,
                box_index=(i, j),
            )
            # Get the channel ids corresponding to the relative channel indices
            # specified in the dimensions. Channel 0 corresponds to the first
            # best channel for the selected cluster, and so on.
            label_x = self._get_axis_label(dim_x)
            label_y = self._get_axis_label(dim_y)
            # Add labels.
            self.text_visual.add_batch_data(
                pos=[1, 1],
                anchor=[-1, -1],
                text=label_y,
                data_bounds=None,
                box_index=(i, j),
            )
            self.text_visual.add_batch_data(
                pos=[0, -1.],
                anchor=[0, 1],
                text=label_x,
                data_bounds=None,
                box_index=(i, j),
            )

    def _plot_axes(self):
        self.line_visual.reset_batch()
        for i, j, dim_x, dim_y in self._iter_subplots():
            self.line_visual.add_batch_data(
                pos=[[-1., 0., +1., 0.],
                     [0., -1., 0., +1.]],
                color=(.5, .5, .5, .5),
                box_index=(i, j),
                data_bounds=None,
            )
        self.canvas.update_visual(self.line_visual)

    def _get_lim(self, bunchs):
        m, M = min(bunch.data.min() for bunch in bunchs), max(bunch.data.max() for bunch in bunchs)
        M = max(abs(m), abs(M))
        return M

    def _get_scaling_value(self):
        return self.feature_scaling

    def _set_scaling_value(self, value):
        self.feature_scaling = value
        self.plot()

    # Public methods
    # -------------------------------------------------------------------------

    def clear_channels(self):
        """Reset the current channels."""
        self.channel_ids = None
        self.plot()

    def get_clusters_data(self, fixed_channels=None, load_all=None):
        # Get the feature data.
        # Specify the channel ids if these are fixed, otherwise
        # choose the first cluster's best channels.
        c = self.channel_ids if fixed_channels else None
        bunchs = [self.features(cluster_id, channel_ids=c) for cluster_id in self.cluster_ids]
        for cluster_id, bunch in zip(self.cluster_ids, bunchs):
            bunch.cluster_id = cluster_id

        # Choose the channels based on the first selected cluster.
        channel_ids = list(bunchs[0].channel_ids) if bunchs else []
        common_channels = list(channel_ids)
        # Intersection (with order kept) of channels belonging to all clusters.
        for bunch in bunchs:
            common_channels = [c for c in bunch.channel_ids if c in common_channels]
        # The selected channels will be (1) the channels common to all clusters, followed
        # by (2) remaining channels from the first cluster (excluding those already selected
        # in (1)).
        n = len(channel_ids)
        not_common_channels = [c for c in channel_ids if c not in common_channels]
        channel_ids = common_channels + not_common_channels[:n - len(common_channels)]
        assert len(channel_ids) > 0

        # Choose the channels automatically unless fixed_channels is set.
        if (not fixed_channels or self.channel_ids is None):
            self.channel_ids = channel_ids
        assert len(self.channel_ids)

        # Channel labels.
        self.channel_labels = {}
        for d in bunchs:
            chl = d.get('channel_labels', ['%d' % ch for ch in d.channel_ids])
            self.channel_labels.update({
                channel_id: chl[i] for i, channel_id in enumerate(d.channel_ids)})

        return bunchs

    def plot(self, **kwargs):
        """Update the view with the selected clusters."""

        # Determine whether the channels should be fixed or not.
        added = kwargs.get('up', {}).get('added', None)
        # Fix the channels if the view updates after a cluster event
        # and there are new clusters.
        fixed_channels = (
            self.fixed_channels or kwargs.get('fixed_channels', None) or added is not None)

        # Get the clusters data.
        bunchs = self.get_clusters_data(fixed_channels=fixed_channels)
        self._lim = self._get_lim(bunchs)

        # Get the background data.
        background = self.features(channel_ids=self.channel_ids)

        # Plot all features.
        self._plot_axes()

        # NOTE: the columns in bunch.data are ordered by decreasing quality
        # of the associated channels. The channels corresponding to each
        # column are given in bunch.channel_ids in the same order.

        # Plot points.
        self.visual.reset_batch()
        self.text_visual.reset_batch()

        self._plot_points(background)  # background spikes

        # Plot each cluster.
        for clu_idx, bunch in enumerate(bunchs):
            self._plot_points(bunch, clu_idx=clu_idx)

        # Upload the data on the GPU.
        self.canvas.update_visual(self.visual)
        self.canvas.update_visual(self.text_visual)
        self.canvas.update()

    def attach(self, gui):
        """Attach the view to the GUI."""
        super(FeatureView, self).attach(gui)

        self.actions.add(
            self.toggle_automatic_channel_selection,
            checked=not self.fixed_channels, checkable=True)
        self.actions.add(self.clear_channels)
        self.actions.separator()

    def toggle_automatic_channel_selection(self, checked):
        """Toggle the automatic selection of channels when the cluster selection changes."""
        self.fixed_channels = not checked

    @property
    def status(self):
        if self.channel_ids is None:  # pragma: no cover
            return ''
        channel_labels = [self.channel_labels[ch] for ch in self.channel_ids[:2]]
        return 'channels: %s' % ', '.join(channel_labels)

    # Dimension selection
    # -------------------------------------------------------------------------

    def on_select_channel(self, sender=None, channel_id=None, key=None, button=None):
        """Respond to the click on a channel from another view, and update the
        relevant subplots."""
        channels = self.channel_ids
        if channels is None:
            return
        if len(channels) == 1:
            self.plot()
            return
        assert len(channels) >= 2
        # Get the axis from the pressed button (1, 2, etc.)
        if key is not None:
            d = np.clip(len(channels) - 1, 0, key - 1)
        else:
            d = 0 if button == 'Left' else 1
        # Change the first or second best channel.
        old = channels[d]
        # Avoid updating the view if the channel doesn't change.
        if channel_id == old:
            return
        channels[d] = channel_id
        # Ensure that the first two channels are different.
        if channels[1 - min(d, 1)] == channel_id:
            channels[1 - min(d, 1)] = old
        assert channels[0] != channels[1]
        # Remove duplicate channels.
        self.channel_ids = _uniq(channels)
        logger.debug("Choose channels %d and %d in feature view.", *channels[:2])
        # Fix the channels temporarily.
        self.plot(fixed_channels=True)
        self.update_status()

    def on_mouse_click(self, e):
        """Select a feature dimension by clicking on a box in the feature view."""
        b = e.button
        if 'Alt' in e.modifiers:
            # Get mouse position in NDC.
            (i, j), _ = self.canvas.grid.box_map(e.pos)
            dim = self.grid_dim[i][j]
            dim_x, dim_y = dim.split(',')
            dim = dim_x if b == 'Left' else dim_y
            other_dim = dim_y if b == 'Left' else dim_x
            if dim not in self.attributes:
                # When a regular (channel, PC) dimension is selected.
                channel_pc = self._get_channel_and_pc(dim)
                if channel_pc is None:
                    return
                channel_id, pc = channel_pc
                logger.debug("Click on feature dim %s, channel id %s, PC %s.", dim, channel_id, pc)
            else:
                # When the selected dimension is an attribute, e.g. "time".
                pc = None
                # Take the channel id in the other dimension.
                channel_pc = self._get_channel_and_pc(other_dim)
                channel_id = channel_pc[0] if channel_pc is not None else None
                logger.debug("Click on feature dim %s.", dim)
            emit('select_feature', self, dim=dim, channel_id=channel_id, pc=pc)

    def on_request_split(self, sender=None):
        """Return the spikes enclosed by the lasso."""
        if (self.canvas.lasso.count < 3 or not len(self.cluster_ids)):  # pragma: no cover
            return np.array([], dtype=np.int64)
        assert len(self.channel_ids)

        # Get the dimensions of the lassoed subplot.
        i, j = self.canvas.layout.active_box
        dim = self.grid_dim[i][j]
        dim_x, dim_y = dim.split(',')

        # Get all points from all clusters.
        pos = []
        spike_ids = []

        for cluster_id in self.cluster_ids:
            # Load all spikes.
            bunch = self.features(cluster_id, channel_ids=self.channel_ids, load_all=True)
            px = self._get_axis_data(bunch, dim_x, cluster_id=cluster_id, load_all=True)
            py = self._get_axis_data(bunch, dim_y, cluster_id=cluster_id, load_all=True)
            points = np.c_[px.data, py.data]

            # Normalize the points.
            xmin, xmax = self._get_axis_bounds(dim_x, px)
            ymin, ymax = self._get_axis_bounds(dim_y, py)
            r = Range((xmin, ymin, xmax, ymax))
            points = r.apply(points)

            pos.append(points)
            spike_ids.append(bunch.spike_ids)
        pos = np.vstack(pos)
        spike_ids = np.concatenate(spike_ids)

        # Find lassoed spikes.
        ind = self.canvas.lasso.in_polygon(pos)
        self.canvas.lasso.clear()
        return np.unique(spike_ids[ind])

max_stars_repo_name: fjflores/phy

max_issues_repo_name: fjflores/phy

max_forks_repo_name: fjflores/phy


text:
import os.path as osp
import numpy as np
import torch

from data.datasets import InMemoryComplexDataset
from data.utils import convert_graph_dataset_with_rings
from torch_geometric.datasets import GNNBenchmarkDataset
from torch_geometric.utils import remove_self_loops

class CSLDataset(InMemoryComplexDataset):
    """This is the CSL (Circular Skip Link) dataset from the Benchmarking GNNs paper.

    The dataset contains 10 isomorphism classes of regular graphs that must be classified.
    """

    def __init__(self, root, transform=None,
                 pre_transform=None, pre_filter=None, max_ring_size=6, fold=0, init_method='sum',
                 n_jobs=2):
        self.name = 'CSL'
        self._max_ring_size = max_ring_size
        self._n_jobs = n_jobs
        super(CSLDataset, self).__init__(root, transform, pre_transform, pre_filter,
                                         max_dim=2, cellular=True, init_method=init_method,
                                         num_classes=10)

        assert 0 <= fold <= 4
        self.fold = fold

        self.data, self.slices = self.load_dataset()

        self.num_node_type = 1
        self.num_edge_type = 1

        # These cross-validation splits have been taken from
        # https://github.com/graphdeeplearning/benchmarking-gnns/tree/master/data/CSL
        train_filename = osp.join(self.root, 'splits', 'CSL_train.txt')
        valid_filename = osp.join(self.root, 'splits', 'CSL_val.txt')
        test_filename = osp.join(self.root, 'splits', 'CSL_test.txt')

        self.train_ids = np.loadtxt(train_filename, dtype=int, delimiter=',')[fold].tolist()
        self.val_ids = np.loadtxt(valid_filename, dtype=int, delimiter=',')[fold].tolist()
        self.test_ids = np.loadtxt(test_filename, dtype=int, delimiter=',')[fold].tolist()

        # Make sure the split ratios are as expected (3:1:1)
        assert len(self.train_ids) == 3 * len(self.test_ids)
        assert len(self.val_ids) == len(self.test_ids)
        # Check all splits contain numbers that are smaller than the total number of graphs
        assert max(self.train_ids) < 150
        assert max(self.val_ids) < 150
        assert max(self.test_ids) < 150

    @property
    def raw_file_names(self):
        return ['data.pt']

    @property
    def processed_file_names(self):
        return ['complexes.pt']

    def download(self):
        # Instantiating this will download and process the graph dataset.
        GNNBenchmarkDataset(self.raw_dir, 'CSL')

    def load_dataset(self):
        """Load the dataset from here and process it if it doesn't exist"""
        print("Loading dataset from disk...")
        data, slices = torch.load(self.processed_paths[0])
        return data, slices

    def process(self):
        # At this stage, the graph dataset is already downloaded and processed
        print(f"Processing cell complex dataset for {self.name}")
        # This dataset has no train / val / test splits and we must use cross-validation
        data = GNNBenchmarkDataset(self.raw_dir, 'CSL')
        assert len(data) == 150

        # Check that indeed there are no features
        assert data[0].x is None
        assert data[0].edge_attr is None

        print("Populating graph with features")
        # Initialise everything with zero as in the Benchmarking GNNs code
        # https://github.com/graphdeeplearning/benchmarking-gnns/blob/ef8bd8c7d2c87948bc1bdd44099a52036e715cd0/data/CSL.py#L144
        new_data = []
        for i, datum in enumerate(data):
            edge_index = datum.edge_index
            num_nodes = datum.num_nodes
            # Make sure we have no self-loops in this dataset
            edge_index, _ = remove_self_loops(edge_index)
            num_edges = edge_index.size(1)

            vx = torch.zeros((num_nodes, 1), dtype=torch.long)
            edge_attr = torch.zeros(num_edges, dtype=torch.long)
            setattr(datum, 'edge_index', edge_index)
            setattr(datum, 'x', vx)
            setattr(datum, 'edge_attr', edge_attr)
            new_data.append(datum)

        assert new_data[0].x is not None
        assert new_data[0].edge_attr is not None

        print("Converting the train dataset to a cell complex...")
        complexes, _, _ = convert_graph_dataset_with_rings(
            new_data,
            max_ring_size=self._max_ring_size,
            include_down_adj=False,
            init_edges=True,
            init_rings=False,
            n_jobs=self._n_jobs)

        path = self.processed_paths[0]
        print(f'Saving processed dataset in {path}....')
        torch.save(self.collate(complexes, 2), path)

    @property
    def processed_dir(self):
        """Overwrite to change name based on edges"""
        directory = super(CSLDataset, self).processed_dir
        suffix1 = f"_{self._max_ring_size}rings" if self._cellular else ""
        return directory + suffix1

max_stars_repo_name: crisbodnar/cwn

max_issues_repo_name: crisbodnar/cwn

max_forks_repo_name: crisbodnar/cwn


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Jun  1 08:42:46 2019

@author: Hassan
"""
import math
import numpy as np
import pandas as pd
import time
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import classification_report, accuracy_score, precision_recall_fscore_support
from sklearn.utils import shuffle

# from sklearn.decomposition import PCA
from sklearn.preprocessing import minmax_scale

from imblearn.over_sampling import RandomOverSampler
from imblearn.under_sampling import RandomUnderSampler
# from sklearn.decomposition import PCA

def euclideanDistance(x1, x2):
    return (math.sqrt(sum((x1 - x2)**2)))

#This function returns the indexes of rows in trainingset for k Nearest neighbours of  test instance
def getNeighbors(xTrainSet, xTestInstance, k):
    dists = np.zeros(len(xTrainSet))
    n = np.zeros(k)
    for j in range(len(xTrainSet)):
        d = euclideanDistance(xTrainSet[j], xTestInstance)
        dists[j] = d
    sortedIndexes = np.argsort(dists)
    for i in range(k):
        n[i] = sortedIndexes[i]
    return n

#This function predicts the class of test instance according to majority rule voting by k nearest neighbours class label
def predict(neighbors, y_train):
    count= 0
    for z in range(len(neighbors)):
        if(y_train[np.int(neighbors[z])] == 1):
            count += 1
        else:
            count -= 1
    if(count > 0):
        return 1
    elif(count < 0):
        return 0

class Dataset:
    def __init__(self, datapath):
        self.datapath = datapath
        self.data = []
        self.labels = []

    def read_data(self):
        df = pd.read_csv(self.datapath, header=None)
        num_features = df.shape[1] - 1
        y = df.iloc[:, -1].values
        y[y==-1] = 0
        self.labels = y
        self.data = df.iloc[:, :-1].values

        print(self.labels.shape)
        print(self.data.shape)
        print(np.count_nonzero(self.labels==1),\
              np.count_nonzero(self.labels==0))

    def shuffle_data(self):
        self.data, self.labels = shuffle(self.data, self.labels, random_state=550)

    def normalize_data(self, min_range, max_range):
        self.data = minmax_scale(self.data, feature_range=(min_range, max_range))

    def prepare(self, n_splits=10, normalize=True, shuffle_data=True,\
                                       oversample=True, undersample=False):
        self.read_data()
        if oversample:
            ros = RandomOverSampler(random_state=55)
            self.data, self.labels = ros.fit_resample(self.data, self.labels)
        elif undersample:
            rus = RandomUnderSampler(random_state=55)
            self.data, self.labels = rus.fit_resample(self.data, self.labels)
        if shuffle_data:
            self.shuffle_data()
        if normalize:
            self.normalize_data(0, 1)

        skf = StratifiedKFold(n_splits=n_splits, shuffle=shuffle_data, random_state=43)
        return skf
# read & split data
dataset = Dataset('C:/Users/Hassan/Desktop/EE485_data/GitHub_Project/DATA_Project/Dataset/dataset.txt')
skf = dataset.prepare(n_splits=10, normalize=False, shuffle_data=True,\
                               oversample=True, undersample=False)

# separate test set (%10) using one of the folds
X_test = np.array([])
y_test = np.array([])
test_skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=43)
for train, test in test_skf.split(dataset.data, dataset.labels):
    dataset.data, X_test = dataset.data[train], dataset.data[test]
    dataset.labels, y_test = dataset.labels[train], dataset.labels[test]
    break

start = time.time()
X_train = dataset.data
y_train = dataset.labels

k = 1
fold = 1
cumm_valid_pred = np.array([])
cumm_valid_y = np.array([])

for train_index, test_index in skf.split(dataset.data, dataset.labels):

    X_train, X_valid = dataset.data[train_index], dataset.data[test_index]
    y_train, y_valid = dataset.labels[train_index], dataset.labels[test_index]
    val_predictions = np.zeros(len(X_valid))

    print("\nFold {}:\ttrain_size:{} valid_size:{}".format(fold, len(X_train), len(X_valid)))
    a = np.count_nonzero(y_train)
    b = np.count_nonzero(y_valid)
    print("Class Ratios(+1/-1):\ttrain:{}/{} valid:{}/{}\n".format(a, len(y_train)-a, b, len(y_valid)-b))

    for i in range(len(X_valid)):
        neighbors = getNeighbors(X_train, X_valid[i], k)
        val_predictions[i] = predict(neighbors, y_train)

    print("Validation Size:",len(X_valid))
    a = np.count_nonzero(y_valid)
    print("Class Ratios(+1/-1):\t{}/{}".format(a, len(y_valid)-a))

    p, r, f1, sup = precision_recall_fscore_support(y_valid, val_predictions)
    total_acc = accuracy_score(y_valid, val_predictions)

    cumm_valid_pred = np.hstack((cumm_valid_pred, val_predictions))
    cumm_valid_y = np.hstack((cumm_valid_y, y_valid))

    print("Total accuracy is {0:.4f}".format(total_acc))
    for i in range(2):
        print("Class {} Accuracy: {:.2f}".format(2*i - 1, r[i]))

    fold += 1

print('\n=================')
print('Overall Valid Report:')
print(classification_report(cumm_valid_y, cumm_valid_pred, target_names=['Class -1', 'Class 1']))
print('Overall Valid Accuracy:', accuracy_score(cumm_valid_y, cumm_valid_pred))
print('=================\n')

# test time
test_predictions = np.zeros(len(X_test))
for i in range(len(X_test)):
    neighbors = getNeighbors(X_train, X_test[i], k)
    test_predictions[i] = predict(neighbors, y_train)

print("Test Size:",len(X_test))
a = np.count_nonzero(y_valid)
print("Class Ratios(+1/-1):\t{}/{}".format(a, len(y_test)-a))

p, r, f1, sup = precision_recall_fscore_support(y_test, test_predictions)
total_acc = accuracy_score(y_test, test_predictions)
print("Test accuracy is {0:.4f}".format(total_acc))
for i in range(2):
    print("Class {} Accuracy: {:.2f}".format(2*i - 1, r[i]))

max_issues_repo_name: hsn-rza/phishing-website-detection

max_forks_repo_name: hsn-rza/phishing-website-detection


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Feb 12 11:30:27 2020

@author: ning
"""

import os
from glob import glob

import pandas as pd
import numpy as np
from psychopy.misc import fromFile

from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator

from matplotlib import pyplot as plt
import seaborn as sns
sns.set_style('whitegrid')
sns.set_context('poster')

sub_name = 'patxi'
working_dir = '../results/'
saving_dir = '../result_figures'
if not os.path.exists(os.path.join(saving_dir,sub_name)):
    os.mkdir(os.path.join(saving_dir,sub_name))
psy_data = glob(os.path.join(working_dir,sub_name,'*.psydat'))
working_data = glob(os.path.join(working_dir,sub_name,'*.csv'))

dfs = {}
for f,g in zip(working_data,psy_data):
    df_temp = pd.read_csv(f)
    temp = fromFile(g)
    language = temp.extraInfo['language']
    dfs[language] = df_temp

fig,axes = plt.subplots(figsize = (8,16),
                        nrows = 3,
                        sharey = True,
                        )
for (language),ax in zip(['spanish','basque','mixed'],axes.flatten()[:2]):
    df = dfs[language]
    ax = sns.barplot(x = 'set_size',
                     y = 'just_press.rt',
                     data = df,
                     ax = ax,
                     seed = 12345,
                     )
    ax.set(
           xlabel = 'Set Size',
           ylabel = 'Reaction Time (sec)',
           title = f"language = {language}")

ax = axes.flatten()[-1]
df = dfs['mixed']
gen = TimeseriesGenerator(df['word_type'].values,
                          df['word_type'].values,
                          length = 1,
                          batch_size = 1,
                          )
context = [False]
for (x_0,x_1) in gen:
    context.append(x_0[0][0]!=x_1[0])
context = np.array(context)
df['Switch'] = context
ax = sns.barplot(x = 'word_type',
                 y = 'just_press.rt',
                 hue = 'Switch',
                 hue_order = [True, False],
                 order = ['Spanish','Basque',],
                 data = df,
                 ax = ax,
                 seed = 12345,
                 )
ax.set(xlabel = 'Language',
       ylabel = 'Reaction Time (sec)',
       title = 'Mixed')
fig.tight_layout()

fig.savefig(os.path.join(saving_dir,sub_name,'results.jpeg'),
            dpi = 300,
            bbox_inches = 'tight')


text:
from datetime import datetime, timedelta
from functools import partial
import logging
import os
import pathlib
from typing import Union

import geopandas as gpd
import numpy as np
import pandas as pd
import pyproj
from pyproj import CRS
import pytz
from shapely import ops
from shapely.geometry import Point

from pyschism.dates import localize_datetime

_logger = logging.getLogger(__name__)

class SourceSinkDataset:

    def __init__(self, data):
        self._data = data

    def __len__(self):
        return len(self.data)

    def __str__(self):
        return str(self.data)

    @property
    def elements(self):
        unique_elements = set()
        for elements in self.data.values():
            for element_id in elements.keys():
                unique_elements.add(element_id)
        return list(map(str, sorted(list(map(int, unique_elements)))))

    @property
    def timevector(self):
        return list(sorted(self.data.keys()))

    @property
    def df(self):
        if not hasattr(self, '_df'):
            data = []
            for time, edata in self.data.items():
                for eid, _edata in edata.items():
                    data.append({
                        'time': time,
                        'element_id': eid,
                        **_edata
                    })
            self._df = pd.DataFrame(data)
        return self._df

    @property
    def data(self):
        return self._data

class Sources(SourceSinkDataset):
    pass

class Sinks(SourceSinkDataset):
    pass

class TimeHistoryFile:

    def __init__(self, data, start_date, rnday, filename):
        self.data = data
        self.filename = filename
        self.start_date = start_date
        self.rnday = rnday

    def __str__(self):
        data = []
        for time in self.data.timevector:
            relative_time = (time - self.start_date).total_seconds()
            if relative_time < 0:
                continue
            line = [f"{relative_time:G}"]
            for element_id in self.data.elements:
                line.append(
                    f'{self.data._data[time][element_id]["flow"]:.4e}')
            data.append(' '.join(line))
        return '\n'.join(data)

    def write(self, path: Union[str, os.PathLike], overwrite: bool = False):
        path = pathlib.Path(path)
        if path.exists() and overwrite is not True:
            raise IOError('File exists and overwrite is not True.')
        open(path / self.filename, 'w').write(str(self))

class Vsource(TimeHistoryFile):

    def __init__(self, sources: Sources, start_date, rnday, filename='vsource.th'):
        super().__init__(sources, start_date, rnday, filename)

class Msource(TimeHistoryFile):

    def __init__(self, sources, start_date, rnday, filename='msource.th'):
        super().__init__(sources, start_date, rnday, filename)

    def __str__(self):
        data = []
        for time in self.data.timevector:
            relative_time = (time - self.start_date).total_seconds()
            if relative_time < 0:
                continue
            line = [f"{relative_time:G}"]
            for element_id in self.data.elements:
                line.append(
                    f'{self.data._data[time][element_id]["temperature"]: .4e}')
            for element_id in self.data.elements:
                line.append(
                    f'{self.data._data[time][element_id]["salinity"]: .4e}')
            data.append(' '.join(line))
        return '\n'.join(data)

class Vsink(TimeHistoryFile):

    def __init__(self, sinks: Sinks, start_date, rnday, filename='vsink.th'):
        super().__init__(sinks, start_date, rnday, filename)

class SourceSink:

    def __init__(
            self,
            sources: Sources,
            sinks: Sinks,
            filename='source_sink.in'
    ):
        self.sources = sources
        self.sinks = sinks
        self.filename = filename

    def __str__(self):
        source_id = self.sources.elements
        data = []
        data.append(f'{len(source_id)}')
        for element_id in source_id:
            data.append(f'{element_id}')
        data.append('')
        sink_id = self.sinks.elements
        data.append(f'{len(sink_id)}')
        for element_id in sink_id:
            data.append(f'{element_id}')
        return '\n'.join(data)

    def write(self, path: Union[str, os.PathLike], overwrite: bool = False):
        path = pathlib.Path(path)
        if path.exists() and overwrite is not True:
            raise IOError('File exists and overwrite is not True.')
        open(path / self.filename, 'w').write(str(self))

class Hydrology:

    # TODO: This class is missing a time interpolator.
    _data = {}

    def __init__(self, start_date: datetime = None, rnday: timedelta = None):
        self.start_date = start_date
        self.rnday = rnday

    def __len__(self):
        return len(self._data)

    def add_data(
            self,
            time: datetime,
            element_id: str,
            flow: float,
            temperature: float = np.nan,
            salinity: float = np.nan,
    ):

        time = localize_datetime(time).astimezone(pytz.utc)
        data_for_element = self._data.get(time, {}).get('element_id', {})

        # TODO: What happens if we have two different flows that both are
        # assigned to the same element? Example: 100 m^3/s @ 1 psu then
        # another flow on the same element of 1 m^3/s @ 100 psu. How do we
        # combine these on a single element? Flow is just simple summation,

        _tmp = data_for_element.get('temperature', np.nan)
        if not np.isnan(_tmp):
            if _tmp != temperature:
                raise NotImplementedError(
                    'Two different values of temperature for same '
                    'time/element.')

        _sal = data_for_element.get('salinity', np.nan)
        if not np.isnan(_sal):
            if _sal != salinity:
                raise NotImplementedError(
                    'Two different values of salinity for same time/element.')

        self._data.setdefault(time, {}).setdefault(element_id, {}).update(
            {
                'flow': np.nansum(
                    [data_for_element.get('flow', np.nan), flow]),
                'temperature': temperature,
                'salinity': salinity
            })
        if hasattr(self, '_df'):
            del self._df

    def get_element_timeseries(self, element_id):
        data = {}
        element_data = self.df[(self.df['element_id'] == element_id)]
        for row in element_data.sort_values(by=['time']).itertuples():
            data.setdefault(row.time, {}).update({
                'flow': row.flow,
                'temperature': row.temperature,
                'salinity': row.salinity})
        return data

    # def get_interpolated_timeseries(self, element_id):
    #     f = self.get_element_interpolator(element_id)
    #     self.timevector

    def remove_element_timeseries(self, element_id):
        for time in self._data:
            self._data[time].pop(element_id)
        if hasattr(self, '_df'):
            del self._df

    def aggregate_by_radius(self, hgrid, radius, nprocs=-1):

        # --- Generate aggregation mapping

        # gather extreme values
        source_max = {element_id: -float('inf') for element_id
                      in self.sources.elements}
        for element_data in self.sources.data.values():
            for element_id, data in element_data.items():
                source_max[element_id] = np.max(
                    [source_max[element_id], data['flow']])

        sink_max = {element_id: float('inf') for element_id
                    in self.sinks.elements}
        for element_data in self.sinks.data.values():
            for element_id, data in element_data.items():
                sink_max[element_id] = np.min(
                    [sink_max[element_id], data['flow']])

        aggregate_gdf = []
        for element_id, maxflow in {**source_max, **sink_max}.items():
            element_index = hgrid.elements.get_index_by_id(element_id)
            aggregate_gdf.append({
                'element_id': element_id,
                'geometry': hgrid.elements.gdf.loc[element_index].geometry,
                'maxflow': maxflow,

            })
        aggregate_gdf = gpd.GeoDataFrame(
            aggregate_gdf, crs=hgrid.crs).sort_values(
            by='maxflow', key=abs, ascending=False)

        aggregation_mapping = {}
        for row in aggregate_gdf.itertuples():
            if row.element_id in aggregation_mapping:
                continue
            aggregation_mapping[row.element_id] = row.element_id
            possible_sources = aggregate_gdf.loc[
                aggregate_gdf.index.difference(
                    np.where(aggregate_gdf['element_id'].isin(
                        list(aggregation_mapping))))]
            point = np.array(row.geometry.centroid)
            circle = get_circle_of_radius(point[0], point[1], radius)
            sources_in_circle = possible_sources.loc[
                possible_sources.within(circle)]
            for row_in_circle in sources_in_circle.itertuples():
                aggregation_mapping[row_in_circle.element_id] = row.element_id

        # --- move data from one element to the other
        for current, target in aggregation_mapping.items():
            for time, data in self.get_element_timeseries(current).items():
                self.add_data(time, target, **data)

        for current, target in aggregation_mapping.items():
            if current != target:
                self.remove_element_timeseries(current)

        if hasattr(self, '_sources'):
            del self._sources

        if hasattr(self, '_sinks'):
            del self._sinks

        if hasattr(self, '_df'):
            del self._df

    def write(
            self,
            path: Union[str, os.PathLike],
            overwrite: bool = False,
            msource: Union[str, bool] = True,
            vsource: Union[str, bool] = True,
            vsink: Union[str, bool] = True,
            source_sink: Union[str, bool] = True,
    ):

        # unpack sources, sinks
        sources, sinks = self.sources, self.sinks

        # write source sink
        if source_sink is True:
            fname = 'source_sink.in'
        elif isinstance(source_sink, str):
            fname = source_sink
        if source_sink is not False:
            SourceSink(sources, sinks, fname).write(path, overwrite)

        if vsource is True:
            fname = 'vsource.th'
        elif isinstance(vsource, str):
            fname = vsource
        if vsource is not False:
            Vsource(
                sources,
                self.start_date,
                self.rnday,
                fname
            ).write(path, overwrite)

        if msource is True:
            fname = 'msource.th'
        elif isinstance(msource, str):
            fname = msource
        if msource is not False:
            Msource(
                sources,
                self.start_date,
                self.rnday,
                fname
            ).write(path, overwrite)

        if vsink is True:
            fname = 'vsink.th'
        elif isinstance(vsink, str):
            fname = vsink
        if vsink is not False:
            Vsink(
                sinks,
                self.start_date,
                self.rnday,
                fname
            ).write(path, overwrite)

    @property
    def sources(self):
        if not hasattr(self, '_sources'):
            sources = {}
            for element_id in list(map(str, list(
                    sorted(map(int, self.df.element_id.unique()))))):
                element_data = self.df[(self.df['element_id'] == element_id)]
                flow_data = element_data['flow']
                if np.all(flow_data > 0.):
                    # TODO:  Are irregular timeseries allowed?
                    # if not, we need an interpolator here.
                    for row in element_data.sort_values(
                            by=['time']).itertuples():
                        sources.setdefault(row.time, {}).setdefault(
                            element_id, {}).update({
                                'flow': row.flow,
                                'temperature': row.temperature,
                                'salinity': row.salinity})
                # handle elements that are both sources and sinks
                elif not np.all(flow_data < 0) and np.any(flow_data > 0.):
                    for row in element_data.sort_values(
                            by=['time']).itertuples():
                        flow = row.flow if row.flow >= 0. else 0.
                        sources.setdefault(row.time, {}).setdefault(
                            element_id, {}).update({
                                'flow': flow,
                                'temperature': row.temperature,
                                'salinity': row.salinity})
            self._sources = Sources(sources)
        return self._sources

    @property
    def sinks(self):
        if not hasattr(self, '_sinks'):
            sinks = {}
            for element_id in list(map(str, list(
                    sorted(map(int, self.df.element_id.unique()))))):
                element_data = self.df[(self.df['element_id'] == element_id)]
                flow_data = element_data['flow']
                if np.all(flow_data < 0.):
                    # TODO:  Are irregular timeseries allowed?
                    # if not, we need an interpolator here.
                    for row in element_data.sort_values(
                            by=['time']).itertuples():
                        sinks.setdefault(row.time, {}).setdefault(
                            element_id, {}).update({
                                'flow': row.flow})
                # handle elements that are both sources and sinks
                elif not np.all(flow_data > 0.) and np.any(flow_data < 0.):
                    for row in element_data.sort_values(
                            by=['time']).itertuples():
                        sinks.setdefault(row.time, {}).setdefault(
                            element_id, {}).update(
                                {'flow': row.flow if row.flow <= 0. else 0.})
            self._sinks = Sinks(sinks)
        return self._sinks

    @property
    def start_date(self):
        if self._start_date is None:
            return self.df.time.min()
        return self._start_date

    @start_date.setter
    def start_date(self, start_date):
        self._start_date = start_date
        if start_date is not None:
            self._start_date = localize_datetime(
                start_date).astimezone(pytz.utc)
        return self._start_date

    @property
    def rnday(self):
        if self._rnday is None:
            return self.df.time.max() - self.df.time.min()
        return self._rnday

    @rnday.setter
    def rnday(self, rnday):
        self._rnday = rnday
        if rnday is not None:
            self._rnday = rnday if isinstance(rnday, timedelta) else \
                timedelta(days=rnday)
        return self._rnday

    @property
    def data(self):
        return self._data

    @property
    def df(self):
        if not hasattr(self, '_df'):
            _data = []
            for time, element_data in self._data.items():
                for element_id, data in element_data.items():
                    _data.append({
                        'time': time,
                        'element_id': element_id,
                        **data
                    })
            self._df = pd.DataFrame(_data)
        return self._df

def get_local_azimutal_projection(lon, lat):
    return CRS(f"+proj=aeqd +R=6371000 +units=m +lat_0={lat} +lon_0={lon}")

def get_circle_of_radius(lon, lat, radius):

    local_azimuthal_projection = "+proj=aeqd +R=6371000 +units=m " \
                                 f"+lat_0={lat} +lon_0={lon}"
    wgs84_to_aeqd = partial(
        pyproj.transform,
        pyproj.Proj("+proj=longlat +datum=WGS84 +no_defs"),
        pyproj.Proj(local_azimuthal_projection),
    )
    aeqd_to_wgs84 = partial(
        pyproj.transform,
        pyproj.Proj(local_azimuthal_projection),
        pyproj.Proj("+proj=longlat +datum=WGS84 +no_defs"),
    )

    center = Point(float(lon), float(lat))
    point_transformed = ops.transform(wgs84_to_aeqd, center)

    return ops.transform(aeqd_to_wgs84, point_transformed.buffer(radius))


text:
"""
Test for the feature read/write functions form `squids/feature.py`.
"""

import numpy as np
import tensorflow as tf
from PIL import Image, ImageDraw

from squids.config import IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_CHANNELS
from squids.tfrecords.feature import items_to_features, features_to_items

EPS = tf.constant(0.001, tf.float32)

# -----------------------------------------------------------------------------
# Tests: Image -> Feature -> Image transformars
# -----------------------------------------------------------------------------

def test_items_to_features():
    """Tests the `items_to_features` function."""
    expected_image_id = 123
    expected_image = Image.new(mode="RGB", size=(IMAGE_WIDTH, IMAGE_HEIGHT))
    expected_image_width = IMAGE_WIDTH
    expected_image_height = IMAGE_HEIGHT
    expected_bboxes = [[1.0, 2.0, 3.0, 4.0]]
    expected_segmentations = [[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]]
    expected_mask = Image.new(
        "1",
        (IMAGE_WIDTH, IMAGE_HEIGHT),
        0,
    )
    drawing = ImageDraw.Draw(expected_mask)
    drawing.polygon(
        expected_segmentations[0],
        fill=1,
        outline=1,
    )

    expected_category_ids = [1]
    expected_category_max_id = 2

    expected_features = {
        "image/id": tf.train.Feature(
            int64_list=tf.train.Int64List(value=[expected_image_id])
        ),
        "image/size": tf.train.Feature(
            int64_list=tf.train.Int64List(
                value=[expected_image_width, expected_image_height]
            )
        ),
        "image/data": tf.train.Feature(
            bytes_list=tf.train.BytesList(
                value=[np.array(expected_image).tostring()]
            )
        ),
        "annotations/number": tf.train.Feature(
            int64_list=tf.train.Int64List(value=[1])
        ),
        "bboxes/data": tf.train.Feature(
            float_list=tf.train.FloatList(value=expected_bboxes[0])
        ),
        "masks/data": tf.train.Feature(
            bytes_list=tf.train.BytesList(
                value=[np.array(expected_mask).astype("byte").tostring()]
            )
        ),
        "category/ids": tf.train.Feature(
            int64_list=tf.train.Int64List(value=expected_category_ids)
        ),
        "category/max": tf.train.Feature(
            int64_list=tf.train.Int64List(value=[expected_category_max_id])
        ),
    }

    actual_features = items_to_features(
        expected_image_id,
        expected_image,
        expected_image_width,
        expected_image_height,
        expected_bboxes,
        expected_segmentations,
        expected_category_ids,
        expected_category_max_id,
    )

    assert actual_features == expected_features

def test_features_to_items():
    """Tests the `features_to_items` function."""
    expected_image_id = 123
    expected_image = Image.new(mode="RGB", size=(IMAGE_WIDTH, IMAGE_HEIGHT))
    expected_image_arr = np.array(expected_image)
    expected_bboxes = [
        [1.0, 2.0, 3.0, 4.0],
        [1.0, 2.0, 3.0, 4.0],
        [1.0, 2.0, 3.0, 4.0],
    ]
    expected_segmentations = [
        [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0],
        [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0],
        [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0],
    ]
    expected_masks = [
        Image.new(
            "1",
            (IMAGE_WIDTH, IMAGE_HEIGHT),
            0,
        ),
        Image.new(
            "1",
            (IMAGE_WIDTH, IMAGE_HEIGHT),
            0,
        ),
        Image.new(
            "1",
            (IMAGE_WIDTH, IMAGE_HEIGHT),
            0,
        ),
    ]
    drawing = ImageDraw.Draw(expected_masks[0])
    drawing.polygon(
        expected_segmentations[0],
        fill=1,
        outline=1,
    )
    drawing = ImageDraw.Draw(expected_masks[1])
    drawing.polygon(
        expected_segmentations[1],
        fill=1,
        outline=1,
    )
    drawing = ImageDraw.Draw(expected_masks[2])
    drawing.polygon(
        expected_segmentations[2],
        fill=1,
        outline=1,
    )
    expected_masks = [
        np.array(expected_masks[0]).flatten().astype("byte"),
        np.array(expected_masks[1]).flatten().astype("byte"),
        np.array(expected_masks[2]).flatten().astype("byte"),
    ]
    expected_category_ids = [1, 2, 3]
    expected_category_onehot = [[0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]
    expected_category_max_id = 3

    features = {
        "image/id": tf.constant([expected_image_id], tf.int64),
        "image/size": tf.constant([IMAGE_WIDTH, IMAGE_HEIGHT], tf.int64),
        "image/data": tf.constant(
            [np.array(expected_image).tostring()], tf.string
        ),
        "annotations/number": [3],
        "bboxes/data": tf.constant(
            np.array(expected_bboxes).flatten(), tf.float32
        ),
        "masks/data": tf.constant(
            [
                np.concatenate(
                    (
                        expected_masks[0],
                        expected_masks[1],
                        expected_masks[2],
                    )
                ).tostring()
            ],
            tf.string,
        ),
        "category/ids": expected_category_ids,
        "category/max": [expected_category_max_id],
    }

    (
        actual_image_id,
        actual_image_arr,
        actual_bboxes,
        actual_masks,
        actual_category_onehot,
    ) = features_to_items(features)

    assert actual_image_id == expected_image_id
    assert actual_image_arr.shape == (
        IMAGE_WIDTH,
        IMAGE_HEIGHT,
        IMAGE_CHANNELS,
    )
    assert tf.math.less(
        tf.reduce_sum(tf.abs(actual_image_arr - expected_image_arr)), EPS
    )
    assert actual_image_arr.shape == (
        IMAGE_WIDTH,
        IMAGE_HEIGHT,
        IMAGE_CHANNELS,
    )
    assert tf.math.less(
        tf.reduce_sum(tf.abs(actual_bboxes - expected_bboxes)), EPS
    )
    assert actual_masks.shape == (
        3,
        IMAGE_WIDTH * IMAGE_HEIGHT,
    )
    assert tf.math.less(
        tf.reduce_sum(tf.abs(actual_masks - expected_masks)),
        EPS,
    )
    assert actual_category_onehot.shape == (3, 4)
    assert tf.math.less(
        tf.reduce_sum(
            tf.abs(actual_category_onehot - expected_category_onehot)
        ),
        EPS,
    )

    for expected_num_detecting_objects in [5, 1]:
        (
            actual_image_id,
            actual_image_arr,
            actual_bboxes,
            actual_masks,
            actual_category_onehot,
        ) = features_to_items(
            features, num_detecting_objects=expected_num_detecting_objects
        )
        assert actual_image_arr.shape == (
            IMAGE_WIDTH,
            IMAGE_HEIGHT,
            IMAGE_CHANNELS,
        )

        # The number of detecting objects equal to 1 is a special case.
        # In this case, the 2D array is squeezed to 1D.
        if expected_num_detecting_objects == 1:
            assert actual_bboxes.shape == 4
            assert actual_masks.shape == IMAGE_WIDTH * IMAGE_HEIGHT
            assert actual_category_onehot.shape == 4
        else:
            assert actual_bboxes.shape == (expected_num_detecting_objects, 4)
            assert actual_masks.shape == (
                expected_num_detecting_objects,
                IMAGE_WIDTH * IMAGE_HEIGHT,
            )
            assert actual_category_onehot.shape == (
                expected_num_detecting_objects,
                4,
            )

max_stars_repo_name: mmgalushka/squids

max_issues_repo_name: mmgalushka/squids

max_forks_repo_name: mmgalushka/squids