text:
# -*- coding: utf-8 -*-
"""
@author: belaloui

This script computes the average fitness value for a validation dataset at
each number of generations 'g'. Analyzing the fitness versus the number g
shows when the genetic algorithm starts to overfit the training data.
An optimum generations number 'g_opt' should give the best (lowest) fitness
value for the validation dataset. Therefore, when using this GA for
predictive purposes, one must stop the algorithm after 'g_opt' generations
to get a good model for prediction.
This is not necessary when using the GA only for fitting the model's
parameters, i.e., to estimate the epidemic parameters from epidemic data.
"""
__version__ = '2.0'

# from region_data import Region
from seiqrdp_model.seiqrdp_suite import load_data
from seiqrdp_model.seiqrdp_suite import GeneticFit
from seiqrdp_model.seiqrdp_solver import SEIQRDPSolver
import matplotlib.pyplot as plt
import numpy as np

# Data

region_name = 'Italy'
pop_size = 60483054

data_size = 30
# Cross-validation
train_data_size = 22
validation_data_size = 8
# Genetic algorithm
init_n_gens = 1
max_n_gens = 100
#
n_fits = 64

def main():
    generations = [g for g in range(init_n_gens, max_n_gens+1)]
    fitnesses = [0]*len(generations)
    # Load all dataset
    region = load_data(region_name, pop_size)

    # Cutoff to the desired size (all data)
    region.cutoff_data(data_size)
    print(f'Loaded all (training + validation) dataset for {region.name} '
          f'from {region.first_day} to {region.last_day}.')

    # Taking out the validation data.
    cq_realdata = region.rcQ[-validation_data_size:]

    # Cutoff to the training data only
    region.cutoff_data(train_data_size)

    print(f'Using training dataset for {region.name} '
          f'from {region.first_day} to {region.last_day}.')

    for _ in range(n_fits):
        print(f'Fitting {_+1} of {n_fits}.')
        # First fit (from new random population)
        fit = GeneticFit(region=region, elite_size=10, max_gen=init_n_gens)
        fit.run()
        solution = fit.elite

        # Predicting data points (n=validation_data_size)
        model = SEIQRDPSolver(solution, region)
        model.solve_model(t_i=0, t_f=data_size)
        cq_prediction = model.cQ[-validation_data_size:]

        # first result
        fitnesses[0] += fit.n_rmse(cq_realdata, cq_prediction)

        # Evolve 1 generation at a time, and log the fitness.
        for g in range(1, len(generations)):
            # Evolve for 1 more generation.
            fit.run(resume=True, n_gens=1)
            solution = fit.elite

            # Predicting data points (n=validation_data_size)
            model = SEIQRDPSolver(solution, region)
            model.solve_model(t_i=0, t_f=data_size)
            cq_prediction = model.cQ[-validation_data_size:]

            # Adding the fitness value to the appropriate generation
            # number. The resulting sum over all fitnesses for generation g
            # will be divided by the number of fits later.
            fitnesses[g] += fit.n_rmse(cq_realdata, cq_prediction)

    # Computing the average fitness per generation.
    fitnesses = [f/n_fits for f in fitnesses]

    # Saving data to csv file.
    np.savetxt(fname='fit_vs_gen.csv', X=np.c_[generations, fitnesses],
               fmt=('%d', '%1.5f'),  delimiter=',')

    # Showing and saving plot.
    plt.plot(generations, fitnesses)
    plt.savefig(fname='fit_vs_gen.svg', format='svg')

if __name__ == '__main__':
    main()


text:
from django.shortcuts import render    #,render_to_response
from django.http import HttpResponse,HttpResponseNotFound,Http404
from django.conf import settings
from django.core.files.storage import FileSystemStorage

from HackathonProject.code_need.GanttChart_original import ganttChart
from HackathonProject.code_need.sweetvis_downtime import createSweetviz
from HackathonProject.code_need.TablePreparation import create_ML_Table
# from HackathonProject.code_need.staff_optimation import staff_assig_main
from HackathonProject.code_need.staffAssignmentApplication import staff_assig_main

from HackathonProject.code_need.nurseSchedulingProblem import nspDjango

import pandas as pd
import os
import numpy as np

import plotly.graph_objs as go
import plotly.io as pio

# from HackathonProject.code_need.blogAPP_pyecharts import bar_line,temp_bar
# from HackathonProject.models import IOpf_formaterMonth,Post
# from HackathonProject.code_need.machineLearningModel import temp_machinelearning
# from HackathonProject.code_need.machineSchedulingGanttChart import Hackathon_GanttChart
# from HackathonProject.code_need.CLS_Solver_GA import GA_CLS_slover
# from pyecharts import options as opts
# from pyecharts.charts import Bar, Line
# from pyecharts.commons.utils import JsCode
# from pyecharts.globals import ThemeType

def viewHomepage(request):
    context_homepage = {"ani":True}

    return render(request,"WelcomePage_django.html",context_homepage)

def viewAbout(request):

    context_aboutpage= {"ab":True}

    return render(request,"AboutPage_django.html",context_aboutpage)

def viewComputation(request):
    context = {}

    if request.method=="POST":
        try:
            if request.FILES["document"]:
                uploaded_file = request.FILES["document"]

                print(uploaded_file.name)
                print(uploaded_file.size)
                print(request.POST.get("texA"))
                print(request.POST.get("texB"))
                # print(request.POST.get("texC"))

                fs = FileSystemStorage()
                url = fs.save(uploaded_file.name,uploaded_file)

                file_ = os.path.join(settings.BASE_DIR,r"HackathonProject\media\%s"%url)
                print("Files_Full",file_)
                # df = pd.read_excel(file_)

                context["url"] = fs.url(url)  #bar.render_embed()

                # tem_lst = []
                # tem_lst.append(file_)

                # Data_in,Data_out = IOpf_formaterMonth(tem_lst).concatDataFrame()

                # context["barEcharts"] = bar_line(Data_in,Data_out)

                # context["selectValues"] = ["item1","item2","item3","item4"]

                # context["selectionResult"] = request.POST.get("selection")

                # context["machineLearning"] = temp_machinelearning()

                # context["ganttChart"]  = Hackathon_GanttChart(save2db_path=None).generate_gantt()
                # context["ganttChart"]  = GA_CLS_slover().Not_Main()
                context["ganttChart"] = ganttChart(file_)

                context["sweetViz"] = createSweetviz(file_,outputName = "Rockwell_sweetViz_1.html")

                # context["nurseSchedulingProblem"] = nspDjango().main()[1]
                context["staff_SchedulingOpt"] = staff_assig_main(file_).final_result_table()

                url = None

        except Exception as err:
            raise Http404("Page does not exist, please make sure to input data inside file section.")
            # return HttpResponseNotFound("
Page Not Found!
")

    # return render(request,"upload01.html",context)
    return render(request,"upload01.html",context)

def viewAnimation(request):

    context_ani = {}

    if request.method == 'POST':
        pass

    return render(request,"animationPage_django.html",context_ani)

ext: py

max_stars_repo_name: Xiangyongluo/Hackathon-Project

max_issues_repo_name: Xiangyongluo/Hackathon-Project

max_forks_repo_name: Xiangyongluo/Hackathon-Project

reason: import numpy


text:
'''
Trains a simple convnet to recognise a smile
'''

from __future__ import print_function

from os.path import exists

import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, BatchNormalization
from keras.layers import Conv2D, MaxPooling2D, Activation, SeparableConv2D, GlobalAveragePooling2D
from keras import backend as K
from keras.applications import *
from keras.models import load_model
import keras.backend as K

import numpy as np
import pandas as pd
from PIL import Image

import argparse

from data_utils import load_data_to_labels
from data_utils import generate_data
from data_utils import Plotter

batch_size = 64
num_target_values = 2
epochs = 20
steps_per_epoch = 32

def mean_absolute_error(y_true, y_pred):
    return K.mean(np.absolute(y_true - y_pred))

def create_model(input_shape, all_layers_trainable = False):
    conv_base = Xception(input_shape = input_shape, include_top = False, weights = 'imagenet')
    is_layer_trainable = False
    for layer in conv_base.layers:
        if layer.name == 'block14_sepconv1': # we can start with some other
            is_layer_trainable = True
        else:
            is_layer_trainable = all_layers_trainable

        layer.trainable = is_layer_trainable

    model = Sequential()
    model.add(conv_base)
    model.add(Flatten())
    model.add(Dense(32, activation='relu'))
    model.add(BatchNormalization())
    model.add(Dense(num_target_values, name="prediction"))

    return (model)

def make_all_layers_trainable(mdl, is_trainable = True):
    for layer in mdl.layers:
        layer.trainable = is_trainable

    return (mdl)

def train(patch_size, label_path, image_path, train_all_layers):
    # input image dimensions: TODO get from data or command-line params
    input_shape = (patch_size, patch_size, 3)

    train, test, valid = load_data_to_labels(label_path, train_fraction = 0.7, test_fraction = 0.15)

    train_len = len(train)
    test_len = len(test)
    valid_len = len(valid)
    print('Input data: train_len: ' + str(train_len) + ", test_len: " + str(test_len) + ", valid_len: " + str(valid_len))

    model = None
    if exists("best_student.mdl"):
        print("Found a pre-trained model, so loading that")
        model = load_model("best_student.mdl")
        print("Making all layers trainable")
        model = make_all_layers_trainable(model)
    else:
        print("No pre-trained model found, creating a new one")
        model = create_model(input_shape)

    if train_all_layers:
        model = make_all_layers_trainable(model)

    model.compile(loss=keras.losses.mean_squared_error,
                  optimizer=keras.optimizers.RMSprop(lr=0.01, clipnorm=1),
    #              optimizer=keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True),
                  metrics=['mae'])

    #if exists("student.mdl"):
    #    model.load_weights("student.mdl")

    reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_mean_absolute_error')
    early_stop = keras.callbacks.EarlyStopping(monitor='val_mean_absolute_error', patience=10)
    model_checkpoint = keras.callbacks.ModelCheckpoint("best_student.mdl", save_best_only=True, monitor='val_mean_absolute_error')
    no_nan = keras.callbacks.TerminateOnNaN()
    tb = keras.callbacks.TensorBoard()
    plotter = Plotter(input_shape[0])

    # TODO New callback to do sample inference after each epoch
    model.fit_generator(generate_data(image_path, train, batch_size, patch_size),
                steps_per_epoch=steps_per_epoch,
                epochs=epochs,
                callbacks=[reduce_lr,early_stop,model_checkpoint, no_nan, tb, plotter],
                verbose=1,
                validation_data=generate_data(image_path, valid, batch_size, patch_size),
                validation_steps=valid_len/batch_size)

    model.save("transfer_student.mdl")

    print('Evaluation on latest model:')
    score = model.evaluate_generator(generate_data(image_path, test, batch_size, patch_size), steps=16)
    print('\tTest loss:', score[0])
    print('\tTest MSE:', score[1])

    best_model = keras.models.load_model('best_student.mdl')
    print('Evaluation on best model:')
    score = best_model.evaluate_generator(generate_data(image_path, test, batch_size, patch_size), steps=16)
    print('\tTest loss:', score[0])
    print('\tTest MSE:', score[1])

    num_test_samples = 512
    # evaluating on latest model
    print("\nCalculating error over " + str(num_test_samples) + " test samples... (using latest model)")
    predictions = model.predict_generator(generate_data(image_path, test, num_test_samples, patch_size), steps=1)
    np.save("predictions_latest.npy", predictions)

    # Make it easy to compare predictions to actuals for test data
    test_vs_predict = []
    for i in range(0, num_test_samples):
        sample = {"file":test[i][0],
                  "brightness":test[i][1], "sharpness":test[i][2],
                  "brightness_student":predictions[i][0], "sharpness_student":predictions[i][1]}
        test_vs_predict.append(sample)

    print("Saving errors on " + str(num_test_samples) + " to csv")
    df = pd.DataFrame(test_vs_predict)
    df.to_csv("predictions_vs_test_latest.csv")

    # evaluating on best model
    print("\nCalculating error over " + str(num_test_samples) + " test samples... (using best model)")
    predictions = best_model.predict_generator(generate_data(image_path, test, num_test_samples, patch_size), steps=1)
    np.save("predictions_best.npy", predictions)

    # Make it easy to compare predictions to actuals for test data
    test_vs_predict = []
    for i in range(0, num_test_samples):
        sample = {"file":test[i][0],
                  "brightness":test[i][1], "sharpness":test[i][2],
                  "brightness_student":predictions[i][0], "sharpness_student":predictions[i][1]}
        test_vs_predict.append(sample)

    print("Saving errors on " + str(num_test_samples) + " to csv")
    df = pd.DataFrame(test_vs_predict)
    df.to_csv("predictions_vs_test_best.csv")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--images", help="Path to images to train on", type=str, required=True)
    parser.add_argument("--patchsize", help="Dimensions for input image", type=int, required=False, default=299)
    parser.add_argument("--labels", help="File containing training labels", type=str, default="image_to_smile.json")
    parser.add_argument("--train")
    parser.add_argument("--trainall", action='store_true',help="Train all layers")
    args = parser.parse_args()

    train(args.patchsize, args.labels, args.images, args.trainall)

max_stars_repo_name: EdwardDixon/ml-and-security

max_issues_repo_name: EdwardDixon/ml-and-security

max_forks_repo_name: EdwardDixon/ml-and-security


text:
from matplotlib.finance import quotes_historical_yahoo
import sys
from datetime import date
import matplotlib.pyplot as plt
import numpy as np

today = date.today()
start = (today.year - 1, today.month, today.day)

symbol = 'DISH'

if len(sys.argv) == 2:
   symbol = sys.argv[1]

quotes = quotes_historical_yahoo(symbol, start, today)
quotes = np.array(quotes)
close = quotes.T[4]
volume = quotes.T[5]
ret = np.diff(close)/close[:-1]
volchange = np.diff(volume)/volume[:-1]

fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(ret, volchange, c=ret * 100, s=volchange * 100, alpha=0.5)
ax.set_title('Close and volume returns')
ax.grid(True)

plt.show()


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Date    : Sep-18-21 15:00
# @Author  : Kan HUANG (kan.huang@connect.ust.hk)

import os
import numpy as np
import tensorflow as tf

def parse_cmd_args():
    # This enables a ctr-C without triggering errors
    import signal
    signal.signal(signal.SIGINT, lambda x, y: sys.exit(0))

    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--verbose", action='store_true', help="Verbose mode")
    parser.add_argument("--restore_path", type=str, default='',
                        help="Location of checkpoint to restore")
    parser.add_argument("--inference", action="store_true",
                        help="Use in inference mode")
    parser.add_argument("--logdir", type=str,
                        default='./logs', help="Location to save logs")

    # Dataset hyperparams:
    parser.add_argument("--problem", type=str, default='cifar10',
                        help="Problem (mnist/cifar10/imagenet")
    parser.add_argument("--category", type=str,
                        default='', help="LSUN category")
    parser.add_argument("--data_dir", type=str, default='',
                        help="Location of data")
    parser.add_argument("--dal", type=int, default=1,
                        help="Data augmentation level: 0=None, 1=Standard, 2=Extra")

    # New dataloader params
    parser.add_argument("--fmap", type=int, default=1,
                        help="# Threads for parallel file reading")
    parser.add_argument("--pmap", type=int, default=16,
                        help="# Threads for parallel map")

    # Optimization hyperparams:
    parser.add_argument("--n_train", type=int,
                        default=50000, help="Train epoch size")
    parser.add_argument("--n_test", type=int, default=-
                        1, help="Valid epoch size")
    parser.add_argument("--n_batch_train", type=int,
                        default=64, help="Minibatch size")
    parser.add_argument("--n_batch_test", type=int,
                        default=50, help="Minibatch size")
    parser.add_argument("--n_batch_init", type=int, default=256,
                        help="Minibatch size for data-dependent init")
    parser.add_argument("--optimizer", type=str,
                        default="adamax", help="adam or adamax")
    parser.add_argument("--lr", type=float, default=0.001,
                        help="Base learning rate")
    parser.add_argument("--beta1", type=float, default=.9, help="Adam beta1")
    parser.add_argument("--polyak_epochs", type=float, default=1,
                        help="Nr of averaging epochs for Polyak and beta2")
    parser.add_argument("--weight_decay", type=float, default=1.,
                        help="Weight decay. Switched off by default.")
    parser.add_argument("--epochs", type=int, default=1000000,
                        help="Total number of training epochs")
    parser.add_argument("--epochs_warmup", type=int,
                        default=10, help="Warmup epochs")
    parser.add_argument("--epochs_full_valid", type=int,
                        default=50, help="Epochs between valid")
    parser.add_argument("--gradient_checkpointing", type=int,
                        default=1, help="Use memory saving gradients")

    # Model hyperparams:
    parser.add_argument("--image_size", type=int,
                        default=-1, help="Image size")
    parser.add_argument("--anchor_size", type=int, default=32,
                        help="Anchor size for deciding batch size")
    parser.add_argument("--width", type=int, default=512,
                        help="Width of hidden layers")
    parser.add_argument("--depth", type=int, default=32,
                        help="Depth of network")
    parser.add_argument("--weight_y", type=float, default=0.00,
                        help="Weight of log p(y|x) in weighted loss")
    parser.add_argument("--n_bits_x", type=int, default=8,
                        help="Number of bits of x")
    parser.add_argument("--n_levels", type=int, default=3,
                        help="Number of levels")

    # Synthesis/Sampling hyperparameters:
    parser.add_argument("--n_sample", type=int, default=1,
                        help="minibatch size for sample")
    parser.add_argument("--epochs_full_sample", type=int,
                        default=50, help="Epochs between full scale sample")

    # Ablation
    parser.add_argument("--learntop", action="store_true",
                        help="Learn spatial prior")
    parser.add_argument("--ycond", action="store_true",
                        help="Use y conditioning")
    parser.add_argument("--seed", type=int, default=0, help="Random seed")
    parser.add_argument("--flow_permutation", type=int, default=2,
                        help="Type of flow. 0=reverse (realnvp), 1=shuffle, 2=invconv (ours)")
    parser.add_argument("--flow_coupling", type=int, default=0,
                        help="Coupling type: 0=additive, 1=affine")

    hparams = parser.parse_args()  # So error if typo

    return hparams

def get_its(hparams):
    """Get number of training and validation iterations
    """
    # These run for a fixed amount of time. As anchored batch is smaller, we've actually seen fewer examples
    train_its = int(np.ceil(hparams.n_train / hparams.n_batch_train))
    test_its = int(np.ceil(hparams.n_test / hparams.n_batch_train))
    train_epoch = train_its * hparams.n_batch_train

    # Do a full validation run
    full_test_its = hparams.n_test // hparams.local_batch_test

    return train_its, test_its, full_test_its

def get_data(hparams, sess):
    if hparams.image_size == -1:
        hparams.image_size = {'mnist': 32, 'cifar10': 32, 'imagenet-oord': 64,
                              'imagenet': 256, 'celeba': 256, 'lsun_realnvp': 64, 'lsun': 256}[hparams.problem]
    if hparams.n_test == -1:
        hparams.n_test = {'mnist': 10000, 'cifar10': 10000,
                          'imagenet-oord': 50000, 'imagenet': 50000}[hparams.problem]
    hparams.n_y = {'mnist': 10, 'cifar10': 10, 'imagenet-oord': 1000,
                   'imagenet': 1000, 'celeba': 1, 'lsun_realnvp': 1, 'lsun': 1}[hparams.problem]

    # Use anchor_size to rescale batch size based on image_size
    s = hparams.anchor_size
    hparams.local_batch_train = hparams.n_batch_train * \
        s * s // (hparams.image_size * hparams.image_size)
    hparams.local_batch_test = {64: 50, 32: 25, 16: 10, 8: 5, 4: 2, 2: 2, 1: 1}[
        hparams.local_batch_train]  # round down to closest divisor of 50
    hparams.local_batch_init = hparams.n_batch_init * \
        s * s // (hparams.image_size * hparams.image_size)

    # datasets which's loading don't need sess
    if hparams.problem in ['mnist', 'cifar10']:
        hparams.direct_iterator = False
        import datasets.get_mnist_cifar as v

        train_iterator, test_iterator, data_init = \
            v.get_data(hparams.problem, hparams.dal, hparams.local_batch_train,
                       hparams.local_batch_test, hparams.local_batch_init, hparams.image_size)

    else:
        raise Exception()

    return train_iterator, test_iterator, data_init

def tensorflow_session():
    """Create tensorflow session with horovod
    """
    # Init session and params
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    # Pin GPU to local rank (one GPU per process)
    # config.gpu_options.visible_device_list = str(hvd.local_rank())
    sess = tf.Session(config=config)
    return sess

def main(hparams):
    # Create tensorflow session
    sess = tensorflow_session()

    # Download and load dataset.
    tf.set_random_seed(hparams.seed)
    np.random.seed(hparams.seed)

    # Get data and set train_its and valid_its
    train_iterator, test_iterator, data_init = get_data(hparams, sess)
    hparams.train_its, hparams.test_its, hparams.full_test_its = get_its(
        hparams)

    # Create log dir
    logdir = os.path.abspath(hparams.logdir) + "/"
    if not os.path.exists(logdir):
        os.mkdir(logdir)

    # Create model
    import models
    model = models.model(sess, hparams, train_iterator,
                         test_iterator, data_init)

    # Initialize visualization functions
    visualise = init_visualizations(hps, model, logdir)

    if not hparams.inference:
        # Perform training
        train(sess, model, hparams, logdir, visualise)
    else:
        infer(sess, model, hparams, test_iterator)

def train(sess, model, hps, logdir, visualise):
    _print(hps)
    _print('Starting training. Logging to', logdir)
    _print('epoch n_processed n_images ips dtrain dtest dsample dtot train_results test_results msg')

    # Train
    sess.graph.finalize()
    n_processed = 0
    n_images = 0
    train_time = 0.0
    test_loss_best = 999999

    if hvd.rank() == 0:
        train_logger = ResultLogger(logdir + "train.txt", **hps.__dict__)
        test_logger = ResultLogger(logdir + "test.txt", **hps.__dict__)

    tcurr = time.time()
    for epoch in tqdm(range(1, hps.epochs)):

        t = time.time()

        train_results = []
        for it in tqdm(range(hps.train_its)):

            # Set learning rate, linearly annealed from 0 in the first hps.epochs_warmup epochs.
            lr = hps.lr * min(1., n_processed /
                              (hps.n_train * hps.epochs_warmup))

            # Run a training step synchronously.
            _t = time.time()
            train_results += [model.train(lr)]  # 这里写train函数model.train
            if hps.verbose and hvd.rank() == 0:
                _print(n_processed, time.time()-_t, train_results[-1])
                sys.stdout.flush()

            # Images seen wrt anchor resolution
            n_processed += hvd.size() * hps.n_batch_train
            # Actual images seen at current resolution
            n_images += hvd.size() * hps.local_batch_train

        train_results = np.mean(np.asarray(train_results), axis=0)

        dtrain = time.time() - t
        ips = (hps.train_its * hvd.size() * hps.local_batch_train) / dtrain
        train_time += dtrain

        if hvd.rank() == 0:
            train_logger.log(epoch=epoch, n_processed=n_processed, n_images=n_images, train_time=int(
                train_time), **process_results(train_results))

        if epoch < 10 or (epoch < 50 and epoch % 10 == 0) or epoch % hps.epochs_full_valid == 0:
            test_results = []
            msg = ''

            t = time.time()
            # model.polyak_swap()

            if epoch % hps.epochs_full_valid == 0:
                # Full validation run
                for it in range(hps.full_test_its):
                    test_results += [model.test()]
                test_results = np.mean(np.asarray(test_results), axis=0)

                if hvd.rank() == 0:
                    test_logger.log(epoch=epoch, n_processed=n_processed,
                                    n_images=n_images, **process_results(test_results))

                    # Save checkpoint
                    if test_results[0] < test_loss_best:
                        test_loss_best = test_results[0]
                        model.save(logdir+f"model_best_loss_epoch{epoch}.ckpt")
                        msg += ' *'

            dtest = time.time() - t

            # Sample
            t = time.time()
            if epoch == 1 or epoch == 10 or epoch % hps.epochs_full_sample == 0:
                visualise(epoch)
            dsample = time.time() - t

            if hvd.rank() == 0:
                dcurr = time.time() - tcurr
                tcurr = time.time()
                _print(epoch, n_processed, n_images, "{:.1f} {:.1f} {:.1f} {:.1f} {:.1f}".format(
                    ips, dtrain, dtest, dsample, dcurr), train_results, test_results, msg)

            # model.polyak_swap()

    if hvd.rank() == 0:
        _print("Finished!")

if __name__ == "__main__":
    hparams = parse_cmd_args()
    main(hparams)


text:
from argparse import ArgumentParser
from pathlib import Path

from coolname import generate_slug
from sagemaker.pytorch import PyTorch

from syne_tune.backend.sagemaker_backend.sagemaker_utils import get_execution_role
import syne_tune
import benchmarking
from syne_tune.util import s3_experiment_path

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument("--experiment_tag", type=str, required=False, default=generate_slug(2))
    args, _ = parser.parse_known_args()
    experiment_tag = args.experiment_tag
    print(experiment_tag)
    est = PyTorch(
        entry_point="benchmark_main.py",
        source_dir=str(Path(__file__).parent),
        # instance_type="local",
        checkpoint_s3_uri=s3_experiment_path(tuner_name=experiment_tag),
        instance_type="ml.c5.4xlarge",
        instance_count=1,
        py_version="py3",
        framework_version='1.6',
        role=get_execution_role(),
        dependencies=syne_tune.__path__ + benchmarking.__path__,
        disable_profiler=True,
        hyperparameters={"experiment_tag": experiment_tag, 'num_seeds': 30},
    )
    est.fit(job_name=experiment_tag)

max_stars_repo_name: ondrejbohdal/syne-tune

max_issues_repo_name: ondrejbohdal/syne-tune

max_forks_repo_name: talesa/syne-tune

reason: from sage


text:
import os
import itertools
import json
import tempfile
import numpy as np
import tensorflow as tf
import blocksparse as bs
import time
import subprocess
from mpi_utils import mpi_rank

def logger(log_prefix):
    'Prints the arguments out to stdout, .txt, and .jsonl files'

    jsonl_path = f'{log_prefix}.jsonl'
    txt_path = f'{log_prefix}.txt'

    def log(*args, pprint=False, **kwargs):
        if mpi_rank() != 0:
            return
        t = time.ctime()
        argdict = {'time': t}
        if len(args) > 0:
            argdict['message'] = ' '.join([str(x) for x in args])
        argdict.update(kwargs)

        txt_str = []
        args_iter = sorted(argdict) if pprint else argdict
        for k in args_iter:
            val = argdict[k]
            if isinstance(val, np.ndarray):
                val = val.tolist()
            elif isinstance(val, np.integer):
                val = int(val)
            elif isinstance(val, np.floating):
                val = float(val)
            argdict[k] = val
            if isinstance(val, float):
                if k == 'lr':
                    val = f'{val:.6f}'
                else:
                    val = f'{val:.4f}'
            txt_str.append(f'{k}: {val}')
        txt_str = ', '.join(txt_str)

        if pprint:
            json_str = json.dumps(argdict, sort_keys=True)
            txt_str = json.dumps(argdict, sort_keys=True, indent=4)
        else:
            json_str = json.dumps(argdict)

        print(txt_str, flush=True)

        with open(txt_path, "a+") as f:
            print(txt_str, file=f, flush=True)
        with open(jsonl_path, "a+") as f:
            print(json_str, file=f, flush=True)

    return log

def go_over(choices):
    return itertools.product(*[range(n) for n in choices])

def get_git_revision():
    git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'])
    return git_hash.strip().decode('utf-8')

def shape_list(x):
    """
    deal with dynamic shape in tensorflow cleanly
    """
    ps = x.get_shape().as_list()
    ts = tf.shape(x)
    return [ts[i] if ps[i] is None else ps[i] for i in range(len(ps))]

def rsync_data(from_path, to_path):
    subprocess.check_output(['rsync', '-r', from_path, to_path,
                             '--update'])

def maybe_download(path):
    '''If a path is a gsutil path, download it and return the local link,
    otherwise return link'''
    if not path.startswith('gs://'):
        return path
    local_dest = tempfile.mkstemp()[1]
    subprocess.check_output(['gsutil', '-m', 'cp', path, local_dest])
    return local_dest

def upload_to_gcp(from_path, to_path, is_async=False):
    if is_async:
        cmd = f'bash -exec -c "gsutil -m rsync -r {from_path} {to_path}"&'
        subprocess.call(cmd, shell=True, stderr=subprocess.DEVNULL)
    else:
        subprocess.check_output(['gsutil', '-m', 'rsync', from_path, to_path])

def check_identical(from_path, to_path):
    try:
        subprocess.check_output(['git', 'diff', '--no-index', '--quiet',
                                 from_path, to_path])
        return True
    except subprocess.CalledProcessError:
        return False

def wait_until_synced(from_path, to_path):
    while True:
        if check_identical(from_path, to_path):
            break
        else:
            time.sleep(5)

def is_gcp():
    try:
        subprocess.check_output(['curl', '-s',
                                 'metadata.google.internal', '-i'])
        return True
    except subprocess.CalledProcessError:
        return False

def backup_files(save_dir, save_dir_gcp, path=None):
    if mpi_rank() == 0:
        if not path:
            print(f'Backing up {save_dir} to {save_dir_gcp}',
                  'Will execute silently in another thread')
            upload_to_gcp(save_dir, save_dir_gcp, is_async=True)
        else:
            upload_to_gcp(path, save_dir_gcp, is_async=True)

def log_gradient_values(grads, variables, global_step, model_dir):
    loggrads = []
    with tf.name_scope("log_gradient_values"):
        for i, (grad, param) in enumerate(zip(grads, variables)):
            name = param.op.name + "_" + "_".join(
                str(x) for x in param.shape.as_list())
            loggrads.append(bs.log_stats(
                grad, step=global_step, name=name,
                logfile=os.path.join(model_dir, 'grad_stats.txt')))
    return loggrads

def tf_print(t, name, summarize=10, first_n=None, mv=False, maxmin=False):
    # Useful for debugging!
    axes = [i for i in range(len(t.shape))]
    if mv:
        m, v = tf.nn.moments(t, axes=axes)
    if maxmin:
        maxi = tf.reduce_max(t)
        mini = tf.reduce_min(t)
    prefix = f'{tf.get_variable_scope().name}-{name}'
    with tf.device('/cpu:0'):
        if mv:
            t = tf.Print(t, [tf.shape(t), m, v], prefix,
                         summarize=summarize, first_n=first_n)
        elif maxmin:
            t = tf.Print(t, [tf.shape(t), mini, maxi, t], prefix,
                         summarize=summarize, first_n=first_n)
        else:
            t = tf.Print(t, [tf.shape(t), t], prefix,
                         summarize=summarize, first_n=first_n)
    return t

def get_variables(trainable=False):
    if trainable:
        variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
    else:
        variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    return variables

def load_variables(sess, weights, ignore=None, trainable=False, ema=True):
    '''ema refers to whether the exponential moving averaged weights are used to
    initialize the true weights or not.'''
    weights = {os.path.normpath(key): value for key, value in weights.items()}
    ops = []
    feed_dict = {}
    if ema:
        gvs_map = {v.name: v for v in tf.global_variables()}
    for i, var in enumerate(get_variables(trainable=trainable)):
        var_name = os.path.normpath(var.name)
        if ignore:
            do_not_load = False
            for ignore_substr in ignore:
                if ignore_substr in var_name:
                    do_not_load = True
            if do_not_load:
                continue
        ph = tf.placeholder(dtype=var.dtype, shape=var.shape)
        ops.append(var.assign(ph))
        if ema:
            ema_name = f'{var_name[:-2]}/Ema/ema:0'
            # We assign the EMA value to the current value
            try:
                feed_dict[ph] = weights[ema_name]
            except KeyError:
                print(f'warning: ema var not found for {var_name}')
                feed_dict[ph] = weights[var_name]
            # We also assign the EMA value to the current EMA, which will otherwise
            # use the initialized value of the variable (random)
            ema_var = gvs_map[ema_name]
            ph = tf.placeholder(dtype=ema_var.dtype, shape=ema_var.shape)
            ops.append(ema_var.assign(ph))
            feed_dict[ph] = weights[ema_name]
        else:
            feed_dict[ph] = weights[var_name]
    sess.run(ops, feed_dict)

def save_params(sess, path):
    if mpi_rank() == 0:
        tf_vars = dict(zip([var.name for var in get_variables()],
                           sess.run(get_variables())))
        np.savez(path + '.npz', **tf_vars)

def load_variables_from_file(sess, path, ignore=None, trainable=False, ema=True):
    weights = dict(np.load(path))
    load_variables(sess, weights, ignore, trainable=trainable, ema=ema)


text:
import pandas as pd
import numpy as np

data_X = pd.read_csv('X.csv', index_col=0, header=0)
data_y = pd.read_csv('y.csv', index_col=0, header=0)
data_X.to_csv(r'X.txt', index=False, header=False, sep=' ')
data_y.to_csv(r'y.txt', index=False, header=False, sep=' ')


text:
from collections import namedtuple

import numpy as np
from numpy.testing import assert_equal
from numpy.testing.decorators import skipif
from skimage import data
from skimage.viewer import ImageViewer, viewer_available
from skimage.viewer.canvastools import (
    LineTool, ThickLineTool, RectangleTool, PaintTool)
from skimage.viewer.canvastools.base import CanvasToolBase
from numpy.testing import assert_equal
from numpy.testing.decorators import skipif

def get_end_points(image):
    h, w = image.shape[0:2]
    x = [w / 3, 2 * w / 3]
    y = [h / 2] * 2
    return np.transpose([x, y])

def create_mouse_event(ax, button=1, xdata=0, ydata=0, key=None):
    """
     *name*
        the event name

    *canvas*
        the FigureCanvas instance generating the event

    *guiEvent*
        the GUI event that triggered the matplotlib event

    *x*
        x position - pixels from left of canvas

    *y*
        y position - pixels from bottom of canvas

    *inaxes*
        the :class:`~matplotlib.axes.Axes` instance if mouse is over axes

    *xdata*
        x coord of mouse in data coords

    *ydata*
        y coord of mouse in data coords

     *button*
        button pressed None, 1, 2, 3, 'up', 'down' (up and down are used
        for scroll events)

    *key*
        the key depressed when the mouse event triggered (see
        :class:`KeyEvent`)

    *step*
        number of scroll steps (positive for 'up', negative for 'down')
    """
    event = namedtuple('Event',
                       ('name canvas guiEvent x y inaxes xdata ydata '
                        'button key step'))
    event.button = button
    event.x, event.y = ax.transData.transform((xdata, ydata))
    event.xdata, event.ydata = xdata, ydata
    event.inaxes = ax
    event.canvas = ax.figure.canvas
    event.key = key
    event.step = 1
    event.guiEvent = None
    event.name = 'Custom'
    return event

@skipif(not viewer_available)
def test_line_tool():
    img = data.camera()
    viewer = ImageViewer(img)

    tool = LineTool(viewer.ax, maxdist=10)
    tool.end_points = get_end_points(img)
    assert_equal(tool.end_points, np.array([[170, 256], [341, 256]]))

    # grab a handle and move it
    grab = create_mouse_event(viewer.ax, xdata=170, ydata=256)
    tool.on_mouse_press(grab)
    move = create_mouse_event(viewer.ax, xdata=180, ydata=260)
    tool.on_move(move)
    tool.on_mouse_release(move)
    assert_equal(tool.geometry, np.array([[180, 260], [341, 256]]))

    # create a new line
    new = create_mouse_event(viewer.ax, xdata=10, ydata=10)
    tool.on_mouse_press(new)
    move = create_mouse_event(viewer.ax, xdata=100, ydata=100)
    tool.on_move(move)
    tool.on_mouse_release(move)
    assert_equal(tool.geometry, np.array([[100, 100], [10, 10]]))

@skipif(not viewer_available)
def test_thick_line_tool():
    img = data.camera()
    viewer = ImageViewer(img)

    tool = ThickLineTool(viewer.ax, maxdist=10)
    tool.end_points = get_end_points(img)

    scroll_up = create_mouse_event(viewer.ax, button='up')
    tool.on_scroll(scroll_up)
    assert_equal(tool.linewidth, 2)

    scroll_down = create_mouse_event(viewer.ax, button='down')
    tool.on_scroll(scroll_down)
    assert_equal(tool.linewidth, 1)

    key_up = create_mouse_event(viewer.ax, key='+')
    tool.on_key_press(key_up)
    assert_equal(tool.linewidth, 2)

    key_down = create_mouse_event(viewer.ax, key='-')
    tool.on_key_press(key_down)
    assert_equal(tool.linewidth, 1)

@skipif(not viewer_available)
def test_rect_tool():
    img = data.camera()
    viewer = ImageViewer(img)

    tool = RectangleTool(viewer.ax, maxdist=10)
    tool.extents = (100, 150, 100, 150)

    assert_equal(tool.corners,
                 ((100, 150, 150, 100), (100, 100, 150, 150)))
    assert_equal(tool.extents, (100, 150, 100, 150))
    assert_equal(tool.edge_centers,
                 ((100, 125.0, 150, 125.0), (125.0, 100, 125.0, 150)))
    assert_equal(tool.geometry, (100, 150, 100, 150))

    # grab a corner and move it
    grab = create_mouse_event(viewer.ax, xdata=100, ydata=100)
    tool.press(grab)
    move = create_mouse_event(viewer.ax, xdata=120, ydata=120)
    tool.onmove(move)
    tool.release(move)
    assert_equal(tool.geometry, [120, 150, 120, 150])

    # create a new line
    new = create_mouse_event(viewer.ax, xdata=10, ydata=10)
    tool.press(new)
    move = create_mouse_event(viewer.ax, xdata=100, ydata=100)
    tool.onmove(move)
    tool.release(move)
    assert_equal(tool.geometry, [10, 100,  10, 100])

@skipif(not viewer_available)
def test_paint_tool():
    img = data.moon()
    viewer = ImageViewer(img)

    tool = PaintTool(viewer.ax, img.shape)

    tool.radius = 10
    assert_equal(tool.radius, 10)
    tool.label = 2
    assert_equal(tool.label, 2)
    assert_equal(tool.shape, img.shape)

    start = create_mouse_event(viewer.ax, xdata=100, ydata=100)
    tool.on_mouse_press(start)
    move = create_mouse_event(viewer.ax, xdata=110, ydata=110)
    tool.on_move(move)
    tool.on_mouse_release(move)
    assert_equal(tool.overlay[tool.overlay == 2].size, 761)

    tool.label = 5
    start = create_mouse_event(viewer.ax, xdata=20, ydata=20)
    tool.on_mouse_press(start)
    move = create_mouse_event(viewer.ax, xdata=40, ydata=40)
    tool.on_move(move)
    tool.on_mouse_release(move)
    assert_equal(tool.overlay[tool.overlay == 5].size, 881)
    assert_equal(tool.overlay[tool.overlay == 2].size, 761)

    enter = create_mouse_event(viewer.ax, key='enter')
    tool.on_mouse_press(enter)

    tool.overlay = tool.overlay * 0
    assert_equal(tool.overlay.sum(), 0)

@skipif(not viewer_available)
def test_base_tool():
    img = data.moon()
    viewer = ImageViewer(img)

    tool = CanvasToolBase(viewer.ax)
    tool.set_visible(False)
    tool.set_visible(True)

    enter = create_mouse_event(viewer.ax, key='enter')
    tool._on_key_press(enter)

    tool.redraw()
    tool.remove()

    tool = CanvasToolBase(viewer.ax, useblit=False)
    tool.redraw()

max_stars_repo_name: matthew-brett/scikit-image

max_issues_repo_name: matthew-brett/scikit-image

max_forks_repo_name: matthew-brett/scikit-image

reason: import numpy,from numpy


text:
""" Demons registration algorithm
"""

import numpy as np
import scipy.ndimage

from .. import Aarray, diffuse2
from .reg_base import GDGRegistration, BaseRegistration, create_grid_image

class BaseDemonsRegistration(object):
    """ BaseDemonsRegistration

    Abstract class that implements the base functionality of the
    Demons algorithm.

    """

    def _get_derivative(self, im, d, o=1, edgeMode='constant'):
        """ _get_derivative(im, d, o=1)

        Calculate the derivative (of order o) of the given image
        in the given dimension.

        """

        # Set edgeMode to constant, because when the image is deformed,
        # its (deformed) edge will give rise to high filter response any way.

        # We can apply differentiation using compact support kernels
        # because we use a scale space pyramid based on discrete
        # diffusion kernels as proposed by Tony Lindeberg. The
        # resulting differentiation is theoretically valid.
        # (Of course, for low scales, the results suffer from discretisation
        # errors, which is why the scale should be at least 0.5/1.0.)

        if o == 0:
            return im # No differentiation
        elif o == 1:
            k = np.array([0.5, 0, -0.5], dtype='float64')
        elif o == 2:
            k = np.array([1, -2, 1], dtype='float64')
        else:
            raise ValueError('Order of differentiation must be {0,1,2}.')

        # For o in [1,2]
        tmp = scipy.ndimage.convolve1d(im, k, d, mode=edgeMode)
        return Aarray(tmp, im.sampling)

    def _get_image_and_gradient(self, image_id, iterInfo):
        """ _get_image_and_gradient(image_id, iterInfo)

        Get the image and the gradient for the given image id.
        Returns a tuple (mass, (gradz, grady, gradx))

        """
        scale = iterInfo[2]

        # Use buffered?
        buffered = self._get_buffered_data(image_id, iterInfo)
        if buffered is not None:
            im = buffered

        else:
            # Get deformed image
            im = self.get_deformed_image(image_id, scale)

            # Buffer
            self._set_buffered_data(image_id, iterInfo, im)

        # Calculate gradient of image.
        gradient = []
        for d in range(im.ndim):
            tmp = self._get_derivative(im, d, 1, 'nearest')
            gradient.append( tmp )

        # Done
        return im, tuple(gradient)

    def _visualize(self, gridStep=10):
        """ _visualize(self)

        Visualize the registration process.

        """

        if self.visualizer.fig is None:
            return

        import visvis as vv

        firstpass = not self.visualizer.fig.children

        if True:
            # Show

            # Apply deformation to image
            im1 = self.get_deformed_image(0, 0)
            im2 = self.get_deformed_image(1, 0)

            # Get grid images
            grid1 = create_grid_image(im1.shape, im1.sampling, gridStep)
            grid2 = grid1.copy()
            deform1, deform2 = self._deforms.get(0, None), self._deforms.get(1, None)
            if deform1 and deform2:
                grid1 = deform1.apply_deformation(grid1)# + 0.1*self._deforms[0][0])
                grid2 = deform2.apply_deformation(grid2)# + 0.1*self._deforms[1][0])

            # Get images
            ims = [ None, im1, grid1,
                    None, im2, grid2]

            # Update textures
            for i in range(len(ims)):
                if ims[i] is not None:
                    self.visualizer.imshow((2,3,i+1), ims[i])

        if firstpass:
            # Init

            # Show originals
            self.visualizer.imshow(231, self._ims[0])
            self.visualizer.imshow(234, self._ims[1])

            # Show titles
            title_map = [   'im 1', 'deformed 1', 'grid 1',
                            'im 2', 'deformed 2', 'grid 2']
            for i in range(len(title_map)):
                a = vv.subplot(2,3,i+1)
                vv.title(title_map[i])
                a.axis.visible = False

        # Draw now
        self.visualizer.fig.DrawNow()

    def _deform_from_image_pair(self, i, j, iterInfo):
        """ _deform_from_image_pair(i, j, iterInfo)

        Calculate the deform for image i to image j.

        """

        # Extract iter info
        level, iter, scale = iterInfo

        # Try using buffered data
        # we can make good use of the fact that our delta deforms are symetric
        buffered = self._get_buffered_data((i,j), iterInfo)
        if buffered is not None:
            return buffered
        buffered = self._get_buffered_data((j,i), iterInfo)
        if buffered is not None:
            for grid in buffered:
                grid._knots = - grid._knots
            return buffered

        # Get images and their gradients
        self.timer.start('getting images')
        im1, grad1 = self._get_image_and_gradient(i, iterInfo)
        im2, grad2 = self._get_image_and_gradient(j, iterInfo)
        self.timer.stop('getting images')

#         # Prevent too high scales
#         if min(im1.shape) < 16:
#             return None

        self.timer.start('calculating vectors')

        # Calculate norms
        norm1 = im1 * 0.0 # copy
        norm2 = im2 * 0.0
        for d in range(im1.ndim):
            norm1 += grad1[d]**2
            norm2 += grad2[d]**2

        # Calculate denumerators
        imd = im1-im2
        imd2 = imd**2
        alpha = float(self.params.noise_factor)
        denum1 = norm1 + alpha**2 * imd2
        denum2 = norm2 + alpha**2 * imd2
        del norm1, norm2, imd2

        # Make sure the division doesnt cause us problems
        denum1[denum1==0] = np.inf
        denum2[denum2==0] = np.inf

        # Find deformation; use the gradient of both images.
        speed_factor = float(self.params.speed_factor)
        if not self.forward_mapping:
            speed_factor *= -1
        dd_ = []
        for d in range(imd.ndim):
            tmp = imd * ( grad1[d]/denum1 + grad2[d]/denum2 ) * speed_factor
            dd_.append(tmp)

        self.timer.stop('calculating vectors')

        if isinstance(self, GDGRegistration):

            # Regularize using a B-spline grid
            deformForce = self.DeformationField(*dd_)
            deform = self._regularize_diffeomorphic(scale, deformForce)

        else:

            # Regularize deformation field using diffusion
            self.timer.start('regularizing')
            for d in range(imd.ndim):
                # Get sigma
                final_scale = float(self.params.final_scale)
                final_smoothing = float(self.params.final_smoothing)
                sigma_reg = scale * final_smoothing / final_scale
                # Diffuse
                dd_[d] = diffuse2(dd_[d], sigma_reg)
                dd_[d] = dd_[d].astype('float32')
            self.timer.stop('regularizing')

            # Make deform
            deform = self.DeformationField(*dd_)

        # Show
        gridSize = 10
        if hasattr(self, '_get_grid_sampling'):
            gridSize = self._get_grid_sampling(scale)
        if i==0 and j==1:
            self._visualize(gridSize)
        elif self.params.deform_wise.lower() == 'pairwise2' and i==1:
            self._visualize(gridSize)

        # Buffer B-spline grid and return
        self._set_buffered_data((i,j), iterInfo, deform)
        return deform

class OriginalDemonsRegistration(BaseRegistration, BaseDemonsRegistration):
    """ OriginalDemonsRegistration(*images)

    Inherits from :class:`pirt.BaseRegistration`.

    The original version of the Demons algorithm. Uses Gaussian diffusion
    to regularize the deformation field. This is the implementation as
    proposed by He Wang et al. in 2005 "Validation of an accelerated 'demons'
    algorithm for deformable image registration in radiation therapy"

    See also :class:`pirt.DiffeomorphicDemonsRegistration`.

    The ``speed_factor`` and ``noise_factor`` parameters are specific to this
    algorithm. Other important parameters are also listed below.

    Parameters
    ----------
    speed_factor : scalar
        The relative force of the transform. This one of the most important
        parameters to tune. Default 3.0.
    noise_factor : scalar
        The noise factor. Default 2.5.
    final_scale : scalar
        The minimum scale used during the registration process. This is the
        scale at which the registration ends. Default 1.0. Because calculating
        differentials suffer from more errors as the scale decreases, the
        minimum value is limited at 0.5.
    scale_levels : integer
        The amount of scale levels to use during the registration. Each level
        represents a factor of two in scale. The default (4) works for
        most images, but for large images or large deformations a larger
        value can be used.
    scale_sampling : scalar
        The amount of iterations for each level (i.e. between each factor
        two in scale). Values between 20 and 30 are reasonable in
        most situations. Default 25. Higher values yield better results in
        general. The speed of the algorithm scales linearly with this value.

    """

    def _defaultParams(self):
        """ Overload to create all default params.
        """
        params = BaseRegistration._defaultParams(self)

        # Iteration speed and noise
        params.speed_factor = 3.0
        params.noise_factor = 2.5

        # Change default values
        params.scale_sampling = 25

        # Set some things to make stuff original
        params.smooth_scale = False
        params.combine_deforms = 'add'
        params.mapping = 'backward'
        #params.deform_wise = 'pairwise' is of GDG

        # Smoothing factor
        params.final_smoothing = 8

        return params

    def _deform_for_image(self, i, iterInfo):
        # implement pairwise registration

        # Get wise
        deform_wise = 'pairwise'
        if 'deform_wise' in self.params:
            deform_wise = self.params.deform_wise.lower()

        # Decide which routine to use
        if deform_wise in ['pairwise', 'pairwise1']:
            if i==0:
                return self._deform_from_image_pair(0, 1, iterInfo)
        elif deform_wise == 'pairwise2':
            if i==1:
                return self._deform_from_image_pair(1, 0, iterInfo)
        elif deform_wise == 'groupwise':
            self._set_buffered_data((0,1), iterInfo, None)
            self._set_buffered_data((1,0), iterInfo, None)
            if i==0:
                return self._deform_from_image_pair(0, 1, iterInfo)
            elif i==1:
                return self._deform_from_image_pair(1, 0, iterInfo)
            else:
                raise ValueError('Classic Demons only supports 2 images.')
        else:
            raise ValueError('Invalid value for param deform_wise: %s' % repr(deform_wise))

class DiffeomorphicDemonsRegistration(GDGRegistration, BaseDemonsRegistration):
    """ DiffeomorphicDemonsRegistration(*images)

    Inherits from :class:`pirt.GDGRegistration`.

    A variant of the Demons algorithm that is diffeomorphic. Based on the
    generice diffeomorphic groupwise registration (GDGRegistration) method .

    See also :class:`pirt.OriginalDemonsRegistration`.

    The ``speed_factor`` parameter is specific to this algorithm. The
    ``noise_factor`` works best set at 1.0, effectively disabling
    its use; it is made redundant by the B-spline based regularization.
    Other important parameters are also listed below.

    Parameters
    ----------
    speed_factor : scalar
        The relative force of the transform. This one of the most important
        parameters to tune. Default 3.0.
    mapping : {'forward', 'backward'}
        Whether forward or backward mapping is used. Default forward.
    final_scale : scalar
        The minimum scale used during the registration process. This is the
        scale at which the registration ends. Default 1.0. Because calculating
        differentials suffer from more errors as the scale decreases, the
        minimum value is limited at 0.5.
    scale_levels : integer
        The amount of scale levels to use during the registration. Each level
        represents a factor of two in scale. The default (4) works for
        most images, but for large images or large deformations a larger
        value can be used.
    scale_sampling : scalar
        The amount of iterations for each level (i.e. between each factor
        two in scale). Values between 20 and 30 are reasonable in
        most situations. Default 25. Higher values yield better results in
        general. The speed of the algorithm scales linearly with this value.
    final_grid_sampling : scalar
        The grid sampling of the grid at the final level. During the
        registration process, the B-spine grid sampling scales along
        with the scale. This parameter is usually best coupled to final_scale.
        (When increasing final scale, this value should often be increased
        accordingly.)
    grid_sampling_factor : scalar between 0 and 1
        To what extent the grid sampling scales with the scale. By making
        this value lower than 1, the grid is relatively fine at the the
        higher scales, allowing for more deformations. The default is 0.5.
        Note that setting this value to 1 when using 'frozenedge' can cause
        the image to be 'stuck' at higher scales.

    """

    def _defaultParams(self):
        """ Overload to create all default params.
        """
        params = GDGRegistration._defaultParams(self)

        # Change default values
        params.scale_sampling = 25

        # Iteration speed and noise
        params.speed_factor = 3.0
        params.noise_factor = 1.0

        return params

max_stars_repo_name: almarklein/pirt

max_issues_repo_name: almarklein/pirt

max_forks_repo_name: almarklein/pirt

reason: import numpy,import scipy


text:
from process import load_names, merge_datasets
from scanorama import *

from time import time
import numpy as np

NAMESPACE = 'panorama'

if __name__ == '__main__':
    from config import data_names

    datasets, genes_list, n_cells = load_names(data_names)

    t0 = time()
    datasets_dimred, datasets, genes = correct(
        datasets, genes_list, ds_names=data_names,
        return_dimred=True
    )
    if VERBOSE:
        print('Integrated and batch corrected panoramas in {:.3f}s'
              .format(time() - t0))

    labels = []
    names = []
    curr_label = 0
    for i, a in enumerate(datasets):
        labels += list(np.zeros(a.shape[0]) + curr_label)
        names.append(data_names[i])
        curr_label += 1
    labels = np.array(labels, dtype=int)

    embedding = visualize(datasets_dimred,
                          labels, NAMESPACE + '_ds', names,
                          multicore_tsne=False)

    # Uncorrected.
    datasets, genes_list, n_cells = load_names(data_names)
    datasets, genes = merge_datasets(datasets, genes_list)
    datasets_dimred = dimensionality_reduce(datasets)

    labels = []
    names = []
    curr_label = 0
    for i, a in enumerate(datasets):
        labels += list(np.zeros(a.shape[0]) + curr_label)
        names.append(data_names[i])
        curr_label += 1
    labels = np.array(labels, dtype=int)

    embedding = visualize(datasets_dimred, labels,
                          NAMESPACE + '_ds_uncorrected', names)

max_stars_repo_name: grst/scanorama

max_issues_repo_name: grst/scanorama

max_forks_repo_name: grst/scanorama


text:
import pandas as pd
import numpy as np
from surprise import accuracy, Dataset, Reader
from surprise.model_selection import train_test_split, GridSearchCV
from surprise import SVD
from collections import defaultdict

class CollaborativeRecommender():

    def __init__(self, data, score_index, user_index, items_index):

        self.items_index = items_index
        self.user_index = user_index
        self.data = data

        scale = (data[score_index].min(), data[score_index].max())
        reader = Reader(rating_scale=scale)
        dataset = Dataset.load_from_df(data[[user_index, items_index, score_index]], reader)

        param_grid = {'n_factors': [50, 100, 150], 'n_epochs': [25, 50, 75], 'lr_all': [0.005, 0.01], 'reg_all': [0.02, 0.1, 0.5]}

        gs = GridSearchCV(SVD, param_grid, measures=['rmse'], cv=3)
        gs.fit(dataset)

        params = gs.best_params['rmse']

        self.model = SVD(reg_all=params['reg_all'], n_factors=params['n_factors'],
                         n_epochs=params['n_epochs'], lr_all=params['lr_all'])
        self.model.fit(dataset.build_full_trainset())

    def predict(self, user, max=5):
        best = []
        for item in self.data[self.data[self.user_index] != user][self.items_index].unique():
            uid, iid, r_ui, est, _ = self.model.predict(uid=user, iid=item)
            best += [(iid, est)]
        best.sort(key=lambda x : x[1] , reverse=True)
        return best[:max]

max_stars_repo_name: rltp/TripRec

max_issues_repo_name: rltp/TripRec

max_forks_repo_name: rltp/TripRec


text:
import sys;
from math import sqrt;
from scipy import stats;
import time;
import math;
import json;
import numpy as np;

readPath = "/datahouse/tripflow/Anomaly/tj-byhour-statics/"
writePath = "/datahouse/tripflow/Anomaly/tj-byhour-ano/";

startIndex = 0;
endIndex = 1;

records_array = [];

#count_zero = np.load('countZero.npy')
#count_total_zero = [0 for i in range(104000)]
zero_rate = 0.5
avg_travel_num = 5
avg_total_num = 30

def avg(sequence):
    if len(sequence) < 1:
        return 0;
    seq = [float(i) for i in sequence];
    return sum(seq) / len(seq);

def stdev(sequence):
    if len(sequence) < 1:
        return 0;
    _avg = avg(sequence);
    sdsq = sum([(i - _avg) ** 2 for i in sequence])/len(sequence);
    return sqrt(sdsq);

def getAvgWithoutSelf(avgnum, n, num):
    return float(avgnum * n - num)/float(n-1)

def getStdWithoutSelf(avgnum, stdnum, n, num):
    return sqrt(float((stdnum**2)*n - (num-avgnum)**2)/float(n-1))

def EVT_score_compute_by_vector(m_star, raw_value, mean, std_var):
    mahalanobis_distance = (np.abs(raw_value - mean) / std_var)
    h_star = mahalanobis_distance

    mean_m = np.sqrt(2 * np.log(m_star)) - (
    (np.log(np.log(m_star)) + np.log(2 * np.pi)) / (2 * np.sqrt(2 * np.log(m_star))))
    # mean_m = np.sqrt(2*np.log(m_star))-((np.log(np.log(m_star))+np.log(2*np.pi))/(2*np.sqrt(2*np.log(m_star))))
    #print mean_m
    std_m = 1.0/np.sqrt(2 * np.log(m_star))
    #std_m = np.sqrt(2*np.log(m_star))
    #print std_m

    y_m = (h_star - mean_m) / std_m
    #print y_m

    p_v = np.exp(-np.exp(-y_m))
    #print p_v
    #expected_highest_anomaly_score = 1.2
    expected_highest_anomaly_score = 8
    #score = np.minimum(np.eye(raw_value.shape[0]),((-np.log(1-p_v))/expected_highest_anomaly_score))
    score = np.minimum(1, ((-np.log(1 - p_v)) / expected_highest_anomaly_score))
    #print(score)
    for i in range(len(score)):
        if raw_value[i] < mean:
            score[i] = -score[i]
    return score

def main():
    # load flowcount
    with open("/datahouse/tripflow/Anomaly/bj-flowcount/flowcount.json", 'r') as stream:
        flowCountDict = json.load(stream);
    flowCount = [];
    for x in xrange(87):
        flowCount.append(sum(flowCountDict[x]['flowCount']));
    df1=[[] for i in range(90)];

    df2=[[] for i in range(24)];	#data_feature2

    for file_index in range(0, 2088):
        filename=str(file_index%24)+"-"+str(file_index);
        content=file(readPath+filename, 'r').read();
        records=content.split("\n");
        records_array.append(records[:len(records)-1]);
        # for k in range(len(records)):
        #     record = records[k];
        #     columns = record.split(",")
        #     if len(columns) == 4:
        #         grid_index = int(columns[0]);
        #         # if columns[1] == '0':
        #         #     count_zero[grid_index][0] = count_zero[grid_index][0] + 1;
        #         if columns[2] == '0':
        #             count_zero[grid_index][0] = count_zero[grid_index][0] + 1;
        #         # if columns[3] == '0':
        #         #     count_zero[grid_index][2] = count_zero[grid_index][2] + 1;

    np.save('recordsArray.npy',records_array)
    #records_array = np.load('recordsArray.npy')
    #count_total_zero = np.load('count_total_zero.npy')
    # for records in records_array:
    #     for record in records:
    #         columns = record.split(",")
    #         grid_index = int(columns[0]);
    #         if columns[3] == '0':
    #             count_total_zero[grid_index] = count_total_zero[grid_index] + 1;
    # np.save('count_total_zero.npy', count_total_zero)
    #cal df1
    # for i in range(90):
    #     if 24 * i > 2087:
    #         break;
    #     for record_index in range(104000):
    #         stay_array = [];
    #         travel_array = [];
    #         total_array = []
    #         for j in range(24):
    #             if j + 24 * i > 2087:
    #                 break;
    #             if record_index < len(records_array[j + 24 * i]):
    #                 record = records_array[j + 24 * i][record_index];
    #                 columns = record.split(",");
    #                 stay_array.append(float(columns[1]));
    #                 travel_array.append(float(columns[2]));
    #                 total_array.append(float(columns[3]));
    #             else:
    #                 stay_array.append(0.0);
    #                 travel_array.append(0.0);
    #                 total_array.append(0.0);
    #
    #         avg_s = avg(stay_array);
    #         stdev_s = stdev(stay_array);
    #
    #         avg_t = avg(travel_array);
    #         stdev_t = stdev(travel_array);
    #
    #         avg_a = avg(total_array)
    #         stdev_a = avg(total_array)
    #         n = len(stay_array)
    #         df1[i].append([avg_s, stdev_s, avg_t, stdev_t, avg_a, stdev_a, n]);

    #print(records_array)
    for i in range(24):
        for record_index in range(104000):
            stay_array = [];
            travel_array = [];
            total_array = []
            j = 7;
            while j * 24 + i < 2088:
                if record_index < len(records_array[j * 24 + i]):
                    record = records_array[j * 24 + i][record_index];
                    columns = record.split(",");
                    stay_array.append(float(columns[1]));
                    travel_array.append(float(columns[2]));
                    total_array.append(float(columns[3]));
                else:
                    stay_array.append(0.0);
                    travel_array.append(0.0);
                    total_array.append(0.0);
                j = j + 1;

            #print(stay_array)
            avg_s = avg(stay_array);
            stdev_s = stdev(stay_array);

            avg_t = avg(travel_array);
            stdev_t = stdev(travel_array);

            avg_a = avg(total_array)
            stdev_a = avg(total_array)

            df2[i].append([avg_s, stdev_s, avg_t, stdev_t, avg_a, stdev_a, stay_array, travel_array, total_array]);
    #np.save('df2.npy',df2)
    # iterate all records
    for file_index in range(startIndex, endIndex + 1):
        filename = str(file_index % 24) + "-" + str(file_index);
        resultFile = file(writePath + "ano-" + filename, 'w');
        for record_index in range(len(records_array[file_index])):
            columns = records_array[file_index][record_index].split(",");
            stay_ = float(columns[1]);
            travel_ = float(columns[2]);
            total_ = float(columns[3]);

            raw_value_list = [stay_]
            raw_value_vector = np.array(raw_value_list, dtype=np.float64)

            ano_s1 = 0.0
            # if df1[file_index / 24][record_index][1] != 0:
            #     ano_s1 = EVT_score_compute_by_vector(24, raw_value_vector, getAvgWithoutSelf(df1[file_index / 24][record_index][0],df1[file_index / 24][record_index][6], stay_),
            #                                          getStdWithoutSelf(df1[file_index / 24][record_index][0], df1[file_index / 24][record_index][1],df1[file_index / 24][record_index][6], stay_))[0]
            # else:
            #     ano_s1 = 0.87

            ano_s2 = 0.0;
            # arr = df2[file_index % 24][record_index][6]
            # index = arr.index(stay_)
            # arr = arr[0:index]+arr[index+1:]
            # std_s = stdev(arr)
            # if std_s != 0:
            #     ano_s2 = EVT_score_compute_by_vector(87, raw_value_vector, avg(arr),std_s)[0]
            # else:
            #     #ano_s2 = 0.0
            #     new_avg = avg(arr)
            #     if new_avg == 0:
            #         ano_s2 = 0.0;
            #     else:
            #         ano_s2 = (stay_-new_avg)/new_avg

            raw_value_list = [travel_]
            raw_value_vector = np.array(raw_value_list, dtype=np.float64)
            ano_t1 = 0.0;
            # if df1[file_index / 24][record_index][3] != 0:
            #     ano_t1 = EVT_score_compute_by_vector(24, raw_value_vector, df1[file_index / 24][record_index][2],
            #                                          df1[file_index / 24][record_index][3])[0]
            # else:
            #     ano_t1 = 0.0;

            #ano_t2 = 0.0;
            arr = df2[file_index % 24][record_index][7]
            index = arr.index(travel_)
            arr = arr[0:index]+arr[index+1:]
            std_t = stdev(arr)
            #std_t = getStdWithoutSelf(df2[file_index / 24][record_index][2], df2[file_index / 24][record_index][3], 87, travel_)
            if  df2[file_index % 24][record_index][2] < avg_travel_num:
                ano_t2 = 0.0
            elif std_t != 0:
                ano_t2 = EVT_score_compute_by_vector(79, raw_value_vector, avg(arr), std_t)[0]
            else:
                new_avg = avg(arr)
                if new_avg == 0:
                    ano_t2 = 0.0;
                else:
                    ano_t2 = (travel_-new_avg)/new_avg

            raw_value_list = [total_]
            raw_value_vector = np.array(raw_value_list, dtype=np.float64)
            ano_a1 = 0.0
            # if df1[file_index / 24][record_index][3] != 0:
            #     ano_a1 = EVT_score_compute_by_vector(24, raw_value_vector, df1[file_index / 24][record_index][4],
            #                                          df1[file_index / 24][record_index][5])[0]
            # else:
            #     ano_a1 = 0.0;

            ano_a2 = 0.0;
            arr = df2[file_index % 24][record_index][8]
            index = arr.index(total_)
            arr = arr[0:index]+arr[index+1:]
            std_a = stdev(arr)
            #std_a = getStdWithoutSelf(df2[file_index / 24][record_index][4], df2[file_index / 24][record_index][5],87, total_)

            if df2[file_index % 24][record_index][4] < avg_total_num:
               ano_t2 = 0.0
            elif std_a != 0:
                ano_a2 = EVT_score_compute_by_vector(79, raw_value_vector,avg(arr),std_a)[0]
            else:
                #ano_a2 = 0.0;
                new_avg = avg(arr)
                if new_avg == 0:
                    ano_a2 = 0.0;
                else:
                    ano_a2 = (total_-new_avg)/new_avg

            resultFile.write(
                str(record_index) + "," + str(columns[1]) + "," + str(columns[2]) + "," + str(ano_s1) + "," + str(
                    ano_t1) + "," + str(ano_a1) + "," + str(ano_s2) + "," + str(ano_t2) + "," + str(ano_a2) + "\n");

        #resultFile.write(str(record_index)+","+str(columns[1])+","+str(columns[2])+","+str(ano_s1)+","+str(ano_t1)+","+str(ano_a1)+","+str(ano_s2)+","+str(ano_t2)+","+str(ano_a2)+"\n");

        resultFile.close();

if __name__ == '__main__':
    start_time = time.time();
    main();
    print "Complete Time: " + str(time.time() - start_time);

max_stars_repo_name: visdata/UrbanMotionAnalysis

max_issues_repo_name: visdata/UrbanMotionAnalysis

max_forks_repo_name: visdata/UrbanMotionAnalysis

reason: import numpy,from scipy


text:
import tvm
import numpy as np
from tvm import relay

do_print = [False]

def show(text):
    if do_print[0]:
        print("---------------------------")
        print(text)

def test_func():
    x = relay.var("x", shape=(3, 2))
    y = relay.var("y")
    one = relay.const(10e10, dtype="float32")
    z = relay.add(x, one)
    z = relay.add(z, z)
    f = relay.Function([x, y], z)
    show(z.astext())
    show(f.astext())

def test_env():
    x = relay.var("x", "float32")
    y = relay.var("y", "float32")
    z = relay.add(x, y)
    z = relay.add(z, z)
    f = relay.Function([x, y], z)
    env = relay.Environment()
    env.add("myf", f)
    text = env.astext()
    assert "def @myf" in text
    assert "%1 = add(%0, %0) # ty=float32" in text
    show(text)

def test_meta_data():
    n, c, h, w = tvm.var("n"), 10, 224, 224
    x = relay.var("x", shape=(n, c, h, w))
    w = relay.var("w")
    z = relay.nn.conv2d(x, w,
                        kernel_size=(3, 3),
                        padding=(1, 1),
                        channels=2)
    f = relay.Function([x, w], z)
    text = f.astext()
    assert "channels=2" in text
    assert "meta.Variable(id=0)" in text
    show(text)

    text = relay.const([1,2,3]).astext()
    assert "meta.relay.Constant(id=0)" in text
    show(text)

def test_call_attrs():
    x = relay.var("x")
    # non default args
    z = relay.nn.softmax(x, axis=2)
    assert "axis=2" in z.astext()
    # default args
    z = relay.nn.softmax(x)
    assert "softmax(%x)" in z.astext()
    # non default args
    z = relay.expand_dims(x, axis=2, num_newaxis=2)
    assert "num_newaxis=2" in z.astext()

def test_let_if_scope():
    x = relay.var("x", "float32")
    y = relay.var("y", "float32")
    cond = relay.var("cond", "bool")
    v1 = relay.var("v")
    v2 = relay.var("v", "float32")
    then_branch = relay.Let(
        v1, relay.const(1, "float32"),
        relay.Let(v2, x, relay.subtract(v1, v2)))
    v3 = relay.var("v")
    let2 = relay.Let(v3, y, v3)
    else_branch = relay.add(let2, let2)
    result = relay.If(cond, then_branch, else_branch)
    f = relay.Function([x, y, cond], result)
    text = f.astext()
    assert text.count("{") == 4
    assert "%cond: bool" in text
    show(f.astext())

if __name__ == "__main__":
    do_print[0] = True
    test_let_if_scope()
    test_func()
    test_env()
    test_meta_data()
    test_call_attrs()


text:
import pandas as pd
import numpy as np
from statsmodels.tsa.stattools import acf

import properties as pr
from Forecasting import Sarima_forecast, Var_forecast

def Seasonality(DataSet,timecolumn,target):
    Data = DataSet                          # Multivariate Data
    uniData = Data.loc[:,[target]]          # Univariate Data
    lag_acf = acf(uniData.diff().dropna(), nlags=50)
    acf1 = []
    acf2 = []
    for i in range(1,5):
        if i%2 == 0:
            acf2.append(lag_acf[i*12])
        else:
            acf1.append(lag_acf[i*12])
    # Check for seasonality with the (12,36) lag and (24,48) lag
    if (abs(acf1[0]-acf1[1]) < pr.seasonalCheck_err) and (abs(acf2[0]-acf2[1])<pr.seasonalCheck_err):
        #print("Data is seasonal")
        Sarima_forecast.SARIMA_forecasting(Data = uniData,Time_Column = timecolumn,Target_Column = target)
    else:
        #print("Data is Not Seasonal")
        Var_forecast.VAR_forecasting(Data=Data, Time_Column= timecolumn,Target_Column=target)

def Read_Dataset(dataset,timecolumn,target):
    """ Read and Modify The DataSet
    dataset variable holds the location and name of the Dataset
    timecolumn holds the TIME column"""

    series = pd.read_csv(dataset)
    series[timecolumn] = pd.to_datetime(series[timecolumn])
    series[pr.column_date] = [d.date() for d in series[timecolumn]]
    series[pr.column_date] = pd.to_datetime(series[pr.column_date])
    data = series.drop(timecolumn, axis=1)
    data = data.drop(pr.column_date,axis =1)
    data = data.set_index(series[pr.column_date])
    #data = data.loc['2010-01-01 00:00:00+00:00':'2011-01-01 00:00:00+00:00']
    weekly = data.resample('D').mean()
    Seasonality(DataSet=weekly,timecolumn=timecolumn,target=target)

reason: import numpy,from statsmodels


text:
# Generate figures that summarize the results of different papers
# that employ context in object detection. Paper results are provided in lit_summary.xlsx.

import numpy as np
import matplotlib.pyplot as plt
from operator import itemgetter

def xlsx(fname):
    import zipfile
    from xml.etree.ElementTree import iterparse
    z = zipfile.ZipFile(fname)
    strings = [el.text for e, el in iterparse(z.open('xl/sharedStrings.xml')) if el.tag.endswith('}t')]
    rows = []
    row = {}
    value = ''
    for e, el in iterparse(z.open('xl/worksheets/sheet1.xml')):
        if el.tag.endswith('}v'):  # <v>84</v>
            value = el.text
        if el.tag.endswith('}c'):  # <c r="A3" t="s"><v>84</v></c>
            if el.attrib.get('t') == 's':
                value = strings[int(value)]
            letter = el.attrib['r']  # AZ22
            while letter[-1].isdigit():
                letter = letter[:-1]
            row[letter] = value
            value = ''
        if el.tag.endswith('}row'):
            rows.append(row)
            row = {}
    return rows

def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def gen_summaries():
    # generate summaries to all algorithms

    file_name = '/Users/ehud/Work/measure2context/lit_summary/lit_summary.xlsx'

    # define important constants
    num_cats = 161  # number of object categories in the XLSX file
    row_alg = 15
    row_avg = 17
    row_start_cat = 19
    # col_cat_name = 'A'
    cols_alg_names = ['B', 'F', 'J', 'N', 'R', 'V', 'Z', 'AD', 'AH', 'AL', 'AP', 'AT', 'AX']
    cols_ap_add = ['D', 'H', 'L', 'P', 'T', 'X', 'AB', 'AF', 'AJ', 'AN', 'AR', 'AV', 'AZ']
    cols_ap_perc = ['E', 'I', 'M', 'Q', 'U', 'Y', 'AC', 'AG', 'AK', 'AO', 'AS', 'AW', 'BA']

    by_percent = False

    # read xlsx file
    rows = xlsx(file_name)

    # collect best and worst results of each algorithm
    best = []
    worst = []

    for alg in range(len(cols_alg_names)):
        alg_name = rows[row_alg][cols_alg_names[alg]]
        orig_avg = float(rows[row_avg][cols_alg_names[alg]])

        if by_percent:
            col = cols_ap_perc[alg]
        else:
            col = cols_ap_add[alg]

        # get average result
        res_avg = float(rows[row_avg][col])

        # collect category results
        res = []
        for i in range(num_cats):
            r = rows[row_start_cat + i]
            # cat = r[col_cat_name]
            if col not in r.keys():
                continue

            val = r[col]
            if not is_number(val):
                continue
            res.append(float(val))
        res.sort(reverse=True)

        # print(res)
        plt.axhline(y=2, color='r')
        plt.bar(np.arange(0, len(res)), res)
        # plt.plot(np.arange(0, len(res)), res)
        plt.title('%s: %f + %f' % (alg_name, orig_avg, res_avg))
        if by_percent:
            plt.ylim((-100, 100))
        else:
            plt.ylim((-15, 15))
        plt.savefig('out/%d.png' % alg)
        plt.close()

        # collect best and worst results
        if res[-1] < 0:
            best.append(res[0])
            worst.append(res[-1])
            best.append(res[1])
            worst.append(res[-2])
            best.append(res[2])
            worst.append(res[-3])

    # best and worst results figure
    # plt.scatter(best,worst)
    # plt.xlim((0, 15))
    # plt.ylim((0, -15))
    # plt.show()

def gen_literature_summary():
    # generate summaries over several algorithms (for the paper)

    file_name = '/Users/ehud/Work/measure2context/lit_summary/lit_summary.xlsx'

    # define important constants
    num_cats = 161  # number of object categories in the XLSX file
    row_alg = 15
    row_avg = 17
    row_start_cat = 19
    # col_cat_name = 'A'
    cols_alg_names = ['R', 'V', 'AD']
    cols_ap_add =    ['T', 'X', 'AF']
    cols_ap_perc =   ['U', 'Y', 'AG']
    labels = ['Choi et al. (PASCAL 2007)','Choi et al. (SUN 09)','Chen et al. (PASCAL 2007)']

    by_percent = False

    # read xlsx file
    rows = xlsx(file_name)

    # prepare figure
    fig = plt.figure()
    ax = plt.gca()

    left, width = .55, .5
    bottom, height = .38, .5
    right = left + width
    top = bottom + height
    ax.text(right, 0.5 * (bottom + top), 'Improved\n Results',
            horizontalalignment='center',
            verticalalignment='center',
            rotation='vertical',
            transform=ax.transAxes,
            fontsize=12)

    left, width = .55, .5
    bottom, height = -.11, .5
    right = left + width
    top = bottom + height
    ax.text(right, 0.5 * (bottom + top), 'Diminished\n Results',
            horizontalalignment='center',
            verticalalignment='center',
            rotation='vertical',
            transform=ax.transAxes,
            fontsize=12)

    plt.axhline(y=2, color='r')
    plt.axhline(y=0, color='black')
    plt.grid()

    for alg in range(len(cols_alg_names)):
        alg_name = rows[row_alg][cols_alg_names[alg]]
        orig_avg = float(rows[row_avg][cols_alg_names[alg]])

        if by_percent:
            col = cols_ap_perc[alg]
        else:
            col = cols_ap_add[alg]

        # get average result
        res_avg = float(rows[row_avg][col])

        # collect category results
        res = []
        for i in range(num_cats):
            r = rows[row_start_cat + i]
            # cat = r[col_cat_name]
            if col not in r.keys():
                continue

            val = r[col]
            if not is_number(val):
                continue
            res.append(float(val))
        res.sort(reverse=True)

        # print(res)

        x_len = len(res)
        x_len_fix = 1
        if x_len > 20:
            x_len_fix = 19/106

        plt.plot(np.arange(0, len(res))*x_len_fix, res, label=labels[alg])

    if by_percent:
        plt.ylim((-100, 100))
    else:
        plt.ylim((-5, 12))
    plt.xlim((0, 19))
    plt.xticks([])
    plt.xlabel('Object categories', fontsize=12)
    plt.ylabel('Change in average precision (AP)', fontsize=12)
    plt.title('Detection improvement with context')
    plt.legend()
    plt.savefig('out/%d.eps' % alg, bbox_inches='tight')
    plt.close()

def per_category():
    # test improvement per category over PASCAL dataset

    file_name = '/Users/ehud/Work/measure2context/lit_summary/lit_summary.xlsx'

    # define important constants
    num_cats = 20  # number of object categories in the XLSX file
    row_alg = 15
    row_avg = 17
    row_start_cat = 19
    col_cat_name = 'A'
    cols_alg_names = ['B', 'F', 'J', 'R', 'Z', 'AD', 'AL', 'AP', 'AT']
    cols_ap_add = ['D', 'H', 'L', 'T', 'AB', 'AF', 'AN', 'AR', 'AV']

    # read xlsx file
    rows = xlsx(file_name)

    cats = []
    res = []
    for i in range(num_cats):
        r = rows[row_start_cat + i]
        cat = r[col_cat_name]
        # print(cat)
        res_cat = []
        for alg in range(len(cols_alg_names)):
            alg_name = rows[row_alg][cols_alg_names[alg]]
            col = cols_ap_add[alg]
            # print(alg_name)
            # print(r[col])
            # print(len(r[col]))
            val = float(r[col])
            res_cat.append(val)

        # print(cat)
        cat = cat[0:-1]
        if cat == 'motorbike':
            cat = 'bike'
        if cat == 'potted plant':
            cat = 'plant'
        if cat == 'tv/monito':
            cat = 'tv'
        if cat == 'bicycl':
            cat = 'cycle'
        cat = cat.capitalize()
        cats.append(cat)
        res.append(res_cat)

    # sort by mean
    meds = np.array([np.median(x) for x in res])
    order = np.argsort(-meds).tolist()
    cats = [cats[i] for i in order]
    res = [res[i] for i in order]
    meds = np.array([np.median(x) for x in res])

    fig = plt.figure()
    ax = plt.gca()

    left, width = .55, .5
    bottom, height = .38, .5
    right = left + width
    top = bottom + height
    ax.text(right, 0.5 * (bottom + top), 'Improved\n Results',
            horizontalalignment='center',
            verticalalignment='center',
            rotation='vertical',
            transform=ax.transAxes,
            fontsize=12)

    left, width = .55, .5
    bottom, height = -.11, .5
    right = left + width
    top = bottom + height
    ax.text(right, 0.5 * (bottom + top), 'Diminished\n Results',
            horizontalalignment='center',
            verticalalignment='center',
            rotation='vertical',
            transform=ax.transAxes,
            fontsize=12)

    plt.axhline(y=0, color='black')
    plt.axhline(y=2, color='r')
    plt.boxplot(res, labels=cats, whis=999999)
    # plt.ylim((-15, 15))
    plt.xticks(rotation=45)
    plt.ylabel('Change in average precision (AP)', fontsize=12)
    plt.title('Summary of improvement by different models')
    # plt.show()
    plt.savefig('out/summary_per_cat_pascal.eps', bbox_inches='tight')
    plt.savefig('out/summary_per_cat_pascal.png', bbox_inches='tight')
    plt.close()

def main():
    gen_summaries()
    gen_literature_summary()
    per_category()

if __name__ == "__main__":
    main()

max_stars_repo_name: EhudBarnea/ContextAnalysis

max_issues_repo_name: EhudBarnea/ContextAnalysis

max_forks_repo_name: EhudBarnea/ContextAnalysis


text:
#!/usr/bin/env python

'''Simple implementations of connectivity measures.'''

# Authors : pravsripad@gmail.com
#           daniel.vandevelden@yahoo.de

import sys
import numpy as np
import matplotlib.pyplot as pl
import matplotlib.mlab as mlab

n_epochs = 120
sfreq, duration = 1000., 1000
times = np.arange(0, duration, 1 / sfreq)
amp , amp2 , nse_amp = 1., 1., 0.5
nfft = 512

nse1 = np.random.rand(times.size) * nse_amp
nse2 = np.random.rand(times.size) * nse_amp
x =  amp * np.sin(2 * np.pi * 200 * times) + nse1
y =  amp * np.sin(2 * np.pi * 200 * times + np.pi/5) + nse2

shift = 100  # integer
assert shift < sfreq * duration, 'Choose a smaller shift.'
#y = amp2 * np.roll(x, shift) + nse2

# coherence using mlab function
cohxy, freqs = mlab.cohere(x, y, Fs=sfreq, NFFT=nfft)

n_freqs = int(nfft/2 + 1)

def compute_mean_psd_csd(x, y, n_epochs, nfft, sfreq):
    '''Computes mean of PSD and CSD for signals.'''
    x2 = np.array_split(x, n_epochs)
    y2 = np.array_split(y, n_epochs)

    Rxy = np.zeros((n_epochs, n_freqs), dtype=np.complex)
    Rxx = np.zeros((n_epochs, n_freqs), dtype=np.complex)
    Ryy = np.zeros((n_epochs, n_freqs), dtype=np.complex)

    for i in range(n_epochs):
        Rxy[i], freqs = mlab.csd(x2[i], y2[i], NFFT=nfft, Fs=sfreq)
        Rxx[i], _ = mlab.psd(x2[i], NFFT=nfft, Fs=sfreq)
        Ryy[i], _ = mlab.psd(y2[i], NFFT=nfft, Fs=sfreq)

    Rxy_mean = np.mean(Rxy, axis=0)
    Rxx_mean = np.mean(Rxx, axis=0)
    Ryy_mean = np.mean(Ryy, axis=0)

    return freqs, Rxy, Rxy_mean, np.real(Rxx_mean), np.real(Ryy_mean)

def my_coherence(n_freqs, Rxy_mean, Rxx_mean, Ryy_mean):
    ''' Computes coherence. '''

    coh = np.zeros((n_freqs))
    for i in range(0, n_freqs):
        coh[i] = np.abs(Rxy_mean[i]) / np.sqrt(Rxx_mean[i] * Ryy_mean[i])

    return coh

def my_imcoh(n_freqs, Rxy_mean, Rxx_mean, Ryy_mean):
    ''' Computes imaginary coherence. '''

    imcoh = np.zeros((n_freqs))
    for i in range(0, n_freqs):
        imcoh[i] = np.imag(Rxy_mean[i]) / np.sqrt(Rxx_mean[i] * Ryy_mean[i])

    return imcoh

def my_cohy(n_freqs, Rxy_mean, Rxx_mean, Ryy_mean):
    ''' Computes coherency. '''

    cohy = np.zeros((n_freqs))
    for i in range(0, n_freqs):
        cohy[i] = np.real(Rxy_mean[i]) / np.sqrt(Rxx_mean[i] * Ryy_mean[i])

    return cohy

def my_plv(n_freqs, Rxy, Rxy_mean):
    ''' Computes PLV. '''

    Rxy_plv = np.zeros((n_epochs, n_freqs), dtype=np.complex)
    for i in range(0, n_epochs):
        Rxy_plv[i] = Rxy[i] / np.abs(Rxy[i])

    plv = np.abs(np.mean(Rxy_plv, axis=0))
    return plv

def my_pli(n_freqs, Rxy, Rxy_mean):
    ''' Computes PLI. '''
    Rxy_pli = np.zeros((n_epochs, n_freqs), dtype=np.complex)
    for i in range(0, n_epochs):
        Rxy_pli[i] = np.sign(np.imag(Rxy[i]))

    pli = np.abs(np.mean(Rxy_pli, axis=0))
    return pli

def my_wpli(n_freqs, Rxy, Rxy_mean):
    ''' Computes WPLI. '''
    Rxy_wpli_1 = np.zeros((n_epochs, n_freqs), dtype=np.complex)
    Rxy_wpli_2 = np.zeros((n_epochs, n_freqs), dtype=np.complex)
    for i in range(0, n_epochs):
        Rxy_wpli_1[i] = np.imag(Rxy[i])
        Rxy_wpli_2[i] = np.abs(np.imag(Rxy[i]))

    # handle divide by zero
    denom = np.mean(Rxy_wpli_2, axis=0)
    idx_denom = np.where(denom == 0.)
    denom[idx_denom] = 1.
    wpli = np.abs(np.mean(Rxy_wpli_1, axis=0)) / denom
    wpli[idx_denom] = 0.
    return wpli

def my_con(x, y, n_epochs, nfft, sfreq, con_name='coh'):
    '''Computes connectivity measure mentioned on provided signal pair and its surrogates.'''

    freqs, Rxy, Rxy_mean, Rxx_mean, Ryy_mean = compute_mean_psd_csd(x, y, n_epochs, nfft, sfreq)

    # compute surrogates
    x_surr = x.copy()
    y_surr = y.copy()
    np.random.shuffle(x_surr)
    np.random.shuffle(y_surr)
    freqs_surro, Rxy_s, Rxy_s_mean, Rxx_s_mean, Ryy_s_mean = compute_mean_psd_csd(x_surr, y_surr, n_epochs, nfft, sfreq)

    if con_name == 'coh':
        coh = my_coherence(n_freqs, Rxy_mean, Rxx_mean, Ryy_mean)
        coh_surro = my_coherence(n_freqs, Rxy_s_mean, Rxx_s_mean, Ryy_s_mean)
        return coh, coh_surro, freqs, freqs_surro

    if con_name == 'imcoh':
        imcoh = my_imcoh(n_freqs, Rxy_mean, Rxx_mean, Ryy_mean)
        imcoh_surro = my_imcoh(n_freqs, Rxy_s_mean, Rxx_s_mean, Ryy_s_mean)
        return imcoh, imcoh_surro, freqs, freqs_surro

    if con_name == 'cohy':
        cohy = my_cohy(n_freqs, Rxy_mean, Rxx_mean, Ryy_mean)
        cohy_surro = my_cohy(n_freqs, Rxy_s_mean, Rxx_s_mean, Ryy_s_mean)
        return cohy, cohy_surro, freqs, freqs_surro

    if con_name == 'plv':
        plv = my_plv(n_freqs, Rxy, Rxy_mean)
        plv_surro = my_plv(n_freqs, Rxy_s, Rxy_s_mean)
        return plv, plv_surro, freqs, freqs_surro

    if con_name == 'pli':
        pli = my_pli(n_freqs, Rxy, Rxy_mean)
        pli_surro = my_pli(n_freqs, Rxy_s, Rxy_s_mean)
        return pli, pli_surro, freqs, freqs_surro

    if con_name == 'wpli':
        wpli = my_wpli(n_freqs, Rxy, Rxy_mean)
        wpli_surro = my_wpli(n_freqs, Rxy_s, Rxy_s_mean)
        return wpli, wpli_surro, freqs, freqs_surro

    if con_name == '':
        print('Please provide the connectivity method to use.')
        sys.exit()
    else:
        print('Connectivity method unrecognized.')
        sys.exit()

con_name = 'wpli'
con, con_surro, freqs, freqs_surro = my_con(x, y, n_epochs, nfft, sfreq, con_name)

# coherence using mlab function
#cohxy, freqs = mlab.cohere(x, y, Fs=sfreq, NFFT=nfft)
#pl.plot(freqs, cohxy)

# plot results
pl.figure('Connectivity')
pl.plot(freqs, con)
pl.plot(freqs_surro, con_surro)
pl.legend(['Con', 'Surrogates'])
pl.tight_layout()
pl.show()

max_stars_repo_name: fboers/jumeg

max_issues_repo_name: fboers/jumeg

max_forks_repo_name: fboers/jumeg