text:
import astropy
import math
from .defs import time_scale

def jd_now():
    return astropy.time.Time.now().jd

def iso2jd(iso):
    return astropy.time.Time(iso, scale=time_scale).jd

def jd2iso(jd):
    """Convert a Julian date into an ISO 8601 date string representation"""
    return astropy.time.Time(jd, format='jd', scale=time_scale, precision=0).iso

def parse_jd_or_iso_date(date):
    for format in ['jd', 'iso', 'isot']:
        try:
            return astropy.time.Time(float(date) if format == 'jd' else date,
                    format=format, scale=time_scale).jd
        except ValueError:
            continue
    raise ValueError('Please pass the date as either a Julian Day decimal string ' +
                     '(e.g. "2456799.9897") or as an ISO8601 string denoting a UTC ' +
                     'time in this format: 2014-05-20 23:37:17.')

def mod360_distance(a, b):
    """distance between a and b in mod(360)"""
    # Surprisingly enough there doesn't seem to be a more elegant way.
    # Check http://stackoverflow.com/questions/6192825/
    a %= 360
    b %= 360

    if a < b:
        return mod360_distance(b, a)
    else:
        return min(a-b, b-a+360)

def days_frac_to_dhms(days_frac):
    """Convert a day float to integer days, hours, minutes and seconds.

    Returns a tuple (days, hours, minutes, seconds).

    >>> days_frac_to_dhms(2.5305)
    (2, 12, 43, 55)
    """
    days = math.floor(days_frac)
    hms_frac = days_frac - days
    hours = math.floor(hms_frac * 24)
    minutes_frac = hms_frac - hours / 24
    minutes = math.floor(minutes_frac * 1440)
    seconds = math.floor((minutes_frac - minutes / 1440) * 86400)

    return (days, hours, minutes, seconds)

def render_pretty_time(jd):
    """Convert jd into a pretty string representation"""
    year, month, day, hour_frac = sweph.revjul(jd)
    _, hours, minutes, seconds = days_frac_to_dhms(hour_frac/24)
    time_ = calendar.timegm((year,month,day,hours,minutes,seconds,0,0,0))
    return time.strftime('%e %b %Y %H:%M UTC', time.gmtime(time_))

def render_delta_days(delta_days):
    """Convert a time delta into a pretty string representation"""
    days, hours, minutes = days_frac_to_dhms(delta_days)[:3]
    result = []

    if days > 0:
        result += ['%d days' % days]

    if hours > 0:
        result += ['%d hours' % hours]

    if days == 0 and minutes > 0:
        result += ['%d minutes' % minutes]

    if days == 0 and hours == 0 and minutes == 0:
        result = ['less than a minute']

    return ' '.join(result);

def angle_to_aspect_name(angle):
    return [aspect[1] for aspect in aspects if aspect[0] == angle]

def aspect_name_to_angle(name):
    "Get the (dexter, if applicable) angle for an aspect name"
    if name == 'conjunction':
        return 0
    if name == 'opposition':
        return 180
    else:
        return [aspect[0] for aspect in dexter_aspects if aspect[1] == name]

max_stars_repo_name: cyfrmedia/cerridwen

max_issues_repo_name: cyfrmedia/cerridwen

max_forks_repo_name: cyfrmedia/cerridwen

reason: import astropy


text:
import numpy as np
import matplotlib.pyplot as plt

# sin(2wt + theta)
def sinusGenerator(amplitudo,frekuensi,tAkhir,theta) :
  t = np.arange(0,tAkhir,0.1)
  y = amplitudo * np.sin(2*frekuensi*t + np.deg2rad(theta))
  return t,y

t1,y1 = sinusGenerator(1,1,4,0)
plt.plot(t1,y1)

t2,y2 = sinusGenerator(1,1,4,30)
plt.plot(t2,y2)

t3,y3 = sinusGenerator(1,1,4,60)
plt.plot(t3,y3, 'b--')

t4,y4 = sinusGenerator(1,1,4,90)
plt.plot(t4,y4, 'c')

plt.show()

ext: py

max_stars_repo_path: Matplotlib/Color & Marker.py

max_issues_repo_path: Matplotlib/Color & Marker.py

max_forks_repo_path: Matplotlib/Color & Marker.py

reason: import numpy


text:
# From this package
from macrospin_gpu.kernels import Macrospin_2DPhaseDiagram
from macrospin_gpu.simulations import Simulation2D

# Numerical libraries
import numpy as np

# Plotting
import matplotlib.pyplot as plt

if __name__ == '__main__':
    min_duration = 0.001
    max_duration = 0.5
    min_current  = 0
    max_current  = 2e8

    mk = Macrospin_2DPhaseDiagram()
    mk.set_magnetic_properties(Ms=640.0, damping=0.1, Hpma=000, initial_m=[-1,0,0])
    mk.set_external_field([0,0,0])
    mk.set_evolution_properties(dt=1e-12)
    mk.set_geometry(100,50,3)
    mk.add_spin_torque([0.0,0.0,1.0], 0.16, 1.49)
    mk.add_spin_torque([-1.0,0.0,0.0], 0.00, 1.49)
    mk.add_thermal_noise(0.0, 1)
    mk.define_phase_diagram("current_density", np.linspace(min_current, max_current, 64),
                            "pulse_duration",  np.linspace(min_duration*1e-9, max_duration*1e-9, 64))
    mk.store_time_traces(interval=5e-12)

    sim = Simulation2D(mk)
    sim.run()
    phase_diagram = sim.get_phase_diagram()

    extent = (min_duration, max_duration, min_current, max_current)
    aspect = (max_duration-min_duration)/(max_current-min_current)

    plt1 = plt.figure(1)
    plt.imshow(phase_diagram, origin='lower', cmap=plt.get_cmap('Blues'),
                 extent=extent, aspect=aspect)
    plt.colorbar()
    plt1.suptitle('Switching Probability')
    plt1.gca().set_xlabel('Pulse Duration (ns)')
    plt1.gca().set_ylabel(r'Current Density (10$^8$A/cm$^2$)')

    plt2 = plt.figure(2)
    time_data = sim.get_time_traces()
    ni, nj, n_blah = time_data.shape
    for i in range(0,ni,4):
        for j in range(0,nj,4):
            plt.plot(time_data[i,j,:]['x'])
    plt.show()


text:
# Targets not used, not possible to condition generated sequences

import data_utils_2
import pandas as pd
import numpy as np
import tensorflow as tf
import math, random, itertools
import pickle
import time
import json
import os
import math
from data_utils_2 import get_data

import tensorflow as tf
import numpy as np
from sklearn.metrics import roc_curve, auc, precision_recall_curve
import matplotlib
# change backend so that it can plot figures without an X-server running.
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import math, time, json, random

import glob
import copy

import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-learning_rate', type=float)
parser.add_argument('-optimizer_str', type=str)
parser.add_argument('-hidden_units_dec', type=int)
parser.add_argument('-hidden_units_enc', type=int)
parser.add_argument('-emb_dim', type=int)
parser.add_argument('-mult', type=float)
parser.add_argument('-experiment_id', type=str)
args = parser.parse_args()

experiment_id = './' + args.experiment_id

# directory where the data will be saved
if not os.path.isdir(experiment_id):
    os.mkdir(experiment_id)

# function for getting one mini batch

def get_batch(samples, batch_idx, batch_size):
    start_pos = batch_idx * batch_size
    end_pos = start_pos + batch_size
    return samples[start_pos:end_pos]

def save_plot_sample(samples, idx, identifier, n_samples=6, num_epochs=None, ncol=2, path='./'):
    assert n_samples <= samples.shape[0]
    assert n_samples % ncol == 0
    sample_length = samples.shape[1]

    if not num_epochs is None:
        col = hsv_to_rgb((1, 1.0*(idx)/num_epochs, 0.8))
    else:
        col = 'grey'

    x_points = np.arange(sample_length)

    nrow = int(n_samples/ncol)
    fig, axarr = plt.subplots(nrow, ncol, sharex=True, figsize=(6, 6))
    for m in range(nrow):
        for n in range(ncol):
            # first column
            sample = samples[n*nrow + m, :, 0]
            axarr[m, n].plot(x_points, sample, color=col)
            axarr[m, n].set_ylim(-1, 1)
    for n in range(ncol):
        axarr[-1, n].xaxis.set_ticks(range(0, sample_length, int(sample_length/4)))
    fig.suptitle(idx)
    fig.subplots_adjust(hspace = 0.15)
    fig.savefig(path + "/" + identifier + "_sig" + str(idx).zfill(4) + ".png")
    print(path + "/" + identifier + "_sig" + str(idx).zfill(4) + ".png")
    plt.clf()
    plt.close()
    return

def sine_wave(seq_length=30, num_samples=28*5*100, num_signals=1,
        freq_low=1, freq_high=5, amplitude_low = 0.1, amplitude_high=0.9,
        random_seed=None, **kwargs):

    ix = np.arange(seq_length) + 1
    samples = []
    for i in range(num_samples):
        signals = []
        for i in range(num_signals):
            f = np.random.uniform(low=freq_high, high=freq_low)     # frequency
            A = np.random.uniform(low=amplitude_high, high=amplitude_low)        # amplitude
            # offset
            offset = np.random.uniform(low=-np.pi, high=np.pi)
            signals.append(A*np.sin(2*np.pi*f*ix/float(seq_length) + offset))
        samples.append(np.array(signals).T)
    # the shape of the samples is num_samples x seq_length x num_signals
    samples = np.array(samples)
    return samples

########################
# DATA LOADING
########################

print ("loading data...")

samples = sine_wave()
inputs_train, inputs_validation, inputs_test = data_utils_2.split(samples, [0.6, 0.2, 0.2])

save_plot_sample(samples[0:7], '0', 'test_RVAE', path=experiment_id)

print ("data loaded.")

# runs the experiment 5 times
#identifiers = ['eICU_RVAE_synthetic_dataset_VAE_r' + str(i) for i in range(1)]
#for identifier in identifiers:
#identifier = identifiers[0]

#training config
batch_size = 32

print ("data loaded.")

seq_length = inputs_train.shape[1]
num_features = inputs_train.shape[2]
# not used
random_seed = 0

########################
# ENCODER
########################

def encoder(hidden_units_enc, emb_dim, mult):

    with tf.variable_scope("encoder") as scope:

        input_seq_enc = tf.placeholder(tf.float32, [batch_size, seq_length, num_features])

        cell = tf.contrib.rnn.LSTMCell(num_units=hidden_units_enc, state_is_tuple=True)
        enc_rnn_outputs, enc_rnn_states = tf.nn.dynamic_rnn(
            cell=cell,
            dtype=tf.float32,
            #sequence_length=[seq_length]*batch_size,
            inputs=input_seq_enc)

        z_mean = tf.layers.dense(enc_rnn_states[1], emb_dim)
        z_log_sigma_sq = tf.layers.dense(enc_rnn_states[1], emb_dim)

        # Draw one sample z from Gaussian distribution with mean 0 and std 1
        eps = tf.random_normal((batch_size, emb_dim), 0, 1, dtype=tf.float32)

        # z = mu + sigma*epsilon
        latent_emb = tf.add(z_mean, tf.multiply(tf.exp(tf.multiply(z_log_sigma_sq,0.5)), eps))

        latent_loss = mult * (-0.5) * tf.reduce_sum(1 + z_log_sigma_sq
                                       - tf.square(z_mean)
                                       - tf.exp(z_log_sigma_sq), 1)

        latent_loss = tf.reduce_mean(latent_loss)

        return input_seq_enc, enc_rnn_outputs, enc_rnn_states, latent_emb, latent_loss

########################
# DECODER
########################

def decoder(hidden_units_dec, latent_emb, input_seq_enc):

    with tf.variable_scope("decoder") as scope:

        W_out_dec = tf.Variable(tf.truncated_normal([hidden_units_dec,num_features]))
        b_out_dec = tf.Variable(tf.truncated_normal([num_features]))

        dec_inputs = tf.zeros(tf.shape(input_seq_enc))

        #use latent embedding as inputs
        #dec_inputs = tf.layers.dense(latent_emb, latent_emb.shape[1].value, activation=tf.nn.tanh)
        #dec_inputs = tf.tile(dec_inputs, [1, seq_length])
        #dec_inputs = tf.reshape(dec_inputs, [batch_size, seq_length, latent_emb.shape[1].value])

        dec_initial_state = tf.layers.dense(latent_emb, hidden_units_dec, activation=tf.nn.tanh)

        #init_state = tf.contrib.rnn.LSTMStateTuple(tf.zeros([batch_size, hidden_units_dec]),tf.zeros([batch_size, hidden_units_dec]))
        #init_state = tf.contrib.rnn.LSTMStateTuple(dec_initial_state, dec_initial_state)

        cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden_units_dec)
        dec_rnn_outputs, dec_rnn_states = tf.nn.dynamic_rnn(
            cell=cell,
            dtype=tf.float32,
            #sequence_length=[seq_length]*batch_size,
            initial_state=dec_initial_state,
            inputs=dec_inputs)
        rnn_outputs_2d = tf.reshape(dec_rnn_outputs, [-1, hidden_units_dec])
        logits_2d = tf.matmul(rnn_outputs_2d, W_out_dec) + b_out_dec
        output_3d = tf.reshape(logits_2d, [-1, seq_length, num_features])
        #output_3d = tf.tanh(output_3d)
        reconstruction_loss = tf.reduce_mean(tf.square(tf.subtract(output_3d,input_seq_enc)))
        #reconstruction_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits_3d, labels=input_seq_enc))

    return reconstruction_loss, output_3d

########################
# TRAINING
########################

learning_rate = [args.learning_rate]
delta_error = [0]
optimizer_str = [args.optimizer_str]
hidden_units_dec = [args.hidden_units_dec]
hidden_units_enc = [args.hidden_units_enc]
emb_dim = [args.emb_dim]
mult = [args.mult]

configs = itertools.product(learning_rate, delta_error, optimizer_str, hidden_units_dec, hidden_units_enc, emb_dim, mult)
config_keys = ['learning_rate', 'delta_error', 'optimizer_str', 'hidden_units_dec', 'hidden_units_enc', 'emb_dim', 'mult']
verbose = 3

max_epochs=10000
patience=5
minibatch_size_train=batch_size
minibatch_size_validation=batch_size
minibatch_size_test=batch_size

for config in configs:

    num_mini_batches_train = int(math.ceil(len(inputs_train) / float(minibatch_size_train)))
    num_mini_batches_validation = int(math.ceil(len(inputs_validation) / float(minibatch_size_validation)))
    num_mini_batches_test = int(math.ceil(len(inputs_test) / float(minibatch_size_test)))

    experiment_random_id = str(int(np.random.rand(1)*1000000))
    config_id = str(config).replace(", ", "_").replace("'", "")[1:-1] + "_" + experiment_random_id
    if verbose > 0:
        print(config_id)

    tf.reset_default_graph()

    learning_rate = config[config_keys.index('learning_rate')]
    delta_error = config[config_keys.index('delta_error')]
    optimizer_str = config[config_keys.index('optimizer_str')]

    with tf.variable_scope("trainer"):

        hidden_units_enc = config[config_keys.index('hidden_units_enc')]
        hidden_units_dec = config[config_keys.index('hidden_units_dec')]
        emb_dim = config[config_keys.index('emb_dim')]
        mult = config[config_keys.index('mult')]

        input_seq_enc, enc_rnn_outputs, enc_rnn_states, latent_emb, latent_loss = encoder(hidden_units_enc, emb_dim, mult)
        # when the network has the same length enc_rnn_states[1] == enc_rnn_outputs[:,-1,:]
        #in LSTM enc_rnn_states[0] is c; enc_rnn_states[1] is h
        #latent_emb = enc_rnn_states[1]
        reconstruction_loss, output_3d_pred = decoder(hidden_units_dec, latent_emb, input_seq_enc)
        cost = reconstruction_loss + latent_loss

        global_step = tf.Variable(np.int64(0), name='global_step', trainable=False)

        if (optimizer_str == "adagrad_epochs") or (optimizer_str == "adagrad_minibatch_iterations"):
            train = tf.train.AdagradDAOptimizer(learning_rate, global_step).minimize(cost)
        elif optimizer_str == "rmsprop":
            train = tf.train.RMSPropOptimizer(learning_rate).minimize(cost)
        elif optimizer_str == "adam":
            train = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
        elif optimizer_str == "sgd":
            train = tf.train.RMSPropOptimizer(learning_rate).minimize(cost)

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    saver = tf.train.Saver()

    #train
    global_steps_count = 0
    keep_training = True
    epoch_counter = 0
    patience_counter = 0
    best_val_cost = 9999999999

    costs_train = []
    costs_val = []
    costs_test = []
    saved = False

    #Initial costs

    # start training
    weighted_cost_sum = 0
    for mbi in range(num_mini_batches_train):
        input_ = get_batch(inputs_train, mbi, minibatch_size_train)
        # FIX THIS! deal with last samples available in the set
        if len(input_) == batch_size:
            feed_dict = {input_seq_enc:input_, global_step:global_steps_count}
            res = sess.run([cost], feed_dict=feed_dict)
            weighted_cost_sum += res[0]*len(input_)
    cost_train = weighted_cost_sum / len(inputs_train)
    costs_train.append(cost_train)
    if verbose > 1:
        print(cost_train)

    # validation cost
    weighted_cost_sum = 0
    for mbi in range(num_mini_batches_validation):
        input_ = get_batch(inputs_validation, mbi, minibatch_size_validation)
        if len(input_) == batch_size:
            feed_dict = {input_seq_enc:input_, global_step:global_steps_count}
            res = sess.run([cost], feed_dict=feed_dict)
            weighted_cost_sum += res[0]*len(input_)
    cost_val = weighted_cost_sum / len(inputs_validation)
    costs_val.append(cost_val)
    if verbose > 1:
        print(cost_val)

    # compute test cost
    # this should be optional since it is not needed in every epoch
    # I am doing it to get the learning curve
    weighted_cost_sum = 0
    for mbi in range(num_mini_batches_test):
        input_ = get_batch(inputs_test, mbi, minibatch_size_test)
        if len(input_) == batch_size:
            feed_dict = {input_seq_enc:input_, global_step:global_steps_count}
            res = sess.run([cost], feed_dict=feed_dict)
            weighted_cost_sum += res[0]*len(input_)
    cost_test = weighted_cost_sum / len(inputs_test)
    costs_test.append(cost_test)
    if verbose > 1:
        print(cost_test)

    print("++++++++++++++++++++++++++++++++++")

    # start training

    while keep_training:
        time_start = time.time()
        #shuffle data
        np.random.shuffle(inputs_train)

        weighted_cost_sum = 0
        for mbi in range(num_mini_batches_train):
            input_ = get_batch(inputs_train, mbi, minibatch_size_train)
            # FIX THIS! deal with last samples available in the set
            if len(input_) == batch_size:
                feed_dict = {input_seq_enc:input_, global_step:global_steps_count}
                res = sess.run([train, cost, reconstruction_loss, latent_loss], feed_dict=feed_dict)
                if config[2] == "adagrad_epochs":
                    global_steps_count += 1
                # since last minibatch can have different lenth, we compute the mean cost as a
                # weighted mean
                weighted_cost_sum += res[1]*len(input_)
        cost_train = weighted_cost_sum / len(inputs_train)
        costs_train.append(cost_train)
        if verbose > 1:
            print(res[2])
            print(res[3])
            print(cost_train)
            print("-")

        # validation cost
        weighted_cost_sum = 0
        for mbi in range(num_mini_batches_validation):
            input_ = get_batch(inputs_validation, mbi, minibatch_size_validation)
            if len(input_) == batch_size:
                feed_dict = {input_seq_enc:input_, global_step:global_steps_count}
                res = sess.run([cost], feed_dict=feed_dict)
                weighted_cost_sum += res[0]*len(input_)
        cost_val = weighted_cost_sum / len(inputs_validation)
        costs_val.append(cost_val)
        if verbose > 1:
            print(cost_val)

        # compute test cost
        # this should be optional since it is not needed in every epoch
        # I am doing it to get the learning curve
        weighted_cost_sum = 0
        for mbi in range(num_mini_batches_test):
            input_ = get_batch(inputs_test, mbi, minibatch_size_test)
            if len(input_) == batch_size:
                feed_dict = {input_seq_enc:input_, global_step:global_steps_count}
                res = sess.run([cost], feed_dict=feed_dict)
                weighted_cost_sum += res[0]*len(input_)
        cost_test = weighted_cost_sum / len(inputs_test)
        costs_test.append(cost_test)

        # check patience
        if cost_val <= best_val_cost - delta_error:
            best_val_cost = cost_val
            patience_counter = 0
            save_path = saver.save(sess, "./" + experiment_id + "/" + config_id + "_model.ckpt")
            saved = True
        else:
            patience_counter += 1
            if patience_counter > patience:
                keep_training = False
                if saved:
                    saver.restore(sess, "./" + experiment_id + "/" + config_id + "_model.ckpt")

        epoch_counter += 1
        if config[2] == "adagrad_minibatch_iterations":
            global_steps_count = epoch_counter
        if epoch_counter >= max_epochs:
            keep_training = False

        if verbose > 1:
            print(time.time() - time_start)
            print('--------------------')

        print((inputs_train[0] == inputs_train[1]).all())

    # save learning curve plots
    plt.figure()
    plt.xlabel('Epoch')
    plt.ylabel('Cost')
    plt.title('Learning curves')
    plt.plot(range(len(costs_train)), costs_train, label='training')
    plt.plot(range(len(costs_val)), costs_val, label='validation')
    plt.plot(range(len(costs_test)), costs_test, label='test')
    plt.legend(loc="upper right")
    plt.savefig("./" + experiment_id + "/" + config_id + "_learning_curves.png", dpi=300)

    # validation costs
    predicted_values = []
    predicted_values_not_flatten = []
    true_values = []
    true_values_not_flatten = []
    other_scores_validation = []
    weighted_cost_sum = 0
    for mbi in range(num_mini_batches_validation):
        input_ = get_batch(inputs_validation, mbi, minibatch_size_validation)
        if len(input_) == batch_size:
            feed_dict = {input_seq_enc:input_, global_step:global_steps_count}
            res = sess.run([cost], feed_dict=feed_dict)
            weighted_cost_sum += res[0]*len(input_)
    cost_val = weighted_cost_sum / len(inputs_validation)
    costs_val.append(cost_val)
    other_scores_validation = {}

    # test costs
    predicted_values = []
    predicted_values_not_flatten = []
    true_values = []
    true_values_not_flatten = []
    other_scores_test = []
    weighted_cost_sum = 0
    for mbi in range(num_mini_batches_test):
        input_ = get_batch(inputs_test, mbi, minibatch_size_test)
        if len(input_) == batch_size:
            feed_dict = {input_seq_enc:input_, global_step:global_steps_count}
            res = sess.run([cost], feed_dict=feed_dict)
            weighted_cost_sum += res[0]*len(input_)

    cost_test = weighted_cost_sum / len(inputs_test)
    costs_test.append(cost_test)
    other_scores_test = {}

    total_time = time.time() - time_start

    # need to convert values to float64 to be able to serialize them
    to_store = {'config': config, 'costs_train': costs_train, 'costs_val': costs_val, 'costs_test': costs_test,
               'best_val_cost': best_val_cost, 'total_time': total_time, 'random_seed': random_seed,
                'experiment_random_id': experiment_random_id, 'other_scores_validation': other_scores_validation,
               'other_scores_test': other_scores_test}

    with open("./" + experiment_id + "/" + config_id + ".json", "w") as f:
        json.dump(to_store, f)

    if verbose > 0:
        print("==========================")
        print(cost_val)
        print(best_val_cost)
        print(cost_test)
        print("==========================")

print((inputs_train[0] == inputs_train[1]).all())

########################
# SYNTHETIC SAMPLES GENERATION
########################

# Generate new samples
generated_samples = []
for i in range(num_mini_batches_train):
    feed_dict = {latent_emb:np.random.normal(size=(32,emb_dim))}
    res = sess.run([output_3d_pred], feed_dict=feed_dict)
    generated_samples.append(res[0])
generated_samples = np.vstack(generated_samples)

save_plot_sample(generated_samples[0:7], '_' + config_id + '_generated', 'test_RVAE', path=experiment_id)

input_ = get_batch(inputs_train, 0, minibatch_size_train)

save_plot_sample(input_[0:7], '_' + config_id + '_input', 'test_RVAE', path=experiment_id)

feed_dict = {input_seq_enc:input_}
res = sess.run([output_3d_pred], feed_dict=feed_dict)

save_plot_sample(res[0][0:7], '_' + config_id + '_reconstuction', 'test_RVAE', path=experiment_id)

max_stars_repo_name: cliohong/RGAN


text:
# System-related & MQTT imports
import time
import sys, os, pickle, argparse, subprocess
from tqdm import tqdm
from profilehooks import timecall
import queue
import paho.mqtt.client as mqtt

# Synthesizer imports
import synthesizer
from synthesizer import inference as sif
import numpy as np
import cv2
from shutil import copy
from glob import glob
from scipy.io import wavfile
import io

##############
# Parameters #
##############

parser = argparse.ArgumentParser()

# Synthesizer params
parser.add_argument("--checkpoint", help="Path to trained checkpoint", required=True)
parser.add_argument("--preset", help="Speaker-specific hyper-params", type=str, required=True)
parser.add_argument("--wav_action", help="What to do with the generated wav files", type=str, required=False, choices=["save", "forward"], default="forward")
parser.add_argument("--results_root", help="Speaker folder path, only needed if wav_action=='save'", required=False)
parser.add_argument("--method_of_synthesis", help="The method of synthesis to used (cpu-based or gpu-based) to generate wav files", type=str, required=False, choices=["cpu", "gpu"], default="gpu")

# Subscribing client params
parser.add_argument("--sub_client_name", help="The name of the MQTT subscribing client", type=str, required=False, default="jetson-face-receiver")
parser.add_argument("--sub_mqtt_host", help="The MQTT host for the subscribing client", type=str, required=True)
parser.add_argument("--sub_mqtt_port", help="The MQTT port for the subscribing client", type=int, required=False, default=1883)
parser.add_argument("--sub_qos", help="The MQTT quality of service for the subscribing client", type=int, required=False, default=2)
parser.add_argument("--sub_topic", help="The MQTT topic the subscribing client should subscribe to", type=str, required=False, default="jetson/faces")

# Publishing client params
parser.add_argument("--pub_client_name", help="The name of the MQTT publishing client", type=str, required=False, default="jetson-audio-sender")
parser.add_argument("--pub_mqtt_host", help="The MQTT host for the publishing client", type=str, required=True)
parser.add_argument("--pub_mqtt_port", help="The MQTT port for the publishing client", type=int, required=False, default=1883)
parser.add_argument("--pub_qos", help="The MQTT quality of service for the publishing client", type=int, required=False, default=2)
parser.add_argument("--pub_topic", help="The MQTT topic the publishing client should publish to", type=str, required=False, default="jetson/audio")

args = parser.parse_args()

##############
# Core Logic #
##############

# Do param-based initializations
with open(args.preset) as f:
   sif.hparams.parse_json(f.read()) ## add speaker-specific parameters
sif.hparams.set_hparam('eval_ckpt', args.checkpoint)

if (args.wav_action == "save"):
   WAVS_ROOT = os.path.join(args.results_root, 'wavs/')
   if not os.path.isdir(WAVS_ROOT):
      os.mkdir(WAVS_ROOT)

# Set params for processing
num_frames = sif.hparams.T

# Define a frame queue
face_queue = queue.Queue()

def on_log(client, userdata, level, buf):
   print(buf)

def on_connect(client, userdata, flags, rc):
   if (rc == 0):
      print("connected OK")
   else:
      print("Bad connection retruned code = ", rc)
      client.loop_stop()

def on_disconnect(client, userdata, rc):
   print("client disconnected ok")

#def on_publish(client, userdata, mid):
#   print("in on_publish callback mid = ", mid)

def on_subscribe(client, userdata, mid, granted_qos):
   print("subscribed")

def on_message(client, userdata, message):
   # Put frame to python queue to be processed when batches of num_frames are available
   face = np.asarray(bytearray(message.payload), dtype="uint8")
   face = cv2.imdecode(face, cv2.IMREAD_COLOR)
   face_queue.put(face, block=True)

# Set up receiver client & callbacks
receiver_client = mqtt.Client(args.sub_client_name)
receiver_client.on_log = on_log
receiver_client.on_connect = on_connect
receiver_client.on_disconnect = on_disconnect
receiver_client.on_message = on_message
receiver_client.on_subscribe = on_subscribe
receiver_client.connect(args.sub_mqtt_host, args.sub_mqtt_port)

# Set up sender client & callbacks
sender_client = mqtt.Client(args.pub_client_name)
#sender_client.on_log = on_log
sender_client.on_connect = on_connect
sender_client.on_disconnect = on_disconnect
#sender_client.on_publish = on_publish
sender_client.connect(args.pub_mqtt_host, args.pub_mqtt_port)

# start clients & subscribe receiver client to topic
receiver_client.loop_start()
sender_client.loop_start()
receiver_client.subscribe(args.sub_topic, args.sub_qos)

class Generator(object):
   def __init__(self, cpu_based):
      super(Generator, self).__init__()
      self.cpu_based = cpu_based

      self.synthesizer = sif.Synthesizer(verbose=False)
      self.synthesizer.load(cpu_based=self.cpu_based)

      # for CPU-based approach
      self.mel_batches_per_wav_file = 1
      self.mel_batch = None
      self.num_mels = 0

   # Run a single round of inference to force model init
   def force_model_init(self):
      # use the same face for simplicity--the inference results doesn't need to be reasonable
      model_init_face = "./forced_model_init_face.jpg"
      fnames = [model_init_face for i in range(0, num_frames)]
      assert len(fnames) == num_frames

      images = [cv2.imread(fname, cv2.IMREAD_COLOR) for fname in fnames]
      self.generate_wav(images)

   def resize_and_nparrize_images(self, images):
      images = [cv2.resize(img, (sif.hparams.img_size, sif.hparams.img_size)) for img in images]
      images = np.asarray(images) / 255.
      return images

   def post_process_wav(self, wav):
      wav *= 32767 / max(0.01, np.max(np.abs(wav)))
      return wav

   def generate_mel_spec(self, images):
      # Synthesize Spectrogram
      mel_spec = self.synthesizer.synthesize_spectrograms(images)[0]

      # Concatenate batches of mel spectrograms (to get longer wav file samples)
      if self.num_mels == 0:
         self.mel_batch = mel_spec
         self.num_mels = 1
      else:
         self.mel_batch = np.concatenate((self.mel_batch, mel_spec[:, sif.hparams.mel_overlap:]), axis=1)
         self.num_mels += 1

   @timecall(immediate=True)
   def generate_wav_cpu_based(self, images):
      '''
      CPU-based method of converting batches of face images to wav files
      '''
      # Generate mel spectrogram first
      images = self.resize_and_nparrize_images(images)
      self.generate_mel_spec(images)

      # Synthesize wav file from spectrogram when ready
      if (self.num_mels != self.mel_batches_per_wav_file):
         print("not generating wav file yet...")
         return None
      else:
         print("Generating wav file of mel spectrograms")
         wav = self.synthesizer.griffin_lim(self.mel_batch)
         wav = self.post_process_wav(wav)

         self.num_mels = 0
         self.mel_batch = None
         return wav

   @timecall(immediate=True)
   def generate_wav_gpu_based(self, images):
      '''
      GPU-based method of converting batches of face images to wav files
      '''
      images = self.resize_and_nparrize_images(images)
      wav = self.synthesizer.synthesize_wavs(images)
      wav = self.post_process_wav(wav)
      return wav

   def generate_wav(self, images):
      if (self.cpu_based):
         return self.generate_wav_cpu_based(images)
      else:
         return self.generate_wav_gpu_based(images)

   def generate_wav_and_save(self, images, root_dir, wav_num):
      '''
      Generates wav files from batches of images and saves the wav to an output file
      '''
      wav = self.generate_wav(images)
      if (wav is None):
         return # not ready yet
      else:
         print("saving wav file")
         outfile = '{}{}.wav'.format(root_dir, wav_num)
         sif.audio.save_wav(wav, outfile, sr=sif.hparams.sample_rate)

   # Inspiration from here: https://gist.github.com/hadware/8882b980907901426266cb07bfbfcd20
   def generate_wav_and_forward(self, images, mqtt_client, topic, qos):
      '''
      Generates wav files from batches of images and forwards them via MQTT
      '''
      wav = self.generate_wav(images)
      if (wav is None):
         return # not ready yet
      else:
         print("forwarding wav file via MQTT")
         byte_io = io.BytesIO(bytes())
         wavfile.write(byte_io, sif.hparams.sample_rate, wav)
         wav_bytes = byte_io.read()
         mqtt_client.publish(topic, payload=wav_bytes, qos=qos)

def process_faces():
   # Initialize audio generator
   generator = Generator(cpu_based = args.method_of_synthesis == "cpu")
   generator.force_model_init()

   # Wait for messages until disconnected by system interrupt
   print("\n########################\n Ready to receive faces \n########################\n")
   audio_sample_num = 1
   num_frames = sif.hparams.T
   while True:
      print("queue size = " + str(face_queue.qsize()))

      # Check to see if queue has enough frames
      if (face_queue.qsize() >= num_frames):
         print("reached " + str(num_frames) + " frames")

         # Fetch num_frames faces to process from the queue
         faces_to_process = []
         while (len(faces_to_process) != num_frames):
            faces_to_process.append(face_queue.get(block=True))

         # Process frames and generate synthesized audio as wav file data
         # Save as wav file or forward via mqtt
         try:
            if (args.wav_action == "save"):
               generator.generate_wav_and_save(faces_to_process, WAVS_ROOT, audio_sample_num)
            elif (args.wav_action == "forward"):
               generator.generate_wav_and_forward(faces_to_process, sender_client, args.pub_topic, args.pub_qos)

            audio_sample_num += 1
         except KeyboardInterrupt:
            exit(0)
         '''
         except Exception as e:
            print(e)
            continue
         '''

# Run the process face function
process_faces()

reason: import numpy,from scipy


text:
import gym, os, time, random, sys
import tensorflow as tf
import numpy as np

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d

from agents import make_agent, get_agent_class

dir = os.path.dirname(os.path.realpath(__file__))

flags = tf.app.flags

# HP search
flags.DEFINE_boolean('randomsearch', False, 'Perform a random search fixing one HP at a time')
flags.DEFINE_boolean('fullsearch', False, 'Perform a full search of hyperparameter space (hyperband -> lr search -> hyperband with best lr)')
flags.DEFINE_string('fixed_params', "{}", 'JSON inputs to fix some params in a random search, ex: \'{"lr": 0.001}\'')
# Hyperband
flags.DEFINE_boolean('hyperband', False, 'Perform a hyperband search of hyperparameters')
flags.DEFINE_boolean('dry_run', False, 'Perform a hyperband dry_run')
flags.DEFINE_integer('nb_process', 4, 'Number of parallel process to perform a hyperband search')
flags.DEFINE_integer('games_per_epoch', 100, 'Number of parallel process to perform a hyperband search')

# Agent
flags.DEFINE_string('agent_name', 'DQNAgent', 'Name of the agent')
flags.DEFINE_boolean('best', False, 'Use the best known configuration')
flags.DEFINE_float('initial_q_value', 0., 'Initial Q values in the Tabular case')
flags.DEFINE_float('initial_mean', 0., 'Initial mean for NN')
flags.DEFINE_float('initial_stddev', 1e-2, 'Initial standard deviation for NN')
flags.DEFINE_float('lambda', .9, 'Lambda parameters used with eligibility traces')
flags.DEFINE_float('discount', .999, 'Discount factor')
flags.DEFINE_float('lr', 1e-3, 'Learning rate')
flags.DEFINE_integer('lr_decay_steps', 50000, 'Learning rate decay steps for tabular methods')
flags.DEFINE_integer('nb_units', 20, 'Number of hidden units in Deep learning agents')
flags.DEFINE_float('q_scale_lr', 1., 'For actor critic agents, scale variables between q loss and policy loss')
flags.DEFINE_integer('n_step', 4, 'Number of step used in TD(n) algorithm')

# Policy
flags.DEFINE_integer('N0', 100, 'Offset used in the decay algorithm of epsilon')
flags.DEFINE_float('min_eps', 1e-2, 'Limit after which the decay stops')

# Experience replay
flags.DEFINE_integer('er_batch_size', 512, 'Batch size of the experience replay learning')
flags.DEFINE_integer('er_epoch_size', 50, 'Number of sampled contained in an epoch of experience replay')
flags.DEFINE_integer('er_rm_size', 20000, 'Size of the replay memory buffer')
flags.DEFINE_integer('update_every', 20, 'Update the fixed Q network every chosen step')

# Environment
flags.DEFINE_string('env_name', 'CartPole-v0', 'The name of gym environment to use')
flags.DEFINE_boolean('debug', False, 'Debug mode')
flags.DEFINE_integer('max_iter', 2000, 'Number of training step')

flags.DEFINE_string('result_dir', dir + '/results/' + flags.FLAGS.env_name + '/' + flags.FLAGS.agent_name + '/' + str(int(time.time())), 'Name of the directory to store/log the agent (if it exists, the agent will be loaded from it)')

flags.DEFINE_boolean('play', False, 'Load an agent for playing')
flags.DEFINE_integer('play_nb', 10, 'Number of games to play')
flags.DEFINE_integer('random_seed', random.randint(0, sys.maxsize), 'Value of random seed')

def main(_):
    config = flags.FLAGS.__flags.copy()

    env = gym.make(config['env_name'])
    agent = make_agent(config, env)

    qs = agent.sess.run(agent.Qs)

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')

    x = np.linspace(0, len(qs) - 1, len(qs))
    y = np.linspace(0, len(qs[0]) - 1, len(qs[0]))
    xpos, ypos = np.meshgrid(x, y, indexing='ij')
    xpos = xpos.flatten('F')
    ypos = ypos.flatten('F')
    num_elements = len(qs) * len(qs[0])
    zpos = np.zeros(num_elements)
    dx = np.ones(num_elements)
    dy = np.ones(num_elements)
    dz = np.abs(np.array(qs).flatten('F'))

    ax.bar3d(xpos, ypos, zpos, dx, dy, dz, color='#00ceaa')
    plt.savefig('test.png')

if __name__ == '__main__':
    tf.app.run()

max_stars_repo_name: morgangiraud/openai-rl

max_issues_repo_name: morgangiraud/openai-rl

max_forks_repo_name: morgangiraud/openai-rl


text:
import numpy as np
import pandas as pd
import pickle

class Qtable:
    def __init__(self, state_size, action_size, zeros=True, minqval=None, maxqval=None, qtable=None):
        """
        If zeros is set to False, then the q-table will be initiated with random values between 0 and 1. If min and
        max values are set, the table is filled with uniform distributed values between the min and max.
        """
        self.state_size = state_size
        self.action_size = action_size
        self.minqval = minqval
        self.maxqval = maxqval
        if qtable is not None:
            self.qtable = qtable
        else:
            if zeros is True:
                self.qtable = np.zeros((state_size, action_size))

            elif minqval is None and maxqval is None and zeros is False:
                self.qtable = np.random.random((state_size, action_size))

            else:
                self.qtable = np.random.uniform(minqval, maxqval, (state_size, action_size))

    def get_qtable(self):
        return self.qtable

    def set_qtable(self, qtable):
        self.qtable = qtable

    def get_qval(self, state, action):
        """
        Returns the q-vlaue given a state acion pair.
        """
        try:
            return self.qtable[state, action]
        except IndexError:
            if action >= self.action_size:
                print("Index error: No such action.")

            elif state >= self.state_size:
                print("Index error: No such state.")

            elif state >= self.state_size and action >= self.action_size:
                print("Index error: No such state nor such action.")

    def set_qval(self, state, action, qval):
        """
        Assign a q-value to a state action pair.
        """
        try:
            self.qtable[state, action] = qval
        except IndexError:
            if action >= self.action_size:
                print("Index error: No such action.")
            elif state >= self.state_size:
                print("Index error: No such state.")
            elif state >= self.state_size and action >= self.action_size:
                print("Index error: No such state nor such action.")

    def maxq(self, state):
        """
        Returns the maximum q-value for a given state.
        """
        try:
            return np.max(self.qtable[state])
        except IndexError:
            print("Index error: No such state.")

    def optaction(self, state):
        """
        Returns the action associated with the highest q-value given a state
        """
        try:
            return np.argmax(self.qtable[state])
        except IndexError:
            print("Index error: No such state.")

    def random_action(self, state):
        """
        Returns a random action to be taken for a given state
        """
        try:
            return np.random.randint(0, self.action_size)
        except IndexError:
            print("Index error: No such state.")

    def optsequence(self):
        """
        Returns the sequence of actions associated with the highest q-values for each state in  order from 0-state_siz
        """
        return np.argmax(self.qtable, axis=1)

    def save_qtable(self, episode):
        """
        Save qtable object in pickle file.
        """
        with open(f'q_table_episode_{episode}.pkl', 'wb') as pkl:
            pickle.dump(self, pkl)
            pkl.close()


text:
import numpy as np
import pytest

def test_floating_point_associativity():
    eps = np.finfo('double').eps

    x = (eps + 1) + (1 + eps)
    y = eps + (1 + 1) + eps

    assert x != y
    assert x == pytest.approx(y)

max_stars_repo_name: wendazhou/cds-bootcamp

max_issues_repo_name: wendazhou/cds-bootcamp

max_forks_repo_name: wendazhou/cds-bootcamp


text:
import numpy as np
import party
import mediator
import interface

n = 10
k = 9

#  generate secret values
secret_values = 1 - np.random.random_sample(n)
public_vector = 1 - np.random.random_sample(n)

#  generate parties
parties = []
for i in range(n):
    parties.append(party.Party(party_index=i, public_vector=public_vector, secret_value=secret_values[i], k=k))

#  generate mediator
mediator = mediator.Mediator(public_vector, k)

#  generate interface
interface = interface.Interface(parties, mediator)

#  run secure aggregation
aggregation_result = interface.perform_secure_aggregation()

print("Comparison (aggregation_result, np.sum(secret_values))", aggregation_result, np.sum(secret_values))

max_stars_repo_name: AminAminifar/Shamir

max_issues_repo_name: AminAminifar/Shamir

max_forks_repo_name: AminAminifar/Shamir


text:
import numpy as np
import networkx as nx
import cPickle as cp
import random
import ctypes
import os
import sys
from tqdm import tqdm

sys.path.append( '%s/tsp2d_lib' % os.path.dirname(os.path.realpath(__file__)) )
from tsp2d_lib import Tsp2dLib

n_valid = 100

def find_model_file(opt):
    max_n = int(opt['max_n'])
    min_n = int(opt['min_n'])
    log_file = None
    if max_n < 100:
        return None
    if min_n == 100 and max_n == 200:
        n1 = 50
        n2 = 100
    else:
        n1 = min_n - 100
        n2 = max_n - 100

    log_file = '%s/log-%d-%d.txt' % (opt['save_dir'], n1, n2)
    if not os.path.isfile(log_file):
        return None
    best_r = 1000000
    best_it = -1
    with open(log_file, 'r') as f:
        for line in f:
            if 'average' in line:
                line = line.split(' ')
                it = int(line[1].strip())
                r = float(line[-1].strip())
                if r < best_r:
                    best_r = r
                    best_it = it
    if best_it < 0:
        return None
    return '%s/nrange_%d_%d_iter_%d.model' % (opt['save_dir'], n1, n2, best_it)

def PrepareGraphs(isValid):
    if isValid:
        n_graphs = 100
        prefix = 'validation_tsp2d'
    else:
        n_graphs = 10000
        prefix = 'train_tsp2d'
    folder = '%s/%s/tsp_min-n=%s_max-n=%s_num-graph=%d_type=%s' % (opt['data_root'], prefix, opt['min_n'], opt['max_n'], n_graphs, opt['g_type'])

    with open('%s/paths.txt' % folder, 'r') as f:
        for line in tqdm(f):
            fname = '%s/%s' % (folder, line.split('/')[-1].strip())
            coors = {}
            in_sec = False
            n_nodes = -1
            with open(fname, 'r') as f_tsp:
                for l in f_tsp:
                    if 'DIMENSION' in l:
                        n_nodes = int(l.split(' ')[-1].strip())
                    if in_sec:
                        idx, x, y = [int(w.strip()) for w in l.split(' ')]
                        coors[idx - 1] = [float(x) / 1000000.0, float(y) / 1000000.0]
                        assert len(coors) == idx
                    elif 'NODE_COORD_SECTION' in l:
                        in_sec = True
            assert len(coors) == n_nodes
            g = nx.Graph()
            g.add_nodes_from(range(n_nodes))
            nx.set_node_attributes(g, 'pos', coors)
            api.InsertGraph(g, is_test=isValid)

if __name__ == '__main__':
    api = Tsp2dLib(sys.argv)

    opt = {}
    for i in range(1, len(sys.argv), 2):
        opt[sys.argv[i][1:]] = sys.argv[i + 1]

    model_file = find_model_file(opt)
    if model_file is not None:
        print 'loading', model_file
        sys.stdout.flush()
        api.LoadModel(model_file)

    PrepareGraphs(isValid=True)
    PrepareGraphs(isValid=False)

    # startup
    for i in range(10):
        api.lib.PlayGame(100, ctypes.c_double(1.0))
    api.TakeSnapshot()

    eps_start = 1.0
    eps_end = 1.0

    eps_step = 10000.0
    api.lib.SetSign(1)

    lr = float(opt['learning_rate'])
    for iter in range(int(opt['max_iter'])):
        eps = eps_end + max(0., (eps_start - eps_end) * (eps_step - iter) / eps_step)
        if iter % 10 == 0:
            api.lib.PlayGame(10, ctypes.c_double(eps))

        if iter % 100 == 0:
            frac = 0.0
            for idx in range(n_valid):
                frac += api.lib.Test(idx)
            print 'iter', iter, 'lr', lr, 'eps', eps, 'average tour length: ', frac / n_valid
            sys.stdout.flush()
            model_path = '%s/nrange_%d_%d_iter_%d.model' % (opt['save_dir'], int(opt['min_n']), int(opt['max_n']), iter)
            api.SaveModel(model_path)

        if iter % 1000 == 0:
            api.TakeSnapshot()
            lr = lr * 0.95

        api.lib.Fit(ctypes.c_double(lr))

reason: import numpy,import networkx


text:
import argparse
import csv
import glob
import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import cv2
import numpy as np
import tensorflow as tf

from cv_object_detector import cvDetector
from tf2_object_detector import tf2_Detector

class capDetector:
    def __init__(self, video_path: str, result_path: str, frozengraph_path: str,
                 config_path: str, ckpt_path: str, tf_version: int):
        self.video_path = video_path
        self.video_name = os.path.splitext(os.path.basename(self.video_path))[0]
        self.result_path = result_path
        self.frozen_graph = frozengraph_path
        self.pbtxt = config_path
        self.threshold = 0.41
        self.tf_version = tf_version
        self.ckpt_path = ckpt_path

        self.label_dict = {1: "Tray",
                           2: "BottleCap_FaceUp",
                           3: "BottleCap_FaceDown",
                           4: "BottleCap_Deformed"}
        if tf_version == 2:
            self.cap_det = tf2_Detector(self.label_dict, self.ckpt_path,
                                        self.threshold)
        else:
            self.cap_det = cvDetector(self.label_dict, self.frozen_graph,
                                      self.pbtxt)

    def stable_frame_extractor(self, viz_image=False):

        videocap = cv2.VideoCapture(self.video_path)
        if not videocap.isOpened():
            print("Error opening video file")
        prev_frame = None
        buf_stable_frame = None
        prev_unique = 0
        rising_edge = False
        falling_edge = False

        length = int(videocap.get(cv2.CAP_PROP_FRAME_COUNT))

        for frame_n in range(length):
            success, cur_frame = videocap.read()
            if success:
                buf_stable_frame = cur_frame
                cur_frame = cv2.cvtColor(cur_frame, cv2.COLOR_BGR2GRAY)
                cur_frame = cv2.resize(cur_frame, (1024, 1024))
                cur_frame = cv2.blur(cur_frame, (5, 5))
                if frame_n == int(length / 2):
                    stable_frame = buf_stable_frame
                    self.frame_number = frame_n

                if prev_frame is not None:
                    diff = cv2.absdiff(prev_frame, cur_frame)
                    unique = len(np.unique(diff))
                    if unique - prev_unique > 30:
                        rising_edge = True
                    if prev_unique - unique > 30:
                        falling_edge = True
                    print("Frame No:", frame_n, " | ", unique)

                    if unique < 30 and rising_edge and falling_edge:
                        self.frame_number = frame_n
                        print(" The stable frame number is : " + str(frame_n))
                        stable_frame_count = frame_n
                        stable_frame = buf_stable_frame
                        break
                    prev_unique = unique
                prev_frame = cur_frame

        if viz_image:
            img_out = os.path.join(self.result_path, str(self.video_name +
                                                         '.png'))
            print(img_out)
            cv2.imwrite(img_out, stable_frame)
        return stable_frame

    def bbox_in_check(self, in_box):
        tray = self.cap_det.tray
        if in_box[0] > tray[0] and in_box[1] > tray[1]:
            if in_box[2] < tray[2] and in_box[3] < tray[3]:
                return True
        else:
            print("Box not in the Tray")
            return False

    def filter_detections(self, boxes_list, overlap_threshold=0.9):
        filtered_boxes = []
        for idx in range(len(boxes_list)):
            x_min = int(boxes_list[idx][0])
            y_min = int(boxes_list[idx][1])
            x_max = int(boxes_list[idx][2])
            y_max = int(boxes_list[idx][3])
            cls = str(boxes_list[idx][4])
            score = str(np.round(boxes_list[idx][-1], 2))
            if cls == "Tray":
                filtered_boxes.append([x_min, y_min, x_max, y_max, cls,
                                       float(score)])
                continue
            box = [x_min, y_min, x_max, y_max, cls, float(score)]
            if self.bbox_in_check(box):
                filtered_boxes.append(box)
        return filtered_boxes

    def perform_inference(self, viz_save=False, to_csv=True):
        self.stable_frame = self.stable_frame_extractor(viz_image=False)

        if self.tf_version == 2:
            self.det_array = self.cap_det.detect_from_image(self.stable_frame)
        else:
            image = self.cap_det.img_resizer(self.stable_frame, op_size=(1024,
                                                                         1024))
            detections = self.cap_det.detect_from_image(image=self.stable_frame)
            self.det_array = self.cap_det.provide_output(detections)

        filtered_boxes = self.filter_detections(self.det_array)
        img = self.display_detections(self.stable_frame, filtered_boxes,
                                      det_time=None)
        if to_csv:
            with open(os.path.join(self.result_path.rstrip(),
                                   str(self.video_name + ".csv")), 'w',
                      newline='') as f:
                writer = csv.writer(f, delimiter=',')
                all_results_array = []
                for arr in filtered_boxes:
                    result_str = ''
                    if arr[4] != "Tray":
                        result_str = result_str + str(self.frame_number) + ','
                        x_mid, y_mid = arr[0] + arr[2] / 2, arr[1] + arr[3] / 2
                        result_str = result_str + str(int(x_mid)) + ','
                        result_str = result_str + str(int(y_mid)) + ','
                        result_str = result_str + (arr[4])
                        all_results_array.append(result_str)
                writer.writerows([[item] for item in all_results_array])

        if viz_save:
            # cv2.imshow('TF2 Detection', img)
            img_out = os.path.join(self.result_path.rstrip(),
                                   str(self.video_name + ".png"))
            cv2.imwrite(img_out, img)
            # cv2.waitKey(0)

    def display_detections(self, image, boxes_list, det_time=None):
        image = self.stable_frame
        image_w, image_h = image.shape[0], image.shape[1]
        # scale_w, scale_h = image_w / 300.0, image_h / 300.0
        scale_w, scale_h = 1, 1
        if not boxes_list:
            print("No objects Found")
            return image  # input list is empty
        img = image.copy()
        for idx in range(len(boxes_list)):
            x_min = int(boxes_list[idx][0] * scale_h)
            y_min = int(boxes_list[idx][1] * scale_w)
            x_max = int(boxes_list[idx][2] * scale_h)
            y_max = int(boxes_list[idx][3] * scale_w)
            cls = str(boxes_list[idx][4])
            score = str(np.round(boxes_list[idx][-1], 2))

            text = cls + ": " + score
            cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0, 255, 0), 1)
            cv2.rectangle(img, (x_min, y_min - 20), (x_min, y_min), (255, 255,
                                                                     255), -1)
            cv2.putText(img, text, (x_min + 5, y_min - 7),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)

        if det_time is not None:
            fps = round(1000. / det_time, 1)
            fps_txt = str(fps) + " FPS"
            cv2.putText(img, fps_txt, (25, 30),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2)

        return img

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--video_path', action='store', type=str,
                        help='Path to Videos collected')
    parser.add_argument('--result_path', action='store', type=str,
                        help='Path in which results are stored')
    parser.add_argument('--to_show', action='store', default=False, type=bool,
                        help='Path to the config .pbtxt file')

    args = parser.parse_args()
    dirname = os.path.dirname(__file__)

    video_path = args.video_path
    result_path = args.result_path
    frozengraph_path = '../models/tf1_model/frozen_inference_graph.pb'
    config_path = '../models/tf1_model/sample.pbtxt'
    ckpt_path = os.path.join(dirname, '../models/tf2_model/saved_model/')
    to_show = args.to_show

    version = tf.__version__
    tf_version = int(version[0])
    # tf_version = 2

    videos = glob.glob(os.path.join(video_path, '*.mp4'))
    print("Number of videos present are {}".format(len(videos)))
    for video in videos:
        cap_det_model = capDetector(video_path=video, result_path=result_path,
                                     frozengraph_path=frozengraph_path,
                                     config_path=config_path,
                                     ckpt_path=ckpt_path, tf_version=tf_version)
        cap_det_model.perform_inference(viz_save=True, to_csv=True)

max_stars_repo_name: jaswanthbjk/BCDC-Net

max_issues_repo_name: jaswanthbjk/BCDC-Net

max_forks_repo_name: jaswanthbjk/BCDC-Net


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Network architecture definition for Singing Voice Detection experiment.

Author: Jan Schlüter
"""

import numpy as np
import theano
import theano.tensor as T
import lasagne
from lasagne.layers import (InputLayer, Conv2DLayer, MaxPool2DLayer,
                            DenseLayer, ExpressionLayer, dropout, batch_norm)
batch_norm_vanilla = batch_norm
try:
    from lasagne.layers.dnn import batch_norm_dnn as batch_norm
except ImportError:
    pass

class MelBankLayer(lasagne.layers.Layer):
    """
    Creates a mel filterbank layer of `num_bands` triangular filters, with
    the first filter initialized to start at `min_freq` and the last one
    to stop at `max_freq`. Expects to process magnitude spectra created
    from samples at a sample rate of `sample_rate` with a window length of
    `frame_len` samples. Learns a vector of `num_bands + 2` values, with
    the first value giving `min_freq` in mel, and remaining values giving
    the distance to the respective next peak in mel.
    """
    def __init__(self, incoming, sample_rate, frame_len, num_bands, min_freq,
                 max_freq, trainable=True, **kwargs):
        super(MelBankLayer, self).__init__(incoming, **kwargs)
        # mel-spaced peak frequencies
        min_mel = 1127 * np.log1p(min_freq / 700.0)
        max_mel = 1127 * np.log1p(max_freq / 700.0)
        spacing = (max_mel - min_mel) / (num_bands + 1)
        spaces = np.ones(num_bands + 2) * spacing
        spaces[0] = min_mel
        spaces = theano.shared(lasagne.utils.floatX(spaces))  # learned param
        peaks_mel = spaces.cumsum()

        # create parameter as a vector of real-valued peak bins
        peaks_hz = 700 * (T.expm1(peaks_mel / 1127))
        peaks_bin = peaks_hz * frame_len / sample_rate
        self.peaks = self.add_param(peaks_bin,
                shape=(num_bands + 2,), name='peaks', trainable=trainable,
                regularizable=False)

        # store what else is needed
        self.num_bands = num_bands

    def get_output_shape_for(self, input_shape):
        return input_shape[:-1] + (self.num_bands,)

    def get_output_for(self, input, **kwargs):
        num_bins = self.input_shape[-1] or input.shape[-1]
        x = T.arange(num_bins, dtype=input.dtype).dimshuffle(0, 'x')
        peaks = self.peaks
        l, c, r = peaks[0:-2], peaks[1:-1], peaks[2:]
        # triangles are the minimum of two linear functions f(x) = a*x + b
        # left side of triangles: f(l) = 0, f(c) = 1 -> a=1/(c-l), b=-a*l
        tri_left = (x - l) / (c - l)
        # right side of triangles: f(c) = 1, f(r) = 0 -> a=1/(c-r), b=-a*r
        tri_right = (x - r) / (c - r)
        # combine by taking the minimum of the left and right sides
        tri = T.minimum(tri_left, tri_right)
        # and clip to only keep positive values
        bank = T.maximum(0, tri)

        # the dot product of the input with this filter bank is the output
        return T.dot(input, bank)

class PowLayer(lasagne.layers.Layer):
    def __init__(self, incoming, exponent=lasagne.init.Constant(0), **kwargs):
        super(PowLayer, self).__init__(incoming, **kwargs)
        self.exponent = self.add_param(exponent, shape=(), name='exponent', regularizable=False)
    def get_output_for(self, input, **kwargs):
        return input ** self.exponent

class TimeDiffLayer(lasagne.layers.Layer):
    def __init__(self, incoming, delta=1, **kwargs):
        super(TimeDiffLayer, self).__init__(incoming, **kwargs)
        self.delta = delta
    def get_output_shape_for(self, input_shape):
        output_shape = list(input_shape)
        if input_shape[2] is not None:
            output_shape[2] -= self.delta
        return tuple(output_shape)
    def get_output_for(self, input, **kwargs):
        return input[:, :, self.delta:] - input[:, :, :-self.delta]

class PCENLayer(lasagne.layers.Layer):
    def __init__(self, incoming,
                 log_s=lasagne.init.Constant(np.log(0.025)),
                 log_alpha=lasagne.init.Constant(0),
                 log_delta=lasagne.init.Constant(0),
                 log_r=lasagne.init.Constant(0),
                 eps=1e-6, **kwargs):
        super(PCENLayer, self).__init__(incoming, **kwargs)
        num_bands = self.input_shape[-1]
        self.log_s = self.add_param(log_s, shape=(num_bands,),
                                    name='log_s', regularizable=False)
        self.log_alpha = self.add_param(log_alpha, shape=(num_bands,),
                                        name='log_alpha', regularizable=False)
        self.log_delta = self.add_param(log_delta, shape=(num_bands,),
                                        name='log_delta', regularizable=False)
        self.log_r = self.add_param(log_r, shape=(num_bands,),
                                        name='log_r', regularizable=False)
        self.eps = eps
    def get_output_for(self, input, **kwargs):
        def smooth_step(current_in, previous_out, s):
            one = T.constant(1)
            return [(one - s) * previous_out + s * current_in]
        init = input[:, :, 0]  # start smoother from first frame
        s = T.exp(self.log_s).dimshuffle('x', 'x', 0)
        smoother = theano.scan(fn=smooth_step,
                               sequences=[input.transpose(2, 0, 1, 3)],
                               non_sequences=[s],
                               outputs_info=[init],
                               strict=True)[0].transpose(1, 2, 0, 3)
        alpha = T.exp(self.log_alpha)
        delta = T.exp(self.log_delta)
        r = T.exp(self.log_r)
        return (input / (self.eps + smoother)**alpha + delta)**r - delta**r

def architecture(input_var, input_shape, cfg):
    layer = InputLayer(input_shape, input_var)

    # filterbank, if any
    if cfg['filterbank'] == 'mel':
        import audio
        filterbank = audio.create_mel_filterbank(
                cfg['sample_rate'], cfg['frame_len'], cfg['mel_bands'],
                cfg['mel_min'], cfg['mel_max'])
        filterbank = filterbank[:input_shape[3]].astype(theano.config.floatX)
        layer = DenseLayer(layer, num_units=cfg['mel_bands'],
                num_leading_axes=-1, W=T.constant(filterbank), b=None,
                nonlinearity=None)
    elif cfg['filterbank'] == 'mel_learn':
        layer = MelBankLayer(layer, cfg['sample_rate'], cfg['frame_len'],
                cfg['mel_bands'], cfg['mel_min'], cfg['mel_max'])
    elif cfg['filterbank'] != 'none':
        raise ValueError("Unknown filterbank=%s" % cfg['filterbank'])

    # magnitude transformation, if any
    if cfg['magscale'] == 'log':
        layer = ExpressionLayer(layer, lambda x: T.log(T.maximum(1e-7, x)))
    elif cfg['magscale'] == 'log1p':
        layer = ExpressionLayer(layer, T.log1p)
    elif cfg['magscale'].startswith('log1p_learn'):
        # learnable log(1 + 10^a * x), with given initial a (or default 0)
        a = float(cfg['magscale'][len('log1p_learn'):] or 0)
        a = T.exp(theano.shared(lasagne.utils.floatX(a)))
        layer = lasagne.layers.ScaleLayer(layer, scales=a,
                                          shared_axes=(0, 1, 2, 3))
        layer = ExpressionLayer(layer, T.log1p)
    elif cfg['magscale'].startswith('pow_learn'):
        # learnable x^sigmoid(a), with given initial a (or default 0)
        a = float(cfg['magscale'][len('pow_learn'):] or 0)
        a = T.nnet.sigmoid(theano.shared(lasagne.utils.floatX(a)))
        layer = PowLayer(layer, exponent=a)
    elif cfg['magscale'] == 'pcen':
        layer = PCENLayer(layer)
        if cfg.get('pcen_fix_alpha'):
            layer.params[layer.log_alpha].remove("trainable")
    elif cfg['magscale'] == 'loudness_only':
        # cut away half a block length on the left and right
        layer = lasagne.layers.SliceLayer(
                layer, slice(cfg['blocklen']//2, -(cfg['blocklen']//2)), axis=2)
        # average over the frequencies and channels
        layer = lasagne.layers.ExpressionLayer(
                layer, lambda X: X.mean(axis=(1, 3), keepdims=True),
                lambda shp: (shp[0], 1, shp[2], 1))
    elif cfg['magscale'] != 'none':
        raise ValueError("Unknown magscale=%s" % cfg['magscale'])

    # temporal difference, if any
    if cfg['arch.timediff']:
        layer = TimeDiffLayer(layer, delta=cfg['arch.timediff'])

    # standardization per frequency band
    if cfg.get('input_norm', 'batch') == 'batch':
        layer = batch_norm_vanilla(layer, axes=(0, 2), beta=None, gamma=None)
    elif cfg['input_norm'] == 'instance':
        layer = lasagne.layers.StandardizationLayer(layer, axes=2)
    elif cfg['input_norm'] == 'none':
        pass
    else:
        raise ValueError("Unknown input_norm=%s" % cfg['input_norm'])

    # convolutional neural network
    kwargs = dict(nonlinearity=lasagne.nonlinearities.leaky_rectify,
                  W=lasagne.init.Orthogonal())
    maybe_batch_norm = batch_norm if cfg['arch.batch_norm'] else lambda x: x
    if cfg['arch.convdrop'] == 'independent':
        maybe_dropout = lambda x: dropout(x, 0.1)
    elif cfg['arch.convdrop'] == 'channels':
        maybe_dropout = lambda x: dropout(x, 0.1, shared_axes=(2, 3))
    elif cfg['arch.convdrop'] == 'bands':
        maybe_dropout = lambda x: dropout(x, 0.1, shared_axes=(1, 2))
    elif cfg['arch.convdrop'] == 'none':
        maybe_dropout = lambda x: x
    else:
        raise ValueError("Unknown arch.convdrop=%s" % cfg['arch.convdrop'])
    if cfg['arch'] == 'dense:16':
        layer = DenseLayer(layer, 16, **kwargs)
        layer = DenseLayer(layer, 1,
                           nonlinearity=lasagne.nonlinearities.sigmoid,
                           W=lasagne.init.Orthogonal())
        return layer
    convmore = cfg['arch.convmore']
    layer = Conv2DLayer(layer, int(64 * convmore), 3, **kwargs)
    if cfg.get('arch.firstconv_zeromean', False) == 'params':
        layer.W = layer.W - T.mean(layer.W, axis=(2, 3), keepdims=True)
    layer = maybe_batch_norm(layer)
    layer = maybe_dropout(layer)
    layer = Conv2DLayer(layer, int(32 * convmore), 3, **kwargs)
    layer = maybe_batch_norm(layer)
    layer = MaxPool2DLayer(layer, 3)
    layer = maybe_dropout(layer)
    layer = Conv2DLayer(layer, int(128 * convmore), 3, **kwargs)
    layer = maybe_batch_norm(layer)
    layer = maybe_dropout(layer)
    layer = Conv2DLayer(layer, int(64 * convmore), 3, **kwargs)
    layer = maybe_batch_norm(layer)
    if cfg['arch'] == 'ismir2015':
        layer = MaxPool2DLayer(layer, 3)
    elif cfg['arch'] == 'ismir2016':
        layer = maybe_dropout(layer)
        layer = Conv2DLayer(layer, int(128 * convmore), (3, layer.output_shape[3] - 3), **kwargs)
        layer = maybe_batch_norm(layer)
        layer = MaxPool2DLayer(layer, (1, 4))
    else:
        raise ValueError('Unknown arch=%s' % cfg['arch'])
    layer = DenseLayer(dropout(layer, 0.5), 256, **kwargs)
    layer = maybe_batch_norm(layer)
    layer = DenseLayer(dropout(layer, 0.5), 64, **kwargs)
    layer = maybe_batch_norm(layer)
    layer = DenseLayer(dropout(layer, 0.5), 1,
                       nonlinearity=lasagne.nonlinearities.sigmoid,
                       W=lasagne.init.Orthogonal())
    return layer

reason: import numpy,import theano


text:
import pandas as pd
import matplotlib as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
from sklearn.linear_model import LinearRegression
import statsmodels.api as sm
import statsmodels.formula.formulatools

def find_abs_min(arr):
    """
    Finds minimum value of the array where each element is absoluted
    :param arr: the input array
    :return: minimum value and its first index
    """
    min_val, min_index = (abs(arr[0]), 0)
    for i in range(len(arr)):
        if min_val > abs(arr[i]):
            min_val, min_index = (abs(arr[i]), i)
    return min_val, min_index

dataset = pd.read_csv("50_Startups.csv")

X = dataset.values[:, : -1]
# X = dataset.iloc[:, : -1].values
y = dataset.values[:, 4]

ct_X = ColumnTransformer(
    [('one_hot_encoder', OneHotEncoder(), [3])],
    remainder='passthrough')
X = np.array(ct_X.fit_transform(X), dtype=np.float)

# Avoiding the dummy variable trap
X = X[:, 1:]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

regressor = LinearRegression()
regressor.fit(X_train, y_train)

# Predicting the results
y_pred = regressor.predict(X=X_test)

# building the optimal model with backwards elimination
# setup variables for loop
X = np.append(arr=np.ones((len(X), 1)).astype(np.float), values=X, axis=1)
columns = [0, 1, 2, 3, 4, 5]  # Used colims
critical_value = 0.1
running = True

# This is working
while running:
    # Setup the new smaller set with more significant variables
    running = False
    X_opt = X[:, columns]
    X_opt = X_opt.astype(float)
    regressor_ols = sm.OLS(endog=y, exog=X_opt).fit()
    print(type(regressor_ols))
    print(regressor_ols.tvalues)

    # Get t values to find the lowest one and remove it from the set
    t_values = regressor_ols.tvalues
    val, index = find_abs_min(t_values)
    if val < critical_value:
        running = True
        columns.remove(index)
print(columns)

max_stars_repo_path: Regression/Multiple Linear Regression/Multiple Linear Regression.py

reason: import numpy,import statsmodels


text:
"""
A dummy package to allow importing numpy and the unit-aware replacements of
numpy functions without having to know which functions are overwritten.

This can be used for example as ``import brian2.numpy_ as np``
"""

from numpy import *
from brian2.units.unitsafefunctions import *

# These will not be imported with a wildcard import to not overwrite the
# builtin names (mimicking the numpy behaviour)
from builtins import bool, int, float, complex, object, bytes, str

from numpy.core import round, abs, max, min

import numpy
import brian2.units.unitsafefunctions as brian2_functions
__all__ = []
__all__.extend(numpy.__all__)
__all__.extend(brian2_functions.__all__)

reason: import numpy,from numpy


text:
import sys
import os
import logging

import cv2
import numpy as np
from omegaconf import DictConfig
from tqdm import tqdm

from utils import (
    get_path_list,
    get_input_data_type,
    load_image,
    load_video,
    save_image,
    save_coordinate,
    save_density_map,
)

# logging setting
logging.basicConfig(
    level=logging.DEBUG, format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s"
)
logger = logging.getLogger(__name__)

# define control key
Q_KEY = 0x71  # q key (end)
P_KEY = 0x70  # p key (pause)
D_KEY = 0x64  # d key (delete)
S_KEY = 0x73  # s key (save data and restart)

class DensityAnnotator:
    """
    DensityAnnotator is a class that labels the position of an object
    with a mouse and calculates a density map for image or video data.

    After annotation, save the raw image, the coordinate of the object,
    and the density map for each target frame.
    """

    def __init__(self, cfg: DictConfig, original_cwd: str) -> None:
        """
        Initialize DensityAnnotator class by hydra config.

        :param cfg: config of DensityAnnotator class
        :param original_cwd: current working directory path
        """
        logger.info(f"Loaded config: {cfg}")
        cv2.namedWindow("click annotation points")
        cv2.setMouseCallback("click annotation points", self.mouse_event)
        self.sigma_pow = cfg.sigma_pow
        self.mouse_event_interval = cfg.mouse_event_interval

        # set frame information
        self.video = None
        self.frame = None
        self.frame_list = []
        self.width = None
        self.height = None
        self.features = None
        self.coordinate_matrix = None
        self.frame_num = 0

        # set file path
        self.input_file_path = None
        self.input_file_path_list = get_path_list(
            cfg.path.input_file_path, original_cwd
        )
        self.save_raw_image_dir = os.path.join(
            original_cwd, cfg.path.save_raw_image_dir
        )
        self.save_annotated_dir = os.path.join(
            original_cwd, cfg.path.save_annotated_dir
        )
        self.save_image_extension = cfg.path.save_image_extension
        self.save_annotated_image_dir = f"{self.save_annotated_dir}/image"
        self.save_annotated_coord_dir = f"{self.save_annotated_dir}/coord"
        self.save_annotated_density_dir = f"{self.save_annotated_dir}/dens"

        # check and create target directory
        os.makedirs(self.save_raw_image_dir, exist_ok=True)
        os.makedirs(self.save_annotated_image_dir, exist_ok=True)
        os.makedirs(self.save_annotated_coord_dir, exist_ok=True)
        os.makedirs(self.save_annotated_density_dir, exist_ok=True)

    def run(self) -> None:
        """
        Select the data type of the image or video from the extension
        of the input data and execute the annotation.

        :return: None
        """
        for file_path in tqdm(self.input_file_path_list, desc="Annotation File Number"):
            # initialization
            self.frame_list = []
            self.features = None
            # load file
            self.input_file_path = file_path
            data_type = get_input_data_type(self.input_file_path)
            logger.info(f"Annotation Data Type: {data_type}")
            if data_type == "image":
                self.image_annotation()
            elif data_type == "video":
                self.video_annotation()
            else:
                logger.error("Data type is invalid. Please check input file.")
                sys.exit(1)

        # end processing
        cv2.destroyAllWindows()

    def annotator_initialization(self) -> None:
        """
        Initialize coordinate matrix that store the clicked position.

        :return: None
        """
        self.width = self.frame.shape[1]
        self.height = self.frame.shape[0]

        self.coordinate_matrix = np.zeros((self.width, self.height, 2), dtype="int64")
        for i in range(self.width):
            for j in range(self.height):
                self.coordinate_matrix[i][j] = [i, j]

    def image_annotation(self) -> None:
        """
        A function to perform annotation on a single image.

        :return: None
        """
        # load input image
        self.frame = load_image(self.input_file_path)
        self.frame_list.append(self.frame.copy())
        # frame number get from input file name
        self.frame_num = os.path.splitext(os.path.basename(self.input_file_path))[0]
        # initialize by frame information
        self.annotator_initialization()
        while True:
            # display frame
            cv2.imshow("click annotation points", self.frame)

            # each key operation
            wait_interval = self.mouse_event_interval
            key = cv2.waitKey(wait_interval) & 0xFF
            if key == D_KEY:
                # delete the previous feature point
                self.delete_point()
            elif key == S_KEY:
                # save current annotated data and go to next frame
                self.save_annotated_data()
                wait_interval = self.mouse_event_interval
                break

    def video_annotation(self) -> None:
        """
        A function to perform annotation on movie.
        This function allow to annotate multiple images cut out
        from the video data at any time.

        :return: None
        """
        # load input video data
        self.video = load_video(self.input_file_path)
        # load first frame and initialize by frame information
        ret, self.frame = self.video.read()
        self.annotator_initialization()

        # read frames at regular intervals and annotate them.
        wait_interval = self.mouse_event_interval
        while ret:
            if wait_interval != 0:
                self.features = None
                self.frame_num += 1
                # display current frame
                cv2.imshow("click annotation points", self.frame)
                # load next frame and status
                ret, self.frame = self.video.read()

            # each key operation
            key = cv2.waitKey(wait_interval) & 0xFF
            if key == Q_KEY:
                # finish the annotation work
                break
            elif key == P_KEY:
                # pause current frame and start annotation
                wait_interval = 0  # wait until the end of annotation
                self.frame_list.append(self.frame.copy())
                # save raw image
                cv2.imwrite(
                    f"{self.save_raw_image_dir}/{self.frame_num}{self.save_image_extension}",
                    self.frame,
                )
            elif key == D_KEY:
                # delete the previous feature point
                self.delete_point()
            elif key == S_KEY:
                # save current annotated data and go to next frame
                self.save_annotated_data()
                wait_interval = self.mouse_event_interval
        # end processing
        self.video.release()

    def mouse_event(self, event: int, x: int, y: int, flags: int, param: dict) -> None:
        """
        Select annotated point by left click of mouse

        :param event: the type of mouse event
        :param x: x coordinate of the clicked position
        :param y: y coordinate of the clicked position
        :param flags: the type of button or key that was pressed during the mouse event
        :param param: the value of param set in the third argument of setMouseCallback
        :return: None
        """
        # other than left click
        if event != cv2.EVENT_LBUTTONDOWN:
            return

        # draw and add feature point
        cv2.circle(self.frame, (x, y), 4, (0, 0, 255), -1, 8, 0)
        self.add_point(x, y)
        cv2.imshow("click annotation points", self.frame)

    def add_point(self, x, y) -> None:
        """
        Add new feature point on stored list

        :param x: x coordinate of the clicked position
        :param y: y coordinate of the clicked position
        :return: None
        """
        if self.features is None:
            self.features = np.array([[x, y]], np.uint16)
        else:
            self.features = np.append(self.features, [[x, y]], axis=0).astype(np.uint16)
        self.frame_list.append(self.frame.copy())

    def delete_point(self) -> None:
        """
        Delete the previous feature point from stored list.

        :return: None
        """
        if (self.features is not None) and (len(self.features) > 0):
            self.features = np.delete(self.features, -1, 0)
            self.frame_list.pop()
            self.frame = self.frame_list[-1].copy()
            cv2.imshow("click annotation points", self.frame)

    def calculate_gaussian_kernel(self) -> np.array:
        """
        Calculate the density map using the Gaussian kernel
        based on the annotated coordinates.

        :return: calculated density map in numpy format
        """
        kernel = np.zeros((self.width, self.height))

        for point in self.features:
            tmp_coord_matrix = np.array(self.coordinate_matrix)
            point_matrix = np.full((self.width, self.height, 2), point)
            diff_matrix = tmp_coord_matrix - point_matrix
            pow_matrix = diff_matrix * diff_matrix
            norm = pow_matrix[:, :, 0] + pow_matrix[:, :, 1]
            kernel += np.exp(-norm / (2 * self.sigma_pow))

        return kernel.T

    def save_annotated_data(self) -> None:
        """
        Save coordinate and raw image. There are feature point information.

        :return: None
        """
        if self.features is None:
            logger.info("None have been annotated.")
        else:
            # save image that added annotated point
            save_image(
                f"{self.save_annotated_image_dir}/{self.frame_num}{self.save_image_extension}",
                self.frame,
            )

            # save the coordinates of the annotated point
            save_coordinate(
                f"{self.save_annotated_coord_dir}/{self.frame_num}.csv", self.features
            )

            # save annotated density map
            annotated_density = self.calculate_gaussian_kernel()
            save_density_map(
                f"{self.save_annotated_density_dir}/{self.frame_num}.npy",
                annotated_density,
            )
            logger.info(f"Annotated and saved frame number: {self.frame_num}")

max_stars_repo_name: kenya-sk/density-annotator

max_issues_repo_name: kenya-sk/density-annotator

max_forks_repo_name: kenya-sk/density-annotator


text:
import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from typing import Tuple
import math
import matplotlib.pyplot as plt
from torch.utils.data import DataLoader, Dataset
from sklearn.preprocessing import MinMaxScaler

def create_constant_time_series(nb_steps: int) -> np.ndarray:
    return np.array([2] * nb_steps, dtype='float32').reshape(-1, 1)

def create_linear_time_series(nb_steps: int) -> np.ndarray:
    return np.linspace(1, nb_steps, num=nb_steps, dtype='float32').reshape(-1, 1)

def create_sinusoidal_time_series(nb_steps: int) -> np.ndarray:
    steps = np.linspace(0, 4, num=nb_steps)
    return np.sin(np.pi * steps, dtype='float32').reshape(-1, 1)

def check_cuda() -> str:
    if torch.cuda.is_available():
        return "cuda"
    else:
        return "cpu"

def normalize_data(data: np.ndarray) -> Tuple[MinMaxScaler, np.ndarray]:
    scaler = MinMaxScaler(feature_range=(0, 1))
    data_norm = scaler.fit_transform(data)
    return scaler, data_norm

def split_data(data: np.ndarray, train_percent_size: float = 0.7) -> Tuple[np.ndarray, np.ndarray]:

    length = data.shape[0]
    train_range = int(length * train_percent_size)

    train_data = data[:train_range, :]
    test_data = data[train_range:, :]

    return train_data, test_data

def calculate_size(L_in: int, padding: int, dilation: int, kernel_size: int, stride: int) -> int:
    res = L_in + 2 * padding - dilation * (kernel_size - 1) - 1
    res = 1 + res / stride
    L_out = math.floor(res)
    return int(L_out)

class SimpleTSDataset(Dataset):
    def __init__(self, data: np.ndarray, sequence_size: int = 5, prediction_size: int = 1,
                 device: torch.device = "cpu"):
        self.data = data

        # We assume the data is in the following form: (n, length) where n is the number of dimension
        # in the time series data (i.e. 1 for univariate and more than 1 for multivariate) and length
        # is the length of the time series.

        self.x = []
        self.y = []

        n_items = self.data.shape[0]
        n_dims = self.data.shape[1]
        self.max_range = n_items - sequence_size - prediction_size

        for i in range(self.max_range):
            x_temp = torch.from_numpy(self.data[i:i + sequence_size, :]).permute(1, 0).to(device)
            y_temp = torch.from_numpy(self.data[i + sequence_size:i + prediction_size + sequence_size, :]).permute(1,
                                                                                0).to(
                device)
            # self.x.append(x_temp.reshape(-1, n_dims))
            self.x.append(x_temp)
            self.y.append(y_temp)

    def __len__(self) -> int:
        return self.max_range

    def __getitem__(self, index) -> Tuple[torch.Tensor, torch.Tensor]:
        item = (self.x[index], self.y[index])
        return item

class DeepAnt(nn.Module):
    def __init__(self, window: int = 5, p_window: int = 1, nb_filters: int = 32, kernel_conv_size: int = 3,
                 kernel_pool_size: int = 2, dimensions: int = 1):
        super(DeepAnt, self).__init__()
        self.window = window
        self.p_window = p_window

        end_size_conv1 = calculate_size(window, 1, 1, kernel_conv_size, 1)
        end_size_pool1 = calculate_size(end_size_conv1, 1, 1, kernel_pool_size, kernel_pool_size)
        end_size_conv2 = calculate_size(end_size_pool1, 1, 1, kernel_conv_size, 1)
        end_size_pool2 = calculate_size(end_size_conv2, 1, 1, kernel_pool_size, kernel_pool_size)

        self.conv1 = nn.Conv1d(dimensions, nb_filters, kernel_conv_size, padding=1, bias=True)
        self.conv2 = nn.Conv1d(nb_filters, nb_filters, kernel_conv_size, padding=1, bias=True)
        self.relu = nn.ReLU()
        self.maxpool = nn.MaxPool1d(kernel_pool_size, padding=1)
        self.fc = nn.Linear(end_size_pool2 * nb_filters, p_window * dimensions)

    def forward(self, input):
        batch_size = input.size(0)
        intermediate = self.conv1(input)
        intermediate = self.relu(intermediate)
        intermediate = self.maxpool(intermediate)

        out = self.conv2(intermediate)
        out = self.relu(out)
        out = self.maxpool(out)

        # Todo: we should resize the output to (N, dimensions, p_window)
        result = self.fc(out.view(batch_size, -1))
        # result = self.relu(result)

        return torch.unsqueeze(result, 2)

if __name__ == '__main__':
    # func defines the function we use to create the time series
    # available choices:
    # * create_constant_time_series
    # * create_linear_time_series
    # * create_sinusoidal_time_series
    # Do NOT put brackets to the name of the function!

    func = create_sinusoidal_time_series
    total_length = 3000
    dev = torch.device(check_cuda())
    series = func(total_length)

    window = 45
    dimensions = 1
    prediction_window = 1
    batch_size = 32
    learning_rate = 0.001
    epochs = 100
    test_train_split = 0.9

    train, test = split_data(series, train_percent_size=test_train_split)

    train_ds = SimpleTSDataset(data=train, sequence_size=window, prediction_size=prediction_window)
    test_ds = SimpleTSDataset(data=test, sequence_size=window, prediction_size=prediction_window)

    train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=False)
    test_dl = DataLoader(test_ds, batch_size=batch_size, shuffle=True)

    model = DeepAnt(window=window, p_window=prediction_window, dimensions=dimensions)
    criterion = nn.L1Loss()

    optimizer = optim.Adam(model.parameters(), lr=learning_rate)
    losses = list()

    for epoch in range(epochs):
        count = 0
        total_loss = 0
        for x, y in train_dl:
            model.train()
            x = x.requires_grad_()

            optimizer.zero_grad()
            result = model(x)
            loss = criterion(result, y)
            loss.backward()
            optimizer.step()
            current_loss = loss.item()
            total_loss += current_loss
            count += 1

        losses.append(total_loss / count)

    plt.plot(np.array(losses))
    plt.title("Training Loss")
    plt.show()

    print("Evaluation")
    results = []
    model.eval()
    for x, y in test_dl:
        result = model(x)
        results.append(result)

    reconstruct = []
    for data in results:
        data = data[:, 0, :]
        res = data.reshape(-1)
        res = res.detach().numpy()
        for r in res:
            reconstruct.append(r)

    plt.plot(np.array(reconstruct))
    plt.title("Output on test data")
    plt.show()


text:
from ..layers import *
import numpy as np

@mx.operator.register('ICD_BottomUp')
class ICD_BottomUpProp(mx.operator.CustomOpProp):
    def __init__(self, grad_scale=1):
        super(ICD_BottomUpProp, self).__init__(need_top_grad=False)
        self.grad_scale = float(grad_scale)

    def list_arguments(self):
        return ['data', 'label', 'weight']

    def list_outputs(self):
        return ['output']

    def infer_shape(self, in_shape):
        n, c, h, w = in_shape[0]
        _, C = in_shape[1]
        in_shape[2] = [C, c]
        return in_shape, [[n, C, h, w]], []

    def infer_type(self, in_type):
        _type = in_type[0]
        return [_type]*3, [_type], []

    def create_operator(self, ctx, shapes, dtypes):
        return ICD_BottomUp(self.grad_scale)

class ICD_BottomUp(mx.operator.CustomOp):
    def __init__(self, grad_scale):
        self.grad_scale = grad_scale

    def forward(self, is_train, req, in_data, out_data, aux):
        data, label, weight = in_data

        weight_normed = mx.nd.L2Normalization(weight, mode='instance')
        dot = mx.nd.Convolution(data, weight=weight_normed.reshape(0, 0, 1, 1),
                num_filter=weight.shape[0], stride=(1, 1), kernel=(1, 1), no_bias=True)
        self.assign(out_data[0], req[0], dot)

    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
        data, label, weight = in_data
        dot = out_data[0]
        n, c, h, w = data.shape
        label = label * (label.sum(axis=1, keepdims=True) < 1.001)

        data_flat = data.transpose((1, 0, 2, 3)).reshape(0, -1)
        dot_flat = (dot * label.reshape(0, 0, 1, 1)).transpose((1, 0, 2, 3)).reshape(0, -1)
        grad = - mx.nd.dot(dot_flat, data_flat, transpose_b=True) * (self.grad_scale / (h * w))

        w_sum_square = (weight**2).sum(axis=1, keepdims=True)
        w_sum_sqrt = mx.nd.sqrt(w_sum_square)
        w_sum = weight.sum(axis=1, keepdims=True)
        wg_sum = (weight * grad).sum(axis=1, keepdims=True)
        grad = -weight * (wg_sum / (w_sum_square * w_sum_sqrt + 1e-5)) + grad / (w_sum_sqrt + 1e-5)

        in_grad[0][:] = 0
        in_grad[1][:] = 0
        self.assign(in_grad[2], req[2], grad)

@mx.operator.register('AccumResign')
class AccumResignProp(mx.operator.CustomOpProp):
    def __init__(self, momentum=0.9, use_global_stats=0):
        super(AccumResignProp, self).__init__(need_top_grad=False)
        self.momentum = float(momentum)
        self.use_global_stats = int(use_global_stats) > 0
        self.kvstore = None

    def list_arguments(self):
        return ['score', 'cam', 'label']

    def list_outputs(self):
        return ['output']

    def list_auxiliary_states(self):
        return ['moving_mean']

    def infer_shape(self, in_shape):
        n, c, h, w = in_shape[0]
        self._moving_mean_shape = (c, 2)
        return in_shape, [in_shape[0]], [[c, 2]]

    @staticmethod
    def create_updater(momentum):
        def wrapper(key, inputs, stored):
            (stored*momentum + inputs.as_in_context(stored.context)*(1 - momentum)).copyto(stored)
        return wrapper

    def create_operator(self, ctx, shapes, dtypes):
        if self.kvstore is None:
            self.kvstore = mx.kv.create('device')
            updater = self.create_updater(self.momentum)
            self.kvstore._set_updater(updater)
            self.kvstore.init(0, mx.nd.zeros(self._moving_mean_shape, ctx=ctx))
        return AccumResign(self.kvstore, self.use_global_stats)

class AccumResign(mx.operator.CustomOp):
    def __init__(self, kvstore, use_global_stats):
        self.kvstore = kvstore
        self.use_global_stats = use_global_stats

    def forward(self, is_train, req, in_data, out_data, aux):
        score, cam, label = in_data
        accum_score = aux[0]

        if is_train and (not self.use_global_stats):
            pmask = score > 0
            nmask = score < 0

            cam = mx.nd.maximum(cam, 0)
            cam_pos = (cam * pmask).sum(axis=(2, 3)) / mx.nd.maximum(pmask.sum(axis=(2, 3)), 1e-5)
            cam_neg = (cam * nmask).sum(axis=(2, 3)) / mx.nd.maximum(nmask.sum(axis=(2, 3)), 1e-5)

            label_cnt = mx.nd.maximum(label.sum(axis=0), 1e-5)
            mean_pos = (cam_pos * label).sum(axis=0) / label_cnt
            mean_neg = (cam_neg * label).sum(axis=0) / label_cnt

            curr_score = mx.nd.concat(mean_pos.reshape(0, 1), mean_neg.reshape(0, 1), dim=1)
            empty_class = (label_cnt < 0.999).reshape(0, 1)
            curr_score = curr_score * (1 - empty_class) + accum_score * empty_class

            self.kvstore.push(0, curr_score)
            self.kvstore.pull(0, accum_score)

        sign = ((accum_score[:, 0] < accum_score[:, 1]) - 0.5) * (-2)
        new_score = score * sign.reshape(1, -1, 1, 1)
        self.assign(out_data[0], req[0], new_score)

    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
        for grad in in_grad:
            grad[:] = 0

@mx.operator.register('ICD_TopDown')
class ICD_TopDownProp(mx.operator.CustomOpProp):
    def __init__(self, grad_scale=1, warmup=0):
        super(ICD_TopDownProp, self).__init__(need_top_grad=False)
        self.grad_scale = float(grad_scale)
        self.warmup = int(warmup)

    def list_arguments(self):
        return ['data', 'label', 'score']

    def list_outputs(self):
        return ['output']

    def infer_shape(self, in_shape):
        n, c, h, w = in_shape[0]
        _, C = in_shape[1]
        return in_shape, [[n, C, h, w]], []

    def create_operator(self, ctx, shapes, dtypes):
        return ICD_TopDown(self.grad_scale, self.warmup)

class ICD_TopDown(mx.operator.CustomOp):
    def __init__(self, grad_scale, warmup):
        self.grad_scale = grad_scale
        self.warmup = warmup
        self.max_warmup = max(warmup, 1)

    def get_warmup_scale(self):
        if self.warmup <= 0: return 1
        scale = np.exp(-10. * self.warmup * 1. / self.max_warmup)
        self.warmup -= 1
        return scale

    def forward(self, is_train, req, in_data, out_data, aux):
        data, label, score = in_data
        data_sigmoid = mx.nd.sigmoid(data)
        self.assign(out_data[0], req[0], data_sigmoid)

    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
        data, label, score = in_data
        data_sigmoid = out_data[0]
        n, c, h, w = data.shape

        score_norm = mx.nd.maximum(score, 0) / mx.nd.maximum(score.max(axis=(2, 3), keepdims=True), 1e-5) + \
                mx.nd.minimum(score, 0) / (-mx.nd.minimum(score.min(axis=(2, 3), keepdims=True), -1e-5))
        mask = mx.nd.abs(score_norm) > 0.2

        target = score_norm > 0
        grad = (data_sigmoid - target) * label.reshape(0, 0, 1, 1) * mask
        grad = grad * (self.grad_scale * self.get_warmup_scale() / (h * w))

        self.assign(in_grad[0], req[0], grad)
        in_grad[1][:] = 0
        in_grad[2][:] = 0

@mx.operator.register('SPRefine')
class SPRefineProp(mx.operator.CustomOpProp):
    def __init__(self):
        super(SPRefineProp, self).__init__(need_top_grad=False)

    def list_arguments(self):
        return ['data', 'superpixel', 'label']

    def list_outputs(self):
        return ['output']

    def infer_shape(self, in_shape):
        return in_shape, [in_shape[0]], []

    def create_operator(self, ctx, shapes, dtypes):
        return SPRefine()

class SPRefine(mx.operator.CustomOp):
    def forward(self, is_train, req, in_data, out_data, aux):
        data, superpixel, label = in_data
        n, c, h, w = data.shape
        _, h2, w2 = superpixel.shape

        sp_num = int(superpixel.max().asnumpy()) + 1
        sp_flat = mx.nd.one_hot(superpixel, depth=sp_num).transpose((0, 3, 1, 2))
        if h != h2:
            sp_flat = mx.nd.contrib.BilinearResize2D(sp_flat, height=h, width=w)
        sp_flat = sp_flat.reshape(0, 0, -3)

        score_flat = data.reshape(0, 0, -3) * label.reshape(0, 0, 1)
        score_sign = mx.nd.sign(score_flat)

        sp_sign = mx.nd.batch_dot(score_sign, sp_flat, transpose_b=True)
        sp_fg = sp_sign.max(axis=1, keepdims=True) > 1e-5

        sp_flat_avg = sp_flat / mx.nd.maximum(sp_flat.sum(axis=2, keepdims=True), 1e-5)
        score_flat_pos = mx.nd.maximum(score_flat, 0) / mx.nd.maximum(score_flat.max(axis=1, keepdims=True), 1e-5)
        sp_fg_score = mx.nd.batch_dot(score_flat_pos, sp_flat_avg, transpose_b=True)
        sp_fg_label = mx.nd.one_hot(sp_fg_score.argmax(axis=1), depth=c).transpose((0, 2, 1))
        sp_fg = sp_fg * sp_fg_label * label.reshape(0, 0, 1)
        sp_bg = 1 - sp_fg

        score_avg_fg = mx.nd.batch_dot(mx.nd.maximum(score_flat, 0), sp_flat_avg, transpose_b=True)
        score_avg_bg = mx.nd.batch_dot(mx.nd.minimum(score_flat, 0), sp_flat_avg, transpose_b=True)
        score_avg = (score_avg_fg * sp_fg) + (score_avg_bg * sp_bg)
        score_avg = mx.nd.batch_dot(score_avg, sp_flat).reshape(0, 0, h, w)

        self.assign(out_data[0], req[0], score_avg)

    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
        pass

max_stars_repo_name: js-fan/ICD

max_issues_repo_name: js-fan/ICD

max_forks_repo_name: js-fan/ICD