text:
import tensorflow as tf
import pvdm_params as params
import random
import traceback
import numpy as np
import re
import os
import pickle
import math

class TrainData:
    def __init__(self, paragraph_index=None):
        self.index = 0
        if paragraph_index is None:
            self.entries_ids = {}
        else:
            self.entries_ids = paragraph_index
        self.total_coint_unk = 0
        self.total_count_ok = 0
    def refine(row):
        row = row.replace('\n','').replace('\r', '').replace("\\n", "").lower()
        row = re.sub(r'((https|http)?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w\.-]*)*\/?\S', '', row, flags=re.MULTILINE)
        remove_chars = ['(', ')', '[', ']', '-' ]
        chars = [".", "?", ',', '!', ':', ';']
        for char in remove_chars:
            row = row.replace(char, "")
        row = row.replace("  ", " ")
        for char in chars:
            row = row.replace(char, " "+char+" ")
        row = row.replace("  ", " ")

        return row
    def process_data(self, data, calc_words_ids = False):
        self.data = []
        append = False
        if len(self.entries_ids) < 1:
            append = True
        for entry in data:
            entry = TrainData.refine(entry)
            if append and entry not in self.entries_ids:
                self.entries_ids[entry] = len(self.entries_ids)
            if entry not in self.entries_ids:
                continue
            paragraph_id = self.entries_ids[entry]
            entry_words = entry.split()
            entry_words = entry_words + [params.padding_word]*max(0,params.word_windows_size - len(entry_words) + 1)
            for i in range(len(entry_words) - params.word_windows_size):
                self.data.append([paragraph_id, entry_words[i:i+params.word_windows_size], entry_words[i+params.word_windows_size]])
                # print("Paragraph: ",self.data[-1][0])
                # print("Context: ", self.data[-1][1])
                # print("Target: ", self.data[-1][2])
                # print("-"*10)
    def calc_words_ids(self):
        words_ids = {}
        inverse_words_ids = {}
        for entry in self.data:
            words = entry[1] + [entry[2]]
            for word in words:
                if word not in words_ids:
                    words_ids[word] = len(words_ids)
                    inverse_words_ids[words_ids[word]] = word
        if params.padding_word not in words_ids:
            words_ids[params.padding_word] = len(words_ids)
            inverse_words_ids[words_ids[params.padding_word]] = params.padding_word
        if params.unknown_token not in words_ids:
            words_ids[params.unknown_token] = len(words_ids)
            inverse_words_ids[words_ids[params.unknown_token]] = params.unknown_token
        return words_ids, inverse_words_ids
    def data_to_ids(self, words_dict):
        self.words_dict = words_dict
        for entry in self.data:
            entry[1] = [ words_dict[word] if word in words_dict else words_dict[params.unknown_token] for word in entry[1] ]
            if entry[2] in words_dict:
                entry[2] = words_dict[entry[2]]
                self.total_count_ok += 1
            else:
                entry[2] = words_dict[params.unknown_token]
                self.total_coint_unk += 1
    def get_next_batch(self, batch_size):
        if self.index + batch_size >= len(self.data):
            self.index = 0
        if self.index == 0:
            random.shuffle(self.data)
        result = self.data[self.index:self.index+batch_size]
        self.index += batch_size
        paragraph_ids, input_words, labels = zip(*result)
        paragraph_ids = np.asarray(paragraph_ids, dtype=np.int32)
        input_words = np.asarray(input_words, dtype=np.int32)
        labels = np.asarray(labels, dtype=np.int32)
        return paragraph_ids, input_words,labels
    def save_dicts(self, logdir):
        words_ids_filename = os.path.join(logdir, params.words_ids_filename)
        paragraphs_ids_filename = os.path.join(logdir, params.paragraphs_ids_filename)

        with open(words_ids_filename, 'wb') as pfile:
            pickle.dump(self.words_dict, pfile, protocol=pickle.HIGHEST_PROTOCOL)
        with open(paragraphs_ids_filename, 'wb') as pfile:
            pickle.dump(self.entries_ids, pfile, protocol=pickle.HIGHEST_PROTOCOL)
        #print("Dicts saved")
    def load_dicts(logdir):
        words_ids_filename = os.path.join(logdir, params.words_ids_filename)
        paragraphs_ids_filename = os.path.join(logdir, params.paragraphs_ids_filename)

        with open(words_ids_filename, "rb") as input_file:
            words_ids = pickle.load(input_file)
        with open(paragraphs_ids_filename, "rb") as input_file:
            paragraphs_ids = pickle.load(input_file)
        return words_ids, paragraphs_ids

class PVDM:
    def __init__(self, is_training, words_embeddings=None, paragraphs_embeddings=None, num_paragraphs=None, num_words=None, logdir="logdir"):
        self.is_training = is_training
        self.set_words_embeddings(words_embeddings, num_words)
        self.set_paragraph_embeddings(paragraphs_embeddings, num_paragraphs)
        self.logdir = logdir
    def create_model(self, words_count):
        self.words = tf.placeholder(dtype=tf.int32, shape=[None, params.word_windows_size])
        self.target_words = tf.placeholder(dtype=tf.int32, shape=[None])
        self.paragraph_id = tf.placeholder(dtype=tf.int32, shape=[None])

        self.words_embedding = self.get_words_embedding()
        self.paragraph_embedding = self.get_paragraph_embedding()
        self.target = tf.one_hot(self.target_words, words_count)

        self.words_embedded = tf.nn.embedding_lookup(self.words_embedding, self.words)
        self.paragraph_embedded = tf.nn.embedding_lookup(self.paragraph_embedding, self.paragraph_id)

        self.feature = tf.reduce_sum(self.words_embedded, axis=1)
        self.feature = (self.feature + self.paragraph_embedded)/(params.word_windows_size + 1)

        self.u_weight = tf.get_variable("softmax_weight", shape=[params.word_embedding_size, words_count],trainable=self.is_training)
        self.bias = tf.get_variable("softmax_bias", shape=[words_count], trainable=self.is_training)

        self.logits = tf.matmul(self.feature,self.u_weight) + self.bias

        self.prediction = tf.nn.softmax(self.logits, dim=-1)

        self.eps = 1e-4
        self.loss = tf.reduce_mean(-tf.reduce_sum(self.target*tf.log(self.prediction+self.eps), axis=-1))
        # self.opt = tf.train.AdamOptimizer(params.learning_rate).minimize(self.loss)
        gs = tf.Variable(0, trainable=False)
        # self.opt = tf.train.AdagradOptimizer(params.learning_rate).minimize(self.loss,global_step=gs)
        if self.is_training:
            self.opt = tf.train.AdamOptimizer(params.learning_rate).minimize(self.loss,global_step=gs)
        else:
            self.opt = tf.train.AdamOptimizer(params.learning_rate).minimize(self.loss, global_step=gs)
            # self.opt = tf.train.GradientDescentOptimizer(params.learning_rate).minimize(self.loss,global_step=gs)
        self.prediction_ids = tf.cast(tf.argmax(self.prediction, axis=-1), tf.int32)
        self.accuracy = tf.reduce_mean(tf.cast(tf.equal(self.prediction_ids, self.target_words), tf.float32))
        # self.opt = tf.train.GradientDescentOptimizer(params.learning_rate).minimize(self.loss,global_step=gs)

    def load_word_embedding(embeddings_filename):
        embeddings = []
        words_ids = {}
        inverse_word_ids = {}
        with open(embeddings_filename) as emb_file:
            for line in emb_file:

                entries = line.strip().split()
                word = entries[0]
                embedding = list(map(float,entries[-params.word_embedding_size:]))
                if not word in words_ids:
                    words_ids[word] = len(words_ids)
                    inverse_word_ids[words_ids[word]] = word
                    embeddings.append(embedding)

            if params.padding_word not in words_ids:
                words_ids[params.padding_word] = len(words_ids)
                inverse_word_ids[words_ids[params.padding_word]] = params.padding_word
                embedding = np.random.normal(0, .1, params.word_embedding_size)
                embeddings.append(embedding)
            if params.unknown_token not in words_ids:
                words_ids[params.unknown_token] = len(words_ids)
                inverse_word_ids[words_ids[params.unknown_token]] = params.unknown_token
                embedding = np.random.normal(0, .1, params.word_embedding_size)
                embeddings.append(embedding)
        return words_ids, inverse_word_ids, embeddings

    def set_words_embeddings(self, init_value=None, num_elements=None):
        if init_value is not None:
            init = tf.constant(init_value, dtype=tf.float32)
            self.word_embeddings_weights = tf.get_variable("words_embeddings_vectors", initializer=init, trainable=self.is_training)
        else:
            self.word_embeddings_weights = tf.get_variable("words_embeddings_vectors", shape=(num_elements, params.word_embedding_size), trainable=self.is_training)
    def set_paragraph_embeddings(self, init_value=None, num_elements=None):
        if init_value is not None:
            init = tf.constant(init_value, dtype=tf.float32)
            self.paragraph_embeddings_weights = tf.get_variable("paragraphs_embeddings_vectors", initializer=init)
        else:
            self.paragraph_embeddings_weights = tf.get_variable("paragraphs_embeddings_vectors", shape=[num_elements, params.word_embedding_size])
    def get_words_embedding(self):
        return self.word_embeddings_weights

    def get_paragraph_embedding(self):
        return self.paragraph_embeddings_weights

class Embedding:
    def __init__(self, feature_name):

        self.words_ids, self.paragraphs_ids  = TrainData.load_dicts(feature_name)
        self.graph = tf.Graph()
        self.session = tf.Session()
        self.words_counts = len(self.words_ids)

        self.feature_name = feature_name
        # with self.session as sess:

        self.load_graph()
    def load_graph(self, paragraps_emb=None, num_paragraphs=None):
        if num_paragraphs is None:
            num_paragraphs=len(self.paragraphs_ids)
        self.graph = tf.Graph()

        with self.graph.as_default():
            self.session = tf.Session()
            self.model = PVDM(False, logdir=self.feature_name, num_words=self.words_counts,
                              num_paragraphs=num_paragraphs, paragraphs_embeddings=paragraps_emb)
            self.model.create_model(self.words_counts)
            variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)

            if paragraps_emb is not None:
                variables = [variable for variable in variables if variable.name not in {"paragraphs_embeddings_vectors:0","paragraphs_embeddings_vectors/Adam:0","paragraphs_embeddings_vectors/Adam_1:0", "beta1_power:0", "beta2_power:0"} and "Adam" not in variable.name]

            self.session.run(tf.global_variables_initializer())

            self.saver = tf.train.Saver(variables)
            self.saver.restore(self.session, os.path.join(self.feature_name, params.best_test_logdir))
            self.saver = tf.train.Saver()
    def append_paragraph_embedding(self):
        with self.graph.as_default():

            embedding = self.session.run(self.model.paragraph_embeddings_weights)

            new_paragraph_vector = np.random.standard_normal( size=(1, params.word_embedding_size))
            new_embedding = np.concatenate([embedding, new_paragraph_vector], axis=0)
        self.session.close()
        tf.reset_default_graph()

        self.load_graph(num_paragraphs=len(self.paragraphs_ids)-1, paragraps_emb=new_embedding)

    def train_new_paragraph(self, paragraph, paragraph_orig):
        data_master = TrainData(paragraph_index=self.paragraphs_ids )
        data_master.process_data([paragraph_orig])
        data_master.data_to_ids(self.words_ids)
        self.append_paragraph_embedding()
        with self.graph.as_default():
            prev_loss = 1e6
            loss = 0

            # with self.session as sess:

            number_batches = len(data_master.data)//params.batch_size
            if number_batches == 0:
                number_batches += 1
            iteration_number = params.min_iteration_number
            print('-'*10,"Start learning new paragraph", '-'*10)
            while math.fabs(loss - prev_loss) > params.eps_learn_stop or iteration_number > 0:
                summed_loss = 0
                summed_accuracy = 0
                prev_loss = loss
                for batch_number in range(number_batches):
                    paragraphs_ids, words_ids, labels_ids = data_master.get_next_batch(params.batch_size)
                    accuracy_b, loss_b, _ = self.session.run([self.model.accuracy, self.model.loss, self.model.opt],
                                                         feed_dict={self.model.paragraph_id: paragraphs_ids,
                                                                    self.model.words: words_ids,
                                                                    self.model.target_words: labels_ids})
                    summed_loss += loss_b
                    summed_accuracy += accuracy_b
                loss = summed_loss/number_batches
                accuracy = summed_accuracy/number_batches
                if iteration_number < params.min_iteration_number - params.max_iterations_number:
                    break

                iteration_number -= 1
                # print("Learning paragraph loss: %f | Previous loss: %f | Accuracy: %f "%(loss,prev_loss,accuracy))

            # if accuracy < 0.2:
            #     print(paragraph)
            # print("Trained new paragraph")
            self.saver.save(self.session, os.path.join(self.feature_name, params.best_test_logdir))
            data_master.save_dicts(self.feature_name)

            # print('-' * 10, "----------------------------", '-' * 10)
            # print()
    def get_embedding(self, paragraph):
        paragraph_id = self.paragraphs_ids[paragraph]
        with self.graph.as_default():
            # with self.session as sess:
            embedding = self.session.run(tf.nn.embedding_lookup(self.model.paragraph_embeddings_weights, paragraph_id))
        return embedding
    def get_embedding_multiple(self, paragraphs):
        paragraphs_ids = [self.paragraphs_ids[paragraph] for paragraph in paragraphs]
        with self.graph.as_default():
            # with self.session as sess:
            embeddings = self.session.run(tf.nn.embedding_lookup(self.model.paragraph_embeddings_weights, paragraphs_ids))
        return embeddings
    def vectorize(self, paragraph):
        paragraph_orig = paragraph
        paragraph = TrainData.refine(paragraph)
        if paragraph not in self.paragraphs_ids:
            # print("Size before: ", len(self.paragraphs_ids))
            self.paragraphs_ids[paragraph] = len(self.paragraphs_ids)
            # print("Size after: ", len(self.paragraphs_ids))
            self.train_new_paragraph(paragraph, paragraph_orig)
        return self.get_embedding(paragraph)
    def vectorize_paragraphs(self, paragraphs):
        paragraphs_orig = paragraphs
        paragraphs = [TrainData.refine(paragraph) for paragraph in paragraphs]
        for paragraph_index in range(len(paragraphs)):
            if paragraphs[paragraph_index] not in self.paragraphs_ids:
                # print("Size before: ", len(self.paragraphs_ids))
                paragraph = paragraphs[paragraph_index]
                paragraph_orig = paragraphs_orig[paragraph_index]

                self.paragraphs_ids[paragraph] = len(self.paragraphs_ids)
                # print("Size after: ", len(self.paragraphs_ids))
                self.train_new_paragraph(paragraph, paragraph_orig)
        return self.get_embedding_multiple(paragraphs)
        # return self.get_embedding(paragraph)


text:
import cairo
import numpy as np
from gi import require_version
import gi

from utils import reshape

gi.require_version("Rsvg", "2.0")
from gi.repository import Rsvg

def rasterize_in_memory(xml_desc):
    img = cairo.ImageSurface(cairo.FORMAT_A8, 28, 28)
    ctx = cairo.Context(img)
    handle = Rsvg.Handle.new_from_data(xml_desc.encode())
    handle.render_cairo(ctx)
    buf = img.get_data()
    img_array = np.ndarray(shape=(28, 28),
                           dtype=np.uint8,
                           buffer=buf)

    img_array = reshape(img_array)

    return img_array

ext: py

max_stars_repo_name: zohdit/DeepJanus

max_issues_repo_name: zohdit/DeepJanus

max_forks_repo_name: zohdit/DeepJanus

reason: import numpy


text:
import pickle
from collections import defaultdict

import numpy as np

from datasets import DATASET
import preprocessing
import generate_folds
import multilabel_classification
import textual_similarity
import evaluation

# Set these booleans to preprocess, classify, or compute similarities.
preprocess = True
gen_folds = True
classify = True
similarity = True

cuts = (5, 10)  # Final rank cuts

precisions = defaultdict(list)
recalls = defaultdict(list)
f1_scores = defaultdict(list)

if preprocess:
    print('Preprocessing')
    preprocessing.main()

if gen_folds:
    print('Generating folds')
    generate_folds.main()

for fold in range(DATASET.total_folds):

    DATASET.set_fold(fold)

    if classify:
        print(f'Multi-label classification fold {fold}')
        multilabel_classification.main()

    if similarity:
        print(f'Textual similarity fold {fold}')
        textual_similarity.main()

    print(f'Evaluating fold {fold}')
    with DATASET.train_set.open('rb') as file:
        train_set = pickle.load(file)
    param_estimate_data = evaluation.prepare_estimate_date(train_set)

    for at in cuts:
        evaluation.at = at
        pr, re, f1 = evaluation.recommend(param_estimate_data)
        precisions[at].append(pr)
        recalls[at].append(re)
        f1_scores[at].append(f1)

for at in cuts:
    print()

    print(f'@{at}')
    print('p:', precisions[at])
    print('mean:', np.mean(precisions[at]))
    print('std:', np.std(precisions[at], ddof=1))

    print('r:', recalls[at])
    print('mean:', np.mean(recalls[at]))
    print('std:', np.std(recalls[at], ddof=1))

    print('f:', f1_scores[at])
    print('mean:', np.mean(f1_scores[at]))
    print('std:', np.std(f1_scores[at], ddof=1))


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Mar 30 19:09:15 2019

@author: DELL
"""

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('low spees a1.csv')
X = dataset.iloc[:,2:3].values
y = dataset.iloc[:, 4].values
z = dataset.iloc[:,0:1].values
h = np.array(X)
k = np.array(y)
multi = []

for value in range(0,2597,1):
    multi.append(k[value]*h[value])

sum = 0

for val in multi:
    sum = sum + val
print(sum)

#q=np.array(z)
#f=np.array(k)

#from sklearn.linear_model import LinearRegression
#lin_reg = LinearRegression()
#lin_reg.fit(q, f)

#from sklearn.preprocessing import PolynomialFeatures
#poly_reg = PolynomialFeatures(degree = 2)
#Z_poly = poly_reg.fit_transform(q)
#poly_reg.fit(Z_poly, f)
#lin_reg_2 = LinearRegression()
#lin_reg_2.fit(Z_poly, f)

# Visualising the Linear Regression results
#plt.scatter(q, f, color = 'red')
#plt.plot(q, lin_reg.predict(q), color = 'blue')
#plt.title('Regression based speed')
#plt.xlabel('time')
#plt.ylabel('Speed')
#plt.show()

#plt.scatter(q, f, color = 'red')
#plt.plot(q, lin_reg_2.predict(poly_reg.fit_transform(q)), color = 'blue')
#plt.title('Regression based analysis')
#plt.xlabel('time')
#plt.ylabel('Speed')
#plt.show()

#lin_reg_2.predict(poly_reg.fit_transform(45.00))


text:
import numpy as np
import tensorflow as tf

import avod.core.format_checker as fc

def anchor_to_offset(anchors, ground_truth):
    """Encodes the anchor regression predictions with the
    ground truth.

    Args:
        anchors: A numpy array of shape (N, 6) representing
            the generated anchors.
        ground_truth: A numpy array of shape (6,) containing
            the label boxes in the anchor format.

    Returns:
        anchor_offsets: A numpy array of shape (N, 6)
            encoded/normalized with the ground-truth, representing the
            offsets.
    """

    fc.check_anchor_format(anchors)

    anchors = np.asarray(anchors).reshape(-1, 6)
    ground_truth = np.reshape(ground_truth, (6,))

    # t_x_gt = (x_gt - x_anch)/dim_x_anch
    t_x_gt = (ground_truth[0] - anchors[:, 0]) / anchors[:, 3]
    # t_y_gt = (y_gt - y_anch)/dim_y_anch
    t_y_gt = (ground_truth[1] - anchors[:, 1]) / anchors[:, 4]
    # t_z_gt = (z_gt - z_anch)/dim_z_anch
    t_z_gt = (ground_truth[2] - anchors[:, 2]) / anchors[:, 5]
    # t_dx_gt = log(dim_x_gt/dim_x_anch)
    t_dx_gt = np.log(ground_truth[3] / anchors[:, 3])
    # t_dy_gt = log(dim_y_gt/dim_y_anch)
    t_dy_gt = np.log(ground_truth[4] / anchors[:, 4])
    # t_dz_gt = log(dim_z_gt/dim_z_anch)
    t_dz_gt = np.log(ground_truth[5] / anchors[:, 5])
    anchor_offsets = np.stack((t_x_gt,
                               t_y_gt,
                               t_z_gt,
                               t_dx_gt,
                               t_dy_gt,
                               t_dz_gt), axis=1)
    return anchor_offsets

def tf_anchor_to_offset(anchors, ground_truth):
    """Encodes the anchor regression predictions with the
    ground truth.

    This function assumes the ground_truth tensor has been arranged
    in a way that each corresponding row in ground_truth, is matched
    with that anchor according to the highest IoU.
    For instance, the ground_truth might be a matrix of shape (256, 6)
    of repeated entries for the original ground truth of shape (x, 6),
    where each entry has been selected as the highest IoU match with that
    anchor. This is different from the same function in numpy format, where
    we loop through all the ground truth anchors, and calculate IoUs for
    each and then select the match with the highest IoU.

    Args:
        anchors: A tensor of shape (N, 6) representing
            the generated anchors.
        ground_truth: A tensor of shape (N, 6) containing
            the label boxes in the anchor format. Each ground-truth entry
            has been matched with the anchor in the same entry as having
            the highest IoU.

    Returns:
        anchor_offsets: A tensor of shape (N, 6)
            encoded/normalized with the ground-truth, representing the
            offsets.
    """

    fc.check_anchor_format(anchors)

    # Make sure anchors and anchor_gts have the same shape
    dim_cond = tf.equal(tf.shape(anchors), tf.shape(ground_truth))

    with tf.control_dependencies([dim_cond]):
        t_x_gt = (ground_truth[:, 0] - anchors[:, 0]) / anchors[:, 3]
        t_y_gt = (ground_truth[:, 1] - anchors[:, 1]) / anchors[:, 4]
        t_z_gt = (ground_truth[:, 2] - anchors[:, 2]) / anchors[:, 5]
        t_dx_gt = tf.log(ground_truth[:, 3] / anchors[:, 3])
        t_dy_gt = tf.log(ground_truth[:, 4] / anchors[:, 4])
        t_dz_gt = tf.log(ground_truth[:, 5] / anchors[:, 5])
        anchor_offsets = tf.stack((t_x_gt,
                                   t_y_gt,
                                   t_z_gt,
                                   t_dx_gt,
                                   t_dy_gt,
                                   t_dz_gt), axis=1)

        return anchor_offsets

def offset_to_anchor(anchors, offsets):
    """Decodes the anchor regression predictions with the
    anchor.

    Args:
        anchors: A numpy array or a tensor of shape [N, 6]
            representing the generated anchors.
        offsets: A numpy array or a tensor of shape
            [N, 6] containing the predicted offsets in the
            anchor format  [x, y, z, dim_x, dim_y, dim_z].

    Returns:
        anchors: A numpy array of shape [N, 6]
            representing the predicted anchor boxes.
    """

    fc.check_anchor_format(anchors)
    fc.check_anchor_format(offsets)

    # x = dx * dim_x + x_anch
    x_pred = (offsets[:, 0] * anchors[:, 3]) + anchors[:, 0]
    # y = dy * dim_y + x_anch
    y_pred = (offsets[:, 1] * anchors[:, 4]) + anchors[:, 1]
    # z = dz * dim_z + z_anch
    z_pred = (offsets[:, 2] * anchors[:, 5]) + anchors[:, 2]

    tensor_format = isinstance(anchors, tf.Tensor)
    if tensor_format:
        # dim_x = exp(log(dim_x) + dx)
        dx_pred = tf.exp(tf.log(anchors[:, 3]) + offsets[:, 3])
        # dim_y = exp(log(dim_y) + dy)
        dy_pred = tf.exp(tf.log(anchors[:, 4]) + offsets[:, 4])
        # dim_z = exp(log(dim_z) + dz)
        dz_pred = tf.exp(tf.log(anchors[:, 5]) + offsets[:, 5])
        anchors = tf.stack((x_pred,
                            y_pred,
                            z_pred,
                            dx_pred,
                            dy_pred,
                            dz_pred), axis=1)
    else:
        dx_pred = np.exp(np.log(anchors[:, 3]) + offsets[:, 3])
        dy_pred = np.exp(np.log(anchors[:, 4]) + offsets[:, 4])
        dz_pred = np.exp(np.log(anchors[:, 5]) + offsets[:, 5])
        anchors = np.stack((x_pred,
                            y_pred,
                            z_pred,
                            dx_pred,
                            dy_pred,
                            dz_pred), axis=1)

    return anchors

max_stars_repo_name: Liaoqing-up/avod

max_issues_repo_name: zhiranyan/AVOD-BottleneckLstm

max_forks_repo_name: zhiranyan/AVOD-BottleneckLstm


text:
from bokeh.plotting import figure, output_file, show
import numpy as np

if __name__ == '__main__':
	x = np.linspace(0, 2*np.pi, 100)
	y_lin = x
	y_sqrt = np.sqrt(x)
	y_log = np.log(x)
	y_sin = np.sin(x)

	output_file("plot.html")

	fig = figure(title="Simple Plots")
	fig.line(x=x, y=y_lin, color='red', legend="Linear")
	fig.line(x=x, y=y_sqrt, color='blue', legend="Square Root")
	fig.line(x=x, y=y_log, color='black', legend="Log")
	fig.line(x=x, y=y_sin, color='green', legend="Sin")

	fig.legend.location = "top_left"
	fig.legend.click_policy="hide"

	show(fig)

max_stars_repo_name: ashikari/tutorial

max_issues_repo_name: ashikari/tutorial

max_forks_repo_name: ashikari/tutorial


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 18 22:59:59 2020

@author: Christian
"""

import hysteresis
import naturalize as nat
import numpy as np
import os

import matplotlib.pyplot as plt

from functions import ftest, fitness

np.random.seed(40)
llims = [np.array([-10, -10, -10])]
ulims = [np.array([10, 10, 10])]
genePool = nat.DefaultGenePool(llims, ulims)

helper = nat.AlgorithmHelper(ftest, genePool = genePool)

Ngen = 1000
Npop = 50
Ncouples = 20
Nsurvive = 2
mutThresold = 0.1

Analysis = nat.GeneticAlgorithm(helper, Npop, Ncouples, Nsurvive, mutThresold)
solution  = Analysis.optimize(Ngen)
# Analysis.optimize(Ngen)

# nat.pickleAnalysis(Analysis, 'TestFile.obj')
# Analysis = nat.readPickle('TestFile.obj')

# fig, ax = plt.subplots()
# nat.plotAvgFitness(fig, ax, Analysis)
# nat.plotMinFitness(fig, ax, Analysis)
# nat.plotTotalFitness(fig, ax, Analysis)

# yAvg = np.zeros(Ngen)
# ybest = np.zeros(Ngen)
# scoreAvg = np.zeros(Ngen)
# scoreBest = np.zeros(Ngen)

# BestOverTime = np.zeros(Ngen)

# for ii in range(Ngen):
#     # Get the genome
#     out = np.array(algorithm.gens[ii].getCurrentGenome())
#     yAvg[ii] = -np.average(out[:,1])
#     ybest[ii] = -np.min(out[:,1])

#     scoreAvg[ii] = -np.average(algorithm.gens[ii].scores)

#     scoreBest[ii] = -np.min(algorithm.gens[ii].scores)
#     scoreBest[ii] = -np.min(algorithm.gens[ii].scores)

# BestOverTime = algorithm.BestValues
# print(algorithm.gens[5].getCurrentGenome())

# fig, ax = plt.subplots()
# plt.plot(yAvg)
# plt.plot(ybest)

# fig, ax = plt.subplots()
# plt.plot(scoreAvg)
# plt.plot(scoreBest)
# plt.plot(-BestOverTime)

max_stars_repo_name: cslotboom/Naturalize

max_issues_repo_name: cslotboom/Naturalize

max_forks_repo_name: cslotboom/Naturalize


text:
from absl import flags
from absl import logging
import math
import numpy as np
from scipy.sparse.csgraph import minimum_spanning_tree
import time

from .utils import *

FLAGS = flags.FLAGS

flags.DEFINE_integer("ghp_approx", None, "Number of nearest distances per node to consider", lower_bound=1)

def eval_from_matrix(train_features, train_labels):
    logging.log(logging.DEBUG, "Start computing distance matrix D with euclidian measure")
    start = time.time()
    d = compute_distance_matrix_loo(train_features, "squared_l2")
    #d = np.sqrt(compute_distance_matrix_loo(train_features, "squared_l2"))

    if FLAGS.ghp_approx:
        indices = np.argpartition(d, FLAGS.ghp_approx, axis=1)[:,:FLAGS.ghp_approx]
        for row_i in range(d.shape[0]):
            mask = np.ones(d.shape[1],dtype=bool)
            mask[indices[row_i,:]] = False
            d[row_i, mask] = 0.0

    end = time.time()
    logging.log(logging.DEBUG, "D computed in {} seconds".format(end - start))

    logging.log(logging.DEBUG, "Start computing MST on matrix d")
    start = time.time()
    d = minimum_spanning_tree(d).tocoo()
    end = time.time()
    logging.log(logging.DEBUG, "MST computed in {} seconds".format(end - start))

    logging.log(logging.DEBUG, "Start computing estimator")
    start = time.time()

    classes, classes_counts = np.unique(train_labels, return_counts=True)

    num_train_samples = train_labels.size
    num_classes = len(classes)

    mapping = {}
    idx = 0
    for c in classes:
        mapping[c] = idx
        idx += 1

    deltas = []
    for i in range(num_classes-1):
        deltas.append([0.0]*(num_classes-i-1))

    # Claculate number of dichotomous edges
    for i in range(num_train_samples - 1):
        label_1 = mapping[train_labels[d.row[i]]]
        label_2 = mapping[train_labels[d.col[i]]]
        if label_1 == label_2:
            continue
        if label_1 > label_2:
            tmp = label_1
            label_1 = label_2
            label_2 = tmp
        deltas[label_1][label_2 - label_1 - 1] += 1

    # Devide the number of dichotomous edges by 2 * num_train_samples to get estimator of deltas
    deltas = [[item / (2.0 * num_train_samples) for item in sublist] for sublist in deltas]

    # Sum up all the deltas
    delta_sum = sum([sum(sublist) for sublist in deltas])

    end = time.time()
    logging.log(logging.DEBUG, "Estimators computed in {} seconds".format(end - start))

    upper = 2.0 * delta_sum

    lower = ((num_classes - 1.0) / float(num_classes)) * (1.0 - math.sqrt(max(0.0, 1.0 - ((2.0 * num_classes)/(num_classes - 1.0) * delta_sum))))

    return {"default": [upper, lower]}

reason: import numpy,from scipy


text:
import tensorflow as tf
import numpy as np
import lpips
import os
import torch
from tqdm import tqdm
import matplotlib.pyplot as plt
from modules.dataset import load_valid_dataset
from modules.models import RRDB_Model, RRDB_Model_16x, RFB_Model_16x
from modules.utils import plot_to_image,load_yaml
lpips_alex = lpips.LPIPS(net='alex')

def evaluate_lpips(sr, hr):
    sr, hr=tf.expand_dims(tf.transpose(sr, [2, 0, 1]), axis=0), tf.expand_dims(tf.transpose(hr, [2, 0, 1]), axis=0)
    res=lpips_alex.forward(torch.Tensor(hr.numpy()), torch.Tensor(sr.numpy())) #Calculate LPIPS Similarity
    return res.detach().numpy().flatten()[0]

def evaluate_psnr(sr, hr):
    def PSNR(y_true,y_pred, image_range = 1):
        mse=tf.reduce_mean( (y_true - y_pred) ** 2 )
        return 20 * log10(image_range / (mse ** 0.5))

    def log10(x):
        numerator = tf.math.log(x)
        denominator = tf.math.log(tf.constant(10, dtype=numerator.dtype))
        return numerator / denominator
    res=PSNR(hr, sr)

    return res.numpy()

def evaluate_ssim(sr, hr):
    return tf.image.ssim(sr, hr, max_val=1, filter_size=11,
                          filter_sigma=1.5, k1=0.01, k2=0.03)

def plot_examples(data_list, plot_size=4):
    # returns a image with all (SR, HR) pair visualized
    plt.close()

    data_list = data_list[:min(14, len(data_list))]       # plot first 14 images if more data exists
    num_data = len(data_list)
    fig=plt.figure(figsize=(plot_size * num_data, plot_size * 2))

    for idx,x in enumerate(data_list):
        plt.subplot(2, num_data, 1 + idx)
        plt.imshow(np.clip(x[0],0,1))
        plt.axis('off')

        plt.subplot(2, num_data, 1 + idx + num_data)
        plt.imshow(x[1])
        plt.axis('off')

    return plot_to_image(fig)

def evaluate_dataset(dataset, model, cfg, return_sum=True):
    # evalutaes ever
    sum_LPIPS, sum_PSNR, sum_SSIM = 0, 0, 0
    list_PSNR, list_SSIM, list_LPIPS = [], [], []
    data_list = []    # save all SR image for plotting

    print("Evaluating on test dataset.")
    for lr, hr in tqdm(dataset, position=0, leave=True):
        sr = model(lr[np.newaxis,:], training=False)[0] #Generate SR image

        if return_sum:
            if cfg['logging']['psnr']:
                sum_PSNR += evaluate_psnr(sr, hr)
            if cfg['logging']['ssim']:
                sum_SSIM += evaluate_ssim(sr, hr)
            if cfg['logging']['lpips']:
                sum_LPIPS += evaluate_lpips(sr, hr)
        else:
            if cfg['logging']['psnr']:
                list_PSNR.append(evaluate_psnr(sr, hr))
            if cfg['logging']['ssim']:
                list_SSIM.append(evaluate_ssim(sr, hr))
            if cfg['logging']['lpips']:
                list_LPIPS.append(evaluate_lpips(sr,hr))

        if cfg['logging']['plot_samples']:
            data_list.append((sr, hr))

    num_data = len(dataset)
    logs={}
    if return_sum:
        if cfg['logging']['psnr']:
            logs['psnr'] = sum_PSNR / num_data
        if cfg['logging']['ssim']:
            logs['ssim'] = sum_SSIM / num_data
        if cfg['logging']['lpips']:
            logs['lpips'] = sum_LPIPS / num_data
    else:
        if cfg['logging']['psnr']:
            logs['psnr'] = list_PSNR
        if cfg['logging']['ssim']:
            logs['ssim'] = list_SSIM
        if cfg['logging']['lpips']:
            logs['lpips'] = list_LPIPS
    if cfg['logging']['plot_samples']:
        logs['samples'] = plot_examples(data_list)

    return logs

def get_noise_layers(generator, plot_layer_wise=True):
    noise_feature=[]
    def recurse_all_layer(layer):
        try:  # pass only when layer is tf.keras.Model
            _=layer.layers
        except: # if layer is tf.keras.Layer
            if hasattr(layer,'applynoise'):     # if layer has noise layer
                noise_feature.append(layer.applynoise.weights[0])
            return

        for sublayer in layer.layers:
            recurse_all_layer(sublayer)

    recurse_all_layer(generator)

    if plot_layer_wise:
        max_arr, mean_arr, noise_arr=[],[],[]
        for x in noise_feature:
            max_arr.append(max(np.abs(x)))
            mean_arr.append(np.median(np.abs(x)))
            noise_arr.append(np.abs(x))

        plt.boxplot(noise_arr)
        plt.plot(np.arange(1, len(max_arr)+1),max_arr, c='black')
        plt.plot(np.arange(1, len(max_arr)+1),mean_arr, c='orange')
        plt.xticks([])
        plt.xlabel('# layer')
        plt.show()

    return noise_feature

def evaluate_with_path(cfg_path, dataset_path, scale=4, return_sum=True):
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    cfg=load_yaml(cfg_path)
    if cfg['network_G']['name']=='RRDB':    # ESRGAN 4x
        model = RRDB_Model(None, cfg['ch_size'], cfg['network_G'])
    elif cfg['network_G']['name']=='RRDB_CIPLAB':
        model = RRDB_Model_16x(None, cfg['ch_size'], cfg['network_G'])
    elif cfg['network_G']['name']=='RFB_ESRGAN':
        model = RFB_Model_16x(None, cfg['ch_size'], cfg['network_G'])

    checkpoint_dir = cfg['log_dir'] + '/checkpoints/'
    checkpoint = tf.train.Checkpoint(step=tf.Variable(0, name='step'), model=model)
    manager = tf.train.CheckpointManager(checkpoint=checkpoint,
                                         directory=checkpoint_dir,
                                         max_to_keep=3)
    if manager.latest_checkpoint:
        checkpoint.restore(manager.latest_checkpoint)
        print('[*] load ckpt from {} at step {}.'.format(
            manager.latest_checkpoint, checkpoint.step.numpy()))
    else:
        print(f"Checkpoint doesn't exist in {checkpoint_dir}.")

    cfg_logging={'logging': {'lpips':True, 'psnr':True, 'ssim': True, 'plot_samples': False}}
    dataset = load_valid_dataset(dataset_path, scale, crop_centor=0)

    logs = evaluate_dataset(dataset, model, cfg_logging, return_sum=return_sum)
    return logs


text:
import sys

import numpy as np
import torch
from torch import optim
from torch.utils.data import DataLoader, TensorDataset

from ..utils.utils_math import CRF
from ..utils.utils_plot import plot_roc_curve, plot_pr_curve, plot_roc_ind, plot_pr_ind
from ..utils.utils_results import performance, final_results_output, print_metric_dict

START_TAG = "<START>"
STOP_TAG = "<STOP>"

def get_partition(feature, target, length, train_index, val_index):
    feature = np.array(feature)
    x_train = feature[train_index]
    x_val = feature[val_index]
    y_train = np.array(target)[train_index]
    y_val = np.array(target)[val_index]
    train_length = np.array(length)[train_index]
    test_length = np.array(length)[val_index]

    return x_train, x_val, y_train, y_val, train_length, test_length

def crf_cv_process(vectors, labels, seq_length_list, folds, out_dir, params_dict):
    results = []
    cv_labels = []
    cv_prob = []

    # predicted_labels = np.zeros(len(seq_length_list))
    # predicted_prob = np.zeros(len(seq_length_list))

    count = 0

    width = vectors.shape[-1]

    for train_index, val_index in folds:
        train_x, test_x, train_y, test_y, train_length, test_length = get_partition(vectors, labels, seq_length_list,
                                                                                train_index, val_index)

        lr = params_dict['lr']
        num_epochs = params_dict['epochs']
        batch_size = params_dict['batch_size']
        # 筛选最后的模型参数
        min_loss = float('inf')  # 无穷大

        opt_test_prob, opt_test_y_hat = [], []
        for num_epoch in range(num_epochs):
            test_loss, test_prob, test_y_hat = crf_main(train_x, test_x, train_y, test_y, width, batch_size, num_epoch,
                                                        lr)
            if test_loss < min_loss:
                min_loss = test_loss
                opt_test_prob = test_prob
                opt_test_y_hat = test_y_hat

        test_label_list, test_prob_list, predict_label_list = preprocess4evaluate(test_y, opt_test_prob,
                                                                                opt_test_y_hat, test_length)

        result = performance(test_label_list, predict_label_list, test_prob_list, bi_or_multi=False, res=True)
        results.append(result)

        cv_labels.append(test_label_list)
        cv_prob.append(test_prob_list)

        count += 1
        print(" Round[%d]: Accuracy = %.3f | minimum loss = %.4f" % (count, result[0], min_loss))
    print('\n')
    plot_roc_curve(cv_labels, cv_prob, out_dir)  # 绘制ROC曲线
    plot_pr_curve(cv_labels, cv_prob, out_dir)  # 绘制PR曲线

    final_results = np.array(results).mean(axis=0)
    print_metric_dict(final_results, ind=False)
    final_results_output(final_results, out_dir, ind=False, multi=False)  # 将指标写入文件

def crf_ind_process(vectors, labels, ind_vectors, ind_labels, ind_seq_length_list, out_dir, params_dict):
    lr = params_dict['lr']
    num_epochs = params_dict['epochs']
    batch_size = params_dict['batch_size']
    # 筛选最后的模型参数
    min_loss = float('inf')  # 无穷大
    width = vectors.shape[-1]

    opt_test_prob, opt_test_y_hat = [], []
    for num_epoch in range(num_epochs):
        test_loss, test_prob, test_y_hat = crf_main(vectors, ind_vectors, labels, ind_labels, width, batch_size,
                                                    num_epoch, lr)
        if test_loss < min_loss:
            min_loss = test_loss
            opt_test_prob = test_prob
            opt_test_y_hat = test_y_hat

    test_label_list, test_prob_list, predict_label_list = preprocess4evaluate(ind_labels, opt_test_prob,
                                                                              opt_test_y_hat, ind_seq_length_list)

    final_result = performance(test_label_list, predict_label_list, test_prob_list, bi_or_multi=False, res=True)
    print_metric_dict(final_result, ind=True)

    plot_roc_ind(test_label_list, test_prob_list, out_dir)  # 绘制ROC曲线
    plot_pr_ind(test_label_list, test_prob_list, out_dir)  # 绘制PR曲线

    final_results_output(final_result, out_dir, ind=True, multi=False)  # 将指标写入文件
    # prob_output_res(final_target_list, final_predict_list, final_prob_list, out_dir)

def preprocess4evaluate(test_y, test_prob, test_y_hat, test_length):
    """ 将正确的测试集标签读取出来，而非全部内容进行评测 """
    test_label_list = []
    test_prob_list = []
    predict_label_list = []
    for i in range(len(test_length)):
        seq_len = test_length[i]
        test_label_list += list(test_y[i][:seq_len])
        test_prob_list += list(test_prob[i][:seq_len])
        predict_label_list += list(test_y_hat[i][:seq_len])
    return test_label_list, test_prob_list, predict_label_list

def make_data(train_x, test_x, train_y, test_y, batch_size):
    # train_x = np.random.normal(0, 0.1, (num_seq, seq_len, width))
    # test_x = np.random.normal(0, 0.1, (num_seq//2, seq_len, width))
    #
    # train_y = np.random.randint(0, 2, size=(num_seq, seq_len))
    # test_y = np.random.randint(0, 2, size=(num_seq//2, seq_len))

    if sys.platform.startswith('win'):
        num_workers = 0
    else:
        num_workers = 4

    train_dataset = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))
    train_data_iter = DataLoader(train_dataset, batch_size, shuffle=True, num_workers=num_workers)

    test_dataset = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y))
    test_data_iter = DataLoader(test_dataset, 1, shuffle=False, num_workers=num_workers)

    return train_data_iter, test_data_iter

def train_crf(model, data_iter, epoch, optimizer=None):
    train_loss_sum = 0.0
    n = 1
    for x, y in data_iter:
        # 步骤1. 记住，pytorch积累了梯度
        # We need to clear them out before each instance
        model.zero_grad()

        # 步骤3. 向前运行
        loss = model.neg_log_likelihood_parallel(x.float(), y.long())

        # 步骤4.通过optimizer.step()
        loss.backward()
        optimizer.step()

        train_loss_sum += loss.item()
        n += y.shape[0]

    print('epoch[%d]: train loss: %.4f ' % (epoch + 1, train_loss_sum / n))
    # for epoch in range(num_epochs):
    # train_loss_sum = 0.0
    # n = 1
    # for x, y in data_iter:
    #     # 步骤1. 记住，pytorch积累了梯度
    #     # We need to clear them out before each instance
    #     model.zero_grad()
    #
    #     # 步骤3. 向前运行
    #     loss = model.neg_log_likelihood_parallel(x.float(), y.long())
    #
    #     # 步骤4.通过optimizer.step()
    #     loss.backward()
    #     optimizer.step()
    #
    #     train_loss_sum += loss.item()
    #     n += y.shape[0]
    #
    # print('epoch[%d]: train loss: %.4f ' % (epoch + 1, train_loss_sum / n))

def test_crf(model, data_iter, test_x):
    tag_hat_list = []
    test_loss_sum = 0.0
    n = 1
    with torch.no_grad():
        for x, y in data_iter:
            score, tag_seq = model(x.float())
            tag_hat_list.append(tag_seq)
            loss = model.neg_log_likelihood_parallel(x.float(), y.long())
            test_loss_sum += loss.item()
            n += y.shape[0]
    test_loss = test_loss_sum / n

    if sys.platform.startswith('win'):
        num_workers = 0
    else:
        num_workers = 4

    test_y_hat = np.array(tag_hat_list)
    test_dataset = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y_hat))
    test_data_iter = DataLoader(test_dataset, 1, shuffle=False, num_workers=num_workers)

    test_prob = []
    with torch.no_grad():
        for x, y_ in test_data_iter:
            prob_list = model.calculate_pro_new(x.float(), y_.long())
            test_prob.append(prob_list)

    # prob_list = []
    # with torch.no_grad():
    #     for x, y in test_data_iter:
    #         prob_list += model.calculate_pro(x.float(), y.long())

    return test_loss, np.array(test_prob), test_y_hat

def crf_main(train_x, test_x, train_y, test_y, width, batch_size, num_epoch, lr):
    train_data_iter, test_data_iter = make_data(train_x, test_x, train_y, test_y, batch_size)
    # for X, y in train_data_iter:
    #     print(X.size())
    #     print(y)
    #     break
    tag_to_ix = {"B": 0, "O": 1, START_TAG: 2, STOP_TAG: 3}
    model = CRF(width, tag_to_ix)

    optimizer = optim.SGD(model.parameters(), lr=lr, weight_decay=1e-4)
    train_crf(model, train_data_iter, num_epoch, optimizer)

    test_loss, test_prob, test_y_hat = test_crf(model, test_data_iter, test_x)
    # print(test_prob)
    return test_loss, test_prob, test_y_hat

# if __name__ == '__main__':
#     START_TAG = "<START>"
#     STOP_TAG = "<STOP>"
#     TAG2INDEX = {"B": 0, "O": 1, START_TAG: 2, STOP_TAG: 3}
#
#     NUM_SEQ, SEQ_LEN, WIDTH, BATCH_SIZE, NUM_EPOCHS = 100, 10, 10, 5, 13
#     print('... CRF processing ...\n')
#
#     crf_main(TAG2INDEX, NUM_SEQ, SEQ_LEN, WIDTH, BATCH_SIZE, NUM_EPOCHS)
#
#     print('\nFinish!')


text:
"""
Utility functions for working with dataframes
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm
from sklearn.model_selection import train_test_split

print("Testing")

def tvt_split(X):
    """
    A function that splits a dataframe into train, valdation, and test
    """

    train, test = train_test_split(X)
    train, val = train_test_split(train)

    return train

def gauss_plot(my_set=None, my_mean=None, my_stdev=None, width=2,
               height=norm.pdf(0), output_plot=False, color='b'):
    """
    This is a function to plot the gaussian distribution, the input can be a
    pandas series, or it can be a specified mean and standard deviation.
    The output can also be specified as plot or xy values.

    Dependencies:
    * scipy.stats.norm.pdf  (from scipy.stats import norm)
    * numpy.arrange         (import numpy as np)
    * matplotlib.pyplot     (import matplotlib.pyplot as plt)
    * pandas                (import pandas as pd)

    Inputs:
    my_set   = a pandas series
    my_mean  = a personally set mean
    my_stdev = a personally set standard deviation
    width = number of standard deviations to plot(bidirectional)
        for example 2 would produce a graph from
        -2 stdevs to 2 stdevs from mean
    height = Scale the y value of the gaussian.
        Note that this does change the shape of the curve, but if you don't
        want to deal with multiple axes then this could be useful
        purely as a visualization tool
        see _insert link here_ for details on how the visualization is affected
        by the height parameter.
    output_plot = should the function output a plot or return x,y values?
        True = plot directly (using matplotlib.pyplot.plot())
        False = return x, and y values (format a,b = thisfunct(your parameters)
    color = color of graph. this only matters if output_plot = True

    Returns:
    if output_plot = True :
      returns nothing, but should print plot (make sure of %matplotlib inline)
    else if output_plot = False:
      returns x and y values of gaussian plot
    """
    try:
        if my_set is None:
            if my_mean is None and my_stdev is None:
                print("You are missing required values")
                return
            else:
                lb = (my_mean - (my_stdev * width))
                ub = (my_mean + (my_stdev * width))
                x_axis = np.arange(lb, ub, 0.01)
                if output_plot is True:
                    plt.plot(x_axis,
                             (norm.pdf(x_axis, my_mean, my_stdev)) * height,
                             color=color)
                    return
                else:
                    return x_axis, (
                        norm.pdf(x_axis, my_mean, my_stdev)) * height
    except ValueError:
        set_mean = my_set.mean()
        set_stdev = my_set.std()
        lb = (set_mean - (set_stdev * width))
        ub = (set_mean + (set_stdev * width))
        x_axis = np.arange(lb, ub, 0.01)
        if output_plot is True:
            plt.plot(x_axis, (norm.pdf(x_axis, set_mean, set_stdev) * height),
                     color=color)
            return
        else:
            return x_axis, (norm.pdf(x_axis, set_mean, set_stdev)) * height

# Testing both "manual" cases
gauss_plot(None, 0, 1, 3, output_plot=True, color='r')
x, y = gauss_plot(None, 1, 3, 3, output_plot=False)
plt.plot(x, y, color='g')


text:
"""
Author: Sage Bionetworks

CP: aryton.tediarjo@sagebase.org

About:
This module is primarily used as a wrapper for PDKit package and rotation detection on gait features.
Additional featurization option added into this module is as follows:
    - Simple Sensor Filters (variance, resampling)
    - Rotation Detection on longitudinal data
    - Window feature computation for PDKit (number of steps per given window)

References:
    Rotation-Detection Paper : https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5811655/
    PDKit Docs               : https://pdkit.readthedocs.io/_/downloads/en/latest/pdf/
    PDKit Gait Source Codes  : https://github.com/pdkit/gait_processor.py
    Freeze of Gait Docs      : https://ieeexplore.ieee.org/document/5325884
"""

# future liibrary imports
from __future__ import unicode_literals
from __future__ import print_function

# standard library import
import pandas as pd
import numpy as np
from sklearn import metrics

# pdkit imports
import pdkit
import pdkit.processor
from pdkit.utils import (butter_lowpass_filter,
                         numerical_integration)

class GaitFeatures:

    """
    Parameters used in gait features pipeline

    Parameters
    ----------
    sensor_variance_cutoff: minimal variance on longitudinal data
    sensor_window_size: desired window size
    sensor_window_overlap: percentage of overlap in window iteration (%)
    sensor_sampling_frequency: sampling frequency of data (Hz)
    detect_rotation: boolean input whether to detect rotation or not
    rotation_detection_axis: axis to detect rotation
    rotation_detection_frequency_cutoff: low-pass filter (Hz) followed based on paper
    rotation_detection_filter_order: filter order on user acceleration
    rotation_detection_aucXt_upper_limit: limit of AUC * time (metrics used in paper) for inferring rotation
    pdkit_gait_features_cutoff_frequency: low-pass filter on user acceleration for pdkit features
    pdkit_gait_features_filter_order: filter order on user acceleration for pdkit features
    pdkit_gait_features_delta: A point is considered a maximum peak if it has the maximal value,
                                and was preceded (to the left) by a value lower by delta (0.5 default).
    pdkit_gait_features_low_freq: lower frequency limit in Hz (2.0 default)
    pdkit_gait_features_upper_freq: upper frequency limit in Hz (2.0 default)
    pdkit_gait_features_step_size: The average step size in centimeters (50.0 default).
    pdkit_gait_features_stride_fraction: <undocumented>
    fog_loco_band: The ratio of the energy in the locomotion band, measured in Hz ([0.5, 3] default)
    fog_freeze_band: The ration of energy in the freeze band, measured in Hz ([3, 8] default)

    Returns
    ----------
    Instantiated class object for gait features
    """

    def __init__(self,
                 sensor_variance_cutoff=1e-4,
                 sensor_window_size=512,
                 sensor_window_overlap=0.2,
                 sensor_sampling_frequency=100,
                 detect_rotation = True,
                 rotation_detection_axis = "y",
                 rotation_detection_frequency_cutoff=2,
                 rotation_detection_filter_order=2,
                 rotation_detection_aucXt_upper_limit=2,
                 pdkit_gait_features_cutoff_frequency=5,
                 pdkit_gait_features_filter_order=4,
                 pdkit_gait_features_delta=0.5,
                 pdkit_gait_features_low_freq = 2,
                 pdkit_gait_features_upper_freq = 10,
                 pdkit_gait_features_step_size = 50,
                 pdkit_gait_features_stride_fraction=0.125,
                 fog_loco_band=[0.5, 3],
                 fog_freeze_band=[3, 8]):

        ### parameters for sensor qc filtering
        self.sensor_sampling_frequency = sensor_sampling_frequency
        self.sensor_variance_cutoff = sensor_variance_cutoff
        self.sensor_window_size = sensor_window_size
        self.sensor_window_overlap = sensor_window_overlap
        self.sensor_sampling_frequency = sensor_sampling_frequency

        ### parameters for rotation detection
        self.detect_rotation = detect_rotation
        self.rotation_detection_axis = rotation_detection_axis
        self.rotation_detection_frequency_cutoff = rotation_detection_frequency_cutoff
        self.rotation_detection_filter_order = rotation_detection_filter_order
        self.rotation_detection_aucXt_upper_limit = rotation_detection_aucXt_upper_limit

        ### parameters for pdkit gait based on gait processor
        self.pdkit_gait_features_cutoff_frequency = pdkit_gait_features_cutoff_frequency
        self.pdkit_gait_features_filter_order = pdkit_gait_features_filter_order
        self.pdkit_gait_features_delta = pdkit_gait_features_delta
        self.pdkit_gait_features_low_freq = pdkit_gait_features_low_freq
        self.pdkit_gait_features_upper_freq = pdkit_gait_features_upper_freq
        self.pdkit_gait_features_step_size = pdkit_gait_features_step_size
        self.pdkit_gait_features_stride_fraction = pdkit_gait_features_stride_fraction

        ### parameters for freeze of gait
        self.fog_loco_band = fog_loco_band
        self.fog_freeze_band = fog_freeze_band

    def format_time_series(self, data):
        """
        Function to format mPower Dataframe after subselecting
        which sensor types it tries to query

        Parameters
        ----------
        data (dtype: pd.dataframe): sensor time-series with at least t,x,y,z info

        Returns
        ----------
        Returns a cleaned and resampled time-series
        """
        try:
            if data.empty:
                return "ERROR: Filepath has Empty Dataframe"
            elif not (set(data.columns) >= set(["x", "y", "z", "t"])):
                return "ERROR: [t, x, y, z] in columns is required"
            else:
                data = data.dropna(subset=["x", "y", "z"])
                data["AA"] = np.sqrt(data["x"]**2 + data["y"]**2 + data["z"]**2)
                data["td"] = data["t"] - data["t"].iloc[0]
                data["t"] = pd.to_datetime(data["td"], unit="s")
                data = data.set_index("t")
                data = data.sort_index()
                data = data[~data.index.duplicated(keep='first')]
                data = self.resample_signal(data[["td", "x", "y", "z", "AA"]])
            return(data)
        except(TypeError, AttributeError) as err:
            return "ERROR: %s" % type(err).__name__

    def resample_signal(self, dataframe):
        """
        Utility method for data resampling,
        Data will be interpolated using linear method

        Parameters
        ----------
        dataframe(dtype: pd.DataFrame): A time-indexed sensor dataframe

        Returns
        ----------
        Returns a pd.Dataframe based on defined sampling frequency
        """
        new_freq = np.round(1 / self.sensor_sampling_frequency, decimals=6)
        df_resampled = dataframe.resample(str(new_freq) + 'S').mean()
        df_resampled = df_resampled.interpolate(method='linear')
        return df_resampled

    def get_freeze_index_features(self, accel_series, sample_rate):
        """
        Modified pdkit FoG freeze index function to be compatible with
        current data gait pipeline

        Parameters
        ----------
        accel_series (dtype: pd.Series): pd.Series of acceleration signal in one axis
        sample_rate  (dtype: float)  : signal sampling rate

        Returns
        ----------
        Returns a list containing (energy freeze index, locomotor freeze index)
        """
        try:
            loco_band = self.fog_loco_band
            freeze_band = self.fog_freeze_band
            window_size = accel_series.shape[0]
            f_res = sample_rate / window_size
            f_nr_LBs = int(loco_band[0] / f_res)
            f_nr_LBe = int(loco_band[1] / f_res)
            f_nr_FBs = int(freeze_band[0] / f_res)
            f_nr_FBe = int(freeze_band[1] / f_res)

            # discrete fast fourier transform
            Y = np.fft.fft(accel_series, int(window_size))
            Pyy = abs(Y*Y) / window_size
            areaLocoBand = numerical_integration(
                Pyy[f_nr_LBs-1: f_nr_LBe],   sample_rate)
            areaFreezeBand = numerical_integration(
                Pyy[f_nr_FBs-1: f_nr_FBe], sample_rate)
            sumLocoFreeze = areaFreezeBand + areaLocoBand
            freezeIndex = areaFreezeBand / areaLocoBand

        except ZeroDivisionError:
            return "Not Enough Samples", "Not Enough Samples"

        else:
            if (freezeIndex == np.inf) or (freezeIndex == -np.inf):
                freezeIndex = np.NaN
            if (sumLocoFreeze == np.inf) or (sumLocoFreeze == -np.inf):
                sumLocoFreeze = np.NaN
            return freezeIndex, sumLocoFreeze

    def get_pdkit_gait_features(self, accel_dataframe):
        """
        Function to featurize dataframe using pdkit package (gait)
        based on all axis orientation (x, y, z, AA).

        Note: Extra error handling in computing step/stride manually
              as pdkit will throw an error when stride is undetected.

        Features from pdkit contains the following:
            1. Number of steps
            2. Cadence
            3. Freeze Index
            4. Energy/Locomotor Freeze Index
            5. Average Step/Stride Duration
            6. Std of Step/Stride Duration
            7. Step/Stride Regularity
            8. Speed of Gait
            9. Symmetry

        Parameters
        -----------
        accel_dataframe (type: pd.DataFrame): user acceleration information containing axis (t (index), td, x, y, z, AA)

        Returns
        -------
        return a listed PDKit gait features as dictionary
        """

        # check if dataframe is valid
        window_start = accel_dataframe.td[0]
        window_end = accel_dataframe.td[-1]
        window_duration = window_end - window_start
        feature_dict = {}
        for axis in ["x", "y", "z", "AA"]:
            gp = pdkit.GaitProcessor(
                duration = window_duration,
                cutoff_frequency=self.pdkit_gait_features_cutoff_frequency,
                filter_order=self.pdkit_gait_features_filter_order,
                delta=self.pdkit_gait_features_delta,
                sampling_frequency=self.sensor_sampling_frequency,
                lower_frequency=self.pdkit_gait_features_low_freq,
                upper_frequency =self.pdkit_gait_features_upper_freq,
                step_size=self.pdkit_gait_features_step_size,
                stride_fraction = self.pdkit_gait_features_stride_fraction)
            series = accel_dataframe[axis]
            var = series.var()
            try:
                if (var) < self.sensor_variance_cutoff:
                    steps = 0
                    cadence = 0
                else:
                    strikes, _ = gp.heel_strikes(series)
                    steps = np.size(strikes)
                    cadence = steps/window_duration
            except (IndexError, ValueError):
                steps = 0
                cadence = 0

            if steps >= 2:
                step_durations = []
                for i in range(1, np.size(strikes)):
                    step_durations.append(strikes[i] - strikes[i-1])
                avg_step_duration = np.mean(step_durations)
                sd_step_duration = np.std(step_durations)
            else:
                avg_step_duration = np.NaN
                sd_step_duration = np.NaN

            if (steps >= 4) and \
                    (avg_step_duration > 1/self.sensor_sampling_frequency):
                strides1 = strikes[0::2]
                strides2 = strikes[1::2]
                stride_durations1 = []
                for i in range(1, np.size(strides1)):
                    stride_durations1.append(strides1[i] - strides1[i-1])
                stride_durations2 = []
                for i in range(1, np.size(strides2)):
                    stride_durations2.append(strides2[i] - strides2[i-1])
                avg_number_of_strides = np.mean(
                    [np.size(strides1), np.size(strides2)])
                avg_stride_duration = np.mean((np.mean(stride_durations1),
                                               np.mean(stride_durations2)))
                sd_stride_duration = np.mean((np.std(stride_durations1),
                                              np.std(stride_durations2)))

                step_regularity, stride_regularity,\
                    symmetry = gp.gait_regularity_symmetry(
                        series,
                        average_step_duration=avg_step_duration,
                        average_stride_duration=avg_stride_duration)
            else:
                avg_number_of_strides = np.NaN
                avg_stride_duration = np.NaN
                sd_stride_duration = np.NaN
                step_regularity = np.NaN
                stride_regularity = np.NaN
                symmetry = np.NaN

            speed_of_gait = gp.speed_of_gait(series, wavelet_level=6)
            energy_freeze_index, \
                locomotor_freeze_index = self.get_freeze_index_features(
                    series, self.sensor_sampling_frequency)
            feature_dict["%s_energy_freeze_index" % axis] = energy_freeze_index
            feature_dict["%s_loco_freeze_index" %
                         axis] = locomotor_freeze_index
            feature_dict["%s_step_duration" % axis] = avg_step_duration
            feature_dict["%s_step_deviation" % axis] = sd_step_duration
            feature_dict["%s_cadence" % axis] = cadence
            feature_dict["%s_stride_duration" % axis] = avg_stride_duration
            feature_dict["%s_stride_deviation" % axis] = sd_stride_duration
            feature_dict["%s_speed_of_gait" % axis] = speed_of_gait
            feature_dict["%s_step_regularity" % axis] = step_regularity
            feature_dict["%s_stride_regularity" % axis] = stride_regularity
            feature_dict["%s_symmetry" % axis] = symmetry
        feature_dict["window_size"] = window_duration
        feature_dict["window_start"] = window_start
        feature_dict["window_end"] = window_end
        return feature_dict

    def detect_rotation_timepoint(self, gyro_dataframe, axis):
        """
        Function to output rotation information (omega, period of rotation)
        in a dictionary format, given a gyroscope dataframe
        and its axis orientation

        Parameters
        -----------
        gyro_dataframe (type: pd.DataFrame): A gyrosocope dataframe
        axis (type: str, default = "y"): Rotation-rate axis

        Returns
        --------
            Returns a dictionary of omega and period of rotaton
            Return Format: {"chunk_1": {omega: value, period: value},
                            "chunk_2": {omega: value, period: value}}
        """
        gyro_dataframe[axis] = butter_lowpass_filter(
            data=gyro_dataframe[axis],
            sample_rate=self.sensor_sampling_frequency,
            cutoff=self.rotation_detection_frequency_cutoff,
            order=self.rotation_detection_filter_order)
        zero_crossings = np.where(np.diff(np.sign(gyro_dataframe[axis])))[0]
        start = 0
        num_rotation_window = 0
        rotation_dict = {}
        for crossing in zero_crossings:
            if (gyro_dataframe.shape[0] >= 2) & (start != crossing):
                duration = gyro_dataframe["td"][crossing] - \
                    gyro_dataframe["td"][start]
                auc = np.abs(metrics.auc(
                    gyro_dataframe["td"][start: crossing + 1],
                    gyro_dataframe[axis][start: crossing + 1]))
                omega = auc/duration
                aucXt = auc * duration

                if aucXt > self.rotation_detection_aucXt_upper_limit:
                    num_rotation_window += 1
                    rotation_dict["rotation_chunk_%s" % num_rotation_window] =\
                        ({"omega": omega,
                          "aucXt": aucXt,
                          "duration": duration,
                          "period": [start, crossing]})
                start = crossing
        return rotation_dict

    def segment_gait_period(self, accel_dataframe, periods):
        """
        Function to chunk dataframe into several periods of rotational
        and non-rotational sequences into List.
        Each dataframe chunk in the list will be in dictionary format
        containing information of the dataframe itself
        and what type of chunk it categorizes as.

        Parameters
        -----------
        accel_dataframe (type: pd.Dataframe): User acceleration dataframe
        periods         (type: List): A list of list of periods
        periods input format:
            [[start_ix, end_ix], [start_ix, end_ix]]

        Parameters
        -----------
        Returns a List of dictionary that has keys containing information
        of what type of chunk, and the dataframe itself.
        Return Format: [{chunk1 : walk_chunk_1, dataframe : data},
                        {chunk2 : rotation_chunk_1, dataframe : data}]
        """
        # instantiate initial values
        chunk_list = []
        pointer = 0
        num_rotation_window = 1
        num_walk_window = 1

        # check if there is rotation
        if len(periods) == 0:
            chunk_list.append(
                {"dataframe": accel_dataframe,
                 "chunk": "walk_chunk_%s" % num_walk_window})
            return chunk_list

        # edge case: if rotation occurs at zero
        if periods[0][0] == pointer:
            chunk_list.append(
                {"dataframe": accel_dataframe.iloc[pointer: periods[0][1] + 1],
                 "chunk": "rotation_chunk_%s" % num_rotation_window})
            num_rotation_window += 1
            pointer = periods[0][1] + 1
            periods = periods[1:]

        # middle case
        for index in periods:
            chunk_list.append(
                {"dataframe": accel_dataframe.iloc[pointer: index[0] + 1],
                 "chunk": "walk_chunk_%s" % num_walk_window})
            num_walk_window += 1
            chunk_list.append(
                {"dataframe": accel_dataframe.iloc[index[0] + 1: index[1] + 1],
                 "chunk": "rotation_chunk_%s" % num_rotation_window})
            num_rotation_window += 1
            pointer = index[1] + 1

        # edge case, last bit of data
        if pointer < accel_dataframe.shape[0]:
            chunk_list.append(
                {"dataframe": accel_dataframe.iloc[pointer:],
                 "chunk": "walk_chunk_%s" % num_walk_window})
        return chunk_list

    def featurize_gait_segment(self,list_of_dataframe_dicts, rotation_dict):
        """
        Function to featurize list of rotation-segmented dataframe chunks with
        moving windows, and returns list of dictionary with all the pdkit
        and rotation features from each moving window

        Notes:
          1. if dataframe chunk is smaller than the window size or is a
          rotation dataframe chunk it will just be treated as one window
          2. if current dataframe size is bigger than window size,
          then iterate through dataframe chunk

        Parameters
        -----------
        list_of_dataframe_dicts (dtype = List): A List filled with
                                                dictionaries containing
                                                key-value pair of the
                                                dataframe and
                                                its chunk type
        rotation_dict (dtype = Dict): A Dictionary containing
                                      rotation information
                                      on each rotation_chunk.
        Returns
        -----------
            RType = List
            Returns a list of features from each window,
            each window features is stored as dictionary
            inside the list

            Return format:
                [{window1_features:...},
                {window2_features:...},
                {window3_features:...}]
        """

        feature_list = []
        num_window = 1
        step_size = int(self.sensor_window_size * self.sensor_window_overlap)
        # separate to rotation and non rotation
        for dataframe_dict in list_of_dataframe_dicts:
            end_window_pointer = self.sensor_window_size
            curr_chunk = dataframe_dict["chunk"]
            curr_dataframe = dataframe_dict["dataframe"]

            if curr_dataframe.shape[0] < 0.5 * self.sensor_sampling_frequency:
                continue

            # define if chunk is a rotation sequence
            if "rotation_chunk" in curr_chunk:
                rotation_omega = rotation_dict[curr_chunk]["omega"]
            else:
                rotation_omega = np.NaN

            # edge case where dataframe is smaller than window, or rotation
            if ((curr_dataframe.shape[0] < end_window_pointer)
                    or (rotation_omega > 0)):
                feature_dict = self.get_pdkit_gait_features(curr_dataframe)
                feature_dict["rotation_omega"] = rotation_omega
                feature_dict["window"] = "window_%s" % num_window
                feature_list.append(feature_dict)
                num_window += 1
                continue

            # ideal case when data chunk is larger than window ##
            while end_window_pointer < curr_dataframe.shape[0]:
                start_window_pointer = end_window_pointer - self.sensor_window_size
                subset = curr_dataframe.iloc[start_window_pointer:
                                             end_window_pointer]
                feature_dict = self.get_pdkit_gait_features(subset)
                feature_dict["rotation_omega"] = rotation_omega
                feature_dict["window"] = "window_%s" % num_window
                feature_list.append(feature_dict)
                end_window_pointer += step_size
                num_window += 1
        return feature_list

    def run_pipeline(self, accel_data, rotation_data):
        """
        main entry point of this featurizaton class, parameter will take in
        pd.DataFrame or the filepath to the dataframe.

        Parameters
        ----------
            accel_data (type: pd.DataFrame): accelerometer sensor time series
            rotation_data (type: pd.DataFrame): rotation rate sensor time series

        Returns
        -------
            A list of dictionary. With each dictionary representing
            gait features on each window.
        """
        error = []
        rotation_dict = {}
        result = pd.DataFrame()
        try:
            resampled_accel = self.format_time_series(accel_data)
            resampled_rotation = self.format_time_series(rotation_data)
            if isinstance(resampled_accel, str):
                error.append(resampled_accel)
            if isinstance(resampled_rotation, str):
                error.append(resampled_rotation)
            if len(error) != 0:
                result["error"] = ["; ".join(error)]
                return result
            if self.detect_rotation == True:
                rotation_dict = self.detect_rotation_timepoint(resampled_rotation,
                                                               self.rotation_detection_axis)
            periods = [v["period"] for k, v in rotation_dict.items()]
            list_df_chunks = self.segment_gait_period(resampled_accel, periods)
            list_of_feature_dict = self.featurize_gait_segment(list_df_chunks, rotation_dict)
            result = pd.DataFrame(list_of_feature_dict)
            result["error"] = np.nan
            return result
        except Exception as err:
            result["error"] = [err]
            return result

max_stars_repo_name: arytontediarjo/PDKitRotationFeatures

max_issues_repo_name: Sage-Bionetworks/PDKitRotationFeatures

max_forks_repo_name: Sage-Bionetworks/PDKitRotationFeatures


text:
import torch
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
import numpy
class Denormalization:
    def __init__(self):
        self.mean_3 = torch.tensor([-1.0, -1.0, -1.0]).to(device)
        self.std_3 = torch.tensor([1 / 0.5, 1 / 0.5, 1 / 0.5]).to(device)
        self.mean_1 = torch.tensor([-1.0]).to(device)
        self.std_1 = torch.tensor([1 / 0.5]).to(device)

        self.mean_norm = torch.tensor([0.485,0.456,0.406]).to(device)
        self.std_norm = torch.tensor([0.229,0.224,0.225]).to(device)
    def __call__(self, inp_tensor):
        #print(inp_tensor)
        if inp_tensor.shape[1] == 3:
            #inp_tensor.sub_(self.mean_3[None, :, None, None]).div_(self.std_3[None, :, None, None])
            inp_tensor_denorm = (inp_tensor - self.mean_3[None, :, None, None])/(self.std_3[None, :, None, None])
        elif inp_tensor.shape[1] == 1:
            #inp_tensor.sub_(self.mean_1[None, :, None, None]).div_(self.std_1[None, :, None, None])
            inp_tensor_denorm = (inp_tensor - self.mean_1[None, :, None, None]) / (self.std_1[None, :, None, None])
        #print(inp_tensor)
        #inp_tensor.sub_(self.mean_norm[None, :, None, None]).div_(self.std_norm[None, :, None, None])
        inp_tensor_renorm = (inp_tensor_denorm - self.mean_norm[None, :, None, None])/(self.std_norm[None, :, None, None])
        return inp_tensor_renorm

max_stars_repo_name: ShuvozitGhose/UDBNET

max_issues_repo_name: ShuvozitGhose/UDBNET

max_forks_repo_name: ShuvozitGhose/UDBNET


text:
import kplr
import numpy as np
# import matplotlib.pyplot as plt

from functools import partial
from lmfit     import Parameters, Minimizer, report_errors, minimize

from matplotlib import rcParams
from time      import time

from sklearn.externals import joblib
from scipy import special
from pandas import DataFrame
from exoparams import PlanetParams
import batman

from bokeh.io       import output_notebook, show
from bokeh.plotting import figure
from bokeh.models   import Span
from bokeh.layouts  import gridplot

# output_notebook()

def batman_wrapper_lmfit(period, tCenter, inc, aprs, rprs, edepth, ecc, omega, u1, u2,
                         intcpt, slope, crvtur, times, ldtype='quadratic', transitType='primary'):

    if intcpt == 1.0 and slope == 0.0 and crvtur == 0.0:
        OoT_crvtur = 1.0 # OoT == Out of Transit
    else:
        OoT_crvtur = intcpt + slope*(times-times.mean()) + crvtur*(times-times.mean())**2

    bm_params           = batman.TransitParams() # object to store transit parameters

    bm_params.per       = period   # orbital period
    bm_params.t0        = tCenter  # time of inferior conjunction
    bm_params.inc       = inc      # inclunaition in degrees
    bm_params.a         = aprs     # semi-major axis (in units of stellar radii)
    bm_params.rp        = rprs     # planet radius (in units of stellar radii)
    bm_params.fp        = edepth   # planet radius (in units of stellar radii)
    bm_params.ecc       = ecc      # eccentricity
    bm_params.w         = omega    # longitude of periastron (in degrees)
    bm_params.limb_dark = ldtype   # limb darkening model # NEED TO FIX THIS
    bm_params.u         = [u1, u2] # limb darkening coefficients # NEED TO FIX THIS

    m_eclipse = batman.TransitModel(bm_params, times, transittype=transitType)# initializes model

    return m_eclipse.light_curve(bm_params)*OoT_crvtur

def transit_line_model(model_params, times):
    intcpt  = model_params['intcpt'].value if 'intcpt' in model_params.keys() else 1.0
    slope   = model_params['slope'].value  if 'slope'  in model_params.keys() else 0.0
    crvtur  = model_params['crvtur'].value if 'crvtur' in model_params.keys() else 0.0

    # Transit Parameters
    period  = model_params['period'].value
    tCenter = model_params['tCenter'].value
    inc     = model_params['inc'].value
    aprs    = model_params['aprs'].value
    edepth  = model_params['edepth'].value
    tdepth  = model_params['tdepth'].value
    ecc     = model_params['ecc'].value
    omega   = model_params['omega'].value
    u1      = model_params['u1'].value
    u2      = model_params['u2'].value

    # delta_phase = deltaphase_eclipse(ecc, omega) if ecc is not 0.0 else 0.5
    # t_secondary = tCenter + period*delta_phase

    rprs  = np.sqrt(tdepth)

    transit_model = batman_wrapper_lmfit(period, tCenter, inc, aprs, rprs, edepth, ecc, omega, u1, u2,
                         intcpt, slope, crvtur, times, ldtype='quadratic', transitType='primary')

    line_model    = intcpt + slope*(times-times.mean()) + crvtur*(times-times.mean())**2.

    return transit_model * line_model

def bokeh_errorbars(xs, ys, yerrs, xerrs=None, color='#1f77b4', size=5, alpha=1, fig=None, show_now = False):

    if xerrs is None:
        xerrs = np.zeros(xs.size)

    if fig is None:
        fig = figure()

    fig.circle(xs, ys, color=color, size=size, alpha=alpha)

    # create the coordinates for the errorbars
    err_xs = []
    err_ys = []

    for x, y, yerr, xerr in zip(xs, ys, yerrs, xerrs):
        err_xs.append((x - xerr, x + xerr))
        err_ys.append((y - yerr, y + yerr))

    # plot them
    fig.multi_line(err_xs, err_ys, color=color, alpha=alpha)

    if show_now: show(fig)

    return fig

def bokeh_hist(data, bins=100, range=None, color='#1f77b4', density=True, alpha=1.0, fig = None, show_now = False):
    if fig is None:
        fig = figure()

    data_sorted = np.sort(data)
    hist, edges = np.histogram(data_sorted, density=density, bins=bins, range=range)
    fig.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:],
             fill_color=color, line_color=color, alpha=alpha)

    if show_now: show(fig)

    return fig

def bokeh_corner_plot(dataset, TOOLS=None, hist_color='orange', kde_color="violet"):
    if isinstance(dataset, np.ndarray):
        dataset = DataFrame(dataset)

    if TOOLS is None:
        TOOLS = "box_select,lasso_select,pan,wheel_zoom,box_zoom,reset,help"

    scatter_plots = []
    y_max = len(dataset.columns) - 1
    for i, y_col in enumerate(dataset):
        for j, x_col in enumerate(dataset):
            df = DataFrame({x_col: dataset[x_col].tolist(), y_col: dataset[y_col].tolist()})
            fig = figure(tools=TOOLS, toolbar_location="below", toolbar_sticky=False)
            if i >= j:
                if i != j:
                    fig.scatter(x=x_col, y=y_col, source=df)
                else:
                    x_now       = np.sort(dataset[x_col].values)
                    mu  , sigma = np.mean(x_now), np.std(x_now)
                    hist, edges = np.histogram(x_now, density=True, bins=len(x_now)//100)
                    pdf         = 1/(sigma * np.sqrt(2*np.pi)) * np.exp(-0.5*(x_now-mu)**2 / sigma**2)
                    cdf         = 0.5*(1+special.erf((x_now-mu)/np.sqrt(2*sigma**2)))

                    fig.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:],fill_color=hist_color, line_color=hist_color, alpha=1.0)
                    fig.line(x_now, pdf, line_color=kde_color, line_width=8, alpha=0.7)#, legend="PDF")
                    #fig.line(x_now, cdf, line_color="black"  , line_width=2, alpha=0.5, legend="CDF")
                if j > 0:
                    fig.yaxis.axis_label = ""
                    fig.yaxis.visible = False
                if i < y_max:
                    fig.xaxis.axis_label = ""
                    fig.xaxis.visible = False
            else:
                fig.outline_line_color = None

            scatter_plots.append(fig)

    grid = gridplot(scatter_plots, ncols = len(dataset.columns))
    show(grid)


text:
# fovea interactive dynamics diagnostics, (c) 2015
from __future__ import division, absolute_import
from fovea import graphics
from fovea import common
from fovea import domain2D
from fovea.diagnostics import *
from fovea.calc_context import calc_context, general_context, make_measure, prep
from matplotlib import pyplot as plt
import numpy as npy

max_stars_repo_name: robclewley/fovea

max_issues_repo_name: robclewley/fovea

max_forks_repo_name: robclewley/fovea


text:
#!/usr/bin/python

"""
Import packages/libraries
"""
import matplotlib.pyplot as plt
import pandas as pd
import statsmodels.formula.api as smf
from my_config import Config as cfg

def main():
    main_title = 'Advertisement Sales Trends and Predictions'

    # read advertisement data sample into DataFrame
    ad = pd.read_csv(
        '{pp}/{pd}/{fn}'.format(
            pp=cfg.PATH_PARENT
            , pd=cfg.PATH_DATA
            , fn=cfg.DATA_ADV
        )
        , index_col=0
    )

    # show the first 5 rows of the imported data
    #print(ad.head())
    # print the shape of the DataFrame
    #print(ad.shape)

    fig, axs = plt.subplots(nrows=1, ncols=3, sharey=True)

    # adding the scatterplots to the grid
    ad.plot(kind='scatter', x='TV', y='sales', ax=axs[0], figsize=(16, 8))
    ad.plot(kind='scatter', x='Radio', y='sales', ax=axs[1])
    ad.plot(kind='scatter', x='Newspaper', y='sales', ax=axs[2])

    #plt.show()

    """
    Simple linear regression model
    >> $$y = \beta_0 + \beta_1x$$

    * $$y$$ is the predicted numerical value (response) -> *sales*
    * x is the feature
    * $$\beta_0$$ is called the *intercept*, which is the value of $$y$$ when $$x = 0$$
    * $$\beta_1$$ is the coefficient of the feature $$x$$ also called the *slope*, which represent the change in $$y$$ divided by the change in $$x$$
        $$\beta_1 = \frac{\delta y}{\delta x}$$

    create a fitted model that represent the least squares line

    TV's Output:
    TV sales's coefficient:
    Intercept    7.032594
    TV           0.047537
    dtype: float64
    """
    tv_lm = smf.ols(formula='sales~TV', data=ad).fit()
    radio_lm = smf.ols(formula='sales~Radio', data=ad).fit()
    newspaper_lm = smf.ols(formula='sales~Newspaper', data=ad).fit()
    # show the trained model coefficients
    #print("TV sales's coefficient:\n{pa}".format(pa=tv_lm.params))
    #print("Radio sales's coefficient:\n{pa}".format(pa=radio_lm.params))
    #print("Newspaper sales's coefficient:\n{pa}".format(pa=newspaper_lm.params))

    """
    A unit increase in the feature (TV ad) spending is associated with 0.047537 unit increase in Sales (response).
    In other words, an additional $100 spent on TV ads is associated with an increase in sales of 4.7537

    TV's Output:
    7.032593549127694 | 0.047536640433019764
    """
    b0 = tv_lm.params[0]
    b1 = tv_lm.params[1]
    #print("{b0} | {b1}".format(b0=b0, b1=b1))

    """
    Make a manual prediction of how much sales will increase from $50,000 of TV advertising

    Output:
    2383.864615200116
    """
    sales = b0 + (b1 * 50000)
    #print(sales)

    """
    creating a new Pandas DataFrame to match Statsmodels interface expectations

    TV's Output:
          TV
    0  50000
    """
    new_tv_ad_spending = pd.DataFrame({'TV': [50000]})
    #print(new_tv_ad_spending.head())

    """
    use the model to make predictions on a new value

    TV's Output:
    0    2383.864615
    """
    preds = tv_lm.predict(new_tv_ad_spending)
    #print(preds)

    """
    least square line. In order to draw the line, need two points: minimum and maximum values from feature

    TV's Output:
          TV
    0    0.7
    1  296.4
    """
    tv_min_max = pd.DataFrame({'TV': [ad['TV'].min(), ad['TV'].max()]})
    radio_min_max = pd.DataFrame({'Radio': [ad['Radio'].min(), ad['Radio'].max()]})
    newspaper_min_max = pd.DataFrame({'Newspaper': [ad['Newspaper'].min(), ad['Newspaper'].max()]})
    #print(tv_min_max)
    #print(radio_min_max)
    #print(newspaper_min_max)

    """
    predictions for x min and max values

    TV's Output:
    0     7.065869
    1    21.122454
    dtype: float64
    """
    tv_predictions = tv_lm.predict(tv_min_max)
    radio_predictions = radio_lm.predict(radio_min_max)
    newspaper_predictions = newspaper_lm.predict(newspaper_min_max)
    #print(tv_predictions)
    #print(radio_predictions)
    #print(newspaper_predictions)

    # plotting the least squares line
    #plt.plot(tv_min_max, tv_predictions, color='red', linewidth=2)
    axs[0].plot(tv_min_max, tv_predictions, color='red', linestyle='-', linewidth=2)
    axs[0].grid(True)

    axs[1].plot(radio_min_max, radio_predictions, color='red', linestyle='-', linewidth=2)
    axs[1].grid(True)

    axs[2].plot(newspaper_min_max, newspaper_predictions, color='red', linestyle='-', linewidth=2)
    axs[2].grid(True)

    fig.tight_layout()
    fig.suptitle(main_title, fontsize=18, fontweight='bold')
    #fig.legend(loc='upper right')

    fig_mgr = plt.get_current_fig_manager()
    fig_mgr.window.showMaximized()

    plt.subplots_adjust(top=0.9)
    plt.show()

if __name__ == '__main__':
    main()

reason: import statsmodels


text:
def kernel_12_first_diff(LEN_1D):
    #! init
    import numpy as np

    x = np.zeros(LEN_1D)
    y = np.random.randn(LEN_1D + 1)

    #! loop
    for k in range(LEN_1D):
        x[k] = y[k+1] - y[k]

    #! array_op
    x = y[1:(LEN_1D + 1)] - y[0:LEN_1D]

max_stars_repo_name: quepas/polyglot-scientific-kernels

max_issues_repo_name: quepas/polyglot-scientific-kernels

max_forks_repo_name: quepas/polyglot-scientific-kernels


text:
import numpy
import pytest

from vlattice.lattice import Atom
from vlattice.lattice import Locator
from vlattice.lattice import Site

@pytest.fixture(scope='module')
def cubic_locator(request):
    return Locator(numpy.eye(3), crystal_coords=False)

@pytest.fixture(scope='module')
def parallel_locator(request):
    return Locator(numpy.diag([1, 2, 3]), crystal_coords=True)

def test_basic_locator(cubic_locator):
    atom = Atom((0, 0, 0), 'A', 0)
    site = Site((0, 0, 0), atom)
    assert ([0, 0, 0] == cubic_locator.locate(site)).all()

def test_parallel_locator(parallel_locator):
    atom = Atom((0, 0, 0), 'A', 0)
    assert ([0, 0, 0] == parallel_locator.locate(Site((0, 0, 0), atom))).all()
    assert ([1, 0, 0] == parallel_locator.locate(Site((1, 0, 0), atom))).all()
    assert ([0, 2, 0] == parallel_locator.locate(Site((0, 1, 0), atom))).all()
    assert ([0, 0, 3] == parallel_locator.locate(Site((0, 0, 1), atom))).all()
    assert ([1, 2, 3] == parallel_locator.locate(Site((1, 1, 1), atom))).all()

def test_crystal_coords(parallel_locator):
    pl = parallel_locator
    atom = Atom((0.5, 0.5, 0.5), 'A', 0)
    assert ([0.5, 1.0, 1.5] == pl.locate(Site((0, 0, 0), atom))).all()
    assert ([1.5, 1.0, 1.5] == pl.locate(Site((1, 0, 0), atom))).all()
    assert ([0.5, 3.0, 1.5] == pl.locate(Site((0, 1, 0), atom))).all()
    assert ([0.5, 1.0, 4.5] == pl.locate(Site((0, 0, 1), atom))).all()
    assert ([1.5, 3.0, 4.5] == pl.locate(Site((1, 1, 1), atom))).all()

max_stars_repo_name: odarbelaeze/vegas-lattice

max_issues_repo_name: odarbelaeze/vegas-lattice

max_forks_repo_name: odarbelaeze/vegas-lattice


text:
import numpy as np

from misc import memList,mergeList,taxoLCA,inf
from taxoTree import TaxoTree
from writeOnFiles import writeMatrix

#@dataArray = [samplesInfoList,infoList,idSequences,filenames,matchingNodes,paths,nodesListTree,taxoTree]

#distance(@sample1,@sample2) = |number of nodes matched in @sample1| + |number of nodes matched in @sample2| - 2*|number of nodes matched in both samples|
#@q is useless here
def distMatched(sample1,sample2,dataArray,q):
    #@dataArray[4] = matchingNodes, dictionary of (key=sample,value=list of nodes matched in this sample)
    nodes1 = dataArray[4].get(sample1)
    nodes2 = dataArray[4].get(sample2)
    _,number,numberl1,numberl2 = memList(nodes1,nodes2)
    return numberl1 + numberl2 - 2*number

#distance(@sample1,@sample2) = |L1| + |L2| - q*(|N1interM2| + |N2interM1|) - |M1interM2| (see README for notations)
def distConsensus(sample1,sample2,dataArray,q):
    #@dataArray[4] = matchingNodes, dictionary of (key=sample,value=list of nodes matched in this sample)
    nodes1 = dataArray[4].get(sample1)
    nodes2 = dataArray[4].get(sample2)
    nodeLCA1 = taxoLCA(dataArray[7].paths,nodes1)
    nodeLCA2 = taxoLCA(dataArray[7].paths,nodes2)
    print nodeLCA1,len(nodes1)
    print nodeLCA2,len(nodes2)
    #@dataArray[7] = taxoTree
    #Looking for the subtree rooted at the LCA of the matched nodes
    #that is the "induced tree"
    if not nodeLCA1 or not nodeLCA2:
        return inf
    try:
        tree1 = dataArray[7].search(nodeLCA1[0],nodeLCA1[1])
        tree2 = dataArray[7].search(nodeLCA2[0],nodeLCA2[1])
    except ValueError:
        return inf
    leaves1,leavesNumber1 = tree1.leaves(False)
    leaves2,leavesNumber2 = tree2.leaves(False)
    leaves = mergeList(leaves1,leaves2)
    matchedBy1,_,_,_ = memList(leaves,nodes1)
    matchedBy2,_,_,_ = memList(leaves,nodes2)
    #M1interM2, N1interM2, N2interM1
    _,numberCommon,number1,number2 = memList(matchedBy1,matchedBy2)
    return leavesNumber1 + leavesNumber2 - q*(number1 + number2) - numberCommon

#__________________________________________________________________________

#@dataArray = [samplesInfoList,infoList,idSequences,filenames,matchingNodes,paths,nodesListTree,taxoTree]

def computeDistanceMatrix(dist,dataArray):
    q = 0.5
    if dist == distConsensus:
        q = float(raw_input("Choose the value of q to compute [q should be between 0 and 1].\n"))
        if q > 1 or q < 0:
            print "\n/!\ ERROR: Wrong value of q [ should be between 0 and 1 ]:",q,"."
            raise ValueError
    n = len(dataArray[3])
    matrix = np.zeros((n,n))
    #distance is symmetric
    for i in range(n):
        for j in range(i+1,n):
            #dataArray[3] = filenames
            print dataArray[3][i],dataArray[3][j]
            distance = dist(dataArray[3][i],dataArray[3][j],dataArray,q)
            matrix[i][j] = distance
            matrix[j][i] = distance
    if dist == distMatched:
        version = "1"
    else:
        version = "2" + str(q)
    writeMatrix("matrix" + version,matrix,"Distance matrix")

max_stars_repo_name: kuredatan/taxocluster

max_issues_repo_name: kuredatan/taxocluster

max_forks_repo_name: kuredatan/taxocluster


text:
import datetime
import os, sys
import numpy as np
import matplotlib.pyplot as plt

import casadi as cas
import pickle
import copy as cp

PROJECT_PATH = '/home/nbuckman/Dropbox (MIT)/DRL/2020_01_cooperative_mpc/mpc-multiple-vehicles/'
sys.path.append(PROJECT_PATH)

import src.MPC_Casadi as mpc
import src.TrafficWorld as tw
import src.IterativeBestResponseMPCMultiple as mibr
import src.car_plotting_multiple as cmplot

np.set_printoptions(precision=2)

subdir_name = "20200226_201330shorter_window_running"
folder = "results/" + subdir_name + "/"
T = 5
# N = int(T/dt) #Number of control intervals

n_other = 4

ibr_sub_it=1

n_total_round = 6

# all_other_x0 = []
for time_step in range(5):
    for n_round in range(n_total_round):
        ibr_sub_it +=1

        for i in range(n_other):
            ibr_sub_it+=1

            suffixes = ["amb_costs_list", "car1_costs_list", "other_svo_cost", "svo_cost", "total_svo_cost", "u0", "u1", "u2", "u3",
                        "x0", "x1", "x2", "x3", "uamb", "xamb", "x_des0", "x_des1", "x_des2", "x_des3", "xamb_des" ]

            for s in suffixes:
                file_name = folder + "data/'%d %03d"%(time_step, ibr_sub_it) + s + ".npy'"
                new_file_name = folder + "data/%d_%03d"%(time_step, ibr_sub_it) + s + ".npy"

                cmd = "mv %s %s"%(file_name, new_file_name)
                print(cmd)
                os.system(cmd)
            # raise Exception

        # if WARM:
        #     br1.opti.set_initial(br1.u_opt, u1)
        #     br1.opti.set_initial(br1.x_opt, x1)
        #     br1.opti.set_initial(br1.x_desired, x1_des)

max_stars_repo_name: noambuckman/mpc-multiple-vehicles

max_issues_repo_name: noambuckman/mpc-multiple-vehicles

max_forks_repo_name: noambuckman/mpc-multiple-vehicles


text:
# SPDX-License-Identifier: MIT
# Copyright (c) 2021 ETH Zurich, Luc Grosheintz-Laval

import numpy as np

from morinth.quadrature import GaussLegendre
from morinth.math_tools import gaussian, l1_error, l1rel_error, linf_error, convergence_rate

class GaussianBumpIC(object):
    def __init__(self, model, equilibrium):
        self.model = model
        self.equilibrium = equilibrium

        self.p_amplitude = 1.0e-5
        self.rho_amplitude = 1.0e-5
        self.sigma = 0.05

        self.x_ref = 0.0
        self.rho_ref = 2.0
        self.p_ref = 1.0

        self.x_mid = 0.5
        self.p_mid = 0.3

        self.quadrature = GaussLegendre(5)

    @property
    def T_ref(self):
        return self.model.temperature(p=self.p_ref, rho=self.rho_ref)

    @property
    def u_ref(self):
        E_int_ref = self.model.internal_energy(p=self.p_ref)
        return np.array([self.rho_ref, 0.0, 0.0, E_int_ref])

    def __call__(self, grid):
        du_lambda = lambda x : self.delta(x, as_conserved_variables=True)
        u0 = self.back_ground(grid)
        du0 = self.quadrature(grid.edges, du_lambda)

        assert np.all(np.isfinite(u0))
        assert np.all(np.isfinite(du0))

        return u0 + du0

    def delta(self, x, as_conserved_variables=False):
        """Perturbation in respective variables."""

        dp = self.p_mid*gaussian(x - self.x_mid, self.sigma)
        drho = self.model.rho(p=dp, T=self.T_ref)

        dw = np.zeros((4,) + dp.shape)
        dw[0,...] = self.rho_amplitude * drho
        dw[3,...] = self.p_amplitude * dp

        if as_conserved_variables:
            dw[3,...] = self.model.internal_energy(p=dw[3,...])

        return dw

    def back_ground(self, grid):
        x = grid.cell_centers[:,0]
        equilibrium = self.equilibrium
        u_bar_ref = equilibrium.cell_averages(self.u_ref, self.x_ref)

        return equilibrium.reconstruct(u_bar_ref, self.x_ref, x)

    def point_values(self, x):
        """Point values of the primitive variables."""

        equilibrium = self.equilibrium
        rho_ref, p_ref, x_ref = self.rho_ref, self.p_ref, self.x_ref

        w = np.zeros((4,) + x.shape)
        w[0,...], w[3,...] = equilibrium.extrapolate(rho_ref, p_ref, x_ref, x)
        dw = self.delta(x)

        return w + dw

class ExtremeGaussianBumpIC(GaussianBumpIC):
    def __init__(self, model, equilibrium):
        self.model = model
        self.equilibrium = equilibrium

        self.p_amplitude = 1.0e-6
        self.rho_amplitude = 1.0e-6

        self.sigma = 0.05*(self.domain[1] - self.domain[0])

        self.x_ref = 0.0
        self.rho_ref = 32.0
        self.p_ref = 10000.0

        self.x_mid = 0.5*(self.domain[0] + self.domain[1])
        self.p_mid = 1000.0

        self.quadrature = GaussLegendre(5)

    @property
    def domain(self):
        return np.array([0.0, 500.0])


text:
# -*- coding: utf-8 -*-
""" RandOm Convolutional KErnel Transform (ROCKET)
"""

__author__ = ["RavenRudi"]
__all__ = ["ROCKETRegressor"]

import numpy as np
from sklearn.linear_model import RidgeCV

from sktime.utils.validation.panel import check_X, check_X_y
from sktime.regression.base import BaseRegressor
from sktime.series_as_features.base.estimators.shapelet_based._rocket_estimator import (
    BaseROCKETEstimator,
)

class ROCKETRegressor(BaseROCKETEstimator, BaseRegressor):
    """
    Regressor wrapped for the ROCKET transformer using RidgeCV as the
    base regressor.

    Parameters
    ----------
    num_kernels             : int, number of kernels for ROCKET transform
    (default=10,000)
    n_jobs                  : int, optional (default=1)
    The number of jobs to run in parallel for both `fit` and `predict`.
    ``-1`` means using all processors.
    random_state            : int or None, seed for random, integer,
    optional (default to no seed)

    Attributes
    ----------
    classifier              : ROCKET classifier
    n_classes               : extracted from the data

    Notes
    -----
    @article{dempster_etal_2019,
      author  = {Dempster, Angus and Petitjean, Francois and Webb,
      Geoffrey I},
      title   = {ROCKET: Exceptionally fast and accurate time series
      classification using random convolutional kernels},
      year    = {2019},
      journal = {arXiv:1910.13051}
    }

    Java version
    https://github.com/uea-machine-learning/tsml/blob/master/src/main/java/
    tsml/classifiers/shapelet_based/ROCKETClassifier.java

    """

    @property
    def base_estimator(self):
        return RidgeCV(alphas=np.logspace(-3, 3, 10), normalize=True)

def fit(self, X, y):
    """Fit regressor to training data.

    Parameters
    ----------
    X : pd.DataFrame, optional (default=None)
        Exogeneous data
    y : pd.Series, pd.DataFrame, or np.array
        Target time series to which to fit the regressor.

    Returns
    -------
    self :
        Reference to self.
    """
    coerce_to_numpy = self.get_tag("coerce-X-to-numpy", False)

    X, y = check_X_y(X, y, coerce_to_numpy=coerce_to_numpy)

    self._fit(X, y)

    # this should happen last
    self._is_fitted = True

def predict(self, X):
    """Predict time series.

    Parameters
    ----------
    X : pd.DataFrame, shape=[n_obs, n_vars]
        A2-d dataframe of exogenous variables.

    Returns
    -------
    y_pred : pd.Series
        Regression predictions.
    """
    coerce_to_numpy = self.get_tag("coerce-X-to-numpy", False)

    X = check_X(X, coerce_to_numpy=coerce_to_numpy)
    self.check_is_fitted()

    y = self._predict(X)

    return y

max_stars_repo_name: RavenRudi/sktime

max_issues_repo_name: RavenRudi/sktime

max_forks_repo_name: RavenRudi/sktime