text:
"""
Main file
We will run the whole program from here
"""

import torch
from train import train
from ray import tune
import numpy as np
from ray.tune import CLIReporter
from ray.tune.schedulers import ASHAScheduler
import json
from train_separately import train as train_separately

torch.backends.cudnn.benchmark = True

def main() -> None:
	# Setting hyperparameters for training
	config = {}
	config["batch_size"] = 16
	config["dropout"] = 0.098203#tune.sample_from(lambda _: np.random.uniform(0, 0.6))
	config["lr_value"] = 0.00034058#tune.loguniform(1e-5, 1e-2)
	config["hidden_bb_dim"] = 298#tune.sample_from(lambda _: np.random.randint(200, 500))
	config["hidden_label_dim"] = 259#tune.sample_from(lambda _: np.random.randint(200, 500))
	config["bb_loss_weight"] = 28#tune.sample_from(lambda _: np.random.randint(10, 100))
	config["step_size"] = 30
	config["train_path"] ='/home/student/HW2/train'

	# tuning model using ray tune, updating tensorboard
	scheduler = ASHAScheduler(
		metric="score",
		mode="max",
		max_t=50,
		grace_period=2,
		reduction_factor=2)
	reporter = CLIReporter(
		metric_columns=["loss", "bb_loss", "bce_loss", "iou_score", "accuracy_score", "training_iteration"])
	# train(config)
	result = tune.run(
		train_separately,
		resources_per_trial={"cpu": 2, "gpu": 1},
		config=config,
		num_samples=1,
		verbose=1,
		scheduler=scheduler,
		progress_reporter=reporter
	)

if __name__ == '__main__':
	main()


text:
# coding: utf-8

# # Dataset boolean6: implicit conjunction - different predicates applied on a person (reference problem)
#
# Generating sentences of the form
#
# - 1) **c, who is a PROFESSION, saw X, c didn't saw X** -- contradiction
# - 1) **c, who is a PROFESSION, visit d, c didn't  visit d ** -- contradiction
# - 1) **c, who is a PROFESSION, bought Y, c didn't  buy Y ** -- contradiction
#
#
# - 2) **c, who is a PROFESSION, saw X, c isn't a PROFESSION ** -- contradiction
# - 2) **c, who is a PROFESSION, visit d, c isn't a PROFESSION ** -- contradiction
# - 2) **c, who is a PROFESSION, bought Y, c isn't a PROFESSION ** -- contradiction
#
#
# - 3) **c, who is a PROFESSION1, saw X, c, who is a PROFESSION2, didn't saw X**  -- non-contradiction
# - 3) **c, who is a PROFESSION1, visit d, c, who is a PROFESSION2, didn't visit d**  -- non-contradiction
# - 3) **c, who is a PROFESSION1, bought Y, c, who is a PROFESSION2, didn't bought Y**  -- non-contradiction
#
#
# - 4) **c, who is a PROFESSION, saw X, c, e's father (mother), didn't saw X** -- non-contradiction
# - 4) **c, who is a PROFESSION, visit d, c, e's father (mother), didn't visit d** -- non-contradiction
# - 4) **c, who is a PROFESSION, bought Y, c, e's father (mother), didn't bought Y** -- non-contradiction
#
#
# - 5) **c saw Z running COMPLEMENT, c didn't saw Z running** -- contradiction
# - 5) **c saw Z driving COMPLEMENT, c didn't saw Z driving** -- contradiction
#
#
#
# - 6) **c saw X running COMPLEMENT1, c didn't saw X running  COMPLEMENT2** -- non-contradiction
# - 6) **c saw X driving COMPLEMENT1, c didn't saw X driving COMPLEMENT2** -- non-contradiction
#
#
# X = [a girl running, a blue plane, the new Tesla sport car, the new Marvel movie, a dog chasing a cat, etc.]
#
# Y = [the new Tesla Roadster, etc.]
#
# Z = [name, a girl, a boy]
#
#

import numpy as np
import pandas as pd
import os

try:
    from word_lists import male_names, female_names, name_list
except ImportError:
    from contra_qa.text_generation.word_lists import male_names, female_names, name_list  # noqa

try:
    from word_lists import professions, color_list, city_list
except ImportError:
    from contra_qa.text_generation.word_lists import professions, color_list, city_list  # noqa

def get_new_item(item_list, src_list):
    size = len(src_list)
    new_item = src_list[np.random.choice(size)]
    while new_item in item_list:
        new_i = np.random.choice(size)
        new_item = src_list[new_i]
    return new_item

def boolean6():

    upper_bound = 11000 / 6

    COMPLEMENT_runnig = ["from a parade car",
                         "from PERSON's car",
                         "from a COLOR car",
                         "from a COLOR bus",
                         "from a special recently built bus",
                         "from a COLOR school bus",
                         "from a bus going to CITY",
                         "from a COLOR bus with a new engine"]

    COMPLEMENT_driving = ["a COLOR car",
                          "in CITY",
                          "PERSON's car",
                          "a COLOR bicycle",
                          "PERSON's bicycle",
                          "an electric COLOR bicycle",
                          "PERSON's new Tesla Roadster"]

    what_I_see = ["a girl running",
                  "a blue plane",
                  "the new Tesla Roadster",
                  "the new Marvel movie from the Russo Brothers",
                  "a dog chasing a cat",
                  "the accident",
                  "the car crash",
                  "a strange add",
                  "the city mayor"]

    what_I_buy = ["the new Tesla Roadster",
                  "the new Marvel movie from the Russo Brothers",
                  "a Chilean wine",
                  "a Macbook",
                  "a cup of coffee",
                  "a pizza",
                  "a japanese novel"]

    # - 1) **c, who is a PROFESSION, saw X, c didn't saw X** -- contradiction
    # - 1) **c, who is a PROFESSION, visit d, c didn't  visit d ** -- contradiction
    # - 1) **c, who is a PROFESSION, bought Y, c didn't  buy Y ** -- contradiction

    all_sentences_1 = []
    for i in range(int(upper_bound)):
        person1 = get_new_item([], name_list)
        person2 = get_new_item([person1], name_list)
        buy = get_new_item([], what_I_buy)
        see = get_new_item([], what_I_see)
        profession = get_new_item([], professions)
        if i % 3 == 0:
            sentence = "{}, who is a {}, saw {};{} didn't saw {}".format(
                person1, profession, see, person1, see)
        elif i % 3 == 1:
            sentence = "{}, who is a {}, visit {};{} didn't visit {}".format(
                person1, profession, person2, person1, person2)
        else:
            sentence = "{}, who is a {}, bought {};{} didn't buy {}".format(
                person1, profession, buy, person1, buy)
        all_sentences_1.append(sentence)

    all_sentences_1 = [sentence.split(";") + [1]
                       for sentence in all_sentences_1]

    # - 2) **c, who is a PROFESSION, saw X, c isn't a PROFESSION ** -- contradiction
    # - 2) **c, who is a PROFESSION, visit d, c isn't a PROFESSION ** -- contradiction
    # - 2) **c, who is a PROFESSION, bought Y, c isn't a PROFESSION ** -- contradiction

    all_sentences_2 = []
    for i in range(int(upper_bound)):
        person1 = get_new_item([], name_list)
        person2 = get_new_item([person1], name_list)
        buy = get_new_item([], what_I_buy)
        see = get_new_item([], what_I_see)
        profession = get_new_item([], professions)
        if i % 3 == 0:
            sentence = "{}, who is a {}, saw {};{} isn't a {}".format(
                person1, profession, see, person1, profession)
        elif i % 3 == 1:
            sentence = "{}, who is a {}, visit {};{} isn't a {}".format(
                person1, profession, person2, person1, profession)
        else:
            sentence = "{}, who is a {}, bought {};{} isn't a {}".format(
                person1, profession, buy, person1, profession)
        all_sentences_2.append(sentence)

    all_sentences_2 = [sentence.split(";") + [1]
                       for sentence in all_sentences_2]

    # - 3) **c, who is a PROFESSION1, saw X, c, who is a PROFESSION2, didn't saw X**  -- non-contradiction
    # - 3) **c, who is a PROFESSION1, visit d, c, who is a PROFESSION2, didn't visit d**  -- non-contradiction
    # - 3) **c, who is a PROFESSION1, bought Y, c, who is a PROFESSION2, didn't bought Y**  -- non-contradiction

    all_sentences_3 = []
    for i in range(int(upper_bound)):
        person1 = get_new_item([], name_list)
        person2 = get_new_item([person1], name_list)
        buy = get_new_item([], what_I_buy)
        see = get_new_item([], what_I_see)
        profession1 = get_new_item([], professions)
        profession2 = get_new_item([profession1], professions)
        assert profession1 != profession2
        if i % 3 == 0:
            sentence = "{}, who is a {}, saw {};{}, who is a {}, didn't saw {}".format(
                person1, profession1, see, person1, profession2, see)
        elif i % 3 == 1:
            sentence = "{}, who is a {}, visit {};{}, who is a {}, didn't visit {}".format(
                person1, profession1, person2, person1, profession2, person2)
        else:
            sentence = "{}, who is a {}, bought {};{}, who is a {}, didn't buy {}".format(
                person1, profession1, buy, person1, profession2, buy)
        all_sentences_3.append(sentence)

    all_sentences_3 = [sentence.split(";") + [0]
                       for sentence in all_sentences_3]

    # - 4) **c, who is a PROFESSION, saw X, c, e's father (mother), didn't saw X** -- non-contradiction
    # - 4) **c, who is a PROFESSION, visit d, c, e's father (mother), didn't visit d** -- non-contradiction
    # - 4) **c, who is a PROFESSION, bought Y, c, e's father (mother), didn't bought Y** -- non-contradiction

    all_sentences_4 = []
    for i in range(int(upper_bound)):
        if i % 2 == 0:
            person1 = get_new_item([], female_names)
            parent = "mother"
        else:
            person1 = get_new_item([], male_names)
            parent = "father"
        person2 = get_new_item([person1], name_list)
        person3 = get_new_item([person1, person2], name_list)
        buy = get_new_item([], what_I_buy)
        see = get_new_item([], what_I_see)
        profession1 = get_new_item([], professions)
        profession2 = get_new_item([profession1], professions)
        if i % 3 == 0:
            sentence = "{}, who is a {}, saw {};{}, {}'s {}, didn't saw {}".format(
                person1, profession1, see, person1, person3, parent, see)
        elif i % 3 == 1:
            sentence = "{}, who is a {}, visit {};{}, {}'s {}, didn't visit {}".format(
                person1, profession1, person2, person1, person3, parent, person2)
        else:
            sentence = "{}, who is a {}, bought {};{}, {}'s {}, didn't buy {}".format(
                person1, profession1, buy, person1, person3, parent, buy)
        all_sentences_4.append(sentence)

    all_sentences_4 = [sentence.split(";") + [0]
                       for sentence in all_sentences_4]

    # - 5) **c saw X running COMPLEMENT, c didn't saw X running** -- contradiction
    # - 5) **c saw X driving COMPLEMENT, c didn't saw X driving** -- contradiction

    all_sentences_5 = []
    for i in range(int(upper_bound + 1)):
        person1 = get_new_item([], name_list)
        object_list = [get_new_item([person1], name_list), "a girl", "a boy"]
        direct_object = get_new_item([], object_list)
        if i % 2 == 0:
            verb = "running"
            complement = get_new_item([], COMPLEMENT_runnig)
        else:
            verb = "driving"
            complement = get_new_item([], COMPLEMENT_driving)
        person2 = get_new_item([person1, direct_object], name_list)
        color = get_new_item([], color_list)
        city = get_new_item([], city_list)
        complement = complement.replace("PERSON's", person2 + "'s")
        complement = complement.replace("COLOR", color)
        complement = complement.replace("CITY", city)
        sentence = "{} saw {} {} {};{} didn't saw {} {}".format(person1,
                                                                direct_object,
                                                                verb,
                                                                complement,
                                                                person1,
                                                                direct_object,
                                                                verb)
        all_sentences_5.append(sentence)

    all_sentences_5 = [sentence.split(";") + [1]
                       for sentence in all_sentences_5]

    # - 6) **c saw X running COMPLEMENT1, c didn't saw X running  COMPLEMENT2** -- non-contradiction
    # - 6) **c saw X driving COMPLEMENT1, c didn't saw X driving COMPLEMENT2** -- non-contradiction

    all_sentences_6 = []
    for i in range(int(upper_bound + 1)):
        person1 = get_new_item([], name_list)
        object_list = [get_new_item([person1], name_list), "a girl", "a boy"]
        direct_object = get_new_item([], object_list)
        if i % 2 == 0:
            verb = "running"
            COMPLEMENT = COMPLEMENT_runnig
        else:
            verb = "driving"
            COMPLEMENT = COMPLEMENT_driving

        complement1 = get_new_item([], COMPLEMENT)
        complement2 = get_new_item([complement1], COMPLEMENT)
        person2 = get_new_item([person1, direct_object], name_list)
        color = get_new_item([], color_list)
        city = get_new_item([], city_list)
        complement1 = complement1.replace("PERSON's", person2 + "'s")
        complement1 = complement1.replace("COLOR", color)
        complement1 = complement1.replace("CITY", city)
        person2 = get_new_item([person1, direct_object, person2], name_list)
        color = get_new_item([], color_list)
        city = get_new_item([], city_list)
        complement2 = complement2.replace("PERSON's", person2 + "'s")
        complement2 = complement2.replace("COLOR", color)
        complement2 = complement2.replace("CITY", city)
        sentence = "{} saw {} {} {};{} didn't saw {} {} {}".format(person1,
                                                                   direct_object,
                                                                   verb,
                                                                   complement1,
                                                                   person1,
                                                                   direct_object,
                                                                   verb,
                                                                   complement2)
        all_sentences_6.append(sentence)

    all_sentences_6 = [sentence.split(";") + [0]
                       for sentence in all_sentences_6]

    np.random.shuffle(all_sentences_1)
    np.random.shuffle(all_sentences_2)
    np.random.shuffle(all_sentences_3)
    np.random.shuffle(all_sentences_4)
    np.random.shuffle(all_sentences_5)
    np.random.shuffle(all_sentences_6)

    size1 = len(all_sentences_1)
    size2 = len(all_sentences_2)
    size3 = len(all_sentences_3)
    size4 = len(all_sentences_4)
    size5 = len(all_sentences_5)
    size6 = len(all_sentences_6)

    all_sentences = all_sentences_1 + all_sentences_2 + \
        all_sentences_3 + all_sentences_4
    all_sentences += all_sentences_5 + all_sentences_6

    # ### Generating a train DataFrame with 10000 examples and a test DataFrame with 1000 examples

    sentence_1 = [triple[0] for triple in all_sentences]
    sentence_2 = [triple[1] for triple in all_sentences]
    label = [triple[2] for triple in all_sentences]

    df_dict = {"sentence1": sentence_1,
               "sentence2": sentence_2,
               "label": label}

    df = pd.DataFrame(df_dict)
    df = df[["sentence1", "sentence2", "label"]]
    df = df.sample(frac=1).reset_index(drop=True)

    df_train = df.iloc[:10000]
    df_test = df.iloc[10000:]

    # ### Saving to CSV

    if not os.path.exists("./data"):
        os.makedirs("data/")

    df_train.to_csv("data/boolean6_train.csv", index=False)
    df_test.to_csv("data/boolean6_test.csv", index=False)

if __name__ == '__main__':
    boolean6()

ext: py

max_stars_repo_name: felipessalvatore/ContraQA

max_issues_repo_name: felipessalvatore/ContraQA

max_forks_repo_name: felipessalvatore/ContraQA

reason: import numpy


text:
#!/usr/bin/env python

# For offline plots ...
import matplotlib
#matplotlib.use('Agg')

import os, sys, glob

from netCDF4 import Dataset

import scipy
import numpy as np
import matplotlib.pyplot as plt
from palettable.colorbrewer import sequential

import utils

# Setup
netcdf_dir = 'netcdfs'

basins_mtn = [1.2, 1.3, 2.2, 3.1, 3.2, 3.3, 4.1, 4.3, 5.0, 6.1, 7.2]
basins_gtl = [1.1, 1.4, 2.1, 4.2, 6.2, 7.1, 8.1, 8.2]

csv_output_file = 'SI_glacier_spreadsheet.csv'

# Processing
glaciers_all = glob.glob(netcdf_dir + '/glacier????.nc')

branches = dict()
for glacier in glaciers_all:
   if glacier.split('/')[-1][7].isalpha():
      branches['0' + glacier.split('/')[-1][8:11]] = [branch for branch in glaciers_all if branch.split('/')[-1][8:11] == glacier.split('/')[-1][8:11]]
   else:
      branches['0' + glacier.split('/')[-1][8:11]] = [glacier]

for parent in sorted(branches.keys()):

   PeX = dict()
   PeY = dict()
   PeD = dict()

   flux = 0.

   for branch in branches[parent]:
      ds = Dataset(branch, 'r')
      basin = float('{:3.1f}'.format(np.asarray(ds['basin'][:])[()]))
      flux = flux + np.asarray(ds['flux'][:])[()]
      name = ds.description.split(':')[1].split('.')[0].strip()

      flowline_groups, iteration_list = utils.get_flowline_groups(ds)
      knickpoint_slope_flowlines = list()
      has_knickpoint_flowlines = list()
      for flowline_group, iteration in zip(flowline_groups, iteration_list):
         x = flowline_group['x'][:]
         y = flowline_group['y'][:]
         d = flowline_group['d'][:]

         Pe1 = flowline_group['Pe']['AERO']['nominal'][:]
         Pe2 = flowline_group['Pe']['GIMP']['nominal'][:]

         # Find where Pe=3 along this flowline
         ID = 'glacier' + branch.split('/')[-1][7:11] + '_' + flowline_group.name.replace('flowline','') + '_' + iteration
         PeX[ID], PeY[ID], PeD[ID] = utils.get_Pe3(ID, x, y, d, Pe1, Pe2, PeThreshold=3.0)

         # Knickpoint
         has_knickpoint = np.asarray(flowline_group['knickpoint']['has_knickpoint'][:])[()]
         if has_knickpoint:
            knickpoint_slope = float(np.asarray(flowline_group['knickpoint']['knickpoint_slope'][:])[()])
            knickpoint_slope_flowlines.append(knickpoint_slope)
         has_knickpoint_flowlines.append(has_knickpoint)

      ds.close()

      # Glacier-wide
      has_knickpoint_flowlines = np.array(has_knickpoint_flowlines)
      knickpoint_slope_flowlines = np.array(knickpoint_slope_flowlines)
      if len(has_knickpoint_flowlines[has_knickpoint_flowlines == 1]) / len(has_knickpoint_flowlines) > 0.5:
         has_knickpoint_str = 'has knickpoint'
         knickpoint_slope = np.median(np.array(knickpoint_slope_flowlines)) * 180/np.pi # radians to degrees
      else:
         has_knickpoint_str = 'no'
         knickpoint_slope = np.nan

   # Calculate glacier-wide thinning limit
   statsDict = utils.get_stats(PeX, PeY, PeD)
   gwtl = utils.glacier_wide_thinning_limit(statsDict)

   sys.stdout.write('{:s}, {:s}, {:3.1f}, {:6.1f}, {:6.1f}, {:s}, {:6.3f}\n'.format(parent, name, basin, gwtl/1000., flux, has_knickpoint_str, knickpoint_slope))

reason: import numpy,import scipy


text:
import numpy as np

#import matplotlib as mpl
import matplotlib.pyplot as plt
#from mpl_toolkits.mplot3d import Axes3D

from sklearn import datasets
from sklearn.decomposition import PCA

iris = datasets.load_iris(as_frame=True)
X = iris.data.values
y = iris.target.values

print(iris.frame.head(2))

X_reduced_2d = PCA(n_components=2).fit_transform(X)

# import matplotlib.gridspec as gridspec
fig = plt.figure(figsize = (20,15))

# ...
ax1 = fig.add_subplot(3,3,1, projection='3d')
ax1.set_title("3D iris, first three features")
ax1.scatter(X[:, 0], X[:, 1],  X[:, 2], c=y, cmap=plt.cm.Set1, edgecolor='k', alpha=.2)

ax2 = fig.add_subplot(3,3,2, projection='3d')
ax2.set_title("3D iris, PCA 2D")
ax2.scatter(X_reduced_2d[:, 0], X_reduced_2d[:, 1], zs=0, c=y, cmap=plt.cm.Set1, edgecolor='k', alpha=.2)

ax3 = fig.add_subplot(3,3,3)
ax3.set_title("2D iris, PCA 2D")
ax3.scatter(X_reduced_2d[:, 0], X_reduced_2d[:, 1], c=y, cmap=plt.cm.Set1, edgecolor='k', alpha=.2)

plt.show()

max_stars_repo_name: SalAlba/coursera-machine-learning

max_issues_repo_name: SalAlba/coursera-machine-learning

max_forks_repo_name: SalAlba/coursera-machine-learning


text:
import sys
from ...utils import ini,util
import numpy as np

def get_config():
    """Return the essential configuration specifics, i.e. number of parameters,
    total number of jobs..
    """
    if not util.fexists('config.ini'):
        print("Can't find file: config.ini")
        sys.exit(1)

    params, settings = ini.parameters_from_ini('config.ini')

    return params, settings

def get_number_params():
    return get_config()[0].get_dim()

def get_number_jobs():
    return get_config()[0].get_maxnum()

def get_parameter_info(parameter_name=None):
    params, settings = get_config()
    names = params.get_names()

    if parameter_name == None:
        # all
#        value_format = " | ".join(["{:3d} - {:10}"]*len(names))
        value_format = "{:3d} - {:10}"
        empty_format = " "*16
        name_format = " | ".join(["{:^16}"]*len(names))
        values = [params.get_values_ax(n) for n in names]
        print(name_format.format(*names))
        print("-|-".join(["-"*16]*len(names)))
        dims = [len(v) for v in values]
        for i in range(max(dims)):
#            cur_vals = [v[i] if i < dims[ind] else "" for ind,v in enumerate(values)]
#            line_values = []
#            for v in cur_vals:
#                line_values.append(i)
#                line_values.append(v)
#            print(value_format.format(*line_values))
            line_strings = [value_format.format(i, v[i]) if i < dims[ind] else empty_format for ind,v in enumerate(values)]
            print(" | ".join(line_strings))
        print("Total number: {}".format(np.prod(dims)))
    else:
        try:
            ind = names.index(parameter_name)
        except ValueError:
            print("Unknown parameter: {}".format(parameter_name))
            sys.exit(1)
        value_format = "{:3d} - {:10}"
        name_format = "{:16}"
        values = params.get_values_ax(names[ind])
        for i,v in enumerate(values):
            print(value_format.format(i, v))


text:
'''
Demo of predictions with pretrained models.

Summary
-------
Reads example input data from provided CSV file
Reads model from provided ONNX file
Performs prediction for each input vector in the provided model,
using an sklearn-like interface
'''

import onnxruntime as rt
import numpy as np
import pandas as pd
import os

from ONNXPretrainedModel import ONNXPretrainedModel

if __name__ == "__main__":

    #resource_dir = "toy_lasso"             # simple task with artificial data
    #resource_dir = "mastre_medianprobhit"  # predict marksmanship
    resource_dir = "mastre_coursetime"      # predict time to complete course

    x_df = pd.read_csv(
        os.path.join(resource_dir, "example_input_features.csv"))
    print("-----------------------")
    print("Example Input X-Factors")
    print("-----------------------")
    print(x_df.head())

    print()
    model = ONNXPretrainedModel(
        os.path.join(resource_dir, "model_trial001.onnx"))
    print("-----------------------")
    print("Predictions")
    print("where predicted Y-Factor is: %s" % model.label_name)
    print("-----------------------")
    print(model.predict(x_df.values))
    print()

max_stars_repo_name: tufts-ml/mastre-reuse-pretrained-models

max_issues_repo_name: tufts-ml/mastre-reuse-pretrained-models

max_forks_repo_name: tufts-ml/mastre-reuse-pretrained-models


text:
"""
********************************
*   Created by mohammed-alaa   *
********************************
This Contains:
Helper function for data loaders and augmentation
the sequence loader class: multiprocess/multithread approach for dataloading
"""
import os

import cv2
import numpy as np
from imgaug import augmenters as iaa
from imgaug import parameters as iap
from tensorflow.python.keras.utils import OrderedEnqueuer

def stack_opticalflow(start_frame_index, video_name, data_root_path, stacked_frames):  # returns numpy (h,w,stacked*2) = one sample
    """
    Stacks "stacked_frames" u/v frames on a single numpy array : (h,w,stacked*2)
    """
    first_optical_frame_u = cv2.imread(os.path.join(data_root_path, "u", "v_" + video_name, 'frame{}'.format(str(start_frame_index).zfill(6)) + '.jpg'), cv2.IMREAD_GRAYSCALE)  # horizontal
    first_optical_frame_v = cv2.imread(os.path.join(data_root_path, "v", "v_" + video_name, 'frame{}'.format(str(start_frame_index).zfill(6)) + '.jpg'), cv2.IMREAD_GRAYSCALE)  # vertical

    stacked_optical_flow_sample = np.zeros(first_optical_frame_u.shape + (2 * stacked_frames,), dtype=np.uint8)  # with channel dimension of  stacked_frames(u)+ stacked_frames(v)

    stacked_optical_flow_sample[:, :, 0] = first_optical_frame_u
    stacked_optical_flow_sample[:, :, 0 + stacked_frames] = first_optical_frame_v

    for index, optical_frame_id in enumerate(range(start_frame_index + 1, start_frame_index + stacked_frames), 1):  # index starts at 1 placed after the first one
        stacked_optical_flow_sample[:, :, index] = cv2.imread(os.path.join(data_root_path, "u", "v_" + video_name, 'frame{}'.format(str(optical_frame_id).zfill(6)) + '.jpg'), cv2.IMREAD_GRAYSCALE)
        stacked_optical_flow_sample[:, :, index + stacked_frames] = cv2.imread(os.path.join(data_root_path, "v", "v_" + video_name, 'frame{}'.format(str(optical_frame_id).zfill(6)) + '.jpg'), cv2.IMREAD_GRAYSCALE)

    return stacked_optical_flow_sample

def get_noise_augmenters(augmenter_level):
    """
    Gets an augmenter object of a given level
    """
    # 0 heavy , 1 medium,2 simple
    if augmenter_level == 0:
        ####################################################### heavy augmentation #########################################################################
        return [iaa.Sometimes(0.9, iaa.Crop(
            percent=((iap.Clip(iap.Normal(0, .5), 0, .6),) * 4)  # random crops top,right,bottom,left
        )),
                # some noise
                iaa.Sometimes(0.9, [iaa.GaussianBlur(sigma=(0, 0.3)), iaa.Sharpen(alpha=(0.0, .15), lightness=(0.5, 1.5)), iaa.Emboss(alpha=(0.0, 1.0), strength=(0.1, 0.2))]),
                iaa.Sometimes(0.9, iaa.Add((-12, 12), per_channel=1))]  # rgb jittering
    elif augmenter_level == 1:
        ####################################################### medium  augmentation #######################################################################
        return [iaa.Sometimes(0.9, iaa.Crop(percent=((0.0, 0.15), (0.0, 0.15), (0.0, 0.15), (0.0, 0.15)))),  # random crops top,right,bottom,left
                # some noise
                iaa.Sometimes(0.5, [iaa.GaussianBlur(sigma=(0, 0.25)), iaa.Sharpen(alpha=(0.0, .1), lightness=(0.5, 1.25)), iaa.Emboss(alpha=(0.0, 1.0), strength=(0.05, 0.1))]),
                iaa.Sometimes(.7, iaa.Add((-10, 10), per_channel=1))]  # rgb jittering
    elif augmenter_level == 2:
        ######################################################## simple augmentation #######################################################################
        return [iaa.Sometimes(0.6, iaa.Crop(percent=((0.0, 0.1), (0.0, 0.1), (0.0, 0.1), (0.0, 0.1)))),  # random crops top,right,bottom,left
                # some noise
                iaa.Sometimes(0.35, [iaa.GaussianBlur(sigma=(0, 0.17)), iaa.Sharpen(alpha=(0.0, .07), lightness=(0.35, 1)), iaa.Emboss(alpha=(0.0, .7), strength=(0.1, 0.7))]),
                iaa.Sometimes(.45, iaa.Add((-7, 7), per_channel=1))]  # rgb jittering
        ###################################################################################################################################################

def get_validation_augmenter(height, width):
    """
    for validation we don't add any stochasticity just resize them to height*width
    """
    aug = iaa.Sequential([
        iaa.Scale({"height": height, "width": width})
    ])

    return aug

def get_training_augmenter(height, width, augmenter_level):
    """
    Get validation augmenter according to the level of stochasticity added
    """
    aug = iaa.Sequential([
        iaa.Fliplr(0.5),  # horizontal flips
        *get_noise_augmenters(augmenter_level),  # noisy heavy or simple
        iaa.Scale({"height": height, "width": width})
    ], random_order=True)  # apply augmenters in random order

    return aug

class SequenceLoader:
    """
    efficient pre fetching multiprocessing wrapper
    """

    def __init__(self, sequence_class, num_workers, queue_size, use_multiprocessing, do_shuffle, **kwargs):
        self.sequence_class = sequence_class
        self.kwargs = kwargs
        self.sequence = None
        self.parallel_generator = None
        self.shuffle = do_shuffle
        self.num_workers = num_workers
        self.queue_size = queue_size
        self.use_multiprocessing = use_multiprocessing

        self.reset()

    def __len__(self):
        """Denotes the number of batches per epoch"""
        return len(self.sequence)

    def get_actual_length(self):
        """Denotes the total number of samples"""
        return self.sequence.get_actual_length()

    def __getitem__(self, index):  # this is linear and inefficient, don't use me
        return self.sequence[index]

    def reset(self):
        self.sequence = self.sequence_class(**self.kwargs)
        enqueuer = OrderedEnqueuer(self.sequence, use_multiprocessing=self.use_multiprocessing)
        enqueuer.start(workers=self.num_workers, max_queue_size=self.queue_size)
        self.parallel_generator = enqueuer.get()

        if self.shuffle:
            self.sequence.shuffle_and_reset()

    def get_epoch_generator(self):  # you can loop on this or send it to generator based training
        for _ in range(self.__len__()):  # makes sure of length
            yield next(self.parallel_generator)

        if self.shuffle:
            self.sequence.shuffle_and_reset()

    def get_sustaining_generator(self):  # you can loop on this or send it to generator based training
        while True:  # any number of epochs
            for _ in range(self.__len__()):  # makes sure of length
                yield next(self.parallel_generator)

            if self.shuffle:
                self.sequence.shuffle_and_reset()

    def get_epochs_generator(self, epochs):  # you can loop on this or send it to generator based training
        for _ in range(epochs):  # any number of epochs
            for _ in range(self.__len__()):  # makes sure of length
                yield next(self.parallel_generator)

            if self.shuffle:
                self.sequence.shuffle_and_reset()


text:
import random
import math
import numpy as np

import pymunk

PI = math.pi
DEBUG_SIZE = 600  # size of the pygame debug screen

def get_force_with_angle(force, angle):
    x = force * math.cos(angle)
    y = force * math.sin(angle)
    return [x, y]

def front_from_corner(width, length, corner_position, angle):
    half_width = width/2
    dx = length * math.cos(angle) + half_width * math.cos(angle + PI/2)  # PI/2 gives a half-rotation for the width component
    dy = length * math.sin(angle) + half_width * math.sin(angle + PI/2)
    front_position = [corner_position[0] + dx, corner_position[1] + dy]
    return np.array([front_position[0], front_position[1], angle])

def corner_from_center(width, length, center_position, angle):
    half_length = length/2
    half_width = width/2
    dx = half_length * math.cos(angle) + half_width * math.cos(angle + PI/2)
    dy = half_length * math.sin(angle) + half_width * math.sin(angle + PI/2)
    corner_position = [center_position[0] - dx, center_position[1] - dy]
    return np.array([corner_position[0], corner_position[1], angle])

def random_body_position(body):
    ''' pick a random point along the boundary'''
    width, length = body.dimensions
    if random.randint(0, 1) == 0:
        # force along ends
        if random.randint(0, 1) == 0:
            # force on the left end
            location = (random.uniform(0, width), 0)
        else:
            # force on the right end
            location = (random.uniform(0, width), length)
    else:
        # force along length
        if random.randint(0, 1) == 0:
            # force on the bottom end
            location = (0, random.uniform(0, length))
        else:
            # force on the top end
            location = (width, random.uniform(0, length))
    return location

class NullScreen(object):
    def update_screen(self):
        pass
    def configure(self, config):
        pass

class PymunkMultibody(object):
    """
    Multibody object for interfacing with pymunk
    """

    defaults = {
        'agent_shape': 'segment',
        # hardcoded parameters
        'elasticity': 0.9,
        'damping': 0.5,  # 1 is no damping, 0 is full damping
        'angular_damping': 0.8,
        'friction': 0.9,  # does this do anything?
        'physics_dt': 0.001,
        'force_scaling': 5e4,  # scales from pN
        # configured parameters
        'jitter_force': 1e-3,  # pN
        'bounds': [20, 20],
        'barriers': False,
        'initial_agents': {},
        # for debugging
        'screen': None,
    }

    def __init__(self, config):
        # hardcoded parameters
        self.elasticity = self.defaults['elasticity']
        self.friction = self.defaults['friction']
        self.damping = self.defaults['damping']
        self.angular_damping = self.defaults['angular_damping']
        self.physics_dt = config.get('physics_dt', self.defaults['physics_dt'])
        self.force_scaling = self.defaults['force_scaling']

        # configured parameters
        self.agent_shape = config.get('agent_shape', self.defaults['agent_shape'])
        self.jitter_force = config.get('jitter_force', self.defaults['jitter_force'])
        self.bounds = config.get('bounds', self.defaults['bounds'])
        barriers = config.get('barriers', self.defaults['barriers'])

        # initialize pymunk space
        self.space = pymunk.Space()

        # debug screen
        self.screen = config.get('screen')
        if self.screen is None:
            self.screen = NullScreen()
        self.screen.configure({
            'space': self.space,
            'bounds': self.bounds})

        # add static barriers
        self.add_barriers(self.bounds, barriers)

        # initialize agents
        initial_agents = config.get('initial_agents', self.defaults['initial_agents'])
        self.bodies = {}
        for agent_id, specs in initial_agents.items():
            self.add_body_from_center(agent_id, specs)

    def run(self, timestep):
        if self.physics_dt > timestep:
            print('timestep skipped by pymunk_multibody: {}'.format(timestep))
            return

        time = 0
        while time < timestep:
            time += self.physics_dt

            # apply forces
            for body in self.space.bodies:
                self.apply_jitter_force(body)
                self.apply_motile_force(body)
                self.apply_viscous_force(body)

            # run for a physics timestep
            self.space.step(self.physics_dt)

        self.screen.update_screen()

    def apply_motile_force(self, body):
        width, length = body.dimensions
        motile_location = (width / 2, 0)  # apply force at back end of body
        thrust = 0.0
        torque = 0.0
        motile_force = [thrust, torque]

        if hasattr(body, 'thrust'):
            thrust = body.thrust
            torque = body.torque
            motile_force = [thrust, 0.0]

            # add to angular velocity
            body.angular_velocity += torque

        scaled_motile_force = [force * self.force_scaling for force in motile_force]
        body.apply_impulse_at_local_point(scaled_motile_force, motile_location)

    def apply_jitter_force(self, body):
        jitter_location = random_body_position(body)
        jitter_force = [
            random.normalvariate(0, self.jitter_force),
            random.normalvariate(0, self.jitter_force)]
        scaled_jitter_force = [
            force * self.force_scaling
            for force in jitter_force]
        body.apply_impulse_at_local_point(
            scaled_jitter_force,
            jitter_location)

    def apply_viscous_force(self, body):
        # dampen velocity
        body.velocity = body.velocity * self.damping + (body.force / body.mass) * self.physics_dt

        # dampen angular velocity
        body.angular_velocity = body.angular_velocity * self.angular_damping + (body.torque / body.moment) * self.physics_dt

    def add_barriers(self, bounds, barriers):
        """ Create static barriers """
        thickness = 50.0
        offset = thickness
        x_bound = bounds[0]
        y_bound = bounds[1]

        static_body = self.space.static_body
        static_lines = [
            pymunk.Segment(
                static_body,
                (0.0-offset, 0.0-offset),
                (x_bound+offset, 0.0-offset),
                thickness),
            pymunk.Segment(
                static_body,
                (x_bound+offset, 0.0-offset),
                (x_bound+offset, y_bound+offset),
                thickness),
            pymunk.Segment(
                static_body,
                (x_bound+offset, y_bound+offset),
                (0.0-offset, y_bound+offset),
                thickness),
            pymunk.Segment(
                static_body,
                (0.0-offset, y_bound+offset),
                (0.0-offset, 0.0-offset),
                thickness),
        ]

        if barriers:
            spacer_thickness = barriers.get('spacer_thickness', 0.1)
            channel_height = barriers.get('channel_height')
            channel_space = barriers.get('channel_space')
            n_lines = math.floor(x_bound/channel_space)

            machine_lines = [
                pymunk.Segment(
                    static_body,
                    (channel_space * line, 0),
                    (channel_space * line, channel_height), spacer_thickness)
                for line in range(n_lines)]
            static_lines += machine_lines

        for line in static_lines:
            line.elasticity = 0.0  # bounce
            line.friction = 0.8
            self.space.add(line)

    def get_shape(self, boundary):
        '''
        shape documentation at: https://pymunk-tutorial.readthedocs.io/en/latest/shape/shape.html
        '''

        if self.agent_shape == 'segment':
            width = boundary['width']
            length = boundary['length']

            half_width = width / 2
            half_length = length / 2 - half_width
            shape = pymunk.Segment(
                None,
                (-half_length, 0),
                (half_length, 0),
                radius=half_width)

        elif self.agent_shape == 'circle':
            length = boundary['length']
            half_length = length / 2
            shape = pymunk.Circle(None, radius=half_length, offset=(0, 0))

        elif self.agent_shape == 'rectangle':
            width = boundary['width']
            length = boundary['length']
            half_length = length / 2
            half_width = width / 2
            shape = pymunk.Poly(None,
                ((-half_length, -half_width),
                 (half_length, -half_width),
                 (half_length, half_width),
                 (-half_length, half_width)))

        return shape

    def get_inertia(self, shape, mass):
        if self.agent_shape == 'rectangle':
            inertia = pymunk.moment_for_poly(mass, shape.get_vertices())
        elif self.agent_shape == 'circle':
            radius = shape.radius
            inertia = pymunk.moment_for_circle(mass, radius, radius)
        elif self.agent_shape == 'segment':
            a = shape.a
            b = shape.b
            radius = shape.radius
            inertia = pymunk.moment_for_segment(mass, a, b, radius)

        return inertia

    def add_body_from_center(self, body_id, specs):
        boundary = specs['boundary']
        mass = boundary['mass']
        center_position = boundary['location']
        angle = boundary['angle']
        angular_velocity = boundary.get('angular_velocity', 0.0)
        width = boundary['width']
        length = boundary['length']

        # get shape, inertia, make body, assign body to shape
        shape = self.get_shape(boundary)
        inertia = self.get_inertia(shape, mass)
        body = pymunk.Body(mass, inertia)
        shape.body = body

        body.position = (
            center_position[0],
            center_position[1])
        body.angle = angle
        body.dimensions = (width, length)
        body.angular_velocity = angular_velocity

        shape.elasticity = self.elasticity
        shape.friction = self.friction

        # add body and shape to space
        self.space.add(body, shape)

        # add body to agents dictionary
        self.bodies[body_id] = (body, shape)

    def update_body(self, body_id, specs):
        boundary = specs['boundary']
        length = boundary['length']
        width = boundary['width']
        mass = boundary['mass']
        thrust = boundary['thrust']
        torque = boundary['torque']

        body, shape = self.bodies[body_id]
        position = body.position
        angle = body.angle

        # get shape, inertia, make body, assign body to shape
        new_shape = self.get_shape(boundary)
        inertia = self.get_inertia(new_shape, mass)
        new_body = pymunk.Body(mass, inertia)
        new_shape.body = new_body

        new_body.position = position
        new_body.angle = angle
        new_body.velocity = body.velocity
        new_body.angular_velocity = body.angular_velocity
        new_body.dimensions = (width, length)
        new_body.thrust = thrust
        new_body.torque = torque

        new_shape.elasticity = shape.elasticity
        new_shape.friction = shape.friction

        # swap bodies
        self.space.remove(body, shape)
        self.space.add(new_body, new_shape)

        # update body
        self.bodies[body_id] = (new_body, new_shape)

    def update_bodies(self, bodies):
        # if an agent has been removed from the agents store,
        # remove it from space and bodies
        removed_bodies = [
            body_id for body_id in self.bodies.keys()
            if body_id not in bodies.keys()]
        for body_id in removed_bodies:
            body, shape = self.bodies[body_id]
            self.space.remove(body, shape)
            del self.bodies[body_id]

        # update agents, add new agents
        for body_id, specs in bodies.items():
            if body_id in self.bodies:
                self.update_body(body_id, specs)
            else:
                self.add_body_from_center(body_id, specs)

    def get_body_position(self, agent_id):
        body, shape = self.bodies[agent_id]
        return {
            'location': [pos for pos in body.position],
            'angle': body.angle,
        }

    def get_body_positions(self):
        return {
            body_id: {
                'boundary': self.get_body_position(body_id)}
            for body_id in self.bodies.keys()}

def test_multibody(
        total_time=2,
        agent_shape='rectangle',
        n_agents=1,
        jitter_force=1e1,
        screen=None):

    bounds = [500, 500]
    center_location = [0.5*loc for loc in bounds]
    agents = {
        str(agent_idx): {
            'boundary': {
                'location': center_location,
                'angle': random.uniform(0,2*PI),
                'volume': 15,
                'length': 30,
                'width': 10,
                'mass': 1,
                'thrust': 1e3,
                'torque': 0.0}}
        for agent_idx in range(n_agents)
    }
    config = {
        'agent_shape': agent_shape,
        'jitter_force': jitter_force,
        'bounds': bounds,
        'barriers': False,
        'initial_agents': agents,
        'screen': screen
    }
    multibody = PymunkMultibody(config)

    # run simulation
    time = 0
    time_step = 1
    while time < total_time:
        time += time_step
        multibody.run(time_step)

if __name__ == '__main__':
    test_multibody(10)

max_stars_repo_name: vivarium-collective/vivarium-cell

max_issues_repo_name: vivarium-collective/vivarium-cell

max_forks_repo_name: vivarium-collective/vivarium-cell


text:
import pdb
import numpy as np
import math as m
import sys

#Read in the tables for each star
referencestarfile2 = sys.argv[1]
referencestarfile3 = sys.argv[2]
referencestarfile4 = sys.argv[3]
referencestarfile5 = sys.argv[4]
referencestarfile6 = sys.argv[5]
referencestarfile7 = sys.argv[6]
referencestarfile8 = sys.argv[7]
referencestarfile11 = sys.argv[8]

#Import the data into arrays.
#We will only work with the first 3 columns
#These are the times, fluxes, and flux errors
data2 = np.loadtxt(referencestarfile2)
data3 = np.loadtxt(referencestarfile3)
data4 = np.loadtxt(referencestarfile4)
data5 = np.loadtxt(referencestarfile5)
data6 = np.loadtxt(referencestarfile6)
data7 = np.loadtxt(referencestarfile7)
data8 = np.loadtxt(referencestarfile8)
data11 = np.loadtxt(referencestarfile11)

#The following is storing the relevent data into arrays
#--------------------------------------
fluxdata2 = data2[:,1]
fluxerrordata2 = data2[:,2]
time2 = data2[:,0]
#-------------------------------------
fluxdata3 = data3[:,1]
fluxerrordata3 = data3[:,2]
time3 = data3[:,0]
#-------------------------------------
fluxdata4 = data4[:,1]
fluxerrordata4 = data4[:,2]
time4 = data4[:,0]
#-------------------------------------
fluxdata5 = data5[:,1]
fluxerrordata5 = data5[:,2]
time5 = data5[:,0]
#--------------------------------------
fluxdata6 = data6[:,1]
fluxerrordata6 = data6[:,2]
time6 = data6[:,0]
#--------------------------------------
fluxdata7 = data7[:,1]
fluxerrordata7 = data7[:,2]
time7 = data7[:,0]
#---------------------------------------
fluxdata8 = data8[:,1]
fluxerrordata8 = data8[:,2]
time8 = data8[:,0]
#-------------------------------------
fluxdata11 = data11[:,1]
fluxerrordata11 = data11[:,2]
time11 = data11[:,0]
#------------------------------------

#Concatanate all of the fluxes and their errors
fluxdata = np.column_stack([fluxdata2, fluxdata3, fluxdata4, fluxdata5, fluxdata6, fluxdata7, fluxdata8])
fluxerrordata = np.column_stack([fluxerrordata2, fluxerrordata3, fluxerrordata4, fluxerrordata5, fluxerrordata6, fluxerrordata7, fluxerrordata8])

#Initialize some variables
mu_ref = np.zeros(fluxdata.shape[0])
sigma_ref = np.zeros(fluxdata.shape[0])
ratio = np.zeros(fluxdata.shape[0])
sigma_ratio = np.zeros(fluxdata.shape[0])

#Calculate average reference star flux for each exposure and its error
for i in range(0, fluxerrordata.shape[0]):     #i = image index
    denominator = 0
    numerator = 0
    for n in range(0, fluxerrordata.shape[1]): #n = star index
        #Ignore certain exposures from each reference star
        if fluxerrordata[i, n] == 0:
            continue

        #Calculate parts of average and sigma
        numerator += fluxdata[i,n]/(fluxerrordata[i,n]**2)
        denominator += (1.0/(fluxerrordata[i, n]**2))

    #Actual average and error for each image
    if (numerator == 0) and (denominator == 0):     #ignore the beginning exposures
        continue
    mu_ref[i] = numerator/denominator
    sigma_ref[i] = m.sqrt(1.0/denominator)

#Get ratio between science flux and average ref star flux
#and its error
for k in range(0, mu_ref.shape[0]):
    #Ignore certain exposures from each reference star
    if (fluxdata11[k] == 0) or (mu_ref[k] == 0):
        continue
    ratio[k] = (fluxdata11[k])/(mu_ref[k])
    sigma_ratio[k] = ratio[k]*m.sqrt((fluxerrordata11[k]/fluxdata11[k])**2+(sigma_ref[k]/mu_ref[k])**2)

#Write everything to a text file
with open('exo_masterrecord.txt', 'w+') as outfile:
    for i in range(0,ratio.shape[0]):
        #pdb.set_trace()
        outfile.write(str(time11[i]) + ' ' + str(fluxdata11[i]) + ' ' + str(fluxerrordata11[0]) + ' ' + str(mu_ref[i]) + ' ' + str(ratio[i]) + ' ' + str(sigma_ratio[i]) + '\n')


text:
from __future__ import print_function, division, absolute_import

import numpy as np

import imgaug as ia
import imgaug.augmenters as iaa

def main():
    image = ia.data.quokka(0.25)

    for N in [1, 2]:
        print("N=%d" % (N,))

        images_aug = []
        for M in [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20]:
            images_aug.extend(
                iaa.RandAugment(n=N, m=M, random_state=1)(images=[image] * 10)
            )
        ia.imshow(ia.draw_grid(images_aug, cols=10))

    for M in [0, 1, 2, 4, 8, 10]:
        print("M=%d" % (M,))
        aug = iaa.RandAugment(m=M, random_state=1)

        images_aug = []
        for _ in np.arange(6):
            images_aug.extend(aug(images=[image] * 16))

        ia.imshow(ia.draw_grid(images_aug, cols=16, rows=6))

if __name__ == "__main__":
    main()

max_stars_repo_name: chroneus/imgaug

max_issues_repo_name: chroneus/imgaug

max_forks_repo_name: chroneus/imgaug


text:
# Importing all the necessary files and functions

import time

import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import confusion_matrix, r2_score
from sklearn.model_selection import train_test_split

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.optim.lr_scheduler as scheduler
import torch.utils.data as data

# The following class is used to create necessary inputs for dataset class and dataloader class used during training process
class ModelDataset():

    def __init__(self, X, Y, batchsize, valset_size, shuffle):

        self.x = X                      # Inputs
        self.y = Y                      # Labels
        self.batchsize = batchsize
        self.valset_size = valset_size
        self.shuffle = shuffle
        self.x_train, self.x_val, self.y_train, self.y_val = train_test_split(
            self.x, self.y, test_size=self.valset_size, shuffle=self.shuffle)

    def get_trainset(self):

        # Method for getting training set inputs and labels

        return self.x_train, self.y_train

    def get_valset(self):

        # Method for getting validation set inputs and labels

        return self.x_val, self.y_val

    def get_batchsize(self):

        # Method for getting batch size for training and validatioin

        return self.batchsize

# The following class is used for creating a dataset class using torch functionality. Its a standard pytorch class
class Dataset(data.Dataset):

    def __init__(self, X, Y):

        self.X = X
        self.Y = Y

    def __len__(self):

        return len(self.Y)

    def __getitem__(self, index):

        x_item = torch.from_numpy(self.X[index]).double()
        y_item = torch.from_numpy(np.array(self.Y[index])).double()

        return x_item, y_item

# The following class builds an LSTM network
class LSTM(nn.Module):

    def __init__(self, if_default):

        super(LSTM, self).__init__()

        self.default_gate = if_default

        print(' ')
        print('1/11 - LSTM input size')
        self._get_input_size()

        print('='*25)
        print('2/11 - LSTM hidden size')
        self._get_hidden_size()

        print('='*25)
        print('3/11 - LSTM number of layers')
        self._get_num_layers()

        print('='*25)
        print('4/11 - LSTM bidirectional')
        self. _get_bidirectional()

        print('='*25)
        print('5/11 - LSTM output size')
        self. _get_output_size()

        self._build_network_architecture()

    def _get_input_size(self):

        # Method for getting an input size parameter for the LSTM network

        gate = 0
        while gate != 1:
            self.input_size = (input(
                'Please enter the input size in int for the LSTM network (input size > 0): ')).replace(' ','')
            if self.input_size.isnumeric() and int(self.input_size) > 0:
                self.input_size = int(self.input_size)
                gate = 1
            else:
                print('Please enter a valid input')
                print(' ')
        print(' ')

    def _get_hidden_size(self):

        # Method for getting an hidden size parameter for the LSTM network

        gate = 0
        while gate != 1:
            if self.default_gate == True:
                print('Default value for hidden size selected: 128')
                self.hidden_size = '128'
            else:
                self.hidden_size = (input(
                    'Please enter the hidden size in int for the LSTM network (hidden size > 0)\n For default size, please directly press enter without any input: ')).replace(' ','')
            if self.hidden_size == '':              # handling default case for hidden size
                print('Default value for hidden size selected: 128')
                self.hidden_size = '128'
            if self.hidden_size.isnumeric() and int(self.hidden_size) > 0:
                self.hidden_size = int(self.hidden_size)
                gate = 1
            else:
                print('Please enter a valid input')
                print(' ')
        print(' ')

    def _get_num_layers(self):

        # Method for getting a number of LSTM layer parameter for the LSTM network

        gate = 0
        while gate != 1:
            if self.default_gate == True:
                print('Default value selected for number of layers: 3')
                self.nlayers = '3'
            else:
                self.nlayers = (input(
                    'Please enter the number of layer for the LSTM network in int (number of layer > 0)\n For default option, please directly press enter without any input: ')).replace(' ','')
            if self.nlayers == '':              # handling default case for number of LSTM layer
                print('Default value selected for number of layers: 3')
                self.nlayers = '3'
            if self.nlayers.isnumeric() and int(self.nlayers) > 0:
                self.nlayers = int(self.nlayers)
                gate = 1
            else:
                print('Please enter a valid input')
                print(' ')
        print(' ')

    def _get_bidirectional(self):

        # Method for getting bidirectional input for the LSTM network

        gate = 0
        while gate != 1:
            if self.default_gate == True:
                print('By default, unidirectional LSTM network selected')
                self.bidirection = '0'
            else:
                self.bidirection = input(
                    'Please enter 1 to have a bidirectional LSTM network else enter 0 \n For default option, please directly press enter without any input: ').replace(' ','')
            if self.bidirection == '':
                print('By default, unidirectional LSTM network selected')
                self.bidirection = '0'
            if self.bidirection.isnumeric() and int(self.bidirection) > -1 and int(self.bidirection) < 2:
                if self.bidirection == '1':
                    self.bidirection_input = True
                else:
                    self.bidirection_input = False
                gate = 1
            else:
                print('Please enter a valid input')
                print(' ')
        print(' ')

    def _get_output_size(self):

        # Method for getting output size of the network
        gate = 0
        while gate != 1:
            self.output_size = (input(
                'Please enter the output size for the LSTM network. \n For regression please enter 1 else enter the number of classes for classification problem: ')).replace(' ','')
            if self.output_size.isnumeric() and int(self.output_size) > 0:
                self.output_size = int(self.output_size)
                gate = 1
            else:
                print('Please enter a valid input')
                print(' ')
        print(' ')

    def _build_network_architecture(self):

        # Method for building a network using all the information provided by a user in above functions
        self.lstm = nn.LSTM(self.input_size, self.hidden_size,
                            bidirectional=self.bidirection_input, num_layers=self.nlayers, batch_first=True)
        if self.bidirection_input:
            self.linear_input = self.hidden_size*2
        else:
            self.linear_input = self.hidden_size
        self.linear1 = nn.Linear(self.linear_input, int(self.linear_input/2))
        self.linear2 = nn.Linear(
            int(self.linear_input/2), int(self.linear_input/4))
        self.linear3 = nn.Linear(int(self.linear_input/4), self.output_size)

    def forward(self, x):
        out, _ = self.lstm(x)
        out = self.linear1(out[:, -1, :])
        out = self.linear2(out)
        out = self.linear3(out)

        return out

    def predict(self, x):
        out, _ = self.lstm(x)
        out = self.linear1(out[:, -1, :])
        out = self.linear2(out)
        out = self.linear3(out)
        return out

# The following class will be called by a user. The class calls other necessary classes to build a complete pipeline required for training
class LSTM():
    """
    Documentation Link:https://manufacturingnet.readthedocs.io/en/latest/

    """

    def __init__(self, X, Y, shuffle=True):

        # Lists used in the functions below
        self.criterion_list = {1: nn.CrossEntropyLoss(), 2: torch.nn.L1Loss(
        ), 3: torch.nn.SmoothL1Loss(), 4: torch.nn.MSELoss()}

        self.x_data = X
        self.y_data = Y
        self.shuffle = shuffle

        self.get_default_paramters()            # getting default parameters argument

        # building a network architecture
        self.net = (LSTM(self.default_gate)).double()

        print('='*25)
        print('6/11 - Batch size input')
        # getting a batch size for training and validation
        self._get_batchsize_input()

        print('='*25)
        print('7/11 - Validation set size')
        self._get_valsize_input()                # getting a train-validation split

        # splitting the data into training and validation sets
        self.model_data = ModelDataset(
            self.x_data, self.y_data, batchsize=self.batchsize, valset_size=self.valset_size, shuffle=self.shuffle)

        print('='*25)
        print('8/11 - Loss function')
        self._get_loss_function()               # getting a loss function

        print('='*25)
        print('9/11 - Optimizer')
        self._get_optimizer()               # getting an optimizer input

        print('='*25)
        print('10/11 - Scheduler')
        self._get_scheduler()               # getting a scheduler input

        self._set_device()              # setting the device to gpu or cpu

        print('='*25)
        print('11/11 - Number of epochs')
        self._get_epoch()           # getting an input for number oftraining epochs

        self.main()             # run function

    def get_default_paramters(self):

        # Method for getting a binary input for default paramters

        gate = 0
        while gate != 1:
            self.default = input(
                'Do you want default values for all the parameters (y/n)? ').replace(' ','')
            if self.default == 'y' or self.default == 'Y' or self.default == 'n' or self.default == 'N':
                if self.default.lower() == 'y':
                    self.default_gate = True
                else:
                    self.default_gate = False
                gate = 1
            else:
                print('Enter a valid input')
                print(' ')
        print(' ')

    def _get_batchsize_input(self):

        # Method for getting batch size input

        gate = 0
        while gate != 1:
            self.batchsize = (
                input('Please enter the batch size int input (greater than 0): ')).replace(' ','')
            if self.batchsize.isnumeric() and int(self.batchsize) > 0:
                self.batchsize = int(self.batchsize)
                gate = 1
            else:
                print('Please enter a valid input')
        print(' ')

    def _get_valsize_input(self):

        # Method for getting validation set size input

        gate = 0
        while gate != 1:
            if self.default_gate == True:
                print('Default value selected : 0.2')
                self.valset_size = '0.2'
            else:
                self.valset_size = (input(
                    'Please enter the train set size float input (size > 0 and size < 1) \n For default size, please directly press enter without any input: ')).replace(' ','')
            if self.valset_size == '':              # handling default case for valsize
                print('Default value selected : 0.2')
                self.valset_size = '0.2'
            if self.valset_size.replace('.', '').isdigit():
                if float(self.valset_size) > 0 and float(self.valset_size) < 1:
                    self.valset_size = float(self.valset_size)
                    gate = 1
            else:
                print('Please enter a valid input')
                print(' ')
        print(' ')

    def _get_loss_function(self):

        # Method for getting a loss function for training

        gate = 0
        while gate != 1:
            self.criterion_input = (input(
                'Please enter the appropriate loss function index for the problem: \n Criterion_list - [1: CrossEntropyLoss, 2: L1Loss, 3: SmoothL1Loss, 4: MSELoss]: ')).replace(' ','')

            if self.criterion_input.isnumeric() and int(self.criterion_input) < 5 and int(self.criterion_input) > 0:
                gate = 1
            else:
                print('Please enter a valid input')
                print(' ')

        self.criterion = self.criterion_list[int(self.criterion_input)]
        print(' ')

    def _get_optimizer(self):

        # Method for getting a optimizer input

        gate = 0
        while gate != 1:
            if self.default_gate == True:
                print('Default optimizer selected : Adam')
                self.optimizer_input = '1'
            else:
                self.optimizer_input = (input(
                    'Please enter the optimizer index for the problem \n Optimizer_list - [1: Adam, 2: SGD] \n For default optimizer, please directly press enter without any input: ')).replace(' ','')
            if self.optimizer_input == '':              # handling default case for optimizer
                print('Default optimizer selected : Adam')
                self.optimizer_input = '1'

            if self.optimizer_input.isnumeric() and int(self.optimizer_input) > 0 and int(self.optimizer_input) < 3:
                gate = 1
            else:
                print('Please enter a valid input')
                print(' ')
        print(' ')
        gate = 0
        while gate != 1:
            if self.default_gate == True:
                print('Default value for learning rate selected : 0.001')
                self.user_lr = '0.001'
            else:
                self.user_lr = input(
                    'Please enter a required value float input for learning rate (learning rate > 0) \n For default learning rate, please directly press enter without any input: ').replace(' ','')
            if self.user_lr == '':               # handling default case for learning rate
                print('Default value for learning rate selected : 0.001')
                self.user_lr = '0.001'
            if self.user_lr.replace('.', '').isdigit():
                if float(self.user_lr) > 0:
                    self.lr = float(self.user_lr)
                    gate = 1
            else:
                print('Please enter a valid input')
                print(' ')

        self.optimizer_list = {1: optim.Adam(self.net.parameters(
        ), lr=self.lr), 2: optim.SGD(self.net.parameters(), lr=self.lr)}
        self.optimizer = self.optimizer_list[int(self.optimizer_input)]
        print(' ')

    def _get_scheduler(self):

        # Method for getting scheduler

        gate = 0
        while gate != 1:
            if self.default_gate == True:
                print('By default no scheduler selected')
                self.scheduler_input = '1'
            else:
                self.scheduler_input = input(
                    'Please enter the scheduler index for the problem: Scheduler_list - [1: None, 2:StepLR, 3:MultiStepLR] \n For default option of no scheduler, please directly press enter without any input: ').replace(' ','')
            if self.scheduler_input == '':
                print('By default no scheduler selected')
                self.scheduler_input = '1'
            if self.scheduler_input.isnumeric() and int(self.scheduler_input) > 0 and int(self.scheduler_input) < 4:
                gate = 1
            else:
                print('Please enter a valid input')
                print(' ')

        if self.scheduler_input == '1':
            print(' ')
            self.scheduler = None

        elif self.scheduler_input == '2':
            print(' ')
            gate = 0
            while gate != 1:
                self.step = (
                    input('Please enter a step value int input (step > 0): ')).replace(' ','')
                if self.step.isnumeric() and int(self.step) > 0:
                    self.step = int(self.step)
                    gate = 1
                else:
                    print('Please enter a valid input')
                    print(' ')
            print(' ')
            gate = 0
            while gate != 1:
                self.gamma = (input(
                    'Please enter a Multiplying factor value float input (Multiplying factor > 0): ')).replace(' ','')
                if self.gamma.replace('.', '').isdigit():
                    if float(self.gamma) > 0:
                        self.gamma = float(self.gamma)
                        gate = 1
                else:
                    print('Please enter a valid input')
                    print(' ')

            self.scheduler = scheduler.StepLR(
                self.optimizer, step_size=self.step, gamma=self.gamma)

        elif self.scheduler_input == '3':
            print(' ')
            gate = 0
            while gate != 1:
                self.milestones_input = (
                    input('Please enter values of milestone epochs int input (Example: 2, 6, 10): ')).replace(' ','')
                self.milestones_input = self.milestones_input.split(',')
                for i in range(len(self.milestones_input)):
                    if self.milestones_input[i].isnumeric() and int(self.milestones_input[i]) > 0:
                        gate = 1
                    else:
                        gate = 0
                        break
                if gate == 0:
                    print('Please enter a valid input')
                    print(' ')

            self.milestones = [int(x)
                               for x in self.milestones_input if int(x) > 0]
            print(' ')

            gate = 0
            while gate != 1:
                self.gamma = (input(
                    'Please enter a Multiplying factor value float input (Multiplying factor > 0): ')).replace(' ','')
                if self.gamma.replace('.', '').isdigit():
                    if float(self.gamma) > 0:
                        self.gamma = float(self.gamma)
                        gate = 1
                else:
                    print('Please enter a valid input')
                    print(' ')
            self.scheduler = scheduler.MultiStepLR(
                self.optimizer, milestones=self.milestones, gamma=self.gamma)

    def _set_device(self):

        # Method for setting device type if GPU is available

        if torch.cuda.is_available():
            self.device = torch.device('cuda')
        else:
            self.device = torch.device('cpu')

    def _get_epoch(self):

        # Method for getting number of epochs for training the model

        gate = 0
        while gate != 1:
            self.numEpochs = (input(
                'Please enter the number of epochs int input to train the model (number of epochs > 0): ')).replace(' ','')
            if self.numEpochs.isnumeric() and int(self.numEpochs) > 0:
                self.numEpochs = int(self.numEpochs)
                gate = 1
            else:
                print('Please enter a valid input')
                print(' ')
        print(' ')

    def main(self):

        # Method integrating all the functions and training the model

        self.net.to(self.device)
        print('='*25)

        print('Network architecture: ')
        print(' ')
        print(self.net)         # printing model architecture
        print('='*25)

        self.get_model_summary()        # printing summaray of the model
        print(' ')
        print('='*25)

        # getting inputs and labels for training set
        xt, yt = self.model_data.get_trainset()

        # getting inputs and labels for validation set
        xv, yv = self.model_data.get_valset()

        # creating the training dataset
        self.train_dataset = Dataset(xt, yt)

        # creating the validation dataset
        self.val_dataset = Dataset(xv, yv)

        self.train_loader = torch.utils.data.DataLoader(self.train_dataset, batch_size=self.model_data.get_batchsize(
        ), shuffle=True)           # creating the training dataset dataloadet

        # creating the validation dataset dataloader
        self.dev_loader = torch.utils.data.DataLoader(
            self.val_dataset, batch_size=self.model_data.get_batchsize())

        self.train_model()          # training the model

        self.get_loss_graph()           # saving the loss graph

        if self.criterion_input == '1':

            self.get_accuracy_graph()           # saving the accuracy graph
            self.get_confusion_matrix()         # printing confusion matrix
        else:

            self.get_r2_score()             # saving r2 score graph

        self._save_model()              # saving model paramters

        print(' Call get_prediction() to make predictions on new data')
        print(' ')
        print('=== End of training ===')

    def _save_model(self):

        # Method for saving the model parameters if user wants to

        gate = 0
        while gate != 1:
            save_model = input(
                'Do you want to save the model weights? (y/n): ').replace(' ','')
            if save_model.lower() == 'y' or save_model.lower() == 'yes':
                path = 'model_parameters.pth'
                torch.save(self.net.state_dict(), path)
                gate = 1
            elif save_model.lower() == 'n' or save_model.lower() == 'no':
                gate = 1
            else:
                print('Please enter a valid input')
        print('='*25)

    def get_model_summary(self):

        # Method for getting the summary of the model
        print('Model Summary:')
        print(' ')
        print('Bidirectional: ', self.net.bidirection_input)
        print('Number of layer: ', self.net.nlayers)
        print('Criterion: ', self.criterion)
        print('Optimizer: ', self.optimizer)
        print('Scheduler: ', self.scheduler)
        print('Validation set size: ', self.valset_size)
        print('Batch size: ', self.batchsize)
        print('Initial learning rate: ', self.lr)
        print('Number of training epochs: ', self.numEpochs)
        print('Device: ', self.device)

    def train_model(self):

        # Method for training the model

        self.net.train()
        self.training_loss = []
        self.training_acc = []
        self.dev_loss = []
        self.dev_accuracy = []
        total_predictions = 0.0
        correct_predictions = 0.0

        print('Training the model...')

        for epoch in range(self.numEpochs):

            start_time = time.time()
            self.net.train()
            print('Epoch_Number: ', epoch)
            running_loss = 0.0

            for batch_idx, (data, target) in enumerate(self.train_loader):

                self.optimizer.zero_grad()
                data = data.to(self.device)
                target = target.to(self.device)

                outputs = self.net(data)

                # calculating the batch accuracy only if the loss function is Cross entropy
                if self.criterion_input == '1':

                    loss = self.criterion(outputs, target.long())
                    _, predicted = torch.max(outputs.data, 1)
                    total_predictions += target.size(0)
                    correct_predictions += (predicted == target).sum().item()

                else:

                    loss = self.criterion(outputs, target)

                running_loss += loss.item()
                loss.backward()
                self.optimizer.step()

            running_loss /= len(self.train_loader)
            self.training_loss.append(running_loss)
            print('Training Loss: ', running_loss)

            # printing the epoch accuracy only if the loss function is Cross entropy
            if self.criterion_input == '1':

                acc = (correct_predictions/total_predictions)*100.0
                self.training_acc.append(acc)
                print('Training Accuracy: ', acc, '%')

            dev_loss, dev_acc = self.validate_model()

            if self.scheduler_input != '1':

                self.scheduler.step()
                print('Current scheduler status: ', self.optimizer)

            end_time = time.time()
            print('Epoch Time: ', end_time - start_time, 's')
            print('#'*50)

            self.dev_loss.append(dev_loss)

            # saving the epoch validation accuracy only if the loss function is Cross entropy
            if self.criterion_input == '1':

                self.dev_accuracy.append(dev_acc)

    def validate_model(self):

        with torch.no_grad():
            self.net.eval()
        running_loss = 0.0
        total_predictions = 0.0
        correct_predictions = 0.0
        acc = 0
        self.actual = []
        self.predict = []

        for batch_idx, (data, target) in enumerate(self.dev_loader):

            data = data.to(self.device)
            target = target.to(self.device)
            outputs = self.net(data)

            if self.criterion_input == '1':

                loss = self.criterion(outputs, target.long())
                _, predicted = torch.max(outputs.data, 1)
                total_predictions += target.size(0)
                correct_predictions += (predicted == target).sum().item()
                self.predict.append(predicted.detach().cpu().numpy())

            else:
                loss = self.criterion(outputs, target)
                self.predict.append(outputs.detach().cpu().numpy())
            running_loss += loss.item()
            self.actual.append(target.detach().cpu().numpy())

        running_loss /= len(self.dev_loader)
        print('Validation Loss: ', running_loss)

        # calculating and printing the epoch accuracy only if the loss function is Cross entropy
        if self.criterion_input == '1':

            acc = (correct_predictions/total_predictions)*100.0
            print('Validation Accuracy: ', acc, '%')

        return running_loss, acc

    def get_loss_graph(self):

        # Method for showing and saving the loss graph in the root directory

        plt.figure(figsize=(8, 8))
        plt.plot(self.training_loss, label='Training Loss')
        plt.plot(self.dev_loss, label='Validation Loss')
        plt.legend()
        plt.title('Model Loss')
        plt.xlabel('Epochs')
        plt.ylabel('loss')
        plt.savefig('loss.png')

    def get_accuracy_graph(self):

        # Method for showing and saving the accuracy graph in the root directory

        plt.figure(figsize=(8, 8))
        plt.plot(self.training_acc, label='Training Accuracy')
        plt.plot(self.dev_accuracy, label='Validation Accuracy')
        plt.legend()
        plt.title('Model accuracy')
        plt.xlabel('Epochs')
        plt.ylabel('acc')
        plt.savefig('accuracy.png')

    def get_confusion_matrix(self):

        # Method for getting the confusion matrix for classification problem
        print('Confusion Matix: ')

        result = confusion_matrix(np.concatenate(
            np.array(self.predict)), np.concatenate(np.array(self.actual)))
        print(result)

    def get_r2_score(self):

        # Method for getting the r2 score for regression problem
        print('r2 score: ')
        result = r2_score(np.concatenate(np.array(self.predict)),
                          np.concatenate(np.array(self.actual)))
        print(result)

        plt.figure(figsize=(8, 8))
        plt.scatter(np.concatenate(np.array(self.actual)), np.concatenate(
            np.array(self.predict)), label='r2 score', s=1)
        plt.legend()
        plt.title('Model r2 score: ' + str(result))
        plt.xlabel('labels')
        plt.ylabel('predictions')
        plt.savefig('r2_score.png')

    def get_prediction(self, x_input):
        """

        Pass in an input numpy array for making prediction.
        For passing multiple inputs, make sure to keep number of examples to be the first dimension of the input.
        For example, 5 data points need to be checked and each point has 14 input size with time sequence dimension of 4, the shape of the array must be (5,4,14).
        For more information, please see documentation.

        """

        # Method to use at the time of inference

        if len(x_input.shape) == 2:             # handling the case of single

            x_input = (x_input).reshape(1, x_input.shape[0], x_input.shape[1])

        x_input = torch.from_numpy(x_input).to(self.device)

        net_output = self.net.predict(x_input)

        if self.criterion_input == '1':             # handling the case of classification problem

            _, net_output = torch.max(net_output.data, 1)

        return net_output

max_stars_repo_name: lalitjg/ManufacturingNet

max_issues_repo_name: BaratiLab/ManufacturingNet

max_forks_repo_name: BaratiLab/ManufacturingNet


text:
description = "a module housing useful functions for the heartbeat monitoring of lvalert_listenMP processes"
author = "Reed Essick (reed.essick@ligo.org)"

#-------------------------------------------------

from numpy import infty
from lvalertHeartbeat import lvalert_heartbeat

from lvalertMP.lvalert import lvalertMPutils

#-------------------------------------------------

class HeartbeatItem(lvalertMPutils.QueueItem):
    '''
    a wrapper around heartbeat functionality for lvalertMP
    '''
    name = 'heartbeat'
    description = 'a response to heartbeat queries'

    def __init__(self, t0, name, alert, logTag='iQ'):
        tasks = [HeartbeatTask(name, alert, logTag=logTag)]
        super(HeartbeatItem, self).__init__(t0, tasks, logTag=logTag)

class HeartbeatTask(lvalertMPutils.Task):
    '''
    a wrapper around heartbeat functionality for lvalertMP
    '''
    name = 'heartbeat'
    description = 'a response to heartbeat queries'

    def __init__(self, name, alert, logTag='iQ'):
        self.name = name
        self.alert = alert
        timeout = -infty ### do this immediately, always
        super(HeartbeatTask, self).__init__(timeout, logTag=logTag)

    def heartbeat(self, verbose=False):
        '''
        delegate to a helper function
        '''
        lvalert_heartbeat.respond( self.name, self.alert, verbose=verbose )

#-------------------------------------------------

def parse_heartbeat( queue, queueByGraceID, alert, t0, config, logTag='iQ' ):
    '''
    a function that parses alerts specifically for heartbeat packets.
    should be called within parse_alert as needed
    '''
    assert alert['uid']=='heartbeat', 'I only know how to parse alerts with uid="heartbeat"'
    queue.insert( HeartbeatItem( t0, config.get('general','process_type'), alert, logTag=logTag) )
    return 0

max_stars_repo_name: reedessick/lvalertHeartbeat

max_issues_repo_name: reedessick/lvalertHeartbeat

max_forks_repo_name: reedessick/lvalertHeartbeat

reason: from numpy


text:
import numpy as np
import pandas as pd
from primitive_interfaces.unsupervised_learning import UnsupervisedLearnerPrimitiveBase
from typing import NamedTuple, Sequence
import copy

def isCat_95in10(col):
    """
    hardcoded rule for identifying (integer/string) categorical column
    """
    return col.value_counts().head(10).sum() / float(col.count()) > .95

Input = pd.DataFrame
Output = pd.DataFrame

Params = NamedTuple('Params', [
    ('mapping', dict),
    ('all_columns', list),
    ('empty_columns', list),
    ('textmapping', dict)
    ])

## reference: https://github.com/scikit-learn/scikit-learn/issues/8136
class Label_encoder(object):
    def __init__(self):
        self.class_index = None

    def fit_pd(self, df, cols=[]):
        '''
        fit all columns in the df or specific list.
        generate a dict:
        {feature1:{label1:1,label2:2}, feature2:{label1:1,label2:2}...}
        '''
        if len(cols) == 0:
            cols = df.columns
        self.class_index = {}
        for f in cols:
            uf = df[f].unique()
            self.class_index[f] = {}
            index = 1
            for item in uf:
                self.class_index[f][item] = index
                index += 1

    def transform_pd(self,df,cols=[]):
        '''
        transform all columns in the df or specific list from lable to index, return an update dataframe.
        '''
        newdf = copy.deepcopy(df)
        if len(cols) == 0:
            cols = df.columns
        for f in cols:
            if f in self.class_index:
                newdf[f] = df[f].apply(lambda d: self.__update_label(f,d))
        return newdf

    def get_params(self):
        return self.class_index

    def set_params(self, textmapping):
        self.class_index = textmapping

    def __update_label(self,f,x):
        '''
        update the label to index, if not found in the dict, add and update the dict.
        '''
        try:
            return self.class_index[f][x]
        except:
            self.class_index[f][x] = max(self.class_index[f].values())+1
            return self.class_index[f][x]

class Encoder(UnsupervisedLearnerPrimitiveBase[Input, Output, Params]):
    """
    An one-hot encoder, which

    1. is given rules or values to identify categorical columns/
       - categorical_features:
            '95in10': is category if 95% of the column fall into 10 values.
       - tex2int: if non-categorical text/string columns be mapped to integers
       - n_limit: max number of distinct values to one-hot encode,
         remaining values with fewer occurence are put in [colname]_other_ column.

    2. feed in data by set_training_data, then apply fit() function to tune the encoder.

    3. produce(): input data would be encoded and return.
    """

    def __repr__(self):
        return "%s(%r)" % ('Encoder', self.__dict__)

    def __init__(self, *, categorical_features='95in10', text2int=True, n_limit=10) -> None:

        self.categorical_features = categorical_features
        self.n_limit = n_limit
        self.text2int = text2int
        #
        self.textmapping = None
        #
        self.mapping = None
        self.all_columns = []
        self.empty_columns = []

        self.training_inputs = None
        self.fitted = False

    def __column_features(self, col, n_limit):

        topn = col.dropna().unique()
        if n_limit:
            if col.nunique() > n_limit:
                topn = col.value_counts().head(n_limit).index
        return col.name, list(topn)+['other_']

    def __process(self, col, categorical_features, n_limit):

        if categorical_features == '95in10':

            # if empty column (all missing/NaN)
            if col.count() == 0:
                print('Warning:',col.name,'is an empty column.')
                print('The encoder will discard it.')
                self.empty_columns.append(col.name)
                return

	        # if dtype = integer
            elif col.dtype.kind in np.typecodes['AllInteger']+'u':
                if isCat_95in10(col):
                    return self.__column_features(col.astype(str), n_limit)

            # if dtype = category
            elif col.dtype.name == 'category':
                return self.__column_features(col, n_limit)

            # for the rest other than float
            elif col.dtype.kind not in np.typecodes['AllFloat']:
                if isCat_95in10(col):
                    return self.__column_features(col, n_limit)

            return

    def get_params(self) -> Params:
        return Params(mapping=self.mapping, all_columns=self.all_columns, empty_columns=self.empty_columns, textmapping=self.textmapping)

    def set_params(self, *, params: Params) -> None:
        self.fitted = True
        self.mapping = params.mapping
        self.all_columns = params.all_columns
        self.empty_columns = params.empty_columns
        self.textmapping = params.textmapping

    def set_training_data(self, *, inputs: Sequence[Input]):
        self.training_inputs = inputs
        self.fitted = False

    def fit(self, *, timeout:float = None, iterations: int = None) -> None:
        """
        Need training data from set_training_data first.
        The encoder would record categorical columns identified and
        the corresponding (with top n occurrence) column values to
        one-hot encode later in the produce step.
        """
        if self.fitted:
            return

        if self.training_inputs is None:
            raise ValueError('Missing training(fitting) data.')

        data_copy = self.training_inputs.copy()

        self.all_columns = set(data_copy.columns)

        if self.categorical_features == '95in10':
            idict = {}
            for column_name in data_copy:
                col = data_copy[column_name]
                p = self.__process(col, self.categorical_features, self.n_limit)
                if p:
                    idict[p[0]] = p[1]
            self.mapping = idict
            #
            if self.text2int:
                texts = data_copy.drop(self.mapping.keys(),axis=1)
                texts = texts.select_dtypes(include=[object])
                le = Label_encoder()
                le.fit_pd(texts)
                self.textmapping = le.get_params()
            #
        self.fitted = True

    def produce(self, *, inputs: Sequence[Input], timeout:float = None, iterations: int = None):
        """
        Convert and output the input data into encoded format,
        using the trained (fitted) encoder.
        Notice that a [colname]_other_ column is always kept for
        one-hot encoded columns.
        Missing(NaN) cells in a column one-hot encoded would give
        out a row of all-ZERO columns for the target column.
        """

        if isinstance(inputs, pd.DataFrame):
            data_copy = inputs.copy()
        else:
            data_copy = inputs[0].copy()

        data_enc = data_copy[list(self.mapping.keys())]
        data_else = data_copy.drop(self.mapping.keys(),axis=1)

        set_columns = set(data_copy.columns)

        if set_columns != self.all_columns:
            raise ValueError('Columns(features) fed at produce() differ from fitted data.')

        data_enc = data_copy[list(self.mapping.keys())]
        data_else = data_copy.drop(self.mapping.keys(),axis=1)

        res = []
        for column_name in data_enc:
            col = data_enc[column_name]
            col.is_copy = False

            chg_t = lambda x: str(int(x)) if type(x) is not str else x
            col[col.notnull()] = col[col.notnull()].apply(chg_t)

            chg_v = lambda x: 'other_' if (x and x not in self.mapping[col.name]) else x
            col = col.apply(chg_v)

            encoded = pd.get_dummies(col, dummy_na=True, prefix=col.name)

            missed = (["%s_%s"%(col.name,str(i)) for i in self.mapping[col.name] if
                    "%s_%s"%(col.name,str(i)) not in list(encoded.columns)])

            for m in missed:
                encoded[m] = 0

            res.append(encoded)

        data_else.drop(self.empty_columns, axis=1, inplace=True)
        if self.text2int:
            texts = data_else.select_dtypes([object])
            le = Label_encoder()
            le.set_params(self.textmapping)
            data_else[texts.columns] = le.transform_pd(texts)
            #for column_name in data_else:
            #    if data_else[column_name].dtype.kind not in np.typecodes['AllInteger']+'uf':
            #        data_else[column_name] = text2int(data_else[column_name])
        res.append(data_else)
        result = pd.concat(res, axis=1)

        return result

# example
if __name__ == '__main__':
    enc = Encoder()
    df = pd.DataFrame({'A':[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],'B':[1,2,3,4,5,1,2,3,4,5,1,2,3,4,5]})
    train_x = df
    enc.set_training_data(inputs=train_x)
    enc.fit()
    print(enc.produce(inputs=df))

    #save model for later use
    model = enc.get_params()

    enc2 = Encoder()
    df2 = pd.DataFrame({'A':[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],'B':[2.0,7,3,1,6,1,2,4,2,5,1,2,4,4,3]})
    enc2.set_params(params=model)
    print(enc2.produce(inputs=df2))


text:
# Tests for explanation dashboard
import numpy as np
import pytest
from common_utils import (create_cancer_data, create_cancer_data_booleans,
                          create_lightgbm_classifier,
                          create_sklearn_svm_classifier)
from constants import owner_email_tools_and_ux
from datasets import retrieve_dataset
from interpret import show
from interpret_community.common.constants import ModelTask
from interpret_community.mimic.models.lightgbm_model import \
    LGBMExplainableModel
from interpret_community.widget import \
    ExplanationDashboard as OldExplanationDashboard
from raiwidgets import ExplanationDashboard
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler

@pytest.mark.owner(email=owner_email_tools_and_ux)
@pytest.mark.usefixtures('clean_dir')
class TestExplanationDashboard:
    def test_raw_timestamp_explanation(self, mimic_explainer):
        df = retrieve_dataset('insurance_claims.csv', na_values='?',
                              parse_dates=['policy_bind_date', 'incident_date'])
        label = 'fraud_reported'
        df_y = df[label]
        df_X = df.drop(columns=label)
        x_train, x_test, y_train, y_test = train_test_split(df_X, df_y, test_size=0.2, random_state=7)
        str_cols = df_X.select_dtypes(exclude=[np.number, np.datetime64]).columns.tolist()
        dt_cols = df_X.select_dtypes(include=[np.datetime64]).columns.tolist()
        numeric_cols = df_X.select_dtypes(include=[np.number]).columns.tolist()
        transforms_list = []
        for str_col in str_cols:
            transforms_list.append((str_col, Pipeline(steps=[
                ('imputer', SimpleImputer(strategy='most_frequent')),
                ('ohe', OneHotEncoder(sparse=False))
                ]), [str_col]
            ))
        for numeric_col in numeric_cols:
            transforms_list.append((numeric_col, Pipeline(steps=[
                ('imputer', SimpleImputer(strategy='mean')),
                ('scaler', StandardScaler())
                ]), [numeric_col]
            ))
        for dt_col in dt_cols:
            transforms_list.append((dt_col, Pipeline(steps=[
                ('scaler', StandardScaler())
                ]), [dt_col]
            ))
        transformations = ColumnTransformer(transforms_list)
        x_train_transformed = transformations.fit_transform(x_train)
        model = create_lightgbm_classifier(x_train_transformed, y_train)
        model_task = ModelTask.Classification
        features = df_X.columns.tolist()
        explainer = mimic_explainer(model, x_train, LGBMExplainableModel, transformations=transformations,
                                    features=features, model_task=model_task)
        explanation = explainer.explain_global(x_train)
        dashboard_pipeline = Pipeline(steps=[('preprocess', transformations), ('model', model)])
        ExplanationDashboard(explanation, dashboard_pipeline, dataset=x_train, true_y=y_train)

    def test_local_explanation(self, mimic_explainer):
        # Validate visualizing ExplanationDashboard with a local explanation
        x_train, x_test, y_train, y_test, feature_names, target_names = create_cancer_data()
        # Fit an SVM model
        model = create_sklearn_svm_classifier(x_train, y_train)
        explainer = mimic_explainer(model, x_train, LGBMExplainableModel,
                                    features=feature_names, classes=target_names)
        explanation = explainer.explain_local(x_test)
        ExplanationDashboard(explanation, model, dataset=x_test, true_y=y_test)

    def test_boolean_labels(self, mimic_explainer):
        # Validate visualizing ExplanationDashboard with a local explanation
        x_train, x_test, y_train, y_test, feature_names, target_names = create_cancer_data_booleans()
        # Fit an SVM model
        model = create_sklearn_svm_classifier(x_train, y_train)
        explainer = mimic_explainer(model, x_train, LGBMExplainableModel,
                                    features=feature_names, classes=target_names)
        explanation = explainer.explain_local(x_test)
        ExplanationDashboard(explanation, model, dataset=x_test, true_y=y_test)

    def test_old_explanation_dashboard(self, mimic_explainer):
        # Validate old explanation dashboard namespace works but only prints a warning
        x_train, x_test, y_train, y_test, feature_names, target_names = create_cancer_data()
        # Fit an SVM model
        model = create_sklearn_svm_classifier(x_train, y_train)
        explainer = mimic_explainer(model, x_train, LGBMExplainableModel,
                                    features=feature_names, classes=target_names)
        explanation = explainer.explain_local(x_test)
        err = ("ExplanationDashboard in interpret-community package is deprecated and removed."
               "Please use the ExplanationDashboard from raiwidgets package instead.")
        with pytest.warns(DeprecationWarning, match=err):
            OldExplanationDashboard(explanation, model, dataset=x_test, true_y=y_test)

    def test_interpret_dashboard(self, mimic_explainer):
        # Validate our explanation works with the interpret dashboard
        x_train, x_test, y_train, y_test, feature_names, target_names = create_cancer_data()
        # Fit an SVM model
        model = create_sklearn_svm_classifier(x_train, y_train)
        explainer = mimic_explainer(model, x_train, LGBMExplainableModel,
                                    features=feature_names, classes=target_names)
        explanation = explainer.explain_global(x_test)
        show(explanation)