text:
import numpy as np
import pandas as pd
import pytest

from typeddfs import TypedDf, TypedDfs, UntypedDf
from typeddfs.abs_dfs import AbsDf
from typeddfs.base_dfs import BaseDf

from . import sample_data

class TestCore:
    def test_wrap(self):
        df = pd.DataFrame({})
        df2 = TypedDfs.wrap(df)
        assert not isinstance(df, AbsDf)
        assert isinstance(df2, BaseDf)

    def test_wrap_multilayer(self):
        # not fully supported yet, but let's check that it's reasonable
        rows = ["yes", "no", "maybe"]
        cols = [("animal", "cat"), ("animal", "armadillo", ("person", "matt"))]
        cols = pd.MultiIndex.from_tuples(cols)
        df = pd.DataFrame(np.zeros((3, 2)), rows, cols)
        df = TypedDfs.wrap(df)
        assert df.column_names() == [
            ("animal", "cat", np.nan),
            ("animal", "armadillo", ("person", "matt")),
        ]

    def test_empty_simple(self):
        new = TypedDfs.untyped("a class")
        df = new.convert(pd.DataFrame())
        assert list(df.columns) == []

    def test_no_name_simple(self):
        with pytest.raises(TypeError):
            # noinspection PyTypeChecker
            TypedDfs.untyped(None)

    def test_no_name_fancy(self):
        with pytest.raises(TypeError):
            # noinspection PyTypeChecker
            TypedDfs.typed(None).build()

    def test_simple(self):
        new = TypedDfs.untyped("a class", doc="A doc")
        assert new.__name__ == "a class"
        assert new.__doc__ == "A doc"
        df = new.convert(pd.DataFrame(sample_data()))
        assert isinstance(df, UntypedDf)
        assert df.__class__.__name__ == "a class"

    def test_fancy(self):
        new = TypedDfs.typed("a class", doc="A doc").build()
        assert new.__name__ == "a class"
        assert new.__doc__ == "A doc"
        df = new.convert(pd.DataFrame(sample_data()))
        assert isinstance(df, TypedDf)
        assert df.__class__.__name__ == "a class"

    def test_fancy_with_index(self):
        new = TypedDfs.typed("a class").require("abc", index=True).build()
        df = new.convert(pd.DataFrame(sample_data()))
        assert df.index_names() == ["abc"]
        assert df.column_names() == ["123", "xyz"]
        assert isinstance(df, TypedDf)
        assert df.__class__.__name__ == "a class"

    def test_fancy_with_col(self):
        new = TypedDfs.typed("a class").require("abc", index=False).build()
        df = new.convert(pd.DataFrame(sample_data()))
        assert df.index_names() == []
        assert df.column_names() == ["abc", "123", "xyz"]
        assert isinstance(df, TypedDf)
        assert df.__class__.__name__ == "a class"

    def test_fancy_with_multiindex(self):
        new = TypedDfs.typed("a class").require("abc", "xyz", index=True).build()
        df = new.convert(pd.DataFrame(sample_data()))
        assert df.index_names() == ["abc", "xyz"]
        assert df.column_names() == ["123"]
        assert isinstance(df, TypedDf)
        assert df.__class__.__name__ == "a class"

    def test_fancy_with_all_index(self):
        new = TypedDfs.typed("a class").require("abc", "xyz", "123", index=True).build()
        df = new.convert(pd.DataFrame(sample_data()))
        assert df.index_names() == ["abc", "xyz", "123"]
        assert df.column_names() == []
        assert isinstance(df, TypedDf)
        assert df.__class__.__name__ == "a class"

    def test_fancy_with_no_index(self):
        new = TypedDfs.typed("a class").require("abc", "123", "xyz", index=False).build()
        df = new.convert(pd.DataFrame(sample_data()))
        assert df.index_names() == []
        assert df.column_names() == ["abc", "123", "xyz"]
        assert isinstance(df, TypedDf)
        assert df.__class__.__name__ == "a class"

    def test_extra_col(self):
        new = TypedDfs.typed("a class").require("abc", index=True).strict().build()
        df = pd.DataFrame(sample_data())
        with pytest.raises(TypedDfs.UnexpectedColumnError):
            new.convert(df)

    def test_extra_index(self):
        new = TypedDfs.typed("a class").require("xyz", index=False).strict().build()
        df = pd.DataFrame(sample_data())
        with pytest.raises(TypedDfs.UnexpectedColumnError):
            new.convert(df)

    def test_missing(self):
        new = TypedDfs.typed("a class").require("qqq", index=False).strict().build()
        df = pd.DataFrame(sample_data())
        with pytest.raises(TypedDfs.MissingColumnError):
            new.convert(df)

if __name__ == "__main__":
    pytest.main()

max_stars_repo_name: dmyersturnbull/typed-dfs

max_issues_repo_name: kokellab/typed-dfs

max_forks_repo_name: kokellab/typed-dfs


text:
import numpy as np

# remarks : official openpose includes more parts
JOINT_NAMES = ['Nose', 'Neck',
                'Shoulder right', 'Elbow right', 'Hand right',
                'Shoulder left', 'Elbow left', 'Hand left',
                'Hip center',
                'Hip right', 'Knee right', 'Foot right',
                'Hip left', 'Knee left',  'Foot left',
                'Eye right', 'Eye left', 'Ear right', 'Ear left']
    # 'toe1 left',
    # 'toe2 left',
    # 'heel left',
    # 'toe1 right',
    # 'toe2 right',
    # 'heel right',

CONNECTIONS = [[1,8],[9,10],[10,11],[8,9],[8,12],[12,13],[13,14],[1,2],
                        [2,3],[3,4],[1,5],[5,6],[6,7],[1,0],
                        [2,17],[5,18],[0,15],[0,16],[15,17],[16,18],[17,18]]

ext: py

reason: import numpy


text:
import pandas as pd
import numpy as np
import re
import sys
import pyth
from bs4 import BeautifulSoup
pd.set_option('display.max_rows', 1000)
pd.set_option('display.width', 500)

r = pd.read_csv("/home/dooku/Downloads/QueryResults.csv")

answers = r["Body"]

def extract_code(answer, lim=1):
    html = BeautifulSoup(answer, 'html.parser')
    snippets = [n.get_text()[:-1] for n in html.find_all('pre') if "  " not in n.get_text() and len(n.get_text()) and n.get_text().count("\n")<3]
    return snippets[:lim] if lim-1 else snippets[0] if snippets else ""

def process_answer(answer, extraction = True):
    code = extract_code(answer)
    code = re.sub("\$.*\$", "", code)

    try:
        data = pyth.general_parse(code)[1]
    except Exception as e:
        print(code)
        print(e)
        data = {i: [0]*3 for i in "bdGHkNTYZ"}

    data = {i: np.array(v + [v[0]+v[2]]) for i, v in data.items()}

    data["Code"] = code

    return pd.Series(data)

answers = answers.apply(process_answer)
aggr = answers.apply(lambda x: x.sum())
aggr.Code = "Total: "
answers = answers.append(aggr, ignore_index = True)
print(answers)


text:
import os, sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import numpy as np

from nn.module import Linear,TanH,Sigmoide
from nn.loss import MSELoss
from tools.basic import *
from tools.nntools import Sequentiel,Optim
import matplotlib.pyplot as plt

size = 500

datax, datay = gen_arti(centerx=1, centery=1, sigma=0.5, nbex=size, data_type=2, epsilon=0.1)
testx, testy = gen_arti(centerx=1, centery=1, sigma=0.5, nbex=size, data_type=2, epsilon=0.1)

datay = np.where(datay==-1,0,1).reshape((-1,1))
testy = np.where(testy==-1,0,1).reshape((-1,1))

n = datax.shape[1]
hidden = 80
hidden2 = 60
d = 1

iteration = 500
gradient_step = 1e-3
batchsize=10

def label_func(x):
    return np.where(x >= 0.5,1, 0)

loss_mse = MSELoss()
lin_layer = Linear(n, hidden,type=1)
lin_layer2 = Linear(hidden, hidden2,type=1)
lin_layer3 = Linear(hidden2, d,type=1)
lin_layer4 = Linear(40, 30,type=1)
lin_layer5 = Linear(30, d,type=1)
act_sig = Sigmoide()
act_tan = TanH()

net = Sequentiel([lin_layer,act_tan,lin_layer2,act_tan,lin_layer3,act_sig],labels=label_func)

opt = Optim(net,loss_mse,eps=gradient_step)
mean, std = opt.SGD(datax,datay,batchsize,iteration)
plt.figure()
plt.plot(mean)
plt.plot(std)
plt.legend(('Mean', 'std'))
plt.show()

acc = opt.score(datax,datay)
print("accuracy : ",acc)
plt.figure()
plot_frontiere(datax, opt._net.predict, step=100)
plot_data(datax, datay.reshape(-1))
plt.title("accuracy = "+str(acc))
plt.show()

max_stars_repo_name: hider-toula/API-CNN

max_issues_repo_name: hider-toula/API-CNN

max_forks_repo_name: hider-toula/API-CNN


text:
import numpy as np

from pandas import DatetimeIndex, Index, MultiIndex, Series, Timestamp

from .pandas_vb_common import tm

def no_change(arr):
    return arr

def list_of_str(arr):
    return list(arr.astype(str))

def gen_of_str(arr):
    return (x for x in arr.astype(str))

def arr_dict(arr):
    return dict(zip(range(len(arr)), arr))

def list_of_tuples(arr):
    return [(i, -i) for i in arr]

def gen_of_tuples(arr):
    return ((i, -i) for i in arr)

def list_of_lists(arr):
    return [[i, -i] for i in arr]

def list_of_tuples_with_none(arr):
    return [(i, -i) for i in arr][:-1] + [None]

def list_of_lists_with_none(arr):
    return [[i, -i] for i in arr][:-1] + [None]

class SeriesConstructors:

    param_names = ["data_fmt", "with_index", "dtype"]
    params = [
        [
            no_change,
            list,
            list_of_str,
            gen_of_str,
            arr_dict,
            list_of_tuples,
            gen_of_tuples,
            list_of_lists,
            list_of_tuples_with_none,
            list_of_lists_with_none,
        ],
        [False, True],
        ["float", "int"],
    ]

    # Generators get exhausted on use, so run setup before every call
    number = 1
    repeat = (3, 250, 10)

    def setup(self, data_fmt, with_index, dtype):
        if data_fmt in (gen_of_str, gen_of_tuples) and with_index:
            raise NotImplementedError(
                "Series constructors do not support using generators with indexes"
            )
        N = 10 ** 4
        if dtype == "float":
            arr = np.random.randn(N)
        else:
            arr = np.arange(N)
        self.data = data_fmt(arr)
        self.index = np.arange(N) if with_index else None

    def time_series_constructor(self, data_fmt, with_index, dtype):
        Series(self.data, index=self.index)

class SeriesDtypesConstructors:
    def setup(self):
        N = 10 ** 4
        self.arr = np.random.randn(N)
        self.arr_str = np.array(["foo", "bar", "baz"], dtype=object)
        self.s = Series(
            [Timestamp("20110101"), Timestamp("20120101"), Timestamp("20130101")]
            * N
            * 10
        )

    def time_index_from_array_string(self):
        Index(self.arr_str)

    def time_index_from_array_floats(self):
        Index(self.arr)

    def time_dtindex_from_series(self):
        DatetimeIndex(self.s)

    def time_dtindex_from_index_with_series(self):
        Index(self.s)

class MultiIndexConstructor:
    def setup(self):
        N = 10 ** 4
        self.iterables = [tm.makeStringIndex(N), range(20)]

    def time_multiindex_from_iterables(self):
        MultiIndex.from_product(self.iterables)

from .pandas_vb_common import setup  # noqa: F401 isort:skip


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 28 08:38:15 2017

@author: jorgemauricio
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from scipy.interpolate import griddata as gd
from time import gmtime, strftime
import time

#%% fecha del pronostico
fechaPronostico = strftime("%Y-%m-%d")

#%% read csv
data1 = pd.read_table('data/d1.txt', sep=',')
data2 = pd.read_table('data/d2.txt', sep=',')
data3 = pd.read_table('data/d3.txt', sep=',')
data4 = pd.read_table('data/d4.txt', sep=',')
data5 = pd.read_table('data/d5.txt', sep=',')

#%% make one dataFrame
data = data1.filter(items=['Long', 'Lat','Rain'])
data['Rain1'] = data1['Rain']
data['Rain2'] = data2['Rain']
data['Rain3'] = data3['Rain']
data['Rain4'] = data4['Rain']
data['Rain5'] = data5['Rain']
data['Acum'] = data['Rain'] + data['Rain2'] + data['Rain3'] + data['Rain4'] + data['Rain5']

#%% get values from Ags
data = data.loc[data['Lat'] > 21.0]
data = data.loc[data['Lat'] < 24.0]
data = data.loc[data['Long'] > -104.0]
data = data.loc[data['Long'] < -100.0]

#%% get x and y values
lons = np.array(data['Long'])
lats = np.array(data['Lat'])

#%% generate arrayFechas
# Generate Days
arrayFechas = []
tanio, tmes, tdia = fechaPronostico.split('-')
anio = int(tanio)
mes = int(tmes)
dia = int(tdia)

for i in range(0,5,1):
	if i == 0:
		newDiaString = '{}'.format(dia)
		if len(newDiaString) == 1:
			newDiaString = '0' + newDiaString
		newMesString = '{}'.format(mes)
		if len(newMesString) == 1:
			newMesString = '0' + newMesString
		fecha = '{}'.format(anio)+"-"+newMesString+"-"+newDiaString
		arrayFechas.append(fecha)
	if i > 0:
		dia = dia + 1
		if mes == 2 and anio % 4 == 0:
			diaEnElMes = 29
		elif mes == 2 and anio % 4 != 0:
			diaEnElMes = 28
		elif mes == 1 or mes == 3 or mes == 5 or mes == 7 or mes == 8 or mes == 10 or mes == 12:
			diaEnElMes = 31
		elif mes == 4 or mes == 6 or mes == 9 or mes == 11:
			diaEnElMes = 30
		if dia > diaEnElMes:
			mes = mes + 1
			dia = 1
		if mes > 12:
			anio = anio + 1
			mes = 1
		newDiaString = '{}'.format(dia)
		if len(newDiaString) == 1:
			newDiaString = '0' + newDiaString
		newMesString = '{}'.format(mes)
		if len(newMesString) == 1:
			newMesString = '0' + newMesString
		fecha = '{}'.format(anio)+"-"+newMesString+"-"+newDiaString
		arrayFechas.append(fecha)

#%% loop diarios
counterFecha = 0
for i in range(1,6,1):
	#%% set up plot
	plt.clf()
	#fig = plt.figure(figsize=(48,24))
	m = Basemap(projection='mill',llcrnrlat=21.3,urcrnrlat=23,llcrnrlon=-103.5,urcrnrlon=-101,resolution='h')

	#%% generate lats, lons
	x, y = m(lons,lats)

	#%% number of cols and rows
	numcols = len(x)
	numrows = len(y)

	#%% generate xi, yi
	xi = np.linspace(x.min(), x.max(), numcols)
	yi = np.linspace(y.min(), y.max(), numrows)

	#%% generate meshgrid
	xi, yi = np.meshgrid(xi,yi)

	#%% genate zi
	tempTitleColumn = "Rain{}".format(i)
	z = np.array(data[tempTitleColumn])
	zi = gd((x,y), z, (xi,yi), method='cubic')

	#%% generate clevs
	def generateClevs(minV, maxV):
		arrayValues = []
		step = (maxV - minV) / 10
		for i in range(10):
			rangeOfValue = int(step * i)
			arrayValues.append(rangeOfValue)
		return arrayValues

	clevs = generateClevs(z.min(), z.max())
	#%% contour plot
	cs = m.contourf(xi,yi,zi, clevs, zorder=4, alpha=0.5, cmap='Spectral_r')
	#%% draw map details
	m.drawcoastlines()
	#m.drawstates(linewidth=0.7)
	m.drawcountries()
	#%% read municipios shape file
	m.readshapefile('shapes/Municipios', 'Municipios')
	#m.readshapefile('shapes/Estados', 'Estados')
	#m.drawmapscale(22, -103, 23, -102, 100, units='km', fontsize=14, yoffset=None, barstyle='fancy', labelstyle='simple', fillcolor1='w', fillcolor2='#000000',fontcolor='#000000', zorder=5)

	#%% colorbar
	cbar = m.colorbar(cs, location='right', pad="5%")
	cbar.set_label('mm')
	tempMapTitle = "Precipitación acumulada en 24h (mm)\nPronóstico válido para: {}".format(arrayFechas[counterFecha])
	plt.title(tempMapTitle)
	tempFileName = "maps/{}.png".format(arrayFechas[counterFecha])
	plt.annotate('INIFAP (WRF -EMS)', xy=(-102,22), xycoords='data', xytext=(-102,21), color='g')
	plt.savefig(tempFileName, dpi=300, transparent=True)
	counterFecha += 1
	print('****** Genereate: {}'.format(tempFileName))

#%% generate Acum
#%% set up plot
plt.clf()
#fig = plt.figure(figsize=(48,24))
m = Basemap(projection='mill',llcrnrlat=21.3,urcrnrlat=23,llcrnrlon=-103.5,urcrnrlon=-101,resolution='h')

#%% generate lats, lons
x, y = m(lons,lats)

#%% number of cols and rows
numcols = len(x)
numrows = len(y)

#%% generate xi, yi
xi = np.linspace(x.min(), x.max(), numcols)
yi = np.linspace(y.min(), y.max(), numrows)

#%% generate meshgrid
xi, yi = np.meshgrid(xi,yi)

#%% genate zi
z = np.array(data['Acum'])
zi = gd((x,y), z, (xi,yi), method='cubic')

#%% generate clevs
def generateClevs(minV, maxV):
	arrayValues = []
	step = (maxV - minV) / 10
	for i in range(10):
		rangeOfValue = int(step * i)
		arrayValues.append(rangeOfValue)
	return arrayValues

clevs = generateClevs(z.min(), z.max())
#%% contour plot
cs = m.contourf(xi,yi,zi, clevs, zorder=4, alpha=0.5, cmap='Spectral_r')
#%% draw map details
m.drawcoastlines()
#m.drawstates(linewidth=0.7)
m.drawcountries()
#%% read municipios shape file
m.readshapefile('shapes/Municipios', 'Municipios')
#m.readshapefile('shapes/Estados', 'Estados')
#m.drawmapscale(22, -103, 23, -102, 100, units='km', fontsize=14, yoffset=None, barstyle='fancy', labelstyle='simple', fillcolor1='w', fillcolor2='#000000',fontcolor='#000000', zorder=5)

#%% colorbar
cbar = m.colorbar(cs, location='right', pad="5%")
cbar.set_label('mm')
tempMapTitle = "Precipitación acumulada en 24h (mm)\nPronóstico válido para {} al {}".format(arrayFechas[0],arrayFechas[-1])
plt.title(tempMapTitle)
plt.annotate('INIFAP (WRF -EMS)', xy=(-102,22), xycoords='data', xytext=(-102,21), color='g')
plt.savefig("maps/acum.png", dpi=300, transparent=True)
print('****** Genereate: Acum')

max_stars_repo_name: jorgemauricio/rainMap

max_issues_repo_name: jorgemauricio/rainMap

max_forks_repo_name: jorgemauricio/rainMap

reason: import numpy,from scipy


text:
import shap
import sklearn
import xgboost

from django.core.management.base import BaseCommand
from sklearn.externals import joblib

from src.core.core import get_encoded_logs
from src.encoding.common import retrieve_proper_encoder
from src.jobs.models import Job
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import numpy as np
import xgboost

class Command(BaseCommand):
    help = 'tries to deliver an explanation of a random prediction of the trained model'

    def handle(self, *args, **kwargs):

        TARGET_MODEL = 68
        job = Job.objects.filter(pk=TARGET_MODEL)[0]
        model = joblib.load(job.predictive_model.model_path)
        model = model[0]
        training_df, test_df = get_encoded_logs(job)

        EXPLANATION_TARGET = 2_3300
        FEATURE_TARGET = 1
        shap.initjs()

        explainer = shap.TreeExplainer(model)
        training_df = training_df.drop(['trace_id','label'], 1)

        shap_values = explainer.shap_values(training_df)

        encoder = retrieve_proper_encoder(job)
        encoder.decode(training_df, job.encoding)

        shap.force_plot(explainer.expected_value, shap_values[EXPLANATION_TARGET, :],training_df.iloc[EXPLANATION_TARGET, :],
                                       show=False, matplotlib=True).savefig('shap_plot_train_1_3.png')

        # encoder.encode(training_df, job.encoding)
        # plt.savefig('dependence_plot.png')
        # a = shap.force_plot(explainer.expected_value[1], shap_values[0], training_df)
        # shap.save_html('shap_plot_train_1_3.html', a)
        # shap.summary_plot(shap_values, training_df.iloc[111], plot_type="bar")
        # plt.savefig('summary_plot.png')
        # show plot
        # shap.force_plot(explainer.expected_value, shap_values[0], training_df)
        # shap.embedding_plot(FEATURE_TARGET, shap_values=shap_values[FEATURE_TARGET])
        # plt.savefig('embedding_plot.png')

        # X_test = training_df.drop(['trace_id', 'label'], 1)
        #
        # X_output = X_test.copy()
        #
        # X_output.loc[:,'predict'] = np.round(model.predict(X_output),2)
        #
        # S = X_output.iloc[1]
        # S


text:
import numpy as np

data_for_tests = dict()

data_for_tests['transition_matrices'] = np.array([[[0.4, 0.6],
                                                   [0.7, 0.3]],
                                                  [[0.1, 0.9],
                                                   [0.5, 0.5]]])

data_for_tests['lambdas'] = np.array([0.4, 0.6])

data_for_tests['indexes'] = [(0, 0, 0),
                             (0, 0, 1),
                             (0, 1, 0),
                             (0, 1, 1),
                             (1, 0, 0),
                             (1, 0, 1),
                             (1, 1, 0),
                             (1, 1, 1)]

data_for_tests['expected_p_array'] = np.array([[8 / 11, 3 / 11],
                                               [12 / 39, 27 / 39],
                                               [8 / 23, 15 / 23],
                                               [12 / 27, 15 / 27],
                                               [14 / 17, 3 / 17],
                                               [2 / 11, 9 / 11],
                                               [14 / 29, 15 / 29],
                                               [2 / 7, 5 / 7]])

expected_p_direct_array = np.array(
    [[[data_for_tests['expected_p_array'][0, 0] + data_for_tests['expected_p_array'][2, 0],
       data_for_tests['expected_p_array'][1, 0] + data_for_tests['expected_p_array'][3, 0]],
      [data_for_tests['expected_p_array'][4, 0] + data_for_tests['expected_p_array'][6, 0],
       data_for_tests['expected_p_array'][5, 0] + data_for_tests['expected_p_array'][7, 0]]],
     [[data_for_tests['expected_p_array'][0, 1] + data_for_tests['expected_p_array'][4, 1],
       data_for_tests['expected_p_array'][1, 1] + data_for_tests['expected_p_array'][5, 1]],
      [data_for_tests['expected_p_array'][2, 1] + data_for_tests['expected_p_array'][6, 1],
       data_for_tests['expected_p_array'][3, 1] + data_for_tests['expected_p_array'][7, 1]]]
     ])
data_for_tests['expected_p_direct_array'] = expected_p_direct_array / expected_p_direct_array.sum(0)

expected_p_direct_array2 = expected_p_direct_array.sum(2)
data_for_tests['expected_p_direct_array2'] = expected_p_direct_array2 / expected_p_direct_array2.sum(1).reshape(-1, 1)

data_for_tests['n_passes'] = np.array([100, 100, 400, 400, 100, 100, 400, 400])

data_for_tests['n_passes_direct'] = np.array([[[500, 500],
                                               [500, 500]],
                                              [[200, 200],
                                               [800, 800]]])

denominator = 1 / (data_for_tests['n_passes'].sum() - 2)
lambda1 = denominator * np.dot(data_for_tests['expected_p_array'][:, 0], data_for_tests['n_passes'])
lambda2 = denominator * np.dot(data_for_tests['expected_p_array'][:, 1], data_for_tests['n_passes'])

lambda_sum = lambda1 + lambda2

lambda1 = lambda1 / lambda_sum
lambda2 = lambda2 / lambda_sum

data_for_tests['expected_lambdas'] = np.array([lambda1, lambda2])

expected_transition_matrices = data_for_tests['n_passes_direct'] * data_for_tests['expected_p_direct_array']
expected_transition_matrices = expected_transition_matrices / expected_transition_matrices.sum(2).reshape(2, -1, 1)

data_for_tests['expected_transition_matrices'] = expected_transition_matrices

log_likelihood1 = 0

log_likelihood1 = log_likelihood1 + data_for_tests['n_passes'][0] * np.log(
    data_for_tests['lambdas'][0] * data_for_tests['transition_matrices'][0, 0, 0] +
    data_for_tests['lambdas'][1] * data_for_tests['transition_matrices'][1, 0, 0])

log_likelihood1 = log_likelihood1 + data_for_tests['n_passes'][1] * np.log(
    data_for_tests['lambdas'][0] * data_for_tests['transition_matrices'][0, 0, 1] +
    data_for_tests['lambdas'][1] * data_for_tests['transition_matrices'][1, 0, 1])

log_likelihood1 = log_likelihood1 + data_for_tests['n_passes'][2] * np.log(
    data_for_tests['lambdas'][0] * data_for_tests['transition_matrices'][0, 0, 0] +
    data_for_tests['lambdas'][1] * data_for_tests['transition_matrices'][1, 1, 0])

log_likelihood1 = log_likelihood1 + data_for_tests['n_passes'][3] * np.log(
    data_for_tests['lambdas'][0] * data_for_tests['transition_matrices'][0, 0, 1] +
    data_for_tests['lambdas'][1] * data_for_tests['transition_matrices'][1, 1, 1])

log_likelihood1 = log_likelihood1 + data_for_tests['n_passes'][4] * np.log(
    data_for_tests['lambdas'][0] * data_for_tests['transition_matrices'][0, 1, 0] +
    data_for_tests['lambdas'][1] * data_for_tests['transition_matrices'][1, 0, 0])

log_likelihood1 = log_likelihood1 + data_for_tests['n_passes'][5] * np.log(
    data_for_tests['lambdas'][0] * data_for_tests['transition_matrices'][0, 1, 1] +
    data_for_tests['lambdas'][1] * data_for_tests['transition_matrices'][1, 0, 1])

log_likelihood1 = log_likelihood1 + data_for_tests['n_passes'][6] * np.log(
    data_for_tests['lambdas'][0] * data_for_tests['transition_matrices'][0, 1, 0] +
    data_for_tests['lambdas'][1] * data_for_tests['transition_matrices'][1, 1, 0])

log_likelihood1 = log_likelihood1 + data_for_tests['n_passes'][7] * np.log(
    data_for_tests['lambdas'][0] * data_for_tests['transition_matrices'][0, 1, 1] +
    data_for_tests['lambdas'][1] * data_for_tests['transition_matrices'][1, 1, 1])

data_for_tests['log_likelihood1'] = log_likelihood1

log_likelihood2 = 0

log_likelihood2 = log_likelihood2 + data_for_tests['n_passes'][0] * np.log(
    data_for_tests['expected_lambdas'][0] * data_for_tests['expected_transition_matrices'][0, 0, 0] +
    data_for_tests['expected_lambdas'][1] * data_for_tests['expected_transition_matrices'][1, 0, 0])

log_likelihood2 = log_likelihood2 + data_for_tests['n_passes'][1] * np.log(
    data_for_tests['expected_lambdas'][0] * data_for_tests['expected_transition_matrices'][0, 0, 1] +
    data_for_tests['expected_lambdas'][1] * data_for_tests['expected_transition_matrices'][1, 0, 1])

log_likelihood2 = log_likelihood2 + data_for_tests['n_passes'][2] * np.log(
    data_for_tests['expected_lambdas'][0] * data_for_tests['expected_transition_matrices'][0, 0, 0] +
    data_for_tests['expected_lambdas'][1] * data_for_tests['expected_transition_matrices'][1, 1, 0])

log_likelihood2 = log_likelihood2 + data_for_tests['n_passes'][3] * np.log(
    data_for_tests['expected_lambdas'][0] * data_for_tests['expected_transition_matrices'][0, 0, 1] +
    data_for_tests['expected_lambdas'][1] * data_for_tests['expected_transition_matrices'][1, 1, 1])

log_likelihood2 = log_likelihood2 + data_for_tests['n_passes'][4] * np.log(
    data_for_tests['expected_lambdas'][0] * data_for_tests['expected_transition_matrices'][0, 1, 0] +
    data_for_tests['expected_lambdas'][1] * data_for_tests['expected_transition_matrices'][1, 0, 0])

log_likelihood2 = log_likelihood2 + data_for_tests['n_passes'][5] * np.log(
    data_for_tests['expected_lambdas'][0] * data_for_tests['expected_transition_matrices'][0, 1, 1] +
    data_for_tests['expected_lambdas'][1] * data_for_tests['expected_transition_matrices'][1, 0, 1])

log_likelihood2 = log_likelihood2 + data_for_tests['n_passes'][6] * np.log(
    data_for_tests['expected_lambdas'][0] * data_for_tests['expected_transition_matrices'][0, 1, 0] +
    data_for_tests['expected_lambdas'][1] * data_for_tests['expected_transition_matrices'][1, 1, 0])

log_likelihood2 = log_likelihood2 + data_for_tests['n_passes'][7] * np.log(
    data_for_tests['expected_lambdas'][0] * data_for_tests['expected_transition_matrices'][0, 1, 1] +
    data_for_tests['expected_lambdas'][1] * data_for_tests['expected_transition_matrices'][1, 1, 1])

data_for_tests['log_likelihood2'] = log_likelihood2

max_stars_repo_name: PiotrekGa/mtd-learn

max_issues_repo_name: PiotrekGa/mtd-learn

max_forks_repo_name: PiotrekGa/mtd-learn


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import os.path as osp
import numpy as np
# `pip install easydict` if you don't have it
from easydict import EasyDict as edict

__C = edict()
# Consumers can get config by:
#   from fast_rcnn_config import cfg
cfg = __C

#
# Training options
#
__C.TRAIN = edict()

# Initial learning rate
__C.TRAIN.LEARNING_RATE = 0.001

# Momentum
__C.TRAIN.MOMENTUM = 0.9

# Weight decay, for regularization
__C.TRAIN.WEIGHT_DECAY = 0.0005

# Factor for reducing the learning rate
__C.TRAIN.GAMMA = 0.1

# Step size for reducing the learning rate, currently only support one step
__C.TRAIN.STEPSIZE = [30000]

# Iteration intervals for showing the loss during training, on command line interface
__C.TRAIN.DISPLAY = 10

# Whether to double the learning rate for bias
__C.TRAIN.DOUBLE_BIAS = True

# Whether to initialize the weights with truncated normal distribution
__C.TRAIN.TRUNCATED = False

# Whether to have weight decay on bias as well
__C.TRAIN.BIAS_DECAY = False

# Whether to add ground truth boxes to the pool when sampling regions
__C.TRAIN.USE_GT = False

# Whether to use aspect-ratio grouping of training images, introduced merely for saving
# GPU memory
__C.TRAIN.ASPECT_GROUPING = False

# The number of snapshots kept, older ones are deleted to save space
__C.TRAIN.SNAPSHOT_KEPT = 3

# The time interval for saving tensorflow summaries
__C.TRAIN.SUMMARY_INTERVAL = 180

# Scale to use during training (can list multiple scales)
# The scale is the pixel size of an image's shortest side
__C.TRAIN.SCALES = (600,)

# Max pixel size of the longest side of a scaled input image
__C.TRAIN.MAX_SIZE = 1000

# Trim size for input images to create minibatch
__C.TRAIN.TRIM_HEIGHT = 600
__C.TRAIN.TRIM_WIDTH = 600

# Images to use per minibatch
__C.TRAIN.IMS_PER_BATCH = 1

# Minibatch size (number of regions of interest [ROIs])
__C.TRAIN.BATCH_SIZE = 128

# Fraction of minibatch that is labeled foreground (i.e. class > 0)
__C.TRAIN.FG_FRACTION = 0.25

# Overlap threshold for a ROI to be considered foreground (if >= FG_THRESH)
__C.TRAIN.FG_THRESH = 0.5

# Overlap threshold for a ROI to be considered background (class = 0 if
# overlap in [LO, HI))
__C.TRAIN.BG_THRESH_HI = 0.5
__C.TRAIN.BG_THRESH_LO = 0.1

# Use horizontally-flipped images during training?
__C.TRAIN.USE_FLIPPED = True

# Train bounding-box regressors
__C.TRAIN.BBOX_REG = True

# Overlap required between a ROI and ground-truth box in order for that ROI to
# be used as a bounding-box regression training example
__C.TRAIN.BBOX_THRESH = 0.5

# Iterations between snapshots
__C.TRAIN.SNAPSHOT_ITERS = 5000

# solver.prototxt specifies the snapshot path prefix, this adds an optional
# infix to yield the path: <prefix>[_<infix>]_iters_XYZ.caffemodel
__C.TRAIN.SNAPSHOT_PREFIX = 'res101_faster_rcnn'
# __C.TRAIN.SNAPSHOT_INFIX = ''

# Use a prefetch thread in roi_data_layer.layer
# So far I haven't found this useful; likely more engineering work is required
# __C.TRAIN.USE_PREFETCH = False

# Normalize the targets (subtract empirical mean, divide by empirical stddev)
__C.TRAIN.BBOX_NORMALIZE_TARGETS = True
# Deprecated (inside weights)
__C.TRAIN.BBOX_INSIDE_WEIGHTS = (1.0, 1.0, 1.0, 1.0)
# Normalize the targets using "precomputed" (or made up) means and stdevs
# (BBOX_NORMALIZE_TARGETS must also be True)
__C.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED = True
__C.TRAIN.BBOX_NORMALIZE_MEANS = (0.0, 0.0, 0.0, 0.0)
__C.TRAIN.BBOX_NORMALIZE_STDS = (0.1, 0.1, 0.2, 0.2)

# Train using these proposals
__C.TRAIN.PROPOSAL_METHOD = 'gt'

# Make minibatches from images that have similar aspect ratios (i.e. both
# tall and thin or both short and wide) in order to avoid wasting computation
# on zero-padding.

# Use RPN to detect objects
__C.TRAIN.HAS_RPN = True
# IOU >= thresh: positive example
__C.TRAIN.RPN_POSITIVE_OVERLAP = 0.7
# IOU < thresh: negative example
__C.TRAIN.RPN_NEGATIVE_OVERLAP = 0.3
# If an anchor statisfied by positive and negative conditions set to negative
__C.TRAIN.RPN_CLOBBER_POSITIVES = False
# Max number of foreground examples
__C.TRAIN.RPN_FG_FRACTION = 0.5
# Total number of examples
__C.TRAIN.RPN_BATCHSIZE = 256
# NMS threshold used on RPN proposals
__C.TRAIN.RPN_NMS_THRESH = 0.7
# Number of top scoring boxes to keep before apply NMS to RPN proposals
__C.TRAIN.RPN_PRE_NMS_TOP_N = 12000
# Number of top scoring boxes to keep after applying NMS to RPN proposals
__C.TRAIN.RPN_POST_NMS_TOP_N = 2000
# Proposal height and width both need to be greater than RPN_MIN_SIZE (at orig image scale)
__C.TRAIN.RPN_MIN_SIZE = 8
# Deprecated (outside weights)
__C.TRAIN.RPN_BBOX_INSIDE_WEIGHTS = (1.0, 1.0, 1.0, 1.0)
# Give the positive RPN examples weight of p * 1 / {num positives}
# and give negatives a weight of (1 - p)
# Set to -1.0 to use uniform example weighting
__C.TRAIN.RPN_POSITIVE_WEIGHT = -1.0
# Whether to use all ground truth bounding boxes for training,
# For COCO, setting USE_ALL_GT to False will exclude boxes that are flagged as ''iscrowd''
__C.TRAIN.USE_ALL_GT = True

# Whether to tune the batch normalization parameters during training
__C.TRAIN.BN_TRAIN = False

#
# Testing options
#
__C.TEST = edict()

# Scale to use during testing (can NOT list multiple scales)
# The scale is the pixel size of an image's shortest side
__C.TEST.SCALES = (600,)

# Max pixel size of the longest side of a scaled input image
__C.TEST.MAX_SIZE = 1000

# Overlap threshold used for non-maximum suppression (suppress boxes with
# IoU >= this threshold)
__C.TEST.NMS = 0.3

# Experimental: treat the (K+1) units in the cls_score layer as linear
# predictors (trained, eg, with one-vs-rest SVMs).
__C.TEST.SVM = False

# Test using bounding-box regressors
__C.TEST.BBOX_REG = True

# Propose boxes
__C.TEST.HAS_RPN = False

# Test using these proposals
__C.TEST.PROPOSAL_METHOD = 'gt'

## NMS threshold used on RPN proposals
__C.TEST.RPN_NMS_THRESH = 0.7
## Number of top scoring boxes to keep before apply NMS to RPN proposals
__C.TEST.RPN_PRE_NMS_TOP_N = 1000
# __C.TEST.RPN_PRE_NMS_TOP_N = 6000

## Number of top scoring boxes to keep after applying NMS to RPN proposals
__C.TEST.RPN_POST_NMS_TOP_N = 100
# __C.TEST.RPN_POST_NMS_TOP_N = 300

# Proposal height and width both need to be greater than RPN_MIN_SIZE (at orig image scale)
__C.TEST.RPN_MIN_SIZE = 16

# Testing mode, default to be 'nms', 'top' is slower but better
# See report for details
__C.TEST.MODE = 'nms'

# Only useful when TEST.MODE is 'top', specifies the number of top proposals to select
__C.TEST.RPN_TOP_N = 5000

#
# ResNet options
#

__C.RESNET = edict()

# Option to set if max-pooling is appended after crop_and_resize.
# if true, the region will be resized to a square of 2xPOOLING_SIZE,
# then 2x2 max-pooling is applied; otherwise the region will be directly
# resized to a square of POOLING_SIZE
__C.RESNET.MAX_POOL = False

# Number of fixed blocks during training, by default the first of all 4 blocks is fixed
# Range: 0 (none) to 3 (all)
__C.RESNET.FIXED_BLOCKS = 1

#
# MobileNet options
#

__C.MOBILENET = edict()

# Whether to regularize the depth-wise filters during training
__C.MOBILENET.REGU_DEPTH = False

# Number of fixed layers during training, by default the first of all 14 layers is fixed
# Range: 0 (none) to 12 (all)
__C.MOBILENET.FIXED_LAYERS = 5

# Weight decay for the mobilenet weights
__C.MOBILENET.WEIGHT_DECAY = 0.00004

# Depth multiplier
__C.MOBILENET.DEPTH_MULTIPLIER = 1.

#
# MISC
#

# The mapping from image coordinates to feature map coordinates might cause
# some boxes that are distinct in image space to become identical in feature
# coordinates. If DEDUP_BOXES > 0, then DEDUP_BOXES is used as the scale factor
# for identifying duplicate boxes.
# 1/16 is correct for {Alex,Caffe}Net, VGG_CNN_M_1024, and VGG16
__C.DEDUP_BOXES = 1. / 16.

# Pixel mean values (BGR order) as a (1, 1, 3) array
# We use the same pixel mean for all networks even though it's not exactly what
# they were trained with
__C.PIXEL_MEANS = np.array([[[102.9801, 115.9465, 122.7717]]])

# For reproducibility
__C.RNG_SEED = 3

# A small number that's used many times
__C.EPS = 1e-14

# Root directory of project
__C.ROOT_DIR = osp.abspath(osp.join(osp.dirname(__file__), '..', '..', '..'))

# Data directory
__C.DATA_DIR = osp.abspath(osp.join(__C.ROOT_DIR, 'data'))

# Name (or path to) the matlab executable
__C.MATLAB = 'matlab'

# Place outputs under an experiments directory
__C.EXP_DIR = 'default'

# Use GPU implementation of non-maximum suppression
__C.USE_GPU_NMS = True

# Default GPU device id
__C.GPU_ID = 0

__C.POOLING_MODE = 'crop'

# Size of the pooled region after RoI pooling
__C.POOLING_SIZE = 7

# Maximal number of gt rois in an image during Training
__C.MAX_NUM_GT_BOXES = 20

# Anchor scales for RPN
__C.ANCHOR_SCALES = [8,16,32]

# Anchor ratios for RPN
__C.ANCHOR_RATIOS = [0.5,1,2]

# Feature stride for RPN
__C.FEAT_STRIDE = [16, ]

__C.CUDA = False

__C.CROP_RESIZE_WITH_MAX_POOL = True

import pdb
def get_output_dir(imdb, weights_filename):
  """Return the directory where experimental artifacts are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'output', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir

def get_output_tb_dir(imdb, weights_filename):
  """Return the directory where tensorflow summaries are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'tensorboard', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir

def _merge_a_into_b(a, b):
  """Merge config dictionary a into config dictionary b, clobbering the
  options in b whenever they are also specified in a.
  """
  if type(a) is not edict:
    return

  for k, v in a.items():
    # a must specify keys that are in b
    if k not in b:
      raise KeyError('{} is not a valid config key'.format(k))

    # the types must match, too
    old_type = type(b[k])
    if old_type is not type(v):
      if isinstance(b[k], np.ndarray):
        v = np.array(v, dtype=b[k].dtype)
      else:
        raise ValueError(('Type mismatch ({} vs. {}) '
                          'for config key: {}').format(type(b[k]),
                                                       type(v), k))

    # recursively merge dicts
    if type(v) is edict:
      try:
        _merge_a_into_b(a[k], b[k])
      except:
        print(('Error under config key: {}'.format(k)))
        raise
    else:
      b[k] = v

def cfg_from_file(filename):
  """Load a config file and merge it into the default options."""
  import yaml
  with open(filename, 'r') as f:
    yaml_cfg = edict(yaml.load(f))

  _merge_a_into_b(yaml_cfg, __C)

def cfg_from_list(cfg_list):
  """Set config keys via list (e.g., from command line)."""
  from ast import literal_eval
  assert len(cfg_list) % 2 == 0
  for k, v in zip(cfg_list[0::2], cfg_list[1::2]):
    key_list = k.split('.')
    d = __C
    for subkey in key_list[:-1]:
      assert subkey in d
      d = d[subkey]
    subkey = key_list[-1]
    assert subkey in d
    try:
      value = literal_eval(v)
    except:
      # handle the case when v is a string literal
      value = v
    assert type(value) == type(d[subkey]), \
      'type {} does not match original type {}'.format(
        type(value), type(d[subkey]))
    d[subkey] = value


text:
import wave
import numpy as np
from .unitgenerator import UnitGenerator

# Simple call to get_frames() to get data in format we like (numpy array, float32)
class WaveFile(object):
    def __init__(self, filepath) :
        super(WaveFile, self).__init__()

        self.wave = wave.open(filepath)
        self.num_channels, self.sampwidth, self.sr, self.end, \
           comptype, compname = self.wave.getparams()

        # for now, we will only accept 16 bit files at 44k
        assert(self.sampwidth == 2)
        assert(self.sr == 44100)

    # read an arbitrary chunk of data from the file
    def get_frames(self, start_frame, end_frame) :
        # get the raw data from wave file as a byte string. If asking for more than is available, it just
        # returns what it can
        self.wave.setpos(start_frame)
        raw_bytes = self.wave.readframes(end_frame - start_frame)

        # convert raw data to numpy array, assuming int16 arrangement
        samples = np.fromstring(raw_bytes, dtype = np.int16)

        # convert from integer type to floating point, and scale to [-1, 1]
        samples = samples.astype(np.float32)
        samples *= (1 / 32768.0)

        return samples

    def get_num_channels(self):
        return self.num_channels

        # generates audio data by asking an audio-source (ie, WaveFile) for that data.
class WaveGenerator(UnitGenerator):
    def __init__(self, wave_source, loop=False):
        super(WaveGenerator, self).__init__()
        self.source = wave_source
        self.num_channels = self.source.num_channels
        self.loop = loop
        self.paused = False
        self._release = False

    def reset(self):
        self.paused = True
        self.frame = 0

    def play_toggle(self):
        self.paused = not self.paused

    def play(self):
        self.paused = False

    def pause(self):
        self.paused = True

    def release(self):
        self._release = True

    def set_gain(self, g):
        self.gain = g

    def get_gain(self):
        return self.gain

    def  __generate__(self, frame_id,num_frames,sample_rate):
        if self.paused:
            output = np.zeros(num_frames * self.source.num_channels)
            return (output, True)

        else:
            # get data based on our position and requested # of frames
            output = self.source.get_frames(self.frame, self.frame + num_frames)
            # check for end-of-buffer condition:
            actual_num_frames = int(len(output) / self.num_channels)

            # looping. If we got to the end of the buffer, don't actually end.
            # Instead, read some more from the beginning

            shortfall = num_frames * self.num_channels - len(output)
            if self.loop and not shortfall>0:
                remainder = num_frames - actual_num_frames
                output = np.append(output, self.source.get_frames(0, remainder))
                self.frame = remainder

            # zero-pad if output is too short (may happen if not looping / end of buffer)
            shortfall = num_frames * self.num_channels - len(output)
            if shortfall > 0:
                output = np.append(output, np.zeros(shortfall))
                self.frame=0

            # return
            return output

class SpeedModulator(object):
    def __init__(self, generator, speed = 1.0):
        super(SpeedModulator, self).__init__()
        self.generator = generator
        self.speed = speed

    def set_speed(self, speed) :
        self.speed = speed

    def generate(self, num_frames, num_channels) :
        # optimization if speed is 1.0
        if self.speed == 1.0:
            return self.generator.generate(num_frames, num_channels)

        # otherwise, we need to ask self.generator for a number of frames that is
        # larger or smaller than num_frames, depending on self.speed
        adj_frames = int(round(num_frames * self.speed))

        # get data from generator
        data, continue_flag = self.generator.generate(adj_frames, num_channels)

        # split into multi-channels:
        data_chans = [ data[n::num_channels] for n in range(num_channels) ]

        # stretch or squash data to fit exactly into num_frames
        from_range = np.arange(adj_frames)
        to_range = np.arange(num_frames) * (float(adj_frames) / num_frames)
        resampled = [ np.interp(to_range, from_range, data_chans[n]) for n in range(num_channels) ]

        # convert back by interleaving into a single buffer
        output = np.empty(num_channels * num_frames, dtype=np.float32)
        for n in range(num_channels) :
            output[n::num_channels] = resampled[n]

        return (output, continue_flag)

# We can generalize the thing that WaveFile does - it provides arbitrary wave
# data. We can define a "wave data providing interface" (called WaveSource)
# if it can support the function:
#
# get_frames(self, start_frame, end_frame)
#
# Now create WaveBuffer. Same WaveSource interface, but can take a subset of
# audio data from a wave file and holds all that data in memory.
class WaveBuffer(object):
    def __init__(self, filepath, start_frame, num_frames):
        super(WaveBuffer, self).__init__()

        # get a local copy of the audio data from WaveFile
        wr = WaveFile(filepath)
        self.data = wr.get_frames(start_frame, start_frame + num_frames)
        self.num_channels = wr.get_num_channels()

    # start and end args are in units of frames,
    # so take into account num_channels when accessing sample data
    def get_frames(self, start_frame, end_frame) :
        start_sample = start_frame * self.num_channels
        end_sample = end_frame * self.num_channels
        return self.data[start_sample : end_sample]

    def get_num_channels(self):
        return self.num_channels


text:
import numpy
import torch
import random
import sklearn.preprocessing
import math
import scipy.stats
import itertools
import random
import matplotlib.pyplot as plt
import matplotlib.collections as mcoll
# from deep_signature.data_generation import CurveDatasetGenerator
# from deep_signature.data_generation import CurveManager
from skimage.util.shape import view_as_windows
import os
from deep_signature.linalg import affine_transform
from deep_signature.linalg import euclidean_transform
from deep_signature.data_manipulation import curve_processing
import bisect

# https://stackoverflow.com/questions/36074455/python-matplotlib-with-a-line-color-gradient-and-colorbar
def colorline(ax, x, y, z=None, cmap='copper', norm=plt.Normalize(0.0, 1.0), linewidth=3, alpha=1.0):
    """
    http://nbviewer.ipython.org/github/dpsanders/matplotlib-examples/blob/master/colorline.ipynb
    http://matplotlib.org/examples/pylab_examples/multicolored_line.html
    Plot a colored line with coordinates x and y
    Optionally specify colors in the array z
    Optionally specify a colormap, a norm function and a line width
    """

    # Default colors equally spaced on [0,1]:
    if z is None:
        z = numpy.linspace(0.0, 1.0, len(x))

    # Special case if a single number:
    # to check for numerical input -- this is a hack
    if not hasattr(z, "__iter__"):
        z = numpy.array([z])

    z = numpy.asarray(z)

    segments = make_segments(x, y)
    lc = mcoll.LineCollection(segments, array=z, cmap=cmap, norm=norm,
                              linewidth=linewidth, alpha=alpha)

    # ax = plt.gca()
    ax.add_collection(lc)

    return lc

def make_segments(x, y):
    """
    Create list of line segments from x and y coordinates, in the correct format
    for LineCollection: an array of the form numlines x (points per line) x 2 (x
    and y) array
    """

    points = numpy.array([x, y]).T.reshape(-1, 1, 2)
    segments = numpy.concatenate([points[:-1], points[1:]], axis=1)
    return segments

def plot_dist(ax, dist):
    x = numpy.array(range(dist.shape[0]))
    y = dist
    ax.set_xlim(x.min(), x.max())
    ax.set_ylim(y.min(), y.max())
    colorline(ax=ax, x=x, y=y, cmap='hsv')

def plot_curve_section_sample(ax, curve, curve_section_sample, dist, sampling_indices, point_size=10):
    x = curve_section_sample[:,0]
    y = curve_section_sample[:,1]
    c = numpy.linspace(0.0, 1.0, dist.shape[0])

    ax.scatter(
        x=x,
        y=y,
        c=c[sampling_indices],
        s=point_size,
        cmap='hsv')

def strided_indexing_roll(a, r):
    # Concatenate with sliced to cover all rolls
    a_ext = numpy.concatenate((a,a[:,:-1]),axis=1)

    # Get sliding windows; use advanced-indexing to select appropriate ones
    n = a.shape[1]
    return view_as_windows(a_ext,(1,n))[numpy.arange(len(r)), (n-r)%n,0]

def insert_sorted(arr, items):
    arr = arr.copy()
    for item in items:
        if item not in arr:
            bisect.insort(arr, item)

    return arr

if __name__ == '__main__':

    from itertools import count

    for i in count(5):
        # ...
        pass

    a = [1, 2, 4, 5]
    bla = numpy.array(a)
    da = numpy.where(bla==4)

    a2 = insert_sorted(a, [2,3])
    print(a)

    bla1 = list(range(-1, 5))
    bla2 = list(range(5))

    bla = numpy.linspace(start=0, stop=150, num=20, endpoint=False, dtype=int)
    bla2 = numpy.arange(start=0, stop=100, step=20)

    bla = numpy.arange(start=0, stop=5)

    indices = [1,2,3,4,5,6,7,8,9]
    for i, (index1, index2, index3) in enumerate(zip(indices, indices[1:], indices[2:])):
        h = 5

    curve = numpy.array([[1, 1], [2, 3], [3, 7], [4, 6], [5, 5], [6, 2]])
    curve_processing.calculate_equiaffine_arclength_by_euclidean_metrics(curve=curve)

    bla = range(2,6)

    arr = numpy.array([1,2,3,4,5,6,7,8,9,10,11])
    for i in numpy.arange(start=1, stop=arr.shape[0] - 1, step=3):
        indices = numpy.arange(i-1,i+4)

    bla = numpy.arange(start=2, stop=10)

    i = 1
    bla = list(range((i-2),(i+3)))

    bla = numpy.square(2)

    from itertools import combinations

    indices = numpy.array([1,2,3,4,5])
    for item in combinations(indices, 3):
        sorted_indices = sorted(item)
        bla = 5

    bla = numpy.array([[9, 4], [2, 3], [2, 4], [2, 5], [7, 7], [8, 8]])
    curve_processing.calculate_equiaffine_curvature(curve=bla)

    curve_processing.transform_curve(curve=bla, transform=affine_transform.generate_random_equiaffine_transform_2d())

    points = bla[[0,1,2]]
    points = numpy.hstack((points, numpy.ones((points.shape[0], 1), dtype=points.dtype)))

    print(points)
    print(numpy.linalg.det(points))

    scale = numpy.random.randint(low=1, high=5, size=2)
    coeffs = numpy.random.random(size=2)
    entries = scale * coeffs

    bla1 = numpy.array([[1, 0], [entries[0], 1]])
    print(bla1)

    bla2 = numpy.array([[1, entries[1]], [0, 1]])
    print(bla2)

    bla = numpy.matmul(bla1, bla2)
    print(bla)

    print(bla[0,0] * bla[1,1] - bla[0, 1] * bla[1, 0])

    bla = list(range(200))[::20]

    indices = [1,2,3,4,5,6]
    indices2 = [7,8,9,10,11,12]
    indices3 = indices + indices2
    for i, [bla1, bla2] in enumerate(zip(indices, indices[1:])):
        j = 5

    bla = indices[:-1]

    h= 5

    # indices = numpy.linspace(
    #     start=start_point_index,
    #     stop=end_point_index,
    #     num=int(numpy.abs(end_point_index - start_point_index)),
    #     endpoint=False,
    #     dtype=int)

    bla = torch.tensor([1,5,4,6,-2,-3,-7,5])
    v = torch.max(-bla, torch.zeros_like(bla))

    bla = numpy.array([[1,1],[2,2],[4,4],[5,5],[7,7],[8,8]])
    bla2 = numpy.diff(a=bla,n=1,axis=0)
    bla3 = numpy.linalg.norm(x=bla2,ord=2,axis=1)
    bla4 = numpy.cumsum(a=numpy.concatenate((numpy.array([0]), bla3),axis=0))

    h = 5

    v = torch.rand([3, 9, 1])
    v2 = v[:, 0, :].unsqueeze(dim=1)
    v3 = v[:, 1:, :]
    v4 = v3[:, 0::2, :]
    v5 = v3[:, 1::2, :]
    v6 = v4 + v5
    v7 = torch.cat((v2, v6), dim=1)

    print('------- V -------')
    print(v)
    print('------- V2 -------')
    print(v2)
    print('------- V3 -------')
    print(v3)
    print('------- V4 -------')
    print(v4)
    print('------- V5 -------')
    print(v5)
    print('------- V6 -------')
    print(v6)
    print('------- V7 -------')
    print(v7)

    bla = 3

    bla = numpy.random.uniform(low=-0.2,high=0.2,size=100)

    bla = numpy.array([1,2,3,4,5,6])

    da = bla[-1]

    bla2 = bla[1:-1]

    bla = numpy.arange(start=60, stop=-20, step=-1)

    start_point_index = 100
    end_point_index = 200
    indices = numpy.linspace(
        start=start_point_index,
        stop=end_point_index,
        num=int(numpy.abs(end_point_index - start_point_index)),
        endpoint=False,
        dtype=int)

    h = 5

    a = numpy.array([5,8,7,4,1,2,3])
    b = numpy.sort(a)

    lengths = numpy.random.randint(low=10, high=100, size=500)
    lengths = lengths[:, None]
    delta = numpy.random.uniform(low=0, high=1, size=[500,2]) *  lengths

    v = torch.tensor([[[1],[2],[3],[4],[5]], [[11],[13],[14],[15],[16]], [[21],[24],[25],[26],[27]], [[31],[35],[36],[37],[38]]]).double()

    v2 = v[:, 0, :]

    v3 = v2.unsqueeze(dim=1)

    # v4 = v3.repeat(1, 3, 1)

    v5 = v - v3

    v6 = v5.abs().squeeze(dim=2)

    v7 = v6[:, 1:]

    v8 = v7[:, 0]

    v9 = v8.unsqueeze(dim=1)

    v10 = v7 - v9

    v11 = v10[:, 1:]

    v12 = v11.exp()

    v13 = v12.sum(dim=1)

    v14 = v13 + 1

    v15 = v14.log()

    v16 = v15.mean(dim=0)

    h = 5

    bla = [1,2,3,4]
    bla.remove(3)

    bla = numpy.random.choice([1,2,3,6,7], 10)

    bla = numpy.random.randint(7)
    bla = numpy.random.randint(7)
    bla = numpy.random.randint(7)
    bla = numpy.random.randint(7)

    bla = itertools.combinations([1,2,3,4,5,6,7,8,9,10], 2)
    for combi in bla:
        bubu = list(combi)
        h = 5

    from numpy.random import default_rng

    rng = numpy.random.default_rng()
    numbers1 = rng.choice(20, size=10, replace=False)
    numbers2 = rng.choice(20, size=10, replace=False)

    bla = random.randrange(10)
    bla = numpy.random.randint(0, 10, size=[20,2]).tolist()
    # lst = [[random.randrange(r1) for _ in range(random.randrange(r2))] for __ in range(random.randrange(r3))]

    bla = numpy.array([1,2,3])
    bla2 = numpy.flip(bla)
    bla3 = numpy.random.randint(0, 2, 15)

    count = 10
    center_point_index=4
    indices = numpy.arange(count)
    indices = numpy.roll(indices, -(center_point_index + 1))
    indices = numpy.delete(indices, count - 1)

    lst = [1,2,3,4,5,6,7]
    mydict = [{"a": 1, "b": 2}] * 7
    bla = list(zip(lst, mydict))

    for x in bla:
        print(x)

    lst2 = lst[:4]

    a = 6

    b = numpy.mod(-4, 3)

    c = 5

    negative_pairs = numpy.load(file=os.path.normpath(os.path.join("C:/deep-signature-data/datasets/dataset2/", "negative_pairs.npy")), allow_pickle=True)
    positive_pairs = numpy.load(file=os.path.normpath(os.path.join("C:/deep-signature-data/datasets/dataset2/", "positive_pairs.npy")), allow_pickle=True)

    bla = numpy.empty((negative_pairs.shape[0] + positive_pairs.shape[0], negative_pairs.shape[1], negative_pairs.shape[2], negative_pairs.shape[3]))

    bla[::2, :] = negative_pairs

    bla[1::2, :] = positive_pairs

    del negative_pairs
    del positive_pairs

    numpy.random.shuffle(bla)

    h = 5

    # bla = numpy.linspace(start=0, stop=40, num=30, endpoint=False, dtype=int)
    # h = 5

    # arr1 = numpy.array([1,2,3])
    # arr2 = numpy.array([4,5,6])
    # arr3 = numpy.vstack((arr1, arr2))
    # arr4 = numpy.vstack((arr2, arr3))
    #
    # h = 5

    # # momo = numpy.mod(-1, 300)
    #
    # arr1 = numpy.array([1,2,3])
    # arr2 = numpy.array([4,5,6])
    # arr = numpy.concatenate((arr1, arr2))
    #
    # bla = numpy.array([[1,2,3],[4,5,6],[7,8,9]])
    #
    # shift = numpy.round(numpy.random.rand(3) * 3).astype(int)
    # # bla2 = numpy.roll(a=bla, axis=(1,1,1), shift=(2,0,1))
    #
    # bla2 = strided_indexing_roll(bla, shift)

    rotation_factor = 10
    sectioning_factor = 20
    sampling_factor = 4
    multimodality_factor = 20
    sampling_points_count = 400
    supporting_points_count = 10

    curve_dataset_generator = CurveDatasetGenerator()
    curves = curve_dataset_generator.load_curves(file_path="C:/deep-signature-data/curves/curves.npy")

    for i, curve in enumerate(curves[:3]):
        curve_data_generator = CurveManager(
            curve=curve,
            rotation_factor=rotation_factor,
            sectioning_factor=sectioning_factor,
            sampling_factor=sampling_factor,
            multimodality_factor=multimodality_factor,
            sampling_points_count=sampling_points_count,
            supporting_points_count=supporting_points_count)

        negative_pairs = curve_data_generator.generate_negative_pairs()

        for negative_pair_index, negative_pair in enumerate(negative_pairs[:3]):
            fig, ax = plt.subplots(1, 2, figsize=(80, 40))

            for label in (ax[0].get_xticklabels() + ax[0].get_yticklabels()):
                label.set_fontsize(30)

            plot_dist(ax=ax[0], dist=negative_pair['dist'])
            plt.show()

    # arr = numpy.array([[2,4,6],[3,6,9],[5,10,15],[6,12,18],[7,14,21],[8,16,24]])
    #
    # arr = arr.reshape(-1,3,2)
    #
    # arr2 = arr.sum(axis=1)
    # arr3 = arr2.sum(axis=1)
    #
    # div = numpy.array([2,3,5])
    #
    # res = arr / div[:,None]
    #

    # a = numpy.arange(6,20)
    # it = numpy.nditer(a)
    # while not it.finished:
    #     bubu = it[0].item()
    #     it.iternext()

    # center = 17
    # delta = 6
    #
    # bibi = numpy.arange(center - delta, center + delta + 1)
    # bibi2 = numpy.mod(bibi, 20)
    #
    # itemindex = numpy.where(bibi2 == 17).item()
    # bla = bibi2[itemindex-3:itemindex+3+1]
    #
    # bla = [1,2,3,4,5,6,7,8]
    #
    # random.shuffle(bla)
    #
    # bla2 = itertools.combinations(bla, 3)
    #
    # for x in bla2:
    #     bla3 = x
    #
    # bla.extend([5,6,7])
    # j = 6

    # bins = 500
    # is_even = bins % 2 == 0
    # half_bins = math.floor(bins / 2)
    #
    # start, stop = -half_bins, half_bins
    # if not is_even:
    #     stop = stop + 1
    #
    # x = numpy.arange(start, stop)
    # x_lower, x_upper = x - 0.5, x + 0.5
    #
    # loc = 0.5
    # scale = 0.25
    # a = 0
    # b = 1
    # truncated_normal = scipy.stats.truncnorm(a=(a - loc) / scale, b=(b - loc) / scale, loc=loc, scale=scale)
    #
    # scales = truncated_normal.rvs(bins) * 10 * numpy.sqrt(bins)
    # locs = [0] * bins
    #
    # # blu = x_lower[:,None]
    #
    # dist_count = 1000
    #
    # cfd_lower = scipy.stats.norm.cdf(x_lower[:,None], loc=[0] * dist_count, scale=[15] * dist_count).transpose()
    # cfd_upper = scipy.stats.norm.cdf(x_upper[:,None], loc=[0] * dist_count, scale=[15] * dist_count).transpose()
    #
    # dist = cfd_upper - cfd_lower
    #
    # du = numpy.sum(dist, axis=1)
    #
    # dist2 = dist / du[:,None]

    # m = 5

    # numpy.seterr(all='raise')
    # x0 = numpy.array([[1,1],[2,2],[3,3],[4,4],[5,5],[6,6]])
    # padded_curve = numpy.pad(x0, ((2, 2), (0, 0)), 'wrap')
    #
    # bla = 2

    # x1 = numpy.array([1,2,3,4,5,6])
    # x2 = x1[:, numpy.newaxis]
    #
    # x3 = x0 * x2
    #
    # bla = 5

    # x1 = sklearn.preprocessing.normalize(x0, axis=1, norm='l2')
    #
    # x2 = numpy.pad(x1, ((1,1),(0,0)), 'wrap')
    # x3 = x2[1:-1]

    # bla = [1,2,3,7,8,12,20]
    # var = numpy.sqrt(numpy.var(bla))
    # mean = numpy.mean(bla)

    # dataset_generator = DatasetGenerator()
    # generated_curves = dataset_generator.generate_curves(dir_path="C:/deep-signature-data/images", plot_curves=False)
    # dataset_generator.save_curves(dir_path="C:/deep-signature-data/curves")
    # loaded_curves = dataset_generator.load_curves(file_path="C:/deep-signature-data/curves/curves.npy")
    # h = 5
    # a = numpy.array([[0., 0.], [0.3, 0.], [1.25, -0.1],
    #               [2.1, -0.9], [2.85, -2.3], [3.8, -3.95],
    #               [5., -5.75], [6.4, -7.8], [8.05, -9.9],
    #               [9.9, -11.6], [12.05, -12.85], [14.25, -13.7],
    #               [16.5, -13.8], [19.25, -13.35], [21.3, -12.2],
    #               [22.8, -10.5], [23.55, -8.15], [22.95, -6.1],
    #               [21.35, -3.95], [19.1, -1.9]])
    #
    # dx_dt = numpy.gradient(a[:, 0])
    # dy_dt = numpy.gradient(a[:, 1])
    # bla = numpy.array([[dx_dt[i], dy_dt[i]] for i in range(dx_dt.size)])
    #
    # bla2 = numpy.gradient(a, axis=0)
    #
    # h = 5

    # print(torch.__version__)
    #
    # t = torch.tensor([[2,1,2],[2,5,2],[1,1,4]], dtype=torch.float32)
    # norms = torch.norm(t)
    #
    #
    # t1 = torch.tensor([1,0,1,1,1,0,1,0,0,1], dtype=torch.float32)
    # t2 = torch.tensor([2,2,2,2,2,2,2,2,2,2], dtype=torch.float32)
    # t3 = 1 - t1
    # t4 = torch.tensor([2,-5,5,1,2,-1,-0.1,8,-7,6], dtype=torch.float32)
    #
    # bla = t1 * t2
    # bla2 = torch.sum(bla)
    #
    # bla3 = torch.max(torch.zeros_like(t1), t4)
    #
    #
    #
    #
    # metadata = numpy.load(file='./dataset/metadata.npy', allow_pickle=True)
    #
    # bla = metadata.item()
    #
    # curve = numpy.load(file='C:\\Users\\Roy\\Documents\\GitHub\\deep-signature\\dataset\\1268\\0\\8\\sample.npy', allow_pickle=True)
    #
    # g = 5
    #
    # arr1 = numpy.array([[1,2],[4,6],[7,5]])
    #
    # curve_torch = torch.from_numpy(curve)
    #
    # print(curve_torch[:, 0])
    #
    # # arr1_da = numpy.concatenate((arr1[-2:],arr1,arr1[:2]), axis=0)
    #
    # arr2 = numpy.array([[7,3],[9,6],[0,3]])
    # #
    # # print(numpy.concatenate((arr1, arr2), axis=1))
    #
    # # print(arr1)
    # bob = numpy.vstack((numpy.transpose(arr1), numpy.transpose(arr2)))
    # print(bob)
    # print(bob.shape)
    #
    # # print(arr[0])
    # # print(arr[1:4])
    # # print(arr[4:8])

max_stars_repo_name: xenomarz/deep-signature

max_issues_repo_name: xenomarz/deep-signature

max_forks_repo_name: xenomarz/deep-signature

reason: import numpy,from numpy,import scipy


text:
# Begin training neural net using default data / parameters
import unittest

import torch
import torchvision
import numpy as np

from CNNScan.Reco import Settings
import CNNScan.Samples

# Our neural net architecture has two output modes.
# If in unique_outputs mode, there is a unique output layer for each (ballot, contest) pair.
# Each output layer has as many neurons as there are options in the associated (ballot, contest) pair.
# Otherwise, there is a single output layer shared between all (ballot, contest) pairs, which is as long as the longest pair.
# This test case ensures that both network configurations work properly.
class TestOutputLabeling(unittest.TestCase):
	def setUp(self):

		self.config = Settings.generate_default_settings()
		self.config['epochs'] = 1
		# Reduce size of NN to speed up testing.
		self.config['recog_full_layers'] = [10]
		self.config['recog_conv_layers'] 	= [
		CNNScan.Settings.conv_def(4, 1, 4, 0, 4, False),
		]
		self.markdb = CNNScan.Mark.MarkDatabase()
		self.markdb.insert_mark(CNNScan.Mark.BoxMark())

		# Create fake data that can be used.
		self.ballot_factory = CNNScan.Ballot.BallotDefinitions.BallotFactory()
		_ = CNNScan.Samples.Random.get_sample_ballot(self.ballot_factory, dpi=40)

		# Create faked marked ballots from ballot factories.
		self.data = CNNScan.Reco.Load.GeneratingDataSet(self.ballot_factory, self.markdb, 2)
		self.load = torch.utils.data.DataLoader(self.data, batch_size=self.config['batch_size'], shuffle=True)

	def test_unique_outputs(self):
		# Configure the NN to have unpooled outputs for each (ballot, contest) pairs.
		self.config['unique_outputs'] = True

		# Attempts to train model.
		model = CNNScan.Reco.ContestRec.BallotRecognizer(self.config, self.ballot_factory)
		model = CNNScan.Reco.ContestRec.train_election(model, self.config, self.ballot_factory, self.load, self.load)

	def test_pooled_outputs(self):
		# Configure the NN to have a pooled output layer shared between all (ballot, contest) pairs.
		self.config['unique_outputs'] = False

		# Attempts to train model.
		model = CNNScan.Reco.ContestRec.BallotRecognizer(self.config, self.ballot_factory)
		model = CNNScan.Reco.ContestRec.train_election(model, self.config, self.ballot_factory, self.load, self.load)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: Matthew-McRaven/ConvVote

max_issues_repo_name: Matthew-McRaven/ConvVote

max_forks_repo_name: Matthew-McRaven/ConvVote


text:
#!/usr/bin/env python3

# Takes a single benchmark output JSON and plots the duration of item executions over time

import json
import matplotlib.patches as mpatches
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sys

if(len(sys.argv) != 2):
    exit('Usage: ' + sys.argv[0] + ' benchmark.json')

with open(sys.argv[1]) as json_file:
    data_json = json.load(json_file)['benchmarks']

# Build flat list of {name, begin, duration} entries for every benchmark item run
data = []
benchmark_names = []
for benchmark_json in data_json:
	name = benchmark_json['name']
	benchmark_names.append(name)
	for run_json in benchmark_json['successful_runs']:
		data.append({'name': name, 'begin': run_json['begin'] / 1e9, 'duration': run_json['duration'], 'success': True})
	for run_json in benchmark_json['unsuccessful_runs']:
		data.append({'name': name, 'begin': run_json['begin'] / 1e9, 'duration': run_json['duration'], 'success': False})

df = pd.DataFrame(data).reset_index()

# Set the colors
benchmark_names.sort()  # Sort the benchmarks for a deterministic color mapping
name_to_color = {}
prop_cycle = plt.rcParams['axes.prop_cycle']
default_colors = prop_cycle.by_key()['color']

fig, ax = plt.subplots()
for i in range(0, len(benchmark_names)):
	benchmark_name = benchmark_names[i]
	color = default_colors[i % len(default_colors)]
	name_to_color[benchmark_name] = color

	filtered_df = df[df['name'] == benchmark_name]

	# Plot into combined graph
	for success in [True, False]:
		if not filtered_df[filtered_df['success']==success].empty:
			filtered_df[filtered_df['success']==success].plot(ax=ax, kind='scatter', x='begin', y='duration', c=color, figsize=(12, 9), s=5, marker=('o' if success else 'x'), linewidth=1)

# Finish combined graph
ax.set_xlabel('Seconds since start')
ax.set_ylabel('Execution duration [ns]')
ax.grid(True, alpha=.3)

# Add legend to combined graph
handles = []
for name, color in name_to_color.items():
	handles.append(mpatches.Patch(label=name, color=color))
plt.legend(handles=handles)

# Write combined graph to file
basename = sys.argv[1].replace('.json', '')
plt.savefig(basename + '.pdf')
plt.savefig(basename + '.png')

# Plot detailed graph
grouped_df = df.reset_index().groupby('name')
fig, axes = plt.subplots(nrows=len(benchmark_names), ncols=1, figsize=(12, 3*len(benchmark_names)), sharex=True)
for (key, ax) in zip(grouped_df.groups.keys(), axes.flatten()):
	filtered_df = grouped_df.get_group(key)
	for success in [True, False]:
		if not filtered_df[filtered_df['success']==success].empty:
			filtered_df[filtered_df['success']==success].plot(ax=ax, kind='scatter', x='begin', y='duration', c=name_to_color[key], s=5, marker=('o' if success else 'x'), linewidth=1)
	ax.set_title(key)
	ax.set_xlabel('Seconds since start')
	ax.set_ylabel('Execution duration [ns]')
	ax.yaxis.set_label_coords(-.08, 0.5)
	ax.grid(True, alpha=.3)
fig.tight_layout()

# Write detailed graph to file
plt.savefig(basename + '_detailed.pdf')
plt.savefig(basename + '_detailed.png')


text:
import matplotlib.pyplot as plt

import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torchvision import datasets, transforms, models
from collections import OrderedDict

from PIL import Image
import numpy as np
import json
import time
from pathlib import Path

import helper
from workspace_utils import active_session
import functions
import argparse

if __name__ == "__main__":
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    current_dir = Path.cwd()

    parser = argparse.ArgumentParser()
    parser.description='arguments for training file'
    parser.add_argument("--data_dir", help="directory of images",type=str, default=str(current_dir / 'flowers'))
    parser.add_argument("--save_dir", help="directory that save the checkpoint files",type=str, default=str(current_dir / 'model_checkpoint'))
    parser.add_argument("--chpt_fn", help="name for the checkpoint file",type=str, default='checkpoint.pth')
    parser.add_argument("--arch", help="model architecture",type=str, default='vgg16')
    parser.add_argument("--drop_p", help="dropout percentage",type=float, default=0.5)
    parser.add_argument("--learn_r", help="learning rate",type=float, default=0.01)
    parser.add_argument("--hid_u", help="hidden layer units",type=int, default=1024)
    parser.add_argument("--epochs", help="epochs for training",type=int, default=5)
    args = parser.parse_args()
    print('arguments are',args)

    arch = args.arch
    dropout_p = args.drop_p
    learn_rate = args.learn_r
    hidden_units = args.hid_u
    class_num = 102

    epochs = args.epochs
    steps = 0
    train_loss = 0
    print_every = 30

    data_dir = Path(args.data_dir)
    train_dir = str(data_dir / 'train')
    valid_dir = str(data_dir / 'valid')
    test_dir = str(data_dir / 'test')

    save_dir = str(Path(args.save_dir) / args.chpt_fn)

    # load data
    trainloader, testloader, validloader, train_data = functions.load_data(train_dir, valid_dir, test_dir)

    # load the model from torchvision
    model = getattr(models, arch)(pretrained=True)

    # Freeze parameters so we don't backprop through them
    for param in model.parameters():
        param.requires_grad = False

    # Define our new classifier
    classifier = nn.Sequential(nn.Linear(25088, hidden_units),
                            nn.Dropout(p=dropout_p),
                            nn.ReLU(),
                            nn.Linear(hidden_units, class_num),
                            nn.LogSoftmax(dim=1))
    model.classifier = classifier

    # Define loss / optimizer
    #criterion = nn.NLLLoss()
    criterion = nn.CrossEntropyLoss()
    #optimizer = optim.Adam(model.classifier.parameters(), lr = learn_rate)
    optimizer = optim.SGD(model.classifier.parameters(), lr = learn_rate)

    model.to(device)

    # train classifier
    start = time.time()
    print('start training')
    for epoch in range(epochs):
        for images, labels in trainloader:
            steps += 1

            # 把张量转移到GPU上
            images, labels = images.to(device), labels.to(device)

            # 编写training循环
            # 首先，清零梯度，这步很重要，不要忘了
            optimizer.zero_grad()

            logps = model(images)
            loss = criterion(logps, labels)
            loss.backward()
            optimizer.step()  # 优化步骤

            train_loss += loss.item()  # 递增train_loss，这样当我们用越来越多当数据训练后，可以跟踪训练损失。

            # print("steps={steps}".format(steps=steps))

            # 跟踪验证集的损失和准确率，以确定最佳超参数
            if steps % print_every == 0:
                model.eval()  # 将模型变成评估推理模型，这样会关闭丢弃
                valid_loss = 0
                accuracy = 0

                print("steps={steps}".format(steps=steps))

                for images, labels in validloader:

                    # 把张量转移到GPU上
                    images, labels = images.to(device), labels.to(device)

                    logps = model(images)
                    loss = criterion(logps, labels)

                    valid_loss += loss.item()  # 跟踪测试损失

                    # calculate our accuracy
                    ps = torch.exp(logps) #模型返回的是logSoftmax，表示类别的对数概率，要得到实际的概率，要用.exp()

                    top_ps, top_class = ps.topk(1, dim=1)

                    equality = top_class == labels.view(*top_class.shape) #检查结果是否与lables匹配
                    accuracy += torch.mean(equality.type(torch.FloatTensor)).item() #根据equality张量更新accuracy

                print(f"Epoch {epoch+1}/{epochs}.. "  #看看where we are
                    f"Train loss: {train_loss/print_every:.3f}.. "   #对训练损失取平均值
                    f"Valid loss: {valid_loss/len(validloader):.3f}.. "   #len(validloader)表示我们从validloader获得了多少批数据
                    f"Valid accuracy: {accuracy/len(validloader):.3f}"   #测试集的平均准确率
                )

                train_loss = 0
                model.train()  #改回train模式

    traing_time = time.time() - start
    print("finish training!")
    print("Total time: {:.0f}m {:.0f}s".format(traing_time//60, traing_time % 60))

    # do validation on test data
    functions.validation_all_test_data(model, testloader)

    # save the model to checkpoint
    checkpoint = {'state_dict': model.state_dict(),
                  'class_to_idx': train_data.class_to_idx,
                  'optimizer_state_dict': optimizer.state_dict(),
                  'classifier': model.classifier,
                  'epochs': epochs,
                  'loss': train_loss,
                  'arch': args.arch
            }
    torch.save(checkpoint, save_dir)
    print("save checkpoint file succesfully!")


text:
    #!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
       _
      /  |     | __  _ __  _
     /   |    /  |_||_|| ||
    /    |   /   |  |\ | ||_
   /____ |__/\ . |  | \|_|\_|
   __________________________ .

███████╗██████╗  ██████╗  ██████╗██╗  ██╗
██╔════╝██╔══██╗██╔═══██╗██╔════╝██║  ██║
█████╗  ██████╔╝██║   ██║██║     ███████║
██╔══╝  ██╔═══╝ ██║   ██║██║     ██╔══██║
███████╗██║     ╚██████╔╝╚██████╗██║  ██║
╚══════╝╚═╝      ╚═════╝  ╚═════╝╚═╝  ╚═╝
Created on Wed May 30 15:34:05 2018

@author: chrisunderwood

To compare the outputted Electron spectrums,
as part of a parameter scan
"""

import numpy as np
import os
import matplotlib.pyplot as plt
import seaborn as sns

#==============================================================================
# A function that replicates os.walk with a max depth level
#==============================================================================
def walklevel(some_dir, level=1):
    some_dir = some_dir.rstrip(os.path.sep)
    assert os.path.isdir(some_dir)
    num_sep = some_dir.count(os.path.sep)
    for root, dirs, files in os.walk(some_dir):
        yield root, dirs, files
        num_sep_this = root.count(os.path.sep)
        if num_sep + level <= num_sep_this:
            del dirs[:]

#==============================================================================
# Creates a list of the folders of interest
#==============================================================================
def listFolders(mainDir):
    listSubFolders =  [x[0] for x in walklevel(mainDir)][1:]
    folderNames = []
    #Modify so useable path
    for i in range(len(listSubFolders)):

        listSubFolders[i] += '/'
        #Add the folder that I am looking into here too!
        listSubFolders[i] += 'Dist_evo/'

        folderNames.append(listSubFolders[i].split('/')[-3])
    listSubFolders = np.array(listSubFolders)
    folderNames = np.array(folderNames)
    return listSubFolders, folderNames

def nearposn(array,value):
	#Find array position of value
    posn = (abs(array-value)).argmin()
    return posn

def subplotPerSpectra(data, Crop):
    sns.set_palette(sns.color_palette("Set1", len(folderNames)))
    sns.set_context("talk")
    sns.set_style('darkgrid')

    fig, axes = plt.subplots(nrows = len(data), sharex = True, figsize = (7,8))

    for d, names, ax in zip(data, folderNames, axes):
            yLims = [1e50, 0]

            px = d[:,0]
            Energy_J = (px ** 2) / (2 * 9.11e-31)

            Energy_eV = Energy_J / 1.6e-19
            Energy_MeV = Energy_eV * 1e-6
            xlow = nearposn(Energy_MeV, Crop[0])
            xhigh = nearposn(Energy_MeV, Crop[1])
        #    print xlow, xhigh
            #    xlow = 50; xhigh = 400
            intensity = d[:,1]
            cropI = intensity[xlow:xhigh]
            if cropI.min() < yLims[0]:
                yLims[0] = cropI.min()
            if cropI.max() > yLims[1]:
                yLims[1] = cropI.max()

        #    print fp
            if plot_MeV:
                xAxis = Energy_MeV
            else:
                xAxis = Energy_J
            ax.plot(xAxis, intensity)
            ax.set_title('Blade Translation '  + names[1:] + 'mm')
            ax.set_ylim(yLims)
#            ax.set_ylabel('Intensity (# of electrons)')
            ax.ticklabel_format(style='sci', axis='y', scilimits=(0,0),useOffset=False)

    if plot_MeV:
        plt.xlabel('Electron Energy (MeV)')
    else:
        plt.xlabel('Electron Energy (J)')
#    plt.ylabel('Intensity (# of electrons)')
    fig.text(0.02, 0.5, 'Intensity (# of electrons)', ha='center', va='center', rotation='vertical')
    #==============================================================================
    # Apply the plotting limits
    #==============================================================================
    #plt.xlim([-1e-14, 1e-13])
    #plt.yscale('log')
    #
#    if logPlot:
#        plt.ylim([yLims[1]/1e5, yLims[1]])
#        plt.yscale('log')
#    else:
#        plt.ylim(yLims)
#
    plt.xlim([xAxis[xlow],xAxis[xhigh]])

    plt.legend()

def createPlotOfAll_e_spectra(folderPaths, folderNames, Crop_X, Crop_Y = False):
    sns.set_palette(sns.color_palette("Set1", len(folderNames)))
    sns.set_context("talk")
    sns.set_style('darkgrid')
    yLims = [1e50, 0]
    data = []
    plt.figure(figsize = (10,7))
    for fp, names in zip(folderPaths, folderNames):
        fp += 'Electron_Spectrum.txt'

        try:
            #Assuming that the first row is currently px
            d = np.loadtxt(fp)
            data.append(d)
            px = d[:,0]
            Energy_J = (px ** 2) / (2 * 9.11e-31)

            Energy_eV = Energy_J / 1.6e-19
            Energy_MeV = Energy_eV * 1e-6
            xlow = nearposn(Energy_MeV, Crop_X[0])
            xhigh = nearposn(Energy_MeV, Crop_X[1])
        #    print xlow, xhigh
            #    xlow = 50; xhigh = 400
            intensity = d[:,1]
            if not Crop_Y:
                cropI = intensity[xlow:xhigh]
                if cropI.min() < yLims[0]:
                    yLims[0] = cropI.min()
                if cropI.max() > yLims[1]:
                    yLims[1] = cropI.max()
            else:
                yLims = Crop_Y

        #    print fp
            if plot_MeV:
                xAxis = Energy_MeV
            else:
                xAxis = Energy_J
            plt.plot(xAxis, intensity, label = names)
            plt.xlim([xAxis[xlow],xAxis[xhigh]])

        except:
            print 'Error Reading File'
            print '    ' + fp

    if plot_MeV:
        plt.xlabel('Electron Energy (MeV)')
    else:
        plt.xlabel('Electron Energy (J)')
    plt.ylabel('Intensity (# of electrons)')

    #==============================================================================
    # Apply the plotting limits
    #==============================================================================
    #plt.xlim([-1e-14, 1e-13])
    #plt.yscale('log')
    #
    if logPlot:
        plt.ylim([yLims[1]/1e5, yLims[1]])
        plt.yscale('log')
    else:
        plt.ylim(yLims)

    plt.legend()
    print 'Crop corresponds to: ', [xAxis[xlow],xAxis[xhigh]], ' MeV'
    print 'Range of inputed data is: ', Energy_MeV[0], Energy_MeV[-1]
    return data

hdrive = '/Volumes/CIDU_passport/2018_Epoch_vega_1/'
gdrive = '/Volumes/GoogleDrive/My Drive/'
gdrive += '2018_Epoch_vega_1/'

#hdrive += '0601_Gaus_for_wavebreak/'
#fileSplice = [8,None]

#hdrive += '0607_Intensity_Scan/'
#fileSplice = [1,-11]

#hdrive += '0612_profileScan/'
#fileSplice = [2,None]

#hdrive = gdrive + '0711_highRes_selfInjection/'
#fileSplice = [-4,None]

#hdrive = gdrive + '0721_HR_Jump/'
#fileSplice = [-4,None]

hdrive = hdrive + '1010_SlurmJob/'
fileSplice = [10,12]

#hdrive = gdrive + '1018_vega1_Jump/'
#fileSplice = [2,None]

folderPaths, folderNames = listFolders(hdrive)
logPlot = False
plot_MeV = True
#==============================================================================
# Search for the set of folders to look at!
#==============================================================================
starts = ''
#starts = ''

fins = 'FS'
#Index_to_save = [i for i in xrange(len(folderNames)) if folderNames[i].endswith(fins)]
Index_to_save = [i for i in xrange(len(folderNames)) if folderNames[i].startswith(starts)]
#Index_to_save = [i for i in xrange(len(folderNames)) if folderNames[i].startswith(starts) and folderNames[i].endswith('23')]

#Modify the both arrays to just be the ones of interest
folderPaths = folderPaths[Index_to_save]
folderNames = folderNames[Index_to_save]
print folderNames

#==============================================================================
# Crop the axis to the interesting data
#==============================================================================
Energy_Crop = [1, 5]    # In MeV
IntensityCrop = [0, 0.5e8]

#==============================================================================
# Slice name for number to sort by
#==============================================================================
Num = []
for f in folderNames:
    Num.append(float(f[fileSplice[0]:fileSplice[1]]))
print Num
sort =  sorted(zip(Num, folderNames, folderPaths))
folderNames = [x[1] for x in sort]
folderPaths = [x[2] for x in sort]

print 'Sorted'
print folderNames

#folderNames = folderNames[:-1]

data = createPlotOfAll_e_spectra(folderPaths, folderNames, Energy_Crop, IntensityCrop)
plt.savefig(hdrive + 'Electron_spectrum.png')
plt.show()

#data = data[:4]
subplotPerSpectra(data, Energy_Crop)
plt.tight_layout()
plt.savefig(hdrive + 'Electron_spectrums_in_subplot.png', dpi = 300)

max_stars_repo_name: chunders/EpochAnalysis

max_issues_repo_name: chunders/EpochAnalysis

max_forks_repo_name: chunders/EpochAnalysis


text:
# -*- coding: utf-8 -*-
# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
#

"""
Created on Sat Sep 19 20:55:56 2015

@author: liangshiyu
"""

from __future__ import print_function
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
import numpy as np
import time
from scipy import misc
import calMetric as m
import calData as d
#CUDA_DEVICE = 0

start = time.time()
#loading data sets

transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((125.3/255, 123.0/255, 113.9/255), (63.0/255, 62.1/255.0, 66.7/255.0)),
])

# loading neural network

# Name of neural networks
# Densenet trained on CIFAR-10:         densenet10
# Densenet trained on CIFAR-100:        densenet100
# Densenet trained on WideResNet-10:    wideresnet10
# Densenet trained on WideResNet-100:   wideresnet100
#nnName = "densenet10"

#imName = "Imagenet"

criterion = nn.CrossEntropyLoss()

def recursion_change_bn(module):
    if isinstance(module, torch.nn.BatchNorm2d):
        module.track_running_stats = 1
    else:
        for i, (name, module1) in enumerate(module._modules.items()):
            module1 = recursion_change_bn(module1)
    return module

def test(nnName, dataName, CUDA_DEVICE, epsilon, temperature):

    net1 = torch.load("../models/{}.pth".format(nnName))
    for i, (name, module) in enumerate(net1._modules.items()):
        module = recursion_change_bn(module)

    optimizer1 = optim.SGD(net1.parameters(), lr = 0, momentum = 0)
    net1.cuda(CUDA_DEVICE)

    if dataName != "Uniform" and dataName != "Gaussian":
        testsetout = torchvision.datasets.ImageFolder("../data/{}".format(dataName), transform=transform)
        testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=1,
                                         shuffle=False, num_workers=2)

    if nnName == "densenet10" or nnName == "wideresnet10":
        testset = torchvision.datasets.CIFAR10(root='../data', train=False, download=True, transform=transform)
        testloaderIn = torch.utils.data.DataLoader(testset, batch_size=1,
                                         shuffle=False, num_workers=2)
    if nnName == "densenet100" or nnName == "wideresnet100":
        testset = torchvision.datasets.CIFAR100(root='../data', train=False, download=True, transform=transform)
        testloaderIn = torch.utils.data.DataLoader(testset, batch_size=1,
                                         shuffle=False, num_workers=2)

    if dataName == "Gaussian":
        d.testGaussian(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderIn, nnName, dataName, epsilon, temperature)
        m.metric(nnName, dataName)

    elif dataName == "Uniform":
        d.testUni(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderIn, nnName, dataName, epsilon, temperature)
        m.metric(nnName, dataName)
    else:
        d.testData(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderOut, nnName, dataName, epsilon, temperature)
        m.metric(nnName, dataName)

max_stars_repo_name: aangelopoulos/ltt

max_issues_repo_name: aangelopoulos/ltt

max_forks_repo_name: aangelopoulos/ltt


text:
# -*- coding: utf-8 -*-
"""
Created on Tue May  9 11:30:42 2017

@author: Mic
"""

# -*- coding: utf-8 -*-
"""
Created on Mon Mar 20 11:56:47 2017

@author: Mic

- Funziona
- la lascio che c'è da correggere in EllipticalMirror pTan e quindi VersosNorm
- la lascio che c'è ancora l'inifto bisticco tra XYCentre e XYCentre, XYkb1 e XYkb1

"""
import warnings
warnings.filterwarnings("ignore")

import importlib
import numpy as np

import wiselib2.Rayman as rm
import wiselib2.Fundation as Fundation
import wiselib2.Optics as Optics
import wiselib2.ToolLib as tl
importlib.reload(Fundation)
importlib.reload(Optics)
importlib.reload(tl)
importlib.reload(rm)

from wiselib2.must import *
from wiselib2.Fundation import OpticalElement

# SOURCE
#----------------------------------------------------
tl.Debug.On = False
Lambda = 5e-9

# Gaussian Source (if used)
FermiFactor = {'fel1' : 1.25, 'fel2':1.5}['fel2']
FermiSigma = FermiFactor * Lambda * 1e9
FermiWaist = FermiSigma * np.sqrt(2)
FermiDivergence = Lambda / np.pi/FermiWaist

s = OpticalElement(
                Optics.SourceGaussian(Lambda, FermiWaist, PropagationAngle = 0*0.0008257924757473173 ),
                PositioningDirectives = Fundation.PositioningDirectives(
                        ReferTo = 'absolute',
                        XYCentre = [0,1],
                        Angle = np.deg2rad(0)),
                Name = 'source', IsSource = True)

#----------------------------------------------------
# kb1
#----------------------------------------------------
kb1 = OpticalElement(
                Optics.MirrorElliptic(
                        f1 = 98.5, f2 = 1.180, L= 0.4, Alpha = np.deg2rad(2)),
                PositioningDirectives = Fundation.PositioningDirectives(
                        ReferTo = 'upstream',
                        PlaceWhat = 'upstream focus',
                        PlaceWhere = 'centre',
                        Distance = 98.5),
                Name = 'kb1')

#----------------------------------------------------
# det (detector)
#----------------------------------------------------
det = OpticalElement(
                Optics.OpticsNumericalDependent(
                        ParentOptics = kb1.CoreOptics,
                        GetXYFunction = kb1.CoreOptics.GetXY_TransversePlaneAtF2,
                        N = 200,
                        L = 50e-6,
                        Defocus = 0,
                        ReferenceFrame = 'lab') ,
                PositioningDirectives = None, # they are not necessary
                Name = 'det')
#----------------------------------------------------
#% Creo la sequenza di elementi
t = Fundation.BeamlineElements()
t.Append(s)
t.Append(kb1)
t.Append(det)
t.RefreshPositions()
#t.Paint()

NManual =OpticalElement.GetNSamples_2Body(s.CoreOptics.Lambda, kb1,det)

kb1.UseCustomSampling = False
kb1.CustomSamples = NManual

det.UseCustomSampling = True
det.CustomSamples = NManual

#%% propagazione automatica
print(t)
t.ComputeFields()

#%%propagazione a mano m1 --> f5
Lambda = s.OpticsCore.Lambda
m1_x, m1_y = m1.GetXY(NManual)
m1.ComputedField = s.OpticsCore.EvalField(Lambda,m1_x, m1_y)
plt.figure(9)
plt.plot(abs(m1.ComputedField))
plt.title('Amplitude on m1')

f5_x, f5_y = f5.GetXY(NManual)
f5.ComputedField = rm.HuygensIntegral_1d_MultiPool(Lambda, m1.ComputedField, m1_x, m1_y, f5_x, f5_y, NPools=1)
plt.figure(91)
plt.plot(abs(f5.ComputedField))
plt.title('Amplitude on f5')

#%% propagazione a mano  f5 ---> det

f5_x, f5_y = f5.GetXY(NManual)
det_x, det_y = det.GetXY(NManual)
det.ComputedField = rm.HuygensIntegral_1d_MultiPool(s_k.Lambda, f5.ComputedField, f5_x, f5_y, det_x, det_y, NPools=5)
#%%
plt.figure(11)
plt.plot(abs(det.ComputedField))
plt.title('Amplitude on the screen')


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Dec 17 12:50:32 2019

The polynomial SVM class.

@author: Dr. Dr. Danny E. P. Vanpoucke
@web   : https://dannyvanpoucke.be
"""
import pandas as pd
from TSVMClass import TSVMClass

class TSVMPolynomialModel(TSVMClass):
    """
    Child class of the SVM-class representing a specific type(kernel) of SVM : polynomial SVM
    """
    def __init__(self,
                 name,
                 Target,
                 Feature: pd.DataFrame,
                 Target_test,
                 Feature_test: pd.DataFrame,
                 degree: int=2,
                 gamma: str=None,
                 coef0: float=0.0,
                 max_iter: int=-1):#poly keywords
        """
        Constructor to set up a linear SVM model.
        Any SVM model has two hyper-parameters: epsilon and C. Both are selected
        using a gridsearch approach, and then storred as "best_e" and "best_C"

        parameters:
         - name : the name of the object instance
         - Feature : The features to use & transform
         - Target     : the training target data
         - Target_test: the test target data
         - Feature_test: the untransformed features for testing.
         - degree : The polynomial degree (integer), DEFAULT=2
         - gamma : Kernel coefficient for ‘rbf’.
                   if gamma='scale' is passed then it uses 1/(n_features*X.var())
                   as value of gamma,if ‘auto’, uses 1 / n_features.
                   [default = scale]
         - coef0 : Independent term in kernel function. [DEFAULT = 0.0]
                   (cf: https://scikit-learn.org/dev/modules/svm.html#kernel-functions
                   polynomial: (gamma<x,x'> + r)**d. d is specified by keyword degree, r by coef0.)
         - max_iter : Hard limit on iterations within solver, or -1 for no limit. [DEFAULT = -1]

        It sets the following properties
            - pipeline : a pipeline object containing the preprocessing transformations (excluding the fitter function)
            #- CVmodel  : the fitter function to be used (should be an sklearn function with "fit" method)
            - model : The model is only set once the CV_model has run a fit-operation
            - feature_tf: the transformed features as obtained by the pipeline
        """
        from sklearn.pipeline import Pipeline
        from sklearn.preprocessing import StandardScaler
        from sklearn.svm import SVR

        super().__init__(name,Target, Feature,Target_test, Feature_test)
        self.nameModel='poly SVM Model'
        self.name=name
        print("Initialising the child class:",self.nameModel)
        #create a pipeline (can be extended to contain more functions, p67)
        self.pipeline = Pipeline([
            ('std_scaler', StandardScaler()),  #scaling to be centered on 0, with unit variance...since the values are quite different, this will help things
        ]) #don't include the fitter

        self.feature_tf = self.pipeline.fit_transform(Feature) #this is a numpy array...

        #to keep track of some options:
        if (gamma == None):
            gamma='scale'

        self.degree=degree
        self.gamma=gamma
        self.coef0=coef0
        self.max_iter=max_iter
        self.model=SVR(kernel='poly',degree=self.degree, gamma=self.gamma,
                       coef0=self.coef0, max_iter=self.max_iter) #we need a temporary starting model for the grid search...here we need to set all parameters except C and epsilon
        self.best_C=0   #C:penalty giving rise to regularization. Little C means big regularization
        self.best_e=0   #epsilon: width of the street...wants to be as big as possible(as many targets n the street without violations)
        self.best_score=0 #the score of the "best" gridrearch result
        self.hyperTuned=False #Hyper parameters are not yet tuned

    #def fit(self, Target: pd.DataFrame):
        """
        Class-method wrapping the fit-method of the sklearn model
           - Target : a pandas dataframe with the Target data belonging to the
                   Features provided upon initialisation.
        """
    #    from sklearn.svm import SVR

    #    self.model=self.CVmodel
    #    self.model.fit(self.feature_tf,Target)
    #    print("did some fitting, Parent-style:",type(self.model).__name__)

    def fitHyper(self):
        """
        Class-method performing a gridsearch to define optimum values for the
        C and epsilon hyper-parameters.
           --> focus on SVM with linear kernel
        """
        from sklearn.model_selection import GridSearchCV
        from sklearn.model_selection import LeaveOneOut
        from sklearn.svm import SVR
        from GridModule import Grid1D
        import numpy as np

        #   [0.001, 0.01, 0.1, 1.0, 10, 100, 1000]
        LoO_CV=LeaveOneOut()
        Clow=-3
        Chigh=3
        NG=7
        Elow=-3
        Ehigh=3
        Clst=Grid1D(Min=Clow,Max=Chigh,NGrid=NG,GridType="log")
        Elst=Grid1D(Min=Elow,Max=Ehigh,NGrid=NG,GridType="log")
        done=False
        oldScore=1.0E10 #something very bad
        thress=1.0E-3

        cnt=0
        while not done:
            cnt+=1
            param_grid=[{'C':Clst,
                         'epsilon':Elst}]

            grid_search=GridSearchCV(self.model,param_grid,
                                 scoring='neg_mean_squared_error', #RMSE^2, neg-> higher value is better
                                 n_jobs=-1,             # parallel, all cores
                                 pre_dispatch='n_jobs', # spawn as many jobs as there will run in parallel
                                 cv=LoO_CV,             # Leave-One-Out Cross-validation (small data)
                                 error_score=np.NaN,    # Some parameter-combos may lead to failure to fit. (Robustness to Failure)
                                                        #  Instead of crashing the entire CV job, just set
                                                        #  their score to NaN, and contine.. as such they
                                                        #  are ignored in the end compilation of the best result

                                 )
            grid_search.fit(self.feature_tf,self.target)
            newScore=grid_search.best_score_
            print(cnt,' best param= ',grid_search.best_params_)
            print(cnt,' best score= ',grid_search.best_score_)
            if abs((newScore-oldScore)/newScore)<thress:
                #score isn't changeing enough we reached the end
                done=True
                print(" +-+-+-+ DONE abs< thress")
            elif (cnt>10):
                #stop running, something bad is going on
                print("WARNING in TSVMLinearModel. Hypertunning takes too many runs. Stopping and hoping for the best.")
                done=True
            else:
                print(" +-+-+-+ ELSE abs< thress")
                #now check if we are not on the edge:
                Cval=grid_search.best_params_.get('C')
                Cpow=int(np.log10(Cval))#get rid of rounding errors
                if ((Cpow>Clow)and(Cpow<Chigh)):#it is on the inside
                    Cdone=True
                    Clst=[Cval]
                else:#it is on an edge
                    Cdone=False
                    if (Cpow==Clow):#lower bound
                        Chigh=Clow+1
                        Clow=Chigh-NG+1#7 points have 6 steps in between
                    else: #upper bound
                        Clow=Chigh-1
                        Chigh=Clow+NG-1
                    Clst=Grid1D(Min=Clow,Max=Chigh,NGrid=NG,GridType="log")

                print("    ++++++ Clow < Cval < Chigh == ",Clow," < ",Cpow," < ",Chigh)
                print("    ++++++ Clst =",Clst )

                Eval=grid_search.best_params_.get('epsilon')
                Epow=int(np.log10(Eval))#get rid of rounding errors
                if ((Epow>Elow)and(Epow<Ehigh)):#it is on the inside
                    Edone=True
                    Elst=[Eval]
                else:#it is on an edge
                    Edone=False
                    if (Epow==Elow):#lower bound
                        Ehigh=Elow+1
                        Elow=Ehigh-NG+1#7 points have 6 steps in between
                    else: #upper bound
                        Elow=Ehigh-1
                        Ehigh=Elow+NG-1
                    Elst=Grid1D(Min=Elow,Max=Ehigh,NGrid=NG,GridType="log")

                print("    ++++++ Elow < Eval < Ehigh == ",Elow," < ",Epow," < ",Ehigh)
                print("    ++++++ Elst =",Elst )

                done=(Cdone and Edone)
            oldScore=newScore #store for the next round of while

        self.best_score=newScore
        print('best param= ',grid_search.best_params_)
        print('best score= ',grid_search.best_score_)
        self.best_C=grid_search.best_params_.get('C')
        self.best_e=grid_search.best_params_.get('epsilon')

        #and now we set the actual model
        self.hyperTuned=True
        self.model=SVR(kernel='poly',
                       degree=self.degree,
                       gamma=self.gamma,
                       coef0=self.coef0,
                       C=self.best_C,
                       epsilon=self.best_e,
                       cache_size=500, #cach size in MB, can improve speed
                       max_iter=self.max_iter,  # maximum number of iterations for the solver...default is -1 (infinite)...which it will sometimes take so it shoul dbe avoided
                       )

    def setCoefficients(self):
        """
        Class-method printing the fitting coefficients for a polynomial regression
        with elastic net regularization
        """
        import numpy as np
        super().setCoefficients()

        #--------- hyper parameters -------------------
        self.modelcoef['header_hyperparameter']=[self.coefindex,"The selected hyper-parameters for the ",self.nameModel," are:"]
        line="  - C-penalty = %0.3e  " % (self.best_C )
        self.modelcoef['C-penalty']=[-(self.coefindex+1),line]
        line="  - epsilon   = %0.3e  " % (self.best_e )
        self.modelcoef['epsilon']=[-(self.coefindex+2),line]
        line="  - score   = %0.3f  " % (self.best_score )
        self.modelcoef['score']=[-(self.coefindex+3),line]
        line="  - fit OK   = %r  " % (self.model.fit_status_ == 0 )
        self.modelcoef['fitOK']=[-(self.coefindex+4),line]
        #------------usual coefficients-----------------
        self.coefindex+=5
        self.modelcoef['header_coef']=[self.coefindex,"The coefficients for each target (one per row) are given by:"]
        self.modelcoef['coef_']=[self.coefindex+1,np.matrix(self.model.dual_coef_)] #there is no coef_ attribute for a polynomial model
        self.modelcoef['header_intercept']=[self.coefindex+2,"The intercepts for each target (one per row) are given by:"]
        self.modelcoef['intercept_']=[self.coefindex+3,np.matrix(self.model.intercept_)]
        self.coefindex+=4

max_stars_repo_name: DannyVanpoucke/Amadeus

max_issues_repo_name: DannyVanpoucke/Amadeus

max_forks_repo_name: DannyVanpoucke/Amadeus