text:
"""Evaluating AutoARIMA model on M4 timeseries

"""

from darts import ModelMode
from darts.models import AutoARIMA, NaiveSeasonal
from darts.utils.statistics import check_seasonality, remove_from_series, extract_trend_and_seasonality
from darts.utils import _build_tqdm_iterator

import numpy as np
import pandas as pd
import pickle as pkl

from M4_metrics import owa_m4, mase_m4, smape_m4

if __name__ == "__main__":

    data_categories = ['Yearly', 'Quarterly', 'Monthly', 'Weekly', 'Daily', 'Hourly']

    info_dataset = pd.read_csv('dataset/M4-info.csv', delimiter=',').set_index('M4id')

    for cat in data_categories[::-1]:
        # Load TimeSeries from M4
        ts_train = pkl.load(open("dataset/train_"+cat+".pkl", "rb"))
        ts_test = pkl.load(open("dataset/test_"+cat+".pkl", "rb"))

        # Test models on all time series
        mase_all = []
        smape_all = []
        m = int(info_dataset.Frequency[cat[0]+"1"])
        for train, test in _build_tqdm_iterator(zip(ts_train, ts_test), verbose=True):
            train_des = train
            seasonOut = 1
            if m > 1:
                if check_seasonality(train, m=m, max_lag=2*m):
                    _, season = extract_trend_and_seasonality(train, m, model=ModelMode.MULTIPLICATIVE)
                    train_des = remove_from_series(train, season, model=ModelMode.MULTIPLICATIVE)
                    seasonOut = season[-m:].shift(m)
                    seasonOut = seasonOut.append_values(seasonOut.values())
                    seasonOut = seasonOut[:len(test)]
            # if len(train_des) < 30:
            #     train_des = train_des.shift(-len(train)).append(train_des)
            autoar = AutoARIMA()
            try:
                autoar.fit(train_des)
            except ValueError:
                autoar = NaiveSeasonal(K=1)
                autoar.fit(train_des)
            forecast_autoar = autoar.predict(len(test)) * seasonOut
            mase_all.append(np.vstack([
                mase_m4(train, test, forecast_autoar, m=m),
                ]))
            smape_all.append(np.vstack([
                smape_m4(test, forecast_autoar),
                ]))
        pkl.dump(mase_all, open("autoar_mase_"+cat+".pkl", "wb"))
        pkl.dump(smape_all, open("autoar_smape_"+cat+".pkl", "wb"))
        print("MASE; ARIMA: {}".format(*tuple(np.nanmean(np.stack(mase_all), axis=(0, 2)))))
        print("sMAPE; ARIMA: {}".format(*tuple(np.nanmean(np.stack(smape_all), axis=(0, 2)))))
        print("OWA: ", owa_m4(cat, np.nanmean(np.stack(mase_all), axis=(0, 2)),
                              np.nanmean(np.stack(smape_all), axis=(0, 2))))

max_stars_repo_name: LeoTafti/darts

max_issues_repo_name: LeoTafti/darts

max_forks_repo_name: LeoTafti/darts


text:
import os
from collections import Counter

import numpy as np
import pandas as pd
from PIL import Image

# Get the file paths
if __name__ == "__main__":
    path = "/hkfs/work/workspace/scratch/im9193-H5/data/imgs/"
    im_files = os.listdir(path)

    # imagine we only want to load PNG files (or JPEG or whatever...)
    EXTENSION = ".png"

    def get_characteristics(path, f):
        image = Image.open(path + f)
        img = np.array(image.getdata()).reshape(image.size)
        name = f
        shape = img.shape
        max = np.max(img)
        min = np.min(img)
        mean = np.mean(img)
        return name, shape, max, min, mean

    # Load using matplotlib
    # images_plt_shape = [plt.imread(path + f).shape for f in im_files if f.endswith(EXTENSION)]
    # convert your lists into a numpy array of size (N, H, W, C)

    images_characteristics = [
        get_characteristics(path, f) for f in im_files if f.endswith(EXTENSION)
    ]
    # [print(image.shape) for image in  images_plt]
    # images = np.array(images_plt)

    df = pd.DataFrame(
        images_characteristics, columns=["name", "shape", "max", "min", "mean"]
    )

    df.to_csv("metadata.tsv", sep="\t", index=False)
    # counter = Counter(images_plt_shape)

    # print(counter)

ext: py

reason: import numpy


text:
"""
In the process of being written

"""
import sys
import os
from os.path import join
import glob
import numpy as n
import astropy.io.fits as fits
import SpectraStackingEBOSS as sse
from scipy.interpolate import interp1d
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as p
from scipy.integrate import quad

# ICM GCM lines absorption

line_names_ICM_CGM_abs = {
	'Mg_1_2853':  'MgI 2853',
	'Mg_2_2796': 'MgII 2796',
	'Mg_2_2804': 'MgII 2804',
	'Mn_2_2577': 'MgII 2577',
	'Mn_2_2594': 'MgII 2594',
	'Mn_2_2606': 'MgII 2606',
    'Fe_2_2600': 'FeII 2600',
	'Fe_2_2587': 'FeII 2587',
	'Fe_2_2383': 'FeII 2383',
	'Fe_2_2374': 'FeII 2374',
	'Fe_2_2344': 'FeII 2344',
	'Fe_2_2261': 'FeII 2261',
	'Fe_2_2250': 'FeII 2250'
}

line_lambda_ICM_CGM_abs = {
	'Mg_1_2853': 2852.96,
	'Mg_2_2796': 2796.35,
	'Mg_2_2804': 2803.53,
	'Mn_2_2577': 2576.88,
	'Mn_2_2594': 2594.50,
	'Mn_2_2606': 2606.46,
	'Fe_2_2600': 2600.17,
	'Fe_2_2587': 2586.55,
	'Fe_2_2383': 2382.76,
	'Fe_2_2374': 2374.46,
	'Fe_2_2344': 2344.21,
	'Fe_2_2261': 2260.78,
	'Fe_2_2250': 2249.88
}

# ICM GCM lines emission
line_names_ICM_CGM_abs = {
	'Fes_2_2632': 'FeII* 2632',
	'Fes_2_2626': 'FeII* 2626',
	'Fes_2_2613': 'FeII* 2613',
	'Fes_2_2396': 'FeII* 2396',
	'Fes_2_2381': 'FeII* 2381',
	'Fes_2_2366': 'FeII* 2366',
	'Fes_2_2281': 'FeII* 2281',
	'Fes_2_2270': 'FeII* 2270',

}

line_lambda_ICM_CGM_abs = {
	'Fes_2_2632': 2632.11,
	'Fes_2_2626': 2626.45,
	'Fes_2_2613': 2612.65,
	'Fes_2_2396': 2396.35,
	'Fes_2_2381': 2381.49,
	'Fes_2_2366': 2365.55,
	'Fes_2_2281': 2280.62,
	'Fes_2_2270': 2269.52,
}

# nebular emission
line_names_nebular_emi = {
	'C2_2324': 'CII] 2324',
	'C2_2325': 'CII] 2325',
	'C2_2326': 'CII] 2326',
	'C2_2328': 'CII] 2328',
	'C2_2329': 'CII] 2329',
	'Ne4_2423': '[NeIV] 2423',
	'Ne4_2425': '[NeIV] 2425',
	'O2_2470': '[OII] 2470',
	'O2_2471': '[OII] 2471'
}

line_lambda_nebular_emi = {
	'C2_2324': 2324.21,
	'C2_2325': 2325.40,
	'C2_2326': 2326.11,
	'C2_2328': 2327.64,
	'C2_2329': 2328.83,
	'Ne4_2423': 2422.56,
	'Ne4_2425': 2425.14,
	'O2_2470': 2470.97,
	'O2_2471': 2471.09
	}

def normalize_C0(x,y,yerr):
	"""
	We then mask out
	absorption and emission features and fit a cubic polyno-
	mial function through the rest of the spectrum.
	Using
	the best-fit polynomial function as an estimate of the un-
	derlying continuum, F lambda
	we normalize the observed spectrum to obtain the continuum-normalized spectrum
	"""
	# masking sky contaminated pixels
	maskLambda = n.loadtxt(os.path.join(os.environ['GIT_SPM'],'data',"dr12-sky-mask.txt"), unpack=True)
	ratio = n.min(abs(10000.*n.log10(n.outer(x, 1./maskLambda))), axis=1)
	margin = 1.5
	veto_sky = ( ratio <= margin )

	# UV mask
	UV_mask = (x>2000)&(x<3600)

	# UV line mask
	ratio = n.min(abs(10000.*n.log10(n.outer(x, 1./line_list_abs))), axis=1)
	margin = 10
	veto_line_abs = ( ratio <= margin )

	ratio = n.min(abs(10000.*n.log10(n.outer(x, 1./line_list_em))), axis=1)
	margin = 10
	veto_line_em = ( ratio <= margin )

	# MASKING BAD DATA
	bad_data = n.isnan(y) | n.isinf(y) | (y <= 0.0) | n.isnan(yerr) | n.isinf(yerr)
	# creating new arrays
	x = x[(UV_mask)&(veto_sky==False)&(bad_data==False)&(veto_line_abs==False)&(veto_line_em==False)]
	y = y[(UV_mask)&(veto_sky==False)&(bad_data==False)&(veto_line_abs==False)&(veto_line_em==False)]
	yerr = yerr[(UV_mask)&(veto_sky==False)&(bad_data==False)&(veto_line_abs==False)&(veto_line_em==False)]

	out=n.polyfit(x, y, 4, w=1/yerr)
	return out

def W0_abs(x, y, l_min, l_max):
	it = interp1d(x,1-y)
	return quad(it, l_min, l_max)[0]

def W0_emi(x, y, l_min, l_max):
	it = interp1d(x,y-1)
	return quad(it, l_min, l_max)[0]

def open_spec(path_2_file):
	return fits.open(path_2_file)[1].data

def create_xy_data(path_2_file):
	# opens spectrum
	dd=open_spec(path_2_file)
	# normalizes with a fourth order polynomial
	wl=dd['wavelength'         ]
	s1 = (wl>2200)&(wl<2900)&(dd['NspectraPerPixel'   ]>0.5*n.max(dd['NspectraPerPixel'   ]))
	#, label= bnl, lw=0.7 )
	pfit = normalize_C0(dd['wavelength'         ][s1], dd['medianStack'        ][s1] , dd['jackknifStackErrors'][s1])
	Fcont = n.polyval(pfit, dd['wavelength'         ][s1])
	wave = dd['wavelength'         ][s1]
	spec = dd['medianStack'        ][s1] / Fcont
	specErr = dd['jackknifStackErrors'        ][s1] / Fcont
	return wave, spec, specErr

# list of spectra

qty = 'mass'
qty = 'g'
qty = 'gr'
qty = 'rz'
qty = 'rw1'

dataList_UV = n.array(glob.glob(join(os.environ['HOME'], "SDSS/stacks", "eboss-elg_*_"+qty+"_*.stack")))

idx=0
x, y, ye = create_xy_data(dataList_UV[idx])
# now fitting line features :

def plot_v_profile(x,y,ye):
	# Figure 8.
	fig=p.figure(0, (5., 8.), frameon=False)
	# top panel
	fig.add_subplot(111, ylabel=r'F/Fcont', xlim=((-800, 600)), ylim=((0.9,2)))
	#p.plot( x-line_lambda_ICM_CGM_abs['Fes_2_2632'],y,label=line_names_ICM_CGM_abs['Fes_2_2632'] )
	p.plot( x-line_lambda_ICM_CGM_abs['Fes_2_2626'],y,label=line_names_ICM_CGM_abs['Fes_2_2626'] )
	p.plot( x-line_lambda_ICM_CGM_abs['Fes_2_2613'],y,label=line_names_ICM_CGM_abs['Fes_2_2613'] )
	p.plot( x-line_lambda_ICM_CGM_abs['Fes_2_2396'],y,label=line_names_ICM_CGM_abs['Fes_2_2396'] )
	p.plot( x-line_lambda_ICM_CGM_abs['Fes_2_2381'],y,label=line_names_ICM_CGM_abs['Fes_2_2381'] )
	p.plot( x-line_lambda_ICM_CGM_abs['Fes_2_2366'],y,label=line_names_ICM_CGM_abs['Fes_2_2366'] )
	p.plot( x-line_lambda_ICM_CGM_abs['Fes_2_2281'],y,label=line_names_ICM_CGM_abs['Fes_2_2281'] )
	p.plot( x-line_lambda_ICM_CGM_abs['Fes_2_2270'],y,label=line_names_ICM_CGM_abs['Fes_2_2270'] )
	p.legend(frameon=False)
	p.grid()

	fig.add_subplot(112, ylabel=r'F/Fcont', xlim=((-800, 600)), ylim=((0.9,2)))

	p.grid()

	p.tight_layout()
	p.savefig(join(os.environ['HOME'], "SDSS/stacks", "eboss-elg_"+qty+".UVstack")+".png")
	p.clf()

	p.figure(0,(9,5))
	p.title(qty)
	for specList in dataList:
		bn = os.path.basename(specList)[10:-8]
		dd=fits.open(specList)[1].data
		wl=dd['wavelength'         ]
		s1 = (dd['NspectraPerPixel'   ]>0.5*n.max(dd['NspectraPerPixel'   ]))
		p.plot(dd['wavelength'         ][s1], dd['medianStack'        ][s1], label= bn )
		#dd['meanStack'          ][s1]
		#dd['meanWeightedStack'  ][s1]
		#dd['jackknifeSpectra'   ][s1]
		#dd['jackknifStackErrors'][s1]
		#dd['NspectraPerPixel'   ][s1]

	p.legend(frameon=False)
	p.tight_layout()
	p.savefig(join(os.environ['HOME'], "SDSS/stacks", "eboss-elg_"+qty+".stack")+".png")
	p.clf()

#plot_me(qty = 'mass')
#plot_me(qty = 'g'   )
#plot_me(qty = 'gr'  )
#plot_me(qty = 'rz'  )
#plot_me(qty = 'rw1' )

#os.system("cp -r ~/SDSS/stacks/*.png ~/wwwDir/sdss/elg/stacks/")

max_stars_repo_name: AndresSixtos/pyeBOSS

max_issues_repo_name: AndresSixtos/pyeBOSS

max_forks_repo_name: AndresSixtos/pyeBOSS

reason: import numpy,from scipy,import astropy


text:
# -*- coding: utf-8 -*-
#******************************************************************************************
# Copyright (c) 2019 Hitachi, Ltd.
# All rights reserved. This program and the accompanying materials are made available under
# the terms of the MIT License which accompanies this distribution, and is available at
# https://opensource.org/licenses/mit-license.php
#
# March 1st, 2019 : First version.
#******************************************************************************************
from keras.models import load_model
from keras.utils import HDF5Matrix
import numpy as np
import sys

from pathlib import Path

_proj_dir = Path(__file__).absolute().parent.parent.parent
_lib_dir = Path(_proj_dir, 'lib')
_examples_gtsrb = Path(_proj_dir, 'examples', 'gtsrb')

sys.path.append(str(_proj_dir))
sys.path.append(str(_lib_dir))
from lib.coverage_verification import main

_model_name = 'gtrsb_image_classification_model.hdf5'
_dataset_name = 'test_dataset_3_32_32.h5'
_conf_name = 'config.json'

def get_gtsrb_model(model_path):
    print('Load Model : %s' % model_path)
    return load_model(model_path)

def get_gtsrb_dataset(dataset_path):
    print('Load DataSet : %s' % dataset_path)
    x_test = HDF5Matrix(dataset_path, "images")
    x_test = np.array(x_test)

    return x_test

if __name__ == '__main__':
    # Model file path to use
    gtsrb_model_path = Path(_examples_gtsrb, 'model', _model_name)

    # Dataset file path to use
    gtsrb_dataset_path = Path(_examples_gtsrb, 'data', _dataset_name)

    # Load model
    model = get_gtsrb_model(str(gtsrb_model_path))

    # Load Dataset
    test_x = get_gtsrb_dataset(str(gtsrb_dataset_path))

    # Config Path
    conf_dir = Path(_lib_dir, 'config')
    conf_path = Path(conf_dir, _conf_name)

    # Run DNN Coverage
    main(model, test_x, str(conf_path))


text:
"""
Robbin Bouwmeester

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This code is used to train retention time predictors and store
predictions from a CV procedure for further analysis.

This project was made possible by MASSTRPLAN. MASSTRPLAN received funding
from the Marie Sklodowska-Curie EU Framework for Research and Innovation
Horizon 2020, under Grant Agreement No. 675132.
"""

from sklearn.model_selection import cross_val_predict
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold

import xgboost as xgb
from sklearn.linear_model import Lasso
from sklearn.ensemble import AdaBoostRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import ARDRegression

from sklearn.base import clone
from sklearn.model_selection import RandomizedSearchCV

from scipy.stats import randint
from scipy.stats import uniform

import pandas as pd

import pickle

def train_model_l1(X,y,params,model,n_jobs=24,cv = None,n_params=25):
    """
    Train a model for Layer 1

    Parameters
    ----------
    X : pd.DataFrame
        dataframe with all the features
    y : pd.Series
        vector with observed retention times
    params : dict
        hyperparameters to test
    model : object
        model to be trained
    n_jobs : int
        number of threads to spawn
    cv : list
        CV to use for hyperparameter optimization and prediction
    n_params : int
        number of hyperparameter sets to try

    Returns
    -------
    object
        prediction model
	list
		predictions for the train set based on the CV
    """
    X = X.apply(lambda x: pd.to_numeric(x, errors="coerce"))
    crossv_mod = clone(model)
    ret_mod = clone(model)

    grid = RandomizedSearchCV(model, params, cv=cv,scoring="neg_mean_absolute_error",verbose=0,n_jobs=n_jobs,n_iter=n_params,refit=False)
    grid.fit(X,y)
    cv_pred = cv
    crossv_mod.set_params(**grid.best_params_)
    preds = cross_val_predict(crossv_mod, X=X, y=y, cv=cv_pred, n_jobs=n_jobs, verbose=0)

    ret_mod.set_params(**grid.best_params_)
    ret_mod.fit(X,y)

    return(ret_mod,preds)

def train_l1_func(sets,
                  names=["random","random1","random2","random3","random4","random5"],
                  adds=["","","","","","","",""],
                  cv = None,
                  n_params=25,
                  outfile_modname="",
                  n_jobs=24):
    """
    Train multiple models for Layer 1

    Parameters
    ----------
    sets : pd.DataFrame
        matrix with features and observed retention times
    names : list
        prefixes to add to each model
    adds : list
        add suffixes to add to model names
    cv : list
        CV to use for hyperparameter optimization and prediction
    n_params : int
        number of hyperparameter sets to try
    outfile_modname : str
        prefix for the model name
    n_jobs : int
        number of threads to spawn

    Returns
    -------
    pd.DataFrame
        return dataframe with predictions for the training set
    """

    ret_preds = []

    ################################# LASSO #################################

    model = Lasso()

    params = {
        "alpha" : uniform(0.0,10.0),
        "copy_X" : [True],
        "fit_intercept" : [True],
        "normalize" : [False],
        "precompute" : [True,False],
        "max_iter" : [5000]
    }

    print("Training Layer 1 LASSO")
    model,preds = train_model_l1(sets.drop(["time","IDENTIFIER","system"],axis=1, errors="ignore"),
                                             sets["time"],params,model,
                                             cv = cv,n_params=n_params,
                                             n_jobs=n_jobs)

    outfile = open("preds_l1/%s_lasso%s.txt" % (names[0],adds[0]),"w")
    for val in zip(list(sets["IDENTIFIER"]),list(sets["time"]),preds):
        outfile.write("%s,%s,%s\n" % val)
    outfile.close()

    with open("mods_l1/%s_lasso.pickle" % (names[0]), "wb") as f:
           pickle.dump(model, f)

    if len(outfile_modname) > 0:
        with open("%s_lasso.pickle" % (outfile_modname), "wb") as f:
           pickle.dump(model, f)

    ret_preds.append(preds)

    ################################# AdaBoost #################################

    model = AdaBoostRegressor()

    params = {
        "n_estimators": randint(10,100), #,1000,3000], #
           "learning_rate": uniform(0.01,0.5)
    }

    print("Training Layer 1 AdaBoost")
    model,preds = train_model_l1(sets.drop(["time","IDENTIFIER","system"],axis=1, errors="ignore"),
                                             sets["time"],params,model,
                                             cv = cv,n_params=n_params,
                                             n_jobs=n_jobs)

    outfile = open("preds_l1/%s_adaboost%s.txt" % (names[1],adds[1]),"w")
    for val in zip(list(sets["IDENTIFIER"]),list(sets["time"]),preds):
        outfile.write("%s,%s,%s\n" % val)
    outfile.close()

    with open("mods_l1/%s_adaboost.pickle" % (names[1]), "wb") as f:
           pickle.dump(model, f)

    if len(outfile_modname) > 0:
        with open("%s_adaboost.pickle" % (outfile_modname), "wb") as f:
           pickle.dump(model, f)

    ret_preds.append(preds)

    ################################# XGBoost #################################

    model = xgb.XGBRegressor()

    params = {
        "n_estimators" : randint(20,100),
        "max_depth" : randint(1,12),
        "learning_rate" : uniform(0.01,0.25),
        "gamma" : uniform(0.0,10.0),
        "reg_alpha" : uniform(0.0,10.0),
        "reg_lambda" : uniform(0.0,10.0)
    }

    print("Training Layer 1 XGBoost")
    model,preds = train_model_l1(sets.drop(["time","IDENTIFIER","system"],axis=1, errors="ignore"),
                                             sets["time"],params,model,
                                             n_jobs=n_jobs,cv = cv,n_params=n_params)

    outfile = open("preds_l1/%s_xgb%s.txt" % (names[2],adds[2]),"w")
    for val in zip(list(sets["IDENTIFIER"]),list(sets["time"]),preds):
        outfile.write("%s,%s,%s\n" % val)
    outfile.close()

    with open("mods_l1/%s_xgb.pickle" % (names[2]), "wb") as f:
           pickle.dump(model, f)

    if len(outfile_modname) > 0:
        with open("%s_xgb.pickle" % (outfile_modname), "wb") as f:
           pickle.dump(model, f)

    ret_preds.append(preds)

    ################################# SVR #################################

    model = SVR()

    params = {
       "epsilon": uniform(0.01,100.0),
       "kernel": ["linear","rbf"],
       "degree": randint(1,12),
       "gamma" : uniform(0.000001,100),
       #"tol" : [1e-10]#,
       #"max_iter" : [2000000000]
    }

    print("Training Layer 1 SVR")
    model,preds = train_model_l1(sets.drop(["time","IDENTIFIER","system"],axis=1, errors="ignore"),
                                             sets["time"],params,model,
                                             cv = cv,n_params=n_params,
                                             n_jobs=n_jobs)

    outfile = open("preds_l1/%s_SVM%s.txt" % (names[3],adds[3]),"w")
    for val in zip(list(sets["IDENTIFIER"]),list(sets["time"]),preds):
        outfile.write("%s,%s,%s\n" % val)
    outfile.close()

    with open("mods_l1/%s_SVM.pickle" % (names[3]), "wb") as f:
           pickle.dump(model, f)

    if len(outfile_modname) > 0:
        with open("%s_SVM.pickle" % (outfile_modname), "wb") as f:
           pickle.dump(model, f)

    ret_preds.append(preds)

    ################################# BRR #################################

    model = ARDRegression()

    params = {
        "n_iter" : [50],
        "alpha_1" : uniform(1e-10,1e-2),
        "lambda_1" : uniform(1e-10,1e-2),
        "threshold_lambda" : randint(1,10000),
	"verbose" : [True]
    }

    print("Training Layer 1 BRR")
    model,preds = train_model_l1(sets.drop(["time","IDENTIFIER","system"],axis=1, errors="ignore"),
                                             sets["time"],params,model,
                                             cv = cv,n_params=n_params,
                                             n_jobs=n_jobs)

    outfile = open("preds_l1/%s_bayesianregr%s.txt" % (names[6],adds[6]),"w")
    for val in zip(list(sets["IDENTIFIER"]),list(sets["time"]),preds):
        outfile.write("%s,%s,%s\n" % val)
    outfile.close()

    with open("mods_l1/%s_brr.pickle" % (names[6]), "wb") as f:
           pickle.dump(model, f)

    if len(outfile_modname) > 0:
        with open("%s_brr.pickle" % (outfile_modname), "wb") as f:
           pickle.dump(model, f)

    ret_preds.append(preds)

    ######################################################################

    ret_preds = pd.DataFrame(ret_preds).transpose()

    ret_preds.columns = ["%s_lasso" % (names[0]),
                         "%s_adaboost"  % (names[1]),
                         "%s_xgb" % (names[2]),
                         "%s_SVM" % (names[3]),
                         "%s_brr"  % (names[4])]

    return(ret_preds)

max_stars_repo_name: compomics/CALLC

max_issues_repo_name: compomics/CALLC

max_forks_repo_name: compomics/CALLC

reason: from scipy


text:
import errno
import os
import time
from os import path
from threading import Thread
from typing import Optional

import cv2
import numpy as np

from ndu_gate_camera.api.ndu_camera_runner import NDUCameraRunner
from ndu_gate_camera.utility import constants, onnx_helper, geometry_helper, string_helper, image_helper
from ndu_gate_camera.utility.geometry_helper import add_padding_rect
from ndu_gate_camera.utility.ndu_utility import NDUUtility

class IntersectorRunner(Thread, NDUCameraRunner):
    _ST_OR = "or"
    _ST_AND = "and"
    _ST_TOUCH = "touch"
    _ST_DIST = "dist"
    _ST_STATIONARY = "stationary"
    _ST_SPEED = "speed"

    def __init__(self, config, _):
        super().__init__()
        self._selection_mode = config.get("ground_dist_selection_mode", False)
        self._fps = config.get("fps", None)
        self._debug_mode = config.get("debug_mode", False)
        if not self._selection_mode:
            self._check_config(config.get("groups", None))
        self._last_data = {}
        self._tracks = {}
        self._frame_index = 0

    def _init_classification(self):
        onnx_fn = path.dirname(path.abspath(__file__)) + "/data/googlenet-9.onnx"
        class_names_fn = path.dirname(path.abspath(__file__)) + "/data/synset.txt"
        if not path.isfile(onnx_fn):
            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), onnx_fn)
        if not path.isfile(class_names_fn):
            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), class_names_fn)
        self.__onnx_classify_names = onnx_helper.parse_class_names(class_names_fn)
        self.sess_tuple = onnx_helper.get_sess_tuple(onnx_fn)

    def _check_config(self, config):

        class ObjDet(object):
            def __init__(self):
                self.ground = None
                self.dist = None
                self.stationary_seconds = None
                self.max_dist_per_seconds = None
                self.debug_speed_value = None
                self.debug_speed_suffix = None
                self.rects = []
                self.all_rect_names = []

        class CssDet(object):
            def __init__(self):
                self.threshold = 0.5
                self.padding = 0
                self.rect_names = []
                self.classify_indexes = []

        class ConfigDef(object):
            def __init__(self):
                self.groups = []
                self.all_rect_names = []

        class GroupDef(object):
            def __init__(self):
                self.name = ""
                self.obj_detection = ObjDet()
                self.classification = CssDet()
                self._M = None
                self._M_size = None
                self._dist_warped_sq = None
                self.active_frame = None

            def _warp_point(self, p):
                matrix = self._M
                px = (matrix[0][0] * p[0] + matrix[0][1] * p[1] + matrix[0][2]) / (matrix[2][0] * p[0] + matrix[2][1] * p[1] + matrix[2][2])
                py = (matrix[1][0] * p[0] + matrix[1][1] * p[1] + matrix[1][2]) / (matrix[2][0] * p[0] + matrix[2][1] * p[1] + matrix[2][2])
                return [int(px), int(py)]

            def _warp_rect(self, rect):
                y1, x1, y2, x2 = geometry_helper.get_rect_values(rect)
                p1 = self._warp_point([x1, y1])
                p2 = self._warp_point([x2, y2])
                return geometry_helper.get_rect(p1, p2)

            def _debug_draw(self, res, rect1, rect2):
                frame = self.active_frame
                dst = cv2.warpPerspective(frame, self._M, self._M_size)

                # cv2.polylines(dst, [np.array(rect1, np.int32)], True, color=(255, 255, 255), thickness=1)
                # cv2.polylines(dst, [np.array(rect2, np.int32)], True, color=(255, 255, 255), thickness=1)
                color = [0, 0, 255] if res else [128, 128, 128]
                image_helper.draw_rect(dst, rect1, color=color)
                image_helper.draw_rect(dst, rect2, color=color)

                cv2.imshow("intersector_runner", dst)
                cv2.waitKey(1)

                ground = self.obj_detection.ground
                g = np.array(ground, np.int32)
                cv2.polylines(frame, [g], True, color=(0, 255, 255), thickness=2)
                dist = self.obj_detection.dist
                d = np.array(dist, np.int32)
                cv2.polylines(frame, [d], False, color=(255, 255, 0), thickness=2)

            def _init_matrix(self):
                ground = self.obj_detection.ground
                dist = self.obj_detection.dist

                # ground: [[112, 1043], [565, 1027], [533, 934], [148, 938]]
                ground1 = []
                for i in [3, 2, 0, 1]:
                    ground1.append(ground[i])
                pts1 = np.float32(ground1)
                x1 = 450
                y1 = 450
                x2 = 550
                y2 = 550
                pts2 = np.float32([[y1, x1], [y2, x1], [y1, x2], [y2, x2]])

                self._M = cv2.getPerspectiveTransform(pts1, pts2)
                self._M_size = (1000, 1000)

                dist_w = [self._warp_point(dist[0]), self._warp_point(dist[1])]
                self._dist_warped_sq = geometry_helper.get_dist_sq(dist_w[0], dist_w[1])
                self._dist_warped = self._dist_warped_sq ** 0.5

            def rects_within_dist(self, r1, r2):
                if self._M is None:
                    self._init_matrix()

                rect1 = self._warp_rect(r1)
                rect2 = self._warp_rect(r2)

                p1 = geometry_helper.get_rect_bottom_center(rect1)
                p2 = geometry_helper.get_rect_bottom_center(rect2)
                d_sq = geometry_helper.get_dist_sq(p1, p2)
                res = d_sq <= self._dist_warped_sq
                if self.active_frame is not None:
                    self._debug_draw(res, rect1, rect2)
                return res

            def rect_speed(self, r1, r2, seconds):
                if self._M is None:
                    self._init_matrix()

                rect1 = self._warp_rect(r1)
                rect2 = self._warp_rect(r2)

                p1 = geometry_helper.get_rect_bottom_center(rect1)
                p2 = geometry_helper.get_rect_bottom_center(rect2)
                d_sq = geometry_helper.get_dist_sq(p1, p2)
                d = d_sq ** 0.5
                unit_speed = d / self._dist_warped / seconds
                if self.active_frame is not None:
                    self._debug_draw(unit_speed >= 1.0, rect1, rect2)
                return unit_speed

        class RectDet(object):
            def __init__(self):
                self.padding = 0
                self.style = IntersectorRunner._ST_OR
                self.class_names = []

        def get_obj_detection(_gr0) -> Optional[ObjDet]:
            if "obj_detection" not in _gr0:
                return None
            else:
                def get_rect(r0_):
                    r_ = RectDet()
                    r_.padding = r0_["padding"] if "padding" in r0_ else 0
                    r_.style = r0_["style"] if "style" in r0_ else "or"
                    if "class_names" in r0_:
                        for class_name in r0_["class_names"]:
                            r_.class_names.append(class_name)
                    return r_

                od0 = _gr0["obj_detection"]
                od = ObjDet()
                od.ground = od0["ground"] if "ground" in od0 else None
                od.dist = od0["dist"] if "dist" in od0 else None
                od.stationary_seconds = od0["stationary_seconds"] if "stationary_seconds" in od0 else None
                od.max_dist_per_seconds = od0["max_dist_per_seconds"] if "max_dist_per_seconds" in od0 else None
                od.debug_speed_value = od0["debug_speed_value"] if "debug_speed_value" in od0 else None
                od.debug_speed_suffix = od0["debug_speed_suffix"] if "debug_speed_suffix" in od0 else None
                if "rects" not in od0:
                    raise Exception("Bad config! no rects in obj_detection node.")
                for r0 in od0["rects"]:
                    r = get_rect(r0)
                    if len(r.class_names) == 0:
                        raise Exception("Bad config! no class_name in rects.")
                    od.rects.append(r)
                    for name1 in r.class_names:
                        if name1 not in od.all_rect_names:
                            od.all_rect_names.append(name1)
                return od

        def get_classification(_gr0) -> Optional[CssDet]:
            if "classification" not in _gr0:
                return None
            else:
                cs0 = _gr0["classification"]
                cs = CssDet()
                cs.threshold = cs0["threshold"] if "threshold" in cs0 else 0.5
                cs.padding = cs0["padding"] if "padding" in cs0 else 0

                for r0 in cs0["rect_names"]:
                    cs.rect_names.append(r0)

                if "classify_names" not in cs0:
                    raise Exception("Bad config! no classify_names in classificaion node.")
                for name1 in cs0["classify_names"]:
                    for i in range(len(self.__onnx_classify_names)):
                        css_name = self.__onnx_classify_names[i]
                        if string_helper.wildcard(css_name, name1):
                            cs.classify_indexes.append(i)
                            break
                return cs

        def enumerate_rect_class_names(_gr) -> str:
            if _gr.obj_detection is not None:
                for r in _gr.obj_detection.rects:
                    for name_ in r.class_names:
                        yield name_
            if _gr.classification is not None:
                for rect_name in _gr.classification.rect_names:
                    yield rect_name

        self.__onnx_classify_names = None
        self._conf = ConfigDef()
        for group_name, gr0 in config.items():
            gr = GroupDef()
            gr.name = group_name
            gr.obj_detection = get_obj_detection(gr0)
            if self.__onnx_classify_names is None and gr.classification is not None:
                self._init_classification()
            gr.classification = get_classification(gr0)
            self._conf.groups.append(gr)
            for name in enumerate_rect_class_names(gr):
                if name not in self._conf.all_rect_names:
                    self._conf.all_rect_names.append(name)

    def get_name(self):
        return "IntersectorRunner"

    def get_settings(self):
        settings = {}
        return settings

    def _classify(self, frame, bbox, threshold, classify_indexes):
        y1 = max(int(bbox[0]), 0)
        x1 = max(int(bbox[1]), 0)
        y2 = max(int(bbox[2]), 0)
        x2 = max(int(bbox[3]), 0)
        image = frame[y1:y2, x1:x2]

        blob = cv2.dnn.blobFromImage(image, 1, (224, 224), (123.68, 116.779, 103.939))
        preds = onnx_helper.run(self.sess_tuple, [blob])[0]

        # cv2.imshow(str(classify_indexes), image)
        # cv2.waitKey(100)

        for classify_index in classify_indexes:
            score = preds[0][classify_index]
            if score > threshold:
                return True
        return False

    def _get_stationary_rects(self, gr, r, rects0):
        def _stationary_check_required(gr_, track_, now_):
            if gr_.obj_detection.stationary_seconds is not None:
                if self._fps is None:
                    diff = now_ - track_["time0"]
                    if diff >= gr_.obj_detection.stationary_seconds:
                        return True, diff
                else:
                    diff = self._frame_index - track_["frame_index0"]
                    max_frame = gr_.obj_detection.stationary_seconds * self._fps
                    if diff >= max_frame:
                        return True, diff / self._fps
            else:
                raise Exception("intersector_runner config has problems! 'stationary' is defined, but stationary_seconds is not defined.")
            return False, None

        now = time.time()
        for class_name, score, rect, item in rects0:
            if class_name in r.class_names:
                track_id = item.get("track_id", None)
                if track_id is not None:
                    track_key = "{}-{}".format(class_name, track_id)
                    if track_key not in self._tracks:
                        self._tracks[track_key] = {
                            "time0": now,
                            "frame_index0": self._frame_index,
                            "rect0": rect,
                            "is_stationary": False
                        }
                    else:
                        track = self._tracks[track_key]
                        req, time_diff = _stationary_check_required(gr, track, now)
                        if req:
                            rect0 = track["rect0"]
                            if gr.rects_within_dist(rect0, rect):
                                if "waiting_time" in track:
                                    track["waiting_time"] += int(time_diff)
                                else:
                                    track["waiting_time"] = int(time_diff)
                                track["time0"] = now
                                track["frame_index0"] = self._frame_index
                                track["rect0"] = rect
                                track["is_stationary"] = True
                            else:
                                track["is_stationary"] = False
        rects = []
        waiting_times = []
        remove_keys = []
        for track_key, track in self._tracks.items():
            req, time_diff = _stationary_check_required(gr, track, now)
            if req:
                remove_keys.append(track_key)
            if track["is_stationary"]:
                rects.append(track["rect0"])
                waiting_times.append(track.get("waiting_time", None))
        for track_key in remove_keys:
            del self._tracks[track_key]
        return rects, waiting_times

    def _get_speed_rects(self, gr, r, rects0):
        def _get_diff_in_seconds(track_, now_):
            if self._fps is None:
                return now_ - track_["time0"]
            else:
                diff = self._frame_index - track_["frame_index0"]
                return diff / self._fps

        now = time.time()
        check_time = 0.5
        for class_name, score, rect, item in rects0:
            if class_name in r.class_names:
                track_id = item.get("track_id", None)
                if track_id is not None:
                    track_key = "{}-{}".format(class_name, track_id)
                    if track_key not in self._tracks:
                        self._tracks[track_key] = {
                            "time0": now,
                            "frame_index0": self._frame_index,
                            "rect0": rect,
                            "unit_speed": 0
                        }
                    else:
                        track = self._tracks[track_key]
                        rect0 = track["rect0"]
                        sec = _get_diff_in_seconds(track, now)
                        if now - track["time0"] > check_time:
                            speed = gr.rect_speed(rect0, rect, sec)
                            is_speeding = False
                            if gr.obj_detection.max_dist_per_seconds is not None:
                                speed /= gr.obj_detection.max_dist_per_seconds
                                is_speeding = speed >= 1.0
                                if gr.obj_detection.debug_speed_value is not None:
                                    speed *= gr.obj_detection.debug_speed_value
                            speed_txt = "{:.2f}".format(speed)
                            if gr.obj_detection.debug_speed_suffix is not None:
                                speed_txt = "{}{}".format(speed_txt, gr.obj_detection.debug_speed_suffix)

                            track["time0"] = now
                            track["frame_index0"] = self._frame_index
                            track["rect0"] = rect
                            track["speed_txt"] = speed_txt
                            track["is_speeding"] = is_speeding
        rects = []
        is_speeding_rects = []
        speed_txts = []
        remove_keys = []
        for track_key, track in self._tracks.items():
            if now - track["time0"] > check_time:
                remove_keys.append(track_key)
            else:
                rects.append(track["rect0"])
                is_speeding_rects.append(track.get("is_speeding", False))
                speed_txts.append(track.get("speed_txt", 0))
        for track_key in remove_keys:
            del self._tracks[track_key]
        return rects, is_speeding_rects, speed_txts

    def process_frame(self, frame, extra_data=None):
        self._frame_index += 1
        if self._selection_mode:
            areas = image_helper.select_areas(frame, "select intersector ground", max_count=1, max_point_count=4, next_area_key="n", finish_key="s")
            gr = []
            for p in areas[0]:
                gr.append(list(p))
            print('"ground": {},'.format(str(gr)))
            lines = image_helper.select_lines(frame, "select intersector dist", max_count=1)
            dist = []
            for p in lines[0]:
                dist.append(list(p))
            print('"dist": {},'.format(str(dist)))
            exit(1)

        super().process_frame(frame)
        res = []
        if self._conf is None:
            return res

        def has_or(r_, rect_names0_):
            _count = 0
            for r_name in r_.class_names:
                if string_helper.wildcard_has_match(rect_names0_, r_name):
                    return True
            return False

        def has_and(r_, rect_names0_):
            _all_ok = False
            for r_name in r_.class_names:
                if not string_helper.wildcard_has_match(rect_names0_, r_name):
                    return False
                else:
                    _all_ok = True
            return _all_ok

        def _has_interaction(r_, rect_names0_, rects0_, touch_check_func, only_if_different_classes_):
            if has_or(r_, rect_names0_):
                class RDef:
                    def __init__(self):
                        self.name = ""
                        self.rect = None

                rects_ = []
                for class_name1, score1, rect1, item1 in rects0_:
                    ok_ = False
                    for r_name in r_.class_names:
                        if string_helper.wildcard(class_name1, r_name):
                            ok_ = True
                            break
                    if ok_:
                        r1_ = RDef()
                        r1_.name = class_name1
                        if r_.padding != 0:
                            rect1 = add_padding_rect(rect1, r_.padding)
                        r1_.rect = rect1
                        rects_.append(r1_)

                for r1_ in rects_:
                    for r2_ in rects_:
                        if r1_ != r2_:
                            can_check = not only_if_different_classes_ or r1_.name != r2_.name
                            if can_check and touch_check_func(r1_.rect, r2_.rect):
                                if self._debug_mode:
                                    cv2.rectangle(frame, (int(r1_.rect[1]), int(r1_.rect[0])), (int(r1_.rect[3]), int(r1_.rect[2])), color=[0, 0, 0], thickness=3)
                                    cv2.rectangle(frame, (int(r2_.rect[1]), int(r2_.rect[0])), (int(r2_.rect[3]), int(r2_.rect[2])), color=[255, 255, 255], thickness=3)
                                return True, r1_, r2_
            return False, None, None

        def has_touch(r_, rect_names0_, rects0_, only_if_different_classes_):
            return _has_interaction(r_, rect_names0_, rects0_, geometry_helper.rects_intersect, only_if_different_classes_)

        def has_dist(gr_, r_, rect_names0_, rects0_, only_if_different_classes_):
            return _has_interaction(r_, rect_names0_, rects0_, gr_.rects_within_dist, only_if_different_classes_)

        def has_classification(gr_, rects0_):
            rects_css = []
            for class_name_, score_, rect_, item_ in rects0_:
                for css_rect_name in gr_.classification.rect_names:
                    if string_helper.wildcard(class_name_, css_rect_name):
                        rects_css.append(rect_)
                        break
            for rect_css in rects_css:
                if self._classify(frame, rect_css, gr.classification.threshold, gr.classification.classify_indexes):
                    return True
            return False

        counts = {}
        rects = {}
        rect_stationary_times = {}
        rect_speed_texts = {}
        rects_at_good_speed = []
        for gr in self._conf.groups:
            rects[gr.name] = []
            count = 0
            if self._debug_mode:
                gr.active_frame = frame
            if gr.obj_detection is not None:
                rects0 = []
                rect_names0 = []
                for class_name, score, rect, item in NDUUtility.enumerate_results(extra_data, gr.obj_detection.all_rect_names, use_wildcard=True, return_item=True):
                    if rect is not None:
                        rects0.append((class_name, score, rect, item))
                        rect_names0.append(class_name)
                if len(rects0) > 0:
                    for r in gr.obj_detection.rects:
                        if r.style == self._ST_OR:
                            if has_or(r, rect_names0):
                                count += 1
                        elif r.style == self._ST_AND:
                            if has_and(r, rect_names0):
                                count += 1
                        elif r.style == self._ST_TOUCH:
                            only_if_different_classes = len(gr.obj_detection.all_rect_names) > 1
                            ok, r1, r2 = has_touch(r, rect_names0, rects0, only_if_different_classes)
                            if ok:
                                count += 1
                                rects[gr.name].extend([r1.rect, r2.rect])
                        elif r.style == self._ST_DIST:
                            only_if_different_classes = len(gr.obj_detection.all_rect_names) > 1
                            ok, r1, r2 = has_dist(gr, r, rect_names0, rects0, only_if_different_classes)
                            if ok:
                                count += 1
                                rects[gr.name].extend([r1.rect, r2.rect])
                        elif r.style == self._ST_STATIONARY:
                            stationary_rects, waiting_times = self._get_stationary_rects(gr, r, rects0)
                            if len(stationary_rects) > 0:
                                count += len(stationary_rects)
                                rects[gr.name].extend(stationary_rects)
                                if gr.name in rect_stationary_times:
                                    rect_stationary_times[gr.name].extend(waiting_times)
                                else:
                                    rect_stationary_times[gr.name] = waiting_times
                        elif r.style == self._ST_SPEED:
                            speed_rects, is_speeding_rects, speed_txts = self._get_speed_rects(gr, r, rects0)
                            for i in range(len(speed_rects)):
                                is_speeding = is_speeding_rects[i]
                                speed_txt = speed_txts[i]
                                if is_speeding:
                                    count += 1
                                    rects[gr.name].append(speed_rects[i])
                                    if gr.name in rect_speed_texts:
                                        rect_speed_texts[gr.name].append(speed_txt)
                                    else:
                                        rect_speed_texts[gr.name] = [speed_txt]
                                else:
                                    rects_at_good_speed.append({"rect": speed_rects[i], "speed_txt": speed_txt})

            if count == 0 and gr.classification is not None:
                rects0 = []
                for class_name, score, rect, item in NDUUtility.enumerate_results(extra_data, gr.classification.rect_names, use_wildcard=True, return_item=True):
                    if rect is not None:
                        rects0.append((class_name, score, rect, item))
                        rects[gr.name].append(rect)
                if len(rects0) > 0:
                    if has_classification(gr, rects0):
                        count += 1

            counts[gr.name] = count
            if count > 0:
                rects = rects[gr.name]
                for i in range(len(rects)):
                    rect = rects[i]
                    name = gr.name
                    score = None
                    if gr.name in rect_stationary_times:
                        t = rect_stationary_times[name][i]
                        if t is not None:
                            score = "{} sec.".format(str(t))
                    if gr.name in rect_speed_texts:
                        score = rect_speed_texts[name][i]
                    res_item = {
                        constants.RESULT_KEY_RECT: rect,
                        constants.RESULT_KEY_CLASS_NAME: name,
                        constants.RESULT_KEY_RECT_COLOR: [0, 0, 255]
                    }
                    if score is not None:
                        res_item[constants.RESULT_KEY_SCORE] = score
                    res.append(res_item)
            for good_speed_item in rects_at_good_speed:
                rect = good_speed_item["rect"]
                speed_txt = good_speed_item["speed_txt"]
                res_item = {
                    constants.RESULT_KEY_RECT: rect,
                    constants.RESULT_KEY_CLASS_NAME: "good_speed",
                    constants.RESULT_KEY_SCORE: speed_txt,
                    constants.RESULT_KEY_RECT_COLOR: [0, 255, 0]
                }
                res.append(res_item)
        for gr_name, count in counts.items():
            if gr_name not in self._last_data:
                self._last_data[gr_name] = None
            val = {constants.RESULT_KEY_DEBUG: "{} - {}".format(gr_name, count)}
            if self._last_data[gr_name] != count:
                self._last_data[gr_name] = count
                val[constants.RESULT_KEY_DATA] = {gr_name: count, "{} exists".format(gr_name).replace(" ", "_"): count > 0}
            res.append(val)
        return res

max_stars_repo_name: netcadlabs/ndu-gate

max_issues_repo_name: netcadlabs/ndu-gate

max_forks_repo_name: netcadlabs/ndu-gate


text:
# Copyright (c) 2021, Technische Universität Kaiserslautern (TUK) & National University of Sciences and Technology (NUST).
# All rights reserved.

# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

"""
    A few helper functions
"""

from __future__ import print_function
from __future__ import division
import os
import numpy as np
import PIL.Image as Image
import scipy.io as sio

def load_weights_from_matfiles(dir_path):
    """
        Uses scipy.io to read .mat files and loads weights into torch model
    :param path_to_file: path to mat file to read
    :return: None, but saves the model dictionary!
    """
    import pickle
    model_file = 'Unet_pretrained_model.pkl'
    if os.path.exists(os.path.join(dir_path, model_file)):
        print('loading saved model dictionary...')
        with open(os.path.join(dir_path, model_file), 'rb') as handle:
            model_dict = pickle.load(handle)
        for i, layer in enumerate(model_dict.keys(), 1):
            print('{}.'.format(i), layer, model_dict[layer].shape)
    else:
        model_dict = {}
        for file in [x for x in os.listdir(dir_path) if x.endswith('.mat')]:
            layer, _ = os.path.splitext(file)
            try:
                read = sio.loadmat(os.path.join(dir_path, file))
            except:
                print(layer)
            print(layer, read[layer].shape)
            model_dict[layer] = read[layer]
        pass
        os.chdir('/home/annus/Desktop/trainedUnet/weightsforpython/')
        with open(model_file, 'wb') as handle:
            pickle.dump(model_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
        print('Saved model!!!')

def show_image():

    def histeq(im):
        """  Histogram equalization of a grayscale image. """
        nbr_bins = 256
        # get image histogram
        imhist, bins = np.histogram(im.flatten(), nbr_bins, normed=True)
        cdf = imhist.cumsum()  # cumulative distribution function
        cdf = 255 * cdf / cdf[-1]  # normalize

        # use linear interpolation of cdf to find new pixel values
        im2 = np.interp(im.flatten(), bins[:-1], cdf)

        return im2.reshape(im.shape)

    os.chdir('/home/annus/Desktop/rit18_data/')
    train_data = np.load('train_data.npy', mmap_mode='r').transpose((2, 1, 0))
    print(train_data.shape)
    w, h, patch = 2000, 2000, 1000
    image = train_data[w:w + patch, h:h + patch, 4:]
    # image = (255 / 65536 * image).astype(np.int8)
    r, g, b = map(histeq, [image[:, :, 0], image[:, :, 1], image[:, :, 2]])
    image = Image.fromarray(np.dstack((r, g, b)), 'RGB')
    # image = cv2.normalize(image, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX,
    #                            dtype=cv2.CV_32F).astype(np.int8)
    # print(image.shape, image.dtype, np.max(np.max(image)), np.min(np.min(image)), np.mean(np.mean(image)))
    # pl.imshow(image)
    # pl.axis('off')
    # pl.show()
    os.chdir('/home/annus/Desktop/')
    image.save('image.png')

max_stars_repo_path: old code/utils.py

max_stars_repo_name: dll-ncai/AI-ForestWatch

max_issues_repo_path: old code/utils.py

max_issues_repo_name: dll-ncai/AI-ForestWatch

max_forks_repo_path: old code/utils.py

max_forks_repo_name: dll-ncai/AI-ForestWatch

reason: import numpy,import scipy


text:
import os
import numpy as np
import pandas as pd
from collections import OrderedDict
import gzip

try:
    import cPickle as pickle  # much faster
except:
    import pickle

try:
    import stfio
    from stfio import StfIOException
except ImportError:
    print("Module stfio is not installed. Make sure .abf files are converted to .pkl in Python 2.")

def load_current_step(abf_file, filetype='abf', channels=[0,1], min_voltage=-140):
    '''
    Load current clamp recordings from pClamp .abf files
    filetype: one of ['abf', 'pkl']
            'pkl' is the pickle file converted from abf
    min_voltage: None or a number (e.g. -130), traces with min below this voltage are not loaded.
    '''
    ch0, ch1 = channels[0], channels[1]
    use_pkl = False
    if filetype == 'abf':
        try:
            rec = stfio.read(abf_file)
        except:
            use_pkl = True
    if filetype == 'pkl' or use_pkl:
        try:
            with gzip.open(abf_file.strip('.' + filetype) + '.pkl', 'rb') as handle: # compressed pkl
                data = pickle.load(handle, encoding='bytes') # for files converted to .pkl
        except IOError:
            with open(abf_file.strip('.' + filetype) + '.pkl', 'rb') as handle:  # non-compressed pkl
                data = pickle.load(handle, encoding='bytes') # for files converted to .pkl
        data = decode_bytes(data)
        return data

    assert((rec[ch0].yunits in ['mV', 'pA']) and (rec[ch1].yunits in ['mV', 'pA']))

    data = OrderedDict()
    data['file_id'] = os.path.basename(abf_file).strip('.' + filetype)
    data['file_directory'] = os.path.dirname(abf_file)
    data['record_date'] = rec.datetime.date()
    data['record_time'] = rec.datetime.time()

    data['dt'] = rec.dt / 1000
    data['hz'] = 1./rec.dt * 1000
    data['time_unit'] = 's'

    data['n_channels'] = len(rec)
    data['channel_names'] = [rec[x].name for x in channels]
    data['channel_units'] = [rec[x].yunits for x in channels]
    data['n_sweeps'] = len(rec[ch0])
    data['sweep_length'] = len(rec[ch0][0])

    data['t'] = np.arange(0, data['sweep_length']) * data['dt']

    if rec[ch0].yunits == 'mV' and rec[ch1].yunits == 'pA':
        data['voltage'] = rec[ch0]
        data['current'] = rec[ch1]
    elif rec[ch1].yunits == 'mV' and rec[ch0].yunits == 'pA':
        data['voltage'] = rec[ch1]
        data['current'] = rec[ch0]
    else:
        raise ValueError("channel y-units must be 'mV' or 'pA'.")
    data['voltage'] = [x.asarray() for x in data['voltage']]
    data['current'] = [x.asarray() for x in data['current']]

    if not min_voltage is None:
        to_pop = []
        for i, x in enumerate(data['voltage']):
            if np.min(x) < min_voltage:
                to_pop.append(i)
        data['voltage'] = [x for i, x in enumerate(data['voltage']) if not i in to_pop]
        data['current'] = [x for i, x in enumerate(data['current']) if not i in to_pop]
        data['n_sweeps'] -= len(to_pop)

    return data

def decode_bytes(data):
    if isinstance(data, bytes):
        return data.decode('ascii')
    elif isinstance(data, dict):
        return dict(map(decode_bytes, data.items()))
    elif isinstance(data, tuple):
        return tuple(map(decode_bytes, data))
    elif isinstance(data, list):
        return list(map(decode_bytes, data))
    else:
        return data

def save_data_as_pickle(data, pkl_file, compress=True):
    if compress:
        with gzip.open(pkl_file, 'wb') as handle:
            pickle.dump(data, handle, protocol=2)
    else:
        with open(pkl_file, 'wb') as handle:
            pickle.dump(data, handle, protocol=2)

def current_clamp_abf_to_pkl(in_file, out_file):
    print("Saving: " + out_file)
    data = load_current_step(in_file, channels=[0,1], min_voltage=-140)
    save_data_as_pickle(data, out_file)

def batch_current_clamp_abf_to_pkl(input_folder, output_folder=None):
    if not output_folder is None and input_folder != output_folder:
        try:
            os.makedirs(output_folder)
        except OSError:
            pass
    if output_folder is None:
        output_folder = input_folder

    filenames = [x for x in os.listdir(input_folder) if x.endswith(".abf")]
    input_file_paths = [os.path.join(input_folder, x) for x in filenames]
    output_file_paths = [os.path.join(output_folder, x[:-4] + ".pkl") for x in filenames]
    for in_file, out_file in zip(input_file_paths, output_file_paths):
        current_clamp_abf_to_pkl(in_file, out_file)


text:
# visualizatioin
import numpy as np
import matplotlib.pyplot as plt
import imageio

# This import registers the 3D projection, but is otherwise unused.
from mpl_toolkits.mplot3d import Axes3D  # noqa: F401 unused import
from second.sphere.conv import depth_to_3D

def depth_from_feature_np(feature, k):
    """
    Args:
        feature: numpy array of shape [5, 64, 512], C, H, W,
            the 3th channel is depth channel

    Return:
        depth map, of shape [64, 512]  scaled to  [0 to k],
        for grey scale plotting
    """
    r = feature[3, :, :]
    depth = ((r - r.min()) * (k) / (r.max() - r.min()))
    return depth

def plot_point_cloud_scatter_2d(x, y):
    """
    given a point cloud, plot the scattered map
    Args:
        x : numpy array of shape [N], x coordinates
        y : numpy array of shape [N], y coordinates
    """
    plt.scatter(x, y, linewidths=0.1, marker='.')
    plt.show()

def plot_point_cloud_scatter_3d(x, y, z):
    """
    given a point cloud, plot the scattered map
    Args:
        x : numpy array of shape [N], x coordinates
        y : numpy array of shape [N], y coordinates
        z : numpy array of shape [N], z coordinates
    """
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.scatter(x, y, z, linewidths=1)
    ax.set_xlabel('X Label')
    ax.set_ylabel('Y Label')
    ax.set_zlabel('Z Label')
    plt.show()

def save_depth_jpg(depth, path: str):
    """
    save depth map to image file
    Args:
        depth : np tensor of shape (H, W)
        path
    """
    depth = depth.cpu().numpy()
    gray = np.interp(depth, (depth.min(), depth.max()), (50, 255))
    H, W = depth.shape
    imageio.imwrite(path, gray.astype(np.uint8))

def save_2d_jpg(feature, name:str):
    """
    save a 2d feature map to image file
    depth : tensor of shape (H, W)
    """
    save_depth_jpg(feature, name)

depth_map_bev_count = 0
def depthmap_bev(feature, depth, D=0):
    """
    aruguments:
        feature: tensor of shape (B,C,H,W)
        depth: (B,H,W)
        D: the max of 'depth', if not given, it will guess
    save image file
    """
    global depth_map_bev_count
    f3d = depth_to_3D(feature, depth, D=D)
    B, C, D, H, W = f3d.shape
    for C_i in range(C):
        f3d_piece = f3d[0][C_i]
        # sum over depth, D,H,W
        f2d = f3d_piece.sum(dim=1)
        save_2d_jpg(f2d, "fmap-%d-%d.jpg" % (depth_map_bev_count , C_i))
        depth_map_bev_count +=1

max_stars_repo_name: reveriel/depconv

max_issues_repo_name: reveriel/depconv

max_forks_repo_name: reveriel/depconv


text:
from matplotlib import pyplot as plt
from IPython.display import clear_output
import pandas as pd
import matplotlib
import numpy as np

def circle(x, y, r, p = 100):

    x_, y_ = [], []
    for i in range(p):
        x_.append(x+r*np.cos(2*i*np.pi/p))
        y_.append(y+r*np.sin(2*i*np.pi/p))
    return x_, y_

def plot_graph(reward, precision, alpha, episode_len, avg_rewards, avg_alphas, avg_precisions, avg_episode_lengths):
    """
    Plot training progress (reward, precision, alpha, episode length and their mean)

    Parameters
    ---------
    reward, precision, alpha, episode_len, avg_rewards, avg_alphas, avg_precisions, avg_episode_lengths: array_like

    Returns
    -------
    None : None
    """
    clear_output(wait=True)
    _, ((ax, ax1), (ax2, ax3)) = plt.subplots(2, 2, figsize=(12,8))
    ax_plot_graph(reward, avg_rewards, ax, 'Episode reward')
    #ax.set_ylim([-8, np.max(avg_rewards)+1])
    ax_plot_graph(precision, avg_precisions, ax1, 'Precision (nm)')
    ax1.set_ylim([0, 1])
    ax1.hlines(0.166, 0, len(precision))
    ax_plot_graph(alpha, avg_alphas, ax2, 'alpha')
    ax_plot_graph(episode_len, avg_episode_lengths, ax3, 'Episode lengths')
    plt.show()

def ax_plot_graph(data, avg_data, ax, y_label):
    """
    Plot data and its rolling mean

    Parameters
    ---------
    data, avg_data: array_like
    ax: axes
    y_label: str

    Returns
    -------
    None : None
    """
    df = pd.DataFrame({'x': range(len(data)), 'data': data, 'average': avg_data})
    ax.plot(df['x'], df['data'], marker='', color='silver', linewidth=0.8, alpha=0.9)
    ax.plot(df['x'], df['average'], marker='', color='DodgerBlue', linewidth=1, alpha=0.9)
    ax.set_xlabel("episode", fontsize=12)
    ax.set_ylabel(y_label, fontsize=12)

def show_reset(img_info, atom_start_position, destination_position, template_nm = None, template_wh = None):
    """
    Show STM image, atom and target position, and template position at the reset step in reinforcement learning

    Parameters
    ---------
    img_info: dict
    atom_start_position, destination_position: array_like
        atom and target position in STM coordinate (nm)
    template_nm, template_wh: array_like, optional
        template position and size in nm

    Returns
    -------
    None : None
    """
    img = img_info['img_forward']
    offset_nm = img_info['offset_nm']
    len_nm = img_info['len_nm']
    _, ax = plt.subplots()
    extent = (offset_nm[0]-0.5*len_nm[0], offset_nm[0]+0.5*len_nm[0], offset_nm[1]+len_nm[0], offset_nm[1])

    ax.imshow(img, extent = extent)
    if (template_nm is not None) and (template_wh is not None):
        rect = matplotlib.patches.Rectangle(template_nm,template_wh[0], template_wh[1], linewidth=1, edgecolor='r', facecolor='none')
        ax.add_patch(rect)
    ax.scatter(atom_start_position[0], atom_start_position[1], s = 20, linewidths=3, edgecolors='#33dbff', color = None, label='start')
    ax.scatter(destination_position[0], destination_position[1], s = 20, linewidths=3, edgecolors='#75ff33', color = None, label='gaol')
    plt.legend(frameon=False, labelcolor= 'white')
    plt.show()

def show_done(img_info, atom_position, atom_start_position, destination_position, reward,
              template_nm = None, template_wh = None):
    """
    Show STM image, atom start and current position, target position, episode reward, and template position when an episode terminates in reinforcement learning

    Parameters
    ---------
    img_info: dict
    atom_position,atom_start_position, destination_position: array_like
        atom start, atom current, and target position in STM coordinate (nm)
    reward: float
        episode reward
    template_nm, template_wh: array_like, optional
        template position and size in nm

    Returns
    -------
    None : None
    """
    img = img_info['img_forward']
    offset_nm = img_info['offset_nm']
    len_nm = img_info['len_nm']
    _, ax = plt.subplots()
    extent = (offset_nm[0]-0.5*len_nm[0], offset_nm[0]+0.5*len_nm[0], offset_nm[1]+len_nm[0], offset_nm[1])
    ax.imshow(img, extent = extent)
    if (template_nm is not None) and (template_wh is not None):
        rect = matplotlib.patches.Rectangle(template_nm,template_wh[0], template_wh[1], linewidth=1, edgecolor='r', facecolor='none')
        ax.add_patch(rect)
    ax.scatter(atom_start_position[0], atom_start_position[1], s = 20, linewidths=3, edgecolors='#33dbff', color = None, label='start')
    ax.scatter(destination_position[0], destination_position[1], s = 20, linewidths=3, edgecolors='#75ff33', color = None, label='gaol')
    ax.scatter(atom_position[0], atom_position[1], s = 20, linewidths=3, edgecolors='#ff5733', color = None, label='atom')

    ax.text(offset_nm[0], offset_nm[1],'reward: {}'.format(reward), ha='center')
    plt.legend(frameon=False, labelcolor= 'white')
    plt.show()

def show_step(img_info, start_nm, end_nm, atom_position, atom_start_position, destination_position, mvolt, pcurrent,
              template_nm = None, template_wh = None):
    """
    Show STM image, atom start and current position, target position,
    bias, current setpoint, and template position when the environment take a step in reinforcement learning

    Parameters
    ---------
    img_info: dict
    atom_position,atom_start_position, destination_position: array_like
        atom start, atom current, and target position in STM coordinate (nm)
    mvolt, pcurrent: float
        bias in mV and current in pA
    template_nm, template_wh: array_like, optional
        template position and size in nm

    Returns
    -------
    None : None
    """
    img = img_info['img_forward']
    offset_nm = img_info['offset_nm']
    len_nm = img_info['len_nm']
    _, ax = plt.subplots()
    extent = (offset_nm[0]-0.5*len_nm[0], offset_nm[0]+0.5*len_nm[0], offset_nm[1]+len_nm[0], offset_nm[1])
    ax.imshow(img, extent = extent)
    if (template_nm is not None) and (template_wh is not None):
        rect = matplotlib.patches.Rectangle(template_nm,template_wh[0], template_wh[1], linewidth=1, edgecolor='r', facecolor='none')
        ax.add_patch(rect)
    ax.scatter(atom_start_position[0], atom_start_position[1], s = 20, linewidths=3, edgecolors='#33dbff', color = None, label='start')
    ax.scatter(atom_position[0], atom_position[1], s = 20, linewidths=3, edgecolors='#ff5733', color = None, label='atom')
    ax.scatter(destination_position[0], destination_position[1], s = 20, linewidths=3, edgecolors='#75ff33', color = None, label='gaol')
    ax.arrow(start_nm[0], start_nm[1], (end_nm - start_nm)[0], (end_nm - start_nm)[1],width=0.1, length_includes_head = True)
    ax.text(offset_nm[0]+0.5*len_nm[0], offset_nm[1]+len_nm[0], 'bias(mV):{:.2f}, current(nA):{:.2f}'.format(mvolt,pcurrent/1000))
    plt.legend(frameon=False, labelcolor= 'white')
    plt.show()

def plot_large_frame(img_info, atom_chosen, design_chosen, anchor_chosen, next_destination, path):
    """
    Used for building multiple atom structures
    Show STM image, atoms, designs, and anchor, next target, and path between atom and design

    Parameters
    ---------
    img_info: dict
    atom_chosen, design_chosen, anchor_chosen, next_destination: array_like
        atom, design, anchor, and next target positions in STM coordinate (nm)
    path: array_like
        path between atom and design

    Returns
    -------
    None : None
    """
    _, ax = plt.subplots()
    img = img_info['img_forward']
    offset_nm = img_info['offset_nm']
    len_nm = img_info['len_nm']
    extent = (offset_nm[0]-0.5*len_nm[0], offset_nm[0]+0.5*len_nm[0], offset_nm[1]+len_nm[0], offset_nm[1])
    ax.imshow(img, extent = extent)
    ax.scatter(atom_chosen[0], atom_chosen[1], color='#7027A0' ,label='atom')
    ax.scatter(design_chosen[0],design_chosen[1], color='#1DB9C3', label='design')
    ax.scatter(anchor_chosen[0], anchor_chosen[1], color = '#F56FAD', label='anchor')

    path = np.array(path)
    ax.plot(path[:,0], path[:,1])
    ax.arrow(atom_chosen[0],atom_chosen[1], (next_destination - atom_chosen)[0], (next_destination - atom_chosen)[1], width=0.1,length_includes_head=True, color='#FFC069')
    ax.legend(frameon = False, labelcolor = '#FAEBE0')
    plt.show()

def plot_atoms_and_design(img_info, all_atoms, design, anchor, show_legend = True):
    _, ax = plt.subplots()
    img = img_info['img_forward']
    offset_nm = img_info['offset_nm']
    len_nm = img_info['len_nm']
    extent = (offset_nm[0]-0.5*len_nm[0], offset_nm[0]+0.5*len_nm[0], offset_nm[1]+len_nm[0], offset_nm[1])
    ax.imshow(img, extent = extent, cmap='Greys')
    if (all_atoms is not None) and (all_atoms.size !=0):
        ax.scatter(all_atoms[:,0], all_atoms[:,1], color='#7027A0' ,label='atom')
    if design is not None:
        ax.scatter(design[:,0],design[:,1], color='#1DB9C3', label='design')
    if anchor is not None:
        ax.scatter(anchor[0], anchor[1], color = '#F56FAD', label='anchor')
    if show_legend:
        ax.legend(frameon = False, labelcolor = '#FAEBE0')
    plt.show()


text:
import os
import numpy as np
import pandas as pd
from GRUD import GRUD
import sklearn.metrics as skm
import re

from prospective_set_up import get_retro, get_prosp

from run_model import *

import pickle as pkl

def main(args):

    arg_suf = args.out_suf

    device='cuda' if torch.cuda.is_available() else 'cpu'

    # Store the set of suffixes attached to the model output folder in home_dir
    home_suffix = 'out_grud' + arg_suf

    assert os.path.exists(os.path.join(args.home_dir, home_suffix, 'train_means.pickle')), 'No train_means file to load in ' + str(args.home_dir) + '/' + home_suffix

    with open(os.path.join(args.home_dir, home_suffix, 'train_means.pickle'), 'rb') as f:
        train_means = pd.read_pickle(f)
        #train_means = pickle.load(f)

    input_size = args.num_feature + args.weekofyear
    cell_size = args.grud_hidden
    hidden_size = args.grud_hidden
    model=GRUD( input_size, cell_size, hidden_size, train_means, device, fp16=not(args.opt_level=='O0'))

    if os.path.exists(os.path.join(args.model_dir, 'checkpoint_best.pth'))&(args.model_dir!=''):
        checkpoint = torch.load(os.path.join(args.model_dir, 'checkpoint_best.pth'))
    # Assumed fit model in sub dir out_grud with suffix if no model_dir is provided.
    else:
        assert os.path.exists(os.path.join(args.home_dir, home_suffix, 'checkpoint_best.pth')), "Missing trained model, expect file: " + str(os.path.join(args.home_dir, home_suffix, 'checkpoint_best.pth'))
        checkpoint = torch.load(os.path.join( args.home_dir, home_suffix, 'checkpoint_best.pth'), map_location=device)
    checkpoint['state_dict']= {k.replace('model.', ''): v for k, v in checkpoint['state_dict'].items()}
    print(checkpoint['state_dict'].keys())
    try:
        model.load_state_dict(checkpoint['state_dict'])
    except:
        print(checkpoint['state_dict'].keys())
        model['model'].load_state_dict(checkpoint['state_dict'])
    model.to(device)

    if (not args.eval_train)|(args.data_dir is None):
        try:
            d_name = 'split_dict_' + args.regularly_sampled*'regular' + (not args.regularly_sampled)*'irregular' + '.pkl'
            with open(os.path.join(args.home_dir, 'test', d_name), 'rb') as f:
                prosp_specs = pickle.load(f)

            f_name = 'split_daily_data_' + args.regularly_sampled*'regular' + (not args.regularly_sampled)*'irregular' + '.hdf'
            assert not os.path.exists(os.path.join(args.home_dir, 'test', f_name)), 'Found a prospective dictionary file and full dataframe file in {}, uncertain which should be used.'.format(args.data_dir)
            with open(DATA_PATH_DICTIONARY_FILE, 'rb') as f:
                tmp = pickle.load(f)
            path = tmp[GET_PATH_DICT_KEY(args)]
            d = os.path.dirname(path)
            f = os.path.basename(path)
            dfs=load_data(d, regular=args.regularly_sampled, only_healthy=args.only_healthy, fname=f)
            dfs = get_prosp(dfs, prosp_specs)

            # Clean up temporary objects
            del path, d, f, tmp
            # Get the number of test_days, update the passed in argument.
            if prosp_specs['test_days'] < 1:
                max_date = dfs['activity'].index.get_level_values('date').max()
                args.test_size = (max_date - prosp_specs['border_day']).days
            else:
                args.test_size = prosp_specs['test_days']
            last_day_str = prosp_specs['border_day']
        except OSError:
            dfs=load_data(os.path.join(args.home_dir, 'test'), regular=args.regularly_sampled, only_healthy=args.only_healthy)
            last_day_str = X_test_flat.index.get_level_values('date').max()
    else:
        dfs=load_data(args.home_dir, regular=args.regularly_sampled, only_healthy=args.only_healthy)

    if args.weekofyear:
        idx=pd.IndexSlice
        dfs['activity']['weekday']=(dfs['activity'].index.get_level_values('date').weekday<5).astype(np.int32) # add week of year
        dfs['activity'].index
        dfs['activity'].index.get_level_values('date')
        dfs['activity'].index.get_level_values('date' ).isocalendar().week.astype(np.int32)
#         dfs['activity'].loc[:, idx['measurement', 'weekofyear']]=dfs['activity'].index.get_level_values('date').weekofyear
#         dfs['activity'].loc[:, idx['measurement_z', 'weekofyear']]=dfs['activity'].index.get_level_values('date').weekofyear
        dfs['activity'].loc[:, idx['measurement', 'weekofyear']]=dfs['activity'].index.get_level_values('date').isocalendar().week.astype(np.int32).values
        dfs['activity'].loc[:, idx['measurement_z', 'weekofyear']]=dfs['activity'].index.get_level_values('date').isocalendar().week.astype(np.int32).values
        dfs['activity'].loc[:, idx['mask', 'weekofyear']]=np.ones(len(dfs['activity']))
        dfs['activity'].loc[:, idx['time', 'weekofyear']]=np.ones(len(dfs['activity']))

    if not(args.eval_train):
        # remove participants who are not in the prospective set
        first_test_day = dfs['survey'].index.get_level_values('date').max() - pd.to_timedelta(args.test_size, 'd')
        # prospective participants were only necessarily removed from the last seven days of the dataframe.
        tmp = dfs['activity'][dfs['activity'].index.get_level_values('date') >= first_test_day]
        test_participants = np.unique(tmp.index.get_level_values('participant_id'))

        test_dfs = {k:v.loc[v.index.get_level_values('participant_id').isin(test_participants), :     ] for k, v in dfs.items()}

    else:
        # reload the training data (but we will save it as test_dfs)
        test_participants = pd.read_csv(os.path.join(args.home_dir, home_suffix, 'train_participants.csv')).values.ravel().tolist()
        test_dfs = {k:v.loc[v.index.get_level_values('participant_id').isin(test_participants), :     ] for k, v in dfs.items()}

    # Assumed data scaler in sub dir out.
    with open(os.path.join(args.home_dir, home_suffix, 'scaler.pkl'), 'rb') as f:
        scaler = pkl.load(f)
    test_dfs, _ = apply_standard_scaler(test_dfs, args, scaler=scaler)
    test_dataset=ILIDataset(test_dfs, args, full_sequence=True, feat_subset=args.feat_subset)
    test_dataloader=DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=args.num_dataloader_workers, collate_fn=id_collate)
    # Need this last_test_day for participants whose data ends before the final date to calculate how many days to keep.
    last_prosp_day = test_dfs['survey'].index.get_level_values('date').max()
    proba_projection=torch.nn.Softmax(dim=1)
    labels=[]
    scores = []
    participants = []
    model.eval()
    batches=tqdm(test_dataloader, total=len(test_dataloader))
    for batch in batches:
        assert test_dataloader.batch_size == 1, "batch size for test_dataloader must be one."
        participant = batch[-1][0]
        batch = batch[0][0]
        batch={k:v.to(device) for k, v in batch.items()}
        #Some participants don't have data all the way till the last day of the prospective test set, so here calculate how many of those days there are take that many days out of the returned scores and labels.
        participant_last_day = test_dfs['survey'].loc[participant]['ili'].index.get_level_values('date').max()
        num_days = ( last_prosp_day - participant_last_day).days
        keep_days = args.test_size - num_days
        #if keep_days != 7:
        #    print('p: {0}; pld: {1}; k: {2}'.format(participant, participant_last_day, keep_days))
        with torch.no_grad():
            prediction, hidden_states = model(batch['measurement_z'].float() if args.zscore else batch['measurement'].float(), batch['measurement'].float(), batch['mask'].float(), batch['time'].float(), pad_mask= batch['obs_mask'], return_hidden=True)

            scores.append(proba_projection(prediction.view(-1, 2))[:, -1].detach().cpu().numpy()[-keep_days:])
#             print((batch[args.target[0]].detach().cpu().numpy()[:, -args.test_size:]).shape)
#             print(batch[args.target[0]].shape)
#             print(args.test_size)
#             raise
        assert len(batch[args.target[0]].detach().cpu().numpy()[:, -keep_days:].ravel())>0
        assert len(batch[args.target[0]].detach().cpu().numpy()[:, -keep_days:].shape)==2
        labels.append(batch[args.target[0]].detach().cpu().numpy()[:, -keep_days:].ravel())
        participants.append([participant]*scores[-1].shape[-1])

#         if len(labels)>10:
#             break

#     print(labels[0])
#     print(len(labels))
#     labels = np.concatenate(tuple(labels), axis=1).squeeze()
    try:
        labels = np.concatenate(tuple(labels), axis=0).squeeze()
    except:
        print(labels)
        print(tuple(labels))
        raise
    scores = np.concatenate(tuple(scores))

    last_day = dfs['activity'].index.get_level_values('date').max()
    prospective_days = [last_day - pd.to_timedelta(args.test_size-1,unit='d') + pd.to_timedelta(n, unit='d') for n in range(args.test_size)]

    dates = np.concatenate(tuple([prospective_days[:len(p)] for p in participants]))

    participants = np.concatenate(tuple(participants))

    auc = skm.roc_auc_score(labels, scores)
    aps = skm.average_precision_score(labels, scores)

    args.output_dir = args.home_dir

    out_df = pd.DataFrame({args.target[0] + '_score': scores, args.target[0]: labels, 'participant_id': participants, 'date': dates})

    assert os.path.exists( os.path.join( args.home_dir,'out_grud' + arg_suf, 'thresholds_activity.json')), print(os.path.join( args.home_dir,'out_grud' + arg_suf, 'thresholds_activity.json'))

    if os.path.exists( os.path.join( args.home_dir,'out_grud' + arg_suf, 'thresholds_activity.json')):
        with open(os.path.join( args.home_dir,'out_grud' + arg_suf, 'thresholds_activity.json'), 'r') as f:
            thresholds = json.load(f)
        # apply this to the data and get the scores.
        for k, v in thresholds.items():
            print(args.target[0])
            out_df[args.target[0]+'_pred_'+k] = np.asarray(scores >= float(v) ).astype(np.int32)

    # Check if an explicit path was passed for the model, if so get the date it was trained till.
    if args.model_dir!='':
        model_date = os.path.split(os.path.split(args.model_dir)[0])[1].replace('_', '-')
        fn = args.target[0] + '_grud_prosp_testset_results_' + arg_suf + model_date +'.csv'
        write_path = os.path.join(args.home_dir, 'test', fn)
    # Since model_dir wasn't passed the model must match the date of the home_dir.
    else:
        fn = args.target[0] + 'grud_prosp_testset_results' + arg_suf +'.csv'
        if args.eval_train:
            fn = args.target[0] + 'grud_prosp_trainset_results'+arg_suf+'.csv'
        write_path = os.path.join(args.home_dir, 'test', fn)

    if args.output_file is not None:
        write_path = args.output_file
    out_df.to_csv(os.path.join(args.home_dir, 'test', fn))

    if args.output_file is not None:
        # we don't need extra aggregation for models that aren't evaluated in a prospective manner so we can quit.
        return

    if args.model_dir!='':
        # last day is the last
        # model dir date
        model_date = os.path.split(args.model_dir)[1].replace('_', '-')
        test_date = os.path.split(os.path.split(args.home_dir)[0])[1].replace('_', '-')
        with open(os.path.join(args.home_dir, '..', 'all_grud_auc'+arg_suf+'.csv'), 'a+') as f:
            f.write(args.model_dir + ', ' + args.home_dir +', '+ str(last_day) + ', ' + str(auc) + ', ' + str(aps) + '\n')
    else:
        if not(args.eval_train):
            with open(os.path.join(args.home_dir, '..', 'all_grud_auc'+arg_suf+'.csv'), 'a+') as f:
                f.write(str(last_day) + ', ' + str(auc) + ', ' + str(aps) + '\n')
        else:
            with open(os.path.join(args.home_dir, '..', 'all_grud_train_auc'+arg_suf+'.csv'), 'a+') as f:
                f.write(str(last_day) + ', ' + str(auc) + ', ' + str(aps) + '\n')

    print('Successfully evaluated')

    return None

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()

    parser.add_argument('--target', type=str, default=('ili',), nargs='+', choices=['ili', 'ili_24', 'ili_48', 'covid', 'covid_24', 'covid_48', 'symptoms__fever__fever'], help='The target')
    parser.add_argument("--home_dir", type=str, required=True, help='Base directory for reading data and fit model. Assumed home_dir will contain an out folder, which contains the fit model on the training part of the split and the scaler used, and the test folder, which contains the test part of the split.')
    parser.add_argument("--model_dir", type=str, default='', help='Add this if the model is in a different folder.')
    parser.add_argument("--data_dir", type=str, default=None, help='If evaluating on the dataset in home_dir/test is not wanted then an alternative directory can be supplied here to override that default behaviour.')
    parser.add_argument("--output_file", type=str, default=None, help='If writing the results to the file home_dir/test/args.target[0]grud_prosp_testset_results.csv is not wanted then an alternative directory can be supplied to override the output location.')
    parser.add_argument("--regularly_sampled", action='store_true', help="Set this flag to have regularly sampled data rather than irregularly sampled data.")
    parser.add_argument('--num_dataloader_workers', type=int, default=4, help='# dataloader workers.')
    parser.add_argument('--test_size', type=int, default=7, help='# of days in test set which were not in the training data')
    parser.add_argument('--max_seq_len', type=int, default=48, help='maximum number of timepoints to feed into the model, required for load_data.')
    parser.add_argument('--zscore', action='store_true', help='Set this flag to train a model using the z score (assume forward fill imputation for missing data)')
    parser.add_argument('--weekofyear', action='store_true', help='Use the week_of_year feature when fitting testing the xgb model.')
    parser.add_argument("--num_feature", type=int, default=48, help="number of features passed to the model.")
    parser.add_argument("--grud_hidden", type=int, default=67, help="learning rate for the model")
    parser.add_argument('--opt_level', type=str, default='O1', choices=['O0', 'O1'], help='The model to train.')
    parser.add_argument("--only_healthy", action='store_true', help='Set this flag to train the model on only healthy measurements before the first onset of the target illness')
    parser.add_argument("--feat_subset", action='store_true', help='in the measurement and measurement_z dataframes only use the subset of features found to work better for xgboost and ridge regression')
    parser.add_argument("--eval_train", action='store_true', help='Set this flag to get the performance on the training set.')
    parser.add_argument("--no_imputation", action='store_true', help='If set use the data prior to imputation preserving missing values as NA.')
    parser.add_argument("--out_suf", default='', type=str, help='Suffix to append (before the extension) to each output file name.')
    args = parser.parse_args()

    print(vars(args))
    with open(os.path.join(args.home_dir, 'test', 'args.json'), 'w') as f:
        f.write(json.dumps(vars(args)))

    main(args)


text:
import os
import numpy as np
import utils
import dataset
import models as ml
import tensorflow as tf
import tensorflow_datasets as tfds
import cv2

import matplotlib.pyplot as plt

print("TensorFlow version: {}".format(tf.__version__))
print("Eager execution: {}".format(tf.executing_eagerly()))
print("Is there a GPU available: "),
print('No GPU is available' if len(tf.config.list_physical_devices("GPU")) == 0 else 'Yes, there is a GPU')

ds = dataset.load_dataset(name="davis", split="train")
print(ds)

# t = ds.take(4)
# for example in tfds.as_numpy(t):
#     print(example['video']['segmentations'].shape, example['video']['frames'].shape)
#
ds = dataset.pad(ds)
print(ds)

t = ds.take(4)
for example in tfds.as_numpy(t):
    print(example['x'].shape, example['y'].shape)
#
# x_indim, _ = utils.shape_extractor(ds)
#
# print(x_indim)
# element = next(iter(ds))
# print(element)

# for example in tfds.as_numpy(ds):
#     print(np.unique(example['video']['segmentations']))
#     print(example['metadata'])
#     print(example['video'])

# utils.frame_show(ds, video_no=[1, 2, 3, 4])

# print(element[])
# print('segmentation label in ', np.unique(element['video']['segmentations'])

# model = ml.VOS_Model(input_x=element, indim=x_indim)
# print(model.summary())

def train():
    pass

def test():
    pass

if __name__ == '__main__':
    pass
    # print('Video Object Segmentation')

# utils.frame_show(ds)

# for example in ds:
#     print(example['video']['frames'].numpy().shape)
#     if example['video']['frames'].numpy().shape[2] != 854:
#         print(example['video']['frames'].numpy().shape)
#         print('not 854 \n')
#         # print(type(example['video']['frames']))
#         # example['video']['frames'] = tf.image.resize_with_crop_or_pad(example['video']['frames'], 480, 854)
#         # print(type(example['video']['frames']))
#         # print('padded', example['video']['frames'].numpy().shape)
#         # print('\n')

max_stars_repo_name: malekijoo/danaxa

max_issues_repo_name: malekijoo/danaxa

max_forks_repo_name: malekijoo/danaxa


text:
"""
Training a seq2bow encoder-decoder model
========================================
"""
from tmnt.estimator import SeqBowEstimator
import numpy as np
import gluonnlp as nlp
import os
import mxnet as mx
import logging
from sklearn.datasets import fetch_20newsgroups
from tmnt.preprocess.vectorizer import TMNTVectorizer
from tmnt.inference import SeqVEDInferencer
from tmnt.bert_handling import get_bert_datasets
from tmnt.utils.log_utils import logging_config
from mxnet.gluon.data import ArrayDataset
from tmnt.selector import BaseSelector
from tmnt.configuration import TMNTConfigSeqBOW, default_seq_config_space
from tmnt.trainer import SeqBowVEDTrainer
import autogluon.core as ag

data, y = fetch_20newsgroups(shuffle=True, random_state=1,
                              remove=('headers', 'footers', 'quotes'),
                              return_X_y=True)

tr_size = 8000
train_data = data[:tr_size]
dev_data   = data[-tr_size:]
train_y    = y[:tr_size]
dev_y      = y[-tr_size:]
model_name = 'bert_12_768_12'
dataset = 'book_corpus_wiki_en_uncased'
batch_size = 32
seq_len = 128
pad = True

use_logging = True

if use_logging:
    logging_config(folder='.', name='f_seqbow_20news', level='info', console_level='info')
    log_method = 'log'
else:
    log_method = 'print'

train_y_s = ['class_'+str(y) for y in train_y]
dev_y_s = ['class_'+str(y) for y in dev_y]

## Or - if training a pure unsupervised topic model:
#train_y_s = [ None for _ in train_y ]
#dev_y_s   = [ None for _ in dev_y ]

#config_space = './ved_config.yaml'
#tmnt_config = TMNTConfigSeqBOW(config_space).get_configspace()
#tmnt_config = ag.space.Dict(**default_seq_config_space)
tmnt_config = default_seq_config_space
selector = BaseSelector(tmnt_config, iterations=2, searcher='random',
                        scheduler='fifo', cpus_per_task=8, log_dir='_exp_out_dir')

trainer = SeqBowVEDTrainer('_exp_out_dir', (train_data, train_y_s), (dev_data, dev_y_s), use_gpu=False, log_interval=1)
selector.select_model(trainer)

max_stars_repo_name: mitre/tmnt

max_issues_repo_name: mitre/tmnt

max_forks_repo_name: mitre/tmnt


text:
#importing some useful packages
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
import cv2
import os
import glob

def renderVideoFFMPEG():
    os.system("ffmpeg -f image2 -s 1280x720 -r 25 -i ./render/frameOut%05d.jpg -vcodec mpeg4 -y ./output.mp4")


text:
import os
import numpy as np
import tensorflow as tf
import torch
import argparse
import collections

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--layers_num", type=int, default="12",
                        help=".")
    parser.add_argument("--input_model_path", type=str, default="models/google_model.bin",
                        help=".")
    parser.add_argument("--output_model_path",
                        type=str,
                        default="models/bert_base_chinese.ckpt",
                        help=".")

    args = parser.parse_args()

    input_model = torch.load(args.input_model_path)

    session = tf.Session()
    output_model = collections.OrderedDict()

    output_model["bert/embeddings/word_embeddings"] = input_model["embedding.word_embedding.weight"]
    output_model["bert/embeddings/position_embeddings"] = input_model["embedding.position_embedding.weight"]
    output_model["bert/embeddings/token_type_embeddings"] = input_model["embedding.segment_embedding.weight"][1:, :]
    output_model["bert/embeddings/LayerNorm/gamma"] = input_model["embedding.layer_norm.gamma"]
    output_model["bert/embeddings/LayerNorm/beta"] = input_model["embedding.layer_norm.beta"]

    for i in range(args.layers_num):
        output_model["bert/encoder/layer_" + str(i) + "/attention/self/query/kernel"] = input_model[
            "encoder.transformer." + str(i) + ".self_attn.linear_layers.0.weight"]
        output_model["bert/encoder/layer_" + str(i) + "/attention/self/query/bias"] = input_model[
            "encoder.transformer." + str(i) + ".self_attn.linear_layers.0.bias"]
        output_model["bert/encoder/layer_" + str(i) + "/attention/self/key/kernel"] = input_model[
            "encoder.transformer." + str(i) + ".self_attn.linear_layers.1.weight"]
        output_model["bert/encoder/layer_" + str(i) + "/attention/self/key/bias"] = input_model[
            "encoder.transformer." + str(i) + ".self_attn.linear_layers.1.bias"]
        output_model["bert/encoder/layer_" + str(i) + "/attention/self/value/kernel"] = input_model[
            "encoder.transformer." + str(i) + ".self_attn.linear_layers.2.weight"]
        output_model["bert/encoder/layer_" + str(i) + "/attention/self/value/bias"] = input_model[
            "encoder.transformer." + str(i) + ".self_attn.linear_layers.2.bias"]
        output_model["bert/encoder/layer_" + str(i) + "/attention/output/dense/kernel"] = input_model[
            "encoder.transformer." + str(i) + ".self_attn.final_linear.weight"]
        output_model["bert/encoder/layer_" + str(i) + "/attention/output/dense/bias"] = input_model[
            "encoder.transformer." + str(i) + ".self_attn.final_linear.bias"]
        output_model["bert/encoder/layer_" + str(i) + "/attention/output/LayerNorm/gamma"] = input_model[
            "encoder.transformer." + str(i) + ".layer_norm_1.gamma"]
        output_model["bert/encoder/layer_" + str(i) + "/attention/output/LayerNorm/beta"] = input_model[
            "encoder.transformer." + str(i) + ".layer_norm_1.beta"]
        output_model["bert/encoder/layer_" + str(i) + "/intermediate/dense/kernel"] = input_model[
            "encoder.transformer." + str(i) + ".feed_forward.linear_1.weight"]
        output_model["bert/encoder/layer_" + str(i) + "/intermediate/dense/bias"] = input_model[
            "encoder.transformer." + str(i) + ".feed_forward.linear_1.bias"]
        output_model["bert/encoder/layer_" + str(i) + "/output/dense/kernel"] = input_model[
            "encoder.transformer." + str(i) + ".feed_forward.linear_2.weight"]
        output_model["bert/encoder/layer_" + str(i) + "/output/dense/bias"] = input_model[
            "encoder.transformer." + str(i) + ".feed_forward.linear_2.bias"]
        output_model["bert/encoder/layer_" + str(i) + "/output/LayerNorm/gamma"] = input_model[
            "encoder.transformer." + str(i) + ".layer_norm_2.gamma"]
        output_model["bert/encoder/layer_" + str(i) + "/output/LayerNorm/beta"] = input_model[
            "encoder.transformer." + str(i) + ".layer_norm_2.beta"]

    output_model["bert/pooler/dense/kernel"] = input_model["target.nsp_linear_1.weight"]
    output_model["bert/pooler/dense/bias"] = input_model["target.nsp_linear_1.bias"]
    output_model["cls/seq_relationship/output_weights"] = input_model["target.nsp_linear_2.weight"]
    output_model["cls/seq_relationship/output_bias"] = input_model["target.nsp_linear_2.bias"]
    output_model["cls/predictions/transform/dense/kernel"] = input_model["target.mlm_linear_1.weight"]
    output_model["cls/predictions/transform/dense/bias"] = input_model["target.mlm_linear_1.bias"]
    output_model["cls/predictions/transform/LayerNorm/gamma"] = input_model["target.layer_norm.gamma"]
    output_model["cls/predictions/transform/LayerNorm/beta"] = input_model["target.layer_norm.beta"]
    output_model["cls/predictions/output_bias"] = input_model["target.mlm_linear_2.bias"]

    def assign_tf_var(tensor: np.ndarray, name: str):
        tmp_var = tf.Variable(initial_value=tensor)
        tf_var = tf.get_variable(dtype=tmp_var.dtype, shape=tmp_var.shape, name=name)
        op = tf.assign(ref=tf_var, value=tmp_var)
        session.run(tf.variables_initializer([tmp_var, tf_var]))
        session.run(fetches=[op, tf_var])
        return tf_var
    tf_vars = []
    tensors_to_transopse = (
        "dense/kernel",
        "attention/self/query",
        "attention/self/key",
        "attention/self/value"
    )
    for k, v in output_model.items():
        tf_name = k
        torch_tensor = v.cpu().numpy()
        if any([x in k for x in tensors_to_transopse]):
            torch_tensor = torch_tensor.T
        tf_tensor = assign_tf_var(tensor=torch_tensor, name=tf_name)
        tf_vars.append(tf_tensor)
        print("{0}{1}initialized".format(tf_name, " " * (60 - len(tf_name))))

    saver = tf.train.Saver(tf_vars)
    saver.save(session, args.output_model_path)

if __name__ == "__main__":
    main()

max_stars_repo_name: tjuwlz/JointCWPDParser

max_issues_repo_name: tjuwlz/JointCWPDParser

max_forks_repo_name: tjuwlz/JointCWPDParser