text:
#
# holt-winters forecasting
#
# H-W is a structural timeseries model with level, trend, and seasonal
# components that are estimated by exponential smoothing as data
# arrives (https://en.wikipedia.org/wiki/Exponential_smoothing).
# Only the additive seasonality model is implemented.
#
from sys import stderr
from copy import deepcopy
from collections import namedtuple
import numpy as np
from scipy.optimize import fmin_l_bfgs_b
import seasonal

HWState = namedtuple('HWState', 't level trend seasons')
HWParams = namedtuple('HWParams', 'alpha beta gamma')

def estimate_state(data):
    """estimate initial state for Holt Winters

    HWState estimates are for t=-1, the step before y[0].

    Parameters
    ----------
    data : ndarray
        observations

    """
    seasons, trended = seasonal.fit_seasons(data)
    if seasons is None:
        seasons = np.zeros(1)
    trend = trended[1] - trended[0]
    level = trended[0] - trend
    return HWState(-1, level, trend, seasons)

def forecast(state, steps=1):
    """return a single or multi-step forecast from the current state

    Parameters
    ----------
    state : HWState
        current model state
    steps : int
        number of steps out to forecast

    """
    season = state.seasons[(state.t + steps) % len(state.seasons)]
    return state.level + state.trend * steps + season

def advance(y, params, state):
    """incorporate the next observation into the state estimate.

    This returns updated state, using Hyndman's error correction form of H-W [1]
    It mutates state's seasonal array.

    Parameters
    ----------
    y : float
        observed value at time state.t + 1
    params : HWParams
        alpha, beta, gamma params for HW
    state : HWState
        current HW state

    Returns
    -------
    state, err : HWState, float
        state: updated state
        one-step forecast error for y

    References
    ----------
    .. [1] https://www.otexts.org/fpp/7/5, Holt-Winters additive method

    """
    seasons = state.seasons
    e = y - forecast(state)
    level = state.level + state.trend + params.alpha * e
    trend = state.trend + params.alpha * params.beta * e
    seasons[(state.t + 1) % len(state.seasons)] += params.gamma * e
    # in a proper implementation, we would enforce seasons being 0-mean.
    return HWState(state.t+1, level, trend, seasons), e

def estimate_params(data, state, alpha0=0.3, beta0=0.1, gamma0=0.1):
    """Estimate Holt Winters parameters from data

    Parameters
    ----------
    data : ndarray
        observations
    state : HWState
        initial state for HW (one step prior to first data value)
    alpha0, beta0, gamma0 : float, float, float
        initial guess for HW parameters

    Returns
    -------
    params : HWParams

    Notes
    -----
    This is a not a demo about estimating Holt Winters parameters, and
    this is not a great way to go about it, because it does not
    produce good out-of-sample error. In this demo, we unrealistically
    train the HW parameters over all the data, not just the training
    prefix used for the initial seasonal state estimate.

    """
    def _forecast_error(x0, state, data):
        """bfgs HW parameter error callback."""
        E = 0
        state = deepcopy(state)
        params = HWParams(*x0)
        for y in data:
            state, e = advance(y, params, state)
            E += e * e
        return E / len(data)

    alpha, beta, gamma = fmin_l_bfgs_b(
        _forecast_error, x0=[alpha0, beta0, gamma0], bounds=[[0, 1]] * 3,
        args=(state, data), approx_grad=True)[0]
    return HWParams(alpha, beta, gamma)

def hw(data, split=None, params=None):
    """fit a HW model and return the 1-step forecast and smoothed series.

    Parameters
    ----------
    data : array of float
        observations
    split : number
        initialize using the leading split*100% of the data (if split <=1.0)
        or N=split points (if split > 1)

    Returns
    -------
    forecast, smoothed : ndarray, ndarray

    """
    if split is None:
        splitidx = len(data)
    elif split > 1.0:
        splitidx = int(split)
    else:
        splitidx = int(split * len(data))
    state = estimate_state(data[:splitidx])
    print "||seasons|| = {:.3f}".format(np.sqrt(np.sum(state.seasons ** 2)))
    if params is None:
        params = estimate_params(data, state)
        print "estimated alpha={:.3f}, beta={:.3f}, gamma={:.3f}".format(*params)
    level = np.empty(len(data))
    fcast = np.empty(len(data))
    for y in data:
        yhat = forecast(state)
        state, _ = advance(y, params, state)
        level[state.t], fcast[state.t] = state.level, yhat
    print "RMSE = ", np.sqrt(np.sum((fcast - data) ** 2) / len(data))
    print "final ||seasons|| = {:.3f}".format(np.sqrt(np.sum(state.seasons ** 2)))
    return fcast, level

def main():
    import os.path
    from optparse import OptionParser
    from textwrap import dedent
    import matplotlib
    import matplotlib.pyplot as plt
    matplotlib.style.use('ggplot')
    from seasonal.application import read_csv

    parser = OptionParser(
        usage="usage: %prog [options] csv-files...",
        description="Holt-Winters demo using the seasonal package"
    )
    parser.add_option("--column", default=-1,
                      help="csv column to use (name or 0-based index, default rightmost)")
    parser.add_option("--split", type="float", default=None,
                      help=("split data at the split*100%  or int(split) " +
                            "point for initialization"))
    parser.add_option("--params",
                      help=("comma-separated list of alpha, beta, gamma. " +
                            "default is to estimate these from ALL the data"))
    parser.add_option("--demo", action="store_true",
                      help="demonstrate with some air passenger data")
    (options, args) = parser.parse_args()

    if options.demo:
        args = [os.path.join(os.path.dirname(seasonal.__file__),
                             "data/airPassengers.csv")]
        if options.split is None:
            options.split = 0.20

    if not args:
        parser.print_help()
        exit(-1)

    if not plt:
        stderr.write(
            "Error: matplotlib must be installed\n")
        exit(-1)

    if options.params is not None:
        try:
            params = [float(p) for p in options.params.split(',')]
            options.params = HWParams(*params)
        except Exception:
            stderr.write("\nError: --params wants alpha,beta,gamma\n")
            parser.print_help()
            exit(-1)

    for csvpath in args:
        index, data, column = read_csv(csvpath, column=options.column)
        fcast, smoothed = hw(data, options.split, params=options.params)
        plt.figure(1)
        plt.subplot(211)
        plt.title("Holt Winters for "+os.path.basename(csvpath))
        plt.plot(index, data, label=column)
        plt.plot(index, fcast, label="forecast")
        plt.plot(index, smoothed, label="smoothed")
        leg = plt.legend(loc='upper left')
        leg.get_frame().set_alpha(0.5)
        plt.subplot(212)
        plt.title("Forecast Error")
        plt.plot(index, fcast - data)
        plt.show()

if __name__ == "__main__":
    main()

max_stars_repo_name: welch/seasonal

reason: import numpy,from scipy


text:
# python codes for empirical hilbert-huang transform

from matplotlib import *
from numpy import *
#from scipy.interpolate import interp1d
from matplotlib import pyplot
from scipy import interpolate, arange
from scipy.signal import signaltools    # including convoluation and hilbert transform
import cmath

#extremaDetection([1,2,3,3,2,1,1,2,3,4,5,6,7,2,3,4,4,3,2,4,5,6,7,9,0])
def extremaDetection(data=[1,2,3,4,5], x=[0,1,2,3,4]):
    start = min(x)
    step = x[1] - x[0]
    upperEnvelop = [(start,data[0])]	#initialisation
    lowerEnvelop = [(start,data[0])]
    j = data[0]
    print (start,data[0]),
    newDirection = 0
    oldDirection = sign(data[1]-data[0])# =1 when the data is going up, 0 when flat
    cache = []       	# to store data for the case when there is a stationary point
    xcount = start
    for i in data[1:]:
        xcount+= step
        print (xcount,i),

        newDirection = sign(i-j)
        if newDirection == 0:
           cache.append((xcount,i))
           j=i
           continue			# got a flat section, keep it in cache
        if (oldDirection == 1) and (newDirection == -1):	#got a max
            upperEnvelop += cache
            cache = [(xcount,i)]
            oldDirection = newDirection
            print '!'
        if (oldDirection ==-1) and( newDirection == 1 ):	#got a min
            lowerEnvelop += cache
            cache = [(xcount,i)]
            oldDirection = newDirection
            print 'v'
        if oldDirection * newDirection == 1:		#equal but nonzero : clear cache
            cache = [(xcount,i)]
        j = i						#keep old datapoint
    upperEnvelop.append((xcount,data[-1]))
    lowerEnvelop.append((xcount,data[-1]))
    print upperEnvelop, lowerEnvelop
    return upperEnvelop, lowerEnvelop

## to construct the cubic spline

def cubicSpline(x, y):
    return interpolate.splrep(x,y)

## to display the spline

def displaySpline(f, x, y):
    a = min(x)
    b = max(x)
    xnew = arange(a,b, (a-b)/50.0)
    ynew = interpolate.splev(xnew, f, der=0)
    pyplot.plot(xnew, ynew, '.', x, y, 'o')
    pyplot.title('displaySpline')
    pyplot.legend(['interpolation','data'])

## stoppage criterion:
def stoppage(y1,y2):
    y2=single(y2)
    SD = sum((y1-y2)**2 / y1**2)
    if SD < 0.2:
        return True
    else:
        return False

##getting the next mode

def getNextMode(y=[1,2,3,2,1], x=[0,1,2,3,4]):  # also called 'sifting' -
                                 #input:  a function;  output: the next mode y1
                                 #                       and the residue m=y-y1

    upper,lower = extremaDetection(y, x)
    Fu = cubicSpline([v[0] for v in upper], [v[1] for v in upper])
    Fl = cubicSpline([v[0] for v in lower], [v[1] for v in lower])
    xnew = arange( min(x), max(x), 0.1)
    ynew = interpolate.splev(xnew, F)
    ynew_u = interpolate.splev(xnew, Fu)
    ynew_l = interpolate.splev(xnew, Fl)
    ynew_m = (ynew_u + ynew_l) /2
    y1 = (ynew - ynew_m)   #first mode=original data interpolated - average trend
    return y1, ynew_m

#empirical mode decomposition (EMP)
def emp(y=[1,2,3,4,5]):       #input: a function;  output: a list of functions

    stoppage = False
    while stoppage == False:
        #########
        #  main loop
        #  F = cubicSpline(x,y) - already done before the function call
        stoppage = True
    return data

##############

##marginal spectrum
def marginalSpectrum(x):
    return x

def main():

  y = array([1,2,3,3,2,1,1,2,3,4,5,6,7,2,3,4,4,3,2,4,5,6,7,9,0])
  x = array([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24])

  y = array([1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2])
  x = array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13])

  y = array([1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1])
  x = array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13])

  y = [1, 2, 1, 2, 1, 2, 1, 3, 0, 2, 1, 2, 2, 1, 2]
  x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14]

  y = [4, 3, 4, 2, 5, 1, 6, 3, 9, 0,10, 1,11, 4,13]
  x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14]

  y = [0, -0.1, 1, 1, 2, 2.1, 2, 3, 3, 2, 2, 0, 0, 5, 5, 2, 2, 6, 6]
  x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18]

  print 'x-y:',
  print len(x)-len(y)

  F = cubicSpline(x,y)
  upper,lower = extremaDetection(y)
  #Fu = cubicSpline([v[0] for v in upper], [v[1] for v in upper])
  #Fl = cubicSpline([v[0] for v in lower], [v[1] for v in lower])
  #xnew = arange( min(x), max(x), 0.1)
  #ynew = interpolate.splev(xnew, F)
  #ynew_u = interpolate.splev(xnew, Fu)
  #ynew_l = interpolate.splev(xnew, Fl)
  #ynew_m = (ynew_u + ynew_l) /2

  #y1 = (ynew - ynew_m)   #first mode=original data interpolated - average trend
  y1, ynew_m = getNextMode(ynew, xnew)

  pyplot.plot(x,y,'.', xnew,ynew,'-', xnew, ynew_u, '--', xnew, ynew_l, '--',
              xnew, ynew_m,'--', xnew, y1, '-', xnew, [0]*len(xnew), '-' )
  pyplot.legend(['data', 'interpolation', 'upper', 'lower', '"mean"','first mode'], loc='best')
  pyplot.show()

  #computing the analytic signal (x + i.hilbertTransformOf(x) )
  Hy1 = signaltools.hilbert(y1)
  #print Hy1

  #extracting the phase
  phase1 = [cmath.phase(i) for i in Hy1]
  pyplot.plot(xnew, phase1, '.')
  pyplot.title('phase plot for the first mode y1, from the analytic signal')
  pyplot.show()
  #plotting the instantaneous frequency omega = dtheta/dt
  omega1=[]
  for i in range(len(phase1)-1):
      om = (phase1[i+1]-phase1[i])
      if om > 3.15:
          om = om- 2*pi
      if om < -3.15:
          om += 2*pi

      om = om/(xnew[i+1]-xnew[i])
      omega1.append(om)

  pyplot.plot(xnew[10:-11], omega1[10:-10],'.')
  pyplot.title('instantanenous frequency omega1 for the first mode')
  pyplot.show()
  #print omega1
  #print xnew

if __name__ == "__main__":
    main()

ext: py

max_stars_repo_name: yaukwankiu/armor

max_issues_repo_name: yaukwankiu/armor

max_forks_repo_name: yaukwankiu/armor

reason: from numpy,from scipy


text:
import unittest
import retrain
import numpy as np

class test_BottleneckCache(unittest.TestCase):
    def test_save_and_load(self):
        """
        testing
        :return:
        """
        file_path = './test_temp_file.h5'
        d0 = {0: np.array([.1, .2, .3]), 1: np.array([.4, .5, .6])}
        d1 = {0: np.array([.2, .3, .4]), 1: np.array([.5, .6, .7]), 2: np.array([.7])}

        bn_cache0 = retrain.BottleneckCache('file1 file2')
        bn_cache0.feed(d0)
        bn_cache1 = retrain.BottleneckCache('file1 file2 file3')
        bn_cache1.feed(d1)
        bn_cache02 = retrain.BottleneckCache('file1 file2')
        bn_cache11 = retrain.BottleneckCache('file1')

        self.assertEqual(bn_cache0.size(), 2)
        self.assertEqual(bn_cache1.size(), 3)
        self.assertTrue(np.allclose(d0[0], bn_cache0.get(0)))
        self.assertTrue(np.allclose(d1[1], bn_cache1.get(1)))
        self.assertEqual(bn_cache02.size(), 2)
        self.assertTrue(np.allclose(d0[1], bn_cache02.get(1)))
        self.assertEqual(bn_cache11.size(), 2)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: MindGeek/inception-retrain-rh

max_issues_repo_name: MindGeek/inception-retrain-rh

max_forks_repo_name: MindGeek/inception-retrain-rh

reason: import numpy


text:
from matplotlib import pyplot as plt
from PIL import Image
import pandas as pd
import matplotlib
import numpy as np

from typing import Optional, Union, Mapping  # Special
from typing import Sequence, Iterable  # ABCs
from typing import Tuple  # Classes

from anndata import AnnData
import warnings

from .classes import GenePlot
from .classes_bokeh import BokehGenePlot
from ..utils import Empty, _empty, _AxesSubplot

from bokeh.io import push_notebook, output_notebook
from bokeh.plotting import show

def gene_plot(
    adata: AnnData,
    gene_symbols: Union[str, list] = None,
    threshold: Optional[float] = None,
    method: str = "CumSum",
    contour: bool = False,
    step_size: Optional[int] = None,
    title: Optional["str"] = None,
    figsize: Optional[Tuple[float, float]] = None,
    cmap: Optional[str] = "Spectral_r",
    use_label: Optional[str] = None,
    list_clusters: Optional[list] = None,
    ax: Optional[_AxesSubplot] = None,
    show_plot: Optional[bool] = True,
    show_axis: Optional[bool] = False,
    show_image: Optional[bool] = True,
    show_color_bar: Optional[bool] = True,
    crop: Optional[bool] = True,
    margin: Optional[bool] = 100,
    size: Optional[float] = 7,
    image_alpha: Optional[float] = 1.0,
    cell_alpha: Optional[float] = 0.7,
    use_raw: Optional[bool] = False,
    fname: Optional[str] = None,
    dpi: Optional[int] = 120,
) -> Optional[AnnData]:

    GenePlot(
        adata,
        gene_symbols=gene_symbols,
        threshold=threshold,
        method=method,
        contour=contour,
        step_size=step_size,
        title=title,
        figsize=figsize,
        cmap=cmap,
        use_label=use_label,
        list_clusters=list_clusters,
        ax=ax,
        show_plot=show_plot,
        show_axis=show_axis,
        show_image=show_image,
        show_color_bar=show_color_bar,
        crop=crop,
        margin=margin,
        size=size,
        image_alpha=image_alpha,
        cell_alpha=cell_alpha,
        use_raw=use_raw,
        fname=fname,
        dpi=dpi,
    )

def gene_plot_interactive(adata: AnnData):
    bokeh_object = BokehGenePlot(adata)
    output_notebook()
    show(bokeh_object.app, notebook_handle=True)


text:
import os
import re
import numpy as np
import pandas as pd
import subprocess
try: # pip install rpy2==3.3.1
    from rpy2.robjects import pandas2ri, r
except ModuleNotFoundError:
    pass
#
from electricityLoadForecasting import paths

def fit_and_predict(inputs_training,
                    Y_training,
                    inputs_validation,
                    hprm,
                    assignments = {},
                    ):
    Y_hat_training   = pd.DataFrame(0,
                                    index   = Y_training.index,
                                    columns = Y_training.columns,
                                    )
    Y_hat_validation = pd.DataFrame(0,
                                    index   = inputs_validation.index,
                                    columns = Y_training.columns,
                                    )

    for ii, site_name in enumerate(Y_training.columns):
        print('\r{0}/{1}'.format(ii, Y_training.shape[1]), end = '\n')

        site_selection = [(inpt, trsfm, prm, location)
                          for inpt, trsfm, prm, location in inputs_training.columns
                          if (   inpt not in assignments
                              or location in assignments[inpt][site_name].values
                              )
                          ]

        site_inputs_training   = inputs_training  [site_selection]
        site_inputs_validation = inputs_validation[site_selection]

        site_inputs_training.columns   = site_inputs_training.columns.remove_unused_levels()
        site_inputs_validation.columns = site_inputs_validation.columns.remove_unused_levels()

        Y_hat_training[site_name], Y_hat_validation[site_name] = call_fitter(site_inputs_training,
                                                                             Y_training[site_name],
                                                                             site_inputs_validation,
                                                                             hprm,
                                                                             )
    return Y_hat_training, Y_hat_validation

def call_fitter(site_inputs_training,
                y_training,
                site_inputs_validation,
                hprm,
                ):
    assert y_training.ndim == 1
    path_R_files = os.path.join(paths.outputs,
                                'R_files/',
                                )
    os.makedirs(path_R_files,
                exist_ok = True,
                )

    ### Data
    data_training   = {**{simplify_inpt_name(inpt, trsfm, prm, location) : site_inputs_training[inpt, trsfm, prm, location].values
                          for inpt, trsfm, prm, location in site_inputs_training
                          },
                       'target' : y_training.values,
                       }

    data_validation = {simplify_inpt_name(inpt, trsfm, prm, location) : site_inputs_validation[inpt, trsfm, prm, location].values
                       for inpt, trsfm, prm, location in site_inputs_validation
                       }

    # Convert arrays
    pandas2ri.activate()
    df_train = pandas2ri.py2rpy(pd.DataFrame.from_dict(data_training))
    df_test  = pandas2ri.py2rpy(pd.DataFrame.from_dict(data_validation))
    pandas2ri.deactivate()

    # Save converted files
    r.assign("data_train", df_train)
    r("save(data_train, file='{0}/temp_dat_for_r_train.gzip', compress=TRUE)".format(path_R_files))
    r.assign("data_test",  df_test)
    r("save(data_test,  file='{0}/temp_dat_for_r_test.gzip',  compress=TRUE)".format(path_R_files))

    nb_unique = {k : len(np.unique(v))
                 for k, v in site_inputs_training.items()
                 }

    string_formula = make_gam_formula(site_inputs_training.columns,
                                      nb_unique,
                                      hprm,
                                      )

    ### Launch the R script
    path2script = os.path.join(os.path.dirname(__file__),
                               'load_fit_predict_savePredictions.R',
                               )
    args        = [string_formula, path_R_files]
    cmd         = ['Rscript', path2script] + args
    # Python will quote what must be quoted in subprocess.check_output

    print('launch Rscript')
    x = subprocess.check_output(cmd, universal_newlines=True)
    print(x)

    y_hat_training   = r['read.table']("{0}/predictions_from_r_train.gzip".format(path_R_files))
    y_hat_training   = pandas2ri.rpy2py(y_hat_training)
    y_hat_training   = y_hat_training.values

    y_hat_validation = r['read.table']("{0}/predictions_from_r_test.gzip".format(path_R_files))
    y_hat_validation = pandas2ri.rpy2py(y_hat_validation)
    y_hat_validation = y_hat_validation.values

    return y_hat_training, y_hat_validation

def simplify_inpt_name(qty,
                       transform,
                       prm,
                       location = '',
                       ):
    ans = re.sub('_+', '_', '_'.join([qty,
                                      (transform[:3]
                                       if
                                       bool(transform)
                                       else
                                       ''
                                       ),
                                      (str(prm.n*prm.hours
                                           if hasattr(prm, 'hours')
                                           else prm
                                           ).replace('-', 'm')
                                       if bool(prm)
                                       else
                                       ''
                                       ),
                                      str(location).replace('-','_'),
                                      ])).strip('_')
    return ans

def make_gam_formula(list_cols,
                     nb_unique,
                     hprm,
                     ):

    ### Formula
    univariate_formula = {simplify_inpt_name(*inpt) : v
                          for inpt, v in hprm['gam.univariate_functions'].items()
                          }

    bivariate_formula  = {(simplify_inpt_name(*inpt1),
                           simplify_inpt_name(*inpt2),
                           ) : v
                          for (inpt1, inpt2), v in hprm['gam.bivariate_functions'].items()
                          }

    ### Init
    formula = 'target~'

    ### Univariate part
    for col in list_cols:
        *inpt, location = col
        simple_cat = simplify_inpt_name(*inpt)
        if simple_cat in univariate_formula:
            formula += (formula[-1]!='~')*'+'
            if nb_unique[col] > 1:
                formula += univariate_formula[simple_cat][0]
            formula += '('
            formula += simplify_inpt_name(*col)
            if univariate_formula[simple_cat][1]:
                formula += ',' + univariate_formula[simple_cat][1]
            formula += ')'

    ### Bivariate part
    for col1 in list_cols:
        for col2 in list_cols:
            *inpt1, location1 = col1
            *inpt2, location2 = col2
            simple_cat1 = simplify_inpt_name(*inpt1)
            simple_cat2 = simplify_inpt_name(*inpt2)
            if (simple_cat1, simple_cat2) in bivariate_formula:
                if len(bivariate_formula[simple_cat1, simple_cat2]) == 1:
                    assert bivariate_formula[simple_cat1, simple_cat2][0] == 'by'
                    formula += (formula[-1]!='~')*'+'
                    if nb_unique[col1] > 1:
                        formula += 's'
                    formula += '({0},by={1})'.format(simplify_inpt_name(*col1),
                                                     simplify_inpt_name(*col2),
                                                     )
                elif len(bivariate_formula[simple_cat1, simple_cat2]) == 2:
                    formula += (formula[-1]!='~')*'+'
                    formula += '{0}({1},{2}{3})'.format(bivariate_formula[simple_cat1, simple_cat2][0],
                                                        simplify_inpt_name(*col1),
                                                        simplify_inpt_name(*col2),
                                                        (',' + bivariate_formula[simple_cat1, simple_cat2][1]
                                                         if bivariate_formula[simple_cat1, simple_cat2][1]
                                                         else
                                                         ''
                                                         ),
                                                        )
                else:
                    raise ValueError

    return formula


text:
import pandas as pd
import os
import numpy as np
import argparse

parser = argparse.ArgumentParser(description='Plot results.')
parser.add_argument('--input-dir', type=str, default='/vision2/u/zixianma/football_results', required=True)
parser.add_argument('--output-dir', type=str, default='/vision2/u/zixianma/football_results/data')
parser.add_argument('--seeds', type=int, nargs='+', help='a list of random seeds', required=True)
parser.add_argument('--eval-last', type=int, default=50000)
# parser.add_argument('--eval-end', type=int, default=50000)
parser.add_argument('--metric', type=str, default='avg')
args = parser.parse_args()

dirpath = args.input_dir
metric_vals = []

for i in args.seeds:
    filename = f'seed{i}.json'
    json_path = os.path.join(os.path.join(dirpath, filename))
    print(json_path)
    df = pd.read_json(json_path, lines=True)
    end = len(df)
    assert end >= args.eval_last, "There aren't enough entries in the input file. \
    Decrease the eval-last number or double check your input file."
    start = end - args.eval_last
    print(f'The are {len(df)} entries in the input file.')
    if args.metric == 'avg':
        avg = np.mean(df['episode_reward_mean'][start:end])
        print(f'The avg episode reward for seed {i} is {avg}.')
        metric_vals.append(avg)
    elif args.metric == 'max':
        max = np.max(df['episode_reward_mean'][start:end])
        print(f'The max episode reward for seed {i} is {max}.')
        metric_vals.append(max)
    else:
        raise NotImplementedError

divide_pos = args.input_dir.rfind('/')
exp_name = args.input_dir[divide_pos+1:] + '.csv'
if not os.path.exists(args.output_dir):
    os.mkdir(args.output_dir)
output_path = os.path.join(args.output_dir, exp_name)
print(f"Outputing to {output_path}...")
data = {}
mu = np.mean(metric_vals)
sigma = np.std(metric_vals) / np.sqrt(len(metric_vals))
print(f"The mean and stderr are {mu} and {sigma}.")
data['exp_name'] = [exp_name + f'_{args.metric}_last{args.eval_last}']
data[args.metric + '_mean'] = [mu]
data[args.metric + '_stderr'] = [sigma]
output_df = pd.DataFrame(data)
output_df.to_csv(output_path, index=False)

max_stars_repo_name: zixianma/football

max_issues_repo_name: zixianma/football

max_forks_repo_name: zixianma/football


text:
import os

import albumentations as A
import cv2
import numpy as np
import pandas as pd
import torch
from pandas import DataFrame
from torch.utils.data import Dataset, DataLoader

from constants import FACE_FORENSICS
from training.datasets.transform import create_generalization_transform, generalization_preprocessing
from training.datasets.transform import create_train_transform, create_val_test_transform

class FaceForensicsDataset(Dataset):

    def __init__(self, data_root, df: DataFrame, mode: str, transform: A.Compose, fake_type: str,
                 compression_version: str, use_generalization=False, label_smoothing=0.01):
        """
        Args:
            fake_type(str): Deepfakes, Face2Face, FaceShifter, FaceSwap, NeuralTextures, All
        """
        self.original_path = os.path.join(data_root, 'original_sequences', 'youtube', compression_version)
        self.manipulated_path = os.path.join(data_root, 'manipulated_sequences')
        self.df = df
        self.mode = mode
        self.transform = transform
        self.generalization_transform = create_generalization_transform()
        self.fake_type = fake_type
        self.compression_version = compression_version
        self.use_generalization = use_generalization
        self.label_smoothing = label_smoothing

    def __getitem__(self, index):
        if self.fake_type == 'All':
            video, img_file, label, ori_video, frame, fake_type = self.df.iloc[index].values
            fake_type_path = os.path.join(self.manipulated_path, fake_type)
        else:
            video, img_file, label, ori_video, frame = self.df.iloc[index].values
            fake_type_path = os.path.join(self.manipulated_path, self.fake_type)

        if label == 1:
            # image
            img_path = os.path.join(fake_type_path, self.compression_version, 'crops', video, img_file)
            image = cv2.imread(img_path, cv2.IMREAD_COLOR)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            # mask
            mask_path = os.path.join(fake_type_path, 'c23', 'diffs', video, f'{img_file[:-4]}_diff.png')
            mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)
        else:
            img_path = os.path.join(self.original_path, 'crops', video, img_file)
            image = cv2.imread(img_path, cv2.IMREAD_COLOR)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            mask = np.zeros((image.shape[0], image.shape[1]), dtype=np.uint8)

        if self.mode == 'train' and self.use_generalization:
            landmark_path = os.path.join(self.original_path, "landmarks", str(ori_video), img_file[:-4] + ".npy")
            image, mask = generalization_preprocessing(landmark_path, image, label, mask, self.generalization_transform)

        # data augmentation
        transformed = self.transform(image=image, mask=mask)
        image = transformed["image"]
        mask = transformed["mask"]
        mask = mask.unsqueeze(0) / 255.

        return {'images': image, 'labels': label, 'masks': mask}

    def __len__(self):
        r = self.df.shape[0]
        return r

def get_face_forensics_dataloader(model_cfg, args, fake_type="Deepfakes"):
    train_df = pd.read_csv(f'data/{FACE_FORENSICS}/{args.compression_version}/data_{FACE_FORENSICS}_{fake_type}_train.csv')
    # row_num = train_df.shape[0]
    # start_index = (row_num - 9600) // 2
    # train_df = train_df.iloc[start_index:start_index + 9600]
    # train_df = train_df.iloc[-1000:]
    train_transform = create_train_transform(model_cfg)
    train_data = FaceForensicsDataset(data_root=args.data_dir, df=train_df, mode='train', transform=train_transform,
                                      fake_type=fake_type, compression_version=args.compression_version,
                                      use_generalization=False)
    if args.distributed:
        train_sampler = torch.utils.data.distributed.DistributedSampler(train_data)
    else:
        train_sampler = None
    train_loader = DataLoader(train_data, batch_size=args.batch_size, shuffle=(train_sampler is None),
                              sampler=train_sampler, num_workers=args.workers, pin_memory=True, drop_last=True)

    val_df = pd.read_csv(f'data/{FACE_FORENSICS}/{args.compression_version}/data_{FACE_FORENSICS}_{fake_type}_val.csv')
    # val_df = val_df.iloc[-1000:]
    val_transform = create_val_test_transform(model_cfg)
    val_data = FaceForensicsDataset(data_root=args.data_dir, df=val_df, mode='validation', transform=val_transform,
                                    fake_type=fake_type, compression_version=args.compression_version)
    val_loader = DataLoader(val_data, batch_size=30, shuffle=False, num_workers=args.workers,
                            pin_memory=True, drop_last=False)

    return train_sampler, train_loader, val_loader

def get_face_forensics_test_dataloader(model_cfg, args, fake_type="Deepfakes"):
    test_df = pd.read_csv(f'data/{FACE_FORENSICS}/{args.compression_version}/data_{FACE_FORENSICS}_{fake_type}_test.csv')
    # test_df = test_df.iloc[-1000:]
    test_transform = create_val_test_transform(model_cfg)
    test_data = FaceForensicsDataset(data_root=args.data_dir, df=test_df, mode='test', transform=test_transform,
                                     fake_type=fake_type, compression_version=args.compression_version)
    test_loader = DataLoader(test_data, batch_size=args.batch_size, shuffle=False, num_workers=args.workers,
                             pin_memory=True, drop_last=False)
    return test_loader

max_stars_repo_name: joizhang/sifdnet

max_issues_repo_name: joizhang/sifdnet

max_forks_repo_name: joizhang/sifdnet


text:
import pandas as pd
import numpy as np
import re
from tqdm.notebook import tqdm
import random
import sklearn.metrics
from sklearn.pipeline import Pipeline

# For XGBoost Regression and Classification
import xgboost as xgb
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, cross_val_score, KFold
from sklearn.metrics import mean_squared_error, f1_score, r2_score, mean_absolute_error

import catboost

from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import AdaBoostRegressor
from sklearn.tree import DecisionTreeRegressor

import lightgbm

from sklearn.linear_model import LinearRegression

from sklearn.ensemble import VotingRegressor

class ModelsParameters:
    def __init__(self, dictionary_params):
        self.dictionary_params = dictionary_params

    """ I need this function for having all keys for the coming functions """
## Functions for creating a dictionary by simply inputting values of the params for
## each type of estimator

    # Create dictionary with all params of RandomForest
    def random_forest_params(
        n_estimators=[100],  # The number of trees in the forest.
        criterion=['mse'],  # {“mse”, “mae”}, default=”mse”. The function to measure the quality of a split.
        max_depth=[None],  # The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples
        min_samples_split=[2],  # The minimum number of samples required to split an internal node
        min_samples_leaf=[1],  # The minimum number of samples required to be at a leaf node.
        min_weight_fraction_leaf=[0.0],  # The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node
        max_features=['auto'],  # {“auto”, “sqrt”, “log2”}, int or float, default=”auto” The number of features to consider when looking for the best split. If auto, == n_features [if not so many]
        max_leaf_nodes=[None],  # pruning? [FIX]
        min_impurity_decrease=[0.0],
        min_impurity_split=[None],
        bootstrap=[True],
        oob_score=[False],  # whether to use out-of-bag samples to estimate the R^2 on unseen data [should be true? FIX but then it will be less comparable]
        n_jobs=[None],
        random_state=[None],
        verbose=[0],
        warm_start=[False],
        ccp_alpha=[0.0],  # Complexity parameter used for Minimal Cost-Complexity Pruning [FIX]. The subtree with the largest cost complexity that is smaller than ccp_alpha will be chosen.
        max_samples=[None],  # bootstrap is True, the number of samples to draw from X to train each base estimator. If None, == to X.shape[0]
    ):

        params_dict = {
            'n_estimators': [n_estimators][0],
            'criterion': [criterion][0],
            'max_depth': [max_depth][0],
            'min_samples_split': [min_samples_split][0],
            'min_samples_leaf': [min_samples_leaf][0],
            'min_weight_fraction_leaf': [min_weight_fraction_leaf][0],
            'max_features': [max_features][0],
            'max_leaf_nodes': [max_leaf_nodes][0],
            'min_impurity_decrease': [min_impurity_decrease][0],
            'min_impurity_split': [min_impurity_split][0],
            'bootstrap': [bootstrap][0],
            'oob_score': [oob_score][0],
            'n_jobs': [n_jobs][0],
            'random_state': [random_state][0],
            'verbose': [verbose][0],
            'warm_start': [warm_start][0],
            'ccp_alpha': [ccp_alpha][0],
            'max_samples': [max_samples][0],
        }

        return params_dict

    def rf_params_pipeline(self, existing_prefix='', prefix_to_add='rf__'):
        params_dict = {
        prefix_to_add+'n_estimators': self.dictionary_params[existing_prefix+'n_estimators'],
        prefix_to_add+'criterion': self.dictionary_params[existing_prefix+'criterion'],
        prefix_to_add+'max_depth': self.dictionary_params[existing_prefix+'max_depth'],
        prefix_to_add+'min_samples_split': self.dictionary_params[existing_prefix+'min_samples_split'],
        prefix_to_add+'min_samples_leaf': self.dictionary_params[existing_prefix+'min_samples_leaf'],
        prefix_to_add+'min_weight_fraction_leaf': self.dictionary_params[existing_prefix+'min_weight_fraction_leaf'],
        prefix_to_add+'max_features': self.dictionary_params[existing_prefix+'max_features'],
        prefix_to_add+'max_leaf_nodes': self.dictionary_params[existing_prefix+'max_leaf_nodes'],
        prefix_to_add+'min_impurity_decrease': self.dictionary_params[existing_prefix+'min_impurity_decrease'],
        prefix_to_add+'min_impurity_split': self.dictionary_params[existing_prefix+'min_impurity_split'],
        prefix_to_add+'bootstrap': self.dictionary_params[existing_prefix+'bootstrap'],
        prefix_to_add+'oob_score': self.dictionary_params[existing_prefix+'oob_score'],
        prefix_to_add+'n_jobs': self.dictionary_params[existing_prefix+'n_jobs'],
        prefix_to_add+'random_state': self.dictionary_params[existing_prefix+'random_state'],
        prefix_to_add+'verbose': self.dictionary_params[existing_prefix+'verbose'],
        prefix_to_add+'warm_start': self.dictionary_params[existing_prefix+'warm_start'],
        prefix_to_add+'ccp_alpha': self.dictionary_params[existing_prefix+'ccp_alpha'],
        prefix_to_add+'max_samples': self.dictionary_params[existing_prefix+'max_samples'],
        }
        return params_dict

    def adaboost_params(
        base_estimator=[None],
        n_estimators=[50],
        learning_rate=[1.0],
        loss=['linear'],
        random_state=[None]
    ):

        params_dict = {
            'base_estimator': [base_estimator][0],
            'n_estimators': [n_estimators][0],
            'learning_rate': [learning_rate][0],
            'loss': [loss][0],
            'random_state': [random_state][0]
        }

        return params_dict

    def ab_params_pipeline(self, existing_prefix='', prefix_to_add='ab__'):
        params_dict = {
        prefix_to_add+'base_estimator': self.dictionary_params[existing_prefix+'base_estimator'],
        prefix_to_add+'n_estimators': self.dictionary_params[existing_prefix+'n_estimators'],
        prefix_to_add+'learning_rate': self.dictionary_params[existing_prefix+'learning_rate'],
        prefix_to_add+'loss': self.dictionary_params[existing_prefix+'loss'],
        prefix_to_add+'random_state': self.dictionary_params[existing_prefix+'random_state'],
        }
        return params_dict

    def gradientboost_params(
        loss=['ls'],
        learning_rate=[0.1],
        n_estimators=[100],
        subsample=[1.0],
        criterion=['friedman_mse'],
        min_samples_split=[2],
        min_samples_leaf=[1],
        min_weight_fraction_leaf=[0.0],
        max_depth=[3],
        min_impurity_decrease=[0.0],
        # min_impurity_split=[None], # deprecated FIX
        init=[None],
        random_state=[None],
        max_features=[None],
        alpha=[0.9],
        verbose=[0],
        max_leaf_nodes=[None],
        warm_start=[False],
        presort=['deprecated'],
        validation_fraction=[0.1],
        n_iter_no_change=[None],
        tol=[0.0001],
        ccp_alpha=[0.0],
    ):

        params_dict = {
            'loss': [loss][0],
            'learning_rate': [learning_rate][0],
            'n_estimators': [n_estimators][0],
            'subsample': [subsample][0],
            'criterion': [criterion][0],
            'min_samples_split': [min_samples_split][0],
            'min_samples_leaf': [min_samples_leaf][0],
            'min_weight_fraction_leaf': [min_weight_fraction_leaf][0],
            'max_depth': [max_depth][0],
            'min_impurity_decrease': [min_impurity_decrease][0],
            # 'min_impurity_split': [min_impurity_split][0],
            'init': [init][0],
            'random_state': [random_state][0],
            'max_features': [max_features][0],
            'alpha': [alpha][0],
            'verbose': [verbose][0],
            'max_leaf_nodes': [max_leaf_nodes][0],
            'warm_start': [warm_start][0],
            'presort': [presort][0],
            'validation_fraction': [validation_fraction][0],
            'n_iter_no_change': [n_iter_no_change][0],
            'tol': [tol][0],
            'ccp_alpha': [ccp_alpha][0],
        }

        return params_dict

    def gb_params_pipeline(self, existing_prefix='', prefix_to_add='gb__'):
        params_dict = {
        prefix_to_add+'loss': self.dictionary_params[existing_prefix+'loss'],
        prefix_to_add+'learning_rate': self.dictionary_params[existing_prefix+'learning_rate'],
        prefix_to_add+'n_estimators': self.dictionary_params[existing_prefix+'n_estimators'],
        prefix_to_add+'subsample': self.dictionary_params[existing_prefix+'subsample'],
        prefix_to_add+'criterion': self.dictionary_params[existing_prefix+'criterion'],
        prefix_to_add+'min_samples_split': self.dictionary_params[existing_prefix+'min_samples_split'],
        prefix_to_add+'min_samples_leaf': self.dictionary_params[existing_prefix+'min_samples_leaf'],
        prefix_to_add+'min_weight_fraction_leaf': self.dictionary_params[existing_prefix+'min_weight_fraction_leaf'],
        prefix_to_add+'max_depth': self.dictionary_params[existing_prefix+'max_depth'],
        prefix_to_add+'min_impurity_decrease': self.dictionary_params[existing_prefix+'min_impurity_decrease'],
        # prefix_to_add+'min_impurity_split': self.dictionary_params[existing_prefix+'min_impurity_split'],
        prefix_to_add+'init': self.dictionary_params[existing_prefix+'init'],
        prefix_to_add+'random_state': self.dictionary_params[existing_prefix+'random_state'],
        prefix_to_add+'max_features': self.dictionary_params[existing_prefix+'max_features'],
        prefix_to_add+'alpha': self.dictionary_params[existing_prefix+'alpha'],
        prefix_to_add+'verbose': self.dictionary_params[existing_prefix+'verbose'],
        prefix_to_add+'max_leaf_nodes': self.dictionary_params[existing_prefix+'max_leaf_nodes'],
        prefix_to_add+'warm_start': self.dictionary_params[existing_prefix+'warm_start'],
        prefix_to_add+'presort': self.dictionary_params[existing_prefix+'presort'],
        prefix_to_add+'validation_fraction': self.dictionary_params[existing_prefix+'validation_fraction'],
        prefix_to_add+'n_iter_no_change': self.dictionary_params[existing_prefix+'n_iter_no_change'],
        prefix_to_add+'tol': self.dictionary_params[existing_prefix+'tol'],
        prefix_to_add+'ccp_alpha': self.dictionary_params[existing_prefix+'ccp_alpha'],
        }
        return params_dict

    # XGBoost
    def xgb_params(
        objective=['reg:squarederror'],
        n_estimators=[100],
        max_depth=[10],
        learning_rate=[0.3],
        verbosity=[0],
        booster=[None],  # 'gbtree'
        tree_method=['auto'],
        n_jobs=[1],
        gamma=[0],
        min_child_weight=[None],
        max_delta_step=[None],
        subsample=[None],
        colsample_bytree=[None],
        colsample_bylevel=[None],
        colsample_bynode=[None],
        reg_alpha=[0],
        reg_lambda=[0],
        scale_pos_weight=[None],
        base_score=[None],
        random_state=[random.randint(0, 500)],
        missing=[np.nan],
        num_parallel_tree=[None],
        monotone_constraints=[None],
        interaction_constraints=[None],
        importance_type=['gain']
    ):
        params_dict = {
            'objective': [objective][0],
            'n_estimators': [n_estimators][0],
            'max_depth': [max_depth][0],
            'learning_rate': [learning_rate][0],
            'verbosity': [verbosity][0],
            'booster': [booster][0],
            'tree_method': [tree_method][0],
            'n_jobs': [n_jobs][0],
            'gamma': [gamma][0],
            'min_child_weight': [min_child_weight][0],
            'max_delta_step': [max_delta_step][0],
            'subsample': [subsample][0],
            'colsample_bytree': [colsample_bytree][0],
            'colsample_bylevel': [colsample_bylevel][0],
            'colsample_bynode': [colsample_bynode][0],
            'reg_alpha': [reg_alpha][0],
            'reg_lambda': [reg_lambda][0],
            'scale_pos_weight': [scale_pos_weight][0],
            'base_score': [base_score][0],
            'random_state': [random_state][0],
            'missing': [missing][0],
            'num_parallel_tree': [num_parallel_tree][0],
            'monotone_constraints': [monotone_constraints][0],
            'interaction_constraints': [interaction_constraints][0],
            'importance_type': [importance_type][0]
        }

        return params_dict

    def xgb_params_pipeline(self, existing_prefix='', prefix_to_add='xgb__'):
        params_dict = {
        prefix_to_add+'objective': self.dictionary_params[existing_prefix+'objective'],
        prefix_to_add+'n_estimators': self.dictionary_params[existing_prefix+'n_estimators'],
        prefix_to_add+'max_depth': self.dictionary_params[existing_prefix+'max_depth'],
        prefix_to_add+'learning_rate': self.dictionary_params[existing_prefix+'learning_rate'],
        prefix_to_add+'verbosity': self.dictionary_params[existing_prefix+'verbosity'],
        prefix_to_add+'booster': self.dictionary_params[existing_prefix+'booster'],
        prefix_to_add+'tree_method': self.dictionary_params[existing_prefix+'tree_method'],
        prefix_to_add+'n_jobs': self.dictionary_params[existing_prefix+'n_jobs'],
        prefix_to_add+'gamma': self.dictionary_params[existing_prefix+'gamma'],
        prefix_to_add+'min_child_weight': self.dictionary_params[existing_prefix+'min_child_weight'],
        prefix_to_add+'max_delta_step': self.dictionary_params[existing_prefix+'max_delta_step'],
        prefix_to_add+'subsample': self.dictionary_params[existing_prefix+'subsample'],
        prefix_to_add+'colsample_bytree': self.dictionary_params[existing_prefix+'colsample_bytree'],
        prefix_to_add+'colsample_bylevel': self.dictionary_params[existing_prefix+'colsample_bylevel'],
        prefix_to_add+'colsample_bynode': self.dictionary_params[existing_prefix+'colsample_bynode'],
        prefix_to_add+'reg_alpha': self.dictionary_params[existing_prefix+'reg_alpha'],
        prefix_to_add+'reg_lambda': self.dictionary_params[existing_prefix+'reg_lambda'],
        prefix_to_add+'scale_pos_weight': self.dictionary_params[existing_prefix+'scale_pos_weight'],
        prefix_to_add+'base_score': self.dictionary_params[existing_prefix+'base_score'],
        prefix_to_add+'random_state': self.dictionary_params[existing_prefix+'random_state'],
        prefix_to_add+'missing': self.dictionary_params[existing_prefix+'missing'],
        prefix_to_add+'num_parallel_tree': self.dictionary_params[existing_prefix+'num_parallel_tree'],
        prefix_to_add+'monotone_constraints': self.dictionary_params[existing_prefix+'monotone_constraints'],
        prefix_to_add+'interaction_constraints': self.dictionary_params[existing_prefix+'interaction_constraints'],
        prefix_to_add+'importance_type': self.dictionary_params[existing_prefix+'importance_type'],
        }
        return params_dict

    # Greedy search?
    def create_spaces(self, prefix_pipeline, estimator_name):
        df = pd.DataFrame(data=[self.dictionary_params])
        params_range = {}

        for col in df.columns:
            number = 0
            string = 0      # not needed so far
            nones = 0
            trees = 0
            string_key = str(col)

            for i in df[col][0]:
                type_i = type(i)
                if (type_i == int) | (type_i == float):
                    number += 1
                elif type_i == str:  # not needed
                    string += 1
                elif i == None:  # not needed?
                    nones += 1
                elif (type_i == DecisionTreeRegressor):
                    trees += 1

            # Ranges for simple numeric values - FIX check upon them
            if (number == len(df)) & (col != prefix_pipeline+'verbose') & \
                    (col != (prefix_pipeline+'random_state')) & (col != (prefix_pipeline+'verbosity')) \
                    & (col != (prefix_pipeline+'n_jobs')) & (trees == 0) & + (col != (prefix_pipeline+'n_iter_no_change')) & \
                    (col != (prefix_pipeline+'missing')) & (col != (prefix_pipeline+'validation_fraction')):
                output = df[col][0][0]

                if estimator_name == 'RandomForest':
                    range_output, lower_output, upper_output = ModelsParameters.rf_ranges(self, col, prefix_pipeline, output)

                elif estimator_name == 'AdaBoost':
                    range_output, lower_output, upper_output = ModelsParameters.ab_ranges(self, col, prefix_pipeline, output, trees)

                elif estimator_name == 'GradientBoosting':
                    range_output, lower_output, upper_output = ModelsParameters.gb_ranges(self, col, prefix_pipeline, output)

                elif estimator_name == 'XGBoost':
                    range_output, lower_output, upper_output = ModelsParameters.xgb_ranges(self, col, prefix_pipeline, output)

                # Further Conditions on the allowed output range and append
                data_to_append = ModelsParameters.create_outputs(self, output, range_output, string_key, lower_output, upper_output)
                params_range.update(data_to_append)

            # Special Range for AdaBoost trees' max_depth
            elif (trees > 0):
                data_to_append = ModelsParameters.range_ab_decision_tree(df, self.dictionary_params, col, prefix_pipeline)
                params_range.update(data_to_append)

            # Else cases - just repeat the same value
            else:
                data_to_append = {string_key: [i]}
                params_range.update(data_to_append)

        return params_range

    def rf_ranges(self, col, prefix_pipeline, output):
        if col == prefix_pipeline+'n_estimators':
            range_output = 5
        elif col == prefix_pipeline+'max_depth':
            range_output = 3
        elif col == prefix_pipeline+'min_samples_split':
            range_output = 2
        elif col == prefix_pipeline+'min_samples_leaf':
            range_output = 1
        elif col == prefix_pipeline+'min_weight_fraction_leaf':
            range_output = 0.05
        elif col == prefix_pipeline+'max_features':
            range_output = 0
        elif col == prefix_pipeline+'max_leaf_nodes':
            range_output = 0
        elif col == prefix_pipeline+'min_impurity_decrease':
            range_output = 0.2
        elif col == prefix_pipeline+'ccp_alpha':
            range_output = 0.2
        elif col == prefix_pipeline+'max_samples':
            range_output = 0
        lower_output = output - range_output
        upper_output = output + range_output
        return range_output, lower_output, upper_output

    def ab_ranges(self, col, prefix_pipeline, output, trees):

        # FIX later: for not needed, thinking of merging with the estimator for tree
        if trees == 0:
            if col == prefix_pipeline+'n_estimators':
                range_output = 5
            elif col == prefix_pipeline+'learning_rate':
                range_output = 0.01 # FIX: is learning rate max == 1?

        else:
            pass

        lower_output = output - range_output
        upper_output = output + range_output

        return range_output, lower_output, upper_output

    def range_ab_decision_tree(df, start_params, col, prefix_pipeline): # # For AdaBoost range of base_estimator max_depth
        tree = df[col][0][0]  # not needed
        for i in start_params[col]:
            x = re.split("\=", str(i))
            y = re.split("\)", str(x[1]))[0]
            max_depth = int(str(y))
        output = sklearn.tree.DecisionTreeRegressor(max_depth=max_depth)

        if col == prefix_pipeline+'base_estimator':
            range_output = 3

        lower_output = max_depth - range_output
        upper_output = max_depth + range_output

        if (range_output != 0) & (lower_output > 0):
            data_to_append = {str(col): [
                sklearn.tree.DecisionTreeRegressor(max_depth=lower_output),
                output,
                sklearn.tree.DecisionTreeRegressor(max_depth=upper_output)
            ]}

        elif (range_output != 0) & (lower_output <= 0):
            data_to_append = {str(col): [
                output,
                sklearn.tree.DecisionTreeRegressor(max_depth=upper_output)
            ]}

        elif (range_output == 0):
            data_to_append = {str(col): [
                output
            ]}

        return data_to_append

    def gb_ranges(self, col, prefix_pipeline, output):

        if col == prefix_pipeline+'learning_rate':
            range_output = 0 # FIX: is learning rate max == 1?
        elif col == prefix_pipeline+'n_estimators':
            range_output = 5
        elif col == prefix_pipeline+'subsample':
            range_output = 0
        elif col == prefix_pipeline+'min_samples_split':
            range_output = 2
        elif col == prefix_pipeline+'min_samples_leaf':
            range_output = 1
        elif col == prefix_pipeline+'min_weight_fraction_leaf':
            range_output = 0.05
        elif col == prefix_pipeline+'max_depth':
            range_output = 3
        elif col == prefix_pipeline+'min_impurity_decrease':
            range_output = 0.2
        elif col == prefix_pipeline+'max_features':
            range_output = 0
        elif col == prefix_pipeline+'alpha':
            range_output = 0
        elif col == prefix_pipeline+'max_leaf_nodes':
            range_output = 0
        elif col == prefix_pipeline+'tol':
            range_output = 0
        elif col == prefix_pipeline+'ccp_alpha':
            range_output = 0

        lower_output = output - range_output
        upper_output = output + range_output

        return range_output, lower_output, upper_output

    def xgb_ranges(self, col, prefix_pipeline, output):

        if col == prefix_pipeline+'n_estimators':
            range_output = 5
        elif col == prefix_pipeline+'max_depth':
            range_output = 3
        elif col == prefix_pipeline+'learning_rate':
            range_output = 0 # FIX: is learning rate max == 1?
        elif col == prefix_pipeline+'gamma':
            range_output = 0
        elif col == prefix_pipeline+'min_child_weight':
            range_output = 0
        elif col == prefix_pipeline+'max_delta_stop':
            range_output = 0
        elif col == prefix_pipeline+'subsample':
            range_output = 0
        elif col == prefix_pipeline+'colsample_bytree':
            range_output = 0
        elif col == prefix_pipeline+'colsample_bylevel':
            range_output = 0
        elif col == prefix_pipeline+'colsample_bynode':
            range_output = 0
        elif col == prefix_pipeline+'reg_alpha':
            range_output = 0
        elif col == prefix_pipeline+'reg_lambda':
            range_output = 0
        elif col == prefix_pipeline+'scale_pos_weight':
            range_output = 0
        elif col == prefix_pipeline+'base_score':
            range_output = 0
        elif col == prefix_pipeline+'monotone_constraints':
            range_output = 0
        elif col == prefix_pipeline+'interaction_constraints':
            range_output = 0

        lower_output = output - range_output
        upper_output = output + range_output

        return range_output, lower_output, upper_output

##
    def create_outputs(self, output, range_output, string_key, lower_output, upper_output):
        if range_output == 0:
            data_to_append = {string_key: [
                output
            ]}

        elif (range_output != 0) & (lower_output > 0):
            data_to_append = {string_key: [
                lower_output,
                output,
                upper_output
            ]}

        # FIX could be controversial in certain instances in case you want lower bound to be 0
        elif (range_output != 0) & (lower_output == 0):
            data_to_append = {string_key: [
                output,
                upper_output
            ]}

        elif (lower_output < 0) & (output != 0):
            data_to_append = {string_key: [
                0,
                output,
                upper_output]}

        elif (lower_output < 0) & (output == 0):
            data_to_append = {string_key: [
                output,
                upper_output]}

        return data_to_append

    def best_model_pipeline(X, Y, pipeline, params_range, cv, scoring='neg_mean_squared_error'):

        optimal_model = GridSearchCV(pipeline,
                                    params_range,
                                    scoring=scoring,
                                    cv=cv,
                                    refit=True)  # when there is a list in scoring, it needs an explicit one. NP because here comes "s", not "scoring"
        print('Below are the params_range')
        print(params_range)

        result = optimal_model.fit(X, Y)
        best_params = result.best_estimator_  # result.best_params_ needed when refit=False

        dict_parameters_pipeline = {}
        for param in params_range:  # list of parameters
            dict_parameters_pipeline[str(param)] = [best_params.get_params()[str(param)]]

        print('Below are the best params')
        print(dict_parameters_pipeline)

        return result, dict_parameters_pipeline

##

    def NestedCV(X, Y, params, pipeline, prefix_pipeline=None, estimator=None, estimator_name=None,
                 NUM_TRIALS=1,  # for repeated. Note that the sample is anew every time CHECK
                 inner_n_splits=5,
                 outer_n_splits=5,
                 adaptive_grid='yes',
                 scoring=['neg_mean_squared_error', 'neg_mean_absolute_error', 'neg_root_mean_squared_error'],
                 ):

        best_params = pd.DataFrame()
        df_feature_importance = pd.DataFrame()
        mse = list()
        mae = list()
        rmse = list()
        score_list = list()
        score_metric = list()

    # PROPOSAL: evaluate with metrics listed. BLOCK for metrics]
    ## WRONG FIX. it is just differnet metric at the end. Shouldn0t be in loop like this
        for s in scoring:

    # PROPOSAL: nested CV
            for i in tqdm(range(NUM_TRIALS)):
                # configure the cross-validation procedure
                cv_outer = KFold(n_splits=outer_n_splits, shuffle=True, random_state=i)

                # [BLOCK for nested CV?
                for train_ix, test_ix in cv_outer.split(X):
                    # split data
                    X_train, X_test = X.iloc[train_ix, :], X.iloc[test_ix, :]
                    Y_train, Y_test = Y.iloc[train_ix], Y.iloc[test_ix]

                    # configure the cross-validation procedure
                    cv_inner = KFold(n_splits=inner_n_splits,
                                     shuffle=True,
                                     random_state=i)

                    if (pipeline==None) & (adaptive_grid=='yes'):
                        params = ModelsParameters(params).create_spaces(prefix_pipeline='', estimator=estimator) # create range RF params in format for pipeline
                        result, best_params_grid = ModelsParameters.best_model_pipeline(X_train, Y_train,
                                                                              pipeline=estimator,
                                                                              params_range=params,
                                                                              scoring=s, cv=cv_inner)
                        #print(best_params_grid)
                        print('done without pipeline')
                        # FIX test this version
                        feature_importances = pd.DataFrame(result.best_estimator_.feature_importances_.reshape(1, -1),
                                                           columns=X_train.columns)
                        df_feature_importance = df_feature_importance.append(
                            feature_importances, ignore_index=True)

                    # FIX features importance of the best model. Excluded with pipeline because there are no
                    elif (pipeline!=None) & (adaptive_grid=='yes'):
                        params = ModelsParameters(params).create_spaces(prefix_pipeline=prefix_pipeline, estimator_name=estimator_name) # create range RF params in format for pipeline
                        result, best_params_grid = ModelsParameters.best_model_pipeline(X_train, Y_train,
                                                                              pipeline=pipeline,
                                                                              params_range=params,
                                                                              scoring=s, cv=cv_inner)

                        for value in best_params_grid:
                            data_to_update = {str(value): best_params_grid[value]}
                            params.update(data_to_update)
                        #print(best_params_grid)
                        print('done one with pipeline')

                    elif (pipeline!=None) & (adaptive_grid=='no'):
                        result, best_params_grid = ModelsParameters.best_model_pipeline(X_train, Y_train,
                                                                              pipeline=pipeline,
                                                                              params_range=params,
                                                                              scoring=s, cv=cv_inner)
                        #print(best_params_grid)
                        print('done one with pipeline with no adaptive grid')

                    elif (pipeline==None) & (adaptive_grid=='no'):
                        result, best_params_grid = ModelsParameters.best_model_pipeline(X_train, Y_train,
                                                                              pipeline=estimator,
                                                                              params_range=params,
                                                                              scoring=s, cv=cv_inner)
                        #print(best_params_grid)
                        print('done without pipeline and without adaptive grid')
                        feature_importances = pd.DataFrame(result.best_estimator_.feature_importances_.reshape(1, -1),
                                                           columns=X_train.columns)
                        df_feature_importance = df_feature_importance.append(
                            feature_importances, ignore_index=True)

    # PROPOSAL: END function for metrics
                    # evaluate model on the hold out dataset (i.e. the test set)

                    # FIX the score_ variables names
                    if s == 'neg_mean_squared_error':
                        #best_model.fit(X_train, Y_train)
                        #score = best_model.score(X_test, Y_test) # for using pipeline?
                        score = mean_squared_error(Y_test, result.predict(X_test))  # result.predict(X_test): needed when refit=False
                        score_list.append(score)                                       # which quickest? FIX
                        score_metric.append('MSE')
                        print('MSE=%.3f, est=%.3f' % (score, -result.best_score_))

                    elif s == 'neg_mean_absolute_error':
                        score = mean_absolute_error(Y_test, result.predict(X_test))
                        score_list.append(score)
                        score_metric.append('MAE')
                        print('MAE=%.3f, est=%.3f' % (score, -result.best_score_))

                    elif s == 'neg_root_mean_squared_error':
                        score = mean_squared_error(Y_test, result.predict(X_test), squared=False)
                        score_list.append(score)
                        score_metric.append('RMSE')
                        print('RMSE=%.3f, est=%.3f' % (score, -result.best_score_))

                    # best parameters
                    best_params = best_params.append(best_params_grid,
                                                     ignore_index=True)

        # [BLOCK for proper output? or just not?
        best_params['Score Value'] = score_list
        best_params['Metric'] = score_metric

        return best_params, df_feature_importance

    """ FIND overall metrics/hyperparameters as sum/count/.. """

    def find_best_params(best_params):
        df = pd.DataFrame()

        # Pretransform the format of the params dataframe
        for col in best_params.loc[:, (best_params.columns != 'Score Value') & (best_params.columns != 'Metric')]: # & (best_params.columns != 'RMSE')]:
            new_list = []
            for i in range(len(best_params)):
                new_item = best_params[str(col)][i][0]
                new_list.append(new_item)
            best_params[str(col)] = new_list

        for metric in best_params['Metric'].unique():
            data = {}
            selected_metric_df = best_params.loc[best_params['Metric']==metric]
            for col in selected_metric_df.columns:
                number = 0
                string = 0
                nones = 0
                trees = 0
                string_col = str(col)

                for i in selected_metric_df[col]:
                    type_i = type(i)
                    if (type_i == int) | (type_i == float):
                        number += 1
                    elif type_i == str:
                        string += 1
                    elif i == None:
                        nones += 1
                    elif (type_i == DecisionTreeRegressor):  # for Adaboost
                        trees += 1

                # If the params are numbers, it reports the mean
                if (number == len(selected_metric_df[col])) & (nones == 0):
                    integer = 0
                    for i in selected_metric_df[col]:
                        type_i = type(i)
                        if (type_i == int):
                            output = int(selected_metric_df[col].mean())
                            data_to_append = {string_col: [output]}
                            data.update(data_to_append)

                        elif (type_i == float):
                            output = float(selected_metric_df[col].mean())
                            data_to_append = {string_col: [output]}
                            data.update(data_to_append)

                # In case of string params, the most frequent one
                elif (string == len(selected_metric_df[col])) & (nones == 0):
                    output = selected_metric_df[col].value_counts().reset_index()['index'][0]
                    data_to_append = {string_col: [output]}
                    data.update(data_to_append)

                # In case of all Nones values, the value will be None
                elif nones == len(selected_metric_df[col]):
                    data_to_append = {string_col: [None]}
                    data.update(data_to_append)

                # In case some are Nones and other not, see below FIX
                elif (nones > 0) & (nones != len(selected_metric_df[col])):
                    if (type(i) == int for i in selected_metric_df[col]):
                        if ((selected_metric_df[col].value_counts().sum() >= len(selected_metric_df)/2) == True):
                            output = int(selected_metric_df[col].value_counts().reset_index()['index'][0])
                            data_to_append = {string_col: [output]}
                            data.update(data_to_append)

                        else:
                            data_to_append = {string_col: [None]}
                            data.update(data_to_append)

                    elif (type(i) == float for i in selected_metric_df[col]):
                        if ((selected_metric_df[col].value_counts().sum() >= len(selected_metric_df)/2) == True):
                            output = float(selected_metric_df[col].value_counts().reset_index()['index'][0])
                            data_to_append = {string_col: [output]}
                            data.update(data_to_append)

                        else:
                            data_to_append = {string_col: [None]}
                            data.update(data_to_append)

            df_partial = pd.DataFrame(data)
            df_partial['Metric'] = metric
            df = df.append(df_partial, ignore_index=True)

        return df


text:
import pytest

import numpy as np

from fanok.selection import adaptive_significance_threshold

@pytest.mark.parametrize(
    "w, q, offset, expected",
    [
        ([1, 2, 3, 4, 5], 0.1, 0, 1),
        ([-1, 2, -3, 4, 5], 0.1, 0, 4),
        ([-3, -2, -1, 0, 1, 2, 3], 0.1, 0, np.inf),
        ([-3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.1, 0, 4),
        ([-3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.15, 0, 3),
        (
            [-1.52, 1.93, -0.76, -0.35, 1.21, -0.39, 0.08, -1.45, 0.31, -1.38],
            0.1,
            0,
            1.93,
        ),
    ],
)
def test_adaptive_significance_threshold(w, q, offset, expected):
    w = np.array(w)
    threshold = adaptive_significance_threshold(w, q, offset=offset)
    assert threshold == expected

max_stars_repo_name: qrebjock/fanok

max_issues_repo_name: qrebjock/fanok

max_forks_repo_name: qrebjock/fanok


text:
import sys
sys.path.insert(0, '../..')

from TheSoundOfAIOSR.stt.wavenet.inference import WaveNet
import argparse
import soundfile as sf
from scipy.signal import resample
import torch
from torchaudio.transforms import Resample

parser = argparse.ArgumentParser(description="ASR with recorded audio (offline)")
parser.add_argument("--recording", "-rec", required=True)
args = parser.parse_args()

wavenet = WaveNet(device='cpu')

print("Loading Models ...")
wavenet.load_model()
print("Models Loaded ...")

USE_TORCHAUDIO_RESAMPLING = True

def main():
    wav, sr = sf.read(args.recording)
    target_sr = 16000
    if USE_TORCHAUDIO_RESAMPLING:
        resampling_transform = Resample(orig_freq=sr,
                                    new_freq=target_sr)

        inputs = resampling_transform(torch.Tensor([wav])).squeeze()
    else:
        inputs = resample(wav, num=int(len(wav)*target_sr/sr))

    print(wavenet.transcribe(inputs))

if __name__=="__main__":
    print("Start Transcribing...")
    main()

reason: from scipy


text:
import numpy as np

def convert_bbox_coordinate(bboxes, old_resolution, new_resolution):
    """Convert bboxes' coordinates from old resolution to new resolution

    Args:
        bboxes (list): list of tlbr sequence of type list
        old_resolution (tuple): old screen size (width, height)
        new_resolution (tuple): new screen size (width, height)

    Returns:
        a list of tlbr sequence in new coordinate system
    """
    if len(bboxes) == 0:
        return []

    ratios = np.array(new_resolution) / np.array(old_resolution)
    x_ratio, y_ratio = ratios[0], ratios[1]

    bboxes = np.array(bboxes)
    bboxes[:, 0] = (bboxes[:, 0]*x_ratio).astype(np.int)
    bboxes[:, 2] = (bboxes[:, 2]*x_ratio).astype(np.int)
    bboxes[:, 1] = (bboxes[:, 1]*y_ratio).astype(np.int)
    bboxes[:, 3] = (bboxes[:, 3]*y_ratio).astype(np.int)
    bboxes = bboxes.tolist()
    return bboxes

max_stars_repo_name: johnnylord/mtmc-testbed

max_issues_repo_name: johnnylord/mtmc-testbed

max_forks_repo_name: johnnylord/mtmc-testbed


text:
import numpy as np
import matplotlib.pyplot as plt
import os
'''
Auxiliary functions module.
'''

def Hooke(strain, E = 150000, b = 0):
        '''
        Defines the linear equation for the linear portion of the stress-strain curve.

        Inputs:

        strain - the vector containing the strain values
        E - default = 150,000; initial guess for the modulus of elasticity in MPa.
        b - default = 0; intercept of the liear equation.

        Output:

        Array with the calculated values of E * strain + b.
        '''

        return E * strain + b

def r_squared(x, y, model, modelParams):

        '''
        Calculates the R² statistics to evaluate the quality of the data regression.

        Inputs:

        x - the vector containing the x values of the independent variable of the function.
        y - the vector containing the dependent (measured) values as a function of x.
        model - the function that receives x as variable and calculates a y vector to model measured y.
        modelParams - the parameters of the model function passed as a list.

        Output:

        r_squared - the R² statistics.
        '''

        res = y - model(x, *modelParams)
        ss_res = np.sum(res ** 2)
        ss_tot = np.sum((y - np.mean(y)) ** 2)
        r_squared = 1 - (ss_res/ss_tot)

        return r_squared

def find_index(array, value):

        '''
        Finds the index of a value in an array.
        '''

        i = 0
        while array[i] < value:
                i = i + 1
        return i

def Hollomon(x, K = 300, n = 0.25):

        '''
        Defines the function to perform flow model fitting.
        '''

        return K * x ** n

def Ludwik(x, sig_o = 600, K = 600, n = 0.24):

        '''
        Defines the function to perform flow model fitting.
        '''

        return sig_o + K * x ** n

def Datsko(x, K = 300, x0 = 300, n = 0.20):

        '''
        Defines the function to perform flow model fitting.
        '''

        return K * (x0 + x) ** n

def create_output_folder(path = 'output'):
        '''
        Creates the output folder where the results will be saved.
        '''

        if not os.path.exists(path):
            os.mkdir(path)
        else:
            print('The path already exists.')


text:
import os.path as op
import time

import numpy as np
import tifffile as tiff

from ..session.status import DummyStatus
from ..img_op.coarse_binarize_phc import binarize_frame

def binarize_phasecontrast_stack(stack, i_channel, outfile=None, status=None, return_result=False):
    if status is None:
        status = DummyStatus()

    stack_bin = np.empty((stack.n_frames, stack.height, stack.width), dtype=np.uint8)
    with status("Binarizing …") as current_status:
        for i_frame in range(stack.n_frames):
            current_status.reset(msg="Binarizing frame", current=i_frame+1, total=stack.n_frames)
            stack_bin[i_frame, ...] = binarize_frame(stack.get_image(frame=i_frame, channel=i_channel))

        if outfile:
            current_status.reset(f"Saving binarized stack to '{outfile}' …")

            ext = op.splitext(outfile)[-1].casefold()
            if ext in ('.tif', '.tiff'):
                tiff.imwrite(outfile, stack_bin[:, None, None, ...], imagej=True)
            elif ext == '.npy':
                np.save(outfile, stack_bin)
            elif ext == '.npz':
                np.savez_compressed(outfile, stack_bin)
            else:
                raise ValueError(f"Unknown file extension '{ext}'")

            current_status.reset(f"Saved binarized stack to '{outfile}'.")

        if return_result:
            return stack_bin

        time.sleep(2)

max_stars_repo_name: BeastyBlacksmith/pyama

max_issues_repo_name: BeastyBlacksmith/pyama

max_forks_repo_name: BeastyBlacksmith/pyama


text:
from typing import List, Tuple

import numpy as np
from ortools.constraint_solver import pywrapcp
from ortools.constraint_solver import routing_enums_pb2

import utility.mad_api
import utility.utility

def distance_matrix(coords):
    # see https://stackoverflow.com/a/29546836
    lat, lng = map(np.radians, zip(*coords))
    # calculate differences, essentially x.T - x
    dlat = lat[..., np.newaxis] - lat
    dlng = lng[..., np.newaxis] - lng
    # haversine
    cos_lat = np.cos(lat)
    a = np.sin(dlat / 2.0) ** 2 + np.multiply(np.outer(cos_lat, cos_lat), np.sin(dlng / 2.0) ** 2)
    c = 2 * np.arcsin(np.sqrt(a))
    return 6367 * c

def recalc_routecalc(routecalc: List[Tuple[float, float]], arbitrary_endpoints: bool = False) -> List[Tuple[float, float]]:
    if not routecalc:
        return []

    num_coords = len(routecalc)

    if arbitrary_endpoints:
        # we need an additional fake point with distance==0 to all others
        num_coords += 1

    # Create the routing index manager.
    manager = pywrapcp.RoutingIndexManager(num_coords, 1, num_coords - 1 if arbitrary_endpoints else 0)
    # Create Routing Model.
    routing = pywrapcp.RoutingModel(manager)

    # calculating distances, since distance matrix is symmetric, only calculate half
    distances = distance_matrix(routecalc)

    def distance_callback(from_index, to_index):
        from_node, to_node = manager.IndexToNode(from_index), manager.IndexToNode(to_index)
        if from_node == to_node:
            # distance of a point to itself is 0
            return 0
        m = max(from_node, to_node)
        if (arbitrary_endpoints and m == num_coords - 1) or m >= len(distances):
            # distance to fake point is always 0, fake point is always last in list
            return 0
        return int(1000000 * distances[from_node][to_node])

    transit_callback_index = routing.RegisterTransitCallback(distance_callback)

    # Define cost of each arc.
    routing.SetArcCostEvaluatorOfAllVehicles(transit_callback_index)

    # Setting first solution heuristic.
    search_parameters = pywrapcp.DefaultRoutingSearchParameters()
    search_parameters.first_solution_strategy = routing_enums_pb2.FirstSolutionStrategy.PATH_CHEAPEST_ARC

    # Solve the problem.
    solution = routing.SolveWithParameters(search_parameters)

    # return original routecalc if ortools failed to find a solution
    if not solution:
        return routecalc

    new_routecalc = []
    index = routing.Start(0)
    # skip fake point
    if arbitrary_endpoints:
        index = solution.Value(routing.NextVar(index))
    while not routing.IsEnd(index):
        new_routecalc.append(routecalc[manager.IndexToNode(index)])
        index = solution.Value(routing.NextVar(index))
    return new_routecalc

if __name__ == '__main__':
    api = utility.mad_api.Api()
    area_ids = utility.utility.query_area_ids(api, "Please input area ids of areas you want to recalculate separated by comma, or 'all' to recalc all areas:\n")
    do_mad_recalc = input("Do you want to trigger a MAD recalc before using ortools to calculate the routes? (Y/n) ").strip().lower() != 'n'
    for i, area_id in enumerate(area_ids):
        area = api.areas[area_id]
        print(f"Recalculating route for area {area.name} ({i + 1}/{len(area_ids)})")
        if do_mad_recalc:
            print(f"Using MAD to recalculate route")
            area.recalculate()
        # get old routecalc
        old_route = area.routecalc
        # check if area has a route (otherwise we can't and don't need to recalculate)
        if not old_route:
            print("Area has no route.")
            continue
        # calculate new route
        if not do_mad_recalc:
            print("Using ortools to recalculate route")
        # calculate and set new route
        area.routecalc = recalc_routecalc(old_route, area.mode == 'pokestops')
    # apply settings if desired
    if input("Do you want to apply the changes and reload? (y/N)  ").strip().lower() == 'y':
        api.apply_settings()

max_stars_repo_name: Tabbomat/MADUtilities

max_issues_repo_name: Tabbomat/MADUtilities

max_forks_repo_name: Tabbomat/MADUtilities


text:
"""
Author: Lucy McLaughlin
Date: 19/02/2021

Acknowledgments:
The Alan Turing Institute for funding the Newcastle Seedcorn project
"Automating visualization", under the EPSRC grant EP/N510129/1 and for
Nick Holliman's Turing Fellowship.

Citation:
"Visual Entropy and the Visualization of Uncertainty", Holliman et al,
arXiv:1907.12879

"""

import numpy as np
from .metofficelimits import *
import matplotlib
import matplotlib.cm as cm

def scale_is_negative(values):
    return any(x<0 for x in values) and not any(x>0 for x in values)

def scale_is_divergent(values):
    return any(x<0 for x in values) and any(x>0 for x in values)

def get_colour_scale(values, max_val, min_val, n_colours,
                     scale_spread, scale_dp):
    if scale_spread is not None and scale_spread < 0:
        scale_spread=abs(scale_spread)
    # Determine min and max scale values
    if max_val==None and min_val==None:
        if scale_spread == None:
            min_val = min(values)
            max_val = max(values)
            scale_spread = max_val - min_val
        else:
            mid_point = (min(values)+max(values))/2
            max_val = mid_point + (scale_spread/2)
            min_val = mid_point - (scale_spread/2)
    elif max_val==None:
        if scale_spread == None:
            max_val = max(values)
            scale_spread = max_val - min_val
        else:
            max_val = min_val + scale_spread
    elif min_val==None:
        if scale_spread == None:
            min_val = min(values)
            scale_spread = max_val - min_val
        else:
            min_val = max_val - scale_spread
    else:
        if min_val>=max_val:
            raise ValueError("minimum colour scale value must be lower than "
                             "maximum colour scale value")
        scale_spread = max_val - min_val
    # The user is warned if their specified values exclude data
    if min_val > min(values) or max_val < max(values):
        print("Warning: specified minimum and maximum colour scale values "
              "or specified colour scale spread exclude some data")
    # Determine intermediate values
    scale_vals = []
    for i in range(n_colours):
        try:
            scale_vals.append(np.round(min_val+scale_spread*(i/(n_colours-1)),
                                       scale_dp))
        except ZeroDivisionError:
            scale_vals.append(np.round((min_val), scale_dp))
    return scale_vals

def get_colour_mapping(colour_scale, colormap):
    if colormap == "metoffice":
        return None
    else:
        norm = matplotlib.colors.Normalize(vmin=min(colour_scale),
                                           vmax=max(colour_scale), clip=True)
        try:
            mapping = cm.ScalarMappable(norm=norm, cmap=colormap)
        except ValueError:
            print("'{0}' is not a valid colormap. See https://matplotlib"
                  ".org/3.1.1/gallery/color/colormap_reference.html "
                  "for available colormaps. Viridis colormap will "
                  "be used by default".format(colormap))
            return cm.ScalarMappable(norm=norm, cmap="viridis")
        else:
            return mapping

def get_colour(value, colormap, mapping):
    if colormap == "metoffice":
        i=0
        try:
            while value > metOfficeLimits[i]:
                i += 1
        except IndexError:
            raise IndexError("Data is outside of the limits of the metoffice "
                             "scale. Select another colormap for this data.")
        return metOfficeColours[i]
    else:
        return mapping.to_rgba(value)

def get_shape_scale(values, max_val, min_val, n_shapes, scale_diverges,
                    scale_spread, scale_dp):
    if scale_spread is not None and scale_spread < 0:
        scale_spread=abs(scale_spread)
    if n_shapes is not None and n_shapes > 7:
        n_shapes = 7
        print("Maximum number of shapes is 7.")
    scale_vals = []
    if n_shapes == None:
        if scale_diverges:
            n_shapes = 4
        else:
            n_shapes = 5

    # diverging scale
    if scale_diverges:
        if max_val==None and min_val==None:
            if scale_spread == None:
                scale_spread = 2
                max_val = max(abs(max(values, key=abs)), scale_spread/2)
                min_val = max_val * -1
            else:
                max_val = scale_spread/2
                min_val = max_val * -1
        elif max_val==None:
            if scale_spread == None:
                max_val = min_val * -1
            else:
                max_val = min_val + scale_spread
        elif min_val==None:
            if scale_spread == None:
                min_val = max_val * -1
            else:
                min_val = max_val - scale_spread
        else:
            if min_val>=max_val:
                raise ValueError("minimum shape scale value must be lower than"
                                 " maximum shape scale value")
        # -ve vals and zero
        scale_spread = 0 - min_val
        for i in reversed(range(n_shapes)):
            scale_vals.append(np.round(0-scale_spread
                                       *(i/(n_shapes-1)), scale_dp))
        # +ve values
        scale_spread = max_val
        for i in range(1, n_shapes):
            scale_vals.append(np.round(0+scale_spread
                                       *(i/(n_shapes-1)), scale_dp))
    # negative scale
    elif scale_is_negative(values):
        if max_val==None and min_val==None:
            if scale_spread == None:
                scale_spread = 1
                max_val = max(max(values),0)
                min_val = max(abs(max(values, key=abs)), scale_spread)*-1
            else:
                max_val = 0
                min_val = -scale_spread
        elif max_val==None:
            if scale_spread == None:
                max_val = 0
            else:
                max_val = min_val + scale_spread
        elif min_val==None:
            if scale_spread == None:
                min_val = min(values)
            else:
                min_val = max(max(values, key=abs), max_val-scale_spread,
                              key=abs)
        else:
            if min_val>=max_val:
                raise ValueError("minimum shape scale value must be lower than"
                                 " maximum shape scale value")
        scale_spread = abs(max_val - min_val)
        for i in reversed(range(n_shapes)):
            scale_vals.append(np.round(max_val-scale_spread
                                       *(i/(n_shapes-1)), scale_dp))
    # normal scale
    else:
        if max_val==None and min_val==None:
            if scale_spread == None:
                scale_spread = 1
                max_val = max(abs(max(values, key=abs)), scale_spread)
                min_val = min(min(values),0)
            else:
                max_val = scale_spread
                min_val = 0
        elif max_val==None:
            if scale_spread == None:
                max_val = max(abs(max(values, key=abs)), 1)
            else:
                max_val = min_val + scale_spread
        elif min_val==None:
            if scale_spread == None:
                min_val = min(min(values),0)
            else:
                min_val = max_val - scale_spread
        else:
            if min_val>=max_val:
                raise ValueError("minimum shape scale value must be lower than"
                                 " maximum shape scale value")
        scale_spread = max_val - min_val
        for i in range(n_shapes):
            try:
                scale_vals.append(np.round(min_val+scale_spread
                                           *(i/(n_shapes-1)), scale_dp))
            except ZeroDivisionError:
                scale_vals.append(np.round((min_val), scale_dp))

    # The user is warned if their specified values exclude data
    if min_val > min(values) or max_val < max(values):
        print("Warning: specified minimum and maximum shape scale values "
              "or specified shape scale spread exclude some data")
    return scale_vals

def get_frequency_scale(shape_scale, scale_diverges):
    frequency_scale = []
    if scale_diverges:
        n_shapes = len(shape_scale)//2 + 1
        for i in reversed(range(n_shapes-1)):
            frequency_scale.append(3*(2**i))
        frequency_scale.append(0)
        for i in range(n_shapes-1):
            frequency_scale.append(3*(2**i))
    else:
        n_shapes = len(shape_scale)
        if scale_is_negative(shape_scale):
            for i in reversed(range(n_shapes-1)):
                frequency_scale.append(3*(2**i))
            frequency_scale.append(0)
        else:
            frequency_scale.append(0)
            for i in range(n_shapes-1):
                frequency_scale.append(3*(2**i))
    return frequency_scale

def get_shape(value, shape=None, divergent=None, shape_pos=None,
              shape_neg=None):
    if divergent:
        if value <= 0:
            return shape_neg
        else:
            return shape_pos
    else:
        return shape

def get_frequency(value, shape_scale, frequency_scale, interval_type):
    if len(shape_scale) == 1:
        return frequency_scale[0]
    if interval_type=="closest":
        i=0
        while value > shape_scale[i]:
            i += 1
            if i==len(shape_scale)-1:
                break
        if abs(value-shape_scale[i]) > abs(value-shape_scale[i-1]):
            i = i - 1
    elif interval_type=="limit":
        i=0
        if value <= 0:
            while value > shape_scale[i]:
                i += 1
        else:
            while value >= shape_scale[i]:
                i += 1
                if i==len(shape_scale):
                    break
            i -= 1
    else:
        raise ValueError("The specified interval type for categorizing shapes "
                         "values does not exist. Choose from 'closest' or "
                         "'limit'")
    return frequency_scale[i]


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__title__ = ''
__author__ = 'HaiFeng'
__mtime__ = '2016/8/16'
"""
# import talib._ta_lib as talib
from hfpy.data import Data
from hfpy.bar import Bar
from hfpy.strategy import Strategy
import numpy as np
import talib as ta

class SMACross(Strategy):

    def __init__(self, jsonfile):
        super().__init__(jsonfile)
        self.p_ma1 = self.Params['MA1']
        self.p_ma2 = self.Params['MA2']
        self.p_lots = self.Params['Lots']

    def OnBarUpdate(self, data=Data, bar=Bar):
        if len(self.C) < self.p_ma2:
            return
        # if len(data.Instrument) > 0:
        #     print(f'{data.Tick.Instrument},{data.Tick.Volume}')

        # print('{0}-{1}'.format(self.D[-1], self.C[-1]))
        ma1 = ta.SMA(np.array(self.C, dtype=float), self.p_ma1)
        ma2 = ta.SMA(np.array(self.C, dtype=float), self.p_ma2)

        self.IndexDict['ma5'] = ma1
        self.IndexDict['ma10'] = ma2

        if len(ma2) < 2 or len(ma1) < 2:
            return
        if self.PositionLong == 0:
            if ma1[-1] >= ma2[-1] and ma1[-2] < ma2[-2]:
                if self.PositionShort > 0:
                    self.BuyToCover(self.O[-1], self.p_lots, '买平')
                self.Buy(self.O[-1], self.p_lots, '买开')
        elif self.PositionShort == 0:
            if ma1[-1] <= ma2[-1] and ma1[-2] > ma2[-2]:
                if self.PositionLong > 0:
                    self.Sell(self.O[-1], self.p_lots, '卖平')
                self.SellShort(self.O[-1], self.p_lots, '卖开')


text:
# -*- coding: utf-8 -*-
"""
@author: HYPJUDY 2019/4/15
https://github.com/HYPJUDY

Decoupling Localization and Classification in Single Shot Temporal Action Detection
-----------------------------------------------------------------------------------

Functions to get train and test data

"""

import numpy as np
import random
from os.path import join
import pickle
from config import get_anno_ath, get_data_x_path
import sys

small_num_data_for_test = False

def read_window_info(path):
    result = []
    with open(path, 'r') as path:
        for line in path:
            sFrame, video_name = line.split(',')
            result.append([int(sFrame), video_name.strip()])
    return result

def read_pickle(path):
    with open(path, 'rb') as path:
        if sys.version_info[0] == 2:  # python2
            result = pickle.load(path)
        else:  # python3
            result = pickle.load(path, encoding='bytes')
    return result

############################# GET TRAIN DATA ##############################

def batch_data_process(batch_data):
    batch_size = len(batch_data)
    new_batch_data = np.array(np.ones([1, batch_data[0].shape[1]]))
    batch_start_index = [0]
    for i in range(batch_size):
        new_batch_data = np.concatenate((new_batch_data, batch_data[i]))
        if i < (batch_size - 1):
            batch_start_index.append(batch_start_index[-1] + len(batch_data[i]))
    new_batch_data = new_batch_data[1:]
    batch_start_index.append(len(new_batch_data))
    return new_batch_data, np.array(batch_start_index)

def get_train_data(config, mode, pretrain_dataset, shuffle=True):
    batch_size = config.batch_size
    split_set = config.train_split_set
    data_x_path = get_data_x_path(config.feature_path, split_set, mode, pretrain_dataset)
    anno_path = get_anno_ath(split_set)
    # Since the dataX is matched with window_info.log,
    # window_info need to load from pre-defined file
    gt_label_file = join(anno_path, 'gt_label.pkl')
    gt_info_file = join(anno_path, 'gt_info.pkl')
    gt_label = read_pickle(gt_label_file)
    gt_info = read_pickle(gt_info_file)

    if not small_num_data_for_test:
        num_data = len(gt_label)
    else:
        num_data = batch_size

    batch_dataX = []
    batch_gt_label = []
    batch_gt_info = []
    batch_index = []

    batch_start_list = [i * batch_size for i in range(int(num_data / batch_size))]
    if (num_data - (batch_start_list[-1] + batch_size)) > (batch_size / 8):
        batch_start_list.append(num_data - batch_size)

    batch_shuffle_list = list(range(num_data))

    if shuffle:
        random.seed(6)
        random.shuffle(batch_shuffle_list)

    for bstart in batch_start_list:
        data_list = batch_shuffle_list[bstart:(bstart + batch_size)]
        tmp_batch_dataX = []
        tmp_batch_gt_label = []
        tmp_batch_gt_info = []
        for idx in data_list:
            adataX = np.load(join(data_x_path, str(idx) + '.npy'))
            tmp_batch_dataX.append(adataX)
            tmp_batch_gt_label.append(gt_label[idx])
            tmp_batch_gt_info.append(gt_info[idx])

        batch_dataX.append(np.array(tmp_batch_dataX))

        tmp_batch_gt_label, start_index = batch_data_process(tmp_batch_gt_label)
        batch_gt_label.append(tmp_batch_gt_label)
        batch_index.append(start_index)

        tmp_batch_gt_info, start_index = batch_data_process(tmp_batch_gt_info)
        batch_gt_info.append(tmp_batch_gt_info)

    return batch_dataX, batch_gt_label, batch_gt_info, batch_index

############################# GET TEST DATA ##############################

def get_test_data(config, mode, pretrain_dataset):
    batch_size = config.batch_size
    split_set = config.test_split_set
    data_x_path = get_data_x_path(config.feature_path, split_set, mode, pretrain_dataset)
    anno_path = get_anno_ath(split_set)

    # Since the dataX is matched with window_info.log,
    # window_info need to load from pre-defined file
    window_info_path = join(anno_path, 'window_info.log')
    window_info = read_window_info(window_info_path)

    if not small_num_data_for_test:
        num_data = len(window_info)
    else:
        num_data = batch_size

    batch_dataX = []
    batch_window_info = []

    batch_start_list = [i * batch_size for i in range(int(num_data / batch_size))]
    if (num_data - (batch_start_list[-1] + batch_size)) > (batch_size / 8):
        batch_start_list.append(num_data - batch_size)

    batch_list = list(range(num_data))

    for bstart in batch_start_list:
        data_list = batch_list[bstart:(bstart + batch_size)]
        tmp_batch_dataX = []
        tmp_batch_window_info = []
        for idx in data_list:
            adataX = np.load(join(data_x_path, str(idx) + '.npy'))
            tmp_batch_dataX.append(adataX)
            tmp_batch_window_info.append(window_info[idx])

        batch_dataX.append(np.array(tmp_batch_dataX))
        batch_window_info.append(tmp_batch_window_info)

    return batch_dataX, batch_window_info

max_stars_repo_name: HYPJUDY/Decouple-SSAD

max_issues_repo_name: bityangke/Decouple-SSAD

max_forks_repo_name: bityangke/Decouple-SSAD