text:
from functools import partial
from re import A

import numpy as np
import matplotlib.pyplot as plt
from scipy.special import gamma, softmax
from scipy.linalg import sqrtm
import scipy.stats
from tqdm import tqdm

PI = 3.1415926535897932384626
E = 2.718281828459045
EPSILON = 0.000001

omega = lambda k: 2 * PI**((k-1)/2) / gamma((k-1)/2)
c = lambda k: 0.39894228040143267793**k

# utils
import numpy as np
from scipy.stats import multivariate_normal

def multidet(M):
  # Determinant of multidimensional matrix
  dm = np.zeros(M.shape[0])
  for _ in range(len(dm)):
    dm[_] = np.linalg.det(M[_])
  return dm[np.newaxis].T

# 90% sure this can be done using broadcasting:
def multimult(A, B):
  # "Broadcast" multiplication of matrices with vector
  dm = np.zeros(A.shape + B.shape)
  for _ in range(len(dm)):
    dm[_] = A[_] * B
  return dm

def sigmabroadcast(sig, samples):
  # "Broadcast" multiplication of matrices with vector
  dm = np.zeros((samples.shape[0], sig.shape[0], sig.shape[0]))
  for _ in range(dm.shape[0]):
    dm[_] = sig @ samples[0][np.newaxis].T
  return dm

def gaussianExpectation(mu, cov, fn, n_samples=1000, presamples=None):
  if type(presamples) == type(None):
    samples = np.random.multivariate_normal(mu, cov, n_samples)
  else:
    samples = presamples
  return np.mean(np.apply_along_axis(fn, 1, samples), axis=0)

def mvNormpdf(x, mu, cov):
  k = mu.shape[0]
  #print(x.shape, mu.shape, cov.shape)
  return 1/((2*np.pi)**(k/2)*(np.linalg.det(cov)**0.5))* np.exp(-0.5 * ((x-mu).T @ (np.linalg.pinv(cov)) @ (x-mu)).sum())

def logRatio(mu1, mu2, sig1, sig2, w1, w2):
  return np.log(1 + (w2 * mvNormpdf(x, mu2, sig2))/(w1 * mvNormpdf(x, mu1, sig1)))

def pairwiseObjective(mu1, mu2, sig1, sig2, w1, w2):
  return -w1 * gaussianExpectation() - w2 * gaussianExpectation()

class Param:
  def __init__(self):
    self.m_prev = 0
    self.v_prev = 0

class VGMM:
    def __init__(self, k=1, d=1):
        """We have a few parameters:
        sigma_0 = base sigma
        lambda_i (i \in range(1, k+1)) = sigma multiple
        w_i (i \in range(1, k+1)) = weight
        mu_i = mean
        """

        self.sigma_0 = np.eye(d)
        self.lambdas = np.random.rand(k, 1)
        self.Ws = np.random.rand(k, 1)
        self.ws = softmax(self.Ws)
        self.mus = np.random.rand(k, d)

        self.k = k
        self.d = d

        self.eta = 0.01 # learning rate

        #adam
        self.beta_1 = 0.9
        self.beta_2 = 0.999
        self.epsilon = 10**(-8)
        self.m_prev = 0
        self.v_prev = 0
        self.adamparams = {}

        self.n_samples = 1000
        self.cgd = False

        self.whist = []
        self.lambdahist = [[] for i in range(k)]
        self.lambdaghist = [[] for i in range(k)]
        self.mughist = [[] for i in range(k)]
        self.muhist = [[] for i in range(k)]
        self.h = []

    def get_grad(self, grad, params=None):
      return self.adam(grad, params=params)

    def adam(self, grad, params):
      if params['id'] not in self.adamparams:
        self.adamparams[params['id']] = Param()
      s = self.adamparams[params['id']]

      s.m_current = self.beta_1 * s.m_prev + (1 - self.beta_1) * grad
      s.v_current = self.beta_2 * s.v_prev + (1 - self.beta_2) * grad**2
      s.m_norm = s.m_current/(1 - self.beta_1**(params['t']+1))
      s.v_norm = s.v_current/(1 - self.beta_2**(params['t']+1))

      s.m_prev = s.m_current * 1.0
      s.v_prev = s.v_current * 1.0

      if params['id'] == 'lambda':
        self.h.append(self.eta * s.m_norm/((s.v_norm**0.5) + self.epsilon))
      return self.eta * s.m_norm/((s.v_norm**0.5) + self.epsilon)

    def setTarget(self, dist):
       # idea: compute sigma_0 based on ELBO
       dist1 = scipy.stats.norm(-1, 0.5).pdf
       dist2 = scipy.stats.norm(1, 2).pdf
       self.targetpdf = lambda x: 0 * dist1(x) + 1 * dist2(x)
       self.logp = lambda x: np.log(self.targetpdf(x))
      #  self.logp = lambda x: np.log(self.targetpdf(x)) if np.log(self.targetpdf(x)) > -100 else -100

    def coordinateDescent(self, rounds=500, entropyRounds=1, energyRounds=1):
        # TODO: allow "change < epsilon" flag
        maxrounds = rounds
        round = 0

        for round in tqdm(range(maxrounds)):
            self.round = round
            for i in range(self.ws.shape[0]):
                self.entropyDescent(rounds=entropyRounds, param=("w", i))
                self.energyDescent(rounds=energyRounds, param=("w", i))
                self.ws = softmax(self.Ws)
            for i in range(self.lambdas.shape[0]):
                self.entropyDescent(rounds=entropyRounds, param=("lambda", i))
                self.energyDescent(rounds=energyRounds, param=("lambda", i))
                pass
            for i in range(self.mus.shape[0]):
                self.entropyDescent(rounds=entropyRounds, param=("mu", i))
                self.energyDescent(rounds=energyRounds, param=("mu", i))

    def entropyDescent(self, rounds=1, param=None):

        # maximize entropy wrt params
        self.ws = softmax(self.Ws)
        for round in range(rounds):
            # update wrt individual entropies

            # det of arrays within array, see https://stackoverflow.com/questions/13393733/determinant-of-multidimensional-array for optimization options
            if round % 10 == 0:
              # print(round, multimult(self.lambdas, self.sigma_0))
              # print(round, self.mus)
              pass
            wGrads = -(np.log(self.ws)+1) + 1/2 * np.log(multidet(2*PI*E*multimult(self.lambdas**2, self.sigma_0))) * self.ws * (1 - self.ws)
            lambdaGrads = self.ws * self.d/(self.lambdas + EPSILON)

            var, ind = param
            # only update one variable at a time
            if var == "w" or not self.cgd:
                # mask = np.zeros(self.Ws.shape)
                # mask[ind] = np.ones(self.Ws[ind].shape)
                self.Ws += self.get_grad(wGrads, params={'t':self.round, 'id':'node-W'})

            if var == "lambda" or not self.cgd:
                # mask = np.zeros(self.lambdas.shape)
                # mask[ind] = np.ones(self.lambdas[ind].shape)
                self.lambdas += self.get_grad(lambdaGrads, params={'t':self.round, 'id':'node-lambdas'})

            # self.whist.append((len(self.whist), self.ws[0][0]))
            self.muhist[ind].append(1.0 * self.mus[ind])
            self.lambdahist[ind].append(1.0 * self.lambdas[ind])
            self.lambdaghist[ind].append(1.0 * lambdaGrads[ind])

            # update wrt pairwise entropies
            Q1SAMPLES = np.random.multivariate_normal(self.mus[ind], self.lambdas[ind]**2 * self.sigma_0, self.n_samples)
            self.Q1SAMPLES = Q1SAMPLES
            for j in range(self.k):
                if j != ind:
                    w1, w2 = self.ws[ind], self.ws[j]
                    sig1, sig2 = self.lambdas[ind]**2 * self.sigma_0, self.lambdas[j]**2 * self.sigma_0
                    sig1inv = np.linalg.inv(self.sigma_0 * self.lambdas[ind]**2)
                    sig2inv = np.linalg.inv(self.sigma_0 * self.lambdas[j]**2)
                    mu1, mu2 = self.mus[ind], self.mus[j]

                    # cache results
                    pdf1 = scipy.stats.multivariate_normal.pdf(Q1SAMPLES, mean=mu1, cov=sig1)#mvNormpdf(Q1SAMPLES, mu1, sig1)
                    pdf2 = scipy.stats.multivariate_normal.pdf(Q1SAMPLES, mean=mu2, cov=sig2)#mvNormpdf(Q1SAMPLES, mu2, sig2)

                    if var == "w" or not self.cgd:
                      #wfn1 = lambda x: np.log(1+(w2*pdf2/(w1 * pdf1))) + 1/(1+(w2*pdf2/(EPSILON + w1 * pdf1))) * (w2 * pdf2/(EPSILON + pdf1)) * (-1/(EPSILON + w1))
                      # evaluate wfn1 at Q1SAMPLES
                      wfn1_exp = np.log(1+(w2*pdf2/(w1 * pdf1))) + 1/(1+(w2*pdf2/(EPSILON + w1 * pdf1))) * (w2 * pdf2/(EPSILON + pdf1)) * (-1/(EPSILON + w1))
                      wfn1_exp = np.mean(wfn1_exp)

                      #wfn2 = lambda x: 1/(1 + (w1 * pdf1)/(w2 * pdf2)) * 1/(EPSILON + w2)
                      # evaluate wfn2 at !sSAMPLES
                      wfn2_exp = 1/(1 + (w1 * pdf1)/(w2 * pdf2)) * 1/(EPSILON + w2)
                      wfn2_exp = np.mean(wfn2_exp)

                      #self.Ws[ind] += self.get_grad(-gaussianExpectation(None, None, wfn1, presamples=Q1SAMPLES) - w2 * gaussianExpectation(None, None, wfn2, presamples=Q1SAMPLES) * w1 * (1 - w1), params={'t':self.round, 'id':'pair-'+str(ind)+','+str(j)+'-W'})

                      # use expectations to calculate gradient
                      self.Ws[ind] += self.get_grad(-wfn1_exp - w2 * wfn2_exp * w1 * (1 - w1), params={'t':self.round, 'id':'pair-'+str(ind)+','+str(j)+'-W'})

                    if var == "mu" or not self.cgd:
                      #omega1 = lambda x: 1/(1+(w2*pdf2/(EPSILON + w1 * pdf1))) * (w2 * pdf2/(EPSILON + pdf1)) * (-1/(EPSILON + pdf1**2)) * pdf1 * sig1inv @ (x - mu1[np.newaxis])
                      omega1 = 1/(1+(w2*pdf2/(EPSILON + w1 * pdf1))) * (w2 * pdf2/(EPSILON + pdf1)) * (-1/(EPSILON + pdf1**2)) * pdf1 * -sig1inv @ (Q1SAMPLES - mu1[np.newaxis])

                      #mufn1 = lambda x: sig1inv @ (x - mu1[np.newaxis]) * np.log(1+(w2*pdf2/(w1 * pdf1))) + omega1(x)
                      mufn1_exp = -sig1inv @ (Q1SAMPLES - mu1[np.newaxis]).T * np.log(1+(w2*pdf2/(w1 * pdf1))) + omega1
                      mufn1_exp = np.mean(mufn1_exp)

                      #mufn2 = lambda x: pdf2 * 1/(1 + (w1 * pdf1)/(w2 * pdf2)) * w1/(EPSILON + w2 * pdf2) * sig1inv @ (x - mu1[np.newaxis])
                      postprefix = -sig1inv @ (Q1SAMPLES - mu1[np.newaxis]).T
                      mufn2_exp = pdf2 * 1/(1 + (w1 * pdf1)/(w2 * pdf2)) * w1/(EPSILON + w2 * pdf2) * postprefix
                      mufn2_exp = np.mean(mufn2_exp)

                      self.mus[ind] += self.get_grad(np.squeeze((-w1 * mufn1_exp - w2 * mufn2_exp), axis=-1), params={'t':self.round, 'id':'pair-'+str(ind)+','+str(j)+'-mu'})

                    if var == "lambda" or not self.cgd:
                      # omega1 = lambda x: 1/(1+(w2*pdf2/(EPSILON + w1 * pdf1))) * (w2 * pdf2/(EPSILON + pdf1)) * (-1/(EPSILON + pdf1**2)) * pdf1 * (sig1inv - sig1inv @ (x - mu1[np.newaxis]) @ (x - mu1[np.newaxis]).T @ sig1inv) @ (self.lambdas[ind] * self.sigma_0)
                      # print(sig1inv.shape, (Q1SAMPLES - mu1[np.newaxis]).shape, (Q1SAMPLES - mu1[np.newaxis]).T.shape, sig1inv)
                      postprefix = (sig1inv - sig1inv @ (Q1SAMPLES - mu1[np.newaxis]).T @ (Q1SAMPLES - mu1[np.newaxis]) @ sig1inv)
                      postpostprefix = (self.lambdas[ind] * self.sigma_0)
                      omega1 = 1/(1+(w2*pdf2/(EPSILON + w1 * pdf1))) * (w2 * pdf2/(EPSILON + pdf1)) * (-1/(EPSILON + pdf1**2)) * pdf1 * postprefix * postpostprefix

                      #lambdafn1 = lambda x: (sig1inv - sig1inv @ (x - mu1[np.newaxis]) @ (x - mu1[np.newaxis]).T @ sig1inv) @ (self.lambdas[ind] * self.sigma_0) * np.log(1+(w2*pdf2/(w1 * pdf1))) + omega1(x)
                      lambdafn1_exp = (sig1inv - sig1inv @ (Q1SAMPLES - mu1[np.newaxis]).T @ (Q1SAMPLES - mu1[np.newaxis]) @ sig1inv) * (self.lambdas[ind] * self.sigma_0) * np.log(1+(w2*pdf2/(w1 * pdf1))) + omega1
                      lambdafn1_exp = np.mean(lambdafn1_exp)

                      # lambdafn2 = lambda x: pdf2 * 1/(1 + (w1 * pdf1)/(w2 * pdf2)) * w1/(EPSILON + w2 * pdf2) * (sig1inv - sig1inv @ (x - mu1[np.newaxis]) @ (x - mu1[np.newaxis]).T @ sig1inv) @ (self.lambdas[ind] * self.sigma_0)
                      lambdafn2_exp = pdf2 * 1/(1 + (w1 * pdf1)/(w2 * pdf2)) * w1/(EPSILON + w2 * pdf2) * (sig1inv - sig1inv @ (Q1SAMPLES - mu1[np.newaxis]).T @ (Q1SAMPLES - mu1[np.newaxis]) @ sig1inv) * (self.lambdas[ind] * self.sigma_0)
                      lambdafn2_exp = np.mean(lambdafn2_exp)

                      self.lambdas[ind] += self.get_grad(-w1 * lambdafn1_exp - w2 * lambdafn2_exp, params={'t':self.round, 'id':'pair-'+str(ind)+','+str(j)+'-lambda'})

    def energyDescent(self, rounds=1, param="dummy variable"):
        # minimize energy $-\int q \log p$ wrt params
        self.ws = softmax(self.Ws)

        for round in range(rounds):
            for i in range(self.mus.shape[0]):
                mu = self.mus[i].T
                siginv = np.linalg.inv(self.sigma_0 * self.lambdas[i]**2)
                sig = self.sigma_0 * self.lambdas[i]** 2
                w = self.ws[i]

                Q1SAMPLES = np.random.multivariate_normal(mu, sig, self.n_samples)

                # mufn = lambda x: self.logp(x) * siginv @ (x - mu[np.newaxis])
                mufn_exp = self.logp(Q1SAMPLES) * sigmabroadcast(-siginv, (Q1SAMPLES - mu[np.newaxis]))
                mufn_exp = np.mean(mufn_exp)

                # lambdafn = lambda x: self.logp(x) * -0.5 * (siginv - siginv @ (x - mu[np.newaxis]) @ (x - mu[np.newaxis]).T @ siginv) @ (2 * self.lambdas[i] * self.sigma_0)
                lambdafn_exp = self.logp(Q1SAMPLES) * -0.5 * (siginv - siginv @ (Q1SAMPLES - mu[np.newaxis]).T @ (Q1SAMPLES - mu[np.newaxis]) @ siginv) * (2 * self.lambdas[i] * self.sigma_0)
                lambdafn_exp = np.mean(lambdafn_exp)

                logp_exp = self.logp(Q1SAMPLES)
                logp_exp = np.mean(logp_exp)

                self.mus[i] -= self.get_grad(np.squeeze(-self.ws[i] * mufn_exp, axis=-1), params={'t':self.round, 'id':'energy-mu-'+str(i)})*1.5
                self.lambdas[i] -= self.get_grad(-1 * self.ws[i] * lambdafn_exp, params={'t':self.round, 'id':'energy-lambda-'+str(i)}) # expectation should be diagonal matrix
                self.Ws -= self.get_grad(-1 * logp_exp * w * (1 - w), params={'t':self.round, 'id':'energy-W'+str(i)})

                # self.lambdahist.append((len(self.lambdahist), self.lambdas[0][0]))
                self.muhist[i].append(1.0 * self.mus[i])
                self.lambdahist[i].append(1.0 * self.lambdas[i])
                self.mughist[i].append(1.0 * np.squeeze(-self.ws[i] * mufn_exp, axis=-1))

    # print results
    def printout(self, plot=False, show_target=True):
        print("weights", self.ws)
        print("mus", self.mus)
        print("sigma_0", self.sigma_0)
        print("lambdas", self.lambdas)

        if plot:
            if self.d > 2:
                raise ValueError("Dimensionality is too large for a plot of the resultant mixture.")

            self.ws = softmax(self.Ws)

            clustersamples = np.random.multinomial(10000, (self.ws).flatten(), size=1)[0]

            data = np.array([])
            for i in range(len(clustersamples)):
                data = np.append(data, np.random.multivariate_normal(self.mus[i], self.lambdas[i]**2 * self.sigma_0, clustersamples[i]))

            plt.hist(data, density=True, bins=30)
            if show_target==True:
              plt.plot(list(np.arange(-5,5,0.05)), [self.targetpdf(x) for x in list(np.arange(-5,5,0.05))])
            plt.show()

max_stars_repo_name: tiwalayo/zobay

max_issues_repo_name: tiwalayo/zobay

max_forks_repo_name: tiwalayo/zobay

reason: import numpy,import scipy,from scipy


text:
import numpy as np
import sys
import logging
from h2o4gpu.decomposition import TruncatedSVDSklearn as sklearnsvd
from h2o4gpu.solvers import TruncatedSVD
import pytest

print(sys.path)

logging.basicConfig(level=logging.DEBUG)

def func(m=5000, n=10, k=9, algorithm="cusolver", convert_to_float32=False):
    np.random.seed(1234)

    X = np.random.rand(m, n)
    if convert_to_float32:
        X = X.astype(np.float32)
    print("SVD on " + str(X.shape[0]) + " by " + str(X.shape[1]) + " matrix")
    print("Original X Matrix")
    print(X)
    print("\n")

    # Exact scikit impl
    print("sklearn run")
    sklearn_tsvd = sklearnsvd(algorithm="arpack", n_components=k, random_state=42)
    sklearn_tsvd.fit(X)
    print("Sklearn Singular Values")
    print(sklearn_tsvd.singular_values_)
    print("Sklearn Components (V^T)")
    print(sklearn_tsvd.components_)
    print("Sklearn Explained Variance")
    print(sklearn_tsvd.explained_variance_)
    print("Sklearn Explained Variance Ratio")
    print(sklearn_tsvd.explained_variance_ratio_)
    print(sklearn_tsvd.get_params())

    print("GPU run through h2o4gpu wrapper")
    h2o4gpu_tsvd_sklearn_wrapper = TruncatedSVD(n_components=k, algorithm=[algorithm, 'randomized'], random_state=42, verbose=True, n_iter=500, tol=1E-7)
    h2o4gpu_tsvd_sklearn_wrapper.fit(X)
    print("h2o4gpu tsvd Singular Values")
    print(h2o4gpu_tsvd_sklearn_wrapper.singular_values_)
    print("h2o4gpu tsvd Components (V^T)")
    print(h2o4gpu_tsvd_sklearn_wrapper.components_)
    print("h2o4gpu tsvd Explained Variance")
    print(h2o4gpu_tsvd_sklearn_wrapper.explained_variance_)
    print("h2o4gpu tsvd Explained Variance Ratio")
    print(h2o4gpu_tsvd_sklearn_wrapper.explained_variance_ratio_)
    print(h2o4gpu_tsvd_sklearn_wrapper.get_params())

    rtol = 0.5
    assert np.allclose(h2o4gpu_tsvd_sklearn_wrapper.singular_values_, sklearn_tsvd.singular_values_, rtol=rtol)
    assert np.allclose(h2o4gpu_tsvd_sklearn_wrapper.components_, sklearn_tsvd.components_, rtol=rtol)
    assert np.allclose(h2o4gpu_tsvd_sklearn_wrapper.explained_variance_, sklearn_tsvd.explained_variance_, rtol=rtol)
    assert np.allclose(h2o4gpu_tsvd_sklearn_wrapper.explained_variance_ratio_, sklearn_tsvd.explained_variance_ratio_, rtol=rtol)

    ######################

    # Exact scikit impl
    print("sklearn run")
    sklearn_tsvd2 = sklearnsvd(algorithm="randomized", n_components=k, random_state=42)
    sklearn_tsvd2.fit(X)
    print("Sklearn Singular Values")
    print(sklearn_tsvd2.singular_values_)
    print("Sklearn Components (V^T)")
    print(sklearn_tsvd2.components_)
    print("Sklearn Explained Variance")
    print(sklearn_tsvd2.explained_variance_)
    print("Sklearn Explained Variance Ratio")
    print(sklearn_tsvd2.explained_variance_ratio_)
    print(sklearn_tsvd2.get_params())

    print("Sklearn run through h2o4gpu wrapper using n_gpus=0")
    #FAILS to agree, seems cusolver solution is diverging or (unlikely) bug in randomized in same way.
    #h2o4gpu_tsvd_sklearn_wrapper2 = TruncatedSVD(n_components=k, algorithm=[algorithm, 'randomized'], random_state=42, verbose=True, n_gpus=0, n_iter=[1000,400], tol=[1E-7, 1E-7])
    h2o4gpu_tsvd_sklearn_wrapper2 = TruncatedSVD(n_components=k, algorithm=[algorithm, 'randomized'], random_state=42, verbose=True, n_gpus=0, n_iter=[1000,5], tol=[1E-7, 1E-4])
    h2o4gpu_tsvd_sklearn_wrapper2.fit(X)
    print("h2o4gpu tsvd Singular Values")
    print(h2o4gpu_tsvd_sklearn_wrapper2.singular_values_)
    print("h2o4gpu tsvd Components (V^T)")
    print(h2o4gpu_tsvd_sklearn_wrapper2.components_)
    print("h2o4gpu tsvd Explained Variance")
    print(h2o4gpu_tsvd_sklearn_wrapper2.explained_variance_)
    print("h2o4gpu tsvd Explained Variance Ratio")
    print(h2o4gpu_tsvd_sklearn_wrapper2.explained_variance_ratio_)
    print(h2o4gpu_tsvd_sklearn_wrapper2.get_params())

    rtol = 1E-2
    assert np.allclose(h2o4gpu_tsvd_sklearn_wrapper2.singular_values_, sklearn_tsvd2.singular_values_, rtol=rtol)
    assert np.allclose(h2o4gpu_tsvd_sklearn_wrapper2.components_, sklearn_tsvd2.components_, rtol=rtol)
    assert np.allclose(h2o4gpu_tsvd_sklearn_wrapper2.explained_variance_, sklearn_tsvd2.explained_variance_, rtol=rtol)
    assert np.allclose(h2o4gpu_tsvd_sklearn_wrapper2.explained_variance_ratio_, sklearn_tsvd2.explained_variance_ratio_, rtol=rtol)

def test_tsvd_error_k2_cusolver_wrappertest(): func(n=50, k=2, algorithm="cusolver")
def test_tsvd_error_k2_cusolver_wrappertest_float32(): func(n=50, k=2, algorithm="cusolver", convert_to_float32=True)
@pytest.mark.skip("Failing")
def test_tsvd_error_k2_power_wrappertest(): func(n=50, k=2, algorithm="power")

ext: py

reason: import numpy


text:
# -*- encoding: utf-8 -*-
"""
@File    : save_json.py
@Time    : 2020/5/4 18:08
@Author  : Alessia K
@Email   : ------
"""
import json
import numpy as np
import os
import csv

def label_list(path):
	with open(path, 'r', encoding='utf-8') as f:
		class_csv = csv.reader(f, delimiter=',')
		result = {}

		for line, row in enumerate(class_csv):
			line += 1

			try:
				class_name, class_id = row
			except ValueError:
				# raise_from(ValueError('line {}: format should be \'class_name,class_id\''.format(line)), None)
				raise (ValueError('line {}: format should be \'class_name,class_id\''.format(line)), None)

			if class_name in result:
				raise ValueError('line {}: duplicate class name: \'{}\''.format(line, class_name))
			result[int(class_id)] = class_name
		return result

def create_class_dict(path):
	with open(os.path.join(path, 'classify_rule.json'), 'r', encoding='utf-8') as f:
		classify_rule = json.load(f)
		class_name = {}
		for keys, vals in classify_rule.items():
			for i in range(len(vals)):
				class_name[str(vals[i])]=str(keys)

	return class_name

def get_classes_name(class_name, class_name_i):
	return class_name[class_name_i] + '/' + class_name_i

def save_result_as_json(img_name, classes, scores, bboxes, time):
	"""
	{
	    "detection_classes":    []
	    "detection_scores":     []  (.4f)
	    "detection_bboxes":     []  (xmin、ymin、xmax、ymax) (.1f)
	    "latency_time":         ""  (str(.1f))
	}
	"""

	scores = np.around(scores.astype(np.float), decimals=4).tolist()
	bboxes = np.around(bboxes.astype(np.float), decimals=1).tolist()
	print(bboxes)

	save_file = {}
	save_file["detection_classes"] 	= classes
	save_file["detection_scores"] 	= scores
	save_file["detection_boxes"] 	= bboxes
	save_file["latency_time"] 		= "{:.1f} ms".format(time)

	return save_file

	# save_name = os.path.join('data', os.path.basename(img_name).replace('.jpg', '.json'))
	# with open(save_name, 'w+', encoding='utf-8')as f:
	# 	json.dump(save_file, f, indent=4, ensure_ascii=False)

if __name__ == '__main__':
	class_name = create_class_dict(r'D:\Work\ohter\SodicData\train_val')
	print(class_name)
	labels = label_list('data/class_name.csv')
	print(labels)


text:
"""JAX functions to compute losses.
Author: Toshinori Kitamura
Affiliation: NAIST & OSX
"""

import chex
import jax
import optax
from chex import Array

@jax.jit
def l2_loss(pred: Array, target: Array) -> float:
    chex.assert_equal_shape((pred, target))
    loss = optax.l2_loss(pred, target)
    return loss.mean()

@jax.jit
def huber_loss(pred: Array, target: Array) -> float:
    chex.assert_equal_shape((pred, target))
    loss = optax.huber_loss(pred, target)
    return loss.mean()

@jax.jit
def cross_entropy_loss(logits: Array, targ_logits: Array) -> float:
    chex.assert_equal_shape((logits, targ_logits))
    targ = jax.nn.softmax(targ_logits, axis=-1)
    loss = optax.softmax_cross_entropy(logits, targ)
    return loss.mean()

@jax.jit
def kl_loss(logits: Array, targ_logits: Array) -> float:
    chex.assert_equal_shape((logits, targ_logits))
    log_pol = jax.nn.log_softmax(logits, axis=1)
    pol = jax.nn.softmax(logits, axis=-1)
    loss = (pol * (log_pol - targ_logits)).sum(-1, keepdims=True)
    return loss.mean()

max_stars_repo_name: omron-sinicx/ShinRL

max_issues_repo_name: omron-sinicx/ShinRL

max_forks_repo_name: omron-sinicx/ShinRL

reason: import jax


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Dec 21 21:45:56 2017

@author: Marcus
"""

import numpy as np

class network:

    def __init__(self, network_type, characteristic_impedance, *args):
        self.Z0 = characteristic_impedance
        self.Y0 = 1 / self.Z0

        #Define network_type names
        self.series             = 'series'
        self.shunt              = 'shunt'
        self.transmission_line  = 'transmission_line'
        self.transformer        = 'transformer'
        self.pi_pad             = 'pi_pad'
        self.tee_pad            = 'tee_pad'
        self.abcd               = 'abcd'
        self.s                  = 's'

        #Create list of network types
        #The key is the network type, as a string
        #The first element is the number of expected arguments for that network
        self.arg_length_idx = 0

        self.network_types = {
            self.series : [1],
            self.shunt : [1],
            self.transmission_line : [2],
            self.transformer : [1],
            self.pi_pad : [3],
            self.tee_pad : [3],
            self.abcd : [1],
            self.s : [1]
        }

        #Check that we have a valid network type
        if (not(network_type in self.network_types)):
            raise ValueError('Invalid network type: %s' % network_type)

        #Check the number of arguments
        if (self.network_types[network_type][self.arg_length_idx] !=
            len(args)):

           raise ValueError('Incorrect number of arguments')

        # convert args to floats (to make sure no integers are used later on)
        fl_args = np.array(list(args), dtype='float')
        args = tuple(fl_args)
        characteristic_impedance = float(characteristic_impedance)

        #Initialise the ABCD matrix based on the arguments
        initial_value = 0
        self.A = initial_value
        self.B = initial_value
        self.C = initial_value
        self.D = initial_value

        self.S11 = initial_value
        self.S12 = initial_value
        self.S21 = initial_value
        self.S22 = initial_value

        #The expected argument for series is impedance
        if (network_type == self.series):
            impedance_idx = 0
            impedance = args[impedance_idx]

            self.A = 1
            self.B = impedance
            self.C = 0
            self.D = 1

        #The expected argument for shunt is impedance
        elif (network_type == self.shunt):
            impedance_idx = 0
            impedance = args[impedance_idx]

            admittance = 1 / impedance

            self.A = 1
            self.B = 0
            self.C = admittance
            self.D = 1

        #The expected arguments for a transmission line are phase constant
        #(B, or k), and length, in metres
        elif (network_type == self.transmission_line):
            phase_constant_idx = 0
            length_idx = 1

            phase_constant = args[phase_constant_idx]
            length = args[length_idx]

            self.A = np.cos(phase_constant * length)
            self.B = 1j * self.Z0 * np.sin(phase_constant * length)
            self.C = 1j * (1 / self.Z0) * np.sin(phase_constant *
                     length)
            self.D = np.cos(phase_constant * length)

        #The expected argument for a transformer is the number of turns
        elif (network_type == self.transformer):
            turns_idx = 0
            turns = args[turns_idx]

            self.A = turns
            self.B = 0
            self.C = 0
            self.D = 1 / turns

        #The expected arguments for a pi-pad are the impedances, from left to
        #right
        elif (network_type == self.pi_pad):
            Z1_idx = 0
            Z2_idx = 2
            Z3_idx = 1

            Z1 = args[Z1_idx]
            Z2 = args[Z2_idx]
            Z3 = args[Z3_idx]

            Y1 = 1 / Z1
            Y2 = 1 / Z2
            Y3 = 1 / Z3

            self.A = 1 + (Y2 / Y3)
            self.B = 1 / Y3
            self.C = Y1 + Y2 + ((Y1 * Y2) / Y3)
            self.D = 1 + (Y1 / Y3)

        #The expected arguments for a tee-pad are the impedances, from left to
        #right
        elif (network_type == self.tee_pad):
            Z1_idx = 0
            Z2_idx = 2
            Z3_idx = 1

            Z1 = args[Z1_idx]
            Z2 = args[Z2_idx]
            Z3 = args[Z3_idx]

            self.A = 1 + (Z1 / Z3)
            self.B = Z1 + Z2 + ((Z1 * Z2) / Z3)
            self.C = 1 / Z3
            self.D = 1 + (Z2 / Z3)

        #The expected argument for abcd is an ABCD matrix
        elif (network_type == self.abcd):
            abcd_matrix_idx = 0

            A_idx = (0, 0)
            B_idx = (0, 1)
            C_idx = (1, 0)
            D_idx = (1, 1)

            self.abcd = args[abcd_matrix_idx]

            self.A = self.abcd[A_idx]
            self.B = self.abcd[B_idx]
            self.C = self.abcd[C_idx]
            self.D = self.abcd[D_idx]

        #The expected argument for s is an S matrix
        elif (network_type == self.s):
            s_matrix_idx = 0

            S11_idx = (0, 0)
            S12_idx = (0, 1)
            S21_idx = (1, 0)
            S22_idx = (1, 1)

            self.s = args[s_matrix_idx]

            self.S11 = self.s[S11_idx]
            self.S12 = self.s[S12_idx]
            self.S21 = self.s[S21_idx]
            self.S22 = self.s[S22_idx]

            self.convert_s_to_abcd()

        self.generate_abcd_matrix()
        self.generate_s_matrix()

    def generate_abcd_matrix(self):
        self.abcd = np.matrix([[self.A, self.B], [self.C, self.D]])

    def generate_s_matrix(self):
        common_denominator = (self.A + (self.B / self.Z0) +
                             (self.C * self.Z0) + self.D)

        self.S11 = (self.A + (self.B / self.Z0) - (self.C * self.Z0) -
                   self.D) / common_denominator

        self.S12 = (2 * ((self.A * self.D) -
                   (self.B * self.C))) / common_denominator

        self.S21 = 2 / common_denominator

        self.S22 = ((-1 * self.A) + (self.B / self.Z0) - (self.C * self.Z0)
                   + self.D) / common_denominator

        self.s = np.matrix([[self.S11, self.S12], [self.S21, self.S22]])

        self.s_db = 20 * np.log10(np.absolute(self.s))

    def convert_s_to_abcd(self):
        self.A = (((1 + self.S11) * (1 - self.S22)) +
                 (self.S12 * self.S21)) / (2 * self.S21)
        self.B = (self.Z0 * (((1 + self.S11) * (1 + self.S22)) -
                 (self.S12 * self.S21))) / (2 * self.S21)
        self.C = (self.Y0 * (((1 - self.S11) * (1 - self.S22)) -
                 (self.S12 * self.S21))) / (2 * self.S21)
        self.D = (((1 - self.S11) * (1 + self.S22)) +
                 (self.S12 * self.S21)) / (2 * self.S21)

        self.generate_abcd_matrix()

class cascaded_network(network):

    def __init__(self, characteristic_impedance, *args):
        temp = 1
        for element in args:
            temp *= element.abcd

        network.__init__(self, 'abcd', characteristic_impedance, temp)


text:
# -*- coding:utf8 -*-
import numpy as np
from PIL import Image, ImageDraw, ImageFont

def draw_txt(background, word, f_path, t_size , t_color, t_cood,t_pad):
    # フォント設定
    font = ImageFont.truetype(f_path, t_size)
    # テキスト入力用クラス宣言
    draw = ImageDraw.Draw(background)
    # 背景に書き込み
    cnt = 0
    for y in range(t_cood[0], background.size[1]-t_cood[0], t_size+t_pad[0]):
        for x in range(t_cood[1], background_size[0]-t_cood[1], t_size+t_pad[1]):
            if len(word) > cnt:
                draw.text((x,y), word[cnt], font=font, fill=t_color)
            else:
                break
            cnt += 1
    return background

if __name__ == "__main__":
    # 背景作成
    background_size = (320,240)
    background_color = (0,0,0,255)
    background = Image.new('RGBA', background_size, background_color)
    #テキスト指定
    word = str(np.genfromtxt('word.txt', delimiter=',', dtype=np.str))
    font_path = '/System/Library/Fonts/ヒラギノ角ゴシック W4.ttc'
    text_size = 20
    text_color = (180,180,180,255)
    #y,x
    text_coordinate = (20,20)
    text_padding = (10,0)
    draw_img = draw_txt(background, word, font_path, text_size, text_color, text_coordinate, text_padding)
    #画像指定
    draw_img.show()
    #画像保存
    save_path = './text.png'
    draw_img.save(save_path)


text:
import os
import numpy as np
import h5py
import json
from torch.utils.data import Dataset

class VRDDataset(Dataset):
	def __init__(self, cfg, logger):
		self.rng = np.random.RandomState(cfg.ETC.RANDOM_SEED)
		self.phase = cfg.MODEL.PHASE
		self.logger = logger
		self.logger.info('loading preprocessed video segments for {}...'.format(self.phase))

		path = os.path.join('vidvrd-baseline-output', 'preprocessed_data')

		if self.phase == 'train':
			self.dataset = h5py.File(os.path.join(path, 'preprocessed_'+self.phase+'_dataset.hdf5'), 'r')
			self.feats = self.dataset['feats']
			self.pred_id = self.dataset['pred_id']

			assert len(self.feats) == len(self.pred_id)
			self.logger.info('total {} preprocessed relation instance proposals'.format(len(self.feats)))
		elif self.phase == 'test':
			self.dataset = h5py.File(os.path.join(path, 'preprocessed_'+self.phase+'_dataset.hdf5'), 'r')
			self.pairs = self.dataset['pairs']
			self.feats = self.dataset['feats']
			# self.trackid = self.dataset['trackid']

			# with open(os.path.join(path, 'preprocessed_'+self.phase+'_dataset.json'), 'r') as f:
			# 	self.dataset2 = json.load(f)
			# self.index = self.dataset2['index']
			# self.iou = self.dataset2['iou']

			assert len(self.pairs) == len(self.feats)
			self.logger.info('total {} preprocessed relation instance proposals'.format(len(self.feats)))
		else:
			raise ValueError('Unknown phase: {}'.format(self.phase))

	def __len__(self):
		return len(self.feats)

	def __getitem__(self, idx):
		if self.phase == 'train':
			return self.feats[idx], self.pred_id[idx]
		else:
			return self.pairs[idx], self.feats[idx]
			# return self.index[idx], self.pairs[idx], self.feats[idx], self.iou[idx], self.trackid[idx]


text:
import re
from numpy import append
import pandas as pd
import copy
"""
Question has a common format PRB-<problem_number_full_document>  CH.PRB- <chapted_number>.<problem_number_chapterwise>.
Example: PRB-4  CH.PRB- 2.1.

Solution has a common format SOL-<problem_number_full_document>  CH.SOL- <chapted_number>.<problem_number_chapterwise>.
Example: SOL-4  CH.SOL- 2.1.
"""
class ExtractDf:
    def __init__(self, path) -> None:
        self.path = path
        self.df_columns_template = {
            "problem_number": "",
            "chapterwise_problem_number" : "",
            "PRB" : [],
            "SOL" : []
        }
        self.read_txt()

    def convert_to_df(self):
        dataset_list = []
        for _,object in self.problem_sol_dict.items():
            problem_dataset = {}
            problem_dataset['problem_number'] = object["problem_number"]
            problem_dataset['chapterwise_problem_number'] = object['chapterwise_problem_number']
            problem_dataset["PRB"] = " ".join(object['PRB']).replace("-","")
            problem_dataset["SOL"] = " ".join(object["SOL"]).replace("-","")
            dataset_list.append(problem_dataset)
        df = pd.DataFrame(dataset_list)
        df.to_csv("dataset_chapter_2.csv")
        return df

    def prepare_problem_sol_dict(self):
        self.problem_sol_dict = {}
        for statement in self.text:
            problem_number,chapterwise_problem_number,type = self.detect_question_number(statement)
            if problem_number!= -1:
                self.problem_number,self.chapterwise_problem_number,self.type = problem_number,chapterwise_problem_number,type
                if self.chapterwise_problem_number not in self.problem_sol_dict.keys():
                    self.problem_sol_dict[self.chapterwise_problem_number] = {}
                    self.problem_sol_dict[self.chapterwise_problem_number]["PRB"] =[]
                    self.problem_sol_dict[self.chapterwise_problem_number]["SOL"] =[]
                self.problem_sol_dict[self.chapterwise_problem_number]["problem_number"] = self.problem_number
                self.problem_sol_dict[self.chapterwise_problem_number]["chapterwise_problem_number"] = self.chapterwise_problem_number

            else:
                try:
                    self.problem_sol_dict[self.chapterwise_problem_number][self.type].append(statement)
                except Exception as AttributeError:
                    pass

    def detect_question_number(self,statement:str):
        #PRB-22  CH.PRB- 2.19.
        if statement.startswith('PRB-') or statement.startswith('SOL-'):
            first = re.findall(r'[0-9]',statement.split()[0])
            second = re.findall(r'[0-9]*\.[0-9]*\.',statement)
            # problem_number,chapterwise_problem_number,type
            return "".join(first),second[0],statement[:3]
        return -1,-1,-1

    def processing(self):
        for idx in range(len(self.text)):
            self.text[idx] = (self.text[idx][:-1] if self.text[idx].endswith('\n') else self.text[idx])
            self.text[idx] = (self.text[idx][1:-1] if self.text[idx].startswith("(") and self.text[idx].endswith(")") else self.text[idx])
            self.text[idx] = self.text[idx].replace("\uf059","")
            self.text[idx] = self.text[idx].replace("\uf14b","")
            self.text[idx] = self.text[idx].replace("\x0c","")
            self.text[idx] = self.text[idx].replace("\x0c","")
            self.text[idx] = self.text[idx].replace("cid:4","")
            self.text[idx] = ("" if self.text[idx].startswith('Chapter') else self.text[idx])
            if self.text[idx].isdigit() or bool(re.match(r'^[-+]?[0-9]*\.[0-9]+$', self.text[idx])):
                self.text[idx] = ""
        while "" in self.text:
            self.text.remove("")

    def read_txt(self):
        f = open(self.path,'r',encoding="utf8")
        self.text = f.readlines()
        self.processing()
        self.prepare_problem_sol_dict()

if __name__ == '__main__':
    path = 'Sample_Chapter_2_Q&A.txt'
    extract_df_from_text_obj = ExtractDf(path)
    df = extract_df_from_text_obj.convert_to_df()

reason: from numpy


text:
from torch import nn
import torch
import numpy as np
import scipy as sp

class InvertibleMM(nn.Module):
    """ An implementation of a invertible matrix multiplication
    layer from Glow: Generative Flow with Invertible 1x1 Convolutions
    (https://arxiv.org/abs/1807.03039).
    """

    def __init__(self, num_inputs):
        super(InvertibleMM, self).__init__()
        self.W = nn.Parameter(torch.Tensor(num_inputs, num_inputs))
        nn.init.orthogonal_(self.W)

    def forward(self, inputs, cond_inputs=None, mode='direct'):
        if mode == 'direct':
            return inputs @ self.W, torch.slogdet(
                self.W)[-1].unsqueeze(0).unsqueeze(0).repeat(
                    inputs.size(0), 1)
        else:
            return inputs @ torch.inverse(self.W), -torch.slogdet(
                self.W)[-1].unsqueeze(0).unsqueeze(0).repeat(
                    inputs.size(0), 1)

class LUInvertibleMM(nn.Module):
    """ An implementation of a invertible matrix multiplication
    layer from Glow: Generative Flow with Invertible 1x1 Convolutions
    (https://arxiv.org/abs/1807.03039).
    """

    def __init__(self, num_inputs):
        super(LUInvertibleMM, self).__init__()
        self.W = torch.Tensor(num_inputs, num_inputs)
        nn.init.orthogonal_(self.W)
        self.L_mask = torch.tril(torch.ones(self.W.size()), -1)
        self.U_mask = self.L_mask.t().clone()

        P, L, U = sp.linalg.lu(self.W.numpy())
        self.P = torch.from_numpy(P)
        self.L = nn.Parameter(torch.from_numpy(L))
        self.U = nn.Parameter(torch.from_numpy(U))

        S = np.diag(U)
        sign_S = np.sign(S)
        log_S = np.log(abs(S))
        self.sign_S = torch.from_numpy(sign_S)
        self.log_S = nn.Parameter(torch.from_numpy(log_S))

        self.I = torch.eye(self.L.size(0))

    def forward(self, inputs, cond_inputs=None, mode='direct'):
        if str(self.L_mask.device) != str(self.L.device):
            self.L_mask = self.L_mask.to(self.L.device)
            self.U_mask = self.U_mask.to(self.L.device)
            self.I = self.I.to(self.L.device)
            self.P = self.P.to(self.L.device)
            self.sign_S = self.sign_S.to(self.L.device)

        L = self.L * self.L_mask + self.I
        U = self.U * self.U_mask + torch.diag(
            self.sign_S * torch.exp(self.log_S))
        W = self.P @ L @ U

        if mode == 'direct':
            return inputs @ W, self.log_S.sum().unsqueeze(0).unsqueeze(
                0).repeat(inputs.size(0), 1)
        else:
            return inputs @ torch.inverse(
                W), -self.log_S.sum().unsqueeze(0).unsqueeze(0).repeat(
                    inputs.size(0), 1)

reason: import numpy,import scipy


text:
"""
Utility functions for analysing the results.
"""

import os
import numpy as np
import pandas as pd

def plot_loss_scale_dist(train_dir, ax):
    """ Plot the distribution of loss scales across all layers.

  The log file is in CSV format. Example rows:

    ,iter,label,key,val
    0,0,AdaLossConvolution2DFunction,unbound,1.0
    1,0,AdaLossConvolution2DFunction,bound,1.0
    2,0,AdaLossConvolution2DFunction,power_of_two,1.0
    3,0,AdaLossConvolution2DFunction,final,1.0
    4,0,AdaLossConvolution2DFunction,unbound,1.0
    5,0,AdaLossConvolution2DFunction,bound,1.0
    6,0,AdaLossConvolution2DFunction,power_of_two,1.0
    7,0,AdaLossConvolution2DFunction,final,1.0
    8,0,AdaLossConvolution2DFunction,unbound,1.0965939

  Explanation:
  - "iter" refers to the iteration ID.
  - "key" stands for the stage name of the loss scaling calculation, including "unbound", "bound", "power_of_two".
  - "label" is the function ID.
  - "val" is the actual loss scale value.

  Args:
    train_dir(str): path to where log files are stored.
      There should exist a file called "loss_scale.csv"
  """
    log_path = os.path.join(train_dir, "loss_scale.csv")
    if not os.path.isfile(log_path):
        raise FileNotFoundError(
            "'loss_scale.csv' does not exist in train_dir: {}".format(train_dir)
        )

    # load data
    df = pd.read_csv(log_path, index_col=0)

    # extract iteration IDs
    iters = np.unique(df["iter"].values)
    keys = np.unique(df["key"].values)

    # calculate how many layers logged in one iteration.
    N = len(df[(df["iter"] == iters[0])].index) // len(keys)

    # all loss scales
    # we only take care of the case when key equals to 'final'
    arr = df[(df["key"] == "final") & (df["label"] == "AdaLossConvolution2DFunction")][
        "val"
    ].values
    arr = arr.reshape([len(iters), N])
    arr = arr[:, ::-1]  # reverse layer ID

    # calculate mean of loss scales for each layer
    mean = np.mean(arr, axis=0)  # mean accross iterations
    std = np.std(arr, axis=0)

    # plot to ax
    ax.bar(np.arange(len(mean)), mean, yerr=std)

max_stars_repo_name: kumasento/gradient-scaling

max_issues_repo_name: kumasento/gradient-scaling

max_forks_repo_name: kumasento/gradient-scaling


text:
"""Pulse penetration ratio and density absolute mean calculations.

See https://github.com/eEcoLiDAR/eEcoLiDAR/issues/23.
"""

import numpy as np

from laserchicken.feature_extractor.base_feature_extractor import FeatureExtractor
from laserchicken.keys import point, normalized_height

# classification according to
# http://www.asprs.org/wp-content/uploads/2010/12/LAS_1-4_R6.pdf
GROUND_TAGS = [2]

def _is_ground(i, point_cloud):
    return point_cloud[point]['raw_classification']["data"][i] in GROUND_TAGS

class PulsePenetrationFeatureExtractor(FeatureExtractor):
    """Feature extractor for the point density."""

    @classmethod
    def requires(cls):
        """
        Get a list of names of the point attributes that are needed for this feature extraction.

        For simple features, this could be just x, y, and z. Other features can build on again
        other features to have been computed first.

        :return: List of feature names
        """
        return []

    @classmethod
    def provides(cls):
        """
        Get a list of names of the feature values.

        This will return as many names as the number feature values that will be returned.
        For instance, if a feature extractor returns the first 3 eigen values, this method
        should return 3 names, for instance 'eigen_value_1', 'eigen_value_2' and 'eigen_value_3'.

        :return: List of feature names
        """
        return ['pulse_penetration_ratio']

    def extract(self, point_cloud, neighborhoods, target_point_cloud, target_indices, volume_description):
        """
        Extract the feature value(s) of the point cloud at location of the target.

        :param point_cloud: environment (search space) point cloud
        :param neighborhoods: list of arrays of indices of points within the point_cloud argument
        :param target_point_cloud: point cloud that contains target point
        :param target_indices: list of indices of the target point in the target point cloud
        :param volume_description: volume object that describes the shape and size of the search volume
        :return: feature values
        """
        return [self._extract_one(point_cloud, neighborhood) for neighborhood in neighborhoods]

    def _extract_one(self, point_cloud, neighborhood):
        if 'raw_classification' not in point_cloud[point]:
            raise ValueError(
                'Missing raw_classification attribute which is necessary for calculating pulse_penetratio and '
                'density_absolute_mean features.')

        ground_indices = [i for i in neighborhood if _is_ground(i, point_cloud)]
        pulse_penetration_ratio = self._get_pulse_penetration_ratio(
            ground_indices, len(neighborhood))

        return pulse_penetration_ratio

    @staticmethod
    def _get_ground_indices(point_cloud, ground_tags):
        index_grd = []
        for ipt, c in enumerate(point_cloud):
            if c in ground_tags:
                index_grd.append(ipt)
        return index_grd

    @staticmethod
    def _get_pulse_penetration_ratio(ground_indices, n_total_points):
        n_total = max(n_total_points, 1)
        n_ground = len(ground_indices)
        return float(n_ground) / n_total

    def get_params(self):
        """
        Return a tuple of parameters involved in the current feature extractor object.

        Needed for provenance.
        """
        return ()

max_stars_repo_name: eEcoLiDAR/eEcoLiDAR

max_forks_repo_name: vishalbelsare/laserchicken


text:
import matplotlib.pyplot as plt
import numpy as np

Y = "y"
DY = "dy"
U = "u"
POINT_OPTIONS = {"marker": "x",
                 "markersize": 5,
                 "linestyle": "",
                 "color": "black"}
LINE_OPTIONS = {"linewidth": 2}
MESH_OPTIONS = {"align": "edge",
                "alpha": 0.5}
X_TITLES = {"y": "States", "dy": "State Derivatives", "u": "Controls"}

def plot_solution(solution, *, iterpolated=True, plot_y=True, plot_dy=True,
                  plot_u=True, render=True):
    if plot_y:
        t_data_phases, y_datas_phases = interpolate_x_solution(solution, Y)
        render_x_solution(solution, Y, t_data_phases, y_datas_phases)
    if plot_dy:
        t_data_phases, dy_datas_phases = interpolate_x_solution(solution, DY)
        render_x_solution(solution, DY, t_data_phases, dy_datas_phases)
    if plot_u:
        t_data_phases, u_datas_phases = interpolate_x_solution(solution, U)
        render_x_solution(solution, U, t_data_phases, u_datas_phases)
    plot_buffered_renders(render)

def interpolate_x_solution(solution, x):
    t_data_phases = []
    x_datas_phases = []
    zipped = zip(solution.phase_polys,
                 solution.phase_data,
                 solution.it.mesh.mesh_index_boundaries)
    for phase_polys, p_data, mesh_index_boundaries in zipped:
        p_data_tau_lower = p_data.tau[mesh_index_boundaries[:-1]]
        p_data_tau_upper = p_data.tau[mesh_index_boundaries[1:]]
        t_start_stops = list(zip(p_data_tau_lower, p_data_tau_upper))
        t_data = []
        x_datas = []
        for i_x, x_polys in enumerate(getattr(phase_polys, x)):
            t_list = []
            x_list = []
            for t_start_stop, x_poly in zip(t_start_stops, x_polys):
                t_linspace = np.linspace(*t_start_stop)[:-1]
                x_linspace = x_poly(t_linspace)
                t_list.extend(t_linspace)
                x_list.extend(x_linspace)
            t_list.append(p_data.tau[-1])
            x_list.append(getattr(p_data, x)[i_x, -1])
            t_data.append(t_list)
            x_datas.append(x_list)
        t_data = np.array(t_data[0]) * p_data.stretch + p_data.shift
        x_datas = np.array(x_datas)
        t_data_phases.append(t_data)
        x_datas_phases.append(x_datas)
    return t_data_phases, x_datas_phases

def render_x_solution(solution, x, t_data_phases, x_datas_phases):
    plt.figure()
    zipped = zip(solution.backend.p,
                 solution.phase_data,
                 t_data_phases,
                 x_datas_phases)
    for p, p_data, t_data, x_datas in zipped:
        for i_x, x_data in enumerate(x_datas):
            plt.plot(p_data.time, getattr(p_data, x)[i_x], **POINT_OPTIONS)
            label = str(getattr(p, f"{x[-1]}_var")[i_x])[1:]
            plt.plot(t_data, x_data, **LINE_OPTIONS, label=label)
    plt.legend(loc="upper left")
    plt.grid(True)
    plt.title(X_TITLES[x])
    plt.xlabel("Time / $s$")
    plt.ylabel(f"{X_TITLES[x]} Values")

def plot_mesh(mesh, *, render=True):
    try:
        iter(mesh)
    except (TypeError, ValueError):
        mesh = [mesh]
    for i, _ in enumerate(mesh[0].p):
        plot_phase_mesh(mesh, i)
    plot_buffered_renders(render)

def plot_phase_mesh(meshes, i):
    plt.figure()
    for mesh in meshes:
        xs = mesh.tau[i][mesh.mesh_index_boundaries[i][:-1]]
        widths = mesh.h_K[i]
        heights = mesh.N_K[i] / mesh.h_K[i]
        plt.bar(x=xs, height=heights, width=widths, **MESH_OPTIONS)
    plt.grid(True)
    plt.title("Phase Mesh Density")
    plt.xlabel("Tau")
    plt.ylabel("Node Density / $node$")

def plot_buffered_renders(render):
    if render:
        plt.show()

max_stars_repo_name: NoNotCar/pycollo

max_issues_repo_name: NoNotCar/pycollo

max_forks_repo_name: NoNotCar/pycollo


text:
import csv
import cv2
import numpy as np
from keras.models import Sequential
from keras.layers import Flatten, Dense, Lambda
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D, Cropping2D
from scipy import ndimage
import matplotlib.pyplot as plt

FLG_USE_LR = False

images = []
measures = []
data_folder = '/opt/carnd_p3/data/'#'../train_data/4/'
image_folder = data_folder + 'IMG/'
print('training folder:', data_folder)
count = 0
with open(data_folder+'driving_log.csv') as csvfile:
    reader = csv.reader(csvfile)
    for line in reader:
        if count is 0:
            count = count +1
            continue
        source_path = line[0]
        file_name = source_path.split('/')[-1]
        current_img_path = image_folder + file_name
        image = ndimage.imread(current_img_path)
        m = float(line[3])
        if FLG_USE_LR:
            left_img_path = image_folder +  line[1].split('/')[-1]
            right_img_path = image_folder + line[2].split('/')[-1]
            left_img = ndimage.imread(left_img_path)
            right_img = ndimage.imread(right_img_path)
            images.append(left_img)
            measures.append(m+0.2)
            images.append(right_img)
            measures.append(m-0.2)
        images.append(image)
        measures.append(m)
        image_flipped = np.fliplr(image)
        images.append(image_flipped)
        measurement_flipped = -m
        measures.append(measurement_flipped)

#         print('source_path',source_path)
#         print('file_name',image)
y_train = np.array(measures)
X_train = np.array(images)
start_train = True
print_history = True

batch_size = 128
print(X_train.shape)
print(X_train[0].shape)
print(y_train.shape)
input_shape=(160, 320, 3)
if start_train:
    model = Sequential()
    # preprocessing
    model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(160,320,3)))
    model.add(Cropping2D(cropping=((70,25),(0,0))))
#     ===== architecture =====
    model.add(Conv2D(24, 5, 5, activation='elu', subsample=(2, 2)))
    model.add(Conv2D(36, 5, 5, activation='elu', subsample=(2, 2)))
    model.add(Conv2D(48, 5, 5, activation='elu', subsample=(2, 2)))
    model.add(Conv2D(64, 3, 3, activation='elu'))
    model.add(Conv2D(64, 3, 3, activation='elu'))
    model.add(Dropout(0.5))
    model.add(Flatten())
    model.add(Dense(100, activation='elu'))
    model.add(Dense(50, activation='elu'))
    model.add(Dense(10, activation='elu'))
    model.add(Dense(1))
    #2nd Layer - Add a fully connected layer
    #takes in the output of the first layer and sets the output dimensions to (100)

#  ===== architecture end =====
    model.compile(loss = 'mse', optimizer = 'adam')
    history_object = model.fit(x=X_train, y=y_train, batch_size=batch_size,validation_split= 0.2, shuffle = True, epochs = 7, verbose = 1)
    model.save('model.h5')
    if print_history:
        ### print the keys contained in the history object
        print(history_object.history.keys())

        ### plot the training and validation loss for each epoch
        plt.plot(history_object.history['loss'])
        plt.plot(history_object.history['val_loss'])
        plt.title('model mean squared error loss')
        plt.ylabel('mean squared error loss')
        plt.xlabel('epoch')
        plt.legend(['training set', 'validation set'], loc='upper right')
        plt.savefig('history')
        plt.close()

max_stars_repo_name: yingCMU/self-driving-behavior-clone

max_issues_repo_name: yingCMU/self-driving-behavior-clone

max_forks_repo_name: yingCMU/self-driving-behavior-clone

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 17 18:42:25 2018

@author: Mohammad Doosti Lakhani
"""

# import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Importing dataset
dataset = pd.read_csv('Ads_CTR_Optimisation.csv')

# Implementing Thomson Sampling
m,n = dataset.shape

ad_selected = np.zeros(m) # selected ads by users

N1 = np.zeros(n,dtype=np.float16) # ad i got reward count
N0 = np.zeros(n,dtype=np.float16) #ad i did not get reward count

total_reward = 0

# Implementation in vectorized form
for i in range(0,m):
    max_index = 0
    theta = np.random.beta(N1+1,N0+1)
    max_index = np.argmax(theta)
    ad_selected[i] = max_index
    reward = dataset.values[i,max_index]
    if reward == 1:
        N1[max_index] += 1
    else:
        N0[max_index] += 1
    total_reward+= reward

# Visualizing selections
plt.hist(ad_selected)
plt.show()

max_stars_repo_path: Part 6 - Reinforcement Learning/Section 19 - Thompson Sampling/ts.py

max_stars_repo_name: Nikronic/Machine-Learning-Models

max_issues_repo_name: Nikronic/Machine-Learning-Models

max_forks_repo_name: Nikronic/Machine-Learning-Models


text:
"""
mpld3 Logo Idea
===============
This example shows how mpld3 can be used to generate relatively intricate
vector graphics in the browser. This is an adaptation of a logo proposal by
github user debjan, in turn based on both the matplotlib and D3js logos.
"""
# Author: Jake VanderPlas

import matplotlib.pyplot as plt
from matplotlib import image, patches, colors
from matplotlib.colors import colorConverter
import numpy as np
import mpld3

imsize = np.array([319, 217])
center = [108.5, 108.5]
max_radius = 108.5
radii = np.linspace(16, max_radius, 5)
angles = np.arange(0, 360, 45)

fig = plt.figure(figsize=imsize / 50.)
ax = fig.add_axes([0, 0, 1, 1], frameon=False, xticks=[], yticks=[])

# Create a clip path for the elements
clip_path = patches.Rectangle((0, 0), imsize[0], imsize[1],
                              transform=ax.transData)

# Create the background gradient
x = np.array([0, 104, 196, 300])
y = np.linspace(150, 450, 86)[:, None]

c = np.cos(-np.pi / 4)
s = np.sin(-np.pi / 4)
X, Y = (c * x - s * y) - 116, (s * x + c * y)
C = np.arange(255).reshape((3, 85)).T
C = C[::-1, :]
cmap = colors.LinearSegmentedColormap.from_list("mpld3",
                                                [[0.97, 0.6, 0.29],
                                                 [0.97, 0.59, 0.27],
                                                 [0.97, 0.58, 0.25],
                                                 [0.95, 0.44, 0.34],
                                                 [0.92, 0.51, 0.29],
                                                 [0.68, 0.21, 0.20]])
mesh = ax.pcolormesh(X, Y, C, cmap=cmap, shading='gourand', zorder=0)
mesh.set_clip_path(clip_path)

# cut-off the background to form the "D" and "3" using white patches
# (this could also be done with a clip path)
kwargs = dict(fc='white', ec='none', zorder=1)
ax.add_patch(patches.Rectangle([0, 0], center[0], imsize[1], **kwargs))

ax.add_patch(patches.Circle(center, radii[2], **kwargs))
ax.add_patch(patches.Wedge(center, 127, -90, 90, width=18.5, **kwargs))

ax.add_patch(patches.Circle((252, 66), 18, **kwargs))
ax.add_patch(patches.Rectangle([216, 48], 36, 36, **kwargs))
ax.add_patch(patches.Wedge((252, 66), 101, -90, 40.1, width=35, **kwargs))

ax.add_patch(patches.Circle((252, 151), 18, **kwargs))
ax.add_patch(patches.Rectangle([216, 133], 36, 36, **kwargs))
ax.add_patch(patches.Wedge((252, 151), 101, -40.1, 90, width=35, **kwargs))

ax.add_patch(patches.Rectangle([-200, -200], 719, 200, **kwargs))
ax.add_patch(patches.Rectangle([-200, -200], 200, 617, **kwargs))
ax.add_patch(patches.Rectangle([-200, imsize[1]], 719, 200, **kwargs))
ax.add_patch(patches.Rectangle([imsize[0], -200], 200, 617, **kwargs))

# plot circles and lines
for radius in radii:
    ax.add_patch(patches.Circle(center, radius, lw=0.5,
                                ec='gray', fc='none', zorder=2))
for angle in angles:
    dx, dy = np.sin(np.radians(angle)), np.cos(np.radians(angle))
    ax.plot([max_radius * (1 - dx), max_radius * (1 + dx)],
            [max_radius * (1 - dy), max_radius * (1 + dy)],
            '-', color='gray', lw=0.5, zorder=2)

# plot wedges within the graph
wedges = [(98, 231, 258, '#FF6600'),
          (85, 170, 205, '#FFC500'),
          (60, 80, 103, '#7DFF78'),
          (96, 45, 58, '#FD7C1A'),
          (73, 291, 308, '#CCFF28'),
          (47, 146, 155, '#28FFCC'),
          (25, 340, 360, '#004AFF')]

for (radius, theta1, theta2, color) in wedges:
    ax.add_patch(patches.Wedge(center, radius, theta1, theta2,
                               fc=color, ec='black', alpha=0.6, zorder=3))

for patch in ax.patches:
    patch.set_clip_path(clip_path)

ax.set_xlim(0, imsize[0])
ax.set_ylim(imsize[1], 0)

#plt.savefig('mpld3.png')
mpld3.show()


text:
import numpy as np

class Quaternion:
    """A base quaternion class with universal quaternion properties."""

    def __init__(self, w, x, y, z):
        """
        Instantiate a quaternion from its component parts.

        Args:
            w (float): scalar component
            x (float): first vector component
            y (float): second vector component
            z (float): third vector component
        """
        self.__scalar = w
        self.__vector = np.array([x, y, z])

    @property
    def array(self):
        """Get the quaternion as a scalar-first array."""
        return np.array([self.w, self.x, self.y, self.z])

    @property
    def w(self):
        return self.__scalar

    @property
    def v(self):
        return self.__vector

    @property
    def x(self):
        return self.v[0]

    @property
    def y(self):
        return self.v[1]

    @property
    def z(self):
        return self.v[2]

    def normalize(self):
        """Normalize the quaternion to have unit length."""
        mag = np.linalg.norm(self.array)
        if mag > 0:
            arr_norm = self.array/mag
            self.__scalar = arr_norm[0]
            self.__vector = arr_norm[1:]
        else:
            raise ValueError(('Cannot normalize quaternion with non-positive '
                              'magnitude.'))

class UnitQuaternion(Quaternion):
    """A quaternion class that enforces unit magnitude."""

    def __init__(self, w, x, y, z, tol=1e-5):
        """
        Create the quaternion with the closest floating point values to the
        inputted components such that the magnitude is as close to 1.0 as
        possible.

        Args:
            w (float): scalar component
            x (float): first vector component
            y (float): second vector component
            z (float): third vector component
            tol (float): if the difference between 1.0 and the norm of the
                inputted components is greater than tol, raise an Exception
        """
        norm = np.linalg.norm([w, x, y, z])
        if norm == 0:
            raise ValueError(('Cannot create UnitQuaternion from components '
                              'with zero magnitude.'))
        if np.abs(1.0 - norm) > tol:
            raise ValueError(('The magnitude of the inputted components is '
                              'greater than the tolerance for rounding to 1.'))

        w, x, y, z = np.array([w, x, y, z])/norm
        super().__init__(w, x, y, z)

        self.__tol = tol

    @property
    def is_unit(self):
        return np.abs(1.0 - np.linalg.norm(self.array)) < self.__tol

    @property
    def tol(self):
        return self.__tol

max_stars_repo_name: mdbernard/quaternions

max_issues_repo_name: mdbernard/quaternions

max_forks_repo_name: mdbernard/quaternions


text:
import jax
import jax.numpy as jnp
from itertools import combinations
from gate_jax import *

from jax.config import config
config.update("jax_enable_x64", True)

def _block(params, qubits, state, n_qubit, rot_axis='Y'):
    if rot_axis == 'Y' or rot_axis == 'y':
        rgate = ry
    elif rot_axis == 'X' or rot_axis == 'x':
        rgate = rx
    elif rot_axis == 'Z' or rot_axis == 'z':
        rgate = rz
    else:
        raise ValueError("rot_axis should be either 'X', 'Y', or 'Z'.")

    # Rotation layer
    for qubit, param in zip(qubits, params):
        state = rgate(param, n_qubit, qubit) @ state

    # CZ layer
    entangler_pairs = sorted(combinations(range(len(qubits)), 2),
                             key=lambda x: abs(x[0] - x[1]), reverse=False)

    for control, target in entangler_pairs:
        state = cz_gate(n_qubit, control, target) @ state

    return state

def alternating_layer_ansatz(params, n_qubit, s_block, n_layer, rot_axis='Y'):

    assert n_qubit % s_block == 0
    assert len(params) == n_qubit * n_layer

    # Initial state
    state = jnp.array([0] * (2 ** n_qubit - 1) + [1], dtype=jnp.complex64)

    for d in range(n_layer):
        block_idx = jnp.arange(n_qubit)
        if d % 2:
            block_idx = jnp.roll(block_idx, -(s_block // 2))
        block_idx = jnp.reshape(block_idx, (-1, s_block))

        for i in range(block_idx.shape[0]):
            state = _block(params=params[block_idx[i] + d * n_qubit],
                           qubits=block_idx[i], state=state, n_qubit=n_qubit, rot_axis=rot_axis)

    return state

reason: import jax,from jax


text:
import numpy as np
import pytest
import xarray as xr

from climpred import HindcastEnsemble, PerfectModelEnsemble
from climpred.constants import HINDCAST_CALENDAR_STR, PM_CALENDAR_STR
from climpred.tutorial import load_dataset
from climpred.utils import convert_time_index

CALENDAR = PM_CALENDAR_STR.strip("Datetime").lower()

@pytest.fixture
def PM_ds3v_initialized_1d():
    """MPI Perfect-model-framework initialized timeseries xr.Dataset with three
    variables."""
    return load_dataset("MPI-PM-DP-1D").isel(area=1, period=-1)

@pytest.fixture
def PM_ds_initialized_1d(PM_ds3v_initialized_1d):
    """MPI Perfect-model-framework initialized timeseries xr.Dataset."""
    return PM_ds3v_initialized_1d.drop_vars(["sos", "AMO"])

@pytest.fixture
def PM_da_initialized_1d(PM_ds_initialized_1d):
    """MPI Perfect-model-framework initialized timeseries xr.DataArray."""
    return PM_ds_initialized_1d["tos"]

@pytest.fixture
def PM_da_initialized_1d_lead0(PM_da_initialized_1d):
    """MPI Perfect-model-framework initialized timeseries xr.DataArray in lead-0
    framework."""
    da = PM_da_initialized_1d
    # Convert to lead zero for testing
    da["lead"] = da["lead"] - 1
    da["init"] = da["init"] + 1
    return da

@pytest.fixture
def PM_ds_initialized_3d_full():
    """MPI Perfect-model-framework initialized global maps xr.Dataset."""
    return load_dataset("MPI-PM-DP-3D")

@pytest.fixture
def PM_da_initialized_3d_full(PM_ds_initialized_3d_full):
    """MPI Perfect-model-framework initialized global maps xr.Dataset."""
    return PM_ds_initialized_3d_full["tos"]

@pytest.fixture
def PM_ds_initialized_3d(PM_ds_initialized_3d_full):
    """MPI Perfect-model-framework initialized maps xr.Dataset of subselected North
    Atlantic."""
    return PM_ds_initialized_3d_full.sel(x=slice(120, 130), y=slice(50, 60))

@pytest.fixture
def PM_da_initialized_3d(PM_ds_initialized_3d):
    """MPI Perfect-model-framework initialized maps xr.DataArray of subselected North
    Atlantic."""
    return PM_ds_initialized_3d["tos"]

@pytest.fixture
def PM_ds3v_control_1d():
    """To MPI Perfect-model-framework corresponding control timeseries xr.Dataset with
    three variables."""
    return load_dataset("MPI-control-1D").isel(area=1, period=-1)

@pytest.fixture
def PM_ds_control_1d(PM_ds3v_control_1d):
    """To MPI Perfect-model-framework corresponding control timeseries xr.Dataset."""
    return PM_ds3v_control_1d.drop_vars(["sos", "AMO"])

@pytest.fixture
def PM_da_control_1d(PM_ds_control_1d):
    """To MPI Perfect-model-framework corresponding control timeseries xr.DataArray."""
    return PM_ds_control_1d["tos"]

@pytest.fixture
def PM_ds_control_3d_full():
    """To MPI Perfect-model-framework corresponding control global maps xr.Dataset."""
    return load_dataset("MPI-control-3D")

@pytest.fixture
def PM_da_control_3d_full(PM_ds_control_3d_full):
    """To MPI Perfect-model-framework corresponding control global maps xr.DataArray."""
    return PM_ds_control_3d_full["tos"]

@pytest.fixture
def PM_ds_control_3d(PM_ds_control_3d_full):
    """To MPI Perfect-model-framework corresponding control maps xr.Dataset of
    subselected North Atlantic."""
    return PM_ds_control_3d_full.sel(x=slice(120, 130), y=slice(50, 60))

@pytest.fixture
def PM_da_control_3d(PM_ds_control_3d):
    """To MPI Perfect-model-framework corresponding control maps xr.DataArray of
    subselected North Atlantic."""
    return PM_ds_control_3d["tos"]

@pytest.fixture
def perfectModelEnsemble_initialized_control_3d_North_Atlantic(
    PM_ds_initialized_3d, PM_ds_control_3d
):
    """PerfectModelEnsemble with `initialized` and `control` for the North Atlantic."""
    return PerfectModelEnsemble(PM_ds_initialized_3d).add_control(PM_ds_control_3d)

@pytest.fixture
def perfectModelEnsemble_initialized_control(PM_ds_initialized_1d, PM_ds_control_1d):
    """PerfectModelEnsemble initialized with `initialized` and `control` xr.Dataset."""
    pm = PerfectModelEnsemble(PM_ds_initialized_1d)
    pm = pm.add_control(PM_ds_control_1d)
    return pm

@pytest.fixture
def perfectModelEnsemble_3v_initialized_control_1d(
    PM_ds3v_initialized_1d, PM_ds3v_control_1d
):
    """PerfectModelEnsemble 1d initialized with `initialized` and `control` xr.Dataset
    with three variables."""
    return PerfectModelEnsemble(PM_ds3v_initialized_1d).add_control(PM_ds3v_control_1d)

@pytest.fixture
def hind_ds_initialized_1d():
    """CESM-DPLE initialized hindcast timeseries mean removed xr.Dataset."""
    ds = load_dataset("CESM-DP-SST")
    ds["init"] = ds.init.astype("int")
    return ds

@pytest.fixture
def hind_ds_initialized_1d_cftime(hind_ds_initialized_1d):
    """CESM-DPLE initialzed hindcast timeseries with cftime initializations."""
    ds = hind_ds_initialized_1d
    ds = convert_time_index(ds, "init", "ds.init", calendar=HINDCAST_CALENDAR_STR)
    ds.lead.attrs["units"] = "years"
    return ds

@pytest.fixture
def hind_ds_initialized_1d_lead0(hind_ds_initialized_1d):
    """CESM-DPLE initialized hindcast timeseries mean removed xr.Dataset in lead-0
    framework."""
    da = hind_ds_initialized_1d
    # Change to a lead-0 framework
    da["init"] = da["init"] + 1
    da["lead"] = da["lead"] - 1
    return da

@pytest.fixture
def hind_da_initialized_1d(hind_ds_initialized_1d):
    """CESM-DPLE initialized hindcast timeseries mean removed xr.DataArray."""
    return hind_ds_initialized_1d["SST"]

@pytest.fixture
def hind_ds_initialized_3d_full():
    """CESM-DPLE initialized hindcast Pacific maps mean removed xr.Dataset."""
    da = load_dataset("CESM-DP-SST-3D")
    return da - da.mean("init")

@pytest.fixture
def hind_ds_initialized_3d(hind_ds_initialized_3d_full):
    """CESM-DPLE initialized hindcast Pacific maps mean removed xr.Dataset."""
    return hind_ds_initialized_3d_full.isel(nlon=slice(0, 10), nlat=slice(0, 12))

@pytest.fixture
def hind_da_initialized_3d(hind_ds_initialized_3d):
    """CESM-DPLE initialized hindcast Pacific maps mean removed xr.DataArray."""
    return hind_ds_initialized_3d["SST"]

@pytest.fixture
def hist_ds_uninitialized_1d():
    """CESM-LE uninitialized historical timeseries members mean removed xr.Dataset."""
    da = load_dataset("CESM-LE")
    # add member coordinate
    da["member"] = range(1, 1 + da.member.size)
    return da - da.mean("time")

@pytest.fixture
def hist_da_uninitialized_1d(hist_ds_uninitialized_1d):
    """CESM-LE uninitialized historical timeseries members mean removed xr.DataArray."""
    return hist_ds_uninitialized_1d["SST"]

@pytest.fixture
def reconstruction_ds_1d():
    """CESM-FOSI historical reconstruction timeseries members mean removed
    xr.Dataset."""
    da = load_dataset("FOSI-SST")
    return da - da.mean("time")

@pytest.fixture
def reconstruction_ds_1d_cftime(reconstruction_ds_1d):
    """CESM-FOSI historical reconstruction timeseries with cftime time axis."""
    ds = reconstruction_ds_1d
    ds = convert_time_index(ds, "time", "ds.init", calendar=HINDCAST_CALENDAR_STR)
    return ds

@pytest.fixture
def reconstruction_da_1d(reconstruction_ds_1d):
    """CESM-FOSI historical reconstruction timeseries members mean removed
    xr.DataArray."""
    return reconstruction_ds_1d["SST"]

@pytest.fixture
def reconstruction_ds_3d_full():
    """CESM-FOSI historical Pacific reconstruction maps members mean removed
    xr.Dataset."""
    ds = load_dataset("FOSI-SST-3D")
    return ds - ds.mean("time")

@pytest.fixture
def reconstruction_ds_3d(reconstruction_ds_3d_full):
    """CESM-FOSI historical reconstruction maps members mean removed
    xr.Dataset."""
    return reconstruction_ds_3d_full.isel(nlon=slice(0, 10), nlat=slice(0, 12))

@pytest.fixture
def reconstruction_da_3d(reconstruction_ds_3d):
    """CESM-FOSI historical reconstruction maps members mean removed
    xr.DataArray."""
    return reconstruction_ds_3d["SST"]

@pytest.fixture
def observations_ds_1d():
    """Historical timeseries from observations matching `hind_da_initialized_1d` and
    `hind_da_uninitialized_1d` mean removed xr.Dataset."""
    da = load_dataset("ERSST")
    return da - da.mean("time")

@pytest.fixture
def observations_da_1d(observations_ds_1d):
    """Historical timeseries from observations matching `hind_da_initialized_1d` and
    `hind_da_uninitialized_1d` mean removed xr.DataArray."""
    return observations_ds_1d["SST"]

@pytest.fixture
def hindcast_recon_3d(hind_ds_initialized_3d, reconstruction_ds_3d):
    """HindcastEnsemble initialized with `initialized`, `reconstruction`(`recon`)."""
    # fix to align coords
    for c in ["TLAT", "TLONG", "TAREA"]:
        reconstruction_ds_3d[c] = hind_ds_initialized_3d[c]
    hindcast = HindcastEnsemble(hind_ds_initialized_3d)
    hindcast = hindcast.add_observations(reconstruction_ds_3d)
    hindcast = hindcast - hindcast.sel(time=slice("1964", "2014")).mean("time").sel(
        init=slice("1964", "2014")
    ).mean("init")
    return hindcast

@pytest.fixture
def hindcast_recon_1d_ym(hind_ds_initialized_1d, reconstruction_ds_1d):
    """HindcastEnsemble initialized with `initialized`, `uninitialzed` and `recon`."""
    hindcast = HindcastEnsemble(hind_ds_initialized_1d)
    hindcast = hindcast.add_observations(reconstruction_ds_1d)
    hindcast = hindcast - hindcast.sel(time=slice("1964", "2014")).mean("time").sel(
        init=slice("1964", "2014")
    ).mean("init")
    return hindcast

@pytest.fixture
def hindcast_hist_obs_1d(
    hind_ds_initialized_1d, hist_ds_uninitialized_1d, observations_ds_1d
):
    """HindcastEnsemble initialized with `initialized`, `uninitialzed` and `obs`."""
    hindcast = HindcastEnsemble(hind_ds_initialized_1d)
    hindcast = hindcast.add_uninitialized(hist_ds_uninitialized_1d)
    hindcast = hindcast.add_observations(observations_ds_1d)
    hindcast = hindcast - hindcast.sel(time=slice("1964", "2014")).mean("time").sel(
        init=slice("1964", "2014")
    ).mean("init")
    return hindcast

@pytest.fixture
def hindcast_recon_1d_mm(hindcast_recon_1d_ym):
    """HindcastEnsemble with initialized and reconstruction (observations) as a monthly
    time series (no grid)."""
    hindcast = hindcast_recon_1d_ym.sel(time=slice("1964", "1970"))
    hindcast._datasets["initialized"].lead.attrs["units"] = "months"
    hindcast._datasets["observations"] = (
        hindcast._datasets["observations"].resample(time="1MS").interpolate("linear")
    )
    return hindcast

@pytest.fixture
def hindcast_recon_1d_dm(hindcast_recon_1d_ym):
    """HindcastEnsemble with initialized and reconstruction (observations) as a daily
    time series (no grid)."""
    hindcast = hindcast_recon_1d_ym.sel(time=slice("1964", "1970"))
    hindcast._datasets["initialized"].lead.attrs["units"] = "days"
    hindcast._datasets["observations"] = (
        hindcast._datasets["observations"].resample(time="1D").interpolate("linear")
    )
    return hindcast

@pytest.fixture
def ds1():
    """Small plain multi-dimensional coords xr.Dataset."""
    return xr.Dataset(
        {"air": (("lon", "lat"), [[0, 1, 2], [3, 4, 5], [6, 7, 8]])},
        coords={"lon": [1, 3, 4], "lat": [5, 6, 7]},
    )

@pytest.fixture
def ds2():
    """Small plain multi-dimensional coords xr.Dataset identical values but with
    different coords compared to ds1."""
    return xr.Dataset(
        {"air": (("lon", "lat"), [[0, 1, 2], [3, 4, 5], [6, 7, 8]])},
        coords={"lon": [1, 3, 6], "lat": [5, 6, 9]},
    )

@pytest.fixture
def da1():
    """Small plain two-dimensional xr.DataArray."""
    return xr.DataArray([[0, 1], [3, 4], [6, 7]], dims=("x", "y"))

@pytest.fixture
def da2():
    """Small plain two-dimensional xr.DataArray with different values compared to
    da1."""
    return xr.DataArray([[0, 1], [5, 6], [6, 7]], dims=("x", "y"))

@pytest.fixture
def da_lead():
    """Small xr.DataArray with coords `init` and `lead`."""
    lead = np.arange(5)
    init = np.arange(5)
    return xr.DataArray(
        np.random.rand(len(lead), len(init)),
        dims=["init", "lead"],
        coords=[init, lead],
    )

@pytest.fixture
def two_dim_da():
    """xr.DataArray with two dims."""
    da = xr.DataArray(
        np.vstack(
            [
                np.arange(0, 5, 1.0),
                np.arange(0, 10, 2.0),
                np.arange(0, 40, 8.0),
                np.arange(0, 20, 4.0),
            ]
        ),
        dims=["row", "col"],
    )
    return da

@pytest.fixture
def multi_dim_ds():
    """xr.Dataset with multi-dimensional coords."""
    ds = xr.tutorial.open_dataset("air_temperature")
    ds = ds.assign(**{"airx2": ds["air"] * 2})
    return ds

@pytest.fixture
def da_SLM():
    """Small xr.DataArray with dims `S`, `M` and  `L` for `init`, `member` and
    `lead`.
    """
    lead = np.arange(5)
    init = np.arange(5)
    member = np.arange(5)
    return xr.DataArray(
        np.random.rand(len(lead), len(init), len(member)),
        dims=["S", "L", "M"],
        coords=[init, lead, member],
    )

@pytest.fixture
def da_dcpp():
    """Small xr.DataArray with coords `dcpp_init_year`, `member_id` and `time` as from
    `intake-esm` `hindcastA-dcpp`."""
    lead = np.arange(5)
    init = np.arange(5)
    member = np.arange(5)
    return xr.DataArray(
        np.random.rand(len(lead), len(init), len(member)),
        dims=["dcpp_init_year", "time", "member_id"],
        coords=[init, lead, member],
    )

@pytest.fixture
def PM_ds_initialized_1d_ym_cftime(PM_ds_initialized_1d):
    """MPI Perfect-model-framework initialized timeseries xr.Dataset with init as
    cftime."""
    PM_ds_initialized_1d = convert_time_index(
        PM_ds_initialized_1d,
        "init",
        "PM_ds_initialized_1d.init",
        calendar=PM_CALENDAR_STR,
    )
    PM_ds_initialized_1d["lead"].attrs["units"] = "years"
    return PM_ds_initialized_1d

@pytest.fixture
def PM_ds_control_1d_ym_cftime(PM_ds_control_1d):
    """To MPI Perfect-model-framework corresponding control timeseries xr.Dataset with
    time as cftime."""
    PM_ds_control_1d = convert_time_index(
        PM_ds_control_1d, "time", "PM_ds_control_1d.time", calendar=PM_CALENDAR_STR
    )
    return PM_ds_control_1d

@pytest.fixture
def perfectModelEnsemble_initialized_control_1d_ym_cftime(
    PM_ds_initialized_1d_ym_cftime, PM_ds_control_1d_ym_cftime
):
    """PerfectModelEnsemble with MPI Perfect-model-framework initialized and control
    timeseries annual mean with cftime coords."""
    pm = PerfectModelEnsemble(PM_ds_initialized_1d_ym_cftime)
    pm = pm.add_control(PM_ds_control_1d_ym_cftime)
    return pm

@pytest.fixture
def PM_ds_initialized_1d_mm_cftime(PM_ds_initialized_1d):
    """MPI Perfect-model-framework initialized timeseries xr.Dataset with init as
    cftime faking all inits with monthly separation in one year and lead units to
    monthly."""
    PM_ds_initialized_1d["init"] = xr.cftime_range(
        start="3004",
        periods=PM_ds_initialized_1d.init.size,
        freq="MS",
        calendar=CALENDAR,
    )
    PM_ds_initialized_1d["lead"].attrs["units"] = "months"
    return PM_ds_initialized_1d

@pytest.fixture
def PM_ds_control_1d_mm_cftime(PM_ds_control_1d):
    """To MPI Perfect-model-framework corresponding control timeseries xr.Dataset with
    time as cftime faking the time resolution to monthly means."""
    PM_ds_control_1d["time"] = xr.cftime_range(
        start="3000", periods=PM_ds_control_1d.time.size, freq="MS", calendar=CALENDAR
    )
    return PM_ds_control_1d

@pytest.fixture
def perfectModelEnsemble_initialized_control_1d_mm_cftime(
    PM_ds_initialized_1d_mm_cftime, PM_ds_control_1d_mm_cftime
):
    """PerfectModelEnsemble with MPI Perfect-model-framework initialized and control
    timeseries monthly mean with cftime coords."""
    pm = PerfectModelEnsemble(PM_ds_initialized_1d_mm_cftime)
    pm = pm.add_control(PM_ds_control_1d_mm_cftime)
    return pm

@pytest.fixture
def PM_ds_initialized_1d_dm_cftime(PM_ds_initialized_1d):
    """MPI Perfect-model-framework initialized timeseries xr.Dataset with init as
    cftime faking all inits with daily separation in one year and lead units to
    daily."""
    PM_ds_initialized_1d["init"] = xr.cftime_range(
        start="3004",
        periods=PM_ds_initialized_1d.init.size,
        freq="D",
        calendar=CALENDAR,
    )
    PM_ds_initialized_1d["lead"].attrs["units"] = "days"
    return PM_ds_initialized_1d

@pytest.fixture
def PM_ds_control_1d_dm_cftime(PM_ds_control_1d):
    """To MPI Perfect-model-framework corresponding control timeseries xr.Dataset with
    time as cftime faking the time resolution to daily means."""
    PM_ds_control_1d = PM_ds_control_1d.isel(
        time=np.random.randint(0, PM_ds_control_1d.time.size, 5000)
    )
    PM_ds_control_1d["time"] = xr.cftime_range(
        start="3000", periods=PM_ds_control_1d.time.size, freq="D", calendar=CALENDAR
    )
    return PM_ds_control_1d

@pytest.fixture
def perfectModelEnsemble_initialized_control_1d_dm_cftime(
    PM_ds_initialized_1d_dm_cftime, PM_ds_control_1d_dm_cftime
):
    """PerfectModelEnsemble with MPI Perfect-model-framework initialized and control
    timeseries daily mean with cftime coords."""
    pm = PerfectModelEnsemble(PM_ds_initialized_1d_dm_cftime)
    pm = pm.add_control(PM_ds_control_1d_dm_cftime)
    return pm

@pytest.fixture
def small_initialized_da():
    """Very small simulation of an initialized forecasting system."""
    inits = [1990, 1991, 1992, 1993]
    lead = [1]
    return xr.DataArray(
        np.random.rand(len(inits), len(lead)),
        dims=["init", "lead"],
        coords=[inits, lead],
    )

@pytest.fixture
def small_verif_da():
    """Very small simulation of a verification product."""
    time = [1990, 1991, 1992, 1993, 1994]
    return xr.DataArray(np.random.rand(len(time)), dims=["time"], coords=[time])

max_stars_repo_name: raybellwaves/climpred

max_issues_repo_name: raybellwaves/climpred

max_forks_repo_name: raybellwaves/climpred


text:
from copy import deepcopy
from typing import Tuple

import numpy as np
import pandas as pd
import pytest

from etna.datasets import generate_const_df
from etna.datasets.tsdataset import TSDataset

@pytest.fixture(autouse=True)
def random_seed():
    """Fixture to fix random state for every test case."""
    import random

    import torch

    SEED = 121  # noqa: N806
    torch.manual_seed(SEED)
    random.seed(SEED)
    np.random.seed(SEED)

@pytest.fixture()
def example_df(random_seed):
    df1 = pd.DataFrame()
    df1["timestamp"] = pd.date_range(start="2020-01-01", end="2020-02-01", freq="H")
    df1["segment"] = "segment_1"
    df1["target"] = np.arange(len(df1)) + 2 * np.random.normal(size=len(df1))

    df2 = pd.DataFrame()
    df2["timestamp"] = pd.date_range(start="2020-01-01", end="2020-02-01", freq="H")
    df2["segment"] = "segment_2"
    df2["target"] = np.sqrt(np.arange(len(df2)) + 2 * np.cos(np.arange(len(df2))))

    return pd.concat([df1, df2], ignore_index=True)

@pytest.fixture
def two_dfs_with_different_timestamps(random_seed):
    """Generate two dataframes with the same segments and different timestamps"""

    def generate_df(start_time):
        df = pd.DataFrame()
        for i in range(5):
            tmp = pd.DataFrame({"timestamp": pd.date_range(start_time, "2021-01-01")})
            tmp["segment"] = f"segment_{i + 1}"
            tmp["target"] = np.random.uniform(0, 10, len(tmp))
            df = df.append(tmp)
        df = df.pivot(index="timestamp", columns="segment")
        df = df.reorder_levels([1, 0], axis=1)
        df = df.sort_index(axis=1)
        df.columns.names = ["segment", "feature"]
        return TSDataset(df, freq="1D")

    df1 = generate_df(start_time="2020-01-01")
    df2 = generate_df(start_time="2019-01-01")

    return df1, df2

@pytest.fixture
def two_dfs_with_different_segments_sets(random_seed):
    """Generate two dataframes with the same timestamps and different segments"""

    def generate_df(n_segments):
        df = pd.DataFrame()
        for i in range(n_segments):
            tmp = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", "2021-01-01")})
            tmp["segment"] = f"segment_{i + 1}"
            tmp["target"] = np.random.uniform(0, 10, len(tmp))
            df = df.append(tmp)
        df = df.pivot(index="timestamp", columns="segment")
        df = df.reorder_levels([1, 0], axis=1)
        df = df.sort_index(axis=1)
        df.columns.names = ["segment", "feature"]
        return TSDataset(df, freq="1D")

    df1 = generate_df(n_segments=5)
    df2 = generate_df(n_segments=10)

    return df1, df2

@pytest.fixture
def train_test_dfs(random_seed):
    """Generate two dataframes with the same segments and the same timestamps"""

    def generate_df():
        df = pd.DataFrame()
        for i in range(5):
            tmp = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", "2021-01-01")})
            tmp["segment"] = f"segment_{i + 1}"
            tmp["target"] = np.random.uniform(0, 10, len(tmp))
            df = df.append(tmp)
        df = df.pivot(index="timestamp", columns="segment")
        df = df.reorder_levels([1, 0], axis=1)
        df = df.sort_index(axis=1)
        df.columns.names = ["segment", "feature"]
        return TSDataset(df, freq="1D")

    df1 = generate_df()
    df2 = generate_df()

    return df1, df2

@pytest.fixture
def simple_df() -> TSDataset:
    """Generate dataset with simple values without any noise"""
    history = 49

    df1 = pd.DataFrame()
    df1["target"] = np.arange(history)
    df1["segment"] = "A"
    df1["timestamp"] = pd.date_range(start="2020-01-01", periods=history)

    df2 = pd.DataFrame()
    df2["target"] = [0, 2, 4, 6, 8, 10, 12] * 7
    df2["segment"] = "B"
    df2["timestamp"] = pd.date_range(start="2020-01-01", periods=history)

    df = pd.concat([df1, df2]).reset_index(drop=True)
    df = TSDataset.to_dataset(df)
    tsds = TSDataset(df, freq="1d")

    return tsds

@pytest.fixture()
def outliers_df():
    timestamp1 = np.arange(np.datetime64("2021-01-01"), np.datetime64("2021-02-01"))
    target1 = [np.sin(i) for i in range(len(timestamp1))]
    target1[10] += 10

    timestamp2 = np.arange(np.datetime64("2021-01-01"), np.datetime64("2021-02-10"))
    target2 = [np.sin(i) for i in range(len(timestamp2))]
    target2[8] += 8
    target2[15] = 2
    target2[26] -= 12

    df1 = pd.DataFrame({"timestamp": timestamp1, "target": target1, "segment": "1"})
    df2 = pd.DataFrame({"timestamp": timestamp2, "target": target2, "segment": "2"})

    df = pd.concat([df1, df2], ignore_index=True)
    return df

@pytest.fixture
def example_df_(random_seed) -> pd.DataFrame:
    periods = 100
    df1 = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=periods)})
    df1["segment"] = ["segment_1"] * periods
    df1["target"] = np.random.uniform(10, 20, size=periods)
    df1["target_no_change"] = df1["target"]

    df2 = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=periods)})
    df2["segment"] = ["segment_2"] * periods
    df2["target"] = np.random.uniform(-15, 5, size=periods)
    df2["target_no_change"] = df2["target"]

    df = pd.concat((df1, df2))
    df = df.pivot(index="timestamp", columns="segment").reorder_levels([1, 0], axis=1).sort_index(axis=1)
    df.columns.names = ["segment", "feature"]
    return df

@pytest.fixture
def example_tsds(random_seed) -> TSDataset:
    periods = 100
    df1 = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=periods)})
    df1["segment"] = "segment_1"
    df1["target"] = np.random.uniform(10, 20, size=periods)

    df2 = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=periods)})
    df2["segment"] = "segment_2"
    df2["target"] = np.random.uniform(-15, 5, size=periods)

    df = pd.concat([df1, df2]).reset_index(drop=True)
    df = TSDataset.to_dataset(df)
    tsds = TSDataset(df, freq="D")

    return tsds

@pytest.fixture
def example_reg_tsds(random_seed) -> TSDataset:
    periods = 100
    df1 = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=periods)})
    df1["segment"] = "segment_1"
    df1["target"] = np.random.uniform(10, 20, size=periods)

    df2 = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=periods)})
    df2["segment"] = "segment_2"
    df2["target"] = np.random.uniform(-15, 5, size=periods)

    exog_weekend_1 = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=periods + 7)})
    exog_weekend_1["segment"] = "segment_1"
    exog_weekend_1["regressor_exog_weekend"] = ((exog_weekend_1.timestamp.dt.dayofweek) // 5 == 1).astype("category")

    exog_weekend_2 = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=periods + 7)})
    exog_weekend_2["segment"] = "segment_2"
    exog_weekend_2["regressor_exog_weekend"] = ((exog_weekend_2.timestamp.dt.dayofweek) // 5 == 1).astype("category")

    df = pd.concat([df1, df2]).reset_index(drop=True)
    exog = pd.concat([exog_weekend_1, exog_weekend_2]).reset_index(drop=True)

    df = TSDataset.to_dataset(df)
    exog = TSDataset.to_dataset(exog)

    tsds = TSDataset(df, freq="D", df_exog=exog)

    return tsds

@pytest.fixture()
def outliers_tsds():
    timestamp1 = np.arange(np.datetime64("2021-01-01"), np.datetime64("2021-02-01"))
    target1 = [np.sin(i) for i in range(len(timestamp1))]
    target1[10] += 10

    timestamp2 = np.arange(np.datetime64("2021-01-01"), np.datetime64("2021-02-10"))
    target2 = [np.sin(i) for i in range(len(timestamp2))]
    target2[8] += 8
    target2[15] = 2
    target2[26] -= 12

    df1 = pd.DataFrame({"timestamp": timestamp1, "target": target1, "segment": "1"})
    df2 = pd.DataFrame({"timestamp": timestamp2, "target": target2, "segment": "2"})

    df = pd.concat([df1, df2], ignore_index=True)

    df = df.pivot(index="timestamp", columns="segment")
    df = df.reorder_levels([1, 0], axis=1)
    df = df.sort_index(axis=1)
    df.columns.names = ["segment", "feature"]

    exog = df.copy()
    exog.columns = pd.MultiIndex.from_arrays([["1", "2"], ["exog", "exog"]])

    tsds = TSDataset(df, "1d", exog)

    return tsds

@pytest.fixture
def outliers_df_with_two_columns() -> TSDataset:
    timestamp1 = np.arange(np.datetime64("2021-01-01"), np.datetime64("2021-02-10"))
    target1 = [np.sin(i) for i in range(len(timestamp1))]
    feature1 = [np.cos(i) for i in range(len(timestamp1))]
    target1[10] += 10
    feature1[7] += 10

    timestamp2 = np.arange(np.datetime64("2021-01-01"), np.datetime64("2021-02-10"))
    target2 = [np.sin(i) for i in range(len(timestamp2))]
    feature2 = [np.cos(i) for i in range(len(timestamp2))]
    target2[8] += 8
    target2[15] = 2
    target2[26] -= 12
    feature2[25] += 10

    df1 = pd.DataFrame({"timestamp": timestamp1, "target": target1, "feature": feature1, "segment": "1"})
    df2 = pd.DataFrame({"timestamp": timestamp2, "target": target2, "feature": feature2, "segment": "2"})

    df = pd.concat([df1, df2], ignore_index=True)

    df = df.pivot(index="timestamp", columns="segment")
    df = df.reorder_levels([1, 0], axis=1)
    df = df.sort_index(axis=1)
    df.columns.names = ["segment", "feature"]

    tsds = TSDataset(df, "1d")

    return tsds

@pytest.fixture
def multitrend_df() -> pd.DataFrame:
    """Generate one segment pd.DataFrame with multiple linear trend."""
    df = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", "2021-05-31")})
    ns = [100, 150, 80, 187]
    ks = [0.4, -0.3, 0.8, -0.6]
    x = np.zeros(shape=(len(df)))
    left = 0
    right = 0
    for i, (n, k) in enumerate(zip(ns, ks)):
        right += n
        x[left:right] = np.arange(0, n, 1) * k
        for _n, _k in zip(ns[:i], ks[:i]):
            x[left:right] += _n * _k
        left = right
    df["target"] = x
    df["segment"] = "segment_1"
    df = TSDataset.to_dataset(df=df)
    return df

@pytest.fixture
def ts_with_different_series_length(example_df: pd.DataFrame) -> TSDataset:
    """Generate TSDataset with different lengths series."""
    df = TSDataset.to_dataset(example_df)
    df.loc[:4, pd.IndexSlice["segment_1", "target"]] = None
    ts = TSDataset(df=df, freq="H")
    return ts

@pytest.fixture
def imbalanced_tsdf(random_seed) -> TSDataset:
    """Generate two series with big time range difference"""
    df1 = pd.DataFrame({"timestamp": pd.date_range("2021-01-25", "2021-02-01", freq="D")})
    df1["segment"] = "segment_1"
    df1["target"] = np.random.uniform(0, 5, len(df1))

    df2 = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", "2021-02-01", freq="D")})
    df2["segment"] = "segment_2"
    df2["target"] = np.random.uniform(0, 5, len(df2))

    df = df1.append(df2)
    df = df.pivot(index="timestamp", columns="segment").reorder_levels([1, 0], axis=1).sort_index(axis=1)
    df.columns.names = ["segment", "feature"]
    ts = TSDataset(df, freq="D")
    return ts

@pytest.fixture
def example_tsdf(random_seed) -> TSDataset:
    df1 = pd.DataFrame()
    df1["timestamp"] = pd.date_range(start="2020-01-01", end="2020-02-01", freq="H")
    df1["segment"] = "segment_1"
    df1["target"] = np.arange(len(df1)) + 2 * np.random.normal(size=len(df1))

    df2 = pd.DataFrame()
    df2["timestamp"] = pd.date_range(start="2020-01-01", end="2020-02-01", freq="H")
    df2["segment"] = "segment_2"
    df2["target"] = np.sqrt(np.arange(len(df2)) + 2 * np.cos(np.arange(len(df2))))

    df = pd.concat([df1, df2], ignore_index=True)
    df = df.pivot(index="timestamp", columns="segment").reorder_levels([1, 0], axis=1).sort_index(axis=1)
    df.columns.names = ["segment", "feature"]
    df = TSDataset(df, freq="H")
    return df

@pytest.fixture
def big_daily_example_tsdf(random_seed) -> TSDataset:
    df1 = pd.DataFrame()
    df1["timestamp"] = pd.date_range(start="2019-01-01", end="2020-04-01", freq="D")
    df1["segment"] = "segment_1"
    df1["target"] = np.arange(len(df1)) + 2 * np.random.normal(size=len(df1))

    df2 = pd.DataFrame()
    df2["timestamp"] = pd.date_range(start="2019-06-01", end="2020-04-01", freq="D")
    df2["segment"] = "segment_2"
    df2["target"] = np.sqrt(np.arange(len(df2)) + 2 * np.cos(np.arange(len(df2))))

    df = pd.concat([df1, df2], ignore_index=True)
    df = df.pivot(index="timestamp", columns="segment").reorder_levels([1, 0], axis=1).sort_index(axis=1)
    df.columns.names = ["segment", "feature"]
    df = TSDataset(df, freq="D")
    return df

@pytest.fixture
def big_example_tsdf(random_seed) -> TSDataset:
    df1 = pd.DataFrame()
    df1["timestamp"] = pd.date_range(start="2020-01-01", end="2021-02-01", freq="D")
    df1["segment"] = "segment_1"
    df1["target"] = np.arange(len(df1)) + 2 * np.random.normal(size=len(df1))

    df2 = pd.DataFrame()
    df2["timestamp"] = pd.date_range(start="2020-01-01", end="2021-02-01", freq="D")
    df2["segment"] = "segment_2"
    df2["target"] = np.sqrt(np.arange(len(df2)) + 2 * np.cos(np.arange(len(df2))))

    df = pd.concat([df1, df2], ignore_index=True)
    df = df.pivot(index="timestamp", columns="segment").reorder_levels([1, 0], axis=1).sort_index(axis=1)
    df.columns.names = ["segment", "feature"]
    df = TSDataset(df, freq="D")
    return df

@pytest.fixture
def simple_df_relevance() -> Tuple[pd.DataFrame, pd.DataFrame]:
    timestamp = pd.date_range("2021-01-01", "2021-02-01")

    df_1 = pd.DataFrame({"timestamp": timestamp, "target": np.arange(32), "segment": "1"})
    df_2 = pd.DataFrame({"timestamp": timestamp[5:], "target": np.arange(5, 32), "segment": "2"})
    df = pd.concat([df_1, df_2], ignore_index=True)
    df = TSDataset.to_dataset(df)

    timestamp = pd.date_range("2020-12-01", "2021-02-11")
    regr1_2 = np.sin(-np.arange(len(timestamp) - 5))
    regr2_2 = np.log(np.arange(1, len(timestamp) - 4))
    df_1 = pd.DataFrame(
        {
            "timestamp": timestamp,
            "regressor_1": np.arange(len(timestamp)),
            "regressor_2": np.zeros(len(timestamp)),
            "segment": "1",
        }
    )
    df_2 = pd.DataFrame({"timestamp": timestamp[5:], "regressor_1": regr1_2, "regressor_2": regr2_2, "segment": "2"})
    df_exog = pd.concat([df_1, df_2], ignore_index=True)
    df_exog = TSDataset.to_dataset(df_exog)

    return df, df_exog

@pytest.fixture
def const_ts_anomal() -> TSDataset:
    df = generate_const_df(periods=15, start_time="2020-01-01", scale=1.0, n_segments=2)
    ts = TSDataset(TSDataset.to_dataset(df), freq="D")
    return ts

@pytest.fixture
def ts_diff_endings(example_reg_tsds):
    ts = deepcopy(example_reg_tsds)
    ts.loc[example_reg_tsds.index[-5] :, pd.IndexSlice["segment_1", "target"]] = np.NAN
    return ts

@pytest.fixture
def toy_dataset_equal_targets_and_quantiles():
    n_periods = 5
    n_segments = 2

    time = list(pd.date_range("2020-01-01", periods=n_periods, freq="1D"))

    df = {
        "timestamp": time * n_segments,
        "segment": ["a"] * n_periods + ["b"] * n_periods,
        "target": np.concatenate((np.array((2, 3, 4, 5, 5)), np.array((3, 3, 3, 5, 2)))).astype(np.float64),
        "target_0.01": np.concatenate((np.array((2, 3, 4, 5, 5)), np.array((3, 3, 3, 5, 2)))).astype(np.float64),
    }
    return TSDataset.to_dataset(pd.DataFrame(df))

@pytest.fixture
def toy_dataset_with_mean_shift_in_target():
    mean_1 = 10
    mean_2 = 20
    n_periods = 5
    n_segments = 2

    time = list(pd.date_range("2020-01-01", periods=n_periods, freq="1D"))

    df = {
        "timestamp": time * n_segments,
        "segment": ["a"] * n_periods + ["b"] * n_periods,
        "target": np.concatenate((np.array((-1, 3, 3, -4, -1)) + mean_1, np.array((-2, 3, -4, 5, -2)) + mean_2)).astype(
            np.float64
        ),
        "target_0.01": np.concatenate((np.array((-1, 3, 3, -4, -1)), np.array((-2, 3, -4, 5, -2)))).astype(np.float64),
    }
    return TSDataset.to_dataset(pd.DataFrame(df))


text:
from random import randint

import numpy as np
import torch
from scipy.ndimage import rotate
from torch.utils.data.dataset import Dataset

from scribbler.generator import DocumentGenerator

class ScribblerDocumentSet(Dataset):

    def __init__(self, helper, loss, transform=True):
        self.loss = loss
        self.document_generator = DocumentGenerator()

    def __getitem__(self, index):
        image, label = self.generate_image_with_label(index)

        image = np.array(image, dtype='float') / 255.0
        label = self.loss.document_to_ytrue(np.array([image.shape[1], image.shape[0]], dtype='int32'),
                                            np.array(label, dtype='int32'))

        # angle = randint(-45, 45)
        # label = rotate(label, angle, order=0)
        # image = rotate(image, angle, order=0)

        return torch.from_numpy(image), torch.from_numpy(label)

    def __len__(self):
        return self.document_generator.count()

    def generate_image_with_label(self, index):
        image, document = self.document_generator.get(index)
        return image, document

max_stars_repo_name: belosthomas/socr

max_issues_repo_name: belosthomas/socr

max_forks_repo_name: belosthomas/socr