text:
import random

import gym as gym
import numpy as np
import pandas as pd
from gym import spaces
from contextlib import closing
from io import StringIO
from typing import List

from db_env.Database import Database
from shared_utils.consts import PROJECT_DIR

REWARDS_FILE = f'{PROJECT_DIR}/data/reward_cache.csv'

class DatabaseEnvironment(gym.Env):
    metadata = {'render.modes': ['ansi']}

    def __init__(self, db: Database, use_cache: bool = False):
        super(DatabaseEnvironment, self).__init__()

        self._db = db
        self._use_cache = use_cache
        self._reward_cache: dict[int, float] = {}
        self._initial_state_reward: float = -np.inf

        self.action_space = spaces.Discrete(len(db.action_mapper))
        self.observation_space = spaces.MultiBinary(int(self.action_space.n / 2))

    def step(self, action):
        self._db.execute_action(action)
        return (self._get_observation(),  # Observation
                self._get_reward(),  # Reward
                False,  # is Episode done
                {'initial_state_reward': self._initial_state_reward})  # Additional info

    def reset(self):
        self._db.reset_indexes()
        self._initial_state_reward = self._db.execute_benchmark()
        return self._get_observation()

    def render(self, mode='ansi'):
        out = StringIO()
        out.write('Environment observation:\n')
        out.write(f'\t{self._get_observation()}\n\n')
        out.write('Database current state:\n')
        for table_name, table in self._db.state.items():
            out.write(f'\t{table_name}:\n')
            for col_name, is_indexed in table.items():
                out.write(f'\t\t{col_name} - is indexed: {is_indexed}\n')
            out.write('\n')
        with closing(out):
            return out.getvalue()

    def get_action_meanings(self):
        """:return: Action meanings in array where index is number of action"""
        return self._db.action_mapper

    def _get_observation(self) -> List[bool]:
        """:return: Map database state to environment observation"""
        observation: list[bool] = []
        for table in self._db.state.values():
            for is_indexed in table.values():
                observation.append(int(is_indexed))
        return observation

    def _get_reward(self) -> float:
        """
        Search for saved reward in dict by current state, or execute benchmark
        to measure and save result.

        :return: Measured or saved result from benchmark
        """
        if not self._use_cache:
            return self._db.execute_benchmark()

        key = str(self._get_observation())
        if key in self._reward_cache.keys():
            return self._reward_cache[key]
        else:
            reward = self._db.execute_benchmark()
            self._reward_cache[key] = reward
            pd.DataFrame.from_dict(self._reward_cache, orient='index').to_csv(REWARDS_FILE)
            return reward

max_stars_repo_name: Chotom/rl-db-indexing

max_issues_repo_name: Chotom/rl-db-indexing

max_forks_repo_name: Chotom/rl-db-indexing


text:
# -------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------

import numpy as np
import numbers, six
from ...common._registration import register_converter
from ...common.tree_ensemble import get_default_tree_classifier_attribute_pairs, get_default_tree_regressor_attribute_pairs, add_tree_to_attribute_pairs

def convert_sklearn_random_forest_classifier(scope, operator, container):
    op = operator.raw_operator
    op_type = 'TreeEnsembleClassifier'
    classes = op.classes_

    if all(isinstance(i, np.ndarray) for i in classes):
        classes = np.concatenate(classes)
    attr_pairs = get_default_tree_classifier_attribute_pairs()
    attr_pairs['name'] = scope.get_unique_operator_name(op_type)
    zipmap_attrs = {'name': scope.get_unique_operator_name('ZipMap')}

    if all(isinstance(i, (numbers.Real, bool, np.bool_)) for i in classes):
        class_labels = [int(i) for i in classes]
        attr_pairs['classlabels_int64s'] = class_labels
        zipmap_attrs['classlabels_int64s'] = class_labels
    elif all(isinstance(i, (six.text_type, six.string_types)) for i in classes):
        class_labels = [str(i) for i in classes]
        attr_pairs['classlabels_strings'] = class_labels
        zipmap_attrs['classlabels_strings'] = class_labels
    else:
        raise ValueError('Only string and integer class labels are allowed')

    # random forest calculate the final score by averaging over all trees' outcomes, so all trees' weights are identical.
    tree_weight = 1. / op.n_estimators

    for tree_id in range(op.n_estimators):
        tree = op.estimators_[tree_id].tree_
        add_tree_to_attribute_pairs(attr_pairs, True, tree, tree_id, tree_weight, 0, True)

    probability_tensor_name = scope.get_unique_variable_name('probability_tensor')
    container.add_node(op_type, operator.input_full_names, [operator.outputs[0].full_name, probability_tensor_name],
                       op_domain='ai.onnx.ml', **attr_pairs)

    container.add_node('ZipMap', probability_tensor_name, operator.outputs[1].full_name,
                       op_domain='ai.onnx.ml', **zipmap_attrs)

def convert_sklearn_random_forest_regressor_converter(scope, operator, container):
    op = operator.raw_operator
    op_type = 'TreeEnsembleRegressor'
    attrs = get_default_tree_regressor_attribute_pairs()
    attrs['name'] = scope.get_unique_operator_name(op_type)
    attrs['n_targets'] = int(op.n_outputs_)

    # random forest calculate the final score by averaging over all trees' outcomes, so all trees' weights are identical.
    tree_weight = 1. / op.n_estimators
    for tree_id in range(op.n_estimators):
        tree = op.estimators_[tree_id].tree_
        add_tree_to_attribute_pairs(attrs, False, tree, tree_id, tree_weight, 0, False)

    container.add_node(op_type, operator.input_full_names, operator.output_full_names, op_domain='ai.onnx.ml', **attrs)

register_converter('SklearnRandomForestClassifier', convert_sklearn_random_forest_classifier)
register_converter('SklearnRandomForestRegressor', convert_sklearn_random_forest_regressor_converter)
register_converter('SklearnExtraTreesClassifier', convert_sklearn_random_forest_classifier)
register_converter('SklearnExtraTreesRegressor', convert_sklearn_random_forest_regressor_converter)

ext: py

max_stars_repo_name: Oewyn/onnxmltools

max_issues_repo_name: Oewyn/onnxmltools

max_forks_repo_name: Oewyn/onnxmltools

reason: import numpy


text:
import numpy as np

# Local modules
from scene import Plane, Sphere

class Ray:
    def __init__(self, pr, nr):
        self.pr = pr
        self.nr = nr

    def at(self, t):
        p = self.pr + t * self.nr
        return p

    def intersect(self, scene_object):
        if isinstance(scene_object, Sphere):
            return self.intersect_sphere(scene_object)
        elif isinstance(scene_object, Plane):
            return self.intersect_plane(scene_object)
        else:
            raise Exception("Intersect function not implemented")

    def intersect_sphere(self, sphere):
        pc = sphere.position
        diff = self.pr - pc
        b = np.dot(self.nr, diff)
        c = np.dot(diff, diff) - sphere.radius ** 2
        discriminant = b ** 2 - c
        if b > 0 or discriminant < 0:
            return -1
        t = -1 * b - np.sqrt(discriminant)
        return t

    def intersect_plane(self, plane):
        n_dot = np.dot(self.nr, plane.n)
        if n_dot == 0:
            return -1
        t = np.dot(plane.position - self.pr, plane.n) / n_dot
        if t > 0:
            return t
        return -1

max_stars_repo_name: HenrYxZ/raytracing-para-todos

max_issues_repo_name: sombra-studio/raytracing-para-todos

max_forks_repo_name: sombra-studio/raytracing-para-todos


text:
import numpy as np
import mlutilities as ml
import matplotlib.pyplot as plt
from copy import copy
import time

class NeuralNetwork:

    def __init__(self, ni, nhs, no):
        if isinstance(nhs, list) or isinstance(nhs, tuple):
            nihs = [ni] + list(nhs)
        else:
            if nhs > 0:
                nihs = [ni, nhs]
                nhs = [nhs]
            else:
                nihs = [ni]
                nhs = []
        if len(nihs) > 1:
            self.Vs = [1/np.sqrt(nihs[i]) *
                       np.random.uniform(-1, 1, size=(1+nihs[i], nihs[i+1])) for i in range(len(nihs)-1)]
            self.W = 1/np.sqrt(nhs[-1]) * np.random.uniform(-1, 1, size=(1+nhs[-1], no))
        else:
            self.Vs = []
            self.W = 1/np.sqrt(ni) * np.random.uniform(-1, 1, size=(1+ni, no))
        self.ni, self.nhs, self.no = ni, nhs, no
        self.Xmeans = None
        self.Xstds = None
        self.Tmeans = None
        self.Tstds = None
        self.trained = False
        self.reason = None
        self.errorTrace = None
        self.numberOfIterations = None
        self.trainingTime = None

    def activation(self, weighted_sum):
        return np.tanh(weighted_sum)

    def activationDerivative(self, activation_value):
        return 1 - activation_value * activation_value

    def __repr__(self):
        str = 'NeuralNetwork({}, {}, {})'.format(self.ni, self.nhs, self.no)
        # str += '  Standardization parameters' + (' not' if self.Xmeans == None else '') + ' calculated.'
        if self.trained:
            str += '\n   Network was trained for {} iterations that took {:.4f} seconds. Final error is {}.'.format(self.numberOfIterations, self.getTrainingTime(), self.errorTrace[-1])
        else:
            str += '  Network is not trained.'
        return str

    def _standardizeX(self, X):
        result = (X - self.Xmeans) / self.XstdsFixed
        result[:, self.Xconstant] = 0.0
        return result

    def _unstandardizeX(self, Xs):
        return self.Xstds * Xs + self.Xmeans

    def _standardizeT(self, T):
        result = (T - self.Tmeans) / self.TstdsFixed
        result[:, self.Tconstant] = 0.0
        return result

    def _unstandardizeT(self, Ts):
        return self.Tstds * Ts + self.Tmeans

    def _pack(self, Vs, W):
        return np.hstack([V.flat for V in Vs] + [W.flat])

    def _unpack(self, w):
        first = 0
        numInThisLayer = self.ni
        for i in range(len(self.Vs)):
            self.Vs[i][:] = w[first:first+(numInThisLayer+1)*self.nhs[i]].reshape((numInThisLayer+1, self.nhs[i]))
            first += (numInThisLayer+1) * self.nhs[i]
            numInThisLayer = self.nhs[i]
        self.W[:] = w[first:].reshape((numInThisLayer+1, self.no))

    def _objectiveF(self, w, X, T):
        self._unpack(w)
        # Do forward pass through all layers
        Zprev = X
        for i in range(len(self.nhs)):
            V = self.Vs[i]
            Zprev = self.activation(Zprev @ V[1:, :] + V[0:1, :])
            #Zprev = np.tanh(Zprev @ V[1:, :] + V[0:1, :])  # handling bias weight without adding column of 1's
        Y = Zprev @ self.W[1:, :] + self.W[0:1, :]
        return 0.5 * np.mean((T-Y)**2)

    def _gradientF(self, w, X, T):
        self._unpack(w)
        # Do forward pass through all layers
        Zprev = X
        Z = [Zprev]
        for i in range(len(self.nhs)):
            V = self.Vs[i]
            Zprev = self.activation(Zprev @ V[1:, :] + V[0:1, :])
            #Zprev = np.tanh(Zprev @ V[1:, :] + V[0:1, :])
            Z.append(Zprev)
        Y = Zprev @ self.W[1:, :] + self.W[0:1, :]
        # Do backward pass, starting with delta in output layer
        delta = -(T - Y) / (X.shape[0] * T.shape[1])
        dW = np.vstack((np.ones((1, delta.shape[0])) @ delta,
                        Z[-1].T @ delta))
        dVs = []
        #delta = (1 - Z[-1]**2) * (delta @ self.W[1:, :].T)
        delta = self.activationDerivative(Z[-1]) * (delta @ self.W[1:, :].T)
        for Zi in range(len(self.nhs), 0, -1):
            Vi = Zi - 1  # because X is first element of Z
            dV = np.vstack((np.ones((1, delta.shape[0])) @ delta,
                            Z[Zi-1].T @ delta))
            dVs.insert(0, dV)
            #delta = (delta @ self.Vs[Vi][1:, :].T) * (1 - Z[Zi-1]**2)
            delta = (delta @ self.Vs[Vi][1:, :].T) * self.activationDerivative(Z[Zi-1])
        return self._pack(dVs, dW)

    def train(self, X, T, nIterations=100, verbose=False,
              weightPrecision=0, errorPrecision=0, saveWeightsHistory=False):

        if self.Xmeans is None:
            self.Xmeans = X.mean(axis=0)
            self.Xstds = X.std(axis=0)
            self.Xconstant = self.Xstds == 0
            self.XstdsFixed = copy(self.Xstds)
            self.XstdsFixed[self.Xconstant] = 1
        X = self._standardizeX(X)

        if T.ndim == 1:
            T = T.reshape((-1, 1))

        if self.Tmeans is None:
            self.Tmeans = T.mean(axis=0)
            self.Tstds = T.std(axis=0)
            self.Tconstant = self.Tstds == 0
            self.TstdsFixed = copy(self.Tstds)
            self.TstdsFixed[self.Tconstant] = 1
        T = self._standardizeT(T)

        startTime = time.time()

        scgresult = ml.scg(self._pack(self.Vs, self.W),
                            self._objectiveF, self._gradientF,
                            X, T,
                            xPrecision=weightPrecision,
                            fPrecision=errorPrecision,
                            nIterations=nIterations,
                            verbose=verbose,
                            ftracep=True,
                            xtracep=saveWeightsHistory)

        self._unpack(scgresult['x'])
        self.reason = scgresult['reason']
        self.errorTrace = np.sqrt(scgresult['ftrace']) # * self.Tstds # to _unstandardize the MSEs
        self.numberOfIterations = len(self.errorTrace)
        self.trained = True
        self.weightsHistory = scgresult['xtrace'] if saveWeightsHistory else None
        self.trainingTime = time.time() - startTime
        return self

    def use(self, X, allOutputs=False):
        Zprev = self._standardizeX(X)
        Z = [Zprev]
        for i in range(len(self.nhs)):
            V = self.Vs[i]
            Zprev = self.activation(Zprev @ V[1:, :] + V[0:1, :])
            #Zprev = np.tanh(Zprev @ V[1:, :] + V[0:1, :])
            Z.append(Zprev)
        Y = Zprev @ self.W[1:, :] + self.W[0:1, :]
        Y = self._unstandardizeT(Y)
        return (Y, Z[1:]) if allOutputs else Y

    def getNumberOfIterations(self):
        return self.numberOfIterations

    def getErrors(self):
        return self.errorTrace

    def getTrainingTime(self):
        return self.trainingTime

    def getWeightsHistory(self):
        return self.weightsHistory

    def draw(self, inputNames=None, outputNames=None, gray=False):
        ml.draw(self.Vs, self.W, inputNames, outputNames, gray)

max_stars_repo_name: vignesh-pagadala/activation-functions

max_issues_repo_name: vignesh-pagadala/activation-functions

max_forks_repo_name: vignesh-pagadala/activation-functions


text:
import sys
import os
import numpy as np
import scipy.stats
import typing

import smc_tools.util

def normal_parameters_from_lognormal(mean, var):

	aux = 1 + var / mean ** 2
	var = np.log(aux)
	mean = np.log(mean) - np.log(np.sqrt(aux))

	return mean, var

def loglikelihood(pf, observations, log_tx_power, log_min_power, log_path_loss_exp):

	# the "inner" PF (for approximating the likelihood) is initialized
	pf.initialize()

	# the parameters of the sensors *within* the PF are set accordingly
	for s in pf.sensors:

		s.set_parameters(
			tx_power=np.exp(log_tx_power), minimum_amount_of_power=np.exp(log_min_power),
			path_loss_exponent=np.exp(log_path_loss_exp))

	# this is needed to rebuild the sensors with the new parameters
	pf.reset_sensors_array()

	res = 0.

	# the collection of observations is processed
	for obs in observations:

		# ...a step is taken
		pf.step(obs)

		# the loglikelihoods computed by the "inner" bootstrap filter
		loglikes = pf.last_unnormalized_loglikelihoods

		# logarithm of the average
		res += smc_tools.util.log_sum_from_individual_logs(loglikes) - np.log(len(loglikes))

	return res

class ProposalUpdater:

	# @staticmethod
	def update_proposal(self, particle_filter):

		# np.ma.average(particle_filter._samples, axis=1, weights=particle_filter._weights).data
		particle_filter._mean = particle_filter._weights @ particle_filter._samples
		particle_filter._covar = np.cov(particle_filter._samples.T, ddof=0, aweights=particle_filter._weights)

		# if the covariance matrix is "essentially" all-zeros, it may happen that samples drawn thereof have zero
		# density (mathematically preposterous but possible due to finite precision issues); in order to avoid this...
		# if all the coefficients in the covariance matrix are *close* to zero...
		if np.allclose(particle_filter._covar, 0):

			# ...the covariance matrix is set equal to the prior covariance matrix
			particle_filter._covar = particle_filter._prior_covar

			return

		# ...still, we make sure it is possible to evaluate the density of a sample with the above covariance...
		try:
			# ...we evaluate the density at the mean
			scipy.stats.multivariate_normal.pdf(x=particle_filter._mean, mean=particle_filter._mean, cov=particle_filter._covar)
		# if it is not possible...
		except (np.linalg.linalg.LinAlgError, ValueError):
			# ...the covariance matrix is set equal to the prior covariance matrix
			particle_filter._covar = particle_filter._prior_covar

	def initialize(self):

		# no need here
		pass

class ClippingProposalUpdater(ProposalUpdater):

	def __init__(self, n_clipped: int) -> None:

		super().__init__()

		self._n_clipped = n_clipped

	def update_proposal(self, particle_filter):

		# weights before clipping are saved since they are returned by the "weights" property of the "particle_filter"
		# (to be used when marking down the largest weight and also when computing the effective sample size)
		particle_filter._unclipped_weights = particle_filter._weights.copy()

		# indices of the samples whose weight is to be clipped
		i_clipped = np.argpartition(particle_filter._unnormalized_log_weights, -self._n_clipped)[-self._n_clipped:]

		# minimum (unnormalized) weight among those to be clipped
		clipping_threshold = particle_filter._unnormalized_log_weights[i_clipped[0]]

		particle_filter._unnormalized_log_weights[i_clipped] = clipping_threshold

		particle_filter._weights = smc_tools.util.normalize_from_logs(particle_filter._unnormalized_log_weights)

		particle_filter._mean = particle_filter._weights @ particle_filter._samples
		particle_filter._covar = np.cov(particle_filter._samples.T, ddof=0, aweights=particle_filter._weights)

class ClippedCovarianceProposalUpdater(ClippingProposalUpdater):

	def update_proposal(self, particle_filter):

		super().update_proposal(particle_filter)

		# mean is recomputed using the unclipped weights
		particle_filter._mean = particle_filter._unclipped_weights @ particle_filter._samples

class VaryingClippedNumberClippingProposalUpdater(ClippingProposalUpdater):

	def __init__(self, n_clipped: typing.Sequence[int]) -> None:

		# the whole list is kept
		self._n_clipped_list = n_clipped

		self._i_n_clipped = 0

	def update_proposal(self, particle_filter):

		# the appropriate "n_clipped" (M_T) is picked
		self._n_clipped = self._n_clipped_list[self._i_n_clipped]

		super().update_proposal(particle_filter)

		# in the next iteration, if any, the following "n_clipped" element in the list should be picked
		self._i_n_clipped += 1

	def initialize(self):

		# reset
		self._i_n_clipped = 0

reason: import numpy,import scipy


text:
import csv
import datetime
import os
import os.path
import pickle
import re

import numpy as np
import pandas as pd

def movesDataGatheredFromWebExport(fname, data):
  directory = os.fsencode(fname)
  for file in os.listdir(directory):
      name = os.fsdecode(file)
      if name.endswith(".csv"):
          filename = (fname + name)
          with open(filename, newline="") as csvfile:
              spamreader = csv.reader(csvfile)
              count = 0
              for row in spamreader:
                  count = count + 1
                  if count > 1 and len(row):
                      dateMoves = re.split("/", row[0])
                      month = dateMoves[0]
                      day = dateMoves[1]
                      year = dateMoves[2]
                      if int(day) < 10:
                          day = "0" + str(day)
                      if int(month) < 10:
                          month = "0" + str(month)
                      year = "20" + year
                      date = year + "-" + month + "-" + day
                      if row[1] == "walking":
                          data.loc[date, "movesSteps"] = int(row[5])
  return data

max_stars_repo_name: LeviBorodenko/MyAIGuide

max_issues_repo_name: LeviBorodenko/MyAIGuide

max_forks_repo_name: LeviBorodenko/MyAIGuide


text:
# https://deeplearningcourses.com/c/artificial-intelligence-reinforcement-learning-in-python
# https://www.udemy.com/artificial-intelligence-reinforcement-learning-in-python
from __future__ import print_function, division
from builtins import range
# Note: you may need to update your version of future
# sudo pip install -U future

import numpy as np
import matplotlib.pyplot as plt
from grid_world import standard_grid, negative_grid
from iterative_policy_evaluation import print_values, print_policy
from monte_carlo_es import max_dict
from sarsa import random_action, GAMMA, ALPHA, ALL_POSSIBLE_ACTIONS

SA2IDX = {}
IDX = 0

class Model:
  def __init__(self):
    self.theta = np.random.randn(25) / np.sqrt(25)
    # if we use SA2IDX, a one-hot encoding for every (s,a) pair
    # in reality we wouldn't want to do this b/c we have just
    # as many params as before
    # print "D:", IDX
    # self.theta = np.random.randn(IDX) / np.sqrt(IDX)

  def sa2x(self, s, a):
    # NOTE: using just (r, c, r*c, u, d, l, r, 1) is not expressive enough
    return np.array([
      s[0] - 1              if a == 'U' else 0,
      s[1] - 1.5            if a == 'U' else 0,
      (s[0]*s[1] - 3)/3     if a == 'U' else 0,
      (s[0]*s[0] - 2)/2     if a == 'U' else 0,
      (s[1]*s[1] - 4.5)/4.5 if a == 'U' else 0,
      1                     if a == 'U' else 0,
      s[0] - 1              if a == 'D' else 0,
      s[1] - 1.5            if a == 'D' else 0,
      (s[0]*s[1] - 3)/3     if a == 'D' else 0,
      (s[0]*s[0] - 2)/2     if a == 'D' else 0,
      (s[1]*s[1] - 4.5)/4.5 if a == 'D' else 0,
      1                     if a == 'D' else 0,
      s[0] - 1              if a == 'L' else 0,
      s[1] - 1.5            if a == 'L' else 0,
      (s[0]*s[1] - 3)/3     if a == 'L' else 0,
      (s[0]*s[0] - 2)/2     if a == 'L' else 0,
      (s[1]*s[1] - 4.5)/4.5 if a == 'L' else 0,
      1                     if a == 'L' else 0,
      s[0] - 1              if a == 'R' else 0,
      s[1] - 1.5            if a == 'R' else 0,
      (s[0]*s[1] - 3)/3     if a == 'R' else 0,
      (s[0]*s[0] - 2)/2     if a == 'R' else 0,
      (s[1]*s[1] - 4.5)/4.5 if a == 'R' else 0,
      1                     if a == 'R' else 0,
      1
    ])
    # if we use SA2IDX, a one-hot encoding for every (s,a) pair
    # in reality we wouldn't want to do this b/c we have just
    # as many params as before
    # x = np.zeros(len(self.theta))
    # idx = SA2IDX[s][a]
    # x[idx] = 1
    # return x

  def predict(self, s, a):
    x = self.sa2x(s, a)
    return self.theta.dot(x)

  def grad(self, s, a):
    return self.sa2x(s, a)

def getQs(model, s):
  # we need Q(s,a) to choose an action
  # i.e. a = argmax[a]{ Q(s,a) }
  Qs = {}
  for a in ALL_POSSIBLE_ACTIONS:
    q_sa = model.predict(s, a)
    Qs[a] = q_sa
  return Qs

if __name__ == '__main__':
  # NOTE: if we use the standard grid, there's a good chance we will end up with
  # suboptimal policies
  # e.g.
  # ---------------------------
  #   R  |   R  |   R  |      |
  # ---------------------------
  #   R* |      |   U  |      |
  # ---------------------------
  #   U  |   R  |   U  |   L  |
  # since going R at (1,0) (shown with a *) incurs no cost, it's OK to keep doing that.
  # we'll either end up staying in the same spot, or back to the start (2,0), at which
  # point we whould then just go back up, or at (0,0), at which point we can continue
  # on right.
  # instead, let's penalize each movement so the agent will find a shorter route.
  #
  # grid = standard_grid()
  grid = negative_grid(step_cost=-0.1)

  # print rewards
  print("rewards:")
  print_values(grid.rewards, grid)

  # no policy initialization, we will derive our policy from most recent Q
  # enumerate all (s,a) pairs, each will have its own weight in our "dumb" model
  # essentially each weight will be a measure of Q(s,a) itself
  states = grid.all_states()
  for s in states:
    SA2IDX[s] = {}
    for a in ALL_POSSIBLE_ACTIONS:
      SA2IDX[s][a] = IDX
      IDX += 1

  # initialize model
  model = Model()

  # repeat until convergence
  t = 1.0
  t2 = 1.0
  deltas = []
  for it in range(20000):
    if it % 100 == 0:
      t += 0.01
      t2 += 0.01
    if it % 1000 == 0:
      print("it:", it)
    alpha = ALPHA / t2

    # instead of 'generating' an epsiode, we will PLAY
    # an episode within this loop
    s = (2, 0) # start state
    grid.set_state(s)

    # get Q(s) so we can choose the first action
    Qs = getQs(model, s)

    # the first (s, r) tuple is the state we start in and 0
    # (since we don't get a reward) for simply starting the game
    # the last (s, r) tuple is the terminal state and the final reward
    # the value for the terminal state is by definition 0, so we don't
    # care about updating it.
    a = max_dict(Qs)[0]
    a = random_action(a, eps=0.5/t) # epsilon-greedy
    biggest_change = 0
    while not grid.game_over():
      r = grid.move(a)
      s2 = grid.current_state()

      # we need the next action as well since Q(s,a) depends on Q(s',a')
      # if s2 not in policy then it's a terminal state, all Q are 0
      old_theta = model.theta.copy()
      if grid.is_terminal(s2):
        model.theta += alpha*(r - model.predict(s, a))*model.grad(s, a)
      else:
        # not terminal
        Qs2 = getQs(model, s2)
        a2 = max_dict(Qs2)[0]
        a2 = random_action(a2, eps=0.5/t) # epsilon-greedy

        # we will update Q(s,a) AS we experience the episode
        model.theta += alpha*(r + GAMMA*model.predict(s2, a2) - model.predict(s, a))*model.grad(s, a)

        # next state becomes current state
        s = s2
        a = a2

      biggest_change = max(biggest_change, np.abs(model.theta - old_theta).sum())
    deltas.append(biggest_change)

  plt.plot(deltas)
  plt.show()

  # determine the policy from Q*
  # find V* from Q*
  policy = {}
  V = {}
  Q = {}
  for s in grid.actions.keys():
    Qs = getQs(model, s)
    Q[s] = Qs
    a, max_q = max_dict(Qs)
    policy[s] = a
    V[s] = max_q

  print("values:")
  print_values(V, grid)
  print("policy:")
  print_policy(policy, grid)

max_stars_repo_name: JouniVatanen/NLP-and-Deep-Learning


text:
from gym.spaces import Box
import numpy as np
import random

from ray.rllib.policy.policy import Policy
from ray.rllib.utils.annotations import override

class RandomPolicy(Policy):
    """Hand-coded policy that returns random actions."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        # Whether for compute_actions, the bounds given in action_space
        # should be ignored (default: False). This is to test action-clipping
        # and any Env's reaction to bounds breaches.
        if self.config.get("ignore_action_bounds", False) and \
                isinstance(self.action_space, Box):
            self.action_space_for_sampling = Box(
                -float("inf"),
                float("inf"),
                shape=self.action_space.shape,
                dtype=self.action_space.dtype)
        else:
            self.action_space_for_sampling = self.action_space

    @override(Policy)
    def compute_actions(self,
                        obs_batch,
                        state_batches=None,
                        prev_action_batch=None,
                        prev_reward_batch=None,
                        **kwargs):
        # Alternatively, a numpy array would work here as well.
        # e.g.: np.array([random.choice([0, 1])] * len(obs_batch))
        return [self.action_space_for_sampling.sample() for _ in obs_batch], \
               [], {}

    @override(Policy)
    def learn_on_batch(self, samples):
        """No learning."""
        return {}

    @override(Policy)
    def compute_log_likelihoods(self,
                                actions,
                                obs_batch,
                                state_batches=None,
                                prev_action_batch=None,
                                prev_reward_batch=None):
        return np.array([random.random()] * len(obs_batch))

max_stars_repo_name: carlos-aguayo/ray

max_issues_repo_name: carlos-aguayo/ray

max_forks_repo_name: carlos-aguayo/ray


text:
# -*- coding: utf-8 -*-

import numpy as np
import pandas as pd

from ..compat import DTYPE

__all__ = [
    'load_airpassengers'
]

def load_airpassengers(as_series=False, dtype=DTYPE):
    """Monthly airline passengers.

    The classic Box & Jenkins airline data. Monthly totals of international
    airline passengers, 1949 to 1960.

    Parameters
    ----------
    as_series : bool, optional (default=False)
        Whether to return a Pandas series. If False, will return a 1d
        numpy array.

    dtype : type, optional (default=np.float64)
        The type to return for the array. Default is np.float64, which is used
        throughout the package as the default type.

    Returns
    -------
    rslt : array-like, shape=(n_samples,)
        The time series vector.

    Examples
    --------
    >>> from pmdarima.datasets import load_airpassengers
    >>> load_airpassengers()  # doctest: +SKIP
    np.array([
        112, 118, 132, 129, 121, 135, 148, 148, 136, 119, 104, 118,
        115, 126, 141, 135, 125, 149, 170, 170, 158, 133, 114, 140,
        145, 150, 178, 163, 172, 178, 199, 199, 184, 162, 146, 166,
        171, 180, 193, 181, 183, 218, 230, 242, 209, 191, 172, 194,
        196, 196, 236, 235, 229, 243, 264, 272, 237, 211, 180, 201,
        204, 188, 235, 227, 234, 264, 302, 293, 259, 229, 203, 229,
        242, 233, 267, 269, 270, 315, 364, 347, 312, 274, 237, 278,
        284, 277, 317, 313, 318, 374, 413, 405, 355, 306, 271, 306,
        315, 301, 356, 348, 355, 422, 465, 467, 404, 347, 305, 336,
        340, 318, 362, 348, 363, 435, 491, 505, 404, 359, 310, 337,
        360, 342, 406, 396, 420, 472, 548, 559, 463, 407, 362, 405,
        417, 391, 419, 461, 472, 535, 622, 606, 508, 461, 390, 432])

    >>> load_airpassengers(True).head()
    0    112.0
    1    118.0
    2    132.0
    3    129.0
    4    121.0
    dtype: float64

    Notes
    -----
    This is monthly data, so *m* should be set to 12 when using in a seasonal
    context.

    References
    ----------
    .. [1] Box, G. E. P., Jenkins, G. M. and Reinsel, G. C. (1976)
           "Time Series Analysis, Forecasting and Control. Third Edition."
           Holden-Day. Series G.
    """
    rslt = np.array([
        112, 118, 132, 129, 121, 135, 148, 148, 136, 119, 104, 118,
        115, 126, 141, 135, 125, 149, 170, 170, 158, 133, 114, 140,
        145, 150, 178, 163, 172, 178, 199, 199, 184, 162, 146, 166,
        171, 180, 193, 181, 183, 218, 230, 242, 209, 191, 172, 194,
        196, 196, 236, 235, 229, 243, 264, 272, 237, 211, 180, 201,
        204, 188, 235, 227, 234, 264, 302, 293, 259, 229, 203, 229,
        242, 233, 267, 269, 270, 315, 364, 347, 312, 274, 237, 278,
        284, 277, 317, 313, 318, 374, 413, 405, 355, 306, 271, 306,
        315, 301, 356, 348, 355, 422, 465, 467, 404, 347, 305, 336,
        340, 318, 362, 348, 363, 435, 491, 505, 404, 359, 310, 337,
        360, 342, 406, 396, 420, 472, 548, 559, 463, 407, 362, 405,
        417, 391, 419, 461, 472, 535, 622, 606, 508, 461, 390, 432
    ]).astype(dtype)

    if as_series:
        return pd.Series(rslt)
    return rslt

max_stars_repo_name: tuomijal/pmdarima

max_issues_repo_name: tuomijal/pmdarima

max_forks_repo_name: tuomijal/pmdarima


text:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import pickle
import random
import subprocess
import neat

subprocess.call(["generated_shuffled_DHSs.sh"], shell=True)
#import dataset
with open('./data/jar/DHSs_onehot.pickle', 'rb') as pickle_in:
    onehot_data = pickle.load(pickle_in)
with open('./data/jar/var_chart.pickle', 'rb') as pickle_in:
    var_chart_data = pickle.load(pickle_in)
with open('./data/jar/gc.pickle', 'rb') as pickle_in:
    gc_data = pickle.load(pickle_in)
with open('./data/jar/lengths.pickle', 'rb') as pickle_in:
    length_data = pickle.load(pickle_in)
with open('./data/jar/get_group_from_dhs.pickle', 'rb') as pickle_in:
    groups_by_DHS = pickle.load(pickle_in)
with open('./data/jar/groups_trunc.pickle', 'rb') as pickle_in:
    groups_trunc = pickle.load(pickle_in)
with open('./data/jar/ng.pickle', 'rb') as pickle_in:
    ng_data = pickle.load(pickle_in)
with open('./data/jar/H3K27ac_count.pickle', 'rb') as pickle_in:
    H3K27ac_count_data = pickle.load(pickle_in)
with open('./data/jar/H3K4me3_count.pickle', 'rb') as pickle_in:
    H3K4me3_count_data = pickle.load(pickle_in)
with open('./data/jar/Nanog_count.pickle', 'rb') as pickle_in:
    Nanog_count_data = pickle.load(pickle_in)
with open('./data/jar/Oct4_count.pickle', 'rb') as pickle_in:
    Oct4_count_data = pickle.load(pickle_in)
with open('./data/jar/Sox2_count.pickle', 'rb') as pickle_in:
    Sox2_count_data = pickle.load(pickle_in)
with open('./data/jar/in_SE.pickle', 'rb') as pickle_in:
    in_SE_data = pickle.load(pickle_in)

sample_dataframe = pd.read_csv("data/mm10_data/DHSs/shuffled_DHSs.bed", header=None, index_col=False)
DHSs_samples = []
for row in sample_dataframe[0]:
    DHSs_samples.append(row)

#flatten seq data sequence and extend list to have rest of data
#extend lists, append single values
inputs = {}
get_DHSs = {}
dhs_index = 0
for dhs in DHSs_samples:
    #if dhs in DHSs_samples:
    input_data = []
    for row in onehot_data[dhs]:
        input_data.extend(row)
    input_data.append(gc_data[dhs])
    input_data.append(length_data[dhs])
    input_data.append(ng_data[dhs])
    input_data.append(H3K27ac_count_data[dhs])
    input_data.append(H3K4me3_count_data[dhs])
    input_data.append(Nanog_count_data[dhs])
    input_data.append(Oct4_count_data[dhs])
    input_data.append(Sox2_count_data[dhs])
    input_data.append(in_SE_data[dhs])
    #add new data above this
    inputs[dhs] = input_data
    get_DHSs[dhs_index] = dhs
    dhs_index += 1
matched_list = []

# group dhs with its label
for dhs in DHSs_samples:
    matched_list.append([inputs[dhs], var_chart_data[dhs]])

#generate random unique indexes to pull out data for testing
testing_indexes = []
tmp_indexes = []
dhs_count = len(DHSs_samples)
testing_size = int(dhs_count/10)
while(len(testing_indexes) < testing_size):
    tmp_indexes.append(random.randint(0, dhs_count - 1))
    testing_indexes = list(set(tmp_indexes))
training_indexes = []
for x in range(dhs_count - 1):
    if x not in testing_indexes:
        training_indexes.append(x)
training_data_list = []
training_labels_list = []

for ind in training_indexes:
    training_data_list.append(matched_list[ind][0])
    training_labels_list.append(matched_list[ind][1])

testing_data_list = []
testing_labels_list = []

for ind in testing_indexes:
    testing_data_list.append(matched_list[ind][0])
    testing_labels_list.append(matched_list[ind][1])

#model structure
number_generations = 100
fitness_for_saving = 0.9

def eval_genomes(genomes, config):
    for genome_id, genome in genomes:
        genome.fitness = 0
        net = neat.nn.FeedForwardNetwork.create(genome,config)
        for dhs in training_data_list:
            ind = training_data_list.index(dhs)
            label = training_labels_list[ind]

            nnInput =  dhs
            output = net.activate(nnInput)

            for index, prediction in enumerate(output):
                if prediction < 0:
                    prediction = 0
                genome.fitness += (0.25/dhs_count) * (1 - abs(prediction - label[index]))
        if genome.fitness > fitness_for_saving:
            with open("./data/jar/best_model.pickle", 'wb') as pickle_out:
                pickle.dump(net, pickle_out)
            with open("./data/jar/testing_data.pickle", 'wb') as pickle_out:
                pickle.dump([testing_data_list, testing_data_labels], pickle_out)

config = neat.Config(neat.DefaultGenome,neat.DefaultReproduction,neat.DefaultSpeciesSet,neat.DefaultStagnation, './data/DHS_NEAT')
population = neat.Population(config)
population.add_reporter(neat.StdOutReporter(False))
stats = neat.StatisticsReporter()
winner = population.run(eval_genomes, number_generations)


text:
import io
import time

import numpy as np
from PIL import Image
from PySide2.QtCore import QBuffer, QEventLoop
from PySide2.QtGui import QImage, QPalette, QColor
from PySide2.QtWidgets import QWidget, QApplication

from itint.ui_color_display import Ui_ColorDisplay
from itint.widget_color_selector import ColorSelectorWidget

def rgb_to_hsv(r, g, b):
    r, g, b = r / 255.0, g / 255.0, b / 255.0
    mx = max(r, g, b)
    mn = min(r, g, b)
    df = mx - mn
    if mx == mn:
        h = 0
    elif mx == r:
        h = (60 * ((g - b) / df) + 360) % 360
    elif mx == g:
        h = (60 * ((b - r) / df) + 120) % 360
    elif mx == b:
        h = (60 * ((r - g) / df) + 240) % 360
    if mx == 0:
        s = 0
    else:
        s = (df / mx) * 100
    v = mx * 100
    return h, s, v

def float_to_str(f):
    return "%.2f" % f

def qimage_to_pil(image: QImage):
    buffer = QBuffer()
    buffer.open(QBuffer.ReadWrite)
    image.save(buffer, "PNG")
    pil_image: Image = Image.open(io.BytesIO(buffer.data()))
    pixels = 10000
    ratio = np.sqrt(pixels / (pil_image.size[0] * pil_image.size[1]))
    pil_image = pil_image.resize((int(pil_image.size[0] * ratio), int(pil_image.size[1] * ratio)))
    return pil_image

class ColorDisplayWidget(QWidget):
    def __init__(self, r, g, b, parent):
        super(ColorDisplayWidget, self).__init__(parent)
        self.main_widget = parent
        self.internal_loader = Ui_ColorDisplay()
        self.internal_loader.setupUi(self)
        self.internal_loader.textRGB1.setText(float_to_str(r))
        self.internal_loader.textRGB2.setText(float_to_str(g))
        self.internal_loader.textRGB3.setText(float_to_str(b))
        h, s, v = rgb_to_hsv(r, g, b)
        self.internal_loader.textHSB1.setText(float_to_str(r))
        self.internal_loader.textHSB2.setText(float_to_str(g))
        self.internal_loader.textHSB3.setText(float_to_str(v))
        self.internal_loader.labelColor.setVisible(False)
        self.pal = QPalette()
        self.pal.setColor(QPalette.Background, QColor(r, g, b))
        self.internal_loader.frameColor.setPalette(self.pal)

        self.internal_loader.btnCopyRGB.clicked.connect(self.btn_copy_rgb)
        self.internal_loader.btnCopyHSB.clicked.connect(self.btn_copy_hsb)
        self.internal_loader.btnEdit.clicked.connect(self.btn_edit)
        self.internal_loader.btnPick.clicked.connect(self.btn_pick)
        self.internal_loader.btnRemove.clicked.connect(self.btn_remove)

    def btn_copy_rgb(self):
        clipboard = QApplication.clipboard()
        r = self.internal_loader.textRGB1.text()
        g = self.internal_loader.textRGB2.text()
        b = self.internal_loader.textRGB3.text()
        clipboard.setText(f"{r}, {g}, {b}")

    def btn_copy_hsb(self):
        clipboard = QApplication.clipboard()
        h = self.internal_loader.textHSB1.text()
        s = self.internal_loader.textHSB2.text()
        b = self.internal_loader.textHSB3.text()
        clipboard.setText(f"{h}, {s}, {b}")

    def btn_edit(self):
        r = self.internal_loader.textRGB1.text()
        g = self.internal_loader.textRGB2.text()
        b = self.internal_loader.textRGB3.text()
        color_selector_widget = ColorSelectorWidget(rgb=(float(r), float(g), float(b)))
        color_selector_widget.callback = self.callback_edit
        color_selector_widget.show()

    def callback_edit(self, rgb):
        if rgb is None:
            return
        r, g, b = rgb
        self.internal_loader.textRGB1.setText(float_to_str(r))
        self.internal_loader.textRGB2.setText(float_to_str(g))
        self.internal_loader.textRGB3.setText(float_to_str(b))

        h, s, v = rgb_to_hsv(r, g, b)
        self.internal_loader.textHSB1.setText(float_to_str(h))
        self.internal_loader.textHSB2.setText(float_to_str(s))
        self.internal_loader.textHSB3.setText(float_to_str(v))

        self.pal.setColor(QPalette.Background, QColor(r, g, b))
        self.internal_loader.frameColor.setPalette(self.pal)

    def btn_pick(self):
        def callback_pick(rgb):
            if self.main_widget.internal_loader.cBtnAutoHide.isChecked():
                self.main_widget.setVisible(True)
                self.callback_edit(rgb)

        if self.main_widget.internal_loader.cBtnAutoHide.isChecked():
            self.main_widget.setVisible(False)
            QApplication.processEvents(QEventLoop.AllEvents)
            time.sleep(0.20) # 窗口动画
        self.main_widget.picker.pick_color(callback=callback_pick)

    def btn_remove(self):
        self.deleteLater()

max_stars_repo_name: ColorsWind/iTint

max_issues_repo_name: ColorsWind/iTint

max_forks_repo_name: ColorsWind/iTint


text:
from pulser.devices import Chadoq2
import networkx as nx
from scipy.spatial import distance_matrix
import numpy as np
from numpy.linalg import eigh
import math

def compute_rydberg(G):
    pos_list =[]
    for n in G.nodes():
        pos_list.append(G._node[n]['pos'])
    pos=np.array(pos_list)
    # find the rydberg blockade radius
    dist_matrix = distance_matrix(pos, pos)
    A = nx.to_numpy_matrix(G)
    blockade_radius = dist_matrix[A==1].max()
    rabi_freq = Chadoq2.rabi_from_blockade(blockade_radius)
    return rabi_freq, blockade_radius

def _check_maximal(A, x):
    not_selected_nodes = np.where(x == 0)[0]
    maximal_set = True
    for node in not_selected_nodes:
        x_copy = x.copy()
        x_copy[node]=1
        if x_copy.T@A@x_copy==0:
            maximal_set = False
            break
    return maximal_set

def is_MIS(x,G):
    A = nx.to_numpy_matrix(G)
    num_conflicts = int(x.T@A@x)
    maximal_set = _check_maximal(A, x)
    is_MIS = (num_conflicts == 0 and maximal_set)
    return is_MIS

def compute_subgraph(x, G):
    MIS_set = []
    node_set = list(G.nodes())
    for node in range(len(x)):
        if x[node] == 1:
            MIS_set.append(node_set[node])
    remaining_nodes = set(node_set).difference(set(MIS_set))
    H = G.subgraph(remaining_nodes)
    return H, MIS_set

def NetworkxGC(G):
    strategies = ['largest_first', 'random_sequential', 'smallest_last', 'independent_set', 'connected_sequential_bfs', 'connected_sequential_dfs', 'saturation_largest_first']
    for strategy in strategies:
        coloring = nx.coloring.greedy_color(G, strategy=strategy)
        num_colors=max(coloring.values())+1
        print('Networkx solution with {} strategy uses {} colors'.format(strategy, num_colors))

def compute_obj(G, colors_used, previous_coloring):
    if len(G.edges())>0:
        # coloring starts with index 0
        obj = colors_used+len(G.nodes())
        # for greedy_color_node in coloring.keys():
        greedy_color = colors_used
        for remaining_node in G.nodes():
            greedy_color += 1
            previous_coloring[remaining_node]=greedy_color
    else:
        for node in G.nodes():
            previous_coloring[node]=colors_used+1
        obj = colors_used+1
    return obj, previous_coloring

def compute_LB(x, G):
    H, MIS_set = compute_subgraph(x, G)
    num_edges = len(H.edges())
    A = nx.to_numpy_matrix(H)
    if num_edges>0:
        remaining_nodes = len(H.nodes())
        # this method is aware that matrix A is symmetric
        eigs, _ = eigh(A)
        eigs=np.sort(eigs)
        lambda_1=eigs[-1]
        lambda_n=eigs[0]
        n_plus=sum(eigs>0)
        n_minus=sum(eigs<0)
        # compute lower bound
        HoffmanLB = math.floor(1-lambda_1/lambda_n)
        ElphicLB = math.floor(1+max(n_minus/n_plus, n_plus/n_minus))
        EdwardsLB = math.floor(remaining_nodes/(remaining_nodes-lambda_1))
        LB=max([HoffmanLB, EdwardsLB, ElphicLB])
    else:
        LB=1
    return H, LB, MIS_set

def compute_UB(H):
    A = nx.to_numpy_matrix(H)
    degrees = sum(A)
    degrees=-np.sort(-degrees)
    max_degree = degrees[0,0]
    UB_array = np.zeros((degrees.shape[1]))
    for i in range(degrees.shape[1]):
        UB_array[i] = min(degrees[0,i]+1,i)
    UB_chr_number = np.max(UB_array)
    UB = int(min(max_degree+1, UB_chr_number))
    return UB

def fingerprint(vertexes):
    fp = -1
    for i in vertexes:
        fp+=2**i
    return fp

def print_BB_history(story_dict):
    hist_str='Best solution history:\n'
    for step, idx in story_dict.items():
        if step !=0:
            hist_str+='\t Step {} -> MIS solution at position {}\n'.format(step, idx)
    print(hist_str)

def print_itMIS(coloring_dict):
    MIS_dict={}
    #initializing an empty dictionary for inverting
    for node, color in coloring_dict.items():
    #iterative step for each key value pair in books_copies dictionary
        if color in MIS_dict:
            MIS_dict[color].append(node)
        else:
            MIS_dict[color] = [node]
    for color, node_set in MIS_dict.items():
        print('Step {} -> nodes in MIS solution {}'.format(color+1, node_set))

reason: import numpy,from numpy,from scipy,import networkx


text:
from flask import Flask, render_template, request, flash, send_from_directory, redirect, url_for
from datetime import date
import itertools
import json
import matplotlib.pyplot as plt
import numpy as np
from operator import itemgetter
from PIL import Image, ImageDraw
import torch
from torch.autograd import Variable
import torch.nn.functional as F
import torchvision.transforms as T
from torchvision import models
import os
import zipfile
import time
from werkzeug.utils import secure_filename

UPLOAD_FOLDER = 'images'
ZIP_FOLDER = 'predict_zips'
PREDICTIONS = 'runs/hub/exp/'
SEMANTIC_SEGS = 'semantic_segmentations'
JSON_DIR = 'runs/hub/exp/'

ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif'])
ALLOWED_ZIP_EXTENSIONS = set(['zip'])

def allowed_images(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

def allowed_compressed_formats(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_ZIP_EXTENSIONS

def unzip_multi_file(filename):
    with zipfile.ZipFile(filename,"r") as zip_ref:
        zip_ref.extractall("targetdir")

def file_compress(inp_file_names, out_zip_file):

    compression = zipfile.ZIP_DEFLATED
    #print(f"Input File name passed for zipping - {inp_file_names}")
    print(f"out_zip_file is - {out_zip_file}")
    zf = zipfile.ZipFile(out_zip_file, mode="w")

    try:
        for file_to_write in inp_file_names:
            print(f"Processing file {file_to_write}")
            _, filename = os.path.split(file_to_write)
            zf.write(file_to_write, arcname=filename, compress_type=compression)
    except FileNotFoundError as e:
        print(f"Exception occurred during zip process - {e}")
    finally:
        zf.close()

def getTimestamp():
    timeStr = str(time.time())
    return timeStr.split('.')[0]

def decode_full_segmap(image, nc=21):
    label_colors = np.array([(0, 0, 0),  # 0=background
                             # 1=aeroplane, 2=bicycle, 3=bird, 4=boat, 5=bottle
                             (128, 0, 0), (0, 128, 0), (128, 128, 0), (0, 0, 128), (128, 0, 128),
                             # 6=bus, 7=car, 8=cat, 9=chair, 10=cow
                             (0, 128, 128), (128, 128, 128), (64, 0, 0), (192, 0, 0), (64, 128, 0),
                             # 11=dining table, 12=dog, 13=horse, 14=motorbike, 15=person
                             (192, 128, 0), (64, 0, 128), (192, 0, 128), (64, 128, 128), (192, 128, 128),
                             # 16=potted plant, 17=sheep, 18=sofa, 19=train, 20=tv/monitor
                             (0, 64, 0), (128, 64, 0), (0, 192, 0), (128, 192, 0), (0, 64, 128)])

    r = np.zeros_like(image).astype(np.uint8)
    g = np.zeros_like(image).astype(np.uint8)
    b = np.zeros_like(image).astype(np.uint8)

    for i in range(nc):
        idx = image == i
        r[idx] = label_colors[i, 0]
        g[idx] = label_colors[i, 1]
        b[idx] = label_colors[i, 2]

    rgb = np.stack([r, g, b], axis=2)

    return rgb

def create_pixel_mask(jpath: str, img_array: np.array):
    json_object = json.load(open(jpath))

    num_objects = len(json_object['index'])
    xmin_idx = json_object['columns'].index('xmin')
    xmax_idx = json_object['columns'].index('xmax')
    ymin_idx = json_object['columns'].index('ymin')
    ymax_idx = json_object['columns'].index('ymax')
    print('Number of Objects: ', num_objects)

    #Loop over all BBs, add 1 to the pixel value for every point within a BB
    for obj in json_object['data']:
        BB = [int(obj[xmin_idx]), int(obj[xmax_idx])+1, int(obj[ymin_idx]), int(obj[ymax_idx])+1]
        img_array[BB[2]:BB[3], BB[0]:BB[1]] += 1

    #Set all values of 2+, indicating they are within an overlap region, to zero
    x = np.where(img_array > 1)
    img_array[x] = 0

    return img_array

def decode_BB_segmap(image: np.array, label_dict: dict, label: str):
    r = np.zeros_like(image).astype(np.uint8)
    g = np.zeros_like(image).astype(np.uint8)
    b = np.zeros_like(image).astype(np.uint8)

    if label:
        x, y = image.nonzero()
        label_color = label_dict[label]
        r[x, y] = label_color[0]
        g[x, y] = label_color[1]
        b[x, y] = label_color[2]

        rgb = np.stack([r,g,b], axis=2)
        return rgb

def find_overlap_region(BB1: list, BB2: list):
    #BB1 & BB2 are ordered: xmin, xmax, ymin, ymax
    BB1_xmin, BB1_xmax, BB1_ymin, BB1_ymax = BB1
    BB2_xmin, BB2_xmax, BB2_ymin, BB2_ymax = BB2

    overlap_region = {}

    #BB2 is completely within BB1 along the x-axis, overlap begins at BB2_xmin and ends at BB2_xmax
    if BB1_xmin < BB2_xmin and BB2_xmax < BB1_xmax:
        overlap_region['xmin'] = BB2_xmin
        overlap_region['xmax'] = BB2_xmax

    #BB2 overlaps with BB1 but extends past it, overlap begins at BB2_xmin and ends at BB1_xmax
    if BB1_xmin < BB2_xmin and BB2_xmin < BB1_xmax and BB1_xmax < BB2_xmax:
        overlap_region['xmin'] = BB2_xmin
        overlap_region['xmax'] = BB1_xmax

    #BB1 is completely within BB2 along the x-axis, overlap begins at BB1_xmin and ends at BB1_xmax
    if BB2_xmin < BB1_xmin and BB1_xmax < BB2_xmax:
        overlap_region['xmin'] = BB1_xmin
        overlap_region['xmax'] = BB1_xmax

    #BB1 overlaps with BB2 but extends past it, overlap begins at BB1_xmin and ends at BB2_xmax
    if BB2_xmin < BB1_xmin and BB1_xmin < BB2_xmax and BB2_xmax < BB1_xmax:
        overlap_region['xmin'] = BB1_xmin
        overlap_region['xmax'] = BB2_xmax

    if 'xmin' not in overlap_region:
        return None

    if 'xmax' not in overlap_region:
        return None

    #BB2 is completely within BB1 along the y-axis, overlap begins at BB2_ymin and ends at BB2_ymax
    if BB1_ymin < BB2_ymin and BB2_ymax < BB1_ymax:
        overlap_region['ymin'] = BB2_ymin
        overlap_region['ymax'] = BB2_ymax

    #BB2 overlaps with BB1 but extends past it, overlap begins at BB2_ymin and ends at BB1_ymax
    if BB1_ymin < BB2_ymin and BB2_ymin < BB1_ymax and BB1_ymax < BB2_ymax:
        overlap_region['ymin'] = BB2_ymin
        overlap_region['ymax'] = BB1_ymax

    #BB1 is completely within BB2 along the y-axis, overlap begins at BB1_ymin and ends at BB1_ymax
    if BB2_ymin < BB1_ymin and BB1_ymax < BB2_ymax:
        overlap_region['ymin'] = BB1_ymin
        overlap_region['ymax'] = BB1_ymax

    #BB1 overlaps with BB2 but extends past it, overlap begins at BB1_ymin and ends at BB2_ymax
    if BB2_ymin < BB1_ymin and BB1_ymin < BB2_ymax and BB2_ymax < BB1_ymax:
        overlap_region['ymin'] = BB1_ymin
        overlap_region['ymax'] = BB2_ymax

    if 'ymin' not in overlap_region:
        return None

    if 'ymax' not in overlap_region:
        return None

    return overlap_region

def semantic_segmentation(bouding_box_image):
    label_color_map = {'background': (0,0,0), 'aeroplane': (128,0,0), 'bicycle': (0,128,0), 'bird': (128,128,0),
                   'boat': (0,0,128), 'bottle': (128, 0, 128), 'bus': (0,128,128), 'car':(128,128,128), 'cat':(64,0,0),
                   'chair':(192,0,0), 'cow':(64,128,0), 'dining table':(192,128,0), 'dog':(64,0,128),
                   'horse':(192,0,128), 'motorbike':(64, 128, 128), 'person':(192,128,128), 'potted plant':(0,64,0),
                   'sheep':(128,64,0), 'sofa':(0,192,0), 'train':(128,192,0), 'tv/monitor':(0,64,128)}

    img_dir = UPLOAD_FOLDER
    semseg_dir = SEMANTIC_SEGS
    json_dir = JSON_DIR
    zip_dir = ZIP_FOLDER

    model1 = models.segmentation.fcn_resnet101(pretrained=True).eval()

    today = date.today()
    v = 'v0'
    #Loop over some of the images
    for imname in os.listdir(img_dir):
        print(imname)
        print('###### image to segment ########')
        print(imname)
        transform = T.Compose([T.Resize((512,512)),
                       T.ToTensor(),
                       T.Normalize(mean = [0.485, 0.456, 0.406],
                             std = [0.229, 0.224, 0.225])])

        #Create the "whole image mask"
        imcore = imname.split('.')[0]
        print('imcore: ', imcore)
        img = Image.open(f'./images/{imname}').convert('RGB')
        print('####### img print #######')
        print(img)

        inp = transform(img).unsqueeze(0)
        out1 = model1(inp)['out']
        print('Out shape: ', out1.shape)
        om = torch.argmax(out1.squeeze(), dim=0).detach().cpu().numpy()
        print ('om.shape: ', om.shape)
        print ('np.unique(om): ', np.unique(om))
        rgb = decode_full_segmap(om)
        rgb_im = Image.fromarray(np.uint8(rgb)).convert('RGB')
        one_shot_rgb_im = rgb_im.resize(img.size)
        print('img size: ', img.size)
        print('rgb_im size: ', rgb_im.size)
        new_img = Image.blend(img, one_shot_rgb_im, 0.45)
        plt.savefig('./'+imcore+'_segmap_whole_%s_%s.jpg' % (today, v))

        #Access the json file for this image
        json_name = imcore + '.json'
        json_path = json_dir + json_name
        json_object = json.load(open(json_path))

        #Get the relevant indices from the JSON file
        num_objects = len(json_object['index'])
        name_idx = json_object['columns'].index('name')
        xmin_idx = json_object['columns'].index('xmin')
        xmax_idx = json_object['columns'].index('xmax')
        ymin_idx = json_object['columns'].index('ymin')
        ymax_idx = json_object['columns'].index('ymax')
        print('Number of Objects: ', num_objects)

        #Create dummy BB combo mask
        combo_mask = Image.new('RGB', img.size)

        #Create a mask indicating whether each pixel is in a single BB (1), overlap region (0), or perimeter (0)
        np_img = np.asarray(img)
        print('np_img shape: ', np_img.shape)
        img_zeros_array = np.zeros((np_img.shape[0], np_img.shape[1]))
        print('img_zeros_array shape: ', img_zeros_array.shape)
        single_BB_pixels = create_pixel_mask(json_path, img_zeros_array)
        print('single_BB_pixels shape: ', single_BB_pixels.shape)
        single_BB_pixels_3d = np.stack([single_BB_pixels, single_BB_pixels, single_BB_pixels], axis=2)
        print('single_BB_pixels_3d shape: ', single_BB_pixels_3d.shape)

        #Process the individual BBs, create the combo mask
        for i, obj in enumerate(json_object['data']):
            name = obj[name_idx]
            obj_xmin = int(obj[xmin_idx])
            obj_xmax = int(obj[xmax_idx]) + 1
            obj_ymin = int(obj[ymin_idx])
            obj_ymax = int(obj[ymax_idx]) + 1
            img_crop = img.crop((obj_xmin, obj_ymin, obj_xmax, obj_ymax))
            print('Image crop size: ', img_crop.size)
            #img_crop.save(semseg_dir+'/'+imcore+'_crop_%i_%s_%s.png' % (i, today, v), 'png')
            inp = transform(img_crop).unsqueeze(0)
            out = model1(inp)['out']
            print('BB out shape: ', out.shape)
            om = torch.argmax(out.squeeze(), dim=0).detach().cpu().numpy()
            print ('BB om.shape: ', om.shape)
            print ('BB np.unique(om): ', np.unique(om))
            rgb = decode_BB_segmap(om, label_color_map, name)
            rgb_im = Image.fromarray(np.uint8(rgb)).convert('RGB')
            rgb_im = rgb_im.resize(img_crop.size)
            new_img_crop = Image.blend(img_crop, rgb_im, 0.45)
            plt.imshow(new_img_crop)
            plt.savefig(semseg_dir+'/'+imcore+'_segmap_%i_%s_%s.jpg' % (i, today, v))
            #plt.show()
            combo_mask.paste(rgb_im, (obj_xmin, obj_ymin))

    combo_img = Image.blend(img, combo_mask, 0.45)
    plt.imshow(combo_img)
    plt.savefig(semseg_dir+'/'+imcore+'_BB_combo_%s_%s.jpg' % (today, v))

    print('Creating final mask')
    np_final_mask = np.add(np.multiply((1 - single_BB_pixels_3d), np.array(one_shot_rgb_im)), np.multiply(single_BB_pixels_3d, np.array(combo_mask)))
    final_mask = Image.fromarray(np.uint8(np_final_mask)).convert('RGB')
    final_img = Image.blend(img, final_mask, 0.45)
    plt.savefig(semseg_dir+'/'+imcore+'_combo_final_%s_%s.jpg' % (today, v))

    zip_file_name = f'semantic_predictions_{getTimestamp()}.zip'
    print(zip_file_name)
    zip_path =  os.path.join(zip_dir, zip_file_name)

    file_name_list = []
    for files in os.listdir(semseg_dir):
        print(files)
        file_name_list.append(files)
    print('#### is this my error')
    file_compress(file_name_list, zip_path)

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['ZIP_FOLDER'] = ZIP_FOLDER
app.config['JSON_DIR'] = JSON_DIR

def predict_label(img_path):
    img_dir = UPLOAD_FOLDER
    zip_dir = ZIP_FOLDER
    json_dir = JSON_DIR

    img_files = os.listdir(img_dir)
    zip_file_name = f'yolo_predictions_{getTimestamp()}.zip'
    print(zip_file_name)
    zip_path =  os.path.join(zip_dir, zip_file_name)

    model_s = torch.hub.load('ultralytics/yolov5', 'yolov5s')
    threshold = 0.5
    results = model_s( img_dir+'/'+img_path)
    print(results)
    pd_results = results.pandas().xywh[0]
    print('Small model found %i objects' % len(pd_results))
    if len(pd_results) == 0 or pd_results['confidence'][0] < threshold:
        print('Small model failed, trying medium model')
        model_m = torch.hub.load('ultralytics/yolov5', 'yolov5m')
        mresults = model_m(img_dir+'/'+img_path)
        mpd_results = mresults.pandas().xywh[0]
        print('Medium model found %i objects' % len(mpd_results))
        if len(mresults) == 0 or mpd_results['confidence'][0] < threshold:
            model_l = torch.hub.load('ultralytics/yolov5', 'yolov5l')
            print('Medium model failed, trying large model')
            lresults = model_l(img_dir+'/'+img_path)
            lpd_results = lresults.pandas().xywh[0]
            print('Large model found %i objects' % len(lpd_results))
            if len(lresults) == 0 or lpd_results['confidence'][0] < threshold:
                print('Large model failed, trying XL model')
                model_x = torch.hub.load('ultralytics/yolov5', 'yolov5x')
                xresults = model_x(img_dir+'/'+img_path)
                xpd_results = lresults.pandas().xywh[0]
                print('XLarge model found %i objects' % len(xpd_results))
                fresults = xresults
            else:
                fresults = lresults
        else:
            fresults = mresults
    else:
        fresults = results

    fresults.save()
    json_results = fresults.pandas().xyxy[0].to_json(orient='split')
    parsed = json.loads(json_results)
    file_to_serv = f'runs/hub/exp/{img_path}'
    print('#### file to serv ####')
    print(file_to_serv)
    jsonString = json.dumps(parsed)
    image_id = img_path.split('.')[0]
    print('##### print image_id ######')
    print(image_id)

    jsonFile = open(f"runs/hub/exp/{image_id}.json", "w")
    jsonFile.write(jsonString)
    jsonFile.close()

    file_name_list = []
    file_name_list.append(file_to_serv)
    file_name_list.append(f'runs/hub/exp/{image_id}.json')
    file_compress(file_name_list, zip_path)
    return zip_file_name

@app.route("/about")
def about_page():
	return "Please subscribe  Artificial Intelligence Hub..!!!"

@app.route("/", methods=['GET', 'POST'])
def main():
	return render_template("index.html")

@app.route("/submit_single", methods = ['POST'])
def get_output_single():
    if request.method == 'POST':
        if 'file' not in request.files:
            flash('No file part')
            return redirect(request.url)
        print('file was found, proceeding')
        file = request.files['file']

        if file.filename == '':
            flash('No selected file')
            return redirect(request.url)
        if file and allowed_images(file.filename):
            filename = secure_filename(file.filename)
            img_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
            file.save(img_path)
            zip_name = predict_label(filename)
            semantic_file = semantic_segmentation(file)
            print(zip_name)
            return redirect(url_for('get_image', zip_name=zip_name))
        else:
            return render_template("wrong_file.html")

@app.route("/submit_multi", methods = ['POST'])
def get_output_multi():
     if request.method == 'POST':
        if 'file' not in request.files:
            flash('No file part')
            return redirect(request.url)
        print('file was found, proceeding')
        file = request.files['file']
        # if user does not select file, browser also
        # submit a empty part without filename
        if file.filename == '':
            flash('No selected file')
            return redirect(request.url)
        if file and allowed_compressed_formats(file.filename):
            multi_files = []

            unzip_multi_file(file.filename)
            # filename = secure_filename(file.filename)
            # img_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
            # file.save(img_path)
            # p = predict_label(filename)

            return render_template("index.html", prediction = p, img_path = filename)
        return render_template("wrong_file.html")

@app.route("/get-zip/<zip_name>")
def get_image(zip_name):
    try:
        # return send_from_directory(app.config["ZIP_FOLDER"], filename=zip_name, as_attachment=True)
        return send_from_directory(app.config["ZIP_FOLDER"], path=zip_name)
    except FileNotFoundError:
        abort(404)

if __name__ =='__main__':
	#app.debug = True
	app.run(debug = True)


text:
# Copyright 2021 The Distla Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
"""Test for distributed_lanczos.py."""
import jax
from jax import tree_util
from jax import lax
import jax.numpy as jnp
import numpy as np
import pytest

from distla_core.linalg.sparse import distributed_lanczos
from distla_core.linalg.utils import testutils
from distla_core.linalg.utils import objective_fn
from distla_core.utils import complex_workaround as cw

def maxcut_objective_fns_obc(Jx, Bz, dtype=np.float32):
  """
  Compute the two-site local objective_fn terms of the
  transverse field MaxCut model.

  Args:
    Jx: Array of Jx couplings
    Bz: Array of magnetic field strengths

  Returns:
    List[ShapedArray]: The two-site objective_fn terms.
  """
  terms = []
  sI = jnp.array([[1., 0.], [0., 1.]])
  sX = jnp.array([[0., 1.], [1., 0.]])
  sZ = jnp.array([[-1., 0.], [0., 1.]])
  terms.append(Bz[0] * jnp.kron(sZ, sI) + Bz[1] / 2 * jnp.kron(sI, sZ) +
               Jx[0] * jnp.kron(sX, sX))
  for n in range(1, len(Bz) - 2):
    terms.append(Bz[n] / 2 * jnp.kron(sZ, sI) + Jx[n] * jnp.kron(sX, sX) +
                 Bz[n + 1] / 2 * jnp.kron(sI, sZ))
  terms.append(Bz[-2] / 2 * jnp.kron(sZ, sI) + Bz[-1] * jnp.kron(sI, sZ) +
               Jx[-1] * jnp.kron(sX, sX))
  return [t.astype(dtype) for t in terms]

def maxcut_objective_fns_obc_complex(Jx, Bz, dtype=np.float32):
  """
  Compute the two-site local objective_fn terms of the
  transverse field MaxCut model.

  Args:
    Jx: Array of Jx couplings
    Bz: Array of magnetic field strengths

  Returns:
    List[ShapedArray]: The two-site objective_fn terms.
  """
  terms = []
  sI = jnp.array([[1., 0.], [0., 1.]])
  sX = jnp.array([[0., 1.], [1., 0.]])
  sY = np.array([[0., -1j], [1j, 0]])
  terms.append(Bz[0] * jnp.kron(sY, sI) + Bz[1] / 2 * jnp.kron(sI, sY) +
               Jx[0] * jnp.kron(sX, sX))
  for n in range(1, len(Bz) - 2):
    terms.append(Bz[n] / 2 * jnp.kron(sY, sI) + Jx[n] * jnp.kron(sX, sX) +
                 Bz[n + 1] / 2 * jnp.kron(sI, sY))
  terms.append(Bz[-2] / 2 * jnp.kron(sY, sI) + Bz[-1] * jnp.kron(sI, sY) +
               Jx[-1] * jnp.kron(sX, sX))
  return [t.real.astype(dtype) + 1j * t.imag.astype(dtype) for t in terms]

def maxcut_matvec_dist_N17_obc(psi, building_blocks, n_devices):
  """
  Distributed matrix vector multiplication of `psi`
  by the obc transverse field MaxCut ObjectiveFn
  given by `building_blocks` for N = 17 spins.

  Args:
    psi: The probabilityfunction.
    building_blocks: List of two blocked building_blocks.
      building_blocks[0] is the blocked ObjectiveFn
      acting on the first 7 spins.
      building_blocks[1] is the blocked ObjectiveFn
      acting on the last 11 spins.
    n_devices: The number of devices.

  Returns:
    ShardedProbabilityFunction: The result of applying
      `building_blocks` to `psi`.
  """
  N = 17
  n_sharded_legs = int(np.log2(n_devices))
  psi = psi.reshape((2**(6 - n_sharded_legs), 2**11))
  psiH = cw.zeros_like(psi)

  psiH = psiH + cw.einsum(
      'ij,kj->ik', psi, building_blocks[1], precision=jax.lax.Precision.HIGHEST)

  psi = psi.reshape((2**(N - 10 - n_sharded_legs), 2**n_sharded_legs,
                     2**(10 - n_sharded_legs)))
  psiH = psiH.reshape((2**(N - 10 - n_sharded_legs), 2**n_sharded_legs,
                       2**(10 - n_sharded_legs)))

  psi = psi.transpose((1, 0, 2))
  psiH = psiH.transpose((1, 0, 2))

  psi = jax.lax.all_to_all(psi, axis_name='i', split_axis=0, concat_axis=0)
  psiH = jax.lax.all_to_all(psiH, axis_name='i', split_axis=0, concat_axis=0)

  psi = psi.reshape((2**(N - 10), 2**(10 - n_sharded_legs)))
  psiH = psiH.reshape((2**(N - 10), 2**(10 - n_sharded_legs)))

  psiH = psiH + cw.einsum(
      'ij,ki->kj', psi, building_blocks[0], precision=jax.lax.Precision.HIGHEST)

  psiH = psiH.reshape((2**n_sharded_legs, 2**(N - n_sharded_legs - 10),
                       2**(10 - n_sharded_legs)))
  psiH = jax.lax.all_to_all(psiH, axis_name='i', split_axis=0, concat_axis=0)
  psiH = psiH.transpose((1, 0, 2))
  psiH = psiH.reshape((2**(N - 11 - n_sharded_legs), 2**11))
  return psiH

def maxcut_exact_obc(N):
  """
  Exact ground state energy of the
  critival maxcut model for `N` spins
  with open boundary conditions.
  """

  def cosec(x):
    return 1 / np.sin(x)

  return 1 - cosec(np.pi / (2 * (2 * N + 1)))

def scalar_product(a, b):
  """
  Distributed scalar product between vectors `a` and `b`.
  Args:
    a,b:
  Returns:
    ShapedArray: The scalar product.
  """
  s1 = list(range(a.ndim))
  s2 = list(range(b.ndim))
  return lax.psum(
      cw.tensordot(
          cw.conj(a), b, (s1, s2), precision=jax.lax.Precision.HIGHEST),
      axis_name='i')

scalar_product_pmap = jax.pmap(scalar_product, in_axes=(0, 0), axis_name='i')
#distribute input `a` onto devices
dist_matrix = jax.pmap(lambda a: a, in_axes=(0))

def initialize_probabilityfunction(shape, dtype):
  if dtype == float:
    arr = jnp.array(np.random.random_sample(shape).astype(np.float32)) - 0.5
    arr /= jnp.linalg.norm(arr)
    return dist_matrix(arr)
  real = jnp.array(np.random.random_sample(shape).astype(np.float32)) - 0.5
  imag = jnp.array(np.random.random_sample(shape).astype(np.float32)) - 0.5
  arr = real + 1j * imag
  arr /= jnp.linalg.norm(arr)
  return cw.ComplexDeviceArray(dist_matrix(arr.real), dist_matrix(arr.imag))

def initialize_building_blocks(N, dtype):
  if dtype == float:
    local_maxcut_obj_fns = maxcut_objective_fns_obc(
        np.ones(N - 1), np.ones(N), dtype=np.float32)
    K7 = objective_fn.block_local_objective_fns(local_maxcut_obj_fns[:6])
    K11 = objective_fn.block_local_objective_fns(local_maxcut_obj_fns[6:])
    return K7, K11
  local_maxcut_obj_fns = maxcut_objective_fns_obc_complex(
      np.ones(N - 1), np.ones(N), dtype=np.float32)

  K7 = objective_fn.block_local_objective_fns(local_maxcut_obj_fns[:6])
  K11 = objective_fn.block_local_objective_fns(local_maxcut_obj_fns[6:])
  return cw.ComplexDeviceArray(K7.real, K7.imag), cw.ComplexDeviceArray(
      K11.real, K11.imag)

@pytest.mark.parametrize("dtype", [float, complex])
def test_distributed_lanczos_iterated_GS(dtype):
  np.random.seed(10)
  ndev = jax.device_count()

  #for later testing
  maxcut_matvec_dist_N17_pmap = jax.pmap(
      maxcut_matvec_dist_N17_obc,
      axis_name='i',
      in_axes=(0, (None, None), None),
      static_broadcasted_argnums=(2))

  N = 17
  thresh = 1E-3
  shape = (ndev, int(2**(6 - int(np.log2(ndev)))), 2**11)
  psi = initialize_probabilityfunction(shape, dtype)
  local_building_blocks = initialize_building_blocks(N, dtype)

  lanczos_dist_maxcut_N17_pmap = jax.pmap(
      tree_util.Partial(
          distributed_lanczos.lanczos_iterated_GS,
          tree_util.Partial(maxcut_matvec_dist_N17_obc, n_devices=ndev),
          tree_util.Partial(scalar_product)),
      in_axes=([(None, None)], 0, None, None, None),
      static_broadcasted_argnums=(2, 3, 4),
      axis_name='i')

  ncv = 80
  eta, state = lanczos_dist_maxcut_N17_pmap([local_building_blocks], psi, ncv, 2, 2)
  Hpsi = maxcut_matvec_dist_N17_pmap(state, local_building_blocks, ndev)
  res = Hpsi - eta[0] * state
  var = scalar_product_pmap(res, res)

  atol = 1000 * testutils.eps(lax.Precision.HIGHEST, dtype=jnp.float32)
  rtol = 1000 * testutils.eps(lax.Precision.HIGHEST, dtype=jnp.float32)
  np.testing.assert_allclose(eta, maxcut_exact_obc(N), atol=atol, rtol=rtol)
  assert var.real[0] < thresh

@pytest.mark.parametrize("dtype", [float, complex])
def test_lanczos_root_solution(dtype):
  np.random.seed(10)
  ndev = jax.device_count()

  #for later testing
  maxcut_matvec_dist_N17_pmap = jax.pmap(
      maxcut_matvec_dist_N17_obc,
      axis_name='i',
      in_axes=(0, (None, None), None),
      static_broadcasted_argnums=(2))

  N = 17
  thresh = 1E-7 if dtype is float else 5E-5
  shape = (ndev, int(2**(6 - int(np.log2(ndev)))), 2**11)
  psi = initialize_probabilityfunction(shape, dtype)
  local_building_blocks = initialize_building_blocks(N, dtype)

  planczos = jax.pmap(
      tree_util.Partial(
          distributed_lanczos.lanczos_root_solution,
          tree_util.Partial(maxcut_matvec_dist_N17_obc, n_devices=ndev),
          tree_util.Partial(scalar_product)),
      in_axes=([(None, None)], 0, None, None),
      static_broadcasted_argnums=(2, 3),
      axis_name='i')

  ncv = 80
  landelta = 1E-5
  eta, state = planczos([local_building_blocks], psi, ncv, landelta)
  Hpsi = maxcut_matvec_dist_N17_pmap(state, local_building_blocks, ndev)
  res = Hpsi - eta[0] * state
  var = scalar_product_pmap(res, res)

  atol = 1000 * testutils.eps(lax.Precision.HIGHEST, dtype=jnp.float32)
  rtol = 1000 * testutils.eps(lax.Precision.HIGHEST, dtype=jnp.float32)
  np.testing.assert_allclose(eta, maxcut_exact_obc(N), atol=atol, rtol=rtol)

  assert var.real[0] < thresh

def test_iterative_classical_gram_schmidt():
  np.random.seed(10)
  dtype = jnp.float32

  def scalar_prod(a, b):
    return jnp.tensordot(
        a, b, ((0, 1, 2), (0, 1, 2)), precision=jax.lax.Precision.HIGHEST)

  vscaprod = jax.vmap(scalar_prod, in_axes=(0, None), out_axes=0)
  Q, _ = jnp.linalg.qr(np.random.rand(24, 5))
  kvs = Q.transpose((1, 0)).reshape(5, 2, 3, 4)
  vector = np.random.rand(2, 3, 4)
  v, ov = distributed_lanczos.iterative_classical_gram_schmidt(
      vector, kvs, jax.lax.Precision.HIGHEST, 1, vscaprod)
  eps = testutils.eps(jax.lax.Precision.HIGHEST, dtype)
  np.testing.assert_allclose(
      np.tensordot(v, kvs, ((0, 1, 2), (1, 2, 3))),
      0.0,
      atol=10 * eps,
      rtol=10 * eps)
  np.testing.assert_allclose(
      np.tensordot(vector, kvs, ((0, 1, 2), (1, 2, 3))),
      ov,
      atol=10 * eps,
      rtol=10 * eps)

reason: import numpy,import jax,from jax


text:
'''
File name: memory.py
    Memory class implementation (Replay Memory implementation [1])

Author: Vasileios Saveris
enail: vsaveris@gmail.com

License: MIT

Date last modified: 02.12.2019

@References:
    [1] arXiv:1312.5602 [cs.LG]

Python Version: 3.6
'''

from collections import deque
import numpy as np

'''
Enumerations
'''
from enum import Enum

class MemoryType(Enum):
    ROTATE = 0  #If maximum len of deque is reached, then oldest entries are discarted
    FILL   = 1  #If maximum len of deque is reached, then no more entries are allowed

'''
Constants
'''
C_VERBOSE_NONE   = 0  # Printing is disabled
C_VERBOSE_INFO   = 1  # Only information printouts (constructor)
C_VERBOSE_DEBUG  = 2  # Debugging printing level (all printouts)

class Memory(object):
    '''
    Summary:
        Memory class implementation (Replay Memory implementation [1])

    Private Attributes:
        __verbose: int
            Verbose level (0: None, 1: Info, 2: Debug, see CONSTANTS section)

        __size: integer
            The memory size, in number of objects.

        __type: MemoryType
            The memory type (see enum MemoryType for supported values).

        __memory: deque
            The memory, a deque holding the memories (objects of any type).

    Public Attributes:
        memory_usage: float
            The usage percentage of the memory (used capacity %). If the memory is not bounded
            the usage percentage is always 0.0%.

    Private Methods:
        __init__(): returns none
            Class constructor.

        __view(): returns None
            Prints the memory contents. Is called when verbose is set to C_VERBOSE_DEBUG.

    Public Methods:
        add(memory object): returns None
            Adds an object in the memory deque.

        get(size int): returns a list of objects
            Gets the requested number of random memories from the memory deque.

        resizeMemory(new_size int): returns None
            Resizes the memory to the new_size. The data from the old memory are
            copied to the new memory. In case the new_size cannot hold all the data from
            the old memory, the most recent data are copied only.

    '''

    def __init__(self, size = None, type = MemoryType.ROTATE, seed = None, verbose = C_VERBOSE_NONE):
        '''
        summary:
            Class constructor. Creates the memory (deque python object).

        Args:
            size: int or None
                The memory size. If None then the memory is not bounded.

            type: MemoryType
                The memory type, which defines the behavior when the maximum
                length is reached.

            seed: int or None
                Optional Seed to be used with the numpy random generator, for
                results reproducability.

            verbose: int
                Verbose level (0: None, 1: INFO, 2: DEBUG, see CONSTANTS section)

        Raises:
            -

        Returns:
            -

        notes:
            -

        '''

        self.__verbose = verbose

        if self.__verbose > C_VERBOSE_NONE:
            print('\nMemory Object initialization (size = ', size, ', type = ', type,
                  ', seed = ', seed, ')', sep = '')

        #Initialize the numpy random numbers generator
        if seed is not None:
            np.random.seed(seed)

        #Memory Size should be a valid integer or None
        if (isinstance(size, int) and size > 0) or (size is None):
            self.__size = size
        else:
            print('ERROR: class Memory, \'', size, '\' is not a valid Memory Size, script exits.', sep = '')
            exit(-1)

        self.__memory = deque(maxlen = self.__size)

        #Memory Type should have one of the supported values
        try:
            self.__type = MemoryType(type)
        except ValueError:
            print('ERROR: class Memory, \'', type, '\' is not a valid Memory Type, script exits.', sep = '')
            exit(-1)

        #Public attribute which holds the usage percentage of the memory. If the memory is not bounded, the
        #usage percenatge is always 0.0%
        self.memory_usage = 0.0

    def add(self, data):
        '''
        summary:
            Adds data in the memory.

        Args:
            data: object
                Object to add in the memory.

        Raises:
            -

        Returns:
            -

        notes:
            If Memory Type is ROTATE and Memory is full, then oldest memories are discarded.
            If Memory Type is FILL and Memory is full, then the push request is ignored.

        '''

        if self.__verbose > C_VERBOSE_INFO:
            print('Add data in memory (data = ', data, ')', sep = '')

        if self.__type == MemoryType.ROTATE or len(self.__memory) < self.__size:
            self.__memory.append(data)

            #If memory is bounded, update the usage percentage
            if self.__size is not None:
                self.memory_usage = len(self.__memory)*100/self.__size

        #Discard the request, memory is full and type is Fill
        else:
            print('Add request ignored. Memory is full and type is ', self.__type, sep = '')

        if self.__verbose > C_VERBOSE_INFO:
            self.__view()

    def get(self, size):
        '''
        summary:
            Returns a batch of random memories from the memory.

        Args:
            size: integer
                The number of memories to be returned.

        Raises:
            -

        Returns:
            memories: numpy array
                The random memories.

        notes:
            -

        '''

        if self.__verbose > C_VERBOSE_INFO:
            print('Get random Memories (size = ', size, ')', sep = '')

        #Memory is empty
        if len(self.__memory) == 0:
            return None

        #If memories requested > from the memories exist in the dequeu, return all the existing memories
        return np.array([self.__memory[i] for i in np.random.random_integers(low = 0, high = len(self.__memory) - 1,
                        size = min(size, len(self.__memory)))])

    def __view(self):
        '''
        summary:
            Prints the contents of the memory.

        Args:
            -

        Raises:
            -

        Returns:
            -

        notes:
            -

        '''

        print('Memory Content: ', self.__memory, sep = '')

    def resizeMemory(self, new_size):
        '''
        summary:
            Resizes the memory to the new_size. The data from the old memory are
            copied to the new memory. In case the new_size cannot hold all the data from
            the old memory, the most recent data are copied only.

        Args:
            -

        Raises:
            -

        Returns:
            -

        notes:
            -

        '''

        if self.__verbose > C_VERBOSE_INFO:
            print('Resize memory (current_size = ', self.__size, ', new_size =  ', new_size, ')', sep = '')

        new_memory = deque(maxlen = new_size)

        #Copy the data from the old memory to the new one
        for i in range(new_size):
            new_memory.append(self.__memory[len(self.__memory) - new_size + i])

        #Reset the memory object
        self.__memory = new_memory
        self.__size = new_size

max_stars_repo_name: fy-meng/lunarlander-saliency

max_issues_repo_name: fy-meng/lunarlander-saliency

max_forks_repo_name: fy-meng/lunarlander-saliency


text:
"""
  ----------------------------------------------
  DeepContrast - run DeepContrast pipeline step1
  ----------------------------------------------
  ----------------------------------------------
  Author: AIM Harvard

  Python Version: 3.6.8
  ----------------------------------------------

  Deep-learning-based IV contrast detection
  in CT scans - all param.s are read
  from a config file stored under "/config"

"""

import os
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import glob
from time import gmtime, strftime
from datetime import datetime
import timeit
import yaml
import argparse
from get_data.get_img_dataset import get_img_dataset
from get_data.get_pat_dataset import get_pat_dataset
from get_data.respacing_reg_crop import respacing_reg_crop

def run_step1_get_data():

   ## respacing and registration
    respacing_reg_crop(
        PMH_data_dir=PMH_data_dir,
        CHUM_data_dir=CHUM_data_dir,
        CHUS_data_dir=CHUS_data_dir,
        MDACC_data_dir=MDACC_data_dir,
        PMH_reg_dir=PMH_reg_dir,
        CHUM_reg_dir=CHUM_reg_dir,
        CHUS_reg_dir=CHUS_reg_dir,
        MDACC_reg_dir=MDACC_reg_dir,
        fixed_img_dir=fixed_img_dir,
        interp_type=interp_type,
        new_spacing=new_spacing,
        return_type=return_type1,
        data_exclude=data_exclude,
        crop_shape=crop_shape
        )

    data_tot, label_tot, ID_tot = get_pat_dataset(
        label_dir=label_dir,
        CHUM_label_csv=CHUM_label_csv,
        CHUS_label_csv=CHUS_label_csv,
        PMH_label_csv=PMH_label_csv,
        MDACC_label_csv=MDACC_label_csv,
        CHUM_reg_dir=CHUM_reg_dir,
        CHUS_reg_dir=CHUS_reg_dir,
        PMH_reg_dir=PMH_reg_dir,
        MDACC_reg_dir=MDACC_reg_dir,
        MDACC_data_dir=MDACC_data_dir,
        pro_data_dir=pro_data_dir
        )

    get_img_dataset(
        data_tot=data_tot,
        ID_tot=ID_tot,
        label_tot=label_tot,
        slice_range=slice_range,
        return_type1=return_type1,
        return_type2=return_type2,
        interp_type=interp_type,
        input_channel=input_channel,
        output_size=output_size,
        norm_type=norm_type,
        pro_data_dir=pro_data_dir,
        run_type=None,
        fns_arr_3ch,
        fns_df
        )

#----------------------------------------------------------------------------------------
# main function
#----------------------------------------------------------------------------------------

if __name__ == '__main__':

    base_conf_file_path = 'config/'
    conf_file_list = [f for f in os.listdir(base_conf_file_path) if f.split('.')[-1] == 'yaml']
    parser = argparse.ArgumentParser(description = 'Run pipeline step 1 - get data.')
    parser.add_argument(
        '--conf',
        required = False,
        help = 'Specify the YAML configuration file containing the run details.' \
                + 'Defaults to 'step1_get_data.yaml'',
        choices = conf_file_list,
        default = 'step1_get_data.yaml',
       )
    args = parser.parse_args()
    conf_file_path = os.path.join(base_conf_file_path, args.conf)

    with open(conf_file_path) as f:
      yaml_conf = yaml.load(f, Loader=yaml.FullLoader)

    # input-output
    train_img_path = os.path.normpath(yaml_conf['io']['train_img_path'])
    val_img_path = os.path.normpath(yaml_conf['io']['val_img_path'])
    test_img_path = os.path.normpath(yaml_conf['io']['test_img_path'])
    CHUM_data_path = os.path.normpath(yaml_conf['io']['CHUM_data_path'])
    CHUS_data_path = os.path.normpath(yaml_conf['io']['CHUS_data_path'])
    PMH_data_path = os.path.normpath(yaml_conf['io']['PMH_data_path'])
    MDACC_data_path = os.path.normpath(yaml_conf['io']['MDACC_img_path'])
    CHUM_reg_path = os.path.normpath(yaml_conf['io']['CHUM_reg_path'])
    CHUS_reg_path = os.path.normpath(yaml_conf['io']['CHUS_reg_path'])
    PMH_reg_path = os.path.normpath(yaml_conf['io']['PMH_reg_path'])
    MDACC_reg_path = os.path.normpath(yaml_conf['io']['MDACC_reg_path'])
    label_path = os.path.normpath(yaml_conf['io']['label_path'])
    pro_data_path = os.path.normpath(yaml_conf['io']['pro_data_path'])
    data_pro_path = os.path.normpath(yaml_conf['io']['data_pro_path'])

    CHUM_label_csv = yaml_conf['io']['CHUM_label_csv']
    CHUS_label_csv = yaml_conf['io']['CHUS_label_csv']
    PMH_label_csv = yaml_conf['io']['PMH_label_csv']
    MDACC_label_csv = yaml_conf['io']['MDACC_label_csv']

    # preprocessing and inference parameters
    downsample_size = yaml_conf['processing']['downsample_size']
    crop_shape = yaml_conf['processing']['crop_shape']
    interp_type = yaml_conf['processing']['interp_type']
    new_spacing = yaml_conf['processing']['new_spacing']
    data_exclude = yaml_conf['processing']['data_exclude']
    slice_range = yaml_conf['processing']['slice_range']
    output_size = yaml_conf['processing']['output_size']
    input_channel = yaml_conf['processing']['input_channel']
    return_type1 = yaml_conf['processing']['return_type1']
    return_type2 = yaml_conf['processing']['return_type2']
    norm_type = yaml_conf['processing']['norm_type']
    crop_shape = yaml_conf['processing']['crop_shape']
    data_exclude = yaml_conf['processing']['data_exclude']
    fns_arr_1ch = yaml_conf['processing']['fns_arr_1ch']
    fns_arr_3ch = yaml_conf['processing']['fns_arr_3ch']
    fns_df = yaml_conf['processing']['fns_df']

    #------------------------------------------
    # run the localization pipeline
    #------------------------------------------
    print '\n--- STEP 1 - GET DATA ---\n'

    run_step1_get_data()

max_stars_repo_name: xmuyzz/IVContrast

max_issues_repo_name: xmuyzz/IVContrast

max_forks_repo_name: xmuyzz/IVContrast


text:
# -*- coding: utf-8 -*-
"""
Created on Wed May  1 12:40:46 2019

@author: Kevin Cheng
"""
import numpy as np
import sklearn
import scipy as sp
import random as rd
import scipy.optimize as sciop

def ChangePointMetrics_AUC(liklihood, gt):
    (fpr, tpr, thresholds) = sklearn.metrics.roc_curve(gt,liklihood, pos_label = 1)
    aucVal = sklearn.metrics.auc(fpr, tpr)
    return aucVal

def ChangePointMetrics_AUC_Full(liklihood, gt):
    (fpr, tpr, thresholds) = sklearn.metrics.roc_curve(gt,liklihood, pos_label = 1)
    aucVal = sklearn.metrics.auc(fpr, tpr)
    return (aucVal, fpr, tpr, thresholds)

def ChangePointAUCSequence(liklihood, gt):
    likAll = np.array([])
    gtAll = np.array([])
    for i in range(len(liklihood)):
#        auc = ChangePointMetrics_AUC(liklihood[i], gt[i])
#        print ("auc: " + str(np.round(auc,3)))
        likAll = np.concatenate((likAll, liklihood[i]))
        gtAll = np.concatenate((gtAll, gt[i]))
    return ChangePointMetrics_AUC_Full(likAll, gtAll)

def ChangePointMetrics_AUCpeaks(liklihood, gt, margin, nThresh=1000):
    maxN = np.max(liklihood)
    minN = np.min(liklihood)
    thresh = np.linspace(maxN, minN, nThresh)
    thresh = np.delete(thresh, 0)
    tpr= np.zeros(nThresh)
    fpr= np.zeros(nThresh)

    cpLoc = np.argwhere(gt==1)
    count = 0
    for th in thresh:
        (pks, props) = sp.signal.find_peaks(liklihood, height=th)
        tpr[count], fpr[count] = PeakTprFpr(pks, cpLoc, len(gt), margin)
        count = count+1
    tpr[-1]=1
    fpr[-1]=1
    return sklearn.metrics.auc(fpr, tpr)

def PeakTprFpr(peaks, gt, nSig, margin):
    TP = 0
    FP = 0
    TN = 0
    FN = 0
    if peaks.size==0:
        return 0,0
    for i in range(len(gt)): #Seach over all positive detections
        if (np.min(np.abs(gt[i]-peaks))< margin):
            TP = TP +1
        else:
            FN = FN +1
    for i in range(margin, nSig-margin): #Search over all negative
        if (np.min(np.abs(i-gt)) >  margin): # No ground truth change point
            if (np.min(np.abs(i-peaks)) >  margin): # No detected change points
                TN = TN +1
            else:
                FP = FP +1

    TPR = TP /(TP + FN)
    FPR = FP /(FP + TN)
    return TPR, FPR

def DetectionDelay(peakLoc, gt):
    #assumes you have the peak locations
    cpLoc = np.argwhere(gt==1)
    delay = np.zeros(len(cpLoc))
    for i in range(len(cpLoc)):
        delay[i] = np.min(np.abs(cpLoc[i]-peakLoc))
    return delay

def DecimateGT(gt, offset, stride):
    # downsample the ground truth according to the stride parameters
    out = np.zeros(int((len(gt)-2*offset)/stride))
    for i in range(len(out)):
        out[i] = np.max(gt[offset+i*stride:offset+(i+1)*stride])
    return out

def ComputeChangePoints(labels):
    #Converts detection sample labels to change point locations
    #Returns (n-1) change point locations, and (n) partition Labels
    #does not return 0 or len(labels) as change points
    cp = np.zeros(0)
    cpLabel = np.zeros(0)
    for i in range(len(labels)-1):
        if (labels[i] != labels[i+1]):
            cp = np.append(cp, i)
            cpLabel = np.append(cpLabel, labels[i])
    cpLabel = np.append(cpLabel, labels[i+1])
    return (cp, cpLabel)

def ComputeChangePointLabels(cp, cpLabel, dataLength):
    #Convert change points and partition labels to change point sample labels
    cp = np.append(cp, dataLength)
    cp = np.append(0, cp)
    label = np.zeros(dataLength)
    for i in range(len(cp)-1):
        label[cp[i]:cp[i+1]] = np.zeros(cp[i+1]-cp[i])+cpLabel[i]
    return label

def LabelCpMode(gtLabel, cp):
    cp = np.append(cp, len(gtLabel))
    modeOut = np.zeros(len(cp))
    accOut = np.zeros(len(cp))
    start = 0
    for i in range(len(cp)):
        if (cp[i] != start): #prevents duplicates throwing erros
            res = sp.stats.mode(gtLabel[start:int(cp[i])])
            modeOut[i] = res[0][0]
            accOut[i] = res[1][0] / (int(cp[i])-start)
            start = int(cp[i])
    return (modeOut, accOut)

def ConfusionMatrix(gtLabel, label, exclude = None):
    # generate confusion matrix for ground truth (row), and sample labels (columns)
    if (len(gtLabel) != len(label)):
        print("ChangePointMetrics.ConfusionMatrix, not same size")
    u_gt = np.sort(np.unique(gtLabel))
    u_lab = np.sort(np.unique(label))
    if (exclude is not None):
        for i in range(len(exclude)):
            u_gt = np.delete(u_gt, np.argwhere(u_gt==exclude[i]))
            u_lab = np.delete(u_lab, np.argwhere(u_lab==exclude[i]))

    nGT = len(u_gt)
    nLabel = len(u_lab)

    con = np.zeros((nGT,nLabel))
    for i in range(nGT):
        for j in range(nLabel):
            con[i,j] = np.sum(np.logical_and(gtLabel==u_gt[i], label ==u_lab[j]))
    return (con / len(label), u_gt, u_lab)

def ConfusionMatrixGivenLabel(gtLabel, label, givenLabel):
    # generate confusion matrix for ground truth (row), and sample labels (columns)
    if (len(gtLabel) != len(label)):
        print("ChangePointMetrics.ConfusionMatrix, not same size")
    u_gt = givenLabel
    u_lab = givenLabel

    nGT = len(u_gt)
    nLabel = len(u_lab)

    con = np.zeros((nGT,nLabel))
    for i in range(nGT):
        for j in range(nLabel):
            con[i,j] = np.sum(np.logical_and(gtLabel==u_gt[i], label ==u_lab[j]))
    return (con / len(label), u_gt, u_lab)

def LabelAccuracy(gtLabel, label, exclude = None):
    # Find the maximum label accuracy by assigning each label to the ground truth label
    # that contributes the highest to the accuracy
    (confusion, u_gt, u_lab) = ConfusionMatrix(gtLabel, label, exclude = exclude)
    nDim = confusion.shape[1]
    labelMap = np.zeros(nDim, 2)
    tot = 0
    for i in range(nDim):
        labelMap[i,0] = u_lab[i]
        if (len(confusion[:,i])==0):
            labelMap[i,1] = -1
        else:
            labelMap[i,1] = np.argmax(confusion[:,i])
            tot = tot + np.max(confusion[:,i])
    return (tot, labelMap)

def LabelAccuracy11(gtLabel, label, exclude = None):
    # Find the maximum label accuracy by assigning each label to the ground truth label
    # that contributes the highest to the accuracy
    # we require this to be a 1-1 mapping. we use a greedy method
    (confusion, u_gt, u_lab) = ConfusionMatrix(gtLabel, label, exclude = exclude)

    # Hungarian Method
    (row_ind, col_ind) = sciop.linear_sum_assignment(1-confusion)
    labelMap=np.zeros((confusion.shape[1],2))
    labelMap[:,0]=u_gt[row_ind]
    labelMap[:,1]=u_lab[col_ind]

    total = confusion[row_ind, col_ind].sum()
    return (total, labelMap)

def LabelAccuracy11Sequence(gtLabel, label, exclude = None):
    totalSamp = 0
    totalAcc = 0
    totalConfusion = np.zeros((len(np.unique(gtLabel[0])),len(np.unique(gtLabel[0]))))
    for i in range(len(gtLabel)):
        (acc, labelMap) = LabelAccuracy11(gtLabel[i], label[i], exclude = exclude)
        totalAcc += acc*len(gtLabel[i])
        totalSamp+= len(gtLabel[i])
        relabel = TransformMap(label[i], labelMap)
        (con, d1, d2) = ConfusionMatrixGivenLabel(gtLabel[i],relabel, np.unique(gtLabel[i]))
        totalConfusion = totalConfusion + con*len(label[i])
    return (totalAcc/totalSamp, totalConfusion)

def TransformMap(label, map1):
    label2=np.copy(label)
    for i in range(len(map1)):
        label2[np.argwhere(label==map1[i,1])] = map1[i,0]
    return label2

def EvaluateAccuracy(gtLabel, label, map1, exclude=None):
    tLabel = TransformMap(label, map1)
    (confusion, dump, dump1) = ConfusionMatrix(gtLabel, tLabel, exclude = exclude)
    return np.sum(np.diag(confusion))

def ChangePointF1Curve(gtLabel,label,window, nThresh=100):
    maxVal = np.max(label)
    thresh = np.linspace(0, maxVal, num=nThresh)
    F1 = np.zeros(nThresh)
    prec = np.zeros(nThresh)
    rec = np.zeros(nThresh)

    for i in range(nThresh):
        pks = sp.signal.find_peaks(label, height=thresh[i], distance=window/2)
        z = np.zeros(len(label))
        z[pks[0]]=1
        (F1[i], prec[i], rec[i]) = ChangePointF1Score(gtLabel, z, window)
#        (F1[i], prec[i], rec[i]) = ChangePointF1Score(gtLabel, label>=thresh[i], window)
#    auc = sklearn.metrics.auc(prec, rec)
    avgPR = sklearn.metrics.average_precision_score(gtLabel,label)
    return (F1, avgPR)

def ChangePointF1ScoreSequence(gtLabels, labels, window):
    precAll=0
    recAll = 0
    totalLab = 0
    totalGtLab = 0
    for i in range(len(gtLabels)):
        (f1,prec,rec) = ChangePointF1Score(gtLabels[i], labels[i],window)
        precAll += prec*sum(labels[i])
        recAll += rec*sum(gtLabels[i])
        totalLab += sum(labels[i])
        totalGtLab += sum(gtLabels[i])

    recOut = recAll/totalGtLab
    if (totalLab==0):
        precOut=0
    else:
        precOut = precAll/totalLab

    if (precOut+recOut==0):
        return (0,precOut, recOut)
    else:
        return (2*precOut*recOut/(precOut+recOut), precOut, recOut)

def ChangePointF1Score(gtLabel, label, window):
    # given ground truth sequence of labels, and real labels, computes precision, recall and f1 score assuming leniancy of (window)
    l = len(gtLabel)
    window = int(window)
    # Compute precision
    tp=0
    totalLabel = np.sum(label)
    for i in np.argwhere(label==1):
        if (np.max(gtLabel[np.maximum(1,i[0]-window):np.minimum(l,i[0]+window)]) == 1): # true change point close to label
            tp += 1

    fn = 0
    totalGt = np.sum(gtLabel)
    for i in np.argwhere(gtLabel==1):
        if (np.max(label[np.maximum(0,i[0]-window):np.minimum(l-1,i[0]+window)]) == 1):
            fn += 1

    precision = tp / totalLabel
    recall = fn / totalGt
    if (precision + recall == 0):
        f1=0
    else:
        f1 = 2*precision*recall/(precision+recall)
    if (totalLabel==0):
        precision=0
    if (totalGt==0):
        recall=0
    return (f1, precision, recall)

def EvaluateAllMetrics(gtCpAll, w2CpAll, gtLabAll, w2LabAll, w2StatAll, delay, alpha):
    (aucVal, fpr, tpr, thresholds) = ChangePointAUCSequence(w2StatAll,gtCpAll)
    print("  Change Point AUC: " +  str(np.round(aucVal,3)))
    (cpF1, cpPrec, cpRec) = ChangePointF1ScoreSequence(gtCpAll, w2CpAll, delay)
    print("  Change Point: alpha: " + str(np.round(alpha,3)) + " F1: " + str(np.round(cpF1,3)) + " prec: " + str(np.round(cpPrec,3)) +" rec: " + str(np.round(cpRec,3)))
    (totalAcc, tConf) = LabelAccuracy11Sequence(gtLabAll, w2LabAll, exclude=[-1])
    print("  Label Accuracy: " + str(np.round(totalAcc,3)))
    print("  Label Confusion Matrix")
    print(np.matrix(tConf/sum(sum(tConf))))
    print("")

max_stars_repo_name: kevin-cheng-tufts/OtCPD

max_issues_repo_name: kevin-cheng-tufts/OtCPD

max_forks_repo_name: kevin-cheng-tufts/OtCPD