text:
#
#  file:  store_analysis.py
#
#  Interpret the store simulation results
#
#  RTK, 07-Jan-2020
#  Last update:  20-Sep-2020
#
################################################################

import os
import sys
import numpy as np
import matplotlib.pylab as plt
from matplotlib.ticker import MaxNLocator

def GetResults(fname):
    """Parse an output file"""

    lines = [i[:-1] for i in open(fname)]
    revenue = float(lines[1].split()[3][1:])
    lower_prob = float(lines[-3].split()[-1])
    lower_value= float(lines[-2].split()[-1])
    upper_prob = float(lines[-5].split()[-1])
    upper_value= float(lines[-4].split()[-1])
    milk_rank = float(lines[-8].split()[-1])
    candy_rank = float(lines[-7].split()[-1])
    return lower_prob, lower_value, upper_prob, upper_value, milk_rank, candy_rank, revenue

def PlotRanking(s,title):
    """Plot milk and candy ranking"""

    t = np.zeros(len(s))
    for i in range(len(s)):
        t[i] = s[i][4]
    milk = t.copy()
    plt.plot(range(len(t)),t, color='r', marker='o', label='Milk')
    for i in range(len(s)):
        t[i] = s[i][5]
    candy = t.copy()
    plt.plot(range(len(t)),t, color='b', marker='s', label='Candy')
    plt.xticks(np.arange(0, 20, 1.0))
    plt.xlabel('Run')
    plt.ylabel('Ranking')
    plt.title(title)
    plt.tight_layout(pad=0, w_pad=0, h_pad=0)
    plt.savefig("exp_%s_ranking.png" % title, dpi=300)
    plt.show()
    return [milk, candy]

def PlotRevenue(pso,de,ro,gwo,jaya,ga):
    """Plot the daily revenue"""

    def revenue(s):
        t = np.zeros(len(s))
        for i in range(len(s)):
            t[i] = s[i][-1]
        return t

    ax0 = plt.subplot(611)
    plt.plot(revenue(pso), marker='.')
    plt.ylabel("PSO")
    plt.setp(ax0.get_xticklabels(), visible=False)
    ax1 = plt.subplot(612, sharex=ax0)
    plt.plot(revenue(de), marker='.')
    plt.ylabel("DE")
    plt.setp(ax1.get_xticklabels(), visible=False)
    ax2 = plt.subplot(613, sharex=ax0)
    plt.plot(revenue(ga), marker='.')
    plt.ylabel("GA")
    plt.setp(ax2.get_xticklabels(), visible=False)
    ax3 = plt.subplot(614, sharex=ax0)
    plt.plot(revenue(gwo), marker='.')
    plt.ylabel("GWO")
    plt.setp(ax3.get_xticklabels(), visible=False)
    ax4 = plt.subplot(615, sharex=ax0)
    plt.plot(revenue(jaya), marker='.')
    plt.ylabel("Jaya")
    plt.setp(ax4.get_xticklabels(), visible=False)
    ax5 = plt.subplot(616, sharex=ax0)
    plt.plot(revenue(ro), marker='.')
    plt.xticks(np.arange(0, 20, 1.0))
    plt.ylabel("RO")
    plt.xlabel("Run")
    plt.tight_layout(pad=0, w_pad=0, h_pad=0)
    plt.savefig("exp_revenue.png", dpi=300)
    plt.show()

    return [
        np.array(revenue(pso)), np.array(revenue(de)),   np.array(revenue(ro)),
        np.array(revenue(gwo)), np.array(revenue(jaya)), np.array(revenue(ga))
    ]

def main():
    """Make the plots"""

    N = len(os.listdir("pso_results"))

    pso = []
    for i in range(N):
        pso.append(GetResults("pso_results/pso_run%d" % i))
    de = []
    for i in range(N):
        de.append(GetResults("de_results/de_run%d" % i))
    ro = []
    for i in range(N):
        ro.append(GetResults("ro_results/ro_run%d" % i))
    gwo = []
    for i in range(N):
        gwo.append(GetResults("gwo_results/gwo_run%d" % i))
    jaya = []
    for i in range(N):
        jaya.append(GetResults("jaya_results/jaya_run%d" % i))
    ga = []
    for i in range(N):
        ga.append(GetResults("ga_results/ga_run%d" % i))

    #  Plot milk and candy rankings
    m0,c0 = PlotRanking(pso, "PSO")
    m1,c1 = PlotRanking(de, "DE")
    m2,c2 = PlotRanking(ro, "RO")
    m3,c3 = PlotRanking(gwo, "GWO")
    m4,c4 = PlotRanking(jaya, "Jaya")
    m5,c5 = PlotRanking(ga, "GA")

    #  Milk and candy stats
    n = np.sqrt(len(m0))
    print()
    print("PSO :  milk: %0.2f +/- %0.2f (%0.0f, %0.0f),  candy: %0.2f +/- %0.2f (%0.0f, %0.0f)" % \
            (m0.mean(),m0.std(ddof=1)/n,m0.max(),m0.min(),c0.mean(),c0.std(ddof=1)/n,c0.max(),c0.min()))
    print("DE  :  milk: %0.2f +/- %0.2f (%0.0f, %0.0f),  candy: %0.2f +/- %0.2f (%0.0f, %0.0f)" % \
            (m1.mean(),m1.std(ddof=1)/n,m1.max(),m1.min(),c1.mean(),c1.std(ddof=1)/n,c1.max(),c1.min()))
    print("RO  :  milk: %0.2f +/- %0.2f (%0.0f, %0.0f),  candy: %0.2f +/- %0.2f (%0.0f, %0.0f)" % \
            (m2.mean(),m2.std(ddof=1)/n,m2.max(),m2.min(),c2.mean(),c2.std(ddof=1)/n,c2.max(),c2.min()))
    print("GWO :  milk: %0.2f +/- %0.2f (%0.0f, %0.0f),  candy: %0.2f +/- %0.2f (%0.0f, %0.0f)" % \
            (m3.mean(),m3.std(ddof=1)/n,m3.max(),m3.min(),c3.mean(),c3.std(ddof=1)/n,c3.max(),c3.min()))
    print("Jaya:  milk: %0.2f +/- %0.2f (%0.0f, %0.0f),  candy: %0.2f +/- %0.2f (%0.0f, %0.0f)" % \
            (m4.mean(),m4.std(ddof=1)/n,m4.max(),m4.min(),c4.mean(),c4.std(ddof=1)/n,c4.max(),c4.min()))
    print("GA  :  milk: %0.2f +/- %0.2f (%0.0f, %0.0f),  candy: %0.2f +/- %0.2f (%0.0f, %0.0f)" % \
            (m5.mean(),m5.std(ddof=1)/n,m5.max(),m5.min(),c5.mean(),c5.std(ddof=1)/n,c5.max(),c5.min()))

    #  Plot revenue
    rev = PlotRevenue(pso,de,ro,gwo,jaya,ga)

    #  Revenue stats
    print()
    print("PSO : $%0.2f +/- %0.2f" % (rev[0].mean(), rev[0].std(ddof=1)/n))
    print("DE  : $%0.2f +/- %0.2f" % (rev[1].mean(), rev[1].std(ddof=1)/n))
    print("RO  : $%0.2f +/- %0.2f" % (rev[2].mean(), rev[2].std(ddof=1)/n))
    print("GWO : $%0.2f +/- %0.2f" % (rev[3].mean(), rev[3].std(ddof=1)/n))
    print("Jaya: $%0.2f +/- %0.2f" % (rev[4].mean(), rev[4].std(ddof=1)/n))
    print("GA  : $%0.2f +/- %0.2f" % (rev[5].mean(), rev[5].std(ddof=1)/n))
    print()

if (__name__ == "__main__"):
    main()


text:
from numpy import array, convolve, ones
from thunder.images import fromtif

def load(path, nplanes=None, engine=None):
    data = fromtif(path, nplanes=nplanes, engine=engine)
    return data

def split(oim, nrois, nlines):
    """
    Splits a single two dimensional plane.
    """
    if not oim.ndim == 2:
      raise Exception('original image must be two dimensional')

    if nrois > 1:
        ntotal = oim.shape[0]
        ngap = (ntotal - nlines*nrois)/(nrois-1)
        return array([oim[int(nlines*i + ngap*i): int(nlines*(i + 1) + ngap*i), :] for i in range(nrois)])
    else:
        return oim

def merge(oim, order):
    """
    Merges a three dimensional volume into a single plane.
    """
    if not type(order) == bool:
        if not oim.ndim == 3:
          raise Exception('original image must be three dimensional')
        if not len(order) == oim.shape[0]:
          raise Exception('length of order must match first image dimension')
        return oim[order].transpose((1, 0, 2)).reshape([oim.shape[1], -1])
    else:
        return oim

def reshape(oim, nrois, nlines, order):
    """
    Split and merge a two dimensional plane or three dimensional volume.
    """
    if oim.ndim == 3:
        return array(map(lambda x : merge(split(x, nrois, nlines), order), oim))
    else:
        return merge(split(oim, nrois, nlines), order)

def normalize(oim):
    """
    Normalize a three dimensional volume.
    """
    if not oim.ndim == 3:
        raise Exception('image must be three dimensional')
    means = oim.mean(axis=(1, 2), dtype='float32')
    maximum = means.max()
    return array([oim[i]*maximum/means[i] for i in range(oim.shape[0])])

def smooth(data, dt):
    data = convolve(data, ones(dt)/dt, 'same').astype('int16')
    return data[::dt]

ext: py

max_stars_repo_name: sofroniewn/Mesoscope

max_issues_repo_name: sofroniewn/Mesoscope

max_forks_repo_name: sofroniewn/Mesoscope

reason: from numpy


text:
'''
main class for bayesian network penalized regression

Tian Gao, 3/6/2020
'''

import numpy as np
from scipy.linalg import expm
import utils
import networkx as nx
from utils import setup_logger
from tqdm import tqdm

from scipy import linalg

import pandas as pd
import scipy.linalg as slin
import scipy.optimize as sopt
from scipy.special import expit as sigmoid
import networkx as nx

class BPR:
    def __init__(self, args):
        self.args = args
        self.rho_max = args.rho_A_max # augmented Langagragian coefficent
        self.h_tol = args.h_tol
        self.lambda1 = args.lambda1
        self.lambda2 = args.lambda2
        self.train_epochs = args.train_epochs
        self.threshold_A = args.graph_threshold
        self.loss_type = 'l2'

    def fit(self, X, method = 'nocurl'):

        if method == 'notear':
            # --original version of notear
            return self.fit_all(X)

        elif method == 'nocurl':
            return self.fit_all_L2proj(X)

        elif method =='CAM':
            return self.fit_cam(X)

        elif method =='GES':
            return self.fit_ges(X)

        elif method == 'MMPC':
            return self.fit_mmpc(X)

        elif method =='FGS':
            return self.fit_fgs(X)

        else:
            print('method is not support')

    def fit_fgs(self, X):
        '''FGS version'''

        '''FGS version'''
        from fges_continuous_yyu import fit_FGS

        d = X.shape[1]
        trueG = nx.to_numpy_array(self.ground_truth_G)
        A = fit_FGS(X, trueG, d, self.pc)

        lossA = self._loss_L2(A, X, loss_type = 'l2')

        return A, [-1], [], [lossA[0]]

    def fit_cam(self, X):
        import cdt
        model = cdt.causality.graph.CAM(score='nonlinear', cutoff=0.001,
                                        variablesel=True, selmethod='gamboost',
                                        pruning=True, prunmethod='gam',
                                        njobs=None, verbose=None)  # causal additive model: Guasisan process + additive noise
        # model = cdt.causality.graph.LiNGAM()  # Linear Non-Gaussian Acyclic model + addtive noise
        #

        data_frame = pd.DataFrame(X)
        output_graph_nc = model.predict(data_frame)
        A = nx.adjacency_matrix(output_graph_nc).todense()

        A = np.asarray(A).astype(np.float64)

        lossA = self._loss_L2(A, X, loss_type='l2')

        return A, [-1], [],[lossA[0]]

    def fit_mmpc(self, X):
        import cdt
        model = cdt.causality.graph.bnlearn.MMPC()

        data_frame = pd.DataFrame(X)
        output_graph_nc = model.predict(data_frame)
        A = nx.adjacency_matrix(output_graph_nc).todense()
        A = np.asarray(A).astype(np.float64)

        lossA = self._loss_L2(A, X, loss_type='l2')

        return A, [-1], [], [lossA[0]]

    def fit_ges(self, X):
        import cdt
        model = cdt.causality.graph.GES()
        data_frame = pd.DataFrame(X)
        output_graph_nc = model.predict(data_frame)
        A = nx.adjacency_matrix(output_graph_nc).todense()
        A = np.asarray(A).astype(np.float64)

        lossA = self._loss_L2(A, X, loss_type='l2')

        return A, [-1], [], [lossA[0]]

    def fit_aug_lagr_AL2proj(self, X, hTol, lambda1, lambda2, threshold):

        def relu(x, derivative=False, alpha=0.1):
            rel = x * (x > 0)
            if derivative:
                return (x > 0)*1
            return rel

        def _h(w):
            W = w.reshape([d, d])
            M = np.eye(d) + W * W / d  # (Yu et al. 2019)
            E = np.linalg.matrix_power(M, d - 1)
            h = (E.T * M).sum() - d
            G_h = E.T * W * 2
            return h, G_h

        def _loss(W):

            return 0.5 / n * np.square(np.linalg.norm(X.dot(np.eye(d, d) - W), 'fro'))

        def _loss_comp(W):

            return 0.5 / n * np.square(np.linalg.norm(X.dot(np.eye(d, d) - W), axis=0))

        def _prefunc(w):
            W = w.reshape([d, d])
            loss = 0.5 / n * np.square(np.linalg.norm(X.dot(np.eye(d, d) - W), 'fro'))
            h,_ = _h(W)
            return loss + 0.5 * rho * h * h + alpha * h

        def _pregrad(w):
            W = w.reshape([d, d])
            loss_grad = - 1.0 / n * X.T.dot(X).dot(np.eye(d, d) - W)
            h, Gh = _h(W)
            obj_grad = loss_grad + (rho * h + alpha)*Gh
            return obj_grad.flatten()

        def build_W(w, d,totalcount):
        # build w
            w1 = np.zeros([d, d])
            lower_index = np.tril_indices(d, -1)
            w1[lower_index] = w[:totalcount]

            return w1 + w1.T - np.diag(w1.diagonal())

        def build_phi(w, totalcount):
            phi = w[totalcount:].reshape(-1,1)
            return phi

        def _func(w, choice = 'relu'):
            totalcount = int(d*(d-1)/2)
            loss = 0

        # build parameters
            w1 = build_W(w, d, totalcount)
            phi = build_phi(w, totalcount)

        # compute loss
        # loss2 = 0
            ones = np.ones_like(phi)
            W = np.multiply(w1,  relu(np.matmul(ones, phi.T) - np.matmul(phi, ones.T), False))
            loss += _loss(W)

            return loss

        def _grad(w, choice='relu'):
            totalcount = int(d * (d - 1) / 2)
            w1 = build_W(w, d, totalcount) # np.zeros([d,d])
            phi = build_phi(w, totalcount)
            ones = np.ones_like(phi)
            W_all = np.multiply(w1,  relu(np.matmul(ones, phi.T) - np.matmul(phi, ones.T), False))

            loss_grad = np.zeros(totalcount+d)
            W_grad = - 1.0 / n * X.T.dot(X).dot(np.eye(d, d) - W_all)

        # loss to W
            dF_dW_2= np.multiply(W_grad, relu(np.matmul(ones, phi.T) - np.matmul(phi, ones.T), False))
            dF_dW = dF_dW_2 + dF_dW_2.T

            loss_grad[:totalcount] = dF_dW[np.tril_indices(d, -1)]

        # loss to phi
            dF_dPhi_2= np.multiply( np.multiply(W_grad, w1),  relu(np.matmul(ones, phi.T) - np.matmul(phi, ones.T), True))

            loss_grad[totalcount:] = np.sum(dF_dPhi_2 - dF_dPhi_2.T, axis=0)

            obj_grad = loss_grad #+ (rho * (np.trace(E) - d) + alpha) * E.T * W * 2
            return obj_grad#.flatten()

        def _func1(w):
            loss = _func(w, choice='relu')
            return loss

        def _grad1(w):
            obj_grad = _grad(w, choice='relu')
            return obj_grad

        def _func_w(w_input, *args):
        #         W = w[0:d*d].reshape([d, d])

            choice = args[0] # 'relu'
            totalcount = int(d * (d - 1) / 2)
            loss = 0

        # build parameters
            phi = args[1]
            w = np.zeros(totalcount + d)
            w[:totalcount] = w_input
            w[totalcount:totalcount+d-1] = phi
            w1 = build_W(w, d, totalcount)
            phi = build_phi(w, totalcount)

        # compute loss
        # loss2 = 0
            ones = np.ones_like(phi)
            W = np.multiply(w1, relu(np.matmul(ones, phi.T) - np.matmul(phi, ones.T), False))

            return  _loss(W)

        def build_w_inv(A, phi, d,totalcount):
        # build w
            w1 = np.zeros([d, d])
            for i in range(d):
                for j in range(d):
                    if ((relu(phi[j]-phi[i])>1e-8)):
                        w1[i,j] = A[i,j]/relu(phi[j]-phi[i])
                    else:
                        w1[i,j] = 0
            w = (w1 + w1.T)/2.
            wnew = np.zeros(totalcount)
            lower_index = np.tril_indices(d, -1)
            wnew[:totalcount] = w[lower_index]

            return wnew

        def _grad_w(w_input, *args):
            #        W = w[0:d*d].reshape([d, d])
            choice = args[0]
            phi = args[1]

            totalcount = int(d * (d - 1) / 2)

            w = np.zeros(totalcount + d)
            w[:totalcount] = w_input
            w[totalcount:totalcount+d-1] = phi

            w1 = build_W(w, d, totalcount)  # np.zeros([d,d])
            phi = build_phi(w, totalcount)
            ones = np.ones_like(phi)
            W_all = np.multiply(w1, relu(np.matmul(ones, phi.T) - np.matmul(phi, ones.T), False))

            loss_grad = np.zeros(totalcount)
            W_grad = - 1.0 / n * X.T.dot(X).dot(np.eye(d, d) - W_all)

        # loss to W
            dF_dW_2 = np.multiply(W_grad, relu(np.matmul(ones, phi.T) - np.matmul(phi, ones.T), False))
            dF_dW = dF_dW_2 + dF_dW_2.T
            loss_grad[:totalcount] = dF_dW[np.tril_indices(d, -1)]
            obj_grad = loss_grad
            return obj_grad  # .flatten()

        n, d = X.shape
        totalcount = int(d * (d - 1) / 2)
        w_est, w_best = np.zeros(totalcount + d), np.zeros(totalcount + d)
        bnds = [(None, None) for i in range(totalcount + d)]
        bnds[totalcount + d - 1] = (0, 0)
        bndsw = [(None, None) for i in range(totalcount)] #bounds for w in g(phi)
        rho = 0.0
        alpha = lambda1
        prew = np.zeros(d * d)
        prebnds = [(0, 0) if i == j else (None, None) for i in range(d) for j in range(d)]

        presol = sopt.minimize(_prefunc, prew, method='L-BFGS-B', jac=_pregrad, bounds=prebnds, options={'ftol': hTol})
        prew = np.copy(presol.x)

        rho = 0.0
        alpha = lambda2

        presol = sopt.minimize(_prefunc, prew, method='L-BFGS-B', jac=_pregrad, bounds=prebnds, options={'ftol': hTol})
        prew = np.copy(presol.x)

        prew[np.abs(prew) < 0.3] = 0

        preX = np.sign(np.abs(prew.reshape([d, d])))
        preX = np.sign(linalg.expm(preX)-np.identity(d))
        asymX = preX - preX.transpose()

        preM = np.ones([d-1,d-1])
        for i in range(d-1):
            preM[i,i] = -(d-1)
        preb = np.sum(asymX, axis=1)

        prephi = np.linalg.solve(preM, preb[0:d-1])

        w_est = np.zeros(totalcount + d)
        w_est[totalcount:totalcount + d - 1] = prephi
        w_phi = np.zeros(totalcount)

###############################
#standard 2-step version
        solw1 = sopt.minimize(_func_w, w_phi, args=('relu', prephi), method='L-BFGS-B', jac=_grad_w, bounds=bndsw, options={'ftol': hTol})
        w_est[0:totalcount] = solw1.x

#full version
#        w_est[0:totalcount + d - 1] = np.random.rand(totalcount + d - 1)
#        sol2 = sopt.minimize(_func1, w_est, method='L-BFGS-B', jac=_grad1, bounds=bnds, options={'ftol': hTol})
#        w_est = np.copy(sol2.x)
################################

        w1 = build_W(w_est, d, totalcount)
        phi = build_phi(w_est, totalcount)

        ones = np.ones_like(phi)
        W = np.multiply(w1, relu(np.matmul(ones, phi.T) - np.matmul(phi, ones.T), False))

        lossW = _loss(W)

#    W[np.abs(W) < threshold] = 0

        return W, lossW

    def fit_all_L2proj(self, X):
        Wstar, lossW = self.fit_aug_lagr_AL2proj(X,
                                    hTol=self.h_tol,
                                    lambda1=self.lambda1,
                                    lambda2=self.lambda2,
                                    threshold=self.threshold_A)

        # compare how sparse it is
        W_original = Wstar.copy()
        W_original[np.abs(W_original) < 1e-5] = 0
        num_non_zero  = np.sum( np.abs(W_original)>0)

        Wstar[np.abs(Wstar) < self.threshold_A] = 0

        return Wstar, [0.], [num_non_zero],  [lossW]

    def fit_all(self,X):
        '''original notear'''

        def _h(w):
            W = w.reshape([d, d])
            M = np.eye(d) + W * W / d  # (Yu et al. 2019)
            E = np.linalg.matrix_power(M, d - 1)
            h = (E.T * M).sum() - d
            return h # np.trace(slin.expm(W * W)) - d

        def _func(w):
            W = w.reshape([d, d])
            loss = 0.5 / n * np.square(np.linalg.norm(X.dot(np.eye(d, d) - W), 'fro'))
            h = _h(W)
            return loss + 0.5 * rho * h * h + alpha * h

        def _loss(w):
            W = w.reshape([d, d])
            loss = 0.5 / n * np.square(np.linalg.norm(X.dot(np.eye(d, d) - W), 'fro'))
            return loss

        def _grad(w):
            W = w.reshape([d, d])
            loss_grad = - 1.0 / n * X.T.dot(X).dot(np.eye(d, d) - W)
            M = np.eye(d) + W * W / d
            E = np.linalg.matrix_power(M, d - 1)
            obj_grad = loss_grad + (rho * _h(w) + alpha) * E.T * W * 2
            return obj_grad.flatten()

        T = int(self.train_epochs)
        n, d = X.shape
        w_est, w_new = np.zeros(d * d), np.zeros(d * d)
        rho, alpha, h, h_new = 1.0, 0.0, np.inf, np.inf
        alpha_iter = np.zeros(T + 1)
        rho_iter = np.zeros(T + 1)
        bnds = [(0, 0) if i == j else (None, None) for i in range(d) for j in range(d)]
        for t in range(T):
            while rho < self.rho_max:
                sol = sopt.minimize(_func, w_est, method='L-BFGS-B', jac=_grad, bounds=bnds)
                w_new = sol.x
                h_new = _h(w_new)
                if h_new > 0.25 * h:
                    rho *= 10
                else:
                    break
            w_est, h = w_new, h_new
            alpha += rho * h
            alpha_iter[t + 1] = alpha
            rho_iter[t + 1] = rho

            if h <= self.h_tol:
                break

        lossA = _loss(w_est)

        w_est[np.abs(w_est) < self.threshold_A] = 0
        A = w_est.reshape([d, d])
        # return
#        lossA = self._loss_L2(w_est, X, loss_type='l2')

        return A, [h], alpha_iter, [lossA]

def sigmoid(x, derivative=False):
    D=10.
    sigm = 1. / (1. + np.exp(-D*(x)))
    if derivative:
        return D*sigm * (1. - sigm)
    return sigm

def softrelu(x, derivative=False, alpha=0.1):
    D=10.
    rel = np.log((1. + np.exp((D*(x+0.)))))/D #x * (x > 0)
    if derivative:
        return 1./ (1. + np.exp(-(D*(x+0.))))#(x > 0)*1
    return rel

def elu(x, derivative=False, alpha=0.1):
    rel = x * (x > 0) + alpha*(np.exp(x)-1) * (x<0)
    if derivative:
        return (x > 0)*1 + alpha*np.exp(x) * (x<0)
    return rel

def relu(x, derivative=False, alpha=0.1):
    rel = x * (x > 0)
    if derivative:
        return (x > 0)*1
    return rel

def activation (x, derivative = False, choice='relu'):
    if choice == 'sigmoid':
        return sigmoid(x, derivative)
    elif choice == 'softrelu':
        return softrelu(x, derivative)
    elif choice == 'relu':
        return relu(x, derivative)
    elif choice == 'relubar':
        return relu(x, derivative)
    elif choice == 'elu':
        return elu(x, derivative)

reason: import numpy,import scipy,from scipy,import networkx


text:
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================

import numpy as np

from tensorflow.keras import layers
from tensorflow.compiler.plugin.poplar.tests import test_utils as tu
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import variables
from tensorflow.python.platform import googletest
from tensorflow.python.training import gradient_descent
from tensorflow.python.ipu import ipu_compiler
from tensorflow.python.ipu import ipu_outfeed_queue
from tensorflow.python.ipu import pipelining_ops
from tensorflow.python.ipu.config import IPUConfig
from tensorflow.compat.v1 import disable_v2_behavior

disable_v2_behavior()

# Various graph constructor helpers
def fc(x, num_units_out):
  return layers.Dense(num_units_out,
                      kernel_initializer=init_ops.constant_initializer(0.1),
                      bias_initializer=init_ops.constant_initializer(0.0))(x)

class PipeliningRecomputationConvClassifyTest(test_util.TensorFlowTestCase):
  @test_util.deprecated_graph_mode_only
  def testTwoMatMuls(self):
    # Check that we get all classifications for a simple conv
    cfg = IPUConfig()
    cfg.ipu_model.compile_ipu_code = False
    cfg.auto_select_ipus = 4
    cfg.allow_recompute = True
    cfg._profiling.enable_ipu_events = True  # pylint: disable=protected-access
    cfg.configure_ipu_system()

    def stage1(x, label):
      with variable_scope.variable_scope("stage1", use_resource=True):
        x = fc(x, 16)
        x = nn.relu(x)
        x = fc(x, 48)
        x = nn.relu(x)
        return x, label

    def stage2(x, label):
      with variable_scope.variable_scope("stage2", use_resource=True):
        x = fc(x, 48)
        x = nn.relu(x)
        x = fc(x, 100)
        x = nn.relu(x)
        return x, label

    def stage3(x, label):
      with variable_scope.variable_scope("stage3", use_resource=True):
        loss = math_ops.reduce_mean(
            nn.sparse_softmax_cross_entropy_with_logits(logits=x,
                                                        labels=label))
        return loss

    def optimizer_function(loss):
      opt = gradient_descent.GradientDescentOptimizer(0.01)
      return pipelining_ops.OptimizerFunctionOutput(opt, loss)

    outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue()

    # Run the pipeline twice.
    def model_pipeline(x, lr):
      return pipelining_ops.pipeline([stage1, stage2, stage3],
                                     12,
                                     inputs=[x, lr],
                                     outfeed_queue=outfeed_queue,
                                     optimizer_function=optimizer_function)

    with ops.device('cpu'):
      x = array_ops.placeholder(np.float32, shape=[1, 224])
      l = array_ops.placeholder(np.int32, shape=[1])

    with tu.ipu_session() as sess:

      with ops.device("/device:IPU:0"):
        compiled_model_pipeline = ipu_compiler.compile(model_pipeline,
                                                       inputs=[x, l])
      outfeed_queue.dequeue()
      tu.move_variable_initialization_to_cpu()

      report_json = tu.ReportJSON(self, sess)
      sess.run(variables.global_variables_initializer())
      report_json.reset()
      sess.run(compiled_model_pipeline, {x: np.ones(x.shape), l: [1]})
      report_json.parse_log()

      # 2x matmul in 2 stages = 4x fwd x recomputation, 3x grads, 4x updates
      self.assertAllEqual(report_json.get_ml_type_counts(), [0, 8, 3, 4])

  @test_util.deprecated_graph_mode_only
  def testTwoParallelMatMuls(self):
    # Check that we get all classifications for a simple conv
    cfg = IPUConfig()
    cfg.ipu_model.compile_ipu_code = False
    cfg.auto_select_ipus = 4
    cfg.allow_recompute = True
    cfg._profiling.enable_ipu_events = True  # pylint: disable=protected-access
    cfg.configure_ipu_system()

    def stage1(x, label):
      with variable_scope.variable_scope("stage1", use_resource=True):
        a = fc(x, 48)
        a = nn.relu(a)
        b = fc(x, 48)
        b = nn.relu(b)
        return a + b, label

    def stage2(x, label):
      with variable_scope.variable_scope("stage2", use_resource=True):
        a = fc(x, 100)
        a = nn.relu(a)
        b = fc(x, 100)
        b = nn.relu(b)
        return a + b, label

    def stage3(x, label):
      with variable_scope.variable_scope("stage3", use_resource=True):
        loss = math_ops.reduce_mean(
            nn.sparse_softmax_cross_entropy_with_logits(logits=x,
                                                        labels=label))
        return loss

    def optimizer_function(loss):
      opt = gradient_descent.GradientDescentOptimizer(0.01)
      return pipelining_ops.OptimizerFunctionOutput(opt, loss)

    outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue()

    # Run the pipeline twice.
    def model_pipeline(x, lr):
      return pipelining_ops.pipeline([stage1, stage2, stage3],
                                     12,
                                     inputs=[x, lr],
                                     outfeed_queue=outfeed_queue,
                                     optimizer_function=optimizer_function)

    with ops.device('cpu'):
      x = array_ops.placeholder(np.float32, shape=[1, 224])
      l = array_ops.placeholder(np.int32, shape=[1])

    with tu.ipu_session() as sess:

      with ops.device("/device:IPU:0"):
        compiled_model_pipeline = ipu_compiler.compile(model_pipeline,
                                                       inputs=[x, l])
      outfeed_queue.dequeue()
      tu.move_variable_initialization_to_cpu()

      report_json = tu.ReportJSON(self, sess)
      sess.run(variables.global_variables_initializer())
      report_json.reset()
      sess.run(compiled_model_pipeline, {x: np.ones(x.shape), l: [1]})
      report_json.parse_log()

      # 2x matmul in 2 stages = 4x fwd x recomputation, 2x grads, 4x updates
      self.assertAllEqual(report_json.get_ml_type_counts(), [0, 8, 2, 4])

if __name__ == "__main__":
  googletest.main()

max_stars_repo_name: pierricklee/tensorflow

max_issues_repo_name: pierricklee/tensorflow

max_forks_repo_name: pierricklee/tensorflow

reason: import numpy


text:
"""
This module provides functions to plot the p-value spectra of the SPADE
results.
"""
from collections import defaultdict

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import matplotlib.cm as cm

import config as cf

def _load_pv_spectrum(setup):
    results = np.load(
        file=f'{setup.pattern_path}patterns_{int(setup.rate.magnitude)}_'
             f'{setup.data_type}_{setup.surr_method_short}.npy',
        allow_pickle=True)
    return [result['pvalue_spectrum'] for result in results]

def _get_optimized_pvalue_spec(setup):
    optimized_pvalue_spec = defaultdict(
        lambda: defaultdict(lambda: defaultdict(list)))

    pv_spectrum_list = _load_pv_spectrum(setup)
    for pv_spec in pv_spectrum_list:
        for entry in pv_spec:
            optimized_pvalue_spec[entry[0]][entry[2]][entry[1]].append(
                entry[3])
    return optimized_pvalue_spec

def _get_mean_optimized_pvalue_spec(setup):
    optimized_pvalue_spec = _get_optimized_pvalue_spec(setup)
    for size in optimized_pvalue_spec.keys():
        for dur in optimized_pvalue_spec[size].keys():
            for occ in optimized_pvalue_spec[size][dur].keys():
                optimized_pvalue_spec[size][dur][occ] = sum(
                    optimized_pvalue_spec[size][dur][occ]
                ) / setup.n_realizations
    return optimized_pvalue_spec

def plot_comparison_of_two_pvalue_spectra(setup1, setup2):
    """
    Function creates the plot comparing to p-value spectra.

    Parameters
    ----------
    setup1: cf.TestCaseSetUp
    setup2: cf.TestCaseSetUp

    """
    cmap = 'Blues_r'
    scatter_size = 45.

    size = setup1.sizes_to_analyze[0]

    color_norm = mcolors.SymLogNorm(
        linthresh=1e-6,
        vmin=1e-6,
        vmax=1.,
        base=10.)

    centimeters = 1/2.54  # centimeters in inches
    fig, axes = plt.subplots(  # 12.7
        2, 1, figsize=(8.5*centimeters, 10.0*centimeters), sharex='all',
        gridspec_kw=dict(hspace=0.32, right=0.85, top=0.93, bottom=0.12,
                         left=0.15))

    # find max_occurrences to fill up with zeroes
    max_occ = []
    for axis_id, (axis, setup) in enumerate(zip(axes, (setup1, setup2))):
        optimized_pvalue_spec = _get_mean_optimized_pvalue_spec(setup)

        for dur in optimized_pvalue_spec[size].keys():
            for occ in optimized_pvalue_spec[size][dur].keys():
                max_occ.append(occ)
    max_occ = max(max_occ)

    for axis_id, (axis, setup) in enumerate(zip(axes, (setup1, setup2))):
        optimized_pvalue_spec = _get_mean_optimized_pvalue_spec(setup)

        durs = []
        occs = []
        pvs = []

        for dur in optimized_pvalue_spec[size].keys():
            fill = list(
                range(list(optimized_pvalue_spec[size][dur].keys())[-1]+1,
                      max_occ+1))
            for occ in list(optimized_pvalue_spec[size][dur].keys()) + fill:
                durs.append(dur)
                occs.append(occ)
                if optimized_pvalue_spec[size][dur][occ]:
                    pvs.append(optimized_pvalue_spec[size][dur][occ])
                else:
                    pvs.append(0)

        durs_zeros = []
        occs_zeros = []
        pvs_zeros = []

        for dur in optimized_pvalue_spec[size].keys():
            fill = list(
                range(list(optimized_pvalue_spec[size][dur].keys())[-1]+1,
                      max_occ+1))
            for occ in list(optimized_pvalue_spec[size][dur].keys()) + fill:
                if not optimized_pvalue_spec[size][dur][occ]:
                    durs_zeros.append(dur)
                    occs_zeros.append(occ)
                    pvs_zeros.append(0)

        axis.scatter(occs, durs, s=scatter_size,
                     c=pvs, marker='s', cmap=cmap,
                     norm=color_norm)

        axis.scatter(occs_zeros, durs_zeros, s=scatter_size,
                     c=pvs_zeros, marker='s',
                     cmap='binary_r')

        if axis_id == 1:
            axis.set_xlabel('number of occurrences')
        axis.set_ylabel('duration', labelpad=1.8)
        if axis_id == 0:
            axis.set_title('Ground Truth')
        else:
            axis.set_title('Uniform Dithering')
        axis.set_yticks(np.arange(1, setup.win_len, 2))
        axis.set_yticklabels(np.arange(2, setup.win_len+1, 2))

    cbar_map = cm.ScalarMappable(
            norm=color_norm,
            cmap=cmap)

    cbar_map.set_array(np.array([]))
    cbar = fig.colorbar(mappable=cbar_map, ax=axes)

    cbar.set_label('p-value')
    fig.savefig('../../plots/fig3_panelC_pvalue_spectrum.svg',
                dpi=300)
    plt.show()

def plot_flattened_pvalue_spectra(setup1, setup2):
    """
    Function creates a plot comparing to p-value spectra, showing the p-values
    as function of each other.

    Parameters
    ----------
    setup1: cf.TestCaseSetUp
    setup2: cf.TestCaseSetUp

    """

    plt.rcParams.update({'font.size': 12,
                         'text.usetex': True})

    fig, ax = plt.subplots(1, 1, figsize=(5., 5.))

    optimized_pvalue_spec1 = _get_mean_optimized_pvalue_spec(setup1)
    optimized_pvalue_spec2 = _get_mean_optimized_pvalue_spec(setup2)

    pvalues1 = []
    pvalues2 = []

    for size in optimized_pvalue_spec1.keys():
        for dur in optimized_pvalue_spec1[size].keys():
            for occ in optimized_pvalue_spec1[size][dur].keys():
                pvalues1.append(optimized_pvalue_spec1[size][dur][occ])
                if isinstance(optimized_pvalue_spec2[size][dur][occ], float):
                    pvalues2.append(optimized_pvalue_spec2[size][dur][occ])
                else:
                    pvalues2.append(0.)

    ax.scatter(np.array(pvalues1), np.array(pvalues2))
    ax.set_xlabel('p-values GT')
    ax.set_ylabel('p-values UD')

if __name__ == '__main__':
    setup_gt = cf.TestCaseSetUp(surr_method='ground_truth')
    setup_ud = cf.TestCaseSetUp(surr_method='dither_spikes')
    plot_comparison_of_two_pvalue_spectra(setup_gt, setup_ud)


text:
import pytest

import numpy as np
import pandas as pd

from numpy.testing import assert_almost_equal

from fake_data_for_learning.fake_data_for_learning import (
    BayesianNodeRV, FakeDataBayesianNetwork
)

# Test static methods
def test_name_in_list():
    assert FakeDataBayesianNetwork.name_in_list('bob', None) == 0
    assert FakeDataBayesianNetwork.name_in_list('alice', ['alice', 'bob']) == 1

def test_parent_idx():
    X = np.array([
        [0, 0, 1],
        [0, 0, 1],
        [0, 0, 0]
    ])

    assert FakeDataBayesianNetwork.get_parent_idx(2, X) == [0, 1]
    assert FakeDataBayesianNetwork.get_parent_idx(0, X) == []

@pytest.fixture
def rv_binary_X0():
    return BayesianNodeRV('X0', np.array([0.1, 0.9]))

@pytest.fixture
def rv_binary_child_X1(rv_binary_X0):
    return BayesianNodeRV(
        'X1',
        np.array([
            [0.2, 0.8],
            [0.7, 0.3]
        ]),
        parent_names=[rv_binary_X0.name]
    )

def test_parents(rv_binary_X0, rv_binary_child_X1):
    # Test for missing parent variable
    with pytest.raises(ValueError):
        FakeDataBayesianNetwork(rv_binary_child_X1)

    # Test for wrong parent name
    with pytest.raises(ValueError):
        FakeDataBayesianNetwork(
            rv_binary_child_X1,
            BayesianNodeRV(
                'X1',
                np.array([
                    [0.2, 0.8],
                    [0.7, 0.3]
                ]),
                parent_names=['geoff']
            )
        )

@pytest.fixture
def non_binary_bayesian_network(rv_binary_X0):
    # X0 -> X2 <- Y1
    return FakeDataBayesianNetwork(
        rv_binary_X0,
        BayesianNodeRV('Y1', np.array([0.1, 0.7, 0.2])),
        BayesianNodeRV(
            'X2',
            np.array([
                [
                    [0., 1.0],
                    [0.2, 0.8],
                    [0.1, 0.9]
                ],
                [
                    [0.5, 0.5],
                    [0.3, 0.7],
                    [0.9, 0.1]
                ],

            ]),
            parent_names=['X0', 'Y1']
        )
    )

@pytest.fixture
def thrifty_bayesian_network():
    r'''
        age     ->      profession
        |                   /
        -> thriftiness <---
    '''
    age = BayesianNodeRV('age', np.array([0.2, 0.5, 0.3]), values=('20', '40', '60'))
    profession = BayesianNodeRV(
        'profession',
        np.array([
            [0.3, 0.4, 0.2, 0.1],
            [0.05, 0.15, 0.3, 0.5],
            [0.15, 0.05, 0.2, 0.6]
        ]),
        values=('salaried', 'self-employed', 'student', 'unemployed'),
        parent_names=['age'])

    thriftiness = BayesianNodeRV(
        'thriftiness',
        np.array([
            [
                [0.6, 0.4],  # 20, salaried
                [0.1, 0.9],  # 20, self-employed
                [0.2, 0.8],  # 20, student
                [0.3, 0.7],  # 20, unemployed
            ],
            [
                [0.2, 0.8],  # 40, salaried
                [0.3, 0.7],  # 40, self-employed
                [0.7, 0.3],  # 40, student
                [0.4, 0.6],  # 40, unemployed
            ],
            [
                [0.25, 0.75],  # 60, salaried
                [0.3, 0.7],  # 60, self-employed
                [0.2, 0.8],  # 60, student
                [0.1, 0.9],  # 60, unemployed
            ],
        ]),
        parent_names=['age', 'profession']
    )

    return FakeDataBayesianNetwork(age, profession, thriftiness)

def test_expected_cpt_dims(
    rv_binary_X0, rv_binary_child_X1,
    non_binary_bayesian_network,
    thrifty_bayesian_network
):
    bn = FakeDataBayesianNetwork(rv_binary_X0, rv_binary_child_X1)
    assert (
        bn.get_expected_cpt_dims([0], len(rv_binary_X0.values))
        == (2, 2)
    )

    # X0 -> X2 <- Y1 with Y1 ternary
    assert(
        non_binary_bayesian_network.get_expected_cpt_dims(
            [0, 1], len(non_binary_bayesian_network.bnrvs[2].values)
        )
        == (2, 3, 2)
    )

    # Thriftiness Bayesian network
    assert (
        thrifty_bayesian_network.get_expected_cpt_dims(
            [0, 1], len(thrifty_bayesian_network.bnrvs[2].values)
        )
        == (3, 4, 2)
    )

def test_adjacency_matrix(
    rv_binary_X0, rv_binary_child_X1,
    non_binary_bayesian_network,
    thrifty_bayesian_network
):
    bn = FakeDataBayesianNetwork(rv_binary_X0, rv_binary_child_X1)
    expected_adj = np.array(
        [[0, 1], [0, 0]]
    )

    np.testing.assert_equal(
        bn.adjacency_matrix,
        expected_adj
    )

    # X0 -> X2 <- Y1
    expected_nonbinary_adj = np.array([
        [0, 0, 1],
        [0, 0, 1],
        [0, 0, 0]
    ])
    np.testing.assert_equal(
        non_binary_bayesian_network.adjacency_matrix,
        expected_nonbinary_adj
    )

    expected_thrifty_adj = np.array([
        [0, 1, 1],
        [0, 0, 1],
        [0, 0, 0]
    ])
    np.testing.assert_equal(
        thrifty_bayesian_network.adjacency_matrix,
        expected_thrifty_adj
    )

def test_topological_sort():
    eve_0 = BayesianNodeRV('E0', np.array([0.2, 0.8]))
    eve_1 = BayesianNodeRV('E1', np.array([0.7, 0.3]))
    descendent = BayesianNodeRV(
        'D',
        np.array([
            [
                [0.3, 0.7],
                [0.7, 0.3]
            ],
            [
                [0.7, 0.3],
                [0.7, 0.3]
            ]
        ]),
        parent_names=['E0', 'E1']
    )
    further_descendent = BayesianNodeRV(
        'FD',
        np.array([
            [0.4, 0.6],
            [0.6, 0.4]
        ]),
        parent_names=['D']
    )
    bn = FakeDataBayesianNetwork(eve_0, eve_1, descendent, further_descendent)

    res = list(bn.get_topological_ordering())
    assert (
        res == ['E0', 'E1', 'D', 'FD'] or
        res == ['E1', 'E0', 'D', 'FD']
    )

def test_rvs_boundary_cases():
    pt_always_0 = np.array([1., 0.])
    always_0 = BayesianNodeRV('X0', pt_always_0)
    always_1c0 = BayesianNodeRV(
        'X1',
        np.array([
            [0., 1.],
            [1., 0.],
        ]),
        parent_names=['X0']
    )

    bn_1c0 = FakeDataBayesianNetwork(always_0, always_1c0)
    print(bn_1c0.rvs(size=1))
    pd.testing.assert_frame_equal(
        bn_1c0.rvs(size=1),
        pd.DataFrame.from_records([{'X0': 0, 'X1': 1}])
    )

    always_0c0 = BayesianNodeRV(
        'X1',
        np.array([
            [1., 0.],
            [0., 1.],
        ]),
        parent_names=['X0']
    )
    bn_0c0 = FakeDataBayesianNetwork(always_0, always_0c0)
    pd.testing.assert_frame_equal(
        bn_0c0.rvs(size=1),
        pd.DataFrame.from_records([{'X0': 0, 'X1': 0}])
    )

def test_pmf():
    X0 = BayesianNodeRV('X0', np.array([0.1, 0.9]))

    X1cX0 = BayesianNodeRV(
        'X1',
        np.array([
            [0.2, 0.8],
            [0.7, 0.3]
        ]),
        parent_names=[X0.name]
    )
    bn = FakeDataBayesianNetwork(X0, X1cX0)

    assert_almost_equal(
        bn.pmf(pd.Series([0, 0])),
        X0.cpt[0] * X1cX0.cpt[0, 0]
    )
    assert_almost_equal(
        bn.pmf(pd.Series([1, 0])),
        X0.cpt[1] * X1cX0.cpt[1, 0]
    )
    assert_almost_equal(
        bn.pmf(pd.Series([0, 1])),
        X0.cpt[0] * X1cX0.cpt[0, 1]
    )
    assert_almost_equal(
        bn.pmf(pd.Series([1, 1])),
        X0.cpt[1] * X1cX0.cpt[1, 1]
    )

def test_pmf_non_default_values():
    level = BayesianNodeRV('Level', np.array([0.1, 0.9]), values=['high', 'low'])

    outcome = BayesianNodeRV(
        'Outcome',
        np.array([
            [0.2, 0.5, 0.3],
            [0.3, 0.4, 0.3],
        ]),
        values=['bad', 'good', 'meltdown'],
        parent_names=['Level']
    )
    bn = FakeDataBayesianNetwork(level, outcome)

    assert pytest.approx(bn.pmf(pd.Series(['high', 'bad']))) == \
        pytest.approx(level.cpt[0] * outcome.cpt[0, 0])
    assert pytest.approx(bn.pmf(pd.Series(['low', 'meltdown']))) == \
        pytest.approx(level.cpt[1] * outcome.cpt[1, 2])

def test_rvs_counts_vs_pmf():
    # X0 -> X1 binary
    pt_X0 = np.array([0.5, 0.5])
    X0 = BayesianNodeRV('X0', pt_X0)
    pt_X1cX0 = np.array([
            [0.25, 0.75],
            [0.75, 0.25],
    ])
    X1 = BayesianNodeRV('X1', pt_X1cX0, parent_names=['X0'])

    bn = FakeDataBayesianNetwork(X0, X1)
    samples = bn.rvs(size=5000)
    sample_ratios = samples.groupby(['X0', 'X1']).size() / samples.shape[0]

    expected_index = pd.MultiIndex.from_tuples(
        [(0, 0), (0, 1), (1, 0), (1, 1)],
        names=['X0', 'X1'])
    expected_ratios = pd.Series(
        [0.125, 0.375, 0.375, 0.125],
        index=expected_index
    )
    pd.testing.assert_series_equal(
        sample_ratios, expected_ratios,
        check_exact=False, check_less_precise=0
    )

max_stars_repo_name: munichpavel/fake-data-for-learning

max_issues_repo_name: munichpavel/fake-data-for-learning

max_forks_repo_name: munichpavel/fake-data-for-learning

reason: import numpy,from numpy


text:
"""
  Name     : c14_23_N_f_vs_N2_f.py
  Book     : Python for Finance (2nd ed.)
  Publisher: Packt Publishing Ltd.
  Author   : Yuxing Yan
  Date     : 6/6/2017
  email    : yany@canisius.edu
             paulyxy@hotmail.com
"""

from math import exp,sqrt,log
import statsmodels.sandbox.distributions.extras as extras
#
def dOne(s,k,r,sigma,T):
    #print(s,k,r,sigma,T)
    a=log(s/k)+(r-0.5*sigma**2)*T
    b=(sigma*sqrt(T))
    return a/b
#
def sigmaA_f(sigma1,sigma2,rho):
    return sqrt(sigma1**2-2*rho*sigma1*sigma2+sigma2**2)
#
def dTwo(d1,sigma,T):
    return d1+sigma*sqrt(T)
#
def rhoTwo(sigma1,sigma2,sigmaA,rho):
    return (rho*sigma2-sigma1)/sigmaA
#
def N2_f(d1,d2,rho):
    import statsmodels.sandbox.distributions.extras as extras
    muStandardNormal=0.0    # mean of a standard normal distribution
    varStandardNormal=1.0   # variance of standard normal distribution
    upper=([d1,d2])         # upper bound for two values
    v=varStandardNormal     # simplify our notations
    mu=muStandardNormal     # simplify our notations
    covM=([v,rho],[rho,v])
    return extras.mvnormcdf(upper,mu,covM)
#
def dOneTwo(s1,s2,sigmaA,T):
    a=log(s2/s1)-0.5*sigmaA**2*T
    b=sigmaA*sqrt(T)
    return a/b
#
def rainbowCallOnMinimum(s1,s2,k,T,r,sigma1,sigma2,rho):
    d1=dOne(s1,k,r,sigma1,T)
    d2=dOne(s2,k,r,sigma2,T)
    d11=dTwo(d1,sigma1,T)
    d22=dTwo(d2,sigma2,T)
    sigmaA=sigmaA_f(sigma1,sigma2,rho)
    rho1=rhoTwo(sigma1,sigma2,sigmaA,rho)
    rho2=rhoTwo(sigma2,sigma1,sigmaA,rho)
    d12=dOneTwo(s1,s2,sigmaA,T)
    d21=dOneTwo(s2,s1,sigmaA,T)
    #
    part1=s1*N2_f(d11,d12,rho1)
    part2=s2*N2_f(d21,d22,rho2)
    part3=k*exp(-r*T)*N2_f(d1,d2,rho)
  #  print("d1=",d1)   # ok
  #  print("d11=",d11)  #ok
  #    print("d2=",d2)  #ok
    print("sigmaA=",sigmaA) # good
    print("d12=",d12)
    print("d21=",d21)
   # print("d22=",d22)   # good
    print("rho1=",rho1)  # good
    print("rho2=",rho2)  # good
    print("part 1=",part1)
    print("part 2=",part2)
    print("part 3=",part3)
    return part1 + part2 - part3
#
s1=100.
s2=95.
k=102.0
T=8./12.
r=0.08
rho=0.75
sigma1=0.15
sigma2=0.20
price=rainbowCallOnMinimum(s1,s2,k,T,r,sigma1,sigma2,rho)
print("price of call based on the minimum of 2 assets=",price)

reason: import statsmodels


text:
import unittest

import numpy as np
import numpy.testing as npt

from cw.tile_coding import GridTileCoding, TileSelection, Tile

class TestGridTileCoding(unittest.TestCase):
    def test_init(self):
        gtc = GridTileCoding(
            center_coordinate=[-2, 1],
            tile_size=[1, 3],
            n_tilings=3,
            initial_value=[0, 1],
            random_offsets=False,
        )

        npt.assert_allclose(gtc.center_coordinates, [[-2., 1.],
                                                     [-1.66666667, 2.],
                                                     [-1.33333333, 3.]])

    def test_get_value(self):
        gtc = GridTileCoding(
            center_coordinate=[0, 0],
            tile_size=[1, 3],
            n_tilings=3,
            initial_value=0,
            random_offsets=False,
        )

        tile_collection = gtc[0.4, 1]
        self.assertIsInstance(tile_collection, TileSelection)

    def test_tile_collection(self):
        tiles = [Tile(np.array([0, 1])), Tile(np.array([1, 2])), Tile(np.array([3, 4]))]
        tc = TileSelection(tiles, [])
        npt.assert_allclose(tc.value, [4/3, 7/3])

    def test_tile_collection_setting(self):
        tiles = [Tile(np.array([0, 1])), Tile(np.array([1, 2])), Tile(np.array([3, 4]))]
        tc = TileSelection(tiles, [])
        tc.value = [9, 10]

        npt.assert_allclose(tc.value, [9, 10])
        npt.assert_allclose(tiles[0].weight, [3., 3.33333333])
        npt.assert_allclose(tiles[1].weight, [3., 3.33333333])
        npt.assert_allclose(tiles[2].weight, [3., 3.33333333])

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: aarondewindt/cw

max_issues_repo_name: aarondewindt/cw

max_forks_repo_name: aarondewindt/cw


text:
# --------------------------------------------------------
# SMNet FaceNet
# Licensed under The MIT License [see LICENSE for details]
# Copyright 2019 smarsu. All Rights Reserved.
# --------------------------------------------------------

import sys
import os.path as osp
import numpy as np
from datasets import CelebA
from facenet import FaceNet
from euclidean import euclidean_distance

#np.random.seed(196)
dataset_root = '/datasets/CelebA'

if __name__ == '__main__':
    parse = sys.argv[1]  # 'train'
    print(parse)
    face_size = (96, 96)

    if parse == 'train':
        batch_size = 8
        celebA = CelebA(dataset_root, face_size=face_size)
        facenet = FaceNet(batch_size=batch_size, face_size=face_size, alpha=0.5)
        # Restore from:
        #   1. 0.2943756_7_8.npz test loss: 0
        #   2. 0.2817538_2_0.8.npz test loss: 0
        #   3. 0.28175184_22_0.08.npz test loss: 0
        #   4. 0.2740853_5_0.008.npz  test loss: 0

        #   1. 0.52165955_1_0.8.npz
        facenet.sess.restore(osp.join(facenet.model_root, '0.33327728476725504_94_1.npz'))
        # Train stage:
        #   1. epoch: 100, lr: 1
        #   2. epoch: 100, lr: 0.1
        #   3. epoch: 100, lr: 0.1
        #   4. epoch: 100, lr: 0.1
        #   5. epoch: 100, lr: 0.01
        #   6. epoch: 100, lr: 0.01
        #   6. epoch: 100, lr: 0.001
        #facenet.train(100, celebA.train_datas, 1 * batch_size, celebA.image2identity, testpaths=['data/demo/zjl0.jpg', 'data/demo/zjl1.jpg', 'data/demo/ffy.jpg'])
        #facenet.train(100, celebA.train_datas, 0.1 * batch_size, celebA.image2identity, testpaths=['data/demo/zjl0.jpg', 'data/demo/zjl1.jpg', 'data/demo/ffy.jpg'])
        facenet.train(100, celebA.train_datas, 1, celebA.image2identity, testpaths=['data/demo/ffy.jpg', 'data/demo/ffy1.jpg', 'data/demo/zjl.jpg'])
        #facenet.train(100, celebA.train_datas, 0.01 * batch_size, celebA.image2identity, testpaths=['data/demo/zjl0.jpg', 'data/demo/zjl1.jpg', 'data/demo/ffy.jpg'])
        #facenet.train(100, celebA.train_datas, 0.001 * batch_size, celebA.image2identity, testpaths=['data/demo/zjl0.jpg', 'data/demo/zjl1.jpg', 'data/demo/ffy.jpg'])
    elif parse == 'test':
        celebA = CelebA(dataset_root, face_size=face_size)
        facenet = FaceNet(face_size=face_size)
        # well-done 0.3703060848329446_30_1.npz
        facenet.sess.restore(osp.join(facenet.model_root, 'nan_99_1.npz'))

        images = ['data/demo/syz.jpg', 'data/demo/syz2.jpg', 'data/demo/zjl.jpg']
        feat0 = facenet.test(images[0])
        feat1 = facenet.test(images[1])
        feat2 = facenet.test(images[2])
        print(euclidean_distance(feat0, feat1))
        print(euclidean_distance(feat0, feat2))
        print(euclidean_distance(feat1, feat2))

        images = celebA.identity2images['1']
        feat0 = facenet.test(images[0])
        feat1 = facenet.test(images[1])
        feat3 = facenet.test(images[2])
        feat5 = facenet.test(images[3])
        images = celebA.identity2images['2']
        feat2 = facenet.test(images[0])
        feat4 = facenet.test(images[1])
        feat6 = facenet.test(images[2])
        # print(feat0)
        # print(feat1)
        # print(feat2)
        print(euclidean_distance(feat0, feat1))
        print(euclidean_distance(feat0, feat2))
        print(euclidean_distance(feat0, feat3))
        print(euclidean_distance(feat0, feat4))
        print(euclidean_distance(feat0, feat5))
        print(euclidean_distance(feat0, feat6))
    else:
        raise NotImplementedError

max_stars_repo_name: smarsu/facenet

max_issues_repo_name: smarsu/facenet

max_forks_repo_name: smarsu/facenet


text:
#!/usr/bin/env python
"""
linear combination fitting
"""
import os
import sys
import time

from itertools import combinations
from collections import OrderedDict

import numpy as np
from numpy.random import randint

try:
    from sklearn.decomposition import PCA
    HAS_SKLEARN = True
except ImportError:
    HAS_SKLEARN = False

from .. import Group
from .utils import interp, index_of

from lmfit import minimize, Parameters

from .lincombo_fitting import get_arrays, get_label, groups2matrix

def nmf_train(groups, arrayname='norm', xmin=-np.inf, xmax=np.inf,
              solver='cd', beta_loss=2):
    """use a list of data groups to train a Non-negative model

    Arguments
    ---------
      groups      list of groups to use as components
      arrayname   string of array name to be fit (see Note 2) ['norm']
      xmin        x-value for start of fit range [-inf]
      xmax        x-value for end of fit range [+inf]
      beta_loss   beta parameter for NMF [2]

    Returns
    -------
      group with trained NMF model, to be used with pca_fit

    Notes
    -----
     1.  The group members for the components must match each other
         in data content and array names.
     2.  arrayname can be one of `norm` or `dmude`
    """
    xdat, ydat = groups2matrix(groups, arrayname, xmin=xmin, xmax=xmax)

    ydat[np.where(ydat<0)] = 0
    opts = dict(n_components=len(groups), solver=solver)
    if solver == 'mu':
        opts.update(dict(beta_loss=beta_loss))
    ret = NMF(**opts).fit(ydat)
    labels = [get_label(g) for g  in groups]

    return Group(x=xdat, arrayname=arrayname, labels=labels, ydat=ydat,
                 components=ret.components_,
                 xmin=xmin, xmax=xmax, model=ret)

def pca_train_sklearn(groups, arrayname='norm', xmin=-np.inf, xmax=np.inf):
    """use a list of data groups to train a Principal Component Analysis

    Arguments
    ---------
      groups      list of groups to use as components
      arrayname   string of array name to be fit (see Note 2) ['norm']
      xmin        x-value for start of fit range [-inf]
      xmax        x-value for end of fit range [+inf]

    Returns
    -------
      group with trained PCA or N model, to be used with pca_fit

    Notes
    -----
     1.  The group members for the components must match each other
         in data content and array names.
     2.  arrayname can be one of `norm` or `dmude`
    """
    xdat, ydat = groups2matrix(groups, arrayname, xmin=xmin, xmax=xmax)
    if not HAS_SKLEARN:
        raise ImportError("scikit-learn not installed")

    ret = PCA().fit(ydat)
    labels = [get_label(g) for g  in groups]

    return Group(x=xdat, arrayname=arrayname, labels=labels, ydat=ydat,
                 xmin=xmin, xmax=xmax, model=ret, mean=ret.mean_,
                 components=ret.components_,
                 variances=ret.explained_variance_ratio_)

def pca_athena(groups, arrayname='norm', subtract_mean=True,
               normalize=True, xmin=-np.inf, xmax=np.inf):
    xdat, data = groups2matrix(groups, arrayname, xmin=xmin, xmax=xmax)
    if subtract_mean:
        data = data - data.mean(axis=0)

    data = data.T
    data = data - data.mean(axis=0)
    if normalize:
        data = data / data.std(axis=0)

    cor = np.dot(data.T, data) / data.shape[0]
    evals, var = np.linalg.eigh(cor)
    iorder = np.argsort(evals)[::-1]
    evals = evals[iorder]
    evec = np.dot(data, var)[:, iorder]
    return evec, evals

def pca_train(groups, arrayname='norm', xmin=-np.inf, xmax=np.inf):
    """use a list of data groups to train a Principal Component Analysis

    Arguments
    ---------
      groups      list of groups to use as components
      arrayname   string of array name to be fit (see Note 2) ['norm']
      xmin        x-value for start of fit range [-inf]
      xmax        x-value for end of fit range [+inf]

    Returns
    -------
      group with trained PCA or N model, to be used with pca_fit

    Notes
    -----
     1.  The group members for the components must match each other
         in data content and array names.
     2.  arrayname can be one of `norm` or `dmude`
    """
    xdat, ydat = groups2matrix(groups, arrayname, xmin=xmin, xmax=xmax)
    labels = [get_label(g) for g  in groups]
    narr, nfreq = ydat.shape

    ymean = ydat.mean(axis=0)
    ynorm = ydat - ymean

    # normalize data to be centered at 0 with unit standard deviation
    ynorm = (ynorm.T - ynorm.mean(axis=1)) / ynorm.std(axis=1)
    eigval, eigvec_ = np.linalg.eigh(np.dot(ynorm.T, ynorm) / narr)
    eigvec = (np.dot(ynorm, -eigvec_)/narr).T
    eigvec, eigval = eigvec[::-1, :], eigval[::-1]

    variances = eigval/eigval.sum()

    # calculate IND statistic
    ind = None
    for r in range(narr-1):
        nr = narr - r - 1
        indval = np.sqrt(nfreq*eigval[r:].sum()/nr)/nr**2
        if ind is None:
            ind = [indval]
        ind.append(indval)
    ind = np.array(ind)

    nsig = np.argmin(ind)
    return Group(x=xdat, arrayname=arrayname, labels=labels, ydat=ydat,
                 xmin=xmin, xmax=xmax, mean=ymean, components=eigvec,
                 eigenvalues=eigval, variances=variances, ind=ind, nsig=nsig)

def pca_statistics(pca_model):
    """return PCA arrays of statistics IND and F

    For data of shape (p, n) (that is, p frequencies/energies, n spectra)

    For index r, and eigv = eigenvalues

      IND(r) =  sqrt( eigv[r:].sum() / (p*(n-r))) / (n-r)**2

      F1R(r) = eigv[r] / (p+1-r)*(n+1-r) / sum_i=r^n-1 (eigv[i] / ((p+1-i)*(n+1-i)))
    """
    p, n = pca_model.ydat.shape
    eigv = pca_model.eigenvalues
    ind, f1r = [], []
    for r in range(n-1):
        nr = n-r-1
        ind.append( np.sqrt(eigv[r:].sum()/ (p*nr))/nr**2)
        f1sum = 0
        for i in range(r, n):
            f1sum += eigv[i]/((p+1-i)*(n+1-i))
        f1sum = max(1.e-10, f1sum)
        f1r.append(eigv[r] / (max(1, (p+1-r)*(n-r+1)) * f1sum))

    pca_model.ind = np.array(ind)
    pca_model.f1r = np.array(f1r)

    return pca_model.ind, pca_model.f1r

def _pca_scale_resid(params, ydat=None, pca_model=None, comps=None):
    scale = params['scale'].value
    weights, chi2, rank, s = np.linalg.lstsq(comps, ydat*scale-pca_model.mean)
    yfit = (weights * comps).sum(axis=1) + pca_model.mean
    return (scale*ydat - yfit)

def pca_fit(group, pca_model, ncomps=None, rescale=True):
    """
    fit a spectrum from a group to a PCA training model from pca_train()

    Arguments
    ---------
      group       group with data to fit
      pca_model   PCA model as found from pca_train()
      ncomps      number of components to included
      rescale     whether to allow data to be renormalized (True)

    Returns
    -------
      None, the group will have a subgroup name `pca_result` created
            with the following members:

          x          x or energy value from model
          ydat       input data interpolated onto `x`
          yfit       linear least-squares fit using model components
          weights    weights for PCA components
          chi_square goodness-of-fit measure
          pca_model  the input PCA model

    """
    # get first nerate arrays and interpolate components onto the unknown x array
    xdat, ydat = get_arrays(group, pca_model.arrayname)
    if xdat is None or ydat is None:
        raise ValueError("cannot get arrays for arrayname='%s'" % arrayname)

    ydat = interp(xdat, ydat, pca_model.x, kind='cubic')

    params = Parameters()
    params.add('scale', value=1.0, vary=True, min=0)

    if ncomps is None:
        ncomps=len(pca_model.components)
    comps = pca_model.components[:ncomps].transpose()

    if rescale:
        weights, chi2, rank, s = np.linalg.lstsq(comps, ydat-pca_model.mean)
        yfit = (weights * comps).sum(axis=1) + pca_model.mean

        result = minimize(_pca_scale_resid, params, method='leastsq',
                          gtol=1.e-5, ftol=1.e-5, xtol=1.e-5, epsfcn=1.e-5,
                          kws = dict(ydat=ydat, comps=comps, pca_model=pca_model))
        scale = result.params['scale'].value
        ydat *= scale
        weights, chi2, rank, s = np.linalg.lstsq(comps, ydat-pca_model.mean)
        yfit = (weights * comps).sum(axis=1) + pca_model.mean

    else:
        weights, chi2, rank, s = np.linalg.lstsq(comps, ydat-pca_model.mean)
        yfit = (weights * comps).sum(axis=1) + pca_model.mean
        scale = 1.0

    group.pca_result = Group(x=pca_model.x, ydat=ydat, yfit=yfit,
                             pca_model=pca_model, chi_square=chi2[0],
                             data_scale=scale, weights=weights)
    return


text:
import os
import sys

# Flask
from flask import Flask, redirect, url_for, request, render_template, Response, jsonify, redirect
from werkzeug.utils import secure_filename
from gevent.pywsgi import WSGIServer

# Tensorflow and tf.keras

#import tensorflow as tf
#from tensorflow import keras

from tensorflow.keras.applications.imagenet_utils import preprocess_input, decode_predictions
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing import image

# Some utilities

import numpy as np

import re
import base64

import numpy as np

from PIL import Image
from io import BytesIO

def base64_to_pil(img_base64):
    """
    Convert base64 image data to PIL image
    """
    image_data = re.sub('^data:image/.+;base64,', '', img_base64)
    pil_image = Image.open(BytesIO(base64.b64decode(image_data)))
    return pil_image

def np_to_base64(img_np):
    """
    Convert numpy image (RGB) to base64 string
    """
    img = Image.fromarray(img_np.astype('uint8'), 'RGB')
    buffered = BytesIO()
    img.save(buffered, format="PNG")
    return u"data:image/png;base64," + base64.b64encode(buffered.getvalue()).decode("ascii")

# Declare a flask app
app = Flask(__name__)

# You can use pretrained model from Keras
from tensorflow.keras.applications import MobileNetV2
Model_path = 'models/model_mobilenetv2.h5'
try:
    model = MobileNetV2(weights = Model_path)
except:
    model = MobileNetV2(weights = 'imagenet')

print("Model loaded. ")

def model_predict(img, model):
    img = img.resize((224, 224))

    # Preprocessing the image
    x = image.img_to_array(img)
    # x = np.true_divide(x, 255)
    x = np.expand_dims(x, axis=0)

    # Be careful how your trained model deals with the input
    # otherwise, it won't make correct prediction!
    x = preprocess_input(x, mode='tf')

    preds = model.predict(x)
    return preds

@app.route('/', methods=['GET'])
def index():
    # Main page
    return render_template('index.html')

@app.route('/predict', methods=['GET', 'POST'])
def predict():
    if request.method == 'POST':
        # Get the image from post request
        img = base64_to_pil(request.json)

        # Save the image to ./uploads
        # img.save("./uploads/image.png")

        # Make prediction
        preds = model_predict(img, model)

        # Process your result for human
        pred_proba = "{:.3f}".format(np.amax(preds))    # Max probability
        pred_class = decode_predictions(preds, top=1)   # ImageNet Decode

        result = str(pred_class[0][0][1])               # Convert to string
        result = result.replace('_', ' ').capitalize()

        # Serialize the result, you can add additional fields
        return jsonify(result=result, probability=pred_proba)

    return None

if __name__ == "__main__":
    app.run(debug = True)

    #http_server = WSGIServer(('0.0.0.0', 5000), app)
    #http_server.serve_forever()

max_stars_repo_name: antoreep-jana/Web-App-ImageNet

max_issues_repo_name: antoreep-jana/Web-App-ImageNet

max_forks_repo_name: antoreep-jana/Web-App-ImageNet


text:
from __future__ import division
import os
import re
import logging
import numpy
import tensorflow as tf
from bob.ip.color import gray_to_rgb
from bob.io.image import to_matplotlib
from bob.extension import rc
import bob.extension.download
import bob.io.base
import multiprocessing

logger = logging.getLogger(__name__)

FACENET_MODELPATH_KEY = "bob.ip.tensorflow_extractor.facenet_modelpath"

def prewhiten(img):
    mean = numpy.mean(img)
    std = numpy.std(img)
    std_adj = numpy.maximum(std, 1.0 / numpy.sqrt(img.size))
    y = numpy.multiply(numpy.subtract(img, mean), 1 / std_adj)
    return y

def get_model_filenames(model_dir):
    # code from https://github.com/davidsandberg/facenet
    files = os.listdir(model_dir)
    meta_files = [s for s in files if s.endswith(".meta")]
    if len(meta_files) == 0:
        raise ValueError("No meta file found in the model directory (%s)" % model_dir)
    elif len(meta_files) > 1:
        raise ValueError(
            "There should not be more than one meta file in the model "
            "directory (%s)" % model_dir
        )
    meta_file = meta_files[0]
    max_step = -1
    for f in files:
        step_str = re.match(r"(^model-[\w\- ]+.ckpt-(\d+))", f)
        if step_str is not None and len(step_str.groups()) >= 2:
            step = int(step_str.groups()[1])
            if step > max_step:
                max_step = step
                ckpt_file = step_str.groups()[0]
    return meta_file, ckpt_file

_semaphore = multiprocessing.Semaphore()
class FaceNet(object):
    """Wrapper for the free FaceNet variant:
    https://github.com/davidsandberg/facenet

    To use this class as a bob.bio.base extractor::

        from bob.bio.base.extractor import Extractor
        class FaceNetExtractor(FaceNet, Extractor):
            pass
        extractor = FaceNetExtractor()

    And for a preprocessor you can use::

        from bob.bio.face.preprocessor import FaceCrop
        # This is the size of the image that this model expects
        CROPPED_IMAGE_HEIGHT = 160
        CROPPED_IMAGE_WIDTH = 160
        # eye positions for frontal images
        RIGHT_EYE_POS = (46, 53)
        LEFT_EYE_POS = (46, 107)
        # Crops the face using eye annotations
        preprocessor = FaceCrop(
            cropped_image_size=(CROPPED_IMAGE_HEIGHT, CROPPED_IMAGE_WIDTH),
            cropped_positions={'leye': LEFT_EYE_POS, 'reye': RIGHT_EYE_POS},
            color_channel='rgb'
        )

    """

    def __init__(
        self,
        model_path=rc[FACENET_MODELPATH_KEY],
        image_size=160,
        layer_name="embeddings:0",
        **kwargs
    ):
        super(FaceNet, self).__init__()
        self.model_path = model_path
        self.image_size = image_size
        self.layer_name = layer_name
        self._clean_unpicklables()

    def _clean_unpicklables(self):
        self.session = None
        self.embeddings = None
        self.graph = None
        self.images_placeholder = None
        self.embeddings = None
        self.phase_train_placeholder = None
        self.session = None

    def _check_feature(self, img):
        img = numpy.ascontiguousarray(img)
        if img.ndim == 2:
            img = gray_to_rgb(img)
        assert img.shape[-1] == self.image_size
        assert img.shape[-2] == self.image_size
        img = to_matplotlib(img)
        img = prewhiten(img)
        return img[None, ...]

    def load_model(self):
        if self.model_path is None:
            self.model_path = self.get_modelpath()
        if not os.path.exists(self.model_path):
            bob.io.base.create_directories_safe(FaceNet.get_modelpath())
            zip_file = os.path.join(FaceNet.get_modelpath(), "20170512-110547.zip")
            urls = [
                # This link only works in Idiap CI to save bandwidth.
                "http://www.idiap.ch/private/wheels/gitlab/"
                "facenet_model2_20170512-110547.zip",
                # this link to dropbox would work for everybody
                "https://www.dropbox.com/s/"
                "k7bhxe58q7d48g7/facenet_model2_20170512-110547.zip?dl=1",
            ]
            bob.extension.download.download_and_unzip(urls, zip_file)

        # code from https://github.com/davidsandberg/facenet
        model_exp = os.path.expanduser(self.model_path)
        with self.graph.as_default():
            if os.path.isfile(model_exp):
                logger.info("Model filename: %s" % model_exp)
                with tf.compat.v1.gfile.FastGFile(model_exp, "rb") as f:
                    graph_def = tf.compat.v1.GraphDef()
                    graph_def.ParseFromString(f.read())
                    tf.import_graph_def(graph_def, name="")
            else:
                logger.info("Model directory: %s" % model_exp)
                meta_file, ckpt_file = get_model_filenames(model_exp)

                logger.info("Metagraph file: %s" % meta_file)
                logger.info("Checkpoint file: %s" % ckpt_file)

                saver = tf.compat.v1.train.import_meta_graph(
                    os.path.join(model_exp, meta_file)
                )
                saver.restore(self.session, os.path.join(model_exp, ckpt_file))
        # Get input and output tensors
        self.images_placeholder = self.graph.get_tensor_by_name("input:0")
        self.embeddings = self.graph.get_tensor_by_name(self.layer_name)
        self.phase_train_placeholder = self.graph.get_tensor_by_name("phase_train:0")
        logger.info("Successfully loaded the model.")

    def __call__(self, img):
        with _semaphore:
            images = self._check_feature(img)
            if self.session is None:
                self.graph = tf.Graph()
                self.session = tf.compat.v1.Session(graph=self.graph)
            if self.embeddings is None:
                self.load_model()
            feed_dict = {
                self.images_placeholder: images,
                self.phase_train_placeholder: False,
            }
            features = self.session.run(self.embeddings, feed_dict=feed_dict)
        return features.flatten()

    @staticmethod
    def get_modelpath():
        """
        Get default model path.

        First we try the to search this path via Global Configuration System.
        If we can not find it, we set the path in the directory
        `<project>/data`
        """

        # Priority to the RC path
        model_path = rc["bob.ip.tensorflow_extractor.facenet_modelpath"]

        if model_path is None:
            import pkg_resources

            model_path = pkg_resources.resource_filename(
                __name__, "data/FaceNet/20170512-110547"
            )

        return model_path

    def __setstate__(self, d):
        # Handling unpicklable objects
        self.__dict__ = d

    def __getstate__(self):
        # Handling unpicklable objects
        with _semaphore:
            self._clean_unpicklables()
        return self.__dict__


text:
# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE

import pytest  # noqa: F401
import numpy as np  # noqa: F401
import awkward as ak  # noqa: F401

def test_iterator():
    content = ak._v2.contents.NumpyArray(np.array([1.1, 2.2, 3.3]))
    offsets = ak._v2.index.Index32(np.array([0, 2, 2, 3], "i4"))
    array = ak._v2.contents.ListOffsetArray(offsets, content)

    assert list(content) == [1.1, 2.2, 3.3]
    assert [np.asarray(x).tolist() for x in array] == [[1.1, 2.2], [], [3.3]]


text:
from .base import BaseProblem
import numpy as np
from scipy.optimize import rosen

class Rosenbrock(BaseProblem):

    def __init__(self, d_int, d_cont, lb=-5, ub=10, dolog=False, noise_seed=None, noise_factor=1e-6):
        self.ub = ub
        self.lb = lb
        self.d_int = d_int
        self.d_cont = d_cont
        d = d_int + d_cont
        self.d = d
        self.scaling = d*(100*((self.ub-self.lb**2)**2)+(self.ub-1)**2)
        self.dolog = dolog

        self.noise_seed = noise_seed
        self.noise_rng = np.random.RandomState(seed=noise_seed)
        self.noise_factor = noise_factor

    def evaluate(self, x):
        assert len(x) == self.d
        if self.dolog:
            return np.log(rosen(x) + 1)  + self.noise_rng.normal(scale=self.noise_factor)
        return rosen(x)/self.scaling + self.noise_rng.normal(scale=self.noise_factor)

    def lbs(self):
        return self.lb*np.ones(self.d, dtype=float)

    def ubs(self):
        return self.ub*np.ones(self.d, dtype=float)

    def vartype(self):
        return np.array(['int'] * self.d_int + ['cont'] * self.d_cont)

    def dims(self):
        return self.d

    def __str__(self):
        return f"Rosenbrock(d_int={self.d_int}, d_cont={self.d_cont}, log={self.dolog})"

max_stars_repo_name: AlgTUDelft/ExpensiveOptimBenchmark

max_issues_repo_name: WholeG/ExpensiveOptimBenchmark

max_forks_repo_name: WholeG/ExpensiveOptimBenchmark

reason: import numpy,from scipy


text:
import cv2
import numpy as np
import cv2
import colorsys

def bbox2mid(bbox):
    """
    get middle point
    :param bbox: top left, right down
    :return: mid point
    """
    [x1, y1, x2, y2] = bbox
    return (int((x1 + x2) / 2), int((y1 + y2) / 2))

def get_ar(bbox):
    """

    :param bbox: top left, right down
    :return: aspect ratio
    """
    [x1, y1, x2, y2] = bbox
    return (y2 - y1) / (x2 - x1)

def draw_mid(img, bbox, radio, color, thickness=2, text='', y_bias=5):
    """

    :param bbox:  top left, right down
    :return:
    """
    (xc, yc) = bbox2mid(bbox)
    cv2.circle(img, (xc, yc), radio, color=color, thickness=thickness)
    if text:
        cv2.putText(img, text,
                    (xc, yc - y_bias), cv2.FONT_HERSHEY_COMPLEX, 0.3, color,
                    thickness=1)

def draw_matched_line(img, gt, dt, line_color):
    """

    :param gt: left top, right down
    :param dt: same as gt
    :param line_color:
    :return:
    """
    # [xl_g, yl_g, xr_g, yr_g] = gt
    # [xl_d, yl_d, xr_d, yr_d] = dt
    def form_corner(x0, y0, x1, y1):
        return np.array([(x0, y0), (x0, y1), (x1, y0), (x1, y1)])
    gt_corners = form_corner(*gt)
    dt_corners = form_corner(*dt)
    for i, gt_corner in enumerate(gt_corners):
        cv2.line(img, tuple(gt_corner), tuple(dt_corners[i]),
                 color=line_color, thickness=1)

def draw_bbox_ltrd(img, bbox, rect_color, thickness=1, text='', font_size=0.5):
    (x1, y1, x2, y2) = bbox
    cv2.rectangle(img, (x1, y1), (x2, y2), rect_color, thickness=thickness)
    if text:
        cv2.putText(img, text,
                    (int(x1), int(y1 - 5)), cv2.FONT_HERSHEY_SIMPLEX, font_size, rect_color, 2)

def draw_bbox(img, bbox, rect_color):
    (x1, y1, x2, y2) = bbox
    (x1, y1) = (int(x1), int(y1))
    (x2, y2) = (int(x2), int(y2))
    cv2.rectangle(img, (x1, y1), (x2, y2), rect_color, thickness=1)

def draw_gt(img, bbox, cat, id_name_map, rect_color, text_color=(0, 200, 200),
            rect_thick=2, font_size=0.5):
    (x1, y1, x2, y2) = bbox
    (x1, y1) = (int(x1), int(y1))
    (x2, y2) = (int(x2), int(y2))
    cv2.rectangle(img, (x1, y1), (x2, y2), rect_color, thickness=rect_thick)
    cv2.putText(img, id_name_map[str(cat)],
                (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, font_size, text_color, 2)

def draw_dt(img, bbox, cat, score, id_name_map, rect_color, text_color=(0, 200, 200),
            no_text=False):
    (x1, y1, x2, y2) = bbox
    (x1, y1) = (int(x1), int(y1))
    (x2, y2) = (int(x2), int(y2))
    cv2.rectangle(img, (x1, y1), (x2, y2), rect_color, thickness=1)
    show_text = '%s%.3f' % (id_name_map[str(cat)], score)
    if not no_text:
        cv2.putText(img, show_text,
                    (x1, y1 - 5), cv2.FONT_HERSHEY_COMPLEX, 0.3, text_color,
                    thickness=1)

def draw_bboxes(img, bbs, color, thickness=1, texts=None, font_size=0.5):
    """

    :param img:
    :param bbs: tensor
    :param color:
    :param thickness:
    :param texts:
    :param font_size:
    :return:
    """
    if texts:
        for b, t in zip(bbs, texts):
            b = b.numpy()
            draw_bbox_ltrd(img, b, color, text=t, thickness=thickness, font_size=font_size)
    else:
        for b in bbs:
            b = b.numpy()
            draw_bbox_ltrd(img, b, color, thickness=thickness, font_size=font_size)

# class BboxDrawer:
#     def __init__(self, type='val2017'):
#         from configs.config import bbox_dict_file_path
#         from utils.sl_utils import jsonload
#         self.bbox_dict = jsonload(bbox_dict_file_path)
#         from configs.id2name import id2name
#         self.id2name = id2name
#         from configs.color_map import gt_colors
#         self.colors = gt_colors['matched']
#
#
#     def draw(self, img_id, org_img=np.empty([0]), scale=(1, 1)):
#         try:
#             info = self.bbox_dict[str(img_id)]
#             file_path = info['file_path']
#             bboxes = info['bbox']
#             if not org_img.any():
#                 org_img = cv2.imread(file_path)
#             bboxes = np.array(bboxes)
#             bboxes[:, 0] = scale[0] * bboxes[:, 0]
#             bboxes[:, 2] = scale[0] * bboxes[:, 2]
#             bboxes[:, 1] = scale[1] * bboxes[:, 1]
#             bboxes[:, 3] = scale[1] * bboxes[:, 3]
#         except IndexError:
#             print(self.bbox_dict[str(img_id)])
#             return org_img
#
#         for i, bbox in enumerate(bboxes):
#             draw_gt(org_img, bbox[0: 4], int(bbox[4]), self.id2name, (255, 200, 0),
#                     text_color=(0, 200, 200), rect_thick=1, font_size=0.5)
#         return org_img
#
#
#     def draw_SML(self, img_id, org_img=np.empty([0]), scale=(1, 1)):
#         """
#         different area with different color
#         :param img_id:
#         :param org_img:
#         :param scale:
#         :return:
#         """
#         try:
#             info = self.bbox_dict[str(img_id)]
#             file_path = info['file_path']
#             org_bboxes = info['bbox']
#             if not org_img.any():
#                 org_img = cv2.imread(file_path)
#             bboxes = np.array(org_bboxes)
#             bboxes[:, 0] = scale[0] * bboxes[:, 0]
#             bboxes[:, 2] = scale[0] * bboxes[:, 2]
#             bboxes[:, 1] = scale[1] * bboxes[:, 1]
#             bboxes[:, 3] = scale[1] * bboxes[:, 3]
#         except IndexError:
#             print(self.bbox_dict[str(img_id)])
#             return org_img
#
#         for i, bbox in enumerate(bboxes):
#             [x, y, w, h] = org_bboxes[i][0: 4]
#             area = w * h
#             ids, label = get_rng(area)
#             color = self.colors[label]
#             draw_gt(org_img, bbox[0: 4], int(bbox[4]), self.id2name, color,
#                     text_color=color, rect_thick=1, font_size=0.5)
#         return org_img

if __name__ == '__main__':
    pass


text:
import cv2
import numpy as np
from matplotlib import pyplot as plt

imagesPath = 'images/'
outputPath = 'output/filters/'
fileExtension = '.jpg'

def convolution2D(img, window):

  kernel = np.ones((5,5),np.float32)/25
  dst = cv2.filter2D(img,-1,kernel)

  imgs = np.hstack((img, dst))

  cv2.imshow(window, imgs)
  cv2.imwrite(outputPath + window + fileExtension, imgs)

def averaging(img, window):

  blur = cv2.blur(img,(5,5))

  imgs = np.hstack((img, blur))

  cv2.imshow(window, imgs)
  cv2.imwrite(outputPath + window + fileExtension, imgs)

def gaussianBlur(img, window):
  blur = cv2.GaussianBlur(img,(5,5),0)

  imgs = np.hstack((img, blur))

  cv2.imshow(window, imgs)
  cv2.imwrite(outputPath + window + fileExtension, imgs)

def medianBlur(img, window):

  median = cv2.medianBlur(img,5)

  imgs = np.hstack((img, median))

  cv2.imshow(window, imgs)
  cv2.imwrite(outputPath + window + fileExtension, imgs)

def bilateralFilter(img, window):

  blur = cv2.bilateralFilter(img,9,75,75)

  imgs = np.hstack((img, blur))

  cv2.imshow(window, imgs)
  cv2.imwrite(outputPath + window + fileExtension, imgs)
  # key = cv2.waitKey(0)

def fastN1MeansDenoising(img, window):

  dst = cv2.fastNlMeansDenoisingColored(img,None,10,10,7,21)

  imgs = np.hstack((img, dst))

  cv2.imshow(window, imgs)
  cv2.imwrite(outputPath + window + fileExtension, imgs)

pcb1 = cv2.imread(imagesPath + 'pcb1.jpg')
pcb2 = cv2.imread(imagesPath + 'pcb2.jpg')

#convolution2D(pcb1, "convolution1")
#convolution2D(pcb2, "convolution2")
#averaging(pcb1, "averaging1")
#averaging(pcb2, "averaging2")
#gaussianBlur(pcb1, "gaussianBlur1")
#gaussianBlur(pcb2, "gaussianBlur2")
#medianBlur(pcb1, "medianBlur")
#bilateralFilter(pcb1, "bilateralFilter1")
#bilateralFilter(pcb2, "bilateralFilter2")
fastN1MeansDenoising(pcb1, "Denoised1")
fastN1MeansDenoising(pcb2, "Denoised2")

key = cv2.waitKey(0)


text:
# -*- coding: UTF-8 -*-
# @Author  : Chenyang Wang
# @Email   : THUwangcy@gmail.com

""" SASRec
Reference:
    "Self-attentive Sequential Recommendation"
    Kang et al., IEEE'2018.
Note:
    When incorporating position embedding, we make the position index start from the most recent interaction.
CMD example:
    python main.py --model_name SASRec --emb_size 64 --num_layers 1 --num_heads 1 --lr 1e-4 --l2 1e-6 \
    --history_max 20 --dataset 'Grocery_and_Gourmet_Food'
"""

import torch
import torch.nn as nn
import numpy as np

from models.BaseModel import SequentialModel
from utils import layers

class SASRec(SequentialModel):
    extra_log_args = ['emb_size', 'num_layers', 'num_heads']

    @staticmethod
    def parse_model_args(parser):
        parser.add_argument('--emb_size', type=int, default=64,
                            help='Size of embedding vectors.')
        parser.add_argument('--num_layers', type=int, default=1,
                            help='Number of self-attention layers.')
        parser.add_argument('--num_heads', type=int, default=4,
                            help='Number of attention heads.')
        return SequentialModel.parse_model_args(parser)

    def __init__(self, args, corpus):
        self.emb_size = args.emb_size
        self.max_his = args.history_max
        self.num_layers = args.num_layers
        self.num_heads = args.num_heads
        super().__init__(args, corpus)
        self.len_range = torch.from_numpy(np.arange(self.max_his)).to(self.device)

    def _define_params(self):
        self.i_embeddings = nn.Embedding(self.item_num, self.emb_size)
        self.p_embeddings = nn.Embedding(self.max_his + 1, self.emb_size)

        self.transformer_block = nn.ModuleList([
            layers.TransformerLayer(d_model=self.emb_size, d_ff=self.emb_size, n_heads=self.num_heads,
                                    dropout=self.dropout, kq_same=False)
            for _ in range(self.num_layers)
        ])

    def forward(self, feed_dict):
        self.check_list = []
        i_ids = feed_dict['item_id']  # [batch_size, -1]
        history = feed_dict['history_items']  # [batch_size, history_max]
        lengths = feed_dict['lengths']  # [batch_size]
        batch_size, seq_len = history.shape

        valid_his = (history > 0).long()
        his_vectors = self.i_embeddings(history)

        # Position embedding
        # lengths:  [4, 2, 5]
        # position: [[4, 3, 2, 1, 0], [2, 1, 0, 0, 0], [5, 4, 3, 2, 1]]
        position = (lengths[:, None] - self.len_range[None, :seq_len]) * valid_his
        pos_vectors = self.p_embeddings(position)
        his_vectors = his_vectors + pos_vectors

        # Self-attention
        causality_mask = np.tril(np.ones((1, 1, seq_len, seq_len), dtype=np.int))
        attn_mask = torch.from_numpy(causality_mask).to(self.device)
        # attn_mask = valid_his.view(batch_size, 1, 1, seq_len)
        for block in self.transformer_block:
            his_vectors = block(his_vectors, attn_mask)
        his_vectors = his_vectors * valid_his[:, :, None].float()

        his_vector = his_vectors[torch.arange(batch_size), lengths - 1, :]
        # his_vector = his_vectors.sum(1) / lengths[:, None].float()
        # ↑ average pooling is shown to be more effective than the most recent embedding

        i_vectors = self.i_embeddings(i_ids)
        prediction = (his_vector[:, None, :] * i_vectors).sum(-1)
        return {'prediction': prediction.view(batch_size, -1)}

max_stars_repo_name: TalonCB/ReChorus

max_issues_repo_name: TalonCB/ReChorus

max_forks_repo_name: TalonCB/ReChorus


text:
import overpy
import csv
import time
import operator
import pandas as pd
import numpy as np
import random

building_types = ["apartments",
                 "bungalow",
                 "cabin",
                 "detached",
                 "dormitory",
                 "farm",
                 "ger",
                 "hotel",
                 "house",
                 "houseboat",
                 "residential",
                 "semidetached_house",
                 "static_caravan",
                 "terrace"]

def sumproduct(vec1, vec2):
    return sum(map(operator.mul, vec1, vec2))

def request_germany(csvfilename):
    api = overpy.Overpass()

    r = api.query("""
    area["ISO3166-1"="DE"][admin_level=2];
    (node["amenity"="biergarten"](area);
     way["amenity"="biergarten"](area);
     rel["amenity"="biergarten"](area);
    );
    out center;
    """)
    coords  = []
    coords += [(float(node.lon), float(node.lat))
               for node in r.nodes]
    coords += [(float(way.center_lon), float(way.center_lat))
               for way in r.ways]
    coords += [(float(rel.center_lon), float(rel.center_lat))
               for rel in r.relations]

    header_name = ['lon', 'lat']
    with open(csvfilename, 'w') as csv_coords_w:
        coords_wr = csv.writer(csv_coords_w)
        coords_wr.writerow(header_name)

        for coord in coords:
            coords_wr.writerow(coord)
#area["ISO3166-1"="GB"][admin_level=2];
def request_coords_to_csv(csvfilename):
    api = overpy.Overpass()
    coords  = []
    for building_type in building_types:
        r = api.query(f"""
        area["ISO3166-2"="GB-LDS"][admin_level=8];
        (nwr["building"="{building_type}"](area);
        );
        out center;
        """)
        coords += [(float(node.lon), float(node.lat), building_type)
                   for node in r.nodes]
        coords += [(float(way.center_lon), float(way.center_lat), building_type)
                   for way in r.ways]
        coords += [(float(rel.center_lon), float(rel.center_lat), building_type)
                   for rel in r.relations]
        time.sleep(5)
    header_name = ['lon', 'lat','building_type']
    with open(csvfilename, 'w', newline='') as csv_coords_w:
        coords_wr = csv.writer(csv_coords_w)
        coords_wr.writerow(header_name)
        for coord in coords:
            coords_wr.writerow(coord)

def get_coords(csvfilename):
    coords  = []
    with open(csvfilename, 'r') as csv_coords_f:
        coords_rd = csv.DictReader(csv_coords_f)
        coords += [[float(coord['lon']), float(coord['lat']), str(coord['building_type'])]
               for coord in coords_rd]
        return coords

def count_coords_for_types(coords, types_count_csv):
    header_name = ['building_type', 'number']
    types_counts = []
    for building_type in building_types:
        count = 0
        for coord in coords:
            if coord[2] == building_type:
                count += 1
        types_counts += [(building_type, count)]
    with open(types_count_csv, 'w', newline='') as csv_types_count_w:
        csv_types_count_wr = csv.writer(csv_types_count_w)
        csv_types_count_wr.writerow(header_name)
        for types_count in types_counts:
            csv_types_count_wr.writerow(types_count)
    return types_counts

def merge_building_types_constraints_to_accommodations(types_count_csv, types_constraints_csv):
    df_types_count = pd.read_csv(types_count_csv)
    df_types_constraints = pd.read_csv(types_constraints_csv)
    return pd.merge(df_types_count, df_types_constraints,on="building_type",how="inner")

def write_list_to_csv(record_list, output_csv, header_name):
    with open(output_csv, 'w') as w:
        wr = csv.writer(w)
        wr.writerow(header_name)
        for record in record_list:
            wr.writerow(record)

def allocate_households_to_each_building(num_of_households, list_types_average_households, list_coords):
    valid = 0
    df_coords_types = pd.DataFrame(list_coords, columns=['lon', 'lat', 'building_type'])

    while valid != num_of_households:
        df_result = pd.DataFrame()
        list_num_households = []
        for types_average_households in list_types_average_households:
            list_num_households = list(types_average_households[3] + np.random.poisson(types_average_households[2],
                                                                                size=types_average_households[
                                                                                1]))
            df_temp = pd.DataFrame(df_coords_types[df_coords_types['building_type'] == types_average_households[0]])
            df_temp['num_of_households'] = list_num_households
            df_result = pd.concat([df_result, df_temp])
        valid = np.sum(df_result['num_of_households'])
    return df_result.values.tolist()
def test_merge_building_types_constraints_to_accommodations(types_count_list, types_constraints_csv):
    df_types_count = pd.DataFrame(types_count_list, columns=['building_type', 'number'])
    df_types_constraints = pd.read_csv(types_constraints_csv)
    return pd.merge(df_types_count, df_types_constraints, on="building_type", how="inner")
def next_household_home(homes_list):
    next_home = random.choice(homes_list)
    homes_list.remove(next_home)
    return next_home

max_stars_repo_name: Sciteb-code/CODiT

max_issues_repo_name: Sciteb-code/CODiT

max_forks_repo_name: Sciteb-code/CODiT


text:
import os
import gym
import math
import random
import numpy as np
from astar import AStar
from .mapper import RealMapper
from ray.rllib.env import MultiAgentEnv
from PIL import Image, ImageDraw, ImageColor

DEFAULT_ACTION_REPEAT = 2

class SingleRobot(gym.Env):

    def __init__(self, env_config={}):
        if 'world' in env_config:
            world = env_config['world']
        else:
            world = Playground()
        self.action_repeat = env_config.get("action_repeat") or DEFAULT_ACTION_REPEAT
        self.env = RealMapper(world, env_config)
        self.action_space = self.env.action_space
        self.observation_space = self.env.observation_space

    def step(self, action):
        """
        Step the simulation forward one timestep
        """
        self.env.act(action)
        for i in range(self.action_repeat):
            self.env.physics.stepSimulation()
            if self.env.is_crashed() or self.env.is_at_target():
                break
        return self.env.observe()

    def reset(self):
        """
        Reset the base environment
        """
        return self.env.reset()

    def render(self, *arg, **kwargs):
        return self.env.render(*arg, **kwargs)

    def render_map(self, *arg, **kwargs):
        return self.env.render_map(*arg, **kwargs)

class MultiRobot(gym.Env, MultiAgentEnv):

    def __init__(self, env_config={}):
        if 'world' in env_config:
            self.world = env_config['world']
        else:
            raise ValueError("The world was not specified")

        if 'num_robots' in env_config:
            self.num_robots = env_config['num_robots']
        else:
            raise ValueError("The number of robots was not specified")

        if "reset_on_target" in env_config:
            env_config["resetOnTarget"] = env_config["reset_on_target"]

        self.dones = set()
        self.action_repeat = env_config.get("action_repeat") or DEFAULT_ACTION_REPEAT
        self.env = {i: RealMapper(self.world, env_config) for i in range(self.num_robots)}
        self.default_env = self.env[0]
        self.action_space = self.default_env.action_space
        self.observation_space = self.default_env.observation_space
        print("Created an environment with %i robots"%len(self.env))

        for env in self.env.values():
            for other in self.env.values():
                if other != env:
                    env.collision_objects.append(other.robot.racecarUniqueId)

    def step(self, actions):
        """
        Step the simulation forward one timestep
        """
        for key, action in actions.items():
            self.env[key].act(action)

        for i in range(self.action_repeat):
            self.world.physics.stepSimulation()
            is_crashed = any(e.is_crashed() for e in self.env.values())
            is_target = any(e.is_at_target() for e in self.env.values())
            if is_crashed or is_target:
                break

        obs, rew, done, info = {}, {}, {}, {}
        for i in actions.keys():
            obs[i], rew[i], done[i], info[i] = self.env[i].observe()
            if done[i]:
                self.dones.add(i)

        # Rllib requires a dones[__all__] key
        done["__all__"] = len(self.dones) == len(self.env)

        return obs, rew, done, info

    def reset(self):
        """
        Reset the base environment
        """
        self.dones = set()
        return {key:env.reset() for key,env in self.env.items()}

    def render(self, *arg, **kwargs):
        return self.default_env.render(*arg, **kwargs)

    def render_map(self, *arg, **kwargs):
        return self.default_env.render_map(*arg, **kwargs)

max_stars_repo_name: maxkferg/dbm

max_issues_repo_name: maxkferg/dbm

max_forks_repo_name: maxkferg/dbm


text:
import math
import sys
import argparse
import numpy as np
import scipy.io as sio
import os
import glob
import h5py
import cv2
import gc

''' Documentation: class Frame_extractor

    This class has a few methods:

    extract
    extract_frame

    extract: calls to extract_frame for it's multiple classes

    extract_frame: extracts all video frames and saves them individually.
'''

class Frame_extractor:

    def __init__(self, classes, extension):
        self.classes = classes
        self.classes_dirs = []
        self.classes_videos = []
        self.fall_dirs = []
        self.class_value = []
        self.extension = extension

    def get_dirs(self, data_folder):

        for c in self.classes:
            self.classes_dirs.append([f for f in os.listdir(data_folder + c)
                        if os.path.isdir(os.path.join(data_folder, c, f))])
            self.classes_dirs[-1].sort()

            self.classes_videos.append([])
            for f in self.classes_dirs[-1]:
                self.classes_videos[-1].append(data_folder + c+ '/' + f +
                                   '/' + f + '.' + self.extension)

            self.classes_videos[-1].sort()

    def extract(self, data_folder):

        self.get_dirs(data_folder)

        for i in range(len(self.classes)):
            # Extracting video frames
            self.extract_frame(data_folder, self.classes_videos[i], self.classes_dirs[i], self.classes[i])

    def extract_frame(self, data_folder, videos, dirs, class_):

        for (video, dir) in zip(videos, dirs):
            print (":::Video:::")
            print (video)

            path = data_folder + class_ +  '/' + dir
            cap = cv2.VideoCapture(video)

            length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
            print( length )
            #width1 = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))   # float
            #height1 = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # float
            #print (width1)
            #print (height1)
            count = 0
            for fi in range (0,length):
                cap.set(1,fi);
                ret, img = cap.read()

                save_path = path + '/depth_' + str(count).zfill(5) + '.jpg'
                print (save_path)
                cv2.imwrite(save_path, img)

                count = count + 1

            cap.release()
            #cv2.destroyAllWindows()

if __name__ == '__main__':
    print("***********************************************************",
            file=sys.stderr)
    print("             SEMANTIX - UNICAMP DATALAB 2018", file=sys.stderr)
    print("***********************************************************",
            file=sys.stderr)
    argp = argparse.ArgumentParser(description='Do feature extraction tasks')

    argp.add_argument("-data", dest='data_folder', type=str, nargs=1,
            help='Usage: -data <path_to_your_data_folder>', required=True)

    argp.add_argument("-class", dest='classes', type=str, nargs='+',
            help='Usage: -class <class0_name> <class1_name>..<n-th_class_name>',
            required=True)

    argp.add_argument("-extension", dest='extension', type=str, nargs='+',
            help='Usage: -extension <video_extension_type>',
            required=True)

    try:
        args = argp.parse_args()
    except:
        argp.print_help(sys.stderr)
        exit(1)

    fr_extractor = Frame_extractor(args.classes, args.extension[0])
    fr_extractor.extract(args.data_folder[0])
    print ("done")

reason: import numpy,import scipy