text:
import pygame
import os
import time
import numpy as np

# Set window position close to top left
os.environ['SDL_VIDEO_WINDOW_POS'] = "%d,%d" % (50, 50)

CELL_SIZE = 48
BORDER_SIZE = 10

MARK = pygame.image.load('game_symbols/pygame_mark.png')
BLANK = pygame.image.load('game_symbols/pygame_blank.png')
CROSS = pygame.image.load('game_symbols/pygame_cross.png')
TEMP = pygame.image.load('game_symbols/pygame_temp.png')

def picross_init(rows, cols):
    # Pull grid size from input
    r = len(rows)
    c = len(cols)

    set_len = [len(row) for row in rows]
    set_len.extend([len(column) for column in cols])
    m = max(set_len)
    print(m)

    # Initialize game window to be appropriate size
    pygame.init()
    height = (r + m) * CELL_SIZE
    width = (c + m) * CELL_SIZE
    board = pygame.display.set_mode((height + 2 * BORDER_SIZE, width + 2 * BORDER_SIZE))
    board.fill((255, 255, 255))
    pygame.display.set_caption('Picross')

    # Draw all necessary squares into the grid
    x_position = m * CELL_SIZE + BORDER_SIZE
    y_position = m * CELL_SIZE + BORDER_SIZE
    pygame.draw.rect(board, (0, 0, 0), (x_position, y_position, r * CELL_SIZE, c * CELL_SIZE), 2)

    for i in range(r):
        for j in range(c):
            pygame.draw.rect(board, (0, 0, 0), (x_position + i * CELL_SIZE, y_position + j * CELL_SIZE, CELL_SIZE,
                                                CELL_SIZE), 1)

    running = True

    pygame.font.init()
    font = pygame.font.Font('JAi_____.TTF', 32)

    x_position = (m - 0.5) * CELL_SIZE + BORDER_SIZE
    y_position = (m + 0.5) * CELL_SIZE + BORDER_SIZE
    for i, row in enumerate(rows):
        for j, run in enumerate(row):
            text = font.render(str(run), True, (0, 0, 0))
            pos = text.get_rect(center=(x_position - j * CELL_SIZE, y_position + i * CELL_SIZE))
            board.blit(text, pos)

    x_position = (m + 0.5) * CELL_SIZE + BORDER_SIZE
    y_position = (m - 0.5) * CELL_SIZE + BORDER_SIZE
    for i, col in enumerate(cols):
        for j, run in enumerate(col):
            text = font.render(str(run), True, (0, 0, 0))
            pos = text.get_rect(center=(x_position + i * CELL_SIZE, y_position - j * CELL_SIZE))
            board.blit(text, pos)

    # board.blit(pygame.image.load('pygame_cross.png'), (m * CELL_SIZE + BORDER_SIZE, m * CELL_SIZE + BORDER_SIZE))

    board_copy = board.copy()

    # add cursor after copy
    pygame.draw.rect(board, (255, 200, 0), (m * CELL_SIZE + BORDER_SIZE, m * CELL_SIZE + BORDER_SIZE, CELL_SIZE, CELL_SIZE), 3)

    pygame.display.update()

    grid = np.array(np.zeros((r, c)))

    cursor_position = [0, 0]
    marking = False
    crossing = False
    temping = False

    removing = False

    clock = pygame.time.Clock()

    while running:

        clock.tick(12)

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                running = False

        keys_pressed = pygame.key.get_pressed()

        if keys_pressed[pygame.K_LEFT]:
            cursor_position[0] -= 1
            if cursor_position[0] < 0:
                cursor_position[0] = c - 1

        elif keys_pressed[pygame.K_RIGHT]:
            cursor_position[0] += 1
            if cursor_position[0] >= c:
                cursor_position[0] = 0

        elif keys_pressed[pygame.K_UP]:
            cursor_position[1] -= 1
            if cursor_position[1] < 0:
                cursor_position[1] = r - 1

        elif keys_pressed[pygame.K_DOWN]:
            cursor_position[1] += 1
            if cursor_position[1] >= r:
                cursor_position[1] = 0

        # Updating the game grid

        if keys_pressed[pygame.K_z] or keys_pressed[pygame.K_x] or keys_pressed[pygame.K_c]:
            # if previously pressed and no longer pressed, booleans are reset
            if marking and not keys_pressed[pygame.K_z]:
                marking = False
                removing = False
            if crossing and not keys_pressed[pygame.K_x]:
                crossing = False
                removing = False
            if temping and not keys_pressed[pygame.K_c]:
                temping = False
                removing = False

            current = grid[cursor_position[0]][cursor_position[1]]

            if marking or crossing or temping:
                if marking and keys_pressed[pygame.K_z]:
                    if removing and current == 1:
                        current = 0
                    elif not removing and current == 0 or current == -2:
                        current = 1

                elif crossing and keys_pressed[pygame.K_x]:
                    if removing and current == -1:
                        current = 0
                    elif not removing and current == 0 or current == -2:
                        current = -1

                elif temping and keys_pressed[pygame.K_c]:
                    if removing and current == -2:
                        current = 0
                    elif not removing and current == 0:
                        current = -2

            elif keys_pressed[pygame.K_z] or keys_pressed[pygame.K_x] or keys_pressed[pygame.K_c]:
                if keys_pressed[pygame.K_z]:
                    if current == 0 or current == -2:
                        current = 1
                    elif current == 1:
                        current = 0
                        removing = True
                    marking = True

                elif keys_pressed[pygame.K_x]:
                    if current == 0 or current == -2:
                        current = -1
                    elif current == -1:
                        current = 0
                        removing = True
                    crossing = True

                elif keys_pressed[pygame.K_c]:
                    if current == 0:
                        current = -2
                    elif current == -2:
                        current = 0
                        removing = True
                    temping = True

            # update board
            update_board(board_copy, cursor_position, m, current)
            grid[cursor_position[0]][cursor_position[1]] = current

        else:
            marking = False
            crossing = False
            temping = False
            removing = False

        board.blit(board_copy, (0, 0))
        pygame.draw.rect(board, (0, 255, 100), ((m + cursor_position[0]) * CELL_SIZE + BORDER_SIZE, (m + cursor_position[1]) * CELL_SIZE + BORDER_SIZE, CELL_SIZE, CELL_SIZE), 3)

        pygame.display.update()

    print(grid)
    pygame.quit()

def update_board(board, cursor_position, m, img):
    if img == 1:
        symbol = MARK
    elif img == 0:
        symbol = BLANK
    elif img == -1:
        symbol = CROSS
    elif img == -2:
        symbol = TEMP
    board.blit(BLANK, ((m + cursor_position[0]) * CELL_SIZE + BORDER_SIZE,
                        (m + cursor_position[1]) * CELL_SIZE + BORDER_SIZE))
    board.blit(symbol, ((m + cursor_position[0]) * CELL_SIZE + BORDER_SIZE,
                        (m + cursor_position[1]) * CELL_SIZE + BORDER_SIZE))

if __name__ == "__main__":
    sa10_r = [[1, 1, 2, 1], [1, 1, 1], [2], [1, 2, 1, 1], [1, 1, 2],
              [1, 1, 1, 1], [1, 1], [2, 1, 1], [1, 1, 2], [1, 1, 1, 1, 1]]

    sa10_c = [[1, 1, 2, 1], [2, 1, 1], [1, 1, 1], [1, 1, 2], [1, 2],
              [1, 1, 1, 1], [2, 1, 1, 1], [2, 2], [1], [1, 1, 1, 1, 1]]

    picross_init(sa10_r, sa10_c)

max_stars_repo_name: Thomas-van-Haastrecht/nonogram-solver

max_issues_repo_name: Thomas-van-Haastrecht/nonogram-solver

max_forks_repo_name: Thomas-van-Haastrecht/nonogram-solver


text:
import cv2
import numpy as np

img = cv2.imread("1.jpeg",1)

kernel = np.array([[0,1,0],[1,-4,1],[0,1,0]])

img = cv2.GaussianBlur(img,-1,kernel=kernel)

cv2.imshow("image",img)
cv2.waitKey(20000)

ext: py

reason: import numpy


text:
import numpy as np
import pandas as pd
from gmpy2 import bit_mask

from rulelist.rulelistmodel.gaussianmodel.gaussiantarget import GaussianTargets

class TestGaussianTargets(object):
    def test_onetarget(self):
        dictoutput = {"target1": np.arange(100)}
        input_target_data = pd.DataFrame(data=dictoutput)

        expected_number_targets = 1
        expected_bit_array = bit_mask(100)
        expected_mean_vector = np.array([49.5])
        expected_variance_vector  = np.var([*range(100)])

        output_gaussiantargets= GaussianTargets(input_target_data)

        assert expected_bit_array ==  output_gaussiantargets.bit_array
        assert expected_number_targets == len(output_gaussiantargets.mean)
        assert expected_number_targets == len(output_gaussiantargets.variance)

        np.testing.assert_array_equal(expected_mean_vector, output_gaussiantargets.mean)
        np.testing.assert_array_equal(expected_variance_vector, output_gaussiantargets.variance)

    def test_twotargets(self):
        dictoutput = {"target1": np.arange(100), "target2": np.ones(100)}
        input_target_data = pd.DataFrame(data=dictoutput)

        expected_number_targets = 2
        expected_bit_array = bit_mask(100)
        expected_mean_vector = np.array([49.5,1])
        expected_variance_vector  = [np.var([*range(100)]),0]

        output_gaussiantargets= GaussianTargets(input_target_data)

        assert expected_bit_array ==  output_gaussiantargets.bit_array
        assert expected_number_targets == len(output_gaussiantargets.mean)
        assert expected_number_targets == len(output_gaussiantargets.variance)
        np.testing.assert_array_equal(expected_mean_vector, output_gaussiantargets.mean)
        np.testing.assert_array_equal(expected_variance_vector, output_gaussiantargets.variance)

    def test_onlyzeros(self):
        dictoutput = {"target1": np.zeros(100)}
        input_target_data = pd.DataFrame(data=dictoutput)

        expected_number_targets = 1
        expected_bit_array = bit_mask(100)
        expected_mean_vector = np.array([0])
        expected_variance_vector = np.array([0])

        output_gaussiantargets = GaussianTargets(input_target_data)

        assert expected_bit_array == output_gaussiantargets.bit_array
        assert expected_number_targets == len(output_gaussiantargets.mean)
        assert expected_number_targets == len(output_gaussiantargets.variance)
        np.testing.assert_array_equal(expected_mean_vector, output_gaussiantargets.mean)
        np.testing.assert_array_equal(expected_variance_vector, output_gaussiantargets.variance)

max_stars_repo_name: HMProenca/RuleList

max_issues_repo_name: HMProenca/RuleList

max_forks_repo_name: HMProenca/RuleList


text:
import json
import networkx as nx

UNIQUE_ID = 0

def get_unique_id():
    global UNIQUE_ID
    UNIQUE_ID += 1
    id_str = str(UNIQUE_ID)
    if len(id_str) != 5:
        id_str = id_str.zfill(5)
    return id_str

def load_graph_network(graph_file):
    with open(graph_file) as f:
        graph = json.load(f)
    G = nx.DiGraph()
    graph_nodes = graph["nodes"]
    for graph_list in graph_nodes:
        node_name = graph_list[0]

        attributes = graph_list[1]
        if not bool(attributes):
            G.add_node(node_name)
        else:
            original_id = attributes["original_id"]
            datatype = attributes["datatype"]
            name = attributes["name"]
            metadata = attributes["metadata"]

            G.add_node(
                node_name,
                original_id=original_id,
                datatype=datatype,
                name=name,
                metadata=metadata,
            )
    graph_edges = graph["edges"]
    for graph_list in graph_edges:
        edge_1 = graph_list[0]
        edge_2 = graph_list[1]

        G.add_edge(edge_1, edge_2)

    return G

def save_graph(G, fname):
    with open(fname, "w") as f:
        graph_dict = dict(
            nodes=[[n, G.nodes[n]] for n in G.nodes()],
            edges=[[u, v, G.edges[u, v]] for u, v in G.edges()],
        )
        json.dump(graph_dict, f, indent=2)

max_stars_repo_name: zahidayar/BRON

max_issues_repo_name: zahidayar/BRON

max_forks_repo_name: zahidayar/BRON

reason: import networkx


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from fipy import *
from numpy import *
import scipy.sparse as sp
import scipy.sparse.linalg as la
import parameterFunctions.immuneResponse as delt
import parameterFunctions.sigmaF as sigmaF
import inspect
from collections import OrderedDict
import pygpc

from pygpc.sobol_saltelli import get_sobol_indices_saltelli
from pygpc.sobol_saltelli import saltelli_sampling

# This function is a modified version of the original pygpc function
def modified_get_sobol_indices(gpc_object, coeffs, n_samples=1e4):
    """
    Calculate the available sobol indices from the gPC coefficients (standard) or by sampling.
    In case of sampling, the Sobol indices are calculated up to second order.

    sobol, sobol_idx, sobol_idx_bool = SGPC.get_sobol_indices(coeffs, algorithm="standard", n_samples=1e4)

    Parameters
    ----------
    coeffs:  ndarray of float [n_basis x n_out]
        GPC coefficients
    algorithm : str, optional, default: "standard"
        Algorithm to determine the Sobol indices
        - "standard": Sobol indices are determined from the gPC coefficients
        - "sampling": Sobol indices are determined from sampling using Saltelli's Sobol sampling sequence [1, 2, 3]
    n_samples : int, optional, default: 1e4
        Number of samples to determine Sobol indices by sampling. The efficient number of samples
        increases to n_samples * (2*dim + 2) in Saltelli's Sobol sampling sequence.

    Returns
    -------
    sobol: ndarray of float [n_sobol x n_out]
        Normalized Sobol indices w.r.t. total variance
    sobol_idx: list of ndarray of int [n_sobol x (n_sobol_included)]
        Parameter combinations in rows of sobol.
    sobol_idx_bool: ndarray of bool [n_sobol x dim]
        Boolean mask which contains unique multi indices.

    Notes
    -----
    .. [1] Sobol, I. M. (2001).  "Global sensitivity indices for nonlinear
           mathematical models and their Monte Carlo estimates."  Mathematics
           and Computers in Simulation, 55(1-3):271-280,
           doi:10.1016/S0378-4754(00)00270-6.
    .. [2] Saltelli, A. (2002).  "Making best use of model evaluations to
           compute sensitivity indices."  Computer Physics Communications,
           145(2):280-297, doi:10.1016/S0010-4655(02)00280-1.
    .. [3] Saltelli, A., P. Annoni, I. Azzini, F. Campolongo, M. Ratto, and
           S. Tarantola (2010).  "Variance based sensitivity analysis of model
           output.  Design and estimator for the total sensitivity index."
           Computer Physics Communications, 181(2):259-270,
           doi:10.1016/j.cpc.2009.09.018.
    """

    if gpc_object.p_matrix is None:
        dim = gpc_object.problem.dim
    else:
        dim = gpc_object.problem_original.dim

    if gpc_object.problem_original is None:
        problem_original = gpc_object.problem
    else:
        problem_original = gpc_object.problem_original

    # generate uniform distributed sobol sequence (parameter space [0, 1])
    coords_norm_01 = saltelli_sampling(n_samples=n_samples, dim=dim, calc_second_order=True)
    coords_norm = zeros(coords_norm_01.shape)

    # transform to respective input pdfs using inverse cdfs
    for i_key, key in enumerate(problem_original.parameters_random.keys()):
        coords_norm[:, i_key] = problem_original.parameters_random[key].icdf(coords_norm_01[:, i_key])

    # run model evaluations
    res = gpc_object.get_approximation(coeffs=coeffs, x=coords_norm)

    # determine sobol indices
    sobol, sobol_idx, sobol_idx_bool = get_sobol_indices_saltelli(y=exp(res),
                                                                  dim=dim,
                                                                  calc_second_order=True,
                                                                  num_resamples=100,
                                                                  conf_level=0.95)

    # sort
    idx = flip(argsort(sobol[:, 0], axis=0))
    sobol = sobol[idx, :]
    sobol_idx = [sobol_idx[i] for i in idx]
    sobol_idx_bool = sobol_idx_bool[idx, :]

    return sobol, sobol_idx, sobol_idx_bool

# @wrap_non_picklable_objects
class MyModel(pygpc.AbstractModel):

    def __init__(self):
        self.fname = inspect.getfile(inspect.currentframe())
        # pass

    # def __reduce__(self):
    #     return (MyModel, (self.fname,))

    def validate(self):
        pass

    def dichotomy(self, mu_a, mu_b, eps, mesK, Q, aDiffE, aConVE, Id, gF, t_s, E, delta_tild):
        while mu_b - mu_a > eps:
            mu = (mu_b + mu_a) / 2.

            bE = (mesK * Q * mu).T
            E_new = la.spsolve((- aDiffE.tocsc() - mu * aConVE + Id.multiply(mesK * gF * t_s)), bE)
            E.setValue(E_new)
            E.updateOld()

            F_mu_m = numerix.sum(mesK * delta_tild.value * E.value) - 1.

            bE = (mesK * Q  * mu_a).T
            E_new = la.spsolve((- aDiffE.tocsc() - mu_a * aConVE + Id.multiply(mesK * gF * t_s)), bE)
            E.setValue(E_new)
            E.updateOld()

            F_mu_a = numerix.sum(mesK * delta_tild.value * E.value) - 1.

            # print ('{0} x {1}'.format(F_mu_m, F_mu_a))

            if F_mu_m * F_mu_a <= 0:
                mu_b = mu
            else:
                mu_a = mu
        return mu

    def not_converge(self, x, y):
        if (abs(x - y) / y) <= 1e-6:
            return True
        else:
            return False

    def simulate(self, process_id=None, matlab_engine=None):
        step = 0
        res = asarray([])
        print(self.p['a'].flatten())
        print("PARAM/LA TAILLE: {0}/{1}".format(self.p["a"], self.p["a"].shape))

        print('HIHIHIHIHIHI  ', float64(self.p["a"]))
        for idx in range(self.p["a"].shape[0]):
            # print(self.p["a"]*self.p['D']*self.p['sF'])
            t_s = 1. / self.p["a"][idx]
            x_s = sqrt(self.p["D"][idx] * t_s)
            c_s = 1. / (t_s * (x_s ** 2) * self.p["delta"][idx])
            # nu = (D/a)*sqrt(D/a)*(S*d*delta)/(chi*sF)
            # mu1_s = mu1_tild
            mu1_s = c_s / (self.p["R"][idx] * t_s)
            # mu0_s = a * mu1_s / V

            phi_s = (x_s ** 2) / (mu1_s * t_s * self.p["chi"][idx])

            # Q = S*mu1_s*t_s/c_s
            Q = 1.
            # print(self.p["sF"] )
            U = (self.p["sF"][idx] * x_s ** 2) / (self.p["K"][idx] * phi_s)
            # print('VOICI LA VALEUR DU PARAMETRE: {0}'.format(self.p["sF"][0]))

            radius = 1. / x_s
            cellSize = radius/10.

            mesh = Gmsh2D('''
                           cellSize = %(cellSize)g;
                           radius = %(radius)g;
                           Point(1) = {0, 0, 0, cellSize};
                           Point(2) = {-radius, 0, 0, cellSize};
                           Point(3) = {0, radius, 0, cellSize};
                           Point(4) = {radius, 0, 0, cellSize};
                           Point(5) = {0, -radius, 0, cellSize};
                           Circle(6) = {2, 1, 3};
                           Circle(7) = {3, 1, 4};
                           Circle(8) = {4, 1, 5};
                           Circle(9) = {5, 1, 2};
                           Line Loop(10) = {6, 7, 8, 9};
                           Plane Surface(11) = {10};
                           ''' % locals())
            # print('je suis ici')
            x = mesh.cellCenters
            xt, yt = mesh.cellCenters
            nVol = mesh.numberOfCells
            nFaces = mesh.numberOfFaces

            intF = mesh.interiorFaceIDs
            extF = arange(0, nFaces, 1)[array(mesh.exteriorFaces)]

            intFacesCells = mesh.faceCellIDs[:, intF]
            extFacesCells = mesh.faceCellIDs[:, extF]

            TKL = mesh._calcFaceAreas() / mesh._calcFaceToCellDistAndVec()[0].sum(axis=0)
            mes_edge = mesh._calcFaceAreas()
            mesK = mesh.cellVolumes
            # ------------------------------------------ The Chemical Potential ------------------------------
            aDiffP = zeros((nVol, nVol))
            aDiffP = sp.csc_matrix(aDiffP)
            aDiffP = aDiffP + sp.coo_matrix((-TKL[intF], (intFacesCells[0], intFacesCells[0])), shape=(nVol, nVol))
            aDiffP = aDiffP + sp.coo_matrix((TKL[intF], (intFacesCells[0], intFacesCells[1])), shape=(nVol, nVol))
            aDiffP = aDiffP + sp.coo_matrix((TKL[intF], (intFacesCells[1], intFacesCells[0])), shape=(nVol, nVol))
            aDiffP = aDiffP + sp.coo_matrix((-TKL[intF], (intFacesCells[1], intFacesCells[1])), shape=(nVol, nVol))

            # -----------------------------------Neumann Boundary condition------------------------------------------
            aDiffP = aDiffP + sp.coo_matrix((0. * TKL[extF], (extFacesCells[0], extFacesCells[0])), shape=(nVol, nVol))

            e = ones((1, nVol))
            EaDiffP = sp.csc_matrix(concatenate((concatenate((aDiffP.T.todense(), (mesK * e).T), axis=1),
                                                    array([append((mesK * e).T, 0.)])), axis=0))

            # -----------------------------------Dirichlet Boundary condition------------------------------------------
            test = CellVariable(mesh=mesh, value=0.)
            phi = CellVariable(name="$\phi(t,x,y)$", mesh=mesh, value=0.0, hasOld=1)
            # sF = sigmaF.SigmaF2D(params.sF, xt, yt, Rs=0.05)
            sF = sigmaF.SigmaF2D(1. / x_s, xt, yt, Rs=0.05 / (x_s ** 2))

            F = sF
            extendedF = append(mesK * U * F, 0.)
            phi_new = la.spsolve(EaDiffP, extendedF)
            phi.setValue(phi_new[0:nVol])
            phi.updateOld()

            # ------------------------------------------ The Chemoattractant ------------------------------
            aDiffE = zeros((nVol, nVol))
            aDiffE = sp.csc_matrix(aDiffE)
            aDiffE = aDiffE + sp.coo_matrix((-TKL[intF], (intFacesCells[0], intFacesCells[0])), shape=(nVol, nVol))
            aDiffE = aDiffE + sp.coo_matrix((TKL[intF], (intFacesCells[0], intFacesCells[1])), shape=(nVol, nVol))
            aDiffE = aDiffE + sp.coo_matrix((TKL[intF], (intFacesCells[1], intFacesCells[0])), shape=(nVol, nVol))
            aDiffE = aDiffE + sp.coo_matrix((-TKL[intF], (intFacesCells[1], intFacesCells[1])), shape=(nVol, nVol))

            # -----------------------------------Dirichlet Boundary condition------------------------------------------
            aDiffE = aDiffE + sp.coo_matrix((-TKL[extF], (extFacesCells[0], extFacesCells[0])), shape=(nVol, nVol))

            aConVE = zeros((nVol, nVol))
            aConVE = sp.csc_matrix(aConVE)

            dPhi_int = numerix.dot(phi.faceGrad.value, mesh.faceNormals)[intF]

            aConVE = aConVE + sp.coo_matrix((mes_edge[intF] * plus(dPhi_int), (intFacesCells[0], intFacesCells[0])),
                                            shape=(nVol, nVol))
            aConVE = aConVE + sp.coo_matrix((-mes_edge[intF] * minus(dPhi_int), (intFacesCells[0], intFacesCells[1])),
                                            shape=(nVol, nVol))
            aConVE = aConVE + sp.coo_matrix((-mes_edge[intF] * plus(dPhi_int), (intFacesCells[1], intFacesCells[0])),
                                            shape=(nVol, nVol))
            aConVE = aConVE + sp.coo_matrix((mes_edge[intF] * minus(dPhi_int), (intFacesCells[1], intFacesCells[1])),
                                            shape=(nVol, nVol))

            dPhi_ext = numerix.dot(phi.faceGrad.value, mesh.faceNormals)[extF]

            aConVE = aConVE + sp.coo_matrix((mes_edge[extF] * plus(dPhi_ext), (extFacesCells[0], extFacesCells[0])),
                                            shape=(nVol, nVol))

            Id = sp.spdiags(numerix.ones(nVol), [0], nVol, nVol)

            # ---------------Variables and parameters for the Immune Cells Displacement equation---------
            # E = CellVariable(name="$E(t,x,y)$", mesh=mesh, value=0.53235e6/c_s, hasOld=1)
            E = CellVariable(name="$E(t,x,y)$", mesh=mesh, value=0., hasOld=1)

            delta_tild = CellVariable(name="$\delta_t(x,y)$", mesh=mesh, value=0.)
            delta_tild.setValue(delt.GaussianImmuneResponse2D(1. / x_s, xt, yt, Ra=0.02 / x_s ** 2))

            gF = self.p["gF"][idx]

            Id = sp.spdiags(numerix.ones(nVol), [0], nVol, nVol)

            # ---------------------------------------------- Dichotomie Method --------------------------------------------

            mu_a = 0.
            mu_b = 1.

            F_mu_m = 0.
            F_mu_a = 0.

            eps = 1e-10

            mu = self.dichotomy(mu_a, mu_b, eps, mesK, Q,  aDiffE, aConVE, Id, gF, t_s, E, delta_tild)

            while self.not_converge(mu, mu_b):
                # print(mu, mu_b)
                mu_b = (mu_a + mu_b) / 2.
                mu = self.dichotomy(mu_a, mu_b, eps, mesK, Q, aDiffE, aConVE, Id, gF, t_s, E, delta_tild)
            print('Step -- > {0}'.format(step))
            step = step + 1
            res = append(res, [mu*mu1_s*1e-9])

            print('mu:{0}'.format(mu))
        # res = np.asarray([mu*mu1_s*1e-9])
        # res[:, newaxis]
        res = log(res[:, newaxis])
        return res

def norm_n(V, dx, n):
    if n == 0:
        c_max = max(abs(V))
        yield c_max
    else:
        norme = sum(abs(dx*V)**n)
        yield norme**(1./n)

def mo(x):
    return numerix.L2norm(x)

def plus(z):
    return 0.5*(z+abs(z))

def minus(z):
    return 0.5*(-z+abs(z))

def alphan(n):
    if n == 0:
        return 1
    return 2 * alphan(n - 1.) / ((2. ** n) - 1.)

def toss(deb, fin):
    return random.uniform(deb, fin)

# --------------- Sensitivity Analysis---------------------------
# Create the coffee cup model
# model = un.Model(run=evaluate_mu_un, labels=["tumor volume($mm^3$)"])

model = MyModel()

parameters = OrderedDict()

parameters["a"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[0.1, 0.5])
# parameters["a"] = pygpc.Norm(pdf_shape=[0.2, 0.09])
parameters["D"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[8.64e-5, 1e-3])
# parameters["D"] = [8.64e-5]
parameters["delta"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[1., 60.])
# parameters["R"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[7.573e-8, 1.231e-6])
# parameters["R"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[6.456e-8, 1.520e-6])#IC1
# parameters["R"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[5.5e-7, 1.036e-6])#IC3_99
parameters["R"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[6.11e-7, 9.74e-7])#IC4_95
# parameters["R"] = pygpc.Norm(pdf_shape=[7.923174114490609e-07, 7.945822739100839e-15])
parameters["chi"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[86.4, 86.4e5])
# parameters["chi"] = [86.4]
parameters["sF"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[5e-17, 0.625e-16])
# parameters["sF"] = [5e-17]
parameters["K"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[1e-2, 1.])
# parameters["K"] = [1e-2]
# parameters["gF"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[2e-2, 1.])
parameters["gF"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[2e-2, 1.])

interval = 'IC4_hetero'
# parameters["chi"] = 86.4
# parameters["sF"] = 5e-17
# parameters["K"] = 1e-2
# parameters["gF"] = 2e-2

problem = pygpc.Problem(model=model, parameters=parameters)

# basis = pygpc.Basis()
# basis.init_basis_sgpc(problem=problem,
#                       order=[5, 5, 5],
#                       order_max=15,
#                       order_max_norm=1,
#                       interaction_order=3)

# basis.plot_basis(dims=[0, 1, 2])

#
fn_results = 'Sensitivity_data/PCE_data'.format(interval)
save_session_format = ".hdf5"
# ---------------------------------- Personnalized Options ------------------------------
options = dict()
options["method"] = "reg"
# options["method"] = "quad"
options["solver"] = "Moore-Penrose"
# options["solver"] = "OMP"
options["settings"] = None
options["order"] = [5] * problem.dim # The univariate polynomials expansion orders
options["order_max"] = 5
options["order_max_norm"] = 0.7
# options["order_max_norm"] = 1.
options["interaction_order"] = 2
# options["interaction_order"] = 2
options["matrix_ratio"] = 2
# options["error_type"] = "nrmsd"
options["error_type"] = "loocv"
options["n_samples_validation"] = 1e3
options["n_cpu"] = 2
options["fn_results"] = fn_results
options["save_session_format"] = save_session_format
options["gradient_enhanced"] = False
options["gradient_calculation"] = "FD_1st2nd"
options["gradient_calculation_options"] = {"dx": 0.001, "distance_weight": -2}
options["backend"] = "omp"
# options["grid"] = pygpc.Random
# options["grid"] = pygpc.LHS(parameters_random=problem.parameters_random, seed=1)
options["grid_options"] = None

n_coeffs = pygpc.get_num_coeffs_sparse(order_dim_max=options["order"],
                                       order_glob_max=options["order_max"],
                                       order_inter_max=options["interaction_order"],
                                       dim=problem.dim)
# problem.dim
grid = pygpc.LHS(parameters_random=problem.parameters_random,
                    n_grid=options["matrix_ratio"] * n_coeffs,
                    seed=1)
# grid = pygpc.Random(parameters_random=problem.parameters_random,
#                     n_grid=options["matrix_ratio"] * n_coeffs,
#                     seed=1)
# print('taille grille', grid.n_grid)
# options["fn_results"] = 'Sensitivity_data/PCE_data_{0}'.format(grid.n_grid)

algorithm = pygpc.Static(problem=problem, options=options, grid=grid)
#
# gpc, coeffs, results = algorithm.run()
session = pygpc.Session(algorithm=algorithm)
# # session.grid = algorithm.grid
#
# # #
# # # # # run gPC session
session, coeffs, results = session.run()

dataPath = 'Sensitivity_data/Pygpc_Sobol_idx.txt'.format(interval)

outF = open(dataPath, "w")

mean = session.gpc[0].get_mean(coeffs)
# outF.write('Mean: '+mean)
print("Mean: {}".format(mean))

std = session.gpc[0].get_std(coeffs)
# outF.write('Std: '+std)
print("Std: {}".format(std))

sobol, sobol_idx, sobol_idx_bool = modified_get_sobol_indices(session.gpc[0], coeffs, n_samples=10)

n_idx = len(sobol_idx)
for i in range(n_idx):
    print("Parameter x{}: {}".format(sobol_idx[i]+1, sobol[i][0]))
    str_tmp = ''
    for k in range(problem.dim):
        if len(sobol_idx[i])==k+1:
            if k+1==1:
                str_tmp = str(sobol_idx[i][k] + 1)
            elif k+1>1:
                for m in range(k):
                    str_tmp = str_tmp + str(sobol_idx[i][m] + 1)+' '
                str_tmp = str_tmp + str(sobol_idx[i][k] + 1)
            outF.write(str_tmp +','+str(sobol[i][0]))
            outF.write('\n')

print(sobol_idx_bool)

outF.close()

pygpc.validate_gpc_plot(session=session,
                        coeffs=coeffs,
                        random_vars=["a", "delta"],
                        n_grid=[25, 25],
                        output_idx=0,
                        fn_out=session.fn_results+'plot',
                        folder="gpc_vs_original_plot",
                        n_cpu=options["n_cpu"])

# Validate gPC approximation vs original model function using Monte Carlo simulation
nrmsd = pygpc.validate_gpc_mc(session=session,
                              coeffs=coeffs,
                              n_samples=1e3,
                              fn_out=session.fn_results+'mc',
                              n_cpu=options["n_cpu"])

reason: from numpy,import scipy


text:
from astropy.coordinates import SkyCoord, match_coordinates_sky
from astropy.coordinates.name_resolve import NameResolveError
from astropy.table import Table, MaskedColumn
from astropy import units as u

import pickle

import targets

try:
    with open("data/target_coords.pickle", "rb") as coords_file:
        target_coords = pickle.load(coords_file)
except (FileNotFoundError, EOFError):
    target_coords = {}

for target_name in targets.TARGETS:
    if target_name not in target_coords:
        try:
            target_coords[target_name] = SkyCoord.from_name(target_name, parse=True)
        except NameResolveError:
            continue

with open("data/target_coords.pickle", "wb") as coords_file:
    pickle.dump(target_coords, coords_file)

target_coords = Table(
    {
        "target_name": list(target_coords.keys()),
        "coords": SkyCoord(list(target_coords.values())),
    }
)

def match_targets(table):
    if "coords" not in table.colnames:
        table["coords"] = SkyCoord.guess_from_table(table)
    nearest_matches, nearest_separations, nearest_distances = match_coordinates_sky(
        table["coords"], target_coords["coords"]
    )
    mask = nearest_separations > (1e-3 * u.deg)
    table["matched target"] = MaskedColumn(
        tuple(target_coords[n]["target_name"] for n in nearest_matches), mask=mask,
    )
    table["matched target separation"] = MaskedColumn(nearest_separations, mask=mask,)

max_stars_repo_name: adammcmaster/pirate-photometry

max_issues_repo_name: adammcmaster/pirate-photometry

max_forks_repo_name: adammcmaster/pirate-photometry

reason: from astropy


text:
import time
import datetime
import pytz
import argparse
import numpy as np
import torch
import matplotlib.pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
import os
from PIL import Image
import torchvision.utils as vutils

from model.net import MonocularDepthModel
from model.loss import LossNetwork, combined_loss, mean_l2_loss
from model.metrics import evaluate_predictions
from model.dataloader import DataLoaders, get_test_dataloader
from utils import *
from evaluate import infer_depth, evaluate

class Trainer():
  def __init__(self, data_path, test_data_path):
    self.dataloaders = DataLoaders(data_path)
    self.test_data_path = test_data_path

  def train_and_evaluate(self, config):
    batch_size = config['batch_size']
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

    train_dataloader = self.dataloaders.get_train_dataloader(batch_size = batch_size)
    num_batches = len(train_dataloader)

    test_dataloader = get_test_dataloader(self.test_data_path, config['test_batch_size'], shuffle=True)
    model = MonocularDepthModel(backbone = config['backbone'])

    model = model.to(device)
    params = [param for param in model.parameters() if param.requires_grad == True]
    optimizer = torch.optim.Adam(params, config['lr'])

    loss_model = LossNetwork().to(device)

    if config['checkpoint']:
      load_checkpoint(config['checkpoint'], model, optimizer)

    print('Training...')

    for epoch in range(config['epochs']):

      accumulated_per_pixel_loss = RunningAverage()
      accumulated_feature_loss = RunningAverage()
      accumulated_iteration_time = RunningAverage()
      epoch_start_time = time.time()

      for iteration, batch in enumerate(train_dataloader):
        model.train()
        time_start = time.time()

        optimizer.zero_grad()
        images, depths = batch['img'], batch['depth']
        images = normalize_batch(torch.autograd.Variable(images.to(device)))
        depths = torch.autograd.Variable(depths.to(device))

        predictions = model(images)

        predictions_normalized = normalize_batch(predictions)
        depths_normalized = normalize_batch(depths)

        feature_losses_predictions = loss_model(predictions_normalized)
        feature_losses_depths = loss_model(depths_normalized)

        per_pixel_loss = combined_loss(predictions, depths)
        accumulated_per_pixel_loss.update(per_pixel_loss, images.shape[0])

        feature_loss = config['perceptual_weight'] * mean_l2_loss(feature_losses_predictions.res1, feature_losses_depths.res1)
        accumulated_feature_loss.update(feature_loss, images.shape[0])

        total_loss = per_pixel_loss + feature_loss
        total_loss.backward()
        optimizer.step()

        time_end = time.time()
        accumulated_iteration_time.update(time_end - time_start)
        eta = str(datetime.timedelta(seconds = int(accumulated_iteration_time() * (num_batches - iteration))))

        if iteration % config['log_interval'] == 0:
          print(datetime.datetime.now(pytz.timezone('Asia/Kolkata')), end = ': ')
          print('Epoch: %d [%d / %d] ; it_time: %f (%f) ; eta: %s' % (epoch, iteration, num_batches, time_end - time_start, accumulated_iteration_time(), eta))
          print('Average per-pixel loss: %f; Average feature loss: %f' % (accumulated_per_pixel_loss(), accumulated_feature_loss()))
          metrics = evaluate_predictions(predictions, depths)

      epoch_end_time = time.time()
      print('Epoch %d complete, time taken: %s' % (epoch, str(datetime.timedelta(seconds = int(epoch_end_time - epoch_start_time)))))
      torch.cuda.empty_cache()

      save_checkpoint({
                  'iteration': num_batches * epoch + iteration,
                  'state_dict': model.state_dict(),
                  'optim_dict': optimizer.state_dict()}, config['checkpoint_dir'])

      print('Epoch %d saved\n\n' % (epoch))

      # EVALUATE ON TEST DATA:

      test_metrics = evaluate(model, test_dataloader, model_upsample = True)

      random_test_batch = next(iter(test_dataloader))
      log_images = random_test_batch[0]['img']
      log_depths = random_test_batch[0]['depth']
      log_preds = torch.cat([infer_depth(img, model, upsample = True)[0].unsqueeze(0) for img in log_images], dim = 0)

  def get_with_colormap(self, plots):
    images = []
    for plot in plots:
      plt.imsave('_.png', plot, cmap='jet')
      img = Image.open('_.png')
      os.remove('_.png')
      images.append(img)
    return images

if __name__ == '__main__':
  parser = argparse.ArgumentParser(description='Training of depth estimation model')
  '''REQUIRED ARGUMENTS'''
  parser.add_argument('--train_dir', help='Train directory path - should contain the \'data\' folder', required = True)
  parser.add_argument('--test_dir', help='Test directory path - should contain 3 files', required = True)
  parser.add_argument('--batch_size', type=int, help='Batch size to process the train data', required = True)
  parser.add_argument('--checkpoint_dir', help='Directory to save checkpoints in', required = True)
  parser.add_argument('--epochs', type = int, help = 'Number of epochs', required = True)
  '''OPTIONAL ARGUMENTS'''
  parser.add_argument('--checkpoint', help='Model checkpoint path', default = None)
  parser.add_argument('--lr', help = 'Learning rate', default = 3e-4)
  parser.add_argument('--log_interval', help = 'Interval to print the avg. loss and metrics', default = 50)
  parser.add_argument('--backbone', type=str, help = 'Model backbone: densenet161 or densenet121', default = 'densenet161')
  parser.add_argument('--test_batch_size', type=int, help='Batch size for frequent testing', default = 2)
  parser.add_argument('--perceptual_weight', type=int, help='Weight for the perceptual loss', default = 0.5)

  args = parser.parse_args()

  if args.backbone not in ('densenet161', 'densenet121'):
    raise Exception('Invalid backbone specified!')

  if not os.path.isdir(args.checkpoint_dir):
    os.mkdir(args.checkpoint_dir)

  trainer = Trainer(args.train_dir, args.test_dir)
  trainer.train_and_evaluate(vars(args))

max_stars_repo_name: aiarjun/Monocular-Depth-Estimation

max_issues_repo_name: aiarjun/Monocular-Depth-Estimation

max_forks_repo_name: aiarjun/Monocular-Depth-Estimation


text:
#!/usr/bin/env python

import sys,time

import rospy
import roslib
import numpy as np

from cv_bridge import CvBridge, CvBridgeError

import cv2

from sensor_msgs.msg import Image, CameraInfo
from std_msgs.msg import Header

#np.set_printoptions(threshold=sys.maxsize)

class CutterNode:
    def __init__(self):
        self.image = np.zeros([480,640])
        self.depth = np.zeros([480,640])
        self.depth_output = np.zeros([480,640])
        self.camera_info_timestamp = 0
        self.camera_info_dump = CameraInfo()

        '''Initialize ros publisher, ros subscriber'''
        # topic where we publish
        self.depth_aligned_pub = rospy.Publisher("/cutter_node_align_depth",Image, queue_size=10)
        self.camera_info_pub = rospy.Publisher("/cutter_node_camera_info",CameraInfo, queue_size=1)
        # cv bridge
        self.cv_bridge = CvBridge()
        # subscribed Topic
        self.image_subscriber = rospy.Subscriber("/grabcut",Image, self.callback_image, queue_size=1)
        self.depth_subscriber = rospy.Subscriber("/align_depth",Image, self.callback_depth, queue_size=1)
        self.camera_info_subscriber = rospy.Subscriber("/camera_info",CameraInfo, self.callback_info, queue_size=1)

    def callback_image(self,data):
        # filter image elementwise numpy
        try:
            cv_image = self.cv_bridge.imgmsg_to_cv2(data, "mono8")
            self.image = cv_image
        except CvBridgeError as e:
            print(e)

        # compare two images

        self.depth_output = np.zeros([480,640])

        ret,thresh1 = cv2.threshold(cv_image,10.0,255.0,cv2.THRESH_BINARY)

        start = time.time()

        self.depth_output =  thresh1 * self.depth * 1.0 / 255.0

        end = time.time()
        #print(end - start)
        try:
            self.align_message = self.cv_bridge.cv2_to_imgmsg(self.depth_output, "32FC1")
            self.align_message.header.frame_id = "map"
            #self.align_message.header.stamp = self.camera_info_dump.header.stamp
            self.depth_aligned_pub.publish(self.align_message)
            #self.camera_info_pub.publish(self.camera_info_dump)
            # before
            #self.depth_aligned_pub.publish(self.cv_bridge.cv2_to_imgmsg(self.depth_output, "32FC1"))
        except CvBridgeError as e:
            print(e)
        ## debug
        #print(thresh1)
        cv2.imshow("cutter_node_depth", self.depth_output)
        cv2.imshow("cutter_node_grey", thresh1)
        cv2.waitKey(3)

    def callback_depth(self,data):
        # filter image elementwise numpy
        try:
            cv_image = self.cv_bridge.imgmsg_to_cv2(data, "32FC1")
            # Convert the depth image to a Numpy array since most cv2 functions require Numpy arrays.
            cv_image_array = np.array(cv_image, dtype = np.dtype('f4'))
            #cv_image_array = cv_image_array * 255.0
            # Normalize the depth image to fall between 0 (black) and 1 (white)
            cv_image_norm = cv2.normalize(cv_image_array, cv_image_array, 0, 1, cv2.NORM_MINMAX)
            self.depth = cv_image_norm
        except CvBridgeError as e:
            print(e)

    def callback_info(self,data):
        try:
            self.camera_info_dump = data
            #self.camera_info_timestamp = data.header.stamp
        except e:
            print(e)

def main(args):
    ## debug
    cv2.namedWindow('cutter_node_depth')
    cv2.namedWindow('cutter_node_grey')
    rospy.init_node('cutter_node',anonymous=True)
    cn = CutterNode()
    try:
        rospy.spin()
    except KeyboardInterrupt:
        print("Shutting down ROS cutter node")
    ## debug
    cv2.destroyAllWindows()

if __name__ == "__main__":
    main(sys.argv)


text:
#encoding=utf-8

import cv2
import keras
from keras.applications.imagenet_utils import preprocess_input
from keras.backend.tensorflow_backend import set_session
from keras.models import Model
from keras.preprocessing import image
import matplotlib.pyplot as plt
import numpy as np
import pickle
from random import shuffle
# from scipy.misc import imread
# from scipy.misc import imresize
import tensorflow as tf
import os
from Util.enhance import random_sized_crop
from Network.SSD300.ssd import SSD300
from Network.SSD300.ssd_training import MultiboxLoss
from Network.SSD300.ssd_utils import BBoxUtility
from Util.metrics import evaluate_detections
from Util.mylogger import MyLogger
from Util.decrypt import decrypt_txt_file

# %matplotlib inline
plt.rcParams['figure.figsize'] = (8, 8)
plt.rcParams['image.interpolation'] = 'nearest'

np.set_printoptions(suppress=True)

# config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
# set_session(tf.Session(config=config))
__MODE__ = 'RELEASE'#'RELEASE'
def debug_print(x):
    if __MODE__ =='DEBUG':
        print(x)

class SSD_Model:
    def __init__(self,class_count=4, input_shape=(300, 300, 3),lr = 3e-4, num_freeze_layer=11):
        '''
        Management of training and predict.
        Parameters are hyperparameters that allow adjustments.

        Args:
            class_count -- number of categories to identify.
            input_shape -- The shape of input data to resized.
            lr -- learning rate
            num_freeze_layer -- The number of layer to be freeze.
        '''
        self.class_names = None
        self.class_count = class_count
        self.input_shape = input_shape
        self.name = 'SSD300'
        self.lr = lr
        self.num_freeze_layer = num_freeze_layer
        print('Freeze:',num_freeze_layer)

        if os.path.exists(os.path.join('Network','SSD300','prior_boxes_ssd300.pkl')):
            priors = pickle.load(open(os.path.join('Network','SSD300','prior_boxes_ssd300.pkl'), 'rb'))
            print('Priors load success.')
        self.bbox_util = BBoxUtility(self.class_count, priors)
        # path_prefix = '../../frames/'

        self.model = SSD300(input_shape, num_classes=self.class_count)
        if os.path.exists(os.path.join('Network','SSD300','weights_SSD300.hdf5')):
            self.model.load_weights( os.path.join('Network','SSD300','weights_SSD300.hdf5'), by_name=True)
            print('Load pre-trained weights success')

        layers = ['input_1', 'conv1_1', 'conv1_2', 'pool1',
                'conv2_1', 'conv2_2', 'pool2',
                'conv3_1', 'conv3_2', 'conv3_3', 'pool3',
                'conv4_1', 'conv4_2', 'conv4_3', 'pool4']

        freeze = layers[:num_freeze_layer]

        self.batch_size=4
        for L in self.model.layers:
            if L.name in freeze:
                L.trainable = False
        self.gt_info_test = None

        # def schedule(epoch, decay=0.9):
        #     return base_lr * decay**(epoch)

        # callbacks = [keras.callbacks.ModelCheckpoint('./checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5',
        #                                             verbose=1,
        #                                             save_weights_only=True),
        #             keras.callbacks.LearningRateScheduler(schedule)]

        optim = keras.optimizers.Adam(lr=lr)
        # optim = keras.optimizers.RMSprop(lr=base_lr)
        # optim = keras.optimizers.SGD(lr=base_lr, momentum=0.9, decay=decay, nesterov=True)
        self.model.compile(optimizer=optim,loss=MultiboxLoss(self.class_count, neg_pos_ratio=2.0).compute_loss)

    def set_dataset(self, dataset):
        self.dataset = dataset
        self.train_keys = dataset.train_keys
        self.val_keys = dataset.val_keys
        self.class_count = dataset.class_count
        self.class_names = dataset.class_names
        self.gt_boxes = dataset.gt_boxes
        self.dataset.bbox_util = self.bbox_util
        self.data_path = dataset.data_path
        self.serval_path = dataset.anno_path

    def train(self):  # Train on the whole train set batch for once.
        print('Start training.')
        assert self.dataset.bbox_util is not None, 'Set dataset before start training.'
        history = self.model.fit_generator(self.dataset.generate_train(), self.dataset.train_batches,
                              1, verbose=2,#callbacks=callbacks,
                              validation_data=self.dataset.generate_vaild(),
                              nb_val_samples=self.dataset.val_batches,
                              nb_worker=1)
        return history

    def get_gt_info_test(self):
        '''for metrics, get annotation info. '''
        if self.gt_info_test==None:
            self.gt_info_test = {}
            for k in self.class_names[1:]:
                self.gt_info_test[k] = []
            for name in self.val_keys:
                gt = self.gt_boxes[name]
                gt.astype(float)
                for k in self.gt_info_test.keys():
                    self.gt_info_test[k].append({'bbox':[],'det':[],'difficult':[]})
                for x in gt:
                    if x[4]==0:
                        continue
                    self.gt_info_test[self.class_names[x[4]]][-1]['bbox'].append(x)
                    self.gt_info_test[self.class_names[x[4]]][-1]['det'].append(False)
                    self.gt_info_test[self.class_names[x[4]]][-1]['difficult'].append(False)
            return self.gt_info_test
        else:
            return self.gt_info_test

    def metrics(self, model_path):
        '''Get mAP, recall, precision // Return: {'mAP':., 'rec':., 'prec':.}'''
        self.gt_val_keys = [os.path.join(self.data_path, l) for l in self.val_keys]
        all_boxes = [[[] for _ in range(len(self.val_keys))] for _ in range(len(self.class_names))]
        print('Val key length:',len(self.gt_val_keys))
        for img_i, test_img_path in enumerate(self.gt_val_keys):

            test_result = self.predict(img_path=test_img_path, model_path=model_path,is_metrics=True)
            print('Finish predict the {} val image for metrics.'.format(img_i))
            #print('test_result:',test_result)
            test_result.sort(key=lambda x:float(x[4]), reverse=True)
            if len(test_result) > 58:
                test_result = test_result[:58]
            img_obj_out = image.load_img(test_img_path, target_size=(300, 300))
            img_obj_out = image.img_to_array(img_obj_out)
            for obj in test_result:
                if obj[-1]>0:
                    cv2.rectangle(img_obj_out, (int(obj[0]),int(obj[1])), (int(obj[2]),int(obj[3])), (255,0,0), 2)
            cv2.imwrite(os.path.join('./test_img_output/',os.path.split(test_img_path)[-1]),img_obj_out)
            if len(test_result)==0:
                for cls_j in range(1,len(self.class_names)):
                    all_boxes[cls_j][img_i] = np.asarray([]).reshape(0,5).astype(np.float32,copy=False)
                continue # fix bug of line 151 shape error.

            test_result = np.asarray(test_result)
            for cls_j in range(1,len(self.class_names)):
                #print ('test_result.shape:',test_result.shape)
                coords_and_label = test_result[np.where(test_result[:,-1]==cls_j)]
                all_boxes[cls_j][img_i] = coords_and_label[:,:-1].astype(np.float32, copy=False)
        all_gt_infos = self.get_gt_info_test().copy()
        mAp,rec,prec = evaluate_detections(all_boxes, all_gt_infos, self.class_names )
        return {'mAP':mAp, 'rec':rec, 'prec':prec}

    def visualize_val_set(self, save_dir, model_path):
        '''Visualize val set when training model'''
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        for name in self.val_keys:
            path = os.path.join(self.data_path,name)
            test_result = self.predict(img_path=path, model_path=model_path,is_metrics=True)
            test_result.sort(key=lambda x:float(x[4]), reverse=True)
            if len(test_result) > 58:
                test_result = test_result[:58]
            img = cv2.imread(path)
            for rec in test_result:
                cv2.rectangle(img, (rec[0],rec[1]),(rec[2],rec[3]),(255,0,0),thickness=2)
            cv2.imwrite(os.path.join(save_dir,name),img)

    def predict(self,img_array=None,img_path=None, model_path=None,is_metrics=False, visualize_path=None, anno_path=None):
        '''
        Predict one image.
        Return:
            for just predict one image: [label, class_name, score, (left, top), (right, bottom)]
            for matrics: [xmin,ymin,xmax,ymax,score,label]
        Args:
            img_array -- None when img_path is not None. Image numpy array.[width, hight, channel]
            img_path -- None when img_array is not None. Image abs path.
            model_path -- SSD300 model path for prediction.
            is_metrics -- Is get info to do metrics when training.(False when you just predict image)
            visualize_path -- Visualize path.(Not None if you want to visualize image prediction results)
        '''
        if model_path is not None:
            self.model.load_weights(model_path)
        if self.class_names is None:
            assert anno_path is not None, "Set class names first."
            _, contents = decrypt_txt_file(anno_path)
            contents = contents.split('\n')
            self.class_names = contents[1].split(',')
            self.class_names = [cn.split(':')[1] for cn in self.class_names]
        inputs = []
        images = []
        try:
            if not img_path==None:
                img = cv2.imread(img_path)
                img = cv2.resize(img,(300,300),interpolation=cv2.INTER_NEAREST)
                images.append(cv2.cvtColor(cv2.imread(img_path),cv2.COLOR_BGR2RGB))
                images.append(cv2.imread(img_path))

                inputs.append(img.copy())
                inputs = preprocess_input(np.array(inputs))
            else:
                print('go')
                img = img_array.copy()
                img = cv2.resize(img,(300,300),interpolation=cv2.INTER_NEAREST)
                images.append(img_array)
                inputs.append(img.copy())
                inputs = preprocess_input(np.array(inputs))
        except Exception as e:
            print('img_path wrong:',e)
            return []

        preds = self.model.predict(inputs, batch_size=1, verbose=0)
        results = self.bbox_util.detection_out(preds)

        if len(results[0])==0:
            return []
        elif len(results)==0:
            return []

        result = []

        if not visualize_path==None:
            img_name = img_path.split(os.sep)[-1]
            print('new visual dir')
            if not os.path.exists(visualize_path):
                print('make new visual dir')
                os.mkdir(visualize_path)
            save_path = os.path.join(visualize_path, img_name)
        for i, img in enumerate (images):
            # Parse the outputs.
            det_label = results[i][:, 0]
            det_conf = results[i][:, 1]
            det_xmin = results[i][:, 2]
            det_ymin = results[i][:, 3]
            det_xmax = results[i][:, 4]
            det_ymax = results[i][:, 5]

            # Get detections with confidence higher than 0.6.
            top_indices = [i for i, conf in enumerate(det_conf) if conf >= 0.6]

            top_conf = det_conf[top_indices]
            top_label_indices = det_label[top_indices].tolist()
            top_xmin = det_xmin[top_indices]
            top_ymin = det_ymin[top_indices]
            top_xmax = det_xmax[top_indices]
            top_ymax = det_ymax[top_indices]

            for i in range(top_conf.shape[0]):
                xmin = int(round(top_xmin[i] * img.shape[1]))
                ymin = int(round(top_ymin[i] * img.shape[0]))
                xmax = int(round(top_xmax[i] * img.shape[1]))
                ymax = int(round(top_ymax[i] * img.shape[0]))
                score = top_conf[i]
                label = int(top_label_indices[i])
                if not visualize_path==None:
                    print('rec')
                    cv2.rectangle(img,(xmin,ymin),(xmax,ymax),(255,0,0),thickness=2)
                if is_metrics==False:
                    result.append([label,self.class_names[label], score, (xmin,ymin),(xmax,ymax)])
                else:   #just for metrics.
                    result.append([xmin,ymin,xmax,ymax,score,label])
            if not visualize_path==None:
                cv2.imwrite(save_path,img)
            print('Finish predict')
            return result

class SSD_DataLoader(object):
    def __init__(self, anno_path,data_path, bbox_util=None,
                 batch_size=8,
                 image_shape=(300,300),
                 do_crop = True,
                 extra_params=None
                 ):
        '''
        Manager of SSD datasets.

        Args:
            anno_path -- Path of annotation file.
            batch_size -- Num of one batch.
            data_path -- Path of dataset.
            image_shape -- The shape of the image to be resized.
            extra_params -- Custom parameter.
        '''
        self.gt = self.serval_to_dict(anno_path, data_path)
        keys = list(self.gt.keys()).copy()
        import random
        random.shuffle(keys)
        num_train = int(round(0.9 * len(keys)))
        self.train_keys = keys[:num_train]
        self.val_keys = keys[num_train:]
        self.anno_path = anno_path
        self.bbox_util = bbox_util
        self.batch_size = batch_size
        self.data_path = data_path
        self.train_batches = (len(self.train_keys)-1)//self.batch_size+1
        self.val_batches = (len(self.val_keys)-1)//self.batch_size+1
        self.image_shape = image_shape
        self.do_crop = do_crop

    def generate_train(self):
        '''Training set generator.'''
        while True:
            shuffle(self.train_keys)
            keys = self.train_keys
            inputs = []
            targets = []
            for key in keys:
                img_path = os.path.join(self.data_path, key)
                img = cv2.imread(img_path).astype('float32')
                if len(img.shape)<3:
                    img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
                y = self.gt[key].copy()
                if self.do_crop:
                    img, y = random_sized_crop(img, y)
                img = cv2.resize(img, self.image_shape).astype('float32')
                y = self.bbox_util.assign_boxes(y)
                inputs.append(img)
                targets.append(y)
                if len(targets) == self.batch_size:
                    tmp_inp = np.array(inputs)
                    tmp_targets = np.array(targets)
                    inputs = []
                    targets = []
                    yield preprocess_input(tmp_inp), tmp_targets
            if(len(targets)>0):
                tmp_inp = np.array(inputs)
                tmp_targets = np.array(targets)
                yield preprocess_input(tmp_inp), tmp_targets

    def generate_vaild(self):
        '''Validation set generator.'''
        while True:
            shuffle(self.val_keys)
            keys = self.val_keys
            inputs = []
            targets = []
            for key in keys:
                img_path = os.path.join(self.data_path, key)
                img = cv2.imread(img_path).astype('float32')
                if len(img.shape)<3:
                    img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
                y = self.gt[key].copy()
                if self.do_crop:
                    img, y = random_sized_crop(img, y)
                img = cv2.resize(img, self.image_shape).astype('float32')
                y = self.bbox_util.assign_boxes(y)
                inputs.append(img)
                targets.append(y)
                if len(targets) == self.batch_size:
                    tmp_inp = np.array(inputs)
                    tmp_targets = np.array(targets)
                    inputs = []
                    targets = []
                    yield preprocess_input(tmp_inp), tmp_targets
            if(len(targets)>0):
                tmp_inp = np.array(inputs)
                tmp_targets = np.array(targets)
                yield preprocess_input(tmp_inp), tmp_targets

    def serval_to_dict(self, anno_path, data_path, max_box=128):
        '''Extract annotation info and do preprocessing'''
        contents = ''
        self.gt_boxes = {}      #for metrices

        _, contents = decrypt_txt_file(anno_path)
        lines = contents.split('\n')
        self.class_count = len(lines[1].split(','))
        self.class_names = lines[1].split(',')
        self.class_names = [cn.split(':')[1] for cn in self.class_names]

        gt = {}
        annos = lines[2:]
        for anno in annos:
            self.gt_boxes_per_img = np.zeros((max_box,5),dtype='int32')     #for matrics
            x = anno.split(':')
            if(len(x)==1) or x[1]=='':
                continue
            name = x[0]
            img = cv2.imread(os.path.join(data_path,name))
            try:
                width = img.shape[0]
                height = img.shape[1]
            except:
                continue
            x = list(map(lambda a:int(float(a)), x[1].split(',')))
            i=0
            boxes = []
            while len(x)>0 and i<max_box:
                box = []
                self.gt_boxes_per_img[i] = [x[1],x[2],x[3],x[4],x[0]]       #for metrics
                label = np.zeros(self.class_count)
                label[x[0]] = float(1)
                xmin = float(x[1])/width
                ymin = float(x[2])/height
                xmax = float(x[3])/width
                ymax = float(x[4])/height
                box = [xmin, ymin, xmax, ymax]
                box.extend(label[1:])
                boxes.append(box)
                x = x[5:]
                i+=1
            self.gt_boxes[name] = self.gt_boxes_per_img     #for metrics
            boxes = np.asarray(boxes,dtype='float64')
            gt[name] = boxes
        return gt     #{'name':[[x1,y1,x2,y2,0,0,1]]}

max_stars_repo_name: generalized-intelligence/Tegu

max_issues_repo_name: generalized-intelligence/Tegu

max_forks_repo_name: generalized-intelligence/Tegu

reason: import numpy,from scipy


text:
import logging
import jellyfish
import numpy as np

def levenshtein(s1, s2):
    '''Levenshtein distance between two strings'''
    m = np.zeros((len(s2) + 1, len(s1) + 1))
    m[:, 0] = np.arange(len(s2) + 1)
    m[0, :] = np.arange(len(s1) + 1)
    for i in range(1, m.shape[0]):
        for j in range(1, m.shape[1]):
            if s2[i - 1] == s1[j - 1]:
                m[i, j] = min(m[i - 1, j - 1], m[i, j - 1] + 1, m[i-1, j] + 1)
            else:
                m[i, j] = min(m[i - 1, j - 1] + 1, m[i, j - 1] + 1, m[i - 1, j] + 1)
    return m[-1, -1]

def phonetic_match(input1, input2, jaro_d, damerau_d):
    '''Phonetic matching of input strings'''
    # Set distances if not passed
    if jaro_d is None:
        jaro_d = 0.9
    if damerau_d is None:
        damerau_d = 1
    logging.info(f'Processing Jaro distance with max distance of {jaro_d}, Damerau distance with max distance of {damerau_d}.')

    # Preprocess and define steps
    input1.replace(" ", "")
    input2.replace(" ", "")
    input1_soundex = jellyfish.soundex(input1)
    input1_nysiis = jellyfish.nysiis(input1)
    input1_match_rating_codex = jellyfish.match_rating_codex(input1)
    input2_soundex = jellyfish.soundex(input2)
    input2_nysiis = jellyfish.nysiis(input2)
    input2_match_rating_codex = jellyfish.match_rating_codex(input2)

    jaro_soundex = jellyfish.jaro_distance(input1_soundex, input2_soundex)
    logging.info(f"Phontetic match - jaro_soundex: {jaro_soundex}")
    if jaro_soundex >= jaro_d:
        return True

    jaro_nysiis = jellyfish.jaro_distance(input1_nysiis, input2_nysiis)
    logging.info(f"Phontetic match - jaro_nysiis: {jaro_nysiis}")
    if jaro_nysiis >= jaro_d:
        return True

    jaro_match_rating_codex = jellyfish.jaro_distance(input1_match_rating_codex, input2_match_rating_codex)
    logging.info(f"Phontetic match - jaro_match_rating_codex: {jaro_nysiis}")
    if jaro_match_rating_codex >= jaro_d:
        return True

    damerau_soundex = jellyfish.damerau_levenshtein_distance(input1_soundex, input2_soundex)
    logging.info(f"Phontetic match - damerau_soundex: {damerau_soundex}")
    if damerau_soundex <= damerau_d:
        return True

    damerau_nysiis = jellyfish.damerau_levenshtein_distance(input1_nysiis, input2_nysiis)
    logging.info(f"Phontetic match - damerau_nysiis: {damerau_nysiis}")
    if jellyfish.damerau_levenshtein_distance(input1_nysiis, input2_nysiis) <= damerau_d:
        return True

    damerau_match_rating_codex = jellyfish.damerau_levenshtein_distance(input1_match_rating_codex, input2_match_rating_codex)
    logging.info(f"Phontetic match - damerau__match_rating_codex: {damerau_match_rating_codex}")
    if damerau_match_rating_codex <= damerau_d:
        return True
    return False

'''CHECKS'''
def check_item_exact(token_input, token_truth):
    if token_input == token_truth:
        return True
    else:
        return False

def check_item_dist(token_input, token_truth, d):
    '''Levenshtein distance'''
    if d is None:
        d = 2
    logging.info(f'Processing Levenshtein distance with max distance of {d}.')
    if levenshtein(token_input, token_truth) > d:
        return False
    else:
        return True

def check_item_phonetic(token_input, token_truth, jaro_d, damerau_d):
    '''Phonetic matching'''
    if not phonetic_match(token_input, token_truth, jaro_d, damerau_d):
        return False
    else:
        return True

'''APPLIED CHECKS'''
def apply_check_exact(customer_data, cleaned, manifest=None):
    '''Apply check for exact match'''
    return {key:(value.lower() == cleaned[key]) for key, value in customer_data.items()}

def apply_check_levensthein(customer_data, cleaned, manifest):
    '''Apply check for Levenshtein distance'''
    return {key:check_item_dist(cleaned[key], value.lower(), manifest[key].get('max_distance_levenshtein')) if not manifest[key]['exact_match'] else (value.lower() == cleaned[key]) for key, value in customer_data.items()}

def apply_check_phonetic(customer_data, cleaned, manifest):
    '''Apply check for phonetic matching'''
    return {key:check_item_phonetic(cleaned[key], value.lower(), manifest[key].get('max_distance_jaro'), manifest[key].get('max_distance_damerau')) if not manifest[key]['exact_match'] else (value.lower() == cleaned[key]) for key, value in customer_data.items()}


text:
# -*- coding: utf-8 -*-

#Mixture of gaussians
#In this examples, the 1d case will be shown

# -*- coding: utf-8 -*-

# =============================================================================
# Here we will be testing with mixtures of student-t, 1d
# =============================================================================
import sys
sys.path.insert(0,"../../src")
import math
import functools
import time
import os

import torch
import numpy as np
from scipy.special import gamma
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

from src.variational_boosting_bmc import VariationalBoosting
from src import vb_utils
from src import sampling
from src.utils import TicToc
from source_2d_likelihood_fn import compute_log_likelihood

np.random.seed(100)
torch.manual_seed(100)
tictoc = TicToc()
#%% priors
#x0 : Unif(0,1)
#Ts : Unif(0,0.5)
lambd_rho = 20.0 #rho : Exp(20)
lambd_q = 0.2 #q : Exp(0.2)
#%%%
def logit_t(x,a=0,b=1):
    return torch.log(((x-a)/(b-a))/(1.0-(x-a)/(b-a)))
def sigmoid(x,a=0,b=1):
    return (b-a)*1.0/(1.0+np.exp(-x)) + a
def dsigmoid(x,a=0,b=1):
    return (b-a)*np.exp(x)/((1+np.exp(x))**2)
def exp(x):
    return np.exp(x)
def dexp(x):
    return np.exp(x)

def unwarped_logjoint_np(x0,y0,Ts,q0,rho):
    rho = 0.05
    Ts = 0.3
    q0 = 6.366197723675814
    ll = compute_log_likelihood(x0,y0,rho,q0,Ts)
#    ll += -np.log(1+(q0/10.0)**2)
#    ll += -np.log(1+(rho/0.1)**2)
    return ll

def logjoint_np(x):
#    x0,y0,Ts,q0,rho = x[0],x[1],x[2],x[3],x[4]
    x0,y0,Ts,q0,rho = x[0],x[1],0.0,0.0,0.0
    ll = unwarped_logjoint_np(sigmoid(x0),sigmoid(y0),
                              sigmoid(Ts,b=0.4),
                              exp(q0),exp(rho)) + \
         np.log(dsigmoid(x0)) + np.log(dsigmoid(y0))
#         np.log(dsigmoid(Ts,b=0.4)) + \
#         np.log(dexp(q0)) + np.log(dexp(rho))
    return ll

def logjoint(sample):
    return torch.tensor(logjoint_np(sample.flatten().numpy()))
#%%

#torch.randn(20)
dim = 2
nsamples = 20*dim
def sampling1(nsamples):
    X1 = logit_t(torch.rand(nsamples,1))
    X2 = logit_t(torch.rand(nsamples,1))
#    X3 = logit_t(torch.rand(nsamples,1)*0.4,b=0.4)
#    X4 = torch.log(torch.distributions.HalfCauchy(scale=10.0).rsample((nsamples,1)))
#    X5 = torch.log(torch.distributions.HalfCauchy(scale=0.1).rsample((nsamples,1)))
    sample = torch.cat([X1,X2],dim=1)
    return sample
samples = sampling1(nsamples)
mu0 = torch.zeros(dim)
cov0 = 20.0*torch.ones(dim)

#%%
#samples = vb.samples
training_interval = 20
acquisitions = ["prospective","mmlt"]
vb = VariationalBoosting(dim,logjoint,samples,mu0,cov0,
                         bmc_type="FM",normalization_mode="normalize",
                         training_space="gspace",noise=1e-4,
                         kernel_function="PMat",matern_coef=1.5,
                         numstab=-50.0,degree_hermite=50)
vb.optimize_bmc_model(maxiter=500,verbose=1,
                      lr=0.05)

#%%
elbo_list = [vb.evidence_lower_bound(nsamples=10000).cpu().numpy()]
kl_list = [vb.kullback_proposal_bmc(10000).item()]
step_list = [0]
time_list = [0.0]
#%%
print("Active sampling...")
for i in range(10*dim):
    vb.update_bmcmodel(acquisition="mmlt",mode="optimizing",vreg=1e-2)
vb.update_full()
folder_name = "testheat1c"
try:
    os.mkdir(folder_name)
except FileExistsError:
    pass
vb.save_distribution("%s/mvn%i"%(folder_name,0))
#%%
print(torch.sigmoid(vb.currentq_mean))
print(elbo_list[-1])
print(kl_list[-1])
#%%
for i in range(100):
    tictoc.tic()
    _ = vb.update(maxiter_nc=300,lr_nc=0.1,b_sn=0.1,
                  n_samples_nc=500,n_samples_sn=300,n_iter_sn=300,
                  max_alpha=1.0,verbose=0)
    try:
        acquisition = np.random.choice(acquisitions)
        vb.update_bmcmodel(acquisition=acquisition,mode="optimizing",
                           acq_reg=1e-1,verbose=0)
    except:
        continue
    if ((i+1)%training_interval) == 0:
        vb.update_full()
    elapsed = tictoc.toc(printing=False)
    elbo_list.append(vb.evidence_lower_bound(nsamples=10000).cpu().numpy())
    kl_list.append(vb.kullback_proposal_bmc(10000).item())
    step_list.append(i+1)
    time_list.append(elapsed)
    print(torch.sigmoid(vb.currentq_mean))
    print(acquisition)
    print(elbo_list[-1])
    print(kl_list[-1])
    print(time_list[-1])
    print("Step %i"%(i+1))
    vb.save_distribution("%s/mvn%i"%(folder_name,i+1))

elbo_np = np.array(elbo_list).astype(float)
step_list_np = np.array(step_list).astype(int)
times_np = np.array(time_list)
np.savez("%s/tracking"%folder_name,
         time=times_np,
         elbo=elbo_np,
         steps=step_list_np)
#%%
Nplot = 41
x,y = torch.linspace(-10,10,Nplot),torch.linspace(-10,10,Nplot)
X,Y = torch.meshgrid(x,y)
XY_ = torch.stack([X,Y],dim=-1).reshape(-1,2)
Zgp_ = vb.scaled_prediction(XY_)
Zgp = Zgp_.reshape(*X.shape)
Zq_ = vb.current_logq(XY_)
Zq = Zq_.reshape(*X.shape)

fig1 = plt.figure()
ax1 = fig1.add_subplot(111, projection='3d')
ax1.plot_surface(X.numpy(),Y.numpy(),Zgp.numpy())
plt.show()

fig2 = plt.figure()
ax2 = fig2.add_subplot(111, projection='3d')
ax2.plot_surface(X.numpy(),Y.numpy(),Zq.numpy())
plt.show()
#%%
Nplot = 41
x = torch.linspace(-5,5,Nplot).reshape(-1,1)
xy_ = torch.cat([x,1.0986122886681098*torch.ones_like(x)],dim=-1)
zgp = vb.scaled_prediction(xy_)
zvb = vb.current_logq(xy_)
fig,ax = plt.subplots()
plt.plot(x.numpy(),zgp.numpy())
fig,ax = plt.subplots()
plt.plot(x.numpy(),zvb.detach().numpy())
#%%
Nplot = 21
x,y = torch.linspace(-10,10,Nplot),torch.linspace(-10,10,Nplot)
X,Y = torch.meshgrid(x,y)
XY_ = torch.stack([X,Y],dim=-1).reshape(-1,2)
Ztrue_ = np.array([logjoint_np(xy) for xy in XY_.numpy()]).reshape(-1,1)
Ztrue = Ztrue_.reshape(*X.shape)
fig3 = plt.figure()
ax3 = fig3.add_subplot(111, projection='3d')
ax3.plot_surface(X.numpy(),Y.numpy(),Ztrue)
plt.show()
#Zgp_ = vb.scaled_prediction(XY_)
#Zgp = Zgp_.reshape(*X.shape)
#Zq_ = vb.current_logq(XY_)
#Zq = Zq_.reshape(*X.shape)
#
#fig1 = plt.figure()
#ax1 = fig1.add_subplot(111, projection='3d')
#ax1.plot_surface(X.numpy(),Y.numpy(),Zgp.numpy())
#plt.show()
#
#fig2 = plt.figure()
#ax2 = fig2.add_subplot(111, projection='3d')
#ax2.plot_surface(X.numpy(),Y.numpy(),Zq.numpy())
#plt.show()

max_stars_repo_name: DFNaiff/Dissertation

max_issues_repo_name: DFNaiff/Dissertation

max_forks_repo_name: DFNaiff/Dissertation


text:
import numpy as np
from .parameters import ParameterList
from .latent import LatentParameters
from collections import OrderedDict
from .vb import MVN, Gamma, variational_bayes, VariationalBayesInterface
from .jacobian import d_model_error_d_named_parameter

class ModelErrorInterface:
    def __init__(self):
        self.parameter_list = ParameterList()

    def __call__(self):
        """
        Evaluate the model error based on `self.parameter_list` as a dict of
        {some_key: numpy.array}.
        """
        raise NotImplementedError("Override this!")

    def jacobian(self, latent_names=None):
        jac = dict()
        latent_names = latent_names or self.parameter_list.names
        for prm_name in latent_names:
            prm_jac = d_model_error_d_named_parameter(self, prm_name)
            for key in prm_jac:
                if key not in jac:
                    jac[key] = dict()

                jac[key][prm_name] = prm_jac[key]

        return jac

class InferenceProblem:
    def __init__(self):
        self.latent = LatentParameters()
        self.model_errors = OrderedDict()  # key : model_error
        self._noise_models = OrderedDict()  # key : noise_model

    @property
    def noise_models(self):
        if not self._noise_models:
            raise RuntimeError(
                "You need to define and add a noise model first! See `bayes.noise` for options and then call `.add_noise_model` to add it to the inference problem."
            )
        return self._noise_models

    def add_model_error(self, model_error, key=None):

        key = key or len(self.model_errors)
        assert key not in self.model_errors

        self.model_errors[key] = model_error
        return key

    def add_noise_model(self, noise_model, key=None):

        key = key or f"noise{len(self._noise_models)}"

        assert key not in self._noise_models
        self._noise_models[key] = noise_model
        return key

    def __call__(self, number_vector):
        self.latent.update(number_vector)
        result = {}
        for key, me in self.model_errors.items():
            result[key] = me()
        return result

    def define_shared_latent_parameter_by_name(self, name):
        for model_error in self.model_errors.values():
            try:
                prm = model_error.parameter_list
            except AttributeError:
                raise AttributeError(
                    "This method requires the `model_error` to have a `parameter_list` attribute!"
                )

            if name in model_error.parameter_list:
                self.latent[name].add(model_error.parameter_list, name)

    def loglike(self, number_vector):
        self.latent.update(number_vector)
        raw_me = {}
        for key, me in self.model_errors.items():
            raw_me[key] = me()

        log_like = 0.0
        for noise_key, noise_term in self.noise_models.items():
            log_like += noise_term.loglike_contribution(raw_me)

        return log_like

class VariationalBayesProblem(InferenceProblem, VariationalBayesInterface):
    def __init__(self):
        super().__init__()
        self.prm_prior = {}
        self.noise_prior = {}

    def set_normal_prior(self, latent_name, mean, sd):
        if latent_name not in self.latent:
            raise RuntimeError(
                f"{latent_name} is not defined as a latent parameter. "
                f"Call InferenceProblem.latent[{latent_name}].add(...) first."
            )
        self.prm_prior[latent_name] = (mean, sd)

    def set_noise_prior(self, name, gamma):
        if name not in self.noise_models:
            raise RuntimeError(
                f"{name} is not associated with noise model.. "
                f"Call InferenceProblem.add_noise_model({name}, ...) first."
            )
        self.noise_prior[name] = gamma

    def run(self, **kwargs):
        MVN = self.prior_MVN()
        info = variational_bayes(self, MVN, self.noise_prior, **kwargs)
        return info

    def jacobian(self, number_vector, concatenate=True):
        """
        overwrites VariationalBayesInterface.jacobian
        """
        self.latent.update(number_vector)
        jac = {}
        for key, me in self.model_errors.items():

            # For each global latent parameter, we now need to find its
            # _local_ name, so the name in the parameter_list of the
            # model_error ...
            latent_names = self.latent.latent_names(me.parameter_list)
            local_latent_names = [n[0] for n in latent_names]
            # ... and only request the jacobian for the latent parameters.
            sensor_parameter_jac = me.jacobian(local_latent_names)
            """
            sensor_parameter_jac contains a
                dict (sensor) of
                dict (parameter)

            We now flatten the last dict (parameter) in the order of the
            latent parameters for a valid VB input.

            This is challenging/ugly because:
                * The "parameter" in sensor_parameter_jac is not the same
                  as the corresponding _global_ parameter in the latent
                  parameters.
                * Some of the latent parameters may not be part of
                  sensor_parameter_jac, because it only is a parameter of a
                  different model error. We have to fill it with zeros of the
                  right dimension

            """
            sensor_jac = {}
            for sensor, parameter_jac in sensor_parameter_jac.items():
                first_jac = list(parameter_jac.values())[0]
                N = len(first_jac)

                # We allocate "stacked_jac" where each column corresponds
                # to a number in the "number_vector".
                stacked_jac = np.zeros((N, len(number_vector)))

                for (local_name, global_name) in latent_names:
                    J = parameter_jac[local_name]

                    # If it is a scalar parameter, the user may have
                    # defined as a vector of length N. We need to
                    # transform it to a matrix Nx1.
                    if len(J.shape) == 1:
                        J = np.atleast_2d(J).T

                    stacked_jac[:, self.latent[global_name].global_index_range()] += J

                sensor_jac[sensor] = stacked_jac

            jac[key] = sensor_jac

        jacs_by_noise = {}
        for key, noise in self.noise_models.items():
            terms = noise.jacobian_terms(jac)
            if concatenate:
                terms = np.concatenate(terms)
            jacs_by_noise[key] = terms

        return jacs_by_noise

    def __call__(self, number_vector, concatenate=True):
        """
        overwrites VariationalBayesInterface.__call__
        """
        me = super().__call__(number_vector)

        errors_by_noise = {}
        for key, noise in self.noise_models.items():
            terms = noise.model_error_terms(me)
            if concatenate:
                terms = np.concatenate(terms)
            errors_by_noise[key] = terms

        return errors_by_noise

    def prior_MVN(self):

        means = []
        precs = []

        for name, latent in self.latent.items():
            if name not in self.prm_prior:
                raise RuntimeError(
                    f"You defined {name} as latent but did not provide a prior distribution!."
                )
            mean, sd = self.prm_prior[name]
            for _ in range(latent.N):
                means.append(mean)
                precs.append(1.0 / sd ** 2)

        return MVN(
            means,
            np.diag(precs),
            name="MVN prior",
            parameter_names=list(self.latent.keys()),
        )

max_stars_repo_name: BAMresearch/BayesianInference

max_issues_repo_name: BAMresearch/BayesianInference

max_forks_repo_name: BAMresearch/BayesianInference


text:
from neon.callbacks.callbacks import Callback
import numpy as np
import logging
import cPickle

logger = logging.getLogger(__name__)

class TrainByStageCallback(Callback):
    """
    0. Show model structure
    1. Calculate and display classification error.
    2. Save classification eror
    3. Store best stage
    4. Train by stages
    5. Early Stopping
    6. Save everything on train end
    """

    def __init__(self, model, eval_set, metric, filename='model.pkl', max_patience=10, epoch_freq=1, max_stages=3):
        super(TrainByStageCallback, self).__init__(epoch_freq=epoch_freq)
        self.filename = filename
        self.model = model
        self.eval_set = eval_set
        self.metric = metric

        self.max_stages = max_stages
        self.best_epoch = 0
        self.best_stats = None
        self.best_model = None
        self.patience = 0
        self.max_patience = max_patience
        self.stage = 1
        self.stats_list = []
        self.cost_list = []

    def on_epoch_end(self, callback_data, model, epoch):
        if (epoch + 1) % self.epoch_freq == 0:
            # Calculate and display classification error
            self.eval_set.reset()
            stats = self.model.eval(self.eval_set, metric=self.metric)
            logger.info('Epoch: %d, Classification error: %.2f%%' % (epoch, stats.flatten() * 100))

            # Save classification error
            self.stats_list.extend(stats)
            self.cost_list.extend(self.model.total_cost)

            # Store best stage
            if self.best_stats > stats or self.best_stats is None:
                self.best_epoch = epoch
                self.best_stats = stats
                self.best_model = self.model.serialize()
                self.patience = 0
                cPickle.dump(self.best_model, open('temp_model.pkl', 'wb'), cPickle.HIGHEST_PROTOCOL)
            else:
                self.patience += 1

            # Train by stages + early stopping
            if self.max_patience <= self.patience:
                if self.stage < self.max_stages:
                    self.stage += 1
                    self.patience = 0
                    if self.model.optimizer.name == 'multiopt':
                        for optimizer in self.model.optimizer.optimizer_mapping:
                            self.model.optimizer.optimizer_mapping[optimizer].learning_rate /= 10
                        logger.info('Recuding learning rate to %f' %
                                    self.model.optimizer.optimizer_mapping['default'].learning_rate)
                    else:
                        self.model.optimizer.learning_rate /= 10
                        logger.info('Recuding learning rate to %f' % self.model.optimizer.learning_rate)
                    # model.deserialize(self.best_model)
                else:
                    self.model.finished = True

    def on_train_begin(self, callback_data, model, epochs):
        logger.info("Model:\n%s", self.model)

    def on_train_end(self,  callback_data, model):
        pdict = dict()
        pdict['model'] = self.best_model
        pdict['misclas'] = self.stats_list
        pdict['cost'] = self.cost_list
        cPickle.dump(pdict, open(self.filename, 'wb'), cPickle.HIGHEST_PROTOCOL)
        logger.info('Best result at epoch %i: %.2f%%' % (self.best_epoch, self.best_stats * 100.))
        model.deserialize(self.best_model)

class MetricCallback(Callback):
    def __init__(self, model, eval_set, metric, epoch_freq=1):
        super(MetricCallback, self).__init__(epoch_freq=epoch_freq)
        self.model = model
        self.eval_set = eval_set
        self.metric = metric

    def on_train_begin(self, callback_data, model, epochs):
        logger.info("Model:\n%s", self.model)

    def on_epoch_end(self, callback_data, model, epoch):
        self.eval_set.reset()
        stats = self.model.eval(self.eval_set, metric=self.metric)
        logger.info('Epoch: %d, Classification error: %.2f%%' % (epoch, stats.flatten() * 100))

class FuzzyPruneCallback(Callback):
    def __init__(self, num_states=10, op='max', start_prune_epoch=5, epoch_freq=1, filename='fprune.pkl',
                 num_prune='auto', one_time=True, model=None):
        super(FuzzyPruneCallback, self).__init__(epoch_freq=epoch_freq)
        self.one_time = one_time
        self.prune = False
        self.start_prune_epoch = start_prune_epoch
        self.num_states = num_states
        self.interval = 1. / num_states
        if isinstance(num_prune, list):
            self.num_prune = num_prune
        elif num_prune == 'auto':
            assert model is not None
            self.num_prune = []
            for idx, l in enumerate(model.layers_to_optimize):
                if l.__class__.__name__ in ['Convolution', 'Linear']:
                    self.num_prune.append(np.prod(l.W.shape, dtype=np.float32))
            self.num_prune = np.sqrt(self.num_prune / np.sum(self.num_prune) / range(1, len(self.num_prune)+1)) * num_states
            print self.num_prune
        else:
            self.num_prune = [num_prune]
        assert op in ['max', 'sum']
        self.op = op
        self.states = []
        self.masks = []
        self.filename = filename

    def reset(self):
        self.states = []
        self.masks = []

    def on_minibatch_end(self, callback_data, model, epoch, minibatch):
        offset = 0
        if len(self.masks) > 0:
            for i, l in enumerate(model.layers_to_optimize):
                if l.__class__.__name__ in ['Convolution', 'Linear']:
                    l.set_params({'params': {'W': l.W.asnumpyarray() * self.masks[i - offset]}})
                    # l.W = l.W * self.masks[i - offset]
                else:
                    offset += 1

    # noinspection PyStringFormat
    def on_train_begin(self, callback_data, model, epochs):
        if self.one_time:
            offset = 0
            for idx, l in enumerate(model.layers_to_optimize):
                if l.__class__.__name__ in ['Convolution', 'Linear']:
                    W = l.W.get()
                    if self.op is 'max':
                        normW = np.abs(W) / np.max(np.abs(W), keepdims=True)
                    elif self.op is 'sum':
                        normW = np.abs(W) / np.sum(np.abs(W), keepdims=True)
                    else:
                        raise Exception('Operation ''%s'' not implemented!' % self.op)
                    normW[normW == 1] = self.num_states
                    for i in range(self.num_states):
                        normW[np.logical_and(normW >= (i * self.interval), normW < ((i + 1) * self.interval))] = i + 1
                    self.states.append(normW)
                    if len(self.num_prune) > 1:
                        masks = (normW > self.num_prune[idx - offset])
                    else:
                        masks = (normW > self.num_prune[0])
                    self.masks.append(masks)
                    l.set_params({'params': {'W': W * masks}})
                    # l.W = l.W * masks
                    print 'Mask %i: %f' % (idx - offset, np.mean(masks))
                else:
                    offset += 1
            num_mask = sum([mask.sum() for mask in self.masks])
            shape_mask = sum([np.prod(mask.shape, dtype=np.float32) for mask in self.masks])
            print 'Overall pruned: %f' % (num_mask / shape_mask)

    # noinspection PyStringFormat
    def on_epoch_end(self, callback_data, model, epoch):
        if self.one_time:
            return
        if (not self.prune) and (epoch + 1) >= self.start_prune_epoch:
            self.prune = True
        if self.prune and (epoch + 1) % self.epoch_freq == 0:
            self.reset()
            offset = 0
            for idx, l in enumerate(model.layers_to_optimize):
                if l.__class__.__name__ in ['Convolution', 'Linear']:
                    W = l.W.asnumpyarray()
                    if self.op is 'max':
                        normW = np.abs(W) / np.max(np.abs(W), keepdims=True)
                    elif self.op is 'sum':
                        normW = np.abs(W) / np.sum(np.abs(W), keepdims=True)
                    else:
                        raise Exception('Operation ''%s'' not implemented!' % self.op)
                    normW[normW == 1] = self.num_states
                    for i in range(self.num_states):
                        normW[np.logical_and(normW >= (i * self.interval), normW < ((i + 1) * self.interval))] = i + 1
                    self.states.append(normW)
                    if len(self.num_prune) > 1:
                        masks = (normW > self.num_prune[idx - offset])
                    else:
                        masks = (normW > self.num_prune[0])
                    self.masks.append(masks)
                    l.set_params({'params': {'W': W * masks}})
                    print 'Mask %i: %f' % (idx - offset, np.mean(masks))
                else:
                    offset += 1

    def on_train_end(self, callback_data, model):
        cPickle.dump({'normW': self.states, 'masks': self.masks}, open(self.filename, 'wb'), cPickle.HIGHEST_PROTOCOL)

max_stars_repo_name: cs-chan/Fuzzy-Compression

max_issues_repo_name: cs-chan/fuzzyDCN

max_forks_repo_name: cs-chan/Fuzzy-Compression