text:
"""
    Copyright (C) 2019. Huawei Technologies Co., Ltd and McGill University. All rights reserved.
    This program is free software; you can redistribute it and/or modify
    it under the terms of the MIT License.
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    MIT License for more details.
    """

import numpy as np
from src.utils import reparameterized_to_beta, reparameterized_to_pi, graph_preparation, metric_perp_avg, \
    bernuli_distrbution, step_size_function, accuracy_avg, initialize_theta_phi_with_better_initialization

class MMSBM_SGMCMC:
    def __init__(self, flags, n, k, edges, nonedges, beta_prior, membership_prior, theta_constant,
                 phi_constant, true_labels, better_initialization_flag, step_size_scalar,
                 node_neighbors_dict, val_set_index,
                 mu=1, max_iter=10000):

        """ follows the notations in the original paper
        :param flags: hyper-parameters for GCN and MMSBM
        :param n: node number
        :param k: class number
        :param edges: edge indices
        :param nonedges: non-edge indices

        :param beta_prior: prior for the community strength
        :param membership_prior: prior for the membership
        :param theta_constant: re-parameterization constant for community strength beta
        :param phi_constant: re-parameterization constant for membership
        :param true_labels: ground truth labels
        :param better_initialization_flag: a flag indicate if we train the MMSBM from scratch or we use the better initialization output from gcn
        :param step_size_scalar: step size for the MMSBM
        :param node_neighbors_dict: a dict for query the neighborhood node indices
        :param val_set_index: indices for the validation set
        """
        self.gamma_scale = flags.gamma_scale
        self.better_initialization_flag = better_initialization_flag
        self.step_size_scalar = step_size_scalar
        self.flags = flags
        self.n = n  # number of nodes
        self.k = k
        self.val_set_index = val_set_index

        self.alpha = 1.0 / k
        self.mu = mu
        self.tao = 1024
        self.n_list_set = np.array([i for i in range(self.n)])

        self.max_iter = max_iter
        self.mini_batch_nodes = flags.batch_size
        self.true_labels = true_labels

        self.sample_n = 20  # sample size for update each local parameters
        self.T = 1  # sample number of pi and beta for each edge during the evaluation process
        self.test_edges_n = 500  # test set edges for the perplexity test
        self.delta = flags.delta
        self.node_neighbors_dict = node_neighbors_dict
        self.avg_predict_label = 0

        # variable initialization (random initialization)
        if not self.better_initialization_flag:
            self.phi = np.random.gamma(self.alpha, 1, size=(self.n, self.k))
            self.theta = np.random.gamma(self.mu, 1, size=(self.k, 2))

            self.beta, self.theta_constant = reparameterized_to_beta(self.theta)
            self.pi, self.phi_constant = reparameterized_to_pi(self.phi, self.n)
            self.initial_prediction_labels = self.pi.argmax(axis=1)

        else:
            self.theta_constant = theta_constant
            self.phi_constant = phi_constant

            self.beta = beta_prior
            self.pi = membership_prior
            self.initial_prediction_labels = membership_prior.argmax(axis=1)
            self.theta, self.phi = initialize_theta_phi_with_better_initialization(self.beta, self.pi,
                                                                                self.theta_constant,
                                                                                self.phi_constant,
                                                                                k)
        self.MCMC_MMSBM_prediction_labels = self.initial_prediction_labels
        self.B = np.ones((self.k, self.k)) * flags.delta

        # Info of the given topology, split into the edges and non-edges
        self.edges = edges
        self.nonedges = nonedges
        self.edges_n, self.nonedges_n, self.test_set, self.y_test_set = graph_preparation(self.edges, self.nonedges,
                                                                                test_edges_n=self.test_edges_n)

        self.sampled_non_edges_ratio = self.flags.sampled_non_edges_ratio
        self.sampled_non_edges_n = int(self.sampled_non_edges_ratio * self.nonedges_n)
        self.dir = 'figures/'

    def Z_constant_mini_batch_phi(self, node_a_membership, node_b_membership, link_index_mini_batch):
        bernuli_delta_mini_batch = np.ones((len(node_a_membership), 1)) * (1 - self.delta)
        bernuli_delta_mini_batch[link_index_mini_batch] = self.delta
        Z_constant_mini_batch = bernuli_delta_mini_batch.copy()
        for k in range(self.k):
            bernuli_beta_k_mini_batch = np.ones((len(node_a_membership), 1)) * (1 - self.beta[k])
            bernuli_beta_k_mini_batch[link_index_mini_batch] = self.beta[k]
            pi_a_k = (node_a_membership[:, k]).reshape(len(node_a_membership), 1)
            pi_b_k = (node_b_membership[:, k]).reshape(len(node_b_membership), 1)
            Z_constant_mini_batch += (bernuli_beta_k_mini_batch - bernuli_delta_mini_batch) * pi_a_k * pi_b_k
        return Z_constant_mini_batch

    def Z_constant_mini_batch(self, node_a_membership, node_b_membership, links_flag):
        if links_flag:
            bernuli_delta_mini_batch = np.ones((len(node_a_membership), 1)) * self.delta
        else:
            bernuli_delta_mini_batch = np.ones((len(node_a_membership), 1)) * (1 - self.delta)
        Z_constant_mini_batch = bernuli_delta_mini_batch.copy()
        for k in range(self.k):
            if links_flag:
                bernuli_beta_k_mini_batch = np.ones((len(node_a_membership), 1)) * self.beta[k]
            else:
                bernuli_beta_k_mini_batch = np.ones((len(node_a_membership), 1)) * (1 - self.beta[k])
            pi_a_k = (node_a_membership[:, k]).reshape(len(node_a_membership), 1)
            pi_b_k = (node_b_membership[:, k]).reshape(len(node_b_membership), 1)
            Z_constant_mini_batch += (bernuli_beta_k_mini_batch - bernuli_delta_mini_batch) * pi_a_k * pi_b_k
        return Z_constant_mini_batch

    def function_f_ab_k_k(self, node_a_membership, node_b_membership, observation_ab, k):
        f_ab_k_k = bernuli_distrbution(observation_ab, self.beta[k]) * node_a_membership[k] * node_b_membership[k]
        return f_ab_k_k

    def function_f_ab_k_mini_batch_pi(self, k, node_a_membership, node_b_membership, links_index):
        bernuli_delta_mini_batch = np.ones((len(node_a_membership), 1)) * (1 - self.delta)
        bernuli_delta_mini_batch[links_index] = self.delta

        bernuli_beta_k_mini_batch = np.ones((len(node_a_membership), 1)) * (1 - self.beta[k])
        bernuli_beta_k_mini_batch[links_index] = self.beta[k]

        node_a_k = (node_a_membership[:, k]).reshape(len(node_a_membership), 1)

        node_b_k = (node_b_membership[:, k]).reshape(len(node_b_membership), 1)

        f_ab_k_mini_batch = node_a_k * (
            bernuli_beta_k_mini_batch * node_b_k + bernuli_delta_mini_batch * (1 - node_b_k))
        return f_ab_k_mini_batch

    def function_f_ab_k_k_mini_batch(self, k, node_a_membership, node_b_membership, link_flag):
        if link_flag:
            f_ab_k_k_mini_batch = np.ones((len(node_a_membership), 1)) * self.beta[k]
        else:
            f_ab_k_k_mini_batch = np.ones((len(node_a_membership), 1)) * (1 - self.beta[k])
        f_ab_k_k_mini_batch = f_ab_k_k_mini_batch * (node_a_membership[:, k]).reshape(len(node_b_membership), 1) * (
            node_b_membership[:, k]).reshape(len(node_b_membership), 1)

        return f_ab_k_k_mini_batch

    def update_phi(self, batch_nodes_index, step_size, n_list_set):
        n = self.mini_batch_nodes
        grad_phi = np.zeros((n, self.k))
        # select edges
        node_a = np.zeros(n * n).astype(int)
        node_b = np.zeros(n * n).astype(int)
        y_mini_batch = np.zeros(n * n).astype(int)
        corrections = np.zeros(n * n)
        for i, node in enumerate(batch_nodes_index):
            # deal with links
            node_neighbors = self.node_neighbors_dict[node]

            links_n = len(node_neighbors)
            node_a[i * n:i * n + n] = node
            node_b[i * n:i * n + links_n] = node_neighbors
            y_mini_batch[i * n:i * n + links_n] = 1

            # deal with non-links
            non_neighbors = np.setdiff1d(n_list_set, node_neighbors)
            non_neighbors = np.setdiff1d(non_neighbors, np.array([node]))
            np.random.shuffle(non_neighbors)
            sampled_node_neighbors = non_neighbors[:n - links_n]
            node_b[(i * n + links_n):(i * n + n)] = sampled_node_neighbors

            corrections[i * n: i * n + links_n] = 1
            corrections[(i * n + links_n):(i * n + n)] = float(self.n - links_n - 1) / (self.mini_batch_nodes - links_n)

        corrections = corrections.reshape(n, n)
        pi_a = self.pi[node_a]
        pi_b = self.pi[node_b]
        phi_a = self.phi[node_a]

        links_index = np.where(y_mini_batch == 1)[0]
        Z_ab_mini_batch = self.Z_constant_mini_batch_phi(pi_a, pi_b, links_index)

        for k in range(self.k):
            f_ab_k_mini_batch = self.function_f_ab_k_mini_batch_pi(k, pi_a, pi_b, links_index)
            phi_a_k = (phi_a[:, k]).reshape(n * n, 1)

            temp_denumerator = Z_ab_mini_batch * phi_a_k
            denumerator = temp_denumerator.copy()

            index_zero = np.where(temp_denumerator == 0)[0]
            denumerator[index_zero] = 10 ** (-25)

            temp = f_ab_k_mini_batch / denumerator - np.ones((n * n, 1)) / ((np.sum(phi_a,
                                                                                axis=1)).reshape(
                n * n, 1))

            temp = temp.reshape(n, n) * corrections
            grad_phi[:, k] = np.sum(temp, axis=1)

        temp_phi = np.abs(self.phi[batch_nodes_index] + step_size / 2 * (
        - self.phi[batch_nodes_index] * self.gamma_scale + grad_phi * self.phi[batch_nodes_index]))
        return temp_phi

    # @do_profile(follow=[])
    def update_theta(self, step_size):
        grad_theta = np.zeros((self.k, 2))
        sample_non_edges_index = np.random.randint(self.nonedges_n, size=self.sampled_non_edges_n)
        non_edges_index_a = self.nonedges[0][sample_non_edges_index]
        non_edges_index_b = self.nonedges[1][sample_non_edges_index]

        pi_a_non_links = self.pi[non_edges_index_a]
        pi_b_non_links = self.pi[non_edges_index_b]
        z_ab_non_links = self.Z_constant_mini_batch(pi_a_non_links, pi_b_non_links, links_flag=False)

        edges_index_a = self.edges[0]
        edges_index_b = self.edges[1]
        pi_a_links = self.pi[edges_index_a]
        pi_b_links = self.pi[edges_index_b]
        z_ab_links = self.Z_constant_mini_batch(pi_a_links, pi_b_links, links_flag=True)

        correction = float(self.nonedges_n) / len(non_edges_index_a)

        for k in range(self.k):
            f_ab_kk_mini_batch_links = self.function_f_ab_k_k_mini_batch(k, pi_a_links, pi_b_links, link_flag=True)

            links_term = f_ab_kk_mini_batch_links / z_ab_links

            f_ab_kk_mini_batch_non_links = self.function_f_ab_k_k_mini_batch(k, pi_a_non_links, pi_b_non_links,
                                                                             link_flag=False)
            non_links_term = (f_ab_kk_mini_batch_non_links / z_ab_non_links) * correction

            theta_k = (np.sum(self.theta, axis=1))[k]

            if self.theta[k][0] < 10 ** (-50):
                self.theta[k][0] = 10 ** (-50)

            if self.theta[k][1] < 10 ** (-50):
                self.theta[k][1] = 10 ** (-50)

            # print(self.theta[k][0], self.theta[k][1], theta_k)
            grad_theta[k][0] = np.sum(links_term * (-1.0 / theta_k)) \
                               + np.sum(non_links_term * (1.0 / self.theta[k][0] - 1.0 / theta_k))

            grad_theta[k][1] = np.sum(links_term * (1.0 / self.theta[k][1] - 1.0 / theta_k)) \
                               + np.sum(non_links_term * (-1.0 / theta_k))

        temp_theta = np.abs(
            self.theta + (step_size / 2) * (-self.theta * self.gamma_scale + grad_theta * self.theta))
        return temp_theta

    def train_one_epoch(self, step_size, n_list_set):
        batch_nodes_index = np.random.choice(self.n, size=self.mini_batch_nodes, replace=False)
        temp_phi = self.update_phi(batch_nodes_index, step_size, n_list_set)

        self.phi[batch_nodes_index] = temp_phi
        self.pi, self.phi_constant = reparameterized_to_pi(self.phi, self.n)

        temp_theta = self.update_theta(step_size)
        self.beta, self.theta_constant = reparameterized_to_beta(temp_theta)
        self.theta = temp_theta

        return

    def evaluation(self, beta_list, pi_list):
        perplexity_score = metric_perp_avg(beta_list, pi_list, self.test_set, self.y_test_set, self.delta)
        ARI, acc, change_from_the_initial_prediction_labels, avg_predict_label = accuracy_avg(pi_list,
                                                                                self.initial_prediction_labels,
                                                                                self.true_labels,
                                                                                self.sample_n,
                                                                                self.k,
                                                                                self.val_set_index)
        return perplexity_score, ARI, acc, change_from_the_initial_prediction_labels, avg_predict_label

    def model_training(self):
        """
        MMSBM model training using stochastic MCMC
        """
        avg_perp = []
        prediction_acc = []
        beta_list = []
        ARI_list = []
        change_of_labels = []
        beta_samples = []
        pi_samples = []

        for itr in range(self.max_iter):
            if self.better_initialization_flag:
                itr += 10000
            step_size = step_size_function(itr, self.tao, self.step_size_scalar)
            self.train_one_epoch(step_size, self.n_list_set)

            if len(beta_samples) < self.T:
                beta_samples.append(self.beta)
                pi_samples.append(self.pi)

            if len(beta_samples) == self.T:
                beta_samples.pop(0)
                pi_samples.pop(0)
                beta_samples.append(self.beta)
                pi_samples.append(self.pi)

            for k in range(self.k):
                self.B[k][k] = self.beta[k]

            if itr % 100 == 99:
                print("========Iteration {}=======".format(itr))
                score, ARI, acc, change_from_the_initial_prediction_labels, avg_predict_label = self.evaluation(
                    beta_samples,
                    pi_samples)
                self.avg_predict_label = avg_predict_label

                print("The averaged perplexity on the held-out test result is {}".format(score))

                avg_perp.append(score)
                change_of_labels.append(change_from_the_initial_prediction_labels)
                prediction_acc.append(acc)
                ARI_list.append(ARI)
                beta_list.append(self.beta)

                self.MCMC_MMSBM_prediction_labels = avg_predict_label

max_stars_repo_name: ebonilla/BGCN

max_issues_repo_name: ebonilla/BGCN

max_forks_repo_name: ebonilla/BGCN


text:
import gdown
import numpy as np
import pandas as pd
from config import Config
from sklearn.model_selection import train_test_split

np.random.seed(Config.RANDOM_SEED)
Config.ORIGINAL_DATASET_FILE_PATH.parent.mkdir(parents=True, exist_ok=True)
Config.DATASET_PATH.mkdir(parents=True, exist_ok=True)

gdown.download(
    "https://drive.google.com/uc?id=1gkYBOIMm8pAGunRoI3OzQHQrgOdaRjfC",
    str(Config.ORIGINAL_DATASET_FILE_PATH),
)

df = pd.read_csv(str(Config.ORIGINAL_DATASET_FILE_PATH))

df_train, df_test = train_test_split(df, test_size=0.2, random_state=Config.RANDOM_SEED)

df_train.to_csv(str(Config.DATASET_PATH / "train.csv"), index=None)
df_test.to_csv(str(Config.DATASET_PATH / "test.csv"), index=None)

ext: py

max_stars_repo_name: aldiak/project

max_issues_repo_name: aldiak/project

max_forks_repo_name: aldiak/project

reason: import numpy


text:
#  Copyright (c) 2021 Jan Westerhoff
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import copy
from collections import defaultdict
from copy import deepcopy
from enum import Flag, auto
from math import floor, ceil, inf
from multiprocessing import Process
from pathlib import Path
from typing import Union, Optional

import networkx as nx
from matplotlib import pyplot as plt

import algos.global_shift as gs
import helper_functions.graph_helper_functions as gh
from digraph import DiGraph

class Direction(Flag):
    UP = auto()
    DOWN = auto()
    LEFT = auto()
    RIGHT = auto()

class Layer:
    def __init__(self):
        # Enthält die selben Daten und dienen nur des schnellen Zugriffs jenach Key
        self.nodes: dict[int, str] = dict()
        self.positions: dict[str, int] = dict()
        self.len = 0

    # def move(self, pos: int, item: str) -> None:
    #     old_node = self.nodes[pos]
    #
    #     # Node, welche nach rechts bewegt werden müssen
    #     temp_dict = dict()
    #     for node in self.nodes.values():
    #         if i := self.positions[node] >= self.positions[old_node]:
    #             self.positions[node] = i + 1
    #             temp_dict[i + 1] = node
    #     self.nodes.update(temp_dict)
    #
    #     self.positions[item] = pos
    #     self.nodes[pos] = item

    def append(self, item: str):
        self.len += 1
        self.nodes[self.len] = item
        self.positions[item] = self.len

        return self.len

    def pos(self, item: str) -> int:
        return self.positions[item]

    def predecessor(self, item: str) -> str:
        if (pos := self.positions[item]) > 1:
            return self.nodes[pos - 1]
        raise ValueError('Für das erste Element gibt es keinen Vorfolger')

    def node_by_index(self, i: int) -> str:
        return self.nodes[i]

    # def set_node_by_index(self, i: int, value) -> None:
    #     self.nodes[i] = value

    def sort(self, G: DiGraph):
        s_list = sorted(self.nodes.values(), key=lambda node: G.nodes[node]['pi'])

        self.nodes.clear()
        self.positions.clear()
        for i, node in enumerate(s_list):
            self.nodes[i + 1] = node
            self.positions[node] = i + 1

    def __len__(self) -> int:
        return self.len

    def __getitem__(self, item: int) -> str:
        # FIX: Brainfuck 2.0
        return self.nodes[item]

    def __setitem__(self, key: str, value: str) -> None:
        # FIX: Brainfuck
        if key in self.positions:
            index = self.positions[key]
        else:
            self.len += 1
            index = self.len

        self.nodes[index] = value

class Layout:
    def __init__(self):
        self.layer: defaultdict[int, Layer] = defaultdict(Layer)
        self.index: dict[Layer, int] = dict()
        self.node_to_layer: dict[str, int] = dict()

        self.marked_segments: defaultdict[tuple[str, str], bool] = defaultdict(lambda: False)

        self.root: dict[str, str] = dict()
        self.align: dict[str, str] = dict()
        self.sink: dict[str, str] = dict()
        self.shift: dict[str, float] = dict()
        self.x: [str, Optional[int]] = defaultdict(lambda: None)

    def add_node(self, layer: int, node: str):
        self.layer[layer].append(node)
        self.index[self.layer[layer]] = layer
        self.node_to_layer[node] = layer

    def mark_segement(self, edge: tuple[str, str]):
        self.marked_segments[edge] = True

    def is_marked(self, edge: tuple[str, str]):
        return self.marked_segments[edge]

    def normalize_and_sort(self, G: DiGraph):
        keys = list(self.layer.keys())
        min_layer = min(keys) - 1

        old = copy.copy(self.layer)
        self.layer.clear()
        self.index.clear()

        for key in keys:
            layer = old[key]

            layer.sort(G)

            self.layer[key - min_layer] = layer
            self.index[layer] = key - min_layer

        for key in self.node_to_layer:
            self.node_to_layer[key] -= min_layer

    def pos(self, node: str):
        return self.layer[self.node_to_layer[node]].pos(node)

    def get_layer(self, node: str):
        return self.layer[self.node_to_layer[node]]

    def L(self, i):
        return self.layer[i]

    def __len__(self):
        return len(self.layer)

    def __getitem__(self, item: int):
        return self.layer[item]

def list_upper_neighbors(G, layers: Layout, i: int, k: int, direction: Direction):
    if direction.DOWN in direction:
        return sorted([u for u, v in G.in_edges(layers[i].node_by_index(k))],
                      key=lambda n: layers.pos(n))
    else:
        return sorted([v for u, v in G.out_edges(layers[i].node_by_index(k))],
                      key=lambda n: layers.pos(n))

def algo_1(G: Union[DiGraph, Path], layers: Layout) -> Layout:
    # ----Algo. 1.------------------------
    for i in range(2, len(layers) - 1):
        k0 = 0
        l = 1
        for l1 in range(1, len(layers[i + 1]) + 1):
            inner_nodes = [(u, v) for u, v in G.in_edges(layers[i + 1][l1]) if "DUMMY_NODE" in u and "DUMMY_NODE" in v]
            if l1 == len(layers[i + 1]) or inner_nodes:
                k1 = len(layers[i])
                if inner_nodes:
                    vl1i1 = layers[i + 1][l1]
                    upper_neighbor, _ = inner_nodes[0]
                    k1 = layers.pos(upper_neighbor)
                while l < l1:
                    for vk, vl in G.in_edges(layers[i + 1][l]):
                        k = layers.pos(vk)
                        if k < k0 or k > k1:
                            layers.mark_segement((vk, vl))

                    l += 1
                k0 = k1
    return layers

def algo_2(G: DiGraph, layout: Layout, direction: Direction) -> Layout:
    ver_range = range(1, len(layout) + 1)

    if Direction.UP in direction:
        ver_range = reversed(ver_range)

    # ----Algo. 2.------------------------
    for i in ver_range:
        r = 0
        hoz_range = range(1, len(layout[i]) + 1)

        sign = 1
        if Direction.LEFT in direction:
            hoz_range = reversed(hoz_range)
            sign = -1

        for k in hoz_range:
            if upper_neighbors := list_upper_neighbors(G, layout, i, k, direction):
                f = floor(((len(upper_neighbors) + 1) / 2))
                c = ceil(((len(upper_neighbors) + 1) / 2))

                vk = layout[i].node_by_index(k)
                for m in [f, c]:
                    if layout.align[vk] == vk:
                        um = upper_neighbors[m - 1]

                        if Direction.LEFT in direction:
                            edge = (vk, um)
                        else:
                            edge = (um, vk)

                        if not layout.is_marked(edge) and (r * sign < sign * layout.pos(um)):
                            layout.align[um] = vk
                            layout.root[vk] = layout.root[um]
                            layout.align[vk] = layout.root[vk]
                            r = layout.pos(um)
    return layout

def algo_3(G: DiGraph, layout: Layout, delta: float, direction: Direction) -> Layout:
    # ----Algo. 3.------------------------
    def place_block(v: str):
        if layout.x[v] is None:
            layout.x[v] = 0
            w = v
            while True:
                if layout.pos(w) > 1:
                    u = layout.root[layout.get_layer(w).predecessor(w)]
                    place_block(u)
                    if layout.sink[v] == v:
                        layout.sink[v] = layout.sink[u]
                    if layout.sink[v] != layout.sink[u]:
                        layout.shift[layout.sink[u]] = min(layout.shift[layout.sink[u]],
                                                           layout.x[v] - layout.x[u] - delta)
                    else:
                        layout.x[v] = max(layout.x[v], layout.x[u] + delta)
                w = layout.align[w]
                if w == v:
                    break

    for v in G.nodes:
        if layout.root[v] == v:
            place_block(v)

    for v in G.nodes:
        layout.x[v] = layout.x[layout.root[v]]
        if layout.shift[layout.sink[layout.root[v]]] < inf:
            layout.x[v] += layout.shift[layout.sink[layout.root[v]]]

        G.nodes[v]['pi'] = layout.x[v]

    # ------------------------------------
    return layout

def _brand_koepf(G: DiGraph, name, path, delta=1.0) -> DiGraph:
    original_layout = Layout()

    # Add Node in to the layers datastructurs
    for node in G.nodes:
        original_layout.add_node(G.nodes[node]['level'], node)
        original_layout.root[node] = node
        original_layout.align[node] = node
        original_layout.sink[node] = node
        original_layout.shift[node] = inf

    directions = [Direction.UP | Direction.LEFT, Direction.UP | Direction.RIGHT, Direction.DOWN | Direction.LEFT,
                  Direction.DOWN | Direction.RIGHT]
    layouts = []
    original_layout.normalize_and_sort(G)
    original_layout = algo_1(G, original_layout)

    i = 0

    for direction in directions:
        l = deepcopy(original_layout)

        l = algo_2(G, l, direction)
        l = algo_3(G, l, delta, direction)

        print_result(G, path.joinpath(f'{i}.png'))
        i += 1
        layouts.append(l)

    # for layer in

    # width = []
    # minimum = []
    # shift = []
    # minWidthLayout = 0
    #
    # for i, layout in enumerate(layouts):
    #     minimum.append(inf)
    #
    #     width.append(layout.size())
    #     if width[minWidthLayout] > width[i]:
    #         minWidthLayout = i
    #
    #     for l in range(1, len(layout) + 1):
    #         for n in range(1, len(layout[l]) + 1):
    #             node = layout[l].node_by_index(n)
    #             minimum[i] = min(minimum[i], layout.x[node])
    #
    # for i, layout in enumerate(layouts):
    #     shift.append(minimum[minWidthLayout] - minimum[i])
    #
    for node in G.nodes:
        calc_x = []

        for i, layout in enumerate(layouts):
            x = layout.x[node]
            calc_x.append(x)

        calc_x = sorted(calc_x)
        G.nodes[node]['pi'] = (calc_x[1] + calc_x[2]) / 2.0

    return G

def _draw_graph(G, path, figsize):
    colors = [G[u][v]['color'] for u, v in G.edges()]

    pos = {}
    for node in G.nodes:
        y = -G.nodes[node]['level']
        x = G.nodes[node]['pi']
        pos[node] = (x, y)

    print(f"\rDrawing {path.stem}...", end='')
    plt.figure(figsize=figsize)
    nx.draw(G, pos=pos, edge_color=colors, with_labels=True)

    print(f"\rWriting {path.stem} to file...", end='')
    plt.savefig(path)
    plt.close()

def print_result(G, path, figsize=(50, 10)):
    p = Process(target=_draw_graph, args=(G, path, figsize))
    p.start()

def run(path, graph_key, figsize, use_results=True, delta=1):
    file_name = Path(path).stem

    result_path = Path.cwd().joinpath(f'result/brand_koepf/{Path(path).stem}/{graph_key}/')
    result_path.mkdir(parents=True, exist_ok=True)

    if use_results:
        G = gs.load_results(path, graph_key)
    else:
        G = gh.load_file(path, key=graph_key)

    print_result(G, result_path.joinpath(f'{file_name}_before.png'))

    print(f"\rLayer assignment and crossing reduction | {graph_key}", end='')
    G = _brand_koepf(G, file_name, result_path, delta=delta)
    print("\rx-coordinate assigment...", end='')

    print_result(G, result_path.joinpath(f'{file_name}_after.png'))

    nx.readwrite.write_gpickle(G, result_path.joinpath('graph.dat'))
    plt.close()

    print(f"\rDone! {file_name} | {graph_key}", end='\n')

reason: import networkx


text:
import numpy as np
import h5py
from tqdm import tqdm, tqdm_notebook
import torch
import faiss

def save_submission(
    embed_qry_d,
    embed_ref_d,
    save_path='./submission.h5',
):

    with h5py.File(save_path, "w") as f:
        f.create_dataset("query", data=embed_qry_d['embedding'])
        f.create_dataset("reference", data=embed_ref_d['embedding'])
        f.create_dataset('query_ids', data=[str(s) for s in embed_qry_d['sample_id']])
        f.create_dataset('reference_ids', data=[str(s) for s in embed_ref_d['sample_id']])

    print(f' - Saved: {save_path}')
    return None

def read_submission(
    save_path='./sub.h5',
    ):
    embed_qry_d = {}
    embed_ref_d = {}

    with h5py.File(save_path, "r") as f:
        embed_qry_d['embedding'] = f['query'][:]
        embed_ref_d['embedding'] = f['reference'][:]
        embed_qry_d['sample_id'] = f['query_ids'][:].astype(str)
        embed_ref_d['sample_id'] = f['reference_ids'][:].astype(str)

    return embed_qry_d, embed_ref_d

def calc_embed_d(
    model,
    dataloader,
    do_simple_augmentation=False,
    device='cuda:0',
):

    model.to(device)
    model.eval()

    with torch.no_grad():
        image_ids_v = []
        embeddings_v = []

        for data in tqdm(dataloader):
            img = data['img'].to(device)

            embed_v = model.predict_embedding(
                img,
                do_simple_augmentation=do_simple_augmentation,
                calc_triplet_cls=False
            ).detach().cpu().numpy()

            image_ids_v.append( data['sample_id'] )
            embeddings_v.append( embed_v )

    image_ids_v = np.concatenate(image_ids_v, axis=0)
    embeddings_v = np.concatenate(embeddings_v, axis=0)

    embed_d = {
        'embedding':embeddings_v,
        'sample_id':image_ids_v
    }

    return embed_d

def add_GT_description_v3(embed_qry_d, embed_ref_d, public_gt, factor=1.0):
    """
    returns a copy of "embed_qry_d" with the ref embedding replaced.
    it points embed_qry towards embed_ref mantaining L2(embed_qry).
    """

    embed_qry_d = {
        'embedding': embed_qry_d['embedding'].copy(),
        'sample_id': embed_qry_d['sample_id'].copy(),
    }

    for query_id, reference_id in tqdm(public_gt[~public_gt.reference_id.isna()].values):

        query_idx = int(query_id[1:])
        reference_idx = int(reference_id[1:])

        norm_qry = np.linalg.norm( embed_qry_d['embedding'][query_idx] )
        embed_qry_d['embedding'][query_idx] = embed_ref_d['embedding'][reference_idx] * norm_qry * factor

    return embed_qry_d

def add_GT_description_v2(embed_qry_d, embed_ref_d, public_gt):
    """
    returns a copy of "embed_qry_d" with the ref embedding replaced.
    it points embed_qry towards embed_ref mantaining L2(embed_qry).
    """

    embed_qry_d = {
        'embedding': embed_qry_d['embedding'].copy(),
        'sample_id': embed_qry_d['sample_id'].copy(),
    }

    for query_id, reference_id in tqdm(public_gt[~public_gt.reference_id.isna()].values):

        query_idx = int(query_id[1:])
        reference_idx = int(reference_id[1:])

        norm_qry = np.linalg.norm( embed_qry_d['embedding'][query_idx] )
        embed_qry_d['embedding'][query_idx] = embed_ref_d['embedding'][reference_idx] * norm_qry

    return embed_qry_d

def add_GT_description(embed_qry_d, embed_ref_d, public_gt):
    """
    returns a copy of "embed_qry_d" with the ref embedding replaced.
    """

    embed_qry_d = {
        'embedding': embed_qry_d['embedding'].copy(),
        'sample_id': embed_qry_d['sample_id'].copy(),
    }

    for query_id, reference_id in tqdm(public_gt[~public_gt.reference_id.isna()].values):

        query_idx = int(query_id[1:])
        reference_idx = int(reference_id[1:])

        embed_qry_d['embedding'][query_idx] = embed_ref_d['embedding'][reference_idx]

    return embed_qry_d

def calc_match_scores(embed_qry_d, embed_ref_d, k=500, steps=100, gpu_id=None):

    dim = embed_ref_d['embedding'].shape[1]

    assert embed_qry_d['embedding'].shape[1] == embed_ref_d['embedding'].shape[1]

    if gpu_id is not None:
        cfg = faiss.GpuIndexFlatConfig()
        cfg.useFloat16 = False
        cfg.device = gpu_id
        index = faiss.GpuIndexFlatL2(
            faiss.StandardGpuResources(),
            dim,
            cfg,
        )

    else:
        index = faiss.IndexFlatL2( dim )

    index.add( embed_ref_d['embedding'] )

    n_samples = len(embed_qry_d['embedding'])
    d_steps   = n_samples // steps

    D_v = []
    I_v = []
    for i in tqdm( range(steps) ):
        i_s = d_steps * i
        i_e = (i_s + d_steps) if (i < steps-1) else  n_samples

        D, I = index.search( embed_qry_d['embedding'][i_s:i_e], k)

        D_v.append(D)
        I_v.append(I.astype(np.int32))

    D = np.concatenate(D_v)
    I = np.concatenate(I_v)

    C = 1.0 - 0.5 * D

    match_scores_d = {'I':I.astype(np.int32), 'C':C}

    return match_scores_d

def sigmoid(x):
    return 1.0/(1.0 + np.exp(-x))

def get_pca_parameters(pca):
    A  = np.zeros( (pca.d_in * pca.d_out) )
    PCAMat = np.zeros( (pca.d_in * pca.d_out) )

    b  = np.zeros( (pca.d_out) )
    ev = np.zeros( (pca.d_in) )

    # for i in range(pca.d_in):
    #     ev[i] = pca.eigenvalues.at(i)
    #     for j in range(pca.d_out):
    #         A[i,j] = pca.A.at(j + i*pca.d_out )
    #         PCAMat[i,j] = pca.PCAMat.at(j + i*pca.d_out )

    for i in range(pca.d_in*pca.d_out):
        A[i] = pca.A.at(i)
        PCAMat[i] = pca.PCAMat.at(i)

    PCAMat = PCAMat.reshape( (-1,pca.d_in))
    A = A.reshape( (-1,pca.d_in))

    for i in range(pca.d_in):
        ev[i] = pca.eigenvalues.at(i)

    for j in range(pca.d_out):
        b[j] = pca.b.at(j)

    ret_d = {
        'A':A,
        'b':b,
        'ev':ev,
        'PCAMat':PCAMat,

    }

#     plt.plot(ev)
#     (PCAMat == A).all()

    return ret_d


text:
#! /usr/local/bin/python3.5
# -*- coding: utf-8 -*-
"""
vbHmm_Common.py
Common VB-HMM engine.

Reference:
  Christopher M. Bishop, "Pattern Recognition and Machine Learning", Springer, 2006

Created by OKAMOTO Kenji, SAKO Yasushi and RIKEN
Copyright 2011-2015
Cellular Informatics Laboratory, Advance Science Institute, RIKEN, Japan.
All rights reserved.

@author: kenji
"""

import math
import numpy as np
import sys

## Structure encapsulating data to analyze
class vbHmm_Data:
    def __init__(self, dLen):
        self.N = dLen       # number of data points
        self.T = None        # total length of data (may not be always necessary)

## Commonly used parameters
class vbHmm_Params:
    def __init__(self, sNo, dLen):
        self.dLen = dLen                                        # number of data points
        self.sNo = sNo                                         # number of states

        self.gmMat = np.zeros((dLen, sNo), dtype=np.float_)      # gamma matrix for E-step
        self.xiMat = np.zeros((dLen, sNo, sNo), dtype=np.float_)  # xi matrix for E-step
        self.aMat = np.zeros((dLen, sNo), dtype=np.float_)       # alpha matrix for Baum-Welch
        self.bMat = np.zeros((dLen, sNo), dtype=np.float_)       # beta matrix for Baum-Welch
        self.cn = np.zeros(dLen, dtype=np.float_)               # scaling factor for Baum-Welch

        # temporary storage of calculation to save time
        self.valpZnZn1 = np.zeros((sNo, sNo), dtype=np.float_)
        self.valpXnZn = np.zeros((dLen, sNo), dtype=np.float_)

## Results of analysis
class vbHmm_Results:
    def __init__(self):
        self.iteration = 0              # calculation steps
        self.maxLq = 0.0                # final lower bound
        self.LqArr = None               # time series of lower bound
        self.maxSumTraj = None          # resulting state transition trajectory

#//////////////////////////////////////////////////////////////////  VB-HMM Execution Functions
class vbHmmMain:

    def __init__(self, modelClass):
        self.modelClass = modelClass

    def modelComparison(self, data, sFrom, sTo, trials, maxIteration, threshold, out_name, logFP):
        logFP.write("  No. of states from {0:d} to {1:d},  trials = {2:d}, ".format(sFrom, sTo, trials))
        logFP.write("  analyze:  maxIteration = {0:d},  threshold = {1} \n\n".format(maxIteration, threshold))

        LqVsK = np.zeros(trials * (sTo - sFrom + 1), dtype=np.float_)

        maxS = 0
        maxLq = -sys.float_info.max
        for s in range(sFrom, sTo+1):
            modelArray = []
            for t in range(trials):
                st = (s - sFrom) * trials + t
                model = self.modelClass(data, s)
                modelArray.append(model)

                LqVsK[st] = self.vbHmm_Main(model, maxIteration, threshold, logFP)

                if LqVsK[st] > maxLq:
                    maxLq = LqVsK[st]
                    maxS = s

            maxLqForS = LqVsK[(s - sFrom) * trials]
            maxT = 0
            for t in range(1, trials):
                st = (s - sFrom) * trials + t
                if LqVsK[st] > maxLqForS:
                    maxLqForS = LqVsK[st]
                    maxT = t

            modelArray[maxT].outputResults(out_name, logFP)

            if s >= (maxS+3):
                sTo = s
                break

        fn = out_name + ".LqVsK"
        with open(fn, "w") as fp:
            for s in range(trials * (sTo - sFrom + 1)):
                fp.write("{0:2d}, {1:.20f}\n".format((s//trials) + sFrom, LqVsK[s]))

        return maxS

    #//////////////////////////////////////////////////////////////////  VB-HMM Common Engine
    def vbHmm_Main(self, m, maxIteration, threshold, logFP):

        m.r.LqArr = np.zeros(maxIteration, dtype=np.float_)

        for i in range(maxIteration):
#            // E-step
            self.forwardBackward(m)

            m.calcStatsVars()
            m.r.LqArr[i] = m.varLowerBound()

#            // End loop if derivative of variational lower bound reaches threshold.
            if i > 0  and (math.fabs((m.r.LqArr[i] - m.r.LqArr[i-1]) / m.r.LqArr[i]) < threshold):
                maxIteration = i + 1
                break

#            // M-step
            m.maximization()

        else:
            logFP.write("MAX iteration ({0:d}) reached.\n".format(maxIteration))

        m.reorderParameters()
        self.maxSum(m)

        m.r.iteration = maxIteration
        m.r.LqArr.resize(maxIteration)
        m.r.maxLq = m.r.LqArr[maxIteration-1]
        logFP.write("  iteration: {0:d}    evidence p(x|K={1:d}) = {2:.20f} \n".format(maxIteration, m.p.sNo, m.r.maxLq))

        return m.r.maxLq

#// Baum-Welch algorithm for E-step calculation
    def forwardBackward(self, m):
        sNo = m.p.sNo
        dLen = m.p.dLen

#        // forward
        m.p.cn[0] = 0.0
        for i in range(sNo):
            m.p.valpXnZn[0][i] = m.pTilde_xn_zn(0, i)
            m.p.aMat[0][i] = m.pTilde_z1(i) * m.p.valpXnZn[0][i]

            m.p.cn[0] += m.p.aMat[0][i]

            for j in range(sNo):
                m.p.valpZnZn1[i][j] = m.pTilde_zn_zn1(i, j)

        for i in range(sNo):
            m.p.aMat[0][i] /= m.p.cn[0]

        for n in range(1, dLen):
            m.p.cn[n] = 0.0
            for j in range(sNo):
                m.p.aMat[n][j] = 0.0
                for i in range(sNo):
                    m.p.aMat[n][j] += m.p.aMat[n-1][i] * m.p.valpZnZn1[i][j]

                m.p.valpXnZn[n][j] = m.pTilde_xn_zn(n, j)
                m.p.aMat[n][j] *= m.p.valpXnZn[n][j]
                m.p.cn[n] += m.p.aMat[n][j]

            for j in range(sNo):
                m.p.aMat[n][j] /= m.p.cn[n]

#        // backward
        for i in range(sNo):
            m.p.bMat[dLen-1][i] = 1.0

        for nn in range(dLen-1):
            n = (dLen - 2) - nn
            for i in range(sNo):
                m.p.bMat[n][i] = 0.0
                for j in range(sNo):
                    m.p.bMat[n][i] += m.p.bMat[n+1][j] *  m.p.valpZnZn1[i][j] *  m.p.valpXnZn[n+1][j]
                m.p.bMat[n][i] /= m.p.cn[n+1]

#        // update gamma
        for n in range(dLen):
            for i in range(sNo):
                m.p.gmMat[n][i] = m.p.aMat[n][i] * m.p.bMat[n][i]

#        // update xi
        for i in range(sNo):
            for j in range(sNo):
                m.p.xiMat[0][i][j] = 0.0

        for n in range(1, dLen):
            for i in range(sNo):
                for j in range(sNo):
                    xiTerm  = m.p.aMat[n-1][i]    * m.p.valpXnZn[n][j]
                    xiTerm *= m.p.valpZnZn1[i][j] * m.p.bMat[n][j]
                    m.p.xiMat[n][i][j] = xiTerm / m.p.cn[n]

#// Viterbi algorithm to construct most likely trajectory
    def maxSum(self, m):
        sNo = m.p.sNo
        dLen = m.p.dLen

        m.r.maxSumTraj = np.zeros(dLen, dtype=np.int_)
        wnMat = np.zeros((dLen, sNo), dtype=np.float_)
        phiMat = np.zeros((dLen, sNo), dtype=np.float_)

#        // forward
        for i in range(sNo):
            wnMat[0][i] = math.log(m.pTilde_z1(i)) + math.log(m.pTilde_xn_zn(0, i))

        for n in range(1, dLen):
            for j in range(sNo):
                maxWn = math.log(m.pTilde_zn_zn1(0, j)) + wnMat[n-1][0]
                maxI = 0
                for i in range(1, sNo):
                    wnTest = math.log(m.pTilde_zn_zn1(i, j)) + wnMat[n-1][i]
                    if wnTest > maxWn:
                        maxWn = wnTest
                        maxI = i

                phiMat[n][j] = maxI
                try:
                    wnMat[n][j] = math.log(m.pTilde_xn_zn(n, j)) + maxWn
                except ValueError:
#                    sys.stderr.write("  -- n:{0:d}, j:{1:d}/{2:d} - {3}\n".format(n, j, sNo, m.pTilde_xn_zn(n, j)))
                    wnMat[n][j] = -sys.float_info.max

#        // backward
        maxWn = wnMat[dLen-1][0]
        maxI = 0
        for i in range(1, sNo):
            if wnMat[dLen-1][i] > maxWn:
                maxWn = wnMat[dLen-1][i]
                maxI = i

        m.r.maxSumTraj[dLen-1] = maxI
        for nn in range(dLen-1):
            n = (dLen - 1) - nn
            m.r.maxSumTraj[n-1] = phiMat[n][m.r.maxSumTraj[n]]

if __name__ == "__main__":
    pass

#########################################################

max_stars_repo_name: okamoto-kenji/varBayes-HMM

max_issues_repo_name: okamoto-kenji/varBayes-HMM

max_forks_repo_name: okamoto-kenji/varBayes-HMM


text:
import numpy as np
import random

letters = ['a','b','c','d','e','f','g','h','i','j','k','l','m',
'n','o','p','q','r','s','t','u','v','w','x','y','z',]

def noise_maker(sentence, threshold):
    '''Relocate, remove, or add characters to create spelling mistakes'''

    noisy_sentence = []
    i = 0
    while i < len(sentence):
        random = np.random.uniform(0,1,1)
        # Most characters will be correct since the threshold value is high
        if random < threshold:
            noisy_sentence.append(sentence[i])
        else:
            new_random = np.random.uniform(0,1,1)
            # ~33% chance characters will swap locations
            if new_random > 0.67:
                if i == (len(sentence) - 1):
                    # If last character in sentence, it will not be typed
                    continue
                else:
                    # if any other character, swap order with following character
                    noisy_sentence.append(sentence[i+1])
                    noisy_sentence.append(sentence[i])
                    i += 1
            # ~33% chance an extra lower case letter will be added to the sentence
            elif new_random < 0.33:
                random_letter = np.random.choice(letters, 1)[0]
                noisy_sentence.append(random_letter)
                noisy_sentence.append(sentence[i])
            # ~33% chance a character will not be typed
            else:
                pass
        i += 1
    return noisy_sentence


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 20 13:18:37 2019

@author: Luke
"""

#==============================================================================
#SUMMARY
#==============================================================================

#This script generates ERA5 lake ice depth signal plots

#==============================================================================
#IMPORT
#==============================================================================

import xarray as xr
import os
import numpy as np
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import matplotlib as mpl

#==============================================================================
#FUNCTIONS
#==============================================================================

def reader(file):
    ds = xr.open_dataset(file, decode_times=False)
    da = ds.licd.squeeze(dim='time',drop=True)
    return da

#==============================================================================
#SETTINGS
#==============================================================================

title_font = 13

tick_font = 11

lats_font = 7

#==============================================================================
#INITIALIZE
#==============================================================================

directory = '/Users/Luke/Documents/PHD/C3S_511/DATA/era5/04_2020/icedepth/depth/signal'

o_directory = '/Users/Luke/Documents/PHD/C3S_511/SPQB/04_2020/era5'

#set directory
os.chdir(directory)

#==============================================================================
#OPEN DATA
#==============================================================================

files = []
for file in sorted(os.listdir(directory)):
    if '.nc' in file:
        files.append(file)

JAN = reader(files[4])
FEB = reader(files[3])
MAR = reader(files[7])
APR = reader(files[0])
MAY = reader(files[8])
JUN = reader(files[6])
JUL = reader(files[5])
AUG = reader(files[1])
SEP = reader(files[11])
OCT = reader(files[10])
NOV = reader(files[9])
DEC = reader(files[2])

months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']

dataset = [JAN,FEB,MAR,APR,MAY,JUN,JUL,AUG,SEP,OCT,NOV,DEC]

sample_da = reader(files[0])

lat = sample_da.lat.values
lon = sample_da.lon.values

#==============================================================================
#DATA TESTS
#==============================================================================

testjan_min = np.nanmin(JAN.values)
testjan_max = np.nanmax(JAN.values)

testfeb_min = np.nanmin(FEB.values)
testfeb_max = np.nanmax(FEB.values)

testmar_min = np.nanmin(MAR.values)
testmar_max = np.nanmax(MAR.values)

#==============================================================================
#PLOT MAIN FIG
#==============================================================================

cmap_whole = plt.cm.get_cmap('RdBu_r')
cmap55 = cmap_whole(0.01)
cmap50 = cmap_whole(0.05)   #blue
cmap45 = cmap_whole(0.1)
cmap40 = cmap_whole(0.15)
cmap35 = cmap_whole(0.2)
cmap30 = cmap_whole(0.25)
cmap25 = cmap_whole(0.3)
cmap20 = cmap_whole(0.35)
cmap10 = cmap_whole(0.4)
cmap0 = 'darkgrey'
cmap_5 = cmap_whole(0.55)
cmap_10 = cmap_whole(0.6)
cmap_20 = cmap_whole(0.65)
cmap_25 = cmap_whole(0.7)
cmap_30 = cmap_whole(0.75)
cmap_35 = cmap_whole(0.8)
cmap_40 = cmap_whole(0.85)
cmap_45 = cmap_whole(0.9)
cmap_50 = cmap_whole(0.95)  #red
cmap_55 = cmap_whole(0.99)

cmap = mpl.colors.ListedColormap([cmap_50,cmap_40,cmap_30,cmap_20,cmap0,
                                  cmap20,cmap30,cmap40,cmap50],N=9)
cmap.set_over(cmap55)
cmap.set_under(cmap_55)

values = [-1.0,-0.75,-0.50,-0.25,-.025,0.025,0.25,0.5,0.75,1.0]
tick_locs = [-1.0,-0.75,-0.50,-0.25,0,0.25,0.5,0.75,1.0]
norm = mpl.colors.BoundaryNorm(values,cmap.N)

#=============================================================================
#SET PLOTS
#=============================================================================

f, axes = plt.subplots(4,3,figsize=(15,15));

lon, lat = np.meshgrid(lon, lat)

count = 0
for month,ax in zip(dataset,axes.flatten()):
    count = count+1
    m = Basemap(projection='npaeqd',round=True,boundinglat=20,\
                lat_0=80,lon_0=0,resolution='l');
    m.ax = ax
    m.drawcoastlines(linewidth=0.4);
    m.drawmapboundary(fill_color='whitesmoke');
    m.fillcontinents(color='white');
    m.pcolormesh(lon,lat,month,latlon=True,cmap=cmap,norm=norm,vmax=1,vmin=-1,zorder=3)
    ax.set_title(months[count-1],loc='left',fontsize=title_font)

#==============================================================================
#COLORBAR
#==============================================================================

cbaxes = f.add_axes([0.2515, 0.165, 0.5, 0.015])
cb = mpl.colorbar.ColorbarBase(ax=cbaxes,cmap=cmap,
                               norm=norm,
                               spacing='uniform',
                               orientation='horizontal',
                               extend='both',
                               ticks=tick_locs)
cb.set_label('Change in ice thickness (m)',size=title_font)
cb.ax.xaxis.set_label_position('top');
cb.ax.tick_params(labelcolor='0.2',labelsize=tick_font,color='0.2',\
                  length=2.5,width=0.35,direction='out'); #change color of ticks?
cb.ax.set_xticklabels([r'$\leq$-1.0','-0.75','-0.50',\
                       '-0.25','0','0.25','0.5','0.75',r'1.0$\leq$'])
cb.outline.set_edgecolor('0.2')
cb.outline.set_linewidth(0.4)

bluelabel = 'More ice cover'
redlabel = 'Less ice cover'

#plot arrows
plt.text(0.75, -2.8, bluelabel, size=title_font, ha='center', va='center')
plt.text(0.25, -2.8, redlabel, size=title_font, ha='center', va='center')

plt.arrow(0.505, -3.5, 0.44, 0, width=0.25, linewidth=0.1, label=bluelabel,\
          shape='right', head_width=0.5, head_length=0.06,\
          facecolor=cmap40, edgecolor='k', clip_on=False)
plt.arrow(0.495, -3.5, -0.44, 0, width=0.25, linewidth=0.1, label=redlabel,\
          shape='left', head_width=0.5, head_length=0.06,\
          facecolor=cmap_40, edgecolor='k', clip_on=False)

plt.subplots_adjust(left=0.25, right=0.75, bottom=0.2, top=0.85, wspace=0.05, hspace=0.05)

plt.show()

#save figure
f.savefig(o_directory+'/'+'D511.6.4.b1_ERA5_lakes_mixedlayertemperature_icedepth_Section_2.4.2_Figure_2.png',bbox_inches='tight',dpi=500)


text:
"""
Usage:  python process.py <coupling_strength> <slope_parameter> <label>
  where <coupling_strength> globally scales the coupling,
        <slope_parameter> determines the slope of the sigmoid, and
        <label> is an integer to label this parameter set.
"""

import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
from scipy import signal
import scipy.io as sio
import numpy as np
import numpy.lib.scimath as sm
import sys
import os
import errno
import glob

datadir = './datafiles'
imgdir = './img'

def load_mat(filename):
    """load array from a MATLAB .mat file"""
    return sio.loadmat(filename)['data']

def sigmoid(x, a, midpoint):
    return 1.0/(1.0 + np.exp(-a*(x-midpoint)))

def siggain(x, a, midpoint):
    """first derivative of sigmoid"""
    return a/(2.0 + 2.0*np.cosh(a*(x-midpoint)))

def inverse_sigmoid(phi, gain, midpoint):
    return midpoint - np.log(1.0/phi - 1.0)/gain

def hilbert_phase(ts, axis=0):
    """Phase of the analytic signal, using the Hilbert transform"""
    return np.angle(signal.hilbert(signal.detrend(ts, axis=axis), axis=axis))

def mod2pi(ts):
    """For a timeseries where all variables represent phases (in radians),
    return an equivalent timeseries where all values are in the range (-pi, pi]
    """
    return np.pi - np.mod(np.pi - ts, 2*np.pi)

def order_param(ts, axis=1):
    """Order parameter of phase synchronization.
    ts is assumed to hold phase values in radians."""
    return np.abs(np.exp(1.0j * ts).mean(axis=axis))

def phase_entropy(ts, bins=20):
    """Entropy of phase distrib estimated by partition of (-pi,pi] into m bins.
    ts is assumed to be a 1D vector of phase values in the range (-pi, pi]."""
    n = ts.shape[0]
    (h, _) = np.histogram(ts, bins, range=(-np.pi, np.pi), density=False)
    p = np.true_divide(h, n)
    q = p.copy()
    q[h==0] = 1.0 # For empty bins, contribution to the sum should be zero.
    return -np.sum(p*np.log(q))

_third = np.float64(1)/3
_ca = (np.float64(686)/3)**_third
_cb = np.float64(18)**_third
# Equilibrium V, as a function of local input I.
def Veq(I):
    s = np.power(72 - 9*I + np.sqrt(9300.0 + 81*I**2 -1296*I), _third)
    return 1.0 + _ca/s - s/_cb

# Jacobian first eigenvalue, as a function of equilibrium V.
def eig(V):
    return -10 + 60*V -30*V**2 + 10*sm.sqrt(-39+12*V +30*V**2 -36*V**3 +9*V**4)

# Make dirs without complaining if they already exist
def mkdirp(path):
    try:
        os.mkdir(path)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

gcs=float(sys.argv[1]) # global coupling strength
slope=float(sys.argv[2]) # sigmoid slope
label=int(sys.argv[3]) # parameter set label
midpoint = 1.5 # horizontal shift of sigmoid

# Create subdirectories for output
for d in ['meancoup', 'meangains', 'meansigin', 'meaninstab', 'meanop',
          'meanpe']:
    mkdirp(datadir + '/' + d)
for d in ['corrgain_a', 'corrgain_g', 'coup', 'gain', 'orbit', 'sig', 'instab',
          'sigin']:
    mkdirp(imgdir + '/' + d)

filenames = glob.glob(datadir + '/raw/Raw_SimplePreSigmoidal_%06f_%06f_*.mat' %
        ( gcs, slope))

# First plot orbits
def plot_orbit(ar0, ar1, num, square=False, color='blue', alpha=0.01):
    """draw a translucent plot of the orbit"""
    index_partition = np.linspace(0, ar0.shape[0], num=num, dtype=int)
    fig = plt.figure(figsize=(10,10))
    ax = fig.add_subplot(111)
    for k in range(num - 1):
        i0 = index_partition[k]
        i1 = index_partition[k+1]
        ax.plot(ar0[i0:i1], ar1[i0:i1], color=color, alpha=alpha)
    (xmin, xmax) = (ar0.min(), ar0.max())
    (ymin, ymax) = (ar1.min(), ar1.max())
    if square:
        width = max(xmax - xmin, ymax - ymin)
        xpad = max(0.0, (width - (xmax - xmin))/2.0)
        ypad = max(0.0, (width - (ymax - ymin))/2.0)
        ax.set_xlim(xmin - xpad, xmax + xpad)
        ax.set_ylim(ymin - ypad, ymax + ypad)
    else:
        ax.set_xlim(xmin, xmax)
        ax.set_ylim(ymin, ymax)
    return fig

# Plotting orbits from first simulation only:
firstarray = load_mat(filenames[0])
# take a slice of 20 seconds from the middle of the 10 minute simulation
# with oscillations of ~10Hz that is about 200 cycles of each node
start = 5*60*2000
end = start + 20*2000
# append time series of all 76 nodes and plot together
toplot = np.vstack([firstarray[start:end,:,i] for i in range(76)])
fig00 = plot_orbit(toplot[:,0], toplot[:,1], num=50*76, alpha=0.006)
# find means
mean_V = firstarray[:,0,:].mean(axis=0)
mean_W = firstarray[:,1,:].mean(axis=0)
plt.scatter(mean_V, mean_W, c='red', alpha=0.1)
ax = fig00.axes[0]
ax.set_xlim(-2.0, 4.5)
ax.set_ylim(-35.0, 2.0)
ax.set_xlabel('V')
ax.set_ylabel('W')
ax.set_title('Orbits of all 76 nodes, $\\gamma$=%.02f, a=%.03f' % (gcs,slope))
fig00.savefig(imgdir + '/orbit/img' + str(label).zfill(4) +'.png', dpi=100)
plt.close()
del firstarray, toplot

# Estimate synchrony
print('Processing %d files for gcs %f, slope %f' % (
	len(filenames), gcs, slope))
# concatenate all trials, V variable only.
arrays = [load_mat(fn)[:,0,:] for fn in filenames]
allv = np.concatenate(arrays, axis=0) # shape (nreps*ntimepoints, nregions)
del arrays
allphase = hilbert_phase(allv)
mean_op = order_param(allphase, axis=1).mean()
ntimepoints = allphase.shape[0] # across all repetitions concatenated
pe = np.ones(ntimepoints) * np.nan
for i in range(ntimepoints):
    pe[i] = phase_entropy(allphase[i,:], bins=19)
mean_pe = pe.mean()
del allphase
np.savetxt(datadir + '/meanop/meanop_SimplePreSigmoidal_%06f_%06f.txt' % (
        gcs, slope), np.array([mean_op]))
np.savetxt(datadir + '/meanpe/meanpe_SimplePreSigmoidal_%06f_%06f.txt' % (
        gcs, slope), np.array([mean_pe]))

# Get mean gain for each node.
allgain = siggain(allv, slope, midpoint)
meangains = allgain.mean(axis=0) # mean over time only. shape (nregions,)
np.savetxt(datadir + '/meangains/meangains_SimplePreSigmoidal_%06f_%06f.txt' % (
        gcs, slope), meangains)
meangain = meangains.mean() # mean over all time and all regions.

# Now plot histograms.
# All trials and all nodes concatenated, V variable only.
allsigin = np.ravel(allv)
meansigin = allsigin.mean()
(min, max) = (-20.0, 20.0)

# sigmoid input distribution juxtaposed with sigmoid and gain curves
fig0 = plt.figure(figsize=(16,9))
# plot sigmoid and its derivative
domain = np.linspace(min, max, num=100)
sigvals = sigmoid(domain, slope, midpoint)
siggainvals = siggain(domain, slope, midpoint)
print('For gcs=%.02f a=%.03f, mean gain = %.03g' % (gcs, slope, meangain))
plt.plot(domain, sigvals, 'b-', linewidth=2.0, label="Sigmoid $S(V)$")
plt.plot(domain, siggainvals*4.5, color='magenta', alpha=0.2,
	 label="gain $S'(V)$ (rescaled)")
w = np.repeat(0.2*(max - min)/allsigin.size, len(allsigin))
plt.hist(allsigin, normed=False, bins=300, range=(min, max), weights=w,
	 histtype='stepfilled', color='grey', alpha=0.3,
	 label="distribution $p(V)$")
ax = fig0.axes[0]
plt.text(0.01, 0.70, ("For $\\gamma$=%.02f,a=%.03f, mean gain "%(gcs,slope)) +
	 "$\int_{-\infty}^{\infty} p(V) S'(V)dV$ = %.03g" %
	 meangain, ha='left', va='bottom', transform=ax.transAxes)
plt.legend(loc='upper left')
plt.title('distribution of input $V$ to Sigmoidal, $\\gamma$=%.02f, a=%.03f' %
        (gcs, slope))
plt.xlabel("sigmoid input $V$")
plt.xlim(min, max)
plt.ylim(0.0, 1.0)
fig0.savefig(imgdir + '/sig/img' + str(label).zfill(4) +'.png', dpi=100)
plt.close()

# sigmoid input distribution
fig1 = plt.figure(figsize=(16,9))
plt.hist(allsigin, normed=True, bins=300, range=(min, max),
         histtype='stepfilled', color='grey', alpha=0.3)
plt.title('distribution of Sigmoidal input, $\\gamma$=%.02f, a=%.03f' % (
	gcs, slope))
plt.xlabel("sigmoid input $V$")
plt.xlim(min, max)
plt.ylim(0.0, 15.0/(max - min))
fig1.savefig(imgdir + '/sigin/img' + str(label).zfill(4) +'.png', dpi=100)
plt.close()
np.savetxt(datadir + '/meansigin/meansigin_SimplePreSigmoidal_%06f_%06f.txt' % (
        gcs, slope), np.array([meansigin]))
del allsigin

# sigmoid gain distribution
(min, max) = (0.0, 0.30)
fig2 = plt.figure(figsize=(16,9))
plt.hist(allgain.ravel(), normed=True, bins=300, range=(min, max),
	 histtype='stepfilled', color='magenta', alpha=0.3)
plt.title('distribution of Sigmoidal gain , $\\gamma$=%.02f, a=%.03f' % (
	gcs, slope))
plt.xlabel("sigmoid gain $S'(V)$")
plt.xlim(min, max)
plt.ylim(0.0, 80.0/(max - min))
fig2.savefig(imgdir + '/gain/img' + str(label).zfill(4) +'.png', dpi=100)
plt.close()
del allgain

# retain only the first simulation for each parameter set
for fn in filenames[1:]:
    os.unlink(fn)

# TEMP: Now do the same for coupling input
filenames = glob.glob('datafiles/coupling/coup_SimplePreSigmoidal_%06f_%06f_*.mat' % (gcs, slope))
print('Processing %d files for coupling, gcs %f, slope %f' % (
	len(filenames), gcs, slope))
# concatenate all trials and all nodes together
arrays = [load_mat(fn).ravel() for fn in filenames]
allcoup = gcs * np.concatenate(arrays, axis=0)
del arrays

# coupling distribution
meancoup = allcoup.mean()
#(min, max) = (allcoup.min(), allcoup.max())
(min, max) = (0.0, 60.0)
fig3 = plt.figure(figsize=(16,9))
plt.hist(allcoup, normed=True, bins=300, range=(min, max),
         histtype='stepfilled', color='grey', alpha=0.3)
plt.title('distribution of coupling input, $\\gamma$=%.02f, a=%.03f' % (
	gcs, slope))
plt.xlabel("coupling input $\\gamma I_{net}$")
plt.xlim(min, max)
#plt.ylim(0.0, 15.0/(max - min))
plt.ylim(0.0, 0.4)
fig3.savefig(imgdir + '/coup/img' + str(label).zfill(4) +'.png', dpi=100)
plt.close()
np.savetxt('datafiles/meancoup/meancoup_SimplePreSigmoidal_%06f_%06f.txt' % (
        gcs, slope), np.array([meancoup]))

# distribution of instability measure
allinstab = np.reciprocal(np.abs(np.real(eig(Veq(allcoup)))))
del allcoup
meaninstab = allinstab.mean()
(min, max) = (0.0, 100.0)
fig4 = plt.figure(figsize=(16,9))
plt.hist(allinstab, normed=True, bins=300, range=(min, max),
         histtype='stepfilled', color='blueviolet', alpha=0.3)
plt.title('distribution of $|Re(eig)|^{-1}$, $\\gamma$=%.02f, a=%.03f' % (
	gcs, slope))
plt.xlabel("$|Re(eig)|^{-1}$ at equilibrium point")
plt.xlim(min, max)
#plt.ylim(0.0, 15.0/(max - min))
plt.ylim(0.0, 0.4)
fig4.savefig(imgdir + '/instab/img' + str(label).zfill(4) +'.png', dpi=100)
plt.close()
np.savetxt('datafiles/meaninstab/meaninstab_SimplePreSigmoidal_%06f_%06f.txt' % (
        gcs, slope), np.array([meaninstab]))
del allinstab

# retain only the first simulation for each parameter set
for fn in filenames[1:]:
    os.unlink(fn)

reason: import numpy,import scipy,from scipy


text:
# -*- coding: utf-8 -*-
'''
Written by Daniel M. Aukes and CONTRIBUTORS
Email: danaukes<at>asu.edu.
Please see LICENSE for full license.
'''

from .shapes import Point, Edge, Triangle
from .advancing_front import AdvancingFront, Node

class Basin(object):

    def __init__(self):
        self.left_node = None
        self.bottom_node = None
        self.right_node = None
        self.width = 0.0
        self.left_highest = False

    def Clear(self):
        self.left_node = None
        self.bottom_node = None
        self.right_node = None
        self.width = 0.0
        self.left_highest = False

class EdgeEvent(object):

    def __init__(self):
        self.constrained_edge = None
        self.right = False

class SweepContext(object):

    def __init__(self, polyline):
        self.basin = Basin()
        self.edge_event = EdgeEvent()
        self.edge_list = []

        self.points_ = polyline
        self.triangles_ = []
        self.map_ = []
        self.kAlpha = .3

        self.InitEdges(self.points_)

        self.front_ = None
        self.head_ = None
        self.tail_ = None
        self.af_head_ = None
        self.af_middle_ = None
        self.af_tail_ = None

    def AddHole(self, polyline):
        self.InitEdges(polyline)
        self.points_.extend(polyline)

    def AddPoint(self, point):
        self.points_.append(point)

    def GetTriangles(self):
        return self.triangles_

    def GetMap(self):
        return self.map_

    def InitTriangulation(self):
        import numpy
        xs = numpy.array([item.x for item in self.points_])
        ys = numpy.array([item.y for item in self.points_])
        xmin = xs.min()
        xmax = xs.max()
        ymin = ys.min()
        ymax = ys.max()

        dx = self.kAlpha * (xmax - xmin)
        dy = self.kAlpha * (ymax - ymin)
        self.head_ = Point(xmax + dx, ymin - dy)
        self.tail_ = Point(xmin - dx, ymin - dy)

        from operator import attrgetter
        self.points_ = sorted(self.points_, key=attrgetter('y', 'x'))
#        self.points_.reverse()

    def InitEdges(self, polyline):
        self.edge_list.extend(
            [Edge(line1, line2) for line1, line2 in zip(polyline, polyline[1:] + polyline[:1])])

    def GetPoint(self, index):
        return self.points_[index]

    def GetPoints(self):
        return self.points_

    def AddToMap(self, triangle):
        self.map_.append(triangle)

    def LocateNode(self, point):
        return self.front_.LocateNode(point.x)

    def CreateAdvancingFront(self, nodes):
        triangle = Triangle(self.points_[0], self.tail_, self.head_)
        self.map_.append(triangle)

        self.af_head_ = Node(triangle.GetPoint(1), triangle)
        self.af_middle_ = Node(triangle.GetPoint(0), triangle)
        self.af_tail_ = Node(triangle.GetPoint(2))
        self.front_ = AdvancingFront(self.af_head_, self.af_tail_)

        self.af_head_.next = self.af_middle_
        self.af_middle_.next = self.af_tail_
        self.af_middle_.prev = self.af_head_
        self.af_tail_.prev = self.af_middle_

    def RemoveNode(self, node):
        del node

    def MapTriangleToNodes(self, t):
        for i in range(3):
            if not t.GetNeighbor(i):
                n = self.front_.LocatePoint(t.PointCW(t.GetPoint(i)))
                if n is not None:
                    n.triangle = t

    def RemoveFromMap(self, triangle):
        self.map_.remove(triangle)

    def MeshClean(self, triangle):
        if triangle is not None and not triangle.IsInterior():
            triangle.IsInterior(True)
            self.triangles_.append(triangle)
            for i in range(3):
                if not triangle.constrained_edge[i]:
                    self.MeshClean(triangle.GetNeighbor(i))

    def front(self):
        return self.front_

    def point_count(self):
        return len(self.points_)

    def set_head(self, p1):
        self.head_ = p1

    def head(self):
        return self.head_

    def set_tail(self, p1):
        self.tail_ = p1

    def tail(self):
        return self.tail_


text:
import numpy as np
from copy import deepcopy
import pickle as pickle
import os

class dispersionSaver:
    def __init__(self, x_array, t_array, last_idx_array, y_array = None):
        self.x_array = x_array
        self.t_array = t_array
        self.last_idx_array = last_idx_array
        if y_array is not None:
            self.y_array = y_array

class caseInfo(object):
    def __init__(self, inj_type, n_realizations, network_name, max_dx, min_dx, max_v, min_v, max_dt, min_dt):
        self.inj_type = inj_type
        self.n_realizations = n_realizations
        self.network_name = network_name
        self.max_velocity = max_v
        self.min_velocity = min_v
        self.max_dx = max_dx
        self.min_dx = min_dx
        self.max_dt = max_dt
        self.min_dt = min_dt

def find_dispersion_object_min_max(dn1):
    dx = np.diff(dn1.x_array)
    dt = np.diff(dn1.time_array)
    max_dt = np.amax(dt)
    dt[dt == 0.0] = max_dt
    v_array = np.divide(dx, dt)
    v_array[np.isnan(v_array)] = 0.0
    max_dx = np.amax(dx)
    min_dx = np.amin(dx)
    min_dt = np.amin(dt)
    max_v = np.amax(v_array)
    min_v = np.amin(v_array)
    return max_v, min_v, max_dx, min_dx, max_dt, min_dt

def choose_extremes(max_v, max_v1, min_v, min_v1, max_dx, max_dx1, min_dx, min_dx1,
                    max_dt, max_dt1, min_dt, min_dt1):
    max_v = max(max_v, max_v1)
    min_v = min(min_v, min_v1)
    max_dx = max(max_dx, max_dx1)
    min_dx = min(min_dx, min_dx1)
    max_dt = max(max_dt, max_dt1)
    min_dt = min(min_dt, min_dt1)
    return max_v, min_v, max_dx, min_dx, max_dt, min_dt

class correlatedSaver(object):
    def __init__(self, trans_matrix, class_velocity, init_class_count,
                 dt, n_neg_class, n_realizations):
        self.trans_matrix = trans_matrix
        self.class_velocity = class_velocity
        self.init_class_count = init_class_count
        self.dt = dt
        self.n_neg_class = n_neg_class
        self.n_realizations = n_realizations

class correlatedSaver3d(object):
     def __init__(self, trans_matrix, init_class_count, dt, mapping, n_realizations,
                  n_binning_realz, n_slow_classes):
        self.trans_matrix = trans_matrix
        self.mapping = mapping
        self.init_class_count = init_class_count
        self.dt = dt
        self.n_binning_realz = n_binning_realz
        self.n_realizations = n_realizations
        self.n_slow_classes = n_slow_classes

def purturb_network(network, multip):
    network2 = deepcopy(network)
    l = network2.l
    theta = network2.theta
    dx = l*np.cos(theta)
    dy = l*np.sin(theta)
    x_array = network2.pores.x
    y_array = network2.pores.y
    left_boundary = (x_array == 0)
    right_boundary = (x_array == np.amax(x_array))
    inner_cells = ~np.logical_or(left_boundary, right_boundary)
    n_inner_cells = len(np.where(inner_cells)[0])
    rand_array = np.random.uniform(-1,1,n_inner_cells)
    x_array[inner_cells] += multip*dx*rand_array
    rand_array = np.random.uniform(-1,1,n_inner_cells)
    y_array[inner_cells] += multip*dy*rand_array
    return network2

def tube_length(network):
    tp_adj = network.tp_adj
    x_array = network.pores.x
    y_array = network.pores.y
    x1 = x_array[tp_adj[:,0]]
    y1 = y_array[tp_adj[:,0]]
    x2 = x_array[tp_adj[:,1]]
    y2 = y_array[tp_adj[:,1]]
    tube_l_array = np.sqrt(np.power(x2-x1,2) + np.power(y2-y1,2))
    return tube_l_array

def save_results_pickle(save_path, x_array, y_array, t_array, last_idx_array=None):
    if last_idx_array == None:
        save_shape = x_array.shape
        last_idx_array = save_shape[1]*np.ones(save_shape[0], dtype=np.int)
    data_holder = dispersionSaver(x_array, t_array, last_idx_array, y_array)
    with open(save_path, 'wb') as output:
        pickle.dump(data_holder, output, pickle.HIGHEST_PROTOCOL)

def save_case_info(save_folder, n_runs, n_particles, n_combine, n_files):
    total_n_particles = n_runs*n_particles
    info_file = os.path.join(save_folder, 'case_info.npz')
    np.savez(info_file, total_n_particles=total_n_particles, n_runs=n_runs, n_combine=n_combine, n_files=n_files)

max_stars_repo_name: amirdel/dispersion-random-network

max_issues_repo_name: amirdel/dispersion-random-network

max_forks_repo_name: amirdel/dispersion-random-network


text:
import tensorflow as tf
import sys, json, base64
import numpy as np
import cv2
import timeit
import scipy.ndimage
import subprocess as sp
import os
import threading
import queue

# This runs in a thread and performs CPU bound postprocessing.
def postprocess(q, answer_key, batch_size):

    nn_shape = (160, 576)
    crop_size = (296, 800)
    org_size = (600, 800)
    crop_offset = 226
    frame = 1

    while True:
        # Read next batch
        seg_map_batch = q.get()
        outarry = seg_map_batch[0]

        outarry = outarry.reshape(batch_size[0], nn_shape[0], nn_shape[1])

        for i in range(batch_size[0]):
            seg_map = outarry[i, ...].astype(np.uint8)

            upscaled = cv2.resize(seg_map, dsize=(crop_size[1], crop_size[0]), interpolation=cv2.INTER_NEAREST)
            # un-crop
            padded = np.zeros(org_size, dtype=np.uint8)  # bg type
            padded[crop_offset:upscaled.shape[0] + crop_offset, :upscaled.shape[1]] = upscaled
            # ready to create binary images
            binary_car_result, binary_road_result = get_binary_image(padded)
            answer_key[frame] = [encode(binary_car_result), encode(binary_road_result)]
            # Increment frame
            frame += 1

        q.task_done()

def get_decoded_vidoe(file, raw_video):
    # NN expects images cropped and resized.
    command = [ "ffmpeg",
        '-i', file,
        '-f', 'image2pipe',
        '-pix_fmt', 'rgb24',
        '-vcodec', 'rawvideo',
        '-filter:v',
        'crop=800:296:0:226,scale=576x160',
        '-']
    FNULL = open(os.devnull, 'w')
    pipe = sp.Popen(command, stdout=sp.PIPE, stderr=FNULL, bufsize=-1)
    raw_video[0] = pipe.stdout.read()

def get_binary_image(seg_map):

    road_class = 1
    car_class = 2
    gt = seg_map
    cargt = gt == car_class
    roadgt = gt == road_class
    '''
    Binary dilation improves Car’s Recall at the cost of reducing
    car’s Precision. This is ok since the scorning formula
    requires higher recall for car and higher precision for road.
    num_iterations is hyper-parameter.
    '''
    cargt = scipy.ndimage.binary_dilation(cargt, iterations=3)

    return cargt.astype(np.uint8), roadgt.astype(np.uint8)

# Define encoder function
def encode(array):
    _, buffer = cv2.imencode('.png', array)
    return base64.b64encode(buffer).decode("utf-8")

def main():

    nn_shape = (160, 576)
    batch_size = [50]
    answer_key = {}
    raw_video = [np.zeros(0)]
    q = queue.Queue()
    file = sys.argv[-1]
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

    myname = __file__
    if file == myname:
        print ("Error loading video")
        exit()
    '''
    Start a thread that starts FFmepg subprocess and reads raw video from pipe. This can
    happen in parallel to TensorFlow initialization and loading of model, saving seconds.
    '''
    download_thread = threading.Thread(target=get_decoded_vidoe, args=[file, raw_video])
    download_thread.start()
    '''
    Start the post processing thread now, so that it is ready when we need it later.
    '''
    postprocess_threads = threading.Thread(target=postprocess, args=(q, answer_key, batch_size))
    postprocess_threads.daemon = True
    postprocess_threads.start()

    with tf.gfile.GFile('./graph_optimized.pb', 'rb') as f:
        graph_def_optimized = tf.GraphDef()
        graph_def_optimized.ParseFromString(f.read())

    G = tf.Graph()

    with tf.Session(graph=G) as sess:
        logits, = tf.import_graph_def(graph_def_optimized, return_elements=['ArgMax:0'])
        image_input = G.get_tensor_by_name('import/image_input:0')
        keep_prob = G.get_tensor_by_name('import/keep_prob:0')

        sess.run(tf.global_variables_initializer())

        start_time = timeit.default_timer()
        download_thread.join()
        elapsed = timeit.default_timer() - start_time
        print("Waited for ffmpeg: ", elapsed, file=sys.stderr)

        video = np.fromstring(raw_video[0], dtype='uint8')
        n_frames = video.size // (nn_shape[0] * nn_shape[1] * 3)
        video = video.reshape((n_frames, nn_shape[0], nn_shape[1], 3))

        start_time = timeit.default_timer()

        if not video.shape[0] % batch_size[0] == 0:
            batch_size[0] = 1
            print("WARNING! Batching disabled.", file=sys.stderr)

        batch_size = batch_size[0]
        loops_req = video.shape[0] // batch_size
        print("Loops needed: ", loops_req, ", Batch size:", batch_size, file=sys.stderr)

        for c in range(loops_req):
            start = c * batch_size
            end = start + batch_size
            clip = video[start:end, ...]
            seg_map_batch = sess.run([logits],
                               {keep_prob: 1.0, image_input: clip})
            q.put(seg_map_batch)
        # Wait for any pending post processing to complete
        q.join()

        elapsed = timeit.default_timer() - start_time
        fps = video.shape[0] / elapsed
        start_time = timeit.default_timer()
        # Print output in proper json format
        print (json.dumps(answer_key))
        elapsed = timeit.default_timer() - start_time
        print("json time: ", elapsed, file=sys.stderr)
        print("Video fps: ", fps, file=sys.stderr)

if __name__ == '__main__':
    main()

max_stars_repo_name: asadziach/CarND-Semantic-Segmentation

max_issues_repo_name: asadziach/CarND-Semantic-Segmentation

max_forks_repo_name: asadziach/CarND-Semantic-Segmentation

reason: import numpy,import scipy


text:
# -*- coding: utf-8 -*-
"""
@author:XuMing(xuming624@qq.com)
@description:
"""
import os
from typing import List, Union
from loguru import logger
import numpy as np
from text2vec.utils.get_file import get_file

class NGram:
    def __init__(self, model_name_or_path=None, cache_folder=os.path.expanduser('~/.pycorrector/datasets/')):
        if model_name_or_path and os.path.exists(model_name_or_path):
            logger.info('Load kenlm language model:{}'.format(model_name_or_path))
            language_model_path = model_name_or_path
        else:
            # 语言模型 2.95GB
            get_file(
                'zh_giga.no_cna_cmn.prune01244.klm',
                'https://deepspeech.bj.bcebos.com/zh_lm/zh_giga.no_cna_cmn.prune01244.klm',
                extract=True,
                cache_subdir=cache_folder,
                verbose=1)
            language_model_path = os.path.join(cache_folder, 'zh_giga.no_cna_cmn.prune01244.klm')
        try:
            import kenlm
        except ImportError:
            raise ImportError('Kenlm not installed, use "pip install kenlm".')
        self.lm = kenlm.Model(language_model_path)
        logger.debug('Loaded language model: %s.' % language_model_path)

    def ngram_score(self, sentence: str):
        """
        取n元文法得分
        :param sentence: str, 输入的句子
        :return:
        """
        return self.lm.score(' '.join(sentence), bos=False, eos=False)

    def perplexity(self, sentence: str):
        """
        取语言模型困惑度得分，越小句子越通顺
        :param sentence: str, 输入的句子
        :return:
        """
        return self.lm.perplexity(' '.join(sentence))

    def encode(self, sentences: Union[List[str], str]):
        """
        将句子转换成ngram特征向量
        """
        if self.lm is None:
            raise ValueError('No model for embed sentence')

        input_is_string = False
        if isinstance(sentences, str) or not hasattr(sentences, '__len__'):
            sentences = [sentences]
            input_is_string = True

        all_embeddings = []
        for sentence in sentences:
            ngram_avg_scores = []
            for n in [2, 3]:
                scores = []
                for i in range(len(sentence) - n + 1):
                    word = sentence[i:i + n]
                    score = self.ngram_score(word)
                    scores.append(score)
                if scores:
                    # 移动窗口补全得分
                    for _ in range(n - 1):
                        scores.insert(0, scores[0])
                        scores.append(scores[-1])
                    avg_scores = [sum(scores[i:i + n]) / len(scores[i:i + n]) for i in range(len(sentence))]
                else:
                    avg_scores = np.zeros(len(sentence), dtype=float)
                ngram_avg_scores.append(avg_scores)
            # 取拼接后的n-gram平均得分
            sent_scores = np.average(np.array(ngram_avg_scores), axis=0)
            all_embeddings.append(sent_scores)
        all_embeddings = np.asarray(all_embeddings, dtype=object)
        if input_is_string:
            all_embeddings = all_embeddings[0]
        return all_embeddings


text:
from sklearn.ensemble import AdaBoostClassifier
import pandas as pd
from sklearn import metrics
import numpy as np
from imblearn.combine import SMOTEENN
import pickle
#import data
df_train = pd.read_csv('/Users/jaadeoye/Desktop/screen_ml/screen_full3.csv')
features = ['V1', 'V3', 'V4', 'V6', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14',
       'V15', 'V16', 'V17', 'V18', 'V20', 'V21', 'V22', 'V23', 'V25', 'V27',
       'V28', 'V29', 'V30', 'V31', 'V33', 'V34', 'V35']
x = df_train[features]
y = df_train.Status
#train model
sm = SMOTEENN(random_state=0)
x_res, y_res = sm.fit_resample(x,y)
logreg = AdaBoostClassifier(random_state=0, n_estimators = 100)
logreg.fit(x_res,y_res)
pickle.dump(logreg, open('ada', 'wb'))

max_stars_repo_name: jaadeoye/oral-cancer-risk-predictor

max_issues_repo_name: jaadeoye/oral-cancer-risk-predictor

max_forks_repo_name: jaadeoye/oral-cancer-risk-predictor


text:
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score
import pandas as pd
import numpy as np

# Algorithm selection with nested cross-validation

df = pd.read_csv('wdbc.data', header=None)

X = df.loc[:, 2:].values
y = df.loc[:, 1].values
le = LabelEncoder()
y = le.fit_transform(y)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, stratify=y, random_state=1)

pipe_svc = make_pipeline(StandardScaler(), SVC(random_state=1))

param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]

param_grid = [{'svc__C': param_range,
               'svc__kernel': ['linear']},
              {'svc__C': param_range,
               'svc__gamma': param_range,
               'svc__kernel': ['rbf']}]
# SVM

gs = GridSearchCV(estimator=pipe_svc,
                  param_grid=param_grid,
                  scoring='accuracy',
                  cv=2)

scores = cross_val_score(gs, X_train, y_train, scoring='accuracy', cv=5)

print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

# Decision Tree

gs = GridSearchCV(estimator=DecisionTreeClassifier(random_state=0),
                  param_grid=[{'max_depth': [1, 2, 3, 4, 5, 6, 7, None]}],
                  scoring='accuracy',
                  cv=2)

scores = cross_val_score(gs, X_train, y_train,
                         scoring='accuracy', cv=5)
print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

max_stars_repo_path: Chapter 06/Fine-tuning machine learning models via grid search/program2.py


text:
#!/usr/bin/env python
# MIT License
#
# Copyright (c) 2020-2022, Alex M. Maldonado
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""Tests for `mbgdml` package."""

import os
from math import isclose
import pytest
import numpy as np

import mbgdml.data as data
import mbgdml.parse as parse
import mbgdml.utils as utils
import mbgdml.qc as qc

# Must be run from mbGDML root directory.

def test_qc_ORCA():
    z_test = np.array([8, 1, 1])
    R_test = np.array(
        [[-0.48381516,  1.17384211, -1.4413092 ],
         [-0.90248552,  0.33071306, -1.24479905],
         [-1.21198585,  1.83409853, -1.4187445 ]]
    )
    coord_string = utils.string_xyz_arrays(z_test, R_test, precision=9)
    test_string = ( "O   -0.483815160    1.173842110   -1.441309200\n"
                    "H   -0.902485520    0.330713060   -1.244799050\n"
                    "H   -1.211985850    1.834098530   -1.418744500\n")
    assert coord_string == test_string

    engrad = qc.ORCA(
        '4h2o.abc0.2.step2',
        '4h2o.abc0.2.step2-orca.engrad-mp2.def2tzvp.tightscf.frozencore',
        '4h2o.abc0.2.step2-orca.engrad-mp2.def2tzvp.tightscf.frozencore'
    )
    slurm_file_name, slurm_file = engrad.submit(
        'smd',
        1,
        6,
        0,
        12,
        qc.pitt_crc_orca_420_submit,
        write=False,
    )

    assert slurm_file_name == 'submit-orca.420.slurm'
    assert slurm_file == (
        "#!/bin/bash\n"
        "#SBATCH --job-name=4h2o.abc0.2.step2\n"
        "#SBATCH --output=4h2o.abc0.2.step2-orca.engrad-mp2.def2tzvp.tightscf.frozencore.out\n"
        "#SBATCH --nodes=1\n"
        "#SBATCH --ntasks-per-node=6\n"
        "#SBATCH --time=0-12:00:00\n"
        "#SBATCH --cluster=smd\n"
        "\n"
        "cd $SBATCH_O_WORKDIR\n"
        "module purge\n"
        "module load openmpi/3.1.4\n"
        "module load orca/4.2.0\n"
        "\n"
        "cp $SLURM_SUBMIT_DIR/*.inp $SLURM_SCRATCH\n"
        "\n"
        "export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib:$LD_LIBRARY_PATH\n"
        "# Suppresses OpenFabrics (openib) absence nonfatal error.\n"
        "export OMPI_MCA_btl_base_warn_component_unused=0\n"
        "# Makes all error messages print.\n"
        "export OMPI_MCA_orte_base_help_aggregate=0"
        "\n"
        "\n"
        "cd $SLURM_SCRATCH"
        "\n"
        "$(which orca) *.inp"
        "\n"
        "\n"
    )

    step_R_string = utils.string_xyz_arrays(z_test, R_test, precision=9)
    assert step_R_string == (
        "O   -0.483815160    1.173842110   -1.441309200\n"
        "H   -0.902485520    0.330713060   -1.244799050\n"
        "H   -1.211985850    1.834098530   -1.418744500\n"
    )
    _, calc_string = engrad.input(
        'EnGrad',
        step_R_string,
        'MP2',
        'def2-TZVP',
        0,
        1,
        6,
        options='TightSCF FrozenCore',
        control_blocks='%scf\n    ConvForced true\nend\n%maxcore 8000\n',
        write=False
    )
    assert calc_string == (
        "# 4h2o.abc0.2.step2\n"
        "! MP2 def2-TZVP EnGrad TightSCF FrozenCore\n"
        "\n"
        "%pal\n"
        "    nprocs 6\n"
        "end\n"
        "\n"
        "%scf\n"
        "    ConvForced true\n"
        "end\n"
        "%maxcore 8000\n"
        "\n"
        "\n"
        "*xyz 0 1\n"
        "O   -0.483815160    1.173842110   -1.441309200\n"
        "H   -0.902485520    0.330713060   -1.244799050\n"
        "H   -1.211985850    1.834098530   -1.418744500\n"
        "*\n"
    )

def test_qc_slurm_engrad_calculation():
    coord_path = './tests/data/md/4h2o.abc0-orca.md-mp2.def2tzvp.300k-1.traj'
    z_all, _, R_list = parse.parse_stringfile(coord_path)
    try:
        assert len(set(tuple(i) for i in z_all)) == 1
    except AssertionError:
        raise ValueError(f'coord_path contains atoms in different order.')
    z = np.array(utils.atoms_by_number(z_all[0]))
    R = np.array(R_list)

    submit_file, input_file = qc.slurm_engrad_calculation(
        'ORCA',
        z,
        R,
        '4h2o.abc0.0,1,2,3',
        '4h2o.abc0.0,1,2,3-orca.engrad-mp2.def2tzvp.tightscf.frozencore',
        '4h2o.abc0.0,1,2,3-orca.engrad-mp2.def2tzvp.tightscf.frozencore',
        theory='MP2',
        basis_set='def2-TZVP',
        charge=0,
        multiplicity=1,
        cluster='smp',
        nodes=1,
        cores=6,
        days=0,
        hours=12,
        calc_dir='.',
        options='TightSCF FrozenCore',
        control_blocks='%scf\n    ConvForced true\nend\n%maxcore 8000\n',
        submit_script=qc.pitt_crc_orca_420_submit,
        write=False,
        submit=False
    )

max_stars_repo_name: KeithLabPitt/mbGDML

max_issues_repo_name: KeithLabPitt/mbGDML

max_forks_repo_name: KeithLabPitt/mbGDML


text:
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import logging
import os
import numpy as np
from fairseq.data import(
    Dictionary,
    IdDataset,
    NestedDictionaryDataset,
    OffsetTokensDataset,
    StripTokenDataset,
    NumSamplesDataset,
    NumelDataset,
    data_utils,
    LeftPadDataset,
    BaseWrapperDataset
)
from fairseq.data.shorten_dataset import maybe_shorten_dataset
from fairseq.tasks import FairseqTask, register_task
from fairseq.dataclass import FairseqDataclass, ChoiceEnum
from dataclasses import dataclass, field
from typing import Optional, List
from omegaconf import II
from fairseq.data.indexed_dataset import (get_available_dataset_impl,
                                          make_dataset,
                                          infer_dataset_impl)
from fairseq.data.molecule.indexed_dataset import MolMMapIndexedDataset
from fairseq.data.molecule.indexed_dataset import make_dataset as make_graph_dataset
from torch_geometric.data import Data, Batch

logger = logging.getLogger(__name__)

@dataclass
class GraphSeqPredictionConfig(FairseqDataclass):
    data: Optional[str] = field(
        default=None,
        metadata={
            "help": "colon separated path to data directories list, will be iterated upon during epochs "
                    "in round-robin manner; however, valid and test data are always in the first directory "
                    "to avoid the need for repeating them in all directories"
        }
    )
    num_classes: int = field(default=2)
    regression_target: bool = field(default=False)
    no_shuffle: bool = field(default=False)
    shorten_method: ChoiceEnum(["none", "truncate", "random_crop"]) = field(
        default="truncate"
    )
    shorten_data_split_list: str = field(default='')
    max_positions: int = II("model.max_positions")
    dataset_impl: Optional[ChoiceEnum(get_available_dataset_impl())] = II(
        "dataset.dataset_impl"
    )
    seed: int = II("common.seed")
    order_noise: int = field(default=5)

@register_task("graphseq_sp", dataclass=GraphSeqPredictionConfig)
class GraphSeqPredicition(FairseqTask):

    cfg: GraphSeqPredictionConfig

    def __init__(self, cfg: GraphSeqPredictionConfig,
                 data_dictionary,
                 label_dictionary):
        super().__init__(cfg)
        self.dictionary = data_dictionary
        self.dictionary.add_symbol("[MASK]")
        self._label_dictionary = label_dictionary
        self._max_positions = cfg.max_positions
        self.seed = cfg.seed
        self.order_noise = cfg.order_noise

    @classmethod
    def setup_task(cls, cfg: GraphSeqPredictionConfig, **kwargs):
        assert cfg.num_classes > 0
        data_dict = cls.load_dictionary(os.path.join(
            cfg.data, 'input0', "dict.txt"
        ))
        logger.info("[input] Dictionary {}: {} types.".format(
            os.path.join(cfg.data, "input0",),
            len(data_dict)
        ))
        if not cfg.regression_target:
            label_dict = cls.load_dictionary(os.path.join(
                cfg.data, "label", "dict.txt"
            ))
            logger.info("[label] Dictionary {}: {} types.".format(
                os.path.join(cfg.data, "label"),
                len(label_dict)
            ))
        else:
            label_dict = data_dict
        return cls(cfg, data_dict, label_dict)

    def load_dataset(
        self,
        split: str,
        combine: bool = False,
        **kwargs
    ):

        def get_path(key, split):
            return os.path.join(self.cfg.data, key, split)

        prefix = get_path('input0', split)
        if not MolMMapIndexedDataset.exists(prefix):
            raise FileNotFoundError("Graph data {} not found.".format(prefix))
        if self.cfg.dataset_impl is None:
            dataset_impl = infer_dataset_impl(prefix)
        else:
            dataset_impl = self.cfg.dataset_impl

        seq_dataset = make_dataset(prefix, impl=dataset_impl)
        assert seq_dataset is not None
        graph_dataset = make_graph_dataset(prefix, impl=dataset_impl)
        assert graph_dataset is not None

        with data_utils.numpy_seed(self.cfg.seed):
            shuffle = np.random.permutation(len(seq_dataset))

        seq_dataset = maybe_shorten_dataset(
            seq_dataset,
            split,
            self.cfg.shorten_data_split_list,
            self.cfg.shorten_method,
            self._max_positions,
            self.cfg.seed
        )
        dataset = {
            "id": IdDataset(),
            "net_input": {
                "src_tokens": LeftPadDataset(
                    seq_dataset,
                    pad_idx=self.dictionary.pad()
                ),
                "src_lengths": NumelDataset(seq_dataset, reduce=False)
            },
            "graph": GraphCollateDataset(graph_dataset),
            "nsentences": NumSamplesDataset(),
            "ntokens": NumelDataset(seq_dataset, reduce=True)
        }
        if not self.cfg.regression_target:
            prefix = get_path('label', split)
            label_dataset = make_dataset(prefix, impl=dataset_impl)
            assert label_dataset is not None
            dataset.update(
                target=OffsetTokensDataset(
                    StripTokenDataset(
                        label_dataset,
                        id_to_strip=self.label_dictionary.eos()
                    ),
                    offset=-self.label_dictionary.nspecial
                )
            )
        else:
            raise NotImplementedError
        nested_dataset = NestedDictionaryDataset(
            dataset,
            sizes=[seq_dataset.sizes]
        )

        if self.cfg.no_shuffle:
            dataset = nested_dataset
        else:
            dataset = NoiseOrderedDataset(
                nested_dataset,
                sort_order=[shuffle],
                seed=self.seed,
                order_noise=self.order_noise
            )

        logger.info("Loaded {} with #samples: {}.".format(split, len(dataset)))
        self.datasets[split] = dataset
        return self.datasets[split]

    def build_model(self, cfg):
        model = super().build_model(cfg)
        model.register_classification_head(
            getattr(cfg, "classification_head_name", "sentence_classification_head"),
            num_classes=self.cfg.num_classes
        )
        return model

    @property
    def source_dictionary(self):
        return self.dictionary

    @property
    def src_dict(self):
        return self.dictionary

    @property
    def target_dictionary(self):
        return self.dictionary

    @property
    def label_dictionary(self):
        return self._label_dictionary

class NoiseOrderedDataset(BaseWrapperDataset):

    def __init__(self, dataset, sort_order,
                 seed, order_noise):
        super().__init__(dataset)
        if not isinstance(sort_order, (list, tuple)):
            sort_order = [sort_order]
        self.sort_order = sort_order
        self.seed = seed
        self.order_noise = order_noise

        assert all(len(so) == len(dataset) for so in sort_order)
        self._epoch = 0

    def ordered_indices(self):
        sort_order = []
        with data_utils.numpy_seed(self.seed + self._epoch):
            for so in self.sort_order:
                sort_order.append(
                    so + np.random.randint(low=-self.order_noise,
                                           high=self.order_noise,
                                           size=so.shape)
                )
            return np.lexsort(self.sort_order)

    def set_epoch(self, epoch):
        self._epoch = epoch
        super().set_epoch(epoch)

class GraphCollateDataset(BaseWrapperDataset):

    def __init__(self, dataset):
        super().__init__(dataset)

    def collater(self, samples):
        data_list = []
        for s in samples:
            data_list.append(Data(
                x=s['node_attr'],
                edge_index=s['edge_index'].T,
                edge_attr=s['edge_attr'],
            ))
        return Batch.from_data_list(data_list)

max_stars_repo_name: microsoft/DVMP

max_issues_repo_name: QizhiPei/DVMP

max_forks_repo_name: QizhiPei/DVMP