text:
import tensorflow as tf
import numpy as np
import argparse
import IPython.display as display
import matplotlib.pyplot as plt
from tensorflow import keras
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
from sklearn.utils import shuffle
import pandas as pd
tf.compat.v1.enable_eager_execution()
import cProfile
from sklearn.model_selection import train_test_split
import copy as cp
from sklearn.model_selection import KFold
from datetime import date
import time
class standard_erm_model:
    def __init__(self, model,  num_epochs, batch_size, learning_rate):

        self.model        = model
        self.num_epochs   = num_epochs
        self.batch_size   = batch_size
        self.learning_rate=learning_rate

    def fit(self, data_tuple_list):
        learning_rate = self.learning_rate
        num_epochs = self.num_epochs
        n_e  = len(data_tuple_list)
        x_in = data_tuple_list[0][0]
        for i in range(1,n_e):
            x_c = data_tuple_list[i][0]
            x_in = np.concatenate((x_in, x_c), axis=0)
        y_in = data_tuple_list[0][1]
        for i in range(1,n_e):
            y_c = data_tuple_list[i][1]
            y_in = np.concatenate((y_in, y_c), axis=0)
        e_in = data_tuple_list[0][2]
        for i in range(1,n_e):
            e_c = data_tuple_list[i][2]
            e_in = np.concatenate((e_in, e_c), axis=0)

    ### fit the model
        model = self.model
        batch_size = self.batch_size

        model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

        model.fit(x_in, y_in,  epochs=num_epochs, batch_size=batch_size)

        self.x_in      = x_in
        self.y_in      = y_in
        self.e_in      = e_in
        self.model     = model

    def evaluate(self, data_tuple_test):
        ##### evaluations jmtd
        x_test = data_tuple_test[0]
        y_test = data_tuple_test[1]
        x_in   = self.x_in
        y_in   = self.y_in

        model = self.model
        train_accuracy= tf.keras.metrics.SparseCategoricalAccuracy()
        test_accuracy= tf.keras.metrics.SparseCategoricalAccuracy()

        ytr_ = model.predict(x_in)
        train_acc =  np.float(train_accuracy(y_in, ytr_))

        yts_ = model.predict(x_test)

        test_acc  =  np.float(test_accuracy(y_test, yts_))

        self.train_acc = train_acc
        self.test_acc  = test_acc

class irm_model:
    def __init__(self, model, learning_rate, batch_size, steps_max, steps_threshold, gamma_new):
        self.model= model                                 # initialized model passed
        self.learning_rate = learning_rate                # learning rate for Adam optimizer
        self.batch_size    = batch_size                   # batch size per gradient update
        self.steps_max     = steps_max                    # maximum number of gradient steps
        self.steps_threshold = steps_threshold            # threshold on the number of steps after which we use penalty gamma_new
        self.gamma_new      = gamma_new                   # penalty value; note penalty is set to 1 initially and gamma_new only kicks in after steps exceeed steps_threshold

    def fit(self,data_tuple_list):

        n_e  = len(data_tuple_list)                       # number of environments
        # combine the data from the different environments, x_in: combined data (features) from different environments
        x_in = data_tuple_list[0][0]
        for i in range(1,n_e):
            x_c = data_tuple_list[i][0]
            x_in = np.concatenate((x_in, x_c), axis=0)
        y_in = data_tuple_list[0][1]
        for i in range(1,n_e):
            y_c = data_tuple_list[i][1]
            y_in = np.concatenate((y_in, y_c), axis=0)
        e_in = data_tuple_list[0][2]
        for i in range(1,n_e):
            e_c = data_tuple_list[i][2]
            e_in = np.concatenate((e_in, e_c), axis=0)

        z_in = data_tuple_list[0][2]
        for i in range(1,n_e):
            z_c = data_tuple_list[i][2]
            z_in = np.concatenate((z_in, z_c), axis=0)

        self.frac = 0.8
        frac = self.frac
        n_tr = np.int(np.shape(x_in)[0]*frac)
        x_in_tr  = x_in[:n_tr]
        x_in_val = x_in[n_tr:]
        y_in_tr  = y_in[:n_tr]
        y_in_val = y_in[n_tr:]
        e_in_tr  = e_in[:n_tr]
        e_in_val = e_in[n_tr:]

        x_in = x_in_tr
        y_in = y_in_tr
        e_in = e_in_tr

        self.x_in = x_in
        self.y_in = y_in
        self.e_in = e_in

        self.x_in_val = x_in_val
        self.y_in_val = y_in_val
        self.e_in_val = e_in_val

        # cross entropy (we do not use the cross entropy from keras because there are issues when computing gradient of the gradient)
        def cross_entropy_manual(y,y_pred):
            y_p   = tf.math.log(tf.nn.softmax(y_pred))
            n_p   = np.float(tf.shape(y_p)[0])
            ind_0 = tf.where(y==0)[:,0]
            ind_1 = tf.where(y==1)[:,0]
            y_p0  = tf.gather(y_p, ind_0)[:,0]
            y_p1  = tf.gather(y_p, ind_1)[:,1]
            ent_0 = tf.reduce_sum(y_p0)
            ent_1 = tf.reduce_sum(y_p1)
            total = -(ent_0 + ent_1)/n_p
            return total

        # cross entropy loss for environment e
        def loss_n(model,x,e,y,w,k):
            index = np.where(e==k)
            y1_ = model(x[index[0]])*w
            y1  = y[index[0]]

            return cross_entropy_manual(y1,y1_)

        # gradient of cross entropy loss w.r.t w for environment e
        def grad_norm_n(model,x,e,y,w,k):
            with tf.GradientTape() as g:
                g.watch(w)
                loss_value = loss_n(model,x,e,y,w,k)
            return g.gradient(loss_value, w)**2

        # total cross entropy loss across all environments
        def loss_0(model, x,e,y,w):
            y_ =model(x)
            loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
            return loss_object(y_true=y, y_pred=y_)

        # sum of cross entropy loss and penalty
        def loss_total(model,x,e,y,w,gamma, n_e):
            loss0 = loss_0(model,x,e,y,w)
            loss_penalty = 0.0
            for k in range(n_e):
                loss_penalty += gamma*grad_norm_n(model,x,e,y,w,k)

            return (loss0 + loss_penalty)*(1/gamma)

        # gradient of sum of cross entropy loss and penalty w.r.t model parameters
        def grad_total_n(model,x,e,y,w,gamma, n_e):
            with tf.GradientTape() as tape:
                loss_value = loss_total(model,x,e,y,w,gamma,n_e)
            return loss_value, tape.gradient(loss_value, model.trainable_variables)

        model = self.model
        learning_rate = self.learning_rate
        optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)

        ## train
        train_loss_results = []
        train_accuracy_results = []
        flag = 'false'
        batch_size = self.batch_size
        num_examples= x_in.shape[0]
        gamma     = 1.0
        w         = tf.constant(1.0)
        steps     = 0
        steps_max  = self.steps_max
        steps_threshold = self.steps_threshold
        gamma_new       = self.gamma_new

        while(steps<=steps_max):
            (xt,yt,et)   = shuffle(x_in,y_in,e_in)
            epoch_loss_avg = tf.keras.metrics.Mean()
            epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()
            count = 0
            if(steps>=steps_threshold):
                gamma = gamma_new
            for offset in range(0,num_examples, batch_size):
                if(steps<=steps_max):
                    end = offset + batch_size

                    batch_x, batch_y, batch_e = xt[offset:end,:], yt[offset:end,:], et[offset:end,:]

                    loss_values, grads = grad_total_n(model,batch_x,batch_e,batch_y,w,gamma,n_e)

                    optimizer.apply_gradients(zip(grads, model.trainable_variables))

                    epoch_loss_avg(loss_values)
                    epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()
                    acc_train = np.float(epoch_accuracy(y_in, model(x_in)))
                    train_loss_results.append(epoch_loss_avg.result())
                    train_accuracy_results.append(epoch_accuracy.result())

                count = count +1
                steps = steps + 1

        self.model = model

    def evaluate(self, data_tuple_test):
        x_test = data_tuple_test[0]
        y_test = data_tuple_test[1]

        x_in   = self.x_in
        y_in   = self.y_in

        x_in_val   = self.x_in_val
        y_in_val   = self.y_in_val

        train_accuracy= tf.keras.metrics.SparseCategoricalAccuracy()
        val_accuracy  = tf.keras.metrics.SparseCategoricalAccuracy()
        test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()

        model = self.model
        ytr_  = model.predict(x_in)
        train_acc =  np.float(train_accuracy(y_in, ytr_))
        self.train_acc = train_acc

        if(self.frac<1.0):
            yv_ = model.predict(x_in_val)
            val_acc =  np.float(train_accuracy(y_in_val, yv_))
            self.val_acc   = train_acc

        yts_ = model.predict(x_test)
        test_acc =  np.float(test_accuracy(y_test, yts_))
        self.test_acc  = test_acc


text:
import sys
import os

import numpy as np
import xgboost as xgb
from cpp_preprocessing import cpp_features, write, weight_nodes, get_neighbors
from os.path import basename
from time import time

start = time()

# only run on first graph of them all
GRAPH_PATH = sys.argv[1]
q = float(sys.argv[2])

bst = xgb.Booster({'nthread': 16})
bst.load_model("2021-07-09_15-05-56.model")

with open(GRAPH_PATH) as graph_file:
    line = graph_file.readline()
    while line[0] == "%":  # skip comments at start
        line = graph_file.readline()
    header = list(map(int, line.split()))
    number_of_nodes = header[0]  # number of nodes is first number in first line
    if len(header) == 3:
        has_node_weights = bool(int(line.split()[2]) & 2)
    else:
        has_node_weights = False

# where the reduced graph is stored
reduction_path = ""
exists = np.ones(number_of_nodes)
total_added = np.array([])

num_stages = 5
for stage in range(1, num_stages + 1):
    #print(f"{stage=}")
    # calculate features and store them in the "/graph_files/" folder, reduce the graph and write the reduced graph
    # to reduction_path
    reduction_path = cpp_features(GRAPH_PATH, "/home/jholten/graph_files/", exists=exists)

    # load features into a dmatrix
    removed = np.where(exists == 0)
    feature_path = "/home/jholten/graph_files/" + os.path.basename(GRAPH_PATH) + (
        "." + reduction_path.split(".")[-1] if len(removed) != 0 else "") + ".feat"
    feature_matrix = xgb.DMatrix(data=np.loadtxt(feature_path))

    # predict based on the features
    label_pred = bst.predict(feature_matrix)

    # remove if prediction lower than threshold
    exists[np.where(label_pred <= 1 - q)[0]] = 0

    # add to MIS if prediction is higher than threshold
    likely = np.where(label_pred >= q)[0]
    # decending sort the likely candidates based on their predicted value
    for node in np.flip(likely[np.argsort(label_pred[likely])]):
        # if it still exists, keep it, and remove its neighbors
        if exists[node]:
            total_added = np.append(total_added, node)
            exists[get_neighbors(GRAPH_PATH, [node])] = 0

KERNEL_FOLDER = "/home/jholten/kernels/ml_reduce_kernels/"

end = time()

print("elapsed time:", end - start)

ml_reduction_path = KERNEL_FOLDER + basename(GRAPH_PATH) + ".ml_kernel" + str(int(q * 100))
print("writing kernel to", ml_reduction_path)
number_removed = write(GRAPH_PATH, ml_reduction_path, exists)
print(f"removed in total {number_removed} nodes ({number_removed/number_of_nodes:.2%})")

# save the added nodes to a file and print offset
offset_weight = weight_nodes(GRAPH_PATH, total_added)
print(f"with offset {offset_weight}")

np.savetxt(KERNEL_FOLDER + basename(GRAPH_PATH) + ".ml_offset" + str(int(q*100)), total_added, header=f"% {offset_weight}")

ext: py

max_stars_repo_name: josephholten/KaMIS

max_issues_repo_name: josephholten/KaMIS

max_forks_repo_name: josephholten/KaMIS

reason: import numpy


text:
import torch
from torch import nn
from torch.nn import functional as F
from torchdiffeq import odeint
import numpy as np

from torch.distributions import Bernoulli, Normal

class ConvEncoder(nn.Module):
    """Maps an (x_i, y_i) pair to a representation r_i where y_i is a
    28*28 length vector representation of an image.

    Parameters
    ----------
    y_dim : int
        Dimension of y values.
    r_dim : int
        Dimension of output representation r.
    """

    def __init__(self, y_dim, r_dim):
        super(ConvEncoder, self).__init__()
        n_filt = 16
        self.y_dim = y_dim
        self.r_dim = r_dim
        self.img_width = int(np.sqrt(y_dim))

        self.conv1 = nn.Conv2d(1, n_filt, kernel_size=5, stride=2, padding=(2, 2))
        self.conv2 = nn.Conv2d(n_filt, n_filt * 2, kernel_size=5, stride=2, padding=(2, 2))
        self.conv3 = nn.Conv2d(n_filt * 2, n_filt * 4, kernel_size=5, stride=2, padding=(2, 2))
        self.conv4 = nn.Conv2d(n_filt * 4, n_filt * 8, kernel_size=5, stride=2, padding=(2, 2))
        self.fc1 = nn.Linear(128 * (2**2) + 128, r_dim)
        self.bn1 = nn.BatchNorm2d(n_filt)
        self.bn2 = nn.BatchNorm2d(n_filt * 2)
        self.bn3 = nn.BatchNorm2d(n_filt * 4)
        self.bn4 = nn.BatchNorm2d(n_filt * 8)

    def forward(self, x, y):
        """
        x : torch.Tensor
            Shape (batch_size * num_points, x_dim)
        y : torch.Tensor
            Shape (batch_size * num_points, y_dim)
        """
        y = y.view(y.shape[0], 1, self.img_width, self.img_width)
        y = F.relu(self.bn1(self.conv1(y)))
        y = F.relu(self.bn2(self.conv2(y)))
        y = F.relu(self.bn3(self.conv3(y)))
        y = F.relu(self.bn4(self.conv4(y)))
        y = y.view(y.size()[0], -1)  # Flatten

        input = torch.cat((x.repeat(1, 128), y), dim=1)
        output = self.fc1(input)
        return output

class Y0ConvEncoder(nn.Module):
    """Maps y0 to L_r

    Parameters
    ----------
    x_dim : int
        Dimension of x values.

    y_dim : int
        Dimension of y values.

    h_dim : int
        Dimension of hidden layer.

    r_dim : int
        Dimension of output representation r.
    """

    def __init__(self, y_dim, r_dim):
        super(Y0ConvEncoder, self).__init__()
        n_filt = 8
        self.y_dim = y_dim
        self.r_dim = r_dim
        self.img_width = int(np.sqrt(y_dim))

        self.conv1 = nn.Conv2d(1, n_filt, kernel_size=5, stride=2, padding=(2, 2))
        self.conv2 = nn.Conv2d(n_filt, n_filt * 2, kernel_size=5, stride=2, padding=(2, 2))
        self.conv3 = nn.Conv2d(n_filt * 2, n_filt * 4, kernel_size=5, stride=2, padding=(2, 2))
        self.conv4 = nn.Conv2d(n_filt * 4, n_filt * 8, kernel_size=5, stride=2, padding=(2, 2))

        self.bn1 = nn.BatchNorm2d(n_filt)
        self.bn2 = nn.BatchNorm2d(n_filt * 2)
        self.bn3 = nn.BatchNorm2d(n_filt * 4)
        self.bn4 = nn.BatchNorm2d(n_filt * 8)

        self.fc1 = nn.Linear(n_filt * 8 * (2**2), r_dim)

    def forward(self, y0):
        """
        y0 : torch.Tensor
            Shape (batch_size, y_dim)
        """
        y = y0.view(y0.shape[0], 1, self.img_width, self.img_width)
        y = F.relu(self.bn1(self.conv1(y)))
        y = F.relu(self.bn2(self.conv2(y)))
        y = F.relu(self.bn3(self.conv3(y)))
        y = F.relu(self.bn4(self.conv4(y)))
        y = y.view(y.size()[0], -1)  # Flatten
        output = self.fc1(y)
        return output

class SingleContextNDPEncoder(nn.Module):
    """Use Context and the same conv network to infer both L(0) and D"""
    def __init__(self, context_encoder):
        super(SingleContextNDPEncoder, self).__init__()
        self.context_encoder = context_encoder

    def forward(self, x, y, _):
        """
        x : torch.Tensor
            Shape (batch_size * num_points, x_dim)
        y : torch.Tensor
            Shape (batch_size * num_points, y_dim)
        """
        output = self.context_encoder(x, y)
        return output, output

class Y0ContextNDPEncoder(nn.Module):
    """Use y0 to infer L(0) and Context to infer D"""
    def __init__(self, y0_encoder, context_encoder):
        super(Y0ContextNDPEncoder, self).__init__()
        self.y0_encoder = y0_encoder
        self.context_encoder = context_encoder

    def forward(self, x, y, y0):
        """
        x : torch.Tensor
            Shape (batch_size * num_points, x_dim)
        y : torch.Tensor
            Shape (batch_size * num_points, y_dim)
        y0 : torch.Tensor
            Shape (batch_size, y_dim)
        """
        L_output = self.y0_encoder(y0)
        D_output = self.context_encoder(x, y)
        return L_output, D_output

class Encoder(nn.Module):
    """Maps an (x_i, y_i) pair to a representation r_i.

    Parameters
    ----------
    x_dim : int
        Dimension of x values.

    y_dim : int
        Dimension of y values.

    h_dim : int
        Dimension of hidden layer.

    r_dim : int
        Dimension of output representation r.
    """

    def __init__(self, x_dim, y_dim, h_dim, r_dim):
        super(Encoder, self).__init__()

        self.x_dim = x_dim
        self.y_dim = y_dim
        self.h_dim = h_dim
        self.r_dim = r_dim

        layers = [nn.Linear(x_dim + y_dim, h_dim),
                  nn.ReLU(inplace=True),
                  nn.Linear(h_dim, h_dim),
                  nn.ReLU(inplace=True),
                  nn.Linear(h_dim, r_dim)]

        self.input_to_hidden = nn.Sequential(*layers)

    def forward(self, x, y):
        """
        x : torch.Tensor
            Shape (batch_size, x_dim)

        y : torch.Tensor
            Shape (batch_size, y_dim)
        """
        input_pairs = torch.cat((x, y), dim=1)
        return self.input_to_hidden(input_pairs)

class MuSigmaEncoder(nn.Module):
    """
    Maps a representation r to mu and sigma which will define the normal
    distribution from which we sample the latent variable z.

    Parameters
    ----------
    r_dim : int
        Dimension of output representation r.

    z_dim : int
        Dimension of latent variable z.
    """

    def __init__(self, r_dim, z_dim):
        super(MuSigmaEncoder, self).__init__()

        self.r_dim = r_dim
        self.z_dim = z_dim

        self.r_to_hidden = nn.Linear(r_dim, r_dim)
        self.hidden_to_mu = nn.Linear(r_dim, z_dim)
        self.hidden_to_sigma = nn.Linear(r_dim, z_dim)

    def forward(self, r):
        """
        r : torch.Tensor
            Shape (batch_size, r_dim)
        """
        hidden = torch.relu(self.r_to_hidden(r))
        mu = self.hidden_to_mu(hidden)
        # Define sigma following convention in "Empirical Evaluation of Neural
        # Process Objectives" and "Attentive Neural Processes"
        sigma = 0.1 + 0.9 * torch.sigmoid(self.hidden_to_sigma(hidden))
        return mu, sigma

class NPMlpDecoder(nn.Module):
    """
    Maps target input x_target and samples z (encoding information about the
    context points) to predictions y_target.

    Parameters
    ----------
    x_dim : int
        Dimension of x values.

    z_dim : int
        Dimension of latent variable z.

    h_dim : int
        Dimension of hidden layer.

    y_dim : int
        Dimension of y values.
    """

    def __init__(self, x_dim, z_dim, h_dim, y_dim):
        super(NPMlpDecoder, self).__init__()

        self.x_dim = x_dim
        self.z_dim = z_dim
        self.h_dim = h_dim
        self.y_dim = y_dim

        layers = [nn.Linear(x_dim + z_dim, h_dim),
                  nn.ReLU(inplace=True),
                  nn.Linear(h_dim, h_dim),
                  nn.ReLU(inplace=True),
                  nn.Linear(h_dim, h_dim),
                  nn.ReLU(inplace=True)]

        self.xz_to_hidden = nn.Sequential(*layers)
        self.hidden_to_mu = nn.Linear(h_dim, y_dim)
        self.hidden_to_sigma = nn.Linear(h_dim, y_dim)

    def forward(self, x, z):
        """
        x : torch.Tensor
            Shape (batch_size, num_points, x_dim)

        z : torch.Tensor
            Shape (batch_size, z_dim)

        Returns
        -------
        Returns mu and sigma for output distribution. Both have shape
        (batch_size, num_points, y_dim).
        """
        batch_size, num_points, _ = x.size()
        # Repeat z, so it can be concatenated with every x. This changes shape
        # from (batch_size, z_dim) to (batch_size, num_points, z_dim)
        z = z.unsqueeze(1).repeat(1, num_points, 1)
        # Flatten x and z to fit with linear layer
        x_flat = x.view(batch_size * num_points, self.x_dim)
        z_flat = z.view(batch_size * num_points, self.z_dim)
        # Input is concatenation of z with every row of x
        input_pairs = torch.cat((x_flat, z_flat), dim=1)
        hidden = self.xz_to_hidden(input_pairs)
        mu = self.hidden_to_mu(hidden)
        pre_sigma = self.hidden_to_sigma(hidden)
        # Reshape output into expected shape
        mu = mu.view(batch_size, num_points, self.y_dim)
        pre_sigma = pre_sigma.view(batch_size, num_points, self.y_dim)
        # Define sigma following convention in "Empirical Evaluation of Neural
        # Process Objectives" and "Attentive Neural Processes"
        sigma = 0.1 + 0.9 * F.softplus(pre_sigma)
        return Normal(mu, sigma)

class ConvDecoderNet(nn.Module):
    """Network doing the convolutional decoding in the NP and NDP decoders."""

    def __init__(self, y_dim):
        super(ConvDecoderNet, self).__init__()

        img_width = int(np.sqrt(y_dim))
        self.feat = 3 if img_width == 28 else 4
        padding1 = (0, 0) if img_width == 28 else (2, 2)
        padding2 = (0, 0) if img_width == 28 else (1, 1)
        stride = 1 if img_width == 28 else 2

        self.decode_conv1 = nn.ConvTranspose2d(8, 128, kernel_size=3, stride=stride,
            padding=padding1)
        self.decode_conv2 = nn.ConvTranspose2d(128, 64, kernel_size=5, stride=2, padding=padding2)
        self.decode_conv3 = nn.ConvTranspose2d(64, 32, kernel_size=5, stride=2, padding=(1, 1),
            output_padding=(1, 1))
        self.bn1 = nn.BatchNorm2d(128)
        self.bn2 = nn.BatchNorm2d(64)
        self.bn3 = nn.BatchNorm2d(32)
        self.hidden_to_mu = nn.ConvTranspose2d(32, 1, kernel_size=5, stride=1, padding=(2, 2))

    def forward(self, x):
        x = x.view(x.shape[0], 8, self.feat, self.feat)
        x = F.relu(self.bn1(self.decode_conv1(x)))
        x = F.relu(self.bn2(self.decode_conv2(x)))
        x = F.relu(self.bn3(self.decode_conv3(x)))
        x = self.hidden_to_mu(x)
        return x

class NPConvDecoder(nn.Module):
    """
    Maps target input x_target and samples z (encoding information about the
    context points) to predictions y_target.

    Parameters
    ----------
    x_dim : int
        Dimension of x values.

    z_dim : int
        Dimension of latent variable z.

    h_dim : int
        Dimension of hidden layer.

    y_dim : int
        Dimension of y values.
    """

    def __init__(self, x_dim, z_dim, h_dim, y_dim, conv_decoder_net):
        super(NPConvDecoder, self).__init__()

        self.x_dim = x_dim
        self.z_dim = z_dim
        self.h_dim = h_dim
        self.y_dim = y_dim

        self.feat = 3 if int(np.sqrt(y_dim)) == 28 else 4
        self.decode_fc = nn.Linear(x_dim + z_dim, self.feat * self.feat * 8)
        self.conv_decoder = conv_decoder_net

    def forward(self, x, z):
        """
        x : torch.Tensor
            Shape (batch_size, num_points, x_dim)

        z : torch.Tensor
            Shape (batch_size, z_dim)

        Returns
        -------
        Returns mu and sigma for output distribution. Both have shape
        (batch_size, num_points, y_dim).
        """
        batch_size, num_points, _ = x.size()
        # Repeat z, so it can be concatenated with every x. This changes shape
        # from (batch_size, z_dim) to (batch_size, num_points, z_dim)
        z = z.unsqueeze(1).repeat(1, num_points, 1)
        # Flatten x and z to fit with linear layer
        x_flat = x.view(batch_size * num_points, self.x_dim)
        z_flat = z.view(batch_size * num_points, self.z_dim)
        # Input is concatenation of z with every row of x
        inp = torch.cat((x_flat, z_flat), dim=1)
        inp = self.decode_fc(inp)

        mu = self.conv_decoder(inp)
        mu = mu.view(batch_size, num_points, self.y_dim)
        return Bernoulli(logits=mu)

class AbstractODEDecoder(nn.Module):
    """
    An Abstract Decoder using a Neural ODE. Child classes must implement decode_latent.

    Parameters
    ----------
    x_dim : int
        Dimension of x values. Currently only works for dimension of 1.
    z_dim : int
        Dimension of latent variable z. Contains [L0, z_].

    h_dim : int
        Dimension of hidden layer in odefunc.
    y_dim : int
        Dimension of y values.

    L_dim : int
        Dimension of latent state L.
    """

    def __init__(self, x_dim, z_dim, h_dim, y_dim, L_dim, initial_t, L_out_dim=None, exclude_time=False):
        super(AbstractODEDecoder, self).__init__()
        # The input is always time.
        assert x_dim == 1

        self.exclude_time = exclude_time
        self.x_dim = x_dim
        self.z_dim = z_dim
        self.h_dim = h_dim
        self.y_dim = y_dim
        self.L_dim = L_dim
        if L_out_dim is None:
            L_out_dim = L_dim

        inp_dim = z_dim if exclude_time else z_dim + x_dim
        ode_layers = [nn.Linear(inp_dim, h_dim),
                      nn.Tanh(),
                      nn.Linear(h_dim, h_dim),
                      nn.Tanh(),
                      nn.Linear(h_dim, L_out_dim)]
        # z = [L0, z_] so dim([L, z_, x]) = dim(z)+1
        self.latent_odefunc = nn.Sequential(*ode_layers)

        self.decode_layers = [nn.Linear(x_dim + z_dim, h_dim),
                              nn.ReLU(inplace=True),
                              nn.Linear(h_dim, h_dim),
                              nn.ReLU(inplace=True),
                              nn.Linear(h_dim, h_dim),
                              nn.ReLU(inplace=True)]
        self.xlz_to_hidden = nn.Sequential(*self.decode_layers)

        self.initial_t = initial_t
        self.nfe = 0

    def integrate_ode(self, t, v):  # v = (L(x), z_)
        self.nfe += 1
        z_ = v[:, self.L_dim:]
        batch_size = v.size()[0]
        vt = v
        if not self.exclude_time:
            time = t.view(1, 1).repeat(batch_size, 1)
            vt = torch.cat((vt, time), dim=1)

        dL = self.latent_odefunc(vt)
        dz_ = torch.zeros_like(z_)
        return torch.cat((dL, dz_), dim=1)

    def decode_latent(self, x, z, latent) -> torch.distributions.Distribution:
        raise NotImplementedError('The decoding of the latent ODE state is not implemented')

    def forward(self, x, z):
        """
        x : torch.Tensor
            Shape (batch_size, num_points, 1)
        z : torch.Tensor
            Shape (batch_size, z_dim)
        Returns
        -------
        Returns mu and sigma for output distribution. Both have shape
        (batch_size, num_points, y_dim).
        """
        self.nfe = 0
        batch_size, num_points, _ = x.size()

        # Append the initial time to the set of supplied times.
        x0 = self.initial_t.repeat(batch_size, 1, 1)
        x_sort = torch.cat((x0, x), dim=1)

        # ind specifies where each element in x ended up in times.
        times, ind = torch.unique(x_sort, sorted=True, return_inverse=True)
        # Remove the initial position index since we don't care about it.
        ind = ind[:, 1:, :]

        # Integrate forward from the batch of initial positions z.
        v = odeint(self.integrate_ode, z, times, method='dopri5')

        # Make shape (batch_size, unique_times, z_dim).
        permuted_v = v.permute(1, 0, 2)
        latent = permuted_v[:, :, :self.L_dim]

        # Extract the relevant (latent, time) pairs for each batch.
        tiled_ind = ind.repeat(1, 1, self.L_dim)
        latent = torch.gather(latent, dim=1, index=tiled_ind)

        return self.decode_latent(x, z, latent)

class MlpNormalODEDecoder(AbstractODEDecoder):
    """
    Maps target times x_target (which we call x for consistency with NPs)
    and samples z (encoding information about the context points)
    to predictions y_target. The decoder is an ODEsolve, using torchdiffeq.
    This version contains no control.
    Models inheriting from MlpNormalODEDecoder *must* either set self.xlz_to_hidden
    in constructor or override decode_latent(). Optionally, integrate_ode
    and forward can also be overridden.

    Parameters
    ----------
    x_dim : int
        Dimension of x values. Currently only works for dimension of 1.
    z_dim : int
        Dimension of latent variable z. Contains [L0, z_].

    h_dim : int
        Dimension of hidden layer in odefunc.
    y_dim : int
        Dimension of y values.

    L_dim : int
        Dimension of latent state L.
    """

    def __init__(self, x_dim, z_dim, h_dim, y_dim, L_dim, initial_t, L_out_dim=None):
        super(MlpNormalODEDecoder, self).__init__(
            x_dim, z_dim, h_dim, y_dim, L_dim, initial_t, L_out_dim)

        self.x_dim = x_dim  # must be 1
        self.z_dim = z_dim
        self.h_dim = h_dim
        self.y_dim = y_dim
        self.L_dim = L_dim

        self.hidden_to_mu = nn.Linear(h_dim + L_dim, y_dim)
        self.hidden_to_sigma = nn.Linear(h_dim + L_dim, y_dim)
        self.initial_t = initial_t
        self.nfe = 0

    def decode_latent(self, x, z, latent) -> torch.distributions.Distribution:
        batch_size, num_points, _ = x.size()
        # compute sigma using mlp (t, L(t), z_)
        z = z[:, self.L_dim:]
        z = z.unsqueeze(1).repeat(1, num_points, 1)
        # Flatten x and z to fit with linear layer
        x_flat = x.view(batch_size * num_points, self.x_dim)
        latent_flat = latent.view(batch_size * num_points, -1)
        z_flat = z.view(batch_size * num_points, self.z_dim - self.L_dim)
        # Input is concatenation of z with every row of x
        input_triplets = torch.cat((x_flat, latent_flat, z_flat), dim=1)
        hidden = self.xlz_to_hidden(input_triplets)
        hidden = torch.cat((latent_flat, hidden), dim=1)
        mu = self.hidden_to_mu(hidden)
        pre_sigma = self.hidden_to_sigma(hidden)

        # Reshape output into expected shape
        mu = mu.view(batch_size, num_points, self.y_dim)
        pre_sigma = pre_sigma.view(batch_size, num_points, self.y_dim)

        # Define sigma following convention in "Empirical Evaluation of Neural
        # Process Objectives" and "Attentive Neural Processes"
        sigma = 0.1 + 0.9 * F.softplus(pre_sigma)

        return Normal(mu, sigma)

# Includes batching, now includes a latent state to go through MLP to get mu/sigma
class MlpSonodeDecoder(MlpNormalODEDecoder):
    def __init__(self, x_dim, z_dim, h_dim, y_dim, L_dim, initial_t):
        super(MlpSonodeDecoder, self).__init__(x_dim, z_dim, h_dim, y_dim, L_dim, initial_t,
            L_out_dim=int(L_dim / 2))

    # x needs to be written as t here to use torchdiffeq
    # v, and therefore L and z_ must be vectors as a torch tensor, so they can concatentate
    def integrate_ode(self, t, v):  # v = (L(x), z_)
        self.nfe += 1
        Lv = v[:, int(self.L_dim / 2):self.L_dim]
        z_ = v[:, self.L_dim:]
        batch_size = v.size()[0]
        time = t.view(1, 1).repeat(batch_size, 1)
        vt = torch.cat((v, time), dim=1)
        dLx = Lv
        dLv = self.latent_odefunc(vt)
        dz_ = torch.zeros_like(z_)
        return torch.cat((dLx, dLv, dz_), dim=1)

class VanillaODEDecoder(MlpNormalODEDecoder):
    def __init__(self, x_dim, z_dim, h_dim, y_dim, L_dim, initial_t):
        super(VanillaODEDecoder, self).__init__(
            x_dim, z_dim, h_dim, y_dim, L_dim, initial_t)

        self.latent_to_mu = nn.Linear(L_dim, y_dim)
        self.latent_to_sigma = nn.Linear(L_dim, y_dim)

    def decode_latent(self, x, z, latent):
        batch_size, num_points, _ = x.size()
        latent_flat = latent.view(batch_size * num_points, -1)
        mu = self.latent_to_mu(latent_flat)
        pre_sigma = self.latent_to_sigma(latent_flat)
        # Reshape output into expected shape
        mu = mu.view(batch_size, num_points, self.y_dim)
        pre_sigma = pre_sigma.view(batch_size, num_points, self.y_dim)
        sigma = 0.1 + 0.9 * F.softplus(pre_sigma)
        return Normal(mu, sigma)

class VanillaSONODEDecoder(MlpNormalODEDecoder):
    def __init__(self, x_dim, z_dim, h_dim, y_dim, L_dim, initial_t):
        super(VanillaSONODEDecoder, self).__init__(
            x_dim, z_dim, h_dim, y_dim, L_dim,
            initial_t, L_out_dim=int(L_dim / 2))

        self.latent_to_mu = nn.Linear(L_dim, y_dim)
        self.latent_to_sigma = nn.Linear(L_dim, y_dim)

    # x needs to be written as t here to use torchdiffeq
    # v, and therefore L and z_ must be vectors as a torch tensor, so they can concatentate
    def integrate_ode(self, t, v):  # v = (L(x), z_)
        self.nfe += 1
        Lv = v[:, int(self.L_dim / 2):self.L_dim]
        z_ = v[:, self.L_dim:]
        batch_size = v.size()[0]
        time = t.view(1, 1).repeat(batch_size, 1)
        vt = torch.cat((v, time), dim=1)
        dLx = Lv
        dLv = self.latent_odefunc(vt)
        dz_ = torch.zeros_like(z_)
        return torch.cat((dLx, dLv, dz_), dim=1)

    def decode_latent(self, x, z, latent):
        batch_size, num_points, _ = x.size()
        latent_flat = latent.view(batch_size * num_points, -1)
        mu = self.latent_to_mu(latent_flat)
        pre_sigma = self.latent_to_sigma(latent_flat)
        # Reshape output into expected shape
        mu = mu.view(batch_size, num_points, self.y_dim)
        pre_sigma = pre_sigma.view(batch_size, num_points, self.y_dim)
        sigma = 0.1 + 0.9 * F.softplus(pre_sigma)
        return Normal(mu, sigma)

class ConvODEDecoder(AbstractODEDecoder):
    """
    MlpNormalODEDecoder with a transposed convolutional decoder.
    """

    def __init__(self, x_dim, z_dim, h_dim, y_dim, L_dim, initial_t, conv_decoder_net,
                 latent_only=False, exclude_time=False):
        super(ConvODEDecoder, self).__init__(x_dim, z_dim, h_dim, y_dim, L_dim, initial_t,
            exclude_time=exclude_time)
        self.latent_only = latent_only
        self.feat = 3 if int(np.sqrt(y_dim)) == 28 else 4

        if self.latent_only:
            self.decode_fc = nn.Linear(L_dim, self.feat * self.feat * 8)
        else:
            self.decode_fc = nn.Linear(x_dim + z_dim, self.feat * self.feat * 8)

        self.conv_decoder = conv_decoder_net

    def decode_latent(self, x, z, latent):
        batch_size, num_points, _ = x.size()
        # compute sigma using mlp (t, L(t), z_)
        z = z[:, self.L_dim:]
        z = z.unsqueeze(1).repeat(1, num_points, 1)
        # Flatten x and z to fit with linear layer
        x_flat = x.view(batch_size * num_points, self.x_dim)
        latent_flat = latent.view(batch_size * num_points, -1)
        z_flat = z.view(batch_size * num_points, self.z_dim - self.L_dim)
        # Input is concatenation of z with every row of x

        input = latent_flat if self.latent_only else torch.cat((x_flat, latent_flat, z_flat), dim=1)
        input = self.decode_fc(input)

        mu = self.conv_decoder(input)
        mu = mu.view(batch_size, num_points, self.y_dim)
        return Bernoulli(logits=mu)


text:
import math
##########################Box Operations#############################
def resize_box(box,size):
    cx,cy,w,h=ltrb_to_ccwh(box)
    nw,nh=size
    l=cx-nw//2
    r=cx+nw//2
    t=cy-nh//2
    b=cy+nh//2
    return box.__class__([l,t,r,b])
def rescale_box(box,scale):
    '''rescale with  image'''
    if isinstance(scale,(tuple,list)):
        scaleX,scaleY=scale
    else:
        scaleX=scaleY=scale
    l,t,r,b=box
    l*=scaleX
    r*=scaleX
    t*=scaleY
    b*=scaleY
    return box.__class__([l,t,r,b])
def get_rotate_info(angle,cx,cy,h,w):
    import math
    import numpy as np
    angle = math.radians(angle)
    cos = math.cos(angle)
    sin = math.sin(angle)
    M = np.array([[cos, sin], [-sin, cos]])
    cos = abs(cos)
    sin = abs(sin)
    nw = w * cos + h * sin
    nh = w * sin + h * cos
    return M,nw,nh
def rotate_points(points,angle,cx,cy,h,w):
    import numpy as np
    M,nw,nh=get_rotate_info(angle,cx,cy,h,w)
    points=np.array(points)
    original_shape=points.shape
    points=points.reshape((-1,2))
    points=M.dot((points-np.array([cx,cy])).T).T+np.array([nw//2,nh//2])
    points=points.reshape(original_shape)
    return points

def rotate_boxes(boxes,angle,cx,cy,h,w):
    '''原创'''
    boxes=[list(box) for box in boxes]
    corners = [ltrb_to_four_corners(box) for box in boxes]
    corners=rotate_points(corners,angle,cx,cy,h,w)
    boxes=[four_coners_to_ltrb(quad_to_four_corners(organize_quad_points(four_corners_to_quad(list(box))))) for box in corners]
    return boxes
def rotate_box(box, angle, cx, cy, h, w):
    return rotate_boxes([box],angle,cx,cy,h,w)
def shift_box(box,offset,limits=None):
    l,t,r,b=box
    ofx,ofy=offset
    l+=ofx
    r+=ofx
    t+=ofy
    b+=ofy
    return limit_box(box.__class__([l,t,r,b]),limits=limits)
def translate_box(box,offset,limits=None):
    return shift_box(box,offset,limits)
def flip_box_horizontal(box,imsize):
    imw,imh=imsize
    l,t,r,b=box
    l=imw-r
    r=imw-l
    return box.__class__([l,t,r,b])
def flip_box_vertical(box,imsize):
    imw,imh=imsize
    l,t,r,b=box
    t=imh-b
    b=imh-t
    return box.__class__([l,t,r,b])
def pad_box(box,pad_size=5,pad_ratio=None,limits=None):
    from math import inf
    limits=limits or (-inf,-inf,inf,inf)
    l,t,r,b=box
    bh=b-t
    bw=r-l
    if pad_ratio is not None:
        if isinstance(pad_ratio,(tuple,list)):
            if len(pad_ratio)==2:
                pad_l=pad_r=int(min(bh,bw)*pad_ratio[0])
                pad_t=pad_b=int(min(bh,bw)*pad_ratio[1])
            else:
                assert len(pad_ratio)==4
                pad_l= int(min(bh, bw) * pad_ratio[0])
                pad_t= int(min(bh, bw) * pad_ratio[1])
                pad_r= int(min(bh, bw) * pad_ratio[2])
                pad_b= int(min(bh, bw) * pad_ratio[3])
        else:
            pad_l=pad_t=pad_r=pad_b=int(min(bh,bw)*pad_ratio)
    else:
        if isinstance(pad_size,(tuple,list)):
            if len(pad_size)==2:
                pad_l=pad_r=pad_size[0]
                pad_t=pad_b=pad_size[1]
            else:
                assert len(pad_size)==4
                pad_l,pad_t,pad_r,pad_b=pad_size
        else:
            pad_l=pad_t=pad_r=pad_b=pad_size
    l-=pad_l
    t-=pad_t
    r+=pad_r
    b+=pad_b
    return limit_box(box.__class__([l,t,r,b]),limits=limits)
def limit_box(box,limits=None):
    if limits is None:return box
    if len(limits)==2:
        ml,mt=0,0
        mr,mb=limits
    else:
        assert len(limits)==4
        ml,mt,mr,mb=limits
    l,t,r,b=box
    l=max(ml,l)
    t=max(mt,t)
    r=min(mr,r)
    b=min(mb,b)
    if l>=r:
        return None
    if t>=b:return None
    return box.__class__([l,t,r,b])
############################Sort Operations#############################
def min_enclosing_box(boxes):
    assert len(boxes)
    ml,mt,mr,mb=boxes[0]
    for box in boxes[1:]:
        l,t,r,b=box
        ml=min(ml,l)
        mt=min(mt,t)
        mr=max(mr,r)
        mb=max(mb,b)
    return [ml,mt,mr,mb]
def box_in_area(box,size):
    l,t,r,b=box
    mr,mb=size
    if l<0 or t<0 or r>mr or b>mb:
        return False
    else:
        return True

def organize_quad_points(box):
    '''
    turn into clock-wise points: quad
    p0:lt,p1:rt,p2:rb,p4:lb
    '''
    p0,p3, p1,p2 = sorted(box, key=lambda p: p[0])
    p03=[p0,p3]
    p12=[p1,p2]
    p0, p3 = sorted(p03, key=lambda p: p[1])
    p1, p2 = sorted(p12, key=lambda p: p[1])
    return box.__class__([p0,p1,p2,p3])
def sort_boxes(boxes):
    from functools import cmp_to_key
    boxes=[ltrb_to_ccwh(box) for box in boxes]
    boxes=[[box] for box in boxes]
    def block_cmp(b1, b2):
        list1 = [b1[0][0], b1[0][1], b1[0][2], b1[0][3]]
        list2 = [b2[0][0], b2[0][1], b2[0][2], b2[0][3]]
        if len(b1[0]) == 4:
            list1[0] += list1[2] / 2
            list1[1] += list1[3] / 2
            list2[0] += list2[2] / 2
            list2[1] += list2[3] / 2

        flag = 1
        if list1[0] > list2[0]:
            list1, list2 = list2, list1
            flag = -1

        if list2[1] + list1[3] / 2 < list1[1]:
            return flag
        return -flag

    boxes.sort(key=cmp_to_key(block_cmp), reverse=False)
    boxes=[box[0] for box in boxes]
    boxes=[ccwh_to_ltrb(box) for box in boxes]
    return boxes
############################ConvertingTools#############################
def ltrb_to_four_corners(box):
    '''four corners: 8-d vector clock-wise'''
    l,t,r,b=box
    box2=[l,t,r,t,r,b,l,b]
    return box.__class__(box2)
def four_coners_to_ltrb(box,sort=False):
    x1,y1,x2,y2,x4,y4,x3,y3=box
    l=min(x1,x3)
    r=max(x2,x4)
    t=min(y1,y2)
    b=max(y3,y4)
    if sort:
        newbox = [min(l, r), min(t, b), max(l, r), max(t, b)]
    else:
        newbox=[l,t,r,b]
    return box.__class__(newbox)
def ltrb_to_ccwh(box):
    l, t, r, b = box
    w = r - l
    h = b - t
    cx = (l + r) // 2
    cy = (t + b) // 2
    return box.__class__([cx,cy,w,h])
def ccwh_to_ltrb(box):
    cx,cy,w,h=box
    l=cx-w//2
    t=cy-h//2
    r=cx+w//2
    b=cy+h//2
    return box.__class__([l,t,r,b])
def oowh_to_ltrb(box):
    '''o:origin'''
    x, y, w, h = box
    return box.__class__([int(i) for i in (x, y, x + w, y + h)])
def ltrb_to_oowh(box):
    l, t, r, b = box
    return box.__class__([l, t, r - l, b - t])
def xywh_to_ltrb(box):
    '''same with oowh'''
    x,y,w,h=box
    return box.__class__([int(i) for i in (x,y,x+w,y+h)])
def ltrb_to_xywh(box):
    l,t,r,b=box
    return box.__class__([l,t,r-l,b-t])
def quad_to_ltrb(box):
    '''quad: clock-wise four points'''
    (x0,y0),(x1,y1),(x2,y2),(x3,y3)=box
    return box.__class__([min(x0,x3),min(y0,y1),max(x1,x2),max(y2,y3)])
def ltrb_to_quad(box):
    x0,y0,x1,y1=box
    return box.__class__([(x0,y0),(x1,y0),(x1,y1),(x0,y1)])
def quad_to_four_corners(quad):
    (x0, y0), (x1, y1), (x2, y2), (x3, y3) = quad
    return quad.__class__([x0,y0,x1,y1,x2,y2,x3,y3])
def four_corners_to_quad(box):
    x0,y0,x1,y1,x2,y2,x3,y3 = box
    return box.__class__([(x0, y0), (x1, y1), (x2, y2), (x3, y3)])
###########################Calculations##############################
def calc_iou(box1,box2):
    l1,t1,r1,b1=box1
    l2,t2,r2,b2=box2
    w1,h1=r1-l1,b1-t1
    w2,h2=r2-l2,b2-t2
    width=min(r1,r2)-max(l1,l2)
    height=min(b1,b2)-max(t1,t2)
    width=max(width,0)
    height=max(height,0)
    area=width*height
    return area/(w1*h1+w2*h2-area)
def calc_iou2(box1,box2):
    width=min(box1[2],box2[2])-max(box1[0],box2[0])
    height=min(box1[3],box2[3])-max(box1[1],box2[1])
    width=max(width,0)
    height=max(height,0)
    area=width*height
    return area/((box1[2]-box1[0])*(box1[3]-box1[1])+(box2[2]-box2[0])*(box2[3]-box2[1])-area)
def calc_iou_batch_with_batch(boxes1,boxes2):
    box1=boxes1
    box2=boxes2
    import numpy as np
    box1=np.array(box1)
    box2=np.array(box2)
    width = np.min(np.vstack([box1[...,2], box2[...,2]]),axis=0) - np.max(np.vstack([box1[...,0], box2[...,0]]),axis=0)
    height = np.min(np.vstack([box1[...,3], box2[...,3]]),axis=0) - np.max(np.vstack([box1[...,1], box2[...,1]]),axis=0)
    width = np.clip(width,0,np.inf)
    height = np.clip(height,0,np.inf)
    area = width * height
    return area / ((box1[...,2] - box1[...,0]) * (box1[...,3] - box1[...,1]) + (box2[...,2] - box2[...,0]) * (box2[...,3] - box2[...,1]) - area)
def calc_iou_batch(boxes,box):
    '''calculate iou between batch bboxes with one bbox , 原创'''
    box1=boxes
    box2=box
    import numpy as np
    box1 = np.array(box1)
    box2 = np.array(box2)
    shape=box1.shape
    width = np.min(np.vstack([box1[..., 2], np.full(shape[:-1],box2[2])]), axis=0) - np.max(np.vstack([box1[..., 0], np.full(shape[:-1],box2[0])]),axis=0)
    height = np.min(np.vstack([box1[..., 3], np.full(shape[:-1],box2[3])]), axis=0) - np.max(np.vstack([box1[..., 1], np.full(shape[:-1],box2[1])]),axis=0)
    width = np.clip(width, 0, np.inf)
    height = np.clip(height, 0, np.inf)
    area = width * height
    return area / ((box1[...,2] - box1[...,0]) * (box1[...,3] - box1[...,1]) + (box2[2] - box2[0]) * (box2[3] - box2[1]) - area)
def calc_quad_angle(quad):
    '''cal angle of a quad'''
    p0,p1,p2,p3=quad
    def center(p1,p2):
        x1,y1=p1
        x2,y2=p2
        return (x1+x2)/2,(y1+y2)/2
    x1,y1=center(p0,p3)
    x2,y2=center(p1,p2)
    angle=math.atan2(y1-y2,x2-x1)*180/math.pi
    return angle
def calc_box_area(box):
    l,t,r,b=box
    return (r-l)*(b-t)
##############################TextBox################################
class TextBox(dict):
    def __init__(self):
        super().__init__()
    def parse(self):
        p0,p1,p2,p3=quad=self['quad']
        xmin,ymin,xmax,ymax=l,t,r,b=ltrb=quad_to_ltrb(quad)
        ox,oy,w,h=oowh=ltrb_to_oowh(ltrb)
        cx,cy,w,h=ccwh=ltrb_to_ccwh(ltrb)
        angle=calc_quad_angle(quad)
        text=None
        prob=None
        self.update(
            ltrb=ltrb,oowh=oowh,ccwh=ccwh,
            xmin=xmin,ymin=ymin,xmax=xmax,ymax=ymax,
            l=l,t=t,r=r,b=b,
            ox=ox,oy=oy,w=w,h=h,
            cx=cx,cy=cy,
            angle=angle,
            text=text,
            prob=prob
        )
        return self

max_stars_repo_name: Peiiii/wk

max_issues_repo_name: Peiiii/wk

max_forks_repo_name: Peiiii/wk


text:
import json
import rasterio

from os.path import join
from rasterio import features
from rasterio.transform import from_origin
import rasterio.mask
import geopandas as gpd
import pandas as pd
import numpy as np
import tempfile

from shapely.geometry import Point

import rasterio
import numpy as np
from pygeogrids.grids import BasicGrid

def get_grid(file_name):

    data_handle = rasterio.open(file_name)
    data_bbox = data_handle.bounds
    data_res = data_handle.res

    lon_1d = np.arange(data_bbox.left, data_bbox.right + np.abs(data_res[0] / 2), np.abs(data_res[0]), float)
    lat_1d = np.arange(data_bbox.bottom, data_bbox.top + np.abs(data_res[1] / 2), np.abs(data_res[1]), float)
    lon_2d, lat_2d = np.meshgrid(lon_1d, lat_1d)

    grid = BasicGrid(lon_2d.flatten(), lat_2d.flatten()).to_cell_grid(cellsize=5.)

    return grid, lon_2d, lat_2d, data_bbox

def create_points_shp(data, x, y, data_format="int64",
                      epsg="EPSG:4326", file_name_shp="points.shp"):

    points_rows = np.array([x, y, data])
    points_cols = np.transpose(points_rows)

    assert np.all(points_rows[0, :] == points_cols[:, 0])
    assert np.all(points_rows[1, :] == points_cols[:, 1])
    assert np.all(points_rows[2, :] == points_cols[:, 2])

    points_df = pd.DataFrame(points_cols, columns=["x", "y", "z"])
    points_df.head()
    points_df = points_df.astype(dtype={"x": "float64", "y": "float64", "z": data_format})

    points_df['geometry'] = points_df.apply(lambda row: Point(row.x, row.y, row.z), axis=1)
    points_df = points_df.drop(["x", "y", "z"], axis=1)
    # create the GeoDatFrame
    points_crs = {'init': epsg}
    points_gdf = gpd.GeoDataFrame(points_df, crs=points_crs, geometry=points_df.geometry)
    # save the GeoDataFrame
    points_gdf.to_file(driver='ESRI Shapefile', filename=file_name_shp)

# Method to create mask
def create_mask(rows, cols, geoms, transform, epsg="EPSG:4326", file_name_masked="masked.tiff"):

    mask = rasterio.features.rasterize(geoms,
                                       out_shape=[rows, cols],
                                       transform=transform,
                                       dtype=rasterio.int32
                                       )

    dst_mask = rasterio.open(file_name_masked, 'w', driver='GTiff',
                                 height=rows, width=cols,
                                 count=1, dtype=rasterio.int32,
                                 crs=epsg,
                                 transform=transform)
    dst_mask.write(mask, 1)
    dst_mask.close()

# Method to create template file
def create_template_file(rows, cols, transform, epsg="EPSG:4326",
                         file_name_template="template.tiff"):

    arr = np.zeros([rows, cols], dtype=rasterio.int32)
    arr[:, :] = 1
    dst_template = rasterio.open(file_name_template, 'w', driver='GTiff',
                                 height=rows, width=cols,
                                 count=1, dtype=rasterio.int32,
                                 crs=epsg,
                                 transform=transform)
    dst_template.write(arr, 1)
    dst_template.close()

# Method to get file shp
def get_file_shp(file_name_shp, file_name_mask="masked.tiff", folder_tmp=tempfile.mkdtemp(),
            cell_size=0.0005, epsg="EPSG:4326", bbox_ext=0):

    shp = gpd.read_file(file_name_shp)
    geoms = ((feature['geometry'], 1) for feature in shp.iterfeatures())

    if 'AREA' in shp:
        area = shp['AREA'][0]
    else:
        area = None

    if bbox_ext > 1:
        bbox_ext = 1

    bbox_minx_shp = shp.bounds['minx'][0] - bbox_ext
    bbox_maxx_shp = shp.bounds['maxx'][0] + bbox_ext
    bbox_miny_shp = shp.bounds['miny'][0] - bbox_ext
    bbox_maxy_shp = shp.bounds['maxy'][0] + bbox_ext

    # Create geographical information
    geox_1d = np.arange(bbox_minx_shp, bbox_maxx_shp + np.abs(cell_size / 2), np.abs(cell_size), float)
    geoy_1d = np.arange(bbox_miny_shp, bbox_maxy_shp + np.abs(cell_size / 2), np.abs(cell_size), float)

    bbox_minx = np.min(geox_1d)
    bbox_maxx = np.max(geox_1d)
    bbox_miny = np.min(geoy_1d)
    bbox_maxy = np.max(geoy_1d)

    geox_2d, geoy_2d = np.meshgrid(geox_1d, geoy_1d)
    rows = int(np.round((bbox_maxy - bbox_miny) / cell_size + 1))
    cols = int(np.round((bbox_maxx - bbox_minx) / cell_size + 1))

    corner_ll_x = bbox_minx - cell_size / 2.0
    corner_ll_y = bbox_maxy + cell_size / 2.0

    # Create template raster
    transform = from_origin(corner_ll_x, corner_ll_y, cell_size, cell_size)
    create_template_file(rows, cols, transform, epsg,
                         file_name_template=join(folder_tmp, "template.tiff"))

    # Load template raster
    dst_reference = rasterio.open(join(folder_tmp, "template.tiff"))
    meta_reference = dst_reference.meta.copy()
    meta_reference.update(compress='lzw')
    dst_reference.close()

    # Create mask
    create_mask(rows, cols, geoms, meta_reference['transform'], epsg,
                file_name_masked=file_name_mask)

    return rows, cols, epsg, transform, meta_reference

# Method to get file json
def get_file_json(file_name_json):
    with open(file_name_json, 'r') as file_handle:
        file_data = json.load(file_handle)
    return file_data

max_stars_repo_name: c-hydro-labs/hsaf

max_issues_repo_name: c-hydro-labs/hsaf

max_forks_repo_name: c-hydro-labs/hsaf


text:
import os.path as op
import networkx as nx
from nipype import Workflow, MapNode, Node, IdentityInterface
from nipype.interfaces.base import (BaseInterfaceInputSpec, TraitedSpec,
                                    File, OutputMultiPath, isdefined)

class InputWrapper(object):
    """Implements connections between DataGrabber and workflow inputs."""
    def __init__(self, workflow, subject_node, grabber_node, input_node):

        self.wf = workflow
        self.subj_node = subject_node
        self.grab_node = grabber_node
        self.in_node = input_node

    def connect_inputs(self):
        """Connect stereotyped inputs to the input IdentityInterface."""

        # Connect subject_id to input and grabber nodes
        self.wf.connect(self.subj_node, "subject_id",
                        self.grab_node, "subject_id")
        if hasattr(self.in_node.inputs, "subject_id"):
            self.wf.connect(self.subj_node, "subject_id",
                            self.in_node, "subject_id")

        # Connect the datagrabber outputs to the workflow inputs
        grabbed = self.grab_node.outputs.get()
        inputs = self.in_node.inputs.get()
        for field in grabbed:
            if field in inputs:
                self.wf.connect(self.grab_node, field,
                                self.in_node, field)

class OutputWrapper(object):
    """Implements connections between workflow outputs and DataSink."""
    def __init__(self, workflow, subject_node, sink_node, output_node):

        self.wf = workflow
        self.subj_node = subject_node
        self.sink_node = sink_node
        self.out_node = output_node

    def set_mapnode_substitutions(self, n_runs):
        """Find mapnode names and add datasink substitutions to sort by run."""

        # First determine top-level mapnode names
        mapnode_names = find_mapnodes(self.wf)

        # Then determine mapnode names for each nested workflow
        # Note that this currently only works for one level of nesting
        nested_workflows = find_nested_workflows(self.wf)
        for wf in nested_workflows:
            mapnode_names.extend(find_mapnodes(wf))

        # Build a list of substitution tuples
        substitutions = []
        for r in reversed(range(n_runs)):
            for name in mapnode_names:
                substitutions.append(("_%s%d" % (name, r), "run_%d" % (r + 1)))

        # Set the substitutions attribute on the DataSink node
        if isdefined(self.sink_node.inputs.substitutions):
            self.sink_node.inputs.substitutions.extend(substitutions)
        else:
            self.sink_node.inputs.substitutions = substitutions

    def add_regexp_substitutions(self, sub_list):
        """Safely set subsitutions implemented with regular expressions."""
        if isdefined(self.sink_node.inputs.regexp_substitutions):
            self.sink_node.inputs.regexp_substitutions.extend(sub_list)
        else:
            self.sink_node.inputs.regexp_substitutions = sub_list

    def set_subject_container(self):
        """Store results by subject at highest level."""
        # Connect the subject_id value as the container
        self.wf.connect(self.subj_node, "subject_id",
                        self.sink_node, "container")

        subj_subs = []
        for s in self.subj_node.iterables[1]:
            subj_subs.append(("/_subject_id_%s/" % s, "/"))

        # Strip the subject_id iterable from the path
        if isdefined(self.sink_node.inputs.substitutions):
            self.sink_node.inputs.substitutions.extend(subj_subs)
        else:
            self.sink_node.inputs.substitutions = subj_subs

    def sink_outputs(self, dir_name=None):
        """Connect the outputs of a workflow to a datasink."""
        outputs = self.out_node.outputs.get()
        prefix = "@" if dir_name is None else dir_name + ".@"
        for field in outputs:
            self.wf.connect(self.out_node, field,
                            self.sink_node, prefix + field)

def find_mapnodes(workflow):
    """Given a workflow, return a list of MapNode names."""
    mapnode_names = []
    wf_nodes = nx.nodes(workflow._graph)
    for node in wf_nodes:
        if isinstance(node, MapNode):
            mapnode_names.append(node.name)

    return mapnode_names

def find_nested_workflows(workflow):
    """Given a workflow, find nested workflow objects."""
    nested_workflows = []
    wf_nodes = nx.nodes(workflow._graph)
    for node in wf_nodes:
        if isinstance(node, Workflow):
            nested_workflows.append(node)

    return nested_workflows

def make_subject_source(subject_list):
    """Generate a source node with iterables over a subject_id list."""
    return Node(IdentityInterface(fields=["subject_id"]),
                iterables=("subject_id", subject_list),
                overwrite=True,
                name="subj_source")

class SingleInFile(TraitedSpec):

    in_file = File(exists=True)

class SingleOutFile(TraitedSpec):

    out_file = File(exists=True)

class ManyOutFiles(TraitedSpec):

    out_files = OutputMultiPath(File(exists=True))

def list_out_file(fname):
    """Return a _list_outputs function for a single out_file."""
    def _list_outputs(self):

        outputs = self._outputs().get()
        outputs["out_file"] = op.abspath(fname)
        return outputs

    return _list_outputs

max_stars_repo_name: sgagnon/lyman

max_issues_repo_name: sgagnon/lyman

max_forks_repo_name: sgagnon/lyman

reason: import networkx


text:
import io
import time
import numpy as np
import cv2
import picamera

video = cv2.VideoCapture('fire1.mp4')
_, img = video.read()
frames = list()
grays = list()
background = None
thresh = np.full((len(img), len(img[0])), 30, np.uint8)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))

while video.isOpened():
    _, img = video.read()
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    _, gray = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)

    frames.append(img)
    grays.append(gray)
    if len(frames) < 3:
        continue
    elif len(frames) == 3:
        background = grays[2]
    else:
        frames.remove(frames[0])
        grays.remove(grays[0])

    diff1 = cv2.absdiff(grays[2], grays[1])
    diff1 = cv2.subtract(diff1, thresh)
    _, diff1 = cv2.threshold(diff1, 0, 255, cv2.THRESH_BINARY)
    diff2 = cv2.absdiff(grays[2], grays[0])
    diff2 = cv2.subtract(diff2, thresh)
    _, diff2 = cv2.threshold(diff2, 0, 255, cv2.THRESH_BINARY)
    mask = cv2.bitwise_and(diff1, diff2)

    alpha = 0.96
    moving = cv2.countNonZero(mask)
    if moving < 1000:
        background = cv2.addWeighted(background, alpha, grays[2], 1-alpha, 0)
        result = img
    else:
        diff = cv2.absdiff(grays[2], background)
        mask = cv2.subtract(diff, thresh)
        thresh = cv2.addWeighted(thresh, alpha, diff, 1-alpha, 0)
        _, mask = cv2.threshold(mask, 0, 255, cv2.THRESH_BINARY)
        #cv2.erode(mask, kernel, mask, iterations = 1)
        cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel, mask)
        _, contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
        #result = cv2.bitwise_and(img, img, mask = mask)
        result = cv2.bitwise_and(img, img)
        print('Number of contours : {}'.format(len(contours)))
        for contour in contours:
            if cv2.arcLength(contour, True) < 30:
                continue
            epsilon = 0.01*cv2.arcLength(contour, True)
            approx = cv2.approxPolyDP(contour, epsilon, True)
            #hull = cv2.convexHull(contour)
            cv2.drawContours(result, [approx], 0, (0, 0, 255), 3)
        #background = cv2.addWeighted(background, 1-alpha, grays[2], alpha, 0)

    show_ori = cv2.resize(img, (640, 360))
    show_back = cv2.resize(background, (640, 360))
    show_res = cv2.resize(result, (640, 360))
    cv2.imshow('original', show_ori)
    cv2.imshow('background', show_back)
    cv2.imshow('result', show_res)

    if cv2.waitKey(2) >= 0:
        break


text:
import argparse
import pickle
from collections import defaultdict
from pprint import pprint
from itertools import combinations

import numpy as np
from scipy.stats import wilcoxon

if __name__ == "__main__":
  np.set_printoptions(precision=2)
  parser = argparse.ArgumentParser()
  parser.add_argument('inputs', help='List of pickle files with an ndarray representing a confusion matrix.')
  args = parser.parse_args()
  table = defaultdict(lambda: defaultdict(dict))
  with open(args.inputs) as g:
      for l in g:
        with open(l.rstrip(), 'rb') as f:
          _cm = pickle.load(f)
          name = l.split('/')[-1].split('.')[0].split('_')[0]
          protocol = l.split('/')[-3]
          corpus = l.split('/')[-2]
          correct = np.trace(_cm)
          total = np.sum(_cm)
          table[protocol][corpus][name] = correct / total
  pprint(table)

  print('Wilcoxon signed-rank test')
  for test_type, corpora in table.items():
    accuracies = defaultdict(list)
    for _, corpus in corpora.items():
      for architecture, accuracy in corpus.items():
        accuracies[architecture].append(accuracy)
    print(test_type)
    for (namex, x),(namey, y) in combinations(accuracies.items(), 2):
      print(f"{namex} - {namey}")
      _, p_value = wilcoxon(x,y, zero_method="pratt", correction=False)
      print("{:.3f}".format(p_value))
      print()
  for ttype, datasets in table.items():
    print(ttype)
    average = defaultdict(int)
    for corpus, res in datasets.items():
      #print(res.keys())
      template_row = R'\textbf{{{}}}' + ' & {:.3f}'*len(res.values()) + R'\\'
      print(template_row.format(corpus, *res.values()))
      for k,v in res.items():
        average[k] += v
    print(R'\textbf{{Average}} ' + ('& {:.3f}'*len(average.values())).format(*[x/6 for x in average.values()]) + R'\\')

reason: import numpy,from scipy


text:
# Copyright 2021 Erik Tollerud and Marie van Staveren

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import io

import numpy as np
import pandas as pd

from bokeh.plotting import figure
from bokeh.models import Span
from bokeh.resources import CDN, INLINE
from bokeh.embed import file_html
from bokeh.io.export import get_screenshot_as_png

import matplotlib.pyplot as plt
import matplotlib.dates as mdates

from flask import Flask, send_file, jsonify
app = Flask(__name__)

app.config['DATA_FILE'] = os.environ.get('TEMP_SERVER_DATA_FILE', 'temp_data')

EMERGE_TEMP_F = 64

app.config['UTC_OFFSET'] =  os.environ.get('TEMP_SERVER_UTC_OFFSET', -4)

app.config['EMERGE_TEMP_F'] =  os.environ.get('TEMP_SERVER_EMERGE_TEMP_F', 64)
app.config['TREND_FPERHR'] =  os.environ.get('TEMP_SERVER_TREND_FPERHR', .1)

app.config['PNG_HOURS'] = os.environ.get('TEMP_SERVER_PNG_HOURS', 48)

def f_to_c(degf):
    return (degf - 32)*5/9
def c_to_f(degc):
    return degc*9/5 + 32

@app.route('/')
def index():
    with open(app.config['DATA_FILE']) as f:
        header = next(iter(f))
        header_names = header.split()
        if 'timestamp' in header_names:
            header_names.remove('timestamp')
    return 'try /latest/<colname>, /latestjson/<colname>, /plot/<colname>, /png/<colname>.  <colname> can be: ' + str(header_names)

@app.route('/latestjson/<colname>')
def latest_json(colname):
    x, y = get_data(colname)[:2]

    latest_time_idx = np.argmax(x)
    latest_val = y[latest_time_idx]

    dt = np.datetime64('now') - x.to_numpy()
    hr_since = dt.astype(float)/3.6e12 + app.config['UTC_OFFSET']

    dct = dict(column_name=colname, latest_val=latest_val,
                sec_since=int(hr_since[latest_time_idx]*3600))

    msk24hr = hr_since < 24
    y24 = y[msk24hr]
    dct['min_24hr'] = y24.min()
    dct['max_24hr'] = y24.max()

    if colname.startswith('temp'):
        if colname == 'temp_c':
            emergence_temp = f_to_c(app.config['EMERGE_TEMP_F'])
            slope_threshold = f_to_c(app.config['TREND_FPERHR'])
        elif colname == 'temp_f':
            emergence_temp = app.config['EMERGE_TEMP_F']
            slope_threshold = app.config['TREND_FPERHR']
        else:
            return f"Invalid temp! {colname}"
        dct['temp_diff'] = emergence_temp - latest_val
        dct['emergence_imminent'] = int(dct['temp_diff'] < 0)

        msk2hr = hr_since < 2
        slope, intercept = np.polyfit(-hr_since[msk2hr], y[msk2hr], 1)
        dct['slope_2hr'] = slope
        dct['trend_2hr'] = 0
        if slope > slope_threshold:
            dct['trend_2hr'] = 1
        elif slope < -slope_threshold:
            dct['trend_2hr'] = -1

    return jsonify(dct)

@app.route('/latest/<colname>')
def latest(colname):
    j = latest_json(colname).json

    extra = ''
    if 'temp_diff' in j:
        extra = ' which is {temp_diff:.3} deg below the emergence temperature.'.format(**j)
        if j['emergence_imminent']:
            extra  = extra + ' EMERGENCE IMMINENT!'

    return ('Latest value for column "{column_name}" was {latest_val:.4}, '
            '{sec_since} seconds ago. '.format(**j) + extra)

@app.route('/plot/<colname>')
def plot_column(colname):
    x, y = get_data(colname)[:2]

    p = figure(plot_width=1000, plot_height=600,
            x_axis_type="datetime", tools="pan,wheel_zoom,box_zoom,reset")

    p.line(x, y)

    span_temp = None
    if colname == 'temp_c':
        span_temp = f_to_c(app.config['EMERGE_TEMP_F'])
    elif colname == 'temp_f':
        span_temp = app.config['EMERGE_TEMP_F']

    if span_temp is not None:
        p.add_layout(Span(location=span_temp,
                          dimension='width', line_color='red',
                          line_dash='dotted', line_width=1))

    return file_html(p, CDN, "temp_server: " + colname)

PRETTY_COLNAMES = {'temp_f': 'Temp (F)', 'temp_c':'Temp (C)'}

@app.route('/png/<colname>')
def png_column(colname):
    x, y = get_data(colname)[:2]

    dt = np.datetime64('now') - x.to_numpy()
    hr_since = dt.astype(float)/3.6e12 + app.config['UTC_OFFSET']
    msk = hr_since < float(app.config['PNG_HOURS'])
    x, y = [a[msk] for a in (x, y)]

    fig, ax = plt.subplots(figsize=(10, 6))
    ax.plot(x, y, c='C0')

    if colname == 'temp_c':
        ax.axhline(f_to_c(app.config['EMERGE_TEMP_F']), c='C1', ls='-', lw=2, alpha=.8)
    elif colname == 'temp_f':
        ax.axhline(app.config['EMERGE_TEMP_F'], c='C1', ls='-', lw=2, alpha=.8)

    ax.set_ylabel(PRETTY_COLNAMES.get(colname, colname), fontsize=24)
    ax.yaxis.set_tick_params(rotation=0, labelsize=16)

    ax.set_xlabel('Date & Time', fontsize=24)
    locator = mdates.AutoDateLocator(minticks=4, maxticks=10)
    formatter = mdates.ConciseDateFormatter(locator)
    ax.xaxis.set_major_locator(locator)
    ax.xaxis.set_major_formatter(formatter)
    ax.xaxis.set_tick_params(labelsize=16)
    ax.xaxis.get_offset_text().set_size(16)

    fig.tight_layout()

    bio = io.BytesIO()
    plt.savefig(bio, format='png')

    bio.seek(0)
    return send_file(bio, mimetype='image/png')

def get_data(column_name):
    df = pd.read_table(app.config['DATA_FILE'], ' ', parse_dates=['timestamp'])

    x = df['timestamp']
    if column_name == 'temp_f' and 'temp_f' not in df:
        y = c_to_f(df['temp_c'])
    else:
        y = df[column_name]

    return x, y, df


text:
from social_epi import sampling_social_networks as ssn
from social_epi import nx_conversion as nxc
from social_epi import CCMnet_constr_py as ccm
import numpy as np
import matplotlib.pyplot as plt
import json,time

def determine_burnin(social_config, contact_network, transmission_network, burnin_list):
    config = json.load(open(social_config))
    for bu in burnin_list:
        starttime = time.time()
        config["burnin"] = bu
        print("Burnin: {}".format(bu))
        social_network, overlap_network = ssn.run(config, contact_network, transmission_network)
        print("The overlap network is conserved: {}".format(check_overlap(social_network,overlap_network)))
        print("Error in degree distribution: {}".format(compare_dists(config,social_network)))
        print("Time taken: {}".format(time.time()-starttime))

def check_overlap(SN,ON):
    return set(ON.edges()).issubset(set(SN.edges()))

def compare_dists(config,SN):
    N = SN.number_of_nodes()
    deg_dict = config["degree_distribution"]
    deg_dist = np.asarray(nxc.pad_deg_dist(deg_dict,config["small_prob"],N))
    sn_degs = [deg for _, deg in SN.degree()]
    sn_deg_dist = np.array([sn_degs.count(i)/N for i in range(N)])
    return sum(np.abs(deg_dist-sn_deg_dist))

def assess_burnin_L1(burnin_config_file,contact_network_file,transmission_network_file=None):
    if isinstance(contact_network_file,str):
        contact_network,_ = nxc.favitescontacttransmission2nx(contact_network_file,transmission_network_file)
    else:
        contact_network =contact_network_file
    if isinstance(burnin_config_file,str):
        config = json.load(open(burnin_config_file))
    else:
        config = burnin_config_file
    N=int(contact_network.number_of_nodes())
    # make CCM config dictionary and then run CCM
    ccmc = ssn.gen_config(config,N)
    _, dd_stats = ccm.CCMnet_constr_py(**ccmc)
    dd_stats.to_csv("deg_hist_results.csv")
    # get desired deg dist
    deg_dict = config["degree_distribution"]
    deg_dist = np.asarray(nxc.pad_deg_dist(deg_dict,config["small_prob"],N))
    L1 = []
    for dd in dd_stats.iterrows():
        dd = np.array(dd[1])
        L1.append(sum(np.abs(deg_dist-dd/N)))
    json.dump(L1,open("L1_results.json","w"))
    plt.scatter(range(len(L1)),L1)
    plt.savefig("convergence.png")
    plt.show()

if __name__ == "__main__":
    # cd tests/chain_test/FAVITES_output_20220524141705
    contact_network = "contact_network.txt"
    transmission_network = "error_free_files/transmission_network.txt"

    # social_config = "../../../src/configs/sampling_social_networks_config.json"
    # burnin_list = np.arange(1000,10000,1000)
    # determine_burnin2(social_config,contact_network,transmission_network,burnin_list)

    bc = "../../../src/configs/burnin_config.json"
    assess_burnin_L1(bc,contact_network,transmission_network)


text:
#!/usr/bin/env python3

"""
    Example using VotedPerceptrons in a MulticlassClassifier for MNIST digit classification.

    Example
    -------
    Prior to training set the desired kernel parameters by changing values in the
    KERNEL_PARAMETERS global variable.

    To train a MulticlassClassifier on MNIST with the following parameters:
        fraction_of_mnist: 0.1 (e.g. 6000 MNIST training images (10% of 60000 total))
        strategy: OVA (one-vs-all)
        error_threshold: 0
        max_epochs: 1
    and save it to multicc_mnist.dill.xz::

        $ python mnist_example.py -f0.1 train OVA 0 1

    To test the MulticlassClassifier multicc_mnist.dill.xz with the following parameters:
        fraction_of_mnist: 0.1 (e.g. 1000 MNIST test images (10% of 10000 total))
    and see the error rate::

        $ python mnist_example.py -f0.1 test

    Notes
    -----
    For this example the following are required:
    -Python 3.3 or higher (>=3.2 for argparse and >=3.3 for lzma).
    -At same level of this script a folder named data containing mnist.hdf5.xz or mnist.hdf5
    -dill module (should already be installed by multiprocess module).
    -h5py module.
    Note on debian based systems to install h5py via pip you may need to do::

        apt-get install libhdf5-dev
        HDF5_DIR=/usr/lib/x86_64-linux-gnu/hdf5/serial/ pip install h5py

    In this script the created instance of MulticlassClassifier is serialized via dill and saved
    to a file.
    Saved MulticlassClassifier and VotedPerceptron instances can be loaded later to:
        -Evaluate its error rate on some test data (as we do here in this example).
        -Trained further by calling its train method with more data.
        -Used in an interactive manner as part of a larger program that calls the instance's
         train and predict methods as needed (e.g. as part of some game AI logic).

         For the last use one should keep in mind that there is no inherit limit set on how many
         prediction vectors a VotedPerceptron can accumulate during training.
         If enforcing a limit is desired (e.g. for resource reasons), one can for example
         periodically check the length of one of the VotedPerceptron attributes:
         prediction_vector_term_coeffs
         prediction_vector_terms
         prediction_vector_votes
         (note they are all in 1-1 correspondence with one another so they have the same size)
         and if beyond a certain size then truncate the first say N entries from each where
         0 <= N <= length(prediction_vector_votes). A theoretical analysis on the pros and cons
         of this truncation strategy is beyond the scope of this note.
"""

import argparse
import lzma
import os
import numpy as np
import h5py
import dill

from context import votedperceptron
from votedperceptron import VotedPerceptron, MulticlassClassifier

# Path to MNIST data.
# Note: If only mnist.hdf5.xz found an uncompressed version, mnist.hdf5, will be created.
DATA_FILE_PATH = os.path.join('.', 'data', 'mnist.hdf5')

# Dictionary of kernel parameters to determine kernel used by the underlying binary classifiers.
KERNEL_PARAMETERS = {'kernel_type': 'polynomial', 'degree': 4, 'gamma': 1, 'coef0': 1}
#KERNEL_PARAMETERS = {'kernel_type': 'gaussian_rbf', 'gamma': 0.03125}

def create_mnist_dictionary(dataset_name, labelset_name):
    """ Creates dictionary of MNIST data and labels for the given dataset_name and labelset_name.

    Parameters
    ----------
    dataset_name : str
        Name of data dataset in file to load. Valid choices are:
        train_images, t10k_images
    labelset_name : str
        Name of label dataset in file to load. Valid choices are:
        train_labels, t10k_labels

    Returns
    -------
    dict
        MNIST data and labels for the given dataset_name and labelset_name.

    Raises
    ------
    FileNotFoundError
        If unable to locate data/mnist.hdf5 or data/mnist.hdf5.xz
    """
    # Initialize mnist, filepath, filepath_compressed.
    mnist = {}
    filepath = DATA_FILE_PATH
    filepath_compressed = filepath + '.xz'

    # Check if uncompressed file exists. If not then make uncompressed version.
    if not os.path.isfile(filepath):
        if os.path.isfile(filepath_compressed):
            with lzma.open(filepath_compressed, 'rb') as compressed_file:
                decoded = compressed_file.read()

            with open(filepath, 'wb') as uncompressed_file:
                uncompressed_file.write(decoded)
        else:
            raise FileNotFoundError('Unable to locate ' + filepath + ' or ' + filepath_compressed)

    # Read uncompressed file.
    with h5py.File(filepath, 'r') as mnist_hdf5:
        data = mnist_hdf5.get(dataset_name)
        labels = mnist_hdf5.get(labelset_name)

        # Copy to mnist dictionary.
        mnist[dataset_name] = np.empty(data.shape, data.dtype)
        mnist[labelset_name] = np.empty(labels.shape, labels.dtype)
        data.read_direct(mnist[dataset_name])
        labels.read_direct(mnist[labelset_name])

    return mnist

def get_data_and_labels(data_type, fraction_of_mnist):
    """ Return appropriate type and amount of mnist data and labels scaled to 0-1 range.

    Parameters
    ----------
    data_type : str
        'train': For training data and labels.
        'test': For test data and labels.
    fraction_of_mnist : float (0-1)
        Fractional amount of data and labels to return.

    Returns
    -------
    2 ndarrays
        data and labels appropriate for the given data_type and fraction_of_mnist.
    """
    # Determine the appropriate datasets to use from the mnist hdf5 file.
    if data_type == 'train':
        dataset_name = 'train_images'
        labelset_name = 'train_labels'

    elif data_type == 'test':
        dataset_name = 't10k_images'
        labelset_name = 't10k_labels'

    # Set mnist dictionary.
    mnist = create_mnist_dictionary(dataset_name, labelset_name)

    # mnist data scaled to 0-1 range.
    num_input_vectors = round(fraction_of_mnist * len(mnist[dataset_name]))
    data = mnist[dataset_name][:num_input_vectors].astype(np.float32) / 255
    labels = mnist[labelset_name][:num_input_vectors]

    return data, labels

def train_mnist(args):
    """ Instantiates and trains a MulticlassClassifier for MNIST and saves it to a file.

    Parameters
    ----------
    args : argparse.Namespace
        argparse.Namespace object containing the command line argument values as attributes.

    Returns
    -------
    list of 2-tuples
        For a given 2-tuple the first element is a binary classifier key and
        the second element is the number of prediction vectors making up the binary classifier.
    """
    # Arguments from the command line.
    strategy = args.strategy
    error_threshold = args.error_threshold
    max_epochs = args.max_epochs
    process_count = args.process_count
    fraction_of_mnist = args.fraction_of_mnist

    # Possible labels are 0-9.
    possible_labels = tuple(range(10))

    # Create instance of MulticlassClassifier.
    multicc = MulticlassClassifier(strategy, possible_labels, VotedPerceptron, (),
                                   {'kernel_parameters':KERNEL_PARAMETERS})
    other_bc_train_args = tuple([error_threshold, max_epochs])
    other_bc_train_kwargs = {}

    # Get data and labels for training.
    print('Loading MNIST data')
    data, labels = get_data_and_labels('train', fraction_of_mnist)

    # Train instance of MulticlassClassifier.
    print('Training')
    multicc.train(data, labels, other_bc_train_args, other_bc_train_kwargs, process_count)

    # Save trained MulticlassClassifier.
    print('Saving MulticlassClassifier')
    save_filepath = 'multicc_mnist.dill.xz'
    with lzma.open(save_filepath, 'wb') as multicc_file:
        dill.dump(multicc, multicc_file)

    # Return number of prediction vectors making up each binary classifier.
    bc_vector_counts = [(k, len(v.prediction_vector_votes))
                        for k, v in multicc.binary_classifiers.items()]

    return bc_vector_counts

def test_mnist(args):
    """ Calculates error rate on MNIST for a previously saved MulticlassClassifier instance.

    Parameters
    ----------
    args : argparse.Namespace
        Arguments from command line.

    Returns
    -------
    error_rate : float
        The calculated error rate.

    Raises
    ------
    FileNotFound
        If unable to locate multicc_mnist.dill.xz
    """
    # Arguments from the command line.
    process_count = args.process_count
    fraction_of_mnist = args.fraction_of_mnist

    # Load trained MulticlassClassifier.
    print('Loading MulticlassClassifier')
    load_filepath = 'multicc_mnist.dill.xz'
    if os.path.isfile(load_filepath):
        with lzma.open(load_filepath, 'rb') as multicc_file:
            multicc = dill.load(multicc_file)
    else:
        raise FileNotFoundError('Unable to locate ' + load_filepath)

    # Set other binary classifier predict arguments.
    # We set option so the voted perceptron predict method returns a real-valued confidence score
    # as opposed to a label. This is required by MulticlassClassifier.
    other_bc_predict_args = tuple(['score'])
    other_bc_predict_kwargs = {}

    # Get data and labels for testing.
    print('Loading MNIST data')
    data, labels = get_data_and_labels('test', fraction_of_mnist)

    # Calculate error rate on test data.
    print('Computing error rate')
    error_rate = multicc.error_rate(data, labels,
                                    other_bc_predict_args, other_bc_predict_kwargs,
                                    process_count)

    return error_rate

def main():
    """ Define and parse command line arguments.

    """
    # Create the top-level parser.
    parser = argparse.ArgumentParser()
    parser.add_argument('-t', '--process_count',
                        help='number of worker processes to use',
                        type=int,
                        choices=range(1, os.cpu_count() + 1),
                        default=os.cpu_count())
    parser.add_argument('-f', '--fraction_of_mnist',
                        help='fraction of MNIST data to use (range 0 to 1)',
                        type=float,
                        default=1)
    subparsers = parser.add_subparsers(help='sub-command help')

    # Create the parser for the train command.
    parser_train = subparsers.add_parser('train',
                                         help='create and train a MulticlassClassifier')
    parser_train.add_argument('strategy',
                              help='strategy',
                              choices=['OVA', 'OVO'],
                              default='OVA')
    parser_train.add_argument('error_threshold',
                              help='error_threshold',
                              type=float,
                              default=0)
    parser_train.add_argument('max_epochs',
                              help='max_epochs',
                              type=int,
                              default=1)
    parser_train.set_defaults(func=train_mnist)

    # Create the parser for the test command.
    parser_test = subparsers.add_parser('test',
                                        help='test trained MulticlassClassifier')
    parser_test.set_defaults(func=test_mnist)

    # Parse arguments and call appropriate function (train or test).
    args = parser.parse_args()
    print(args.func(args))

if __name__ == '__main__':
    main()

max_stars_repo_name: bmgee/votedperceptron

max_issues_repo_name: bmgee/votedperceptron

max_forks_repo_name: bmgee/votedperceptron


text:
import os

import pytest
import numpy as np

import pyansys
from pyansys.examples import rstfile, fullfile
from pyvista.plotting import system_supports_plotting

test_path = os.path.dirname(os.path.abspath(__file__))
testfiles_path = os.path.join(test_path, 'testfiles')

HAS_FFMPEG = True
try:
    import imageio_ffmpeg
except ImportError:
    HAS_FFMPEG = False

def test_loadresult():
    result = pyansys.read_binary(rstfile)

    # check result is loaded
    assert result.nsets
    assert result.nnum.size

    # check geometry is genreated
    grid = result.grid
    assert grid.points.size
    assert grid.cells.size
    assert 'ansys_node_num' in grid.point_arrays

    # check results can be loaded
    nnum, disp = result.nodal_solution(0)
    assert nnum.size
    assert disp.size

    nnum, disp = result.nodal_solution(0)
    assert nnum.size
    assert disp.size

    nnum, disp = result.principal_nodal_stress(0)
    assert nnum.size
    assert disp.size

    nnum, disp = result.nodal_stress(0)
    assert nnum.size
    assert disp.size

    element_stress, enum, enode = result.element_stress(0)
    assert element_stress[0].size
    assert enum.size
    assert enode[0].size

    element_stress, enum, enode = result.element_stress(0, principal=True)
    assert element_stress[0].size
    assert enum.size
    assert enode[0].size

@pytest.mark.skipif(not system_supports_plotting(), reason="Requires active X Server")
@pytest.mark.skipif(not HAS_FFMPEG, reason="requires imageio_ffmpeg")
def test_animate_nodal_solution(tmpdir):
    result = pyansys.read_binary(rstfile)
    temp_movie = str(tmpdir.mkdir("tmpdir").join('tmp.mp4'))
    result.animate_nodal_solution(0, nangles=20, movie_filename=temp_movie,
                                  off_screen=True)
    assert np.any(result.grid.points)
    assert os.path.isfile(temp_movie)

def test_loadbeam():
    linkresult = os.path.join(testfiles_path, 'link1.rst')
    result = pyansys.read_binary(linkresult)
    assert np.any(result.grid.cells)

def test_fullreader():
    fobj = pyansys.read_binary(fullfile)
    dofref, k, m = fobj.load_km()
    assert dofref.size
    assert k.size
    assert m.size

max_stars_repo_name: shazimy/pyansys

max_issues_repo_name: OpenCDR/pyansys

max_forks_repo_name: OpenCDR/pyansys


text:
#!/usr/bin/env python
# encoding: utf-8
r"""
Solitary wave formation in periodic nonlinear elastic media
===========================================================

Solve a one-dimensional nonlinear elasticity system:

.. math::
    \epsilon_t + u_x & = 0 \\
    (\rho(x) u)_t + \sigma(\epsilon,x)_x & = 0.

Here :math:`\epsilon` is the strain, :math:`\sigma` is the stress,
u is the velocity, and :math:`\rho(x)` is the density.
We take the stress-strain relation :math:`\sigma = e^{K(x)\epsilon}-1`;
:math:`K(x)` is the linearized bulk modulus.
Note that the density and bulk modulus may depend explicitly on x.

The problem solved here is based on [LeVYon03]_.  An initial hump
evolves into two trains of solitary waves.

"""
from __future__ import absolute_import
from __future__ import print_function
import numpy as np
from six.moves import range

def qinit(state,ic=2,a2=1.0,xupper=600.):
    x = state.grid.x.centers

    if ic==1: #Zero ic
        state.q[:,:] = 0.
    elif ic==2:
        # Gaussian
        sigma = a2*np.exp(-((x-xupper/2.)/10.)**2.)
        state.q[0,:] = np.log(sigma+1.)/state.aux[1,:]
        state.q[1,:] = 0.

def setaux(x,rhoB=4,KB=4,rhoA=1,KA=1,alpha=0.5,xlower=0.,xupper=600.,bc=2):
    aux = np.empty([3,len(x)],order='F')
    xfrac = x-np.floor(x)
    #Density:
    aux[0,:] = rhoA*(xfrac<alpha)+rhoB*(xfrac>=alpha)
    #Bulk modulus:
    aux[1,:] = KA  *(xfrac<alpha)+KB  *(xfrac>=alpha)
    aux[2,:] = 0. # not used
    return aux

def b4step(solver,state):
    #Reverse velocity at trtime
    #Note that trtime should be an output point
    if state.t>=state.problem_data['trtime']-1.e-10 and not state.problem_data['trdone']:
        #print 'Time reversing'
        state.q[1,:]=-state.q[1,:]
        state.q=state.q
        state.problem_data['trdone']=True
        if state.t>state.problem_data['trtime']:
            print('WARNING: trtime is '+str(state.problem_data['trtime'])+\
                ' but velocities reversed at time '+str(state.t))
    #Change to periodic BCs after initial pulse
    if state.t>5*state.problem_data['tw1'] and solver.bc_lower[0]==0:
        solver.bc_lower[0]=2
        solver.bc_upper[0]=2
        solver.aux_bc_lower[0]=2
        solver.aux_bc_upper[0]=2

def zero_bc(state,dim,t,qbc,auxbc,num_ghost):
    """Set everything to zero"""
    if dim.on_upper_boundary:
        qbc[:,-num_ghost:]=0.

def moving_wall_bc(state,dim,t,qbc,auxbc,num_ghost):
    """Initial pulse generated at left boundary by prescribed motion"""
    if dim.on_lower_boundary:
        qbc[0,:num_ghost]=qbc[0,num_ghost]
        t=state.t; t1=state.problem_data['t1']; tw1=state.problem_data['tw1']
        a1=state.problem_data['a1'];
        t0 = (t-t1)/tw1
        if abs(t0)<=1.: vwall = -a1*(1.+np.cos(t0*np.pi))
        else: vwall=0.
        for ibc in range(num_ghost-1):
            qbc[1,num_ghost-ibc-1] = 2*vwall*state.aux[1,ibc] - qbc[1,num_ghost+ibc]

def setup(use_petsc=0,kernel_language='Fortran',solver_type='classic',outdir='./_output'):
    from clawpack import riemann

    if use_petsc:
        import clawpack.petclaw as pyclaw
    else:
        from clawpack import pyclaw

    if kernel_language=='Python':
        rs = riemann.nonlinear_elasticity_1D_py.nonlinear_elasticity_1D
    elif kernel_language=='Fortran':
        rs = riemann.nonlinear_elasticity_fwave_1D

    if solver_type=='sharpclaw':
        solver = pyclaw.SharpClawSolver1D(rs)
        solver.char_decomp=0
    else:
        solver = pyclaw.ClawSolver1D(rs)

    solver.kernel_language = kernel_language

    solver.bc_lower[0] = pyclaw.BC.custom
    solver.bc_upper[0] = pyclaw.BC.extrap

    #Use the same BCs for the aux array
    solver.aux_bc_lower[0] = pyclaw.BC.extrap
    solver.aux_bc_upper[0] = pyclaw.BC.extrap

    xlower=0.0; xupper=300.0
    cells_per_layer=12; mx=int(round(xupper-xlower))*cells_per_layer
    x = pyclaw.Dimension(xlower,xupper,mx,name='x')
    domain = pyclaw.Domain(x)
    state = pyclaw.State(domain,solver.num_eqn,num_aux=3)
    state.problem_data['stress_relation'] = 'exponential'

    #Set global parameters
    alpha = 0.5
    KA    = 1.0
    KB    = 4.0
    rhoA  = 1.0
    rhoB  = 4.0
    state.problem_data = {}
    state.problem_data['t1']    = 10.0
    state.problem_data['tw1']   = 10.0
    state.problem_data['a1']    = 0.1
    state.problem_data['alpha'] = alpha
    state.problem_data['KA'] = KA
    state.problem_data['KB'] = KB
    state.problem_data['rhoA'] = rhoA
    state.problem_data['rhoB'] = rhoB
    state.problem_data['trtime'] = 999999999.0
    state.problem_data['trdone'] = False

    #Initialize q and aux
    xc=state.grid.x.centers
    state.aux[:,:] = setaux(xc,rhoB,KB,rhoA,KA,alpha,xlower=xlower,xupper=xupper)
    qinit(state,ic=1,a2=1.0,xupper=xupper)

    tfinal=500.; num_output_times = 20;

    solver.max_steps = 5000000
    solver.fwave = True
    solver.before_step = b4step
    solver.user_bc_lower=moving_wall_bc
    solver.user_bc_upper=zero_bc

    claw = pyclaw.Controller()
    claw.output_style = 1
    claw.num_output_times = num_output_times
    claw.tfinal = tfinal
    claw.solution = pyclaw.Solution(state,domain)
    claw.solver = solver
    claw.setplot = setplot
    claw.keep_copy = True

    return claw

#--------------------------
def setplot(plotdata):
#--------------------------
    """
    Specify what is to be plotted at each frame.
    Input:  plotdata, an instance of visclaw.data.ClawPlotData.
    Output: a modified version of plotdata.
    """
    plotdata.clearfigures()  # clear any old figures,axes,items data

    # Figure for q[0]
    plotfigure = plotdata.new_plotfigure(name='Stress', figno=1)

    # Set up for axes in this figure:
    plotaxes = plotfigure.new_plotaxes()
    plotaxes.title = 'Stress'
    plotaxes.ylimits = [-0.1,1.0]

    # Set up for item on these axes:
    plotitem = plotaxes.new_plotitem(plot_type='1d_plot')
    plotitem.plot_var = stress
    plotitem.kwargs = {'linewidth':2}

    # Figure for q[1]
    plotfigure = plotdata.new_plotfigure(name='Velocity', figno=2)

    # Set up for axes in this figure:
    plotaxes = plotfigure.new_plotaxes()
    plotaxes.xlimits = 'auto'
    plotaxes.ylimits = [-.5,0.1]
    plotaxes.title = 'Velocity'

    # Set up for item on these axes:
    plotitem = plotaxes.new_plotitem(plot_type='1d_plot')
    plotitem.plot_var = velocity
    plotitem.kwargs = {'linewidth':2}

    return plotdata

def velocity(current_data):
    """Compute velocity from strain and momentum"""
    aux=setaux(current_data.x,rhoB=4,KB=4)
    velocity = current_data.q[1,:]/aux[0,:]
    return velocity

def stress(current_data):
    """Compute stress from strain and momentum"""
    from clawpack.riemann.nonlinear_elasticity_1D_py import sigma
    aux=setaux(current_data.x)
    epsilon = current_data.q[0,:]
    stress = sigma(current_data.q,aux,{'stress_relation':'exponential'})
    return stress

if __name__=="__main__":
    from clawpack.pyclaw.util import run_app_from_main
    output = run_app_from_main(setup,setplot)

max_stars_repo_name: gyanz/pyclaw

max_issues_repo_name: gyanz/pyclaw

max_forks_repo_name: gyanz/pyclaw