text:
from __future__ import absolute_import
from .Node import Op
import ctypes
import numpy as np
from .._base import DNNL_LIB
#from ..cpu_links import dropout as cpu_dropout
#from ..cpu_links import dropout_gradient as cpu_dropout_gradient
from ..gpu_links import dropout2d_gradient
from ..gpu_links import dropout2d

class Dropout2dOp(Op):
    def __init__(self, node_in, keep_prob, ctx=None):
        super().__init__(Dropout2dOp, [node_in], ctx)
        self.seed = ctypes.c_ulonglong(0)
        self.mask = None
        self.keep_prob = keep_prob

    def compute(self, input_vals, output_val, stream_handle=None, inference=False):
        if inference:
            if self.on_cpu:
                output_val[:] = input_vals[0].asnumpy()
            else:
                input_vals[0].copyto(output_val)
        else:
            if self.on_cpu:
                raise NotImplementedError
            else:
                dropout2d(input_vals[0], 1 - self.keep_prob,
                          output_val, self.seed, stream_handle)

    def gradient(self, output_grad):
        return [dropout2d_gradient_op(output_grad, self.keep_prob, self.seed, ctx=self.raw_ctx)]

    def infer_shape(self, input_shapes):
        return input_shapes[0]

class Dropout2d_GradientOp(Op):
    def __init__(self, node_in, keep_prob, seed, ctx=None):
        super().__init__(Dropout2d_GradientOp, [node_in], ctx)
        self.seed = seed
        self.keep_prob = keep_prob

    def compute(self, input_vals, output_val, stream_handle=None):
        if self.on_cpu:
            raise NotImplementedError
        else:
            dropout2d_gradient(
                input_vals[0], 1 - self.keep_prob, output_val, self.seed, stream_handle)

    def gradient(self, output_grad):
        raise NotImplementedError

    def infer_shape(self, input_shapes):
        return input_shapes[0]

def dropout2d_op(node_in, keep_prob, ctx=None):
    """Drops elements of input variable randomly.
    Parameters:
    ----
    node_in : Node
        Input variable.
    keep_prob : float
        Probability of the results to be kept.
    Returns:
    ----
    A new Node instance created by Op.
    """
    return Dropout2dOp(node_in, keep_prob, ctx=ctx)

def dropout2d_gradient_op(node_in, keep_prob, seed, ctx=None):
    """Gradient node of dropout2d operation.
    Parameters:
    ----
    node_in : Node
        Input variable.
    keep_prob : float
        Probability of the results to be kept.
    Returns:
    ----
    A new Node instance created by Op.
    """
    return Dropout2d_GradientOp(node_in, keep_prob, seed, ctx=ctx)

max_stars_repo_name: axiserr/Hetu

max_issues_repo_name: axiserr/Hetu

max_forks_repo_name: axiserr/Hetu


text:
import cv2
import numpy as np
import matplotlib.pyplot as plt
import os
import pandas as pd
import imutils

from my_paths import directory, save_dir

name_list = []
number = []
scores = []
for filename in os.listdir(directory):
    if filename.endswith(".jpg"):
        names = filename.split('_')[1].split(".")[0][-3:]
        name_list.append(names)
y_pos = np.arange(len(name_list))

df = pd.read_csv(save_dir + "otsu_metrics.csv")

for i, key in enumerate(name_list):
    scores.append(df["dice"][i])
    number.append(i)

plt.bar(y_pos, scores, align='center', alpha=0.5)
plt.xticks(y_pos, name_list, rotation=75)
plt.axhline(0.83, color='#999999')
plt.ylabel('Dice Similarity Score')
plt.title('Bar-chart of Dice score and pictures with 0.83 mean dice score')

plt.show()

ext: py

max_stars_repo_path: barchart_from csv_metrics.py

max_stars_repo_name: parisChatz/Image-Segmentation

max_issues_repo_path: barchart_from csv_metrics.py

max_issues_repo_name: parisChatz/Image-Segmentation

max_forks_repo_path: barchart_from csv_metrics.py

max_forks_repo_name: parisChatz/Image-Segmentation

reason: import numpy


text:
import numpy as np

def trid_func(x, d):
    """
    Compute Trid function at a given point with given arguments.

    Parameters
    ----------
    point : 1-D array with shape (d, )
            A point used to evaluate the function.
    d : integer
        Dimension

    Returns
    -------
    function value : float
    """
    s_1 = np.sum((x - 1) ** 2)
    s_2 = np.sum(x[1:] * x[:-1])
    return s_1 - s_2

def trid_grad(x, d):
    """
    Compute Trid gradient at a given point with given arguments.

    Parameters
    ----------
    point : 1-D array with shape (d, )
            A point used to evaluate the function.
    d : integer
        Dimension

    Returns
    -------
    grad : 1-D array with shape (d,)
    """
    grad = np.zeros((d))
    for i in range(d):
        if i == 0:
            grad[i] = 2 * (x[i] - 1) - x[i+1]
        elif i > 0 and i < (d - 1):
            grad[i] = 2 * (x[i] - 1) - x[i-1] - x[i+1]
        elif i == (d-1):
            grad[i] = 2 * (x[i] - 1) - x[i-1]
    return grad

max_stars_repo_name: Megscammell/METOD-Algorithm

max_issues_repo_name: Megscammell/METOD-Algorithm

max_forks_repo_name: Megscammell/METOD-Algorithm


text:
import torch
import numpy as np
import sys
import os
# import matplotlib.pyplot as plt

sys.path.append("../")
from utils import plot_stroke
from utils.constants import Global
from utils.dataset import HandwritingDataset
from utils.data_utils import (
    data_denormalization,
    data_normalization,
    valid_offset_normalization,
)
from models.models import HandWritingSynthesisNet
from generate import generate_conditional_sequence

def generate_handwriting(
    char_seq="hello world",
    real_text="",
    style_path="../app/static/mobile/style.npy",
    save_path="",
    app_path="",
    n_samples=1,
    bias=10.0,
):
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    data_path = os.path.join(app_path, "../data/")
    model_path = os.path.join(app_path, "../results/synthesis/best_model_synthesis.pt")
    # seed = 194
    # print("seed:",seed)
    # torch.manual_seed(seed)
    # np.random.seed(seed)
    # print(np.random.get_state())

    train_dataset = HandwritingDataset(data_path, split="train", text_req=True)

    prime = True
    is_map = False
    style = np.load(style_path, allow_pickle=True, encoding="bytes").astype(np.float32)
    # plot the sequence
    # plot_stroke(style, os.path.join(save_path, "original.png"))

    print("Priming text: ", real_text)
    mean, std, style = data_normalization(style)
    style = torch.from_numpy(style).unsqueeze(0).to(device)
    # style = valid_offset_normalization(Global.train_mean, Global.train_std, style[None,:,:])
    # style = torch.from_numpy(style).to(device)
    print("Priming sequence size: ", style.shape)
    ytext = real_text + " " + char_seq + "  "

    for i in range(n_samples):
        gen_seq, phi = generate_conditional_sequence(
            model_path,
            char_seq,
            device,
            train_dataset.char_to_id,
            train_dataset.idx_to_char,
            bias,
            prime,
            style,
            real_text,
            is_map,
        )
        # if is_map:
        #     plt.imshow(phi, cmap="viridis", aspect="auto")
        #     plt.colorbar()
        #     plt.xlabel("time steps")
        #     plt.yticks(np.arange(phi.shape[0]), list(ytext), rotation="horizontal")
        #     plt.margins(0.2)
        #     plt.subplots_adjust(bottom=0.15)
        #     plt.show()

        # denormalize the generated offsets using train set mean and std
        print("data denormalization...")
        end = style.shape[1]
        # gen_seq[:,:end] = data_denormalization(mean, std, gen_seq[:, :end])
        # gen_seq[:,end:] = data_denormalization(Global.train_mean, Global.train_std, gen_seq[:,end:])
        gen_seq = data_denormalization(Global.train_mean, Global.train_std, gen_seq)
        # plot the sequence
        print(gen_seq.shape)
        # plot_stroke(gen_seq[0, :end])
        plot_stroke(
            gen_seq[0], os.path.join(save_path, "gen_stroke_" + str(i) + ".png")
        )
        print(save_path)


text:
import numpy as np
import pandas as pd

class CrowdingDistanceSorter:

    def __init__(self, greater_is_better_dict):
        self.greater_is_better_dict = greater_is_better_dict

    def _calculate_crowding_distance(self, values, max_value, min_value):
        values = pd.Series(values).sort_values()
        previous_values = values.shift(1)
        next_values = values.shift(-1)
        neighbor_delta = np.abs(next_values - previous_values)
        min_max_delta = np.abs(max_value - min_value)
        crowding_distance = (neighbor_delta / min_max_delta).fillna(float("inf"))
        crowding_distance = crowding_distance.to_dict()
        return crowding_distance

    def _front_sort(self, front, min_max_dict):
        crowding_distance = {key: 0 for key in front.keys()}
        for dim_key, min_max in min_max_dict.items():
            max_value = min_max["max"]
            min_value = min_max["min"]
            values = {obs_key: obs_val[dim_key] for obs_key, obs_val in front.items()}
            temp_cd = self._calculate_crowding_distance(values, max_value, min_value)
            for cd_key, cd_val in temp_cd.items():
                crowding_distance[cd_key] += cd_val
        return crowding_distance

    def sort(self, pareto_fronts):
        min_max_dict = {}
        for key in self.greater_is_better_dict.keys():
            min_max_dict[key] = {}
            min_max_dict[key]["min"] = np.min([phenotype[key] for front in pareto_fronts for phenotype in front.values()])
            min_max_dict[key]["max"] = np.max([phenotype[key] for front in pareto_fronts for phenotype in front.values()])
        return [self._front_sort(front, min_max_dict) for front in pareto_fronts]


text:
import numpy as np
import mistree as mist

def test_construct_mst_2d():
    x = np.random.random_sample(100)
    y = np.random.random_sample(100)
    edge_length, edge_x, edge_y, edge_index = mist.construct_mst(x, y, two_dimensions=True)
    assert len(edge_length) == 99

def test_construct_mst_3d():
    x = np.random.random_sample(100)
    y = np.random.random_sample(100)
    z = np.random.random_sample(100)
    edge_length, edge_x, edge_y, edge_z, edge_index = mist.construct_mst(x, y, z=z, two_dimensions=False)
    assert len(edge_length) == 99

def test_construct_mst_k_neighbours():
    x = np.random.random_sample(100)
    y = np.random.random_sample(100)
    edge_length, edge_x, edge_y, edge_index = mist.construct_mst(x, y, two_dimensions=True)
    edge_length2, edge_x2, edge_2, edge_index2 = mist.construct_mst(x, y, k_neighbours=30, two_dimensions=True)
    condition = np.where(np.sort(edge_length) == np.sort(edge_length2))[0]
    assert len(condition) == len(edge_length)

def test_construct_mst_2d_scale_cut():
    x = np.random.random_sample(100)
    y = np.random.random_sample(100)
    edge_length, edge_x, edge_y, edge_index, num_removed_edges = mist.construct_mst(x, y, two_dimensions=True, scale_cut_length=0.01)
    condition = np.where(edge_length >= 0.01)[0]
    assert len(condition) == len(edge_length)

def test_construct_mst_3d_scale_cut():
    x = np.random.random_sample(100)
    y = np.random.random_sample(100)
    z = np.random.random_sample(100)
    edge_length, edge_x, edge_y, edge_z, edge_index, num_removed_edges = mist.construct_mst(x, y, z=z, two_dimensions=False, scale_cut_length=0.01)
    condition = np.where(edge_length >= 0.01)[0]
    assert len(condition) == len(edge_length)

def test_construct_mst_tomo_scale_cut():
    phi = 360.*np.random.random_sample(100)
    theta = 180.*np.random.random_sample(100)
    x, y, z = mist.spherical_2_unit_sphere(phi, theta)
    edge_length, edge_x, edge_y, edge_z, edge_index, num_removed_edges = mist.construct_mst(x, y, z=z, two_dimensions=False, scale_cut_length=0.2)
    condition = np.where(edge_length >= 0.2)[0]
    assert len(condition) == len(edge_length)


text:
from random import shuffle
from sklearn.model_selection import train_test_split
import numpy as np
from sklearn.metrics import accuracy_score
from sklearn.tree import DecisionTreeClassifier, export_graphviz
from sklearn import tree
from IPython.display import SVG
from graphviz import Source
from IPython.display import display
from sklearn.externals.six import StringIO
from matplotlib import pyplot as plt
import pydotplus

translator = {
    'n': 0,
    'y': 1
}

def getValuesReady(arr):
    coreX = []
    coreY = []

    for line in arr:
        coreX.append(line[:-1])
        coreY.append(translator[line[-1]])

    return (np.array(coreX).astype(np.float), np.array(coreY).astype(np.int))

f = open("data/spam7.csv", "r")
answ = f.readlines()
f.close()
answ = answ[1:]
shuffle(answ)

for i in range(0, len(answ)):
    line = answ[i]
    line = line.rstrip("\n")
    line = line.replace('"', '')
    arr = line.split(",")
    answ[i] = arr

(X, y) = getValuesReady(answ)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
# maxDepths = [i for i in range(5,18)]
# accuracy = []

# for depth in maxDepths:
clf = DecisionTreeClassifier(max_depth=10, min_samples_split=128,
                            min_samples_leaf= 10,
                            max_features="log2",
                             class_weight="balanced",criterion="entropy", )
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(accuracy_score(y_test, y_pred))
print("Depth", clf.get_depth())
print("Params", clf.get_params())
titles = ("crl.tot", "dollar", "bang", "money", "n000", "make")
classes = ("1", "2", "3", "4", "5")

# plt.plot(maxDepths, accuracy)
# plt.title("Точность от максимальной глубины")
# plt.show()

dot_data = StringIO()
tree.export_graphviz(clf, out_file=dot_data,
                     feature_names=titles, class_names=classes)
graph = pydotplus.graph_from_dot_data(dot_data.getvalue())
graph.write_pdf("./iris.pdf")

max_stars_repo_name: alekseiliachko/w-ml

max_issues_repo_name: alekseiliachko/w-ml

max_forks_repo_name: alekseiliachko/w-ml


text:
from collections import namedtuple
# from eeea.genetic.optimizer import Optimizer
from eeea.firefly.optimizer import FAOptimizer
import logging
from tqdm import tqdm
import csv
import numpy as np
import os
import argparse
import eeea.utils.utils as utils
import glob
import sys
import math
import time
import hashlib
from tinydb import TinyDB, Query
import random
import eeea.utils.nasnet_setup as nasnet_setup
import json
import eeea.genetic.nsga2 as nsga2
import copy

result_acc_csv = []
result_params_csv = []
result_hash_csv = []
result_flops_csv = []
result_front_csv = []

parser = argparse.ArgumentParser("search")

# GA Config
parser.add_argument('--generations', type=int, default=30, help='generations')
parser.add_argument('--population', type=int, default=40, help='population')
parser.add_argument('--dataset', type=str, default='cifar10', choices=['cifar10', 'cifar100','imagenet'])
parser.add_argument('--search', type=str, default='ee', choices=['normal','ee', 'ultra'])
parser.add_argument('--search_type', type=str, default='normal', choices=['normal','progressive'])
parser.add_argument('--p_layers', type=int, default=2, help='progressive incresing every 2 layers')
parser.add_argument('--n_blocks', type=int, default=5, help='number of blocks in a cell')
parser.add_argument('--th_param', type=float, default=5.0, help='early exit')
parser.add_argument('--selection', type=str, default='tournament', choices=['random','tournament'])
parser.add_argument('--objective', type=str, default='multi', choices=['single','multi'])

parser.add_argument('--crossover_prob', type=float, default=0.1, help='crossover probability ')
parser.add_argument('--mutate_prob', type=float, default=0.1, help='mutate probability ')
parser.add_argument('--reject_prob', type=float, default=0.0, help='reject probability ')

parser.add_argument('--increment', type=int, default=6, help='filter increment')
parser.add_argument('--pyramid', action='store_true', default=True, help='pyramid')
parser.add_argument('--SE', action='store_true', default=True, help='enable SE')
parser.add_argument('--init_channels', type=int, default=36, help='init channels')
parser.add_argument('--init_channels_train', type=int, default=16, help='init channels')
parser.add_argument('--layers', type=int, default=8, help='layers')
parser.add_argument('--epochs', type=int, default=1, help='epochs to train')

# TinyDB
parser.add_argument('--load_db', action='store_true', default=True, help='load tinydb for fast search')

# God mode
parser.add_argument('--god_mode', action='store_true', default=True, help='god mode')

# Train Config

parser.add_argument('--data', type=str, default='datasets', help='location of the data corpus')
parser.add_argument('--tmp_data_dir', type=str, default='/home/mllab/proj/ILSVRC2015/Data/CLS-LOC', help='temp data dir')

parser.add_argument('--workers', type=int, default=8, help='number of workers to load dataset')
parser.add_argument('--batch_size_train', type=int, default=128, help='batch size')
parser.add_argument('--batch_size_val', type=int, default=128, help='batch size')
parser.add_argument('--learning_rate', type=float, default=0.025, help='init learning rate')
parser.add_argument('--momentum', type=float, default=0.9, help='momentum')
parser.add_argument('--weight_decay', type=float, default=5e-4, help='weight decay')
parser.add_argument('--gpu', type=int, default=0, help='gpu device id')
parser.add_argument('--auxiliary_weight', type=float, default=0.4, help='weight for auxiliary loss')
parser.add_argument('--cutout', action='store_true', default=True, help='enable cutout')
parser.add_argument('--cutout_length', type=int, default=16, help='cutout length')
parser.add_argument('--drop_path_prob', type=float, default=0.2, help='drop path probability')
parser.add_argument('--patience', type=int, default=5, help='patience of early stopping')

parser.add_argument('--auxiliary', action='store_true', default=True, help='enable auxiliary')

parser.add_argument('--activation', type=str, default='relu', choices=['relu'])
parser.add_argument('--grad_clip', type=float, default=5., help='gradient clipping')
parser.add_argument('--optimizer', type=str, default='sgd', choices=['sgd'])
parser.add_argument('--dropout_rate', type=float, default=0.3, help='dropout rate')
parser.add_argument('--autoaugment', action='store_true', default=False, help='enable Autoaugment')
parser.add_argument('--parallel', action='store_true', default=False, help='enable multi gpu parallel')
parser.add_argument('--mode', type=str, default='FP32', choices=['FP32', 'FP16'])
parser.add_argument('--seed', type=int, default=2020, help='seed random')
parser.add_argument('--save', type=str, default='Search', help='Search name')
parser.add_argument('--cutmix', action='store_true', default=False, help='use se')

args = parser.parse_args()

args.save = 'outputs/{}-{}-{}-{}-{}-{}'.format(args.save, args.objective, args.search, args.th_param, args.epochs, time.strftime("%Y%m%d-%H%M%S"))

utils.create_exp_dir(args.save, scripts_to_save=glob.glob('*.py'))

log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG,
    format=log_format, datefmt='%m/%d %I:%M:%S %p')
fh = logging.FileHandler(os.path.join(args.save, 'log.txt'))
fh.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(fh)

Genotype = namedtuple('Genotype', 'normal normal_concat reduce reduce_concat')

db_file = 'db/db_' + str(args.epochs) + ".json"
db = TinyDB(db_file)

best_genotypes = None

def train_networks(networks, generation):
    """Train each network.
    Args:
        networks (list): Current population of networks
        dataset (str): Dataset to use for training/evaluating
    """
    pbar = tqdm(total=len(networks))
    for network in networks:
        network.update_hash()

        if args.load_db == True:
            db_network = Query()
            match_network = db.search(db_network.hash == network.hash)

            if len(match_network) > 0:
                network.load_infomation(match_network[0]['accuracy'], match_network[0]['params'], match_network[0]['flops'])
                # Show Log
                print(match_network[0]['network'])
                network_mixed = nasnet_setup.decode_cell(match_network[0]['network'])
                logging.info(network_mixed)
                logging.info(match_network[0]['hash'])
                logging.info("param size = %fMB", match_network[0]['params'])
                logging.info('valid_acc %f', match_network[0]['accuracy'])

        if network.train_able:
            network.train(args, generation, db)

        pbar.update(1)
    pbar.close()

def train_one_networks(network, generation):
    network.update_hash()

    if args.load_db == True:
        db_network = Query()
        match_network = db.search(db_network.hash == network.hash)

        if len(match_network) > 0:
            network.load_infomation(match_network[0]['accuracy'], match_network[0]['params'], match_network[0]['flops'])
            # Show Log
            network_mixed = nasnet_setup.decode_cell(match_network[0]['network'])
            logging.info(network_mixed)
            logging.info(match_network[0]['hash'])
            logging.info("param size = %fMB", match_network[0]['params'])
            logging.info('valid_acc %f', match_network[0]['accuracy'])

    if network.train_able:
        network.train(args, generation, db)

def run_firefly(args, nn_param_choices, primitives):
    beta_init= 1
    beta_min=0.2
    gamma = 0.97

    max_bound = len(primitives)
    min_bound = 0
    alpha=0.2

    logging.info('init pop')

    optimizer = FAOptimizer(nn_param_choices, args)
    networks = optimizer.create_population()

    for generation in range(args.generations):
        logging.info('-'*80)
        logging.info("***Doing generation %d of %d***" %
                     (generation + 1, args.generations))
        logging.info('-'*80)

        train_networks(networks, generation)

        graded = [(fitness(args,network), network) for network in networks]
        population  = nsga2.RankAndCrowdingSurvival(graded , len(graded))

        alpha = _modify_alpha(args.generations, alpha)

        tmp_population = copy.deepcopy(population)
        for i in range(len(population)):
            for j in range(len(tmp_population)):
                if fitness(args, population[i]) < fitness(args, tmp_population[j]):
                    #print(population[j].hash)
                    #print(tmp_population[j].hash)
                    print(i,j)
                    pop_a = change_genome_to_index(population[i].network, primitives)
                    pop_b = change_genome_to_index(tmp_population[j].network, primitives)
                    problem_dim = len(pop_a)

                    r = math.sqrt(np.sum((pop_a - pop_b) ** 2))
                    beta = (beta_init - beta_min) * math.exp(-gamma * r ** 2) + beta_min
                    tmp = alpha * (np.random.random_sample((1, problem_dim))[0] - 0.5) * (max_bound - min_bound)
                    #tmp = alpha * (random_sample(problem_dim) - 0.5) * (max_bound - min_bound)
                    pop_tmp = check_position(pop_a * (1 - beta) + pop_b * beta + tmp, min_bound, max_bound)
                    pop_tmp = np.floor(pop_tmp).astype(int)

                    population[j].network = change_index_to_genome(pop_tmp, primitives)
                    population[j].update_hash()
                    population[j].train_able = True

                    print(population[j].network)

                    train_one_networks(population[j], generation)
                    #train_networks(population, generation)

        networks = population

        allacc, allparams, allhash, allflops, allfront = get_fitness_generation(networks, generation+1)

        result_acc_csv.append(allacc)
        result_params_csv.append(allparams)
        result_hash_csv.append(allhash)
        result_flops_csv.append(allflops)
        result_front_csv.append(allfront)

        save_population_json(networks, generation+1)

def fitness(args, network):
    if args.objective == 'single':
        return network.accuracy
    elif args.objective == 'multi':
        return ( 100-network.accuracy, network.params, network.flops)

def is_duplicate(genome, population):
    for i in range(0,len(population)):
        if (genome.hash == population[i].hash):
            return True
    return False

def check_position(position, min_bound, max_bound):
    position[position > max_bound] = max_bound
    position[position < min_bound] = min_bound
    return position

def _modify_alpha(generations, alpha):
    delta = 1 - (10 ** (-4) / 0.9) ** (1 / generations)
    alpha = (1 - delta) * alpha
    return alpha

def random_sample(size):
  return np.array(random.sample(range(100000000), k=size)) / 100000000

def change_genome_to_index(network, primitives):
  """
  Network -> "dict" {'0': 'dil_conv_5x5', '1': 0, '2': 'max_pool_3x3', '3': 0, ....
  Result ->  "list" [6, 0, 0, 0, 9, 0, 8, 0, 6, 2, 8, ......
  """
  result =[]
  for i in network:
      if int(i) % 2 == 0:
          network[i] = primitives.index(network[i])
  for i in network:
      result.append(network[i])
  return np.array(result)

def change_index_to_genome(network, primitives):
  """
  Network -> "list" [6, 0, 0, 0, 9, 0, 8, 0, 6, 2, 8, ......
  Result ->  "dict" {'0': 'dil_conv_5x5', '1': 0, '2': 'max_pool_3x3', '3': 0, ....
  """
  output = {}
  for i in range(len(network)):
    if i % 2 == 0:
      output[str(i)] = primitives[network[i]]
    else:
      output[str(i)] = network[i]
  return output

def save_population_json(networks, generation):
    json_db = TinyDB(os.path.join(args.save, 'data.json'))
    for network in networks:
        save_model = {
            "hash" : network.hash,
            "network" : network.network,
            "params"  : round(network.params, 2),
            "accuracy" : round(network.accuracy, 2),
            "flops" : int(network.flops),
            "generation" : generation,
            "front" : network.front,
            "father" : network.father,
            "mother" : network.mother,
        }
        json_db.insert(save_model)

def save_best_network(networks):
    if args.objective == 'single':
        best_network = sorted(networks, key=lambda x: x.accuracy, reverse=True)[0]
    elif args.objective == 'multi':
        best_network = networks[0]

    global best_genotypes
    best_genotypes = "G"+str(best_network.hash)

    # file_genotypes = open('genotypes.py', 'a')
    # file_genotypes.write("\nG"+str(best_network.hash) + " = " + str(nasnet_setup.decode_cell(best_network.network))+"\n")
    # file_genotypes.close()

    logging.info("Best Network")

    logging.info(nasnet_setup.decode_cell(best_network.network))
    logging.info(best_network.hash)
    logging.info('valid_acc %f', best_network.accuracy)
    logging.info('params %f', best_network.params)
    logging.info('flops %f', best_network.flops)

def get_fitness_generation(networks, generation):
    result_accuracy = [generation]
    result_params = [generation]
    result_hash = [generation]
    result_flops = [generation]
    result_front = [generation]

    if args.objective == 'single':
        sortedNetwork = sorted(networks, key=lambda x: x.accuracy, reverse=True)
    elif args.objective == 'multi':
        sortedNetwork = networks

    for network in sortedNetwork:
        result_accuracy.append(round(float(network.accuracy), 2))
        result_params.append(round(float(network.params), 2))
        result_hash.append(str(network.hash))
        result_flops.append(round(float(network.flops), 2))
        result_front.append(int(network.front))

    output_accuracy = [",".join(str(x) for x in result_accuracy)]
    output_params = [",".join(str(x) for x in result_params)]
    output_hash = [",".join(str(x) for x in result_hash)]
    output_flops = [",".join(str(x) for x in result_flops)]
    output_front = [",".join(str(x) for x in result_front)]
    return output_accuracy, output_params, output_hash, output_flops, output_front

def get_average_att_network(networks, select='accuracy'):
    """Get the average accuracy for a group of networks.
    Args:
        networks (list): List of networks
    Returns:
        float: The average accuracy of a population of networks.
    """
    total = 0
    for network in networks:
        if select == 'accuracy':
            total += network.accuracy
        elif select == 'params':
            total += network.params
        elif select == 'flops':
            total += network.flops

    return total / len(networks)

def print_networks(networks):
    """Print a list of networks.
    Args:
        networks (list): The population of networks
    """
    logging.info('-'*80)
    for network in networks:
        network.print_network()

def main():
    np.random.seed(args.seed)
    random.seed(args.seed)

    utils.save_seed(args.seed, 'db/seed_val.txt')

    assert args.crossover_prob * args.population > 2, "Please increase crossover prob or population"

    logging.info("args = %s", args)

    primitives = [
    'max_pool_3x3', 'avg_pool_3x3',
    'skip_connect', 'sep_conv_3x3',
    'sep_conv_5x5', 'dil_conv_3x3',
    'dil_conv_5x5', 'conv_7x1_1x7',
    'inv_res_3x3', 'inv_res_5x5',
    'inv_res_3x3_sh', 'inv_res_5x5_sh',
    'std_gn_3x3', 'std_gn_5x5',
    'mbconv_k3_t1', 'mbconv_k5_t1',
    'blur_pool_3x3'
    ]

    nas_setup = nasnet_setup.setup_NAS(args.n_blocks,len(primitives))

    nn_param_choices = nasnet_setup.create_param_choices(primitives, nas_setup)

    logging.info("***Evolving %d generations with population %d***" %
                 (args.generations, args.population))

    run_firefly(args, nn_param_choices, primitives)

if __name__ == '__main__':
    start= time.time()
    main()
    end = time.time()
    logging.info("Time = %fSec", (end - start))

    np.savetxt(os.path.join(args.save, 'result_acc.csv'), np.array(result_acc_csv), delimiter=',', header="Accuracy,Network", comments="", fmt='%s')
    np.savetxt(os.path.join(args.save, 'result_params.csv'), np.array(result_params_csv), delimiter=',', header="Params,Network", comments="", fmt='%s')
    np.savetxt(os.path.join(args.save, 'result_hash.csv'), np.array(result_hash_csv), delimiter=',', header="Hash,Network", comments="", fmt='%s')
    np.savetxt(os.path.join(args.save, 'result_flops.csv'), np.array(result_flops_csv), delimiter=',', header="Flops,Network", comments="", fmt='%s')
    np.savetxt(os.path.join(args.save, 'result_front.csv'), np.array(result_front_csv), delimiter=',', header="Front,Network", comments="", fmt='%s')

    #import train_cifar
    #train_cifar.main(best_genotypes)

max_stars_repo_name: chakkritte/EEEA-Net

max_issues_repo_name: chakkritte/EEEA-Net

max_forks_repo_name: chakkritte/EEEA-Net


text:
import paddle.v2 as paddle
import paddle.fluid as fluid
import os
import reader
import numpy as np
import load_model as load_model
from mobilenet_ssd import mobile_net

def train(train_file_list,
          val_file_list,
          data_args,
          learning_rate,
          batch_size,
          num_passes,
          model_save_dir='model',
          init_model_path=None):
    image_shape = [3, data_args.resize_h, data_args.resize_w]

    image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
    gt_box = fluid.layers.data(
        name='gt_box', shape=[4], dtype='float32', lod_level=1)
    gt_label = fluid.layers.data(
        name='gt_label', shape=[1], dtype='int32', lod_level=1)
    difficult = fluid.layers.data(
        name='gt_difficult', shape=[1], dtype='int32', lod_level=1)

    mbox_locs, mbox_confs, box, box_var = mobile_net(image, image_shape)
    nmsed_out = fluid.layers.detection_output(
        mbox_locs, mbox_confs, box, box_var, nms_threshold=0.45)
    loss_vec = fluid.layers.ssd_loss(mbox_locs, mbox_confs, gt_box, gt_label,
                                     box, box_var)
    loss = fluid.layers.nn.reduce_sum(loss_vec)

    map_eval = fluid.evaluator.DetectionMAP(
        nmsed_out,
        gt_label,
        gt_box,
        difficult,
        21,
        overlap_threshold=0.5,
        evaluate_difficult=False,
        ap_version='11point')
    map, accum_map = map_eval.get_map_var()

    test_program = fluid.default_main_program().clone(for_test=True)
    with fluid.program_guard(test_program):
        test_program = fluid.io.get_inference_program([loss, map, accum_map])

    optimizer = fluid.optimizer.DecayedAdagrad(
        learning_rate=fluid.layers.exponential_decay(
            learning_rate=learning_rate,
            decay_steps=40000,
            decay_rate=0.1,
            staircase=True),
        regularization=fluid.regularizer.L2Decay(0.0005), )

    opts = optimizer.minimize(loss)

    place = fluid.CUDAPlace(0)
    exe = fluid.Executor(place)
    exe.run(fluid.default_startup_program())

    #load_model.load_and_set_vars(place)
    train_reader = paddle.batch(
        reader.train(data_args, train_file_list), batch_size=batch_size)
    test_reader = paddle.batch(
        reader.test(data_args, val_file_list), batch_size=batch_size)
    feeder = fluid.DataFeeder(
        place=place, feed_list=[image, gt_box, gt_label, difficult])

    def test(pass_id):
        map_eval.reset(exe)
        test_map = None
        for _, data in enumerate(test_reader()):
            test_map = exe.run(test_program,
                               feed=feeder.feed(data),
                               fetch_list=[accum_map])
        print("Test {0}, map {1}".format(pass_id, test_map[0]))

    #print fluid.default_main_program()
    for pass_id in range(num_passes):
        map_eval.reset(exe)
        for batch_id, data in enumerate(train_reader()):
            loss_v, map_v, accum_map_v = exe.run(
                fluid.default_main_program(),
                feed=feeder.feed(data),
                fetch_list=[loss, map, accum_map])
            print(
                "Pass {0}, batch {1}, loss {2}, cur_map {3}, map {4}"
                .format(pass_id, batch_id, loss_v[0], map_v[0], accum_map_v[0]))
        test(pass_id)

        if pass_id % 10 == 0:
            model_path = os.path.join(model_save_dir, str(pass_id))
            print 'save models to %s' % (model_path)
            fluid.io.save_inference_model(model_path, ['image'], [nmsed_out],
                                          exe)

if __name__ == '__main__':
    data_args = reader.Settings(
        data_dir='./data',
        label_file='label_list',
        resize_h=300,
        resize_w=300,
        mean_value=[127.5, 127.5, 127.5])
    train(
        train_file_list='./data/trainval.txt',
        val_file_list='./data/test.txt',
        data_args=data_args,
        learning_rate=0.001,
        batch_size=32,
        num_passes=300)

max_stars_repo_name: jhtsing/models

max_issues_repo_name: jhtsing/models

max_forks_repo_name: jhtsing/models


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""DataGroup1D: work with 1D data sets (line scans)
===================================================

"""
### IMPORTS ###
import os, sys
import numpy as np
from datetime import datetime

# Larch
HAS_LARCH = False
try:
    import larch
    from larch import use_plugin_path, Group
    # Larch Plugins
    use_plugin_path('io')
    from columnfile import _read_ascii
    from larch_plugins.io import write_ascii
    use_plugin_path('wx')
    from plotter import _plot, _scatterplot, _plot_text
    use_plugin_path('math')
    from mathutils import _interp
    from fitpeak import fit_peak
    use_plugin_path('xafs')
    from xafsft import xftf, xftr, xftf_prep, xftf_fast, xftr_fast, ftwindow
    from pre_edge import pre_edge
    HAS_LARCH = True
except ImportError:
    pass

from ..io.specfile_reader import _str2rng as str2rng
from ..io.specfile_reader import spec_getmap2group, spec_getmrg2group
from .datagroup import DataGroup, _norm, MODNAME

# PyMca
HAS_PYMCA = False
try:
    from PyMca5.PyMcaGui import ScanWindow
    HAS_PYMCA = True
except ImportError:
    try:
        from PyMca import ScanWindow
        HAS_PYMCA = True
    except ImportError:
        pass

# DEBUG
if 'DEBUG' in globals():
    pass
else:
    DEBUG = False

class DataGroup1D(DataGroup):
    """1D version of DataGroup"""

    def __init__(self, kwsd=None, _larch=None):
        super(DataGroup1D, self).__init__(kwsd=kwsd, _larch=_larch)

    def read_ascii(self, fname, labels=None, sort=False, sort_column=0):
        """see 'read_ascii' in Larch"""
        return _read_ascii(fname, labels=labels,
                           sort=sort, sort_column=sort_column,
                           _larch=self._larch)

    def write_ascii_xy(self, fname, g, xattr='x', yattr='y', label=None):
        """write a two-columns ascii file

        Parameters
        ==========

        fname : str
                output file name

        xattr, yattr: str
                      attributes to use as X/Y columns to write
        """
        _x = getattr(self.gs[g], xattr)
        _y = getattr(self.gs[g], yattr)
        if label is None:
            try:
                _lab = self.gs[g].label
            except:
                _lab = 'Unknown'
        return write_ascii(fname, _x, _y, label=_lab, _larch=self._larch)

    def getxy(self, fname, xattr='x', yattr='y', scanlab=None, **kws):
        """load two colums ascii data """
        g = _read_ascii(fname, labels='{0} {1}'.format(xattr, yattr),
                        _larch=self._larch)
        g.norint = self.norint(getattr(g, yattr))#, x=getattr(g, xattr))
        g.normax = self.normax(getattr(g, yattr))
        g.label = str(scanlab)
        return g

    def getspecscan(self, fname, scans, scanlab=None, **kws):
        """load two colums data from SPEC file"""
        cntx = kws.get('cntx', self.kwsd['spec']['cntx'])
        cnty = kws.get('cnty', self.kwsd['spec']['cnty'])
        csig = kws.get('csig', self.kwsd['spec']['csig'])
        cmon = kws.get('cmon', self.kwsd['spec']['cmon'])
        csec = kws.get('csec', self.kwsd['spec']['csec'])
        norm = kws.get('norm', self.kwsd['spec']['norm'])
        action = kws.get('action', self.kwsd['spec']['action'])
        g = spec_getmrg2group(fname,
                              scans=scans,
                              cntx=cntx,
                              csig=csig,
                              cmon=cmon,
                              csec=csec,
                              norm=norm,
                              action=action,
                              _larch=self._larch)
        g.label = str(scanlab)
        return g

    def getcom(self, g, xattr='x', yattr='y'):
        """center of mass (com) for the given group and x,w attributes"""
        return np.average(getattr(self.gs[g], xattr), weights=getattr(self.gs[g], yattr))

    def norint(self, y, x=None):
        """simple normalization by area"""
        return _norm(y, norm="area", x=x)

    def normax(self, y):
        """simple normalization by maximum minus offset"""
        return _norm(y, norm="max-min")

    def norxafs(self, g, xattr='x', yattr='y', outattr=None, **kws):
        """XAFS normalization on a given group (g)

        Keyword arguments
        -----------------
        xattr : ['x'] attribute for the energy array
        yattr : ['y'] attribute for the mu array
        outattr : ['flat'] attribute for output normalize array

        **kws : [''self.kwsd[pre_edge][*]''] as in pre_edge()
        """
        e0 = kws.get('e0', self.kwsd['pre_edge']['e0'])
        pre1 = kws.get('pre1', self.kwsd['pre_edge']['pre1'])
        pre2 = kws.get('pre2', self.kwsd['pre_edge']['pre2'])
        norm1 = kws.get('norm1', self.kwsd['pre_edge']['norm1'])
        norm2 = kws.get('norm2', self.kwsd['pre_edge']['norm2'])
        nvict = kws.get('nvict', self.kwsd['pre_edge']['nvict'])
        nnorm = kws.get('nnorm', self.kwsd['pre_edge']['nnorm'])
        pre_edge(getattr(g, xattr), getattr(g, yattr), group=g,
                 nnorm=nnorm, nvict=nvict, pre1=pre1, pre2=pre2,
                 norm1=norm1, norm2=norm2, _larch=self._larch)
        if outattr == 'flat':
            return g.flat
        elif outattr == 'norm':
            return g.norm

    def xcalib(self, g, ref=0, method='com', set_attr=False):
        """find the x-shift to calibrate two spectra

        Keyword arguments
        -----------------
        g : spectrum to shift (index in datagroup)
        ref : reference spectrum (in datagroup)
        method : the method to perform this task
                 'com' -> overlap the center of mass
        set_attr : [False] write 'xcalib' attr for the calibrated group

        """
        if method == 'com':
            xmin = max(min(self.gs[g].x), min(self.gs[ref].x))
            xmax = min(max(self.gs[g].x), max(self.gs[ref].x))
            self.mkinterpxy(xmin=xmin, xmax=xmax)
            cmdiff = self.getcom(ref, xattr='xnew', yattr='ynew') - self.getcom(g, xattr='xnew', yattr='ynew')
            self.gs[g].x = self.gs[g].x + cmdiff
            print('{0}.x shifted by {1}'.format(self.gs[g].label, cmdiff))
            if set_attr:
                self.gs[g].xcalib = cmdiff
        else:
            pass

    def xshift(self, g, xshift, xattr='x'):
        """simply apply the shift to group g"""
        _x = getattr(self.gs[g], xattr)
        setattr(self.gs[g], xattr, _x + xshift)

    def mkinterpxy(self, ref=0, sel='*', **kws):
        """interpolate (xattr, yattr) to (xnew, ynew) on a selected list of
        groups (sel); xnew is taken from the reference group or as
        linear space -- xnew = np.linspace(xmin, xmax,
        (xmax-xmin)/xstep) -- if xmin,xmax,xstep are given as input

        Keyword arguments
        -----------------
        ref : reference group [0]
        sel : list of groups to interpolate ['*']
        xmin, xmax, xstep : extracted from ref group if not given
        kind : type of interpolation method

        Returns
        -------
        None -- output written to attributes: 'xnew', 'ynew'

        """
        xattr = kws.get('xattr', 'x')
        yattr = kws.get('yattr', 'y')
        xref = self.get(xattr, sel=[ref])[0]
        yref = self.get(yattr, sel=[ref])[0]
        xmin = kws.get('xmin', None)
        xmax = kws.get('xmax', None)
        xstep = kws.get('xstep', None)
        kind =  kws.get('kind', self.kwsd['interp']['kind'])
        if (xmin is None) or (xmax is None) or (xstep is None):
            xnew = xref
        else:
            if (xmin is None):
                xmin = xref.min()
            if (xmax is None):
                xmax = xref.max()
            if (xstep is None):
                xstep = min(np.diff(xref))
            xnew = np.linspace(xmin, xmax, (xmax-xmin)/xstep)
        if DEBUG:
            print('DEBUG: {0} interp, {1} to {2}, {3} xstep = {4} points'.format(kind, xmin, xmax, xstep, len(xnew)))
        self.selector(sel)
        for _n, _g in enumerate(self.gs_sel):
            try:
                setattr(_g, 'xnew', xnew)
                setattr(_g, 'ynew', _interp(getattr(_g, xattr), getattr(_g, yattr), xnew, kind=kind))
                if DEBUG:
                    print('DEBUG: group {0} interpolated'.format(_n))
            except AttributeError:
                pass

    def mksum(self, sel, **kws):
        """make a new group as sum of selected groups

        Keyword arguments
        -----------------
        sel : an ordered list of selected groups
              sel[0] is the reference for the interpolation
        xattr : attribute for x-axis ['x']
        yattr : attribute for y-axis ['y']
        label : the label for the new sum group ['sum_sel_indx']

        Returns
        -------
        None -- output written to given attributes: 'xattr', 'yattr'
                of a new group in datagroup
        """
        iref = sel[0]
        xattr = kws.get('xattr', 'x')
        yattr = kws.get('yattr', 'y')
        xref = self.get(xattr, sel=[iref])[0]
        yref = self.get(yattr, sel=[iref])[0]
        label = kws.get('label', 'sum_of_{}'.format(sel))
        self.gs.append(Group())
        gsum = self.gs[-1]
        gsum.label = label
        self.mkinterpxy(ref=iref, sel=sel, xattr=xattr, yattr=yattr)
        for s in sel:
            if (s == iref):
                setattr(gsum, str(xattr), self.gs[s].xnew)
                ysum = self.gs[s].ynew
            else:
                ysum = ysum + self.gs[s].ynew
        setattr(gsum, str(yattr), ysum)

    def plotxy(self, **kws):
        """wrap to plot() in Larch (if available) otherwise PyMca is
        used (if available)

        Keyword arguments
        -----------------
        sel : a list of selected groups
        xattr : attribute for x-axis
        yattr : attribute for y-axis
        replace : True acts as newplot()
        norm : type of normalization to apply
               'area', 'max-min', 'xflat'
        **kws : as in plot()
        """
        # init keyword arguments with educated guesses
        sel = kws.get('sel', self.sel)
        xattr = kws.get('xattr', self.kwsd['plot']['xattr'])
        yattr = kws.get('yattr', self.kwsd['plot']['yattr'])
        norm =  kws.get('norm', self.kwsd['plot']['norm'])
        replace = kws.get('replace', self.kwsd['plot']['replace'])
        win = kws.get('win', self.kwsd['plot']['win'])
        title = kws.get('title', self.kwsd['plot']['title'])
        show_legend = kws.get('show_legend', self.kwsd['plot']['show_legend'])
        legend_loc = kws.get('legend_loc', self.kwsd['plot']['legend_loc'])
        xlabel = kws.get('xlabel', self.kwsd['plot']['xlabel'])
        ylabel = kws.get('ylabel', self.kwsd['plot']['ylabel'])
        xmin = kws.get('xmin', self.kwsd['plot']['xmin'])
        xmax = kws.get('xmax', self.kwsd['plot']['xmax'])
        ymin = kws.get('ymin', self.kwsd['plot']['ymin'])
        ymax = kws.get('ymax', self.kwsd['plot']['ymax'])
        xshift = kws.get('xshift', self.kwsd['plot']['xshift'])
        ystack = kws.get('ystack', self.kwsd['plot']['ystack'])
        xscale = kws.get('xscale', self.kwsd['plot']['xscale'])
        yscale = kws.get('yscale', self.kwsd['plot']['yscale'])
        ###
        if (HAS_PYMCA and ( not self._inlarch )):
            if not hasattr(self, 'pw'):
                self.pw = ScanWindow.ScanWindow()
                # geometry good for >1280x800 resolution
                self.pw.setGeometry(50, 50, 700, 700)
                self.pw.show()
            if title:
                self.pw.setGraphTitle(title)
            if xlabel:
                self.pw.setGraphXTitle(xlabel)
            if ylabel:
                self.pw.setGraphYTitle(ylabel)
            if (xmin and xmax):
                self.pw.setGraphXLimits(xmin, xmax)
            if (ymin and ymax):
                print(ymin, ymax)
                self.pw.setGraphYLimits(ymin, ymax)
            # force replot
            self.pw.replot()
        if sel == '*':
            print('Plotting all...')
            self.show()
            sel = range(len(self.gs))
        if replace:
            _n = 0
        else:
            _n = 1
        _m = 0
        for _s in sel:
            # assign x,y
            x = getattr(self.gs[_s], xattr)
            y = getattr(self.gs[_s], yattr)
            # normalize y
            if norm == 'area':
                y = self.norint(y, x=x)
            elif norm == 'max-min':
                y = self.normax(y)
            elif norm == 'xflat':
                y = self.norxafs(self.gs[_s], outattr='flat', **kws)
            elif norm == 'xnorm':
                y = self.norxafs(self.gs[_s], outattr='norm', **kws)
            # plot
            if _n == 0:
                if self._inlarch:
                    _plot(x * xscale + xshift,
                          y * yscale + ystack * _m,
                          label=self.gs[_s].label, title=title, win=win,
                          xlabel=xlabel, ylabel=ylabel,
                          xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
                          show_legend=show_legend, legend_loc=legend_loc,
                          new=True, _larch=self._larch)
                if (HAS_PYMCA and ( not self._inlarch )):
                    self.pw.addCurve(x * xscale + xshift,
                                     y * yscale + ystack * _m,
                                     legend=self.gs[_s].label,
                                     replace=True)
                _n += 1
                _m += 1
            else:
                if self._inlarch:
                    _plot(x * xscale + xshift,
                          y * yscale + ystack * _m,
                          label=self.gs[_s].label, title=title, win=win,
                          show_legend=show_legend, legend_loc=legend_loc,
                          new=False, _larch=self._larch)
                if (HAS_PYMCA and ( not self._inlarch )):
                    self.pw.addCurve(x * xscale + xshift,
                                     y * yscale + ystack * _m,
                                     legend=self.gs[_s].label,
                                     replace=False)
                _m += 1

### LARCH ###
def datagroup1D(kwsd=None, _larch=None):
    """utility to perform wrapped operations on a list of 1D data
    groups"""
    return DataGroup1D(kwsd=kwsd, _larch=_larch)

def registerLarchPlugin():
    return (MODNAME, {'datagroup1D' : datagroup1D})

if __name__ == '__main__':
    pass

max_stars_repo_name: maurov/xraysloth

max_issues_repo_name: maurov/xraysloth

max_forks_repo_name: maurov/xraysloth


text:
from collections import Counter
from functools import reduce
from pathlib import Path

import numpy as np
import torch
import torchaudio
from fastai.core import listify, warnings

def _channel_first(sig):
    """Use the hack that maximum channel number is 5"""
    return sig.T if sig.shape[1] > 5 else sig

def _signal_first(sig):
    """Use the hack that maximum channel number is 5"""
    return sig.T if sig.shape[0] > 5 else sig

def _to_numpy(array):
    if isinstance(array, torch.Tensor):
        res = array.numpy()
    elif isinstance(array, np.ndarray):
        res = array
    else:
        raise TypeError(f'conversion unsupported for {type(array)}')
    return res

def one_hot_ndarray(signal, n_classes):
    assert(signal.ndim == 1)
    res = np.zeros(shape=(n_classes, signal.size), dtype=np.float32)
    res[signal, np.arange(signal.size)] = 1.0
    return res

def one_hot_decode(tens:torch.Tensor, axis=-2, default_value=None):
    """A function that converts a one-hot encoded tensor into an array pointing to the indice with maximum value in each
    axis. Reverse to one hot encoding.
    @axis indicates the axis that was unrolled to produce one-hot encoding. By default its one-before-the-last axis.
    @default_value is used if the tensor is not properly one-hot-encoded, meaning that there isn't a single maximum value
    alongside given axis. By default it's tens.shape[axis]"""
    # TODO check this for whole batch
    maxes, indices = torch.max(tens, dim=axis)
    # mins, mindices = torch.min(tens, dim=axis)
    # how many times maxvalue appears along given row
    maxval_repetitions = (tens >= maxes).sum(dim=axis)
    if (mask := maxval_repetitions > 1).any():
        warnings.warn(f'Tensor is not correctly one hot encoded, some max values repeat along chosen axis. '
                      f'Defaulting to {default_value}')
        if default_value is not None: indices.masked_fill_(mask, default_value)
    # below tests if there is no equal items along chosen axis. Number of zeros in an array should equal number of maxes
    # if len((tens - maxes.unsqueeze(axis)).nonzero()) != tens.numel()-maxes.numel():
    #     pass
    return indices

def one_hot_tensor(signal, n_classes):
    assert(signal.ndim<4), 'Tensor shape not supported'
    res = torch.zeros(size=(n_classes, signal.numel()), dtype=torch.float, device=signal.device)
    res.scatter_(0, signal.view(1,-1).long(), 1.0)
    return res

def sequences(tens, include_edges=False, c2i: dict = None, return_counter=False, ignore_values=None):
    """Calculate the number of consecutive values in a tensor signal.
    If c2i mapping is provided, silence signal is ignored in stats.
    :param return_counter: """
    tens = tens.squeeze()
    if tens.ndim == 2:
        tens = one_hot_decode(tens, default_value=ignore_values)
    elif tens.ndim == 1:
        pass
    else:
        raise NotImplementedError('Unknown tensor shape')
    if include_edges:
        i, f = None, None
    else:
        i, f = 1, -1
    if ignore_values is not None:
        ignore_values = listify(ignore_values)
        if tens[0] in ignore_values: i = None
        if tens[-1] == ignore_values: f = None
        filter_tensors = (tens != val for val in ignore_values)
        filter = reduce(torch.mul, filter_tensors)
        tens = tens[filter]

    # TODO consider moving this to reconstruct
    s = ((tens[1:] - tens[:-1]).nonzero().squeeze() + 1).view(-1)
    split = s - torch.nn.functional.pad(s[:-1], (1, 0))
    chunks = torch.split(tens, [*split.tolist(), len(tens) - split.sum()])
    if return_counter:
        counter = [list(*Counter(chunk.tolist()).items()) for chunk in chunks[i:f]]
        return counter
    lengths = np.array([t.numel() for t in chunks[i:f]])
    # need to filter out zeros, which result from padding of empty tensor
    return lengths[lengths > 0]

def get_json(path:Path):
    path = Path(path)
    return path.parent / (path.stem + '_annot.json')

def get_file_info(p):
    if isinstance(p, Path): p = p.as_posix()
    signal_info, _ = torchaudio.info(p)
    return signal_info


text:
import collections
import logging
import os

import torch

from torch.utils.data import DataLoader

import criteria
from dataloaders.clip_loader import get_dataset
from models_.gcn_final import Model
# from models_.latent_attention import Model
from optimizer.adam_optimizer import AdamOptimizer
from optimizer.lr_scheduler.inverse_square_root_schedule import InverseSquareRootSchedule
from utils import load_word2vec, AverageMeter, TimeMeter, CountMeter
import numpy as np

class Runner:
    def __init__(self, args):
        self.num_updates = 0
        self.args = args
        self.word2vec = load_word2vec(args.word2vec_path)
        self._build_loader()
        self._build_model()
        self._build_optimizer()

    def _build_loader(self):
        args = self.args
        train = get_dataset(args.dataset, args.feature_path, args.train_data,
                            self.word2vec, args.max_num_frames, args.max_num_words,
                            args.max_num_nodes, is_training=True)
        val = get_dataset(args.dataset, args.feature_path, args.val_data,
                          self.word2vec, args.max_num_frames, args.max_num_words,
                          args.max_num_nodes, is_training=False)
        test = get_dataset(args.dataset, args.feature_path, args.test_data,
                           self.word2vec, args.max_num_frames, args.max_num_words,
                           args.max_num_nodes, is_training=False)
        self.train_loader = DataLoader(dataset=train, batch_size=self.args.batch_size, num_workers=4, shuffle=True)
        self.val_loader = DataLoader(dataset=val, batch_size=self.args.batch_size, num_workers=4,
                                     shuffle=False) if val else None
        self.test_loader = DataLoader(dataset=test, batch_size=self.args.batch_size, num_workers=4,
                                      shuffle=False) if test else None

    def _build_model(self):
        self.model = Model(self.args)
        print(self.model)
        device_ids = [0]
        # self.inference = self.model.inference
        self.model = self.model.to(torch.device('cuda:%d' % device_ids[0]))
        self.model = torch.nn.DataParallel(self.model, device_ids=device_ids)
        if self.args.model_load_path:
            self.model.load_state_dict(torch.load(self.args.model_load_path))

    def _build_optimizer(self):
        l = list(self.model.parameters())
        self.optimizer = AdamOptimizer(self.args, list(self.model.parameters()))
        self.lr_scheduler = InverseSquareRootSchedule(self.args, self.optimizer)

    def train(self):
        if not os.path.exists(self.args.model_saved_path):
            # os.mkdir(self.args.model_saved_path)
            os.makedirs(self.args.model_saved_path)
        for epoch in range(1, self.args.max_num_epochs + 1):
            logging.info('Start Epoch {}'.format(epoch))
            self._train_one_epoch(epoch)
            path = os.path.join(self.args.model_saved_path, 'model-%d' % epoch)
            torch.save(self.model.state_dict(), path)
            logging.info('model saved to %s' % path)
            self.eval()
        logging.info('Done.')

    def _train_one_epoch(self, epoch):
        self.model.train()
        loss_meter = AverageMeter()
        time_meter = TimeMeter()
        for bid, (video, video_mask, words, word_mask,
                  label, scores, scores_mask, id2pos, node_mask, adj_mat) in enumerate(self.train_loader, 1):
            self.optimizer.zero_grad()

            model_input = {
                'frames': video.cuda(),
                'frame_mask': video_mask.cuda(), 'words': words.cuda(), 'word_mask': word_mask.cuda(),
                'label': scores.cuda(), 'label_mask': scores_mask.cuda(), 'gt': label.cuda(),
                'node_pos': id2pos.cuda(), 'node_mask': node_mask.cuda(), 'adj_mat': adj_mat.cuda()
            }

            predict_boxes, loss, _, _, _ = self.model(**model_input)
            loss = torch.mean(loss)
            self.optimizer.backward(loss)

            self.optimizer.step()
            self.num_updates += 1
            curr_lr = self.lr_scheduler.step_update(self.num_updates)

            loss_meter.update(loss.item())
            time_meter.update()

            if bid % self.args.display_n_batches == 0:
                logging.info('Epoch %d, Batch %d, loss = %.4f, lr = %.5f, %.3f seconds/batch' % (
                    epoch, bid, loss_meter.avg, curr_lr, 1.0 / time_meter.avg
                ))
                loss_meter.reset()

    def eval(self):
        data_loaders = [self.val_loader, self.test_loader]
        meters = collections.defaultdict(lambda: AverageMeter())

        self.model.eval()
        with torch.no_grad():
            for data_loader in data_loaders:
                for bid, (video, video_mask, words, word_mask,
                          label, scores, scores_mask, id2pos, node_mask, adj_mat) in enumerate(data_loader, 1):
                    self.optimizer.zero_grad()

                    model_input = {
                        'frames': video.cuda(),
                        'frame_mask': video_mask.cuda(), 'words': words.cuda(), 'word_mask': word_mask.cuda(),
                        'label': scores.cuda(), 'label_mask': scores_mask.cuda(), 'gt': label.cuda(),
                        'node_pos': id2pos.cuda(), 'node_mask': node_mask.cuda(), 'adj_mat': adj_mat.cuda()
                    }

                    predict_boxes, loss, _, _, _ = self.model(**model_input)
                    loss = torch.mean(loss)

                    meters['loss'].update(loss.item())
                    video_mask = video_mask.cpu().numpy()
                    gt_boxes = model_input['gt'].cpu().numpy()
                    predict_boxes = np.round(predict_boxes.cpu().numpy()).astype(np.int32)
                    gt_starts, gt_ends = gt_boxes[:, 0], gt_boxes[:, 1]
                    predict_starts, predict_ends = predict_boxes[:, 0], predict_boxes[:, 1]
                    predict_starts[predict_starts < 0] = 0
                    seq_len = np.sum(video_mask, -1)
                    predict_ends[predict_ends >= seq_len] = seq_len[predict_ends >= seq_len] - 1
                    IoUs = criteria.calculate_IoU_batch((predict_starts, predict_ends),
                                                        (gt_starts, gt_ends))
                    meters['mIoU'].update(np.mean(IoUs), IoUs.shape[0])
                    for i in range(1, 10, 2):
                        meters['IoU@0.%d' % i].update(np.mean(IoUs >= (i / 10)), IoUs.shape[0])
                print('| ', end='')
                for key, value in meters.items():
                    print('{}, {:.4f}'.format(key, value.avg), end=' | ')
                    meters[key].reset()
                print()

    def eval_new(self):
        data_loaders = [self.test_loader]
        meters = collections.defaultdict(lambda: AverageMeter())
        meters_5 = collections.defaultdict(lambda: CountMeter())

        self.model.eval()
        with torch.no_grad():
            for data_loader in data_loaders:
                for bid, (video, video_mask, words, word_mask,
                          label, scores, scores_mask, id2pos, node_mask, adj_mat) in enumerate(data_loader, 1):
                    self.optimizer.zero_grad()

                    model_input = {
                        'frames': video.cuda(),
                        'frame_mask': video_mask.cuda(), 'words': words.cuda(), 'word_mask': word_mask.cuda(),
                        'label': scores.cuda(), 'label_mask': scores_mask.cuda(), 'gt': label.cuda(),
                        'node_pos': id2pos.cuda(), 'node_mask': node_mask.cuda(), 'adj_mat': adj_mat.cuda()
                    }

                    predict_boxes, loss, predict_flatten_old, _, _ = self.model(**model_input)
                    loss = torch.mean(loss)

                    meters['loss'].update(loss.item())
                    video_mask_old = video_mask.cpu().numpy()
                    gt_boxes_old = model_input['gt'].cpu().numpy()
                    predict_boxes_old = np.round(predict_boxes.cpu().numpy()).astype(np.int32)
                    for k in range(predict_boxes.shape[0]):
                        gt_boxes = gt_boxes_old[k]
                        predict_boxes = predict_boxes_old[k]
                        video_mask = video_mask_old[k]
                        predict_flatten = predict_flatten_old[k]
                        gt_starts, gt_ends = gt_boxes[0], gt_boxes[1]
                        predict_starts, predict_ends = predict_boxes[:, 0], predict_boxes[:, 1]
                        predict_starts[predict_starts < 0] = 0
                        seq_len = np.sum(video_mask, -1)
                        # if self.args.dataset == 'ActivityNet':
                        #     seq_len = np.tile(seq_len,(1400,1))
                        #     # seq_len = seq_len[adj_mat.cpu().numpy()>0]
                        #     gt_boxes = np.tile(gt_boxes,(1400,1))
                        #     # gt_boxes = gt_boxes[adj_mat.cpu().numpy()>0]
                        # elif dataset == 'TACOS':
                        #     seq_len = np.tile(seq_len,(800,1)).transpose(1,0)
                        #     seq_len = seq_len[adj_mat.cpu().numpy()>0]
                        #     gt_boxes = np.tile(gt_boxes,(800,1,1)).transpose(1,0,2)
                        #     gt_boxes = gt_boxes[adj_mat.cpu().numpy()>0]
                        predict_ends[predict_ends >= seq_len] = seq_len - 1
                        # IoUs = criteria.calculate_IoU_batch((predict_starts, predict_ends),
                        #                                     (gt_starts, gt_ends))
                        # meters['mIoU'].update(np.mean(IoUs), IoUs.shape[0])
                        predict_flatten = predict_flatten.cpu().numpy()
                        predict_boxes[:, 0], predict_boxes[:, 1] = predict_starts, predict_ends

                        #index = np.argpartition(predict_flatten,-10)[-10:]

                        topn_IoU_matric = criteria.compute_IoU_recall(predict_flatten, predict_boxes, gt_boxes)
                        meters_5['mIoU'].update(topn_IoU_matric, 1)
                    # for i in range(1, 10, 2):
                    #     meters['IoU@0.%d' % i].update(np.mean(IoUs >= (i / 10)), IoUs.shape[0])
                print('| ', end='')
                # for key, value in meters.items():
                #     print('{}, {:.4f}'.format(key, value.avg), end=' | ')
                #     meters[key].reset()
                print('---------------')
                IoU_threshs = [0.1, 0.3, 0.5, 0.7]
                top_n_list = [1,5]
                topn_IoU_matric, count = meters_5['mIoU'].val, meters_5['mIoU'].count
                for i in range(2):
                    for j in range(4):
                        print('{}, {:.4f}'.format('IoU@'+str(top_n_list[i])+'@'+str(IoU_threshs[j]), topn_IoU_matric[i,j]/count), end=' | ')
                meters_5['mIoU'].reset()
                print()
    def eval_save(self):
        data_loaders = [self.test_loader]
        meters = collections.defaultdict(lambda: AverageMeter())
        time_meter = TimeMeter()
        f = open('./our.txt','w')
        self.model.eval()
        with torch.no_grad():
            for data_loader in data_loaders:
                for bid, (video, video_mask, words, word_mask,
                          label, scores, scores_mask, id2pos, node_mask, adj_mat) in enumerate(data_loader, 1):
                    self.optimizer.zero_grad()

                    model_input = {
                        'frames': video.cuda(),
                        'frame_mask': video_mask.cuda(), 'words': words.cuda(), 'word_mask': word_mask.cuda(),
                        'label': scores.cuda(), 'label_mask': scores_mask.cuda(), 'gt': label.cuda(),
                        'node_pos': id2pos.cuda(), 'node_mask': node_mask.cuda(), 'adj_mat': adj_mat.cuda()
                    }

                    predict_boxes, loss, _, a1, a2 = self.model(**model_input)
                    loss = torch.mean(loss)
                    time_meter.update()
                    if bid % self.args.display_n_batches == 0:
                        logging.info('%.3f seconds/batch' % (
                            1.0 / time_meter.avg
                        ))
                    meters['loss'].update(loss.item())
                    a1, a2 = a1.cpu().numpy(), a2.cpu().numpy()
                    np.save('a1.npy',a1)
                    np.save('a2.npy',a2)
                    video_mask = video_mask.cpu().numpy()
                    gt_boxes = model_input['gt'].cpu().numpy()
                    predict_boxes = np.round(predict_boxes.cpu().numpy()).astype(np.int32)
                    gt_starts, gt_ends = gt_boxes[:, 0], gt_boxes[:, 1]
                    predict_starts, predict_ends = predict_boxes[:, 0], predict_boxes[:, 1]
                    predict_starts[predict_starts < 0] = 0
                    seq_len = np.sum(video_mask, -1)
                    predict_ends[predict_ends >= seq_len] = seq_len[predict_ends >= seq_len] - 1
                    IoUs = criteria.calculate_IoU_batch((predict_starts, predict_ends),
                                                        (gt_starts, gt_ends))
                    for kk in range(predict_starts.shape[0]):
                        f.write('IoU: '+str(IoUs[kk])+' start: '+str(predict_starts[kk])+' ends: '+str(predict_ends[kk])+' gt: '+str(gt_starts[kk])+' '+str(gt_ends[kk])+'\n')
                    meters['mIoU'].update(np.mean(IoUs), IoUs.shape[0])
                    for i in range(1, 10, 2):
                        meters['IoU@0.%d' % i].update(np.mean(IoUs >= (i / 10)), IoUs.shape[0])
                if data_loaders.index(data_loader) == 0:
                    print('--------val')
                else:
                    print('--------test')
                print('| ', end='')
                for key, value in meters.items():
                    print('{}, {:.4f}'.format(key, value.avg), end=' | ')
                    meters[key].reset()
                print()

max_stars_repo_name: liudaizong/IA-Net

max_issues_repo_name: liudaizong/IA-Net

max_forks_repo_name: liudaizong/IA-Net


text:
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/detector.ipynb (unless otherwise specified).

__all__ = ['FormatDetector']

# Cell
import numpy as np
import pandas as pd

from typing import Any, Dict, Callable
from tqdm import tqdm

from .judge import FormatJudge
from .utils import PatternGenerator

# Cell
class FormatDetector:
    def __init__(self, skip: list = None):
        self.skip = skip

    def fit(self, df: pd.DataFrame, generator: PatternGenerator or List['str': PatternGenerator],
            n: int = 3, dim: int = 1):
        self.judges = {}

        self.df = df
        with tqdm(total=len(self.df.columns)) as pbar:
            if isinstance(generator, PatternGenerator):
                for col in self.df.columns:
                    if col in self.skip:
                        continue
                    col_values = self.df[col].tolist()
                    format_judge = FormatJudge(generator, n, dim)
                    format_judge.fit(col_values)
                    self.judges[col] = format_judge
                    pbar.update(1)
            else:
                for col in self.df.columns:
                    if col in self.skip:
                        continue
                    col_values = self.df[col].tolist()
                    gen = generator.get(col, PatternGenerator())
                    format_judge = FormatJudge(gen, n, dim)
                    format_judge.fit(col_values)
                    self.judges[col] = format_judge
                    pbar.update(1)

    def detect(self, reduction: Callable = np.min, softmax: bool = True) -> dict:
        scores = []

        with tqdm(total=len(self.df)) as pbar:
            for index, row in self.df.iterrows():
                tuple_score = []
                for col in self.df.columns:
                    if col in self.skip:
                        continue
                    judge = self.judges[col]
                    score = np.mean(judge(row[col]))
                    tuple_score.append(score)
                if softmax:
                    tuple_score = np.exp(tuple_score)
                    softmax_tuple_score = [score / sum(tuple_score) for score in tuple_score]
                    if reduction == np.ptp:
                        scores.append(reduction(softmax_tuple_score))
                    else:
                        scores.append(1 - reduction(softmax_tuple_score))
                else:
                    if reduction == np.ptp:
                        scores.append(reduction(tuple_score))
                    else:
                        scores.append(1 - reduction(tuple_score))
                pbar.update(1)
        assessed_df = self.df.copy()
        assessed_df['p'] = scores
        return assessed_df

max_stars_repo_name: dpoulopoulos/forma

max_issues_repo_name: dpoulopoulos/forma

max_forks_repo_name: dpoulopoulos/forma


text:
from typing import Type, Any, Callable, Union, List, Optional
import torch
import torch.nn as nn
import torchvision.models.resnet
import torch.nn.modules.rnn
from torch import Tensor
import scipy.io as scio

def conv3x3_2d(inplanes: int,
               outplanes: int,
               stride: int = 1) -> nn.Conv2d:
    """3x3二维卷积"""
    return nn.Conv2d(
        inplanes,
        outplanes,
        kernel_size=(3, 3),
        stride=(stride, stride),
        padding=1,
        bias=True
    )

def conv1x1_2d(inplanes: int,
               outplanes: int,
               stride: int = 1) -> nn.Conv2d:
    """1x1二维卷积，用于短接时降采样"""
    return nn.Conv2d(
        inplanes,
        outplanes,
        kernel_size=(1, 1),
        stride=(stride, stride),
        padding=0,
        bias=False
    )

def conv3x3_1d(inplanes: int,
               outplanes: int,
               stride: int = 1) -> nn.Conv1d:
    """3x3一维卷积"""
    return nn.Conv1d(
        inplanes,
        outplanes,
        kernel_size=(3,),
        stride=(stride,),
        padding=1,
        bias=True)

def conv1x1_1d(inplanes: int,
               outplanes: int,
               stride: int = 1) -> nn.Conv1d:
    """1x1一维卷积，用于短接时降采样"""
    return nn.Conv1d(
        inplanes,
        outplanes,
        kernel_size=(1,),
        stride=(stride,),
        padding=0,
        bias=False
    )

class BasicBlock2D(nn.Module):
    """二维基本残差块"""

    def __init__(self,
                 inplanes: int,
                 planes: int,
                 stride: int = 1) -> None:

        super(BasicBlock2D, self).__init__()
        # 每次经过此块，通道的扩增倍数
        self.expansion = 1

        self.norm_layer = nn.BatchNorm2d
        self.conv1 = conv3x3_2d(inplanes, planes, stride)
        self.bn1 = self.norm_layer(planes)
        # inplace=True, 改变输入数据，节省申请内存开销
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3_2d(planes, planes, stride=1)
        self.bn2 = self.norm_layer(planes)

        # 降采样匹配通道数
        self.downsample = None

        if stride != 1 or inplanes != planes:
            self.downsample = nn.Sequential(
                conv1x1_2d(inplanes, planes, stride),
                self.norm_layer(planes)
            )

    def forward(self, x: Tensor) -> Tensor:
        identity = x

        # 第一层卷积
        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)
        # 第二层卷积
        out = self.conv2(out)
        out = self.bn2(out)
        # 降采样以维度匹配：尺寸和通道
        if self.downsample is not None:
            identity = self.downsample(x)
        # 短接
        out += identity
        out = self.relu(out)

        return out

class BasicBlock1D(nn.Module):
    """一维基本残差块"""

    def __init__(self,
                 inplanes: int,
                 planes: int,
                 stride: int = 1) -> None:
        super(BasicBlock1D, self).__init__()

        self.expansion = 1

        self.norm_layer = nn.BatchNorm1d
        self.conv1 = conv3x3_1d(inplanes, planes, stride)
        self.bn1 = self.norm_layer(planes)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3_1d(planes, planes, stride=1)
        self.bn2 = self.norm_layer(planes)

        # 降采样匹配通道数
        self.downsample = None

        if stride != 1 or inplanes != planes:
            self.downsample = nn.Sequential(
                conv1x1_1d(inplanes, planes, stride),
                self.norm_layer(planes)
            )

    def forward(self, x: Tensor) -> Tensor:
        identity = x

        # 第一层卷积
        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)
        # 第二层卷积
        out = self.conv2(out)
        out = self.bn2(out)
        # 降采样以维度匹配：尺寸和通道
        if self.downsample is not None:
            identity = self.downsample(x)
        # 短接
        out += identity
        out = self.relu(out)

        return out

class ResnetSP(nn.Module):
    def __init__(self, num_classes: int) -> None:
        super(ResnetSP, self).__init__()

        self.block1_2d = BasicBlock2D(1, 1, 1)
        self.conv1_2d = nn.Conv2d(in_channels=1, out_channels=2, kernel_size=(4, 4), stride=(1, 1), padding=0,
                                  bias=False)

        self.dimension_squeeze = torch.flatten

        self.block1_1d = BasicBlock1D(inplanes=2, planes=2, stride=1)
        self.block2_1d = BasicBlock1D(inplanes=2, planes=4, stride=1)
        self.avgpool1 = nn.AdaptiveAvgPool1d(18)
        self.block3_1d = BasicBlock1D(inplanes=4, planes=4, stride=1)
        self.block4_1d = BasicBlock1D(inplanes=4, planes=8, stride=2)
        self.block5_1d = BasicBlock1D(inplanes=8, planes=8, stride=1)
        self.block6_1d = BasicBlock1D(inplanes=8, planes=16, stride=1)

        self.avgpool2 = nn.AdaptiveAvgPool1d(3)
        self.fc = nn.Linear(3*16, num_classes)

    def forward(self, x: Tensor) -> Tensor:
        # C H W
        # 1x40x4
        out = self.block1_2d(x)
        # 2x37x1
        out = self.conv1_2d(out)
        # C W
        # 维度压缩，2x37
        out = self.dimension_squeeze(out, 2)
        # 2x37
        out = self.block1_1d(out)
        # 4x37
        out = self.block2_1d(out)
        # 平均池化，4x18
        out = self.avgpool1(out)
        # 4x18
        out = self.block3_1d(out)
        # 8x9
        out = self.block4_1d(out)
        # 8x9
        out = self.block5_1d(out)
        # 16x9
        out = self.block6_1d(out)
        # 平均池化，16x3
        out = self.avgpool2(out)
        # 维度压缩，1x48
        out = torch.flatten(out, 1)
        # 全连接，num_classes
        out = self.fc(out)

        return out

if __name__ == '__main__':
    model = ResnetSP(9)

    datafile = 'datas/sample_nor.mat'
    data = scio.loadmat(datafile)
    data = data['sample_nor']
    sample = data[0, 0]
    print(sample.shape)

    x = torch.from_numpy(sample)
    print(x.shape)
    x = torch.unsqueeze(x, 0)
    x = torch.unsqueeze(x, 0)
    x = x.float()
    print(x.dtype)

    ans = model.forward(x)
    print(ans.shape)
    print(ans)

max_stars_repo_name: HahhForest/PV-FDD

max_issues_repo_name: HahhForest/PV-FDD

max_forks_repo_name: HahhForest/PV-FDD

reason: import scipy


text:
#!/usr/bin/env python

import io
import numpy as np
from config import *
from get_tile_data import get_tile_data

class InputRead(caffe.Layer):

    def initialize(self):
        self.batch_size = 100
        self.channels = 3
        self.height = 64
        self.width = 64
        self.data_type = 'TRAIN'

        self.num_tops = 2
        self.top_names = ['data', 'label']
        self.top_channels = [3, 1]

        self.data = None
        self.label = None

        self.idx = 0
        self.num_images = 0

    def setup(self, bottom, top):

        params = self.param_str
        if len(params) < 1:
            params = ['TRAIN']
            print("Using standard initialization of params:", params)

        self.initialize()
        self.data_type = str(params)

        [self.data, self.label] = get_tile_data(NUM_DATA[self.data_type],
                                                RAND_SEED[self.data_type])

        self.num_images = NUM_DATA[self.data_type]

        print 'Outputs:', self.top_names
        if len(top) != len(self.top_names):
            raise Exception('Incorrect number of outputs (expected %d, got %d)' %
                            (len(self.top_names), len(top)))

    def reshape(self, bottom, top):
        for top_index, name in enumerate(self.top_names):
            shape = (self.batch_size, self.top_channels[top_index],
                    self.height, self.width)
            top[top_index].reshape(*shape)
        pass

    def forward(self, bottom, top):

        top[0].data[...] = self.data[self.idx : self.idx + self.batch_size, :, :, :]
        top[1].data[...] = self.label[self.idx : self.idx + self.batch_size, :, :, :]

        self.idx += self.batch_size
        if self.idx >= self.num_images:
            self.idx = self.idx - self.num_images

        pass

    def backward(self, top, propagate_down, bottom):
        pass

max_stars_repo_name: MPI-IS/bilateralNN

max_issues_repo_name: MPI-IS/bilateralNN

max_forks_repo_name: MPI-IS/bilateralNN


text:
import configparser as cf
import os
import time
from math import exp
import numpy as np
import pygame

import component

world_count = 0
count_solid = 0
count_liquid = 1  # 开始有空气
count_planet = 0
file_saved = 0

"""生成高级物理参数"""

class PhyConstant(object):

    def __init__(self, gravity=None, atom_height=None, height_default=None, temperature=None):
        self.gravity = gravity  # 获取重力
        self.atom_height = atom_height  # 获取大气高度
        self.temperature = temperature  # 获取温度
        self.height_default = height_default  # 获取默认高度
        self.air_list = []  # 空气列表初始化
        self.P = 0
        self.h = 0
        self.t = 0
        self.density = 0
        self.miu = 1

    def GetAir(self, pos_y):
        return self.air_list[int(pos_y)]

    def DensityInit(self):
        gravity = self.gravity
        if gravity is not None:
            P = 8000 * (1 + (self.temperature / 273))
            self.P = P  # 自动生成气压值
            self.h = self.atom_height
            self.t = self.temperature
            self.density = 0.05 * (P / 1.01e+5) * (25 / self.temperature) * 1.2  # 自动生成空气密度

    def AirInit(self):  # 自动生成空气随高度的减小情况
        self.air_list = [exp(-x) for x in range(int(self.atom_height))]
        component.AirList = self.air_list
        for h in range(len(self.air_list)):
            if self.air_list[h] >= self.atom_height:
                self.air_list[h] = 0

        component.InitPhyConfig.append(self)

    def SavePhysics(self):
        """

        保存配置至ini文件

        :return: None
        """
        config = cf.ConfigParser()
        with open("WorldConfig.ini", "r+") as configure:
            config.read(configure)
            config.add_section("Physics")
            for item in dir(self):
                if str(item).find("__") == -1:
                    config.set("Physics", str(item), str(getattr(self, item)))

class World(object):

    def __init__(self, d_time=None, Gravity=None, atom_height=None, default_height=None, temperature=None,
                 get_data_tick=None, floor=None, friction=None):
        """
                    初始化世界参数和物理常数
                    使用牛顿经典力学时空观(支持以场的方式进行扩展)
                    支持多体引力计算
                    采用多线程（一个天体一个线程）
                    保证性能

        """
        pygame.init()
        pygame.display.set_caption('1')
        self.screen = pygame.display.set_mode([640, 480])  # 定义窗口大小为640*480
        self.screen.fill([255, 255, 255])  # 用白色填充窗口

        self.time_set = d_time
        self.tim_rec = 0
        self._able = True
        self.Gravity = Gravity  # 重力初始化
        self.atom_height = atom_height  # 大气层高度初始化
        self.default_height = default_height  # 默认海拔初始化
        self.temperature = temperature  # 温度初始化
        self._Solids_list = []  # 固体初始化
        self._Liquids_list = []  # 流体初始化
        self._Planet_list = []  # 星球初始化
        self._filed_list = []
        self.solid_count = count_solid  # 固体计数
        self.liquid_count = count_liquid  # 流体计数
        self.tick = get_data_tick  # 设置参数获取频率
        self.floor = floor  # 设置地面（如果物体和地面接触则反弹）
        self.floor_friction = friction  # 默认摩擦力为0
        self.collide_depth = 256
        self.now_time = time.time()
        """创造一个世界,没有输入的初级参数由下面自动填充"""
        if d_time is None:
            self.time_set = 0.001  # 设置默认dt
        if Gravity is None:
            self.Gravity = 9.8  # 默认重力
        if atom_height is None:
            self.atom_height = 100000  # 默认大气高度
        if default_height is None:
            self.default_height = 0
        if temperature is None:
            self.temperature = 25  # 设置默认温度
        if floor is None:
            self.floor = 0  # 设置默认最低高度
        self.physics = PhyConstant(gravity=self.Gravity, atom_height=self.atom_height,
                                   height_default=self.default_height,
                                   temperature=self.temperature)  # 存储物理参数
        PhyConstant.DensityInit(self.physics)
        PhyConstant.AirInit(self.physics)
        if self.tick is None:
            self.tick = 100
        component.world = self
        if friction is None:
            self.floor_friction = 1
        self.gravity = self.Gravity
        """
        保存

        """
        self.config = None
        with open("WorldConfig.ini", "w+") as config:
            self._c = config

    def __del__(self):  # 删除世界
        self._able = False

    def __str__(self):
        return self.time_set, self.atom_height, self.now_time, self.gravity

    def Execute(self, running_time=None):  # 开始模拟
        """

        :param running_time float'
        模拟时间dt长度可以自行设定

        """
        import TimeManager
        if running_time is None or running_time == "me":
            TimeManager.Process().Run()
        TimeManager.Process().Run(float(running_time) * 1 / self.time_set)
        return

    @staticmethod
    def GetTimeData(switch_time, observe_type=None):
        """

        获取时间字典内的信息
        :param switch_time: 运行的秒钟
        :param observe_type: 观察的种类，Solid或Liquid或Planet或Field
        :return:ReloadDict

        """
        return component.ReloadDict(this_time=switch_time, reload_type=observe_type)

    def SetBasicSolid(self, solid_type=None, init_pos=None, speed=None, angel=None, solid_round=None,
                      name=None,
                      mass=None, electronic=None, magnet=None, friction=1):
        """

        添加基本固体（圆和正方形）

        :param magnet: Solid
        :param solid_type: C or S
        :param init_pos: list or tuple初始位置
        :param speed: 初速率
        :param angel: 初始运动方向
        :param solid_round: Solid的特征长度
        :param name: 名字
        :param mass: 质量
        :param electronic:是否有电场
        :param friction 摩檫系数
        :return: Solid

        """
        import Solids
        if init_pos is not None:
            init_x, init_y = init_pos[0], init_pos[1]
        else:
            init_x, init_y = None, None
        solid_set = Solids.Solid(solid_type=solid_type, init_x=init_x, init_y=init_y, vector_length=speed,
                                 vector_angle=angel,
                                 name=name, solid_len=solid_round, mass=mass, electronic=electronic,
                                 mag_material=magnet, friction=friction)  # 添加固体实例
        self._Solids_list.append(solid_set)
        self.solid_count += 1
        component.SolidList.append(solid_set)  # 存储至组件文件
        return solid_set

    def SetPoly(self, init_pos, solid_v, name, mass, electronic=0, *solid_pos):
        """

        添加多边形

        :param init_pos: 同上
        :param solid_v: 同上
        :param name: 同上
        :param mass: 同上
        :param electronic: 同上
        :param solid_pos: 多个点的二位列表
        :return: Solid

        """
        import Solids
        if init_pos is not None:
            init_x, init_y = init_pos[0], init_pos[1]
        else:
            init_x, init_y = None, None
        solid_set = Solids.Solid(init_x=init_x, init_y=init_y, vector_angle=solid_v[0], vector_length=solid_v[1],
                                 name=name, solid_len=None, mass=mass, electronic=electronic, poly=True,
                                 poly_pos=solid_pos)
        solid_set._SetPolyShape()  # 激活多边形
        self._Solids_list.append(solid_set)

    def SetLiquid(self, pos=None, speed=None, name=None):  # 添加水实例
        import Liquids
        self.solid_count += 1
        liquid_set = Liquids.Liquid(density=1, point=pos, speed=speed, name=name)
        self._Liquids_list.append(liquid_set)
        component.LiquidList.append(liquid_set)  # 存储至组件文件
        liquid_set._InitCalculate()
        return liquid_set

    def SetPlanet(self, mass=None, planet_radios=None, planet_shell_ball_radios=None, plant_init_x=None,
                  planet_init_y=None, name=None):  # 添加行星
        import Planets
        planet_set = Planets.Planet(mass=mass, planet_radios=planet_radios,
                                    planet_shell_ball_radios=planet_shell_ball_radios,
                                    plant_init_x=plant_init_x, planet_init_y=planet_init_y, name=name)
        self._Planet_list.append(planet_set)  # 生成一个星球实例
        component.PlanetList.append(planet_set)  # 存储至组件文件
        return planet_set

    def SetField(self, field_type, filed_data, field_name):
        """

        添加一个场

        :param field_type: 场类型
        :param filed_data: 场向量或标量
        :param field_name: 场名
        :return:field

        """
        from Functions import _AutoSelectFieldName
        field = _AutoSelectFieldName(field_type=field_type, filed_data=filed_data, name=field_name)
        self._filed_list.append(field)
        component.FieldList.append(field)
        return field

    @staticmethod
    def GetSolid(*name):
        """

        :param name:Solid.name
        获取名字对应的Solid

        :return Solid
        """
        result = []
        for item_name in name:
            result.append(component.Gets("Solid", item_name).NameToObject())
        return result

    @staticmethod
    def GetLiquid(*name):
        """

        :param name
        根据名字选取流体
        :return Liquid

        """
        result = []
        for item_name in name:
            result.append(component.Gets("Liquid", item_name).NameToObject())
        return result

    @staticmethod
    def GetPlanet(*name):
        """

        :param name str
        根据名字选取天体
        :return Planet

        """
        result = []
        for item_name in name:
            result.append(component.Gets("Planet", item_name).NameToObject())
        return result

    @staticmethod
    def GetField(*name):
        """

        名字获得场

        :param name: str
        :return: Field
        """
        result = []
        for item_name in name:
            result.append(component.Gets("Liquid", item_name).NameToObject())
        return result

    @staticmethod
    def GetSpeedNeedsSolid(speed: int or float or list or tuple):
        return component.Gets(type_name="Solid", needs=(component.Gets("Solid", needs=speed).SpeedToName()))

    @staticmethod
    def GetMassNeedsSolid(mass: int or float or list or tuple):
        """

                根据质量选择固体
                :param mass: int or iterable
                :return: Solid

        """
        return component.Gets(type_name="Solid", needs=(component.Gets("Solid", needs=mass).MassToName()))

    @staticmethod
    def GetMassNeedsPlanet(mass: int or float or list or tuple):
        """

        根据质量选择天体
        :param mass: int
        :return: Planet

        """
        return component.Gets(type_name="Planet", needs=(component.Gets("Planet", needs=mass).MassToName()))

    @staticmethod
    def GetLaPoint(planet1, planet2, point_type):
        """

                        :param planet2: Planet
                        :param planet1: Planet
                        :param point_type:int or list or tuple
                        根据输入的要求获得拉格朗日点：
                        L1，L2，L3拉格朗日点位于两行星中线上，L4和L5位于行星两侧

        """
        from Functions import _GetLaPoint

        return _GetLaPoint(planet1, planet2, point_type)

    def RemoveSolid(self, *item_name):
        """

        删除对应名字的Solid

        :param item_name: 名字（可以有多个）
        :return: Bool

        """
        for name in item_name:
            for solid_ in self.GetSolid(name):
                component.SolidList.remove(solid_)
        return True

    def RemoveField(self, *item_name):
        """

        删除对应名字的Field

        :param item_name: 名字（可以有多个）
        :return: Bool
        """
        for name in item_name:
            for solid_ in self.GetField(name):
                component.FieldList.remove(solid_)
            return True

    @staticmethod
    def RemoveTrack():
        component.ObserveList = []

    def DestroyPlanet(self, *item_name):
        """

        碎星者（划掉）
        删除对应名字的Planet

        :param item_name:
        :return: Bool
        """
        for name in item_name:
            for solid_ in self.GetPlanet(name):
                component.PlanetList.remove(solid_)
            return True

    def RemoveLiquid(self, *item_name):
        """

        删除对应名字的Liquid

        :param item_name:
        :return:
        """
        for name in item_name:
            for solid_ in self.GetLiquid(name):
                component.LiquidList.remove(solid_)
            return True

    def SetCollideDepth(self, depth):
        """

        设置碰撞检测深度，超出范围则不检测
        默认为1024m

        """
        self.collide_depth = depth

    @staticmethod
    def SetTrack(pos: list, radio, observe_mode, requirement=None, command=None):
        """

        :param command:
        :param requirement:
        :param observe_mode: "P" or "W"
        :param radio int or float
        :param pos:list
        添加“观测点”，这种观测点会无条件记录所有进入观测点的实体数据,并且根据条件执行命令

        """
        track = component.Observe(pos=pos, radio=radio, observe_mode=observe_mode, requirement=requirement,
                                  command=command)
        component.ObserveList.append(track)
        return track

    def LoadConfig(self, file_name):
        """

        加载配置
        :return:Bool

        """
        from Functions import _Recovery
        _Recovery()
        with open(file_name, "r") as input_config:
            config = cf.ConfigParser()
            config.read(input_config)
            self.time_set = config.get("World", "time_set")
            self.gravity = config.get("World", "gravity")
            self.atom_height = config.get("World", "atom_height")

    def ReSave(self):
        import FileManager
        self.config = FileManager.ConfigureControl()
        self.config.SaveWorld()
        print("保存成功,配置文件在%s" % os.path)
        self.physics.SavePhysics()

    @staticmethod
    def SaveSolid():
        for solid in component.SolidList:
            solid.SaveSolid()
        print("固体保存成功，配置文件在：%s" % os.path)

    def get_all_data(self) -> np.ndarray:
        """
        向量化
        :return:
        """
        for planet in self._Planet_list:
            planet.data_matrix.concatenate(np.array([planet.]), axis=1)

max_stars_repo_name: pronoobe/PhyDataSimulator

max_issues_repo_name: pronoobe/PhyDataSimulator

max_forks_repo_name: pronoobe/PhyDataSimulator


text:
import flowws
from flowws import Argument as Arg
import numpy as np

from .FrameClassificationTask import FrameClassificationTask

@flowws.add_stage_arguments
class FrameRegressionTask(FrameClassificationTask):
    """Generate training data to identify from which frame a sample came, as a continuous quantity"""

    ARGS = [
        Arg(
            'x_scale', '-x', float, 2.0, help='Scale by which to divide input distances'
        ),
        Arg('seed', '-s', int, 13, help='RNG seed for data generation'),
        Arg('subsample', None, float, help='Take only the given fraction of data'),
        Arg('shuffle', None, bool, True, help='If True, shuffle data'),
        Arg(
            'per_cloud',
            '-p',
            bool,
            False,
            help='If True, classify clouds rather than individual bonds',
        ),
    ]

    def run(self, scope, storage):
        super().run(scope, storage)
        class_remap = np.linspace(0, 1, len(scope['label_remap']))
        scope['y_train'] = class_remap[scope['y_train'][..., 0]]
        scope['loss'] = 'mse'
        scope['metrics'].remove('accuracy')
        scope['metrics'].append('mean_absolute_error')

max_stars_repo_name: klarh/flowws-structure-pretraining

max_issues_repo_name: klarh/flowws-structure-pretraining

max_forks_repo_name: klarh/flowws-structure-pretraining