text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec  5 18:55:22 2018
@author: kyungdoehan
"""

import pandas as pd
import os
import numpy as np
#%% Importing lake coordinates and levels
class lake_data_read:
    def __init__(self, Ma, nz, ny, nx, grid, rch, dx, dy, lakeK, top, bot, lakeb):
        self.btop = top - bot[0, :, :]
        self.dir = os.getcwd()
        self.df = pd.read_csv(self.dir + "/tisc_output/lakes_" + str(Ma) + "0Ma.csv",\
                 sep=",", header=None)
        self.df.columns = ["x", "y", "level", "type", "id", "nlakes", "evap", "inflow"]
        self.x = self.df.loc[:, "x"].values.astype('int')
        self.y = self.df.loc[:, "y"].values.astype('int')
        self.level = self.df.loc[:, "level"].values.astype('float32')
        self.type = self.df.loc[:, "type"].values.astype('str')
        self.id = self.df.loc[:, "id"].values.astype('int')
        self.nlakes = self.df.loc[:, "nlakes"].values.astype('int')[0]
        self.evap = self.df.loc[:, "evap"].values.astype('float32')
        self.inflow = self.df.loc[:, "inflow"].values.astype('float32')
        self.stages = [0 for i in range(self.nlakes)]
        self.levap = [0 for i in range(self.nlakes)]
        self.linflow = [0 for i in range(self.nlakes)]
        self.bdlknc = np.zeros((nz, ny, nx), dtype=np.float32)
        for i in range(1, self.nlakes+1):
            for j in range(len(self.id)):
                if self.id[j] == i:
                    self.stages[i-1] = self.level[j]
                    self.levap[i-1] = self.evap[j]
                    self.linflow[i-1] = self.inflow[j]
        self.lakarr = np.zeros((nz, ny, nx), dtype=np.int16)
        self.ghb = [0 for i in range(len(self.id))]
        for lake in range(len(self.x)):
            self.ycoord = np.abs(self.y[lake] - int((grid-1)/2))
            self.xcoord = np.abs(self.x[lake] + int((grid-1)/2))
#            if self.ycoord != 0 and self.xcoord != 0:
#                if self.ycoord != ny-1 and self.xcoord != 0:
            self.lakarr[0, self.ycoord, self.xcoord] = self.level[lake]
            self.ghb[lake] = [0, self.ycoord, self.xcoord, self.level[lake],\
                             dx*dy*lakeK/(lakeb)]
        self.ghb = {0: self.ghb}
def lake(Ma, nz, ny, nx, grid, rch, dx, dy, lakeK, top, bot, lakeb):
    return lake_data_read(Ma, nz, ny, nx, grid, rch, dx, dy, lakeK, top, bot, lakeb)

ext: py

reason: import numpy


text:
from datetime import datetime
import os.path
import functools
import collections
import copy
import re
import time
import logging

import numpy as np
import tensorflow as tf

from train_common import get_global_step, get_lr_and_max_steps, get_ops, run_op

import prune_algorithm.prune_common as pc

logging.basicConfig(level=logging.ERROR)

FLAGS = tf.app.flags.FLAGS
tf.app.flags.DEFINE_string('train_dir', '/tmp/tmp_train', """Directory where to write event logs and checkpoint.""")
tf.app.flags.DEFINE_string('data_dir', './data', """Path to the data directory.""")
tf.app.flags.DEFINE_string('dataset', 'cifar10', """Dataset name""")
tf.app.flags.DEFINE_string('network', 'vgg', """Network name""")
tf.app.flags.DEFINE_string('impt_method', 'correlation', """should be one of `correlation`, `cosine` and `inner_product`""")
tf.app.flags.DEFINE_string('normalize_method', 'max', """should be one of `max`, `l1` and `l2`""")
tf.app.flags.DEFINE_bool('conv_dense_separate', False, """whether pruning conv and dense layers separately""")
tf.app.flags.DEFINE_bool('merge_all', False, """only for networks with residual design, whether mean the importance within the same block""")
tf.app.flags.DEFINE_float('prune_rate', 0.01, """The global pruned ratio for network""")
tf.app.flags.DEFINE_integer('top_k', 3, """The global pruned ratio for network""")
tf.app.flags.DEFINE_bool('weight_decay_growing', False, """Whether use a larger weight_decay when finetuning than training""")
tf.app.flags.DEFINE_float('alpha', 1.0, """The weight of 'correlation' when calculating the importance""")
tf.app.flags.DEFINE_float('beta', 1.0, """The weight of 'computational cost' when calculating the importance""")
tf.app.flags.DEFINE_float('gamma', 1.0, """The weight of 'parameters' when calculating the importance""")

import config
# import prune_config

network, dataset, top_name, PruneAlg = config.parse_net_and_dataset()

def train_with_graph(weights_dict, channel_num_after_pruned, weight_decay, store_model_path):
    train_args = config.args
    num_gpus = train_args.num_gpus
    train_batch_size = train_args.train_batch_size
    test_batch_size = train_args.test_batch_size
    init_lr = train_args.initial_learning_rate
    epochs_per_decay = train_args.num_epochs_per_decay
    lr_decay_factor = train_args.learning_rate_decay_factor
    lr_staircase = train_args.get("staircase")
    max_epochs = train_args.max_epochs

    num_classes = dataset.num_classes
    image_size = [dataset.height, dataset.width]
    examples_for_train = dataset.num_examples_for_train
    examples_for_test = dataset.num_examples_for_test
    with tf.Graph().as_default(), tf.device('/cpu:0'):
        ###### get global step
        global_step = get_global_step(store_model_path)

        ###### get learning rate and max_steps
        lr, max_steps = get_lr_and_max_steps(examples_for_train, train_batch_size, num_gpus,
            lr_decay_factor, epochs_per_decay, init_lr, global_step, lr_staircase, max_epochs)

        ###### get optimizer
        opt = config.args.optimizer(lr)

        ###### Get data
        tf_training = tf.placeholder(tf.bool, shape=())
        train_dataset = dataset.train_input_fn(FLAGS.data_dir, train_batch_size, max_epochs, **config.args.data_augmentation_args).make_one_shot_iterator()
        test_dataset = dataset.test_input_fn(FLAGS.data_dir, test_batch_size, **config.args.data_augmentation_args).make_one_shot_iterator()

        ###### put op on different GPU
        train_args.learning_rate = lr
        train_args.data_queue = [train_dataset, test_dataset]
        train_args.global_step = global_step
        train_args.max_steps = max_steps
        train_args.examples_per_epoch_for_test = examples_for_test
        train_args.weights_dict = weights_dict
        train_args.channels_num = channel_num_after_pruned
        train_args.weight_decay = weight_decay

        ops = get_ops(opt, tf_training, network, dataset, num_classes, top_name, train_args)

        ###### run on session
        run_op(ops, tf_training, store_model_path, train_args)

def train(_):
    # corr_normal_factor = collections.OrderedDict()
    ckpt = tf.train.get_checkpoint_state(os.path.join(FLAGS.train_dir))
    store_model_path = ckpt.model_checkpoint_path
    init_weight_decay = config.args.weight_decay

    importance_coefficient = [FLAGS.alpha, FLAGS.beta, FLAGS.gamma]

    print("store_model_path: " + store_model_path)
    print("importance_coefficient: alpha %.2f, beta %.2f, gamma %.2f" % (FLAGS.alpha, FLAGS.beta, FLAGS.gamma))
    print("weight decay growing: %d" % FLAGS.weight_decay_growing)
    print("top_k: %d" % FLAGS.top_k)
    print("prune rate: %.2f" % FLAGS.prune_rate)

    ## get old weights
    if "resnet" in top_name:
        weights_dict = network.get_weights_from_model(store_model_path, config.args.resnet_version)
    else:
        weights_dict = network.get_weights_from_model(store_model_path)
    ## get pruned channels
    prune_args = {
        "image_size": [dataset.height, dataset.width],
        "importance_method": FLAGS.impt_method,
        "importance_coefficient": [FLAGS.alpha, FLAGS.beta, FLAGS.gamma],
        "top_k": FLAGS.top_k,
        "num_classes": dataset.num_classes,
        "normalize_method": FLAGS.normalize_method,
        "conv_dense_separate": False if FLAGS.conv_dense_separate == 0 else True,
        "merge_all": FLAGS.merge_all
    }
    prune_alg = PruneAlg(weights_dict, **prune_args)

    cut_channels = prune_alg.get_prune_channels(FLAGS.prune_rate)

    ## get pruned weights
    if "resnet" in top_name:
        pruned_weights_dict = prune_alg.get_pruned_weights(cut_channels, config.args.resnet_version)
    else:
        pruned_weights_dict = prune_alg.get_pruned_weights(cut_channels)
    cal_ratio, params_ratio = prune_alg.get_pruned_ratio()

    pruned_cared_weights = prune_alg.get_pruned_cared_weights(pruned_weights_dict)
    channel_num_after_pruned = prune_alg.get_channels_nums(pruned_cared_weights, channel_type='output')
    ## cal weight_decay
    weight_decay = 1.1e-3 if FLAGS.weight_decay_growing else init_weight_decay

    print("The number of channels after pruned: ", channel_num_after_pruned.values())
    # print("Use correlation normalization factor: " + str(corr_normal_factor.values()))
    print("Use weight decay: " + str(weight_decay))

    ## finetune the model
    store_model_path = os.path.join(FLAGS.train_dir, "prune%.2f" % FLAGS.prune_rate) # model dir
    train_with_graph(pruned_weights_dict, channel_num_after_pruned, weight_decay, store_model_path)

if __name__ == '__main__':
    tf.app.run(main=train)

max_stars_repo_name: cheerss/COP

max_issues_repo_name: cheerss/COP

max_forks_repo_name: cheerss/COP


text:
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import numpy as np
import matplotlib.pyplot as plt
from sklearn import cluster
from sklearn import datasets

# 加载iris数据
iris = datasets.load_iris()
data = iris["data"]

# 定义初始中心点
init_centers=np.array([
       [4,2.5,3,0],
       [5,3  ,3,1],
       [6,4  ,3,2]])

# 数据定义和值的取出
x_index = 1
y_index = 2

data_x=data[:,x_index]
data_y=data[:,y_index]

# 定义图表的比例和标签
x_max = 4.5
x_min = 2
y_max = 7
y_min = 1
x_label = iris["feature_names"][x_index]
y_label = iris["feature_names"][y_index]

def show_result(cluster_centers,labels):
    # 绘制cluster 0和中心点
    plt.scatter(data_x[labels==0], data_y[labels==0],c='black' ,alpha=0.3,s=100, marker="o",label="cluster 0")
    plt.scatter(cluster_centers[0][x_index], cluster_centers[0][y_index],facecolors='white', edgecolors='black', s=300, marker="o")

     # 绘制cluster 1和中心点
    plt.scatter(data_x[labels==1], data_y[labels==1],c='black' ,alpha=0.3,s=100, marker="^",label="cluster 1")
    plt.scatter(cluster_centers[1][x_index], cluster_centers[1][y_index],facecolors='white', edgecolors='black', s=300, marker="^")

     # 绘制cluster 2和中心点
    plt.scatter(data_x[labels==2], data_y[labels==2],c='black' ,alpha=0.3,s=100, marker="*",label="cluster 2")
    plt.scatter(cluster_centers[2][x_index], cluster_centers[2][y_index],facecolors='white', edgecolors='black', s=500, marker="*")

    # 设置图表缩放和轴标签
    plt.xlim(x_min, x_max)
    plt.ylim(y_min, y_max)
    plt.xlabel(x_label,fontsize='large')
    plt.ylabel(y_label,fontsize='large')
    plt.show()

# 显示初始状态
labels=np.zeros(len(data),dtype=np.int)
show_result(init_centers,labels)

for i in range(5):
	model = cluster.KMeans(n_clusters=3,max_iter=1,init=init_centers).fit(data)
	labels = model.labels_
	init_centers=model.cluster_centers_
	show_result(init_centers,labels)

max_stars_repo_name: whlll-coder/Python-

max_issues_repo_name: whlll-coder/Python-

max_forks_repo_name: whlll-coder/Python-


text:
# Usage: python lines.py
# then run gnuplot on the individual gnuplot files to generate the plots in PDF.

import numpy as np

params = [('PAN500KD_1', 17, 2), # done 000
          ('PAN500KD_2',  5, 2), # done 0.86
          ('PES200KD_1',  7, 3), # done 0.77
          ('PES200KD_2',  9, 3), # done 1.51
          ('PES300KD_1',  2, 3), # done 0.37
          ('PES300KD_2',  6, 2), # done 1.22
           ('PVDF04u_1',  2, 4), # done 0.42
           ('PVDF04u_2',  3, 3), # done 0.66
           ('PVDF15u_1',  5, 3), # done 0.57
           ('PVDF15u_2',  2, 2)] # done 0.52

for case in params:
    file = case[0]
    data = np.loadtxt('data/' + file + '.txt')
    elem = len(data)-1

    start_data = np.take(data, list(range(case[1])), 0)
    start_line = np.polyfit(start_data[:,0],start_data[:,2],1)

    end_data = np.take(data, list(range(elem, elem-case[2], -1)), 0)
    end_line = np.polyfit(end_data[:,0],end_data[:,2],1)

    intersect_x = (end_line[1] - start_line[1])/\
                  (start_line[0] - end_line[0])
    intersect_y = (start_line[0]*end_line[1] - end_line[0]*start_line[1])/\
                  (start_line[0] - end_line[0])
    intersection = [intersect_x, intersect_y]

    with open('gnuplot_' + str(file) + '.gp', 'w') as outfile:
        print('set terminal pdfcairo  transparent enhanced fontscale 0.5 size 5.00in, 3.00in', file=outfile)
        print('set output "{0}.pdf"'.format(file), file=outfile)
        # print('set terminal pngcairo  background "#ffffff" enhanced fontscale 1.0 size 1200, 720', file=outfile)
        # print('set output "{0}.png"'.format(file), file=outfile)
        # print('set locale "en_US.UTF-8"', file=outfile)
        print('GNUTERM = "wxt"', file=outfile)
        print('set grid', file=outfile)
        print('set xrange [0:*]', file=outfile)
        print('set yrange [0:*]', file=outfile)
        print('set xlabel "TMP (bar)"', file=outfile)
        print('set ylabel "J (L/h/m^{2})"', file=outfile)
        print('f(x) = (x < ({0:.4f} + {0:.4f}*0.1) ) ? {1:.4f}*x + {2:.4f} : 1/0'.format(intersection[0], start_line[0], start_line[1]), file=outfile)
        print('g(x) = (x > ({0:.4f} - {0:.4f}*0.1) ) ? {1:.4f}*x + {2:.4f} : 1/0'.format(intersection[0], end_line[0], end_line[1]), file=outfile)
        print('set label at {0:.4f},{1:.4f} "" point lw 2 pt 6 ps 1 front'.format(intersection[0], intersection[1]), file=outfile)
        print('set label at {0:.4f},0 "{0:.2f}" offset 1,2 front'.format(intersection[0], intersection[1]), file=outfile)
        print('set arrow from {0:.4f},{1:.4f} to {0:.4f},0 nohead lw 2 dt 2 front'.format(intersection[0], intersection[1]), file=outfile)
        print('p "data/{0}.txt" u 1:3:4 t "" w errorbars lw 1.5 pt 7 ps 0.5 lc rgb "#268bd2",\\'.format(file), file=outfile)
        # print('"" u 1:3 t "" lw 1.5 lc rgb "#268bd2" w l,\\'.format(file), file=outfile)
        print('f(x) t "" lw 1.5 lc rgb "#dc322f" w l,\\', file=outfile)
        print('g(x) t "" lw 1.5 lc rgb "#dc322f" w l', file=outfile)

max_stars_repo_name: roguephysicist/PyPSD

max_issues_repo_name: roguephysicist/PyPSD

max_forks_repo_name: roguephysicist/PyPSD


text:
import torch
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
import torch.nn as nn
import torch.optim as optim
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
from datetime import datetime
from torch.utils.data import WeightedRandomSampler, Dataset
import time
import os
os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
from PIL import Image
import random
from matplotlib import pyplot as plt
import torch.utils.data
from sklearn.metrics import confusion_matrix
import itertools
import json
from imgaug import augmenters as iaa
import imgaug as ia

Image.MAX_IMAGE_PIXELS = None

class ClassificationMapedDataset(Dataset):
    def __init__(self, dataset_path, class_to_class_map, transform=None, ignore_list=['']):
        self.ignore_list = ignore_list
        classes, class_to_idx = self._find_classes(dataset_path, class_to_class_map)
        self.classes = classes
        self.class_to_idx = class_to_idx
        samples = self._make_dataset(dataset_path, class_to_idx)
        self.samples = samples
        self.targets = [s[1] for s in samples]
        self.transform = transform

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, index):
        path, target = self.samples[index]
        sample = datasets.folder.default_loader(path)
        if self.transform is not None:
            sample = self.transform(sample)
        return sample, target

    def _find_classes(self, dir, class_to_class_map):
        if class_to_class_map is not None:
            classes = [d.name for d in os.scandir(dir) if d.is_dir()]
            classes.sort()
            class_to_idx = {classes[i]: i for i in range(len(classes))}
            # remove ignore classes
            for ignore_class in self.ignore_list:
                try:
                    classes.remove(ignore_class)
                    class_to_idx.pop(ignore_class, None)
                except:
                    pass

            uniq_clases_key_map = []
            uniq_clases_value_map = []
            for key, value in class_to_class_map.items():
                uniq_clases_key_map.append(key)
                uniq_clases_value_map.append(value)
            uniq_clases_key_map = list(set(uniq_clases_key_map))
            uniq_clases_value_map = list(set(uniq_clases_value_map))
            # swap value in self.class_to_idx for values in
            for key, value in class_to_idx.items():
                print(class_to_class_map)
                class_to_idx[key] = class_to_idx[class_to_class_map[key]]
            # classes reduce to map calsses
            classes = uniq_clases_value_map
        else:
            classes = [d.name for d in os.scandir(dir) if d.is_dir()]
            classes.sort()
            class_to_idx = {classes[i]: i for i in range(len(classes))}
        return classes, class_to_idx

    def _make_dataset(self, dataset_path, class_to_idx):
        images = []
        dir = os.path.expanduser(dataset_path)
        for target in sorted(class_to_idx.keys()):
            d = os.path.join(dir, target)
            if not os.path.isdir(d):
                continue
            for root, _, fnames in sorted(os.walk(d, followlinks=True)):
                for fname in sorted(fnames):
                    path = os.path.join(root, fname)
                    item = (path, class_to_idx[target])
                    images.append(item)
        return images

class ImgAugTransform:
    def __init__(self, agumentation = {}):
        try:
            self.aug = iaa.Sequential([
                iaa.Resize((224, 224)),
                iaa.Sometimes(
                    float(agumentation['GaussianBlur_P']),
                    iaa.GaussianBlur(sigma=(agumentation["GaussianBlur_sigma"]))
                ),
                iaa.Sometimes(
                    float(agumentation['MedianBlur_P']),
                    iaa.imgcorruptlike.MotionBlur(severity=agumentation['MedianBlur_Severity'])
                ),
                iaa.Sometimes(
                    agumentation['DefocusBlur_P'],
                    iaa.imgcorruptlike.DefocusBlur(severity=agumentation['DefocusBlur_Severity'])
                ),
                iaa.Sometimes(
                    agumentation['Cutout_P'],
                    iaa.Cutout(nb_iterations=agumentation['Cutout_cum_of_cuts']),
                ),
                iaa.ChannelShuffle(agumentation['ChannelSuffle_P']),
                iaa.Sometimes(
                    agumentation['ElasticTransformation_P'],
                    iaa.ElasticTransformation(alpha=agumentation['ElasticTransformation_alpha'],
                                            sigma = agumentation['ElasticTransformation_sigma']
                    ),
                ),
                iaa.Sometimes(
                    agumentation['PerspectiveTransform_P'],
                    iaa.PerspectiveTransform(scale=agumentation['PerspectiveTransform_scale']),
                ),
            ], random_order=True)
        except:
            self.aug = iaa.Sequential([
                iaa.Resize((224, 224)),
                iaa.Sometimes(0.25, iaa.GaussianBlur(sigma=(0, 3.0))),
                iaa.Fliplr(0.5),
                iaa.Affine(rotate=(-20, 20), mode='symmetric'),
                iaa.Dropout2d(p=0.5),
                iaa.Sometimes(0.5,iaa.Cutout(nb_iterations=2)),
                iaa.Sometimes(0.25,iaa.Invert(0.25, per_channel=0.5)),
                iaa.Sometimes(0.25,iaa.Add((-40, 40))),
                iaa.AddToHueAndSaturation(value=(-10, 10), per_channel=True)
            ])

    def __call__(self, img):
        img = np.array(img)
        return self.aug.augment_image(img).copy()

class ImbalancedDatasetSampler(torch.utils.data.sampler.Sampler):
    """Samples elements randomly from a given list of indices for imbalanced dataset
    Arguments:
        indices (list, optional): a list of indices
        num_samples (int, optional): number of samples to draw
        callback_get_label func: a callback-like function which takes two arguments - dataset and index
    """

    def __init__(self, dataset, indices=None, num_samples=None, callback_get_label=None):
        self.indices = list(range(len(dataset))) \
            if indices is None else indices
        self.callback_get_label = callback_get_label
        self.num_samples = len(self.indices) \
            if num_samples is None else num_samples
        label_to_count = {}
        for idx in self.indices:
            label = self._get_label(dataset, idx)
            if label in label_to_count:
                label_to_count[label] += 1
            else:
                label_to_count[label] = 1
        weights = [1.0 / label_to_count[self._get_label(dataset, idx)]
                   for idx in self.indices]
        self.weights = torch.DoubleTensor(weights)

    def _get_label(self, dataset, idx):
        return dataset[idx][1]

    def __iter__(self):
        return (self.indices[i] for i in torch.multinomial(
            self.weights, self.num_samples, replacement=True))

    def __len__(self):
        return self.num_samples

class Train:
    def __init__(self, model_type='vgg16', hyperparams={
        "model_type": 'vgg16',
        "batch_size": 64,
        "learning_rate": 0.001,
        "epochs": 10,
        "train_valid_test_split": [0.85, 0.10, 0.05],
    }, folders_structur = {
        "models_folder": "models",
        "graph_folder": "graph_folder",
        "confusion_matrix_folder": "confusion_matrix",
        "test_img_folder": "test_img_folder",
        "metadata_json_folder": "metadata_json"
    }, agumentations = {}, class_map = None, ignore_list =['Object_Detection_Annotations']
    ):
        self.transforms = transforms.Compose([
                ImgAugTransform(),
                lambda x: Image.fromarray(x),
                transforms.Resize((224, 224)),
                transforms.ToTensor(),
                transforms.Normalize([0.485, 0.456, 0.406],
                                     [0.229, 0.224, 0.225])
            ])
        self.data_dim = 5
        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.model_type = hyperparams['model_type']
        self.batch_size = hyperparams['batch_size']
        self.learning_rate = hyperparams['learning_rate']
        self.epochs = hyperparams['epochs']
        self.train_valid_test_split = hyperparams['train_valid_test_split']
        self.model_folder = folders_structur["models_folder"]
        self.graph_folder = folders_structur["graph_folder"]
        self.test_img_folder = folders_structur["test_img_folder"]
        self.confusion_matrix_folder = folders_structur["confusion_matrix_folder"]
        self.json_folder = folders_structur["metadata_json_folder"]
        self.class_map = class_map
        self.ignore_list = ignore_list

        os.makedirs(self.model_folder, exist_ok=True)
        os.makedirs(self.graph_folder, exist_ok=True)
        os.makedirs(self.test_img_folder, exist_ok=True)
        os.makedirs(self.confusion_matrix_folder, exist_ok=True)
        os.makedirs(self.json_folder, exist_ok=True)

    def train(self, dataset_path):
        dataset = ClassificationMapedDataset(dataset_path=dataset_path,
                                            class_to_class_map=self.class_map,
                                            transform=self.transforms,
                                            ignore_list=self.ignore_list)
        # dataset = datasets.ImageFolder(dataset_path, self.transforms)
        train_loader, valid_loader, test_loader, classes = self._split_dataset_to_dataloaders_and_return_classes(dataset)
        model = self._load_model(classes)
        # criterion and optim
        criterion = nn.CrossEntropyLoss()
        try:
            optimizer = torch.optim.Adam(model.fc.parameters(), lr=self.learning_rate)
        except Exception as e:
            optimizer = torch.optim.Adam(model.classifier.parameters(), lr=self.learning_rate)
        # start train loop
        scaler = torch.cuda.amp.GradScaler()
        valid_loss_min = np.Inf
        valid_acc_min = 0.0
        graph_train_loss = []
        graph_valid_loss = []
        graph_valid_acc = []
        for epoch in range(self.epochs):
            print("epoch start :", epoch)
            start = time.time()
            train_loss = 0.0
            valid_loss = 0.0
            model.train()
            for inputs, labels in train_loader:
                # Move input and label tensors to the default device
                inputs, labels = inputs.to(self.device), labels.to(self.device)
                optimizer.zero_grad()
                with torch.cuda.amp.autocast():
                    logps = model(inputs)
                loss = criterion(logps, labels)
                scaler.scale(loss).backward()
                scaler.step(optimizer)
                scaler.update()
                train_loss += loss.item()
            model.eval()
            with torch.no_grad():
                accuracy = 0
                for inputs, labels in valid_loader:
                    inputs, labels = inputs.to(self.device), labels.to(self.device)
                    logps = model(inputs)
                    batch_loss = criterion(logps, labels)
                    valid_loss += batch_loss.item()
                    # Calculate accuracy
                    ps = torch.exp(logps)
                    top_p, top_class = ps.topk(1, dim=1)
                    equals = top_class == labels.view(*top_class.shape)
                    accuracy += torch.mean(equals.type(torch.FloatTensor)).item()
            # calculate average losses
            train_loss = train_loss/len(train_loader)
            valid_loss = valid_loss/len(valid_loader)
            valid_accuracy = accuracy/len(valid_loader)
            graph_train_loss.append(train_loss)
            graph_valid_loss.append(valid_loss)
            graph_valid_acc.append(valid_accuracy)
            # print training/validation statistics
            print('Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f} \tValidation Accuracy: {:.6f}'.format(
                epoch + 1, train_loss, valid_loss, valid_accuracy))
            if valid_loss_min >= valid_loss:
                print('valid_loss  decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(
                valid_loss_min ,
                valid_loss))
                now = datetime.now()
                data = now.strftime("%d_%m_%Y")
                model_save_name = 'resnet50_' + data + '_valid_acc_' + str(round(valid_accuracy, 4)) + '.pth'
                path =  os.path.join(self.model_folder, model_save_name)
                torch.save(model.state_dict(), path)
                valid_loss_min = valid_loss
            print(f"Time per epoch: {(time.time() - start):.3f} seconds")
        model.eval()
        test_loss = 0.0
        test_accuracy = []
        test_images = []
        true_label_list = []
        pred_label_list = []
        with torch.no_grad():
            for inputs, true_label in test_loader:
                test_images.append(inputs)
                true_label = true_label.tolist()[0]
                true_label_list.append(true_label)
                inputs, labels = inputs.to(self.device), labels.to(self.device)
                logps = model(inputs)
                # Calculate accuracy
                ps = torch.exp(logps)
                top_p, top_class = ps.topk(1, dim=1)
                top_class = top_class.tolist()[0]
                top_class = top_class.pop()
                pred_label_list.append(top_class)
                equals = top_class == true_label
                test_accuracy.append(equals)
        test_accuracy = round(np.mean(test_accuracy), 4)
        print(test_accuracy)
        now = datetime.now()
        data = now.strftime("%d_%m_%Y")
        model_save_name = 'resnet50_final_' + data + '_test_acc_' + str(round(test_accuracy, 4)) + '.pth'
        path =  os.path.join(self.model_folder, model_save_name)
        torch.save(model.state_dict(), path)
        test_images = self._save_and_convert_test_images_to_paths(test_images)
        # ploting data
        graph_plot_path = os.path.join(self.graph_folder,'plot.jpg')
        fig = plt.figure()
        plt.plot(list(range(0, self.epochs)), graph_train_loss, label='train_loss')
        plt.plot(list(range(0, self.epochs)), graph_valid_loss, label='valid_loss')
        plt.plot(list(range(0, self.epochs)), graph_valid_acc, label='valid_accuracy')
        plt.legend()
        plt.xlabel("epochs")
        plt.ylabel("values")
        fig.savefig(graph_plot_path, dpi=fig.dpi)
        # confusion matrix
        confusion_matrix_path = os.path.join(self.confusion_matrix_folder, 'plot.jpg')
        conf_matrix = confusion_matrix(true_label_list, pred_label_list)
        plot_confusion_matrix(cm=conf_matrix, target_names=[x for x in range(len(classes))],
                                save_path=confusion_matrix_path)
        conf_matrix = conf_matrix.tolist()
        output_dict = self._generate_output_dict(conf_matrix, confusion_matrix_path, graph_plot_path, test_accuracy, test_images, true_label_list, pred_label_list, path,
                                                 graph_train_loss, graph_valid_loss, graph_valid_acc)
        json_path = os.path.join(self.json_folder, 'metadata.json')
        self._dump_dict_to_json(output_dict, json_path)
        return output_dict

    def _split_dataset_to_dataloaders_and_return_classes(self, dataset):
        classes = dataset.classes
        self.classes = classes
        train_size = int(self.train_valid_test_split[0] * len(dataset))
        valid_size = int(self.train_valid_test_split[1] * len(dataset))
        test_size = int(self.train_valid_test_split[2] * len(dataset))
        rest = len(dataset) - train_size - valid_size - test_size
        train_size =  train_size + rest
        if (train_size + valid_size + test_size == len(dataset)):
            train_set, valid_set, test_set = torch.utils.data.random_split(dataset,
                                                        [train_size, valid_size, test_size])
        else:
            try:
                train_set, valid_set, test_set = torch.utils.data.random_split(dataset,
                                                        [train_size+1, valid_size, test_size])
            except:
                train_set, valid_set, test_set = torch.utils.data.random_split(dataset,
                                                        [train_size, valid_size+1, test_size])
        train_loader = torch.utils.data.DataLoader(train_set, sampler=ImbalancedDatasetSampler(train_set),
                                                batch_size=self.batch_size,
                                                   num_workers=0, pin_memory=True)
        valid_loader = torch.utils.data.DataLoader(valid_set, batch_size=self.batch_size,
                                                 shuffle=True, num_workers=0, pin_memory=True)

        test_loader = torch.utils.data.DataLoader(test_set, batch_size=1)
        return train_loader, valid_loader, test_loader, classes

    def _load_model(self, classes, model_path = None):
        if model_path is None:
            model = self._load_specific_model(classes)
        else:
            # TODO load model from path
            model = self._load_specific_model(classes)
        return model

    def _load_specific_model(self, classes):
        if self.model_type == 'resnet50':
            model = models.resnet50(pretrained=True).to(self.device)
            for param in model.parameters():
                param.requires_grad = False
            model.fc = nn.Sequential(
                        nn.Linear(2048, 128),
                        nn.ReLU(inplace=True),
                        nn.Linear(128, len(classes))).to(self.device)
            model = model.to(self.device)
        if self.model_type == 'resnet18':
            model = models.resnet18(pretrained=True).to(self.device)
            for param in model.parameters():
                param.requires_grad = False
            model.fc = nn.Sequential(
                        nn.Linear(512, 128),
                        nn.ReLU(inplace=True),
                        nn.Linear(128, len(classes))).to(self.device)
            model = model.to(self.device)
        elif self.model_type == 'resnet34':
            model = models.resnet34(pretrained=True).to(self.device)
            for param in model.parameters():
                param.requires_grad = False
            model.fc = nn.Sequential(
                        nn.Linear(2048, 128),
                        nn.ReLU(inplace=True),
                        nn.Linear(128, len(classes))).to(self.device)
            model = model.to(self.device)
        elif self.model_type == 'resnet101':
            model = models.resnet101(pretrained=True).to(self.device)
            for param in model.parameters():
                param.requires_grad = False
            model.fc = nn.Sequential(
                        nn.Linear(2048, 128),
                        nn.ReLU(inplace=True),
                        nn.Linear(128, len(classes))).to(self.device)
            model = model.to(self.device)
        elif self.model_type == 'resnet152':
            model = models.resnet152(pretrained=True).to(self.device)
            for param in model.parameters():
                param.requires_grad = False
            model.fc = nn.Sequential(
                        nn.Linear(2048, 128),
                        nn.ReLU(inplace=True),
                        nn.Linear(128, len(classes))).to(self.device)
            model = model.to(self.device)
        elif self.model_type == 'vgg16':
            model = models.vgg16(pretrained=True)
            for param in model.parameters():
                param.requires_grad = False
            # model.classifier[6].out_features = nn.Linear(4096, len(self.classes))
            model.classifier = nn.Sequential(
                        nn.Linear(25088, 128),
                        nn.ReLU(inplace=True),
                        nn.Linear(128, len(classes))).to(self.device)
            model = model.to(self.device)
        return model

    def _generate_output_dict(self, conf_matrix, confusion_matrix_path, graph_plot_path,
                                test_accuracy, test_images, true_label_list, pred_label_list,
                                model_path, graph_train_loss, graph_valid_loss, graph_valid_acc):
        output_dict = {
            'conf_matrix': conf_matrix,
            "confusion_matrix_path": confusion_matrix_path,
            "graph_plot_path": graph_plot_path,
            "test_accuracy": test_accuracy,
            "test_images": test_images,
            "true_label_list": true_label_list,
            "pred_label_list": pred_label_list,
            "model_path": model_path,
            "epochs": [x for x in range(len(graph_train_loss))],
            "train_loss": graph_train_loss,
            "valid_loss": graph_valid_loss,
            "valid_acc": graph_valid_acc,
        }
        return output_dict

    def _save_and_convert_test_images_to_paths(self, test_images):
        test_images_paths = []
        for idx, test_image in enumerate(test_images):
            test_image = self._inv_normalize_tensor(test_image)
            img_path = os.path.join(self.test_img_folder, str(idx)+'.jpg')
            torchvision.utils.save_image(test_image, img_path)
            test_images_paths.append(img_path)
        return test_images_paths

    def _inv_normalize_tensor(self, pytorch_tensor):
        pytorch_tensor = torch.squeeze(pytorch_tensor, 0)
        inv_normalize = transforms.Normalize(
        mean=[-0.485/0.229, -0.456/0.224, -0.406/0.255],
        std=[1/0.229, 1/0.224, 1/0.255]
        )
        inv_pytorch_tensor = inv_normalize(pytorch_tensor)
        return inv_pytorch_tensor

    def _dump_dict_to_json(self, dict_to_dump, json_path):
        with open(json_path, 'w') as fp:
            json.dump(dict_to_dump, fp,  indent=4)

def plot_confusion_matrix(cm, save_path,
                          target_names,
                          title='Confusion matrix',
                          cmap=None,
                          normalize=False):
    accuracy = np.trace(cm) / np.sum(cm).astype('float')
    misclass = 1 - accuracy
    if cmap is None:
        cmap = plt.get_cmap('Blues')
    fig = plt.figure(figsize=(8, 6))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()

    if target_names is not None:
        tick_marks = np.arange(len(target_names))
        plt.xticks(tick_marks, target_names, rotation=45)
        plt.yticks(tick_marks, target_names)
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    thresh = cm.max() / 1.5 if normalize else cm.max() / 2
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        if normalize:
            plt.text(j, i, "{:0.4f}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")
        else:
            plt.text(j, i, "{:,}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))
    fig.savefig(save_path, dpi=fig.dpi)
    return fig

if __name__ == '__main__':
    print('start')
    kjn = Train()
    output_dict = kjn.train(dataset_path='D:/universal image classification 2/test_dataset')
    # dataset = datasets.ImageFolder('test_dataset')
    # model = kjn._load_model()
    # print(model)


text:
import torch
import torch.backends.cudnn as cudnn

import torchvision
import torchvision.transforms as transforms

import os
import argparse

import data.imagenet as imagenet
from models import *
from utils import progress_bar
import numpy as np

parser = argparse.ArgumentParser(description='Rank extraction')

parser.add_argument(
    '--data_dir',
    type=str,
    default='./data',
    help='dataset path')
parser.add_argument(
    '--dataset',
    type=str,
    default='cifar10',
    choices=('cifar10','imagenet'),
    help='dataset')
parser.add_argument(
    '--job_dir',
    type=str,
    default='result/tmp',
    help='The directory where the summaries will be stored.')
parser.add_argument(
    '--arch',
    type=str,
    default='vgg_16_bn',
    choices=('resnet_50','vgg_16_bn','resnet_56','resnet_110','densenet_40','googlenet'),
    help='The architecture to prune')
parser.add_argument(
    '--resume',
    type=str,
    default=None,
    help='load the model from the specified checkpoint')
parser.add_argument(
    '--limit',
    type=int,
    default=5,
    help='The num of batch to get rank.')
parser.add_argument(
    '--train_batch_size',
    type=int,
    default=128,
    help='Batch size for training.')
parser.add_argument(
    '--eval_batch_size',
    type=int,
    default=100,
    help='Batch size for validation.')
parser.add_argument(
    '--start_idx',
    type=int,
    default=0,
    help='The index of conv to start extract rank.')
parser.add_argument(
    '--gpu',
    type=str,
    default='0',
    help='Select gpu to use')
parser.add_argument(
    '--adjust_ckpt',
    action='store_true',
    help='adjust ckpt from pruned checkpoint')
parser.add_argument(
    '--compress_rate',
    type=str,
    default=None,
    help='compress rate of each conv')

args = parser.parse_args()

os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
cudnn.benchmark = True

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Data
print('==> Preparing data..')
if args.dataset=='cifar10':
    transform_train = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

    transform_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])
    trainset = torchvision.datasets.CIFAR10(root=args.data_dir, train=True, download=True, transform=transform_train)
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.train_batch_size, shuffle=True, num_workers=2)

    testset = torchvision.datasets.CIFAR10(root=args.data_dir, train=False, download=True, transform=transform_test)
    testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2)
elif args.dataset=='imagenet':
    data_tmp = imagenet.Data(args)
    trainloader = data_tmp.loader_train
    testloader = data_tmp.loader_test

if args.compress_rate:
    import re
    cprate_str=args.compress_rate
    cprate_str_list=cprate_str.split('+')
    pat_cprate = re.compile(r'\d+\.\d*')
    pat_num = re.compile(r'\*\d+')
    cprate=[]
    for x in cprate_str_list:
        num=1
        find_num=re.findall(pat_num,x)
        if find_num:
            assert len(find_num) == 1
            num=int(find_num[0].replace('*',''))
        find_cprate = re.findall(pat_cprate, x)
        assert len(find_cprate)==1
        cprate+=[float(find_cprate[0])]*num

    compress_rate=cprate
else:
    default_cprate={
        'vgg_16_bn': [0.7]*7+[0.1]*6,
        'densenet_40': [0.0]+[0.1]*6+[0.7]*6+[0.0]+[0.1]*6+[0.7]*6+[0.0]+[0.1]*6+[0.7]*5+[0.0],
        'googlenet': [0.10]+[0.7]+[0.5]+[0.8]*4+[0.5]+[0.6]*2,
        'resnet_50':[0.2]+[0.8]*10+[0.8]*13+[0.55]*19+[0.45]*10,
        'resnet_56':[0.1]+[0.60]*35+[0.0]*2+[0.6]*6+[0.4]*3+[0.1]+[0.4]+[0.1]+[0.4]+[0.1]+[0.4]+[0.1]+[0.4],
        'resnet_110':[0.1]+[0.40]*36+[0.40]*36+[0.4]*36
    }
    compress_rate=default_cprate[args.arch]

# Model
print('==> Building model..')
print(compress_rate)
net = eval(args.arch)(compress_rate=compress_rate)
net = net.to(device)

if len(args.gpu)>1 and torch.cuda.is_available():
    device_id = []
    for i in range((len(args.gpu) + 1) // 2):
        device_id.append(i)
    net = torch.nn.DataParallel(net, device_ids=device_id)

if args.resume:
    # Load checkpoint.
    print('==> Resuming from checkpoint..')
    checkpoint = torch.load(args.resume, map_location='cuda:'+args.gpu)
    from collections import OrderedDict
    new_state_dict = OrderedDict()
    if args.adjust_ckpt:
        for k, v in checkpoint.items():
            new_state_dict[k.replace('module.', '')] = v
    else:
        for k, v in checkpoint['state_dict'].items():
            new_state_dict[k.replace('module.', '')] = v
    net.load_state_dict(new_state_dict)

criterion = nn.CrossEntropyLoss()
feature_result = torch.tensor(0.)
total = torch.tensor(0.)
def get_feature_hook(self, input, output):
    global feature_result
    global entropy
    global total
    a = output.shape[0]
    b = output.shape[1]
    c = torch.tensor([torch.matrix_rank(output[i,j,:,:]).item() for i in range(a) for j in range(b)])

    c = c.view(a, -1).float()
    c = c.sum(0)
    feature_result = feature_result * total + c
    total = total + a
    feature_result = feature_result / total

def get_feature_hook_densenet(self, input, output):
    global feature_result
    global total
    a = output.shape[0]
    b = output.shape[1]
    c = torch.tensor([torch.matrix_rank(output[i,j,:,:]).item() for i in range(a) for j in range(b-12,b)])

    c = c.view(a, -1).float()
    c = c.sum(0)
    feature_result = feature_result * total + c
    total = total + a
    feature_result = feature_result / total

def get_feature_hook_googlenet(self, input, output):
    global feature_result
    global total
    a = output.shape[0]
    b = output.shape[1]
    c = torch.tensor([torch.matrix_rank(output[i,j,:,:]).item() for i in range(a) for j in range(b-12,b)])

    c = c.view(a, -1).float()
    c = c.sum(0)
    feature_result = feature_result * total + c
    total = total + a
    feature_result = feature_result / total

def test():
    global best_acc
    net.eval()
    test_loss = 0
    correct = 0
    total = 0
    limit = args.limit

    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(trainloader):
            if batch_idx >= limit:  # use the first 6 batches to estimate the rank.
               break
            inputs, targets = inputs.to(device), targets.to(device)
            outputs = net(inputs)
            loss = criterion(outputs, targets)

            test_loss += loss.item()
            _, predicted = outputs.max(1)
            total += targets.size(0)
            correct += predicted.eq(targets).sum().item()

            progress_bar(batch_idx, limit, 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))#'''

if args.arch=='vgg_16_bn':

    if len(args.gpu) > 1:
        relucfg = net.module.relucfg
    else:
        relucfg = net.relucfg

    for i, cov_id in enumerate(relucfg):
        cov_layer = net.features[cov_id]
        handler = cov_layer.register_forward_hook(get_feature_hook)
        test()
        handler.remove()

        if not os.path.isdir('rank_conv/'+args.arch+'_limit%d'%(args.limit)):
            os.mkdir('rank_conv/'+args.arch+'_limit%d'%(args.limit))
        np.save('rank_conv/'+args.arch+'_limit%d'%(args.limit)+'/rank_conv' + str(i + 1) + '.npy', feature_result.numpy())

        feature_result = torch.tensor(0.)
        total = torch.tensor(0.)

elif args.arch=='resnet_56':

    cov_layer = eval('net.relu')
    handler = cov_layer.register_forward_hook(get_feature_hook)
    test()
    handler.remove()

    if not os.path.isdir('rank_conv/' + args.arch+'_limit%d'%(args.limit)):
        os.mkdir('rank_conv/' + args.arch+'_limit%d'%(args.limit))
    np.save('rank_conv/' + args.arch+'_limit%d'%(args.limit)+ '/rank_conv%d' % (1) + '.npy', feature_result.numpy())
    feature_result = torch.tensor(0.)
    total = torch.tensor(0.)

    # ResNet56 per block
    cnt=1
    for i in range(3):
        block = eval('net.layer%d' % (i + 1))
        for j in range(9):
            cov_layer = block[j].relu1
            handler = cov_layer.register_forward_hook(get_feature_hook)
            test()
            handler.remove()
            np.save('rank_conv/' + args.arch +'_limit%d'%(args.limit)+ '/rank_conv%d'%(cnt + 1)+'.npy', feature_result.numpy())
            cnt+=1
            feature_result = torch.tensor(0.)
            total = torch.tensor(0.)

            cov_layer = block[j].relu2
            handler = cov_layer.register_forward_hook(get_feature_hook)
            test()
            handler.remove()
            np.save('rank_conv/' + args.arch +'_limit%d'%(args.limit)+ '/rank_conv%d'%(cnt + 1)+'.npy', feature_result.numpy())
            cnt += 1
            feature_result = torch.tensor(0.)
            total = torch.tensor(0.)

elif args.arch=='densenet_40':

    if not os.path.isdir('rank_conv/' + args.arch+'_limit%d'%(args.limit)):
        os.mkdir('rank_conv/' + args.arch+'_limit%d'%(args.limit))

    feature_result = torch.tensor(0.)
    total = torch.tensor(0.)

    # Densenet per block & transition
    for i in range(3):
        dense = eval('net.dense%d' % (i + 1))
        for j in range(12):
            cov_layer = dense[j].relu
            if j==0:
                handler = cov_layer.register_forward_hook(get_feature_hook)
            else:
                handler = cov_layer.register_forward_hook(get_feature_hook_densenet)
            test()
            handler.remove()

            np.save('rank_conv/' + args.arch +'_limit%d'%(args.limit) + '/rank_conv%d'%(13*i+j+1)+'.npy', feature_result.numpy())
            feature_result = torch.tensor(0.)
            total = torch.tensor(0.)

        if i<2:
            trans=eval('net.trans%d' % (i + 1))
            cov_layer = trans.relu

            handler = cov_layer.register_forward_hook(get_feature_hook_densenet)
            test()
            handler.remove()

            np.save('rank_conv/' + args.arch +'_limit%d'%(args.limit) + '/rank_conv%d' % (13 * (i+1)) + '.npy', feature_result.numpy())
            feature_result = torch.tensor(0.)
            total = torch.tensor(0.)#'''

    cov_layer = net.relu
    handler = cov_layer.register_forward_hook(get_feature_hook_densenet)
    test()
    handler.remove()
    np.save('rank_conv/' + args.arch +'_limit%d'%(args.limit) + '/rank_conv%d' % (39) + '.npy', feature_result.numpy())
    feature_result = torch.tensor(0.)
    total = torch.tensor(0.)

elif args.arch=='googlenet':

    if not os.path.isdir('rank_conv/' + args.arch+'_limit%d'%(args.limit)):
        os.mkdir('rank_conv/' + args.arch+'_limit%d'%(args.limit))
    feature_result = torch.tensor(0.)
    total = torch.tensor(0.)

    cov_list=['pre_layers',
              'inception_a3',
              'maxpool1',
              'inception_a4',
              'inception_b4',
              'inception_c4',
              'inception_d4',
              'maxpool2',
              'inception_a5',
              'inception_b5',
              ]

    # branch type
    tp_list=['n1x1','n3x3','n5x5','pool_planes']
    for idx, cov in enumerate(cov_list):

        if idx<args.start_idx:
            continue
        cov_layer=eval('net.'+cov)

        handler = cov_layer.register_forward_hook(get_feature_hook)
        test()
        handler.remove()

        if idx>0:
            for idx1,tp in enumerate(tp_list):
                if idx1==3:
                    np.save('rank_conv/' + args.arch+'_limit%d'%(args.limit) + '/rank_conv%d_'%(idx+1)+tp+'.npy',
                            feature_result[sum(net.filters[idx-1][:-1]) : sum(net.filters[idx-1][:])].numpy())
                #elif idx1==0:
                #    np.save('rank_conv1/' + args.arch + '/rank_conv%d_'%(idx+1)+tp+'.npy',
                #            feature_result[0 : sum(net.filters[idx-1][:1])].numpy())
                else:
                    np.save('rank_conv/' + args.arch+'_limit%d'%(args.limit) + '/rank_conv%d_' % (idx + 1) + tp + '.npy',
                            feature_result[sum(net.filters[idx-1][:idx1]) : sum(net.filters[idx-1][:idx1+1])].numpy())
        else:
            np.save('rank_conv/' + args.arch+'_limit%d'%(args.limit) + '/rank_conv%d' % (idx + 1) + '.npy',feature_result.numpy())
        feature_result = torch.tensor(0.)
        total = torch.tensor(0.)

elif args.arch=='resnet_110':

    cov_layer = eval('net.relu')
    handler = cov_layer.register_forward_hook(get_feature_hook)
    test()
    handler.remove()

    if not os.path.isdir('rank_conv/' + args.arch+'_limit%d'%(args.limit)):
        os.mkdir('rank_conv/' + args.arch+'_limit%d'%(args.limit))
    np.save('rank_conv/' + args.arch+'_limit%d'%(args.limit) + '/rank_conv%d' % (1) + '.npy', feature_result.numpy())
    feature_result = torch.tensor(0.)
    total = torch.tensor(0.)

    cnt = 1
    # ResNet110 per block
    for i in range(3):
        block = eval('net.layer%d' % (i + 1))
        for j in range(18):
            cov_layer = block[j].relu1
            handler = cov_layer.register_forward_hook(get_feature_hook)
            test()
            handler.remove()
            np.save('rank_conv/' + args.arch  + '_limit%d' % (args.limit) + '/rank_conv%d' % (
            cnt + 1) + '.npy', feature_result.numpy())
            cnt += 1
            feature_result = torch.tensor(0.)
            total = torch.tensor(0.)

            cov_layer = block[j].relu2
            handler = cov_layer.register_forward_hook(get_feature_hook)
            test()
            handler.remove()
            np.save('rank_conv/' + args.arch  + '_limit%d' % (args.limit) + '/rank_conv%d' % (
                cnt + 1) + '.npy', feature_result.numpy())
            cnt += 1
            feature_result = torch.tensor(0.)
            total = torch.tensor(0.)

elif args.arch=='resnet_50':

    cov_layer = eval('net.maxpool')
    handler = cov_layer.register_forward_hook(get_feature_hook)
    test()
    handler.remove()

    if not os.path.isdir('rank_conv/' + args.arch+'_limit%d'%(args.limit)):
        os.mkdir('rank_conv/' + args.arch+'_limit%d'%(args.limit))
    np.save('rank_conv/' + args.arch+'_limit%d'%(args.limit) + '/rank_conv%d' % (1) + '.npy', feature_result.numpy())
    feature_result = torch.tensor(0.)
    total = torch.tensor(0.)

    # ResNet50 per bottleneck
    cnt=1
    for i in range(4):
        block = eval('net.layer%d' % (i + 1))
        for j in range(net.num_blocks[i]):
            cov_layer = block[j].relu1
            handler = cov_layer.register_forward_hook(get_feature_hook)
            test()
            handler.remove()
            np.save('rank_conv/' + args.arch+'_limit%d'%(args.limit) + '/rank_conv%d'%(cnt+1)+'.npy', feature_result.numpy())
            cnt+=1
            feature_result = torch.tensor(0.)
            total = torch.tensor(0.)

            cov_layer = block[j].relu2
            handler = cov_layer.register_forward_hook(get_feature_hook)
            test()
            handler.remove()
            np.save('rank_conv/' + args.arch + '_limit%d' % (args.limit) + '/rank_conv%d' % (cnt + 1) + '.npy',
                    feature_result.numpy())
            cnt += 1
            feature_result = torch.tensor(0.)
            total = torch.tensor(0.)

            cov_layer = block[j].relu3
            handler = cov_layer.register_forward_hook(get_feature_hook)
            test()
            handler.remove()
            if j==0:
                np.save('rank_conv/' + args.arch + '_limit%d' % (args.limit) + '/rank_conv%d' % (cnt + 1) + '.npy',
                        feature_result.numpy())#shortcut conv
                cnt += 1
            np.save('rank_conv/' + args.arch + '_limit%d' % (args.limit) + '/rank_conv%d' % (cnt + 1) + '.npy',
                    feature_result.numpy())#conv3
            cnt += 1
            feature_result = torch.tensor(0.)
            total = torch.tensor(0.)

max_stars_repo_name: lehduong/NPTM

max_issues_repo_name: lehduong/NPTM

max_forks_repo_name: lehduong/NPTM


text:
import numpy as np
from pom.constants import vertical_layers

class BfmStateVariableData:
    def __init__(self, current=np.zeros(vertical_layers), forward=np.zeros(vertical_layers), backward=np.zeros(vertical_layers),
                 surface_value=0, surface_flux=0, bottom_flux=0):
        self.current = current
        self.forward = forward
        self.backward = backward
        self.surface_value = surface_value
        self.surface_flux = surface_flux
        self.bottom_flux = bottom_flux

class NutrientData:
    def __init__(self,NO3surf,NH4surf,PO4surf,SIO4surf,O2bott,NO3bott,PO4bott,PONbott_grad):
        self.NO3surf = NO3surf
        self.NH4surf = NH4surf
        self.PO4surf = PO4surf
        self.SIO4surf = SIO4surf
        self.O2bott = O2bott
        self.NO3bott = NO3bott
        self.PO4bott = PO4bott
        self.PONbott_grad = PONbott_grad


text:
import numpy as np

def pca(X):
    """computes eigenvectors of the covariance matrix of X
      Returns the eigenvectors U, the eigenvalues (on diagonal) in S
    """

    # Useful values
    m, n = X.shape
    sigma = X.T.dot(X) / float(m)
    U, S, V = np.linalg.svd(sigma)
    return U, np.diag(S), V

max_stars_repo_name: robotenique/mlAlgorithms

max_issues_repo_name: robotenique/mlAlgorithms

max_forks_repo_name: robotenique/mlAlgorithms


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jun  7 17:44:28 2018

@author: vinusankars
"""

import cv2 as cv
import os
import pickle
import numpy as np

print('Initiated.')

def rot(img, ang):
    M = cv.getRotationMatrix2D((180,180), ang, 1)
    return cv.warpAffine(img, M, (360,360))

data = np.array([[[[0]*3]*36]*36], dtype='float16')
target = np.array([[0,0]], dtype='float16')
loc = os.getcwd()+'/dataSet/'

print('Accessing files...')
print('Processing...')

w = 0
b = 0

for i in os.listdir(loc):
    print('*',i)
    if i!= 'Normal':
        data1 = np.array([[[[0]*3]*36]*36], dtype='float16')
        target1 = np.array([[0,0]], dtype='float16')
        for j in os.listdir(loc+i+'/data/'):
            print(j)

            img = cv.imread(loc+i+'/data/'+j)
            img = cv.cvtColor(img, cv.COLOR_BGR2Lab)
            imm = cv.imread(loc+i+'/annotations/'+j.split('.')[0]+'m.png')
            break
#                for r in range(0, 360, 22):
#                    img1 = rot(img, r)
#                    imm1 = rot(imm, r)
#                    for x in range(0,360,36):
#                        for y in range(0,360,36):
#                            count = np.reshape(imm1[x: x+36, y: y+36],(-3,3)).tolist().count([255,255,255])
#                            if count >= 36*18:
#                                X = cv.normalize(img1[x: x+36, y: y+36],
#                                         img1[x: x+36, y: y+36],
#                                         alpha=0,
#                                         beta=1,
#                                         norm_type=cv.NORM_MINMAX,
#                                         dtype=cv.CV_32F)
#
#                                data1 = np.concatenate((data1, [X,X[::-1,:,:],X[:,::-1,:]]))
#                                target1 = np.concatenate((target1, [[0,1],[0,1],[0,1]]))
#                                w += 1
#
#                            if count < 36*18 and r==0:
#                                X = cv.normalize(img1[x: x+36, y: y+36],
#                                         img1[x: x+36, y: y+36],
#                                         alpha=0,
#                                         beta=1,
#                                         norm_type=cv.NORM_MINMAX,
#                                         dtype=cv.CV_32F)
#
#                                data1 = np.concatenate((data1, [X,X[::-1,:,:],X[:,::-1,:]]))
#                                target1 = np.concatenate((target1, [[1,0],[1,0],[1,0]]))
#                                b += 1
#            except:
#                continue
#
#        print('Pickling',i,'...',len(data1),w,b)
#        with open(i+'.pckl', 'wb') as f:
#            pickle.dump([data1[1:], target1[1:]], f)
#
##            data = np.concatenate((data, data1))
##            target = np.concatenate((target, target1))
#
#    else:
#        for j in os.listdir(loc+i+'/data/'):
#            print(j)
#            try:
#                img = cv.imread(loc+i+'/data/'+j)
#                img = cv.cvtColor(img, cv.COLOR_BGR2Lab)
#                for x in range(0,360,36):
#                    for y in range(0,360,36):
#                        b += 1
#
#                        X = cv.normalize(img[x: x+36, y: y+36],
#                                 img[x: x+36, y: y+36],
#                                 alpha=0,
#                                 beta=1,
#                                 norm_type=cv.NORM_MINMAX,
#                                 dtype=cv.CV_32F)
#
#                        data = np.concatenate((data, [X,X[::-1,:,:], X[:,::-1,:]]))
#                        target = np.concatenate((target, [[1,0],[1,0],[1,0]]))
#            except:
#                continue
#            with open('Normal.pckl', 'wb') as f:
#                pickle.dump([data[1:], target[1:]], f)
#
#print('Pickling...')
#


text:
"""Tokenizers, encoders, and decoders.

TODO: Should there be a special tokenizer for BERT sequences? It would
allow us to represent different encoding/decodings for BERT-like tasks.
"""

from abc import ABC, abstractmethod
from typing import List, Union

import numpy as np

from profit.utils.data_utils.vocabs import VOCABS

class BaseTokenizer(ABC):
    """Base tokenizer."""

    def __init__(self):
        pass

    @abstractmethod
    def encode(self, text: str) -> np.ndarray:
        """Encode text into ints (based off vocab)."""
        raise NotImplementedError

    @abstractmethod
    def decode(self, ids: List[int]) -> str:
        """Decode token ids back into tokens representation."""
        raise NotImplementedError

class AminoAcidTokenizer(BaseTokenizer):
    """Tokenizer for amino acid sequences."""

    def __init__(self, vocab: str):
        super(AminoAcidTokenizer, self).__init__()

        if vocab in VOCABS.keys():
            self.vocab = VOCABS[vocab]
        else:
            raise ValueError(f"vocab={vocab} is unavailable.")
        self.flipped_vocab = {v:k for k, v in self.vocab.items()}
        self._vocab_type = vocab

    @property
    def vocab_size(self) -> int:
        return len(self.vocab)

    @property
    def start_token(self) -> str:
        if "<cls>" in self.vocab:
            return "<cls>"
        raise RuntimeError(f"{self._vocab_type} vocab does not support BERT "
                           "classification (start) token.")

    @property
    def stop_token(self) -> str:
        if "<sep>" in self.vocab:
            return "<sep>"
        raise RuntimeError(f"{self._vocab_type} vocab does not support seperation.")

    @property
    def mask_token(self) -> str:
        if "<mask>" in self.vocab:
            return "<mask>"
        raise RuntimeError(f"{self._vocab_type} vocab does not support masking.")

    @property
    def pad_token(self) -> str:
        if "<pad>" in self.vocab:
            return "<pad>"
        raise RuntimeError(f"{self._vocab_type} vocab does not support padding.")

    @property
    def unknown_token(self) -> str:
        if "<unk>" in self.vocab:
            return "<unk>"
        raise RuntimeError(f"{self._vocab_type} vocab does not support unknown tokens.")

    @staticmethod
    def tokenize(text: str) -> List[str]:
        """Tokenize (seperate) input."""
        return list(text)

    def convert_token_to_id(self, token: str) -> int:
        """Converts a token (str/unicode) into an id using the vocab.

        NOTE: Tokens are case-sensitive, i.e. 'Ala' != 'ALA'.
        """
        try:
            return self.vocab[token]
        except KeyError:
            try:
                print(f"Unrecognized token `{token}`, using {self.unknown_token} instead!")
                return self.vocab[self.unknown_token]
            except RuntimeError as e:
                # If there is no unknown token, return an error (since we still
                # want to warn the user that the <unk> token was not used).
                raise RuntimeError(f"Unrecognized token `{token}` - {str(e)}")

    def convert_id_to_token(self, id_: int) -> str:
        """Converts an id into its token representation using the vocab."""
        try:
            return self.flipped_vocab[id_]
        except IndexError:
            raise IndexError(f"Unrecognized index: '{id_}'")

    def convert_tokens_to_ids(self, tokens: List[str]) -> List[int]:
        return [self.convert_token_to_id(token) for token in tokens]

    def add_special_tokens(self, token_ids: List[str]) -> List[str]:
        """Adds special tokens to the sequence.

        Used for sequence classification tasks (specifically BERT-like
        tasks). A BERT sequence has the following format: [CLS] X [SEP].
        """
        cls_token = [self.start_token]
        sep_token = [self.stop_token]
        return cls_token + token_ids + sep_token

    def encode(self, text: Union[str, List[str]]) -> np.ndarray:
        """Encode text into ints (based off vocab)."""
        tokens = AminoAcidTokenizer.tokenize(text) if isinstance(text, str) else text
        token_ids = self.convert_tokens_to_ids(tokens)
        return np.array(token_ids, dtype=np.float)

    def decode(self, ids: List[int]) -> List[str]:
        """Decode token ids back into tokens representation."""
        return [self.convert_id_to_token(id_) for id_ in ids]

max_stars_repo_name: ayushkarnawat/profit

max_issues_repo_name: ayushkarnawat/profit

max_forks_repo_name: ayushkarnawat/profit


text:
import numpy as np
import pylab as py
from scipy.fftpack import rfft
from scipy.signal import hann,resample

def tune_audio_data(data):
    C=130.81
    Cs=138.59
    D=146.83
    Ds=155.56
    E=164.81
    F=174.61
    Fs=185.0
    G=196.0
    Gs=207.65
    A=220.0
    As=233.08
    B=246.94

    scale = "None"
    scale1=np.array((C,Cs,D, Ds,E, F, Fs, G,Gs,A, As, B))
    scale2=scale1*2.0
    scale3=scale1/2.0

    if scale=="Fmin" or scale=="G#maj" or scale=="Abmaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((C,Cs,Ds,F,G,Gs,As))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="F#min" or scale=="Gbmin" or scale=="Amaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((Cs,D,E,Fs,Gs,A,B))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="Gmin" or scale=="Bbmaj" or scale=="A#maj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((C,D,Ds,F,G,A,As))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="G#min" or scale=="Abmin" or scale=="Bmaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((Cs,Ds,E,Fs,Gs,As,B))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="Amin" or scale=="Cmaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((C,D,E,F,G,A,B))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="A#min" or scale=="Bbmin" or scale=="C#maj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((C,Cs,Ds,F,Fs,Gs,As))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="Bmin" or scale=="Dmaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((Cs,D,E,Fs,G,A,B))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="Cmin" or scale=="D#maj" or scale=="Ebmaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((C,D,Ds,F,G,Gs,As))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="C#min" or scale=="Dbmin" or scale=="Emaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((Cs,Ds,E,Fs,Gs,A,B))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="Dmin" or scale=="Fmaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((C,D,E,F,G,A,As))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="D#min" or scale=="F#maj" or scale=="Gbmaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((Cs,Ds,F,Fs,Gs,As,B))
        scale2=scale1*2.0
        scale3=scale1/2.0
    if scale=="Emin" or scale=="Gmaj":
        #first scale starting on for example F3 as the middle octave
        scale1=np.array((C,D,E,Fs,G,A,B))
        scale2=scale1*2.0
        scale3=scale1/2.0
    #combine the octaves for one big frequency list reference.
    scale_comb=np.concatenate((scale1,scale2,scale3))

    block = len(data)
    sr=1

    #number of pieces
    pieces=len(data)//block

    #total length of data
    n=len(data)

    #size of each chunk
    chunk=n//pieces

    #zeros for zero padding
    nz=100000

    #array for the data blocks
    data_cut=np.zeros((pieces,n//pieces))

    #array for adjusted data which is resampled
    data_adj=[None]*pieces

    #array for x frequencies
    xfreq_adj=[None]*pieces

    #Arrays for FFT cuts
    fft_cut=np.zeros((pieces,nz+n//pieces))

    #adjusted FFT list
    fft_adj=[None]*pieces
    #xfreq array for frequency look up
    xfreq=np.linspace(0,sr//2,n//pieces+nz)

    #cut the data into pieces
    for i in range (pieces):
        data_cut[i][0:n//pieces]=data[i*(n//pieces):(i+1)*(n//pieces)]

    #find the fundamental frequency.
    def peakfind(x,y):
        return y[np.argmax(x)]

    #array containing the resampling factors
    factor=np.zeros(pieces)

    print('Tuning your input to the key of ' + scale + '...')
    for i in range (pieces):
        fft_cut[i]=rfft(data_cut[i],chunk+nz)
        xfreq_adj[i]=np.linspace(0,sr//2,len(fft_cut[i]))
        factor[i]=scale_comb[np.argmin(abs(peakfind(x=abs(fft_cut[i]),\
                        y=xfreq_adj[i])-scale_comb))]/(peakfind(abs(fft_cut[i]),\
                                                            y=xfreq_adj[i]))
        if factor[i]<0.8:
            factor[i]=1.0
        if factor[i]>1.2:
            factor[i]=1.0
        data_adj[i]=resample(data_cut[i],int(chunk/factor[i]))

    #connect all the pieces
    dataP=np.concatenate((data_adj))

    return dataP

max_stars_repo_name: hovjdev/CyprusVitalSigns

max_issues_repo_name: hovjdev/CyprusVitalSigns

max_forks_repo_name: hovjdev/CyprusVitalSigns

reason: import numpy,from scipy


text:
""" Module for display related operations. """
import warnings
import math

import numpy as np
import cv2

import tensorflow as tf

def grid_display(array, num_rows=None, num_columns=None):
    """
    Display a list of images as a grid.

    Args:
        array (numpy.ndarray): 4D Tensor (batch_size, height, width, channels)

    Returns:
        numpy.ndarray: 3D Tensor as concatenation of input images on a grid
    """
    if num_rows is not None and num_columns is not None:
        total_grid_size = num_rows * num_columns
        if total_grid_size < len(array):
            warnings.warn(
                Warning(
                    "Given values for num_rows and num_columns doesn't allow to display "
                    "all images. Values have been overrided to respect at least num_columns"
                )
            )
            num_rows = math.ceil(len(array) / num_columns)
    elif num_rows is not None:
        num_columns = math.ceil(len(array) / num_rows)
    elif num_columns is not None:
        num_rows = math.ceil(len(array) / num_columns)
    else:
        num_rows = math.ceil(math.sqrt(len(array)))
        num_columns = math.ceil(math.sqrt(len(array)))

    number_of_missing_elements = num_columns * num_rows - len(array)
    # We fill the array with np.zeros elements to obtain a perfect square
    array = np.append(
        array,
        np.zeros((number_of_missing_elements, *array[0].shape)).astype(array.dtype),
        axis=0,
    )

    grid = np.concatenate(
        [
            np.concatenate(
                array[index * num_columns : (index + 1) * num_columns], axis=1
            )
            for index in range(num_rows)
        ],
        axis=0,
    )

    return grid

def filter_display(array, num_rows=None, num_columns=None):
    """
    Display a list of filter outputs as a greyscale images grid.

    Args:
        array (numpy.ndarray): 4D Tensor (batch_size, height, width, channels)

    Returns:
        numpy.ndarray: 3D Tensor as concatenation of input images on a grid
    """
    return grid_display(
        np.concatenate(np.rollaxis(array, 3, 1), axis=0), num_rows, num_columns
    )

def display_all_imgs(array):
    """
    Display a list of filter outputs as a greyscale images.

    Args:
        array (numpy.ndarray): 4D Tensor (batch_size, height, width, channels)

    Returns:
        numpy.ndarray: 4D Tensor (batch_size, height, width, 1)
    """
    # The input array comes with a lot of channels (depending on the network architecture).
    # So, this function changes the batch_size and channels values.

    return np.swapaxes(array, 0, 3)

    """
    return np.reshape(
        array,
        (array.shape[0] * array.shape[3], array.shape[1], array.shape[2], 1)
    )
    """

def image_to_uint_255(image):
    """
    Convert float images to int 0-255 images.

    Args:
        image (numpy.ndarray): Input image. Can be either [0, 255], [0, 1], [-1, 1]

    Returns:
        numpy.ndarray:
    """
    if image.dtype == np.uint8:
        return image

    if image.min() < 0:
        image = (image + 1.0) / 2.0

    return (image * 255).astype("uint8")

def heatmap_display(
    heatmap, original_image, colormap=cv2.COLORMAP_VIRIDIS, image_weight=0.7
):
    """
    Apply a heatmap (as an np.ndarray) on top of an original image.

    Args:
        heatmap (numpy.ndarray): Array corresponding to the heatmap
        original_image (numpy.ndarray): Image on which we apply the heatmap
        colormap (int): OpenCV Colormap to use for heatmap visualization
        image_weight (float): An optional `float` value in range [0,1] indicating the weight of
            the input image to be overlaying the calculated attribution maps. Defaults to `0.7`

    Returns:
        np.ndarray: Original image with heatmap applied
    """
    heatmap = cv2.resize(heatmap, (original_image.shape[1], original_image.shape[0]))

    image = image_to_uint_255(original_image)

    heatmap = (heatmap - np.min(heatmap)) / (heatmap.max() - heatmap.min())

    heatmap = cv2.applyColorMap(
        cv2.cvtColor((heatmap * 255).astype("uint8"), cv2.COLOR_GRAY2BGR), colormap
    )

    output = cv2.addWeighted(
        cv2.cvtColor(image, cv2.COLOR_RGB2BGR), image_weight, heatmap, 1, 0
    )

    return cv2.cvtColor(output, cv2.COLOR_BGR2RGB)


text:
"""
External module tests
"""

import os
import pickle
import unittest

import numpy as np

from txtai.vectors import VectorsFactory

class TestExternalVectors(unittest.TestCase):
    """
    ExternalVectors tests
    """

    @classmethod
    def setUpClass(cls):
        """
        Create single ExternalVectors instance.
        """

        cls.model = VectorsFactory.create({"method": "external"}, None)

    def testIndex(self):
        """
        Test indexing with external vectors
        """

        # Generate dummy data
        data = np.random.rand(1000, 768)

        # Generate enough volume to test batching
        documents = [(x, data[x], None) for x in range(1000)]

        ids, dimension, batches, stream = self.model.index(documents)

        self.assertEqual(len(ids), 1000)
        self.assertEqual(dimension, 768)
        self.assertEqual(batches, 2)
        self.assertIsNotNone(os.path.exists(stream))

        # Test shape of serialized embeddings
        with open(stream, "rb") as queue:
            self.assertEqual(pickle.load(queue).shape, (500, 768))

max_stars_repo_name: techthiyanes/txtai


text:
# coding: utf-8

# 不加正则化项的logistic回归,代码进行了简化处理

import numpy as np
import pandas as pd

class LogisticRegression():
    def __init__(self, alpha=1.0, epsilon=0.01, maxIteration=1000):
        self.alpha = float(alpha)
        self.epsilon = float(epsilon)
        self.maxIteration = int(maxIteration)
        self.theta = None

    def fit(self, X_, Y):
        m = X_.shape[0]
        n = X_.shape[1] + 1
        X = np.hstack((np.ones((m,1)), X_))
        self.theta = np.zeros((n, 1))
        for i in range(self.maxIteration):
            dTheta = -(X.T @ (Y - self.sigmoid(X))) / m
            self.theta -= self.alpha * dTheta
            if np.linalg.norm(dTheta) <= self.epsilon:
                break
        acc = 1 - np.count_nonzero(Y - self.predict(X_)) / m
        print(acc, i)

    def sigmoid(self, X):
        return 1. / (1. + np.exp(-(X @ self.theta)))

    def predict(self, X):
        X = np.hstack((np.ones((X.shape[0],1)), X))
        return (self.sigmoid(X) > 0.5) + 0


text:
import numpy as np
import numpy.random as npr
import os

data_dir = '.'
#anno_file = os.path.join(data_dir, "anno.txt")

size = 48

if size == 12:
    net = "PNet"
elif size == 24:
    net = "RNet"
elif size == 48:
    net = "ONet"

with open(os.path.join(data_dir, '%s/pos_%s.txt' % (size, size)), 'r') as f:
    pos = f.readlines()

with open(os.path.join(data_dir, '%s/neg_%s.txt' % (size, size)), 'r') as f:
    neg = f.readlines()

with open(os.path.join(data_dir, '%s/part_%s.txt' % (size, size)), 'r') as f:
    part = f.readlines()

with open(os.path.join(data_dir, '%s/landmark_%s_aug.txt' % (size, size)), 'r') as f:
    landmark = f.readlines()

dir_path = os.path.join(data_dir, 'imglists',"ONet")
if not os.path.exists(dir_path):
    os.makedirs(dir_path)
#write all data
with open(os.path.join(dir_path, "train_%s_landmark.txt" % (net)), "w") as f:
    print (len(neg))
    print (len(pos))
    print (len(part))
    print (len(landmark))
    for i in np.arange(len(pos)):
        f.write(pos[i])
    for i in np.arange(len(neg)):
        f.write(neg[i])
    for i in np.arange(len(part)):
        f.write(part[i])
    for i in np.arange(len(landmark)):
        f.write(landmark[i])


text:
import json
import base64
import cv2
import numpy as np
import http.client
import time

keypoint_connections = [
    ["right_eye", "right_ear"],
    ["left_eye", "left_ear"],
    ["nose", "right_eye"],
    ["nose", "left_eye"],
    ["neck", "nose"],
    ["neck", "right_shoulder"],
    ["neck", "left_shoulder"],
    ["right_shoulder", "right_elbow"],
    ["left_shoulder", "left_elbow"],
    ["right_elbow", "right_wrist"],
    ["left_elbow", "left_wrist"],
    ["neck", "right_hip"],
    ["neck", "left_hip"],
    ["right_hip", "right_knee"],
    ["left_hip", "left_knee"],
    ["right_knee", "right_ankle"],
    ["left_knee", "left_ankle"]
]

def render_result(skeletons, img, confidence_threshold):
    skeleton_color = (100, 254, 213)
    for skeleton in skeletons["skeletons"]:
        for connection in keypoint_connections:
            if skeleton["keypoints"][connection[0]]["coordinates"]["x"] > 0:
                if skeleton["keypoints"][connection[1]]["coordinates"]["x"] > 0:
                    pt_1 = (
                        int(skeleton["keypoints"][connection[0]]["coordinates"]["x"]),
                        int(skeleton["keypoints"][connection[0]]["coordinates"]["y"])
                    )
                    pt_2 = (
                        int(skeleton["keypoints"][connection[1]]["coordinates"]["x"]),
                        int(skeleton["keypoints"][connection[1]]["coordinates"]["y"])
                    )
                    cv2.line( img, pt_1, pt_2, skeleton_color, thickness=2, lineType=cv2.LINE_AA )

        actions = skeleton["actions"]
        if len(actions) > 0:
            start_pos_y = 40
            for action in actions:
                cv2.putText(frame, action, (20,start_pos_y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,0,0), 1, cv2.LINE_AA)
                start_pos_y = start_pos_y + 20

try:
    # connect to the cubemos API
    conn = http.client.HTTPSConnection("api.cubemos.com")

    # start the webcam
    cap = cv2.VideoCapture(0)
    hasFrame, frame = cap.read()

    # continous acquisition
    frame_count = 0
    fps = 0
    fps_text = "calculating fps"
    while hasFrame:
        hasFrame, frame = cap.read()

        start_time = time.time()
        # get the opencv frame as a base64 byte stream
        retval, buffer = cv2.imencode('.jpg', frame)
        byte_stream = base64.b64encode(cv2.imencode('.jpg', frame)[1]).decode()

        # provide the input payload
        payload = {'image': {'bytes_b64': byte_stream}}
        headers = { 'x-api-key': "YOUR-API-KEY" }

        # make the POST call and get the response
        conn.request("POST", "/skeletons/estimate", json.dumps(payload), headers)
        res = conn.getresponse()
        data = res.read()

        if res.status == 200:
            skeletons = data.decode("utf-8")
            skeletons = json.loads(skeletons)

            # measure the fps for displaying
            end_time = time.time() - start_time

            fps = fps + (1/end_time)
            if frame_count == 10:
                fps = fps/frame_count
                fps_text = str(round(fps, 2)) + "fps"
                frame_count = 0
                fps = 0

            # render the result onto the image using the response obtained
            render_result(skeletons, frame, 0.1)
            cv2.putText(frame, fps_text, (20,20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,0,0), 1, cv2.LINE_AA)

        cv2.imshow("cubemos skeleton estimation api", frame)

        keyPressed = cv2.waitKey(1)
        if keyPressed == 27:
            break;
        frame_count = frame_count + 1

finally:
    cv2.destroyAllWindows()


text:
import scipy as sp
import numpy as np

class MathHelper:
    @staticmethod
    def sigmoid(x):
        return 1. / (1 + sp.exp(-x))

    def sigmoid_grad(self, x):
        sig = self.sigmoid(x)
        return np.multiply(sig, 1 - sig)

max_stars_repo_name: sasoana/multiclassANN

max_issues_repo_name: sasoana/multiclassANN

max_forks_repo_name: sasoana/multiclassANN

reason: import numpy,import scipy


text:
import os
import glob
import random
from PIL import Image
import numpy as np
import trimesh
from im2mesh.data.core import Field
import torch
import time

class IndexField(Field):
    ''' Basic index field.'''
    # def load(self, model_path, idx, category):

    def load(self, model_path, idx, category, start_idx=0, **kwargs):
        ''' Loads the index field.

        Args:
            model_path (str): path to model
            idx (int): ID of data point
            category (int): index of category
            start_idx (int): id of sequence start
        '''
        return idx

    def check_complete(self, files):
        ''' Check if field is complete.

        Args:
            files: files
        '''
        return True

class CategoryField(Field):
    ''' Basic category field.'''

    def load(self, model_path, idx, category):
        ''' Loads the category field.

        Args:
            model_path (str): path to model
            idx (int): ID of data point
            category (int): index of category
        '''
        return category

    def check_complete(self, files):
        ''' Check if field is complete.

        Args:
            files: files
        '''
        return True

class PointsSubseqField(Field):
    ''' Points subsequence field class.

    Args:
        folder_name (str): points folder name
        transform (transform): transform
        seq_len (int): length of sequence
        all_steps (bool): whether to return all time steps
        fixed_time_step (int): if and which fixed time step to use
        unpackbits (bool): whether to unpack bits
    '''

    def __init__(self, folder_name, transform=None, seq_len=17, select_steps=None,
                 all_steps=False, fixed_time_step=None, unpackbits=False, return_loc_scale=False,
                 **kwargs):
        self.folder_name = folder_name
        self.transform = transform
        self.seq_len = seq_len
        self.select_steps = select_steps
        self.all_steps = all_steps
        self.sample_padding = 0.1
        self.fixed_time_step = fixed_time_step
        self.unpackbits = unpackbits
        self.return_loc_scale = return_loc_scale

    def get_loc_scale(self, mesh):
        ''' Returns location and scale of mesh.

        Args:
            mesh (trimesh): mesh
        '''
        bbox = mesh.bounding_box.bounds

        # Compute location and scale with padding of 0.1
        loc = (bbox[0] + bbox[1]) / 2
        scale = (bbox[1] - bbox[0]).max() / (1 - self.sample_padding)

        return loc, scale

    def normalize_mesh(self, mesh, loc, scale):
        ''' Normalize mesh.

        Args:
            mesh (trimesh): mesh
            loc (tuple): location for normalization
            scale (float): scale for normalization
        '''
        # Transform input mesh
        mesh.apply_translation(-loc)
        mesh.apply_scale(1 / scale)

        return mesh

    def load_files(self, model_path, start_idx):
        ''' Loads the model files.

        Args:
            model_path (str): path to model
            start_idx (int): id of sequence start
        '''
        folder = os.path.join(model_path, self.folder_name)
        files = glob.glob(os.path.join(folder, '*.npz'))
        files.sort()
        files = files[start_idx:start_idx+self.seq_len]

        return files

    def load_all_steps(self, files, points_dict, loc0, scale0):
        ''' Loads data for all steps.

        Args:
            files (list): list of files
            points_dict (dict): points dictionary for first step of sequence
            loc0 (tuple): location of first time step mesh
            scale0 (float): scale of first time step mesh
        '''
        p_list = []
        o_list = []
        t_list = []
        loc_seq = []
        scale_seq = []
        for i, f in enumerate(files):
            if self.select_steps is not None:
                if i not in self.select_steps:
                    continue
            points_dict = np.load(f)

            # Load points
            points = points_dict['points']
            if (points.dtype == np.float16):
                # break symmetry (nec. for some version?)
                points = points.astype(np.float32)
                points += 1e-4 * np.random.randn(*points.shape)
            points = points.astype(np.float32)
            occupancies = points_dict['occupancies']
            if self.unpackbits:
                occupancies = np.unpackbits(occupancies)[:points.shape[0]]
            occupancies = occupancies.astype(np.float32)
            loc = points_dict['loc'].astype(np.float32)
            scale = points_dict['scale'].astype(np.float32)
            # Transform to loc0, scale0
            points = (loc + scale * points - loc0) / scale0
            if self.seq_len > 1:
                time = np.array(i / (self.seq_len - 1), dtype=np.float32)
            else:
                time = np.array(0, dtype=np.float32)

            p_list.append(points)
            o_list.append(occupancies)
            t_list.append(time)
            loc_seq.append(loc)
            scale_seq.append(scale)

        data = {
            None: np.stack(p_list),
            'occ': np.stack(o_list),
            'time': np.stack(t_list),
        }
        if self.return_loc_scale:
            data['loc'] = np.stack(loc_seq)
            data['scale'] = np.array(scale_seq)
        return data

    def load_single_step(self, files, points_dict, loc0, scale0):
        ''' Loads data for a single step.

        Args:
            files (list): list of files
            points_dict (dict): points dictionary for first step of sequence
            loc0 (tuple): location of first time step mesh
            scale0 (float): scale of first time step mesh
        '''
        if self.fixed_time_step is None:
            # Random time step
            if self.select_steps is not None:
                len_select = len(self.select_steps)
                time_step = self.select_steps[np.random.choice(len_select)]
            else:
                time_step = np.random.choice(self.seq_len)
        else:
            time_step = int(self.fixed_time_step)

        if time_step != 0:
            points_dict = np.load(files[time_step])
        # Load points
        points = points_dict['points'].astype(np.float32)
        occupancies = points_dict['occupancies']
        if self.unpackbits:
            occupancies = np.unpackbits(occupancies)[:points.shape[0]]
        occupancies = occupancies.astype(np.float32)
        loc = points_dict['loc'].astype(np.float32)
        scale = points_dict['scale'].astype(np.float32)
        # Transform to loc0, scale0
        points = (loc + scale * points - loc0) / scale0

        if self.seq_len > 1:
            time = np.array(
                time_step / (self.seq_len - 1), dtype=np.float32)
        else:
            time = np.array([0], dtype=np.float32)

        data = {
            None: points,
            'occ': occupancies,
            'time': time,
        }
        return data

    def load(self, model_path, idx, c_idx=None, start_idx=0, **kwargs):
        ''' Loads the points subsequence field.

        Args:
            model_path (str): path to model
            idx (int): ID of data point
            category (int): index of category
            start_idx (int): id of sequence start
        '''
        files = self.load_files(model_path, start_idx)
        # Load loc and scale from t_0
        points_dict = np.load(files[0])
        loc0 = points_dict['loc'].astype(np.float32)
        scale0 = points_dict['scale'].astype(np.float32)

        if self.all_steps:
            data = self.load_all_steps(files, points_dict, loc0, scale0)
        else:
            data = self.load_single_step(files, points_dict, loc0, scale0)

        if self.transform is not None:
            data = self.transform(data)
        return data

class ImageSubseqField(Field):
    ''' Image subsequence field class.

    Args:
        folder_name (str): points folder name
        transform (transform): transform
        seq_len (int): length of sequence
        extension (str): image extension
        random_view (bool): whether to return a random view
        only_end_points (bool): whether to only return end points
    '''

    def __init__(self, folder_name, transform=None, seq_len=17,
                 extension='jpg', random_view=True, only_end_points=False,
                 **kwargs):
        self.folder_name = folder_name
        self.transform = transform
        self.seq_len = seq_len
        self.extension = extension
        self.only_end_points = only_end_points
        self.random_view = random_view
        self.num_views = 4

    def get_img_seq_idx(self):
        ''' Returns image sequence idx.
        '''
        if self.random_view:
            idx_img_seq = random.randint(0, self.num_views - 1)
        else:
            idx_img_seq = 0
        return idx_img_seq

    def get_file_paths(self, model_path, start_idx):
        ''' Returns file paths.

        Args:
            model_path (str): model path
            start_idx (int): id of sequence start
        '''
        idx_img_seq = self.get_img_seq_idx()
        folder = os.path.join(model_path, self.folder_name)
        folder = os.path.join(folder, '%03d' % idx_img_seq)
        files = glob.glob(os.path.join(folder, '*.%s' % self.extension))
        files.sort()
        files = files[start_idx:start_idx+self.seq_len]
        if self.only_end_points:
            files = [files[0], files[-1]]

        return files

    def load(self, model_path, idx, c_idx=None, start_idx=0, **kwargs):
        ''' Loads the image sequence field.

        Args:
            model_path (str): path to model
            idx (int): ID of data point
            c_idx (int): index of category
            start_idx (int): id of sequence start
        '''
        files = self.get_file_paths(model_path, start_idx)
        imgs = []
        for f in files:
            image = Image.open(f).convert('RGB')
            if self.transform is not None:
                image = self.transform(image)
            imgs.append(image)

        data = {
            None: torch.stack(imgs),
        }

        return data

class PointCloudSubseqField(Field):
    ''' Point cloud subsequence field class.

    Args:
        folder_name (str): points folder name
        transform (transform): transform
        seq_len (int): length of sequence
        only_end_points (bool): whether to only return end points
        scale_pointcloud (bool): whether to scale the point cloud
            w.r.t. the first point cloud of the sequence
    '''

    def __init__(self, folder_name, transform=None, seq_len=17, select_steps=None,
                 only_end_points=False,
                 scale_pointcloud=True,
                 return_loc_scale=False):
        self.folder_name = folder_name
        self.transform = transform
        self.seq_len = seq_len
        self.select_steps = select_steps
        self.only_end_points = only_end_points
        self.scale_pointcloud = scale_pointcloud
        self.return_loc_scale = return_loc_scale

    # def return_loc_scale(self, mesh):
    #     ''' Returns location and scale of mesh.

    #     Args:
    #         mesh (trimesh): mesh
    #     '''
    #     bbox = mesh.bounding_box.bounds
    #     # Compute location and scale
    #     loc = (bbox[0] + bbox[1]) / 2
    #     scale = (bbox[1] - bbox[0]).max() / (1 - 0)
    #     return loc, scale

    # def apply_normalization(self, mesh, loc, scale):
    #     ''' Normalizes the mesh.

    #     Args:
    #         mesh (trimesh): mesh
    #         loc (tuple): location for normalization
    #         scale (float): scale for normalization
    #     '''
    #     mesh.apply_translation(-loc)
    #     mesh.apply_scale(1/scale)
    #     return mesh

    def load_files(self, model_path, start_idx):
        ''' Loads the model files.

        Args:
            model_path (str): path to model
            start_idx (int): id of sequence start
        '''
        folder = os.path.join(model_path, self.folder_name)
        files = glob.glob(os.path.join(folder, '*.npz'))
        files.sort()
        files = files[start_idx:start_idx+self.seq_len]

        if self.only_end_points:
            files = [files[0], files[-1]]

        return files

    def load_single_file(self, file_path):
        ''' Loads a single file.

        Args:
            file_path (str): file path
        '''
        pointcloud_dict = np.load(file_path)
        points = pointcloud_dict['points'].astype(np.float32)
        loc = pointcloud_dict['loc'].astype(np.float32)
        scale = pointcloud_dict['scale'].astype(np.float32)

        return points, loc, scale

    def get_time_values(self):
        ''' Returns the time values.
        '''
        if self.seq_len > 1:
            time = \
                np.array([i/(self.seq_len - 1) for i in range(self.seq_len)],
                         dtype=np.float32)
        else:
            time = np.array([0]).astype(np.float32)
        return time

    def load(self, model_path, idx, c_idx=None, start_idx=0, **kwargs):
        ''' Loads the point cloud sequence field.

        Args:
            model_path (str): path to model
            idx (int): ID of data point
            c_idx (int): index of category
            start_idx (int): id of sequence start
        '''
        pc_seq = []
        loc_seq = []
        scale_seq = []
        # Get file paths
        files = self.load_files(model_path, start_idx)
        # Load first pcl file
        _, loc0, scale0 = self.load_single_file(files[0])
        for i, f in enumerate(files):
            if self.select_steps is not None:
                if i not in self.select_steps:
                    continue
            points, loc, scale = self.load_single_file(f)
            # Transform mesh to loc0 / scale0
            if self.scale_pointcloud:
                points = (loc + scale * points - loc0) / scale0

            pc_seq.append(points)
            loc_seq.append(loc)
            scale_seq.append(scale)

        data = {
            None: np.stack(pc_seq),
            'time': self.get_time_values(),
        }
        if self.return_loc_scale:
            data['loc'] = np.stack(loc_seq)
            data['scale'] = np.array(scale_seq)

        if self.transform is not None:
            data = self.transform(data)
        return data

class MeshSubseqField(Field):
    ''' Mesh subsequence field class.

    Args:
        folder_name (str): points folder name
        seq_len (int): length of sequence
        only_end_points (bool): whether to only return end points
        only_start_point (bool): whether to only return the start point
        scale (bool): whether to scale the meshes w.r.t. the first mesh of the
            sequence
        file_ext (str): mesh file extension
    '''

    def __init__(self, folder_name, seq_len=17, only_end_points=False, select_steps=None,
                 only_start_point=False, scale=True, file_ext='obj'):
        self.folder_name = folder_name
        self.seq_len = seq_len
        self.only_end_points = only_end_points
        self.only_start_point = only_start_point
        self.scale = scale
        self.file_ext = file_ext
        self.select_steps = select_steps

    def return_loc_scale(self, mesh):
        ''' Returns location and scale of mesh.

        Args:
            mesh (trimesh): mesh
        '''
        bbox = mesh.bounding_box.bounds
        # Compute location and scale
        loc = (bbox[0] + bbox[1]) / 2
        scale = (bbox[1] - bbox[0]).max() / (1 - 0)
        return loc, scale

    def apply_normalization(self, mesh, loc, scale):
        ''' Normalizes the mesh.

        Args:
            mesh (trimesh): mesh
            loc (tuple): location for normalization
            scale (float): scale for normalization
        '''
        mesh.apply_translation(-loc)
        mesh.apply_scale(1/scale)
        return mesh

    def load(self, model_path, idx, c_idx=None, start_idx=0, **kwargs):
        ''' Loads the mesh sequence field.

        Args:
            model_path (str): path to model
            idx (int): ID of data point
            c_idx (int): index of category
            start_idx (int): id of sequence start
        '''
        folder = os.path.join(model_path, self.folder_name)
        mesh_files = glob.glob(os.path.join(folder, '*.%s' % self.file_ext))
        mesh_files.sort()
        mesh_files = mesh_files[start_idx:start_idx+self.seq_len]
        if self.only_end_points:
            mesh_files = [mesh_files[0], mesh_files[-1]]
        elif self.only_start_point:
            mesh_files = mesh_files[[0]]
        elif self.select_steps is not None:
            select_mesh_files = []
            for index in self.select_steps:
                #index = self.select_steps[i]
                select_mesh_files.append(mesh_files[index])

        if self.scale:
            mesh_0 = trimesh.load(mesh_files[0], process=False)
            loc, scale = self.return_loc_scale(mesh_0)

            vertices = []
            for mesh_p in mesh_files:
                mesh = trimesh.load(mesh_p, process=False)
                mesh = self.apply_normalization(mesh, loc, scale)
                vertices.append(np.array(mesh.vertices, dtype=np.float32))

        faces = np.array(trimesh.load(
            mesh_files[0], process=False).faces, dtype=np.float32)
        data = {
            'vertices': np.stack(vertices),
            'faces': faces
        }

        return data

max_stars_repo_name: tangjiapeng/LPDC-Net

max_issues_repo_name: Gorilla-Lab-SCUT/LPDC-Net

max_forks_repo_name: tangjiapeng/LPDC-Net