text:
import numpy as np
import theano
import theano.tensor as T

import lasagne
from lasagne.utils import floatX

class TemperatureSoftmax (object):
    """
    A softmax function with a temperature setting; increasing it smooths the resulting probabilities.
    """
    def __init__(self, temperature=1.0):
        self._temperature = theano.shared(lasagne.utils.floatX(temperature), 'temperature')

    @property
    def temperature(self):
        return 1.0 / self._temperature.get_value()

    @temperature.setter
    def temperature(self, value):
        self._temperature.set_value(lasagne.utils.floatX(1.0 / value))

    def __call__(self, x):
        return lasagne.nonlinearities.softmax(x * self._temperature)

def _flatten_spatial_lasagne_layer(layer):
    out_shape = layer.output_shape
    ndims = len(out_shape)
    spatial = tuple(range(2, ndims))
    n_channels = out_shape[1]
    spatial_shape = out_shape[2:]
    if ndims > 2:
        # (Sample,Channel,Spatial...) -> (Channel,Sample,Spatial...)
        layer = lasagne.layers.DimshuffleLayer(layer, (1, 0) + spatial)
        # (Channel,Sample,spatial...) -> (Channel,Sample:spatial...)
        layer = lasagne.layers.FlattenLayer(layer, outdim=2)
        # (Channel,Sample:spatial...) -> (Sample:spatial...,Channel)
        layer = lasagne.layers.DimshuffleLayer(layer, (1, 0))
    return layer, spatial_shape, n_channels

def _flatten_spatial_theano(x, n_channels):
    ndim = x.ndim
    if n_channels is None:
        if ndim > 1:
            # (Sample,Spatial...) -> (Sample:Spatial...)
            x = x.flatten()
    else:
        if ndim > 2:
            spatial = tuple(range(2, ndim))
            # (Sample,Channel,Spatial...) -> (Sample,Spatial...,Channel)
            x = x.dimshuffle((0,) + spatial + (1,))
            # (Sample,Spatial...,Channel) -> (Sample:Spatial...,Channel)
            x = x.reshape((-1, n_channels))
    return x

def _unflatten_spatial_theano(x, spatial_shape, n_channels):
    n_spatial = len(spatial_shape)
    if n_spatial > 0:
        if n_channels is None:
            # (Sample:spatial...) -> (Sample,Spatial...)
            x = x.reshape((-1,) + spatial_shape)
        else:
            # (Sample:spatial...,Channel) -> (Sample,Spatial...,Channel)
            x = x.reshape((-1,) + spatial_shape + (n_channels,))
            # (Sample,Spatial...,Channel) -> (Sample,Channel,Spatial...)
            x = x.dimshuffle((0, n_spatial+1,) + tuple(range(1,n_spatial+1)))
    return x

def _flatten_spatial(x, n_channels):
    ndim = len(x.shape)
    if n_channels is None:
        if ndim > 1:
            # (Sample,Spatial...) -> (Sample:Spatial...)
            x = x.reshape((-1,))
    else:
        if ndim > 2:
            # (Sample,Channel,Spatial...) -> (Sample,Spatial...,Channel)
            x = np.rollaxis(x, 1, ndim)
            # (Sample,Spatial...,Channel) -> (Sample:Spatial...,Channel)
            x = x.reshape((-1, n_channels))
    return x

def _unflatten_spatial(x, spatial_shape, n_channels):
    n_spatial = len(spatial_shape)
    if n_spatial > 0:
        if n_channels is None:
            # (Sample:spatial...) -> (Sample,Spatial...)
            x = x.reshape((-1,) + spatial_shape)
        else:
            # (Sample:spatial...,Channel) -> (Sample,Spatial...,Channel)
            x = x.reshape((-1,) + spatial_shape + (n_channels,))
            # (Sample,Spatial...,Channel) -> (Sample,Channel,Spatial...)
            x = np.rollaxis(x, 3, 1)
    return x

class ObjectiveOutput (object):
    def __init__(self, train_cost, train_results, train_results_str_fn,
                 eval_results, eval_results_str_fn, prediction):
        self.train_cost = train_cost
        self.train_results = train_results
        self.train_results_str_fn = train_results_str_fn
        self.eval_results = eval_results
        self.eval_results_str_fn = eval_results_str_fn
        self.prediction = prediction

class AbstractObjective (object):
    transform_input_target = None
    inv_transform_prediction = None

    def __init__(self, name, cost_weight=1.0):
        """
        Abstract objective

        :param name: name of objective
        :param cost_weight: (default=1.0) weight applied to this objectives cost
        """
        self.name = name
        self.cost_weight = lasagne.utils.floatX(cost_weight)

    def build(self):
        raise NotImplementedError('Abstract for {}'.format(type(self)))

    def score_improved(self, new_results, best_so_far_results):
        raise NotImplementedError('Abstract for {}'.format(type(self)))

class ClassifierObjective (AbstractObjective):
    SCORE_ERROR = 'err'
    SCORE_JACCARD = 'jaccard'
    SCORE_PRECISION = 'precision'
    SCORE_RECALL = 'recall'
    SCORE_F1 = 'f1'

    def __init__(self, name, objective_layer, target_expr, mask_expr=None, n_target_spatial_dims=0,
                 target_channel_index=None, score=SCORE_ERROR, includes_softmax=False, cost_weight=1.0):
        """
        Multi-class classifier objective

        :param name: objective name
        :param objective_layer: Lasagne layer that with a softmax non-linearity added to it will predict the class
        probabilities
        :param target_expr: ground truth target expression
        :param mask_expr: [optional] mask expression
        :param n_target_spatial_dims: (default=0) number of spatial dimensions for the target
        :param target_channel_index: (default=None) if the target has a channel dimension, this is the index into
        that channel for this objective
        :param score: (default=`'err'`) how to evaluate this objective; one of `'err'`, `'jaccard'`, `'precision'`,
        `'recall'`, `'f1'` or use the constants `ClassifierObjective.SCORE_ERROR`, `ClassifierObjective.SCORE_JACCARD`,
        `ClassifierObjective.SCORE_PRECISION`, `ClassifierObjective.SCORE_RECALL`, `ClassifierObjective.SCORE_F1`
        respectively
        :param includes_softmax: `True` indicates that the objective_layer includes the softmax non-linearity,
        `False` indicates that it does not, in which case a non-linearity layer will be added
        :param cost_weight: (default=1.0) weight applied to the cost of this objective
        """
        super(ClassifierObjective, self).__init__(name, cost_weight)
        self.objective_layer = objective_layer
        self.target_expr = target_expr
        self.mask_expr = mask_expr
        self.n_target_spatial_dims = n_target_spatial_dims
        self.target_channel_index = target_channel_index
        self.score = score
        # The value of the `softmax` attribute indicates if the network came with it already in place
        if includes_softmax:
            self.softmax = None
        else:
            self.softmax = TemperatureSoftmax()

    @property
    def temperature(self):
        if self.softmax is not None:
            return self.softmax.temperature
        else:
            return 1.0

    @temperature.setter
    def temperature(self, t):
        if self.softmax is not None:
            self.softmax.temperature = t

    def build(self):
        if self.target_channel_index is not None:
            target_expr = self.target_expr[:,self.target_channel_index]
            mask_expr = self.mask_expr[:,self.target_channel_index] if self.mask_expr is not None else None
        else:
            target_expr = self.target_expr
            mask_expr = self.mask_expr

        flat_target = _flatten_spatial_theano(target_expr, None)
        flat_mask = _flatten_spatial_theano(mask_expr, None) if self.mask_expr is not None else None

        # Flatten the objective layer (if the objective layer generates an
        # output with 2 dimensions then this is a no-op)
        obj_flat_layer, spatial_shape, n_classes = \
            _flatten_spatial_lasagne_layer(self.objective_layer)

        n_spatial = np.prod(spatial_shape)
        inv_n_spatial = lasagne.utils.floatX(1.0 / float(n_spatial))

        # Predicted probability layer
        if self.softmax is not None:
            prob_layer = lasagne.layers.NonlinearityLayer(obj_flat_layer, self.softmax)
        else:
            prob_layer = obj_flat_layer

        # Get an expression representing the predicted probability
        train_pred_prob = lasagne.layers.get_output(prob_layer)

        # Create a loss expression for training, i.e., a scalar objective we want
        # to minimize (for our multi-class problem, it is the cross-entropy loss):
        train_loss = lasagne.objectives.categorical_crossentropy(train_pred_prob, flat_target)
        if flat_mask is not None:
            train_loss = train_loss * flat_mask

        # Create prediction expressions; use deterministic forward pass (disable
        # dropout layers)
        eval_pred_prob = lasagne.layers.get_output(prob_layer, deterministic=True)
        # Create evaluation loss expression
        eval_loss = lasagne.objectives.categorical_crossentropy(eval_pred_prob, flat_target)
        if flat_mask is not None:
            eval_loss = eval_loss * flat_mask
        # Predicted class
        eval_pred_cls = T.argmax(eval_pred_prob, axis=1)

        eval_scores = []

        if self.score == self.SCORE_ERROR:
            # Create an expression for error count
            errors = T.neq(eval_pred_cls, flat_target).astype(theano.config.floatX)
            if flat_mask is not None:
                errors = errors * flat_mask

            if self.n_target_spatial_dims == 0:
                eval_scores = [errors.sum()]
            else:
                eval_scores = [errors.sum() * inv_n_spatial]
        elif self.score in {self.SCORE_JACCARD, self.SCORE_PRECISION, self.SCORE_RECALL, self.SCORE_F1}:
            for cls_i in range(n_classes):
                truth = T.eq(flat_target, cls_i).astype(theano.config.floatX)
                pred = T.eq(eval_pred_cls, cls_i).astype(theano.config.floatX)
                inv_truth = T.neq(flat_target, cls_i).astype(theano.config.floatX)
                inv_pred = T.neq(eval_pred_cls, cls_i).astype(theano.config.floatX)
                true_neg = inv_truth * inv_pred
                true_pos = truth * pred
                false_neg = truth * inv_pred
                false_pos = inv_truth * pred
                if flat_mask is not None:
                    true_neg = true_neg * flat_mask
                    true_pos = true_pos * flat_mask
                    false_neg = false_neg * flat_mask
                    false_pos = false_pos * flat_mask

                eval_scores.extend([true_neg.sum(), false_neg.sum(), false_pos.sum(), true_pos.sum()])
        else:
            raise ValueError('score is not valid ({})'.format(self.score))

        if self.n_target_spatial_dims == 0:
            train_loss_batch = train_loss.sum()
            eval_loss_batch = eval_loss.sum()
        else:
            train_loss_batch = train_loss.sum() * inv_n_spatial
            eval_loss_batch = eval_loss.sum() * inv_n_spatial

        # Unflatten prediction
        pred_prob = _unflatten_spatial_theano(eval_pred_prob, spatial_shape, n_classes)

        def train_results_str_fn(train_res):
            return '{} loss={:.6f}'.format(self.name, train_res[0])

        def eval_results_str_fn(eval_res):
            return '{} loss={:.6f} {}={:.2%}'.format(self.name, eval_res[0], self.score, self._compute_score(eval_res))

        return ObjectiveOutput(train_cost=train_loss.mean() * self.cost_weight,
                               train_results=[train_loss_batch],
                               train_results_str_fn=train_results_str_fn,
                               eval_results=[eval_loss_batch] + eval_scores,
                               eval_results_str_fn=eval_results_str_fn,
                               prediction=pred_prob)

    def _score_frac(self, numerator, denominator):
        if denominator == 0.0:
            return 0.0
        else:
            return numerator / denominator

    def _compute_score(self, eval_results):
        if self.score == self.SCORE_ERROR:
            return eval_results[1]
        elif self.score in {self.SCORE_JACCARD, self.SCORE_PRECISION, self.SCORE_RECALL, self.SCORE_F1}:
            components = eval_results[1:]
            cls_scores = []
            for cls_i in range(len(components) // 4):
                true_neg = components[cls_i * 4 + 0]
                false_neg = components[cls_i * 4 + 1]
                false_pos = components[cls_i * 4 + 2]
                true_pos = components[cls_i * 4 + 3]

                cls_n = false_neg + false_pos + true_pos

                if cls_n > 0:
                    if self.score == self.SCORE_JACCARD:
                        cls_scores.append(self._score_frac(true_pos, false_pos + false_neg + true_pos))
                    elif self.score == self.SCORE_PRECISION:
                        cls_scores.append(self._score_frac(true_pos, false_pos + true_pos))
                    elif self.score == self.SCORE_RECALL:
                        cls_scores.append(self._score_frac(true_pos, false_neg + true_pos))
                    elif self.score == self.SCORE_F1:
                        precision = self._score_frac(true_pos, false_pos + true_pos)
                        recall = self._score_frac(true_pos, false_neg + true_pos)
                        f1 = self._score_frac(2.0 * (precision * recall), precision + recall)
                        cls_scores.append(f1)
                    else:
                        raise ValueError('score is not valid ({})'.format(self.score))
            return np.mean(cls_scores)

    def score_improved(self, new_results, best_so_far_results):
        if self.score == self.SCORE_ERROR:
            return new_results[1] < best_so_far_results[1]
        else:
            return self._compute_score(new_results) > self._compute_score(best_so_far_results)

class RegressorObjective (AbstractObjective):
    def __init__(self, name, objective_layer, target_expr, mask_expr=None, n_target_spatial_dims=0,
                 cost_weight=1.0):
        """
        Regression objective

        :param name: objective name
        :param objective_layer: Lasagne layer that will predict the output
        :param target_expr: ground truth target expression
        :param mask_expr: [optional] mask expression
        :param n_target_spatial_dims: (default=0) number of spatial dimensions for the target
        :param cost_weight: (default=1.0) weight applied to the cost of this objective
        """
        super(RegressorObjective, self).__init__(name, cost_weight)
        self.objective_layer = objective_layer
        self.target_expr = target_expr
        # Broadcast dimension 1
        self.mask_expr = T.addbroadcast(mask_expr, 1)
        self.n_target_spatial_dims = n_target_spatial_dims

    def build(self):
        # Get an expression representing the predicted probability
        train_pred = lasagne.layers.get_output(self.objective_layer)

        # Create a loss expression for training, i.e., a scalar objective we want
        # to minimize; squared error
        train_loss = lasagne.objectives.squared_error(train_pred, self.target_expr)
        if self.mask_expr is not None:
            train_loss = train_loss * self.mask_expr

        # Create prediction expressions; use deterministic forward pass (disable
        # dropout layers)
        eval_pred = lasagne.layers.get_output(self.objective_layer, deterministic=True)
        # Create evaluation loss expression
        eval_loss = lasagne.objectives.squared_error(eval_pred, self.target_expr)
        if self.mask_expr is not None:
            eval_loss = eval_loss * self.mask_expr

        if self.n_target_spatial_dims == 0:
            train_loss_batch = train_loss.sum()
            eval_loss_batch = eval_loss.sum()
        else:
            train_loss_batch = train_loss.mean(axis=tuple(range(2,2+self.n_target_spatial_dims))).sum()
            eval_loss_batch = eval_loss.mean(axis=tuple(range(2,2+self.n_target_spatial_dims))).sum()

        def train_results_str_fn(train_res):
            return '{} loss={:.6f}'.format(self.name, train_res[0])

        def eval_results_str_fn(eval_res):
            return '{} loss={:.6f}'.format(self.name, eval_res[0])

        return ObjectiveOutput(train_cost=train_loss.mean() * self.cost_weight,
                               train_results=[train_loss_batch],
                               train_results_str_fn=train_results_str_fn,
                               eval_results=[eval_loss_batch],
                               eval_results_str_fn=eval_results_str_fn,
                               prediction=eval_pred)

    def score_improved(self, new_results, best_so_far_results):
        return new_results[0] < best_so_far_results[0]

max_issues_repo_name: Britefury/britefury-lasagne-helpers

max_forks_repo_name: Britefury/britefury-lasagne-helpers

reason: import numpy,import theano


text:
from __future__ import division
import math
from operator import itemgetter
import operator
import random
import numpy as np
import sklearn
from sklearn.cluster import KMeans
from scipy.special import gammaln, psi

# runs bayesian fraud detection; returns vector of suspiciousness
# (in same order as rating_arr and iat_arr)

def fit_alpha(D):
	(m, S) = D.shape
	S = int(S)
	if m <= 1: return [1] * S
	alpha = np.array([3] * S)
	alpha_next = np.array([None] * S)
	row_tot = np.sum(D, axis=1)
	MAX_FIT_ITERS = 100
	for it in range(MAX_FIT_ITERS):
		alpha_sum = np.sum(alpha)
		for s in range(S):
			D_col = D[:, s]
			numer = np.sum(D_col / (D_col + (-1 + alpha[s]) * np.ones(m)))
			denom = np.sum(row_tot / (row_tot + (-1 + alpha_sum) * np.ones(m)))
			alpha_next[s] = alpha[s] * numer / denom
		if np.sum(np.abs(alpha - alpha_next)) < 0.01:
			print("fitting alpha iteration stopped early at iteration", it)
			break
		alpha = alpha_next.copy()
		print (alpha)
	return alpha + 1

# def lbeta(alpha):
#     return sum(math.lgamma(a) for a in alpha) - math.lgamma(sum(alpha))

# def ldirichlet_pdf(theta, alpha):
#     kernel = sum((a - 1) * math.log(t) for a, t in zip(alpha, theta))
#     return kernel - lbeta(alpha)

def ldirich_multi_pdf(z, alpha):
	npalpha = alpha + z
	return (gammaln(np.sum(alpha)) - gammaln(np.sum(npalpha)) +
		   np.sum(gammaln(npalpha)) - np.sum(gammaln(alpha)))

def kl_denom(alpha, beta):
    # denomiator of KL divergence, i.e. E_P 1/log Q(x), where P and Q have Dirich params alpha, beta;
    # http://bariskurt.com/kullback-leibler-divergence-between-two-dirichlet-and-beta-distributions/
	psi_diff = np.sum(beta * (psi(sum(alpha)) - psi(alpha)))
	return -math.lgamma(np.sum(beta)) + np.sum([math.lgamma(x) for x in np.squeeze(beta)]) + psi_diff

def detect(rating_arr, iat_arr, use_times, K):
	m = rating_arr.shape[0]

	rating_sums = rating_arr.sum(axis=1) + 0.1
	rating_normal = rating_arr / rating_sums[:, np.newaxis]
	iat_sums = iat_arr.sum(axis=1) + 0.1
	iat_normal = iat_arr / iat_sums[:, np.newaxis]
	user_normal = np.concatenate((rating_normal, iat_normal), axis=1)

	est = KMeans(n_clusters=K)
	est.fit(user_normal)
	z = np.array(est.labels_)
	#z = np.zeros(m)

	pi = [None] * K
	zn = np.array([-1] * m) # z_next
	(m, S1) = rating_arr.shape
	(m, S2) = iat_arr.shape
	alpha1 = np.array([[0] * S1 for _ in range(K)], dtype=float)
	alpha2 = np.array([[0] * S2 for _ in range(K)], dtype=float)

	NUM_FIT_ITERS = 100
	for it in range(NUM_FIT_ITERS):
		print ("iteration ", it)
		for k in range(K):
			cur_idx = np.array((z == k))
			rating_sub = rating_arr[cur_idx, :]
			iat_sub = iat_arr[cur_idx, :]
			n_k = np.sum(cur_idx)
			pi[k] = n_k / m
			if n_k > 1000:
				sample_idx = np.array(random.sample(range(n_k), 1000))
			else:
				sample_idx = range(n_k)
			print ("subset size is ", len(sample_idx))
			rating_sub = rating_sub[sample_idx, :]
			iat_sub = iat_sub[sample_idx, :]
			alpha1[k,:] = fit_alpha(rating_sub)
			alpha2[k,:] = fit_alpha(iat_sub)
		# print (" alpha1 = ", alpha1)
		# print (" alpha2 = ", alpha2)

		print ("fitting points")
		for i in range(m):
			log_likes = [(ldirich_multi_pdf(rating_arr[i, :], alpha1[k]) +
						  ldirich_multi_pdf(iat_arr[i, :], alpha2[k])) for k in range(K)]
			zn[i] = log_likes.index(max(log_likes))
		num_diff = sum(abs(zn - z))
		z = zn
		if num_diff == 0:
			print ("Outer iteration stopped early at iteration ", it)
			break

	post_rating = np.array(rating_arr, dtype='float')
	post_iat = np.array(iat_arr, dtype='float')
	for i in range(m):
		post_rating[i,:] += alpha1[z[i]]
		post_iat[i,:] += alpha2[z[i]]

	susp1 = np.zeros(m)
	susp2 = np.zeros(m)
	for i in range(m):
		if i % 100000 == 0: print (i)
		susp1[i] = max([kl_denom(post_rating[i,:], alpha1[k,:]) for k in range(K)])
		susp2[i] = max([kl_denom(post_iat[i,:], alpha2[k,:]) for k in range(K)])

	susp1n = susp1 / np.std(susp1)
	susp2n = susp2 / np.std(susp2)
	suspn = susp1n + (susp2n if use_times else 0)
	return suspn

ext: py

reason: import numpy,from scipy


text:
# extract_features.py
# Chapter 3.2 Practitioner Bundle
# Extracting Features using VGG16 and convert to HDF5

# USAGE:
# python sample_code/extract_features.py --dataset ../datasets/animals --output ../datasets/HDF5/animals.hdf5

import os
import random
import argparse
import progressbar
import numpy as np
from imutils import paths
from juggle.io import HDF5DatasetWriter
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.applications import VGG16
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.applications import imagenet_utils

# argument parser
ap = argparse.ArgumentParser()
ap.add_argument("-d", "--dataset", required = True,
        help = "path to input dataset")
ap.add_argument("-o", "--output", required = True,
        help = "pathto output HDF5 file")
ap.add_argument("-b", "--batch-size", type = int, default = 32,
        help = "batch size of images to be passed through network")
ap.add_argument("-s", "--buffer-size", type = int, default = 1000,
        help = "size of feature extraction buffer")
args = vars(ap.parse_args())

# grab the list of images that we'll be describing then randomly
# shuffle them to allow for easy training and testing splits via
# array slicing during training time
print("[INFO] loading images...")
imagePaths = list(paths.list_images(args["dataset"]))
random.shuffle(imagePaths)

# extract the class labels from the image paths then encode the
# labels
labels = [path.split(os.path.sep)[-2] for path in imagePaths]
le = LabelEncoder()
labels = le.fit_transform(labels)

# load the VGG16 Network
print("[INFO] loading network...")
model = VGG16(weights = "imagenet", include_top = False)

# initialize the HDF5 dataset writer, then store the class label
# names in the dataset
dataset = HDF5DatasetWriter(dims = (len(imagePaths), 512 * 7 * 7),
                    outputPath = args["output"],
                    dataKey = "features",
                    bufSize = args["buffer_size"])
dataset.storeClassLabels(le.classes_)

# initialize the progress bar
widgets = ["Extracting Features: ", progressbar.Percentage(), " ",
            progressbar.Bar(), " ", progressbar.ETA()]
pbar = progressbar.ProgressBar(maxval = len(imagePaths), widgets = widgets).start()
batchSize = args["batch_size"]

# loop over the images in patches
for i in np.arange(0, len(imagePaths), batchSize):
    # extract the batch of images and labels,
    # then initialize the list of actual iamges that will be
    # passed through the network foe feature extraction
    batchPaths = imagePaths[i:i + batchSize]
    batchLabels = labels[i:i + batchSize]
    batchImages = []

    # loop over the iamges and labels in the current batch
    for (j, imagePath) in enumerate(batchPaths):
        # load the input image using the Keras helper utility
        # while ensuring thenimage is resized to 224 * 224 pixels
        image = load_img(imagePath, target_size = (224, 244))
        image = img_to_array(image)

        # preprocess the image by (1) expanding the dimenions
        # and (2)subtractinng the mean RGB pixel intensity from
        # the ImageNet dataset
        image = np.expand_dims(image, axis = 0)
        image = imagenet_utils.preprocess_input(image)

        # add the image to the batch
        batchImages.append(image)

    # pass the images through the network and use the output as
    # our actual features
    batchImages = np.vstack(batchImages)
    features = model.predict(batchImages, batch_size = batchSize)

    # reshape the features so that each image is represented by
    # a flattened feature vector of the "MaxPooling2D" outputs
    features = features.reshape((features.shape[0], 512 * 7 * 7))

    # add the features and la els to out HDF5 dataset
    dataset.add(features, batchLabels)
    pbar.update(i)

dataset.close()
pbar.finish()

max_stars_repo_name: laygond/Deep-Tools

max_issues_repo_name: laygond/Deep-Tools

max_forks_repo_name: laygond/Deep-Tools

reason: import numpy


text:
"""
 Copyright (c) 2021 Intel Corporation
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
      http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

from functools import partial
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type

import math
import numpy as np

from nncf.common.graph import NNCFGraph
from nncf.common.graph import NNCFNode
from nncf.common.graph import NNCFNodeName
from nncf.common.tensor import NNCFTensor
from nncf.common.graph.layer_attributes import ConvolutionLayerAttributes
from nncf.common.graph.operator_metatypes import OperatorMetatype
from nncf.common.pruning.clusterization import Cluster
from nncf.common.pruning.clusterization import Clusterization
from nncf.common.pruning.structs import PrunedLayerInfoBase
from nncf.common.utils.registry import Registry

def is_grouped_conv(node: NNCFNode) -> bool:
    return isinstance(node.layer_attributes, ConvolutionLayerAttributes) \
           and node.layer_attributes.groups != 1

def get_sources_of_node(nncf_node: NNCFNode, graph: NNCFGraph, sources_types: List[str]) -> List[NNCFNode]:
    """
    Source is a node of source such that there is path from this node to `nncf_node` and on this path
    no node has one of `sources_types` type.

    :param sources_types: List of sources types.
    :param nncf_node: NNCFNode to get sources.
    :param graph: NNCF graph to work with.
    :return: List of all sources nodes.
    """
    visited = {node_id: False for node_id in graph.get_all_node_ids()}
    partial_traverse_function = partial(traverse_function, type_check_fn=lambda x: x in sources_types,
                                        visited=visited)
    nncf_nodes = [nncf_node]
    if nncf_node.node_type in sources_types:
        nncf_nodes = graph.get_previous_nodes(nncf_node)

    source_nodes = []
    for node in nncf_nodes:
        source_nodes.extend(graph.traverse_graph(node, partial_traverse_function, False))
    return source_nodes

def find_next_nodes_not_of_types(graph: NNCFGraph, nncf_node: NNCFNode, types: List[str]) -> List[NNCFNode]:
    """
    Traverse nodes in the graph from nncf node to find first nodes that aren't of type from types list.
    First nodes with some condition mean nodes:
        - for which this condition is true
        - reachable from `nncf_node` such that on the path from `nncf_node` to
          this nodes there are no other nodes with fulfilled condition

    :param graph: Graph to work with.
    :param nncf_node: NNCFNode to start search.
    :param types: List of types.
    :return: List of next nodes for nncf_node of type not from types list.
    """
    visited = {node_id: False for node_id in graph.get_all_node_ids()}
    partial_traverse_function = partial(traverse_function, type_check_fn=lambda x: x not in types,
                                        visited=visited)
    nncf_nodes = [nncf_node]
    if nncf_node.node_type not in types:
        nncf_nodes = graph.get_next_nodes(nncf_node)

    next_nodes = []
    for node in nncf_nodes:
        next_nodes.extend(graph.traverse_graph(node, partial_traverse_function))
    return next_nodes

def get_next_nodes_of_types(graph: NNCFGraph, nncf_node: NNCFNode, types: List[str]) -> List[NNCFNode]:
    """
    Looking for nodes with type from types list from `nncf_node` such that there is path
    from `nncf_node` to this node and on this path no node has one of types type.

    :param graph: Graph to work with.
    :param nncf_node: NNCFNode to start search.
    :param types: List of types to find.
    :return: List of next nodes of nncf_node with type from types list.
    """
    sources_types = types
    visited = {node_id: False for node_id in graph.get_all_node_ids()}
    partial_traverse_function = partial(traverse_function, type_check_fn=lambda x: x in sources_types,
                                        visited=visited)
    nncf_nodes = [nncf_node]
    if nncf_node.node_type in sources_types:
        nncf_nodes = graph.get_next_nodes(nncf_node)

    next_nodes = []
    for node in nncf_nodes:
        next_nodes.extend(graph.traverse_graph(node, partial_traverse_function))
    return next_nodes

def get_rounded_pruned_element_number(total: int, sparsity_rate: float, multiple_of: int = 8) -> int:
    """
    Calculates number of sparsified elements (approximately sparsity rate) from total such as
    number of remaining items will be multiple of some value.
    Always rounds number of remaining elements up.

    :param total: Total elements number.
    :param sparsity_rate: Prorortion of zero elements in total.
    :param multiple_of: Number of remaining elements must be a multiple of `multiple_of`.
    :return: Number of elements to be zeroed.
    """
    remaining_elems = math.ceil((total - total * sparsity_rate) / multiple_of) * multiple_of
    return max(total - remaining_elems, 0)

def traverse_function(node: NNCFNode, output: List[NNCFNode], type_check_fn, visited) \
        -> Tuple[bool, List[NNCFNode]]:
    if visited[node.node_id]:
        return True, output
    visited[node.node_id] = True

    if not type_check_fn(node.node_type):
        return False, output

    output.append(node)
    return True, output

def get_first_nodes_of_type(graph: NNCFGraph, op_types: List[str]) -> List[NNCFNode]:
    """
    Looking for first node in graph with type in `op_types`.
    First == layer with type in `op_types`, that there is a path from the input such that there are no other
    operations with type in `op_types` on it.

    :param op_types: Types of modules to track.
    :param graph: Graph to work with.
    :return: List of all first nodes with type in `op_types`.
    """
    graph_roots = graph.get_input_nodes()  # NNCFNodes here

    visited = {node_id: False for node_id in graph.get_all_node_ids()}
    partial_traverse_function = partial(traverse_function,
                                        type_check_fn=lambda x: x in op_types,
                                        visited=visited)

    first_nodes_of_type = []
    for root in graph_roots:
        first_nodes_of_type.extend(graph.traverse_graph(root, partial_traverse_function))
    return first_nodes_of_type

def get_last_nodes_of_type(graph: NNCFGraph, op_types: List[str]) -> List[NNCFNode]:
    """
    Looking for last node in graph with type in `op_types`.
    Last == layer with type in `op_types`, that there is a path from this layer to the model output
    such that there are no other operations with type in `op_types` on it.

    :param op_types: Types of modules to track.
    :param graph: Graph to work with.
    :return: List of all last pruned nodes.
    """
    graph_outputs = graph.get_output_nodes()  # NNCFNodes here

    visited = {node_id: False for node_id in graph.get_all_node_ids()}
    partial_traverse_function = partial(traverse_function,
                                        type_check_fn=lambda x: x in op_types,
                                        visited=visited)
    last_nodes_of_type = []
    for output in graph_outputs:
        last_nodes_of_type.extend(graph.traverse_graph(output, partial_traverse_function, False))

    return last_nodes_of_type

def get_previous_convs(graph: NNCFGraph, nncf_node: NNCFNode,
                       pruning_types: List[str], stop_propagation_ops: List[str]) -> Optional[NNCFNode]:
    """
    Returns source convolutions of the node.

    :return: List of source convolutions of node.
    """
    sources = get_sources_of_node(nncf_node, graph, pruning_types + stop_propagation_ops)
    sources = [source for source in sources if source.node_type in pruning_types]
    return sources

def get_conv_in_out_channels(graph: NNCFGraph):
    """
    Collects the number of input and output channels for each convolution in the graph.

    :param graph: NNCFGraph
    :return Dictionary with the number of input channels to convolution layers:
            {node_name: input_channels_num}
            Dictionary with the number of output channels from convolution layers:
            {node_name: output_channels_num}
    """
    in_channels, out_channels = {}, {}
    for node in graph.get_all_nodes():
        if isinstance(node.layer_attributes, ConvolutionLayerAttributes):
            name = node.node_name
            if name in in_channels and name in out_channels:
                continue
            in_channels[name] = node.layer_attributes.in_channels
            out_channels[name] = node.layer_attributes.out_channels
    return in_channels, out_channels

def get_cluster_next_nodes(graph: NNCFGraph, pruned_groups_info: Clusterization[PrunedLayerInfoBase],
                           prunable_types: List[str]) -> Dict[int, List[NNCFNodeName]]:
    """
    Finds nodes of `prunable_types` types that receive the output of a pruned cluster as input.

    :param graph: NNCFGraph.
    :param pruned_groups_info: `Clusterization` of pruning groups.
    :param prunable_types: Types of nodes that will be returned.
    :return Dictionary of next node names by cluster {cluster_id: [node_name]}.
    """
    next_nodes = {}
    for cluster in pruned_groups_info.get_all_clusters():
        next_nodes_cluster = set()
        cluster_nodes = set()
        for pruned_layer_info in cluster.elements:
            nncf_cluster_node = graph.get_node_by_id(pruned_layer_info.nncf_node_id)
            cluster_nodes.add(nncf_cluster_node.node_name)
            curr_next_nodes = get_next_nodes_of_types(graph, nncf_cluster_node, prunable_types)

            next_nodes_idxs = [n.node_name for n in curr_next_nodes]
            next_nodes_cluster = next_nodes_cluster.union(next_nodes_idxs)
        next_nodes[cluster.id] = list(next_nodes_cluster - cluster_nodes)
    return next_nodes

def count_flops_and_weights(graph: NNCFGraph,
                            input_shapes: Dict[NNCFNodeName, List[int]],
                            output_shapes: Dict[NNCFNodeName, List[int]],
                            conv_op_metatypes: List[Type[OperatorMetatype]],
                            linear_op_metatypes: List[Type[OperatorMetatype]],
                            input_channels: Dict[NNCFNodeName, int] = None,
                            output_channels: Dict[NNCFNodeName, int] = None) -> Tuple[int, int]:
    """
    Counts the number weights and FLOPs in the model for convolution and fully connected layers.

    :param graph: NNCFGraph.
    :param input_shapes: Dictionary of input dimension shapes for convolutions and
        fully connected layers. E.g {node_name: (height, width)}
    :param output_shapes: Dictionary of output dimension shapes for convolutions and
        fully connected layers. E.g {node_name: (height, width)}
    :param conv_op_metatypes: List of metatypes defining convolution operations.
    :param linear_op_metatypes: List of metatypes defining linear/fully connected operations.
    :param input_channels: Dictionary of input channels number in convolutions.
        If not specified, taken from the graph. {node_name: channels_num}
    :param output_channels: Dictionary of output channels number in convolutions.
        If not specified, taken from the graph. {node_name: channels_num}
    :return number of FLOPs for the model
            number of weights (params) in the model
    """
    flops_pers_node, weights_per_node = count_flops_and_weights_per_node(graph,
                                                                         input_shapes, output_shapes,
                                                                         conv_op_metatypes, linear_op_metatypes,
                                                                         input_channels, output_channels)
    return sum(flops_pers_node.values()), sum(weights_per_node.values())

def count_flops_and_weights_per_node(graph: NNCFGraph,
                                     input_shapes: Dict[NNCFNodeName, List[int]],
                                     output_shapes: Dict[NNCFNodeName, List[int]],
                                     conv_op_metatypes: List[Type[OperatorMetatype]],
                                     linear_op_metatypes: List[Type[OperatorMetatype]],
                                     input_channels: Dict[NNCFNodeName, int] = None,
                                     output_channels: Dict[NNCFNodeName, int] = None) -> \
        Tuple[Dict[NNCFNodeName, int], Dict[NNCFNodeName, int]]:
    """
    Counts the number weights and FLOPs per node in the model for convolution and fully connected layers.

    :param graph: NNCFGraph.
    :param input_shapes: Dictionary of input dimension shapes for convolutions and
        fully connected layers. E.g {node_name: (height, width)}
    :param output_shapes: Dictionary of output dimension shapes for convolutions and
        fully connected layers. E.g {node_name: (height, width)}
    :param conv_op_metatypes: List of metatypes defining convolution operations.
    :param linear_op_metatypes: List of metatypes defining linear/fully connected operations.
    :param input_channels: Dictionary of input channels number in convolutions.
        If not specified, taken from the graph. {node_name: channels_num}
    :param output_channels: Dictionary of output channels number in convolutions.
        If not specified, taken from the graph. {node_name: channels_num}
    :return Dictionary of FLOPs number {node_name: flops_num}
            Dictionary of weights number {node_name: weights_num}
    """
    flops = {}
    weights = {}
    input_channels = input_channels or {}
    output_channels = output_channels or {}
    for node in graph.get_nodes_by_metatypes(conv_op_metatypes):
        name = node.node_name
        num_in_channels = input_channels.get(name, node.layer_attributes.in_channels)
        num_out_channels = output_channels.get(name, node.layer_attributes.out_channels)
        filters_per_channel = num_out_channels // node.layer_attributes.groups
        flops_numpy = 2 * np.prod(node.layer_attributes.kernel_size) * \
                      num_in_channels * filters_per_channel * np.prod(output_shapes[name])
        weights_numpy = np.prod(node.layer_attributes.kernel_size) * num_in_channels * filters_per_channel
        flops[name] = flops_numpy.astype(int).item()
        weights[name] = weights_numpy.astype(int).item()

    for node in graph.get_nodes_by_metatypes(linear_op_metatypes):
        name = node.node_name
        flops_numpy = 2 * np.prod(input_shapes[name]) * np.prod(output_shapes[name])
        weights_numpy = np.prod(input_shapes[name]) * np.prod(output_shapes[name])
        flops[name] = flops_numpy.astype(int).item()
        weights[name] = weights_numpy.astype(int).item()

    return flops, weights

def calculate_in_out_channels_in_uniformly_pruned_model(pruning_groups: List[Cluster[PrunedLayerInfoBase]],
                                                        pruning_rate: float,
                                                        full_input_channels: Dict[str, int],
                                                        full_output_channels: Dict[str, int],
                                                        pruning_groups_next_nodes: Dict[int, List[str]]):
    """
    Imitates filters pruning by removing `pruning_rate` percent of output filters in each pruning group
    and updating corresponding input channels number in `pruning_groups_next_nodes` nodes.

    :param pruning_groups: A list of pruning groups.
    :param pruning_rate: Target pruning rate.
    :param full_input_channels:  A dictionary of input channels number in original model.
    :param full_output_channels: A dictionary of output channels number in original model.
    :param pruning_groups_next_nodes: A dictionary of next nodes of each pruning group.
    :return Dictionary of new input channels number {node_name: channels_num}
    :return Dictionary of new output channels number {node_name: channels_num}
    """
    tmp_in_channels = full_input_channels.copy()
    tmp_out_channels = full_output_channels.copy()

    for group in pruning_groups:
        layer_name = group.elements[0].node_name
        assert all(tmp_out_channels[layer_name] == tmp_out_channels[node.node_name] for node in
                   group.elements)
        # Prune all nodes in cluster (by output channels)
        old_out_channels = full_output_channels[layer_name]
        num_of_sparse_elems = get_rounded_pruned_element_number(old_out_channels, pruning_rate)
        new_out_channels_num = old_out_channels - num_of_sparse_elems

        for minfo in group.elements:
            tmp_out_channels[minfo.node_name] = new_out_channels_num

        # Prune in_channels in all next nodes of cluster
        for node_name in pruning_groups_next_nodes[group.id]:
            tmp_in_channels[node_name] -= num_of_sparse_elems

    return tmp_in_channels, tmp_out_channels

class PruningOperationsMetatypeRegistry(Registry):
    def __init__(self, name):
        super().__init__(name)
        self._op_name_to_op_class = {}

    def register(self, name=None):
        name_ = name
        super_register = super()._register

        def wrap(obj):
            cls_name = name_
            if cls_name is None:
                cls_name = obj.__name__

            super_register(obj, cls_name)
            op_names = obj.get_all_op_aliases()
            for name in op_names:
                if name not in self._op_name_to_op_class:
                    self._op_name_to_op_class[name] = obj
                else:
                    assert self._op_name_to_op_class[name] == obj, \
                        'Inconsistent operator type registry - single patched op name maps to multiple metatypes!'
            return obj

        return wrap

    def get_operator_metatype_by_op_name(self, op_name: str):
        if op_name in self._op_name_to_op_class:
            return self._op_name_to_op_class[op_name]
        return None

def is_depthwise_conv(node: NNCFNode) -> bool:
    return isinstance(node.layer_attributes, ConvolutionLayerAttributes) \
           and node.layer_attributes.groups == node.layer_attributes.in_channels \
           and (node.layer_attributes.out_channels % node.layer_attributes.in_channels == 0) \
           and node.layer_attributes.in_channels > 1

def is_conv_with_downsampling(node: NNCFNode) -> bool:
    layer_attrs = node.layer_attributes
    if isinstance(layer_attrs, ConvolutionLayerAttributes):
        return not np.all(np.array(layer_attrs.stride) == 1) \
           and not layer_attrs.transpose
    return False

def get_input_masks(node: NNCFNode, graph: NNCFGraph) -> List[Optional[NNCFTensor]]:
    """
    Returns input masks for all inputs of given NNCFNode.

    :param node: Given NNCFNode.
    :param graph: Graph to work with.
    :return: Input masks.
    """
    input_masks = [input_node.data['output_mask'] for input_node in graph.get_previous_nodes(node)]
    return input_masks

def identity_mask_propagation(node: NNCFNode, graph: NNCFGraph) -> None:
    """
    Propagates input mask through NNCFNode.

    :param node: Graph node to perform identity mask propagation on.
    :param graph: Graph to work with.
    """
    input_masks = get_input_masks(node, graph)
    if not input_masks:
        # In case for disconnected NNCFGraph
        input_masks = [None]
    assert len(input_masks) == 1
    node.data['input_masks'] = input_masks
    node.data['output_mask'] = input_masks[0]

max_stars_repo_name: openvinotoolkit/nncf

max_issues_repo_name: openvinotoolkit/nncf

max_forks_repo_name: openvinotoolkit/nncf


text:
from helper import TelecomHelper
import pandas as pd
import numpy as np

class OverViewAnalysis:

    def __init__(self, df):

        self.df = df

    def read_head(self, top=5):
        return self.df.head(top)

    # returning the number of rows columns and column information
    def get_info(self):
        row_count, col_count = self.df.shape

        print(f"Number of rows: {row_count}")
        print(f"Number of columns: {col_count}")
        print("================================")

        return (row_count, col_count), self.df.info()

    # gets number of distnict values in a given coumn
    def get_count(self, column_name):
        return self.df[column_name].value_counts()

    # getting the null count for every column
    def get_null_count(self, column_name):
        print("Null values count")
        print(self.df.isnull().sum())
        return self.df.isnull().sum()

    # getting the percentage of missing values
    def get_percent_missing(self):
        Helper = TelecomHelper()

        percent_missing = Helper.percent_missing(self.df)

        null_percent_df = pd.DataFrame(columns = ['column', 'null_percent'])
        columns = self.df.columns.values.tolist()

        null_percent_df['column'] = columns
        null_percent_df['null_percent'] = null_percent_df['column'].map(lambda x: Helper.percent_missing_for_col(self.df, x))

        return null_percent_df.sort_values(by=['null_percent'], ascending = False), percent_missing

    # returning the top handseet
    def top_handset_type(self, top=5):

        return self.df['handset_type'].value_counts().head(top)

    # returning the top handset manufactruer
    def top_manufacturer(self, top=5):

        return self.df['handset_manufacturer'].value_counts().head(top)

    # top handset by manfuctruer
    def top_handset_by_manufacturer(self, manufacturer, top=5):

        return self.df.groupby('handset_manufacturer')['handset_type'].value_counts()[manufacturer].head(top)

max_stars_repo_licenses: MIT Unlicense

max_issues_repo_licenses: MIT Unlicense

max_forks_repo_licenses: Unlicense MIT


text:
import warnings
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import List, Tuple, Dict

import numpy as np
import torch

from nebullvm.base import (
    DeepLearningFramework,
    ModelParams,
    ModelCompiler,
    InputInfo,
)
from nebullvm.converters import ONNXConverter
from nebullvm.utils.torch import (
    get_outputs_sizes_torch,
    create_model_inputs_torch,
)
from nebullvm.inference_learners.base import PytorchBaseInferenceLearner
from nebullvm.measure import compute_optimized_running_time
from nebullvm.optimizers import ApacheTVMOptimizer
from nebullvm.optimizers.multi_compiler import MultiCompilerOptimizer

def optimize_torch_model(
    model: torch.nn.Module,
    batch_size: int,
    input_sizes: List[Tuple[int, ...]],
    save_dir: str,
    input_types: List[str] = None,
    extra_input_info: List[Dict] = None,
    use_torch_api: bool = False,
    dynamic_axis: Dict = None,
    ignore_compilers: List[str] = None,
) -> PytorchBaseInferenceLearner:
    """Basic function for optimizing a torch model.

    This function saves the output model as well in a nebuly-readable format
    in order to avoid temporary-files corruptions which would prevent the model
    saving later in the process.

    Args:
        model (torch.nn.Module): Pytorch model that needs optimization.
        batch_size (int): The model batch size. Note that nebullvm does not
            support at the moment dynamic batch size, so a valid input should
            be given.
        input_sizes (List[Tuple]]): List containing the size of all the input
            tensors of the model. Note that even just a single tensor is needed
            as model input, this field must be a list containing (in the
            exposed case) a single element). The tuple must contain all the
            input tensor dimensions excluding the batch size. This means that
            the final input tensor size will be considered as
            `(batch_size, *input_tensor_size)`, where `input_tensor_size` is
            one list element of `input_sizes`.
        save_dir (str): Path to the directory where saving the final model.
        input_types (List[str], optional): List of input types. If no value is
            given all the inputs will be considered as float type. The
            supported string values are "int" and "float".
        extra_input_info (List[Dict], optional): List of extra information
            needed for defining the input tensors, e.g. max_value and min_value
            the tensors can get.
        use_torch_api (bool): Parameter for using the torch api of compilers
            when available. The actual implementation supports only the torch
            interface for TVM. Note that when running the torch interface
            nebullvm will ignore the ONNX one once the torch implementation
            succeeds. Clearly, in case of failure of the torch API, a second
            tentative will be done with the ONNX interface.
        dynamic_axis (Dict, optional): Dictionary containing info about the
            dynamic axis. It should contain as keys both "inputs" and "outputs"
            and as values two lists of dictionaries where each dictionary
            represents the dynamic axis information for an input/output tensor.
            The inner dictionary should have as key an integer, i.e. the
            dynamic axis (considering also the batch size) and as value a
            string giving a "tag" to it, e.g. "batch_size".
        ignore_compilers (List[str]): List of DL compilers we want to ignore
            while running the optimization. Compiler name should be one
            between "tvm", "tensor RT", "openvino" and "onnxruntime".

    Returns:
        PytorchBaseInferenceLearner: Optimized model usable with the classical
            Pytorch interface. Note that as a torch model it takes as input
            and it gives as output `torch.Tensor`s.
    """
    if input_types is None:
        input_types = ["float"] * len(input_sizes)
    if extra_input_info is None:
        extra_input_info = [{}] * len(input_sizes)
    if not len(input_sizes) == len(input_types) == len(extra_input_info):
        raise ValueError(
            f"Mismatch in the input list lengths. Given {len(input_sizes)} "
            f"sizes, {len(input_types)} input types and "
            f"{len(extra_input_info)} extra input infos."
        )
    input_infos = [
        InputInfo(size=input_size, dtype=input_type, **extra_info)
        for input_size, input_type, extra_info in zip(
            input_sizes, input_types, extra_input_info
        )
    ]
    dl_library = DeepLearningFramework.PYTORCH
    model_params = ModelParams(
        batch_size=batch_size,
        input_infos=input_infos,
        output_sizes=get_outputs_sizes_torch(
            model,
            input_tensors=create_model_inputs_torch(batch_size, input_infos),
        ),
        dynamic_info=dynamic_axis,
    )
    ignore_compilers = (
        []
        if ignore_compilers is None
        else [ModelCompiler(compiler) for compiler in ignore_compilers]
    )
    with TemporaryDirectory() as tmp_dir:
        if use_torch_api:
            (
                torch_api_model,
                torch_api_latency,
                used_compilers,
            ) = _torch_api_optimization(model, model_params)
            ignore_compilers.extend(used_compilers)
        model_converter = ONNXConverter()
        model_optimizer = MultiCompilerOptimizer(
            ignore_compilers=ignore_compilers,
        )
        if model_optimizer.usable:
            onnx_path = model_converter.convert(
                model, model_params, Path(tmp_dir)
            )
            model_optimized = model_optimizer.optimize(
                str(onnx_path), dl_library, model_params
            )
        else:
            model_optimized = None
        if model_optimized is None and not ignore_compilers:
            raise RuntimeError(
                "No valid compiled model has been produced. "
                "Look at the logs for further information about the failure."
            )
        if use_torch_api:
            model_optimized = _compare_optimized_models(
                model_optimized,
                torch_api_model,
                torch_api_latency,
            )
        model_optimized.save(save_dir)
    return model_optimized.load(save_dir)

def _torch_api_optimization(
    model: torch.nn.Module, model_params: ModelParams
) -> Tuple[PytorchBaseInferenceLearner, float, List]:
    try:
        best_torch_opt_model = ApacheTVMOptimizer().optimize_from_torch(
            torch_model=model, model_params=model_params
        )
        best_latency = compute_optimized_running_time(best_torch_opt_model)
        used_compilers = [ModelCompiler.APACHE_TVM]
    except Exception as ex:
        warnings.warn(
            f"Compilation failed with torch interface of TVM. "
            f"Got error {ex}. The compilation will be re-scheduled "
            f"with the ONNX interface."
        )
        best_torch_opt_model = None
        best_latency = np.inf
        used_compilers = []
    return best_torch_opt_model, best_latency, used_compilers

def _compare_optimized_models(
    new_model: PytorchBaseInferenceLearner,
    previous_best_model: PytorchBaseInferenceLearner,
    previous_latency: float,
) -> PytorchBaseInferenceLearner:
    if new_model is not None:
        new_latency = compute_optimized_running_time(new_model)
        if new_latency < previous_latency:
            return new_model
    return previous_best_model

max_stars_repo_name: zeyus/nebullvm

max_issues_repo_name: zeyus/nebullvm

max_forks_repo_name: zeyus/nebullvm


text:
# Copyright 2018 The Simons Foundation, Inc. - All Rights Reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import cProfile

import netket as nk
import numpy as np

from netket.operator import local_values

# 1D Lattice
g = nk.graph.Hypercube(length=40, n_dim=1, pbc=True)

# Hilbert space of spins on the graph
hi = nk.hilbert.Spin(s=0.5, graph=g)

# Ising spin hamiltonian
ha = nk.operator.Ising(h=1.0, hilbert=hi)

# Hilbert space of spins on the graph
py_hi = nk.hilbert.PySpin(s=0.5, graph=g)

# Ising spin hamiltonian
py_ha = nk.operator.PyIsing(h=1.0, hilbert=py_hi)

# RBM Spin Machine
ma = nk.machine.RbmSpin(alpha=1, hilbert=hi)
ma.init_random_parameters(seed=1234, sigma=0.01)

# Metropolis Local Sampling
sa = nk.sampler.MetropolisLocal(machine=ma, n_chains=16)

n_samples = 1000
samples = np.zeros((n_samples, sa.sample_shape[0], sa.sample_shape[1]))
for i, sample in enumerate(sa.samples(n_samples)):
    samples[i] = sample

loc = np.empty(samples.shape[0:2], dtype=np.complex128)

def compute_locals(ha, n_times):
    for k in range(n_times):
        for i, sample in enumerate(samples):
            local_values(ha, ma, sample, out=loc[i])

compute_locals(ha, 1)
cProfile.run("compute_locals(ha,10)")

compute_locals(py_ha, 1)
cProfile.run("compute_locals(py_ha,10)")

max_stars_repo_name: ChenAo-Phys/netket

max_issues_repo_name: ChenAo-Phys/netket

max_forks_repo_name: yannra/netket


text:
import numpy
from packetPerSecond import generateFeatureFromPackets

def split(l, n):
    subsetSize = len(l) / n
    for i in range(n):
        yield l[(subsetSize * i):((subsetSize * i) + subsetSize)]

def generateFeature(data):
    feature = []
    dataSubset = split(data['packets'], 20)
    for subset in dataSubset:
        feature.extend(generateFeatureFromPackets(subset))
    return feature


text:
import os, sys, math, argparse
import torch
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from copy import copy
import numpy as np
import matplotlib.pyplot as plt

from metaopt.optimizer import SGD_Multi_LR
from itertools import product, cycle
import pickle

from mlp import *
from metaopt.util import *
from metaopt.util_ml import *
from metaopt.mnist.main import main, parse_args, load_mnist

ifold=9
RNG = np.random.RandomState(ifold)

TRAIN=0
VALID=1
TEST =2
basepath = '/scratch/ji641'

def ideal_hyper_exp(args, sampler_type='uniform', thrd=0.):

    dataset = load_mnist(args)
    trial, contF = 0, 1
    te_loss_list = []
    #quotient_list = [1,2,3,4,5]
    quotient_list = [3]
    for quotient in quotient_list:
        args.quotient = quotient
        args.sampler_type = None
        args.model_type = 'qmlp'
        print('Model Type: %s Opt Type: %s meta-lr %f lr %f l2 %f, Update Freq %d Reset Freq %d |Nvl| %d Quotient %d' \
            % (args.model_type, args.opt_type, args.mlr,  args.lr, \
                args.lambda_l2, args.update_freq, args.reset_freq, args.valid_size, args.quotient))

        trial +=1
        print('<<---- Trial %d Lr %f L2 %f Epoch %d--->>'  % (trial, args.lr, args.lambda_l2, args.num_epoch))
        te_loss = main(args, trial=trial, ifold=ifold, quotient=quotient, device=device)
        te_loss_list.append(te_loss)
        print('*** Trial %d Test loss %f Lr %f L2 %f ***' \
                    % (trial, te_loss_list[-1], args.lr, args.lambda_l2))

    print(quotient_list)
    print(te_loss_list)

if __name__ == '__main__':

    args = parse_args()
    is_cuda = 1
    args.is_cuda = is_cuda
    device = 'cuda' if is_cuda else 'cpu'
    ideal_hyper_exp(args)


text:
import numpy as np
import pandas as pd
import dash
import dash_core_components as dcc
import dash_bootstrap_components as dbc
import dash_html_components as html
from dash.dependencies import Input, Output, State, MATCH, ALL
from dash.exceptions import PreventUpdate
from ai4good.webapp.apps import dash_app, facade, model_runner, _redis
import ai4good.webapp.run_model_page as run_model_page
import ai4good.webapp.common_elements as common_elements
from ai4good.webapp.model_runner import InputParameterCache
import ai4good.utils.path_utils as pu

age = ['0 - 5', '6 - 9', '10 - 19', '20 - 29', '30 - 39', '40 - 49', '50 - 59', '60 - 69', '70+']
id_age_popu =['age-population-' + x.replace(' ','') for x in age]
age_perc_start = [15, 15, 10, 10, 10, 10, 10, 10, 10] # starting age group population percentage
total_popu = 20000 # starting total population
err_group_total_not_equal_popu = 'Group total must equal to total population or 100%'

id_gender_perc = ['gender-perc-female', 'gender-perc-male']
id_ethnic_no = ['ethnic-no-'+str(i+1) for i in range(10)]

accommodation_info = ['Type 1', 'Type 2', 'Type 3']
accommodation_info_full = ['Accommodation Type 1', 'Accommodation Type 2', 'Accommodation Type 3']
accommodation_info_required = ['Optional', 'Optional', 'Optional']
accommodation_info_detail = ['Area covered (m²)', 'No. of existing units of accommodation', 'No. of total camp residents in this type of accommodation']
tab_id_accommodation_info = ['tab-accommodation-info-' + x.replace(' ','').lower() for x in accommodation_info]
id_accommodation_area = ['accommodation-area-' + x.replace(' ','').lower() for x in accommodation_info]
id_accommodation_no_unit = ['accommodation-no-unit-' + x.replace(' ','').lower() for x in accommodation_info]
id_accommodation_no_person = ['accommodation-no-person-' + x.replace(' ','').lower() for x in accommodation_info]

def int_perc_1dp(nom, dem):
    if None in [nom, dem]:
        return None
    else:
        perc = nom/dem*100 if dem !=0 else 0
        perc_1dp = "{:.1f}".format(perc)
        return str(perc_1dp)+'%'

def generate_html_age_group(age, id_age_popu, age_perc, total_popu):
    return html.Div([
               html.Header(''),
               html.Header([str(age)], className='card-text'),
               dbc.Input(id={
                   'type':'age-popu-input',
                   'index':id_age_popu
               }, placeholder='', type='number', value=0, min=0, max=total_popu, step=1, bs_size='sm', style={'justify':'right', 'margin-bottom':'5px'}),
               dcc.Slider(id={
                   'type':'age-popu-slider',
                   'index':id_age_popu
               }, value=0, min=0, max=total_popu, step=1, updatemode='drag'),
               dbc.Label(str(age_perc)+'%', id={
                   'type':'age-perc-label',
                   'index':id_age_popu}),
    ], id=id_age_popu, style={'display':'grid', 'grid-template-columns':'5% 20% 20% 43% 12%', 'margin-bottom':'5px'})

def generate_ethnic_popu_children(id_two_no, id_ethnic_no, str_info, max_no, margin_bottom='25px'):
    children = html.Div([
        html.B(''),
        html.B(str_info + str(id_two_no[0]+1)),
        dbc.Input(id=id_ethnic_no[id_two_no[0]], placeholder='', type='number', min=0, max=max_no, step=1, n_submit=0, bs_size='sm'),
        html.B(''),
        html.B(str_info + str(id_two_no[1]+1)),
        dbc.Input(id=id_ethnic_no[id_two_no[1]], placeholder='', type='number', min=0, max=max_no, step=1, n_submit=0, bs_size='sm'),
        html.B(''),
    ], style={'display':'grid', 'grid-template-columns':'7% 18% 20% 12% 18% 20% 5%', 'margin-bottom':margin_bottom})
    return children

def generate_accommodation_info_children(id_index):
    children = [html.Div([
        html.P(''),
        dbc.Label(accommodation_info_full[id_index], color='secondary'),
        html.Header(accommodation_info_detail[0], className='card-text'),
        dbc.Input(id=id_accommodation_area[id_index], placeholder=accommodation_info_required[id_index], type='number', min=0, max=1000000, step=100, bs_size='sm', style={'margin-bottom':'25px'}),
        html.Header(accommodation_info_detail[1], className='card-text'),
        dbc.Input(id=id_accommodation_no_unit[id_index], placeholder=accommodation_info_required[id_index], type='number', min=0, max=10000, step=1, bs_size='sm', style={'margin-bottom':'25px'}),
        html.Header(accommodation_info_detail[2], className='card-text'),
        dbc.Input(id=id_accommodation_no_person[id_index], placeholder=accommodation_info_required[id_index], type='number', min=0, max=100000, step=1, bs_size='sm', style={'margin-bottom':'25px'}),
    ], style={'margin-left':'25px', 'margin-right':'25px', 'margin-bottom':'20px'})]
    return children

layout = html.Div(
    [
        common_elements.nav_bar(),

        html.Div([
            dbc.Container([
                dbc.Row(
                    dbc.Col([
                        dbc.Card([
                            html.H4('COVID-19 Simulator', className='card-title'),
                            html.Center(html.Img(src='/static/input_step2.png', title='Step 2 of 4', style={'width':'50%'}, className="step_counter")),
                            html.P('Fill in the following about the age structure accommodation type of the settlement',className='card-text'),
                            html.H5('Population', className='card-text'),
                            html.Header('Total Population', className='card-text'),
                            html.Header('What is the total population in the camp?',className='card-text', style={'color':'darkgray'}),
                            dbc.Input(id='total-population', placeholder='', value=20000, type='number', min=0, max=100000, step=1, n_submit=0, bs_size='sm', style={'margin-bottom':'25px'}),
                            html.Header('Age Group Structure', className='card-text'),
                            html.Header('Enter the percentage or actual population that each age range represents',className='card-text', style={'color':'darkgray'}),
                            html.Div([
                                html.Header(''),
                                html.Header('Age Group (years)'),
                                html.Header('Population'),
                                html.Header('Distribution'),
                                html.Header('(%)')],
                                style={'display':'grid', 'grid-template-columns':'5% 20% 35% 33% 7%'}),
                            generate_html_age_group(age[0], id_age_popu[0], age_perc_start[0], total_popu),
                            generate_html_age_group(age[1], id_age_popu[1], age_perc_start[1], total_popu),
                            generate_html_age_group(age[2], id_age_popu[2], age_perc_start[2], total_popu),
                            generate_html_age_group(age[3], id_age_popu[3], age_perc_start[3], total_popu),
                            generate_html_age_group(age[4], id_age_popu[4], age_perc_start[4], total_popu),
                            generate_html_age_group(age[5], id_age_popu[5], age_perc_start[5], total_popu),
                            generate_html_age_group(age[6], id_age_popu[6], age_perc_start[6], total_popu),
                            generate_html_age_group(age[7], id_age_popu[7], age_perc_start[7], total_popu),
                            generate_html_age_group(age[8], id_age_popu[8], age_perc_start[8], total_popu),
                            html.Div([
                                html.B(''),
                                html.B('Group Total:'),
                                html.B('', id='age_population_total'),
                                html.B('Percentage Total:'),
                                html.B('', id='age_percentage_total')], className='card-text',
                                style={'display':'grid', 'grid-template-columns':'5% 25% 26% 30% 14%'}),
                            html.Div([
                                html.B(''),
                                dbc.Label('Must equal to total population or 100%', id='age-group-continue-warning', color='secondary'),
                                dbc.Button('Default %', size='sm', color='secondary', id='age-default-perc', style={'float':'right'})], className='card-text', style={'display':'grid', 'grid-template-columns':'5% 77% 18%'}),
                            html.P(''),
                            html.Header('Male and Female Population', className='card-text'),
                            html.Div([
                                html.B(''),
                                html.B('Female (%)'),
                                html.B('Male (%)')],
                                style={'display':'grid', 'grid-template-columns':'0% 85% 15%', 'color':'darkgray'}),
                            html.Div([
                                html.B(''),
                                html.Label('', id='gender-perc-female'),
                            dcc.Slider(id='slider-gender-perc', min=0, max=100, step=1, value=50, included=False, updatemode='drag', marks={50: {'label':'50'}}),
                                html.B(''),
                                html.Label('', id='gender-perc-male')],
                                style={'display':'grid', 'grid-template-columns':'5% 10% 70% 5% 10%', 'margin-bottom':'25px'}),
                            html.Header('Population by Ethnicity', className='card-text'),
                            html.Header('Enter the population represented by each ethnic group (optional)', className='card-text', style={'color':'darkgray'}),
                            generate_ethnic_popu_children([0, 1], id_ethnic_no, 'Group ', 100000, '10px'),
                            generate_ethnic_popu_children([2, 3], id_ethnic_no, 'Group ', 100000, '10px'),
                            generate_ethnic_popu_children([4, 5], id_ethnic_no, 'Group ', 100000, '10px'),
                            generate_ethnic_popu_children([6, 7], id_ethnic_no, 'Group ', 100000, '10px'),
                            generate_ethnic_popu_children([8, 9], id_ethnic_no, 'Group ', 100000),
                            html.P(''),
                            html.H5('Accommodation Information', className='card-text'),
                            html.Header('Population Accommodation', className='card-text'),
                            html.Header('Accommodation details provide population density estimate and suggest mobility pattern', className='card-text', style={'color':'darkgray'}),
                            html.Div([
                                html.B(''),
                                dcc.Slider(id='slider-accommodation-info', min=0, max=2, step=1, value=0, updatemode='drag', marks={i: {'label': str(i+1), 'style': {'font-size': '1.2em'}} for i in range(3)}),
                                html.B('')],
                                style={'display':'grid', 'grid-template-columns':'5% 90% 5%', 'margin-bottom':'10px'}),
                            html.Div([
                                dbc.Tabs([
                                    dbc.Tab(label=accommodation_info[0], tab_id=tab_id_accommodation_info[0], children=generate_accommodation_info_children(0), disabled=True),
                                    dbc.Tab(label=accommodation_info[1], tab_id=tab_id_accommodation_info[1], children=generate_accommodation_info_children(1), disabled=True),
                                    dbc.Tab(label=accommodation_info[2], tab_id=tab_id_accommodation_info[2], children=generate_accommodation_info_children(2), disabled=True),
                                ], id='tabs-accommodation-info', active_tab=tab_id_accommodation_info[0]),
                            ], style={'border':'1px lightgray solid'}),
                            html.P(''),
                            dbc.CardFooter(dbc.Button('Next', id='page-2-button', color='secondary', disabled=False, href='/sim/input_page_3', style={'float':'right'})),
                            dbc.Label('',id='page-2-continue-warning', color='danger', style={'text-align':'right'}),
                            html.Div(id='input-page-2-alert')
                            ], body=True),
                        html.Br()], width=6
                    ), justify='center', style={'margin-top':'50px'}
                )
            ])
        ])
    ]
)

@dash_app.callback(
    [Output('tabs-accommodation-info', 'active_tab')],
    [Input('slider-accommodation-info','value')])
def update_accommodation_info_tab(value):
    return [tab_id_accommodation_info[value]]

@dash_app.callback(
    [Output('gender-perc-female','children'), Output('gender-perc-male','children')],
    [Input('slider-gender-perc','value')])
def update_gender_perc_label(slider_value):
    perc_female = slider_value
    perc_male = 100 - slider_value
    return perc_female, perc_male

@dash_app.callback(
    [Output({'type':'age-perc-label', 'index':MATCH}, 'children')],
    [Input({'type':'age-popu-input', 'index':MATCH}, 'value')],
    [State('total-population', 'value')])
def update_age_group_label(input_value, total_value):
    updated_perc_str = int_perc_1dp(input_value, total_value)
    return [updated_perc_str]

@dash_app.callback(
    [Output({'type':'age-popu-input', 'index':MATCH}, 'value')],
    [Input({'type':'age-popu-slider', 'index':MATCH}, 'value')])
def update_age_group_input(input_value):
    return [input_value]

@dash_app.callback(
    [Output('age_population_total', 'children'), Output('age_percentage_total', 'children'), Output('page-2-button', 'disabled'), Output('page-2-continue-warning', 'children'), Output('age-group-continue-warning', 'color')],
    [Input({'type':'age-popu-input', 'index':ALL}, 'value'), Input({'type':'age-popu-slider', 'index':ALL}, 'value')],
    [State('total-population', 'value')])
def update_age_group_total(input_values, slider_values, total_value):
    sum_input = sum(input_values)
    sum_slider = sum(slider_values)
    sum_perc_str = int_perc_1dp(sum_input, total_value)
    if (sum_input==total_value):
        return str(sum_input), sum_perc_str, False, '', 'secondary'
    else:
        return str(sum_input), sum_perc_str, True, err_group_total_not_equal_popu, 'danger'

@dash_app.callback(
    [Output({'type':'age-popu-input', 'index':ALL}, 'max'), Output({'type':'age-popu-slider', 'index':ALL}, 'max'), Output({'type':'age-popu-slider', 'index':ALL}, 'value'), Output('slider-gender-perc','value')],
    [Input('total-population', 'value'), Input('age-default-perc','n_clicks')],
    [State({'type':'age-popu-input', 'index':ALL}, 'value'), State('total-population', 'min'), State('total-population', 'max'), State('gender-perc-female', 'children'), State('gender-perc-male','children'), State('page-2-button', 'n_clicks')])
def update_age_popu_max(total_value, default_n_clicks, input_values, total_min, total_max, gender_perc_female, gender_perc_male, next_n_clicks):
    inputParameterCache = InputParameterCache(_redis)
    context = dash.callback_context
    if total_value is None:
        raise PreventUpdate
    elif total_value <= total_min:
        raise PreventUpdate
    elif total_value > total_max:
        raise PreventUpdate
    else:
        updated_maxs = [total_value]*len(input_values)
        gender_values = [gender_perc_female, gender_perc_male]
        gender_reset_values = [50, 50]
        slider_reset_values = [round(total_value*x/100) for x in age_perc_start]
        if sum(slider_reset_values) != total_value:  # reset last age group population due to rounding off
            slider_reset_values[-1] = total_value - sum(slider_reset_values) + slider_reset_values[-1]
        if not context.triggered:
            slider_values = slider_reset_values
        else:
            id_trig = context.triggered[0]['prop_id'].split('.')[0]
            if id_trig == 'total-population':
                if next_n_clicks is None:  # age group population and gender percentage update from cache
                    input_param_age_group = {
                        id_age_popu[0]: 0,
                        id_age_popu[1]: 0,
                        id_age_popu[2]: 0,
                        id_age_popu[3]: 0,
                        id_age_popu[4]: 0,
                        id_age_popu[5]: 0,
                        id_age_popu[6]: 0,
                        id_age_popu[7]: 0,
                        id_age_popu[8]: 0,
                    }
                    input_param_gender = {
                        'gender-perc-female': 0,
                        'gender-perc-male': 0,
                    }
                    slider_values = inputParameterCache.cache_get(input_param_age_group.keys())
                    gender_values = inputParameterCache.cache_get(input_param_gender.keys())
                    if None in slider_values:
                        slider_values = slider_reset_values
                    if None in gender_values:
                        gender_values = gender_reset_values
                else:
                    slider_values = slider_reset_values
            elif id_trig == 'age-default-perc':
                slider_values = slider_reset_values
            else: # should not happen
                raise PreventUpdate
            return updated_maxs, updated_maxs, slider_values, gender_values[0]

@dash_app.callback(
    [Output('total-population', 'value'),
     Output('ethnic-no-1', 'value'), Output('ethnic-no-2', 'value'), Output('ethnic-no-3', 'value'),
     Output('ethnic-no-4', 'value'), Output('ethnic-no-5', 'value'), Output('ethnic-no-6', 'value'),
     Output('ethnic-no-7', 'value'), Output('ethnic-no-8', 'value'), Output('ethnic-no-9', 'value'),
     Output('ethnic-no-10', 'value'),
     Output('accommodation-area-type1', 'value'), Output('accommodation-area-type2', 'value'), Output('accommodation-area-type3', 'value'),
     Output('accommodation-no-unit-type1', 'value'), Output('accommodation-no-unit-type2', 'value'), Output('accommodation-no-unit-type3', 'value'),
     Output('accommodation-no-person-type1', 'value'), Output('accommodation-no-person-type2', 'value'), Output('accommodation-no-person-type3', 'value')],
    [Input('page-2-button', 'n_clicks')],
    [State('total-population', 'value'), State({'type':'age-popu-input','index':ALL}, 'value'),
     State('gender-perc-female', 'children'), State('gender-perc-male', 'children'),
     State('ethnic-no-1', 'value'), State('ethnic-no-2', 'value'), State('ethnic-no-3', 'value'),
     State('ethnic-no-4', 'value'), State('ethnic-no-5', 'value'), State('ethnic-no-6', 'value'),
     State('ethnic-no-7', 'value'), State('ethnic-no-8', 'value'), State('ethnic-no-9', 'value'),
     State('ethnic-no-10', 'value'),
     State('accommodation-area-type1', 'value'), State('accommodation-area-type2', 'value'), State('accommodation-area-type3', 'value'),
     State('accommodation-no-unit-type1', 'value'), State('accommodation-no-unit-type2', 'value'), State('accommodation-no-unit-type3', 'value'),
     State('accommodation-no-person-type1', 'value'), State('accommodation-no-person-type2', 'value'), State('accommodation-no-person-type3', 'value')])
def update_input_parameter_page_2(
    n_clicks, total_population_value, age_popu_input_values,
    gender_perc_female_value, gender_perc_male_value,
    ethnic_no_1_value, ethnic_no_2_value, ethnic_no_3_value,
    ethnic_no_4_value, ethnic_no_5_value, ethnic_no_6_value,
    ethnic_no_7_value, ethnic_no_8_value, ethnic_no_9_value, ethnic_no_10_value,
    accommodation_area_type1_value, accommodation_area_type2_value, accommodation_area_type3_value,
    accommodation_no_unit_type1_value, accommodation_no_unit_type2_value, accommodation_no_unit_type3_value,
    accommodation_no_person_type1_value, accommodation_no_person_type2_value, accommodation_no_person_type3_value
):
    inputParameterCache = InputParameterCache(_redis)
    input_param = {
        'total-population': total_population_value,
        id_age_popu[0]: age_popu_input_values[0],
        id_age_popu[1]: age_popu_input_values[1],
        id_age_popu[2]: age_popu_input_values[2],
        id_age_popu[3]: age_popu_input_values[3],
        id_age_popu[4]: age_popu_input_values[4],
        id_age_popu[5]: age_popu_input_values[5],
        id_age_popu[6]: age_popu_input_values[6],
        id_age_popu[7]: age_popu_input_values[7],
        id_age_popu[8]: age_popu_input_values[8],
        'gender-perc-female': gender_perc_female_value,
        'gender-perc-male': gender_perc_male_value,
        'ethnic-no-1': ethnic_no_1_value,
        'ethnic-no-2': ethnic_no_2_value,
        'ethnic-no-3': ethnic_no_3_value,
        'ethnic-no-4': ethnic_no_4_value,
        'ethnic-no-5': ethnic_no_5_value,
        'ethnic-no-6': ethnic_no_6_value,
        'ethnic-no-7': ethnic_no_7_value,
        'ethnic-no-8': ethnic_no_8_value,
        'ethnic-no-9': ethnic_no_9_value,
        'ethnic-no-10': ethnic_no_10_value,
        'accommodation-area-type1': accommodation_area_type1_value,
        'accommodation-area-type2': accommodation_area_type2_value,
        'accommodation-area-type3': accommodation_area_type3_value,
        'accommodation-no-unit-type1': accommodation_no_unit_type1_value,
        'accommodation-no-unit-type2': accommodation_no_unit_type2_value,
        'accommodation-no-unit-type3': accommodation_no_unit_type3_value,
        'accommodation-no-person-type1': accommodation_no_person_type1_value,
        'accommodation-no-person-type2': accommodation_no_person_type2_value,
        'accommodation-no-person-type3': accommodation_no_person_type3_value,
    }
    if n_clicks is None:
        value = inputParameterCache.cache_get(input_param.keys())  # get cached value
        for i,j in enumerate(value):
            if j is None:  # if first time loading, get default value
                if 'total-population' == list(input_param.keys())[i]:
                    value[i] = total_popu
                else:
                    value[i] = None
        del value[1:12]  # remove age group population and gender percentage cache
        return value
    else:
        inputParameterCache.cache_set(input_param, 2)  # put all input parameters in input page 2 to cache
        raise PreventUpdate

max_stars_repo_name: AIforGoodSimulator/model-server

max_issues_repo_name: AIforGoodSimulator/model-server

max_forks_repo_name: AIforGoodSimulator/model-server


text:
# -*- coding: utf-8 -*-
"""
Defines a function to randomly generate particle positions according to
the desired surface density profile (sigma vs r) and the vertical profile
(rho vs r,z).

Created on Mon Jan 27 18:48:04 2014

@author: ibackus
"""

import pynbody
SimArray = pynbody.array.SimArray
import numpy as np
import cPickle as pickle
from scipy.interpolate import RectBivariateSpline
from scipy.interpolate import interp1d
import scipy.optimize as opt

# LOCAL IMPORTING
import calc_sigma
import calc_rho_zr
import isaac

class pos:

    def __init__(self, ICobj, method = None):

        self._parent = ICobj
        # Check that sigma and rho have been generated
        if not hasattr(ICobj, 'rho'):

            raise NameError,'rho could not be found in the IC object'

        if not hasattr(ICobj,'sigma'):

            raise NameError,'sigma could not be found in the IC object'

        if method == None:

            self.method = ICobj.settings.pos_gen.method

        else:

            self.method = method
            # Update settings in ICobj
            ICobj.settings.pos_gen.method = method

        self.nParticles = ICobj.settings.pos_gen.nParticles
        print 'Generating {} particle positions using method: {}'.format(\
        self.nParticles, self.method)

        # Generate positions
        self._generate_r()
        self._generate_z()
        self._generate_theta()
        self._cartesian_pos()

    def __getstate__(self):
        """
        This is required to make the object pickle-able
        """

        # Define a dictionary containing everything needed.  Ignore self.parent
        state = self.__dict__.copy()
        state.pop('_parent', None)

        return state

    def _generate_r(self):
        """
        Generate radial positions
        """

        print 'Generating r positions'
        cdf_inv_r = self._parent.sigma.cdf_inv

        if self.method == 'grid':

            # Generate linearly increasing values of m, using 2 more than
            # necessary to avoid boundary issues
            m = np.linspace(0,1,self.nParticles + 2)
            # Calculate r from inverse CDF
            r = cdf_inv_r(m[1:-1])
            # Assign output
            self.r = r

        if self.method == 'random':

            m = np.random.rand(self.nParticles)
            r = cdf_inv_r(m)
            self.r = r

    def _generate_z(self):
        """
        Generate z positions
        """

        print 'Generating z positions'
        # The inverse CDF over z as a function of r
        cdf_inv_z = self._parent.rho.cdf_inv
        # Random numbers between 0 and 1
        m = np.random.rand(self.nParticles)
        # Calculate z
        z = cdf_inv_z(m, self.r)
        # Randomly select sign of z
        z = z * np.random.choice(np.array([-1,1]), self.nParticles)
        # Assign output
        self.z = z

    def _generate_theta(self):
        """
        Generate angular positions
        """

        nParticles = self.nParticles

        if self.method == 'grid':

            r = self.r

            dtheta = np.sqrt(2*np.pi*(1 - r[0:-1]/r[1:]))
            dtheta = isaac.strip_units(dtheta)
            theta = np.zeros(nParticles)

            for n in range(nParticles - 1):

                # NOTE: it's import to subtract (not add) dtheta.  The particles
                # will be moving counter-clockwise.  To prevent the particle
                # spirals from kinking, the particle spirals must go out
                # clockwise
                theta[n+1] = theta[n] - dtheta[n]

            self.theta = theta

        if self.method == 'random':

            theta = 2*np.pi*np.random.rand(nParticles)
            self.theta = theta

    def _cartesian_pos(self):
        """
        Generate x,y
        """

        r = self.r
        z = self.z
        theta = self.theta
        x = r*np.cos(theta)
        y = r*np.sin(theta)

        xyz = np.zeros([self.nParticles, 3])
        xyz = isaac.match_units(xyz, r)[0]

        xyz[:,0] = x
        xyz[:,1] = y
        xyz[:,2] = isaac.match_units(z, r)[0]

        self.x = x
        self.y = y
        self.xyz = xyz

def make(rhoFileName,sigFileName,nParticles,rlim = None,zlim = None,\
savename = None):
    """
    Generate particle positions on a pseudo-grid according to the surface density
    profile contained in sigFileName and the vertical profile contained in
    rhoFileName.  PDFs are interpolated using a cubic spline, and are
    assumed to be 0 outside the regions defined the the input files

    * Arguments *
    rhoFileName - file name for a file containing a pickled dictionary
        for rho calculated at points on a grid.  The dictionary should
        contain:
            dict['rho'] : 2D array, rho at all pairs of points (z,r)
            dict['z']   : a 1D array of z points
            dict['r']   : a 1D array of r points

    sigFileName - file name for a file containing a pickled dictionary
        for sigma (surface density) calculated at points r.  Dictionary
        should contain:
            dict['sigma']   : 1D array, rho at all points r
            dict['r']       : 1D array of r points

    nParticles - total number of particles to generate

    rlim - cylindrical radius bounds for generating particles.  if None,
        then the default rlims are the range of values contained in
        rhoFileName.

    zlim - positive z-bounds for generating particles.  Particles will be
        always generated symmetrically, within these bounds (the sign of
        z is randomly chosen).  If None, bounds are taken from rhoFileName

    savename - if None, no file is saved.  Otherwise, the output is pickled
        and saved to 'savename'

    * Output *
    Returns a dictionary containing the randomly generated positions, with
    the following keys:

        'x'     : x-positions of the particles
        'y'     : y-positions
        'z'     : z-positions
        'r'     : monotonically increasing R-positions
        'theta' : angular positions

    !!!Everything should be in units of Msol and au!!!
    pynbody simarrays are acceptable
    """
    # ------------------------------------------------------------
    # Loading
    # ------------------------------------------------------------
    a = pickle.load(open(rhoFileName,'rb'))
    rho = a['rho']
    z = a['z']
    r = a['r']
    if rlim == None:
        rlim = [r.min(),r.max()]
    if zlim == None:
        zlim = [z.min(),z.max()]
    # ------------------------------------------------------------
    # Generate PDFs
    # ------------------------------------------------------------
    #print('Generating rho(z,r) spline interpolation')
    #rho_zr = RectBivariateSpline(z,r,rho)
    cdfinv_zr = calc_rho_zr.cdfinv_zr(rhoFileName)

    # Probability as a function of r (2*pi*r*sigma, up to a constant factor)
    # Pr is normalized, but we need the MAX value of it
    print 'Calculating probability vs r spline'
    Pr = calc_sigma.prob(sigFileName)
    def PrNeg(x):
        return -1.0*Pr(x)
    print 'Finding maximum probability for r = {0} to {1}'.format(rlim[0],rlim[1])
    #print PrNeg(rlim[1])
    xmax = opt.fminbound(PrNeg,float(rlim[0]),float(rlim[1]))
    Prmax = Pr(xmax)
    print 'Max. prob = {0} \nat r = {1}'.format(Prmax,xmax)
    # ------------------------------------------------------------
    # Generate random R values according to PDF from sigma vs r
    # ------------------------------------------------------------
    print('Calculating R')
    R = np.zeros(nParticles)
    cdfinv_r = calc_sigma.cdfinv_r(sigFileName,Pr)
    # Calculate R.  Note, we drop the first particle position because it is
    # at the origin at the last particle position to keep it from being at the
    # edge of our radial bins
    m = np.linspace(0,1,nParticles + 2)
    R = cdfinv_r(m[1:-1])
    # ------------------------------------------------------------
    # Generate random Z values for the R values found above,
    # according to rho(z,r)
    # ------------------------------------------------------------
    print('Calculating Z')
    Z = np.zeros(nParticles)
    cdfvals = np.random.uniform(size=nParticles)
    rbins = np.digitize(R,r)
    dr = r[1]-r[0]
    for i in range(nParticles):
        rbin = rbins[i]
        zlo = cdfinv_zr[rbin-1](cdfvals[i])
        zhi = cdfinv_zr[rbin](cdfvals[i])
        Z[i] = zlo + ((zhi-zlo)/dr)*(R[i] - r[rbin-1])
    # ------------------------------------------------------------
    # Generate angles
    # ------------------------------------------------------------
    dtheta = np.sqrt(2*np.pi*(1 - R[0:-1]/R[1:]))
    theta = np.zeros(nParticles)
    for n in range(0,nParticles-1):
        theta[n+1] = theta[n] + dtheta[n]
    # ------------------------------------------------------------
    # Format the output
    # ------------------------------------------------------------
    # Assign units
    if pynbody.units.has_units(r):
        rUnits = r.units
    else:
        rUnits = pynbody.units.au
    if pynbody.units.has_units(z):
        zUnits = z.units
    else:
        zUnits = pynbody.units.au
    if pynbody.units.has_units(Z):
        Z.convert_units(zUnits)
    else:
        Z = SimArray(Z,zUnits)
    if pynbody.units.has_units(R):
        R.convert_units(rUnits)
    else:
        R = SimArray(R,rUnits)
    Z = Z*np.sign(np.random.randn(nParticles))
    x = R*np.cos(theta)
    y = R*np.sin(theta)
    # Add random noise to positions
    var = np.zeros(nParticles)
    var[1:] = (x[1:]-x[0:-1])**2 + (y[1:]-y[0:-1])**2
    var[0] = var[1]
    mu = np.zeros(2)
    xydelta = np.zeros([nParticles,2])
    rand_scale = 0.01
    for n in range(nParticles):
        cov = var[n]*np.identity(2)
        xydelta[n,:] = np.random.multivariate_normal(mu,cov)
    xydelta *= rand_scale
    x += xydelta[:,0]
    y += xydelta[:,1]
    R = np.sqrt(x**2 + y**2)

    # Output
    outDict = {'x': x,'y': y, 'z': Z, 'r': R, 'theta': theta}
    if savename is not None:
        pickle.dump(outDict,open(savename,'wb'))
    return outDict

max_stars_repo_name: trquinn/ICgen

max_issues_repo_name: trquinn/ICgen

max_forks_repo_name: trquinn/ICgen

reason: import numpy,import scipy,from scipy


text:
"""
Metrics for using classification and regression on plaintext
"""
from typing import Union

import numpy as np
import sklearn.metrics

from tno.mpc.mpyc.secure_learning.utils import NumpyNumberArray, NumpyOrVector

# Classification metrics
def accuracy_score(y_real: NumpyOrVector, y_pred: NumpyOrVector) -> float:
    """
    Computes the accuracy of the predicted labels. Accuracy is computed as
    the ratio of all correctly predicted labels over the number of predicted
    labels.

    :param y_real: Real labels
    :param y_pred: Predicted labels
    :return: Accuracy of given predictions
    """
    return sklearn.metrics.accuracy_score(y_real, y_pred)

def precision_score(
    y_real: NumpyOrVector, y_pred: NumpyOrVector, pos_label: Union[str, int]
) -> float:
    """
    Computes the precision of the predicted labels of category
    pos_label. Precision is computed as the ratio of all correctly
    predicted pos_label over the number of predicted pos_label. This is
    an indication how precise the predictions of pos_label are: given
    prediction pos_label, how likely is it that the true label is
    pos_label.

    :param y_real: Real labels
    :param y_pred: Predicted labels
    :param pos_label: Label to compute precision of
    :return: Precision of given predictions
    """
    return sklearn.metrics.precision_score(y_real, y_pred, pos_label=pos_label)

def recall_score(
    y_real: NumpyOrVector, y_pred: NumpyOrVector, pos_label: Union[str, int]
) -> float:
    """
    Computes the recall of the predicted labels of category pos_label.
    Recall is computed as the ratio of all correctly predicted pos_label
    over the number of real pos_label. This is an indication how many
    actual pos_label we misclassified.

    :param y_real: Real labels
    :param y_pred: Predicted labels
    :param pos_label: Label to compute recall of
    :return: Recall of given predictions
    """
    return sklearn.metrics.recall_score(y_real, y_pred, pos_label=pos_label)

def f1_score(
    y_real: NumpyOrVector, y_pred: NumpyOrVector, pos_label: Union[str, int]
) -> float:
    """
    F1-score for given predicted and real target labels.

    :param y_real: Real labels
    :param y_pred: Predicted labels
    :param pos_label: Label to compute f1 for
    :return: f1 score
    """
    return sklearn.metrics.f1_score(y_real, y_pred, pos_label=pos_label)

# Regression metrics
def r2_score(y_real: NumpyOrVector, y_pred: NumpyOrVector) -> float:
    """
    R-squared value for given predicted and real target values.

    :param y_real: Real target values
    :param y_pred: Predicted target values
    :return: R-squared value
    """
    return sklearn.metrics.r2_score(y_real, y_pred)

def adj_r2_score(
    y_real: NumpyOrVector, y_pred: NumpyOrVector, n_features: int
) -> float:
    """
    Adjusted R-squared value for given predicted and real target values.

    :param y_real: Real target values
    :param y_pred: Predicted target values
    :param n_features: Number of features
    :return: Adjusted R-squared value
    """
    n_samples = len(y_pred)
    r2 = r2_score(y_real, y_pred)
    return 1 - (1 - r2) * (n_samples - 1) / (n_samples - n_features - 1)

def mean_squared_error(y_real: NumpyOrVector, y_pred: NumpyOrVector) -> float:
    """
    Compute residual mean of squares. Residual mean of squares equals the
    mean of squares of deviations between predicted and real values.

    :param y_real: Real target values
    :param y_pred: Predicted target values
    :return: Residual mean of squares
    """
    return sklearn.metrics.mean_squared_error(y_real, y_pred)

def mean_squared_model(y_real: NumpyOrVector) -> float:
    """
    Compute explained mean of squares. Explained mean of squares equals the
    mean of squares of deviations from the mean.

    :param y_real: Input data
    :return: Explained mean of squares
    """
    y: NumpyNumberArray = np.asarray(y_real)
    dev_from_mean = y - y.mean()
    return float(np.inner(dev_from_mean, dev_from_mean)) / len(y)


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import datetime as dt
from pandas_datareader import data
import statsmodels.api as sm
from statsmodels.tsa.seasonal import STL

def get_fx(stock,start,end):
    df = data.DataReader(stock, 'stooq',start)["Close"]
    df = df.iloc[::-1]
    return df[start:end]

bunseki = "trend"
start = dt.date(2020,1,1)
end = dt.date(2020,6,11)

import pandas_datareader.data as DataReader

#fx_list = ['ZAR=X','JPY=X', 'EUR=X', 'GBP=X', 'AUD=X']
fx_list = ['JPY=X']
for j in fx_list:
    stock0= j
    df0=DataReader.get_data_yahoo("{}".format("JPY=X"),start,end)
    #df1=DataReader.get_data_yahoo("{}".format(stock0),start,end)
    df=df0 #/df1
    print(df0)
    #print(df1)
    print(df)

    series=df['Close']
    cycle, trend = sm.tsa.filters.hpfilter(series, 144)
    df['trend']=  trend

    def EMA1(x, n):
        #k = 3.45*(n+1)
        a= 2/(n+1)
        return pd.Series(x).ewm(alpha=a).mean()

    series2 = df['trend'].values.tolist()
    print(series2[len(series2)-10:len(series2)])

    df['Close']=series  #series" #cycle" #trend
    df['Close2']=series2
    df['y12'] = EMA1(df['Close2'], 12)
    df['y26'] =  EMA1(df['Close2'], 26)
    df['MACD'] = df['y12'] -df['y26']
    df['MACD2'] = df['Close2'] -df['y26'] #12
    df['signal2'] = EMA1(df['MACD2'], 9)
    df['signal'] = EMA1(df['MACD'], 9)
    df['hist_']=df['MACD2']-df['signal2']
    date_df=df['Close'].index.tolist()
    print(df[len(series)-10:len(series)])

    fig, (ax1,ax2) = plt.subplots(2,1,figsize=(1.6180 * 8, 4*2),dpi=200)
    ax1.plot(df['Close'],label="series")
    ax1.plot(df['Close2'],label="series2")
    ax1.plot(df['y12'],label="y12")
    ax1.plot(df['y26'],label="y26")
    ax2.plot(df['MACD2'],label="MACD2")
    #ax2.plot(df['MACD'],label="MACD")
    ax2.plot(df['signal2'],label="signal2")
    #ax2.plot(df['signal'],label="signal")
    ax2.bar(date_df,df['hist_'])
    ax1.set_title("{}/{}".format(stock0,"JPY"))
    ax1.legend()
    ax2.legend()
    ax1.grid()
    ax2.grid()
    #ax2.set_ylim(-5,5)
    plt.savefig("./fx/{}_{}_{}_.png".format(stock0,"JPN=X",end))
    plt.pause(1)
    plt.close()

reason: import numpy,import statsmodels,from statsmodels