text:
# ------------------------------------------------------------------------------
# Python API to access CodeHawk Binary Analyzer analysis results
# Author: Henny Sipma
# ------------------------------------------------------------------------------
# The MIT License (MIT)
#
# Copyright (c) 2016-2019 Kestrel Technology LLC
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ------------------------------------------------------------------------------

from scipy.sparse import dok_matrix
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.metrics.pairwise import cosine_similarity
from scipy import mat

from scs.x86x.retrieval.IndexedPostings import IndexedPostings
from scs.x86x.retrieval.IndexedVocabulary import IndexedVocabulary

class FindSimilar(object):

    def __init__(self,searchindex,searchterms):
        self.searchindex = searchindex
        self.searchterms = searchterms
        self.featuresets = self.searchterms.get_featuresets()
        self.postings = {}
        self.vocabulary = {}
        self.weightings = {}
        self.similarity = {}
        self._initialize(self.featuresets)

    def search(self):
        doccount = self.searchindex.get_doc_count()
        termcount = self.searchterms.get_term_count()
        print('Creating a ' + str(doccount) + ' by ' + str(termcount) + ' matrix')
        xpattern = self.searchterms.get_expanded_pattern(self.vocabulary)
        print('Expanded pattern: ' + str(xpattern))
        self.dokmatrix = dok_matrix((doccount,termcount),dtype=int)
        self.docids = list(self.searchindex.get_doc_ids())
        for dx in range(doccount):
            tx = 0
            for fs in sorted(xpattern):
                for termix in xpattern[fs]:
                    if self.postings[fs].has_term(self.docids[dx],termix):
                        self.dokmatrix[dx,tx] = 1
                        self.postings[fs].use_term(self.docids[dx],termix)
                    tx += 1
        tfidf = TfidfTransformer(norm='l2',smooth_idf=True)
        tfidf.fit(self.dokmatrix)
        self.set_weightings(xpattern,tfidf.idf_)
        self.similarity = cosine_similarity(mat([tfidf.idf_]),self.dokmatrix)

    def set_weightings(self,xpattern,idfvector):
        index = 0
        for fs in sorted(xpattern):
            for termix in xpattern[fs]:
                t = self.vocabulary[fs].get_term(termix)
                self.weightings[index] = (idfvector[index],t,fs)
                index += 1

    def get_similarity_results_properties(self,propertyformats,mincutoff=0.0,maxcutoff=1.0):
        featuresets = propertyformats.get_featuresets()
        results = self._get_similarity_results(propertyformats,mincutoff,maxcutoff)
        properties = {}   # fs -> termix -> count
        for fs in results:
            properties[fs] = {}
            for docix in results[fs]:
                for termix in results[fs][docix]:
                    properties[fs].setdefault(termix, 0)
                    properties[fs][termix] += results[fs][docix][termix]
        xproperties = {}   # fs ->  term -> count
        for fs in properties:
            xproperties[fs] = {}
            for termix in properties[fs]:
                term = self.vocabulary[fs].get_term(int(termix))
                xproperties[fs][term] = properties[fs][termix]
        return xproperties

    def get_similarity_results_representatives(self,propertyformats,mincutoff=0.0,maxcutoff=1.0):
        featuresets = propertyformats.get_featuresets()
        results = self._get_similarity_results(propertyformats,mincutoff,maxcutoff)
        properties = {}   # fs -> termix -> count
        for fs in results:
            properties[fs] = {}
            for docix in results[fs]:
                for termix in results[fs][docix]:
                    document = self.searchindex.get_document(docix)[0]
                    properties[fs].setdefault(termix, document)
        xproperties = {}
        for fs in properties:
            xproperties[fs] = {}
            for termix in properties[fs]:
                term = self.vocabulary[fs].get_term(int(termix))
                xproperties[fs][term] = properties[fs][termix]
        return xproperties

    def get_similarity_results_structured(self,mincutoff=0.0,maxcutoff=1.0):
        qresults = []
        for i in range(self.searchindex.get_doc_count()):
            if self.similarity[0,i] >= mincutoff and self.similarity[0,i] <= maxcutoff:
                qresults.append((i,self.similarity[0,i]))
        qresults = sorted(qresults,key=lambda x:x[1],reverse=True)
        results = []
        for i in range(len(qresults)):
            docix = self.docids[qresults[i][0]]
            document = self.searchindex.get_document(docix)
            results.append((qresults[i][1],document))
        return results

    def _initialize(self,featuresets):
        for fs in featuresets:
            fsvoc = self.searchindex.get_featureset_vocabulary(fs)
            if not fsvoc is None:
                self.vocabulary[fs] = IndexedVocabulary(fsvoc)
            fspostings = self.searchindex.get_all_featureset_postings(fs)
            self.postings[fs] = IndexedPostings(fspostings)

    def _get_similarity_results(self,propertyformats,mincutoff=0.0,maxcutoff=1.0):
        qresults = []
        featuresets = propertyformats.get_featuresets()
        self._initialize(featuresets)
        for i in range(self.searchindex.get_doc_count()):
            if self.similarity[0,i] >= mincutoff and self.similarity[0,i] <= maxcutoff:
                qresults.append((i,self.similarity[0,i]))
        results = {}        # fs -> docix - > termix -> count
        for i in range(len(qresults)):
            docix = self.docids[qresults[i][0]]
            for fs in featuresets:
                if fs in self.vocabulary and fs in self.postings:
                    results.setdefault(fs,{})
                    results[fs][docix] = self.postings[fs].get_doc_term_frequency(docix)
        return results

reason: from scipy


text:
"""
MomentGenerator generates moment matrices used in mean-variance optimization problems.
Covariance Matrix (of Moment 2) and Higher Moment Matrices can all be generated using the class.

Higher Moment Implementation Reference
https://cran.r-project.org/web/packages/PerformanceAnalytics/vignettes/EstimationComoments.pdf
"""

import itertools
import warnings

import numpy as np
import pandas as pd
from sklearn.covariance import *

from .Exceptions import *

class MomentGenerator:

    def __init__(self, return_data, assets=None):
        """
        Initializes MomentGenerator
        :param return_data: pd.DataFrame/np.ndarray, return data with return values in percentage terms
        :param assets: List[str] optional, list of assets/tickers corresponding to the np.ndarray
        """

        if isinstance(return_data, pd.DataFrame):
            self.return_mat = return_data.values.transpose()
            self.assets = return_data.columns
        elif isinstance(return_data, np.ndarray):
            self.return_mat = return_data
            self.assets = assets
        else:
            raise FormatException("Invalid Format. Return Data options: pd.DataFrame, np.ndarray")

    def calc_cov_mat(self, technique='sample', semi=False, method='default', time_scaling=252,
                     builtin=False, weights=None, bm_return=0.00025, assume_zero=False, normalize=False, ret_format='df', **kwargs):
        """
        Calculates covariance matrix given the return data input.

        :param technique: str, default='sample'
                additional_options: ["EmpiricalCovariance", "EllipticEnvelope", "GraphicalLasso", "GraphicalLassoCV",
                                    "LedoitWolf", "MinCovDet", "OAS", "ShrunkCovariance"]
                Specifies the calculation technique for the covariance matrix
        :param semi: bool, default=False
                If True, returns a semivariance matrix that emphasizes on downside portfolio variance
        :param method: str, default='default'
                additional_options: ["exponential", "custom"]
                Default implies all returns are weighted equally,
                Exponential requires the specification of decay factors and timespan of decay
                Custom requires a customized weight array
        :param time_scaling: int, default=252
                Default annualizes the covariance matrix (assuming daily return is the input)
        :param builtin: bool, default=False,
                If True then calls np.cov() to calculate, otherwise use matrix calculation method written in the class.
                Note that calling builtin function versus using calculation methods yield identical result
        :param weights: np.ndarray, default=None
                If choose parameter method='custom', then input the customized weight array
        :param bm_return: float, default=0.00025,
                Additional parameter for calculating semivariance matrix.
                Ignores all individual asset returns above the bm_return when calculating covariance
        :param assume_zero: bool, default=False
                Additional parameter for calculating semivariance matrix.
                Long term daily mean return for an individual asset is sometiimes assumed to be 0.
                However, this may not hold true if sample size is not sufficiently large.
        :param normalize: bool, default=False
                To normalize the covariance matrix. In the specific case for covariance matrix, a normalized covariance
                matrix is a correlation matrix.
        :param ret_format: str, default='df'
                Additional Options: ["raw"]
                Returns a covariance matrix in the form of a pd.DataFrame, or a tuple of asset names and np.ndarray
        :param kwargs: decay, span (Check MomentGenerator.exp_factor for usage)
        :return: Covariance Matrix
        """

        return_mat = self.return_mat

        if semi:
            return_mat = self.semi_cov(return_mat, bm_return=bm_return, assume_zero=assume_zero)

        if technique == "sample":
            cov_mat = self.sample_cov(return_mat, method, time_scaling, builtin=builtin, weights=weights, **kwargs)
        else:
            cov_mat = self.sk_technique(return_mat, technique, time_scaling, **kwargs)

        if normalize:
            cov_mat = cov_mat * np.dot(((np.diag(cov_mat)) ** -0.5).reshape(-1, 1),
                                       ((np.diag(cov_mat)) ** -0.5).reshape(1, -1))
        if ret_format == 'df':
            return pd.DataFrame(cov_mat, index=self.assets, columns=self.assets)
        elif ret_format == 'raw':
            return self.assets, cov_mat
        else:
            raise FormatException("Invalid Format. Valid options are: df, raw")

    def calc_beta(self, beta_vec, technique="sample", semi=False, method='default', builtin=False, weights=None, ret_format='series', **kwargs):
        """
        Calculates beta of each asset given benchmark return.

        Beta = Covariance(Asset, Benchmark)/Variance(Benchmark)

        The parameters' usage are identical to that of calling method calc_cov_mat as the method creates a
        covariance matrix combining the benchmark return and the individual asset return.

        In this case, setting semi=True gives the downside beta.

        :param beta_vec: pd.DataFrame/pd.Series/np.ndarray, benchmark return vector
        :param ret_format: str, default='series'
                Additional option = ["raw"]
                Returns a covariance matrix in the form of a pd.Series, or a tuple of asset names and np.ndarray
        :return: beta values for each asset
        """

        if isinstance(beta_vec, (pd.DataFrame, pd.Series)):
            beta_vec = beta_vec.values.T.reshape(1, -1)
        elif isinstance(beta_vec, np.ndarray):
            beta_vec = beta_vec.reshape(1, -1)

        if beta_vec.shape[1] != self.return_mat.shape[1]:
            raise DimException("""Dimension of benchmark (beta） data is not in the same length as the return data""")

        return_mat = np.concatenate([self.return_mat, beta_vec])

        if semi:
            return_mat = self.semi_cov(return_mat, bm_return=beta_vec.mean(), assume_zero=False)

        if technique == "sample":
            cov_mat = self.sample_cov(return_mat, method, unit_time=1, builtin=builtin, weights=weights, **kwargs)
        else:
            cov_mat = self.sk_technique(return_mat, technique, time_scaling=1, **kwargs)

        beta_arr = cov_mat[-1, :-1]/cov_mat[-1, -1]

        if ret_format == 'series':
            return pd.Series(beta_arr, index=self.assets)
        elif ret_format == 'raw':
            return self.assets, beta_arr
        else:
            raise FormatException("Invalid Format. Valid Options are: series, raw")

    def calc_coskew_mat(self, semi=False, method='default', weights=None, bm_return=0.001, assume_zero=False, normalize=True, ret_format='df', **kwargs):

        """
        Calculates the coskewness matrix given the return data.
        Dimension (N, N^2) N: num_assets

        The parameters' usage are identical to that of calling method calc_cov_mat.
        Parameter normalize default=True as coskewness matrix is typically used in its normalized form
        """
        return self.calc_comoment_mat(moment=3, semi=semi, method=method, weights=weights, bm_return=bm_return, assume_zero=assume_zero, normalize=normalize, ret_format=ret_format, **kwargs)

    def calc_cokurt_mat(self, semi=False, method='default', weights=None, bm_return=0.001, assume_zero=False, normalize=True, ret_format='df', **kwargs):

        """
        Calculates the cokurtosis matrix given the return data.
        Dimension (N, N^3) N: num_assets

        The parameters' usage are identical to that of calling method calc_cov_mat.
        Parameter normalize default=True as cokurtosis matrix is typically used in its normalized form
        """
        return self.calc_comoment_mat(moment=4, semi=semi, method=method, weights=weights, bm_return=bm_return, assume_zero=assume_zero, normalize=normalize, ret_format=ret_format, **kwargs)

    def calc_comoment_mat(self, moment, semi=False, method='default', weights=None, bm_return=0.001, assume_zero=False, normalize=True, ret_format='df', **kwargs):

        """
        Calculates higher moment matrices matrix given the return data.
        Dimension (N, N^(moment-1)) N: num_assets

        :param moment: int, moment number

        *Moments higher than 4 (cokurtosis) are not commonly used in portfolio constructions.
        *Calling this method to calculate covariance matrix by setting moment=2 will yield identical result

        The parameters' usage are identical to that of calling method calc_cov_mat.
        """
        return_mat = self.return_mat

        if semi:
            return_mat = self.semi_cov(return_mat, bm_return=bm_return, assume_zero=assume_zero)

        weight_factor = self.construct_weight(method, return_mat, weights, **kwargs)

        weight_mat = MomentGenerator.calc_weight_mat(return_mat, weight_factor)
        comoment_mat = MomentGenerator.calc_moment_mat(moment, return_mat, weight_mat, normalize)

        if ret_format == 'df':
            return pd.DataFrame(comoment_mat, index=self.assets, columns=tuple(itertools.product(*[self.assets for i in range(moment - 1)])))
        elif ret_format == 'raw':
            return self.assets, comoment_mat
        else:
            raise FormatException("Invalid Format. Valid options are: df, raw")

    def sk_technique(self, return_mat, technique, time_scaling=252, **kwargs):

        """
        Using sklearn.covariance methods to construct covariance matrix
        :param return_mat: np.ndarray, return matrix
        :param technique: str, options to select sklearn.covariance methods
        :param time_scaling: int, default=252, annualize covariance matrix (assuming daily input)
        :param kwargs: additional arguments used in sklearn methods
        :return: np.ndarray, covariance matrix in its raw form
        """
        technique_dict = {"EmpiricalCovariance": EmpiricalCovariance,
                          "EllipticEnvelope": EllipticEnvelope,
                          "GraphicalLasso": GraphicalLasso,
                          "GraphicalLassoCV": GraphicalLassoCV,
                          "LedoitWolf": LedoitWolf,
                          "MinCovDet": MinCovDet,
                          "OAS": OAS,
                          "ShrunkCovariance": ShrunkCovariance}
        try:
            return technique_dict[technique](**kwargs).fit(return_mat.T).covariance_ * time_scaling
        except KeyError:
            raise MethodException("""Invalid Technique. Options are EmpiricalCovariance,
                                  EllipticEnvelope, GraphicalLasso, GraphicalLassoCV, LedoitWolf, MinCovDet,
                                  OAS, ShrunkCovariance""")

    def sample_cov(self, return_mat, method, unit_time, weights=None, builtin=False, **kwargs):

        """
        Calculates the sample covariance
        """

        weights = self.construct_weight(method, return_mat, weights, **kwargs)
        return MomentGenerator.find_cov(return_mat, weights, builtin) * unit_time

    def construct_weight(self, method, return_mat, weights, **kwargs):

        """
        Constructing return weight based on method
        """

        if method == 'default':
            weights = np.repeat(np.divide(1, return_mat.shape[1]), repeats=return_mat.shape[1])
        elif method == 'exp':
            weights = MomentGenerator.exp_factor(return_mat, **kwargs)
        elif method == 'custom':
            if weights is None:
                warnings.warn("""Weight factor not defined. will use equal weight to calculate covariance.""")
                weights = np.repeat(1 / return_mat.shape[1], repeats=return_mat.shape[1])
            elif weights.shape[0] > return_mat.shape[1]:
                warnings.warn(
                    f"""Weight factor vector is longer than return series shape. Will only use {return_mat.shape[1]} values""")
                weights = weights[:return_mat.shape[1]]
            elif weights.shape[0] < return_mat.shape[1]:
                warnings.warn(
                    f"""Weight factor vector is shorter than return series shape. Will default remaining weights to 0""")
                weights = np.pad(weights, (0, return_mat.shape[1] - weights.shape[0]), 'constant')
        else:
            raise MethodException("""Invalid Method, Valid options are: default (equal weight), exp (exponential decay),
                                    custom (custom weight)""")
        # print(weights)
        return weights

    def semi_cov(self, return_mat, bm_return=0.0001, assume_zero=False):

        """
        Calculates semivariance matrix given bm_return
        """

        _return_mat_copy = return_mat.copy()

        def adj_return_vec(return_vec, bm_return):
            """
            Does not assume daily mean return of 0
            """
            return_vec[return_vec >= bm_return] = np.mean(return_vec[return_vec < bm_return])
            return return_vec

        if not assume_zero:
            _return_mat_copy = np.apply_along_axis(adj_return_vec, axis=1, arr=_return_mat_copy, bm_return=bm_return)
        else:
            _return_mat_copy = np.fmin(_return_mat_copy - bm_return, 0)

        return _return_mat_copy

    @staticmethod
    def find_cov(return_mat, weight_factor, builtin):

        """
        Calculates variance based on weight array and return matrix
        """

        if builtin:
            return np.cov(return_mat, aweights=weight_factor)

        diff_mat = return_mat - np.mean(return_mat, axis=1, keepdims=True)
        weight_mat = MomentGenerator.calc_weight_mat(return_mat, weight_factor)
        return np.dot(weight_mat * diff_mat * (return_mat.shape[1]/(return_mat.shape[1] - 1)), diff_mat.T)

    @staticmethod
    def calc_moment_mat(moment, return_mat, weight_mat, normalize):

        """
        Calculates moment matrix
        """

        # Difference
        diff_mat = return_mat - np.mean(return_mat, axis=1, keepdims=True)
        num_obs = diff_mat.shape[1]
        num_assets = diff_mat.shape[0]

        # Kronecker Product
        temp_mat = diff_mat.T
        for iteration in range(moment - 2):
            temp_mat = np.kron(diff_mat.T, temp_mat)[::num_obs + 1, :]

        # DDOF calculation (May cause overflow if moment is too high)
        unbias_factor = np.prod(np.repeat(num_obs, moment-1)/(num_obs - np.arange(1, moment, 1))) #(num_obs ** (moment - 1)) / (np.prod(num_obs - np.arange(1, moment, 1)))
        weighted_diff_mat = np.multiply(weight_mat, diff_mat) * unbias_factor

        moment_mat = np.dot(weighted_diff_mat, temp_mat)

        # Normalizing each value in the matrix with standard deviations
        if normalize:
            std_arr = np.std(diff_mat, ddof=1, axis=1).reshape(-1, 1)
            temp_std_mat = std_arr.T
            for iteration in range(moment - 2):
                temp_std_mat = np.kron(std_arr.T, temp_std_mat)[::num_assets + 1, :]
            std_arr_mat = np.dot(std_arr, temp_std_mat)
            moment_mat = moment_mat / std_arr_mat
        return moment_mat

    @staticmethod
    def calc_weight_mat(return_mat, weight_factor):
        """
        Construct a weight matrix
        """
        weights = weight_factor/np.sum(weight_factor)
        weight_mat = np.repeat(weights.reshape(1, -1), repeats=return_mat.shape[0], axis=0)
        return weight_mat

    @staticmethod
    def exp_factor(return_mat, decay=0.94, span=30):
        """
        Constructs weight factor based on decay and span

        A decay of 0.94 and a span of 2 will construct weight factor as based on proximity to benchmark date
        as [0.94, 0.94, 0.94^2, 0.94^2, ...]
        """
        decay_factor = decay ** np.arange(0, return_mat.shape[1]//span + 1)
        decay_factor = np.repeat(decay_factor, repeats=span)[:return_mat.shape[1]]
        return decay_factor[::-1]

ext: py

max_stars_repo_name: hermian/pymarkowitz

max_issues_repo_name: hermian/pymarkowitz

max_forks_repo_name: hermian/pymarkowitz

reason: import numpy


text:
import numpy as np

from sklearn.base import ClassifierMixin, BaseEstimator
from sklearn.preprocessing import LabelEncoder
from sklearn.utils import check_random_state
from sklearn.exceptions import NotFittedError

import tensorflow as tf

from muffnn.core import TFPicklingBase, affine

class ConvNetClassifier(TFPicklingBase, BaseEstimator, ClassifierMixin):
    """Scikit-learn compatible convolutional neural network classifier.

    Parameters
    ----------
    conv_hidden_units : list of tuples, optional
        Indicates the size and number of the convolutional filters.
    max_pool_size : int
        Size of max polling regions applied after each convolutional
        layer.
    dense_hidden_units : tuple or list, optional
        A list of integers indicating the number of hidden layers and their
        sizes for the dense layers at the end of the network.
    batch_size : int, optional
        The batch size for learning and prediction. If there are fewer
        examples than the batch size during fitting, then the the number of
        examples will be used instead.
    n_epochs : int, optional
        The number of epochs (iterations through the training data) when
        fitting.
    keep_prob : float, optional
        The probability of keeping values in dropout. A value of 1.0 means that
        dropout will not be used. Only used on the dense layers.
    activation : callable, optional
        The activation function. Any elementwise TensorFlow function is
        allowed.
    random_state : int, RandomState instance or None, optional
        If int, the random number generator seed. If RandomState instance,
        the random number generator itself. If None, then `np.random` will be
        used.
    solver : a subclass of `tf.train.Optimizer`, optional
        The solver to use to minimize the loss.
    solver_kwargs : optional
        Additional keyword arguments to pass to `solver` upon construction.
        See the TensorFlow documentation for possible options. Typically,
        one would want to set the `learning_rate`.

    Attributes
    ----------
    input_size_ : int
        The dimensionality of the input (i.e., number of features).
    graph_ : tensorflow.Graph
        The TensorFlow graph for the model
    n_classes_ : int
        The total number of classes.
    classes_ : array-like
        The class labels.
    """

    def __init__(self, conv_hidden_units=((5, 32), (5, 64)), max_pool_size=2,
                 dense_hidden_units=(1024,), batch_size=64, n_epochs=1,
                 keep_prob=1.0, activation=tf.nn.relu,
                 random_state=None, solver=tf.train.AdamOptimizer,
                 solver_kwargs=None):
        self.conv_hidden_units = conv_hidden_units
        self.max_pool_size = max_pool_size
        self.dense_hidden_units = dense_hidden_units
        self.batch_size = batch_size
        self.n_epochs = n_epochs
        self.keep_prob = keep_prob
        self.activation = activation
        self.random_state = random_state
        self.solver = solver
        self.solver_kwargs = solver_kwargs

    def __getstate__(self):
        # Handles TF persistence
        state = super().__getstate__()

        # Add attributes of this estimator
        state.update(dict(
            conv_hidden_units=self.conv_hidden_units,
            max_pool_size=self.max_pool_size,
            dense_hidden_units=self.dense_hidden_units,
            activation=self.activation,
            batch_size=self.batch_size,
            keep_prob=self.keep_prob,
            random_state=self.random_state,
            n_epochs=self.n_epochs,
            solver=self.solver,
            solver_kwargs=self.solver_kwargs))

        # Add fitted attributes if the model has been fitted.
        if self._is_fitted:
            state['input_size_'] = self.input_size_
            state['_random_state'] = self._random_state
            state['classes_'] = self.classes_
            state['n_classes_'] = self.n_classes_
            state['_label_encoder'] = self._label_encoder
            state['_image_size'] = self._image_size
            state['_num_channels'] = self._num_channels

        return state

    def _set_up_graph(self):
        """Initialize TF objects (needed before fitting or restoring)."""

        # Inputs
        self._keep_prob = tf.placeholder(
            dtype=np.float32, shape=(), name="keep_prob")
        self._input_targets = tf.placeholder(np.int32,
                                             [None],
                                             "input_targets")
        self._input_values = tf.placeholder(
            np.float32, [None, self.input_size_], "input_values")
        t = tf.reshape(
            self._input_values,
            [-1, self._image_size, self._image_size, self._num_channels])

        # Conv. layers
        prev_feats = self._num_channels
        for i, (cdim, num_feats) in enumerate(self.conv_hidden_units):

            with tf.variable_scope('conv_layer_%d' % i):
                W = tf.get_variable(
                    "weights", [cdim, cdim, prev_feats, num_feats])
                b = tf.get_variable(
                    "bias",
                    [num_feats], initializer=tf.constant_initializer(0.0))

            t = tf.nn.conv2d(t, W, strides=[1, 1, 1, 1], padding='SAME') + b
            t = t if self.activation is None else self.activation(t)
            t = tf.nn.max_pool(
                t,
                ksize=[1, self.max_pool_size, self.max_pool_size, 1],
                strides=[1, self.max_pool_size, self.max_pool_size, 1],
                padding='SAME')

            prev_feats = num_feats

        # Flatten to final size.
        final_img_size = (
            self._image_size //
            (self.max_pool_size ** len(self.conv_hidden_units)))
        t = tf.reshape(t, [-1, final_img_size * final_img_size * num_feats])

        # Dense layers.
        for i, layer_sz in enumerate(self.dense_hidden_units):
            if self.keep_prob != 1.0:
                t = tf.nn.dropout(t, keep_prob=self._keep_prob)
            t = affine(t, layer_sz, scope='dense_layer_%d' % i)
            t = t if self.activation is None else self.activation(t)

        # Final layer.
        if self.keep_prob != 1.0:
            t = tf.nn.dropout(t, keep_prob=self._keep_prob)
        t = affine(t, self.n_classes_, scope='output_layer')

        # Probs for each class.
        self._output_layer = tf.nn.softmax(t)

        # Objective function.
        self._obj_func = tf.reduce_mean(
            tf.nn.sparse_softmax_cross_entropy_with_logits(
                labels=self._input_targets, logits=t))

        # Training.
        sk = self.solver_kwargs if self.solver_kwargs is not None else {}
        self._train_step = self.solver(**sk).minimize(self._obj_func)

    def _make_feed_dict(self, X, y=None):
        feed_dict = {
            self._input_values: X}

        if y is None:
            feed_dict[self._keep_prob] = 1.0
        else:
            feed_dict[self._input_targets] = y
            feed_dict[self._keep_prob] = self.keep_prob

        return feed_dict

    def fit(self, X, y, monitor=None):
        """Fit the model.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data
        y : array-like, shape (n_samples, n_targets)
            Target values
        monitor : callable, optional
            The monitor is called after each iteration with the current
            iteration, a reference to the estimator, and a dictionary with
            {'loss': loss_value} representing the loss calculated by the
            objective function at this iteration.
            If the callable returns True the fitting procedure is stopped.
            The monitor can be used for various things such as computing
            held-out estimates, early stopping, model introspection,
            and snapshotting.

        Returns
        -------
        self : returns an instance of self.
        """
        self._is_fitted = False
        return self.partial_fit(X, y, monitor=monitor)

    def partial_fit(self, X, y, monitor=None, classes=None, **kwargs):
        """Fit the model on a minibatch of training data.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data
        y : array-like, shape (n_samples, n_targets)
            Target values
        monitor : callable, optional
            The monitor is called after each iteration with the current
            iteration, a reference to the estimator, and a dictionary with
            {'loss': loss_value} representing the loss calculated by the
            objective function at this iteration.
            If the callable returns True the fitting procedure is stopped.
            The monitor can be used for various things such as computing
            held-out estimates, early stopping, model introspection,
            and snapshotting.
        classes : array-like
            Array of labels to use for the label encoder.

        Returns
        -------
        self : returns an instance of self.
        """
        assert self.batch_size > 0, "batch_size <= 0"

        # Initialize the model if it hasn't been already by a previous call.
        if not self._is_fitted:
            self._random_state = check_random_state(self.random_state)

            self.input_size_ = int(np.prod(X.shape[1:]))
            if X.ndim == 2:
                self._image_size = int(np.sqrt(self.input_size_))
                self._num_channels = 1
            elif X.ndim == 3:
                if X.shape[1] == X.shape[2]:
                    self._image_size = X.shape[1]
                    self._num_channels = 1
                else:
                    self._image_size = int(np.sqrt(X.shape[1]))
                    self._num_channels = X.shape[2]
            elif X.ndim == 4:
                self._image_size = X.shape[1]
                self._num_channels = X.shape[3]

            if classes is not None:
                self._label_encoder = LabelEncoder()
                self._label_encoder.classes_ = np.sort(classes)
            else:
                self._label_encoder = LabelEncoder().fit(y)
            self.classes_ = self._label_encoder.classes_
            self.n_classes_ = len(self.classes_)

            self.graph_ = tf.Graph()
            with self.graph_.as_default():
                tf.set_random_seed(self._random_state.randint(0, 10000000))
                tf.get_variable_scope().set_initializer(
                    tf.contrib.layers.xavier_initializer())
                self._build_tf_graph()
                self._session.run(tf.global_variables_initializer())

            self._is_fitted = True

        yenc = self._label_encoder.transform(y)
        Xrs = X.reshape((X.shape[0], -1))

        # Train the model with the given data.
        with self.graph_.as_default():
            n_examples = X.shape[0]
            indices = np.arange(n_examples)

            for epoch in range(self.n_epochs):
                self._random_state.shuffle(indices)

                for start_idx in range(0, n_examples, self.batch_size):
                    batch_ind = indices[
                        start_idx:min(start_idx + self.batch_size, n_examples)]
                    feed_dict = self._make_feed_dict(
                        Xrs[batch_ind], yenc[batch_ind])
                    obj_val, _ = self._session.run(
                        [self._obj_func, self._train_step],
                        feed_dict=feed_dict)

                if monitor:
                    stop_early = monitor(epoch, self, {'loss': obj_val})
                    if stop_early:
                        print("stopping early due to monitor function!")
                        return self

        return self

    def predict_proba(self, X):
        """Predict probabilities for each class.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Examples to make predictions about.

        Returns
        -------
        C : array, shape = (n_samples, output_size_)
            Predicted probabilities for each class.
        """

        if not self._is_fitted:
            raise NotFittedError("Call fit before prediction!")

        Xrs = X.reshape((X.shape[0], -1))

        # Make predictions in batches.
        pred_batches = []
        start_idx = 0
        n_examples = X.shape[0]
        with self.graph_.as_default():
            while start_idx < n_examples:
                X_batch = Xrs[
                    start_idx:min(start_idx + self.batch_size, n_examples)]
                feed_dict = self._make_feed_dict(X_batch)
                start_idx += self.batch_size
                pred_batches.append(self._session.run(
                    self._output_layer, feed_dict=feed_dict))
        y_pred = np.concatenate(pred_batches)

        if len(y_pred.shape) == 1:
            y_pred = np.column_stack((1.0 - y_pred, y_pred))

        return y_pred

    def predict(self, X):
        """Make predictions.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Examples to make predictions about.

        Returns
        -------
        C : array, shape = (n_samples, output_size_)
            Predicted classes for each label.
        """
        return self.classes_[self.predict_proba(X).argmax(axis=1)]

max_stars_repo_name: civisanalytics/civis-compute

max_issues_repo_name: civisanalytics/civis-compute

max_forks_repo_name: civisanalytics/civis-compute


text:
#
# Author: penhe@microsoft.com
# Date: 01/25/2019
#

from glob import glob
from collections import OrderedDict,defaultdict,Sequence
import copy
import math
from scipy.special import softmax
import numpy as np
import pdb
import os
import sys
import csv

import random
import torch
import re
import ujson as json
from DeBERTa.apps.tasks.metrics import *
from DeBERTa.apps.tasks import EvalData, Task,register_task
from DeBERTa.utils import xtqdm as tqdm
from DeBERTa.data import ExampleInstance, ExampleSet, DynamicDataset,example_to_feature
from DeBERTa.data.example import *
from DeBERTa.utils import get_logger
from DeBERTa.data.example import _truncate_segments
from DeBERTa.apps.models.multi_choice import MultiChoiceModel

logger=get_logger()

__all__ = ["MyRACETask"]

@register_task(name="MyRACE", desc="ReAding Comprehension dataset collected from English Examinations, http://www.qizhexie.com/data/RACE_leaderboard.html")
class MyRACETask(Task):
  def __init__(self, data_dir, tokenizer, args, **kwargs):
    super().__init__(tokenizer, args, **kwargs)
    self.data_dir = data_dir

  def train_data(self, max_seq_len=512, dataset_size=None, epochs=1, mask_gen=None, **kwargs):
    middle = self.load_jsonl(os.path.join(self.data_dir, 'train_middle.jsonl'))
    high = self.load_jsonl(os.path.join(self.data_dir, 'train_high.jsonl'))
    examples = ExampleSet(middle + high)
    if dataset_size is None:
      dataset_size = len(examples)*epochs
    return DynamicDataset(examples, feature_fn = self.get_feature_fn(max_seq_len=max_seq_len, mask_gen=mask_gen), \
dataset_size = dataset_size, shuffle=True, **kwargs)

  def eval_data(self, max_seq_len=512, dataset_size=None, **kwargs):
    ds = [
        self._data('test-high', 'test_high.jsonl', 'test', ignore_metric=True),
        self._data('test-middle', 'test_middle.jsonl', 'test', ignore_metric=True),
        self._data('test', ['test_middle.jsonl', 'test_high.jsonl'], 'test'),
        ]

    for d in ds:
      if dataset_size is None:
        _size = len(d.data)
      d.data = DynamicDataset(d.data, feature_fn = self.get_feature_fn(max_seq_len=max_seq_len), dataset_size = _size, **kwargs)
    return ds

  def test_data(self,max_seq_len=512, dataset_size = None, **kwargs):
    """See base class."""
    ds = [
        self._data('high', 'test_high.jsonl', 'test'),
        self._data('middle', 'test_middle.jsonl', 'test'),
        ]

    for d in ds:
      if dataset_size is None:
        _size = len(d.data)
      d.data = DynamicDataset(d.data, feature_fn = self.get_feature_fn(max_seq_len=max_seq_len), dataset_size = _size, **kwargs)
    return ds

  def _data(self, name, path, type_name = 'dev', ignore_metric=False):
    if isinstance(path, str):
      path = [path]
    data = []
    for p in path:
      input_src = os.path.join(self.data_dir, p)
      assert os.path.exists(input_src), f"{input_src} doesn't exists"
      data.extend(self.load_jsonl(input_src))

    predict_fn = self.get_predict_fn()
    examples = ExampleSet(data)
    return EvalData(name, examples,
      metrics_fn = self.get_metrics_fn(), predict_fn = predict_fn, ignore_metric=ignore_metric, critial_metrics=['accuracy'])

  def get_metrics_fn(self):
    """Calcuate metrics based on prediction results"""
    def metrics_fn(logits, labels):
      metrics =  OrderedDict(accuracy= metric_accuracy(logits, labels))
      return metrics
    return metrics_fn

  def get_labels(self):
    """See base class."""
    return ["A", "B", "C", "D"]

  def load_jsonl(self, path):
    examples = []
    with open(path, encoding='utf-8') as fs:
      data = [json.loads(l) for l in fs]
      for d in data:
        page = d["article"]
        for q,o,a in zip(d["questions"], d["options"], d["answers"]):
          example = ExampleInstance(segments=[page, q, *o], label=self.label2id(a))
          examples.append(example)
    return examples

  def get_feature_fn(self, max_seq_len = 512, mask_gen = None):
    def _example_to_feature(example, rng=None, ext_params=None, **kwargs):
      return self.example_to_feature(self.tokenizer, example, max_seq_len = max_seq_len, \
        rng = rng, mask_generator = mask_gen, ext_params = ext_params, **kwargs)
    return _example_to_feature

  def example_to_feature(self, tokenizer, example, max_seq_len=512, rng=None, mask_generator = None, ext_params=None, label_type='int', **kwargs):
    if not rng:
      rng = random
    max_num_tokens = max_seq_len - 3
    def _normalize(text):
      text = re.sub(r'\s+', ' ', text.strip('\t \r\n_').replace('\n', ' ')).strip()
      return text

    # page,question,options
    context = tokenizer.tokenize(_normalize(example.segments[0]))
    features = OrderedDict(input_ids = [],
        type_ids = [],
        position_ids = [],
        input_mask = [])
    for option in example.segments[2:]:
      #TODO: truncate
      question = example.segments[1]
      qa_cat = " ".join([question, option])
      qa_cat = tokenizer.tokenize(_normalize(qa_cat))[:160]

      segments = [context, qa_cat]
      segments = _truncate_segments(segments, max_num_tokens, rng)
      tokens = ['[CLS]']
      type_ids = [0]
      for i,s in enumerate(segments):
        tokens.extend(s)
        tokens.append('[SEP]')
        type_ids.extend([i]*(len(s)+1))
      token_ids = tokenizer.convert_tokens_to_ids(tokens)
      pos_ids = list(range(len(token_ids)))
      rel_pos = []
      input_mask = [1]*len(token_ids)
      features['input_ids'].append(token_ids)
      features['type_ids'].append(type_ids)
      features['position_ids'].append(pos_ids)
      features['input_mask'].append(input_mask)
      padding_size = max(0, max_seq_len - len(token_ids))
      for f in features:
        features[f][-1].extend([0]*padding_size)

    for f in features:
      features[f] = torch.tensor(features[f], dtype=torch.int)
    if example.label is not None:
      label_type = torch.int if label_type=='int' else torch.float
      features['labels'] = torch.tensor(example.label, dtype=label_type)
    return features

  def get_model_class_fn(self):
    def partial_class(*wargs, **kwargs):
      return MultiChoiceModel.load_model(*wargs, **kwargs)
    return partial_class

  @classmethod
  def add_arguments(cls, parser):
    """Add task specific arguments
      e.g. parser.add_argument('--data_dir', type=str, help='The path of data directory.')
    """
    parser.add_argument('--task_example_arg', type=str, default=None, help='An example task specific argument')

    return parser

max_stars_repo_name: vilhub/DeBERTa

max_issues_repo_name: vilhub/DeBERTa

max_forks_repo_name: vilhub/DeBERTa

reason: import numpy,from scipy


text:
import os
import re
import torch
import lmdb
import pickle
import itertools
import argparse
import numpy as np
import torch.nn as nn
import pickle as pkl
import collections
import time

from tqdm import tqdm
from collections import Counter
from sklearn import utils, metrics
from src.datasets import load_datasets

from torch.utils.data import DataLoader, Dataset
import torch.nn.functional as F

from src.net import SVDCNN

# multiprocessing workaround
import resource
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (2048, rlimit[1]))

# Random seed
np.random.seed(0)
torch.manual_seed(0)
torch.cuda.manual_seed_all(0)

def count_parameters(model):
    total_param = 0
    total_param_conv = 0
    total_param_fc = 0

    for name, param in model.named_parameters():
            num_param = np.prod(param.size())
            # if param.dim() > 1:
            #     print(name, ':', 'x'.join(str(x) for x in list(param.size())), '=', num_param)
            # else:
            #     print(name, ':', num_param)

            if ('fc' in name ):
                total_param_fc += num_param
            else:
                total_param_conv += num_param

            total_param += num_param

    return total_param, total_param_conv, total_param_fc

def get_args():
    parser = argparse.ArgumentParser("""Squeezed Very Deep Convolutional Neural Networks for Text Classification""")
    parser.add_argument("--dataset", type=str, default='ag_news')
    parser.add_argument("--model_folder", type=str, default="models/svdcnn/ag_news")
    parser.add_argument("--data_folder", type=str, default="datasets/ag_news/svdcnn")
    parser.add_argument("--depth", type=int, choices=[9, 17, 29], default=9, help="Depth of the network tested in the paper (9, 17, 29)")
    parser.add_argument("--maxlen", type=int, default=1024)
    parser.add_argument("--shortcut", type=bool)
    parser.add_argument("--batch_size", type=int, default=128, help="number of example read by the gpu")
    parser.add_argument("--epochs", type=int, default=1000)
    parser.add_argument("--lr", type=float, default=0.01)
    parser.add_argument("--lr_halve_interval", type=float, default=100, help="Number of iterations before halving learning rate")
    parser.add_argument("--snapshot_interval", type=int, default=1)
    parser.add_argument("--gamma", type=float, default=0.9)
    parser.add_argument("--gpuid", type=int, default=0)
    parser.add_argument("--nthreads", type=int, default=4)
    args = parser.parse_args()
    return args

class Preprocessing():

    def __init__(self, lowercase=True):
        self.lowercase = lowercase

    def transform(self, sentences):
        """
        sentences: list(str)
        output: list(str)
        """
        return [s.lower() for s in sentences]

class CharVectorizer():
    def __init__(self,char_dict=None, max_features=69, maxlen=1024, padding='pre', truncating='pre'):
        self.char_dict = char_dict
        self.max_features = max_features
        self.maxlen = maxlen
        self.padding = padding
        self.truncating = truncating
        self.char_counter = Counter()

        self.n_transform = 0

        if self.char_dict:
            self.n_transform += 1

    def partial_fit(self, sentences):
        """
        sentences: list of list
        """
        for sentence in sentences:
            self.char_counter.update(sentence)

    def transform(self,sentences):
        """
        sentences: list of string
        list of review, review is a list of sequences, sequences is a list of int
        """

        values = list(range(2,self.max_features))
        keys = list("”abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:’\"/| #$%ˆ&*˜‘+=<>()[]{}")

        self.char_dict = dict(zip(keys, values))
        self.char_dict["_pad_"] = 0
        self.char_dict["_unk_"] = 1

        assert self.char_dict, "No dictionnary to vectorize text \n-> call method build_dict \n-> or set a word_dict attribute \n first"

        sequences = []

        for sentence in sentences:
            seq = [self.char_dict.get(char, self.char_dict["_unk_"]) for char in sentence]

            if self.maxlen:
                length = len(seq)
                if self.truncating == 'pre':
                    seq = seq[-self.maxlen:]
                elif self.truncating == 'post':
                    seq = seq[:self.maxlen]

                if length < self.maxlen:

                    diff = np.abs(length - self.maxlen)

                    if self.padding == 'pre':
                        seq = [self.char_dict['_pad_']] * diff + seq

                    elif self.padding == 'post':
                        seq = seq + [self.char_dict['_pad_']] * diff
            sequences.append(seq)

        return sequences

    def get_params(self):
        params = vars(self)
        if 'char_counter' in params:
            del params['char_counter']
        return params

class TupleLoader(Dataset):

    def __init__(self, path=""):
        self.path = path

        self.env = lmdb.open(path, max_readers=opt.nthreads, readonly=True, lock=False, readahead=False, meminit=False)
        self.txn = self.env.begin(write=False)

    def __len__(self):
        return list_from_bytes(self.txn.get('nsamples'.encode()))[0]

    def __getitem__(self, i):
        xtxt = list_from_bytes(self.txn.get(('txt-%09d' % i).encode()), np.int)
        lab = list_from_bytes(self.txn.get(('lab-%09d' % i).encode()), np.int)[0]
        return xtxt, lab

def get_metrics(cm, list_metrics):
    """Compute metrics from a confusion matrix (cm)
    cm: sklearn confusion matrix
    returns:
    dict: {metric_name: score}

    """
    dic_metrics = {}
    total = np.sum(cm)

    if 'accuracy' in list_metrics:
        out = np.sum(np.diag(cm))
        dic_metrics['accuracy'] = out/total

    if 'pres_0' in list_metrics:
        num = cm[0, 0]
        den = cm[:, 0].sum()
        dic_metrics['pres_0'] =  num/den if den > 0 else 0

    if 'pres_1' in list_metrics:
        num = cm[1, 1]
        den = cm[:, 1].sum()
        dic_metrics['pres_1'] = num/den if den > 0 else 0

    if 'recall_0' in list_metrics:
        num = cm[0, 0]
        den = cm[0, :].sum()
        dic_metrics['recall_0'] = num/den if den > 0 else 0

    if 'recall_1' in list_metrics:
        num = cm[1, 1]
        den = cm[1, :].sum()
        dic_metrics['recall_1'] =  num/den if den > 0 else 0

    return dic_metrics

def train(epoch,net,dataset,device,msg="val/test",optimize=False,optimizer=None,scheduler=None,criterion=None):

    net.train() if optimize else net.eval()

    epoch_loss = 0
    nclasses = len(list(net.parameters())[-1])
    cm = np.zeros((nclasses,nclasses), dtype=int)

    with tqdm(total=len(dataset),desc="Epoch {} - {}".format(epoch, msg)) as pbar:
        for iteration, (tx, ty) in enumerate(dataset):

            data = (tx, ty)
            data = [x.to(device) for x in data]

            if optimize:
                optimizer.zero_grad()

            out = net(data[0])
            ty_prob = F.softmax(out, 1) # probabilites

            #metrics
            y_true = data[1].detach().cpu().numpy()
            y_pred = ty_prob.cpu().max(1)[1]

            cm += metrics.confusion_matrix(y_true, y_pred, labels=range(nclasses))
            dic_metrics = get_metrics(cm, list_metrics)

            loss =  criterion(out, data[1])
            epoch_loss += loss.item()
            dic_metrics['logloss'] = epoch_loss/(iteration+1)

            if optimize:
                loss.backward()
                optimizer.step()
                dic_metrics['lr'] = optimizer.state_dict()['param_groups'][0]['lr']

            pbar.update(1)
            pbar.set_postfix(dic_metrics)

    if scheduler:
        scheduler.step()

def save(net, txt_dict, path):
    """
    Saves a model's state and it's embedding dic by piggybacking torch's save function
    """
    dict_m = net.state_dict()
    dict_m["txt_dict"] = txt_dict
    torch.save(dict_m,path)

def list_to_bytes(l):
    return np.array(l).tobytes()

def list_from_bytes(string, dtype=np.int):
    return np.frombuffer(string, dtype=dtype)

if __name__ == "__main__":

    opt = get_args()
    print("\nparameters: {}\n".format(vars(opt)))

    os.makedirs(opt.model_folder, exist_ok=True)
    os.makedirs(opt.data_folder, exist_ok=True)

    dataset = load_datasets(names=[opt.dataset])[0]
    dataset_name = dataset.__class__.__name__
    n_classes = dataset.n_classes
    print("dataset: {}, n_classes: {}".format(dataset_name, n_classes))

    variables = {
        'train': {'var': None, 'path': "{}/train.lmdb".format(opt.data_folder)},
        'test': {'var': None, 'path': "{}/test.lmdb".format(opt.data_folder)},
        'txt_dict': {'var': None, 'path': "{}/txt_dict.pkl".format(opt.data_folder)},
    }

    # check if datasets exis
    all_exist = True if os.path.exists(variables['txt_dict']['path']) else False
    if all_exist:
        # print("  - Loading: {}".format(variables['txt_dict']['path']))
        variables['txt_dict']['var'] = pkl.load(open(variables['txt_dict']['path'],"rb"))
        n_tokens = len(variables['txt_dict']['var']['char_dict'])
    else:
        print("Creating datasets")
        tr_sentences = [txt for txt,lab in tqdm(dataset.load_train_data(), desc="counting train samples")]
        te_sentences = [txt for txt,lab in tqdm(dataset.load_test_data(), desc="counting test samples")]

        n_tr_samples = len(tr_sentences)
        n_te_samples = len(te_sentences)

        print("[{}/{}] train/test samples".format(n_tr_samples, n_te_samples))

        ################
        # fit on train #
        ################
        preprocessor = Preprocessing()
        vectorizer = CharVectorizer(maxlen=opt.maxlen, padding='post', truncating='post')

        for sentence, label in tqdm(dataset.load_train_data(), desc="fit on train...", total= n_tr_samples):
            s_prepro = preprocessor.transform([sentence])
            vectorizer.partial_fit(s_prepro)

        del tr_sentences
        del te_sentences

        ###################
        # transform train #
        ###################
        with lmdb.open(variables['train']['path'], map_size=1099511627776) as env:
            with env.begin(write=True) as txn:
                for i, (sentence, label) in enumerate(tqdm(dataset.load_train_data(), desc="transform train...", total= n_tr_samples)):

                    xtxt = vectorizer.transform(preprocessor.transform([sentence]))[0]
                    lab = label

                    txt_key = 'txt-%09d' % i
                    lab_key = 'lab-%09d' % i

                    txn.put(lab_key.encode(), list_to_bytes([lab]))
                    txn.put(txt_key.encode(), list_to_bytes(xtxt))

                txn.put('nsamples'.encode(), list_to_bytes([i+1]))

        ##################
        # transform test #
        ##################
        with lmdb.open(variables['test']['path'], map_size=1099511627776) as env:
            with env.begin(write=True) as txn:
                for i, (sentence, label) in enumerate(tqdm(dataset.load_test_data(), desc="transform test...", total= n_te_samples)):

                    xtxt = vectorizer.transform(preprocessor.transform([sentence]))[0]
                    lab = label

                    txt_key = 'txt-%09d' % i
                    lab_key = 'lab-%09d' % i

                    txn.put(lab_key.encode(), list_to_bytes([lab]))
                    txn.put(txt_key.encode(), list_to_bytes(xtxt))

                txn.put('nsamples'.encode(), list_to_bytes([i+1]))

        variables['txt_dict']['var'] = vectorizer.get_params()
        n_tokens = len(variables['txt_dict']['var']['char_dict'])

        ###############
        # saving data #
        ###############
        print("  - saving to {}".format(variables['txt_dict']['path']))
        pkl.dump(variables['txt_dict']['var'],open(variables['txt_dict']['path'],"wb"))

    tr_loader = DataLoader(TupleLoader(variables['train']['path']), batch_size=opt.batch_size, shuffle=False, num_workers=opt.nthreads, pin_memory=True)
    te_loader = DataLoader(TupleLoader(variables['test']['path']), batch_size=opt.batch_size, shuffle=False, num_workers=opt.nthreads, pin_memory=False)

    # select cpu or gpu
    device = torch.device("cuda:{}".format(opt.gpuid) if opt.gpuid >= 0 else "cpu")
    list_metrics = ['accuracy']

    net = SVDCNN(n_classes=n_classes, num_embedding=n_tokens, embedding_dim=16, depth=opt.depth, shortcut=True)

    print("\nCreating model...")
    criterion = torch.nn.CrossEntropyLoss()
    net.to(device)

    total_param, total_param_conv, total_param_fc = count_parameters(net)
    print('\nNum of parameters: ')
    print('#conv (M): %0.2f' % (total_param_conv/pow(10,6)))
    print('#fc (M): %0.2f' % (total_param_fc/pow(10,6)))
    print("#total (M): %0.2f" % (total_param/pow(10,6)))
    print("Size (MB): %0.2f\n" % abs(total_param * 4. / (1024 ** 2.)))

    print(" - optimizer: sgd\n")
    optimizer = torch.optim.SGD(net.parameters(), lr = opt.lr, momentum=0.9, weight_decay=0.001)

    scheduler = None
    if opt.lr_halve_interval > 0:
        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, opt.lr_halve_interval, gamma=opt.gamma, last_epoch=-1)

    for epoch in range(1, opt.epochs + 1):
        train(epoch,net, tr_loader, device, msg="training", optimize=True, optimizer=optimizer, scheduler=scheduler, criterion=criterion)
        train(epoch,net, te_loader, device, msg="testing ", criterion=criterion)

        if (epoch % opt.snapshot_interval == 0) and (epoch > 0):
            path = "{}/model_epoch_{}".format(opt.model_folder,epoch)
            print("snapshot of model saved as {}".format(path))
            save(net, variables['txt_dict']['var'], path=path)

    if opt.epochs > 0:
        path = "{}/model_epoch_{}".format(opt.model_folder,opt.epochs)
        print("snapshot of model saved as {}".format(path))
        save(net, variables['txt_dict']['var'], path=path)

    print('\n\n\n\n\n')

max_stars_repo_name: lazarotm/SVDCNN

max_issues_repo_name: lazarotm/SVDCNN

max_forks_repo_name: lazarotm/SVDCNN


text:
# -*- coding: utf-8 -*-

"""
@author: Quoc-Tuan Truong <tuantq.vnu@gmail.com>
"""

from ..data import MatrixTrainSet, TestSet
from ..utils.generic_utils import validate_data_format
from ..metrics.rating import RatingMetric
from ..metrics.ranking import RankingMetric
from collections import OrderedDict
import numpy as np

class BaseMethod:
    """Base Evaluation Method

    Parameters
    ----------

    train_set: :obj:`TrainSet<cornac.data.TrainSet>`, optional, default: None
        The training data.

    test_set: :obj:`TestSet<cornac.data.TestSet>`, optional, default: None
        The test data.

    total_users: int, optional, default: None
        Total number of unique users in the data including train, val, and test sets

    total_users: int, optional, default: None
        Total number of unique items in the data including train, val, and test sets

    rating_threshold: float, optional, default: 1
        The minimum value that is considered to be a good rating used for ranking, \
        e.g, if the ratings are in {1, ..., 5}, then good_rating = 4.

    exclude_unknowns: bool, optional, default: False
        Ignore unknown users and items (cold-start) during evaluation and testing

    verbose: bool, optional, default: False
        Output running log
    """

    def __init__(self, data=None, data_format='UIR', train_set=None, test_set=None,
                 total_users=None, total_items=None, rating_threshold=1.0,
                 exclude_unknowns=False, verbose=False):
        self._data = data
        self.data_format = validate_data_format(data_format)
        self.train_set = train_set
        self.test_set = test_set
        self.total_users = total_users
        self.total_items = total_items
        self.rating_threshold = rating_threshold
        self.exclude_unknowns = exclude_unknowns
        self.verbose = verbose

        if verbose:
            print('rating_threshold = {:.1f}'.format(rating_threshold))
            print('exclude_unknowns = {}'.format(exclude_unknowns))

    def _organize_metrics(self, metrics):
        if isinstance(metrics, dict):
            rating_metrics = metrics.get('rating', [])
            ranking_metrics = metrics.get('ranking', [])
        elif isinstance(metrics, list):
            rating_metrics = [mt for mt in metrics if isinstance(mt, RatingMetric)]
            ranking_metrics = [mt for mt in metrics if isinstance(mt, RankingMetric)]
        else:
            raise ValueError('Type of metrics has to be either dict or list!')

        return rating_metrics, ranking_metrics

    def evaluate(self, model, metrics, user_based):
        rating_metrics, ranking_metrics = self._organize_metrics(metrics)

        if self.train_set is None:
            raise ValueError('train_set is required but None!')

        if self.test_set is None:
            raise ValueError('test_set is required but None!')

        if self.total_users is None:
            self.total_users = len(set(self.train_set.get_uid_list() + self.test_set.get_uid_list()))

        if self.total_items is None:
            self.total_items = len(set(self.train_set.get_iid_list() + self.test_set.get_iid_list()))

        if self.verbose:
            print("Training started!")

        model.fit(self.train_set)

        if self.verbose:
            print("Evaluation started!")

        all_rating_gts = []
        all_rating_pds = []
        metric_user_results = {}
        for mt in (rating_metrics + ranking_metrics):
            metric_user_results[mt.name] = {}

        num_eval_users = len(self.test_set.get_users())
        for i, user_id in enumerate(self.test_set.get_users()):
            if self.verbose:
                if i % 1000 == 0 or (i+1) == num_eval_users:
                    print(i, "users evaluated")

            # ignore unknown users when self.exclude_unknown
            if self.exclude_unknowns and self.train_set.is_unk_user(user_id):
                continue

            u_rating_gts = []
            u_rating_pds = []
            if self.exclude_unknowns:
                u_ranking_gts = np.zeros(self.train_set.num_items)
                candidate_item_ids = None  # all known items
            else:
                u_ranking_gts = np.zeros(self.total_items)
                candidate_item_ids = np.arange(self.total_items)

            for item_id, rating in self.test_set.get_ratings(user_id):
                # ignore unknown items when self.exclude_unknown
                if self.exclude_unknowns and self.train_set.is_unk_item(item_id):
                    continue

                if len(rating_metrics) > 0:
                    all_rating_gts.append(rating)
                    u_rating_gts.append(rating)

                    rating_pred = model.rate(user_id, item_id)
                    all_rating_pds.append(rating_pred)
                    u_rating_pds.append(rating_pred)

                # constructing ranking ground-truth
                if rating >= self.rating_threshold:
                    u_ranking_gts[item_id] = 1

            # per user evaluation of rating metrics
            if len(rating_metrics) > 0 and len(u_rating_gts) > 0:
                for mt in rating_metrics:
                    mt_score = mt.compute(np.asarray(u_rating_gts), np.asarray(u_rating_pds))
                    metric_user_results[mt.name][user_id] = mt_score

            # evaluation of ranking metrics
            if len(ranking_metrics) > 0 and u_ranking_gts.sum() > 0:
                u_ranking_pds = model.rank(user_id, candidate_item_ids)
                for mt in ranking_metrics:
                    mt_score = mt.compute(u_ranking_gts, u_ranking_pds)
                    metric_user_results[mt.name][user_id] = mt_score

        metric_avg_results = {}

        # avg results of rating metrics
        for mt in rating_metrics:
            if user_based:  # averaging over users
                user_results = list(metric_user_results[mt.name].values())
                metric_avg_results[mt.name] = np.asarray(user_results).mean()
            else:  # averaging over ratings
                metric_avg_results[mt.name] = mt.compute(np.asarray(all_rating_gts), np.asarray(all_rating_pds))

        # avg results of ranking metrics
        for mt in ranking_metrics:
            user_results = list(metric_user_results[mt.name].values())
            metric_avg_results[mt.name] = np.asarray(user_results).mean()

        return metric_avg_results, metric_user_results

    def _build_from_uir_format(self, train_data, test_data, val_data=None):
        global_uid_map = OrderedDict()
        global_iid_map = OrderedDict()
        global_ui_set = set()  # avoid duplicate ratings in the data

        if train_data is None:
            raise ValueError('train_data is required but None!')

        if test_data is None:
            raise ValueError('test_data is required but None!')

        if self.verbose:
            print('Building training set')
        self.train_set = MatrixTrainSet.from_uir_triplets(
            train_data, global_uid_map, global_iid_map, global_ui_set, self.verbose)

        if self.verbose:
            print('Building test set')
        self.test_set = TestSet.from_uir_triplets(
            test_data, global_uid_map, global_iid_map, global_ui_set, self.verbose)

        if not val_data is None:
            if self.verbose:
                print('Building validation set')
            self.val_set = TestSet.from_uir_triplets(
                val_data, global_uid_map, global_iid_map, global_ui_set, self.verbose)

        self.total_users = len(global_uid_map)
        self.total_items = len(global_iid_map)

    @classmethod
    def from_provided(cls, train_data, test_data, val_data=None, data_format='UIR',
                      rating_threshold=1.0, exclude_unknowns=False, verbose=False):
        method = cls(data_format=data_format, rating_threshold=rating_threshold,
                     exclude_unknowns=exclude_unknowns, verbose=verbose)
        if method.data_format == 'UIR':
            method._build_from_uir_format(train_data, test_data, val_data)

        return method

max_stars_repo_name: Andrew-DungLe/cornac

max_issues_repo_name: Andrew-DungLe/cornac

max_forks_repo_name: Andrew-DungLe/cornac


text:
import pytest
from pymilvus.client.check import check_pass_param
from pymilvus.client.utils import generate_timestamp

class TestCheckPassParam:
    def test_check_pass_param_valid(self):
        a = [[i * j for i in range(20)] for j in range(20)]
        check_pass_param(search_data=a)

        import numpy as np
        a = np.float32([[1, 2, 3, 4], [1, 2, 3, 4]])
        check_pass_param(search_data=a)

    def test_check_param_invalid(self):
        with pytest.raises(Exception):
            a = {[i * j for i in range(20) for j in range(20)]}
            check_pass_param(search_data=a)

        with pytest.raises(Exception):
            a = [{i * j for i in range(40)} for j in range(40)]
            check_pass_param(search_data=a)

class TestGenTS:
    def test_gen_timestamp(self):
        t = generate_timestamp(426152581543231492, 1000)

        print(f"Generated ts: {t}")

max_stars_repo_name: jingkl/pymilvus

max_issues_repo_name: jingkl/pymilvus

max_forks_repo_name: jingkl/pymilvus


text:
"""
number of resnet/densenet/pool
                    ----add resnet/densenet/pool
                    ----remove resnet/densenet/pool
properties of resnet/dense/pool
                    ----in_channel/out_channel of resnet
                    ----amount in one resnet
                    ----in_channel/out_channel of densenet
                    ----amount in one densenet
                    ----k of densenet
                    ----pooling type

firstly, three basic operations:add, remove, alter
secondly, the particular operation is chosen based on a probability
"""
import random
import numpy as np
import copy
from algs.aecnn.genetic.population import Individual
from algs.aecnn.utils import Utils
from algs.aecnn.genetic.statusupdatetool import StatusUpdateTool

class CrossoverAndMutation(object):
    def __init__(self, prob_crossover, prob_mutation, _log, individuals, gen_no, _params=None):
        self.prob_crossover = prob_crossover
        self.prob_mutation = prob_mutation
        self.individuals = individuals
        self.gen_no = gen_no
        self.params = _params  # storing other parameters if needed, such as the index for SXB and polynomial mutation
        self.log = _log
        self.offspring = []

    def process(self):
        crossover = Crossover(self.individuals, self.prob_crossover, self.log)
        offspring = crossover.do_crossover()
        self.offspring = offspring
        Utils.save_population_after_crossover(self.individuals_to_string(), self.gen_no)

        mutation = Mutation(self.offspring, self.prob_mutation, self.log)
        mutation.do_mutation()

        for i, indi in enumerate(self.offspring):
            indi_no = 'indi%05d_%05d' % (self.gen_no, i)
            indi.id = indi_no

        Utils.save_population_after_mutation(self.individuals_to_string(), self.gen_no)
        return self.offspring

    def individuals_to_string(self):
        _str = []
        for indi in self.offspring:
            _str.append(str(indi))
            _str.append('-' * 100)
        return '\n'.join(_str)

class Crossover(object):
    def __init__(self, individuals, prob_, _log):
        self.individuals = individuals
        self.prob = prob_
        self.log = _log
        self.pool_limit = StatusUpdateTool.get_pool_limit()[1]
        # get max_pool_limit

    def _choose_one_parent(self):
        count_ = len(self.individuals)
        idx1 = int(np.floor(np.random.random() * count_))
        idx2 = int(np.floor(np.random.random() * count_))
        while idx2 == idx1:
            idx2 = int(np.floor(np.random.random() * count_))

        if self.individuals[idx1].acc > self.individuals[idx1].acc:
            return idx1
        else:
            return idx2

    """
    binary tournament selection
    """

    def _choose_two_diff_parents(self):
        idx1 = self._choose_one_parent()
        idx2 = self._choose_one_parent()
        while idx2 == idx1:
            idx2 = self._choose_one_parent()

        assert idx1 < len(self.individuals)
        assert idx2 < len(self.individuals)
        return idx1, idx2

    def get_pos_and_calculate_pool_numbers(self, parent1, parent2):
        """
        Determine the position of the crossover
        and calculate the number of pooling units after the crossover is done
        """
        len1, len2 = len(parent1.units), len(parent2.units)
        # Determine the position of the crossover
        pos1, pos2 = np.random.randint(len1 + 1), np.random.randint(len2 + 1)
        # pos1 <= len1      pos2 <= len2
        p1_left, p1_right, p2_left, p2_right = 0, 0, 0, 0
        for i in range(0, pos1):
            if parent1.units[i].type == 2:
                p1_left += 1
        for i in range(pos1, len1):
            if parent1.units[i].type == 2:
                p1_right += 1

        for i in range(0, pos2):
            if parent2.units[i].type == 2:
                p2_left += 1
        for i in range(pos2, len2):
            if parent2.units[i].type == 2:
                p2_right += 1

        new_pool_number1 = p1_left + p2_right
        new_pool_number2 = p2_left + p1_right
        return pos1, pos2, new_pool_number1, new_pool_number2

    def do_crossover(self):
        _stat_param = {'offspring_new': 0, 'offspring_from_parent': 0}
        new_offspring_list = []
        for _ in range(len(self.individuals) // 2):
            ind1, ind2 = self._choose_two_diff_parents()

            parent1, parent2 = copy.deepcopy(self.individuals[ind1]), copy.deepcopy(self.individuals[ind2])
            p_ = random.random()
            if p_ < self.prob:
                _stat_param['offspring_new'] += 2
                """
                exchange their units from these parent individuals, the exchanged units must satisfy
                --- the number of pooling layer should not be more than the predefined setting
                --- if their is no changing after this crossover, keep the original acc -- a mutation should be given [to do---]
                """
                first_begin_is_pool, second_begin_is_pool = True, True
                while first_begin_is_pool is True or second_begin_is_pool is True:
                    pos1, pos2, pool_len1, pool_len2 = self.get_pos_and_calculate_pool_numbers(parent1, parent2)
                    try_count = 1
                    # to avoid pool_len more than the predefined setting and to avoid generating a NULL indi
                    while pool_len1 > self.pool_limit or pool_len2 > self.pool_limit or (
                            pos1 == 0 and pos2 == len(parent2.units) or (pos2 == 0 and pos1 == len(parent1.units))):
                        pos1, pos2, pool_len1, pool_len2 = self.get_pos_and_calculate_pool_numbers(parent1, parent2)
                        try_count += 1
                        self.log.warn('The %d-th try to find the position for do crossover' % (try_count))
                    self.log.info('Position %d for %s, positions %d for %s' % (pos1, parent1.id, pos2, parent2.id))
                    unit_list1, unit_list2 = [], []
                    for i in range(0, pos1):
                        unit_list1.append(parent1.units[i])
                    for i in range(pos2, len(parent2.units)):
                        unit_list1.append(parent2.units[i])

                    for i in range(0, pos2):
                        unit_list2.append(parent2.units[i])
                    for i in range(pos1, len(parent1.units)):
                        unit_list2.append(parent1.units[i])
                    first_begin_is_pool = True if unit_list1[0].type == 2 else False
                    second_begin_is_pool = True if unit_list2[0].type == 2 else False

                    if first_begin_is_pool is True:
                        self.log.warn('Crossovered individual#1 starts with a pooling layer, redo...')
                    if second_begin_is_pool is True:
                        self.log.warn('Crossovered individual#2 starts with a pooling layer, redo...')

                # reorder the number of each unit based on its order in the list
                for i, unit in enumerate(unit_list1):
                    unit.number = i
                for i, unit in enumerate(unit_list2):
                    unit.number = i

                # re-adjust the in_channel of the next layer
                last_output_from_list1 = Individual.get_last_output_channel(pos1, unit_list1)
                unit_list1 = Individual.update_channel_after_pos(pos1, unit_list1, last_output_from_list1, 0, self.log)

                last_output_from_list2 = Individual.get_last_output_channel(pos2, unit_list2)
                unit_list2 = Individual.update_channel_after_pos(pos2, unit_list2, last_output_from_list2, 0, self.log)

                parent1.units = unit_list1
                parent2.units = unit_list2
                offspring1, offspring2 = parent1, parent2
                offspring1.reset_acc()
                offspring2.reset_acc()
                new_offspring_list.append(offspring1)
                new_offspring_list.append(offspring2)
            else:
                _stat_param['offspring_from_parent'] += 2
                new_offspring_list.append(parent1)
                new_offspring_list.append(parent2)

        self.log.info('CROSSOVER-%d offspring are generated, new:%d, others:%d' % (
            len(new_offspring_list), _stat_param['offspring_new'], _stat_param['offspring_from_parent']))
        return new_offspring_list

class Mutation(object):

    def __init__(self, individuals, prob_, _log):
        self.individuals = individuals
        self.prob = prob_
        self.log = _log

    def do_mutation(self):
        _stat_param = {'offspring_new': 0, 'offspring_from_parent': 0, 'ADD': 0, 'REMOVE': 0, 'ALTER': 0,
                       'RESNET_OUT_CHANNEL': 0, 'RESNET_AMOUNT': 0, 'DENSENET_AMOUNT': 0, 'POOLING_TYPE': 0}
        # the resnet has two alter type, densenet and pool have only one alter type as shown in above param

        mutation_list = StatusUpdateTool.get_mutation_probs_for_each()
        for indi in self.individuals:
            p_ = random.random()
            if p_ < self.prob:
                _stat_param['offspring_new'] += 1
                mutation_type = self.select_mutation_type(mutation_list)
                if mutation_type == 0:
                    _stat_param['ADD'] += 1
                    self.do_add_unit_mutation(indi)
                elif mutation_type == 1:
                    _stat_param['REMOVE'] += 1
                    self.do_remove_unit_mutation(indi)
                elif mutation_type == 2:
                    mutation_p_type, mutation_p_count = self.do_alter_mutation(indi)
                    _stat_param[mutation_p_type] += mutation_p_count
                    _stat_param['ALTER'] += mutation_p_count
                    if mutation_p_count == 0:
                        _stat_param['offspring_new'] -= 1
                        _stat_param['offspring_from_parent'] += 1
                else:
                    raise TypeError('Error mutation type :%d, validate range:0-4' % (mutation_type))
            else:
                _stat_param['offspring_from_parent'] += 1
        self.log.info(
            'MUTATION-mutated individuals:%d[ADD:%2d,REMOVE:%2d,ALTER:%2d,RESNET_OUT_CHANNEL:%2d, RESNET_AMOUNT:%2d, DENSENET_AMOUNT:%2d, POOLING_TYPE:%2d, no_changes:%d' % (
                _stat_param['offspring_new'], \
                _stat_param['ADD'], _stat_param['REMOVE'], _stat_param['ALTER'], _stat_param['RESNET_OUT_CHANNEL'],
                _stat_param['RESNET_AMOUNT'], _stat_param['DENSENET_AMOUNT'], _stat_param['POOLING_TYPE'],
                _stat_param['offspring_from_parent']))

    def do_add_unit_mutation(self, indi):
        self.log.info('Do the ADD mutation for indi:%s' % (indi.id))
        """
        choose one position to add one unit, adding one resnet/densenet or pooling unit is determined by a probability of 1/3.
        However, if the maximal number of pooling units have been added into the current individual, only
        resnet/densenet unit will be add here
        """
        # determine the position where a unit would be added
        mutation_position = np.random.randint(len(indi.units) + 1)
        self.log.info('Mutation position occurs at %d' % (mutation_position))
        # determine the unit type for adding
        u_ = random.random()
        if u_ < 0.333:
            type_ = 1
        elif u_ < 0.666:
            type_ = 2
        else:
            type_ = 3
        type_string_list = ['RESNET', 'POOLING', 'DENSENET']
        self.log.info('A %s unit would be added due to the probability of %.2f' % (type_string_list[type_ - 1], u_))
        # considering the max pool limit and pool can't be the first unit
        if type_ == 2:
            # the max pool limit
            num_exist_pool_units = indi.get_pool_number()
            if num_exist_pool_units > StatusUpdateTool.get_pool_limit()[1] - 1:
                u_ = random.random()
                type_ = 1 if u_ < 0.5 else 3
                self.log.info(
                    'The added unit is changed to %s because the existing number of POOLING exceeds %d, limit size:%d' % (
                        'RESNET' if type_ == 1 else 'DENSENET', num_exist_pool_units,
                        StatusUpdateTool.get_pool_limit()[1]))

            # can't be the first unit
            if mutation_position == 0:
                mutation_position = np.random.randint(1, len(indi.units) + 1)

        # do the details
        if type_ == 2:
            add_unit = indi.init_a_pool(mutation_position, _max_or_avg=None)
        else:
            _in_channel = Individual.get_last_output_channel(mutation_position, indi.units)
            if type_ == 1:
                add_unit = indi.init_a_resnet(mutation_position, _amount=None, _in_channel=_in_channel,
                                              _out_channel=None)
            if type_ == 3:
                add_unit = indi.init_a_densenet(mutation_position, _amount=None, _k=None, _max_input_channel=None,
                                                _in_channel=_in_channel)

            indi.units = Individual.update_channel_after_pos(mutation_position, indi.units, add_unit.out_channel, 1,
                                                             self.log)

        new_unit_list = []
        # add to the new list and update the number
        for i in range(mutation_position):
            new_unit_list.append(indi.units[i])
        new_unit_list.append(add_unit)
        for i in range(mutation_position, len(indi.units)):
            unit = indi.units[i]
            unit.number += 1
            new_unit_list.append(unit)
        indi.number_id += 1
        indi.units = new_unit_list
        indi.reset_acc()

    def do_remove_unit_mutation(self, indi):
        self.log.info('Do the REMOVE mutation for indi:%s' % (indi.id))
        if len(indi.units) > 1:
            mutation_position = int(
                np.floor(np.random.random() * (len(indi.units) - 1))) + 1  # the first unit would not be removed
            self.log.info('Mutation position occurs at %d' % (mutation_position))
            if indi.units[mutation_position].type == 1 or indi.units[mutation_position].type == 3:
                indi.units = Individual.update_channel_after_pos(mutation_position+1, indi.units,
                                                                 indi.units[mutation_position].in_channel, 1, self.log)

            else:
                self.log.info('A POOLING at %d is removed' % (mutation_position))
            new_unit_list = []
            for i in range(mutation_position):
                new_unit_list.append(indi.units[i])
            for i in range(mutation_position + 1, len(indi.units)):
                unit = indi.units[i]
                unit.number -= 1
                new_unit_list.append(unit)
            indi.number_id -= 1
            indi.units = new_unit_list
            indi.reset_acc()
        else:
            self.log.warn('REMOVE mutation can not be performed due to it has only one unit')

    def do_alter_mutation(self, indi):
        """
                ----out_channel of resnet
                ----amount in one resnet
                ----amount in one densenet
                ----pooling type
        """
        self.log.info('Do the ALTER mutation for indi:%s' % (indi.id))
        mutation_position = int(np.floor(np.random.random() * len(indi.units)))
        mutation_unit = indi.units[mutation_position]
        if mutation_unit.type == 1:
            mutation_unit_name = 'RESNET'
            mutation_p_type, mutation_p_count = self.do_alter_resnet_mutation(mutation_position, indi)
        elif mutation_unit.type == 2:
            mutation_unit_name = 'POOLING'
            mutation_p_type, mutation_p_count = self.do_alter_pooling_mutation(mutation_position, indi)
        else:
            mutation_unit_name = 'DENSENET'
            mutation_p_type, mutation_p_count = self.do_alter_densenet_mutation(mutation_position, indi)
        self.log.info(
            'Do the %s mutation for indi:%s at position %d' % (mutation_unit_name, indi.id, mutation_position))

        return mutation_p_type, mutation_p_count

    def do_alter_resnet_mutation(self, position, indi):
        """
        ----out_channel of resnet
        ----amount in one resnet
        """
        mutation_p_count = 0

        u_ = random.random()
        if u_ < 0.5:
            mutation_p_type = 'RESNET_OUT_CHANNEL'
            channel_list = StatusUpdateTool().get_output_channel()
            index_ = int(np.floor(np.random.random() * len(channel_list)))
            if indi.units[position].out_channel != channel_list[index_]:
                self.log.info('Unit at %d changes its output channel from %d to %d' % (
                    position, indi.units[position].out_channel, channel_list[index_]))
                indi.units[position].out_channel = channel_list[index_]

                indi.units = Individual.update_channel_after_pos(position + 1, indi.units, channel_list[index_], 1,
                                                                 self.log)

                mutation_p_count = 1
                indi.reset_acc()
        else:
            mutation_p_type = 'RESNET_AMOUNT'
            min_resnet_unit, max_resnet_unit = StatusUpdateTool.get_resnet_unit_length_limit()
            amount = np.random.randint(min_resnet_unit, max_resnet_unit + 1)
            if amount != indi.units[position].amount:
                self.log.info(
                    'Unit at %d changes its amount from %d to %d' % (position, indi.units[position].amount, amount))
                indi.units[position].amount = amount
                mutation_p_count = 1
                indi.reset_acc()
        return mutation_p_type, mutation_p_count

    def do_alter_densenet_mutation(self, position, indi):
        mutation_p_type = 'DENSENET_AMOUNT'
        mutation_p_count = 0

        k = indi.units[position].k
        if k == 12:
            max_input_channel, amount_lower_limit, amount_upper_limit = StatusUpdateTool.get_densenet_k12()
        elif k == 20:
            max_input_channel, amount_lower_limit, amount_upper_limit = StatusUpdateTool.get_densenet_k20()
        else:
            max_input_channel, amount_lower_limit, amount_upper_limit = StatusUpdateTool.get_densenet_k40()
        amount = np.random.randint(amount_lower_limit, amount_upper_limit + 1)
        if amount != indi.units[position].amount:
            self.log.info(
                'Unit at %d changes its amount from %d to %d' % (position, indi.units[position].amount, amount))
            indi.units[position].amount = amount
            if indi.units[position].in_channel > max_input_channel:
                true_input_channel = max_input_channel
            else:
                true_input_channel = indi.units[position].in_channel
            new_out_channel = indi.units[position].amount * k + true_input_channel
            self.log.info('Due to the above mutation, unit at %d changes its output channel from %d to %d' % (
                position, indi.units[position].out_channel, new_out_channel))
            indi.units[position].out_channel = new_out_channel
            indi.units = Individual.update_channel_after_pos(position + 1, indi.units, new_out_channel, 1, self.log)

            mutation_p_count = 1
            indi.reset_acc()
        return mutation_p_type, mutation_p_count

    def do_alter_pooling_mutation(self, position, indi):
        mutation_p_type = 'POOLING_TYPE'
        mutation_p_count = 1

        if indi.units[position].max_or_avg > 0.5:
            indi.units[position].max_or_avg = 0.25
            self.log.info('Pool type from avg_pool (>0.5) to max_pool (<0.5)')
        else:
            indi.units[position].max_or_avg = 0.75
            self.log.info('Pool type from max_pool (<0.5) to avg_pool (>0.5)')
        indi.reset_acc()
        return mutation_p_type, mutation_p_count

    def select_mutation_type(self, _a):
        a = np.asarray(_a)
        sum_a = np.sum(a).astype(np.float)
        rand = np.random.random() * sum_a
        _sum = 0
        mutation_type = -1
        for i in range(len(a)):
            _sum += a[i]
            if _sum > rand:
                mutation_type = i
                break
        assert mutation_type != -1
        return mutation_type

max_stars_repo_name: benchenas/BenchENAS

max_issues_repo_name: benchenas/BenchENA

max_forks_repo_name: benchenas/BenchENAS


text:
# The MIT License
# Copyright (c) 2018 Kamil Jurek
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

def round_to(x, _round_to):
    return int(round(x / _round_to)) * _round_to

def print_separator():
    print("-----------------------------------------------------------------------------------------------------------")

def print_rules(rules_sets, support):
    print_separator()
    for rules_set in rules_sets:
        #for rule in sorted(rules_set, key=lambda x: (x.number_of_occurrences, len(x.lhs), x.rhs.len, x.lhs[0].len), reverse=True):
        for rule in sorted(rules_set, key=lambda r: (r.get_rule_score()),reverse=True):
            if rule.rule_support >= support:
                print(rule)
                print_separator()
        print_separator()
    print()

def print_rules_for_attr(rules_sets, attr_name, support):
    print_separator()
    for rules_set in rules_sets:
        #for rule in sorted(rules_set, key=lambda x: (x.number_of_occurrences, len(x.lhs), x.rhs.len, x.lhs[0].len), reverse=True):
        if list(rules_set)[0].lhs[0].attr_name_ != attr_name:
            continue

        for rule in sorted(rules_set, key=lambda r: (r.get_rule_score()),reverse=True):
            if rule.rule_support >= support:
                print(rule)
                print_separator()
        print_separator()
    print()

def print_best_rules(rules_sets):
    print_separator()
    for rules_set in rules_sets:
        best_rule = sorted(rules_set, key=lambda r: (r.get_rule_score()), reverse=True)[0]
        print(best_rule)
        print_separator()
    print()

def print_combined_rules(combined_rules, support):
    print_separator()
    for combined_rule in sorted(combined_rules, key=lambda x: min(x[i].rule_support for i in range(len(x))), reverse=True):
        # print(combined_rule)
        nr_of_occu = min([combined_rule[i].rule_support for i in range(len(combined_rule))])
        if nr_of_occu >= support:
            for i, rule in enumerate(combined_rule[:-1]):
                print(rule.lhs, "{supp:", rule.rule_support, "conf:", rule.get_confidence(), "} AND" if i < len(combined_rule)-1 else "}")
            print(" ==>", )
            print(combined_rule[0].rhs)
            print("number of occurenes:", nr_of_occu)
            print_separator()

def print_detected_change_points(detected_changes):
    print_separator()
    for attr_detected_change in detected_changes:
        for point in attr_detected_change:
            print(point)
        print_separator()

def evaluate_change_detectors(simulator, actual_change_points):
    for k in range(len(simulator.sequences)):
        detected_change_points = np.array(simulator.get_detected_changes())[k]
        detected_change_points = [x.at_ for x in detected_change_points]
        actual_change_points2 = list(actual_change_points)

        print("actual:",actual_change_points)
        print("detect:",detected_change_points)
        if len(detected_change_points) < len(actual_change_points):
            for i in range(len(actual_change_points)):
                if np.abs(detected_change_points[i] - actual_change_points[i]) > 305:
                    detected_change_points.insert(i, 0)
            actual_change_points2 = np.array(actual_change_points)

        elif len(detected_change_points) > len(actual_change_points):
            for i in range(len(detected_change_points)):
                if np.abs(detected_change_points[i] - actual_change_points2[i]) > 205:
                    actual_change_points2.insert(i, 0)
            actual_change_points2 = np.array(actual_change_points2)
        else:
            actual_change_points2 = np.array(actual_change_points)
        #actual_change_points2 = np.array(actual_change_points)
        print("====== ", simulator.sequences_names[k], " ===============================")
        print("Actual change points:   ", list(actual_change_points2))
        print("Detected change points: ", detected_change_points)

        delta = np.abs(actual_change_points2 - detected_change_points)

        print("Difference: ", delta)
        print("Errors sum: ", np.sum(delta))
        print("Errors avg: ", np.sum(delta) / len(detected_change_points))

        rmse = np.sqrt(((detected_change_points - actual_change_points2) ** 2).mean())
        print('Mean Squared Error: {}'.format(round(rmse, 5)))
        print("==============================================================================")

def generate_classical_dataset(detected_changes, discretization_step=100, save_csv=False):
    classical = [[] for i in range(len(detected_changes))]
    for j, seq_changes in enumerate(detected_changes):
        for point in seq_changes:
            for i in range(round_to(point.at_ - point.prev_value_len, discretization_step), round_to(point.at_ - discretization_step, discretization_step), discretization_step):
                classical[j].append(
                    str(str(point.attr_name) + ":" + str(point.prev_value) + "->" + str(point.prev_value)))
                print(point.attr_name, ":", point.prev_value, "->", point.prev_value)
            classical[j].append(str(str(point.attr_name) + ":" + str(point.prev_value) + "->" + str(point.curr_value)))
            print(point.attr_name, ":", point.prev_value, "->", point.curr_value)

    df = pd.DataFrame()
    df['attr_1'] = classical[0][1:-2]
    df['attr_2'] = classical[1][1:-2]
    df['attr_3'] = classical[2][1:-2]
    df['attr_4'] = classical[3][1:-2]

    if save_csv:
        df.to_csv('calssical_dataset.csv', sep=',')
    print(df)

def generate_discretized_sequence(change_points):
    sequences = [[] for i in range(len(change_points))]
    for k, attr_p in enumerate(change_points):
        attr_p = attr_p[1:]
        attr_name = attr_p[0].attr_name
        for i, p in enumerate(attr_p):
            prev = round_to(attr_p[i-1].at_,100) if i > 0 else 0

            if i < len(attr_p)-1:
                for j in range(prev, round_to(p.at_,100), 100):
                    elem = attr_name + ":" + str(p.prev_value)
                    sequences[k].append(elem)
    return sequences

def plot_sequences_on_one_figure(sequences, seq_names, simulator, target_seq_index):
    fig, axes = plt.subplots(nrows=len(seq_names), ncols=1, sharex=False,
                                     figsize=(12, 2*3))
    seq_len = len(sequences[0])
    step = round_to(seq_len / 30, 100)
    ticks_to_use = range(0, seq_len, step)
    for i in range(len(seq_names)):
        axes[i].plot(sequences[i], 'b.', markersize=3)
        axes[i].plot(sequences[i], 'b-', alpha=0.25)
        for change_point in simulator.detected_change_points[i]:
            axes[i].axvline(change_point.at_, color='r', linestyle='--')
        axes[i].set_title(seq_names[i])
        if i == target_seq_index:
            axes[i].plot(simulator.predictor.predicted[:seq_len], 'r.', markersize=3)
        labels = [j for j in ticks_to_use]
        axes[i].set_xticks(ticks_to_use)
        axes[i].set_xticklabels(labels, rotation=0)

max_stars_repo_name: stworekAsiorek/CRDiS