text:
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 13 19:02:19 2021

@author: jsalm
"""

import numpy as np
import matplotlib.pyplot as plt
from os.path import dirname, abspath, join
from os import mkdir
import dill as pickle

from sklearn.svm import SVC
from sklearn.model_selection import cross_val_score
from sklearn.metrics import   accuracy_score
from sklearn.ensemble import  RandomForestClassifier

from xgboost import XGBClassifier

import src.localModules.DataManager as DataManager

#%% PATHS
# Path to file
cfpath = dirname(__file__)
# Path to images to be processed
folderName = abspath(join(cfpath,"..","a_dataGeneration","rawData"))
# Path to save bin : saves basic information
saveBin = join(cfpath,"saveBin")
# Path to training files
trainDatDir = abspath(join(cfpath,"..","b_dataAggregation","processedData","EL-11122021"))
# Path to model
modelDir = abspath(join(saveBin,"saveDT"))
# Path to cross-validated files
cvDatDir = abspath(join(cfpath,"..","c_dataValidation","saveBin"))
# Make directory for saves
mkdir(abspath(join(modelDir)))

#%% DEFINITIONS & PARAMS
def robust_save(fname):
    plt.savefig(join(saveBin,'overlayed_predictions.png',dpi=200,bbox_inches='tight'))
#enddef

#%% PARAMS
dTime = '12242021' #date.today().strftime('%d%m%Y')

#%% Load k-split Data (k=10)
tmpSaveDir = join(cvDatDir, ('CVjoined_data_'+dTime+'.pkl'))
tmpSave = DataManager.load_obj(tmpSaveDir)
X_train = tmpSave[0]
X_test = tmpSave[1]
y_train = tmpSave[2]
y_test = tmpSave[3]
X = np.vstack((X_train,X_test))
y = np.vstack((y_train,y_test))
print("y_train: " + str(np.unique(y_train)))
print("y_test: " + str(np.unique(y_test)))

#%% RANDOM FOREST ALGORITHM
print('Random Forest:')

#%% CREATE RANDOMFOREST PIPELINE
print("starting modeling career...")
coef = [671,10,68,3,650,87,462]
RFmodel = RandomForestClassifier(max_depth = coef[0], min_samples_split = coef[1],
                                       max_leaf_nodes = coef[2], min_samples_leaf = coef[3],
                                       n_estimators = coef[4], max_samples = coef[5],
                                       max_features = coef[6])

#%% MODEL FITTING
print('fitting...')
model = RFmodel.fit(X_train,y_train)
y_score = model.decision_function(X_test)
print(model.score(X_test,y_test))
filename = join(modelDir,('fittedRF_'+dTime+'.sav'))
pickle.dump(model, open(filename, 'wb'))
print('done')

y_predict = model.predict(X_test)
y_train_predict = model.predict(X_train)
print('RF Train accuracy',accuracy_score(y_train, y_train_predict))
print('RF Test accuracy',accuracy_score(y_test,y_predict))

#%% Cross Validate
scores = cross_val_score(estimator = model,
                          X = X,
                          y = y,
                          cv = 10,
                          scoring = 'roc_auc',
                          verbose = True,
                          n_jobs=-1)

print('RF CV accuracy scores: %s' % scores)
print('RF CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

#Best coefficients so far:
    #coef = [671,10,68,3,650,87,462]

"""
#%% SAMPLE CODE FOR OPTIMIZING PARAMETERS
score = 0.75
coef = [671,10,68,3,650,192,462]
for ii in range(2,500,10):
        model = RandomForestClassifier(max_depth = coef[0], min_samples_split = coef[1],
                                       max_leaf_nodes = coef[2], min_samples_leaf = coef[3],
                                       n_estimators = coef[4], max_samples = coef[5],
                                       max_features = ii)
        model.fit(X_train,y_train)
        y_predict = model.predict(X_test)
        y_train_predict = model.predict(X_train)
        newscore = roc_auc_score(y_test, model.predict_proba(X_test)[:,1])
        print(ii,newscore, end="")
        if newscore > score:
            print(' best so far')
            score = newscore
        else:
            print()
"""

#%% XGBOOST ALGORITHM
print('XGBoost:')

##XGBoost with Optimal HyperParameters
print("starting modeling career...")
coef = [2,0.28,150,0.57,0.36,0.1,1,0,0.75,0.42]
XGBmodel = XGBClassifier(max_depth = coef[0],subsample = coef[1],n_estimators = coef[2],
                      colsample_bylevel = coef[3], colsample_bytree = coef[4],learning_rate=coef[5],
                      min_child_weight = coef[6], random_state = coef[7],reg_alpha = coef[8],
                      reg_lambda = coef[9])

#%% MODEL FITTING
print('fitting...')
model = XGBmodel.fit(X_train,y_train)
y_score = model.decision_function(X_test)
print(model.score(X_test,y_test))
filename = join(modelDir,('fittedXGB_'+dTime+'.sav'))
pickle.dump(model, open(filename, 'wb'))
print('done')

y_predict = model.predict(X_test)
y_train_predict = model.predict(X_train)
print('XGB Train accuracy',accuracy_score(y_train, y_train_predict))
print('XGB Test accuracy',accuracy_score(y_test,y_predict))

#%% CROSS VALIDATE k-fold (k=10)
scores = cross_val_score(estimator = model,
                          X = X_train,
                          y = y_train,
                          cv = 10,
                          scoring = 'roc_auc',
                          verbose = True,
                          n_jobs=-1)

print('XGB CV accuracy scores: %s' % scores)
print('XGB CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

#Best coefficients so far:
    #coef = [2,0.28,150,0.57,0.36,0.1,1,0,0.75,0.42]

"""
#%% SAMPLE CODE FOR OPTIMIZING PARAMETERS
score = 0.7159090909090909
coef = [2,0.4,150,.8,1,.1,1,0,1,0.5]
for ii in range(2,31):
    model = XGBClassifier(max_depth = ii,subsample = coef[1],n_estimators = coef[2],
                        colsample_bylevel = coef[3], colsample_bytree = coef[4],learning_rate=coef[5],
                        min_child_weight = coef[6], random_state = coef[7],reg_alpha = coef[8],
                        reg_lambda = coef[9])
    model.fit(X_train,y_train)
    y_predict = model.predict(X_test)
    y_train_predict = model.predict(X_train)
    newscore = roc_auc_score(y_test, model.predict_proba(X_test)[:,1])
    print(ii,newscore, end="")
    if newscore > score:
        print(' best so far')
        score = newscore
    else:
        print()
"""

max_stars_repo_name: GML-UF/Generalized-Sklearn-ML-Pipeline

max_issues_repo_name: GML-UF/Generalized-Sklearn-ML-Pipeline

max_forks_repo_name: GML-UF/Generalized-Sklearn-ML-Pipeline


text:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib

SPINE_COLOR = 'grey'
def latexify(fig_width=None, fig_height=None, columns=1, largeFonts=False,font_scale=1):
    """Set up matplotlib's RC params for LaTeX plotting.
    Call this before plotting a figure.

    Parameters
    ----------
    fig_width : float, optional, inches
    fig_height : float,  optional, inches
    columns : {1, 2}
    """

    # code adapted from http://www.scipy.org/Cookbook/Matplotlib/LaTeX_Examples

    # Width and max height in inches for IEEE journals taken from
    # computer.org/cms/Computer.org/Journal%20templates/transactions_art_guide.pdf

    assert(columns in [1,2])

    if fig_width is None:
        fig_width = 3.39 if columns == 1 else 6.9 # width in inches

    if fig_height is None:
        golden_mean = (np.sqrt(5)-1.0)/2.0    # Aesthetic ratio
        fig_height = fig_width*golden_mean # height in inches

    MAX_HEIGHT_INCHES = 8.0
    if fig_height > MAX_HEIGHT_INCHES:
        print("WARNING: fig_height too large:" + fig_height +
              "so will reduce to" + MAX_HEIGHT_INCHES + "inches.")
        fig_height = MAX_HEIGHT_INCHES

    params = {'backend': 'ps',
              'text.latex.preamble': ['\\usepackage{gensymb}'],
              'axes.labelsize': font_scale*10 if largeFonts else font_scale*7, # fontsize for x and y labels (was 10)
              'axes.titlesize': font_scale*10 if largeFonts else font_scale*7,
              'font.size': font_scale*10 if largeFonts else font_scale*7, # was 10
              'legend.fontsize': font_scale*10 if largeFonts else font_scale*7, # was 10
              'xtick.labelsize': font_scale*10 if largeFonts else font_scale*7,
              'ytick.labelsize': font_scale*10 if largeFonts else font_scale*7,
              'text.usetex': True,
              'figure.figsize': [fig_width,fig_height],
              'font.family': 'serif',
              'xtick.minor.size': 0.5,
              'xtick.major.pad': 1.5,
              'xtick.major.size': 1,
              'ytick.minor.size': 0.5,
              'ytick.major.pad': 1.5,
              'ytick.major.size': 1,
              'lines.linewidth': 0.9,
              'lines.markersize': 0.1,
              'hatch.linewidth': 0.5
    }

    matplotlib.rcParams.update(params)
    plt.rcParams.update(params)

def format_axes(ax):

    for spine in ['top', 'right']:
        ax.spines[spine].set_visible(False)

    for spine in ['left', 'bottom']:
        ax.spines[spine].set_color(SPINE_COLOR)
        ax.spines[spine].set_linewidth(0.5)

    ax.xaxis.set_ticks_position('bottom')
    ax.yaxis.set_ticks_position('left')

    for axis in [ax.xaxis, ax.yaxis]:
        axis.set_tick_params(direction='out', color=SPINE_COLOR)

    return ax

ext: py

max_stars_repo_name: musically-ut/memorize

max_issues_repo_name: musically-ut/memorize

max_forks_repo_name: musically-ut/memorize

reason: import numpy


text:
import numpy as np

a = np.array([1, 3, 5])
print(a)

max_stars_repo_name: jrottersman/snap-numpytest

max_issues_repo_name: jrottersman/snap-numpytest

max_forks_repo_name: jrottersman/snap-numpytest


text:
from copy import copy
from datetime import timedelta
from multiprocessing import Manager, Process
from typing import Callable, List, Optional, Tuple, Union
import pandas as pd
import numpy as np

from fedot.core.composer.cache import OperationsCache
from fedot.core.dag.graph import Graph
from fedot.core.data.data import InputData, data_has_categorical_features, data_has_missing_values
from fedot.core.data.multi_modal import MultiModalData
from fedot.core.log import Log, default_log
from fedot.core.operations.evaluation.operation_implementations.data_operations.sklearn_transformations import \
    DataOperationImplementation, ImputationImplementation, OneHotEncodingImplementation
from fedot.core.optimisers.timer import Timer
from fedot.core.optimisers.utils.population_utils import input_data_characteristics
from fedot.core.pipelines.node import Node, PrimaryNode
from fedot.core.pipelines.template import PipelineTemplate
from fedot.core.pipelines.tuning.unified import PipelineTuner
from fedot.core.data.data import data_type_is_table

# The allowed empirical partition limit of the number of rows to delete.
# Rows that have 'string' type, instead of other 'integer' observes.
# Example: 90% objects in column are 'integer', other are 'string'. Then
# we will try to convert 'string' data to 'integer', otherwise delete it.
EMPIRICAL_PARTITION = 0.1
ERROR_PREFIX = 'Invalid pipeline configuration:'

class Pipeline(Graph):
    """
    Base class used for composite model structure definition

    :param nodes: Node object(s)
    :param log: Log object to record messages
    :param tag: uniq part of the repository filename

    .. note::
        fitted_on_data stores the data which were used in last pipeline fitting (equals None if pipeline hasn't been
        fitted yet)
    """

    def __init__(self, nodes: Optional[Union[Node, List[Node]]] = None,
                 log: Log = None):

        self.computation_time = None
        self.template = None
        self.fitted_on_data = {}
        self.pre_proc_encoders = {}

        self.log = log
        if not log:
            self.log = default_log(__name__)
        else:
            self.log = log
        super().__init__(nodes)

    def fit_from_scratch(self, input_data: Union[InputData, MultiModalData] = None):
        """
        Method used for training the pipeline without using saved information

        :param input_data: data used for operation training
        """
        # Clean all saved states and fit all operations
        self.log.debug('Fit pipeline from scratch')
        self.unfit()
        self.fit(input_data, use_fitted=False)

    def update_fitted_on_data(self, data: InputData):
        characteristics = input_data_characteristics(data=data, log=self.log)
        self.fitted_on_data['data_type'] = characteristics[0]
        self.fitted_on_data['features_hash'] = characteristics[1]
        self.fitted_on_data['target_hash'] = characteristics[2]

    def _fitted_status_if_new_data(self, new_input_data: InputData, fitted_status: bool):
        new_data_params = input_data_characteristics(new_input_data, log=self.log)
        if fitted_status and self.fitted_on_data:
            params_names = ('data_type', 'features_hash', 'target_hash')
            are_data_params_different = any(
                [new_data_param != self.fitted_on_data[param_name] for new_data_param, param_name in
                 zip(new_data_params, params_names)])
            if are_data_params_different:
                info = 'Trained operation is not actual because you are using new dataset for training. ' \
                       'Parameter use_fitted value changed to False'
                self.log.info(info)
                fitted_status = False
        return fitted_status

    def _fit_with_time_limit(self, input_data: Optional[InputData] = None, use_fitted_operations=False,
                             time: timedelta = timedelta(minutes=3)) -> Manager:
        """
        Run training process with time limit. Create

        :param input_data: data used for operation training
        :param use_fitted_operations: flag defining whether use saved information about previous executions or not,
        default True
        :param time: time constraint for operation fitting process (seconds)
        """
        time = int(time.total_seconds())
        manager = Manager()
        process_state_dict = manager.dict()
        fitted_operations = manager.list()
        p = Process(target=self._fit,
                    args=(input_data, use_fitted_operations, process_state_dict, fitted_operations),
                    kwargs={})
        p.start()
        p.join(time)
        if p.is_alive():
            p.terminate()
            raise TimeoutError(f'Pipeline fitness evaluation time limit is expired')

        self.fitted_on_data = process_state_dict['fitted_on_data']
        self.computation_time = process_state_dict['computation_time']
        for node_num, node in enumerate(self.nodes):
            self.nodes[node_num].fitted_operation = fitted_operations[node_num]
        return process_state_dict['train_predicted']

    def _fit(self, input_data: InputData, use_fitted_operations=False, process_state_dict: Manager = None,
             fitted_operations: Manager = None):
        """
        Run training process in all nodes in pipeline starting with root.

        :param input_data: data used for operation training
        :param use_fitted_operations: flag defining whether use saved information about previous executions or not,
        default True
        :param process_state_dict: this dictionary is used for saving required pipeline parameters (which were changed
        inside the process) in a case of operation fit time control (when process created)
        :param fitted_operations: this list is used for saving fitted operations of pipeline nodes
        """

        # InputData was set directly to the primary nodes
        if input_data is None:
            use_fitted_operations = False
        else:
            use_fitted_operations = self._fitted_status_if_new_data(new_input_data=input_data,
                                                                    fitted_status=use_fitted_operations)

            if not use_fitted_operations or not self.fitted_on_data:
                # Don't use previous information
                self.unfit()
                self.update_fitted_on_data(input_data)

        with Timer(log=self.log) as t:
            computation_time_update = not use_fitted_operations or not self.root_node.fitted_operation or \
                                      self.computation_time is None

            train_predicted = self.root_node.fit(input_data=input_data)
            if computation_time_update:
                self.computation_time = round(t.minutes_from_start, 3)

        if process_state_dict is None:
            return train_predicted
        else:
            process_state_dict['train_predicted'] = train_predicted
            process_state_dict['computation_time'] = self.computation_time
            process_state_dict['fitted_on_data'] = self.fitted_on_data
            for node in self.nodes:
                fitted_operations.append(node.fitted_operation)

    def fit(self, input_data: Union[InputData, MultiModalData], use_fitted=True,
            time_constraint: Optional[timedelta] = None):
        """
        Run training process in all nodes in pipeline starting with root.

        :param input_data: data used for operation training
        :param use_fitted: flag defining whether use saved information about previous executions or not,
            default True
        :param time_constraint: time constraint for operation fitting (seconds)
        """
        if not use_fitted:
            self.unfit()

        # Make copy of the input data to avoid performing inplace operations
        copied_input_data = copy(input_data)
        copied_input_data = self._preprocessing_fit_data(copied_input_data)
        copied_input_data = self._assign_data_to_nodes(copied_input_data)

        if time_constraint is None:
            train_predicted = self._fit(input_data=copied_input_data,
                                        use_fitted_operations=use_fitted)
        else:
            train_predicted = self._fit_with_time_limit(input_data=copied_input_data,
                                                        use_fitted_operations=use_fitted,
                                                        time=time_constraint)
        return train_predicted

    def _preprocessing_fit_data(self, data: Union[InputData, MultiModalData]):
        has_imputation_operation, has_encoder_operation = pipeline_encoders_validation(self)

        data = _custom_preprocessing(data)

        if data_has_missing_values(data) and not has_imputation_operation:
            data = _imputation_implementation(data)

        if data_has_categorical_features(data) and not has_encoder_operation:
            self.pre_proc_encoders = _encode_data_for_fit(data)
        return data

    @property
    def is_fitted(self):
        return all([(node.fitted_operation is not None) for node in self.nodes])

    def unfit(self):
        """
        Remove fitted operations for all nodes.
        """
        for node in self.nodes:
            node.unfit()

    def fit_from_cache(self, cache: OperationsCache):
        for node in self.nodes:
            cached_state = cache.get(node)
            if cached_state:
                node.fitted_operation = cached_state.operation
            else:
                node.fitted_operation = None

    def predict(self, input_data: Union[InputData, MultiModalData], output_mode: str = 'default'):
        """
        Run the predict process in all nodes in pipeline starting with root.

        :param input_data: data for prediction
        :param output_mode: desired form of output for operations. Available options are:
                'default' (as is),
                'labels' (numbers of classes - for classification) ,
                'probs' (probabilities - for classification =='default'),
                'full_probs' (return all probabilities - for binary classification).
        :return: OutputData with prediction
        """

        if not self.is_fitted:
            ex = 'Pipeline is not fitted yet'
            self.log.error(ex)
            raise ValueError(ex)

        # Make copy of the input data to avoid performing inplace operations
        copied_input_data = copy(input_data)
        has_imputation_operation, has_encoder_operation = pipeline_encoders_validation(self)

        copied_input_data = _custom_preprocessing(copied_input_data)

        if data_has_missing_values(copied_input_data) and not has_imputation_operation:
            copied_input_data = _imputation_implementation(copied_input_data)

        if data_has_categorical_features(copied_input_data) and not has_encoder_operation:
            _encode_data_for_prediction(copied_input_data, self.pre_proc_encoders)

        copied_input_data = self._assign_data_to_nodes(copied_input_data)

        result = self.root_node.predict(input_data=copied_input_data, output_mode=output_mode)
        return result

    def fine_tune_all_nodes(self, loss_function: Callable,
                            loss_params: dict = None,
                            input_data: Union[InputData, MultiModalData] = None,
                            iterations=50, timeout: int = 5,
                            cv_folds: int = None,
                            validation_blocks: int = 3) -> 'Pipeline':
        """ Tune all hyperparameters of nodes simultaneously via black-box
            optimization using PipelineTuner. For details, see
        :meth:`~fedot.core.pipelines.tuning.unified.PipelineTuner.tune_pipeline`
        """
        # Make copy of the input data to avoid performing inplace operations
        copied_input_data = copy(input_data)

        timeout = timedelta(minutes=timeout)
        pipeline_tuner = PipelineTuner(pipeline=self,
                                       task=copied_input_data.task,
                                       iterations=iterations,
                                       timeout=timeout)
        self.log.info('Start tuning of primary nodes')

        tuned_pipeline = pipeline_tuner.tune_pipeline(input_data=copied_input_data,
                                                      loss_function=loss_function,
                                                      loss_params=loss_params,
                                                      cv_folds=cv_folds,
                                                      validation_blocks=validation_blocks)
        self.log.info('Tuning was finished')

        return tuned_pipeline

    def save(self, path: str = None) -> Tuple[str, dict]:
        """
        Save the pipeline to the json representation with pickled fitted operations.

        :param path to json file with operation
        :return: json containing a composite operation description
        """
        self.template = PipelineTemplate(self, self.log)
        json_object, dict_fitted_operations = self.template.export_pipeline(path, root_node=self.root_node)
        return json_object, dict_fitted_operations

    def load(self, source: Union[str, dict], dict_fitted_operations: dict = None):
        """
        Load the pipeline the json representation with pickled fitted operations.

        :param source path to json file with operation
        :param dict_fitted_operations dictionary of the fitted operations
        """
        self.nodes = []
        self.template = PipelineTemplate(self, self.log)
        self.template.import_pipeline(source, dict_fitted_operations)

    def __eq__(self, other) -> bool:
        return self.root_node.descriptive_id == other.root_node.descriptive_id

    def __str__(self):
        description = {
            'depth': self.depth,
            'length': self.length,
            'nodes': self.nodes,
        }
        return f'{description}'

    @property
    def root_node(self) -> Optional[Node]:
        if len(self.nodes) == 0:
            return None
        root = [node for node in self.nodes
                if not any(self.operator.node_children(node))]
        if len(root) > 1:
            raise ValueError(f'{ERROR_PREFIX} More than 1 root_nodes in pipeline')
        return root[0]

    def _assign_data_to_nodes(self, input_data) -> Optional[InputData]:
        if isinstance(input_data, MultiModalData):
            for node in [n for n in self.nodes if isinstance(n, PrimaryNode)]:
                if node.operation.operation_type in input_data.keys():
                    node.node_data = input_data[node.operation.operation_type]
                    node.direct_set = True
                else:
                    raise ValueError(f'No data for primary node {node}')
            return None
        return input_data

    def print_structure(self):
        """ Method print information about pipeline """
        print('Pipeline structure:')
        print(self.__str__())
        for node in self.nodes:
            print(f'{node.operation.operation_type} - {node.custom_params}')

def pipeline_encoders_validation(pipeline: Pipeline) -> (bool, bool):
    """ Check whether Imputation and OneHotEncoder operation exist in pipeline.

    :param data Pipeline: data to check
    :return (bool, bool): has Imputation and OneHotEncoder in pipeline
    """

    has_imputers, has_encoders = [], []

    def _check_imputer_encoder_recursion(root: Optional[Node], has_imputer: bool = False, has_encoder: bool = False):
        node_type = root.operation.operation_type
        if node_type == 'simple_imputation':
            has_imputer = True
        if node_type == 'one_hot_encoding':
            has_encoder = True

        if has_imputer and has_encoder:
            return has_imputer, has_encoder
        elif root.nodes_from is None:
            return has_imputer, has_encoder

        for node in root.nodes_from:
            answer = _check_imputer_encoder_recursion(node, has_imputer, has_encoder)
            if answer is not None:
                imputer, encoder = answer
                has_imputers.append(imputer)
                has_encoders.append(encoder)

    _check_imputer_encoder_recursion(pipeline.root_node)

    if not has_imputers and not has_encoders:
        return False, False

    has_imputer = len([_ for _ in has_imputers if not _]) == 0
    has_encoder = len([_ for _ in has_encoders if not _]) == 0
    return has_imputer, has_encoder

def _is_numeric(s) -> bool:
    """ Check if variable converted to float.

    :param s: any type variable
    :return: is variable convertable to float
    """
    try:
        float(s)
        return True
    except ValueError:
        return False

def is_np_array_has_nan(array):
    for x in array:
        if x is np.nan:
            return True
    return False

def _try_convert_to_numeric(values):
    try:
        values = pd.to_numeric(values)
        values = values.astype(np.number)
    except ValueError:
        pass
    return values

def _custom_preprocessing(data: Union[InputData, MultiModalData]):
    if isinstance(data, InputData):
        if data_type_is_table(data):
            data = _preprocessing_input_data(data)
    elif isinstance(data, MultiModalData):
        for data_source_name, values in data.items():
            if data_type_is_table(values):
                data[data_source_name] = _preprocessing_input_data(values)

    return data

def _preprocessing_input_data(data: InputData) -> InputData:
    features = data.features
    target = data.target

    # delete rows with equal target None
    if target is not None and len(target.shape) != 0 and is_np_array_has_nan(target):
        target_index_with_nan = np.hstack(np.argwhere(np.isnan(target)))
        data.features = np.delete(features, target_index_with_nan, 0)
        data.target = np.delete(data.target, target_index_with_nan, 0)
        data.idx = np.delete(data.idx, target_index_with_nan, 0)

    source_shape = features.shape
    columns_amount = source_shape[1] if len(source_shape) > 1 else 1

    for i in range(columns_amount):
        values = pd.Series(features[:, i])
        # check for each column, if values converted to numeric. remember index of rows that not converted
        if any(list(map(lambda x: isinstance(x, str), values))):
            not_numeric = list(map(lambda x: not _is_numeric(x), values))
            rows_to_nan = list(values.index[not_numeric])
            partition_not_numeric = len(rows_to_nan) / source_shape[0]

            # if partition of numerical rows less then EMPIRICAL_PARTITION,
            # then convert to numerical and others to Nan
            if partition_not_numeric < EMPIRICAL_PARTITION:
                values[rows_to_nan] = np.nan
                data.features[:, i] = _try_convert_to_numeric(values)
            # if EMPIRICAL_PARTITION < partition < 1, then some data in column are
            # integer and some data are string, can not handle this case
            elif partition_not_numeric < 0.9:
                raise ValueError("The data in the column has a different type. Need to preprocessing data manually.")

    return data

def nodes_with_operation(pipeline: Pipeline, operation_name: str) -> list:
    """ The function return list with nodes with the needed operation

    :param pipeline: pipeline to process
    :param operation_name: name of operation to search
    :return : list with nodes, None if there are no nodes
    """

    # Check if model has decompose operations
    appropriate_nodes = filter(lambda x: x.operation.operation_type == operation_name, pipeline.nodes)

    return list(appropriate_nodes)

def _encode_data_for_fit(data: Union[InputData, MultiModalData]) -> \
        Union[List[DataOperationImplementation], DataOperationImplementation]:
    """ Encode categorical features to numerical. In additional,
    save encoders to use later for prediction data.

    :param data: data to transform
    :return encoders: operation preprocessing categorical features or list of it
    """

    encoders = None
    if isinstance(data, InputData):
        transformed, encoder = _create_encoder(data)
        encoders = encoder
        data.features = transformed
    elif isinstance(data, MultiModalData):
        encoders = {}
        for data_source_name, values in data.items():
            if data_source_name.startswith('data_source_table'):
                transformed, encoder = _create_encoder(values)
                if encoder is not None:
                    encoders[data_source_name] = encoder
                data[data_source_name].features = transformed

    return encoders

def _encode_data_for_prediction(data: Union[InputData, MultiModalData],
                                encoders: Union[dict, DataOperationImplementation]):
    """ Transformation the prediction data inplace. Use the same transformations as for the training data.

    :param data: data to transformation
    :param encoders: encoders f transformation
    """
    if encoders:
        if isinstance(data, InputData):
            transformed = encoders.transform(data, True).predict
            data.features = transformed
        elif isinstance(data, MultiModalData):
            for data_source_name, encoder in encoders.items():
                transformed = encoder.transform(data[data_source_name], True).predict
                data[data_source_name].features = transformed

def _imputation_implementation(data: Union[InputData, MultiModalData]) -> Union[InputData, MultiModalData]:
    if isinstance(data, InputData):
        return _imputation_implementation_unidata(data)
    if isinstance(data, MultiModalData):
        for data_source_name, values in data.items():
            if data_source_name.startswith('data_source_table') or data_source_name.startswith('data_source_ts'):
                data[data_source_name].features = _imputation_implementation_unidata(values)
        return data
    raise ValueError(f"Data format is not supported.")

def _imputation_implementation_unidata(data: InputData):
    """ Fill in the gaps in the data inplace.

    :param data: data for fill in the gaps
    """
    imputer = ImputationImplementation()
    output_data = imputer.fit_transform(data)
    transformed = InputData(features=output_data.predict, data_type=output_data.data_type,
                            target=output_data.target, task=output_data.task, idx=output_data.idx)
    return transformed

def _create_encoder(data: InputData):
    """ Fills in the gaps, converts categorical features using OneHotEncoder and create encoder.

    :param data: data to preprocess
    :return tuple(array, Union[OneHotEncodingImplementation, None]): tuple of transformed and [encoder or None]
    """

    encoder = None
    if data_has_categorical_features(data):
        encoder = OneHotEncodingImplementation()
        encoder.fit(data)
        transformed = encoder.transform(data, True).predict
    else:
        transformed = data.features

    return transformed, encoder

max_stars_repo_name: vkirilenko/FEDOT

max_issues_repo_name: vkirilenko/FEDOT

max_forks_repo_name: vkirilenko/FEDOT


text:
'''
Created on 10 Nov 2016

@author: bogdan
'''

# import numpy as np

class clCognateMatch(object):
	'''
	classdocs
	'''

	def __init__(self):
		'''
		Constructor
		'''

if __name__ == '__main__':
	OCognateMatch = clCognateMatch()

max_stars_repo_name: bogdanbabych/morphosyntax

max_issues_repo_name: bogdanbabych/morphosyntax

max_forks_repo_name: bogdanbabych/morphosyntax


text:
import numpy as np
import warnings
from os import path as op
import pytest
from numpy.testing import assert_equal, assert_array_equal

from pyeparse import read_raw
from pyeparse._event import Discrete
from pyeparse.utils import _requires_edfapi

warnings.simplefilter('always')  # in case we hit warnings

fname = op.join(op.dirname(__file__), 'data', 'test_2_raw.edf')

@_requires_edfapi
def test_find_custom_events():
    """Test finding user-defined events."""
    raw = read_raw(fname)
    events = raw.find_events('TRIALID', 1)
    assert len(events) > 0
    pytest.raises(ValueError, raw.find_events, list(), 1)
    events_2 = raw.find_events(lambda x: 'TRIALID' in x, 1)
    assert_array_equal(events, events_2)

def test_discrete():
    """Test discrete events container."""
    dis = Discrete()
    dis.extend([np.array([1]), 'aaaa'])
    myrepr = '%s' % dis
    checksum = sum([int(d) for d in myrepr if d.isdigit()])
    assert_equal(checksum, 5 + len(dis))
    dis = Discrete('aaaaa')
    assert_equal(len(dis), 5)

reason: import numpy,from numpy


text:
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
""" test_training """
import numpy as np

import mindspore.nn as nn
from mindspore import context
from mindspore.common.tensor import Tensor
from mindspore.nn import WithGradCell, WithLossCell
from mindspore.ops import operations as P
from ..ut_filter import non_graph_engine

def setup_module(module):
    context.set_context(mode=context.PYNATIVE_MODE)

class LeNet5(nn.Cell):
    """ LeNet5 definition """

    def __init__(self):
        super(LeNet5, self).__init__()
        self.conv1 = nn.Conv2d(1, 6, 5, pad_mode='valid')
        self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid')
        self.fc1 = nn.Dense(16 * 5 * 5, 120)
        self.fc2 = nn.Dense(120, 84)
        self.fc3 = nn.Dense(84, 10)
        self.relu = nn.ReLU()
        self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2)
        self.flatten = P.Flatten()

    def construct(self, x):
        x = self.max_pool2d(self.relu(self.conv1(x)))
        x = self.max_pool2d(self.relu(self.conv2(x)))
        x = self.flatten(x)
        x = self.relu(self.fc1(x))
        x = self.relu(self.fc2(x))
        x = self.fc3(x)
        return x

@non_graph_engine
def test_loss_cell_wrapper():
    """ test_loss_cell_wrapper """
    data = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01)
    label = Tensor(np.ones([1, 10]).astype(np.float32))
    net = LeNet5()
    loss_fn = nn.SoftmaxCrossEntropyWithLogits()
    loss_net = WithLossCell(net, loss_fn)
    loss_out = loss_net(data, label)
    assert loss_out.asnumpy().dtype == 'float32' or loss_out.asnumpy().dtype == 'float64'

@non_graph_engine
def test_grad_cell_wrapper():
    """ test_grad_cell_wrapper """
    data = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01)
    label = Tensor(np.ones([1, 10]).astype(np.float32))
    dout = Tensor(np.ones([1]).astype(np.float32))
    net = LeNet5()
    loss_fn = nn.SoftmaxCrossEntropyWithLogits()
    grad_net = WithGradCell(net, loss_fn, dout)
    gradients = grad_net(data, label)
    assert isinstance(gradients[0].asnumpy()[0][0][0][0], (np.float32, np.float64))

max_stars_repo_name: GuoSuiming/mindspore


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Mar  9 09:42:00 2021

@author: barraly
"""

import sabs_pkpd
import numpy as np
import matplotlib.pyplot as plt
import os

# Select the folder in which this repo is downloaded in the line below
os.chdir('The/location/of/the/root/folder/of/this/repo')

# In[Load the model]
filename = './Models/Ohara CiPA - analytical voltage.mmt'
s = sabs_pkpd.load_model.load_simulation_from_mmt(filename)
s.set_tolerance(1e-08, 1e-08)
default_state = s.state()

# Save the initial conditions published in OHara CiPA model
Ohara_init_conds = default_state.copy()

# In[Defin the needed functions]
# Define the functions to make sure there is consistency between the initial conditions
def G0_calc(Ki = 144.65559, Kss = 144.65556, Nai = 7.268, Nass = 7.26809,
            Cai = 8.6e-5, Cansr = 1.61957, Cajsr = 1.571234014, Cass = 8.49e-5,
            V=-88, extraK = 5.4, extraNa = 140, extraCa = 1.8):
    tot_cai = Cai * (1 + 0.05 / (Cai + 0.00238) + 0.07/(Cai + 0.0005))
    tot_cass = Cass * (1 + 0.047 / (Cass + 0.00087) + 1.124/(Cass + 0.0087))
    tot_cajsr = Cajsr * (1 + 10 / (Cajsr + 0.8))
    return V / (96485 * 2.583592e-05) * 0.0001533576 - (Ki + Kss * 0.029411764705882353 + Nai + Nass * 0.029411764705882353 + 2*(tot_cai + tot_cass * 0.029411764705882353 + Cansr * 0.08117647059 + tot_cajsr * 0.007059) - extraK - extraNa - 2 * extraCa)

def Ki_calc(G0, Nai = 7.268, Nass = 7.26809, Cai = 8.6e-5, Cansr = 1.61957, Cajsr = 1.571234014, Cass = 8.49e-5, V=-88, extraK = 5.4, extraNa = 140, extraCa = 1.8):
    tot_cai = Cai * (1 + 0.05 / (Cai + 0.00238) + 0.07/(Cai + 0.0005))
    tot_cass = Cass * (1 + 0.047 / (Cass + 0.00087) + 1.124/(Cass + 0.0087))
    tot_cajsr = Cajsr * (1 + 10 / (Cajsr + 0.8))
    return (V / (96485 * 2.583592e-05) * 0.0001533576 + extraK + extraNa + 2 * extraCa - G0 - Nai - Nass * 0.029411764705882353 - 2*(tot_cai + tot_cass * 0.029411764705882353 + Cansr * 0.08117647059 + tot_cajsr * 0.007059)) / 1.029411764705882353

def compute(Gamma_0):
    # Reinitialise the myokit.Simulation
    s.reset()

    # Set the initial conditions for Ki and Kss so that the initial conditions match with the value of Gamma_0
    initial_state = default_state.copy()
    initial_K = Ki_calc(Gamma_0,
                       Nai = default_state[1],
                       Nass = default_state[2],
                       Cai = default_state[5],
                       Cansr = default_state[7],
                       Cajsr = default_state[8],
                       Cass = default_state[6])
    initial_state[3] = initial_K
    initial_state[4] = initial_K
    s.set_state(initial_state)

    # Set the value of Gamma_0 in the myokit.Simulation
    s.set_constant('membrane.c0', Gamma_0)

    # Record the action potential at the limit cycle
    s.pre(2000000)
    out = s.run(1000, log_interval = 1)
    print('Potassium at steady-state: ' + str(np.round(out['intracellular_ions.ki'][-1], decimals = 2)))

    return np.array(out['membrane.V'])

# In[Reuse the fitting instructions]
# Define the time points on which to read the voltage
time_points = np.linspace(0, 999, 1000)

# Define the fitted parameters and initial point
parameters_to_fit = ['ical.rescale', 'ikr.rescale', 'IKs.rescale', 'INa.rescale', 'INaL.rescale']
true_values = np.array([1, 1, 1, 1, 1])

# In[Compute the fitted data]
# Set the parameters values
for p, label in enumerate(parameters_to_fit):
    s.set_constant(label, true_values[p])
# Run the model with the published original initial conditions and the Gamma_0 value associated with it
Gamma_0_for_fitting = -7.80116
data_to_fit = compute(Gamma_0_for_fitting)

# For validation with 50% IKr inhibition
s.set_constant(parameters_to_fit[1], 0.5 * true_values[1])
validation_data = compute(Gamma_0_for_fitting)

# In[Report the results from the fitting with Ohara initial conditions]
default_state = Ohara_init_conds.copy()
found_parameters = [1.000, 1.000, 1.000, 1.000, 1.000]
for p, label in enumerate(parameters_to_fit):
    s.set_constant(label, found_parameters[p])
fitting_with_Ohara_ICs = compute(Gamma_0_for_fitting)

print('Gamma_0 for fitting : ' + str(Gamma_0_for_fitting))
APD90 = sabs_pkpd.cardiac.compute_APD(AP = fitting_with_Ohara_ICs, time_points= time_points, upstroke_time = 50)
print('APD_90 at baseline : ' + str(APD90))

# Predict IKr block AP
s.set_constant(parameters_to_fit[1], 0.5 * found_parameters[1])
fitting_with_Ohara_ICs_Kr_blocked = compute(Gamma_0_for_fitting)
APD90 = sabs_pkpd.cardiac.compute_APD(AP = fitting_with_Ohara_ICs_Kr_blocked, time_points= time_points, upstroke_time = 50)
print('APD_90 after 50% IKr block : ' + str(APD90))

# In[Report the results from the fitting with TT06 initial conditions]
default_state[1] = 10.134
default_state[2] = 10.134
default_state[3] = 135.369
default_state[4] = 135.369
default_state[5] = 1.058e-04
default_state[6] = 2.142e-04
default_state[7] = 3.556
default_state[8] = 3.556
initial_voltage = -84.936

Gamma_0_for_fitting = G0_calc(Nai = default_state[1],
                              Nass = default_state[2],
                              Ki = default_state[3],
                              Kss = default_state[4],
                              Cai = default_state[5],
                              Cass = default_state[6],
                              Cansr = default_state[7],
                              Cajsr = default_state[8],
                              V=initial_voltage)

found_parameters = [0.8278844, 1.13276793, 0.74292672, 1.08754243, 1.4459109]
for p, label in enumerate(parameters_to_fit):
    s.set_constant(label, found_parameters[p])
fitting_with_TT06_ICs = compute(Gamma_0_for_fitting)

print('Gamma_0 for fitting : ' + str(Gamma_0_for_fitting))
APD90 = sabs_pkpd.cardiac.compute_APD(AP = fitting_with_TT06_ICs, time_points= time_points, upstroke_time = 50)
print('APD_90 at baseline : ' + str(APD90))

# Predict IKr block AP
s.set_constant(parameters_to_fit[1], 0.5 * found_parameters[1])
fitting_with_TT06_ICs_Kr_blocked = compute(Gamma_0_for_fitting)
APD90 = sabs_pkpd.cardiac.compute_APD(AP = fitting_with_TT06_ICs_Kr_blocked, time_points= time_points, upstroke_time = 50)
print('APD_90 after 50% IKr block : ' + str(APD90))

# In[Report the results from the fitting with TT06 initial conditions and Gamma_0]
default_state = Ohara_init_conds.copy()
found_parameters = [0.9999947, 0.99999936, 0.99995396, 0.999993485, 0.9999772, -7.801077]
for p, label in enumerate(parameters_to_fit):
    s.set_constant(label, found_parameters[p])
fitting_with_TT06_ICs_gamma_0 = compute(found_parameters[-1])

APD90 = sabs_pkpd.cardiac.compute_APD(AP = fitting_with_TT06_ICs_gamma_0, time_points= time_points, upstroke_time = 50)
print('APD_90 at baseline : ' + str(APD90))

# Predict IKr block AP
s.set_constant(parameters_to_fit[1], 0.5 * found_parameters[1])
fitting_with_TT06_ICs_gamma_0_Kr_blocked = compute(found_parameters[-1])
APD90 = sabs_pkpd.cardiac.compute_APD(AP = fitting_with_TT06_ICs_gamma_0_Kr_blocked, time_points= time_points, upstroke_time = 50)
print('APD_90 after 50% IKr block : ' + str(APD90))

# In[Plot the comparison]
def place_caption_label(ax, label, loc='upper left', fontsize=35):
    from matplotlib.offsetbox import AnchoredText
    at = AnchoredText(label, loc=loc, prop=dict(size=fontsize), frameon=True, borderpad = 0)
    ax.add_artist(at)
    return None

from mpl_toolkits.axes_grid1.inset_locator import inset_axes
fig, ax = plt.subplots(1, 2, figsize=[15, 7])

size_ticks = 22
size_labels = 25
# Shift x-axis so that AP starts at 0 ms (in the simulations, the stimulus fires at t=50 ms)
x = np.linspace(-50, 599, 650)

# Plot the fitted APs
ax[0].plot(x, data_to_fit[:650], label = 'Data to fit', color = 'k', linewidth = 5)
ax[0].plot(x, fitting_with_Ohara_ICs[:650], label = 'Fitting #1', linestyle = '--', linewidth = 3)
ax[0].plot(x, fitting_with_TT06_ICs[:650], label = 'Fitting #2', linestyle = '--', linewidth = 3)
ax[0].plot(x, fitting_with_TT06_ICs_gamma_0[:650], label = 'Fitting #3', linestyle = '--', linewidth = 3)

ax[0].legend(fontsize = 25)
ax[0].set_xlabel('Time (ms)', fontsize = size_labels)
ax[0].set_ylabel('Voltage (mV)', fontsize = size_labels)
ax[0].tick_params(axis = 'both', labelsize = size_ticks)
place_caption_label(ax[0], 'A', 'lower right')

# Add an inset to zoom into the short pacing periods
axins1 = ax[0].inset_axes(bounds = [0.7, 0.2, 0.3, 0.3])
x_inset = np.linspace(270, 299, 30)

axins1.plot(x_inset, data_to_fit[320:350], color = 'k', linewidth = 5)
axins1.plot(x_inset, fitting_with_Ohara_ICs[320:350], linestyle = '--', linewidth = 3)
axins1.plot(x_inset, fitting_with_TT06_ICs[320:350], linestyle = '--', linewidth = 3)
axins1.plot(x_inset, fitting_with_TT06_ICs_gamma_0[320:350], linestyle = '--', linewidth = 3)

# set up the inset ticks
axins1.set_xticks([270, 285, 300])
axins1.tick_params(axis = 'both', labelsize = 15)

# Plot the predicted APs with Kr block
ax[1].plot(x, validation_data[:650], label = 'Validation data', linestyle = '-', linewidth = 5, color = 'k')
ax[1].plot(x, fitting_with_Ohara_ICs_Kr_blocked[:650], label = 'Prediction #1', linestyle = '-', linewidth = 3)
ax[1].plot(x, fitting_with_TT06_ICs_Kr_blocked[:650], label = 'Prediction #2', linestyle = '-', linewidth = 3)
ax[1].plot(x, fitting_with_TT06_ICs_gamma_0_Kr_blocked[:650], label = 'Prediction #3', linestyle = '-', linewidth = 3)

ax[1].legend(fontsize = 25, loc = 'upper right')
ax[1].set_xlabel('Time (ms)', fontsize = size_labels)
ax[1].set_ylabel('Voltage (mV)', fontsize = size_labels)
ax[1].tick_params(axis = 'both', labelsize = size_ticks)
place_caption_label(ax[1], 'B', 'lower right')

# Add an inset to zoom into the short pacing periods
axins2 = ax[1].inset_axes(bounds = [0.7, 0.2, 0.3, 0.3])
x_inset = np.linspace(375, 424, 50)

axins2.plot(x_inset, validation_data[425:475], linestyle = '-', linewidth = 5, color = 'k')
axins2.plot(x_inset, fitting_with_Ohara_ICs_Kr_blocked[425:475], linestyle = '-', linewidth = 3)
axins2.plot(x_inset, fitting_with_TT06_ICs_Kr_blocked[425:475], linestyle = '-', linewidth = 3)
axins2.plot(x_inset, fitting_with_TT06_ICs_gamma_0_Kr_blocked[425:475], linestyle = '-', linewidth = 3)

# set up the inset ticks
axins2.set_xticks([375, 400, 425])
axins2.tick_params(axis = 'both', labelsize = 15)

# Save
plt.tight_layout()
plt.savefig('./Figures/Comparison of optimal APs.png', dpi = 300)

max_stars_repo_path: Scripts/Fig 8 - Compare optimal APs.py

max_issues_repo_path: Scripts/Fig 8 - Compare optimal APs.py

max_forks_repo_path: Scripts/Fig 8 - Compare optimal APs.py


text:
# check pytorch installation:
import torch, torchvision
print(torch.__version__, torch.cuda.is_available())
assert torch.__version__.startswith("1.8")   # please manually install torch 1.8 if Colab changes its default version

# Some basic setup:
# Setup detectron2 logger
import detectron2
from detectron2.utils.logger import setup_logger
setup_logger()

# import some common libraries
import numpy as np
import os, json, cv2, random
from google.colab.patches import cv2_imshow

# import some common detectron2 utilities
from detectron2 import model_zoo
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog, DatasetCatalog

import os
from os import listdir
from os.path import isfile, join
dir = '/content/detectron2/images/'
images = [f for f in listdir(dir) if isfile(join(dir, f))]
for image in images:
    im = cv2.imread("./images/" + image)
    cfg = get_cfg()
    # add project-specific config (e.g., TensorMask) here if you're not running a model in detectron2's core library
    cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model
    # Find a model from detectron2's model zoo. You can use the https://dl.fbaipublicfiles... url as well
    cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
    predictor = DefaultPredictor(cfg)
    outputs = predictor(im)

    instances = outputs["instances"]
    category_detections = instances[instances.pred_classes == 0]
    print(category_detections.pred_boxes)
    v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)
    out = v.draw_instance_predictions(outputs["instances"][outputs["instances"].pred_classes == 0].to("cpu"))
    cv2_imshow(out.get_image()[:, :, ::-1])


text:
import numpy
import matplotlib.animation as animation
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import likely_chainer.models
import likely_chainer.optimizers

# シード値固定
numpy.random.seed(seed=10)

# モデル定義
dim = 2
alpha = 0.02
epoch = 100
model = likely_chainer.models.LSM(dimension=dim, learning_rate=alpha)
optimizer = likely_chainer.optimizers.SGD()
optimizer.setup(model)

# とりあえずランダムな重みのときをプロットしてみる
plot_x = numpy.linspace(-5, 10)
plot_y = model(plot_x).data

# とりあえずデータを用意する
train_x = [1, 2, 3]
train_y = [3, 2, 0]

# 簡単に学習させてみる
# エポックで回してみる
fig, ax = plt.subplots()
artists = []
for ep in range(epoch):
    # まずはデータをシャッフル
    perm = numpy.random.permutation(len(train_x))
    train_x = numpy.array(train_x)[perm]
    train_y = numpy.array(train_y)[perm]

    for (x, y) in zip(train_x, train_y):
        # 完璧なChainer風に仕上げた
        model.zerograds()
        loss = model([x], [y])
        loss.backward()
        optimizer.update()

    ax.set_ylim(-5, 5)
    pred_y = model(plot_x).data

    # 動的にアニメーション化
    if (ep + 1 == epoch):
        # 最後だけラベルを振る
        ax.scatter(train_x, train_y, color="RED", label="data")
        ax.plot(plot_x, plot_y, color="BLUE", label="initial line")
        line = ax.plot(plot_x, pred_y, color="GREEN", label="optimized line")
    else:
        ax.scatter(train_x, train_y, color="RED")
        ax.plot(plot_x, plot_y, color="BLUE")
        line = ax.plot(plot_x, pred_y, color="GREEN")
        title = ax.text(
                0.5, 1.01,
                "learning rate = {}, {} epoch score".format(alpha, ep + 1),
                transform=ax.transAxes, ha="center")

    artists.append(line + [title])

# GIFで保存
ax.legend()
anime = animation.ArtistAnimation(fig, artists, interval=100)
anime.save('images/anim.gif', writer="imagemagick")


text:
# - * - coding: utf-8 - * -
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.patches

def ecg_fixpeaks(rpeaks, sampling_rate=1000, iterative=True, show=False):
    """Correct R-peaks location based on their interval (RRi).

    Identify erroneous inter-beat-intervals. Lipponen & Tarvainen (2019).

    Parameters
    ----------
    rpeaks : dict
        The samples at which the R-peak occur. Dict returned by
        `ecg_findpeaks()`.
    sampling_rate : int
        The sampling frequency of the signal that contains the peaks (in Hz,
        i.e., samples/second).
    iterative : bool
        Whether or not to apply the artifact correction repeatedly (results
        in superior artifact correction).
    show : bool
        Whether or not to visualize artifacts and artifact thresholds.

    Returns
    -------
    artifacts : dict
        A dictionary containing the indices of artifacts, accessible with the
        keys "ectopic", "missed", "extra", and "longshort".

    See Also
    --------
    ecg_clean, ecg_findpeaks, ecg_peaks, ecg_rate, ecg_process, ecg_plot

    Examples
    --------
    >>> import neurokit2 as nk
    >>> import matplotlib.pyplot as plt

    >>> ecg = nk.ecg_simulate(duration=240, noise=0.1, heart_rate=70,
    >>>                       random_state=41)
    >>> rpeaks_uncorrected = nk.ecg_findpeaks(ecg)
    >>> artifacts, rpeaks_corrected = nk.ecg_fixpeaks(rpeaks_uncorrected,
    >>>                                               iterative=True,
    >>>                                               show=True)
    >>> rate_corrected = nk.ecg_rate(rpeaks_uncorrected,
    >>>                              desired_length=len(ecg))
    >>> rate_uncorrected = nk.ecg_rate(rpeaks, desired_length=len(ecg_signal))
    >>>
    >>> fig, ax = plt.subplots()
    >>> ax.plot(rate_uncorrected, label="heart rate without artifact correction")
    >>> ax.plot(rate_corrected, label="heart rate with artifact correction")
    >>> ax.legend(loc="upper right")

    References
    ----------
    - Lipponen, J. A., & Tarvainen, M. P. (2019). A robust algorithm for heart
    rate variability time series artefact correction using novel beat
    classification. Journal of medical engineering & technology, 43(3),
    173-181. 10.1080/03091902.2019.1640306

    """
    # Format input.
    rpeaks = rpeaks["ECG_R_Peaks"]
    # Get corrected peaks and normal-to-normal intervals.
    artifacts, subspaces = _find_artifacts(rpeaks, sampling_rate=sampling_rate)
    peaks_clean = _correct_artifacts(artifacts, rpeaks)

    if iterative:

        # Iteratively apply the artifact correction until the number of artifact
        # reaches an equilibrium (i.e., the number of artifacts does not change
        # anymore from one iteration to the next).
        n_artifacts_previous = np.inf
        n_artifacts_current = sum([len(i) for i in artifacts.values()])

        previous_diff = 0

        while n_artifacts_current - n_artifacts_previous != previous_diff:

            previous_diff = n_artifacts_previous - n_artifacts_current

            artifacts, subspaces = _find_artifacts(peaks_clean,
                                                   sampling_rate=sampling_rate)
            peaks_clean = _correct_artifacts(artifacts, peaks_clean)

            n_artifacts_previous = n_artifacts_current
            n_artifacts_current = sum([len(i) for i in artifacts.values()])

    if show:
        _plot_artifacts_lipponen2019(artifacts, subspaces)

    return artifacts, {"ECG_R_Peaks": peaks_clean}

# =============================================================================
# Lipponen & Tarvainen (2019).
# =============================================================================
def _find_artifacts(rpeaks, c1=0.13, c2=0.17, alpha=5.2, window_width=91,
                    medfilt_order=11, sampling_rate=1000):

    peaks = np.ravel(rpeaks)

    # Compute period series (make sure it has same numer of elements as peaks);
    # peaks are in samples, convert to seconds.
    rr = np.ediff1d(peaks, to_begin=0) / sampling_rate
    # For subsequent analysis it is important that the first element has
    # a value in a realistic range (e.g., for median filtering).
    rr[0] = np.mean(rr[1:])

    # Artifact identification #################################################
    ###########################################################################

    # Compute dRRs: time series of differences of consecutive periods (dRRs).
    drrs = np.ediff1d(rr, to_begin=0)
    drrs[0] = np.mean(drrs[1:])
    # Normalize by threshold.
    th1 = _compute_threshold(drrs, alpha, window_width)
    drrs /= th1

    # Cast dRRs to subspace s12.
    # Pad drrs with one element.
    padding = 2
    drrs_pad = np.pad(drrs, padding, "reflect")

    s12 = np.zeros(drrs.size)
    for d in np.arange(padding, padding + drrs.size):

        if drrs_pad[d] > 0:
            s12[d - padding] = np.max([drrs_pad[d - 1], drrs_pad[d + 1]])
        elif drrs_pad[d] < 0:
            s12[d - padding] = np.min([drrs_pad[d - 1], drrs_pad[d + 1]])

    # Cast dRRs to subspace s22.
    s22 = np.zeros(drrs.size)
    for d in np.arange(padding, padding + drrs.size):

        if drrs_pad[d] >= 0:
            s22[d - padding] = np.min([drrs_pad[d + 1], drrs_pad[d + 2]])
        elif drrs_pad[d] < 0:
            s22[d - padding] = np.max([drrs_pad[d + 1], drrs_pad[d + 2]])

    # Compute mRRs: time series of deviation of RRs from median.
    df = pd.DataFrame({'signal': rr})
    medrr = df.rolling(medfilt_order, center=True,
                       min_periods=1).median().signal.to_numpy()
    mrrs = rr - medrr
    mrrs[mrrs < 0] = mrrs[mrrs < 0] * 2
    # Normalize by threshold.
    th2 = _compute_threshold(mrrs, alpha, window_width)
    mrrs /= th2

    # Artifact classification #################################################
    ###########################################################################

    # Artifact classes.
    extra_idcs = []
    missed_idcs = []
    ectopic_idcs = []
    longshort_idcs = []

    i = 0
    while i < rr.size - 2:    # The flow control is implemented based on Figure 1

        if np.abs(drrs[i]) <= 1:    # Figure 1
            i += 1
            continue
        eq1 = np.logical_and(drrs[i] > 1, s12[i] < (-c1 * drrs[i] - c2))    # Figure 2a
        eq2 = np.logical_and(drrs[i] < -1, s12[i] > (-c1 * drrs[i] + c2))    # Figure 2a

        if np.any([eq1, eq2]):
            # If any of the two equations is true.
            ectopic_idcs.append(i)
            i += 1
            continue
        # If none of the two equations is true.
        if ~np.any([np.abs(drrs[i]) > 1, np.abs(mrrs[i]) > 3]):    # Figure 1
            i += 1
            continue
        longshort_candidates = [i]
        # Check if the following beat also needs to be evaluated.
        if np.abs(drrs[i + 1]) < np.abs(drrs[i + 2]):
            longshort_candidates.append(i + 1)

        for j in longshort_candidates:
            # Long beat.
            eq3 = np.logical_and(drrs[j] > 1, s22[j] < -1)    # Figure 2b
            # Long or short.
            eq4 = np.abs(mrrs[j]) > 3    # Figure 1
            # Short beat.
            eq5 = np.logical_and(drrs[j] < -1, s22[j] > 1)    # Figure 2b

            if ~np.any([eq3, eq4, eq5]):
                # If none of the three equations is true: normal beat.
                i += 1
                continue
            # If any of the three equations is true: check for missing or extra
            # peaks.

            # Missing.
            eq6 = np.abs(rr[j] / 2 - medrr[j]) < th2[j]    # Figure 1
            # Extra.
            eq7 = np.abs(rr[j] + rr[j + 1] - medrr[j]) < th2[j]    # Figure 1

            # Check if extra.
            if np.all([eq5, eq7]):
                extra_idcs.append(j)
                i += 1
                continue
            # Check if missing.
            if np.all([eq3, eq6]):
                missed_idcs.append(j)
                i += 1
                continue
            # If neither classified as extra or missing, classify as "long or
            # short".
            longshort_idcs.append(j)
            i += 1

    # Prepare output
    artifacts = {"ectopic": ectopic_idcs, "missed": missed_idcs,
                 "extra": extra_idcs, "longshort": longshort_idcs}

    subspaces = {"rr": rr, "drrs": drrs, "mrrs": mrrs, "s12": s12, "s22": s22,
                 "c1": c1, "c2": c2}

    return artifacts, subspaces

def _compute_threshold(signal, alpha, window_width):

    df = pd.DataFrame({'signal': np.abs(signal)})
    q1 = df.rolling(window_width, center=True,
                    min_periods=1).quantile(.25).signal.to_numpy()
    q3 = df.rolling(window_width, center=True,
                    min_periods=1).quantile(.75).signal.to_numpy()
    th = alpha * ((q3 - q1) / 2)

    return th

def _correct_artifacts(artifacts, peaks):

    # Artifact correction
    #####################
    # The integrity of indices must be maintained if peaks are inserted or
    # deleted: for each deleted beat, decrease indices following that beat in
    # all other index lists by 1. Likewise, for each added beat, increment the
    # indices following that beat in all other lists by 1.
    extra_idcs = artifacts["extra"]
    missed_idcs = artifacts["missed"]
    ectopic_idcs = artifacts["ectopic"]
    longshort_idcs = artifacts["longshort"]

    # Delete extra peaks.
    if extra_idcs:
        peaks = _correct_extra(extra_idcs, peaks)
        # Update remaining indices.
        missed_idcs = _update_indices(extra_idcs, missed_idcs, -1)
        ectopic_idcs = _update_indices(extra_idcs, ectopic_idcs, -1)
        longshort_idcs = _update_indices(extra_idcs, longshort_idcs, -1)

    # Add missing peaks.
    if missed_idcs:
        peaks = _correct_missed(missed_idcs, peaks)
        # Update remaining indices.
        ectopic_idcs = _update_indices(missed_idcs, ectopic_idcs, 1)
        longshort_idcs = _update_indices(missed_idcs, longshort_idcs, 1)

    if ectopic_idcs:
        peaks = _correct_misaligned(ectopic_idcs, peaks)

    if longshort_idcs:
        peaks = _correct_misaligned(longshort_idcs, peaks)

    return peaks

def _correct_extra(extra_idcs, peaks):

    corrected_peaks = peaks.copy()
    corrected_peaks = np.delete(corrected_peaks, extra_idcs)

    return corrected_peaks

def _correct_missed(missed_idcs, peaks):

    corrected_peaks = peaks.copy()
    missed_idcs = np.array(missed_idcs)
    # Calculate the position(s) of new beat(s). Make sure to not generate
    # negative indices. prev_peaks and next_peaks must have the same
    # number of elements.
    valid_idcs = np.logical_and(missed_idcs > 1,
                                missed_idcs < len(corrected_peaks))
    missed_idcs = missed_idcs[valid_idcs]
    prev_peaks = corrected_peaks[[i - 1 for i in missed_idcs]]
    next_peaks = corrected_peaks[missed_idcs]
    added_peaks = prev_peaks + (next_peaks - prev_peaks) / 2
    # Add the new peaks before the missed indices (see numpy docs).
    corrected_peaks = np.insert(corrected_peaks, missed_idcs, added_peaks)

    return corrected_peaks

def _correct_misaligned(misaligned_idcs, peaks):

    corrected_peaks = peaks.copy()
    misaligned_idcs = np.array(misaligned_idcs)
    # Make sure to not generate negative indices, or indices that exceed
    # the total number of peaks. prev_peaks and next_peaks must have the
    # same number of elements.
    valid_idcs = np.logical_and(misaligned_idcs > 1,
                                misaligned_idcs < len(corrected_peaks))
    misaligned_idcs = misaligned_idcs[valid_idcs]
    prev_peaks = corrected_peaks[[i - 1 for i in misaligned_idcs]]
    next_peaks = corrected_peaks[[i + 1 for i in misaligned_idcs]]
    half_ibi = (next_peaks - prev_peaks) / 2
    peaks_interp = prev_peaks + half_ibi
    # Shift the R-peaks from the old to the new position.
    corrected_peaks = np.delete(corrected_peaks, misaligned_idcs)
    corrected_peaks = np.concatenate((corrected_peaks,
                                      peaks_interp)).astype(int)
    corrected_peaks.sort(kind="mergesort")

    return corrected_peaks

def _update_indices(source_idcs, update_idcs, update):
    """
    For every element s in source_idcs, change every element u in update_idcs
    according to update, if u is larger than s.
    """
    if not update_idcs:
        return update_idcs

    for s in source_idcs:
        update_idcs = [u + update if u > s else u for u in update_idcs]

    return update_idcs

def _plot_artifacts_lipponen2019(artifacts, info):
    """
    """
    # Extract parameters
    longshort_idcs = artifacts["longshort"]
    ectopic_idcs = artifacts["ectopic"]
    extra_idcs = artifacts["extra"]
    missed_idcs = artifacts["missed"]

    rr = info["rr"]
    drrs = info["drrs"]
    mrrs = info["mrrs"]
    s12 = info["s12"]
    s22 = info["s22"]
    c1 = info["c1"]
    c2 = info["c2"]

    # Visualize artifact type indices.

    # Set grids
    gs = matplotlib.gridspec.GridSpec(ncols=4, nrows=3,
                                      width_ratios=[1, 2, 2, 2])
    fig = plt.figure(constrained_layout=False)
    ax0 = fig.add_subplot(gs[0, :-2])
    ax1 = fig.add_subplot(gs[1, :-2])
    ax2 = fig.add_subplot(gs[2, :-2])
    ax3 = fig.add_subplot(gs[:, -1])
    ax4 = fig.add_subplot(gs[:, -2])

    ax0.set_title("Artifact types", fontweight="bold")
    ax0.plot(rr, label="heart period")
    ax0.scatter(longshort_idcs, rr[longshort_idcs], marker='x', c='m',
                s=100, zorder=3, label="long/short")
    ax0.scatter(ectopic_idcs, rr[ectopic_idcs], marker='x', c='g', s=100,
                zorder=3, label="ectopic")
    ax0.scatter(extra_idcs, rr[extra_idcs], marker='x', c='y', s=100,
                zorder=3, label="false positive")
    ax0.scatter(missed_idcs, rr[missed_idcs], marker='x', c='r', s=100,
                zorder=3, label="false negative")
    ax0.legend(loc="upper right")

    # Visualize first threshold.
    ax1.set_title("Consecutive-difference criterion", fontweight="bold")
    ax1.plot(np.abs(drrs), label="difference consecutive heart periods")
    ax1.axhline(1, c='r', label="artifact threshold")
    ax1.legend(loc="upper right")

    # Visualize second threshold.
    ax2.set_title("Difference-from-median criterion", fontweight="bold")
    ax2.plot(np.abs(mrrs), label="difference from median over 11 periods")
    ax2.axhline(3, c="r", label="artifact threshold")
    ax2.legend(loc="upper right")

    # Visualize subspaces.
    ax4.set_title("Subspace 1", fontweight="bold")
    ax4.set_xlabel("S11")
    ax4.set_ylabel("S12")
    ax4.scatter(drrs, s12, marker="x", label="heart periods")
    verts0 = [(min(drrs), max(s12)),
              (min(drrs), -c1 * min(drrs) + c2),
              (-1, -c1 * -1 + c2),
              (-1, max(s12))]
    poly0 = matplotlib.patches.Polygon(verts0, alpha=0.3, facecolor="r",
                                       edgecolor=None, label="ectopic periods")
    ax4.add_patch(poly0)
    verts1 = [(1, -c1 * 1 - c2),
              (1, min(s12)),
              (max(drrs), min(s12)),
              (max(drrs), -c1 * max(drrs) - c2)]
    poly1 = matplotlib.patches.Polygon(verts1, alpha=0.3, facecolor="r",
                                       edgecolor=None)
    ax4.add_patch(poly1)
    ax4.legend(loc="upper right")

    ax3.set_title("Subspace 2", fontweight="bold")
    ax3.set_xlabel("S21")
    ax3.set_ylabel("S22")
    ax3.scatter(drrs, s22, marker="x", label="heart periods")
    verts2 = [(min(drrs), max(s22)),
              (min(drrs), 1),
              (-1, 1),
              (-1, max(s22))]
    poly2 = matplotlib.patches.Polygon(verts2, alpha=0.3, facecolor="r",
                                       edgecolor=None, label="short periods")
    ax3.add_patch(poly2)
    verts3 = [(1, -1),
              (1, min(s22)),
              (max(drrs), min(s22)),
              (max(drrs), -1)]
    poly3 = matplotlib.patches.Polygon(verts3, alpha=0.3, facecolor="y",
                                       edgecolor=None, label="long periods")
    ax3.add_patch(poly3)
    ax3.legend(loc="upper right")

max_stars_repo_name: TiagoTostas/NeuroKit

max_issues_repo_name: TiagoTostas/NeuroKit

max_forks_repo_name: TiagoTostas/NeuroKit


text:
import os
from collections import Counter
import re
import json
import numpy as np

MAX_TITLE_LEN = 30
MAX_SNI_LEN=100
MAX_VERT_NUM=3
MAX_ENTITY_NUM = 10

def word_tokenize(sent):
    pat = re.compile(r'[\w]+|[.,!?;|]')
    if isinstance(sent, str):
        return pat.findall(sent.lower())
    else:
        return []

def read_news(path,attr,filenames,filer_num=3):
    news={}
    category=[]
    news_index={}
    index=1
    word_cnt=Counter()

    entity_dict = {}
    entity_index = 1

    entity_satorid = {}

    with open(os.path.join(path,filenames)) as f:
        lines=f.readlines()
    for line in lines:
        splited = line.strip('\n').split('\t')
        doc_id,vert,title,snipplet, _, _, _, entity= splited[0:8]
        news_index[doc_id]=index
        index+=1

        if 'vert' in attr:
            vert = [v for v in vert.split(',') if v.startswith('hp1')]
            category.extend(vert)
        else:
            vert = None
        title = title.lower()
        title=word_tokenize(title)
        if 'sni' in attr:
            snipplet = snipplet.lower()
            snipplet=word_tokenize(snipplet)
        else:
            snipplet = []

        entities = []
        if 'entity' in attr:
            entity = json.loads(entity)
            for e in entity:
                label = e['Label']
                label = e['SatoriId']
                if not label in entity_dict:
                    entity_dict[label] = entity_index
                    entity_index += 1
                entities.append(label)
        else:
            entities = None

        news[doc_id]=[vert,title,snipplet,entities]
        word_cnt.update(snipplet+title)

    word = [k for k , v in word_cnt.items() if v > filer_num]
    word_dict = {k:v for k, v in zip(word, range(1,len(word)+1))}
    category=list(set(category))
    category_dict={}
    index=1
    for c in category:
        category_dict[c]=index
        index+=1

    return news,news_index,category_dict,word_dict,entity_dict,

def get_doc_input(attr,news,news_index,category,word_dict,entity_dict):
    news_num=len(news)+1
    news_title=np.zeros((news_num,MAX_TITLE_LEN),dtype='int32')
    if 'sni' in attr:
        news_sni=np.zeros((news_num, MAX_SNI_LEN),dtype='int32')
    else:
        news_sni = None
    if 'vert' in attr:
        news_vert=np.zeros((news_num,MAX_VERT_NUM),dtype='int32')
    else:
        news_vert = None
    if 'entity' in attr:
        news_entity = np.zeros((news_num,MAX_ENTITY_NUM),dtype='int32')
    else:
        news_entity = None

    for key in news:
        vert,title,snipplet,entity=news[key]
        doc_index=news_index[key]
        if 'vert' in attr:
            for vert_id in range(min(MAX_VERT_NUM,len(vert))):
                news_vert[doc_index,vert_id] = category[vert[vert_id]]

        for word_id in range(min(MAX_TITLE_LEN,len(title))):
            if title[word_id] in word_dict:
                news_title[doc_index,word_id]=word_dict[title[word_id].lower()]

        if 'sni' in attr:
            for word_sid in range(min(MAX_SNI_LEN,len(snipplet))):
                if snipplet[word_sid] in word_dict:
                    news_sni[doc_index, word_sid]=word_dict[snipplet[word_sid].lower()]

        if 'entity' in attr:
            for ei in range(min(MAX_ENTITY_NUM,len(entity))):
                e = entity[ei]
                if e in entity_dict:
                    news_entity[doc_index,ei] = entity_dict[e]

    return news_title, news_vert, news_sni, news_entity

def parse_train_user(news_index,root_path,filename):
    with open(os.path.join(root_path,filename)) as f:
        s = f.read()
    user = json.loads(s)

    TrainUsers = {}
    UserId2Index = {}
    UserIndex2Id = {}
    index = 1

    for userid in user:
        UserId2Index[userid] = index
        UserIndex2Id[index] = userid
        index += 1
        TrainUsers[userid] = [[],[]]
        info = user[userid]
        for i in range(len(info)):
            if news_index[info[i][0]] in TrainUsers[userid][0]:
                continue
            TrainUsers[userid][0].append(news_index[info[i][0]])
            TrainUsers[userid][1].append(info[i][1])
        TrainUsers[userid][1] = np.array(TrainUsers[userid][1])
    return TrainUsers,UserId2Index,UserIndex2Id

def parse_train_samples(news_index,Train_UserId2Index,root_path,filename = 'train.json',):
    with open(os.path.join(root_path,filename)) as f:
        s = f.read()
        samples = json.loads(s)

    imp_docids = []
    imp_labels = []
    imp_userindex = []
    imp_time = []

    cnt = 0
    for userid in samples:
        sample = samples[userid]
        uindex = Train_UserId2Index[userid]

        for i in range(len(sample)):
            pid,nid,eventime = sample[i]
            pid = news_index[pid]
            nid = news_index[nid]
            imp_docids.append([pid,nid])
            imp_time.append(eventime)
            imp_userindex.append(uindex)
            imp_labels.append([1,0])
        cnt += 1

    imp_docids = np.array(imp_docids)
    imp_labels = np.array(imp_labels)
    imp_userindex = np.array(imp_userindex,dtype='int32')
    imp_time = np.array(imp_time,dtype='int32')

    return imp_docids, imp_labels, imp_userindex,imp_time

def parse_test_impression(news_index,root_path,filename='test.json',max_clicked_news=50):
    with open(os.path.join(root_path,filename)) as f:
        s = f.read()
        samples = json.loads(s)

    impressions = []
    users = []
    g1 = 0
    g2 = 0
    for sid in range(len(samples)):
        pos,neg,imp_time,click_news,click_time = samples[sid]

        docids = pos+neg
        labels = [1]*len(pos) + [0]*len(neg)
        for i in range(len(docids)):
            if not docids[i] in news_index:
                docids[i] = 0
                g1 += 1
            else:
                docids[i] = news_index[docids[i]]
            g2 += 1

        for i in range(len(click_news)):
            if not click_news[i] in news_index:
                click_news[i] = 0
            else:
                click_news[i] = news_index[click_news[i]]
        if len(click_news)>max_clicked_news:
            click_news = click_news[-max_clicked_news:]
        else:
            click_news = [0]*(max_clicked_news-len(click_news))+click_news
        docids = np.array(docids)
        labels = np.array(labels)
        click_news = np.array(click_news)
        impressions.append([docids,labels,i])
        users.append(click_news)
    users = np.array(users)
    print(g1,g2,g1/g2)
    return impressions,users

def parse_test_impression(news_index,root_path,filename='test.json'):
    with open(os.path.join(root_path,filename)) as f:
        s = f.read()
        samples = json.loads(s)

    impressions = []
    g1 = 0
    g2 = 0
    for sid in range(len(samples)):
        pos,neg,imp_time,uid = samples[sid]

        docids = pos+neg
        labels = [1]*len(pos) + [0]*len(neg)
        for i in range(len(docids)):
            if not docids[i] in news_index:
                docids[i] = 0
                g1 += 1
            else:
                docids[i] = news_index[docids[i]]
            g2 += 1

        docids = np.array(docids)
        labels = np.array(labels)
        impressions.append([docids,labels,uid,imp_time])
    return impressions

def load_matrix(embedding_path,word_dict):
    embedding_matrix = np.zeros((len(word_dict)+1,300))
    have_word=[]
    with open(os.path.join(embedding_path,'glove.840B.300d.txt'),'rb') as f:
        while True:
            l=f.readline()
            if len(l)==0:
                break
            l=l.split()
            word = l[0].decode()
            if word in word_dict:
                index = word_dict[word]
                tp = [float(x) for x in l[1:]]
                embedding_matrix[index]=np.array(tp)
                have_word.append(word)
    return embedding_matrix,have_word

def load_entity_matrix(root_data_path,entity_dict,entity2satorid):

    with open(os.path.join(root_data_path,'entity','entity2id.txt')) as f:
        lines = f.readlines()

    satorid2line = {}

    for i in range(1,len(lines)):
        sid, index = lines[i].strip('\n').split('\t')
        index = int(index)
        satorid2line[sid] = index

    with open(os.path.join(root_data_path,'entity','entity2vec.vec')) as f:
        lines = f.readlines()

    stat = 0
    entity_matrix = np.zeros((len(entity2satorid)+1,100))
    for e in entity2satorid:
        sid = entity2satorid[e]
        if not sid in satorid2line:
            stat += 1
            continue
        eindex = entity_dict[e]
        index = satorid2line[sid]
        line = lines[index]
        line = line.strip('\n').split('\t')[:-1]
        for j in range(len(line)):
            line[j] = float(line[j])
            entity_matrix[eindex,j] = line[j]

    print(stat/len(entity_dict))

    return entity_matrix


text:
# -*- coding: utf-8 -*-
import warnings
warnings.filterwarnings("ignore")

from util_widgets import *
from IPython.display import display
import random
import pandas as pd
import numpy as np
import joblib
import os

from sklearn.pipeline import Pipeline
from sklearn.base import BaseEstimator, TransformerMixin

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.attribute_names].values

class Visual():
    def __init__(self):
        dir_result = './early_prediction_deploy'#结果保存路径
        final_allsave_filename = os.path.join(dir_result, 'all_python_vars.npy')
        result_vars_first = np.load(final_allsave_filename, allow_pickle=True)
        self.sc_params_first = result_vars_first.item()['sc_params']#数据标准化预处理：均值和标注差
        self.clf_first = joblib.load(os.path.join(dir_result,'Simplified_Esb_Clf.model'))

        dir_result = './recent_prediction_deploy'#结果保存路径
        final_allsave_filename = os.path.join(dir_result, 'all_python_vars.npy')
        result_vars_last = np.load(final_allsave_filename, allow_pickle=True)
        self.sc_params_last = result_vars_last.item()['sc_params']#数据标准化预处理：均值和标注差
        self.clf_last = joblib.load(os.path.join(dir_result,'Simplified_Esb_Clf.model'))

        # grid_page_0
        self.grid_p0 = GridspecLayout(10, 2)
        self.p0_left = [
            w_p0_Age,
            w_p0_Morphine_Sulfate, # Y/N
            w_p0_Albumin,
            w_p0_GCS_total,
            w_p0_PH,# Y/N
            w_p0_Cefazolin,
            w_p0_Ca,
            w_p0_Mechanical_Ventilation, # Y/N
            w_p0_Free_Ca,
            w_p0_Kcl, # Y/N
        ]
        self.p0_right = [w_p0_submit, w_p0_result, w_p0_c1, w_p0_c2, w_p0_c3]
        for i, w in enumerate(self.p0_left):
            self.grid_p0[i, 0] = w
        for i, w in enumerate(self.p0_right):
            self.grid_p0[i, 1] = w

        # grid_page_1
        self.grid_p1 = GridspecLayout(10, 2)
        self.p1_left = [
            w_p1_Los_Hospital,
            w_p1_Age,
            w_p1_Fio2,
            w_p1_Albumin,
            w_p1_BUN,
            w_p1_Glucose_Blood,
            w_p1_RDW,
            w_p1_GCS_Total,
            w_p1_HR,
            w_p1_Sum_Diagnosis
        ]
        self.p1_right = [w_p1_submit, w_p1_result, w_p1_c1, w_p1_c2, w_p1_c3]
        for i, w in enumerate(self.p1_left):
            self.grid_p1[i, 0] = w
        for i, w in enumerate(self.p1_right):
            self.grid_p1[i, 1] = w

        # tab:
        self.tab = widgets.Tab()
        self.tab.children=[self.grid_p0, self.grid_p1]
        self.tab.set_title(0, 'Early Prediction')# Early Prediction of ICU Discharge Location of SCI patient
        self.tab.set_title(1, 'Recent Prediction')# Recent Prediction of ICU Discharge Location of SCI patient
        # inference:
        w_p0_submit.on_click(self.p0_inference)
        w_p1_submit.on_click(self.p1_inference)
        # value control(input number)
        ## tab 0:(text)
        w_p0_Age.observe(self.on_p0_Age_change, names='value')
        w_p0_Albumin.observe(self.on_p0_Albumin_change, names='value')
        w_p0_GCS_total.observe(self.on_p0_GCS_total_change, names='value')
        w_p0_PH.observe(self.on_p0_PH_change, names='value')
        w_p0_Ca.observe(self.on_p0_Ca_change, names='value')
        w_p0_Free_Ca.observe(self.on_p0_Free_Ca_change, names='value')
        ## tab 1:(text)
        w_p1_Los_Hospital.observe(self.on_p1_Los_Hospital_change, names='value')
        w_p1_Age.observe(self.on_p1_Age_change, names='value')
        w_p1_Fio2.observe(self.on_p1_Fio2_change, names='value')
        w_p1_Albumin.observe(self.on_p1_Albumin_change, names='value')
        w_p1_BUN.observe(self.on_p1_BUN_change, names='value')
        w_p1_Glucose_Blood.observe(self.on_p1_Glucose_Blood_change, names='value')
        w_p1_RDW.observe(self.on_p1_RDW_change, names='value')
        w_p1_GCS_Total.observe(self.on_p1_GCS_Total_change, names='value')
        w_p1_HR.observe(self.on_p1_HR_change, names='value')
        w_p1_Sum_Diagnosis.observe(self.on_p1_Sum_Diagnosis_change, names='value')

        # lock class value
        w_p0_c1.observe(self.on_p0_c1_change, names='value')
        w_p0_c2.observe(self.on_p0_c2_change, names='value')
        w_p0_c3.observe(self.on_p0_c3_change, names='value')
        w_p1_c1.observe(self.on_p1_c1_change, names='value')
        w_p1_c2.observe(self.on_p1_c2_change, names='value')
        w_p1_c3.observe(self.on_p1_c3_change, names='value')
        # init
        self.p0_c1_value = 0
        self.p0_c2_value = 0
        self.p0_c3_value = 0
        self.p1_c1_value = 0
        self.p1_c2_value = 0
        self.p1_c3_value = 0
        display(self.tab, w_out)

    def p0_inference(self, b):
        w_out.clear_output()
        with w_out:
            X_data = [w.value for w in self.p0_left]
            X_description = [w.description for w in self.p0_left]
        X_data = np.array([X_data])
        features = ['age', 'Morphine.Sulfate', 'Albumin', 'GCS.total', 'PH',
                    'cefazolin', 'Ca', 'Mechanical.ventilation', 'Free.Ca', 'Kcl']
        X_deploy_test = pd.DataFrame(data=X_data, index=[0] ,columns=features)
        with w_out:
            display(X_deploy_test)

        proba = self.predict_proba_discharge_location_first(X_deploy_test)
        self.p0_c1_value = proba[0]
        self.p0_c2_value = proba[1]
        self.p0_c3_value = proba[2]
        w_p0_c1.value = self.p0_c1_value
        w_p0_c2.value = self.p0_c2_value
        w_p0_c3.value = self.p0_c3_value
    def p1_inference(self, b):
        w_out.clear_output()
        with w_out:
            X_data = [w.value for w in self.p1_left]
            X_description = [w.description for w in self.p1_left]
        X_data = np.array([X_data])
        features = ['los_hospital', 'age', 'Fio2', 'Albumin', 'BUN',
                    'Glucose.Blood', 'RDW', 'GCS.total', 'HR', 'sum_diagnosis']
        X_deploy_test = pd.DataFrame(data=X_data, index=[0] ,columns=features)
        with w_out:
            display(X_deploy_test)
        proba = self.predict_proba_discharge_location_last(X_deploy_test)
        self.p1_c1_value = proba[0]
        self.p1_c2_value = proba[1]
        self.p1_c3_value = proba[2]
        w_p1_c1.value = self.p1_c1_value
        w_p1_c2.value = self.p1_c2_value
        w_p1_c3.value = self.p1_c3_value
    # left"
    ## tab 0:
    def on_p0_Age_change(self, change):
        if w_p0_Age.value < 0:
            w_p0_Age.value = 0
        if w_p0_Age.value > 120:
            w_p0_Age.value = 120
    def on_p0_Albumin_change(self, change):
        if w_p0_Albumin.value < 0:
            w_p0_Albumin.value = 0
    def on_p0_GCS_total_change(self, change):
        if w_p0_GCS_total.value < 0:
            w_p0_GCS_total.value = 0
    def on_p0_Albumin_change(self, change):
        if w_p0_Albumin.value < 0:
            w_p0_Albumin.value = 0
    def on_p0_PH_change(self, change):
        if w_p0_PH.value < 0:
            w_p0_PH.value = 0
    def on_p0_Ca_change(self, change):
        if w_p0_Ca.value < 0:
            w_p0_Ca.value = 0
    def on_p0_Free_Ca_change(self, change):
        if w_p0_Free_Ca.value < 0:
            w_p0_Free_Ca.value = 0

    ## tab 1:
    def on_p1_Los_Hospital_change(self, change):
        if w_p1_Los_Hospital.value < 0:
            w_p1_Los_Hospital.value = 0
    def on_p1_Age_change(self, change):
        if w_p1_Age.value < 0:
            w_p1_Age.value = 0
        if w_p1_Age.value > 120:
            w_p1_Age.value = 120
    def on_p1_Fio2_change(self, change):
        if w_p1_Fio2.value < 0:
            w_p1_Fio2.value = 0
    def on_p1_Albumin_change(self, change):
        if w_p1_Albumin.value < 0:
            w_p1_Albumin.value = 0
    def on_p1_BUN_change(self, change):
        if w_p1_BUN.value < 0:
            w_p1_BUN.value = 0
    def on_p1_Glucose_Blood_change(self, change):
        if w_p1_Glucose_Blood.value < 0:
            w_p1_Glucose_Blood.value = 0
    def on_p1_RDW_change(self, change):
        if w_p1_RDW.value < 0:
            w_p1_RDW.value = 0
    def on_p1_GCS_Total_change(self, change):
        if w_p1_GCS_Total.value < 0:
            w_p1_GCS_Total.value = 0
    def on_p1_HR_change(self, change):
        if w_p1_HR.value < 0:
            w_p1_HR.value = 0
    def on_p1_Sum_Diagnosis_change(self, change):
        if w_p1_Sum_Diagnosis.value < 0:
            w_p1_Sum_Diagnosis.value = 0

    # right:
    def on_p0_c1_change(self, change):
        w_p0_c1.value = self.p0_c1_value
    def on_p0_c2_change(self, change):
        w_p0_c2.value = self.p0_c2_value
    def on_p0_c3_change(self, change):
        w_p0_c3.value = self.p0_c3_value
    def on_p1_c1_change(self, change):
        w_p1_c1.value = self.p1_c1_value
    def on_p1_c2_change(self, change):
        w_p1_c2.value = self.p1_c2_value
    def on_p1_c3_change(self, change):
        w_p1_c3.value = self.p1_c3_value

    def predict_proba_discharge_location_first(self,X):
        """问题0的预测"""
        X = (X-self.sc_params_first.loc['mean_'].values)/self.sc_params_first.loc['scale_'].values# standard scale
        proba = self.clf_first.predict_proba(X).squeeze()
        return proba
    def predict_proba_discharge_location_last(self,X):
        """问题1的预测"""
        X = (X-self.sc_params_last.loc['mean_'].values)/self.sc_params_last.loc['scale_'].values# standard scale
        proba = self.clf_last.predict_proba(X).squeeze()
        return proba


text:
import numpy as np
import pytest

from sklearn.neighbors._kd_tree import KDTree

DIMENSION = 3

METRICS = {"euclidean": {}, "manhattan": {}, "chebyshev": {}, "minkowski": dict(p=3)}

def test_array_object_type():
    """Check that we do not accept object dtype array."""
    X = np.array([(1, 2, 3), (2, 5), (5, 5, 1, 2)], dtype=object)
    with pytest.raises(ValueError, match="setting an array element with a sequence"):
        KDTree(X)


text:
from keras.preprocessing import image
from keras.models import model_from_json
import numpy as np
import os

# Module for testing the transfer learning model for car type classification
def load_image(img_path, show=False):
    img = image.load_img(img_path, target_size=(150, 150))
    img_tensor = image.img_to_array(img)                    # (height, width, channels)
    img_tensor = np.expand_dims(img_tensor, axis=0)         # (1, height, width, channels), add a dimension because the model expects this shape: (batch_size, height, width, channels)
    img_tensor /= 255.                                      # imshow expects values in the range [0, 1]
    return img_tensor

# Loading the trained model
json_file = open('model.json','r')
loaded_model = json_file.read()
json_file.close()
model = model_from_json(loaded_model)
model.load_weights('model.h5')

# Compiling the model
model.compile(optimizer='Adam',loss='categorical_crossentropy',metrics=['accuracy'])
c1 = 0
c2 = 0

# Testing the model on the test set of each car type
for filename in os.listdir('cars/images/sedan'):
    img = load_image('cars/images/sedan/' + filename)
    pred = model.predict(img)
    if(pred[0][1] > 0.9):
        c1 += 1
    print("Sedan", pred)

print("Correct",c1)
for filename in os.listdir('cars/images/hatchback'):
    img = load_image('cars/images/hatchback/' + filename)
    pred = model.predict(img)
    if(pred[0][0] > 0.9):
        c2 += 1
    print("Hatchback", pred)
print("Correct", c2)

max_stars_repo_name: Suruchidgupta/ComputerVisionPipeline

max_issues_repo_name: Suruchidgupta/ComputerVisionPipeline

max_forks_repo_name: Suruchidgupta/ComputerVisionPipeline


text:
import numpy as np
import pytest
import tensorflow as tf
import tensorflow_text as text

from tftokenizers.tokenizer import compare_tokenizers

s1 = "sponge bob squarepants is an avenger"
s2 = "Huggingface to Tensorflow tokenizers"
s3 = "HelLo!"
swe = "Pippi Långstrump, en karaktär skapad av Astrid Lindgren"

def test_bert_base_cased():
    # Define out custom tokenizer
    pretrained_model_name = "bert-base-cased"
    out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
    np.testing.assert_array_almost_equal(
        x=out["tf_tokens"]["input_ids"].numpy(),
        y=out["hf_tokens"]["input_ids"].numpy(),
    )

def test_bert_base_multilingual_cased():
    # Define out custom tokenizer
    pretrained_model_name = "bert-base-multilingual-cased"
    out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
    np.testing.assert_array_almost_equal(
        x=out["tf_tokens"]["input_ids"].numpy(),
        y=out["hf_tokens"]["input_ids"].numpy(),
    )

def test_distilbert_base_cased():
    # Define out custom tokenizer
    pretrained_model_name = "distilbert-base-cased"
    out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
    np.testing.assert_array_almost_equal(
        x=out["tf_tokens"]["input_ids"].numpy(),
        y=out["hf_tokens"]["input_ids"].numpy(),
    )

def test_distilbert_base_multilingual_cased():
    # Define out custom tokenizer
    pretrained_model_name = "distilbert-base-multilingual-cased"
    out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
    np.testing.assert_array_almost_equal(
        x=out["tf_tokens"]["input_ids"].numpy(),
        y=out["hf_tokens"]["input_ids"].numpy(),
    )

def test_kb_bert_base_swedish_cased():
    # Define out custom tokenizer
    pretrained_model_name = "KB/bert-base-swedish-cased"
    out = compare_tokenizers(pretrained_model_name, [s1, s2, swe])
    np.testing.assert_array_almost_equal(
        x=out["tf_tokens"]["input_ids"].numpy(),
        y=out["hf_tokens"]["input_ids"].numpy(),
    )

def test_bert_base_chinese():
    # Define out custom tokenizer
    pretrained_model_name = "bert-base-chinese"
    out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
    np.testing.assert_array_almost_equal(
        x=out["tf_tokens"]["input_ids"].numpy(),
        y=out["hf_tokens"]["input_ids"].numpy(),
    )

def test_bert_base_uncased():
    pretrained_model_name = "bert-base-uncased"
    out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
    np.testing.assert_array_almost_equal(
        x=out["tf_tokens"]["input_ids"].numpy(),
        y=out["hf_tokens"]["input_ids"].numpy(),
    )

def test_bert_base_multilingual_uncased():
    pretrained_model_name = "bert-base-multilingual-uncased"
    out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
    np.testing.assert_array_almost_equal(
        x=out["tf_tokens"]["input_ids"].numpy(),
        y=out["hf_tokens"]["input_ids"].numpy(),
    )

def test_minilm_l12_uncased():
    pretrained_model_name = "microsoft/MiniLM-L12-H384-uncased"
    out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
    np.testing.assert_array_almost_equal(
        x=out["tf_tokens"]["input_ids"].numpy(),
        y=out["hf_tokens"]["input_ids"].numpy(),
    )

# def test_alberta_base():
#     pretrained_model_name = "albert-base-v2"
#     out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
#     np.testing.assert_array_almost_equal(
#         x=out["tf_tokens"]["input_ids"].numpy(),
#         y=out["hf_tokens"]["input_ids"].numpy(),
#     )
#
#
# def test_bio_clinical_bert():
#     # Trained on Medical MIMIC dataset
#     pretrained_model_name = "emilyalsentzer/Bio_ClinicalBERT"
#     out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
#     np.testing.assert_array_almost_equal(
#         x=out["tf_tokens"]["input_ids"].numpy(),
#         y=out["hf_tokens"]["input_ids"].numpy(),
#     )
#
#
# def test_bio_pubmed_base_uncased_abstract():
#     pretrained_model_name = "microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"
#     out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
#     np.testing.assert_array_almost_equal(
#         x=out["tf_tokens"]["input_ids"].numpy(),
#         y=out["hf_tokens"]["input_ids"].numpy(),
#     )
#
#
# def test_all_minilm_l12():
#     pretrained_model_name = "sentence-transformers/all-MiniLM-L12-v2"
#     out = compare_tokenizers(pretrained_model_name, [s1, s2, s3])
#     np.testing.assert_array_almost_equal(
#         x=out["tf_tokens"]["input_ids"].numpy(),
#         y=out["hf_tokens"]["input_ids"].numpy(),
#     )

max_stars_repo_name: Hugging-Face-Supporter/TFTokenizers

max_issues_repo_name: Hugging-Face-Supporter/TFTokenizers

max_forks_repo_name: Hugging-Face-Supporter/tftokenizers


text:
import tensorflow as tf
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix

# Load pretrained model
model = tf.keras.models.load_model('CNN/CNN.model')

# Show the model archtecture
model.summary()

# Load dataset
(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()

# Test
test_images = test_images.reshape(test_images.shape[0], 28, 28, 1)
predict_result = model.predict(test_images)
predict_labels = [predict_result[i].argmax() for i in range(predict_result.shape[0])]
mat = confusion_matrix(test_labels, predict_labels)
sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False)
plt.xlabel('true label')
plt.ylabel('predicted label')
plt.show()

max_stars_repo_name: RedMakeUp/DataMiningCourseDesign

max_issues_repo_name: RedMakeUp/DataMiningCourseDesign

max_forks_repo_name: RedMakeUp/DataMiningCourseDesign


text:
import os
import sys
import pyBigWig
import numpy as np

# 30mins per assay

chr_all=['chr1','chr2','chr3','chr4','chr5','chr6','chr7','chr8','chr9','chr10','chr11','chr12','chr13','chr14','chr15','chr16','chr17','chr18','chr19','chr20','chr21','chr22','chrX']
num_bp=[248956422,242193529,198295559,190214555,181538259,170805979,159345973,145138636,138394717,133797422,135086622,133275309,114364328,107043718,101991189,90338345,83257441,80373285,58617616,64444167,46709983,50818468,156040895]
num_25bp=[9958257, 9687742, 7931823, 7608583, 7261531, 6832240, 6373839, 5805546, 5535789, 5351897, 5403465, 5331013, 4574574, 4281749, 4079648, 3613534, 3330298, 3214932, 2344705, 2577767, 1868400, 2032739, 6241636]

chr_len={}
for i in np.arange(len(chr_all)):
    chr_len[chr_all[i]]=num_bp[i]

chr_len25={}
for i in np.arange(len(chr_all)):
    chr_len25[chr_all[i]]=int(np.ceil(num_bp[i]/25.0))

path1='./orange_rank/'
path2='./signal_anchored_final/'
os.system('mkdir -p ' + path2)

assay_all=['M01','M02','M16','M17','M18','M20','M22','M29']

for the_assay in assay_all:
    for i in np.arange(1,52):
        the_id='C' + '%02d' % i + the_assay
        if os.path.isfile(path1 + the_id + '.txt'):
            print(the_id)
            bw = pyBigWig.open(path2 + 'orange_'+ the_id + '.bigwig', 'w')
            bw.addHeader(list(zip(chr_all , num_25bp)), maxZooms=0) # zip two turples
            orange=np.loadtxt(path1 + the_id + '.txt')
            start_txt=0
            for the_chr in chr_all:
                print(the_chr)
                end_txt = chr_len25[the_chr] + start_txt
                x=orange[start_txt:end_txt]
                # pad two zeroes
                z=np.concatenate(([0],x,[0]))
                # find boundary
                starts=np.where(np.diff(z)!=0)[0]
                ends=starts[1:]
                starts=starts[:-1]
                vals=x[starts]
                if starts[0]!=0:
                    ends=np.concatenate(([starts[0]],ends))
                    starts=np.concatenate(([0],starts))
                    vals=np.concatenate(([0],vals))
                if ends[-1]!=chr_len25[the_chr]:
                    starts=np.concatenate((starts,[ends[-1]]))
                    ends=np.concatenate((ends,[chr_len25[the_chr]]))
                    vals=np.concatenate((vals,[0]))
#                if ends[-1]!=chr_len[the_chr]:
#                    starts=np.concatenate((starts,[ends[-1]]))
#                    ends=np.concatenate((ends,[chr_len[the_chr]]))
#                    vals=np.concatenate((vals,[0]))
                # write
                chroms = np.array([the_chr] * len(vals))
                bw.addEntries(chroms, starts, ends=ends, values=vals)
                start_txt = end_txt
            bw.close()