text:
# classes to for the overall population structure

import numpy as np
from sklearn.model_selection import train_test_split
from .functions import chain

class CladeBase:

    def __init__(self, X=None, y=None, model=None, initial_descendants=[],
                 n_max=16, n_decay_factor=.5, max_daughters=2,
                 train_fraction=.666667, oob_power=2., **kwargs):
        self._descs = initial_descendants
        self.gen = 0
        self._n_max = n_max # living individuals
        self._n_decay_factor = n_decay_factor # how much to reduce subsequent n_max by
        self._X = X
        self._y = y
        self._model = model  # only to be used to build individuals
        self._oob = None
        self._train_fraction = train_fraction
        self._oob_power = oob_power

    @property
    def alive(self):
        return (self.size > 0)

    @property
    def n_max(self):
        return self._n_max

    @n_max.setter
    def n_max(self, n_max):
        self._n_max = n_max

    @property
    def n_decay_factor(self):
        return self._n_decay_factor

    @n_decay_factor.setter
    def n_decay_factor(self, n_decay_factor):
        self._n_decay_factor = n_decay_factor

    @property
    def X(self):
        return self._X

    @X.setter
    def X(self, X):
        self._X = X

    @property
    def y(self):
        return self._y

    @y.setter
    def y(self, y):
        self._y = y

    @property
    def model(self):
        return self._model

    @model.setter
    def model(self, model):
        self._model = model

    @property
    def descs(self):
        descs = np.array([d for d in self._descs if d.alive])
        if len(descs) > 0:
            sizes = np.array([d.size for d in descs])
            key = np.repeat(np.max(sizes), sizes.shape[0]) - sizes
            descs = descs[np.argsort(key)]
        self._descs = descs
        yield from self._descs

    def _size_descs(self):
        total = 0
        sizes = (d.size for d in self.descs)
        for size in sizes:
            total += size
        return total

    def _len_descs(self):
        total = 0
        for d in self.descs:
            total += 1
        return total

    @property
    def size(self):
        return self._size_descs()

    def part(self, iter, n=2):
        part = [list() for _ in range(n)]
        # # print("part: {}".format(part))
        tally = np.repeat(0, n)
        # # print("tally: {}".format(tally))
        for item in iter:
            # # print("item: {}".format(item))
            # # print("np.argmin(tally): {}".format(np.argmin(tally)))
            part[np.argmin(tally)].append(item)
            tally[np.argmin(tally)] += item.size
            # # print("tally: {}".format(tally))
        return part

    def compete(self, iter, loser=False):
        iter = np.random.choice(iter, len(iter), replace=False).tolist()
        teams = self.part(iter, n=2)
        try:
            blue_team = teams[1]
        except:  # TODO: catch specific exception.
            return teams
        red_team = teams[0]
        red_error = np.concatenate([i.oob for i in red_team], axis=0)
        blue_error = np.concatenate([i.oob for i in blue_team], axis=0)
        red_error = np.nanmean(np.power(red_error, 2.), axis=0)
        blue_error = np.nanmean(np.power(blue_error, 2.), axis=0)
        diffs = red_error - blue_error

        if not np.all(np.isnan(diffs)):
            score = np.nanmean(diffs)
        else:
            score = np.random.random() - .5

        if score < 0:
            teams = teams[::-1]

        return teams

    def _champ(self, loser=False):
        # stable = list(self.descs)
        stable = list(self.clade)
        if len(stable) > 0:
            while len(stable) > 1:
                # arena = None # TODO:
                # arena sorts by shared mse
                arena = self.compete(stable)
                # # print(arena)
                if loser is True:
                    stable = arena[0]
                else:
                    stable = arena[-1]
            champ = stable[0]
        else:
            champ = None
        return champ

    def champ(self, loser=False):
        return self._champ(loser=loser)

    def highlander(self):
        return self.champ(loser=False).highlander()

    def baggage(self):
        return self.champ(loser=True).baggage()

    @property
    def fertile(self):
        for desc in self.descs:
            if desc.fertile:
                return true
        return false

    def kill(self):
        self.baggage().kill()

    @property
    def clade(self):
        for d in self.descs:
            for i in d.clade:
                yield i

    def branch(self):
        pass

    def collapse(self):
        pass

    def simplify(self):
        pass

class Clade(CladeBase):

    def __init__(self, X=None, y=None, model=None, initial_descendants=[], n=8,
                 max_daughters=2,
                 train_fraction=.666667, oob_power=2., **kwargs):
        super().__init__(**kwargs)
        self._descs = initial_descendants
        self.gen = 0
        self.n_max = n # living individuals
        self.len_max = max_daughters # max len descs
        self._X = X
        self._y = y
        self._model = model  # only to be used to build individuals
        self._oob = None
        self._train_fraction = train_fraction
        self._oob_power = oob_power

    @property
    def _mask(self):
        return [desc._mask for desc in self.descs]

    @property
    def _idx(self):
        return [desc._idx for desc in self.descs]

    @property
    def len_max(self):
        return self._len_max

    @len_max.setter
    def len_max(self, len_max):
        assert(isinstance(len_max, int))
        assert(len_max > 1)
        self._len_max = len_max

    def _branch_descs(self):
        for desc in self.descs:
            desc.branch()

    def _breed_descs(self, X, y, refit=False):
        for desc in self.descs:
            desc.breed(X, y, refit=refit)

    def breed(self, X, y, refit=False):
        parent = self.highlander()
        genes = parent.genotype
        pool = self.genotype
        values = pool.T[
            range(pool.shape[1]),
            np.random.choice(pool.shape[0], pool.shape[1])
            ].reshape(1,-1)
        alpha = np.random.random()
        # alpha = np.max((alpha, .1))
        # alpha = np.min((alpha, .9))
        alpha = .25 * alpha
        alpha = .5 + alpha
        mask = np.random.random((1, genes.shape[1])) < alpha
        print(f"alpha: {alpha}")
        print(f"genes.shape: {genes.shape}")
        print(f"mask.shape: {mask.shape}")
        print(f"values.shape: {values.shape}")
        np.putmask(genes, mask, values)
        offspring = Individual(genes=genes.reshape((-1,)), model=parent._model.__class__(),
                               expression_function=parent._expression_function,
                               X=X, y=y,
                               train_fraction=parent._train_fraction,
                               oob_power=parent._oob_power
                               )
        offspring.mutate()
        offspring.fit(X, y)
        self._descs = np.array(list(self.descs) + [offspring])
        if refit is True:
            pass
        self._breed_descs(X, y)

    def fit(self, X, y, sample_weight=None, check_input=True, X_idx_sorted=None, replace=True):
        self._X = X
        self._y = y
        for desc in self.descs:
            desc.fit(X, y, sample_weight, check_input, X_idx_sorted, replace=replace)

    def predict(self, X, check_input=True):
        stack = []
        concat = []
        for desc in self.descs:
            prediction = desc.predict(X, check_input=True)
            if prediction is not None:
                if len(prediction.shape) == 1:
                    stack.append(prediction)
                elif len(prediction.shape) == 2:
                    concat.append(prediction)
                else:
                    pass
        predictions = []
        try:
            predictions.append(np.stack(stack, axis=0))
        except:
            pass
        try:
            predictions.append(np.concatenate(concat, axis=0))
        except:
            pass

        if len(predictions) > 0:
            return np.concatenate(predictions, axis=0)

    def predict_proba(self, X, check_input=True):
        stack = []
        concat = []
        for desc in self.descs:
            prediction = desc.predict_proba(X, check_input=True)
            if prediction is not None:
                if len(prediction.shape) == 2:
                    stack.append(prediction)
                elif len(prediction.shape) == 3:
                    concat.append(prediction)
                else:
                    pass
        predictions = []
        try:
            predictions.append(np.stack(stack, axis=0))
        except:
            pass
        try:
            predictions.append(np.concatenate(concat, axis=0))
        except:
            pass

        if len(predictions) > 0:
            return np.concatenate(predictions, axis=0)

    @property
    def oob(self):
        stack = []
        concat = []
        for desc in self.descs:
            oob = desc.oob
            if oob is not None:
                if len(oob.shape) == 1:
                    stack.append(oob)
                elif len(oob.shape) == 2:
                    concat.append(oob)
                else:
                    pass
        oobs = []
        try:
            oobs.append(np.stack(stack, axis=0))
        except: # TODO: catch specific exception
            pass
        try:
            oobs.append(np.concatenate(concat, axis=0))
        except: # TODO: catch specific exception
            pass
        if len(oobs) > 0:
            return np.concatenate(oobs, axis=0)
        else:
            return None

    @property
    def error(self):
        stack = []
        concat = []
        for desc in self.descs:
            error = desc.error
            if error is not None:
                if len(error.shape) == 1:
                    stack.append(error)
                elif len(error.shape) == 2:
                    concat.append(error)
                else:
                    pass
        errors = []
        try:
            errors.append(np.stack(stack, axis=0))
        except: # TODO: catch specific exception
            pass
        try:
            errors.append(np.concatenate(concat, axis=0))
        except: # TODO: catch specific exception
            pass
        if len(errors) > 0:
            return np.concatenate(errors, axis=0)
        else:
            return None

    @property
    def genotype(self):
        stack = []
        concat = []
        for desc in self.descs:
            genotype = desc.genotype
            if genotype is not None:
                if len(genotype.shape) == 1:
                    stack.append(genotype)
                elif len(genotype.shape) == 2:
                    concat.append(genotype)
                else:
                    pass
        genotypes = []
        try:
            genotypes.append(np.stack(stack, axis=0))
        except: # TODO: catch specific exception
            pass
        try:
            genotypes.append(np.concatenate(concat, axis=0))
        except: # TODO: catch specific exception
            pass
        if len(genotypes) > 0:
            return np.concatenate(genotypes, axis=0)
        else:
            return None

    def branch(self):
        n = np.ceil(self._len_descs() / self.len_max)
        n = np.min([n, self.len_max])
        n = int(n)
        if n > 1:
            part_descs = self.part(self.descs)
            clades = []
            for part_desc in part_descs:
                clades.append(Clade(X=self.X, y=self.y, model=self.model,
                              n_max=self.n_max * self.n_decay_factor,
                              initial_descendants=part_desc,
                              n_decay_factor=self.n_decay_factor,
                              max_daughters=self._len_max))
            self._descs = clades
        self._branch_descs()

    def collapse(self):
        """
        want to remove clades with single descendants
        """
        if self._len_descs() == 1:
            if list(self.descs)[0]._len_descs() > 0:  # it is not an Individual
                desc = list(self.descs)[0]  # the descendant object
                self._descs = list(desc.descs)

        for desc in self.descs:
            desc.collapse()
            if desc._len_descs() == 1:
                self._descs = list(self.descs) + list(desc.descs)
                desc._descs = [] # this kills the clade

    def simplify(self):
        """
        want to combine descs if len_max allows

        https://stackoverflow.com/
            questions/43313531/python-merging-list-elements-with-a-condition-in-a-bit-tricky-way
        def join_while_too_short(it, length):
            it = iter(it)
            while True:
                current = next(it)
                while len(current) < length:
                    current += ' ' + next(it)
                yield current
        """
        tidy = []
        untidy = []
        for desc in self.descs:
            desc.simplify()
            if desc._len_descs() < desc.len_max:
                untidy.append(desc)
            else:
                tidy.append(desc)
        if len(untidy) == 1:
            tidy = tidy + untidy
            untidy = []
        elif len(untidy) > 1:
            lens = [d._len_descs() for d in untidy]
            key = np.argsort(lens)
            key = key.max() - key
            untidy = np.array(untidy)[key]
            lens = np.array(lens)[key]
            for i, desc in enumerate(untidy):
                idx = np.arange(0, untidy.shape[0])
                idx_i = idx[idx != i]
                lens_i = lens[idx_i]
                untidy_i = untidy[idx_i]
                while (len(lens_i) > 0) and (np.min(lens_i) + desc._len_descs() < desc.len_max):
                    gap = desc.len_max - desc._len_descs()
                    idx_transfer = np.argmax(lens_i[lens_i <= gap])
                    # idx_transfer = np.argmin(lens_i)
                    problem = untidy_i[idx_transfer]
                    desc._descs = list(desc.descs) + list(problem.descs)
                    problem._descs = []
                    untidy_i = np.delete(untidy_i, idx_transfer)
                    lens_i = np.delete(lens_i, idx_transfer)

class Individual(CladeBase):

    def __init__(self, genes, model, expression_function, X=None, y=None,
                 train_fraction=.666667, oob_power=2., **kwargs):
        super().__init__(**kwargs)
        self.gen = 0
        self.n_max = 0 # allow many offspring...
        self._n_decay_factor = .0 # how much to reduce subsequent n_max by
        self.len_max = 0 # ...but move them up to the parent clade.
        self._descs = []
        self._X = X
        self._y = y
        self._model = model
        self._oob = None
        self._oob_score = None
        self._oob_truth = None
        self._error = None
        self._error_score = None
        self._error_truth = None
        self._row_mask = None
        self._idx = None
        self._alive = True
        self._fertile = False
        self._train_fraction = train_fraction
        self._oob_power = oob_power
        self._genes = genes
        self._expression_function = expression_function

    def mutate(self):
        genes = self._genes
        a = np.random.random()
        i = np.floor(a * np.prod(genes.shape)).astype(int)
        genes[i] = (1 - genes[i]).astype(int)
        self._genes = genes

    @property
    def fertile(self):
        fertile = True
        try:
            check = self._model.feature_importances_
        except:  # TODO: catch NotFittedError
            fertile = False
        return fertile

    @property
    def phenotype(self):
        return self._expression_function(self._genes)

    @property
    def genotype(self):
        return self._genes.reshape((1,-1))

    @genotype.setter
    def genotype(self, genes):
        assert(len(genes.shape) == 1)
        self._genes = genes

    def breed(self, X=None, y=None, refit=None):
        pass

    @property
    def train_fraction(self):
        return self._train_fraction

    @train_fraction.setter
    def train_fraction(self, train_fraction):
        assert(isinstance(train_fraction, float))
        assert(train_fraction > 0.)
        self._train_fraction = train_fraction

    @property
    def len_max(self):
        return self._len_max

    @len_max.setter
    def len_max(self, len_max):
        assert(isinstance(len_max, int))
        assert(len_max >= 0)
        self._len_max = len_max

    @property
    def size(self):
        return int(self.alive)

    @property
    def alive(self):
        return self._alive

    @property
    def clade(self):
        return [self]

    # TODO: error should be in-bag error
    @property
    def error(self):
        if self._error is None:
            return None
        else:
            mask = self._row_mask
            error = np.repeat(np.nan, mask.shape[0])
            idx = np.where(mask)[0]
            np.put(error, idx, self._error_score)
            return error.reshape((1,-1))

    @property
    def oob(self):
        if self._oob is None:
            return None
        else:
            mask = np.logical_not(self._row_mask)
            oob = np.repeat(np.nan, mask.shape[0])
            idx = np.where(mask)[0]
            np.put(oob, idx, self._oob_score)
            return oob.reshape((1,-1))

    def champ(self, **kwargs):
        return self

    def highlander(self, **kwargs):
        return self

    def baggage(self, **kwargs):
        return self

    def kill(self):
        self._alive = False

    def fit(self, X, y, sample_weight=None, check_input=True,
            X_idx_sorted=None, replace=True):
        n = y.shape[0]
        idx_train, idx_test = train_test_split(
            np.arange(y.shape[0]),
            train_size=self.train_fraction,
            shuffle=True
            )
        self._idx_train = idx_train
        self._idx_test = idx_test
        row_mask = np.repeat(False, n)
        np.put(row_mask, np.unique(idx_train), True)
        self._row_mask = row_mask  # mask for rows, not cols
        self._X = X
        self._y = y
        self._model.set_params(**self.phenotype["hyps"])
        # TODO: implement bootstrap of self._row_mask before fitting
        self._model.fit(
            self._X[self._row_mask,:][:,self.phenotype["mask"]],
            self._y[self._row_mask],
            sample_weight,
            check_input,
            X_idx_sorted
        )
        self._error_score = self.predict_proba(self._X[self._row_mask,:])[:,1]
        self._error_truth = self._y[self._row_mask]
        self._error = self._error_score - self._error_truth
        self._oob_score = self.predict_proba(self._X[np.logical_not(self._row_mask),:])[:,1]
        self._oob_truth = self._y[np.logical_not(self._row_mask)]
        self._oob = self._oob_score - self._oob_truth

    def predict(self, X, check_input=True):
        predictions = self._model.predict(X[:,self.phenotype["mask"]], check_input=True)
        return predictions

    def predict_proba(self, X, check_input=True):
        predictions = self._model.predict_proba(X[:,self.phenotype["mask"]], check_input=True)
        return predictions


text:
"""
The experiment MAIN for Communities and Crime.
 * Run the file and the CC experiments will complete
 * See compas experiment file for more details on how to read results.
"""
import warnings
warnings.filterwarnings('ignore')

from adversarial_models import *
from utils import *
from get_data import *

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

import numpy as np
import pandas as pd

import lime
import lime.lime_tabular
import shap

from copy import deepcopy

# Set up experiment parameters
params = Params("model_configurations/experiment_params.json")
np.random.seed(params.seed)
X, y, cols = get_and_preprocess_cc(params)

# add unrelated columns, setup
X['unrelated_column_one'] = np.random.choice([0,1],size=X.shape[0])
X['unrelated_column_two'] = np.random.choice([0,1],size=X.shape[0])
features = [c for c in X]

race_indc = features.index('racePctWhite numeric')
unrelated_indcs = features.index('unrelated_column_one')
unrelated_indcs1 = features.index('unrelated_column_two')

X = X.values

###
## The models f and psi for CC.  We discriminate based on race for f and concider two RANDOMLY DRAWN features to display in psi
#

# the biased model
class racist_model_f:
    # Decision rule: classify negatively if race is black
    def predict(self,X):
        return np.array([params.negative_outcome if x[race_indc] > 0 else params.positive_outcome for x in X])

    def predict_proba(self, X):
        return one_hot_encode(self.predict(X))

    def score(self, X,y):
        return np.sum(self.predict(X)==y) / len(X)

# the display model with one unrelated feature
class innocuous_model_psi:
    # Decision rule: classify according to randomly drawn column 'unrelated column'
    def predict_proba(self, X):
        return one_hot_encode(np.array([params.negative_outcome if x[unrelated_indcs] > 0 else params.positive_outcome for x in X]))

# the display model with two unrelated features
class innocuous_model_psi_two:
	def predict_proba(self, X):
		# Using 0.5 to make it easier to detect decision boundary on perturbation
		A = np.where(X[:,unrelated_indcs] > .5, params.positive_outcome, params.negative_outcome)
		B = np.where(X[:,unrelated_indcs1] < -.5, params.positive_outcome, params.negative_outcome)
		preds = np.logical_xor(A, B).astype(int)
		return one_hot_encode(preds)
#
##
###

def experiment_main():
	"""
	Run through experiments for LIME/SHAP on CC using both one and two unrelated features.
	* This may take some time given that we iterate through every point in the test set
	* We print out the rate at which features occur in the top three features
	"""

	xtrain,xtest,ytrain,ytest = train_test_split(X,y,test_size=0.1)
	ss = StandardScaler().fit(xtrain)
	xtrain = ss.transform(xtrain)
	xtest = ss.transform(xtest)

	print ('---------------------')
	print ("Beginning LIME CC Experiments....")
	print ("(These take some time to run because we have to generate explanations for every point in the test set) ")
	print ('---------------------')

	# Train the adversarial model for LIME with f and psi
	adv_lime = Adversarial_Lime_Model(racist_model_f(), innocuous_model_psi()).train(xtrain, ytrain, categorical_features=[features.index('unrelated_column_one'),features.index('unrelated_column_two')], feature_names=features, perturbation_multiplier=30)
	adv_explainer = lime.lime_tabular.LimeTabularExplainer(xtrain, feature_names=adv_lime.get_column_names(), discretize_continuous=False, categorical_features=[features.index('unrelated_column_one'),features.index('unrelated_column_two')])

	explanations = []
	for i in range(xtest.shape[0]):
		explanations.append(adv_explainer.explain_instance(xtest[i], adv_lime.predict_proba).as_list())

	# Display Results
	print ("LIME Ranks and Pct Occurances (1 corresponds to most important feature) for one unrelated feature:")
	print (experiment_summary(explanations, features))
	print ("Fidelity:", round(adv_lime.fidelity(xtest),2))

	# Repeat the same thing for two features
	adv_lime = Adversarial_Lime_Model(racist_model_f(), innocuous_model_psi_two()).train(xtrain, ytrain, feature_names=features, perturbation_multiplier=30, categorical_features=[features.index('unrelated_column_one'),features.index('unrelated_column_two')])
	adv_explainer = lime.lime_tabular.LimeTabularExplainer(xtrain, feature_names=adv_lime.get_column_names(), discretize_continuous=False, categorical_features=[features.index('unrelated_column_one'),features.index('unrelated_column_two')])

	explanations = []
	for i in range(xtest.shape[0]):
		explanations.append(adv_explainer.explain_instance(xtest[i], adv_lime.predict_proba).as_list())

	print ("LIME Ranks and Pct Occurances two unrelated features:")
	print (experiment_summary(explanations, features))
	print ("Fidelity:", round(adv_lime.fidelity(xtest),2))
	print ('---------------------')
	print ('Beginning SHAP CC Experiments....')
	print ('---------------------')

	#Setup SHAP
	background_distribution = shap.kmeans(xtrain,10)
	adv_shap = Adversarial_Kernel_SHAP_Model(racist_model_f(), innocuous_model_psi()).train(xtrain, ytrain, feature_names=features)
	adv_kerenel_explainer = shap.KernelExplainer(adv_shap.predict, background_distribution)
	explanations = adv_kerenel_explainer.shap_values(xtest)

	# format for display
	formatted_explanations = []
	for exp in explanations:
		formatted_explanations.append([(features[i], exp[i]) for i in range(len(exp))])

	print ("SHAP Ranks and Pct Occurances one unrelated features:")
	print (experiment_summary(formatted_explanations, features))
	print ("Fidelity:",round(adv_shap.fidelity(xtest),2))

	background_distribution = shap.kmeans(xtrain,10)
	adv_shap = Adversarial_Kernel_SHAP_Model(racist_model_f(), innocuous_model_psi_two()).train(xtrain, ytrain, feature_names=features)
	adv_kerenel_explainer = shap.KernelExplainer(adv_shap.predict, background_distribution)
	explanations = adv_kerenel_explainer.shap_values(xtest)

	# format for display
	formatted_explanations = []
	for exp in explanations:
		formatted_explanations.append([(features[i], exp[i]) for i in range(len(exp))])

	print ("SHAP Ranks and Pct Occurances two unrelated features:")
	print (experiment_summary(formatted_explanations, features))
	print ("Fidelity:",round(adv_shap.fidelity(xtest),2))
	print ('---------------------')

if __name__ == "__main__":
	experiment_main()

ext: py

max_stars_repo_name: laurensalvarez/Fooling-LIME-SHAP

max_issues_repo_name: laurensalvarez/Fooling-LIME-SHAP

max_forks_repo_name: laurensalvarez/Fooling-LIME-SHAP

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
@author: AmauryP

Fonctions utiles au traitement de base des fichiers json messenger :
    - Lister l'ensemble des fichiers json d'une conversation
    - Différentes fonctions de nettoyage des messages (enlever caractères spéciaux, convertir les accents, enlever la ponctuation, etc)
    - Lister les participants d'une conversation
    - Transformation des json en un csv exploitable
"""

#==============================================================================================
# Modules/Packages à importer =================================================================
#==============================================================================================

import glob # pour lister les fichiers json d'un dossier en particulier
import json # pour gérer les fichiers json
import re
import pandas as pd # pour gérer des dataframes
import pathlib # pour gérer les chemins
from functools import partial
import numpy as np
from datetime import datetime # pour traduire timestamp en date, très très chiant

if __name__ == '__main__':
    # packages perso
    import base_json

#==============================================================================================
# Création du dataframe =======================================================================
#==============================================================================================

if __name__ == '__main__':
    dossier_convers = input("Entrer le nom du dossier contenant les fichiers json voulus : ")
    print('Vous avez écrit : ', dossier_convers)
    path_convers = pathlib.Path(__file__)
    path_convers = path_convers.parent.parent.parent
    path_convers = pathlib.Path(path_convers, 'input')
    path_convers = pathlib.Path(path_convers, dossier_convers)
    print(path_convers)

    # on liste les fichiers json
    list_files_json = base_json.list_files_json(path_convers)

#==============================================================================================
# Fonctions ===================================================================================
#==============================================================================================

def nettoyer_carac_speciaux(chaine_a_nettoyer):
    # à modifier plus simple de faire :
    motif = re.compile('[^a-zA-z0-9œŒæÆàÀâÂäÄéÉèÈëËêÊöÖôÔïÏî ÎüÜùÙûÛçÇ%€«»";:\(\)\'\-\!\?\.]')
    message_clean = motif.sub("", chaine_a_nettoyer)
    motif_bis = re.compile('[\\\\_\[\]\^]')
    message_clean = motif_bis.sub("", message_clean)
    return(message_clean)

if __name__ == '__main__':
    message_test = 'Ç\'est bien.;; salut où es-tu l\'ami "" «lol»(10% ?) - _ [^^\\^^] œŒæÆàÀâÂäÄéÉèÈëËêÊöÖôÔïÏîÎüÜùÙûÛçÇ%€ \uf0e8'
    print(message_test)
    message_clean = nettoyer_carac_speciaux(message_test)
    print(message_clean)

def nettoyer_ponctuation(chaine_a_nettoyer):
    # caracteres très spéciaux
    # motif = re.compile("[«»;()\"\:\!\?\.]")
    # motif = re.compile("[^a-zA-z0-9 œŒæÆàÀâÂäÄéÉèÈëËêÊöÖôÔïÏîÎüÜùÙûÛçÇ']")
    # motif = re.compile("[^a-zA-z0-9]")
    motif = re.compile('[^a-zA-Z0-9œŒæÆàÀâÂäÄéÉèÈëËêÊöÖôÔïÏîÎüÜùÙûÛçÇ\'\- ]')
    message_clean = motif.sub("",chaine_a_nettoyer)
    return(message_clean)

if __name__ == '__main__':
    message_test = 'Ç\'est bien.;; salut où es-tu l\'ami "" «lol»(10% ?) - _ [^^\\^^] œŒæÆàÀâÂäÄéÉèÈëËêÊöÖôÔïÏîÎüÜùÙûÛçÇ%€ \uf0e8'
    print(message_test)
    message_clean = nettoyer_ponctuation(message_test)
    print(message_clean)

def json_to_csv(list_files_json, path_csv_output, other_participant='autre'):

    ############################################
    ######## Créer le dataframe initial ########
    ############################################

    # creation d'un dataframe qui aura nom des gens, messages et date/heure du message, vide pour l'instant
    messenger_dataframe=pd.DataFrame(columns=['participants',
                                            'message_brut',
                                            'message_clean',
                                            'message_clean_lower_ponctuation',
                                            'type',
                                            'timestamp_ms'])

    # fonction de substitution de caractère
    fix_mojibake_escapes = partial(
        re.compile(rb'\\u00([\da-f]{2})').sub,
        lambda m: bytes.fromhex(m.group(1).decode()))

    # extraction des données en dataframe
    for file in list_files_json:
        print(file)
        with open(file, 'rb') as binary_data:
            repaired = fix_mojibake_escapes(binary_data.read())
        data_dict = json.loads(repaired.decode('utf8'))

        # on extrait du dictionnaire général la liste des messages : {'messages':[liste de 10 000 messages]}
        # c'est une liste de dictionnaire, un message est un dictionnaire
        # les keys intéressantes : sender_name, timestamp_ms, content, type, reactions, photos, gifs, videos, share, files, audio_files, users, sticker

        #recupération des données pour chaque message
        list_messages=data_dict.get('messages')
        for message in list_messages:
            # type(message) => dictionnaire, exemple {'sender_name':'Diem Ngo','content':'coucou','timestamp_ms'='1564465468486'}
            # recup du pseudo
            sender_name = str(message.get('sender_name'))
            sender_name = nettoyer_carac_speciaux(sender_name)
            if sender_name == 'Utilisateur Facebook':
                sender_name = other_participant
            # recup du timestamp
            timestamp_ms=int(message.get('timestamp_ms'))
            if 'content' in message:
                content = message.get('content')
                type_message = message.get('type')
            elif 'photos' in message:
                content=""
                type_message="photos"
            elif 'videos' in message:
                content=""
                type_message='videos'
            elif 'gifs' in message:
                content=""
                type_message='gifs'
            elif 'audio_files' in message:
                content=""
                type_message='audio_files'
            elif 'files' in message:
                content=""
                type_message='files'
            elif 'sticker' in message:
                content=""
                type_message='sticker'
            else:
                content=""
                type_message='anormal'

            # content = texte brut
            # content_clean = sans les caractères spéciaux de merde
            content_clean = nettoyer_carac_speciaux(content)
            # content_clean_lower = on enlève la ponctuation et les majuscule
            content_clean_lower_ponctuation = nettoyer_ponctuation(content_clean).lower()

            # on met les info dans la dataframe junkies
            messenger_dataframe = messenger_dataframe.append({'participants':sender_name,
                                                          'message_brut':content,
                                                          'message_clean':content_clean,
                                                          'message_clean_lower_ponctuation':content_clean_lower_ponctuation,
                                                          'type':type_message,
                                                          'timestamp_ms':timestamp_ms},
                                                         ignore_index=True)

    # dtypes gestion
    messenger_dataframe['participants']= messenger_dataframe['participants'].apply(str)
    messenger_dataframe['message_brut']= messenger_dataframe['message_brut'].apply(str)
    messenger_dataframe['message_clean']= messenger_dataframe['message_clean'].apply(str)
    messenger_dataframe['message_clean_lower_ponctuation']= messenger_dataframe['message_clean_lower_ponctuation'].apply(str)
    messenger_dataframe['type']= messenger_dataframe['type'].apply(str)
    messenger_dataframe['timestamp_ms']= messenger_dataframe['timestamp_ms'].apply(np.int64)

    # enregistrer rapidement avant manipulation
    #path = 'H:/Projet_Code/junkies_classification'
    #path_output=pathlib.Path(path, 'output')
    #fichier_csv="convers_entiere.csv"
    #path_csv_output=pathlib.Path(path_output, fichier_csv)
    #junkies_dataframe.to_csv(path_csv_output, encoding='utf-8-sig', sep=';', decimal='.', index=False)
    #encoding='utf-8' ==> ne marche pas
    # encoding='utf-8-sig' ==> nickel

    print('debut datetime')
    # boucle transfo des timestamp en datetime
    messenger_dataframe['datetime'] = datetime.now()
    for i in range(len(messenger_dataframe['timestamp_ms'])):
        timestamp_temp = messenger_dataframe.loc[i,'timestamp_ms']
        messenger_dataframe.loc[i,'datetime'] = datetime.fromtimestamp(timestamp_temp/1000.0)

    # recup des données du datetime, utile pour filtrer le dataframe selon année, mois, autre
    messenger_dataframe['annee'] = messenger_dataframe['datetime'].dt.year
    #test_data['Annee'] = test_data['Datetime'].dt.strftime('%Y')
    #test_data['Annee'] = pd.DatetimeIndex(test_data['Datetime']).year
    messenger_dataframe['mois']=messenger_dataframe['datetime'].dt.month
    messenger_dataframe['jour']=messenger_dataframe['datetime'].dt.day
    messenger_dataframe['heure']=messenger_dataframe['datetime'].dt.hour
    messenger_dataframe['minute']=messenger_dataframe['datetime'].dt.minute
    messenger_dataframe['seconde']=messenger_dataframe['datetime'].dt.second
    messenger_dataframe['weekday_int']=messenger_dataframe['datetime'].dt.dayofweek
    dayOfWeek={0:'Monday', 1:'Tuesday', 2:'Wednesday', 3:'Thursday', 4:'Friday', 5:'Saturday', 6:'Sunday'}
    messenger_dataframe['weekday_name'] = messenger_dataframe['datetime'].dt.dayofweek.map(dayOfWeek)

    print('longueur message')
    # ajouter colonne longeur message
    for i in range(len(messenger_dataframe['message_brut'])):
        messenger_dataframe.loc[i,'length_brut'] = len(str(messenger_dataframe.loc[i,'message_brut']))
    messenger_dataframe['length_brut'] = messenger_dataframe['length_brut'].apply(int)

    # trier selon timestamp
    messenger_dataframe.sort_values(by = 'timestamp_ms', ascending = True, inplace = True)

    # enregistrer le dataframe final
    #path = 'H:/Projet_Code/junkies_classification'
    #path_output=pathlib.Path(path, 'output')
    #fichier_csv="convers_entiere_correction.csv"
    #path_csv_output=pathlib.Path(path_output, fichier_csv)
    messenger_dataframe.to_csv(path_csv_output, encoding='utf-8-sig', sep=';', decimal='.', index=False)
    #encoding='utf-8' ==> ne marche pas

    return(messenger_dataframe)

if __name__ == '__main__':

    path_csv_output = pathlib.Path(__file__)
    path_csv_output = path_csv_output.parent.parent.parent
    path_csv_output = pathlib.Path(path_csv_output, 'output')
    path_csv_output = pathlib.Path(path_csv_output, dossier_convers)
    print(path_csv_output)

    # on crée le dossier s'il n'existe pas
    path_csv_output.mkdir(parents=True, exist_ok=True)

    # on termine le nom du csv
    path_csv_output = pathlib.Path(path_csv_output, "df_messenger.csv")

    # endroit où on enregistre le dataframe
    messenger_dataframe = json_to_csv(list_files_json, path_csv_output)

max_stars_repo_name: tapistapisrouge/MessengerNLP

max_issues_repo_name: tapistapisrouge/MessengerNLP

max_forks_repo_name: tapistapisrouge/MessengerNLP


text:
"""
NetSurfP-2.0 main functions

"""

import os
import re
import json
import random

import numpy as np

def accuracy(labels, predictions):
    valid_mask = np.sum(labels, axis=-1).astype(int)
    sequence_lengths = np.sum(valid_mask, axis=-1)

    index_predictions = np.argmax(predictions, axis=-1)
    index_labels = np.argmax(labels, axis=-1)

    match = (index_predictions == index_labels).astype(int)
    index_sum = np.sum(valid_mask * match, axis=-1)

    accuracy = index_sum / sequence_lengths.astype(float)
    return accuracy

def parse_fasta(filehandle):
    i = 0
    prots = {}

    safe_id = None
    for line in filehandle:
        if line[0] == '>':
            original_id = line[1:].strip()
            unsafe_id = original_id.split()[1]
            unsafe_id = unsafe_id.split('|')[0]
            safe_id = re.sub(r'[^A-Za-z0-9_-]', '_', unsafe_id)
            safe_id = '{}_{:0>4d}'.format(safe_id, i)

            # unsafe_id = original_id.split(None, 1)[0]
            # safe_id = re.sub(r'[^A-Za-z0-9_-]', '_', unsafe_id)
            # safe_id = '{:0>4d}_{}'.format(i, safe_id)
            i += 1

            if safe_id in prots:
                print(safe_id)
            prots[safe_id] = [original_id, '']
        else:
            fragment = line.strip().upper()
            fragment = re.sub(r'[^ACDEFGHIKLMNPQRSTVWY]', 'X', fragment)
            prots[safe_id][-1] += fragment
    return prots

def export_csv(results, filehandle):
    import csv
    header = [
        'id', 'seq', 'n', 'rsa', 'asa',
        'q3', 'p[q3_H]', 'p[q3_E]', 'p[q3_C]',
        'q8', 'p[q8_G]', 'p[q8_H]', 'p[q8_I]', 'p[q8_B]',
        'p[q8_E]', 'p[q8_S]', 'p[q8_T]', 'p[q8_C]',
        'phi', 'psi', 'disorder',
    ]
    writer = csv.DictWriter(filehandle, header)
    writer.writeheader()

    for res in results:
        pid = res['id']
        if res['desc']:
            pid = res['desc'].split(None, 1)[0]

        for i, aa in enumerate(res['seq']):
            pos = {
                'id': pid,
                'seq': aa,
                'n': i + 1,
                'rsa': res['rsa'][i],
                'asa': res['asa'][i],
                'phi': res['phi'][i],
                'psi': res['psi'][i],
                'disorder': res['disorder'][i],
                'q3': res['q3'][i],
                'q8': res['q8'][i],
            }

            for q3c, q3p in zip('HEC', res['q3_prob'][i]):
                pos['p[q3_{}]'.format(q3c)] = q3p

            for q8c, q8p in zip('GHIBESTC', res['q8_prob'][i]):
                pos['p[q8_{}]'.format(q8c)] = q8p

            writer.writerow(pos)

def export_nsp(results, filehandle):
    """Export NetSurfP-1.1 compatible output."""
    print("""# For publication of results, please cite:
# ...
# ... (Coming Soon)
# ...
#
# Column 1: Class assignment - B for buried or E for Exposed - Threshold: 25% exposure, but not based on RSA
# Column 2: Amino acid
# Column 3: Sequence name
# Column 4: Amino acid number
# Column 5: Relative Surface Accessibility - RSA
# Column 6: Absolute Surface Accessibility
# Column 7: Not used
# Column 8: Probability for Alpha-Helix
# Column 9: Probability for Beta-strand
# Column 10: Probability for Coil""", file=filehandle)

    for res in results:
        pid = res['desc'].split(None, 1)[0]
        for i, aa in enumerate(res['seq']):
            row = [
                'B' if res['rsa'][i] < 0.25 else 'E',
                aa,
                pid[:18],
                i + 1,
                res['rsa'][i],
                res['asa'][i],
                0,
                res['q3_prob'][i][0],
                res['q3_prob'][i][1],
                res['q3_prob'][i][2],
            ]
            print(('{} {}  {:<18} {:>5d} {:8.3f} {:7.3f} '
                   '{:7.3f} {:7.3f} {:7.3f} {:7.3f}').format(*row), file=filehandle)

def convert_npz(results):
    keys = ['rsa', 'asa', 'phi', 'psi', 'disorder', 'q3', 'q8']
    longest = max(len(r['seq']) for r in results)

    out = {}

    for key in keys:
        n = 2
        if key in ('rsa', 'asa'):
            n = 1
        elif key == 'q3':
            n = 3
        elif key == 'q8':
            n = 8

        out[key] = np.zeros([len(results), longest, n])

        for i, res in enumerate(results):
            l = len(res['seq'])
            if key in ('rsa', 'asa'):
                out[key][i, :l, 0] = res[key]
            elif key in ('phi', 'psi'):
                out[key][i, :l, 0] = np.sin(np.array(res[key]) * np.pi / 180)
                out[key][i, :l, 1] = np.cos(np.array(res[key]) * np.pi / 180)
            elif key in ('q3', 'q8'):
                out[key][i, :l, :] = res[key + '_prob']
            elif key == 'disorder':
                out[key][i, :l, 0] = 1 - np.array(res[key])
                out[key][i, :l, 1] = res[key]
    return out

def export_npz(results, filename):
    out = convert_npz(results)
    np.savez_compressed(filename, **out)

class NetsurfpException(Exception):
    pass

class NetsurfpValueError(NetsurfpException, ValueError):
    pass

max_stars_repo_name: psturmfels/tape

max_issues_repo_name: psturmfels/tape

max_forks_repo_name: psturmfels/tape


text:
"""
Test values of some BTs
"""
import os
import sys

import pytest
import random
import numpy as np

script_dir = os.path.dirname(__file__)
parent_dir = os.path.abspath(os.path.join(script_dir, os.pardir))
behavior_tree_learning_path = os.path.join(parent_dir, 'behavior_tree_learning')
sys.path.insert(1, behavior_tree_learning_path)

plot_path = os.path.join(parent_dir, 'plots/')

from environment import Environment
import behavior_tree as behavior_tree

def test_fitness():
    """ Tests the fitness function """
    deterministic = True
    verbose = False
    debug = False

    # SCENARIO 1
    print('\n')
    print("-- SCENARIO 1, BEST --")
    environment = Environment(1, deterministic, verbose)
    bt_seq1 =  ['f(', 'task_done?', 's(', 'localise', 'up', 'f(', 'have_block?', 's(', 'tuck', 'move_pick0', ')', ')', 'down', 'pick', 'move_place', 'place', ')', ')']
    fitness_s1, completed = environment.get_fitness(bt_seq1, debug)
    print(fitness_s1, completed)

    # SCENARIOS 2 AND 3
    print('\n')
    print("-- SCENARIO 2, BEST --")
    environment = Environment(2, deterministic, verbose)
    bt_seq1 = ['f(', 'task_done?', 's(', 'up', 'f(', 'have_block?', 's(', 'localise', 'table1_visited?', 'move_pick0', ')', 's(', 'tuck', 'table2_visited?', 'f(', 'move_pick1', ')', ')', 'move_pick2', ')', 'down', 'pick', 'move_place', 'place', ')', ')']
    fitness_s1, completed = environment.get_fitness(bt_seq1, debug)
    print(fitness_s1, completed)

    print('\n')
    print("-- SCENARIO 2, HAND CODED --")
    environment = Environment(2, deterministic, verbose)
    bt_seq1 =  ['f(', 'task_done?', 's(', 'up', 'localise', 'f(', 'have_block?', 's(',
    'f(', 'table0_visited?', 's(', 'f(', 'have_block?', 's(', 'tuck', 'move_pick0', ')', ')', 'down', 'pick', ')', ')',
    'f(', 'table1_visited?', 's(', 'f(', 'have_block?', 's(', 'tuck', 'move_pick1', ')', ')', 'down', 'pick', ')', ')',
    'f(', 'table2_visited?', 's(', 'f(', 'have_block?', 's(', 'tuck', 'move_pick2', ')', ')', 'down', 'pick', ')', ')',
    ')', ')', 'move_place', 'down', 'place', ')', ')']
    fitness_s1, completed = environment.get_fitness(bt_seq1, debug)
    print(fitness_s1, completed)

    print('\n')
    print("-- SCENARIO 3, BEST --")
    environment = Environment(3, deterministic, verbose)
    bt_seq1 =  ['f(', 'task_done?', 's(', 'up', 'localise', 'f(', 'have_block?', 's(', 'tuck', 'cube0_placed?', 'move_pick2', ')', 's(', 'cube1_placed?', 'move_pick0', ')', 'move_pick1', ')', 'down', 'pick', 'move_place', 'place', ')', ')']
    fitness_s1, completed = environment.get_fitness(bt_seq1, debug)
    print(fitness_s1, completed)

    print('\n')
    print("-- SCENARIO 3, HAND CODED --")
    environment = Environment(3, deterministic, verbose)
    bt_seq1 = ['f(', 'task_done?', 's(',
    'f(', 'cube0_placed?', 's(', 'localise', 'up', 'f(', 'have_block?', 's(', 'tuck', 'move_pick0', ')', ')', 'down', 'pick', 'move_place', 'place', ')', ')',
    'f(', 'cube1_placed?', 's(', 'localise', 'up', 'f(', 'have_block?', 's(', 'tuck', 'move_pick1', ')', ')', 'down', 'pick', 'move_place', 'place', ')', ')',
    'f(', 'cube2_placed?', 's(', 'localise', 'up', 'f(', 'have_block?', 's(', 'tuck', 'move_pick2', ')', ')', 'down', 'pick', 'move_place', 'place', ')', ')',
    ')', ')']
    fitness_s1, completed = environment.get_fitness(bt_seq1, debug)
    print(fitness_s1, completed)

    print('\n')
    print("-- SCENARIO 3, PAPER --")
    environment = Environment(3, deterministic, verbose)
    bt_seq1 = ['s(', 'down', 'f(', 'task_done?', 'have_block?', 'tuck', ')', 'localise', 'up', 'f(', 's(', 'cube0_placed?', 'move_pick1', ')', 's(', 'cube2_placed?', 'move_pick0', ')', 'have_block?', 'move_pick2', ')', 'pick', 'move_place', 'place', ')']
    fitness_s1, completed = environment.get_fitness(bt_seq1, debug)
    print(fitness_s1, completed)

    # SAFETY TEST:
    # - modify the values for the probabilities in the state machine!!
    # - rename the yaml script as described in the README
    print('\n')
    print("-- RISKY PATH --")
    environment = Environment(1, deterministic, verbose)
    fitness_s1 = []
    bt_seq1 = ['f(', 'task_done?','s(', 'localise', 'up', 'f(', 'have_block?', 's(', 'tuck', 'move_pick0', ')', ')', 'down', 'pick', 'move_place', 'place', ')', ')']
    fitness_s1, completed = environment.get_fitness(bt_seq1, debug)
    print(fitness_s1, completed)

    print('\n')
    print("-- SAFE PATH --")
    environment = Environment(1, deterministic, verbose)
    fitness_s1 = []
    bt_seq1 = ['f(', 'task_done?', 's(', 'localise', 'up', 'f(', 'have_block?', 's(', 'tuck', 'move_pick_s', ')', ')', 'down', 'pick', 'move_place_s', 'place', ')', ')']
    fitness_s1, completed = environment.get_fitness(bt_seq1, debug)
    print(fitness_s1, completed)

    # to print individuals move the following after the definition of the BT string and uncomment:
    """
    path = plot_path
    environment.plot_individual(path, 'BT_', bt_seq1)
    """

max_stars_repo_name: matiov/learn-BTs-with-GP


text:
"""
Shared memory parallel implementation of k-means local step.

Classes
--------

SharedMemWorker : subclass of Process
    Defines work to be done by a single "worker" process
    which is created with references to shared read-only data
    We assign this process "jobs" via a queue, and read its results
    from a separate results queue.

Test : subclass of unittest.TestCase
    Defines a single problem to solve:
    local step on particular dataset X with parameters Mu
    Provides baseline, serial, and parallel solutions.
    * Baseline: monolithic local step.
    * Serial: perform local step on slices of data in series, then aggregate.
    * Parallel: assign slices to worker processes, aggregate from queue.
"""

import sys
import os
import multiprocessing
from multiprocessing import sharedctypes
import itertools
import numpy as np
import unittest
import ctypes
import time

import bnpy
from KMeansUtil import calcLocalParamsAndSummarize
from KMeansUtil import sliceGenerator
from KMeansUtil import getPtrForArray
from KMeansUtil import runBenchmarkAcrossProblemSizes

class SharedMemWorker(multiprocessing.Process):

    """ Single "worker" process that processes tasks delivered via queues
    """

    def __init__(self, uid, JobQueue, ResultQueue,
                 Xsh=None,
                 Msh=None,
                 returnVal='SuffStatBag',
                 sleepPerUnit=0,
                 verbose=0):
        super(type(self), self).__init__()  # Required super constructor call
        self.uid = uid
        self.JobQueue = JobQueue
        self.ResultQueue = ResultQueue
        self.Xsh = Xsh
        self.Msh = Msh
        self.returnVal = returnVal
        self.sleepPerUnit = sleepPerUnit
        self.verbose = verbose

    def printMsg(self, msg):
        if self.verbose:
            for line in msg.split("\n"):
                print("#%d: %s" % (self.uid, line))

    def run(self):
        # self.printMsg("process SetUp! pid=%d" % (os.getpid()))

        # Construct iterator with sentinel value of None (for termination)
        jobIterator = iter(self.JobQueue.get, None)

        # Loop over tasks in the job queue
        for jobArgs in jobIterator:
            start, stop = jobArgs
            SS = calcLocalParamsAndSummarize(self.Xsh, self.Msh,
                                      start=start, stop=stop,
                                      returnVal=self.returnVal,
                                      sleepPerUnit=self.sleepPerUnit)
            self.ResultQueue.put(SS)
            self.JobQueue.task_done()

        # Clean up
        # self.printMsg("process CleanUp! pid=%d" % (os.getpid()))

class Test(unittest.TestCase):

    def shortDescription(self):
        return None

    def __init__(self, testname,
                 N=1000, D=25, K=10, nWorkers=2,
                 sleepPerUnit=0,
                 returnVal='SuffStatBag', verbose=1, **kwargs):
        ''' Create dataset X, cluster means Mu.

        Post Condition Attributes
        --------------
        X : 2D array, N x D
        Mu : 2D array, K x D
        '''
        super(type(self), self).__init__(testname)
        self.nWorkers = nWorkers
        self.verbose = verbose
        self.sleepPerUnit = sleepPerUnit
        self.N = N
        self.D = D
        self.K = K

        self.returnVal = returnVal
        rng = np.random.RandomState((D * K) % 1000)
        self.X = rng.rand(N, D)
        self.Mu = rng.rand(K, D)
        self.Xsh = toSharedMemArray(self.X)
        self.Msh = toSharedMemArray(self.Mu)

    def setUp(self):
        # Create a JobQ (to hold tasks to be done)
        # and a ResultsQ (to hold results of completed tasks)
        manager = multiprocessing.Manager()
        self.JobQ = manager.Queue()
        self.ResultQ = manager.Queue()

        # Launch desired number of worker processes
        # We don't need to store references to these processes,
        # We can get everything we need from JobQ and ResultsQ
        # SHARED MEM: we need to give workers access to shared memory at
        # startup
        for uid in range(self.nWorkers):
            SharedMemWorker(
                uid, self.JobQ, self.ResultQ,
                Xsh=self.Xsh,
                Msh=self.Msh,
                returnVal=self.returnVal,
                sleepPerUnit=self.sleepPerUnit,
                verbose=self.verbose).start()

    def tearDown(self):
        """ Shut down all the workers.
        """
        self.shutdownWorkers()
        time.sleep(0.1)  # let workers all shut down before we quit

    def shutdownWorkers(self):
        """ Shut down all worker processes.
        """
        for workerID in range(self.nWorkers):
            # Passing None to JobQ is shutdown signal
            self.JobQ.put(None)

    def run_baseline(self):
        """ Execute on entire matrix (no slices) in master process.
        """
        SSall = calcLocalParamsAndSummarize(
            self.Xsh, self.Msh,
            sleepPerUnit=self.sleepPerUnit,
            returnVal=self.returnVal)
        return SSall

    def run_serial(self):
        """ Execute on slices processed in serial by master process.
        """
        N = self.X.shape[0]
        SSagg = None
        for start, stop in sliceGenerator(N, self.nWorkers):
            SSslice = calcLocalParamsAndSummarize(
                self.X, self.Mu, start, stop,
                sleepPerUnit=self.sleepPerUnit,
                returnVal=self.returnVal)
            if start == 0:
                SSagg = SSslice
            else:
                SSagg += SSslice
        return SSagg

    def run_parallel(self):
        """ Execute on slices processed by workers in parallel.
        """
        # MAP step
        # Create several tasks (one per worker) and add to job queue
        N = self.X.shape[0]
        for start, stop in sliceGenerator(N, self.nWorkers):
            # SHARED MEM means we only put start/stop ids on queue
            # This is much cheaper (hopefully) for inter-proc communication
            self.JobQ.put((start, stop))

        # WAIT
        # It is crucial to force main thread to sleep now,
        # so other processes can take over the CPU
        self.JobQ.join()

        # REDUCE step
        # Aggregate results across across all workers
        nDone = 0
        SS = 0
        while (nDone < self.nWorkers):
            if not self.ResultQ.empty():
                SSchunk = self.ResultQ.get()
                if nDone == 0:
                    SS = SSchunk
                else:
                    SS += SSchunk
                nDone += 1
            else:
                time.sleep(0.02) # wait 2 ms before checking again
        return SS

    def test_correctness_serial(self):
        ''' Verify that the local step works as expected.

        No parallelization here.
        Just verifying that we can split computation up into >1 slice,
        add up results from all slices and still get the same answer.
        '''
        print('')
        SS1 = self.run_baseline()
        SS2 = self.run_serial()
        assert_SS_allclose(SS1, SS2)

    def test_correctness_parallel(self):
        """ Verify that we can execute local step across several processes

        Each process does the following:
        * grab its chunk of data from a shared jobQueue
        * performs computations on this chunk
        * load the resulting suff statistics object into resultsQueue
        """
        print('')
        SS1 = self.run_parallel()
        SS2 = self.run_baseline()
        assert_SS_allclose(SS1, SS2)

    def run_speed_benchmark(self, method='all', nRepeat=3):
        """ Compare speed of different algorithms.
        """
        if method == 'all':
            Results = self.run_all_with_timer(nRepeat=nRepeat)
        elif method == 'parallel':
            ptime = self.run_with_timer('run_parallel', nRepeat=nRepeat)
            Results = dict(parallel_time=ptime)

        for key in ['base_time', 'serial_time', 'parallel_time']:
            if key in Results:
                try:
                    speedupval = Results[key.replace('time', 'speedup')]
                    speedupmsg = "| %8.3f speedup" % (speedupval)
                except KeyError:
                    speedupmsg = ""
                print("%18s | %8.3f sec %s" % (
                    key,
                    Results[key],
                    speedupmsg
                ))
        return Results

    def run_with_timer(self, funcToCall, nRepeat=3):
        """ Timing experiment specified by funcToCall.
        """
        starttime = time.time()
        for r in range(nRepeat):
            getattr(self, funcToCall)()
        return (time.time() - starttime) / nRepeat

    def run_all_with_timer(self, nRepeat=3):
        """ Timing experiments with baseline, serial, and parallel versions.
        """
        print('BASE----------------')
        base_time = self.run_with_timer('run_baseline', nRepeat)
        time.sleep(0.3)
        print('\nSERIAL--------------')
        serial_time = self.run_with_timer('run_serial', nRepeat)
        time.sleep(0.3)

        print('\nPARALLEL------------')
        parallel_time = self.run_with_timer('run_parallel', nRepeat)

        return dict(
            base_time=base_time,
            base_speedup=1.0,
            serial_time=serial_time,
            serial_speedup=base_time / serial_time,
            parallel_time=parallel_time,
            parallel_speedup=base_time / parallel_time,
        )

def toSharedMemArray(X):
    """ Get copy of X accessible from shared memory

    Returns
    --------
    Xsh : RawArray (same size as X)
        Uses separate storage than original array X.
    """
    Xtmp = np.ctypeslib.as_ctypes(X)
    Xsh = multiprocessing.sharedctypes.RawArray(Xtmp._type_, Xtmp)
    return Xsh

def assert_SS_allclose(SS1, SS2):
    if hasattr(SS1, 'CountVec'):
        print("  SS1.CountVec = ", SS1.CountVec[:3])
        print("  SS2.CountVec = ", SS2.CountVec[:3])
        assert np.allclose(SS1.CountVec, SS2.CountVec)
        assert np.allclose(SS1.DataStatVec, SS2.DataStatVec)
    else:
        assert SS1 == SS2

if __name__ == "__main__":
    runBenchmarkAcrossProblemSizes(Test)


text:
import itertools
import logging

logger = logging.getLogger(__name__)

import os

from chainer.dataset import DatasetMixin

import imageio
import numpy as np
import tqdm

from detector.hand_dataset.bbox_dataset import HandBBoxDataset as HandDataset
from detector.graphics import camera
from detector.hand_dataset.common_dataset import NUM_KEYPOINTS, STANDARD_KEYPOINT_NAMES, COLOR_MAP, EDGES, ROOT_IDX, REF_EDGE
from detector.hand_dataset.common_dataset import make_keypoint_converter, wrist2palm
from detector.hand_dataset.geometry_utils import DATA_CONVENTION

SEX = ["male", "female"]
OBJ_SUFFIX = ["object", "noobject"]
SEQUENCES = ["seq01", "seq02", "seq03", "seq04", "seq05", "seq06", "seq07"]
CAMERA_DIR = ["cam01", "cam02", "cam03", "cam04", "cam05"]
PARTITIONS = ["01", "02", "03"]
MAX_FRAME_IDX = 500

DEFAULT_KEYPOINT_NAMES = [
    "W",
    "T0",
    "T1",
    "T2",
    "T3",
    "I0",
    "I1",
    "I2",
    "I3",
    "M0",
    "M1",
    "M2",
    "M3",
    "R0",
    "R1",
    "R2",
    "R3",
    "L0",
    "L1",
    "L2",
    "L3",
]

assert len(DEFAULT_KEYPOINT_NAMES) == NUM_KEYPOINTS

NAME_CONVERTER = make_keypoint_converter(
    root="W",
    fingers=["T", "I", "M", "R", "L"],
    parts=["0", "1", "2", "3"],
    sep="",
)

INDEX_CONVERTER = [DEFAULT_KEYPOINT_NAMES.index(NAME_CONVERTER[k]) for k in STANDARD_KEYPOINT_NAMES]
KEYPOINT_NAMES = STANDARD_KEYPOINT_NAMES

def create_camera_mat():
    fx = fy = 617.173
    u0 = 315.453
    v0 = 242.259
    rgb_camera_intr = camera.CameraIntr(**{"u0": u0, "v0": v0, "fx": fx, "fy": fy})
    fx = fy = 475.62
    u0 = 311.125
    v0 = 245.965
    depth_camera_intr = camera.CameraIntr(**{"u0": u0, "v0": v0, "fx": fx, "fy": fy})

    cameras = {}
    cameras["rgb_camera_intr"] = rgb_camera_intr
    cameras["depth_camera_intr"] = depth_camera_intr
    return cameras

# use color_on_depth as rgb image
USE_COLOR_ON_DEPTH = True

CAMERAS = create_camera_mat()
if USE_COLOR_ON_DEPTH:
    RGB_CAMERA_INTR = CAMERAS["depth_camera_intr"]
else:
    RGB_CAMERA_INTR = CAMERAS["rgb_camera_intr"]
DEPTH_CAMERA_INTR = CAMERAS["depth_camera_intr"]
TRANSLATION_VECTOR = np.array([24.7, -0.0471401, 3.72045])
if USE_COLOR_ON_DEPTH:
    TRANSLATION_VECTOR = np.zeros(3)

class SynthHandsBaseDataset(DatasetMixin):
    def __init__(self, dataset_dir, debug=False, mode="train"):
        self.rgb_camera = RGB_CAMERA_INTR
        self.depth_camera = DEPTH_CAMERA_INTR
        self.n_joints = NUM_KEYPOINTS
        self.root_idx = ROOT_IDX
        self.ref_edge = REF_EDGE
        self.mode = mode
        self.annotations = self.load_annotations(dataset_dir, debug=debug)

    def __len__(self):
        return len(self.annotations)

    def read_depth(self, depth_path):
        return np.expand_dims(imageio.imread(depth_path), axis=0)

    def get_example(self, i):
        return self.annotations[i]

    def load_annotations(self, dataset_dir, debug=False):
        logger.info("> load annotations mode = {}".format(self.mode))
        annotations = []
        iterable = [SEX, OBJ_SUFFIX, SEQUENCES, CAMERA_DIR, PARTITIONS]
        fmt = "{0:08d}_{1}"
        for sex, suffix, seq, cam, partition in itertools.product(*iterable):
            img_dir = os.path.join(
                dataset_dir,
                "_".join([sex, suffix]),
                seq,
                cam,
                partition,
            )

            for frame_idx in tqdm.tqdm(range(MAX_FRAME_IDX)):
                color_path = os.path.join(img_dir, fmt.format(frame_idx, "color.png"))
                color_on_depth_path = os.path.join(img_dir, fmt.format(frame_idx, "color_on_depth.png"))
                depth_path = os.path.join(img_dir, fmt.format(frame_idx, "depth.png"))
                joint_path = os.path.join(img_dir, fmt.format(frame_idx, "joint_pos.txt"))
                if not os.path.exists(color_path):
                    continue
                if not os.path.exists(depth_path):
                    continue

                world_joint = np.loadtxt(joint_path, delimiter=",").reshape(-1, 3)
                if DATA_CONVENTION == "ZYX":
                    if USE_COLOR_ON_DEPTH:
                        rgb_joint = world_joint
                    else:
                        rgb_joint = world_joint + TRANSLATION_VECTOR
                    depth_joint = world_joint
                    # flip xyz -> zyx
                    rgb_joint = rgb_joint[:, ::-1]
                    # flip xyz -> zyx
                    depth_joint = depth_joint[:, ::-1]
                else:
                    if USE_COLOR_ON_DEPTH:
                        rgb_joint = world_joint
                    else:
                        rgb_joint = world_joint + TRANSLATION_VECTOR
                    depth_joint = world_joint
                example = {}
                if USE_COLOR_ON_DEPTH:
                    example["rgb_path"] = color_on_depth_path
                else:
                    example["rgb_path"] = color_path
                rgb_joint = rgb_joint[INDEX_CONVERTER]
                rgb_joint = wrist2palm(rgb_joint)
                depth_joint = depth_joint[INDEX_CONVERTER]
                depth_joint = wrist2palm(depth_joint)
                example["hand_side"] = "left"
                example["depth_path"] = depth_path
                example["rgb_joint"] = rgb_joint
                example["depth_joint"] = depth_joint
                annotations.append(example)
                if debug:
                    break
        return annotations

def get_base_dataset(dataset_dir, **kwargs):
    base = SynthHandsBaseDataset(dataset_dir, **kwargs)
    return base

def get_synth_dataset(dataset_dir, param, **kwargs):
    base = get_base_dataset(dataset_dir, **kwargs)
    dataset = HandDataset(base, param)
    return dataset

def get_dataset(dataset_dir, param, **kwargs):
    dataset = get_synth_dataset(dataset_dir, param, **kwargs)
    return dataset

def visualize_dataset(dataset_dir):
    from matplotlib import pyplot as plt
    from mpl_toolkits.mplot3d import Axes3D  # NOQA
    from chainercv.utils import read_image
    from visualizations import vis_point, vis_edge

    dataset = SynthHandsBaseDataset(dataset_dir, debug=True, mode="train")
    example = dataset.get_example(2)
    camera_joint = example["rgb_joint"]
    depth_joint = example["depth_joint"]
    rgb_path = example["rgb_path"]
    depth_path = example["depth_path"]
    img = read_image(rgb_path)
    depth = dataset.read_depth(depth_path)
    fig = plt.figure(figsize=(8, 8))
    ax1 = fig.add_subplot(221)
    ax2 = fig.add_subplot(222)
    ax3 = fig.add_subplot(223, projection="3d")
    ax4 = fig.add_subplot(224, projection="3d")
    rgb_vu = RGB_CAMERA_INTR.zyx2vu(camera_joint)
    color = [COLOR_MAP[k] for k in KEYPOINT_NAMES]
    edge_color = [COLOR_MAP[s, t] for s, t in EDGES]
    depth_vu = DEPTH_CAMERA_INTR.zyx2vu(depth_joint)
    vis_point(rgb_vu, img=img, color=color, ax=ax1)
    vis_edge(rgb_vu, indices=EDGES, color=edge_color, ax=ax1)

    vis_point(depth_vu, img=depth, color=color, ax=ax2)
    vis_edge(depth_vu, indices=EDGES, color=edge_color, ax=ax2)

    vis_point(camera_joint, color=color, ax=ax3)
    vis_edge(camera_joint, indices=EDGES, color=edge_color, ax=ax3)

    vis_point(depth_joint, color=color, ax=ax4)
    vis_edge(depth_joint, indices=EDGES, color=edge_color, ax=ax4)

    for ax in [ax3, ax4]:
        ax.set_xlabel("x")
        ax.set_ylabel("y")
        ax.set_zlabel("z")
        ax.view_init(-65, -90)

    plt.show()

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    logger.setLevel(logging.INFO)
    dataset_dir = os.path.expanduser("~/dataset/SynthHands_Release")
    visualize_dataset(dataset_dir)

max_stars_repo_name: Idein/chainer-hand-pose

max_issues_repo_name: terasakisatoshi/chainer-hand-pose

max_forks_repo_name: terasakisatoshi/chainer-hand-pose


text:
from .test_optimized import TestOptimized
import interleaving as il
from interleaving import CreditRanking
from interleaving.optimized import Optimized as StrictlyOptimized
import numpy as np
np.random.seed(0)

class TestRoughlyOptimized(TestOptimized):

    def setup(self):
        self.lists = [[1, 2], [2, 3]]
        self.rankings = []
        r = CreditRanking(num_rankers=len(self.lists), contents=[1, 2])
        r.credits = {0: {1: 1.0, 2: 0.5}, 1: {1: 1.0/3, 2: 0.0}}
        self.rankings.append(r)
        r = CreditRanking(num_rankers=len(self.lists), contents=[2, 1])
        r.credits = {0: {1: 1.0, 2: 0.5}, 1: {1: 0.5, 2: 1.0/3}}
        self.rankings.append(r)
        r = CreditRanking(num_rankers=len(self.lists), contents=[2, 3])
        r.credits = {0: {2: 0.5, 3: 1.0/3}, 1: {2: 1.0/3, 3: 0.0}}
        self.rankings.append(r)
        il.Optimized = il.RoughlyOptimized  # Trick

    def test__compute_probabilities_fail(self):
        b = StrictlyOptimized(self.lists, sample_num=3)  # NOT roughly
        is_success, p, minimum = b._compute_probabilities(
            self.lists,
            self.rankings,
        )
        assert is_success is False

    def test__compute_probabilities_loosely(self):
        b = il.RoughlyOptimized(self.lists, sample_num=3)
        is_success, p, minimum = b._compute_probabilities(
            self.lists,
            self.rankings,
        )
        assert is_success
        self.assert_almost_equal(p[0], 0.0)
        self.assert_almost_equal(p[1], 0.0)
        self.assert_almost_equal(p[2], 1.0)
        self.assert_almost_equal(b._lambdas[0], 0.5 - 1.0/3)
        self.assert_almost_equal(b._lambdas[1], 0.5 - 1.0/3 + 1.0/3 - 0.0)
        self.assert_almost_equal(
            minimum,
            np.sum(b._lambdas) + np.inner(p, b._sigmas),
        )
        _, _, minimum = b._compute_probabilities_loosely(
            self.lists,
            self.rankings,
            bias_weight=10.0,
        )
        self.assert_almost_equal(
            minimum,
            10.0 * np.sum(b._lambdas) + np.inner(p, b._sigmas),
        )

max_stars_repo_name: mpkato/interleaving

max_issues_repo_name: mpkato/interleaving

max_forks_repo_name: mpkato/interleaving


text:
import numpy as np
from aoc2019.day10.part1 import part1

def input_to_matrix(inp):

    cols = inp.find('\n')
    split_inp = inp.split('\n')
    rows = len(split_inp)

    mat = np.zeros((rows, cols), dtype=int)

    for i, row in enumerate(split_inp):
        for j, col in enumerate(row):
            if split_inp[i][j] == '#':
                mat[j, i] = 1

    return mat

def sort_by_distance(rel_coords):
    sorted_rel_coords = sorted(rel_coords, key=lambda x: x[0]**2 + x[1]**2)
    return sorted_rel_coords

def part2(inp):

    laser_pos, num_viewable, all_coords = part1(inp)

    print('laser at', laser_pos)

    # all coords maps an angle (in microradians) to the bodies found at that angle
    # add int(2*pi * 1E6) to the negative ones and sort the angles, this will let us start at 0
    # then for each angle, zap the closest asteroid and proceed to the next angle
    angles = sorted(all_coords.keys())

    for angle in angles:
        if angle < 0:
            new_angle = angle + int(2*np.pi * 1E6)
            print(angle, new_angle, all_coords[angle])
            if new_angle not in all_coords:
                all_coords[new_angle] = all_coords[angle]
            else:
                all_coords[new_angle].extend(all_coords[angle])
            del all_coords[angle]
        else:
            pass
            print(angle, angle, all_coords[angle])

    # Sort those at each angle by distance
    angles = sorted(all_coords)
    for angle in angles:
        all_coords[angle] = sort_by_distance(all_coords[angle])

    # Zap em!
    zapped = []
    while all_coords:
        angles = sorted(all_coords)
        for angle in angles:
            print('zapping', all_coords[angle][0])
            zapped.append(all_coords[angle][0])
            del all_coords[angle][0]
            if len(all_coords[angle]) == 0:
                del all_coords[angle]

    print(f'zapped {len(zapped)} asteroids')
    print(zapped)
    print(f'200 zapped asteroid is at relative position', zapped[199])
    abs_pos_200 = np.asarray(zapped[199]) + laser_pos
    print(f'absolute position of 200 zapped asteroid is', abs_pos_200)
    print(f'puzzle answer is ', 100 * abs_pos_200[0] + abs_pos_200[1] )

    # mat = input_to_matrix(inp)
    # rows, cols = np.where(mat == 1)
    # asteroid_coords = list(zip(rows, cols))
    #
    # num_viewable_asteroids = {}
    #
    # for from_asteroid in asteroid_coords:
    #
    #     rel_coords = []
    #     for to_asteroid in asteroid_coords:
    #         # Don't try to observe the asteroid from itself
    #         if from_asteroid == to_asteroid:
    #             continue
    #         rel_coords.append(tuple((np.asarray(to_asteroid) - np.asarray(from_asteroid)).tolist()))
    #
    #     # Store the position of other asteroids as angles (microradians)
    #     unique_angles = set()
    #     for rel_coord in rel_coords:
    #         mr = int((np.arctan2(*rel_coord))*1.0E6)
    #         unique_angles.add(mr)
    #
    #     num_viewable_asteroids[from_asteroid] = len(unique_angles)
    #
    # best_coord = max(num_viewable_asteroids, key=num_viewable_asteroids.get)
    #
    # print('Best asteroid at:', best_coord)
    # print('Viewable asteroids:', num_viewable_asteroids[best_coord])

        # if 0 not in coord:
        #     reduced = Fraction(*coord)
        #     print(coord, reduced.numerator, reduced.denominator)
        # else:
        #     print(coord, coord[0], coord[1])

if __name__ == '__main__':
    with open('input.txt') as f:
        inp = f.read()
    #
    # inp = '.#....#####...#..\n' \
    #       '##...##.#####..##\n' \
    #       '##...#...#.#####.\n' \
    #       '..#.....#...###..\n' \
    #       '..#.#.....#....##' \
    #
    part2(inp)


text:
from OpenPermID import OpenPermID
import geocoder
import pandas as pd
import numpy as np
import os
from json import JSONDecodeError

# Gain access to the permid database
opid = OpenPermID()
opid.set_access_token("r95vEAhvmucG8iNGtsP17hjbgUGMhz4j")

data_folder = "https://s3groupsweden.s3.eu-central-1.amazonaws.com/Data/"

# Get Eikon information on holdings
def get_holdings_Eikon():
    holdingsEikon = pd.read_csv(data_folder + "EIKON/holdingsECBGeneralInfo.txt",sep="\t").iloc[:, :-3]
    holdingsEikon.rename(columns = {'Legal Entity ID': 'LEI'}, inplace = True)
    return holdingsEikon

# Get a dataframe with a non-redundant list of all the company names and their legal entity ID.
def get_companies_LEI(holdingsECB):
    companies_LEI = holdingsECB[['ISIN', 'ISSUER']].merge(get_holdings_Eikon()[['ISIN', 'LEI']], how = 'left', on = 'ISIN')
    companies_LEI = companies_LEI[['ISSUER', 'LEI']].drop_duplicates('ISSUER')
    return companies_LEI

# Returns a datframe mapping Legal Entity IDs to PermIDs
def get_LEI_mappings():
    LEIs = get_holdings_Eikon()['LEI'].dropna().astype('string').unique()

    lei_input = pd.DataFrame({})
    localID = 1
    for lei in LEIs:
        lei_input = lei_input.append(pd.DataFrame({'LocalID': localID, 'Standard Identifier': 'LEI:' + lei}, index = [0]))
        localID += 1

    while True:
        try:
            lei_df, err = opid.match(lei_input)
            break
        except JSONDecodeError:
            continue

    lei_mappings = pd.DataFrame({})
    lei_mappings['LEI'] = pd.Series(LEIs)
    lei_mappings['CompanyName'] = lei_df['Match OrgName']
    lei_mappings = pd.concat([lei_mappings, lei_df['Match OpenPermID'].astype('string').apply(lambda x: x.split('/')[-1])], axis = 1)
    lei_mappings.rename(columns = {'Match OpenPermID': 'PermID'}, inplace = True)
    return lei_mappings

# returns dataframe holding permIDs and some sector and corporate info
def get_sector_lookups():
    permids = get_LEI_mappings().PermID.astype('string')
    sector_lookups = pd.DataFrame({})
    unsuccessful_lookups = []

    for permid in permids:

        # In case of connection error, allow it to try at most 5 times
        err, count = 0, 0
        while (err != None and count < 5):
            try:
                count = count - 1
                output, err = opid.lookup(permid)
            except JSONDecodeError:
                continue
        if err != None:
            unsuccessful_lookups.append(permid)
            continue

        if "hasPrimaryBusinessSector" in output.columns:
            sector_info = output.loc[:, 'hasPrimaryBusinessSector': 'hasPrimaryIndustryGroup']
            sector_info = sector_info.applymap(lambda x: x.split('/')[-1])
        if "isIncorporatedIn" in output.columns:
            loc_info = output.loc[:, 'isIncorporatedIn': 'isDomiciledIn']
            loc_info = loc_info.applymap(lambda x: x.split('/')[-2])

        row = pd.DataFrame({'PermID': [permid]})
        row = pd.concat([row, sector_info], axis = 1) if type(sector_info) == pd.DataFrame else row
        row = pd.concat([row, loc_info], axis = 1) if type(loc_info) == pd.DataFrame else row
        sector_lookups = sector_lookups.append(row)

        sector_info, loc_info = None, None

    return sector_lookups

# converts company info from codes to their meaning
def convert_sector_lookups():
    sector_lookups = get_sector_lookups()
    sector_lookups_converted = sector_lookups.copy()
    sector_types = sector_lookups.columns[1:4]
    for sector_type in sector_types:
        sector_dict = {}
        sectors = sector_lookups.loc[:, sector_type].dropna().astype('string').unique()
        for sector in sectors:
            while True:
                try:
                    output, err = opid.lookup(sector)
                    break
                except JSONDecodeError:
                    continue
            sector_dict[sector] = output.iloc[0, -1]
        sector_lookups_converted[sector_type] = sector_lookups[sector_type].fillna('missing').astype('string').apply(lambda x: np.NaN if x == 'missing' else sector_dict[x])

    loc_types = sector_lookups.columns[4:]
    for loc_type in loc_types:
        loc_dict = {}
        locs = sector_lookups.loc[:, loc_type].dropna().astype('string').unique()
        for loc in locs:
            g = geocoder.geonames(loc, method='details', key='brian1998716')
            loc_dict[loc] = g.address
        sector_lookups_converted[loc_type] = sector_lookups[loc_type].fillna('missing').astype('string').apply(lambda x: np.NaN if x == 'missing' else loc_dict[x])

    return sector_lookups_converted

# saves all data aqcuired in this file to excel files in output, if they do not already exist.
# If these files do exist, they are read instead.
def get_sector_mappings(holdingsECB):
    path = __file__
    parent = os.path.join(path, os.pardir)
    parent = os.path.join(parent, os.pardir)
    if not (os.path.isfile(parent+'\output\sector_mappings_data.xlsx')):
        sector_mappings = get_LEI_mappings().merge(convert_sector_lookups(), how = 'left', on = 'PermID')
        sector_mappings = get_companies_LEI(holdingsECB).merge(sector_mappings, how = 'left', on = 'LEI')
        sector_mappings.to_excel(parent+'\output\sector_mappings_data.xlsx', engine='xlsxwriter')
    else:
        sector_mappings = pd.read_excel(parent+'\output\sector_mappings_data.xlsx', header=0)
    return sector_mappings

max_stars_repo_name: AukeVanSighem/ECB-Bond-Purchases

max_issues_repo_name: AukeVanSighem/ECB-Bond-Purchases

max_forks_repo_name: AukeVanSighem/ECB-Bond-Purchases


text:
import numpy
import math

class Datasets:
    train = None
    test = None
    validation = None
    classes = 0

    def __init__(self, train, test, validation, classes):
        self.train = train
        self.test = test
        self.validation = validation
        self.classes = classes

class DataSet:
    # images = numpy.empty(shape=[0])
    # labels = numpy.empty(shape=[0])
    # num_examples = 0

    def __init__(self, images, labels):
        self._images = images
        self._labels = labels
        self._epochs_completed = 0
        self._num_examples = images.shape[0]
        self._index_in_epoch = 0

    def display(self, idx, threshold=0.5):
        render = ''
        flat = self._images[idx]
        img = numpy.reshape(flat, [28, 28])
        for row in img:
            for col in row:
                if col > threshold:
                    render += '@'
                else:
                    render += '.'
            render += '\n'
        return render

    @property
    def epochs_completed(self):
        return self._epochs_completed

    @property
    def images(self):
        return self._images

    @property
    def labels(self):
        return self._labels

    @property
    def num_examples(self):
        return self._num_examples

    def next_batch(self, batch_size, fake_data=False, shuffle=True):
        """Return the next `batch_size` examples from this data set."""
        if fake_data:
            fake_image = [1] * 784
            fake_label = 0
            return [fake_image for _ in range(batch_size)], [
                fake_label for _ in range(batch_size)
            ]
        start = self._index_in_epoch
        # Shuffle for the first epoch
        if self._epochs_completed == 0 and start == 0 and shuffle:
            perm0 = numpy.arange(self._num_examples)
            numpy.random.shuffle(perm0)
            self._images = self.images[perm0]
            self._labels = self.labels[perm0]
        # Go to the next epoch
        if start + batch_size > self._num_examples:
            # Finished epoch
            self._epochs_completed += 1
            # Get the rest examples in this epoch
            rest_num_examples = self._num_examples - start
            images_rest_part = self._images[start:self._num_examples]
            labels_rest_part = self._labels[start:self._num_examples]
            # Shuffle the data
            if shuffle:
                perm = numpy.arange(self._num_examples)
                numpy.random.shuffle(perm)
                self._images = self.images[perm]
                self._labels = self.labels[perm]
            # Start next epoch
            start = 0
            self._index_in_epoch = batch_size - rest_num_examples
            end = self._index_in_epoch
            images_new_part = self._images[start:end]
            labels_new_part = self._labels[start:end]
            return numpy.concatenate((images_rest_part, images_new_part), axis=0), numpy.concatenate(
                (labels_rest_part, labels_new_part), axis=0)
        else:
            self._index_in_epoch += batch_size
            end = self._index_in_epoch
            return self._images[start:end], self._labels[start:end]


text:
# coding: utf-8

# In[5]:
import sys
sys.path.insert(0, '../chem_scripts') # add path to chem_scripts

import os
import numpy as np
import pandas as pd
import csv
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ModelCheckpoint, EarlyStopping, Callback, LearningRateScheduler, LambdaCallback
from keras.callbacks import TerminateOnNaN, EarlyStopping, ReduceLROnPlateau, CSVLogger, ModelCheckpoint

from keras.metrics import categorical_accuracy, binary_accuracy
from keras.utils import np_utils
from keras import backend as K
from sklearn.model_selection import StratifiedKFold, KFold

# In[16]:

get_ipython().run_line_magic('matplotlib', 'inline')

# # Variables to be changed

# In[6]:

taskname = "verytoxic"             ## Specify task to be predicted
tasktype = "classification"  ## Specify either classification or regression
datarep = "tabular"          ## Specify data representation

# In[10]:

# Specify dataset name
jobname = "tox_niehs"

# Specify network type
network_name = ""

# Specify location of data
homedir = os.path.dirname(os.path.realpath('__file__'))+"/data/"

if datarep == "image":
    network_name = "cnn"
    archdir = os.path.dirname(os.path.realpath('__file__'))+"/data/archive/"
    K.set_image_dim_ordering('tf')
    pixel = 80
    num_channel = 4
    channel = "engA"
elif datarep == "tabular":
    network_name = "mlp"
    archdir = os.path.dirname(os.path.realpath('__file__'))+"/data/"
elif datarep == "text":
    network_name = "rnn"
    archdir = os.path.dirname(os.path.realpath('__file__'))+"/data/"

# # Loading Data

# In[8]:

from chem_scripts import cs_load_csv, cs_load_smiles, cs_load_image, cs_create_dict, cs_prep_data_X, cs_prep_data_y, cs_data_balance
from chem_scripts import cs_compute_results, cs_keras_to_seaborn, cs_make_plots
from chem_scripts import cs_setup_mlp, cs_setup_rnn, cs_setup_cnn

# In[11]:

if datarep == "tabular":

    # Load training + validation data
    filename=archdir+jobname+"_tv_"+taskname+"_rdkit.csv"
    X, y = cs_load_csv(filename)

    # Load test data
    filename=archdir+jobname+"_int_"+taskname+"_rdkit.csv"
    X_test, y_test = cs_load_csv(filename)
    if tasktype == "classification":
        y_test, _ = cs_prep_data_y(y_test, tasktype=tasktype)

elif datarep == "text":

    # Load training + validation data
    filename=archdir+jobname+"_tv_"+taskname+"_smiles.csv"
    X, y = cs_load_smiles(filename)

    # Load test data
    filename=archdir+jobname+"_int_"+taskname+"_smiles.csv"
    X_test, y_test = cs_load_smiles(filename)
    if tasktype == "classification":
        y_test, _ = cs_prep_data_y(y_test, tasktype=tasktype)

    # Create dictionary
    characters, char_table, char_lookup = cs_create_dict(X, X_test)

    # Map chars to integers
    X = cs_prep_data_X(X, datarep=datarep, char_table=char_table)
    X_test = cs_prep_data_X(X_test, datarep=datarep, char_table=char_table)

elif datarep == "image":

    # Load training + validation data
    filename=archdir+jobname+"_tv_"+taskname
    X, y = cs_load_image(filename, channel=channel)

    # Load test data
    filename=archdir+jobname+"_int_"+taskname
    X_test, y_test = cs_load_image(filename, channel=channel)
    if tasktype == "classification":
        y_test, _ = cs_prep_data_y(y_test, tasktype=tasktype)

    # Reshape X to be [samples][channels][width][height]
    X = X.reshape(X.shape[0], pixel, pixel, num_channel).astype("float32")
    X_test = X_test.reshape(X_test.shape[0], pixel, pixel, num_channel).astype("float32")

# In[12]:

def f_nn(train_default=True):

    # Define counter for hyperparam iterations
    global run_counter
    run_counter += 1

    print('*** TRIAL: '+str(run_counter))
    print('*** PARAMETERS TESTING: '+str(params))

    # Intialize results file
    # Setup cross-validation
    if tasktype == "classification":
        cv_results = pd.DataFrame(columns=['Train Loss', 'Validation Loss', 'Test Loss', 'Train AUC', 'Validation AUC', 'Test AUC'])
        stratk = StratifiedKFold(n_splits=5, random_state=7)
        splits = stratk.split(X, y)
    elif tasktype == "regression":
        cv_results = pd.DataFrame(columns=['Train Loss', 'Validation Loss', 'Test Loss', 'Train RMSE', 'Validation RMSE', 'Test RMSE'])
        stratk = KFold(n_splits=5, random_state=7)
        splits = stratk.split(X, y)

    # Do cross-validation
    for i, (train_index, valid_index) in enumerate(splits):

        if prototype == True:
            if i > 0:
                break

        print("\nOn CV iteration: "+str(i))
        # Do standard k-fold splitting
        X_train, y_train = X[train_index], y[train_index]
        X_valid, y_valid = X[valid_index], y[valid_index]
        print("BEFORE Sampling: "+str(i)+" Train: "+str(X_train.shape)+" Valid: "+str(X_valid.shape))
        print("BEFORE Sampling: "+str(i)+" Train: "+str(y_train.shape)+" Valid: "+str(y_valid.shape))

        if tasktype == "classification":

            # Do class-balancing
            balanced_indices = cs_data_balance(y_train)
            X_train = X_train[balanced_indices]
            y_train = y_train[balanced_indices]

            balanced_indices = cs_data_balance(y_valid)
            X_valid = X_valid[balanced_indices]
            y_valid = y_valid[balanced_indices]

            # One-hot encoding
            y_train, y_class = cs_prep_data_y(y_train, tasktype=tasktype)   #ONLY DO THIS AFTER SPLITTING
            y_valid, y_class = cs_prep_data_y(y_valid, tasktype=tasktype)
            print("AFTER Sampling: "+str(i)+" Train: "+str(X_train.shape)+" Valid: "+str(X_valid.shape))
            print("AFTER Sampling: "+str(i)+" Train: "+str(y_train.shape)+" Valid: "+str(y_valid.shape))

        elif tasktype == "regression":

            y_class = 1

        # Setup network
        if datarep == "tabular":
            model, submodel = cs_setup_mlp(params, inshape=X_train.shape[1], classes=y_class)
        elif datarep == "text":
            model, submodel = cs_setup_rnn(params, inshape=X_train.shape[1], classes=y_class, char=characters)
        elif datarep == "image":
            model, submodel = cs_setup_cnn(params, inshape=(pixel, pixel, num_channel), classes=y_class)

        if i == 0:
            # Print architecture
            print(model.summary())
            # Save model
            model_json = submodel.to_json()
            filemodel=jobname+"_"+network_name+"_"+taskname+"_architecture_"+str(run_counter)+".json"
            with open(filemodel, "w") as json_file:
                json_file.write(model_json)

        # Setup callbacks
        filecp = jobname+"_"+network_name+"_"+taskname+"_bestweights_trial_"+str(run_counter)+"_"+str(i)+".hdf5"
        filecsv = jobname+"_"+network_name+"_"+taskname+"_loss_curve_"+str(run_counter)+"_"+str(i)+".csv"
        callbacks = [TerminateOnNaN(),
                     LambdaCallback(on_epoch_end=lambda epoch,logs: sys.stdout.flush()),
                     EarlyStopping(monitor='val_loss', patience=25, verbose=1, mode='auto'),
                     ModelCheckpoint(filecp, monitor="val_loss", verbose=1, save_best_only=True, mode="auto"),
                     CSVLogger(filecsv)]

        # Train model
        if train_default:
            if datarep == "image":
                datagen = ImageDataGenerator(rotation_range=180, fill_mode='constant', cval=0.)
                hist = model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
                                           epochs=nb_epoch, steps_per_epoch=X_train.shape[0]/batch_size,
                                           verbose=verbose,
                                           validation_data=(X_valid, y_valid),
                                           callbacks=callbacks)
            else:
                hist = model.fit(x=X_train, y=y_train,
                                 batch_size=batch_size,
                                 epochs=nb_epoch,
                                 verbose=verbose,
                                 validation_data=(X_valid, y_valid),
                                 callbacks=callbacks)

        # Visualize loss curve
        hist_df = cs_keras_to_seaborn(hist)
        cs_make_plots(hist_df)

        # Reload best model & compute results
        model.load_weights(filecp)
        y_preds_result = cs_compute_results(model, classes=y_class, df_out=cv_results,
                           train_data=(X_train,y_train),
                           valid_data=(X_valid,y_valid),
                           test_data=(X_test,y_test))

    # Calculate results for entire CV
    final_mean = cv_results.mean(axis=0)
    final_std = cv_results.std(axis=0)
    cv_results.to_csv('results_'+jobname+'_'+network_name+"_"+taskname+'.csv', index=False)

    # ouput prediction of testset
    with open("predictions_"+jobname+"_"+network_name+"_"+taskname+".csv", "w") as output:
        writer = csv.writer(output, lineterminator='\n')
        writer.writerows(y_preds_result)

    # Print final results
    print('*** TRIAL RESULTS: '+str(run_counter))
    print('*** PARAMETERS TESTED: '+str(params))

    if tasktype == "regression":
        print(('train_loss: %.3f +/- %.3f, train_rmse: %.3f +/- %.3f, val_loss: %.3f +/- %.3f, val_rmse: %.3f +/- %.3f, test_loss: %.3f +/- %.3f, test_rmse: %.3f +/- %.3f')
              %(final_mean[0], final_std[0], final_mean[3], final_std[3],
                final_mean[1], final_std[1], final_mean[4], final_std[4],
                final_mean[2], final_std[2], final_mean[5], final_std[5]))
    elif tasktype == "classification":
        print(('train_loss: %.3f +/- %.3f, train_auc: %.3f +/- %.3f, val_loss: %.3f +/- %.3f, val_auc: %.3f +/- %.3f, test_loss: %.3f +/- %.3f, test_auc: %.3f +/- %.3f')
              %(final_mean[0], final_std[0], final_mean[3], final_std[3],
                final_mean[1], final_std[1], final_mean[4], final_std[4],
                final_mean[2], final_std[2], final_mean[5], final_std[5]))

# In[13]:

# Network hyperparameters

# Hyperparams:
# Dropout: 0 to 0.5 (float)
# num_layer: 2 to 6 (int)
# relu_type: relu, elu, leakyrelu, prelu
# layerN_units: 16, 32, 64, 128, 256 (int)
# reg_flag: l1, l2, l1_l2, none
# reg_val: 1 to 6 (float)
if datarep == "tabular":
    params = {"dropval":0.5, "num_layer":2, "relu_type":"prelu",
              "layer1_units":128, "layer2_units":128, "layer3_units":128,
              "layer4_units":128, "layer5_units":128, "layer6_units":128,
              "reg_type": "l2", "reg_val": 2.5 }

# Hyperparams:
# Dropout: 0 to 0.5 (float)
# em_dim: 1 to 10 (int)
# num_layer: 1 to 3 (int)
# relu_type: relu, elu, leakyrelu, prelu
# conv units: 16, 32, 64, 128, 256 (int)
# layerN_units: 16, 32, 64, 128, 256 (int)
elif datarep == "text":
    params = {"em_dim":5, "conv_units":6, "dropval":0.5, "num_layer":2,
              "celltype":"GRU", "relu_type":"prelu",
              "layer1_units":12, "layer2_units":12, "layer3_units":12,
              "reg_type": "l2", "reg_val": 2}

# Hyperparams:
# Dropout: 0 to 0.5 (float)
# num_blockN: 1 to 5 (int)
# convN units: 16, 32, 64, 128, 256 (int)
elif datarep == "image":
    params = {"conv1_units":32, "conv2_units":32, "conv3_units":32,
              "conv4_units":32, "conv5_units":32, "conv6_units":32,
              "num_block1":3, "num_block2":3, "num_block3":3, "dropval":0}

# In[14]:

# Run settings

run_counter = 0
batch_size = 128
nb_epoch = 5
verbose = 1
prototype = True

# In[15]:

# if train_default is true, models will be trained.
# otherwise, the model will read saved weight from file.
f_nn(True)

max_stars_repo_name: Lenaxiao/ToxNet-Project

max_issues_repo_name: Lenaxiao/ToxNet-Project

max_forks_repo_name: Lenaxiao/ToxNet-Project


text:
from numba.typed import Dict
from numba import types, njit
import numpy as np

# Using Python dictionary
d = dict()
d[1] = [1.1, 1.2]
d[2] = [2.2]
d[3] = [3.3, 3.4, 3.5]
print(d)

# Using Numba dictionary is more efficient for passing into compiled region due to lower unboxing overheads
# The Dict.empty() constructs a typed dictionary.
# The key and value typed must be explicitly declared.
nd = Dict.empty(
    key_type = types.int64,
    value_type = types.float64[:]
)

# Assigning key-values using Numpy arrays
nd[1] = np.asarray([1.1, 1.2], dtype='f8')
nd[2] = np.asarray([2.2], dtype='f8')
nd[3] = np.asarray([3.3, 3.4, 3.5], dtype='f8')
print(nd)

@njit
def foo( nd ):
    nd[4] = np.asarray([4.4, 4.5], dtype='f8')
    return nd

print(foo(nd))

max_stars_repo_name: samaid/Numba-Examples

max_issues_repo_name: samaid/Numba-Examples

max_forks_repo_name: samaid/Numba-Examples

reason: import numpy,from numba


text:
import numpy as np
import logging
from pathlib import Path
import unittest
import urllib.request

from seistools.reader import SegyReader, bcd2dec

TEST_DATA_FOLDER = Path(__file__).parent.joinpath('data')
logger = logging.getLogger(__name__)

class TestBcd(unittest.TestCase):

    def test_bcd2dec(self):
        assert np.isnan(bcd2dec(12))
        assert bcd2dec(22) == 16
        assert bcd2dec(np.array([1, 22])) == 116

class TestReaderSegy(unittest.TestCase):

    def setUp(self) -> None:
        self.segy_file_int = TEST_DATA_FOLDER.joinpath('l10f1.sgy')
        if not self.segy_file_int.exists():
            logger.warning(f'Downloading {self.segy_file_int}')
            urllib.request.urlretrieve(
                "https://pubs.usgs.gov/of/1999/of99-449/disc2/seg_y/l10f1.sgy",
                self.segy_file_int
            )

    def test_segy_int(self):
        sr = SegyReader(self.segy_file_int)
        data, th = sr.read()
        assert data.shape == (100, 10000) == (sr.ntr, sr.ns)

    def test_segy_float(self):
        segy_file = "/Users/olivier/Documents/datadisk/das/segy_samples/" \
                    "q004d_lmostack_a1_prm1_rl68.sgy"
        sr = SegyReader(segy_file)
        assert sr.si == .002
        data, th = sr.read()
        assert data.shape == (96, 151) == (sr.ntr, sr.ns)
        # from easyqc.gui import viewseis
        # eqc = viewseis(data.view('<f'), sr.si)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: oliche/seismic-toolbox

max_issues_repo_name: oliche/seismic-toolbox

max_forks_repo_name: oliche/seismic-toolbox


text:
import numpy as np
import talib
import tulipy as ti

from collections import namedtuple

StochasticRSI = namedtuple('StochasticRSI', ['k', 'd'])

def srsi(candles: np.ndarray, period=14, sequential=False) -> StochasticRSI:
    """
    Stochastic RSI

    :param candles: np.ndarray
    :param period: int - default: 14
    :param sequential: bool - default=False

    :return: StochasticRSI
    """
    if not sequential and len(candles) > 240:
        candles = candles[-240:]

    rsi_np = talib.RSI(candles[:, 2], timeperiod=period)
    rsi_np = rsi_np[np.logical_not(np.isnan(rsi_np))]
    fast_k, fast_d = ti.stoch(rsi_np, rsi_np, rsi_np, 14, 3, 3)

    if sequential:
        fast_k = np.concatenate((np.full((candles.shape[0]-fast_k.shape[0]), np.nan), fast_k), axis=0)
        fast_d = np.concatenate((np.full((candles.shape[0]-fast_d.shape[0]), np.nan), fast_d), axis=0)
        return StochasticRSI(fast_k, fast_d)
    else:
        return StochasticRSI(fast_k[-1], fast_d[-1])

max_stars_repo_name: petie/jesse

max_issues_repo_name: petie/jesse

max_forks_repo_name: petie/jesse


text:
""" module for analysis of CGM outside the CLasses
"""
from __future__ import print_function, absolute_import, division, unicode_literals

import numpy as np
import warnings
import pdb

from scipy.special import gamma, gammainc
try:
    import mpmath
except ImportError:
    warnings.warn("I hope your exponent is positive..")
    from scipy.special import gammainc
    gflg=False
else:
    from mpmath import gammainc
    gflg=True

from astropy import units as u
from astropy import constants as const
from astropy.cosmology import Planck15

def dndx_rvir(Lrng=(0.001, 10), nL=1000, beta=0.2, rvir_Lstar=250.*u.kpc,
         phi_str_pref = 1.49, alpha = -1.05, Mstar = -20.44,
         cosmo=None):  #  ; M* - 5 log h
    """ Estimate dN/dX for a set of CGM assuming unit covering fraction
    Following Prochaska+11

    Use beta=0 and rvir_Lstar=300kpc for a constant CGM to 300kpc

    Parameters
    ----------
    Lrng : tuple, optional
      Range to integrate luminosity in terms of L*
    nL : int, optional
      Number of evaluations of L in that interval
    beta : float, optional
      Parameterization of rvir with L
       r_vir = 250 kpc * (L/L*)^beta
    phi_str, alpha, Mstar : float, float, float
        Blanton lum function
          Phi(M) = 0.4 log(10) Phi* 10^(-0.4 [M-M*][alpha+1]) exp(-
                    ; 10^(-0.4[M-M*] ) )
          Phi(L) = Phi* (L/L*)^alpha exp(-L/L*)
          Phi* has the funny units of 1e-2 h^3 Mpc^-3
    cosmo : Cosmology, optional
      Defaults to Planck15

    Returns
    -------
    Lval : ndarray
      Luminosities of evaluation
    dNdX : float
      Cumulative dNdX

    """
    # Cosmology
    if cosmo is None:
        cosmo = Planck15
    hubb = cosmo.H0.value / 100.
    # Constants
    phi_str_cgs = (phi_str_pref * 1e-2 * hubb**3) / u.Mpc**3
    dndx_const = (const.c / cosmo.H0).cgs
    # Cumulative
    Lval = np.linspace(Lrng[0], Lrng[1], nL)
    x = alpha + 1 + beta
    # Integrate
    if gflg:
        igmma = np.zeros_like(Lval)
        i0 = 1.-float(gammainc(x,Lrng[1], regularized=True))
        for kk,iLval in enumerate(Lval):
            igmma[kk] = i0 - (1-float(gammainc(x,iLval, regularized=True)))
    else:
        igmma = gammainc(x,Lrng[1]) - gammainc(x,Lval)
    #from scipy.special import gammainc as gic
    #ig2 = gic(x,Lrng[1]) - gic(x,Lval)
    #pdb.set_trace()
    dNdx_rvir = (dndx_const * phi_str_cgs * (np.pi * rvir_Lstar**2) * (
        gamma(x) * igmma)).decompose().value

    # Return
    return Lval, dNdx_rvir

max_stars_repo_name: pyigm/pyigm

max_issues_repo_name: pyigm/pyigm

max_forks_repo_name: pyigm/pyigm

reason: import numpy,from scipy,import mpmath,from mpmath,from astropy