text:
import io
import os
from os.path import join, dirname, abspath, exists
import sys
import json
import numpy as np
from utils.config_loader import path_parser
import utils.tools as tools

sys.path.insert(0, dirname(dirname(abspath(__file__))))

def get_summ_comp_root(model_name, n_iter, mode):
    """

    :param model_name:
    :param attn_weigh:
    :param doc_weigh:
    :param n_iter:
    :param mode: w or r
    :return:
    """
    dn_items = tools.get_dir_name_items(model_name, n_iter)
    root_dp = join(path_parser.graph, '-'.join(dn_items))

    if mode == 'r':
        if not exists(root_dp):
            raise ValueError('root_dp does not exists: {}'.format(root_dp))
    elif mode == 'w':
        if exists(root_dp):
            raise ValueError('root_dp already exists: {}'.format(root_dp))
        os.mkdir(root_dp)
    else:
        raise ValueError('Invalid mode: {}'.format(mode))

    return root_dp

def get_sim_mat_dp(summ_comp_root, mode):
    sim_mat_dp = join(summ_comp_root, 'sim_mat')

    if mode == 'r':
        if not exists(sim_mat_dp):
            raise ValueError('sim_mat_dp does not exists: {}'.format(sim_mat_dp))
    elif mode == 'w':
        if exists(sim_mat_dp):
            raise ValueError('sim_mat_dp already exists: {}'.format(sim_mat_dp))
        os.mkdir(sim_mat_dp)
    else:
        raise ValueError('Invalid mode: {}'.format(mode))

    return sim_mat_dp

def get_rel_vec_dp(summ_comp_root, mode):
    rel_vec_dp = join(summ_comp_root, 'rel_vec')

    if mode == 'r':
        if not exists(rel_vec_dp):
            raise ValueError('rel_vec_dp does not exists: {}'.format(rel_vec_dp))
    elif mode == 'w':
        if exists(rel_vec_dp):
            raise ValueError('rel_vec_dp already exists: {}'.format(rel_vec_dp))
        os.mkdir(rel_vec_dp)
    else:
        raise ValueError('Invalid mode: {}'.format(mode))

    return rel_vec_dp

def get_sid2abs_dp(summ_comp_root, mode):
    sid2abs_dp = join(summ_comp_root, 'sid2abs')

    if mode == 'r':
        if not exists(sid2abs_dp):
            raise ValueError('sid2abs_dp does not exists: {}'.format(sid2abs_dp))
    elif mode == 'w':
        if exists(sid2abs_dp):
            raise ValueError('sid2abs_dp already exists: {}'.format(sid2abs_dp))
        os.mkdir(sid2abs_dp)
    else:
        raise ValueError('Invalid mode: {}'.format(mode))

    return sid2abs_dp

def get_sid2score_dp(summ_comp_root, mode):
    sid2score_dp = join(summ_comp_root, 'sid2score')

    if mode == 'r':
        if not exists(sid2score_dp):
            raise ValueError('sid2score_dp does not exists: {}'.format(sid2score_dp))
    elif mode == 'w':
        if exists(sid2score_dp):
            raise ValueError('sid2score_dp already exists: {}'.format(sid2score_dp))
        os.mkdir(sid2score_dp)
    else:
        raise ValueError('Invalid mode: {}'.format(mode))

    return sid2score_dp

def dump_sim_mat(sim_mat, sim_mat_dp, cid):
    np.save(join(sim_mat_dp, cid), sim_mat)

def load_sim_mat(sim_mat_dp, cid):
    return np.load(join(sim_mat_dp, '{}.npy'.format(cid)))

def dump_rel_vec(rel_vec, rel_vec_dp, cid):
    np.save(join(rel_vec_dp, cid), rel_vec)

def load_rel_vec(rel_vec_dp, cid):
    return np.load(join(rel_vec_dp, '{}.npy'.format(cid)))

def dump_sid2abs(sid2abs, sid2abs_dp, cid):
    fp = '{}.json'.format(join(sid2abs_dp, cid))
    with io.open(fp, 'a') as f:
        json.dump(sid2abs, f)

def load_sid2abs(sid2abs_dp, cid):
    fp = '{}.json'.format(join(sid2abs_dp, cid))
    with io.open(fp, 'r') as f:
        return json.load(f)

def dump_sid2score(sid2score, sid2score_dp, cid):
    fp = '{}.json'.format(join(sid2score_dp, cid))
    with io.open(fp, 'a') as f:
        json.dump(sid2score, f)

def load_sid2score(sid2score_dp, cid):
    fp = '{}.json'.format(join(sid2score_dp, cid))
    with io.open(fp, 'r') as f:
        return json.load(f)

def load_components(sim_mat_dp, rel_vec_dp, sid2abs_dp, cid):
    sim_mat = load_sim_mat(sim_mat_dp, cid)
    rel_vec = load_rel_vec(rel_vec_dp, cid)
    sid2abs = load_sid2abs(sid2abs_dp, cid)

    components = {
        'sim_mat': sim_mat,
        'rel_vec': rel_vec,
        'sid2abs': sid2abs,
    }
    return components

max_stars_repo_name: yumoxu/querysum

max_issues_repo_name: yumoxu/querysum

max_forks_repo_name: yumoxu/querysum


text:
import numpy as np
from typing import List, Tuple
from numba import njit, prange
from numba.typed import List as nbList
from lightRaven.policy import PolicyBase, FAPolicy, TabularPolicy
from lightRaven.sampling import SamplingBase

class IS(SamplingBase):
    """
    The implementation of Importance Sampling Estimator
    """
    def __init__(self, dataset: List[Tuple[np.ndarray]], gamma=0.9):
        """
        Parameters
        ----------
        dataset : List[Tuple[np.ndarray]]
            A list of trajectories sampled by the behavioral policy.
            Contains (s, a, r, pi_b) for each timestamp
        gamma : float
            The discount factor. Mostly predefined by the MDP.
        """

        super(IS, self).__init__(dataset, gamma)

    @staticmethod
    @njit(parallel=True)
    def _calc_tb_est(dataset_s: nbList, dataset_a: nbList, dataset_r: nbList, dataset_pi: nbList,
                     n_data: int, pi_e_full: np.ndarray) -> np.ndarray:
        est = np.empty(n_data)

        for i in prange(n_data):
            traj_size = dataset_s[i].size

            # Calculating all importance weights across the trajectory
            importance_weights = np.array(
                [pi_e_full[dataset_s[i][j]][dataset_a[i][j]] / dataset_pi[i][j] for j in prange(traj_size)]
            )
            # This code is basically equivalent to:
            # """
            # importance_weight = 1.0
            # for j in prange(traj_size):
            #     s, a, pi_b = dataset_s[i][j], dataset_a[i][j], dataset_pi[i][j]
            #     importance_weight *= (pi_e_full[s][a] / pi_b)
            # """

            est[i] = importance_weights.prod() * np.sum(dataset_r[i])

        return est

    @staticmethod
    @njit(parallel=True)
    def _calc_fa_est(dataset_s: nbList, dataset_r: nbList, dataset_pi: nbList,
                     n_data: int, pi_e_full: nbList) -> np.ndarray:
        est = np.empty(n_data)

        for i in prange(n_data):
            traj_size = dataset_s[i].shape[0]

            # Calculating all importance weights across the trajectory
            importance_weights = np.array(
                [pi_e_full[i][j] / dataset_pi[i][j] for j in prange(traj_size)]
            )

            est[i] = importance_weights.prod() * np.sum(dataset_r[i])

        return est

    def get_episodic_est(self, idx=None) -> float:
        """ Calculate the IS estimate of the evaluation policy over one trajectory

        Parameters
        ----------
        idx : int
            The index of a specific trajectory.
        """
        assert isinstance(self.eval_policy, PolicyBase)
        if idx is None:
            trajectory_idx = int(np.random.randint(self.n_data, size=1))
        else:
            assert isinstance(idx, int)
            trajectory_idx = idx

        s_history = self.dataset_s[trajectory_idx]
        a_hisotry = self.dataset_a[trajectory_idx]
        r_history = self.dataset_r[trajectory_idx]
        pi_history = self.dataset_pi[trajectory_idx]

        importance_weights = np.array(list(map(
            self.get_importance_weights, enumerate(zip(s_history, a_hisotry, pi_history)))))
        return importance_weights.prod() * r_history.sum()

    def get_est(self) -> np.ndarray:
        if isinstance(self.eval_policy, FAPolicy):
            return self._calc_fa_est(self.dataset_s, self.dataset_r, self.dataset_pi,
                                     self.n_data, self.eval_policy_pi)
        elif isinstance(self.eval_policy, TabularPolicy):
            return self._calc_tb_est(self.dataset_s, self.dataset_a, self.dataset_r, self.dataset_pi,
                                     self.n_data, self.eval_policy_pi)

if __name__ == "__main__":
    # import pickle
    # from lightRaven.policy import TabularPolicy
    #
    # n_proc = 6
    # safe_size = 10000000
    #
    # dataset = pickle.load(open('../../dataset.pickle', "rb"))
    # print("Dataset Loaded!")
    # sampler_safe = IS(dataset, gamma=1, n_proc=n_proc)
    #
    # policy_theta = np.array([-1.11036236, 2.65686747, 2.72182632, 1.55307054, 1.12477341,
    #                          1.59304398, 1.05309918, -0.5498818, -0.84351466, -2.81680243,
    #                          0.60726548, 4.09403887, 1.77653363, 4.90496071, 0.84392019,
    #                          -1.13692561, -1.4197318, 0.46920946, -0.17280731, 1.84314176,
    #                          -0.4245307, 3.85206167, 0.62269739, 0.38913152, 1.45244381,
    #                          2.77060771, 2.03162951, -1.98119988, 1.89080223, -1.00368014,
    #                          0.42235472, 0.52445275, -0.96903325, 1.23880848, -1.65574226,
    #                          2.78958083, -2.34229196, 1.61340813, 0.35849971, 1.86086199,
    #                          -2.36073352, 2.98946907, -5.0315063, -0.22035491, -1.17246592,
    #                          0.3808939, 2.99011244, -1.59308376, 1.47085365, -0.9353486,
    #                          -0.45585766, -0.61237927, 1.75028373, 0.94675292, 1.38176854,
    #                          2.08527732, -1.8067575, -0.04508603, 0.35713237, 3.03952345,
    #                          1.94232556, 0.58669684, 2.35990571, 2.05242542, -0.29257493,
    #                          0.4249482, 2.37701991, 1.22865922, 1.28674961, -1.17742659,
    #                          -1.06667484, 5.5789368])
    # test_policy = TabularPolicy(18, 4, policy_theta)
    # sampler_safe.load_eval_policy(test_policy)
    # print(sampler_safe.get_est())

    import gym

    from lightRaven.utils import generate_dataset, safety_test
    from lightRaven.sampling import IS

    env_id = 'CartPole-v0'
    n_proc = 8
    train_size = 5000
    test_size = 5000

    env = gym.make(env_id)
    dataset_test = generate_dataset(env_id, test_size, n_proc=n_proc)

    obs_shape = env.observation_space.shape[0]
    act_shape = env.action_space.n

    theta = np.ones(obs_shape * act_shape)
    policy = FAPolicy(obs_shape, act_shape, theta)

    sampler_test = IS(dataset_test, gamma=1, n_proc=n_proc)
    sampler_test.load_eval_policy(policy)
    est = sampler_test.get_est()
    print(est.mean())

ext: py

reason: import numpy,from numba


text:
import torch
import numpy as np
from collections import defaultdict

from flatland.envs.agent_utils import RailAgentStatus

MOVEMENT_ARRAY = ((-1, 0), (0, 1), (1, 0), (0, -1))
#  @njit
def get_new_position(position, movement):
    return (position[0] + MOVEMENT_ARRAY[movement][0], position[1] + MOVEMENT_ARRAY[movement][1])

class Contradictions():
    def __init__(self):
        pass

    def reset(self, env):
        self.env = env
        self._build()

    def _build(self):
        self.rails = set()
        self.essential = defaultdict(set)

        for h in range(self.env.height):
            for w in range(self.env.width):
                pos = (h, w)
                transition_bit = bin(self.env.rail.get_full_transitions(*pos))
                total_transitions = transition_bit.count("1")
                if total_transitions > 0:
                    self.rails.add(pos)

        for sh, sw in self.rails:
            for sd in range(4):
                h, w, d = sh, sw, sd
                while True:
                    self.essential[(sh, sw, sd)].add((h, w))
                    cell_transitions = self.env.rail.get_transitions(h, w, d)
                    num_transitions = np.count_nonzero(cell_transitions)
                    if num_transitions != 1: break
                    for nd in range(4):
                        if cell_transitions[nd]:
                            h, w = get_new_position((h, w), nd)
                            d = nd
                            break

        self.bad = defaultdict(set)
        for h, w in self.rails:
            for d in range(4):
                for th, tw in self.essential[(h, w, d)]:
                    for td in range(4):
                        if (h, w) in self.essential[(th, tw, td)]:
                            self.bad[(h, w, d)].add((th, tw, td))

    def start_episode(self):
        self.cur_bad = set()

    def is_bad(self, handle, action):
        if self.env.agents[handle].status == RailAgentStatus.READY_TO_DEPART:
            pos = self.env.agents[handle].initial_position
            d = self.env.agents[handle].initial_direction
        else:
            pos = self.env.agents[handle].position
            d = self.env.agents[handle].initial_direction
        approx_pos, approx_dir = self.env.get_env_actions_new_pos(pos, self.env.agents[handle].direction, action)
        if (approx_pos[0], approx_pos[1], approx_dir) in self.cur_bad:
            return True
        return False

    def add_elem(self, handle, action):
        if self.env.agents[handle].status == RailAgentStatus.READY_TO_DEPART:
            pos = self.env.agents[handle].initial_position
        else:
            pos = self.env.agents[handle].position
        approx_pos, approx_dir = self.env.get_env_actions_new_pos(pos, self.env.agents[handle].direction, action)
        for e in self.bad[(approx_pos[0], approx_pos[1], approx_dir)]:
            self.cur_bad.add(e)

max_stars_repo_name: jbr-ai-labs/mamba

max_issues_repo_name: jbr-ai-labs/mamba

max_forks_repo_name: jbr-ai-labs/mamba

reason: import numpy


text:
import unittest
import numpy as np
from numpy.testing import assert_array_almost_equal

from .test_base import BaseSparrayTest, dense2d

class TestTrueDivision(BaseSparrayTest):

  def test_truediv(self):
    c = 3
    assert_array_almost_equal(dense2d / c, (self.sp2d / c).toarray())
    with np.errstate(divide='ignore'):
      assert_array_almost_equal(c / dense2d, c / self.sp2d)

  def test_itruediv(self):
    self.sp2d /= 1
    assert_array_almost_equal(dense2d, self.sp2d.toarray())
    b = np.random.random(dense2d.shape)
    self.sp2d /= b
    assert_array_almost_equal(dense2d / b, self.sp2d.toarray())

if __name__ == '__main__':
  unittest.main()

max_stars_repo_name: perimosocordiae/sparray

max_issues_repo_name: perimosocordiae/sparray

max_forks_repo_name: perimosocordiae/sparray

reason: import numpy,from numpy


text:
import cupy as _np

from minitf.kernel.core import primitive

# ----- Non-differentiable functions -----
nograd_functions = [
    # _np.ndim,
    # _np.shape,
    _np.size,
    _np.zeros_like,
    _np.ones_like,
    _np.sum,  # temporarily put it here as nograd function
    _np.random,
    _np.array,
    _np.linspace,
    _np.transpose,
]

# ----- Differentiable functions -----
grad_functions = [
    _np.add,
    _np.subtract,
    _np.multiply,
    _np.divide,
    _np.dot,
    _np.square,
    _np.average,
    _np.exp,
    _np.negative,
]

def ndim(x):
    return x.ndim

def shape(x):
    return x.shape

def asnumpy(x):
    return _np.asnumpy(x)

def wrap_namespace(old, new):
    for name, obj in old.items():
        if obj in nograd_functions:
            new[name] = obj
        elif obj in grad_functions:
            new[name] = primitive(obj)

wrap_namespace(_np.__dict__, globals())

max_stars_repo_name: guocuimi/minitf

max_issues_repo_name: guocuimi/minitf

max_forks_repo_name: guocuimi/minitf

reason: import cupy


text:
import logging
from pathlib import Path

import keras
import numpy as np
from keras.callbacks import LearningRateScheduler
from keras.layers import Dense, Activation
from keras.models import Sequential

from examples.gridworld.exceptions.CannotCloneWeightsOfDifferentModelException import \
    CannotCloneWeightsOfDifferentModelException
from reflrn.Interface.Model import Model

class GridWorldQValNNModel(Model):

    #
    # At init time we need
    # - An arbitrary name for the model, used also for save/load filename.
    # - The NN input dimension.
    # - The max number of possible actions from a grid location.
    # - A logging channel.
    #
    def __init__(self,
                 model_name: str,
                 input_dimension: int,
                 num_actions: int,
                 num_grid_cells: int,
                 lg: logging,
                 batch_size: int = 32,
                 num_epoch: int = 1,
                 lr_0: float = 0.001,
                 lr_min: float = 0.001):
        self.__lg = lg
        self.__agent_name = model_name
        self.__input_dimension = input_dimension
        self.__num_actions = num_actions
        self.__num_grid_cells = num_grid_cells
        self.__model = None
        self.__batch_size = batch_size
        self.__num_epoch = num_epoch
        self.__model_compiled = False
        self.__epochs = 10

        self.__lr_0 = lr_0
        self.__lr_min = lr_min
        self.__lr = lr_0
        self.__lr_epoch = 1

        return

    #
    # Return an un-compiled Keras model that has an architecture capable of learning
    # q-values of number and complexity for a grid-world problem given the number of
    # actions and specified input dimension.
    #
    def new_model(self):
        model = Sequential()
        # One Input Layer
        # Three Hidden Layers
        # Output layer has no activation as this a a QVal regression net.
        #
        # Used on QVals from a test grid 10 by 10 no regularization is needed and testing
        # with L2.Reg - kernel and dropout decreased the final accuracy even allowing significant training
        # past the point where the cost function had stabilised.
        #
        model.add(Dense(25, input_dim=self.__input_dimension, kernel_initializer='he_uniform'))
        model.add(Activation('relu'))
        model.add(Dense(50, kernel_initializer='he_uniform'))
        model.add(Activation('relu'))
        model.add(Dense(100, kernel_initializer='he_uniform'))
        model.add(Activation('relu'))
        model.add(Dense(200, kernel_initializer='he_uniform'))
        model.add(Activation('relu'))
        model.add(Dense(units=self.__num_actions, kernel_initializer='normal'))
        self.__model = model
        return self.__model

    #
    # Compile the model with an appropriate loss function and optimizer.
    #
    def compile(self):
        self.__model.compile(loss='mean_squared_error', optimizer='adam')
        self.__model_compiled = True
        return

    #
    # Predict the Q Values for the action space given the vector encoding of the
    # grid curr_coords.
    #
    def predict(self, x) -> [np.float]:
        self.__bootstrap_model()
        return self.__model.predict_on_batch(x)

    #
    # Given the replay memory train the model
    #
    def train(self, x, y) -> None:
        self.__bootstrap_model()
        self.__model.fit(x=x,
                         y=y,
                         batch_size=self.__batch_size,
                         epochs=self.__epochs,
                         verbose=2,
                         callbacks=[LearningRateScheduler(self.__lr_step_down_decay)])
        self.__inc_lr_epoch()  # count a global fitting call.
        return

    #
    # If the model has not been created or complied, do so.
    #
    def __bootstrap_model(self):
        if self.__model is None:
            self.new_model()
        if not self.__model_compiled:
            self.compile()
        return

    #
    # Reset the learning curr_coords of the model.
    #
    def reset(self) -> None:
        self.__lr = self.__lr_0
        self.__lr_epoch = 1

    #
    # Increment the global episode count. This can be used when the learning rate etc are driven
    # by an episode cycle that is not linked directly to model training calls.
    #
    def __inc_lr_epoch(self):
        self.__lr_epoch += 1

    #
    # Step Down decay of learning rate - use the global episode not the local one passed in
    #
    def __lr_step_down_decay(self, _) -> float:
        if self.__lr_epoch % 200 == 0:
            self.__lr -= 0.0001
            self.__lr = max(self.__lr_min, self.__lr)
        return self.__lr

    #
    # Clone the weights of the given model, or throw an exception
    # if it is not an instance of this class as we can only copy
    # weights of identical architecture
    #
    def clone_weights(self, model: 'Model') -> None:
        if not isinstance(model, type(self)):
            raise CannotCloneWeightsOfDifferentModelException(str(type(self)) + " <- " + str(type(model)))
        if self.__model is None:
            raise RuntimeError("Internal Model is value (None) as has not been initialised")
        else:
            self.__model.set_weights(model.get_weights())
        return

    #
    # Return the model weights.
    #
    def get_weights(self):
        if self.__model is None:
            raise RuntimeError("Internal Model is value (None) as has not been initialised")
        return self.__model.get_weights()

    #
    # Save the model using Keras built in save capability.
    #
    def save(self, filename: str) -> None:
        try:
            self.__model.save(filename)
        except Exception as exc:
            err = "Failed to save Keras model to file [" + filename + ": " + str(exc)
            self.__lg.error(err)
            raise RuntimeError(err)
        finally:
            pass
        return

    #
    # Load the current curr_coords of the model from a file that was saved
    # from the same Keras model architecture as the new_model method
    # creates.
    #
    def load(self, filename: str) -> None:
        try:
            if Path(filename).is_file():
                model = keras.models.load_model(filename)
            else:
                raise FileExistsError("Keras Model File: [" + filename + "] Not found")
        except Exception as exc:
            err = "Failed to load Keras Deep NN model from file [" + filename + ": " + str(exc)
            self.__lg.error(err)
            raise RuntimeError(err)
        finally:
            pass
        self.__model = model
        return

    @property
    def explain(self) -> bool:
        False

max_stars_repo_name: parrisma/TicTacToe-DeepLearning

max_issues_repo_name: parrisma/TicTacToe-DeepLearning

max_forks_repo_name: parrisma/TicTacToe-DeepLearning


text:
# Sebastian Raschka 2014-2020
# mlxtend Machine Learning Library Extensions
#
# Author: Sebastian Raschka <sebastianraschka.com>
#
# License: BSD 3 clause

import numpy as np
from sklearn.model_selection import train_test_split

class RandomHoldoutSplit(object):
    """Train/Validation set splitter for sklearn's GridSearchCV etc.

    Provides train/validation set indices to split a dataset
    into train/validation sets using random indices.

    Parameters
    ----------
    valid_size : float (default: 0.5)
        Proportion of examples that being assigned as
        validation examples. 1-`valid_size` will then automatically
        be assigned as training set examples.
    random_seed : int (default: None)
        The random seed for splitting the data
        into training and validation set partitions.
    stratify : bool (default: False)
        True or False, whether to perform a stratified
        split or not

    """

    def __init__(self, valid_size=0.5, random_seed=None, stratify=False):
        self.valid_size = valid_size
        self.random_seed = random_seed
        self.stratify = stratify

    def split(self, X, y, groups=None):
        """Generate indices to split data into training and test set.

        Parameters
        ----------
        X : array-like, shape (num_examples, num_features)
            Training data, where num_examples is the number of
            training examples and num_features is the number of features.

        y : array-like, shape (num_examples,)
            The target variable for supervised learning problems.
            Stratification is done based on the y labels.

        groups : object
            Always ignored, exists for compatibility.

        Yields
        ------
        train_index : ndarray
            The training set indices for that split.

        valid_index : ndarray
            The validation set indices for that split.
        """
        ind = np.arange(X.shape[0])
        if self.stratify:
            train_index, valid_index, _, _ = \
                    train_test_split(ind, y,
                                     test_size=self.valid_size,
                                     shuffle=True,
                                     stratify=y,
                                     random_state=self.random_seed)

        else:
            train_index, valid_index, _, _ = \
                    train_test_split(ind, y,
                                     test_size=self.valid_size,
                                     shuffle=True,
                                     stratify=y,
                                     random_state=self.random_seed)

        for i in range(1):
            yield train_index, valid_index

    def get_n_splits(self, X=None, y=None, groups=None):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        groups : object
            Always ignored, exists for compatibility.

        Returns
        -------
        n_splits : 1
            Returns the number of splitting iterations in the cross-validator.
            Always returns 1.
        """
        return 1

class PredefinedHoldoutSplit(object):
    """Train/Validation set splitter for sklearn's GridSearchCV etc.

    Uses user-specified train/validation set indices to split a dataset
    into train/validation sets using user-defined or random
    indices.

    Parameters
    ----------
    valid_indices : array-like, shape (num_examples,)
        Indices of the training examples in the training set
        to be used for validation. All other indices in the
        training set are used to for a training subset
        for model fitting.

    """

    def __init__(self, valid_indices):
        self.valid_indices = valid_indices

    def split(self, X, y, groups=None):
        """Generate indices to split data into training and test set.

        Parameters
        ----------
        X : array-like, shape (num_examples, num_features)
            Training data, where num_examples is the number of examples
            and num_features is the number of features.

        y : array-like, shape (num_examples,)
            The target variable for supervised learning problems.
            Stratification is done based on the y labels.

        groups : object
            Always ignored, exists for compatibility.

        Yields
        ------
        train_index : ndarray
            The training set indices for that split.

        valid_index : ndarray
            The validation set indices for that split.
        """

        ind = np.arange(X.shape[0])
        train_mask = np.ones(X.shape[0], dtype=np.bool)
        train_mask[self.valid_indices] = False
        valid_mask = np.where(train_mask, False, True)

        for i in range(1):
            yield ind[train_mask], ind[valid_mask]

    def get_n_splits(self, X=None, y=None, groups=None):
        """Returns the number of splitting iterations in the cross-validator

        Parameters
        ----------
        X : object
            Always ignored, exists for compatibility.

        y : object
            Always ignored, exists for compatibility.

        groups : object
            Always ignored, exists for compatibility.

        Returns
        -------
        n_splits : 1
            Returns the number of splitting iterations in the cross-validator.
            Always returns 1.
        """
        return 1


text:
"""
-------------------------------------------------------------------------
Module lattice
-------------------------------------------------------------------------
Generate various crystal lattice structures.

.. module:: lattice
    :platform:
    :synopsis: Module for the generation of various crystal lattice structures.

.. moduleauthor:: Michael Barbier <michael@unam.bilkent.edu.tr>
"""

# -------------------------------------------------------------------------
# Import libraries
# -------------------------------------------------------------------------
import cv2 as cv
import numpy as np
import math
from matplotlib import pyplot as plt
from skimage import io, filters

# -------------------------------------------------------------------------
# Functions
# -------------------------------------------------------------------------
def generate_lattice_square(a, n, dot_radius, shift_x, shift_y):
    """
    Generates an image with dots on a period square lattice with lattice parameter a
    and dot size given by the dot_radius.

    :param a: The lattice length parameter
    :param n: Number of the repeating the basic lattice in both x and y direction
    :param dot_radius: radius of the plotted dots (should be non-negative < a/2), if zero, single pixels are used
    :param shift_y: Shift the dot in x direction (should be non-negative < a)
    :param shift_x: Shift the dot in y direction (should be non-negative < a)
    :return: The generated img
    """
    sz = a * n
    shape = (sz, sz)

    img = np.zeros(shape, np.dtype('float'))
    for ix in range(0, int(sz/a)):
        for iy in range(0, int(sz/a)):
            if dot_radius == 0:
                img[ix*a+shift_x, iy*a+shift_y] = 1.0
            else:
                cv.circle(img, (ix*a+shift_x, iy*a+shift_y), radius=dot_radius, color=1, thickness=cv.FILLED)
    return img

def image_lattice(rr, shape, dot_radius):
    """
     Generates an image from given lattice points rr and lattice image size: shape, and radii of the dots: dot_radius

     :param rr: The lattice points as a list of (x, y) tuples
     :param shape: The corresponding lattice boundaries
     :param dot_radius: the radius of the plotted dots
     :return: The image of size shape with plotted lattice dots
    """
    img = np.zeros(shape, np.dtype('float'))
    n_rr = len(rr)
    for ii in range(0, n_rr):
        if dot_radius == 0:
            img[round(rr[ii][0]), round(rr[ii][1])] = 1.0
        else:
            rx = int(round(rr[ii][0]))
            ry = int(round(rr[ii][1]))
            cv.circle(img, (ry, rx), radius=dot_radius, color=1, thickness=cv.FILLED)
            #img = cv.circle(img, center, radius, color[, thickness[, lineType[, shift]]]    )

    return img

def generate_lattice(a1, a2, angle, n1, n2, shift_x, shift_y):
    """
    Generates a lattice of coordinates on a period lattice with lattice parameters a1, a2, under an angle angle
    TODO: incorrect boundary conditions, not all lattice points are there, take also extra border into account for
    lattice points which are outside but when plotted with finite radius could be partially inside.

    :param a1: The lattice length parameter of the first basis vector (parallel with the x-axis)
    :param a2: The lattice length parameter second basis vector (under angle with the x-axis)
    :param n1: Number of repeating the basic lattice in x direction
    :param n2: Number of repeating the basic lattice in y direction
    :param angle: The angle between the 2 basis vectors of the unit lattice
    :param shift_x: Shift the dot in x direction (should be non-negative < a)
    :param shift_y: Shift the dot in y direction (should be non-negative < a)
    :return: The generated lattice
    """
    shape = (a1*n1, round(a2*math.sin(angle)*n2))
    rr = []
    #np.zeros((n1*n2, 2), np.dtype('float'))
    ii = 0
    for ix in range(-1, n1+1):
        for iy in range(-1, n2+1):
            rx = ix * a1 + iy * a2 * math.cos(angle) + shift_x
            ry = iy * a2 * math.sin(angle) + shift_y
            if rx < 0:
                rx = shape[0]+rx
            if rx > shape[0]:
                rx = rx-shape[0]
            if ry < 0:
                ry = shape[1]+ry
            if ry > shape[1]:
                ry = ry-shape[1]
            rr.append((rx, ry))
            ii = ii + 1
    return rr, shape


text:
import numpy as np

def main1():
    # Before optimization
    x = np.array(range(10**7))
    y = np.array(np.random.uniform(0, 100, size=10**8))

def main2():
    # After optimization, spurious `np.array` removed.
    x = np.array(range(10**7))
    y = np.random.uniform(0, 100, size=10**8)

main1()
main2()

max_stars_repo_name: edisga/scalene

max_issues_repo_name: edisga/scalene

max_forks_repo_name: edisga/scalene


text:
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from rlschool.liftsim.environment.mansion.person_generators.generator_proxy import set_seed
from rlschool.liftsim.environment.mansion.person_generators.generator_proxy import PersonGenerator
from rlschool.liftsim.environment.mansion.mansion_config import MansionConfig
from rlschool.liftsim.environment.mansion.mansion_manager import MansionManager
from rlschool.liftsim.environment.mansion.utils import ElevatorAction

NoDisplay = False
try:
    from rlschool.liftsim.environment.animation.rendering import Render
except Exception as e:
    NoDisplay = True

import argparse
import configparser
import random
import sys
import os

class LiftSim():
    """
    environmentation Environment
    """
    def __init__(self, config_file=os.path.join(os.path.dirname(__file__)+'/../config.ini'), **kwargs):
        file_name = config_file

        config = configparser.ConfigParser()
        config.read(file_name)

        time_step = float(config['Configuration']['RunningTimeStep'])
        assert time_step <= 1, 'RunningTimeStep in config.ini must be less than 1 in order to ensure accuracy'

        # Readin different person generators
        gtype = config['PersonGenerator']['PersonGeneratorType']
        person_generator = PersonGenerator(gtype)
        person_generator.configure(config['PersonGenerator'])

        self._config = MansionConfig(
            dt=time_step,
            number_of_floors=int(config['MansionInfo']['NumberOfFloors']),
            floor_height=float(config['MansionInfo']['FloorHeight'])
        )

        if('LogLevel' in config['Configuration']):
            assert config['Configuration']['LogLevel'] in ['Debug', 'Notice', 'Warning'],\
                        'LogLevel must be one of [Debug, Notice, Warning]'
            self._config.set_logger_level(config['Configuration']['LogLevel'])
        if('Lognorm' in config['Configuration']):
            self._config.set_std_logfile(config['Configuration']['Lognorm'])
        if('Logerr' in config['Configuration']):
            self._config.set_err_logfile(config['Configuration']['Logerr'])

        self._mansion = MansionManager(
            int(config['MansionInfo']['ElevatorNumber']),
            person_generator,
            self._config,
            config['MansionInfo']['Name']
        )

        self.viewer = None

    def seed(self, seed=None):
        set_seed(seed)

    def step(self, action):
        assert type(action) is list, "Type of action should be list"
        assert len(action) == self._mansion.attribute.ElevatorNumber*2, \
            "Action is supposed to be a list with length ElevatorNumber * 2"
        action_tuple = []
        for i in range(self._mansion.attribute.ElevatorNumber):
            action_tuple.append(ElevatorAction(action[i*2], action[i*2+1]))
        time_consume, energy_consume, given_up_persons = self._mansion.run_mansion(action_tuple)
        reward = - (time_consume + 5e-4 * energy_consume +
                    300 * given_up_persons) * 1.0e-4
        info = {'time_consume':time_consume, 'energy_consume':energy_consume, 'given_up_persons': given_up_persons}
        return (self._mansion.state, reward, False, info)

    def reset(self):
        self._mansion.reset_env()
        return self._mansion.state

    def render(self):
        if self.viewer is None:
            if NoDisplay:
                raise Exception('[Error] Cannot connect to display screen. \
                    \n\rYou are running the render() function on a manchine that does not have a display screen')
            self.viewer = Render(self._mansion)
        self.viewer.view()

    def close(self):
        pass

    @property
    def attribute(self):
        return self._mansion.attribute

    @property
    def state(self):
        return self._mansion.state

    @property
    def statistics(self):
        return self._mansion.get_statistics()

    @property
    def log_debug(self):
        return self._config.log_notice

    @property
    def log_notice(self):
        return self._config.log_notice

    @property
    def log_warning(self):
        return self._config.log_warning

    @property
    def log_fatal(self):
        return self._config.log_fatal

max_stars_repo_name: ANCL/QuadPPO

max_issues_repo_name: ANCL/QuadPPO

max_forks_repo_name: ANCL/QuadPPO


text:
# --------------
#Importing header files
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

#Path of the file
path

#Code starts here

# Loding data
data = pd.read_csv(path)

# Renaming column Total to Total_Medals
data.rename(columns={"Total":"Total_Medals"},inplace=True)

# displaying first 10 records
print(data.head(10))

# --------------
#Code starts here

# creating column Better_Event which stores 'summer', 'winter' or 'both'
data['Better_Event'] = np.where(data['Total_Summer']>data['Total_Winter'],'Summer',np.where(data['Total_Summer']<data['Total_Winter'],'Winter','Both'))

# Finding better event wrt all performing countries
max_value = ((data.Better_Event.value_counts()).max())
better_event = data.Better_Event.value_counts()[data.Better_Event.value_counts()==max_value].index[0]
print(better_event)

# --------------
#Code starts here

# Creating dataframe top_countries with columns 'Country_Name','Total_Summer', 'Total_Winter','Total_Medals'
top_countries = data[['Country_Name','Total_Summer', 'Total_Winter','Total_Medals']]

# dropping last row from top_countries
top_countries.drop(index = top_countries.tail(1).index, inplace = True)

# definition of function that return list of top 10 countries
def top_ten(top_countries, col_name):
    """-------Function top ten--------
    input: dataframe, column name
    returns: Returns list of top 10 rows of dataframe based on the input column"""

    country_list = list((top_countries.nlargest(10, col_name))['Country_Name'])
    return country_list

# calculating top_10_summer, top_10_winter, top_10
top_10_summer = top_ten(top_countries,'Total_Summer')
top_10_winter = top_ten(top_countries,'Total_Winter')
top_10 = top_ten(top_countries,'Total_Medals')

# finding common contries in all the three lists
common = list(set(top_10_summer) & set(top_10_winter) & set(top_10))
print(common)

# --------------
#Code starts here
# Subsetting dataframe wrt top_10_summer, top_10_winter, top_10
summer_df = data[data['Country_Name'].isin(top_10_summer)]
winter_df = data[data['Country_Name'].isin(top_10_winter)]
top_df = data[data['Country_Name'].isin(top_10)]

# Plotting bar graphs
fig, ax = plt.subplots(3, sharex=True)
ax[0].bar(summer_df['Country_Name'],summer_df['Total_Summer'])
ax[0].set_title('top 10 country vs summer medals')
ax[1].bar(winter_df['Country_Name'],winter_df['Total_Winter'])
ax[1].set_title('top 10 country vs winter medals')
ax[2].bar(top_df['Country_Name'],top_df['Total_Medals'])
ax[2].set_title('top 10 country vs total medals')

# --------------
#Code starts here

# creating column Golden_Ratio in summerdf
summer_df['Golden_Ratio'] = summer_df['Gold_Summer']/summer_df['Total_Summer']

# finding country with maximum golden ratio in summer dtaframe
summer_max_ratio = np.max(summer_df['Golden_Ratio'])
summer_country_gold = str((summer_df[summer_df['Golden_Ratio']==summer_max_ratio].Country_Name).iloc[0])

# creating column Golden_Ratio in winterdf
winter_df['Golden_Ratio'] = winter_df['Gold_Winter']/winter_df['Total_Winter']

# finding country with maximum golden ratio in winter dtaframe
winter_max_ratio = np.max(winter_df['Golden_Ratio'])
winter_country_gold = str((winter_df[winter_df['Golden_Ratio']==winter_max_ratio].Country_Name).iloc[0])

# creating column Golden_Ratio in top_df
top_df['Golden_Ratio'] = top_df['Gold_Total']/top_df['Total_Medals']

# finding country with maximum golden ratio in top dtaframe
top_max_ratio = np.max(top_df['Golden_Ratio'])
top_country_gold = str((top_df[top_df['Golden_Ratio']==top_max_ratio].Country_Name).iloc[0])

# --------------
#Code starts here

# dropping last row from data and asigning it to data_1
data_1 = data[:-1]

# finding weighted sum of gold, silver and bronze medals
data_1['Total_Points'] = (3*(data_1['Gold_Total']))+(2*(data_1['Silver_Total']))+data_1['Bronze_Total']

# finding best country and its total points
most_points = data_1['Total_Points'].max()
best_country = str((data_1[data_1['Total_Points']==most_points].Country_Name).iloc[0])

# --------------
#Code starts here

# Creating best dataframe
best = data[data['Country_Name']==best_country]

# selecting only necessary columns
best = best[['Gold_Total','Silver_Total','Bronze_Total']]

# Creating a bar graph
best.plot.bar(stacked = True)

# adding labels
plt.xlabel('United States')
plt.ylabel('Medals Tally')
plt.xticks(rotation = 45)

max_stars_repo_name: reshmastadas/olympic-hero

max_issues_repo_name: reshmastadas/olympic-hero

max_forks_repo_name: reshmastadas/olympic-hero


text:
"""Test random_ginibre."""
import numpy as np

from toqito.random import random_ginibre

def test_random_ginibre_dims():
    """Generate random Ginibre matrix and check proper dimensions."""
    gin_mat = random_ginibre(2, 2)
    np.testing.assert_equal(gin_mat.shape[0], 2)
    np.testing.assert_equal(gin_mat.shape[1], 2)

if __name__ == "__main__":
    np.testing.run_module_suite()

max_stars_repo_name: paniash/toqito

max_issues_repo_name: paniash/toqito

max_forks_repo_name: paniash/toqito


text:
import os
import argparse
from itertools import combinations
import logging

import pandas as pd
import numpy as np

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data-dir', type=str, default='/opt/ml/processing/input')
    parser.add_argument('--output-dir', type=str, default='/opt/ml/processing/output')
    parser.add_argument('--transactions', type=str, default='transaction.csv', help='name of file with transactions')
    parser.add_argument('--identity', type=str, default='identity.csv', help='name of file with identity info')
    #columns with identity information related to a user or a transaction: IP, Phone Number... These column types become edge types in the graph, and the entries
    #in these columns become the nodes
    parser.add_argument('--id-cols', type=str, default='', help='comma separated id cols in transactions table')
    #columns with categorical information, such as age range. These columns are used as node attributes in the heterogeneous graph
    parser.add_argument('--cat-cols', type=str, default='', help='comma separated categorical cols in transactions')
    parser.add_argument('--train-data-ratio', type=float, default=0.8, help='fraction of data to use in training set')
    parser.add_argument('--construct-homogeneous', action="store_true", default=False,
                        help='use bipartite graphs edgelists to construct homogenous graph edgelist')
    return parser.parse_args()

    #numerical columns: columns with numerical features like number of times a transaction has been tried. Entries are used as node attributes

def get_logger(name):
    logger = logging.getLogger(name)
    log_format = '%(asctime)s %(levelname)s %(name)s: %(message)s'
    logging.basicConfig(format = log_format, level = logging.INFO)
    return logger

def load_data(data_dir, transaction_data, identity_data, train_data_ratio, output_dir):
    #load the transaction file for training
    transaction_df = pd.read_csv(os.path.join(data_dir, transaction_data), sep = ';')
    logging.info("Shape of transaction data is {}".format(transaction_df.shape))
    logging.info("# Tagged transactions: {}".format(len(transaction_df) - transaction_df.Risk.isnull().sum()))

    #load the identity attributes for training
    identity_df = pd.read_csv(os.path.join(data_dir, identity_data), sep = ';')
    logging.info("Shape of identity data is {}".format(identity_df.shape))
    print(identity_df)

    #split for test/validation
    n_train = int(transaction_df.shape[0] * train_data_ratio)
    test_ids = transaction_df.id.values[n_train:]

    #determinate the proportion of fraud transaction in the proposed data
    get_risk_frac = lambda x: 100 * sum(x)/len(x)
    logging.info("Percent risk for train transactions: {}".format(get_risk_frac(transaction_df.Risk[:n_train])))
    logging.info("Percent risk for test transactions: {}".format(get_risk_frac(transaction_df.Risk[n_train:])))
    logging.info("Percent risk for all transactions: {}".format(get_risk_frac(transaction_df.Risk)))

    #in the test.csv file hosted in the output folder it writes the TransactionIDs from the transactions that will be used for testing
    with open(os.path.join(output_dir, 'test.csv'), 'w') as f:
        f.writelines(map(lambda y: str(y) + '\n', test_ids))
    logging.info("Wrote test to file: {}".format(os.path.join(output_dir, 'test_csv')))

    return transaction_df, identity_df, test_ids

def get_features_and_labels(transactions_df, transactions_id_cols, transactions_cat_cols, output_dir):
    #get features
    #isFraud and TransactionDT are not features: one determines whether a transaction is labeled as fraud or not and the other one is an associated timedelta
    #columns in transactions_id_cols are related to identity attributes, not real features
    non_feature_cols = ['Risk'] + transactions_id_cols.split(",")
    #rest of the columns are feature ones
    feature_cols = [col for col in transactions_df.columns if col not in non_feature_cols]
    #make a distintion between categorical and numerical columns
    logging.info("Categorical columns: {}".format(transactions_cat_cols.split(",")))
    #convert each categorical variable into dummy variables. Fill NA with 0 (not every entry has values in its columns)
    features = pd.get_dummies(transactions_df[feature_cols], columns = transactions_cat_cols.split(",")).fillna(0)
    #apply logarithmic transformation to the amount value --> easier to interpret
    features['Credit amount'] = features['Credit amount'].apply(np.log10)
    logging.info("Transformed feature columns: {}".format(list(features.columns)))
    logging.info("Shape of features: {}".format(features.shape))
    #save the transformed features into a file in the output folder
    features.to_csv(os.path.join(output_dir, 'features.csv'), index = False, header = False)
    logging.info("Wrote features to file: {}".format(os.path.join(output_dir, 'features.csv')))

    #get labels
    transactions_df[['id','Risk']].to_csv(os.path.join(output_dir, 'tags.csv'), index = False)
    logging.info("Wrote labels to file: {}".format(os.path.join(output_dir, 'tags.csv')))

def get_relations_and_edgelist(transactions_df, identity_df, transactions_id_cols, output_dir):
    #get relations
    #transactions_id_cols are the identity columns specified in the command line (the take part of the transactions data)
    edge_types = list(identity_df.columns)
    logging.info("Found the following distinct relation types: {}".format(edge_types))
    #define id_cols as the columns with identity information from the transactions data
    #id_cols = ['id'] + transactions_id_cols.split(",")
    #merge the identity information from the transaction data with the whole identity data by the TransactionID column
    full_identity_df = identity_df
    logging.info("Shape of identity columns: {}".format(full_identity_df.shape))

    #get edges
    edges = {}
    for etype in edge_types:
        edgelist = full_identity_df[['id', etype]].dropna()
        edgelist.to_csv(os.path.join(output_dir, 'relation_{}_edgelist.csv').format(etype), index = False, header = True)
        logging.info("Wrote edgelist to: {}".format(os.path.join(output_dir, 'relation_{}_edgelist.csv').format(etype)))
        edges[etype] = edgelist

    return edges

def create_homogeneous_edgelist(edges, output_dir):
    homogeneous_edges = []
    for etype, relations in edges.items():
        for frame in relations.groupby(etype):
            new_edges = [(a, b) for (a, b) in combinations(frame.id.values,2)
                        if (a, b) not in homogeneous_edges and (b, a) not in homogeneous_edges]
            homogeneous_edges.extend(new_edges)

    with open(os.path.join(output_dir, 'homogeneous_edgelist.csv'), 'w') as f:
        f.writelines(map(lambda x: "{}, {}\n".format(x[0], x[1]), homogeneous_edges))
    logging.info("Wrote homogeneous edgelist to file: {}".format(os.path.join(output_dir, 'homogeneous_edgelist.csv')))

if __name__ == '__main__':
    logging = get_logger(__name__)

    args = parse_args()

    transactions, identity, test_transactions = load_data(args.data_dir,
                                                            args.transactions,
                                                            args.identity,
                                                            args.train_data_ratio,
                                                            args.output_dir)

    get_features_and_labels(transactions, args.id_cols, args.cat_cols, args.output_dir)
    relational_edges = get_relations_and_edgelist(transactions, identity, args.id_cols, args.output_dir)

    if args.construct_homogeneous:
        create_homogeneous_edgelist(relational_edges, args.output_dir)


text:
import glob
import os
import shutil
import collections
import timeit
import random
import shutil

import numpy as np
import torch

from envs.registration import make as gym_make
from .make_agent import make_agent
from .filewriter import FileWriter
from envs.wrappers import ParallelAdversarialVecEnv, VecMonitor, \
    VecNormalize, VecPreprocessImageWrapper, TimeLimit

class DotDict(dict):
    __getattr__ = dict.__getitem__
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__

    def __init__(self, dct):
        for key, value in dct.items():
            if hasattr(value, 'keys'):
                value = DotDict(value)
            self[key] = value

    def __getstate__(self):
        return self

    def __setstate__(self, state):
        self.update(state)
        self.__dict__ = self

def cprint(condition, *args, **kwargs):
    if condition:
        print(*args, **kwargs)

def init(module, weight_init, bias_init, gain=1):
    weight_init(module.weight.data, gain=gain)
    bias_init(module.bias.data)
    return module

def safe_checkpoint(state_dict, path, index=None, archive_interval=None):
    filename, ext = os.path.splitext(path)
    path_tmp = f'{filename}_tmp{ext}'
    torch.save(state_dict, path_tmp)

    os.replace(path_tmp, path)

    if index is not None and archive_interval is not None and archive_interval > 0:
        if index % archive_interval == 0:
            archive_path = f'{filename}_{index}{ext}'
            shutil.copy(path, archive_path)

def cleanup_log_dir(log_dir, pattern='*'):
    try:
        os.makedirs(log_dir)
    except OSError:
        files = glob.glob(os.path.join(log_dir, pattern))
        for f in files:
            os.remove(f)

def seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(seed)

def str2bool(v):
    if isinstance(v, bool):
       return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def get_obs_at_index(obs, i):
    if isinstance(obs, dict):
        return {k: obs[k][i] for k in obs.keys()}
    else:
        return obs[i]

def set_obs_at_index(obs, obs_, i):
    if isinstance(obs, dict):
        for k in obs.keys():
            obs[k][i] = obs_[k].squeeze(0)
    else:
        obs[i] = obs_[0].squeeze(0)

def is_discrete_actions(env, adversary=False):
    if adversary:
        return env.adversary_action_space.__class__.__name__ == 'Discrete'
    else:
        return env.action_space.__class__.__name__ == 'Discrete'

def _make_env(args):
    env_kwargs = {'seed': args.seed}
    if args.singleton_env:
        env_kwargs.update({
            'fixed_environment': True})

    env = gym_make(args.env_name, **env_kwargs)
    return env

def create_parallel_env(args, adversary=True):
    is_multigrid = args.env_name.startswith('MultiGrid')
    is_minihack = args.env_name.startswith('MiniHack')

    make_fn = lambda: _make_env(args)

    venv = ParallelAdversarialVecEnv([make_fn]*args.num_processes, adversary=adversary)
    venv = VecMonitor(venv=venv, filename=None, keep_buf=100)
    venv = VecNormalize(venv=venv, ob=False, ret=args.normalize_returns)

    obs_key = None
    scale = None
    transpose_order = None
    if is_multigrid:
        obs_key = 'image'
        scale = 10.0
        transpose_order = [2,0,1] # Channels first

    elif is_minihack:
        ued_venv = VecPreprocessImageWrapper(venv=venv)

    venv = VecPreprocessImageWrapper(venv=venv, obs_key=obs_key,
        transpose_order=transpose_order, scale=scale)

    if is_multigrid:
        ued_venv = venv

    if args.singleton_env:
        seeds = [args.seed]*args.num_processes
    else:
        seeds = [i for i in range(args.num_processes)]
    venv.set_seed(seeds)

    return venv, ued_venv

max_stars_repo_name: narendasan/paired

max_issues_repo_name: narendasan/paired

max_forks_repo_name: narendasan/paired


text:
import time

from numpy import where, average, std, isnan, inf, hstack, median, argmax, percentile
from corems import chunks
import warnings

#from matplotlib import pyplot
__author__ = "Yuri E. Corilo"
__date__ = "Jun 27, 2019"

class NoiseThresholdCalc:

    def get_noise_threshold(self) -> ( (float, float), (float,float) ):
        ''' return two tuples (min_mz, max_mz) , (noise_threshold, noise_threshold)'''

        if self.is_centroid:

            x = min(self.mz_exp), max((self.mz_exp))

            if self.settings.threshold_method == 'auto':

                abundance_threshold = self.baselise_noise + (self.settings.noise_threshold_std * self.baselise_noise_std)
                y = (abundance_threshold, abundance_threshold)

            elif self.settings.threshold_method == 'signal_noise':

                normalized_threshold = (self.max_abundance * self.settings.s2n_threshold )/self.max_signal_to_noise
                y = (normalized_threshold, normalized_threshold)

            elif self.settings.threshold_method == "relative_abundance":

                normalized_threshold = (max(self.abundance)/100)*self.settings.relative_abundance_threshold
                y = (normalized_threshold, normalized_threshold)

            else:
                    raise  Exception("%s method was not implemented, please refer to corems.mass_spectrum.calc.NoiseCalc Class" % self.settings.threshold_method)

            return x, y

        else:

            if self.baselise_noise and self.baselise_noise_std:

                x = (self.mz_exp_profile.min(), self.mz_exp_profile.max())
                y = (self.baselise_noise_std, self.baselise_noise_std)

                if self.settings.threshold_method == 'auto':

                    #print(self.settings.noise_threshold_std)
                    abundance_threshold = self.baselise_noise + (self.settings.noise_threshold_std * self.baselise_noise_std)

                    y = (abundance_threshold, abundance_threshold)

                elif self.settings.threshold_method == 'signal_noise':

                    max_sn = self.abundance_profile.max()/self.baselise_noise_std

                    normalized_threshold = (self.abundance_profile.max() * self.settings.s2n_threshold )/max_sn
                    y = (normalized_threshold, normalized_threshold)

                elif self.settings.threshold_method == "relative_abundance":

                    normalized_threshold = (self.abundance_profile.max()/100)*self.settings.relative_abundance_threshold
                    y = (normalized_threshold, normalized_threshold)

                else:
                    raise  Exception("%s method was not implemented, \
                        please refer to corems.mass_spectrum.calc.NoiseCalc Class" % self.settings.threshold_method)

                return x, y

            else:

                warnings.warn(
                    "Noise Baseline and Noise std not specified,\
                    defaulting to 0,0 run process_mass_spec() ?"
                )
                return (0,0) , (0,0)

    def cut_mz_domain_noise(self, auto):

        min_mz_whole_ms = self.mz_exp_profile.min()
        max_mz_whole_ms = self.mz_exp_profile.max()

        if auto:

            # this calculation is taking too long (about 2 seconds)
            number_average_molecular_weight = self.weight_average_molecular_weight(
                profile=True)

            # +-200 is a guess for testing only, it needs adjustment for each type of analysis
            # need to check min mz here or it will break
            min_mz_noise = number_average_molecular_weight - 100
            # need to check max mz here or it will break
            max_mz_noise = number_average_molecular_weight + 100

        else:

            min_mz_noise = self.settings.min_noise_mz
            max_mz_noise = self.settings.max_noise_mz

        if min_mz_noise < min_mz_whole_ms:
            min_mz_noise = min_mz_whole_ms

        if max_mz_noise > max_mz_whole_ms:
            max_mz_noise = max_mz_whole_ms

        #print(min_mz_noise, max_mz_noise)
        low_mz_index = (where(self.mz_exp_profile >= min_mz_noise)[0][0])
        #print(self.mz_exp_profile[low_mz_index])
        # low_mz_index = (argmax(self.mz_exp_profile <= min_mz_noise))

        high_mz_index = (where(self.mz_exp_profile <= max_mz_noise)[-1][-1])

        #high_mz_index = (argmax(self.mz_exp_profile <= max_mz_noise))

        if high_mz_index > low_mz_index:
            # pyplot.plot(self.mz_exp_profile[low_mz_index:high_mz_index], self.abundance_profile[low_mz_index:high_mz_index])
            # pyplot.show()
            return self.mz_exp_profile[high_mz_index:low_mz_index], self.abundance_profile[low_mz_index:high_mz_index]
        else:
            # pyplot.plot(self.mz_exp_profile[high_mz_index:low_mz_index], self.abundance_profile[high_mz_index:low_mz_index])
            # pyplot.show()
            return self.mz_exp_profile[high_mz_index:low_mz_index], self.abundance_profile[high_mz_index:low_mz_index]

    def from_posterior(self, param, samples):

        import pymc3 as pm
        import numpy as np
        import theano.tensor as tt
        from theano import as_op
        from scipy.stats import gaussian_kde

        smin, smax = np.min(samples), np.max(samples)
        width = smax - smin
        x = np.linspace(smin, smax, 100)
        y = gaussian_kde(samples)(x)

        # what was never sampled should have a small probability but not 0,
        # so we'll extend the domain and use linear approximation of density on it
        x = np.concatenate([[x[0] - 3 * width], x, [x[-1] + 3 * width]])
        y = np.concatenate([[0], y, [0]])

        return pm.distributions.Interpolated(param, x, y)

    def error_model_from_trace(self, trace, ymincentroid):

        import pymc3 as pm
        #from pymc3 import traceplot, plot_posterior

        with pm.Model() as model2:

            sd = self.from_posterior('sd', trace['sd'])
            y = pm.HalfNormal('y', sd=sd, observed=ymincentroid)
            start = pm.find_MAP()
            step = pm.NUTS() # Hamiltonian MCMC with No U-Turn Sampler
            trace = pm.sample(1000, step, start, random_seed=123, progressbar=True, tune=1000)
            pm.summary(trace)
            #plot_posterior(trace)
            #traceplot(trace)
            return pm.summary(trace)['mean'].values[0]

    def simple_model_error_dist(self,  ymincentroid):

        import pymc3 as pm
        # from pymc3 import traceplot, plot_posterior
        #import seaborn as sns
        #f, ax = pyplot.subplots(figsize=(6, 6))
        #sns.distplot(ymincentroid)
        #sns.kdeplot(ymincentroid, ax=ax, shade=True, color="g")
        #sns.rugplot(ymincentroid, color="black", ax=ax)
        #ax.set(xlabel= "Peak Minima Magnitude", ylabel= "Density")
        #pyplot.show()

        with pm.Model() as model:

            #mu = pm.Uniform('mu', lower=-1, upper=1)
            lower = ymincentroid.min()
            upper = ymincentroid.max()

            sd = pm.Uniform('sd', lower=lower , upper=upper)

            y = pm.HalfNormal('y', sd=sd, observed=ymincentroid)

            start = pm.find_MAP()
            step = pm.NUTS() # Hamiltonian MCMC with No U-Turn Sampler
            trace = pm.sample(1000, step, start, random_seed=123, progressbar=True, tune=1000)

            return pm.summary(trace)['mean'].values[0]

    def get_noise_average(self, ymincentroid, auto=True, bayes=False):
        # assumes noise to be gaussian and estimate noise level by
        # calculating the valley. If bayes is enable it will
        # model the valley distributuion as half-Normal and estimate the std

        average_noise = median((ymincentroid))*2 if auto else median(ymincentroid)

        if bayes:

            s_deviation = self.simple_model_error_dist(ymincentroid)

        else:

            s_deviation = ymincentroid.std()*3 if auto else ymincentroid.std()

        return average_noise, s_deviation

    def get_abundance_minima_centroid(self, mz_cut, abun_cut):

        maximum = self.abundance_profile.max()
        threshold_min = (maximum * 1.00)

        y = -abun_cut

        dy = y[1:] - y[:-1]
        '''replaces NaN for Infinity'''
        indices_nan = where(isnan(y))[0]

        if indices_nan.size:

            y[indices_nan] = inf
            dy[where(isnan(dy))[0]] = inf

        indices = where((hstack((dy, 0)) < 0) & (hstack((0, dy)) > 0))[0]

        if indices.size and threshold_min is not None:
            indices = indices[abun_cut[indices] <= threshold_min]

        # pyplot.plot(mz_cut[indices], abun_cut[indices], linewidth=0, marker='o', color='red', markersize=2)
        # pyplot.show()

        # pyplot.hist(x=abun_cut[indices], bins='auto', color='#0504aa',
                            # alpha=0.7, rwidth=0.85)

        # cutoff = percentile(abun_cut[indices], 0.98)

        # print(cutoff)

        # pyplot.show()
        return abun_cut[indices]

    def run_noise_threshold_calc(self, auto, bayes=False):

        if self.is_centroid:
            # calculates noise_baseline and noise_std
            # needed to run auto noise threshold mode
            # it is not used for signal to noise nor
            # relative abudance methods
            abundances_chunks = chunks(self.abundance, 50)
            each_min_abund = [min(x) for x in abundances_chunks]

            return average(each_min_abund), std(each_min_abund)

        else:

            mz_cut, abundance_cut = self.cut_mz_domain_noise(auto)

            if auto:

                yminima = self.get_abundance_minima_centroid(mz_cut, abundance_cut)

                return self.get_noise_average(yminima, auto=auto, bayes=bayes)

            else:

                # pyplot.show()
                return self.get_noise_average(abundance_cut,auto=auto, bayes=bayes)

max_stars_repo_name: Kzra/CoreMS

max_issues_repo_name: Kzra/CoreMS

max_forks_repo_name: Kzra/CoreMS

reason: import numpy,from numpy,from scipy,import theano,from theano,import pymc3,from pymc3


text:
import numpy as np

def file_lines(fp):
    nlines = 0
    with open(fp, 'r') as f:
        for line in f:
            nlines += 1
    return nlines

sfrlo_fname = "sfr_suberr.npy"
sfrmid_fname = "sfr_middle.npy"
sfrhi_fname = "sfr_adderr.npy"

logm = np.linspace(7, 16, 46)

# Set up output arrays
nmass = logm.size
nsnap = file_lines("sfr_7.0.dat")
scale = np.loadtxt("sfr_7.0.dat", unpack=True)[0]
redshift = 1./scale - 1.

out_logm        = logm.copy()
out_z           = redshift.copy()
out_sfr_merr    = np.zeros((nmass, nsnap))
out_sfr_perr    = np.zeros((nmass, nsnap))
out_sfr_mean    = np.zeros((nmass, nsnap))

for i, lm in enumerate(logm):
    fname = "sfr_%.1f.dat" % (lm)

    try:
        scale, sfr, errup, errdown = np.loadtxt(fname, unpack=True)
        startsnap = nsnap - scale.size

        out_sfr_merr[i, startsnap:] = sfr - errdown
        out_sfr_perr[i, startsnap:] = sfr + errup
        out_sfr_mean[i, startsnap:] = sfr
    except:
        pass

# z needs to be monotonically increasing for later interpolation
out_z = out_z[::-1]
out_sfr_merr = out_sfr_merr[:, ::-1]
out_sfr_perr = out_sfr_perr[:, ::-1]
out_sfr_mean = out_sfr_mean[:, ::-1]

np.save(sfrlo_fname, (out_logm, out_z, out_sfr_merr))
np.save(sfrhi_fname, (out_logm, out_z, out_sfr_perr))
np.save(sfrmid_fname, (out_logm, out_z, out_sfr_mean))


text:
'''
This file contains configuration settings for the neural network.
'''

import json
from copy import deepcopy
import numpy as np

class ConfigNN():
    def __str__(self):
        ss = []
        for k,v in self.__dict__.items():
            ss.append('%s: %s'%(k,str(v)))
        return '\n'.join(ss)

    def set(self, attrs, value):
        for attr in attrs:
            self.__dict__[attr] = value

    def ser(self):
        d = {k:v for k,v in self.__dict__.items() if k[:2] != '__'}
        s = json.dumps(d)
        return s

    def des(self, s):
        d = json.loads(s)
        for k,v in d.items():
            setattr(self, k, v)

    def validate_attr(self):
        valid_attr = ['name', 'description', 'seed', 'epochs', 'lr',
            'train', 'validate', 'train_1ep_fp', 'dataset',
            'add_drift', 'pt_init', 'qbits', 'use_bn', 'norm_b',
            'upc1', 'upc2', 'upc3', 'upc4', 'upd1', 'upd2',
            'train_b', 'train_W', 'quant_regen'
        ]
        for attr in self.__dict__.keys():
            if attr[:2] == '__' or attr in valid_attr:
                continue
            print('WARNING: invalid ConfigNN attribute found in %s: %s'%(self.name, attr))

conf_base = ConfigNN()

### Primary Settings ###

# Every exeriment should have a name.
conf_base.name = 'lrt-base'

# Every experiment should provide a description that possibly references another experiment.
conf_base.description = 'Base configuration.'

# Random seed for reproducibility (None for non-deterministic).
conf_base.seed = 0

# Number of passes through dataset. For online training, should set to 1.
conf_base.epochs = 1

# Whether to run with online training or only inference.
conf_base.train = True

# Whether to validate the runs (faster if no validation).
conf_base.validate = True

# Whether to start with an epoch of floating point training.
conf_base.train_1ep_fp = False

# There are several datasets.
# Normal/standard datasets: MNIST, SVHN, CIFAR10.
# Augmented datasets:
#   MNISTATR: Similar to MNIST, but for the other MNISTA* datasets.
#   MNISTAON: MNIST augmented to 500k samples with elastic transforms.
#   MNISTADS: MNISTAON with input distribution shifts every 10k images.
#   MNISTAON2k: 2k-length version of MNISTAON for testing (also 10k, 100k).
conf_base.dataset = 'MNISTAON2k'

# Adds noise to weights after each sample. There are two versions (version, param):
#   version = 'analog': Adds Gaussian noise such that over 1e6 samples, the total
#       standard deviation of added noise is equal to param. Mathematically,
#       Gaussian N(mu = 0, sigma = param / sqrt(1e6)) is added at each step.
#   version = 'digital': Adds bit-wise flips such that over 1e6 samples, approximately
#       param fraction of binary bits have been flipped.
# Set to `None` for no drift.
conf_base.add_drift = None

# Initializes from the PyTorch model file provided.
# Set to `None` for no initialization.
conf_base.pt_init = None

### Detailed Settings ###

# Quantization settings.
conf_base.qbits = {
    'w':-8,
    'b':-16,
    'a':+8,
    'g':-8,
    'wmax':1.0,
    'bmax':8.0,
    'amax':2.0,
    'gmax':1.0,
}

# Use batchnorm after conv2/4.
conf_base.use_bn = True

# Base learning rate.
conf_base.lr = 1e-2

# Weight update settings.
conf_base.norm_b = True

upConv = ('SKS', {
    'lr':1e-2, 'rank':4, 'zerovar':False, 'kappa_th':100,
    'pseudo_batch':10, 'lr_pb_pow':0.5,
    'norm_uv':'post', 'lr_pb_pow_post':0.0,
    'min_density':1e-2, 'discount':1.0,
    'rho_samp':1.0, 'lookahead':False})
upFC = ('SKS', {
    'lr':1e-2, 'rank':4, 'zerovar':False, 'kappa_th':100,
    'pseudo_batch':100, 'lr_pb_pow':0.5,
    'norm_uv':'post', 'lr_pb_pow_post':0.0,
    'min_density':1e-2, 'discount':1.0,
    'rho_samp':1.0, 'lookahead':False})

upConvStandard = ('Standard', {'lr':1e-2, 'norm_uv':'post', 'count_version':2})
upFCStandard = upConvStandard

convs = ['upc1', 'upc2', 'upc3', 'upc4']
fcs = ['upd1', 'upd2']

conf_base.set(convs, upConv)
conf_base.set(fcs, upFC)

conf_base.train_b = True
conf_base.train_W = True
conf_base.quant_regen = True

# Reference to config.
configs = {
    conf_base.name:conf_base,
}

# Another baseline model.
conf = deepcopy(conf_base)
conf.name = 'lrt-000'
conf.description = 'lrt-base with small online dataset.'
conf.dataset = 'MNISTAON10k'
configs[conf.name] = conf

### Hyperparameter Selection ###

# Baseline and standard weight update learning rate.
i = 0
for lr in 1e-5 * 2**np.arange(0, 21, 2):
    conf = deepcopy(configs['lrt-000'])
    conf.name = 'lrt-001-%03d'%i
    conf.description = 'lrt-000 with base update LR %f.'%(lr)
    conf.validate = False
    conf.lr = lr
    conf.norm_b = False
    stand = deepcopy(upConvStandard)
    stand[1]['lr'] = lr
    stand[1]['norm_uv'] = None
    for l in convs + fcs:
        conf.__dict__[l] = stand
    configs[conf.name] = conf

    conf = deepcopy(configs['lrt-000'])
    conf.name = 'lrt-005-%03d'%i
    conf.description = 'lrt-000 with base update LR %f and post-norming.'%(lr)
    conf.validate = False
    conf.lr = lr
    stand = deepcopy(upConvStandard)
    stand[1]['lr'] = lr
    for l in convs + fcs:
        conf.__dict__[l] = stand
    configs[conf.name] = conf
    i += 1

# Learning rate / batch size experiments for SKS.
i = 0
for pbc, pbd in [(1, 10), (3, 30), (10, 100), (30, 300), (100, 1000)]:
    for lr in 1e-3 * 2**np.arange(0, 15, 2):
        conf = deepcopy(configs['lrt-000'])
        conf.name = 'lrt-002-%03d'%i
        conf.description = 'lrt-000 with LR %f; Pseudobatch Conv %d, FC %d.'%(lr, pbc, pbd)
        conf.validate = False
        conf.norm_b = False
        for l in convs:
            conf.__dict__[l][1]['lr'] = lr
            conf.__dict__[l][1]['norm_uv'] = None
            conf.__dict__[l][1]['pseudo_batch'] = pbc
        for l in fcs:
            conf.__dict__[l][1]['lr'] = lr
            conf.__dict__[l][1]['norm_uv'] = None
            conf.__dict__[l][1]['pseudo_batch'] = pbd
        configs[conf.name] = conf

        conf = deepcopy(configs['lrt-000'])
        conf.name = 'lrt-006-%03d'%i
        conf.description = 'lrt-000 with LR %f; Pseudobatch Conv %d, FC %d and post-norming.'%(lr, pbc, pbd)
        conf.validate = False
        for l in convs:
            conf.__dict__[l][1]['lr'] = lr
            conf.__dict__[l][1]['pseudo_batch'] = pbc
        for l in fcs:
            conf.__dict__[l][1]['lr'] = lr
            conf.__dict__[l][1]['pseudo_batch'] = pbd
        configs[conf.name] = conf
        i += 1

### Ablations ###

# Ablations - bitwidths / ranks.
i = 0
for w_bl in [3, 4, 5, 6, 7, 8, 1, 2]:
    for rank in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:
        conf = deepcopy(configs['lrt-000'])
        conf.name = 'lrt-010-%03d'%i
        conf.description = 'lrt-000 with %d/16/8/8 w/b/a/g and rank %d.'%(w_bl, rank)
        conf.dataset = 'MNISTAON2k'
        conf.validate = False
        conf.qbits['w'] = -w_bl
        conf.norm_b = False
        for l in convs + fcs:
            conf.__dict__[l][1]['rank'] = rank
            conf.__dict__[l][1]['norm_uv'] = None
        configs[conf.name] = conf

        conf = deepcopy(configs['lrt-000'])
        conf.name = 'lrt-015-%03d'%i
        conf.description = 'lrt-000 with %d/16/8/8 w/b/a/g and rank %d and max-norming.'%(w_bl, rank)
        conf.dataset = 'MNISTAON2k'
        conf.validate = False
        conf.qbits['w'] = -w_bl
        for l in convs + fcs:
            conf.__dict__[l][1]['rank'] = rank
        configs[conf.name] = conf
        i += 1

# Ablations - OK vs SVD.
for seed in range(5):
    v = 'a'
    for conv_zv in [True, False]:
        for fc_zv in [True, False]:
            conf = deepcopy(configs['lrt-000'])
            conf.name = 'lrt-011%s%d'%(v,seed)
            conf.description = 'lrt-000 with conv_zv %s and fc_zv %s.'%(str(conv_zv), str(fc_zv))
            conf.validate = False
            conf.seed = seed
            for l in convs:
                conf.__dict__[l][1]['zerovar'] = conv_zv
            for l in fcs:
                conf.__dict__[l][1]['zerovar'] = fc_zv
            configs[conf.name] = conf

            conf = deepcopy(configs['lrt-000'])
            conf.name = 'lrt-017%s%d'%(v,seed)
            conf.description = 'lrt-000 with conv_zv %s and fc_zv %s (no-norm).'%(str(conv_zv), str(fc_zv))
            conf.validate = False
            conf.seed = seed
            conf.norm_b = False
            for l in convs:
                conf.__dict__[l][1]['norm_uv'] = None
                conf.__dict__[l][1]['zerovar'] = conv_zv
            for l in fcs:
                conf.__dict__[l][1]['norm_uv'] = None
                conf.__dict__[l][1]['zerovar'] = fc_zv
            configs[conf.name] = conf
            v = chr(ord(v) + 1)

# Ablations - Clear various binary flags.
for seed in range(5):
    v = 'a'
    for clear in ['use_bn', 'train_b', 'train_W']:
        conf = deepcopy(configs['lrt-000'])
        conf.name = 'lrt-012%s%d'%(v, seed)
        conf.description = 'lrt-000 with %s False.'%clear
        conf.validate = False
        conf.seed = seed
        conf.__dict__[clear] = False
        configs[conf.name] = conf

        conf = deepcopy(configs['lrt-000'])
        conf.name = 'lrt-018%s%d'%(v, seed)
        conf.description = 'lrt-000 with %s False (no-norm).'%clear
        conf.validate = False
        conf.seed = seed
        conf.norm_b = False
        for l in convs + fcs:
            conf.__dict__[l][1]['norm_uv'] = None
        conf.__dict__[clear] = False
        configs[conf.name] = conf

        v = chr(ord(v) + 1)
    conf = deepcopy(configs['lrt-000'])
    conf.name = 'lrt-012%s%d'%(v, seed)
    conf.description = 'lrt-000 again.'
    conf.seed = seed
    configs[conf.name] = conf

# Ablations - misc SKS-related ablations.
modifications = [
    ({}, {}),
    ({'kappa_th':10}, {'kappa_th':10}),
    ({'kappa_th':1e8}, {'kappa_th':1e8}),
    ({'min_density':1e-1}, {'min_density':1e-1}),
    ({'min_density':0}, {'min_density':0}),
    ({'discount':0.9}, {'discount':0.9}),
    ({'norm_uv':None}, {'norm_uv':None}),
]
for seed in range(5):
    for i, mod in enumerate(modifications):
        vv = chr(ord('a') + i)
        conf = deepcopy(configs['lrt-000'])
        conf.name = 'lrt-013%s%d'%(vv, seed)
        conf.description = 'lrt-000 with conv mods %s; fc mods %s.'%(str(mod[0]), str(mod[1]))
        conf.validate = False
        conf.seed = seed
        for l in convs:
            for k,v in mod[0].items():
                conf.__dict__[l][1][k] = v
        for l in fcs:
            for k,v in mod[1].items():
                conf.__dict__[l][1][k] = v
        configs[conf.name] = conf

        conf = deepcopy(configs['lrt-000'])
        conf.name = 'lrt-019%s%d'%(vv, seed)
        conf.description = 'lrt-000 with conv mods %s; fc mods %s (no-norm).'%(str(mod[0]), str(mod[1]))
        conf.validate = False
        conf.seed = seed
        conf.norm_b = False
        for l in convs:
            conf.__dict__[l][1]['norm_uv'] = None
            for k,v in mod[0].items():
                conf.__dict__[l][1][k] = v
        for l in fcs:
            conf.__dict__[l][1]['norm_uv'] = None
            for k,v in mod[1].items():
                conf.__dict__[l][1][k] = v
        configs[conf.name] = conf

### Main Experiments ###

# Different training levels for weights for different MNIST Aug datasets.
modifications = [
    {'dataset':'MNISTAON100k'},
    {'dataset':'MNISTAON100k', 'drift':('analog', 10.0)},
    {'dataset':'MNISTAON100k', 'drift':('digital', 10.0)},
    {'dataset':'MNISTADS100k'},
]
for i, mod in enumerate(modifications):
    v = i+20
    conf = deepcopy(conf_base)
    conf.name = 'lrt-%03da'%v
    conf.description = 'lrt-base with PT init, inference only, and %s.'%str(mod)
    conf.pt_init = 'MNISTATR_model.pt'
    conf.dataset = mod['dataset']
    conf.add_drift = mod.get('drift')
    conf.train = False
    configs[conf.name] = conf

    conf = deepcopy(configs['lrt-%03da'%v])
    conf.name = 'lrt-%03db'%v
    conf.description = 'lrt-%03da with bias-only/no-norm.'%v
    conf.train = True
    conf.norm_b = False
    conf.train_W = False
    configs[conf.name] = conf

    conf = deepcopy(configs['lrt-%03da'%v])
    conf.name = 'lrt-%03dc'%v
    conf.description = 'lrt-%03da with standard SGD/no-norm.'%v
    conf.train = True
    stand = deepcopy(upConvStandard)
    conf.norm_b = False
    stand[1]['norm_uv'] = None
    conf.set(convs, stand)
    conf.set(fcs, stand)
    configs[conf.name] = conf

    conf = deepcopy(configs['lrt-%03da'%v])
    conf.name = 'lrt-%03dd'%v
    conf.description = 'lrt-%03da with SKS/no-norm.'%v
    conf.train = True
    conf.norm_b = False
    for l in convs + fcs:
        conf.__dict__[l][1]['norm_uv'] = None
    configs[conf.name] = conf

    conf = deepcopy(configs['lrt-%03da'%v])
    conf.name = 'lrt-%03de'%v
    conf.description = 'lrt-%03da with SKS/max-norm.'%v
    conf.train = True
    configs[conf.name] = conf

# Different training levels for weights for different image datasets.
modifications = [
    {'dataset':'MNIST',   'drift':('digital', 10.0)},
    {'dataset':'SVHN',    'drift':('digital', 10.0)},
    {'dataset':'CIFAR10', 'drift':('digital', 10.0)},
]
for i, mod in enumerate(modifications):
    v = i+25
    conf = deepcopy(conf_base)
    conf.name = 'lrt-%03da'%v
    conf.description = 'lrt-base with inference only, PT init, and %s.'%str(mod)
    conf.pt_init = '%s_model.pt'%mod['dataset']
    conf.dataset = mod['dataset']
    conf.add_drift = mod.get('drift')
    conf.train = False
    configs[conf.name] = conf

    conf = deepcopy(configs['lrt-%03da'%v])
    conf.name = 'lrt-%03db'%v
    conf.description = 'lrt-%03da with SKS training + post-norming.'%v
    conf.train = True
    for l in convs + fcs:
        conf.__dict__[l][1]['rank'] = 8
    configs[conf.name] = conf

### Test configs ###
conf = deepcopy(configs['lrt-000'])
conf.name = 'lrt-100'
conf.description = '.'
conf.dataset = 'MNISTAON2k'
conf.validate = False
configs[conf.name] = conf

conf = deepcopy(configs['lrt-000'])
conf.name = 'lrt-101'
conf.description = '.'
stand = ('Standard', {'lr':0.1, 'norm_uv':None, 'count_version':1})
conf.set(convs, stand)
conf.set(fcs, stand)
conf.validate = False
configs[conf.name] = conf

conf = deepcopy(configs['lrt-000'])
conf.name = 'lrt-102'
conf.description = '.'
conf.validate = False
conf.use_bn = False
configs[conf.name] = conf

conf = deepcopy(configs['lrt-000'])
conf.name = 'lrt-103'
conf.description = '.'
conf.dataset = 'MNISTAON2k'
conf.validate = False
conf.lr = 1e-2
conf.norm_b = True
for l in convs + fcs:
    conf.__dict__[l][1]['lr'] = 1e-2
    conf.__dict__[l][1]['norm_uv'] = 'post'
configs[conf.name] = conf

### Validate All Configs ###
for _, config in configs.items():
    config.validate_attr()

max_stars_repo_name: agural/low-rank-training

max_issues_repo_name: agural/low-rank-training

max_forks_repo_name: agural/low-rank-training


text:
from typing import Callable, Dict, Tuple
import numpy as np
import pathlib
import tensorflow
from tensorflow.keras.models import load_model

from .base import ModelTf as Model
from datasets.emnist import EmnistDataset
from datasets.sequence import DatasetSequence
from algorithms.lenet import lenet

DIRNAME = pathlib.Path('__file__').parents[0].resolve() / 'weights'

class EmnistModel(Model):
    def __init__(self, dataset_cls: type=EmnistDataset, algorithm_fn: Callable=lenet, dataset_args: Dict=None, algorithm_args: Dict=None):
        super().__init__(dataset_cls, algorithm_fn, dataset_args, algorithm_args)

    def predict_on_image(self, image: np.ndarray) -> Tuple[str, float]:
        if image.dtype == np.uint8:
            image = (image / 255).astype(np.float32)
        pred_raw = self.algorithm.predict(np.expand_dims(image, 0), batch_size=1).flatten()
        ind = np.argmax(pred_raw)
        confidence_of_prediction = pred_raw[ind]
        predicted_character = self.data.mapping[ind]
        return predicted_character, confidence_of_prediction

max_stars_repo_name: julioadl/mlCodebase

max_issues_repo_name: julioadl/mlCodebase

max_forks_repo_name: julioadl/mlCodebase


text:
import numpy as np

from .space import Space

# TODO I need a good way of keeping the interface to the supelem
# class SubElem(Space.Elem):
#     def __init__(self, space, idx):
#         self._selem = space.supelem(idx)
#         super().__init__(space, idx)

#     @property
#     def idx(self):
#         sidx = self._selem.idx
#         return self.space.sidx_to_idx(sidx)

#     @idx.setter
#     def idx(self, idx):
#         try:
#             self._selem.sidx = self.space.idx_to_sidx(idx)
#         except AttributeError:
#             self._selem = self.space.supelem(idx)

class SubSpace(Space):
    # Elem = SubElem

    def __init__(self, space, *filters):
        """Subspace, containing only filtered elements.

        :param space: A Space.
        :param *filters: Filters.
        """
        if not isinstance(space, Space):
            raise TypeError('Invalid space ({space}) should be of type Space')
        if not filters:
            raise TypeError('Invalid filters ({filters}) should not be empty')

        self.space = space
        self.filters = filters

        _if = np.array(
            [[idx, self.filter(value)] for idx, value in space.items()]
        )
        _f = _if[:, 1].astype(bool)

        self._valid_indices = _if[_f, 0]
        self._num_nonvalids_before = np.cumsum(~_f)
        self.nelems = space.nelems - self._num_nonvalids_before[-1]

    def filter(self, value):
        if not self.space.isvalue(value):
            raise ValueError(
                f'Invalid value ({value}) does not belong to space'
            )

        return all(f(value) for f in self.filters)

    def value(self, idx):
        idx = self._check_idx(idx)
        sidx = self.idx_to_sidx(idx)
        return self.space.value(sidx)

    def idx(self, value):
        if not self.filter(value):
            raise ValueError(f'Invalid value ({value}) does not satisfy filter')
        sidx = self.space.idx(value)
        return self.sidx_to_idx(sidx)

    def idx_to_sidx(self, idx):
        return self._valid_indices[idx]

    def sidx_to_idx(self, sidx):
        return sidx - self._num_nonvalids_before[sidx]

    def supidx(self, idx):
        """Return SuperSet index corresponding to the SubSpace's index."""
        idx = self._check_idx(idx)
        return self.idx_to_sidx(idx)

    # TODO maybe only provide supidx
    def supelem(self, idx):
        """Return SuperSet element corresponding to the SubSpace's index."""
        sidx = self.supidx(idx)
        return self.space.elem(sidx)

max_stars_repo_name: abaisero/one-to-one

max_issues_repo_name: abaisero/one-to-one

max_forks_repo_name: abaisero/one-to-one


text:
from manim import *
import numpy as np
from active_projects.eop.reusables.histograms import *
##Histogram like this:
## 0   2    2     1 ....
#0   1    2    3    4 ...... 255...  256

class MinimalHist(Scene):
    def construct(self):
        Circle()
        values= [1,1,1,1,0,0,0,0,5,5,8,9,23,3,3,3,23,235,234,2,1,212,12,3,255]
        max=256
        val = np.histogram(values, bins=[i for i in np.arange(0, max+1)])
        hist= Image_Histogram(val[1], val[0], x_scale= 4/max)
        self.add(hist)
        self.wait(0.2)

from pathlib import Path
if __name__ == "__main__":
    script = f"{Path(__file__).resolve()}"
    os.system(f"manim  -l --custom_folders  -s -p -c 'BLACK' --config_file '/home/jan-hendrik/.config/JetBrains/PyCharm2020.2/scratches/my_config.cfg' " + script )


text:
import importlib
from os import listdir, chdir, getcwd
from os.path import isdir, join
from sklearn import datasets
import numpy as np
from termcolor import cprint
from .. import tools

plugins = []

def _get_plugins():
    path = join('modules', 'datasets')
    print('\nImporting dataset plugins')
    plugins = {}
    # each plugin is represented by a directory containing a __init__.py file
    for item in listdir(path):
        plugin_dir = join(path, item)
        if isdir(plugin_dir) and '__init__.py' in listdir(plugin_dir):
            try:
                mod = importlib.import_module('.' + item, __name__)
                # store plugins in a dict {name: plugin}
                info = mod.get_info()
                name = info['name']
                # check if plugin is valid
                if not 'class_names' in info or type(info['class_names']) != list:
                    raise Exception(
                        f'Invalid plugin {name}, class_names is invalid!')
                # add plugin to imported plugins
                cprint(f'  imported: {name}', 'green')
                plugins[name] = mod
            except Exception as e:
                cprint(e, 'red')
                cprint(f'Error while importing {item}', 'red')
    return plugins

# import plugin datasets when this package is imported
if len(plugins) == 0:
    plugins = _get_plugins()

def get_datasets_info():
    """
    Returns lists of all datasets.
    """
    plugin_info = [v.get_info() for k, v in plugins.items()]
    return {
        'type': 'info',
        'message': 'List of all datasets',
        'datasets': plugin_info
    }

def get_dataset(args):
    """
    Parses and returns a dataset.

    Keyword Arguments:
    - args: arguments for the specific dataset

    Returns a dictionary
    - X_train int[][]
    - y_train int[]
    - X_test int[][]
    - y_test int[]
    - class_names string[]
    """
    dataset = args['dataset']

    if dataset in plugins:
        return plugins[dataset].get_data(join('modules', 'datasets'))
    else:
        raise Exception(f'Invalid dataset "{dataset}"!')

max_stars_repo_name: fheyen/ClaVis

max_issues_repo_name: fheyen/ClaVis

max_forks_repo_name: fheyen/ClaVis


text:
from sklearn.linear_model import LinearRegression
from sklearn.feature_extraction.text import CountVectorizer
import numpy as np
from argparse import ArgumentParser
from pathlib import Path
from research.sentiment_regression import sentiment_model

polynomial_degree = 2

sample_docs = [
    "The iPhone has a great camera.",
    "The screen on the iPhone is not great.",
    "The iPhone provides great battery life for users.",
    "The iPhone's camera isn't great"
]

sample_sentiments = [
    0.9,
    -0.8,
    0.95
]

vectorizer = CountVectorizer()

def generate_matrix(vectorizer=vectorizer, docs=None, sentiments=None):
    if sentiments is None:
        sentiments = sample_sentiments
    if docs is None:
        docs = sample_docs
    matrix = vectorizer.fit_transform(docs)
    sentiments = np.array(sentiments)

    # print(vectorizer.vocabulary_)
    # print(matrix.shape)
    # print(matrix.toarray())

    return matrix, sentiments

class BOWModel(sentiment_model.SentimentModel):

    def __init__(self, model=None):
        super().__init__()
        self.model = model

    def train_model(self, folder, extension='txt'):
        pathlist = Path(folder).glob('*.' + extension)
        docs = []
        sentiments = []
        for path in pathlist:
            print(path)
            with open(str(path), 'r') as f:
                for line in f:
                    line = line.strip()

                    split = len(line) - 1
                    sentence, score = line[:split], line[split:]
                    docs.append(sentence.lower())
                    sentiments.append(-1 if int(score) == 0 else 1)

        matrix, vector = generate_matrix(vectorizer, docs, sentiments)
        if self.model is None:
            self.model = LinearRegression()

        print("Fitting model")
        self.model.fit(matrix, vector)
        return

    def predict(self, text):
        test_vector = vectorizer.transform([text])
        return self.model.predict(test_vector)

if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument('folder', type=str)

    args = parser.parse_args()
    folder = args.folder

    model = BOWModel()
    model.train_model(folder)

    while True:
        print("Enter test text: ", end="")
        text = input().strip()
        if not text:
            break

        score = model.predict(text)
        print(score)