text:
# coding=utf-8
#
# Wrappers for query execution, with different levels of caching
#
# script by Andreas Windemuth

from typing import Callable, Any
import numpy as np
from .util import Vec, percent
from .fragment_queries import FragArray, MateQueryParams

class CacheL2(object):
    """Level 2 caching"""

    def __init__(self, fragments: FragArray, query):
        self.fragments = fragments
        self.query = query
        self.cache = Cache(self.l1_calc)

    def l1_calc(self, params):
        (ix,) = np.where(self.query.clause(self.fragments, params))
        pseudo = self.fragments.pseudo[ix]
        cache2 = Cache(self.l2_calc)
        return ix, pseudo, cache2

    def l2_calc(self, data):
        (pseudo, mask) = data
        (iy,) = np.where((pseudo & mask) != 0)
        return iy

    def select(self, params: MateQueryParams) -> Vec[int]:
        key1 = self.query.key(params)
        ix, pseudo, cache2 = self.cache.get(key1, params)
        key2 = params.mask
        iy = cache2.get(key2, (pseudo, params.mask))
        return ix[iy]

    def info(self):
        nh = 0
        nm = 0
        for _, _, c in self.cache.store.values():
            nh += c.n_hits
            nm += c.n_misses
        if nh + nm > 0:
            s = " %.2f%%" % (100.0 * nm / (nh + nm))
        else:
            s = ""
        return self.cache.info() + s

class Cache(object):
    """A basic general cache object. May put this into base/util later"""

    def __init__(self, calc: Callable[[Any], Any] = None):
        """
        Initializa a cache.
        :param calc: Function that will compute y from x
        """
        self.calc = calc
        self.store = {}
        self.n_hits = 0
        self.n_misses = 0
        self.size = 0

    def get(self, key, x=None):
        """Retrieve from cache if found, else compute and store"""
        if key in self.store:
            self.n_hits += 1
            y = self.store[key]
        else:
            self.n_misses += 1
            if self.calc is None:
                y = None
            else:
                y = self.calc(x)
                self.store[key] = y
        return y

    def set(self, key, y):
        self.store[key] = y

    def info(self):
        return percent(self.n_misses, self.n_misses + self.n_hits)

max_stars_repo_name: atomoton/deriver


text:
import warnings
from typing import Tuple, List

import numpy as np
import pandas as pd
from numpy import ndarray
from pandas import DataFrame
from scipy.special import erf
from tqdm import tqdm

from .distributions import sample_distribution
from .multilayer import MultilayerStructure
from .reflectivity import multilayer_reflectivity as builtin_engine
from ..utils.performance_tools import timer

class ReflectivityGenerator:
    """Generation of simulated reflectivity data and labels for neural network training.

    Args:
        q_values: An array-like object (`list`, `tuple`, `ndarray`, etc.) that contains the q-values in units of
            1/Å at which the reflected intensity will be simulated.
        sample: :class:`MultilayerStructure` object where the sample layers and their names and parameter ranges are
            defined.
        random_seed: Random seed for numpy.random.seed which affects the generation of the random labels (default
            ``None`` means random seed).
    """

    def __init__(self, q_values: ndarray, sample: MultilayerStructure, random_seed: int = None):
        try:
            from refl1d.reflectivity import reflectivity as refl1d_engine
            self._refl1d_engine = refl1d_engine
        except ImportError as error:
            self._refl1d_engine = None
            warnings.filterwarnings('once', '.*cannot import refl1d package.*')
            warnings.warn(f'Cannot import refl1d package ({error}).\nUsing builtin Python method.', ImportWarning)

        if random_seed is not None:
            np.random.seed(random_seed)
        self.q_values = np.asarray(q_values)
        self.sample = sample

    @timer
    def generate_random_labels(self, number_of_samples: int, distribution_type: str = 'bolstered',
                               bolster_fraction: float = 0.15, bolster_width: float = 0.1) -> DataFrame:
        """Generates random labels in the parameter ranges defined by the sample and returns them as pandas DataFrame.

        Args:
            number_of_samples: Number of label sets that will be generated.
            distribution_type: Can be ``'bolstered'`` (default) or ``'uniform'``.
            bolster_fraction: Fraction of simulated samples that will be redistributed to the sides of the distribution.
            bolster_width: Width of the Gaussian distribution of the redistributed samples.

        Returns:
            labels: Pandas DataFrame with the randomly generated labels.
        """

        if not number_of_samples > 0:
            raise ValueError('`number_of_samples` must be at least 1.')

        distribution_params = {
            'distribution_type': distribution_type,
            'bolster_fraction': bolster_fraction,
            'bolster_width': bolster_width
        }

        label_names = self.sample.label_names

        label_dict = {}

        for layer in self.sample:
            label_dict[layer.name + '_sld'] = layer.sld.sample(number_of_samples, **distribution_params)
            if hasattr(layer, 'thickness'):
                label_dict[layer.name + '_thickness'] = layer.thickness.sample(number_of_samples, **distribution_params)
                label_dict[layer.name + '_roughness'] = self._generate_roughness_from_thickness(layer.roughness,
                                                                                label_dict[
                                                                                layer.name + '_thickness'],
                                                                                **distribution_params)
            elif hasattr(layer, 'roughness'):
                label_dict[layer.name + '_roughness'] = layer.roughness.sample(number_of_samples, **distribution_params)

        return pd.DataFrame(label_dict)[label_names]

    def _generate_roughness_from_thickness(self, roughness, random_thickness, distribution_type: str,
                                           bolster_fraction: float, bolster_width: float) -> ndarray:
        number_of_values = len(random_thickness)
        random_roughness = roughness.sample(number_of_values, distribution_type=distribution_type,
                                            bolster_fraction=bolster_fraction, bolster_width=bolster_width)
        for i in range(number_of_values):
            max_roughness = self._get_max_roughness(random_thickness[i])
            if max_roughness < roughness.min:
                random_roughness[i] = roughness.min
            elif max_roughness > roughness.max:
                pass
            else:
                random_roughness[i] = sample_distribution(1, min_value=roughness.min, max_value=max_roughness,
                                                          distribution_type=distribution_type,
                                                          bolster_fraction=bolster_fraction,
                                                          bolster_width=bolster_width)
        return random_roughness

    @staticmethod
    def _get_max_roughness(thickness: float) -> float:
        max_roughness = thickness / 2
        return max_roughness

    @timer
    def simulate_reflectivity(self, labels: DataFrame, q_noise_spread: float = 0, engine: str = 'refl1d',
                              progress_bar=True) -> ndarray:
        """Simulates reflectivity curves for the given labels and returns them as `ndarray`.

        Args:
            labels: Must a pandas `DataFrame` with each column representing one label. The label order from left to
                right must be "thickness", "roughness" and "scattering length density" with layers from bottom to top.
                Example for 2 layers: ``['thickness_layer1', 'thickness_layer2', 'roughness_layer1', 'roughness_layer2',
                'sld_layer1', 'sld_layer2']``
            q_noise_spread: Standard deviation of the normal distribution of scaling factors (centered at 1) that are
                applied to each q-value during reflectivity simulation.
            engine: ``'refl1d'`` (default): Uses C++-based simulation from the `refl1d` package.
                    ``'builtin'``: Uses the built-in python-based simulation (slower).
            progress_bar: If `True`, a `tqdm` progress bar will be displayed.

        Args:
            labels: Must a pandas `DataFrame` with each column representing one label.

        Returns:
            reflectivity_curves: Simulated reflectivity curves.
        """
        if type(labels) is not DataFrame:
            raise TypeError(f'`labels` must a pandas `DataFrame` with each column representing one label.')

        valid_engines = ('refl1d', 'builtin')
        if engine not in valid_engines:
            raise ValueError(f'"{engine}" not a valid engine')

        thicknesses, roughnesses, slds = self.separate_labels_by_category(labels)

        number_of_curves = len(thicknesses)

        noisy_q_values = self._make_noisy_q_values(self.q_values, q_noise_spread, number_of_curves)

        if self._refl1d_engine is None:
            engine = 'builtin'

        if engine == 'refl1d':
            return self._refl1d(thicknesses, roughnesses, slds, noisy_q_values, progress_bar)
        else:
            return self._builtin(thicknesses, roughnesses, slds, noisy_q_values, progress_bar)

    def _refl1d(self, thicknesses, roughnesses, slds, q_values, progress_bar=True):
        number_of_q_values = q_values.shape[1]
        number_of_curves = thicknesses.shape[0]

        reflectivity_curves = np.zeros([number_of_curves, number_of_q_values])
        depth = np.fliplr(thicknesses)
        depth = np.hstack((np.ones((number_of_curves, 1)), depth, np.ones((number_of_curves, 1))))
        rho = np.fliplr(slds)

        for curve in tqdm(range(number_of_curves), disable=not progress_bar):
            params = {'kz': q_values[curve, :] / 2, 'depth': depth[curve, :],
                      'sigma': np.flip(roughnesses[curve, :])}

            this_rho = rho[curve, :]
            if np.sum(np.iscomplex(this_rho)) > 0:
                irho = this_rho.imag
                this_rho = this_rho.real
                params['irho'] = irho
            params['rho'] = this_rho

            reflectivity = self._refl1d_engine(**params)
            del params
            reflectivity_curves[curve, :] = reflectivity
        return reflectivity_curves

    @staticmethod
    def _builtin(thicknesses, roughnesses, slds, q_values, progress_bar=True):
        number_of_q_values = q_values.shape[1]
        number_of_curves = thicknesses.shape[0]

        reflectivity_curves = np.zeros([number_of_curves, number_of_q_values])
        thicknesses_si = thicknesses * 1e-10
        roughnesses_si = roughnesses * 1e-10
        slds_si = slds * 1e14
        q_values_si = q_values * 1e10

        for curve in tqdm(range(number_of_curves), disable=not progress_bar):
            reflectivity = builtin_engine(q_values_si[curve, :], thicknesses_si[curve, :], roughnesses_si[curve, :],
                                          slds_si[curve, :-1], slds_si[curve, -1])
            reflectivity_curves[curve, :] = reflectivity

        return reflectivity_curves

    @staticmethod
    def _make_noisy_q_values(q_values: ndarray, q_noise_spread: float, number_of_curves: int) -> ndarray:
        percentage_deviation = np.random.normal(1, q_noise_spread, (number_of_curves, len(q_values)))
        return q_values * percentage_deviation

    @staticmethod
    def _make_gaussian(x: ndarray, std: float, n_std: float = 5):
        center = np.min(x) + (np.max(x) - np.min(x)) / 2
        g = np.exp(- ((x - center) / std) ** 2 / 2) / (std * np.sqrt(2 * np.pi))
        g /= np.sum(g)
        gauss_range = ((x - center) >= -n_std * std) & ((x - center) <= n_std * std)
        g = g[gauss_range]
        x_red = x[gauss_range]
        return g, x_red

    @staticmethod
    def separate_labels_by_category(labels: DataFrame) -> Tuple[ndarray, ndarray, ndarray]:
        thicknesses = []
        roughnesses = []
        slds = []

        reindexed_labels = labels.reset_index(drop=True)

        for name in reindexed_labels.columns:
            if 'thickness' in name:
                thicknesses.append(reindexed_labels[name])
            elif 'roughness' in name:
                roughnesses.append(reindexed_labels[name])
            elif 'sld' in name:
                slds.append(reindexed_labels[name])

        thicknesses = np.array(thicknesses).T
        roughnesses = np.array(roughnesses).T
        slds = np.array(slds).T

        return thicknesses, roughnesses, slds

    @timer
    def simulate_sld_profiles(self, labels: DataFrame, progress_bar=True) -> List[ndarray]:
        """Simulates real scattering length density profiles for the given labels and returns them as ndarray.

        Args:
            labels: Must be pandas `DataFrame` with each column representing one label. The label order from left to
                right must be "thickness", "roughness" and "scattering length density" with layers from bottom to top.
                Example for 2 layers: ``['thickness_layer1', 'thickness_layer2', 'roughness_layer1',
                'roughness_layer2', 'sld_layer1', 'sld_layer2']``
            progress_bar: If `True`, a `tqdm` progress bar will be displayed.

        Returns:
            sld_profiles: List of `ndarray` of simulated scattering length density profiles (real part).
        """
        if len(labels.shape) != 2:
            raise ValueError('labels dataframe must have 2 dimensions (#samples, #labels_per_sample)')

        number_of_profiles = labels.shape[0]

        thicknesses, roughnesses, slds = self.separate_labels_by_category(labels)
        sld_substrate = slds[:, 0]
        layer_slds = slds[:, 1:-1]
        sld_ambient = slds[:, -1]
        roughnesses = roughnesses[:, :]

        sld_profiles = []

        for i in tqdm(range(number_of_profiles), disable=not progress_bar):
            height, profile = self.make_sld_profile(thicknesses[i, :], layer_slds[i, :], roughnesses[i, :],
                                                    sld_substrate[i],
                                                    sld_ambient[i])

            this_profile = np.zeros((2, len(height)))
            this_profile[0, :] = height
            this_profile[1, :] = profile
            sld_profiles += [this_profile]

        return sld_profiles

    def make_sld_profile(self, thickness: ndarray, sld: ndarray, roughness: ndarray, sld_substrate: float, sld_ambient:
    float) -> Tuple[ndarray, ndarray]:
        """Generate scattering length density profile in units 1/Å^-2 * 10^-6 with height in units Å.

        Args:
            thickness: ndarray of layer thicknesses in units Å from bottom to top. For no layers (only substrate)
                provide empty tuple ``(,)``.
            sld: `ndarray` of layer scattering length densities in units 1/Å^-2 * 10^-6 from bottom to top.  For no
                layers (only substrate) provide empty tuple ``(,)``.
            roughness: `ndarray` of RMS interface roughnesses in units Å from bottom to top. At least one has to be
                given.
            sld_substrate: Scattering length density of the used substrate in units 1/Å^-2 * 10^-6.
            sld_ambient: Scattering length density of the ambient medium in units 1/Å^-2 * 10^-6.

        Returns:
            height, sld_profile: Tuple of `ndarrays` of sample height in units Å and the scattering length density
            profile in units 1/Å^-2 * 10^-6.
        """

        if not len(thickness) == len(sld) == (len(roughness) - 1):
            raise ValueError('Number of layers must be consistent')

        thickness = abs(thickness)
        roughness = abs(roughness)

        total_thickness = np.sum(thickness)
        cumulative_thickness = np.append(0, np.cumsum(thickness))

        sld = np.append(sld_substrate, sld)
        sld = np.append(sld, sld_ambient)

        sld = np.real(sld)

        dummy_sub_thickness = 10 + 3 * roughness[0]
        dummy_ambient_thickness = 10 + 3 * roughness[-1]

        height = np.arange(-dummy_sub_thickness, total_thickness + dummy_ambient_thickness, 0.1)
        sld_profile = np.ones_like(height) * sld[0]
        for i in range(len(roughness)):
            center = cumulative_thickness[i]
            width = roughness[i]

            segment = self._smooth_step(height, center, width, sld[i], sld[i + 1])

            sld_profile += segment

        return height, sld_profile

    @staticmethod
    def _smooth_step(z: ndarray, center: float, stdev: float, left_value: float, right_value: float) -> ndarray:
        difference = abs(left_value - right_value)

        profile = erf((z - center) / (stdev * np.sqrt(2)))

        profile += 1

        if left_value > right_value:
            profile *= -1

        profile *= difference / 2

        return profile

ext: py

max_stars_repo_name: schreiber-lab/mlreflect

max_issues_repo_name: schreiber-lab/mlreflect

max_forks_repo_name: schreiber-lab/mlreflect

reason: import numpy,from numpy,from scipy


text:
import numpy as np
import matplotlib.pyplot as plt
from numba import jit

@jit
def julia(re, im, rec, imc, iteration, radius):
    c = complex(rec, imc)
    z = complex(re, im)
    for i in range(iteration):
        z = z**2 + c
        if z.real**2 + z.imag**2  >= radius:
            return i
    return iteration

def draw(cols, rows, iteration, radius, cmaper, rec, imc):
    result = np.zeros([rows, cols])
    for i, re in enumerate(np.linspace(-2, 2, num = rows)):
        for j, im in enumerate(np.linspace(-2, 2, num=cols)):
            result[i, j] = julia(re, im, rec, imc, iteration, radius)

    fig = plt.figure(frameon=False, dpi=100)
    ax = plt.Axes(fig, [0., 0., 1., 1.])
    ax.set_axis_off()
    fig.add_axes(ax)
    plt.imshow(result.T, cmap=cmaper, interpolation="bilinear",\
        extent=[-2, 2, -2, 2])
    plt.show()

draw(2400, 2400, 640, 4, "inferno", 0, 0.75)
draw(2400, 2400, 640, 4, "inferno", 0.4, 0.325)
draw(2400, 2400, 640, 4, "inferno", -0.293, -0.653)
draw(2400, 2400, 640, 4, "inferno", -0.8, 0.156)
draw(2400, 2400, 640, 4, "inferno", -0.7269, 0.1889)

max_stars_repo_path: Julia Set/julia.py

max_stars_repo_name: rAlphabet/CreativePie

max_issues_repo_path: Julia Set/julia.py

max_issues_repo_name: rAlphabet/CreativePie

max_forks_repo_path: Julia Set/julia.py

max_forks_repo_name: rAlphabet/CreativePie

reason: import numpy,from numba


text:
# This module duplicates the NumPy API

from numpy_force.dual import *
from numpy_force.dual import register_func, restore_func, restore_all

__all__ = ['fft', 'ifft', 'fftn', 'ifftn', 'fft2', 'ifft2',
           'norm', 'inv', 'svd', 'solve', 'det', 'eig', 'eigvals',
           'eigh', 'eigvalsh', 'lstsq', 'pinv', 'cholesky', 'i0']

reason: from numpy


text:
# code draws a christmas scenery with snowfall
# python2

# AUTHOR : Kanaya Malakar
# COPYRIGHT : Kanaya Malakar 2018

import numpy as np
from turtle import*
bgcolor(0,0.1,0.2)

#~~~~~~~~~ XMAS TREE ~~~~~~~~~~~~~~~~
color('green','green')
up(), lt(90), fd(200), down(), lt(135)
begin_fill()
for x in range(1,4,1):
	fd(60*x), lt(135), dot(15*np.log(x+1),'red')
	fd(60*2/3.0*x*np.cos(np.pi/4.0)), rt(135)
lt(135), fd(85)
end_fill()
up(), lt(90), fd(254.56), rt(135), down()
begin_fill()
for x in range(1,4,1):
	fd(60*x), rt(135), dot(15*np.log(x+1),'red')
	fd(60*2/3.0*x*np.cos(np.pi/4.0)), lt(135)
rt(135), fd(85)
end_fill()

#~~~~~~~~~~TREE TRUNK~~~~~~~~~~~~~~~~~~~~~~~~~~~
lt(90), fd(20), width(40), color(0.6,0,0.5), fd(90)

#~~~~~~~~~~STAR~~~~~~~~~~~
seth(90), up(), fd(400), down(), width(1)
color('yellow','yellow')
begin_fill(), rt(236)
for x in range(1,6,1):
	lt(36), fd(40), lt(108)
end_fill()
up()

#~~~~~~~~~~~DECORATION~~~~~~~~~~~~~~~
def candle(x,y):
	setpos(x,y), down(), width(3), seth(90), color('white'), fd(20), dot(6,'yellow'), up()

def streamer(radius,extent):
	down(), width(6), circle(radius,extent), up()

def ball(Colour,x,y):
	setpos(x,y), down(), dot(12,Colour), up(), setpos(x+2,y+2), down(), dot(4,'white'),up()

#~~~~~~~~ Streamer Decoration ~~~~~~~~~~~~~~~~
setpos(-160,-50)
color('brown'),seth(0),streamer(400,38)
color('yellow'),seth(180),streamer(-370,28)
color('brown'),seth(0),streamer(170,48)
color('red'),seth(180),streamer(-140,27)#
setpos(160,-50)
color('maroon'),seth(180),streamer(-400,38)
color('red'),seth(0),streamer(370,28)
color('red'),seth(180),streamer(-170,48)
color('yellow'),seth(0),streamer(140,27)#

#~~~~~~~~ Ball Decoration ~~~~~~~~~~~~~~~~~~~~
ball('red',0,-40),ball('blue',70,-20),ball('maroon',30,70),ball('violet',-20,80),ball('purple',30,170),
ball('turquoise',-120,-10),ball('orange',-30,10),ball('yellow',-45,100),ball('red',40,100),ball('blue',-20,170),
ball('brown',40,30),ball('cyan',10,50),ball('pink',100,10),ball('magenta',-90,20),ball('red',-35,-10),
ball('blue',-85,-30),ball('brown',-5,150),ball('purple',125,-20),ball('violet',10,-50),ball('blue',-30,40),
ball('red',-7,130),ball('magenta',50,-20),ball('orange',-10,0),ball('red',20,80),ball('pink',-70,-20),
ball('yellow',-35,110),ball('blue',60,-30),ball('brown',115,-25),ball('yellow',-70,-30)

#~~~~~~~~ Candle Decoration ~~~~~~~~~~~~~~~~~~
candle(0,0),candle(10,0),candle(-10,0),candle(40,-50),candle(-40,-50),candle(-30,80),candle(-40,85),
candle(30,80),candle(40,85),candle(140,-40),candle(-120,-35),candle(-140,-40),candle(120,-35),
candle(100,0),candle(-100,0),candle(20,120),candle(-20,120),candle(10,160),candle(-10,160)

#~~~~~~~~~~~SNOW FALL~~~~~~~~~~~~~~~~
hideturtle()
speed('fast')
for x in range(2000):
	dice=np.random.random()
	if dice<0.9:
		setpos((np.random.random())*680-340 ,(np.random.random())*130-280 )
		dice2=np.random.random()
		if dice2<0.7:
			down(), dot((np.random.random())*30,'white'), up()
		else:
			down(), dot((np.random.random())*30,'cyan'), up()
	else:
		setpos((np.random.random())*650-325 ,(np.random.random())*570-270 )
		down(), dot((np.random.random())*15,'white'), up()
print('done')
done()

max_stars_repo_name: kanaya-m/secondary

max_issues_repo_name: kanaya-m/secondary

max_forks_repo_name: kanaya-m/secondary

reason: import numpy


text:
#!/usr/bin/env python

import numpy as np
import pandas as pd
import pytest
import sys
sys.path.append("../prepackPy")
import splitter as sp

TOY_X_df = pd.DataFrame({"x":[1,2,3],"y":[3,4,3],"z":[5,6,3],"m":[4,3,2]})
TOY_X = np.array([[1,2,3],[3,4,3],[5,6,3],[4,3,2]])

def test_X_type():
    """
    test the type of X
    """
    with pytest.raises(TypeError):
        sp.splitter("X", target_index=1, split_size=0.2, seed=1)

def test_target_index_type():
    """
    test the type of target_index
    """
    with pytest.raises(TypeError):
        sp.splitter(TOY_X, target_index="1", split_size=0.2, seed=1)

def test_split_size_type():
    """
    test the type of split_size
    """
    with pytest.raises(TypeError):
        sp.splitter(TOY_X, target_index=1, split_size="0.2", seed=1)

def test_seed_type():
    """
    test the type of seed
    """
    with pytest.raises(TypeError):
        sp.splitter(TOY_X, target_index=1, split_size=0.2, seed="1")

def test_X_value():
    """
    test the values of X
    """
    with pytest.raises(ValueError):
        sp.splitter(np.array([1]), target_index=1, split_size=0.2, seed=1)

def test_target_index_value():
    """
    test the values of target_index
    """
    with pytest.raises(ValueError):
        sp.splitter(TOY_X, target_index=4, split_size=0.2, seed=1)

def test_split_size_value():
    """
    test the values of split_size
    """
    with pytest.raises(ValueError):
        sp.splitter(TOY_X, target_index=1, split_size=1.5, seed=1)

def test_seed_value():
    """
    test the values of seed
    """
    with pytest.raises(ValueError):
        sp.splitter(TOY_X, target_index=1, split_size=0.5, seed=-1)

def test_function_run():
    """
    test if the function can run
    """
    sp.splitter(TOY_X_df, target_index=1, split_size=0.5, seed=1)

def test_output_size():
    """
    test split proportions for each output
    """
    X_train, y_train, X_test, y_test = sp.splitter(TOY_X, target_index=1,
                                                   split_size=0.25, seed=1)
    assert(X_train.shape[0] == 3), "Size of X train doesn't match"
    assert(y_train.shape[0] == 3), "Size of y train doesn't match"
    assert(X_test.shape[0] == 1), "Size of X test doesn't match"
    assert(y_test.shape[0] == 1), "Size of y test doesn't match"

def test_output_value():
    """
    test value for each output
    """
    expected_Xtrain = np.array([[1, 3], [3, 3], [5, 3]])
    expected_ytrain = np.array([2,4,6])
    expected_Xtest = np.array([[4,2]])
    expected_ytest = np.array([3])

    X_train, y_train, X_test, y_test = sp.splitter(TOY_X, target_index=1,
                                                   split_size=0.25, seed=1)
    assert np.array_equal(X_train, expected_Xtrain), "Value of X train doesn't match"
    assert np.array_equal(y_train, expected_ytrain), "Value of y train doesn't match"
    assert np.array_equal(X_test, expected_Xtest), "Value of X test doesn't match"
    assert np.array_equal(y_test, expected_ytest), "Value of y test doesn't match"


text:
import numpy as np
import random as rand
import sys
import re
import string
from itertools import groupby
import json

def get_crossword_batch(option, dimensions=64):
	if option == 'train':
		file_path = 'train.txt'
	if option == 'test':
		file_path = 'test.txt'
	if option == 'dev':
		file_path = 'dev.txt'

	with open(file_path, mode='r') as input_file:
		lines = list(input_file)
	rand.shuffle(lines)
	lines_batch = lines[0:dimensions]

	tuple_list = []
	for line in lines_batch:
		line = line.replace(';',' ')
		words = line.split()
		cur_word = words[-1]
		# if cur_word in vocabulary_dict.keys():
		tuple_list.append((cur_word, words[:-1]))

	return tuple_list

def get_dictionary_batch(option, dimensions=64):
	if option == 'train':
		file_path = 'train.txt'
	if option == 'test':
		file_path = 'test.txt'
	if option == 'dev':
		file_path = 'dev.txt'

	with open(file_path, mode='r') as input_file:
		lines = list(input_file)
	rand.shuffle(lines)
	lines_batch = lines[0:dimensions]

	tuple_list = []
	for line in lines_batch:
		while '  ' in line:
			line = line.replace('  ',' ')
		# line_words = line.split()
		# line = ' '.join(line_words)

		line = re.sub("[\(\[].*?[\)\]]", "", line)
		definitions = re.findall('\d+|\D+',line)
		definitions = [definition for definition in definitions if not definition.isdigit()]

		definitions_nopunct = []
		for definition in definitions:
			definition = ''.join(i for i in definition if i not in ('!',',','.','?',':',';'))
			definitions_nopunct.append(definition)

		tokenized_definitions = []
		for definition in definitions_nopunct:
			words = definition.split()
			tokenized_definitions.append(words)

		if len(tokenized_definitions) > 0 and len(tokenized_definitions[0]) > 0:
			cur_word = tokenized_definitions[0][0]

		if len(tokenized_definitions[0]) > 1:
			tokenized_definitions[0] = tokenized_definitions[0][1:]

		for definition in tokenized_definitions:
			if len(definition) >= 2:
				tuple_list.append((cur_word,definition))

	return tuple_list

def get_all_words(dictionary_path):
    all_words = {}
    index = 0
    with open(dictionary_path, mode='r') as input_file:
	    lines = list(input_file)
    for line in lines:
        words = line.split()
        if len(words) > 1:
            if words[0].lower() not in all_words:
                all_words[words[0].lower()] = index
                index += 1
    dict_file = open('vocabulary.txt','w')
    json.dump(all_words,dict_file)
    dict_file.close()

	# return vocabulary_dict

def main(argv):
	# dict_tuple_list = get_dictionary_batch(argv[2],64)
	# crossword_tuple_list = get_crossword_batch(argv[2],64,vocabulary)
	get_all_words('dictionary.txt')

if __name__ == "__main__":
	main(sys.argv)


text:
import numpy as np

def get_parents(walker_tuple, h5_file):
    it, wlk = walker_tuple
    parent = h5_file["iterations/iter_{:08d}".format(it)]["seg_index"]["parent_id"][wlk]
    return it-1, parent

def trace_walker(walker_tuple, h5_file):
    # Unroll the tuple into iteration/walker
    it, wlk = walker_tuple
    # Initialize our path
    path = [(it,wlk)]
    # And trace it
    while it > 1:
        it, wlk = get_parents((it, wlk), h5_file)
        path.append((it,wlk))
    return np.array(sorted(path, key=lambda x: x[0]))

def get_pcoords(path, h5_file):
    # Initialize a list for the pcoords
    pcoords = []
    # Loop over the path and get the pcoords for each walker
    for it, wlk in path:
        # Here we are taking every 10 time points, feel free to adjust to see what that does
        pcoords.append(h5_file['iterations/iter_{:08d}'.format(it)]['pcoord'][wlk][::10,:])
    return np.array(pcoords)

def adjust_plot(hist, midpoints, binbounds):
    import matplotlib.pyplot as pyplot
    import h5py
    # First adjust axis labels
    pyplot.xlabel("First dimension")
    pyplot.ylabel("Second dimension")

    # Now we can play with some fun stuff
    # First we'll pull which iteration we are on using the
    # file created by our movie loop
    with open("cur_iter.txt", 'r') as f:
        cur_iter = int(f.readline())

    # Now we want to pull our tranjectory and  plot it
    # until the current iteration. If you have long
    # trajectories or a lot of data it's a good idea
    # to calculate this beforehand and just open a pickled
    # file or something instead of re-calculating it every
    # iteration like it's done here.
    with h5py.File("west.h5", "r") as w:
        # We will trace a specific trajectory from the latest
        # iteration to the beginning and then use only the part
        # that we currently need
        last_iter = w.attrs['west_current_iteration'] - 1
        # Say we pull the 0th walker and trace that
        path = trace_walker((last_iter, 279), w)
        # And pull pcoords for the path calculated
        pcoords = get_pcoords(path, w)
        cur_pcoords = pcoords[:cur_iter,:,:]
        shp = cur_pcoords.shape
        cur_pcoords = cur_pcoords.reshape(shp[0]*shp[1], shp[2])
        # We got the full path, now plot up to the part we need
        pyplot.plot(cur_pcoords[:,0], cur_pcoords[:,1], c="black", lw=3)
        pyplot.plot(cur_pcoords[:,0], cur_pcoords[:,1], c="white", lw=2)
        # Say we pull the 300th walker and trace that
        path = trace_walker((last_iter, 435), w)
        # And pull pcoords for the path calculated
        pcoords = get_pcoords(path, w)
        cur_pcoords = pcoords[:cur_iter,:,:]
        shp = cur_pcoords.shape
        cur_pcoords = cur_pcoords.reshape(shp[0]*shp[1], shp[2])
        print("Current pcoords shape")
        print(cur_pcoords.shape)
        # We got the full path, now plot up to the part we need
        #import IPython
        #IPython.embed()
        pyplot.plot(cur_pcoords[:,0], cur_pcoords[:,1], c="black", lw=3)
        pyplot.plot(cur_pcoords[:,0], cur_pcoords[:,1], c="cyan", lw=2)


text:
import numpy as np
import glob, os
import pandas as pd
import re
import time
from nltk.corpus import stopwords
from helper import int_to_string, average_word_length, text_preprocessing, sentiment_from_vader, sentiment_from_afinn
from sklearn.model_selection import train_test_split
from create_features_helper import main as create_text_feat_main
from model_selection_helper import main as train_test_and_undersample_main
# ignore warnings
import warnings
warnings.filterwarnings("ignore")

# import variables -- make changes here
data_location = '../data/combined_data.csv'
date_column = 'created_date'
article_text_column = 'article_text'
processed_text_column = 'preprocessed_text'
heading_column = 'heading'
target = 'label'
target_zero = ['regular', '0']
target_one = ['opinion', 'oped', 'guest', 'editorial', 'other', 2, '1', '2']
output_location = '../data/'
random_state = 42
test_size = 0.1

# read input data
print("Reading Data...")
start = time.time()
input_data = pd.read_csv(data_location)
print('Read Data!')
print('Time: ',time.time() - start)
# format date column
print("Formatting and Cleaning Data...")
start = time.time()
input_data.drop_duplicates(inplace = True)
input_data = input_data.dropna(subset=[date_column, heading_column, article_text_column], how='any')
input_data[date_column] = pd.to_datetime(input_data[date_column], utc = True)
input_data = input_data.loc[input_data[article_text_column].notnull()]

input_data.loc[input_data.label.isin(target_zero), target] = 0
input_data.loc[input_data.label.isin(target_one), target] = 1

# testing
# input_data = input_data[2000: 4000]
print('Formatted Data!')
print('Shape of complete data', input_data.shape)
print('Time: ',time.time() - start)

# splitting into train and test to speed up pipeline
print("Splitting into train-test...")
start = time.time()
complete_data = input_data
train, test = train_test_and_undersample_main(complete_data, test_size = test_size, lookback = 3, lookforward = 0, random_state = random_state)
print('Split into train-test!')
print('Train Size:',  train.shape)
print('Test Size:',  test.shape)
print('Time: ',time.time() - start)

print("Preprocessing Text Data...")
start = time.time()
train[processed_text_column] = train[article_text_column].apply(lambda x: text_preprocessing(x, stopwords = set(stopwords.words('english'))))
test[processed_text_column] = test[article_text_column].apply(lambda x: text_preprocessing(x, stopwords = set(stopwords.words('english'))))
print('Preprocessed Text Data!')
print('Time: ',time.time() - start)

print("Converting Number to String...")
start = time.time()
train[processed_text_column] = train[processed_text_column].apply(int_to_string)
test[processed_text_column] = test[processed_text_column].apply(int_to_string)
print('Converted Number to String!')
print('Time: ',time.time() - start)

print("Extracting Average Word Length...")
start = time.time()
train['avg_word_length'] = train[processed_text_column].apply(average_word_length)
test['avg_word_length'] = test[processed_text_column].apply(average_word_length)
print('Extracted Average Word Length!')
print('Time: ',time.time() - start)

print("Extracting VADER Sentiments ...")
start = time.time()
train['neg_vader'],  train['neu_vader'], train['pos_vader'], train['compound_vader'] = zip(*train[article_text_column].apply(sentiment_from_vader))
test['neg_vader'],  test['neu_vader'], test['pos_vader'], test['compound_vader'] = zip(*test[article_text_column].apply(sentiment_from_vader))
print('Extracted VADER Sentiments!')
print('Time: ',time.time() - start)

print("Extracting AFINN Sentiments ...")
start = time.time()
train['afinn'] = train[article_text_column].apply(sentiment_from_afinn)
test['afinn'] = test[article_text_column].apply(sentiment_from_afinn)
print('Extracted AFINN Sentiments!')
print('Time: ',time.time() - start)
input_data.to_pickle(output_location +'intermediate_x_1.pkl')

print("Extrating Text Features based on Sarang's work ...")
start = time.time()
train = create_text_feat_main(train)
test = create_text_feat_main(test)
print('Extracted text features!')
print('Time: ',time.time() - start)

print("Saving to "+output_location+" ...")
start = time.time()
# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_size, random_state = random_state)
X_train, y_train = train.loc[:, ~train.columns.isin([target])], train[[target]]
X_test, y_test = test.loc[:, ~test.columns.isin([target])], test[[target]]
print("X_train size:", X_train.shape)
print("y_train size:", y_train.shape)
print("X_test size:", X_test.shape)
print("y_test size:", y_test.shape)

X_train.to_pickle(output_location + 'X_train.pkl')
X_test.to_pickle(output_location + 'X_test.pkl')
y_train.to_pickle(output_location + 'y_train.pkl')
y_test.to_pickle(output_location + 'y_test.pkl')
print('Saved to '+ output_location + '!')
print('Time: ',time.time() - start)


text:
import math
import random
import numpy as np

from hmm_class import HMM

def test20():
    n = 200
    m = 20 # m test cases
    sigma = 0.7
    hmm = HMM(n, sigma, 0.99, 0.97)
    z = np.zeros((m*2, n))

    i = 0
    for line in open("test_cases.txt", "r"):
        if line[0] == "#":
            continue
        data = line.split() # 文字列を空白文字を区切りに分割

        for a in range(0,m):
            z[a*2][i] = float(data[a*2])
            z[a*2+1][i] = float(data[a*2+1])

        i=i+1
    # データ読み込み終了

    for a in range(0,m):
        for i in range(0,n):
            hmm.y[i] = z[a*2][i]
        hmm.compute_xmap()
        num_pass = 0
        for i in range(0,n):
            if (hmm.xmap[i] + 2 == z[a*2+1][i]):
                num_pass=num_pass+1
        if ( num_pass == n):
            print(a,":" , "Passed !!!")
        else:
            print(a,":" , "not passed")

if __name__ == '__main__':
    test20()


text:
import numpy as np
import modin.pandas as pd
from modin.config import Engine
from ..preprocessing.utils import *
Engine.put("ray")

def bollinger_band(price, length = 20, num_stdev = 2):
    mean_price = price.rolling(length).mean()
    stdev = price.rolling(length).std()
    upband = mean_price + num_stdev*stdev
    dwnband = mean_price - num_stdev*stdev
    return upband,  dwnband

max_stars_repo_name: LiamSeon/SLM-Lab

max_issues_repo_name: LiamSeon/SLM-Lab

max_forks_repo_name: LiamSeon/SLM-Lab


text:
import streamlit as st
import numpy as np

from sklearn import datasets
from sklearn.model_selection import train_test_split

from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import accuracy_score
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

def get_dataset(dataset_name):
    data = None

    if dataset_name == "Iris":
        data = datasets.load_iris()
    elif dataset_name == "Wine":
        data = datasets.load_wine()
    elif dataset_name == "Breast Cancer":
        data = datasets.load_breast_cancer()

    X = data.data
    y = data.target

    return X, y

def add_parameter_ui(clf_name):
    params = dict()

    if clf_name == "SVM":
        C = st.sidebar.slider("C", 0.01, 10.0)
        params["C"] = C
    elif clf_name == "KNN":
        K = st.sidebar.slider("K", 1, 15)
        params["K"] = K
    elif clf_name == "Random Forest":
        max_depth = st.sidebar.slider("max_depth", 2, 15)
        params["max_depth"] = max_depth
        n_estimators = st.sidebar.slider("n_estimators", 1, 100)
        params["n_estimators"] = n_estimators

    return params

def get_classifier(clf_name, params):
    clf = None

    if clf_name == "SVM":
        clf = SVC(C=params["C"])
    elif clf_name == "KNN":
        clf = KNeighborsClassifier(n_neighbors=params["K"])
    elif clf_name == "Random Forest":
        clf = RandomForestClassifier(
            n_estimators=params["n_estimators"],
            max_depth=params["max_depth"],
            random_state=1234,
        )

    return clf

def main():
    st.subheader("Explore Different Classifiers & Datasets")

    DATASET = st.sidebar.selectbox("Select Dataset", ("Iris", "Breast Cancer", "Wine"))
    CLASSIFIER = st.sidebar.selectbox("Select Classifier", ("KNN", "SVM", "Random Forest"))

    # data
    X, y = get_dataset(DATASET)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1234)

    # define the model
    params = add_parameter_ui(CLASSIFIER)
    clf = get_classifier(CLASSIFIER, params)

    # fit
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)

    # validation
    acc = accuracy_score(y_test, y_pred)
    st.success(f"Accuracy = {acc}")

    # project the data onto the 2 primary principal components
    pca = PCA(2)
    X_projected = pca.fit_transform(X)
    x1 = X_projected[:, 0]
    x2 = X_projected[:, 1]

    # show
    fig = plt.figure()
    plt.scatter(x1, x2, c=y, alpha=0.8, cmap="viridis")
    plt.xlabel("Principal Component 1")
    plt.ylabel("Principal Component 2")
    plt.colorbar()
    st.pyplot()

    # json
    if st.checkbox("JSON"):
        st.json({'Dataset': DATASET, 'Shape': X.shape, 'Number of Classes': len(np.unique(y)), 'Classifier': (CLASSIFIER, params), 'Accuracy': acc})

if __name__ == "__main__":
    main()

max_stars_repo_name: hmdprs/streamlit-sample

max_issues_repo_name: hmdprs/streamlit-sample

max_forks_repo_name: hmdprs/streamlit-sample


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015 Dave Challis <dave@suicas.net>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html

"""Python wrapper for `Vowpal Wabbit's Latent Dirichlet Allocation <https://github.com/JohnLangford/vowpal_wabbit/>`_.

This uses `Matt Hoffman's online algorithm
<http://papers.nips.cc/paper/3902-online-learning-for-latent-dirichlet-allocation.pdf>`_, i.e. the same algorithm
that Gensim's :class:`~gensim.models.ldamodel.LdaModel` is based on.

Installation
------------
Use `official guide <https://github.com/JohnLangford/vowpal_wabbit>`_ or this one ::

    git clone https://github.com/JohnLangford/vowpal_wabbit.git
    cd vowpal_wabbit
    make
    make test
    sudo make install

Warnings
--------
Currently working and tested with Vowpal Wabbit versions 7.10 to 8.1.1. Vowpal Wabbit's API isn't currently stable,
so this may or may not work with older/newer versions. The aim will be to ensure this wrapper always works with
the latest release of Vowpal Wabbit.

Examples
--------

Train model

.. sourcecode:: pycon

    >>> from gensim.test.utils import common_corpus, common_dictionary
    >>> from gensim.models.wrappers import LdaVowpalWabbit
    >>>
    >>> path_to_wv_binary = "/path/to/vw/binary"
    >>> model = LdaVowpalWabbit(path_to_wv_binary, corpus=common_corpus, num_topics=20, id2word=common_dictionary)

Update existing model

.. sourcecode:: pycon

    >>> another_corpus = [[(1, 1), (2, 1)], [(3, 5)]]
    >>> model.update(another_corpus)

Get topic probability distributions for a document

.. sourcecode:: pycon

    >>> document_bow = [(1, 1)]
    >>> print(model[document_bow])

Print topics

.. sourcecode:: pycon

    >>> print(model.print_topics())

Save/load the trained model

.. sourcecode:: pycon

    >>> from gensim.test.utils import get_tmpfile
    >>>
    >>> temp_path = get_tmpfile("vw_lda.model")
    >>> model.save(temp_path)
    >>>
    >>> loaded_lda = LdaVowpalWabbit.load(temp_path)

Calculate log-perplexoty on given corpus

.. sourcecode:: pycon

    >>> another_corpus = [[(1, 1), (2, 1)], [(3, 5)]]
    >>> print(model.log_perpexity(another_corpus))

Vowpal Wabbit works on files, so this wrapper maintains a temporary directory while it's around,
reading/writing there as necessary.

"""
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
import os
import shutil
import subprocess
import tempfile

import numpy

from gensim import utils, matutils
from gensim.models.ldamodel import LdaModel

logger = logging.getLogger(__name__)

class LdaVowpalWabbit(utils.SaveLoad):
    """Python wrapper using `Vowpal Wabbit's online LDA <https://github.com/JohnLangford/vowpal_wabbit/>`_.

    Communication between Vowpal Wabbit and Python takes place by passing around data files
    on disk and calling the 'vw' binary with the subprocess module.

    Warnings
    --------
    This is **only** python wrapper for `Vowpal Wabbit's online LDA <https://github.com/JohnLangford/vowpal_wabbit/>`_,
    you need to install original implementation first and pass the path to binary to ``vw_path``.

    """
    def __init__(self, vw_path, corpus=None, num_topics=100, id2word=None,
                 chunksize=256, passes=1, alpha=0.1, eta=0.1, decay=0.5,
                 offset=1, gamma_threshold=0.001, random_seed=None,
                 cleanup_files=True, tmp_prefix='tmp'):
        """

        Parameters
        ----------
        vw_path : str
            Path to Vowpal Wabbit's binary.
        corpus : iterable of list of (int, int), optional
            Collection of texts in BoW format. If given, training will start immediately,
            otherwise, you should call :meth:`~gensim.models.wrappers.ldavowpalwabbit.LdaVowpalWabbit.train` or
            :meth:`~gensim.models.wrappers.ldavowpalwabbit.LdaVowpalWabbit.update` manually for training.
        num_topics : int, optional
            Number of requested latent topics to be extracted from the training corpus.
            Corresponds to VW's ``--lda <num_topics>`` argument.
        id2word : :class:`~gensim.corpora.dictionary.Dictionary`, optional
            Mapping from word ids (integers) to words (strings).
        chunksize : int, optional
            Number of documents examined in each batch.
            Corresponds to VW's ``--minibatch <batch_size>`` argument.
        passes : int, optional
            Number of passes over the dataset to use.
            Corresponds to VW's ``--passes <passes>`` argument.
        alpha : float, optional
            Float effecting sparsity of per-document topic weights.
            This is applied symmetrically, and should be set higher to when documents are thought to look more similar.
            Corresponds to VW's ``--lda_alpha <alpha>`` argument.
        eta : float, optional
            Affects the sparsity of topic distributions.
            This is applied symmetrically, and should be set higher when topics
            are thought to look more similar.
            Corresponds to VW's ``--lda_rho <rho>`` argument.
        decay : float, optional
            Learning rate decay, affects how quickly learnt values are forgotten.
            Should be set to a value between 0.5 and 1.0 to guarantee convergence.
            Corresponds to VW's ``--power_t <tau>`` argument.
        offset: int, optional
            Learning offset, set to higher values to slow down learning on early iterations of the algorithm.
            Corresponds to VW's ``--initial_t <tau>`` argument.
        gamma_threshold : float, optional
            Affects when learning loop will be broken out of, higher values will result in earlier loop completion.
            Corresponds to VW's ``--epsilon <eps>`` argument.
        random_seed : int, optional
            Sets random seed when learning.
            Corresponds to VW's ``--random_seed <seed>`` argument.
        cleanup_files : bool, optional
            Whether or not to delete temporary directory and files used by this wrapper.
            Setting to False can be useful for debugging, or for re-using Vowpal Wabbit files elsewhere.
        tmp_prefix : str, optional
            To prefix temporary working directory name.

        """
        # default parameters are taken from Vowpal Wabbit's defaults, and
        # parameter names changed to match Gensim's LdaModel where possible
        self.vw_path = vw_path
        self.id2word = id2word

        if self.id2word is None:
            if corpus is None:
                raise ValueError(
                    "at least one of corpus/id2word must be specified, to establish input space dimensionality"
                )
            logger.warning("no word id mapping provided; initializing from corpus, assuming identity")
            self.id2word = utils.dict_from_corpus(corpus)
            self.num_terms = len(self.id2word)
        elif len(self.id2word) > 0:
            self.num_terms = 1 + max(self.id2word.keys())
        else:
            self.num_terms = 0

        if self.num_terms == 0:
            raise ValueError("cannot compute LDA over an empty collection (no terms)")

        # LDA parameters
        self.num_topics = num_topics
        self.chunksize = chunksize
        self.passes = passes
        self.alpha = alpha
        self.eta = eta
        self.gamma_threshold = gamma_threshold
        self.offset = offset
        self.decay = decay
        self.random_seed = random_seed
        self._initial_offset = offset

        # temporary files used for Vowpal Wabbit input/output
        self.tmp_dir = None
        self.tmp_prefix = tmp_prefix
        self.cleanup_files = cleanup_files
        self._init_temp_dir(tmp_prefix)

        # used for saving/loading this model's state
        self._model_data = None
        self._topics_data = None

        # cache loaded topics as numpy array
        self._topics = None

        if corpus is not None:
            self.train(corpus)

    def train(self, corpus):
        """Clear any existing model state, and train on given `corpus`.

        Parameters
        ----------
        corpus : iterable of list of (int, int)
            Collection of texts in BoW format.

        """
        logger.debug('Training new model from corpus')

        # reset any existing offset, model, or topics generated
        self.offset = self._initial_offset
        self._topics = None

        corpus_size = write_corpus_as_vw(corpus, self._corpus_filename)

        cmd = self._get_vw_train_command(corpus_size)

        _run_vw_command(cmd)

        # ensure that future updates of this model use correct offset
        self.offset += corpus_size

    def update(self, corpus):
        """Update existing model with `corpus`.

        Parameters
        ----------
        corpus : iterable of list of (int, int)
            Collection of texts in BoW format.

        """
        if not os.path.exists(self._model_filename):
            return self.train(corpus)

        logger.debug('Updating exiting model from corpus')

        # reset any existing topics generated
        self._topics = None

        corpus_size = write_corpus_as_vw(corpus, self._corpus_filename)

        cmd = self._get_vw_update_command(corpus_size)

        _run_vw_command(cmd)

        # ensure that future updates of this model use correct offset
        self.offset += corpus_size

    def log_perplexity(self, chunk):
        """Get per-word lower bound on log perplexity.

        Parameters
        ----------
        chunk : iterable of list of (int, int)
            Collection of texts in BoW format.

        Returns
        -------
        bound : float
            Per-word lower bound on log perplexity.

        """
        vw_data = self._predict(chunk)[1]
        corpus_words = sum(cnt for document in chunk for _, cnt in document)
        bound = -vw_data['average_loss']
        logger.info(
            "%.3f per-word bound, %.1f perplexity estimate based on a held-out corpus of %i documents with %i words",
            bound, numpy.exp2(-bound), vw_data['corpus_size'], corpus_words
        )
        return bound

    def get_topics(self):
        """Get topics X words matrix.

        Returns
        -------
        numpy.ndarray
            `num_topics` x `vocabulary_size` array of floats which represents the learned term topic matrix.

        """
        topics = self._get_topics()
        return topics / topics.sum(axis=1)[:, None]

    def print_topics(self, num_topics=10, num_words=10):
        """Alias for :meth:`~gensim.models.wrappers.dtmmodel.DtmModel.show_topics`.

        Parameters
        ----------
        num_topics : int, optional
            Number of topics to return, set `-1` to get all topics.
        num_words : int, optional
            Number of words.

        Returns
        -------
        list of str
            Topics as a list of strings

        """
        return self.show_topics(num_topics, num_words, log=True)

    def show_topics(self, num_topics=10, num_words=10, log=False, formatted=True):
        """Get the `num_words` most probable words for `num_topics` number of topics.

        Parameters
        ----------
        num_topics : int, optional
            Number of topics to return, set `-1` to get all topics.
        num_words : int, optional
            Number of words.
        log : bool, optional
            If True - will write topics with logger.
        formatted : bool, optional
            If `True` - return the topics as a list of strings, otherwise as lists of (weight, word) pairs.

        Returns
        -------
        list of str
            Topics as a list of strings (if formatted=True) **OR**
        list of (float, str)
            Topics as list of (weight, word) pairs (if formatted=False)

        """
        if num_topics < 0 or num_topics >= self.num_topics:
            num_topics = self.num_topics
        else:
            num_topics = min(num_topics, self.num_topics)

        chosen_topics = range(num_topics)
        shown = []

        for i in chosen_topics:
            if formatted:
                topic = self.print_topic(i, topn=num_words)
            else:
                topic = self.show_topic(i, topn=num_words)

            shown.append(topic)

            if log:
                logger.info("topic #%i (%.3f): %s", i, self.alpha, topic)

        return shown

    def print_topic(self, topicid, topn=10):
        """Get text representation of topic.

        Parameters
        ----------
        topicid : int
            Id of topic.
        topn : int, optional
            Top number of words in topic.

        Returns
        -------
        str
            Topic `topicid` in text representation.

        """
        return ' + '.join('{0:.3f}*{1}'.format(v[0], v[1]) for v in self.show_topic(topicid, topn))

    def show_topic(self, topicid, topn=10):
        """Get `num_words` most probable words for the given `topicid`.

        Parameters
        ----------
        topicid : int
            Id of topic.
        topn : int, optional
            Top number of topics that you'll receive.

        Returns
        -------
        list of (str, float)
            Sequence of probable words, as a list of `(word, word_probability)` for `topicid` topic.

        """
        topics = self._get_topics()
        topic = topics[topicid]
        bestn = matutils.argsort(topic, topn, reverse=True)
        return [(topic[t_id], self.id2word[t_id]) for t_id in bestn]

    def save(self, fname, *args, **kwargs):
        """Save model to file.

        Parameters
        ----------
        fname : str
            Path to output file.

        """
        if os.path.exists(self._model_filename):
            # Vowpal Wabbit uses its own binary model file, read this into
            # variable before serialising this object - keeps all data
            # self contained within a single serialised file
            logger.debug("Reading model bytes from '%s'", self._model_filename)
            with utils.open(self._model_filename, 'rb') as fhandle:
                self._model_data = fhandle.read()

        if os.path.exists(self._topics_filename):
            logger.debug("Reading topic bytes from '%s'", self._topics_filename)
            with utils.open(self._topics_filename, 'rb') as fhandle:
                self._topics_data = fhandle.read()

        if 'ignore' not in kwargs:
            kwargs['ignore'] = frozenset(['_topics', 'tmp_dir'])

        super(LdaVowpalWabbit, self).save(fname, *args, **kwargs)

    @classmethod
    def load(cls, fname, *args, **kwargs):
        """Load model from `fname`.

        Parameters
        ----------
        fname : str
            Path to file with :class:`~gensim.models.wrappers.ldavowpalwabbit.LdaVowpalWabbit`.

        """
        lda_vw = super(LdaVowpalWabbit, cls).load(fname, *args, **kwargs)
        lda_vw._init_temp_dir(prefix=lda_vw.tmp_prefix)

        if lda_vw._model_data:
            # Vowpal Wabbit operates on its own binary model file - deserialise
            # to file at load time, making it immediately ready for use
            logger.debug("Writing model bytes to '%s'", lda_vw._model_filename)
            with utils.open(lda_vw._model_filename, 'wb') as fhandle:
                fhandle.write(lda_vw._model_data)
            lda_vw._model_data = None  # no need to keep in memory after this

        if lda_vw._topics_data:
            logger.debug("Writing topic bytes to '%s'", lda_vw._topics_filename)
            with utils.open(lda_vw._topics_filename, 'wb') as fhandle:
                fhandle.write(lda_vw._topics_data)
            lda_vw._topics_data = None

        return lda_vw

    def __del__(self):
        """Cleanup the temporary directory used by this wrapper."""
        if self.cleanup_files and self.tmp_dir:
            logger.debug("Recursively deleting: %s", self.tmp_dir)
            shutil.rmtree(self.tmp_dir)

    def _init_temp_dir(self, prefix='tmp'):
        """Create a working temporary directory with given prefix.

        Parameters
        ----------
        prefix : str
            Prefix of the temporary directory.

        """
        self.tmp_dir = tempfile.mkdtemp(prefix=prefix)
        logger.info('using %s as temp dir', self.tmp_dir)

    def _get_vw_predict_command(self, corpus_size):
        """Get list of command line arguments for running prediction.

        Parameters
        ----------
        corpus_size : int
            Size of the corpus.

        """
        cmd = [
            self.vw_path,
            '--testonly',  # don't update model with this data
            '--lda_D', str(corpus_size),
            '-i', self._model_filename,  # load existing binary model
            '-d', self._corpus_filename,
            '--learning_rate', '0',  # possibly not needed, but harmless
            '-p', self._predict_filename
        ]

        if self.random_seed is not None:
            cmd.extend(['--random_seed', str(self.random_seed)])

        return cmd

    def _get_vw_train_command(self, corpus_size, update=False):
        """Get list of command line arguments for running model training.

        Parameters
        ----------
        corpus_size : int
            Size of corpus.
        update : bool
            Set `True` to further train an existing model.

        Returns
        -------
        list of str
            Sequence of all training parameters.

        """
        cmd = [
            self.vw_path,
            '-d', self._corpus_filename,
            '--power_t', str(self.decay),
            '--initial_t', str(self.offset),
            '--minibatch', str(self.chunksize),
            '--lda_D', str(corpus_size),
            '--passes', str(self.passes),
            '--cache_file', self._cache_filename,
            '--lda_epsilon', str(self.gamma_threshold),
            '--readable_model', self._topics_filename,
            '-k',  # clear cache
            '-f', self._model_filename
        ]

        if update:
            cmd.extend(['-i', self._model_filename])
        else:
            # these params are read from model file if updating
            cmd.extend([
                '--lda', str(self.num_topics),
                '-b', str(self.num_terms.bit_length()),
                '--lda_alpha', str(self.alpha),
                '--lda_rho', str(self.eta)
            ])

        if self.random_seed is not None:
            cmd.extend(['--random_seed', str(self.random_seed)])

        return cmd

    def _get_vw_update_command(self, corpus_size):
        """Get list of command line arguments to update a model.
        Alias for :meth:`~gensim.models.wrappers.dtmmodel.DtmModel._get_vw_train_command`

        Parameters
        ----------
        corpus_size : int
            Size of the corpus.

        Returns
        -------
        list of str
            Sequence of all training parameters.

        """
        return self._get_vw_train_command(corpus_size, update=True)

    def _load_vw_topics(self):
        """Read topics file generated by Vowpal Wabbit, convert to numpy array."""
        topics = numpy.zeros((self.num_topics, self.num_terms), dtype=numpy.float32)

        with utils.open(self._topics_filename, 'rb') as topics_file:
            found_data = False

            for line in topics_file:
                # look for start of data
                if not found_data:
                    if line.startswith(b'0 ') and b':' not in line:
                        found_data = True
                    else:
                        continue

                fields = line.split()
                word_id = int(fields[0])

                # output contains entries for 2**b terms, where b was set
                # by the '-b' option, ignore anything past num_terms
                if word_id >= self.num_terms:
                    break

                topics[:, word_id] = fields[1:]

        # normalise to probability distribution
        self._topics = topics / topics.sum(axis=1, keepdims=True)

    def _get_topics(self):
        """Get topics matrix, load from file if necessary."""
        if self._topics is None:
            self._load_vw_topics()
        return self._topics

    def _predict(self, chunk):
        """Run given chunk of documents against currently trained model.

        Parameters
        ----------
        chunk : iterable of list of (int, int)
            Sequence of documents in BoW format.

        Returns
        -------
        predictions : ndarray
            Tuple of prediction matrix.
        vw_data : dict
            Vowpal Wabbit data.

        """
        corpus_size = write_corpus_as_vw(chunk, self._corpus_filename)

        cmd = self._get_vw_predict_command(corpus_size)
        vw_data = _parse_vw_output(_run_vw_command(cmd))
        vw_data['corpus_size'] = corpus_size

        predictions = numpy.zeros((corpus_size, self.num_topics), dtype=numpy.float32)

        with utils.open(self._predict_filename, 'rb') as fhandle:
            for i, line in enumerate(fhandle):
                predictions[i, :] = line.split()

        predictions = predictions / predictions.sum(axis=1, keepdims=True)

        return predictions, vw_data

    def __getitem__(self, bow, eps=0.01):
        """Convert document or corpus in BoW format to LDA vectors in BoW format

        Parameters
        ----------
        bow : {list of (int, int), iterable of list of (int, int)}
            Document or corpus in BoW format.
        eps : float
            Threshold value (all topics with probability < `eps` will be ignored.

        Returns
        -------
        list of (int, float)
            LDA vector for document **OR**
        list of list of (int, float)
            LDA vectors for corpus.

        """
        is_corpus, dummy_corpus = utils.is_corpus(bow)
        if not is_corpus:
            bow = [bow]

        predictions = self._predict(bow)[0]

        topics = []
        for row in predictions:
            row_topics = []
            for topic_id, val in enumerate(row):
                if val > eps:
                    row_topics.append((topic_id, val))
            topics.append(row_topics)

        return topics if is_corpus else topics[0]

    def _get_filename(self, name):
        """Get path to given filename in temp directory.

        Parameters
        ----------
        name : str
            Name of the file.

        Returns
        -------
        str
            Path to a file.

        """
        return os.path.join(self.tmp_dir, name)

    @property
    def _model_filename(self):
        """Get path to file to write Vowpal Wabbit model to.

        Returns
        -------
        str
            Path to file to write Vowpal Wabbit model to.

        """
        return self._get_filename('model.vw')

    @property
    def _cache_filename(self):
        """Get path to file to write Vowpal Wabbit cache to.

        Returns
        -------
        str
            Path to file to write Vowpal Wabbit cache to.

        """
        return self._get_filename('cache.vw')

    @property
    def _corpus_filename(self):
        """Get path to file to write Vowpal Wabbit corpus to.

        Returns
        -------
        str
            Path to file to write Vowpal Wabbit corpus to.

        """
        return self._get_filename('corpus.vw')

    @property
    def _topics_filename(self):
        """Get path to file to write Vowpal Wabbit topics to.

        Returns
        -------
        str
            Path to file to write Vowpal Wabbit topics to.

        """
        return self._get_filename('topics.vw')

    @property
    def _predict_filename(self):
        """Get path to file to write Vowpal Wabbit predictions to.

        Returns
        -------
        str
            Path to file to write Vowpal Wabbit predictions to.

        """
        return self._get_filename('predict.vw')

    def __str__(self):
        """Get text representation of model."""
        fields = ['num_terms', 'num_topics', 'chunksize', 'alpha', 'eta']
        kv = ["{0}={1}".format(field, getattr(self, field)) for field in fields]
        return "{0}({1})".format(self.__class__.__name__, ', '.join(kv))

def corpus_to_vw(corpus):
    """Convert corpus to Vowpal Wabbit format.

    Parameters
    ----------
    corpus : iterable of list of (int, int)
        Collection of texts in BoW format.

    Notes
    -----

    Vowpal Wabbit format ::

        | 4:7 14:1 22:8 6:3
        | 14:22 22:4 0:1 1:3
        | 7:2 8:2

    Yields
    ------
    str
        Corpus in Vowpal Wabbit, line by line.

    """
    for entries in corpus:
        line = ['|']
        for word_id, count in entries:
            line.append("{0}:{1}".format(word_id, count))
        yield ' '.join(line)

def write_corpus_as_vw(corpus, filename):
    """Covert `corpus` to  Vowpal Wabbit format and save it to `filename`.

    Parameters
    ----------
    corpus : iterable of list of (int, int)
        Collection of texts in BoW format.
    filename : str
        Path to output file.

    Returns
    -------
    int
        Number of lines in `filename`.

    """
    logger.debug("Writing corpus to: %s", filename)

    corpus_size = 0
    with utils.open(filename, 'wb') as corpus_file:
        for line in corpus_to_vw(corpus):
            corpus_file.write(line.encode('utf-8') + b'\n')
            corpus_size += 1

    return corpus_size

def _parse_vw_output(text):
    """Get dict of useful fields from Vowpal Wabbit's output.

    Parameters
    ----------
    text : str
        Text from vw file.

    Returns
    -------
    dict of (str, float)
        Dictionary with field "average_loss", lower bound on mean per-word log-perplexity.

    """
    data = {}
    for line in text.splitlines():
        if line.startswith('average loss'):
            data['average_loss'] = float(line.split('=')[1])
            break

    return data

def _run_vw_command(cmd):
    """Execute given Vowpal Wabbit command, log stdout and stderr.

    Parameters
    ----------
    cmd : str
        Given Vowpal Wabbit command to execute.

    Returns
    -------
    str
        Stdout and stderr.

    Raises
    ------
    subprocess.CalledProcessError
        If something goes wrong.

    """
    logger.info("Running Vowpal Wabbit command: %s", ' '.join(cmd))
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    output = proc.communicate()[0].decode('utf-8')
    logger.debug("Vowpal Wabbit output: %s", output)

    if proc.returncode != 0:
        raise subprocess.CalledProcessError(proc.returncode, ' '.join(cmd), output=output)

    return output

def vwmodel2ldamodel(vw_model, iterations=50):
    """Convert :class:`~gensim.models.wrappers.ldavowpalwabbit.LdaVowpalWabbit` to
    :class:`~gensim.models.ldamodel.LdaModel`.

    This works by simply copying the training model weights (alpha, beta...) from a trained vwmodel
    into the gensim model.

    Parameters
    ----------
    vw_model : :class:`~gensim.models.wrappers.ldavowpalwabbit.LdaVowpalWabbit`
        Trained Vowpal Wabbit model.
    iterations : int
        Number of iterations to be used for inference of the new :class:`~gensim.models.ldamodel.LdaModel`.

    Returns
    -------
    :class:`~gensim.models.ldamodel.LdaModel`.
        Gensim native LDA.

    """
    model_gensim = LdaModel(
        num_topics=vw_model.num_topics, id2word=vw_model.id2word, chunksize=vw_model.chunksize,
        passes=vw_model.passes, alpha=vw_model.alpha, eta=vw_model.eta, decay=vw_model.decay,
        offset=vw_model.offset, iterations=iterations, gamma_threshold=vw_model.gamma_threshold,
        dtype=numpy.float32
    )
    model_gensim.expElogbeta[:] = vw_model._get_topics()
    return model_gensim


text:
import matplotlib.pyplot as plt
import matplotlib.colors as colors
from mpl_toolkits.axes_grid1.inset_locator import inset_axes

import numpy as np
import itertools as itt
import pathlib as pl

import src.data.rasters
from src.data.load import load
from src.metrics.reliability import signal_reliability
from src.data import dPCA as cdPCA

from src.metrics import dprime as cDP

from src.visualization import fancy_plots as cplt

"""
plots the weights over time for the dPCA context marginalization projection, showing a chunk of the preceding context.
The dPCA has been fitted idependently for the context and the probe.
Aligned to the weights are all the pairwise d' and finally
the hybrid raster/psth of the most weighted neuron at the time with the highest d' across all d' paris
"""

# set the colormap and centre the colorbar
class MidpointNormalize(colors.Normalize):
    """
    Normalise the colorbar so that diverging bars work there way either side from a prescribed midpoint value)

    e.g. im=ax1.imshow(array, norm=MidpointNormalize(midpoint=0.,vmin=-100, vmax=100))
    """

    def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False):
        self.midpoint = midpoint
        colors.Normalize.__init__(self, vmin, vmax, clip)

    def __call__(self, value, clip=None):
        # I'm ignoring masked values and all kinds of edge cases to make a
        # simple example...
        x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]
        return np.ma.masked_array(np.interp(value, x, y), np.isnan(value))

def fit_transform(site, probe, meta, part):
    recs = load(site)

    if len(recs) > 2:
        print(f'\n\n{recs.keys()}\n\n')

    rec = recs['trip0']
    sig = rec['resp']

    # calculates response realiability and select only good cells to improve analysis
    r_vals, goodcells = signal_reliability(sig, r'\ASTIM_*', threshold=meta['reliability'])
    goodcells = goodcells.tolist()

    raster = src.data.rasters.raster_from_sig(sig, probe, channels=goodcells, contexts=meta['transitions'],
                                              smooth_window=meta['smoothing_window'], raster_fs=meta['raster_fs'],
                                              part=part, zscore=meta['zscore'])

    # trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
    trialR, R, _ = cdPCA.format_raster(raster)
    trialR, R = trialR.squeeze(), R.squeeze()  # squeezes out probe

    _, dPCA_projection, _, dpca = cdPCA._cpp_dPCA(R, trialR, significance=False, dPCA_parms={})
    dPCA_projection = dPCA_projection['ct'][:, 0, ...]
    dPCA_weights = np.tile(dpca.D['ct'][:, 0][:, None, None], [1, 1, R.shape[-1]])

    dprime = cDP.pairwise_dprimes(dPCA_projection)

    return dprime, dPCA_projection, dPCA_weights, dpca

def transform(site, probe, meta, part, dpca):
    recs = load(site)

    if len(recs) > 2:
        print(f'\n\n{recs.keys()}\n\n')

    rec = recs['trip0']
    sig = rec['resp']

    # calculates response realiability and select only good cells to improve analysis
    r_vals, goodcells = signal_reliability(sig, r'\ASTIM_*', threshold=meta['reliability'])
    goodcells = goodcells.tolist()

    raster = src.data.rasters.raster_from_sig(sig, probe, channels=goodcells, contexts=meta['transitions'],
                                              smooth_window=meta['smoothing_window'], raster_fs=meta['raster_fs'],
                                              part=part, zscore=meta['zscore'])

    # trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
    trialR, R, _ = cdPCA.format_raster(raster)
    trialR, R = trialR.squeeze(), R.squeeze()  # squeezes out probe

    # transforms context trials withe probe fit
    trialZ = cdPCA.transform_trials(dpca, trialR)
    dPCA_projection = trialZ['ct'][:, 0, ...]
    dPCA_weights = np.tile(dpca.D['ct'][:, 0][:, None, None], [1, 1, R.shape[-1]])

    dprime = cDP.pairwise_dprimes(dPCA_projection)

    return dprime, dPCA_projection, dPCA_weights, dpca

CB_color_cycle = ['#377eb8', '#ff7f00', '#4daf4a', '#a65628',  # blue, orange, green, brow,
                  '#984ea3', '#999999', '#e41a1c', '#dede00']  # purple, gray, scarlet, lime

trans_color_map = {'silence': '#377eb8',  # blue
                   'continuous': '#ff7f00',  # orange
                   'similar': '#4daf4a',  # green
                   'sharp': '#a65628'}  # brown

ci_color = {'shuffled': 'orange',
            'simulated': 'purple'}

transitions = {'P2': {'silence': 0,
                      'continuous': 1,
                      'similar': 3,
                      'sharp': 6},
               'P3': {'silence': 0,
                      'continuous': 2,
                      'similar': 1,
                      'sharp': 5},
               'P5': {'silence': 0,
                      'continuous': 4,
                      'similar': 6,
                      'sharp': 3},
               'P6': {'silence': 0,
                      'continuous': 5,
                      'similar': 4,
                      'sharp': 2}}

# transferable plotting parameters
plt.rcParams['svg.fonttype'] = 'none'
sup_title_size = 15
sub_title_size = 12
ax_lab_size = 12
ax_val_size = 11

meta = {'reliability': 0.1,  # r value
        'smoothing_window': 0,  # ms
        'raster_fs': 30,
        'transitions': ['silence', 'continuous', 'similar', 'sharp'],
        'significance': False,
        'montecarlo': 1000,
        'zscore': False,
        'same_trans': False}

analysis_name = 'dPCA_weights'
analysis_parameters = '_'.join(['{}-{}'.format(key, str(val)) for key, val in meta.items()])
code_to_name = {'t': 'Probe', 'ct': 'Context'}

all_probes = [2, 3, 5, 6]
all_sites = ['ley070a',  # good site. A1
             'ley072b',  # Primary looking responses with strong contextual effects
             'AMT028b',  # good site
             'AMT029a',  # Strong response, somehow visible contextual effects
             'AMT030a',  # low responses, Ok but not as good
             # 'AMT031a', # low response, bad
             'AMT032a']  # great site. PEG

# all_sites = ['AMT029a']
# all_probes = [5]
for site, probe in zip(['AMT029a', 'ley070a'], [5, 2]):
    # for site, probe in itt.product(all_sites, all_probes):

    # gets signal for hybridplot and toe select goodcellss
    recs = load(site)
    if len(recs) > 2:
        print(f'\n\n{recs.keys()}\n\n')
    rec = recs['trip0']
    sig = rec['resp']
    # calculates response realiability and select only good cells to improve analysis
    r_vals, goodcells = signal_reliability(sig, r'\ASTIM_*', threshold=meta['reliability'])
    goodcells = goodcells.tolist()

    # fits and concatenates context probe analysis along the time axis
    dprimesL = list();
    projectionsL = list();
    weightsL = list()

    if meta['same_trans'] is False:
        for part in ['context', 'probe']:
            try:
                badflag = False
                d, p, w, dpca = fit_transform(site, probe, meta, part)
                dprimesL.append(d);
                projectionsL.append(p);
                weightsL.append(w)
            except:
                print(f'failed to analize {site} probe{probe}')
                badflag = True
                break

    elif meta['same_trans'] is True:
        try:
            badflag = False
            # fits transfomrs probe response, append to list
            d, p, w, dpca = fit_transform(site, probe, meta, 'probe')
            dprimesL.append(d);
            projectionsL.append(p);
            weightsL.append(w)

            # transforms  context response, prepends to list
            d, p, w, dpca = transform(site, probe, meta, 'context', dpca)
            dprimesL.insert(0, d);
            projectionsL.insert(0, p);
            weightsL.insert(0, w)

        except:
            print(f'failed to analize {site} probe{probe}')
            badflag = True

    if badflag: continue

    half = dprimesL[0].shape[1]
    dprimes = np.concatenate(dprimesL, axis=-1)
    projections = np.concatenate(projectionsL, axis=-1)
    weights = np.concatenate(weightsL, axis=-1)

    # cuts the whole lenght of the recording to a defined time length
    start = 0.8
    end = 1.5
    offset = start - 1
    stbin = int(np.floor(start * meta['raster_fs']))
    enbin = int(np.floor(end * meta['raster_fs']))
    osbin = int(np.floor(offset * meta['raster_fs']))

    dprimes = dprimes[..., stbin:enbin]
    projections = projections[..., stbin:enbin]
    weights = weights[..., stbin:enbin]

    half = -osbin

    # defines time bins ins seconds
    t = (np.arange(0, dprimes.shape[1]) / meta['raster_fs']) + offset

    # finds highest dprime during probe
    topDbin = np.where(dprimes == np.max(dprimes[:, half:]))[1][0]
    topDtime = topDbin / meta['raster_fs'] + offset

    # formats weights to have consistent signs with the top dprime bin as reference
    toflip = (np.dot(weights[:, 0, :].T, weights[:, 0, topDbin]) < 0) * -2 + 1
    fweights = weights * toflip[None, None, :]

    fig, axes = plt.subplots(3, 1, sharex=True)
    axes = np.ravel(axes)

    # transformation weights
    trans_ax = axes[0]
    wmax = np.max(np.abs(fweights))
    wmin = -wmax
    trans_im = trans_ax.imshow(fweights.squeeze(), aspect='auto',
                               extent=[offset, end + offset - start, fweights.shape[0], 0],
                               cmap='PuOr', clim=(wmin, wmax), norm=MidpointNormalize(midpoint=0, vmin=wmin, vmax=wmax))
    trans_ax.axvline(0, linestyle=':', color='gray')
    trans_ax.axvline(topDtime, linestyle='--', color='Black')

    cbar_ax = inset_axes(trans_ax,
                         width="1%",  # width = 50% of parent_bbox width
                         height="50%",  # height : 5%
                         loc='upper right')

    fig.colorbar(trans_im, cax=cbar_ax, orientation='vertical', ticks=[wmin, 0, wmax])
    cbar_ax.yaxis.set_ticks_position("left")

    # dprimes
    dprime_ax = axes[1]
    for ii, (c0, c1) in enumerate(itt.combinations(meta['transitions'], 2)):
        dprime_ax.plot(t, dprimes[ii, :], label=f"{c0} vs {c1}")
    dprime_ax.legend()
    dprime_ax.axvline(0, linestyle=':', color='gray')
    dprime_ax.axvline(topDtime, linestyle='--', color='black')

    # raw raster
    epoch_names = [f"C{transitions[f'P{probe}'][trans]}_P{probe}" for trans in meta['transitions']]
    topcell_idx = np.argmax(np.abs(weights[:, 0, topDbin]))
    topcell = goodcells[topcell_idx]
    trans_colors = [trans_color_map[trans] for trans in meta['transitions']]

    raster_ax = axes[2]
    cplt.hybrid(sig, epoch_names, channels=topcell, psth_fs=meta['raster_fs'],
                time_strech=[start, end], time_offset=offset, axes=[raster_ax],
                legend=True, labels=meta['transitions'], colors=trans_colors)
    raster_ax.axvline(0, linestyle=':', color='gray')
    raster_ax.axvline(topDtime, linestyle='--', color='black')

    # horizonta line in weightplot indicating most weighted cell
    trans_ax.axhline(topcell_idx + 0.5, linestyle='--', color='Black')

    # Formatting

    for ax in [trans_ax, dprime_ax, raster_ax]:
        ax.tick_params(labelsize=ax_val_size)
        ax.spines['right'].set_visible(False)
        ax.spines['top'].set_visible(False)

    # ax labels
    trans_ax.set_ylabel('neuron\nweights', fontsize=ax_lab_size)
    dprime_ax.set_ylabel("pairwise\nd'", fontsize=ax_lab_size)

    raster_ax.set_ylabel('top cell\nspike rate (Hz)', fontsize=ax_lab_size)
    raster_ax.set_xlabel('time (s)', fontsize=ax_lab_size)

    raster_ax.set_title('')

    # suptitle = f"{site} probe {probe} dPCA weights {meta['raster_fs']}Hz zscore {meta['zscore']} same_trans-{meta['same_trans']}"
    suptitle = f"{site} probe {probe} dPCA"
    fig.suptitle(suptitle, fontsize=20)
    fig.set_size_inches(7, 4)
    # fig.set_size_inches(14, 7)

    # Export figures
    analysis = f"dPCA_weights_{meta['raster_fs']}Hz_zscore-{meta['zscore']}_same_trans-{meta['same_trans']}"

    root = pl.Path(f'/home/mateo/Pictures/APAM/final/{analysis}')
    if not root.exists(): root.mkdir(parents=True, exist_ok=True)

    png = root.joinpath(suptitle).with_suffix('.png')
    fig.savefig(png, transparent=False, dpi=100)

    svg = root.joinpath(suptitle).with_suffix('.svg')
    fig.savefig(svg, transparent=True)


text:
# Diffusion Maps Framework implementation as part of MSc Data Science Project of student
# Napoleon Koskinas at University of Southampton, MSc Data Science course

# Script 5: Visulisation-scatterplot

import openpyxl

import numpy as np
import pandas as pd

import matplotlib
import matplotlib.cm as cm
import matplotlib.pyplot as plt

from itertools import cycle

matplotlib.style.use('ggplot')
matplotlib.rcParams['legend.scatterpoints'] = 1

datasource = './data/pcaData.xlsx'
dtsource = './data/datetimes.xlsx'

def main():
	# Read PCA data, get names of the 6 sheets, for different types of bacteria
	xlData = pd.ExcelFile(datasource)
	sheetNames = xlData.sheet_names

	dtExcel = pd.ExcelFile(dtsource)

	fig = plt.figure()

	no = 331
	inc = [0,1,2,1,2,1]

	licycle = cycle(inc)
	for bactName in sheetNames:
		no += licycle.next()
		worksheet = xlData.parse(bactName)

		# Keep only the actual timeseries data, last 30 columns
		X = pd.DataFrame(worksheet.ix[:,:1]).as_matrix()

		dtDf = dtExcel.parse(bactName)
		dt = pd.DataFrame(dtDf).as_matrix()

		scatterplot(X, bactName, no, fig, dt)

	plt.tight_layout()
	plt.show()

def  scatterplot(X, bactName, no, fig, datetimes):

	markers = ['d', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', '^', '^', '^', '^', '^', '^', '^', '^', '^', '^', '+']
	colors = ['purple', 'b', 'aqua', 'lightseagreen', 'green', 'lightgreen', 'orangered', 'magenta', 'orchid', 'purple', 'darkblue', 'b', 'g', 'lightgreen', 'y', 'orange', 'r', 'magenta', 'purple', 'b', 'aqua', 'lightseagreen', 'g', 'lightgreen', 'orangered', 'magenta', 'orchid', 'purple', 'darkblue', 'b']

	ax = fig.add_subplot(no)
	ax.set_title(bactName)
	points = []
	names = []

	for name_arr in datetimes.tolist():
		names.append(name_arr[0])

	for m, c, i, j in zip(markers, colors, X[:,0], X[:,1]):
		points.append(ax.scatter(i, j, s = 50, marker=m, c=c))

	fig.legend(points, names, 'center right', ncol=1)

	plt.xlabel('PC1')
	plt.ylabel('PC2')

	return fig

if __name__ == '__main__':
	main()

max_stars_repo_path: exploration scripts/scatterplot.py

max_stars_repo_name: koskinap/diffusion-maps

max_issues_repo_path: exploration scripts/scatterplot.py

max_issues_repo_name: koskinap/diffusion-maps

max_forks_repo_path: exploration scripts/scatterplot.py

max_forks_repo_name: koskinap/diffusion-maps


text:
import numpy as np
import os, math, sys
from structure.minibatch import *
from structure.sample import *

def bytes_to_int(bytes_array):
    result = 0
    for b in bytes_array:
        result = result * 256 + int(b)
    return result

class BatchPatcher(object):
    def __init__(self, size_of_data, batch_size, num_of_classes, replacement = False):
        self.size_of_data = size_of_data
        self.batch_size = batch_size
        self.num_of_classes = num_of_classes
        self.num_iters_per_epoch = int(math.ceil(float(size_of_data) / float(batch_size)))

        # For in-memory data
        self.loaded_data = []

        # Replacement in mini-batch for random batch selection
        self.replacement = replacement

        # Custom noise
        self.noise_rate = 0.0
        self.transition_matrix = None

    def bulk_load_in_memory(self, sess, ids, images, labels):
        # initialization
        self.loaded_data.clear()

        for i in range(self.size_of_data):
            self.loaded_data.append(None)

        # load data set in memory
        set_test = set()
        for i in range(self.num_iters_per_epoch * 2):
            mini_ids, mini_images, mini_labels = sess.run([ids, images, labels])
            mini_images /= 255.0
            for j in range(self.batch_size):
                id = bytes_to_int(mini_ids[j])

                if not id in set_test:
                    self.loaded_data[id] = Sample(id, mini_images[j], bytes_to_int(mini_labels[j]))
                    set_test.add(id)

        print("# of samples: ", len(self.loaded_data))

    def get_next_random_mini_batch(self, num_of_samples, table=None, normalized=True):
        if table is None:
            selected_sample_ids = np.random.choice(self.size_of_data, num_of_samples, self.replacement)
        else:
            if normalized:
                prob_table = np.zeros(self.size_of_data, dtype=float)
                for i in range(len(table)):
                    prob_table[i] = table[i]

                total_sum = np.sum(prob_table)
                prob_table = prob_table / total_sum
            else:
                prob_table = table

            selected_sample_ids = np.random.choice(self.size_of_data, num_of_samples, self.replacement, p=prob_table)

        # Fetch mini-batch samples from loaded_data in main memory
        mini_batch = MiniBatch()
        for id in selected_sample_ids:
            sample = self.loaded_data[id]
            mini_batch.append(sample.id, sample.image, sample.label)

        return mini_batch.ids, mini_batch.images, mini_batch.labels

    def get_init_mini_batch(self, init_id):
        # init_id from 0~self.num_iters_per_epoch
        selected_sample_ids = list(range(init_id*self.batch_size, init_id*self.batch_size + self.batch_size))

        # Fetch mini-batch samples from loaded_data in main memory
        mini_batch = MiniBatch()
        for id in selected_sample_ids:
            if id >= self.size_of_data:
                continue
            else:
                sample = self.loaded_data[id]
                mini_batch.append(sample.id, sample.image, sample.label)

        return mini_batch.ids, mini_batch.images, mini_batch.labels

    def set_noise(self, noise_rate, noise_type):
        if noise_type != "symmetry" and noise_type != "pair" and noise_type != "none":
            print("Noise type error!")
            sys.exit(1)
        print("Noise Injection: ", noise_type)

        self.noise_rate = noise_rate
        self.transition_matrix = []
        for i in range(self.num_of_classes):
            self.transition_matrix.append([])
            for j in range(self.num_of_classes):
                self.transition_matrix[i].append(0.0)

        if noise_type == "symmetry":
            for i in range(self.num_of_classes):
                for j in range(self.num_of_classes):
                    if i == j:
                        self.transition_matrix[i][j] = 1.0 - self.noise_rate
                    else:
                        self.transition_matrix[i][j] = noise_rate / float(self.num_of_classes - 1)
        elif noise_type == "pair":
            for i in range(self.num_of_classes):
                self.transition_matrix[i][(i + 1) % self.num_of_classes] = noise_rate
                for j in range(self.num_of_classes):
                    if i == j:
                        self.transition_matrix[i][j] = 1.0 - noise_rate
        elif noise_type == "none":
            for i in range(self.num_of_classes):
                for j in range(self.num_of_classes):
                    if i == j:
                        self.transition_matrix[i][j] = 1.0

        for sample in self.loaded_data:
            sample.label = self.get_noise_label(self.transition_matrix[sample.true_label])
            sample.last_corrected_label = sample.label
            sample.first_label = sample.label

    def set_custom_noise(self, path):
        if not os.path.exists(path):
            print("Custom noise transition matrix is not exist in the path!")

        self.transition_matrix = []
        with open(path) as fp:
            for line in fp:
                temp = []
                for item in line.split(","):
                    temp.append(float(item))
                self.transition_matrix.append(temp)

        for sample in self.loaded_data:
            sample.label = self.get_noise_label(self.transition_matrix[sample.true_label])
            sample.last_corrected_label = sample.label
            sample.first_label = sample.label

    def get_noise_label(self, transition_array):

        return np.random.choice(self.num_of_classes, 1, True, p=transition_array)[0]

    def print_transition_matrix(self, transition_matrix, log = None):
        log_str = ""
        for i in range(len(transition_matrix)):
            for j in range(len(transition_matrix[i])):
                print(str(transition_matrix[i][j]) + " ", end=',')
                log_str += str(transition_matrix[i][j]) + ", "
            print("\n")
            log_str += "\n"
        log_str += "\n"
        if log is not None:
            log.append(log_str)

    def get_current_noise_matrix(self, entire=True):
        noise_matrix = np.zeros([self.num_of_classes, self.num_of_classes], dtype = int)
        for sample in self.loaded_data:
            if entire or sample.corrected:
                noise_matrix[sample.true_label][sample.last_corrected_label] += 1
        return noise_matrix

max_stars_repo_name: kaist-dmlab/RecencyBias


text:
"""
    Name: test_NMFconv
    Date of Revision: Jul 2019
    Programmer: Christian Dittmar, Yiğitcan Özer

    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    If you use the 'NMF toolbox' please refer to:
    [1] Patricio López-Serrano, Christian Dittmar, Yiğitcan Özer, and Meinard
        Müller
        NMF Toolbox: Music Processing Applications of Nonnegative Matrix
        Factorization
        In Proceedings of the International Conference on Digital Audio Effects
        (DAFx), 2019.

    License:
    This file is part of 'NMF toolbox'.
    https://www.audiolabs-erlangen.de/resources/MIR/NMFtoolbox/
    'NMF toolbox' is free software: you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by the
    the Free Software Foundation, either version 3 of the License, or (at
    your option) any later version.

    'NMF toolbox' is distributed in the hope that it will be useful, but
    WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
    Public License for more details.

    You should have received a copy of the GNU General Public License along
    with 'NMF toolbox'. If not, see http://www.gnu.org/licenses/.
"""

import os
import numpy as np
import scipy.io.wavfile as wav

from NMFtoolbox.forwardSTFT import forwardSTFT
from NMFtoolbox.initTemplates import initTemplates
from NMFtoolbox.initActivations import initActivations
from NMFtoolbox.utils import make_monaural, pcmInt16ToFloat32Numpy
from NMFtoolbox.NMFconv import NMFconv

def run_NMFconv():
    inpPath = '../data'
    filename = 'runningExample_AmenBreak.wav'

    # read signals
    fs, x = wav.read(os.path.join(inpPath, filename))

    # make monaural if necessary
    x = make_monaural(x)

    # convert wav from int16 to float32
    x = pcmInt16ToFloat32Numpy(x)

    paramSTFT = dict()
    paramSTFT['blockSize'] = 2048
    paramSTFT['hopSize'] = 512
    paramSTFT['winFunc'] = np.hanning(paramSTFT['blockSize'])
    paramSTFT['reconstMirror'] = True
    paramSTFT['appendFrame'] = True
    paramSTFT['numSamples'] = len(x)

    # STFT computation
    X, A, P = forwardSTFT(x, paramSTFT)

    # get dimensions and time and freq resolutions
    numBins, numFrames = X.shape
    deltaT = paramSTFT['hopSize'] / fs
    deltaF = fs / paramSTFT['blockSize']

    # 3. apply NMF variants to STFT magnitude
    # set common parameters
    numComp = 3
    numIter = 3
    numTemplateFrames = 8

    # generate initial guess for templates
    paramTemplates = dict()
    paramTemplates['deltaF'] = deltaF
    paramTemplates['numComp'] = numComp
    paramTemplates['numBins'] = numBins
    paramTemplates['numTemplateFrames'] = numTemplateFrames
    initW = initTemplates(paramTemplates, 'drums')

    # generate initial activations
    paramActivations = dict()
    paramActivations['numComp'] = numComp
    paramActivations['numFrames'] = numFrames
    initH = initActivations(paramActivations, 'uniform')

    # NMFconv parameters
    paramNMFconv = dict()

    paramNMFconv['numComp'] = numComp
    paramNMFconv['numFrames'] = numFrames
    paramNMFconv['numIter'] = numIter
    paramNMFconv['numTemplateFrames'] = numTemplateFrames
    paramNMFconv['initW'] = initW
    paramNMFconv['initH'] = initH
    paramNMFconv['beta'] = 0

    # NMFconv core method
    nmfconvW, nmfconvH, nmfconvV, divBeta = NMFconv(A, paramNMFconv)

    python_res = {'nmfconvW': nmfconvW,
                  'nmfconvH': nmfconvH,
                  'nmfconvV': nmfconvV,
                  'divBeta': divBeta.reshape(1, -1)}

    return python_res

reason: import numpy,import scipy