text:
from numpy import ones, pi, array
from unittest import TestCase
from os.path import join
import matplotlib.pyplot as plt
from pyleecan.Tests import save_validation_path as save_path

from pyleecan.Classes.Simu1 import Simu1
from pyleecan.Tests.Validation.Machine.IPMSM_xxx import IPMSM_xxx

from pyleecan.Classes.InCurrent import InCurrent
from pyleecan.Classes.ImportGenVectLin import ImportGenVectLin
from pyleecan.Classes.ImportMatrixVal import ImportMatrixVal

from pyleecan.Classes.MagFEMM import MagFEMM
from pyleecan.Classes.Output import Output

simu = Simu1(name="EM_IPMSM_FL_001", machine=IPMSM_xxx)

# Definition of the enforced output of the electrical module
Nr = ImportMatrixVal(value=ones(4) * 3000)
Is = ImportMatrixVal(
    value=array(
        [
            [6.97244193e-06, 2.25353053e02, -2.25353060e02],
            [-2.60215295e02, 1.30107654e02, 1.30107642e02],
            [-6.97244208e-06, -2.25353053e02, 2.25353060e02],
            [2.60215295e02, -1.30107654e02, -1.30107642e02],
        ]
    )
)
time = ImportGenVectLin(start=0, stop=0.015, num=4, endpoint=True)
angle = ImportGenVectLin(start=0, stop=2 * pi, num=1024, endpoint=False)

simu.input = InCurrent(
    Is=Is,
    Ir=None,  # No winding on the rotor
    Nr=Nr,
    angle_rotor=None,  # Will be computed
    time=time,
    angle=angle,
    angle_rotor_initial=0.5216 + pi,
)

# Definition of the magnetic simulation (no symmetry)
simu.mag = MagFEMM(
    is_stator_linear_BH=2, is_rotor_linear_BH=2, is_symmetry_a=False, is_antiper_a=True
)
simu.struct = None
# Copy the simu and activate the symmetry
simu_sym = Simu1(init_dict=simu.as_dict())
simu_sym.mag.is_symmetry_a = True
simu_sym.mag.sym_a = 4
simu_sym.mag.is_antiper_a = False

class test_EM_IPMSM_FL_001(TestCase):
    """unittest FEMM machine IPMSM_xxx
    """

    def test_Magnetic_FEMM_sym(self):
        """Test compute the Flux in FEMM, with and without symmetry
		"""

        out = Output(simu=simu)
        out.post.legend_name = "No symmetry"
        simu.run()

        out2 = Output(simu=simu_sym)
        out2.post.legend_name = "1/2 symmetry"
        out2.post.line_color = "r--"
        simu_sym.run()

        # Plot the result by comparing the two simulation
        plt.close("all")
        out.plot_B_space(out_list=[out2])

        fig = plt.gcf()
        fig.savefig(join(save_path, "test_EM_IPMSM_FL_001_sym.png"))

reason: from numpy


text:
from typing import List
import matplotlib.pyplot as plt
plt.style.use('bmh')
import numpy as np
import pandas as pd
import mplcursors

plt.rcParams["figure.figsize"] = (16, 8)

def filterZeros(nums: List[int]):
    return [x if x > 0 else float('nan') for x in nums]

def prepYData(days, data):
    res = []
    for d in days:
        x = list(data[(data['date'] == d)]['cases'])
        res.append(0 if len(x) == 0 else x[0])
    return res

# states = ['California', 'New York', 'Massachusetts', 'Louisiana', 'Texas', 'Alabama']
states = ['California', 'Washington', 'New York', 'New Jersey', 'Massachusetts', 'Louisiana', 'Texas', 'Florida', 'Michigan', 'Illinois']

# source: https://github.com/nytimes/covid-19-data
data = pd.read_csv('../../covid-19-data/us-states.csv')

exts = [float('nan')] * 5
xexts = [str(x) for x in range(5)]
days = data['date'].drop_duplicates(keep='last')
x = [d[5:] for d in days] + xexts

yStates = {}
for state in states:
    stateData = data[(data['state'] == state)][['date', 'cases']]
    yStates[state] = prepYData(days, stateData) + exts

filterOutDays = 0

print('California Y: {}'.format(yStates['California']))

fig, ax = plt.subplots()
ax.set_yscale('log')
ax.set_xlim([0, 50])
ax.set_ylim([0, 1000000])
ax.minorticks_on()
ax.grid(color='gray')

# build the timeline
timeline = {
    '01-23': 'Wuhan lockdown',
    '01-30': 'WHO: Global Emergency',
    '02-25': 'Mardi Gras',
    '03-08': 'LA marathon',
    '03-11': 'WHO declared Pandemic',
    '03-13': 'National Emergency',
    '03-17': 'Bay Area Shelter-in-Place',
    '03-19': 'CA Shelter-in-Place',
    '03-20': 'NY Stay-at-Home'
}
quotes, q = {}, 1

for i, d in enumerate(x):
    # print('d = {}'.format(d))
    if d in timeline.keys():
        ax.annotate(str(q), xy=(i, 100), xytext=(i, 55), arrowprops=dict(facecolor='red', shrink=0.05))
        quotes[i] = timeline[d]
        ax.text(70, (1.5)**(len(timeline)-q), str(q) + ': ' + timeline[d])
        q += 1

ax.text(70, 50, 'Notes:')

print('x: {}'.format(x[filterOutDays:]))
for state in states:
    ax.plot(x[filterOutDays:], yStates[state][filterOutDays:], label=state)

plt.legend()
plt.xlabel('Days')
plt.ylabel('# of confirmed cases')
plt.title('States of US Confirmed COVID-19 Cases (based on NYTimes)')
plt.xticks(rotation=45)
plt.xticks(x[::2])

# mplcursors.cursor(line)

plt.show()

ext: py

reason: import numpy


text:
from sklearn.datasets import fetch_20newsgroups
import torchvision
from sklearn.feature_extraction.text import TfidfVectorizer
from os.path import join
import numpy as np
import pickle

#TODO: Update mnist examples!!!

def dataset_loader(dataset_path=None, dataset='mnist', seed=1):
    """
    Loads a dataset and creates the appropriate train/test splits
    :param dataset_path: path to the datasets
    :param dataset: the dataset to be loaded
    :param seed: seed for creating train/test splits (when appropriate)
    :return:
    """

    np.random.seed(seed)
    n_train = 5000

    if dataset == 'mnist':
        train_data, train_labels, test_data, test_labels = load_mnist(dataset_path)
        train_data = train_data[:n_train, :]
        train_labels = train_labels[:n_train]
    elif dataset == '20ng':
        train_data, train_labels, test_data, test_labels = load_20ng_dataset_bow()
        train_data = train_data[:n_train, :]
        train_labels = train_labels[:n_train]
    elif dataset == '15scene':
        data, labels = load_15_scene_bow_features(dataset_path)
        # Get a split with 100 samples per class
        train_idx, test_idx = [], []
        idx = np.random.permutation(data.shape[0])
        data = data[idx]
        labels = labels[idx]
        for i in range(15):
            class_idx = np.where(labels == i)[0]
            train_idx.append(class_idx[:100])
            test_idx.append(class_idx[100:])
        train_idx = np.int64(np.concatenate(train_idx))
        test_idx = np.int64(np.concatenate(test_idx))
        # Get the actual split
        train_data = data[train_idx, :]
        train_labels = labels[train_idx]
        test_data = data[test_idx, :]
        test_labels = labels[test_idx]

    elif dataset == 'corel':
        data, labels = load_corel_bow_features(dataset_path)
        idx = np.random.permutation(data.shape[0])
        data = data[idx]
        labels = labels[idx]

        # Get the actual split
        train_data = data[:4800, :]
        train_labels = labels[:4800]
        test_data = data[4800:, :]
        test_labels = labels[4800:]
    elif dataset == 'yale':
        data, labels = load_yale_dataset(dataset_path)

        # Get a split with 30 per sample
        train_idx, test_idx = [], []
        idx = np.random.permutation(data.shape[0])
        data = data[idx]
        labels = labels[idx]
        for i in range(38):
            class_idx = np.where(labels == i)[0]
            train_idx.append(class_idx[:30])
            test_idx.append(class_idx[30:])
        train_idx = np.int64(np.concatenate(train_idx))
        test_idx = np.int64(np.concatenate(test_idx))

        # Get the actual split
        train_data = data[train_idx, :]
        train_labels = labels[train_idx]
        test_data = data[test_idx, :]
        test_labels = labels[test_idx]

    elif dataset == 'kth':
        train_data, train_labels, test_data, test_labels = load_kth(dataset_path)
        idx = np.random.permutation(train_data.shape[0])
        train_data, train_labels = train_data[idx], train_labels[idx]
        idx = np.random.permutation(test_data.shape[0])
        test_data, test_labels = test_data[idx], test_labels[idx]
    else:
        print("Unknown dataset!")
        assert False

    return train_data, train_labels, test_data, test_labels

def load_mnist(dataset_path):
    """
    Loads the MNIST dataset
    :return:
    """

    # Get the train split
    mnist = torchvision.datasets.MNIST(root=dataset_path, download=True, train=True)
    x_train, y_train = mnist.train_data.numpy(), mnist.train_labels.numpy()

    # Get the test split
    mnist = torchvision.datasets.MNIST(root=dataset_path, download=True, train=False)
    x_test, y_test = mnist.test_data.numpy(), mnist.test_labels.numpy()

    x_train = x_train.reshape((x_train.shape[0], -1)) / 255.0
    x_test = x_test.reshape((x_test.shape[0], -1)) / 255.0

    return np.float32(x_train), y_train, np.float32(x_test), y_test

def load_20ng_dataset_bow():
    """
    Loads the 20NG dataset
    :return:
    """

    newsgroups_train = fetch_20newsgroups(subset='train')
    newsgroups_test = fetch_20newsgroups(subset='test')

    # Convert data to tf-idf

    vectorizer = TfidfVectorizer(min_df=0.01, max_df=0.95)
    train_data = vectorizer.fit_transform(newsgroups_train.data)
    test_data = vectorizer.transform(newsgroups_test.data)
    train_data = train_data.todense()
    test_data = test_data.todense()
    train_labels = newsgroups_train.target
    test_labels = newsgroups_test.target

    return train_data, train_labels, test_data, test_labels

def load_15_scene_bow_features(datasets_path):
    """
    Loads the pre-extracted BoF features for the 15-scene dataset
    :return:
    """
    datafile = join(datasets_path, 'scenes.pickle')
    with open(datafile, 'rb') as f:
        features = pickle.load(f, encoding='latin1')
        labels = pickle.load(f, encoding='latin1')
    labels = np.asarray(np.squeeze(labels), dtype='int')
    features = np.asarray(features, dtype='float32')
    return features, labels

def load_corel_bow_features(datasets_path):
    """
    Loads the pre-extracted BoF features for the Corel dataset
    :return:
    """
    datafile = join(datasets_path, 'corel.pickle')

    with open(datafile, 'rb') as f:
        features = pickle.load(f, encoding='latin1')
        labels = pickle.load(f, encoding='latin1')
    labels = np.asarray(np.squeeze(labels), dtype='int')
    features = np.asarray(features, dtype='float32')

    return features, labels

def load_kth(datasets_path):
    """
    Loads the HoF/HoG features for the KTH dataset
    :return:
    """
    datafile = join(datasets_path, 'kth.pickle')
    with open(datafile, 'rb') as f:
        train_data, train_labels = pickle.load(f, encoding='latin1'), pickle.load(f, encoding='latin1')
        test_data, test_labels = pickle.load(f, encoding='latin1'), pickle.load(f, encoding='latin1')

    return train_data, train_labels, test_data, test_labels

def load_yale_dataset(datasets_path):
    """
    Loads the ORL dataset
    """
    datafile = join(datasets_path, 'yale.pickle')

    with open(datafile, 'rb') as f:
        features = pickle.load(f)
        labels = pickle.load(f)

    features = [x for x in features]
    features = np.asarray(features) / 255.0
    labels = np.asarray(np.squeeze(labels), dtype='int')

    return features, labels

max_stars_repo_name: passalis/sef

max_issues_repo_name: passalis/sef

max_forks_repo_name: passalis/sef


text:
# ----- Z99 Factor Analysis Template -------
from quantopian.research import run_pipeline
from quantopian.pipeline import Pipeline
from quantopian.pipeline.factors import Latest
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline.data import morningstar
from quantopian.pipeline.factors import CustomFactor, SimpleMovingAverage, AverageDollarVolume, Returns, RSI
from quantopian.pipeline.classifiers.morningstar import Sector
from quantopian.pipeline.filters import Q500US, Q1500US
from time import time

import talib
import pandas as pd
import numpy as np

import alphalens as al
import pyfolio as pf
from scipy import stats
import matplotlib.pyplot as plt

# ----- morningstar symbols for factor calculation -------
bs = morningstar.balance_sheet
cfs = morningstar.cash_flow_statement
is_ = morningstar.income_statement
or_ = morningstar.operation_ratios
er = morningstar.earnings_report
v = morningstar.valuation
vr = morningstar.valuation_ratios

# --- Factor Algorithm Below -----------------------------
#---------------------------------------------------------
class Price_To_Book(CustomFactor):

    # Pre-declare inputs and window_length
    inputs = [USEquityPricing.close, vr.book_value_per_share]
    window_length = 1

        # Compute market cap value
    def compute(self, today, assets, out, close, bv):
        out[:] = close[-1] / bv[-1]

#---------------------------------------------------------
# --- Factor Algorithm Above -----------------------------

class Volatility (CustomFactor):

    inputs = [USEquityPricing.close]
    window_length = 252

    def compute(self, today, assets, out, close):
        close = pd.DataFrame(data=close, columns=assets)
        # Going to rank largest we'll need to invert the sdev.
        # Annualized volatility is the standard deviation of logarithmic returns
        # as a Ratio of its mean divided by 1/ sqrt of the number of trading days
        out[:] = 1 / np.log(close).diff().std()

        universe = Q1500US()

        #----- Variables to modify each run -------
vol = Volatility()
vol_factor = 'Volatility'

#----- Variables to modify each run -------
factor = Price_To_Book()
select_factor = 'Price_To_Book'

# ----- Pipeline construction ----------
pipe = Pipeline(screen=universe)
pipe.add(factor, select_factor)

# ------- Ranking factor top to bottom -------
factor_rank = factor.rank()

# ------- Ranking factor top to bottom -------
factor_rank = factor.rank()

# ------- Adding Factor Rank to Pipeline as separate column -------
pipe.add(factor_rank, 'factor_rank')

# ------- Measurement inteval for the analysis -------
#start = pd.Timestamp("2015-01-01")
start = pd.Timestamp("2015-01-01")

#end = pd.Timestamp("2017-06-30")
#start = pd.Timestamp("2017-06-30")
#start = pd.Timestamp("2017-08-01")

end = pd.Timestamp("2017-08-29")

# ------- Run the pipeline and measure runtime -------
start_timer = time()
results = run_pipeline(pipe, start_date=start, end_date=end)
end_timer = time()
#results.fillna(value=0);

#results.count() #results.first_valid_index() #results.get_ftype_counts() #results.abs() #results.add_prefix("Z99-")
#results.add_suffix("-Z99") #results.describe() #results.sort #results.tail(50)

# ------- Run the pipeline and measure Runtime to construct pipeline wiht pricing and factor -------
results.head(100)

# A MultiIndex DataFrame indexed by date (level 0) and asset (level 1) # [n Date rows x m Equity columns]
assets = results.index.levels[1].unique()

# ------ Get pricing for dates and equities ---------
pricing = get_pricing(assets, start - pd.Timedelta(days=30), end + pd.Timedelta(days=30), fields="close_price")
# FYI... end + pd.Timedelta(days=30) = Timestamp('2017-07-30 00:00:00')
#pricing

#al.tears.create_factor_tear_sheet(results[select_factor], pricing, )
#select_factor = 'Price_To_Book2'

# Ingest and format data
factor_data = al.utils.get_clean_factor_and_forward_returns(results[select_factor], pricing)

# Show factor_data snapshot before running al
factor_data.head(15)

# Run AlphaLens Analysis
al.tears.create_full_tear_sheet(factor_data)

reason: import numpy,from scipy


text:
import torch
import numpy as np
#import pathos.multiprocessing as mp
import multiprocessing as mp

class EvolutionaryStrategy:

    def __init__(self, model, fitness, impact, processes=4, populationsize=10, learning_rate=0.5):
        self.model = model(transfer = True)
        self.processes = processes
        self.fitness = fitness(individuals=populationsize)
        self.learning_rate = learning_rate
        self.populationsize = populationsize
        self.impact = impact

    def evolution(self, id, output):
        seed = int(torch.randint(0,100000,(1,)))
        torch.manual_seed(seed)
        epsilon = {}
        for key, shape in self.model.shape().items():
            if self.model.params[key].type() == "torch.FloatTensor":
                epsilon[key] = torch.randn(shape).float()
            elif self.model.params[key].type() == "torch.LongTensor":
                epsilon[key] = torch.randn(shape).long()
            else:
                epsilon[key] = torch.randn(shape)
        # fitness function
        reward = self.fitness.evaluate(self.model, epsilon,self.learning_rate, self.impact,5 ,id)
        output.put((reward , seed))    # book keeping
        return

    def play_game(self,num_episode):
        reward = self.fitness.evaluate(self.model, 0, 0, 0, num_episode, 0)
        return reward

    def parallel_evolution(self):

        pool = mp.Pool(self.processes)
        results = pool.map(self.evolution,range(self.populationsize))
        #pool.close()

        # Setup a list of processes that we want to run
        #processes = [mp.Process(target=self.evolution, args=(x, output)) for x in range(permutations)]
        #self.model.update_params(epsilons, self.learning_rate)
        # Run processes
        '''for p in processes:
            p.start()

        # Exit the completed processes
        for p in processes:
            p.join()

        # Get process results from the output queue
        results = [output.get() for p in processes]
        '''
        #print(results)
    def sequential_evolution(self):
        epsilons = []
        rewards = []
        for _ in range(self.populationsize):
            reward, epsilon = self.evolution(impact,1)
            epsilons.append(epsilon)
            rewards.append(reward)
        self.model.update_params(epsilons, rewards, self.learning_rate)

max_stars_repo_name: ChristianIngwersen/BombermanRL

max_issues_repo_name: ChristianIngwersen/BombermanRL

max_forks_repo_name: ChristianIngwersen/BombermanRL


text:
#!/usr/bin/python
"""
FlowCal Python API example, without using calibration beads data.

This script is divided in two parts. Part one processes data from ten cell
samples and generates plots of each one.

Part two exemplifies how to use the processed cell sample data with
FlowCal's plotting and statistics modules to produce interesting plots.

For details about the experiment, samples, and instrument used, please
consult readme.txt.

"""
import os
import os.path
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import FlowCal

###
# Definition of constants
###

# Names of the FCS files containing data from cell samples
samples_filenames = ['FCFiles/sample006.fcs',
                     'FCFiles/sample007.fcs',
                     'FCFiles/sample008.fcs',
                     'FCFiles/sample009.fcs',
                     'FCFiles/sample010.fcs',
                     'FCFiles/sample011.fcs',
                     'FCFiles/sample012.fcs',
                     'FCFiles/sample013.fcs',
                     'FCFiles/sample014.fcs',
                     'FCFiles/sample015.fcs']

# DAPG concentration of each cell sample, in micromolar.
dapg = np.array([0, 2.33, 4.36, 8.16, 15.3, 28.6, 53.5, 100, 187, 350])

# Plots will be generated after gating and transforming cell samples. These
# will be stored in the following folder.
samples_plot_dir = 'plot_samples'

if __name__ == "__main__":

    # Check that plot directory exists, create if it does not.
    if not os.path.exists(samples_plot_dir):
        os.makedirs(samples_plot_dir)

    ###
    # Part 1: Processing cell sample data
    ###
    print("\nProcessing cell samples...")

    # We will use the list ``samples`` to store processed, transformed flow
    # cytometry data of each sample.
    samples = []

    # Iterate over cell sample filenames
    for sample_id, sample_filename in enumerate(samples_filenames):

        # Load flow cytometry data from the corresponding FCS file.
        # ``FlowCal.io.FCSData(filename)`` returns an object that represents
        # flow cytometry data loaded from file ``filename``.
        print("\nLoading file \"{}\"...".format(sample_filename))
        sample = FlowCal.io.FCSData(sample_filename)

        # Data loaded from an FCS file is in "Channel Units", the raw numbers
        # reported from the instrument's detectors. The FCS file also contains
        # information to convert these into Relative Fluorescence Intensity
        # (RFI) values, commonly referred to as arbitrary fluorescence units
        # (a.u.). The function ``FlowCal.transform.to_rfi()`` performs this
        # conversion.
        print("Performing data transformation...")
        sample = FlowCal.transform.to_rfi(sample)

        # Gating

        # Gating is the process of removing measurements of irrelevant
        # particles while retaining only the population of interest.
        print("Performing gating...")

        # ``FlowCal.gate.start_end()`` removes the first and last few events.
        # Transients in fluidics can make these events slightly different from
        # the rest. This may not be necessary in all instruments.
        sample_gated = FlowCal.gate.start_end(sample,
                                              num_start=250,
                                              num_end=100)

        # ``FlowCal.gate.high_low()`` removes events outside a range specified
        # by a ``low`` and a ``high`` value. If these are not specified (as
        # shown below), the function removes events outside the channel's range
        # of detection.
        # Detectors in a flow cytometer have a finite range of detection. If the
        # fluorescence of a particle is higher than the upper limit of this
        # range, the instrument will incorrectly record it with a value equal to
        # this limit. The same happens for fluorescence values lower than the
        # lower limit of detection. These saturated events should be removed,
        # otherwise statistics may be calculated incorrectly.
        # Note that this might not be necessary with newer instruments that
        # record data as floating-point numbers (and in fact it might eliminate
        # negative events). To see the data type stored in your FCS files, run
        # the following instruction: ``print sample_gated.data_type``.
        # We will remove saturated events in the forward/side scatter channels,
        # and in the fluorescence channel FL1.
        sample_gated = FlowCal.gate.high_low(sample_gated,
                                             channels=['FSC','SSC','FL1'])

        # ``FlowCal.gate.density2d()`` preserves only the densest population as
        # seen in a 2D density diagram of two channels. This helps remove
        # particle aggregations and other sparse populations that are not of
        # interest (i.e. debris).
        # We use the forward and side scatter channels and preserve 85% of the
        # events. Finally, setting ``full_output=True`` instructs the function
        # to return additional outputs in the form of a named tuple.
        # ``gate_contour`` is a curve surrounding the gated region, which we
        # will use for plotting later.
        density_gate_output = FlowCal.gate.density2d(
            data=sample_gated,
            channels=['FSC','SSC'],
            gate_fraction=0.85,
            full_output=True)
        sample_gated = density_gate_output.gated_data
        gate_contour = density_gate_output.contour

        # Plot forward/side scatter 2D density plot and 1D fluorescence
        # histograms
        print("Plotting density plot and histogram...")

        # Parameters for the forward/side scatter density plot
        density_params = {}
        # We use the "scatter" mode, in which individual particles will be
        # plotted individually as in a scatter plot, but with a color
        # proportional to the particle density around.
        density_params['mode'] = 'scatter'

        # Parameters for the fluorescence histograms
        hist_params = {}
        hist_params['xlabel'] = 'FL1 Fluorescence (a.u.)'

        # Plot filename
        # The figure can be saved in any format supported by matplotlib (svg,
        # jpg, etc.) by just changing the extension.
        plot_filename = '{}/density_hist_{}.png'.format(
            samples_plot_dir,
            'S{:03}'.format(sample_id + 1))

        # Plot and save
        # The function ``FlowCal.plot.density_and_hist()`` plots a combined
        # figure with a 2D density plot at the top and an arbitrary number of
        # 1D histograms below. In this case, we will plot the forward/side
        # scatter channels in the density plot and a histogram of the
        # fluorescence channel FL1 below.
        # Note that we are providing data both before (``sample``) and after
        # (``sample_gated``) gating. The 1D histogram will display the ungated
        # dataset with transparency and the gated dataset in front with a solid
        # color. In addition, we are providing ``gate_contour`` from the
        # density gating step, which will be displayed in the density diagram.
        # This will result in a convenient representation of the data both
        # before and after gating.
        FlowCal.plot.density_and_hist(
            sample,
            sample_gated,
            density_channels=['FSC','SSC'],
            hist_channels=['FL1'],
            gate_contour=gate_contour,
            density_params=density_params,
            hist_params=hist_params,
            savefig=plot_filename)

        # Save cell sample object
        samples.append(sample_gated)

    ###
    # Part 3: Examples on how to use processed cell sample data
    ###

    # Plot 1: Histogram of all samples
    #
    # Here, we plot the fluorescence histograms of all ten samples in the same
    # figure using ``FlowCal.plot.hist1d``. Note how this function can be used
    # in the context of accessory matplotlib functions to modify the axes
    # limits and labels and to add a legend, among other things.

    # Color each histogram according to its DAPG concentration. Linearize the
    # color transitions using a logarithmic normalization to match the
    # logarithmic spacing of the DAPG concentrations. (Concentrations are also
    # augmented slightly to move the 0.0 concentration into the log
    # normalization range.)
    cmap = mpl.cm.get_cmap('gray_r')
    norm = mpl.colors.LogNorm(vmin=1e0, vmax=3500.)
    colors = [cmap(norm(dapg_i+4.)) for dapg_i in dapg]

    plt.figure(figsize=(6,3.5))
    FlowCal.plot.hist1d(samples,
                        channel='FL1',
                        histtype='step',
                        bins=128,
                        edgecolor=colors)
    plt.ylim((0,2500))
    plt.xlim((0,5e3))
    plt.xlabel('FL1 Fluorescence (a.u.)')
    plt.legend(['{} $\mu M$ DAPG'.format(i) for i in dapg],
               loc='upper left',
               fontsize='small')
    plt.tight_layout()
    plt.savefig('histograms.png', dpi=200)
    plt.close()

    # Plot 2: Dose response curve
    #
    # Here, we illustrate how to obtain statistics from the fluorescence of
    # each sample and how to use them in a plot. The stats module contains
    # functions to calculate different statistics such as mean, median, and
    # standard deviation. In this example, we calculate the mean from channel
    # FL1 of each sample and plot them against the corresponding DAPG
    # concentrations.
    samples_fluorescence = [FlowCal.stats.mean(s, channels='FL1')
                            for s in samples]

    dapg_color = '#ffc400'  # common color used for DAPG-related plots

    plt.figure(figsize=(3,3))
    plt.plot(dapg,
             samples_fluorescence,
             marker='o',
             color=dapg_color)

    # Illustrate min and max bounds. Because some of our control samples were
    # measured at a different cytometer gain setting and we aren't using MEF
    # calibration here, we will use the 0uM and 350uM DAPG concentration
    # samples instead.
    plt.axhline(samples_fluorescence[0],
                color='gray',
                linestyle='--',
                zorder=-1)
    plt.text(s='Min', x=2e2, y=2.0e1, ha='left', va='bottom', color='gray')
    plt.axhline(samples_fluorescence[-1],
                color='gray',
                linestyle='--',
                zorder=-1)
    plt.text(s='Max', x=-0.7, y=2.1e2, ha='left', va='top', color='gray')

    plt.yscale('log')
    plt.ylim((5e0,5e2))
    plt.xscale('symlog')
    plt.xlim((-1e0, 1e3))
    plt.xlabel('DAPG Concentration ($\mu M$)')
    plt.ylabel('FL1 Fluorescence (a.u.)')
    plt.tight_layout()
    plt.savefig('dose_response.png', dpi=200)
    plt.close()

    # Plot 3: Dose response violin plot
    #
    # Here, we use a violin plot to show the fluorescence of (almost) all
    # cells as a function of DAPG. (The `upper_trim_fraction` and
    # `lower_trim_fraction` parameters eliminate the top and bottom 1% of
    # cells from each violin for aesthetic reasons. The summary statistic,
    # which is illustrated as a horizontal line atop each violin, is
    # calculated before cells are removed, though.) We again use the 0uM and
    # 350uM DAPG concentration samples as the min and max data in lieu of
    # controls. We also set `yscale` to 'log' because the cytometer used to
    # collect this data produces positive integer data (as opposed to
    # floating-point data, which can sometimes be negative), so the added
    # complexity of a logicle y-scale (which is the default) is not necessary.
    plt.figure(figsize=(4,3.5))
    FlowCal.plot.violin_dose_response(
        data=samples,
        channel='FL1',
        positions=dapg,
        min_data=samples[0],
        max_data=samples[-1],
        violin_kwargs={'facecolor':dapg_color,
                       'edgecolor':'black'},
        violin_width_to_span_fraction=0.075,
        xscale='log',
        yscale='log',
        ylim=(1e0,2e3))
    plt.xlabel('DAPG Concentration ($\mu M$)')
    plt.ylabel('FL1 Fluorescence (a.u.)')
    plt.tight_layout()
    plt.savefig('dose_response_violin.png', dpi=200)
    plt.close()

    print("\nDone.")

max_stars_repo_name: hema-to/FlowCal

max_issues_repo_name: hema-to/FlowCal

max_forks_repo_name: hema-to/FlowCal


text:
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test for fusion of fc and activation."""
from __future__ import print_function

import unittest
import numpy as np

import paddle.fluid as fluid
from inference_pass_test import InferencePassTest
from paddle import enable_static
from paddle.fluid.core import PassVersionChecker

enable_static()

class FCGeluTanhOneDnnFusePassTest(InferencePassTest):
    def setUp(self):
        self.set_params()
        with fluid.program_guard(self.main_program, self.startup_program):
            data = fluid.data(
                name="data", shape=[-1, 128, 768], dtype="float32")
            fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2)
            gelu_out = fluid.layers.gelu(fc_out, approximate=False)

        self.feeds = {"data": np.random.random((1, 128, 768)).astype("float32")}

        self.fetch_list = [gelu_out]
        self.enable_mkldnn = True

    def set_params(self):
        self.pass_name = "fc_act_mkldnn_fuse_pass"

    def test_check_output(self):
        self.check_output()

class FCGeluErfOneDnnFusePassTest(InferencePassTest):
    def setUp(self):
        self.set_params()
        with fluid.program_guard(self.main_program, self.startup_program):
            data = fluid.data(
                name="data", shape=[-1, 128, 768], dtype="float32")
            fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2)
            gelu_out = fluid.layers.gelu(fc_out, approximate=True)

        self.feeds = {"data": np.random.random((1, 128, 768)).astype("float32")}

        self.fetch_list = [gelu_out]
        self.enable_mkldnn = True

    def set_params(self):
        self.pass_name = "fc_act_mkldnn_fuse_pass"

    def test_check_output(self):
        self.check_output()
        self.assertTrue(PassVersionChecker.IsCompatible(self.pass_name))

class FCTanhOneDnnFusePassTest(InferencePassTest):
    def setUp(self):
        self.set_params()
        with fluid.program_guard(self.main_program, self.startup_program):
            data = fluid.data(
                name="data", shape=[-1, 128, 768], dtype="float32")
            fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2)
            tanh_out = fluid.layers.tanh(fc_out)

        self.feeds = {"data": np.random.random((1, 128, 768)).astype("float32")}

        self.fetch_list = [tanh_out]
        self.enable_mkldnn = True

    def set_params(self):
        self.pass_name = "fc_act_mkldnn_fuse_pass"

    def test_check_output(self):
        self.check_output()
        self.assertTrue(PassVersionChecker.IsCompatible(self.pass_name))

class FCSigmoidOneDnnFusePassTest(InferencePassTest):
    def setUp(self):
        self.set_params()
        with fluid.program_guard(self.main_program, self.startup_program):
            data = fluid.data(
                name="data", shape=[-1, 128, 768], dtype="float32")
            fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2)
            sigmoid_out = fluid.layers.sigmoid(fc_out)

        self.feeds = {"data": np.random.random((1, 128, 768)).astype("float32")}

        self.fetch_list = [sigmoid_out]
        self.enable_mkldnn = True

    def set_params(self):
        self.pass_name = "fc_act_mkldnn_fuse_pass"

    def test_check_output(self):
        self.check_output()
        self.assertTrue(PassVersionChecker.IsCompatible(self.pass_name))

class FCHardSwishOneDnnFusePassTest(InferencePassTest):
    def setUp(self):
        self.set_params()
        with fluid.program_guard(self.main_program, self.startup_program):
            data = fluid.data(
                name="data", shape=[-1, 128, 768], dtype="float32")
            fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2)
            hardswish_out = fluid.layers.hard_swish(fc_out)

        self.feeds = {"data": np.random.random((1, 128, 768)).astype("float32")}

        self.fetch_list = [hardswish_out]
        self.enable_mkldnn = True

    def set_params(self):
        self.pass_name = "fc_act_mkldnn_fuse_pass"

    def test_check_output(self):
        self.check_output()
        self.assertTrue(PassVersionChecker.IsCompatible(self.pass_name))

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: zmxdream/Paddle

max_issues_repo_name: zmxdream/Paddle

max_forks_repo_name: zmxdream/Paddle


text:
# -*- coding: utf-8 -*-
r"""
Helper functions.
"""

__author__ = "Stefan Ulbrich"
__copyright__ = "Copyright 2020-2021, All rights reserved."
# __credits__ = []
__license__ = "Confidential"
__version__ = "0.1"
__maintainer__ = "Stefan Ulbrich"
__email__ = "Stefan.Ulbrich@gmail.com"
__status__ = "alpha"
__date__ = "2020-01-27"

import numpy as np
import logging

logger = logging.getLogger(__name__)

def lemniscate(rotation: float = 0.0, alpha: float = 1.0, num: int = 1000) -> np.ndarray:
    """
    Sample a Bernoulli lemniscate.

    Code shamelessly copied from `here <https://stackoverflow.com/a/27803052>`_
    and `here <https://scipython.com/book/chapter-6-numpy/examples/creating-a-rotation-matrix-in-numpy/>`_

    Parameters
    ----------
    rotation : float, optional
        Radian by which the function is rotated, by default 0.0
    alpha : float, optional
        Scale factor of the lemniscate, by default 1.0
    num : int, optional
        Number of samples, by default 1000

    Returns
    -------
    np.ndarray
        2xnum numpy array
    """

    t = np.linspace(0.5 * np.pi, 2.5 * np.pi, num=num)

    c, s = np.cos(rotation), np.sin(rotation)
    R = np.array(((c, -s), (s, c)))

    values = np.array(
        (
            alpha * np.sqrt(2) * np.cos(t) / (np.sin(t) ** 2 + 1),
            alpha * np.sqrt(2) * np.cos(t) * np.sin(t) / (np.sin(t) ** 2 + 1),
        )
    )

    return R @ values

def get_dymmy_2D_data(n: int, std: float = 1.0, num: int = 1000) -> np.ndarray:
    """
    Create dummy 2-D data from overlapping lemniscates.
    """

    return np.hstack(
        [lemniscate(0, x, num) for x in np.random.randn(n) * std + 10]
        + [lemniscate(np.pi / 2, x, num) for x in np.random.randn(n) * std + 10]
    )

max_stars_repo_name: StefanUlbrich/MergeGNG

max_issues_repo_name: StefanUlbrich/MergeGNG

max_forks_repo_name: StefanUlbrich/MergeGNG


text:
"""
A bidirectional LSTM with optional CRF and character-based presentation for NLP sequence tagging used for multi-task learning.

Author: Nils Reimers
License:  Apache-2.0
"""

# CHANGES by Claudia Schulz:
# skicit learn F1 scores, detailed output path

from __future__ import print_function
from util import BIOF1Validation

from keras.optimizers import *
from keras.models import Model
from keras.layers import *
import math
import numpy as np
import sys
import gc
import time
import os
import random
import logging
import sklearn
from .keraslayers.ChainCRF import ChainCRF

class MultiTaskLSTM:
    earlyStopping = 0
    mainModelName = None

    addFeatureDimensions = 10
    miniBatchSize = 32
    learning_rate_updates = {'adam': {1:0.01, 4:0.005, 7:0.001}, 'sgd': {1: 0.1, 3:0.05, 5:0.01} }
    epoch = 0

    def __init__(self, embeddings, datasetTuples, params=None):
        """
        datasetTuples: dict {name: (dataset, labelKey)}
        """
        self.embeddings = embeddings
        self.modelNames = list(datasetTuples.keys())
        self.evaluateModelNames = []
        self.datasets = {}
        self.labelKeys = {}
        self.models = {}
        self.idx2Labels = {}
        self.idx2Word = {}

        for modelName in self.modelNames:
            dataset = datasetTuples[modelName][0]
            labelKey = datasetTuples[modelName][1]
            evaluateModel = datasetTuples[modelName][2]

            self.datasets[modelName] = dataset
            self.labelKeys[modelName] = labelKey
            self.models[modelName] = None
            self.idx2Labels[modelName] = {v: k for k, v in dataset['mappings'][labelKey].items()}
            self.idx2Word[modelName] = {v: k for k, v in dataset['mappings']['tokens'].items()}

            if evaluateModel:
                self.evaluateModelNames.append(modelName)

            logging.info("--- %s ---" % modelName)
            logging.info("%d train sentences" % len(dataset['trainMatrix']))
            logging.info("%d dev sentences" % len(dataset['devMatrix']))
            logging.info("%d test sentences" % len(dataset['testMatrix']))

        if len(self.evaluateModelNames) == 1:
            self.mainModelName = self.evaluateModelNames[0]

        self.casing2Idx = self.datasets[self.modelNames[0]]['mappings']['casing']

        self.resultsOut = None

# CHANGED
        defaultParams = {'dropout': 0, 'classifier': ['Softmax'], 'LSTM-Size': (100,), 'customClassifier': {}, 'optimizer': 'adam',
                         'charEmbeddings': None, 'charEmbeddingsSize':30, 'charFilterSize': 30, 'charFilterLength':3, 'charLSTMSize': 25,
                         'useTaskIdentifier': False, 'clipvalue': 0, 'clipnorm': 1, 'detailedOutput' : "tmp/", 'mainModelName': None}
        if params != None:
            defaultParams.update(params)
        self.params = defaultParams

        self.mainModelName = self.params['mainModelName']

        self.buildModel()

    def buildModel(self):

        tokens_input = Input(shape=(None,), dtype='int32', name='words_input')
        tokens = Embedding(input_dim=self.embeddings.shape[0], output_dim=self.embeddings.shape[1], weights=[self.embeddings], trainable=False)(tokens_input)

        casing_input = Input(shape=(None,), dtype='int32', name='casing_input')
        caseMatrix = np.identity(len(self.casing2Idx), dtype='float32')
        casing = Embedding(input_dim=caseMatrix.shape[0], output_dim=caseMatrix.shape[1], weights=[caseMatrix], trainable=False)(casing_input)

        self.featureNames = ['tokens', 'casing']
        mergeInputLayers = [tokens, casing]
        inputNodes = [tokens_input, casing_input]

        # :: Character Embeddings ::
        if self.params['charEmbeddings'] not in [None, "None", "none", False, "False", "false"]:
            logging.info("Pad characters to uniform length")
            self.padCharacters()
            logging.info("Words padded to %d charachters" % (self.maxCharLen))

            charset = None
            for dataset in self.datasets.values():
                if charset == None:
                    charset = dataset['mappings']['characters']
                else: #Ensure that the character to int mapping is equivalent
                    tmpCharset = dataset['mappings']['characters']
                    for key, value in charset.items():
                        if key not in tmpCharset or tmpCharset[key] != value:
                            logging.info("Two dataset with different characters mapping have been passed to the model")
                            assert("False")

            charEmbeddingsSize = self.params['charEmbeddingsSize']
            maxCharLen = self.maxCharLen
            charEmbeddings= []
            for _ in charset:
                limit = math.sqrt(3.0/charEmbeddingsSize)
                vector = np.random.uniform(-limit, limit, charEmbeddingsSize)
                charEmbeddings.append(vector)

            charEmbeddings[0] = np.zeros(charEmbeddingsSize) #Zero padding
            charEmbeddings = np.asarray(charEmbeddings)

            chars_input = Input(shape=(None,maxCharLen), dtype='int32', name='char_input')
            chars = TimeDistributed(Embedding(input_dim=charEmbeddings.shape[0], output_dim=charEmbeddings.shape[1],  weights=[charEmbeddings], trainable=True, mask_zero=True), name='char_emd')(chars_input)

            if self.params['charEmbeddings'].lower() == 'lstm': #Use LSTM for char embeddings from Lample et al., 2016
                charLSTMSize = self.params['charLSTMSize']
                chars = TimeDistributed(Bidirectional(LSTM(charLSTMSize, return_sequences=False)), name="char_lstm")(chars)
            else: #Use CNNs for character embeddings from Ma and Hovy, 2016
                charFilterSize = self.params['charFilterSize']
                charFilterLength = self.params['charFilterLength']
                chars = TimeDistributed(Convolution1D(charFilterSize, charFilterLength, border_mode='same'), name="char_cnn")(chars)
                chars = TimeDistributed(GlobalMaxPooling1D(), name="char_pooling")(chars)

            mergeInputLayers.append(chars)
            inputNodes.append(chars_input)
            self.featureNames.append('characters')

        # :: Task Identifier ::
        if self.params['useTaskIdentifier']:
            self.addTaskIdentifier()

            taskID_input = Input(shape=(None,), dtype='int32', name='task_id_input')
            taskIDMatrix = np.identity(len(self.modelNames), dtype='float32')
            taskID_outputlayer = Embedding(input_dim=taskIDMatrix.shape[0], output_dim=taskIDMatrix.shape[1], weights=[taskIDMatrix], trainable=False)(taskID_input)

            mergeInputLayers.append(taskID_outputlayer)
            inputNodes.append(taskID_input)
            self.featureNames.append('taskID')

        merged_input = merge(mergeInputLayers, mode='concat')

        # Add LSTMs
        shared_layer = merged_input
        logging.info("LSTM-Size: %s" % str(self.params['LSTM-Size']))
        cnt = 1
        for size in self.params['LSTM-Size']:
            # if dropout specified
            if isinstance(self.params['dropout'], (list, tuple)):
                shared_layer = Bidirectional(LSTM(size, return_sequences=True, dropout_W=self.params['dropout'][0], dropout_U=self.params['dropout'][1]), name='shared_varLSTM_'+str(cnt))(shared_layer)
            else:
                """ Naive dropout """
                shared_layer = Bidirectional(LSTM(size, return_sequences=True), name='shared_LSTM_'+str(cnt))(shared_layer)
                if self.params['dropout'] > 0.0:
                    shared_layer = TimeDistributed(Dropout(self.params['dropout']), name='shared_dropout_'+str(self.params['dropout'])+"_"+str(cnt))(shared_layer)

            cnt += 1

        for modelName in self.modelNames:
            output = shared_layer

            modelDecoder = self.params['customClassifier'][modelName] if modelName in self.params['customClassifier'] else self.params['classifier']

            cnt = 1
            for decoder in modelDecoder:
                if decoder == 'Softmax':
                    output = TimeDistributed(Dense(len(self.datasets[modelName]['mappings'][self.labelKeys[modelName]]), activation='softmax'), name=modelName+'_softmax')(output)
                    lossFct = 'sparse_categorical_crossentropy'
                elif decoder == 'CRF':
                    output = TimeDistributed(Dense(len(self.datasets[modelName]['mappings'][self.labelKeys[modelName]]), activation=None), name=modelName+'_hidden_lin_layer')(output)
                    crf = ChainCRF(name=modelName+'_CRF')
                    output = crf(output)
                    lossFct = crf.sparse_loss
                elif decoder == 'Tanh-CRF':
                    output = TimeDistributed(Dense(len(self.datasets[modelName]['mappings'][self.labelKeys[modelName]]), activation='tanh'), name=modelName+'_hidden_tanh_layer')(output)
                    crf = ChainCRF()
                    output = crf(output)
                    lossFct = crf.sparse_loss
                elif decoder[0] == 'LSTM':

                    size = decoder[1]
                    if isinstance(self.params['dropout'], (list, tuple)):
                        output = Bidirectional(LSTM(size, return_sequences=True, dropout_W=self.params['dropout'][0], dropout_U=self.params['dropout'][1]), name=modelName+'_varLSTM_'+str(cnt))(output)
                    else:
                        """ Naive dropout """
                        output = Bidirectional(LSTM(size, return_sequences=True), name=modelName+'_LSTM_'+str(cnt))(output)
                        if self.params['dropout'] > 0.0:
                            output = TimeDistributed(Dropout(self.params['dropout']), name=modelName+'_dropout_'+str(self.params['dropout'])+"_"+str(cnt))(output)
                else:
                    assert(False) #Wrong decoder

                cnt += 1

            optimizerParams = {}
            if 'clipnorm' in self.params and self.params['clipnorm'] != None and  self.params['clipnorm'] > 0:
                optimizerParams['clipnorm'] = self.params['clipnorm']

            if 'clipvalue' in self.params and self.params['clipvalue'] != None and  self.params['clipvalue'] > 0:
                optimizerParams['clipvalue'] = self.params['clipvalue']

            if self.params['optimizer'].lower() == 'adam':
                opt = Adam(**optimizerParams)
            elif self.params['optimizer'].lower() == 'nadam':
                opt = Nadam(**optimizerParams)
            elif self.params['optimizer'].lower() == 'rmsprop':
                opt = RMSprop(**optimizerParams)
            elif self.params['optimizer'].lower() == 'adadelta':
                opt = Adadelta(**optimizerParams)
            elif self.params['optimizer'].lower() == 'adagrad':
                opt = Adagrad(**optimizerParams)
            elif self.params['optimizer'].lower() == 'sgd':
                opt = SGD(lr=0.1, **optimizerParams)

            model = Model(input=inputNodes, output=[output])
            model.compile(loss=lossFct, optimizer=opt)

            model.summary(line_length=200)
            logging.info(model.get_config())
            logging.info("Optimizer: %s - %s" % (str(type(model.optimizer)), str(model.optimizer.get_config())))

            self.models[modelName] = model

    def trainModel(self):
        self.epoch += 1

        if self.params['optimizer'] in self.learning_rate_updates and self.epoch in self.learning_rate_updates[self.params['optimizer']]:
            logging.info("Update Learning Rate to %f" % (self.learning_rate_updates[self.params['optimizer']][self.epoch]))
            for modelName in self.modelNames:
                K.set_value(self.models[modelName].optimizer.lr, self.learning_rate_updates[self.params['optimizer']][self.epoch])

        for batch in self.minibatch_iterate_dataset():
            for modelName in self.modelNames:
                nnLabels = batch[modelName][0]
                nnInput = batch[modelName][1:]
                self.models[modelName].train_on_batch(nnInput, nnLabels)

    trainMiniBatchRanges = None
    trainSentenceLengthRanges = None
    def minibatch_iterate_dataset(self, modelNames = None):
        """ Create based on sentence length mini-batches with approx. the same size. Sentences and
        mini-batch chunks are shuffled and used to the train the model """

        if self.trainSentenceLengthRanges == None:
            """ Create mini batch ranges """
            self.trainSentenceLengthRanges = {}
            self.trainMiniBatchRanges = {}
            for modelName in self.modelNames:
                trainData = self.datasets[modelName]['trainMatrix']
                trainData.sort(key=lambda x:len(x['tokens'])) #Sort train matrix by sentence length
                trainRanges = []
                oldSentLength = len(trainData[0]['tokens'])
                idxStart = 0

                #Find start and end of ranges with sentences with same length
                for idx in range(len(trainData)):
                    sentLength = len(trainData[idx]['tokens'])

                    if sentLength != oldSentLength:
                        trainRanges.append((idxStart, idx))
                        idxStart = idx

                    oldSentLength = sentLength

                #Add last sentence
                trainRanges.append((idxStart, len(trainData)))

                #Break up ranges into smaller mini batch sizes
                miniBatchRanges = []
                for batchRange in trainRanges:
                    rangeLen = batchRange[1]-batchRange[0]

                    bins = int(math.ceil(rangeLen/float(self.miniBatchSize)))
                    binSize = int(math.ceil(rangeLen / float(bins)))

                    for binNr in range(bins):
                        startIdx = binNr*binSize+batchRange[0]
                        endIdx = min(batchRange[1],(binNr+1)*binSize+batchRange[0])
                        miniBatchRanges.append((startIdx, endIdx))

                self.trainSentenceLengthRanges[modelName] = trainRanges
                self.trainMiniBatchRanges[modelName] = miniBatchRanges

        if modelNames == None:
            modelNames = self.modelNames

        #Shuffle training data
        for modelName in modelNames:
            #1. Shuffle sentences that have the same length
            x = self.datasets[modelName]['trainMatrix']
            for dataRange in self.trainSentenceLengthRanges[modelName]:
                for i in reversed(range(dataRange[0]+1, dataRange[1])):
                    # pick an element in x[:i+1] with which to exchange x[i]
                    j = random.randint(dataRange[0], i)
                    x[i], x[j] = x[j], x[i]

            #2. Shuffle the order of the mini batch ranges
            random.shuffle(self.trainMiniBatchRanges[modelName])

        #Iterate over the mini batch ranges
        if self.mainModelName != None:
            rangeLength = len(self.trainMiniBatchRanges[self.mainModelName])
            logging.info("Main Model:", self.mainModelName)
            logging.info("Main Model range length", rangeLength)
        else:
            rangeLength = min([len(self.trainMiniBatchRanges[modelName]) for modelName in modelNames])

        batches = {}
        for idx in range(rangeLength):
            batches.clear()

            for modelName in modelNames:
                trainMatrix = self.datasets[modelName]['trainMatrix']
                dataRange = self.trainMiniBatchRanges[modelName][idx % len(self.trainMiniBatchRanges[modelName])]
                labels = np.asarray([trainMatrix[idx][self.labelKeys[modelName]] for idx in range(dataRange[0], dataRange[1])])
                labels = np.expand_dims(labels, -1)

                batches[modelName] = [labels]

                for featureName in self.featureNames:
                    inputData = np.asarray([trainMatrix[idx][featureName] for idx in range(dataRange[0], dataRange[1])])
                    batches[modelName].append(inputData)

            yield batches

    def online_iterate_dataset(self):
        """ Iterate dataset sentence-by-sentence (online training) """

        ranges = {}

        for modelName in self.modelNames:
            rndRange = list(range(len(self.datasets[modelName]['trainMatrix'])))
            random.shuffle(rndRange)
            ranges[modelName] = rndRange

        rangeLength = max([len(ranges[modelName]) for modelName in self.modelNames])

        for idx in range(rangeLength):
            batches = {}

            for modelName in self.modelNames:
                modelIdx = ranges[modelName][idx % len(ranges[modelName])]
                tokens = self.datasets[modelName]['trainMatrix'][modelIdx]['tokens']
                casing = self.datasets[modelName]['trainMatrix'][modelIdx]['casing']
                labelKey = self.labelKeys[modelName]
                labels = np.expand_dims([self.datasets[modelName]['trainMatrix'][modelIdx][labelKey]], -1)
                batches[modelName] = [labels, np.asarray([tokens]), np.asarray([casing])]

            yield batches

    def storeResults(self, resultsFilepath):
        if resultsFilepath != None:
            directory = os.path.dirname(resultsFilepath)
            if not os.path.exists(directory):
                os.makedirs(directory)

            self.resultsOut = open(resultsFilepath, 'w')
        else:
            self.resultsOut = None

    def evaluate(self, epochs):

        total_train_time = 0
        max_dev_score = {modelName:0 for modelName in self.models.keys()}
        max_test_score = {modelName:0 for modelName in self.models.keys()}
        no_improvement_since = 0

        for epoch in range(epochs):
            sys.stdout.flush()
            logging.info("--------- Epoch %d -----------" % (epoch+1))

            start_time = time.time()
            self.trainModel()
            time_diff = time.time() - start_time
            total_train_time += time_diff
            logging.info("%.2f sec for training (%.2f total)" % (time_diff, total_train_time))

            start_time = time.time()
            for modelName in self.evaluateModelNames:
                logging.info("-- %s --" % (modelName))
                dev_score, test_score = self.computeScore(modelName, self.datasets[modelName]['devMatrix'], self.datasets[modelName]['testMatrix'])

                if dev_score > max_dev_score[modelName]:
                    max_dev_score[modelName] = dev_score
                    max_test_score[modelName] = test_score
                    no_improvement_since = 0
                else:
                    no_improvement_since += 1

                if self.resultsOut != None:
                    self.resultsOut.write("\t".join(map(str, [epoch+1, modelName, dev_score, test_score, max_dev_score[modelName], max_test_score[modelName]])))
                    self.resultsOut.write("\n")
                    self.resultsOut.flush()

                logging.info("Max: %.4f dev; %.4f test" % (max_dev_score[modelName], max_test_score[modelName]))
                logging.info("")

            logging.info("%.2f sec for evaluation" % (time.time() - start_time))

            if self.earlyStopping > 0 and no_improvement_since >= self.earlyStopping:
                logging.info("!!! Early stopping, no improvement after ",no_improvement_since," epochs !!!")
                break

    def getSentenceLengths(self, sentences):
        sentenceLengths = {}
        for idx in range(len(sentences)):
            sentence = sentences[idx]['tokens']
            if len(sentence) not in sentenceLengths:
                sentenceLengths[len(sentence)] = []
            sentenceLengths[len(sentence)].append(idx)

        return sentenceLengths

    def predictLabels(self, model, sentences):

        predLabels = [None]*len(sentences)

        sentenceLengths = self.getSentenceLengths(sentences)

        for indices in sentenceLengths.values():
            nnInput = []
            for featureName in self.featureNames:
                inputData = np.asarray([sentences[idx][featureName] for idx in indices])
                nnInput.append(inputData)

            predictions = model.predict(nnInput, verbose=False)
            predictions = predictions.argmax(axis=-1) #Predict classes

            predIdx = 0
            for idx in indices:
                predLabels[idx] = predictions[predIdx]
                predIdx += 1

        return predLabels

    def computeScore(self, modelName, devMatrix, testMatrix):
        if self.labelKeys[modelName].endswith('_BIO') or self.labelKeys[modelName].endswith('_IOBES') or self.labelKeys[modelName].endswith('_IOB'):
            return self.computeF1Scores(modelName, devMatrix, testMatrix)
        else:
            return self.computeAccScores(modelName, devMatrix, testMatrix)

    def computeF1Scores(self, modelName, devMatrix, testMatrix):
        #train_pre, train_rec, train_f1 = self.computeF1(modelName, self.datasets[modelName]['trainMatrix'])
        #print "Train-Data: Prec: %.3f, Rec: %.3f, F1: %.4f" % (train_pre, train_rec, train_f1)

        dev_f1 = self.computeF1(modelName, devMatrix, 'dev')
        test_f1 = self.computeF1(modelName, testMatrix, 'test')
        # dev_pre, dev_rec, dev_f1 = self.computeF1(modelName, devMatrix, 'dev')
        # logging.info("Dev-Data: Prec: %.3f, Rec: %.3f, F1: %.4f" % (dev_pre, dev_rec, dev_f1))
        #
        # test_pre, test_rec, test_f1 = self.computeF1(modelName, testMatrix, 'test')
        # logging.info("Test-Data: Prec: %.3f, Rec: %.3f, F1: %.4f" % (test_pre, test_rec, test_f1))

        return dev_f1, test_f1

    def computeAccScores(self, modelName, devMatrix, testMatrix):
        dev_acc = self.computeAcc(modelName, devMatrix)
        test_acc = self.computeAcc(modelName, testMatrix)

        logging.info("Dev-Data: Accuracy: %.4f" % (dev_acc))
        logging.info("Test-Data: Accuracy: %.4f" % (test_acc))

        return dev_acc, test_acc

     # CHANGED
    def computeF1(self, modelName, sentences, name):
        labelKey = self.labelKeys[modelName]
        model = self.models[modelName]
        idx2Label = self.idx2Labels[modelName]

        correctLabels = [sentences[idx][labelKey] for idx in range(len(sentences))]
        predLabels = self.predictLabels(model, sentences)

        labelKey = self.labelKeys[modelName]
        encodingScheme = labelKey[labelKey.index('_')+1:]

        correctLabelsFlat = [item for sublist in correctLabels for item in sublist]
        predLabelsFlat = [item for sublist in predLabels for item in sublist]
        f1 = sklearn.metrics.f1_score(correctLabelsFlat, predLabelsFlat, average='macro', sample_weight=None)
        #pre, rec, f1 = BIOF1Validation.compute_f1(predLabels, correctLabels, idx2Label, 'O', encodingScheme)
        #pre_b, rec_b, f1_b = BIOF1Validation.compute_f1(predLabels, correctLabels, idx2Label, 'B', encodingScheme)

        # if f1_b > f1:
        #     logging.debug("Setting wrong tags to B- improves from %.4f to %.4f" % (f1, f1_b))
        #     pre, rec, f1 = pre_b, rec_b, f1_b
        #
        #self.writeOutputToFile(sentences, predLabels, name, modelName)

        return f1

# NEW
    def writeOutputToFile(self, sentences, predLabels, name, modelName):
        outputName = self.params['detailedOutput'] + modelName + name + str(self.epoch)
        fOut = open(outputName, 'w')

        for sentenceIdx in range(len(sentences)):
            for tokenIdx in range(len(sentences[sentenceIdx]['tokens'])):
                token = self.idx2Word[modelName][sentences[sentenceIdx]['tokens'][tokenIdx]]
                label = self.idx2Labels[modelName][sentences[sentenceIdx][self.labelKeys[modelName]][tokenIdx]]
                predLabel = self.idx2Labels[modelName][predLabels[sentenceIdx][tokenIdx]]

                fOut.write("\t".join([token, label, predLabel]))
                fOut.write("\n")

            fOut.write("\n")

        fOut.close()

    def computeAcc(self, modelName, sentences):
        correctLabels = [sentences[idx][self.labelKeys[modelName]] for idx in range(len(sentences))]
        predLabels = self.predictLabels(self.models[modelName], sentences)

        numLabels = 0
        numCorrLabels = 0
        for sentenceId in range(len(correctLabels)):
            for tokenId in range(len(correctLabels[sentenceId])):
                numLabels += 1
                if correctLabels[sentenceId][tokenId] == predLabels[sentenceId][tokenId]:
                    numCorrLabels += 1

        return numCorrLabels/float(numLabels)

    def padCharacters(self):
        """ Pads the character representations of the words to the longest word in the dataset """
        #Find the longest word in the dataset
        maxCharLen = 0
        for dataset in self.datasets.values():
            for data in [dataset['trainMatrix'], dataset['devMatrix'], dataset['testMatrix']]:
                for sentence in data:
                    for token in sentence['characters']:
                        maxCharLen = max(maxCharLen, len(token))

        for dataset in self.datasets.values():
            for data in [dataset['trainMatrix'], dataset['devMatrix'], dataset['testMatrix']]:
                #Pad each other word with zeros
                for sentenceIdx in range(len(data)):
                    for tokenIdx in range(len(data[sentenceIdx]['characters'])):
                        token = data[sentenceIdx]['characters'][tokenIdx]
                        data[sentenceIdx]['characters'][tokenIdx] = np.pad(token, (0,maxCharLen-len(token)), 'constant')

        self.maxCharLen = maxCharLen

    def addTaskIdentifier(self):
        """ Adds an identifier to every token, which identifies the task the token stems from """
        taskID = 0
        for modelName in self.modelNames:
            dataset = self.datasets[modelName]
            for dataName in ['trainMatrix', 'devMatrix', 'testMatrix']:
                for sentenceIdx in range(len(dataset[dataName])):
                    dataset[dataName][sentenceIdx]['taskID'] = [taskID] * len(dataset[dataName][sentenceIdx]['tokens'])

            taskID += 1


text:
import unittest

from chainer import backend
from chainer.backends import cuda
from chainer import initializers
from chainer import testing
from chainer.testing import attr
import numpy

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestIdentity(unittest.TestCase):

    scale = 0.1
    shape = (2, 2)

    def setUp(self):
        self.check_options = {}
        if self.dtype == numpy.float16:
            self.check_options = {'atol': 1e-4, 'rtol': 1e-3}

    def check_initializer(self, w):
        initializer = initializers.Identity(scale=self.scale)
        initializer(w)
        testing.assert_allclose(
            w, self.scale * numpy.identity(len(self.shape)),
            **self.check_options)

    def test_initializer_cpu(self):
        w = numpy.empty(self.shape, dtype=self.dtype)
        self.check_initializer(w)

    @attr.gpu
    def test_initializer_gpu(self):
        w = cuda.cupy.empty(self.shape, dtype=self.dtype)
        self.check_initializer(w)

    def check_shaped_initializer(self, xp):
        initializer = initializers.Identity(
            scale=self.scale, dtype=self.dtype)
        w = initializers.generate_array(initializer, self.shape, xp)
        self.assertIs(backend.get_array_module(w), xp)
        self.assertTupleEqual(w.shape, self.shape)
        self.assertEqual(w.dtype, self.dtype)
        testing.assert_allclose(
            w, self.scale * numpy.identity(len(self.shape)),
            **self.check_options)

    def test_shaped_initializer_cpu(self):
        self.check_shaped_initializer(numpy)

    @attr.gpu
    def test_shaped_initializer_gpu(self):
        self.check_shaped_initializer(cuda.cupy)

@testing.parameterize(
    {'shape': (2, 3)},
    {'shape': (2, 2, 4)},
    {'shape': ()},
    {'shape': 0})
class TestIdentityInvalid(unittest.TestCase):

    def setUp(self):
        self.initializer = initializers.Identity()

    def test_invalid_shape(self):
        w = numpy.empty(self.shape, dtype=numpy.float32)
        with self.assertRaises(ValueError):
            self.initializer(w)

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestConstant(unittest.TestCase):

    fill_value = 0.1
    shape = (2, 3)

    def setUp(self):
        self.check_options = {}
        if self.dtype == numpy.float16:
            self.check_options = {'atol': 1e-4, 'rtol': 1e-3}

    def check_initializer(self, w):
        initializer = initializers.Constant(fill_value=self.fill_value)
        initializer(w)
        testing.assert_allclose(
            w, numpy.full(self.shape, self.fill_value),
            **self.check_options)

    def test_initializer_cpu(self):
        w = numpy.empty(self.shape, dtype=self.dtype)
        self.check_initializer(w)

    @attr.gpu
    def test_initializer_gpu(self):
        w = cuda.cupy.empty(self.shape, dtype=self.dtype)
        self.check_initializer(w)

    def check_shaped_initializer(self, xp):
        initializer = initializers.Constant(
            fill_value=self.fill_value, dtype=self.dtype)
        w = initializers.generate_array(initializer, self.shape, xp)
        self.assertIs(backend.get_array_module(w), xp)
        self.assertTupleEqual(w.shape, self.shape)
        self.assertEqual(w.dtype, self.dtype)
        testing.assert_allclose(
            w, numpy.full(self.shape, self.fill_value),
            **self.check_options)

    def test_shaped_initializer_cpu(self):
        self.check_shaped_initializer(numpy)

    @attr.gpu
    def test_shaped_initializer_gpu(self):
        self.check_shaped_initializer(cuda.cupy)

testing.run_module(__name__, __file__)

max_stars_repo_name: tkerola/chainer

max_issues_repo_name: nolfwin/chainer

max_forks_repo_name: nolfwin/chainer