text:
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras import backend as K
from keras.utils import np_utils
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
import numpy as np
import pandas as pd
import os
import random

os.environ["CUDA_VISIBLE_DEVICES"] = "1"

# 读取一个路径下所有的.dat文件，默认路径设为..\\DATA\\xxxx
os.chdir(os.path.realpath(__file__) + "\\..")
train_path_list = os.listdir("DATA\\pre3")
os.chdir("DATA\\pre3")

data = pd.read_csv(r'20210725Apre3.csv')

print(data.shape)

list = range(1, 75)
randomlist = random.sample(list, 15)

print(randomlist)
data1 = data[data['th'].isin(randomlist)]
data2 = data[~data['th'].isin(randomlist)]

X_train = data1.iloc[:, 5:905]
Y_train = data1.iloc[:, data.columns == "bin_type"]

X_test = data2.iloc[:, 5:905]
Y_test = data2.iloc[:, data.columns == "bin_type"]

standard_scaler = MinMaxScaler()
standard_scaler.fit(X_train)

X_train = standard_scaler.transform(X_train)
X_test = standard_scaler.transform(X_test)

X_train = X_train.reshape((X_train.shape[0], 30, 30, 1)).astype('float32')
X_test = X_test.reshape((X_test.shape[0], 30, 30, 1)).astype('float32')

print(X_train.shape)

Y_train = np_utils.to_categorical(Y_train)
Y_test = np_utils.to_categorical(Y_test)

print(Y_test.shape)

num_classes = Y_test.shape[1]

model = Sequential()

# 一层卷积层 包含64个卷积核 大小5*5
model.add(Conv2D(64, (5, 5), input_shape=(30, 30, 1), activation='relu', data_format="channels_last", padding="same"))
model.add(Conv2D(64, (5, 5), input_shape=(30, 30, 1), activation='relu', data_format="channels_last", padding="same"))

# 一个最大池化层 池化大小为2*2
model.add(MaxPooling2D(pool_size=(2, 2)))

# 一个卷积层包含128个卷积核 3*3
model.add(Conv2D(128, (3, 3), activation='relu', data_format="channels_last", padding="same"))
model.add(Conv2D(128, (3, 3), activation='relu', data_format="channels_last", padding="same"))

# 一个池化层
model.add(MaxPooling2D(pool_size=(2, 2)))

# 遗忘层
model.add(Dropout(0.25))

# 压平层
model.add(Flatten())

# 全连接
model.add(Dense(128, activation='relu'))
# 分类
model.add(Dense(num_classes, activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs=20, batch_size=256, verbose=2)

# Final evaluation of the model
scores = model.evaluate(X_test,Y_test, verbose=0)
print("CNN Error: %.2f%%" % (100-scores[1]*100))

model.save('weights.model')
model.save_weights("model.h5")
model_json = model.to_json()
with open("model.json", "w") as json_file:
    json_file.write(model_json)


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 25 19:23:03 2019

@author: wmy
"""

import numpy as np
import matplotlib.pyplot as plt
import pylab

pylab.rcParams['figure.figsize'] = (10.0, 10.0)

class Conv2D(object):

    def __init__(self, name=None):
        self.name = name
        pass

    def prepare(self, X, F, b, strides=(1, 1), padding=True):
        self.X = X
        self.F = F
        self.b = b
        self.n_b, self.n_h, self.n_w, self.n_c = self.X.shape
        self.f_h, self.f_w, self.f_c_prev, self.f_c = self.F.shape
        assert(self.f_c_prev==self.n_c)
        self.s_h, self.s_w = strides
        if padding:
            self.p_h = ((self.n_h-1)*self.s_h + self.f_h - self.n_h)/2.0
            self.p_w = ((self.n_w-1)*self.s_w + self.f_w - self.n_w)/2.0
            y = int(self.p_h) + 1
            x = int(self.p_w) + 1
            h = int(self.n_h+2*self.p_h)
            w = int(self.n_w+2*self.p_w)
            self.X_padding = self.padding(self.X, y, x, h, w)
            return self.X_padding
        else:
            self.p_h = 0.0
            self.p_w = 0.0
            pass
        return self.X

    def padding(self, content, y, x, h, w):
        n_b, n_h, n_w, n_c = content.shape
        padding = np.pad(content, ((0, 0), (y-1, h-y-n_h+1), (x-1, w-x-n_w+1), (0, 0)), \
                         'constant', constant_values=0)
        return padding

    def step(self, slice_area, W, b):
        s = np.multiply(slice_area, W) + b
        Z = np.sum(s)
        return Z

    def forward(self, X, F, b, strides=(1, 1), padding=False):
        X = self.prepare(X, F, b, strides, padding)
        n_H = 1 + int((self.n_h + 2 * self.p_h - self.f_h) / self.s_h)
        n_W = 1 + int((self.n_w + 2 * self.p_w - self.f_w) / self.s_w)
        Z = np.zeros((self.n_b, n_H, n_W, self.f_c))
        for i in range(self.n_b):
            X_i = X[i]
            for h in range(n_H):
                for w in range(n_W):
                    for c in range(self.f_c):
                        vert_start = h * self.s_h
                        vert_end = vert_start + self.f_h
                        horiz_start = w * self.s_w
                        horiz_end = horiz_start + self.f_w
                        slice_area = X_i[vert_start:vert_end, horiz_start:horiz_end, :]
                        Z[i, h, w, c] = self.step(slice_area, F[:, :, :, c], b[:, :, :, c])
                        pass
                    pass
                pass
            pass
        return Z

    pass

image = plt.imread('./test.jpg')
X = np.array([image])

plt.imshow(image)
plt.show()

f1 = np.array([[-1, -1, -1], [0, 0, 0], [1, 1, 1]])
f1 = f1.reshape((3, 3, 1, 1))
f1 = f1.repeat(3, axis=2)

b1 = np.array([0])
b1 = b1.reshape((1, 1, 1, 1))

f2 = np.array([[-1, 0, 1], [-1, 0, 1], [-1, 0, 1]])
f2 = f2.reshape((3, 3, 1, 1))
f2 = f2.repeat(3, axis=2)

b2 = np.array([0])
b2 = b2.reshape((1, 1, 1, 1))

F = np.concatenate([f1, f2], axis=3)
b = np.concatenate([b1, b2], axis=3)

c = Conv2D()

images = c.forward(X, F, b, (2, 2), padding=False)
images = np.max(images, 0, keepdims=True)
images = (images - np.min(images)) / (np.max(images) - np.min(images))
images = images > 0.5

plt.imshow(images[0, :, :, 0])
plt.show()

plt.imshow(images[0, :, :, 1])
plt.show()

ext: py

max_stars_repo_name: wmylxmj/CAST-Deep-Learning

max_issues_repo_name: wmylxmj/CAST-Deep-Learning

max_forks_repo_name: wmylxmj/CAST-Deep-Learning

reason: import numpy


text:
# Random Forests

Fitting the random forest is essentially identical to fitting the bagging model with one exception: we tell the decision trees to consider only a subset of the predictors at each split. The code to do this is actually written in the {doc}`decision tree </content/c5/construction>` construction section.

As in the bagging model, let's import the regression tree construction and load the {doc}`tips </content/appendix/data>` dataset. This is done in the hidden code cell below.

## Import decision trees
import import_ipynb
import regression_tree as rt;

## Import numpy and visualization packages
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import datasets

## Load data
tips = sns.load_dataset('tips')
X = np.array(tips.drop(columns = 'tip'))
y = np.array(tips['tip'])

## Train-test split
np.random.seed(1)
test_frac = 0.25
test_size = int(len(y)*test_frac)
test_idxs = np.random.choice(np.arange(len(y)), test_size, replace = False)
X_train = np.delete(X, test_idxs, 0)
y_train = np.delete(y, test_idxs, 0)
X_test = X[test_idxs]
y_test = y[test_idxs]

We now build the `RandomForest` class in the same way we built the `Bagger` with one difference: we add the argument `C` which controls the number of predictors the decision trees consider at each split.

class RandomForest:

    def fit(self, X_train, y_train, B, C, max_depth = 100, min_size = 2, seed = None):

        self.X_train = X_train
        self.N, self.D = X_train.shape
        self.y_train = y_train
        self.B = B
        self.C = C
        self.seed = seed
        self.trees = []

        np.random.seed(seed)
        for b in range(self.B):

            sample = np.random.choice(np.arange(self.N), size = self.N, replace = True)
            X_train_b = X_train[sample]
            y_train_b = y_train[sample]

            tree = rt.DecisionTreeRegressor()
            tree.fit(X_train_b, y_train_b, max_depth = max_depth, min_size = min_size, C = C)
            self.trees.append(tree)

    def predict(self, X_test):

        y_test_hats = np.empty((len(self.trees), len(X_test)))
        for i, tree in enumerate(self.trees):
            y_test_hats[i] = tree.predict(X_test)

        return y_test_hats.mean(0)

We can now build a random forest and compare the observed with fitted values, as follows.

## Build model
rf = RandomForest()
rf.fit(X_train, y_train, B = 30, C = 4, max_depth = 20, min_size = 5, seed = 123)
y_test_hat = rf.predict(X_test)

## Plot
fig, ax = plt.subplots(figsize = (7, 5))
sns.scatterplot(y_test, y_test_hat)
ax.set(xlabel = r'$y$', ylabel = r'$\hat{y}$', title = r'Random Forest Observed vs. Fitted Values')
sns.despine()

max_stars_repo_name: curioushruti/mlbook

max_issues_repo_name: curioushruti/mlbook

max_forks_repo_name: curioushruti/mlbook


text:
import torch
import numpy as np
from .vitali import *
from .utils import *
#
# def shapley(C, control_cutoff):
#
def adjust_for_external_ownership(ol, g):
    assert torch.min(ol) >= 0 and torch.max(ol) <= 1, "ol was outside bounds"
    C = g.ownership.clone()
    non_root_nodes = g.identify_controllable() # only non-roots should be adjusted
    tot_shares = reduce_from_mask(g.total_shares_in_network, non_root_nodes)
    perc_avail_shares = torch.clamp(reduce_from_mask(ol, non_root_nodes) / tot_shares, min=0, max=1) # to be certain we clip
    C[:,non_root_nodes] = C[:,non_root_nodes] * (1 - perc_avail_shares) # remains the same if we don't own (ol=0)
    return C

def make_control_cutoff(C, control_cutoff):
    if control_cutoff is None:
        return C
    full_control = C >= control_cutoff
    C_adj = torch.zeros_like(C)
    is_full_control = full_control.sum(axis=1)
    C_adj[:, ~is_full_control] = C[:, ~is_full_control]
    C_adj[full_control] = 1.0
    # rest is zero
    return C_adj

def compute_control_with_external(ol, g, source_mask=None, target_mask=None, weight_control=False, control_cutoff=None, as_matrix=False):

    C = adjust_for_external_ownership(ol, g)

    if control_cutoff is not None:
        C = make_control_cutoff(C, control_cutoff)

    #core
    control = compute_control(ol, C, reach=torch.arange(g.number_of_nodes), as_matrix=as_matrix) # get the control of my node
    if as_matrix:
        assert not weight_control, "not implemented"
        return control

    assert control.shape[0] == g.number_of_nodes
    if control_cutoff:
        control[control >= control_cutoff] = 1.0

    if weight_control:
        value = g.value
        assert value is not None, "cannot weight if we don't get value"
        control *= value

    if target_mask is not None:
        control = reduce_from_mask(control, target_mask)

    return control

def compute_control_loss(ol, g, as_array=False, source_mask=None, target_mask=None, weight_control=False, control_cutoff=None): # as low as possible
    """
        D : mask of what has something in what
        g : network containing control matrix
        ol : list of % of stocks that I control in each company
    """
    assert ol.shape[0] == g.number_of_nodes
    if source_mask is not None:
        assert source_mask.shape[0] == g.number_of_nodes, "source mask has incorrect shape"
    if target_mask is not None:
        assert target_mask.shape[0] == g.number_of_nodes, "target mask has incorrect shape"

    tot_control = compute_control_with_external(ol, g, source_mask=source_mask, target_mask=target_mask, weight_control=weight_control, control_cutoff=control_cutoff)
    tot_control = torch.clamp(tot_control, min=0, max=1).flatten() # numerical issues otherwise, shape is target set
    if target_mask is not None:
        assert tot_control.shape[0] == sum(target_mask)
    else:
        assert tot_control.shape[0] == g.number_of_nodes

    if weight_control:
        value = g.value
        assert value is not None, "value cannot be none if we need to weight with it"
        value = reduce_from_mask(value, source_mask)
        cost = value*(1.0 - tot_control)
    else:
        cost = 1.0 - tot_control

    if as_array:
        return cost
    else:
        return torch.sum(cost) # should sum up tot total number of nodes

def compute_owned_cost(ol, g, as_array=False, source_mask=None, scale_by_total=True):
    assert ol.shape[0] == g.number_of_nodes
    if source_mask is not None:
        assert source_mask.shape[0] == g.number_of_nodes

    value = g.value
    s_cost_per_comp = reduce_from_mask(ol, source_mask)

    if g.value is not None: # then weight
        value = reduce_from_mask(value, source_mask)
        s_cost_per_comp *= value
    #print(s_cost_per_comp.flatten())
    if as_array:
        return s_cost_per_comp.flatten()
    else:
        return s_cost_per_comp.sum()

def compute_sparse_loss(
        ol, g, lambd=1,
        as_array=False, as_separate=False,
        source_mask=None, target_mask=None,
        weight_control=False, control_cutoff=None):

    c_loss = compute_control_loss(
        ol, g,
        as_array=as_array,
        source_mask=source_mask, target_mask=target_mask,
        weight_control=weight_control, control_cutoff=control_cutoff
        )
    s_loss = compute_owned_cost(ol, g, as_array=as_array, source_mask=source_mask)

    # s_loss += no_shares_cost(ol, g, cutoff=1e-8, source_mask=source_mask)
    if as_separate:
        return c_loss, s_loss
    else:
        return c_loss + lambd*s_loss

def compute_control_from_loss(cs, g, target_mask, normalize=''):
    #print(sum(g.identify_uncontrollable()[target_mask]))
    tot_target_shares = g.compute_total_target_shares(target_mask)
    n_target_nodes = sum(target_mask)
    #print(n_target_nodes)
    #print(tot_target_shares)
    tot_control = n_target_nodes-cs
    #print(tot_control)
    c = tot_control
    if normalize == 'shares':
        c /= tot_target_shares
    elif normalize == 'nodes':
        c /= sum(target_mask)
    else:
        pass
    return c

max_stars_repo_name: CSI-ADS/OptimControl

max_issues_repo_name: CSI-ADS/OptimControl

max_forks_repo_name: CSI-ADS/OptimControl


text:
# -*- coding: utf-8 -*-

import pprint

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from math import log10
import yaml

import bag
import bag.tech.mos

#parameters
pmos_type='pch'
nmos_type='nch'
#env_list = ['tt', 'ff', 'ss', 'sf', 'fs', 'ff_hot', 'ss_hot']
env_list = ['tt']
l = 16e-9
intent = 'ulvt'
pw = 4
nw = 4
csamp = 60e-15
m_list = [2,2,2,2,2,2,4,8]
num_bits = 8
m_min = 2
m_max = 8

load_from_file=True
save_to_file=True #update sizing yaml file
yamlfile_spec="adc_sar_spec.yaml"
#yamlfile_spec_output="adc_sar_spec_output.yaml"
yamlfile_size="adc_sar_size.yaml"
yamlfile_output="adc_sar_output.yaml"

vdd = 0.8
vincm = 0.15
if load_from_file==True:
    with open(yamlfile_spec, 'r') as stream:
        specdict = yaml.load(stream)
    with open(yamlfile_output, 'r') as stream:
        outdict = yaml.load(stream)
    with open(yamlfile_size, 'r') as stream:
        sizedict = yaml.load(stream)
    l=sizedict['lch']
    pw=sizedict['pw']
    nw=sizedict['nw']
    intent=sizedict['device_intent']
    cmax=sizedict['capdac']['c_m']*(2**(specdict['n_bit']-1))*specdict['c_unit']/2
    vincm=specdict['v_in_cm']
    vdd=specdict['vdd']
    tdac_target=outdict['system']['t_dac']
    num_bits=specdict['n_bit']-1

mos_config = bag.BagProject().tech_info.tech_params['mos']
root_dir = mos_config['mos_char_root']
pmos_db = bag.tech.mos.MosCharDB(root_dir, pmos_type, ['intent', 'l'],
                                 env_list, intent=intent, l=l,
                                 method='spline')
nmos_db = bag.tech.mos.MosCharDB(root_dir, nmos_type, ['intent', 'l'],
                                 env_list, intent=intent, l=l,
                                 method='spline')

#calculate
# sw transistor
vbs = vincm
vgs = vdd-vincm
vds = vincm
mdrv = nmos_db.query(w=pw, vbs=vbs, vgs=vgs, vds=vds)

# parameter calculation
m0 = 2.3*cmax/(tdac_target*np.abs(mdrv['gds'])-2.3*mdrv['cdd']) #90% settling (+cal). Need to be updated if other settling target is needed
m0 = 2**(np.ceil(np.log2(m0)))
m_opt=min(max(m0, m_min), m_max)
c_in_opt=mdrv['cgg']*m_opt
m_iter=int(m_opt)
m_list_new=[]
for i in range(num_bits):
    m_list_new = [m_iter]+m_list_new
    m_iter=min(max(int(m_iter/2), m_min), m_max)
m_list = m_list_new

if save_to_file==True:
    with open(yamlfile_output, 'r') as stream:
        outdict = yaml.load(stream)
    outdict['capdrv']['c_in']=float(c_in_opt)
    with open(yamlfile_output, 'w') as stream:
        yaml.dump(outdict, stream)
    with open(yamlfile_size, 'r') as stream:
        sizedict = yaml.load(stream)
    sizedict['capdrv']['m_list']=m_list_new
    with open(yamlfile_size, 'w') as stream:
        yaml.dump(sizedict, stream)

print('capdrv size results - m_list:'+str(m_list_new))

max_stars_repo_name: tinapiao/Software-IC-Automation

max_issues_repo_name: tinapiao/Software-IC-Automation

max_forks_repo_name: tinapiao/Software-IC-Automation


text:
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# (C) British Crown Copyright 2017-2019 Met Office.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
"""Unit tests for the cube_units utility."""

import unittest
from datetime import datetime

import iris
import numpy as np
from iris.tests import IrisTest

import improver.units
from improver.tests.set_up_test_cubes import (
    set_up_variable_cube, set_up_probability_cube, set_up_percentile_cube)
from improver.utilities import cube_units

class Test_enforce_units_and_dtypes(IrisTest):
    """Test checking with option of enforcement or failure"""

    def setUp(self):
        """Set up some conformant cubes of air_temperature to test"""
        data = 275*np.ones((3, 3), dtype=np.float32)
        self.data_cube = set_up_variable_cube(data, spatial_grid='equalarea')

        data = np.ones((3, 3, 3), dtype=np.float32)
        thresholds = np.array([272, 273, 274], dtype=np.float32)
        self.probability_cube = set_up_probability_cube(data, thresholds)

        data = np.array([274*np.ones((3, 3), dtype=np.float32),
                         275*np.ones((3, 3), dtype=np.float32),
                         276*np.ones((3, 3), dtype=np.float32)])
        percentiles = np.array([25, 50, 75], np.float32)
        self.percentile_cube = set_up_percentile_cube(data, percentiles)
        # set to real source here, to test consistency of setup functions
        # with up-to-date metadata standard
        cube_units.DEFAULT_UNITS = improver.units.DEFAULT_UNITS

    def test_basic(self):
        """Test function returns a CubeList"""
        cubelist = [self.data_cube]
        result = cube_units.enforce_units_and_dtypes(cubelist)
        self.assertIsInstance(result, iris.cube.CubeList)

    def test_cube_input(self):
        """Test function behaves sensibly with a single cube"""
        result = cube_units.enforce_units_and_dtypes(self.data_cube)
        self.assertIsInstance(result, iris.cube.CubeList)
        self.assertArrayAlmostEqual(result[0].data, self.data_cube.data)
        self.assertEqual(result[0].metadata, self.data_cube.metadata)

    def test_conformant_cubes(self):
        """Test conformant data, percentile and probability cubes are all
        passed when enforce=False (ie set to fail on non-conformance)"""
        cubelist = [
            self.data_cube, self.probability_cube, self.percentile_cube]
        result = cube_units.enforce_units_and_dtypes(cubelist, enforce=False)
        self.assertIsInstance(result, iris.cube.CubeList)
        for cube, ref in zip(result, cubelist):
            self.assertArrayAlmostEqual(cube.data, ref.data)
            self.assertEqual(cube.metadata, ref.metadata)

    def test_data_units_enforce(self):
        """Test units are changed on the returned cube and the input cube is
        unmodified"""
        self.data_cube.convert_units('Fahrenheit')
        result, = cube_units.enforce_units_and_dtypes(self.data_cube)
        self.assertEqual(result.units, 'K')
        self.assertEqual(self.data_cube.units, 'Fahrenheit')

    def test_coord_units_enforce(self):
        """Test coordinate units are enforced and the input cube is
        unmodified"""
        test_coord = 'projection_x_coordinate'
        self.data_cube.coord(test_coord).convert_units('km')
        result, = cube_units.enforce_units_and_dtypes(self.data_cube)
        self.assertEqual(self.data_cube.coord(test_coord).units, 'km')
        self.assertEqual(result.coord(test_coord).units, 'm')

    def test_data_units_fail(self):
        """Test error is raised when enforce=False"""
        self.data_cube.convert_units('Fahrenheit')
        msg = "does not conform"
        with self.assertRaisesRegex(ValueError, msg):
            cube_units.enforce_units_and_dtypes(self.data_cube, enforce=False)

    def test_coord_units_fail(self):
        """Test error is raised when enforce=False"""
        self.probability_cube.coord(
            'air_temperature').convert_units('Fahrenheit')
        msg = "does not conform"
        with self.assertRaisesRegex(ValueError, msg):
            cube_units.enforce_units_and_dtypes(
                self.probability_cube, enforce=False)

    def test_data_datatype_enforce(self):
        """Test dataset datatypes are enforced"""
        self.data_cube.data = self.data_cube.data.astype(np.float64)
        result, = cube_units.enforce_units_and_dtypes(self.data_cube)
        self.assertEqual(result.dtype, np.float32)
        # check input is unchanged
        self.assertEqual(self.data_cube.dtype, np.float64)

    def test_coord_datatype_enforce(self):
        """Test coordinate datatypes are enforced (using substring processing)
        """
        test_coord = 'forecast_reference_time'
        self.data_cube.coord(test_coord).points = (
             self.data_cube.coord(test_coord).points.astype(np.float64))
        result, = cube_units.enforce_units_and_dtypes(self.data_cube)
        self.assertEqual(result.coord(test_coord).dtype, np.int64)
        # check input is unchanged
        self.assertEqual(self.data_cube.coord(test_coord).dtype, np.float64)

    def test_data_datatype_fail(self):
        """Test error is raised when enforce=False"""
        self.percentile_cube.data = (
            self.percentile_cube.data.astype(np.float64))
        msg = "does not conform"
        with self.assertRaisesRegex(ValueError, msg):
            cube_units.enforce_units_and_dtypes(
                self.percentile_cube, enforce=False)

    def test_coord_datatype_fail(self):
        """Test error is raised when enforce=False"""
        self.percentile_cube.coord('percentile').points = (
            self.percentile_cube.coord('percentile').points.astype(np.int32))
        msg = "does not conform"
        with self.assertRaisesRegex(ValueError, msg):
            cube_units.enforce_units_and_dtypes(
                self.percentile_cube, enforce=False)

    def test_coordinates_correctly_identified(self):
        """Test all coordinates in a heterogeneous cube list are identified and
        corrected"""
        self.percentile_cube.coord('percentile').points = (
            self.percentile_cube.coord('percentile').points.astype(np.int32))
        self.probability_cube.coord(
            'air_temperature').convert_units('Fahrenheit')
        result = cube_units.enforce_units_and_dtypes(
            [self.percentile_cube, self.probability_cube])
        self.assertEqual(result[0].coord('percentile').dtype, np.float32)
        self.assertEqual(result[1].coord('air_temperature').units, 'K')

    def test_subset_of_coordinates(self):
        """Test function can enforce on a selected subset of coordinates and
        leave all others unchanged"""
        self.percentile_cube.coord('percentile').points = (
            self.percentile_cube.coord('percentile').points.astype(np.int32))
        self.percentile_cube.coord('time').convert_units(
            'hours since 1970-01-01 00:00:00')
        self.probability_cube.coord(
            'air_temperature').convert_units('Fahrenheit')
        self.probability_cube.coord('forecast_period').convert_units('h')

        result = cube_units.enforce_units_and_dtypes(
            [self.percentile_cube, self.probability_cube],
            coords=["time", "forecast_period"])
        self.assertEqual(result[0].coord('percentile').dtype, np.int32)
        self.assertEqual(
            result[0].coord('time').units, 'seconds since 1970-01-01 00:00:00')
        self.assertEqual(
            result[1].coord('air_temperature').units, 'Fahrenheit')
        self.assertEqual(result[1].coord('forecast_period').units, 's')

    def test_quantity_unavailable(self):
        """Test error raised if the named quantity is not listed in the
        dictionary standard"""
        self.data_cube.rename("number_of_fish")
        self.data_cube.units = "1"
        msg = "Name 'number_of_fish' is not uniquely defined in units.py"
        with self.assertRaisesRegex(KeyError, msg):
            cube_units.enforce_units_and_dtypes(self.data_cube)

    def test_multiple_errors(self):
        """Test a list of errors is correctly caught and re-raised"""
        self.data_cube.convert_units('Fahrenheit')
        self.probability_cube.coord(
            'air_temperature').convert_units('degC')
        msg = ("The following errors were raised during processing:\n"
               "air_temperature with units Fahrenheit and datatype float32 "
               "does not conform to expected standard \\(units K, datatype "
               "\\<class 'numpy.float32'\\>\\)\n"
               "air_temperature with units degC and datatype float32 "
               "does not conform to expected standard \\(units K, datatype "
               "\\<class 'numpy.float32'\\>\\)\n")
        with self.assertRaisesRegex(ValueError, msg):
            cube_units.enforce_units_and_dtypes(
                [self.data_cube, self.probability_cube], enforce=False)

class LimitedDictTest(IrisTest):
    """Set up limited item dictionary to point to in smaller tests"""

    def setUp(self):
        """Define dictionary"""
        self.units_dict = {
            "time": {
                "unit": "seconds since 1970-01-01 00:00:00",
                "dtype": np.int64},
            "probability": {"unit": "1"},
            "temperature": {"unit": "K"},
            "rainfall": {"unit": "m s-1"},
            "rate": {"unit": "m s-1"}
        }

class Test__find_dict_key(LimitedDictTest):
    """Test method to find suitable substring keys in dictionary"""

    def setUp(self):
        """Redirect to dummy dictionary"""
        super().setUp()
        cube_units.DEFAULT_UNITS = self.units_dict

    def test_match(self):
        """Test correct identification of single substring match"""
        result = cube_units._find_dict_key("air_temperature")
        self.assertEqual(result, "temperature")

    def test_probability_match(self):
        """Test the correct substring is returned for an IMPROVER-style
        probability cube name that matches multiple substrings"""
        result = cube_units._find_dict_key(
            "probability_of_air_temperature_above_threshold")
        self.assertEqual(result, "probability")

    def test_no_matches_error(self):
        """Test a KeyError is raised if there is no matching substring"""
        msg = "Name 'kittens' is not uniquely defined in units.py"
        with self.assertRaisesRegex(KeyError, msg):
            cube_units._find_dict_key("kittens")

    def test_multiple_matches_error(self):
        """Test a KeyError is raised if there are multiple matching substrings
        """
        msg = "Name 'rainfall_rate' is not uniquely defined in units.py"
        with self.assertRaisesRegex(KeyError, msg):
            cube_units._find_dict_key("rainfall_rate")

class Test__get_required_units_and_dtype(LimitedDictTest):
    """Test method to read requirements from dictionary"""

    def setUp(self):
        """Redirect to dummy dictionary"""
        super().setUp()
        cube_units.DEFAULT_UNITS = self.units_dict

    def test_match(self):
        """Test correct requirements identified"""
        result = cube_units._get_required_units_and_dtype("air_temperature")
        self.assertEqual(result[0], "K")
        self.assertEqual(result[1], np.float32)

    def test_probability_match(self):
        """Test correct requirements for probability (substring) diagnostic"""
        result = cube_units._get_required_units_and_dtype(
            "probability_of_air_temperature_above_threshold")
        self.assertEqual(result[0], "1")
        self.assertEqual(result[1], np.float32)

class Test__check_units_and_dtype(IrisTest):
    """Test method to check object conformance"""

    def setUp(self):
        """Set up test cube"""
        self.cube = set_up_variable_cube(
            data=275.*np.ones((3, 3), dtype=np.float32),
            spatial_grid='equalarea')
        self.coord = self.cube.coord('projection_x_coordinate')

    def test_pass_cube(self):
        """Test return value for compliant cube"""
        result = cube_units._check_units_and_dtype(self.cube, 'K', np.float32)
        self.assertTrue(result)

    def test_fail_cube(self):
        """Test return value for non-compliant cube"""
        result = cube_units._check_units_and_dtype(
            self.cube, 'degC', np.float32)
        self.assertFalse(result)

    def test_pass_coord(self):
        """Test return value for compliant coordinate"""
        result = cube_units._check_units_and_dtype(
            self.coord, 'm', np.float32)
        self.assertTrue(result)

    def test_fail_coord(self):
        """Test return value for non-compliant coordinate"""
        result = cube_units._check_units_and_dtype(self.coord, 'm', np.int32)
        self.assertFalse(result)

class Test__convert_coordinate_dtype(IrisTest):
    """Test method to convert coordinate datatypes"""

    def setUp(self):
        """Set up cubes for testing"""
        self.cube = set_up_variable_cube(np.ones((5, 5), dtype=np.float32),
                                         spatial_grid='equalarea')
        self.cube_non_integer_intervals = set_up_variable_cube(
            np.ones((5, 5), dtype=np.float32), spatial_grid='equalarea',
            time=datetime(2017, 11, 10, 4, 30))
        crd = self.cube_non_integer_intervals.coord('projection_x_coordinate')
        crd.points = crd.points * 1.0005

    def test_time_coordinate_to_hours_valid(self):
        """Test that a cube with a validity time on the hour can be converted
        to integer hours."""
        target_units = "hours since 1970-01-01 00:00:00"
        coord = self.cube.coord('time')
        expected = 419524

        coord.convert_units(target_units)
        cube_units._convert_coordinate_dtype(coord, np.int64)

        self.assertEqual(coord.points[0], expected)
        self.assertEqual(coord.units, target_units)
        self.assertIsInstance(coord.points[0], np.int64)

    def test_time_coordinate_to_hours_invalid(self):
        """Test that a cube with a validity time on the half hour cannot be
        converted to integer hours."""
        target_units = "hours since 1970-01-01 00:00:00"
        coord = self.cube_non_integer_intervals.coord('time')
        coord.convert_units(target_units)

        msg = ('Data type of coordinate "time" could not be'
               ' enforced without losing significant precision.')
        with self.assertRaisesRegex(ValueError, msg):
            cube_units._convert_coordinate_dtype(coord, np.int64)

    def test_time_coordinate_to_hours_float(self):
        """Test that a cube with a validity time on the half hour can be
        converted to float hours."""
        target_units = "hours since 1970-01-01 00:00:00"
        coord = self.cube_non_integer_intervals.coord('time')
        expected = 419524.5

        coord.convert_units(target_units)
        cube_units._convert_coordinate_dtype(coord, np.float64)

        self.assertEqual(coord.points[0], expected)
        self.assertEqual(coord.units, target_units)
        self.assertIsInstance(coord.points[0], np.float64)

class Test__convert_diagnostic_dtype(IrisTest):
    """Test method to convert diagnostic (cube.data) datatypes"""

    def setUp(self):
        """Set up cubes for testing"""
        self.cube = set_up_variable_cube(np.ones((5, 5), dtype=np.float32),
                                         spatial_grid='equalarea')
        self.cube_non_integer_intervals = set_up_variable_cube(
            np.ones((5, 5), dtype=np.float32), spatial_grid='equalarea')
        self.cube_non_integer_intervals.data *= 1.5

    def test_temperature_to_integer_kelvin_valid(self):
        """Test that a cube with temperatures at whole kelvin intervals can
        be converted to integer kelvin"""
        expected = np.ones((5, 5), dtype=np.int32)
        cube_units._convert_diagnostic_dtype(self.cube, np.int32)
        self.assertArrayEqual(self.cube.data, expected)
        self.assertEqual(self.cube.data.dtype, np.int32)

    def test_temperature_to_integer_kelvin_invalid(self):
        """Test that a cube with temperatures not at whole kelvin intervals
        cannot be converted to integer kelvin"""
        msg = ('Data type of diagnostic "air_temperature" could not be'
               ' enforced without losing significant precision.')
        with self.assertRaisesRegex(ValueError, msg):
            cube_units._convert_diagnostic_dtype(
                self.cube_non_integer_intervals, np.int32)

class Test_check_precision_loss(IrisTest):

    """Test the check_precision_loss function behaves as expected."""
    def setUp(self):
        """Make an instance of the plugin that is to be tested."""
        self.plugin = cube_units.check_precision_loss

    def test_non_lossy_float_to_integer(self):
        """Test that the function returns true when whole numbers in float type
        are checked for loss upon conversion to integers. This means that the
        conversion can go ahead without loss."""

        data = np.full((3, 3), 1, dtype=np.float64)
        dtype = np.int32

        result = self.plugin(dtype, data)

        self.assertTrue(result)

    def test_lossy_float_to_integer(self):
        """Test that the function returns false when non-whole numbers in float
        type are checked for loss upon conversion to integers. This means that
        the conversion cannot go ahead without loss."""

        data = np.full((3, 3), 1.5, dtype=np.float64)
        dtype = np.int32

        result = self.plugin(dtype, data)

        self.assertFalse(result)

    def test_lossy_float_to_integer_low_precision(self):
        """Test that the function returns True when non-whole numbers in float
        type are checked for loss upon conversion to integers but the
        fractional component is smaller than the given precision. This means
        that the conversion can go ahead without significant loss."""

        data = np.full((3, 3), 1.005, dtype=np.float64)
        dtype = np.int32

        result = self.plugin(dtype, data, precision=2)

        self.assertTrue(result)

    def test_lossy_float_to_float(self):
        """Test that the function returns true even when a float conversion
        will result in the loss of some precision. The function is not
        designed to prevent loss when converting floats to floats. In this
        test the conversion will result in the values becoming 1.0"""

        data = np.full((3, 3), 1.00000005, dtype=np.float64)
        dtype = np.float32

        result = self.plugin(dtype, data)

        self.assertTrue(result)

    def test_non_lossy_int_to_int_down(self):
        """Test that the function returns true if an integer type is changed
        to a lower precision integer type but no information is lost."""

        data = np.full((3, 3), 1234567891, dtype=np.int64)
        dtype = np.int32

        result = self.plugin(dtype, data)

        self.assertTrue(result)

    def test_non_lossy_int_to_int_up(self):
        """Test that the function returns true if an integer type is changed
        to a higher precision integer type but no information is lost."""

        data = np.full((3, 3), 1234567891, dtype=np.int32)
        dtype = np.int64

        result = self.plugin(dtype, data)

        self.assertTrue(result)

    def test_lossy_int_to_int(self):
        """Test that the function returns false if an integer type is changed
        to a lower precision integer type that results in the loss of
        information."""

        data = np.full((3, 3), 12345678910, dtype=np.int64)
        dtype = np.int32

        result = self.plugin(dtype, data)

        self.assertFalse(result)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: TomekTrzeciak/improver

max_issues_repo_name: TomekTrzeciak/improver

max_forks_repo_name: TomekTrzeciak/improver


text:
import numpy as np
import itertools

def rref(matrix):
    """Row reduced echelon form of matrix with mod 2 coefficients

    Parameters
    ==========

    matrix :
        2-d numpy array with boolean type

    Returns
    =======

    a pair of numpy arrays R, U satisfying as matrices M = UR where M is
    the original matrix R is in row echelon form and U is upper triangular
    and invertible.

    Examples
    ========

    >>> M = np.array([[0, 0, 0], \
                      [0, 1, 0], \
                      [1, 1, 0]], dtype=np.bool)

    >>> reduced, triangular, pivots = rref(M)

    >>> reduced.astype(np.int)
    array([[1, 0, 0],
           [0, 1, 0],
           [0, 0, 0]])

    >>> triangular.astype(np.int)
    array([[0, 1, 1],
           [0, 1, 0],
           [1, 0, 0]])

    >>> pivots
    [[0, 1], [0, 1]]
    """

    pivots = [[], []]  # col-row
    lead = 0
    rowCount, columnCount = matrix.shape
    triangular = np.eye(rowCount, dtype=np.bool)
    for r in range(rowCount):
        if lead >= columnCount:
            break
        if np.any(matrix[r:, lead]):
            i = min(filter(lambda j: matrix[j, lead], range(r, rowCount)))
            matrix[[i, r], :] = matrix[[r, i], :]
            triangular[[i, r], :] = triangular[[r, i], :]
            indicesExcept = itertools.chain(range(0, r), range(r + 1, rowCount))
            for j in filter(lambda j: matrix[j, lead], indicesExcept):
                matrix[j, :] = np.logical_xor(matrix[r, :],
                                              matrix[j, :])
                triangular[j, :] = np.logical_xor(triangular[r, :],
                                                  triangular[j, :])
            pivots[0].append(r)
            pivots[1].append(lead)

        lead += 1

    return matrix, triangular, pivots

def solve(vector, matrix, reduced=None, triangular=None, pivots=None):
    '''Solves the linear system Mx=v, returning one solution and a basis
    for the kernel of M.

    >>> M = np.array([[1, 1, 1], \
                      [0, 1, 0], \
                      [0, 0, 0]])
    >>> v = np.array([0, 0, 1])
    >>> solve(v, M)
    Traceback (most recent call last):
    ValueError: equation has no solutions

    '''
    m, n = matrix.shape
    if reduced is None:
        reduced, triangular, pivots = rref(matrix)
    # Apply row operations to input vector
    w = np.matmul(triangular.astype(np.int8), vector.astype(np.int8))
    w = (w % 2).astype(np.bool)
    # Solvable?
    nonZeroRows = [i for i, b in enumerate(w) if b]
    if not all(i in pivots[0] for i in nonZeroRows):
        raise ValueError('equation has no solutions')
    # Find one solution
    nonZeroColumns = [col for row, col in zip(*pivots) if w[row]]
    solution = np.zeros(n, dtype=np.bool)
    for col in nonZeroColumns:
        solution[col] = True

    # Solutions to homogeneous system WRONG
    # freeColumns = [i for i in range(n) if i not in pivots[1]]
    # kernel_basis = np.zeros((n, len(freeColumns)), dtype=np.bool)
    # for idx, i in enumerate(freeColumns):
    #     freeColumn = reduced[:, i]
    #     for row in filter(lambda row: freeColumn[row], range(m)):
    #         col = pivots[1][pivots[0].index(row)]
    #         kernel_basis[col, idx] = True
    #     kernel_basis[i, idx] = True

    return solution  # , kernel_basis

if __name__ == "__main__":
    import doctest
    doctest.testmod()


text:
import math
import numpy as np
import python_speech_features
from .. import features

class Spectrogram(features.FeaturesExtractor):

    def __init__(self,
                 features_num: int,
                 samplerate: int,
                 winlen: float,
                 winstep: float,
                 winfunc=None,
                 is_standardization=True,
                 pad_audio_to: int = 0):
        self.features_num = features_num
        self.winfunc = winfunc
        self.frame_len = int(winlen * samplerate)
        self.frame_step = int(winstep * samplerate)
        self.is_standardization = is_standardization
        self.pad_to = pad_audio_to

    def make_features(self, audio: np.ndarray) -> np.ndarray:
        """ Use `python_speech_features` lib to extract log-spectrogram's. """
        audio = self.normalize(audio.astype(np.float32))
        audio = (audio * np.iinfo(np.int16).max).astype(np.int16)
        audio = self.pad(audio) if self.pad_to else audio
        frames = python_speech_features.sigproc.framesig(
            audio, self.frame_len, self.frame_step, self.winfunc
        )
        features = python_speech_features.sigproc.logpowspec(
            frames, self.frame_len, norm=1
        )
        features = features[:, :self.features_num]  # Cut high frequency part
        return self.standardize(features) if self.is_standardization else features

    def pad(self, audio: np.ndarray) -> np.ndarray:
        """ Padding signal is required if you play with mixed precession. """
        length = 1 + int((len(audio) - self.frame_len) // self.frame_step + 1)
        pad_size = (self.pad_to - length % self.pad_to) * self.frame_step
        return np.pad(audio, (0, pad_size), mode='constant')


text:
# Generative Adversarial Networks using 1-D input signal

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from matplotlib import style
from scipy.stats import norm

style.use('ggplot')

# Input Data Sample (P_data)
mu, sigma = -1,1
xs=np.linspace(-5,5,1000)
# plt.plot(xs, norm.pdf(xs,loc=mu,scale=sigma))

numTrainIters = 10000
M = 200

# ------------------------------ Multi Layer Perceptron -------------------------
# MultiLayer Perceptron Layers Weights and Bias Value Initialization
# 4-layer ANN with 1 Hidden Layer (6 neurons) and 1 Deeply Connected Layer (5 neurons)
# Total Number of Neurons = 11
# Input => Hidden Layer => Deeply Connected Layer => Output
# Weights are between the layers.
# Biases are on the neurons of the Hidden Layers.
def MultiLayerPerceptron(input, output_dim):
    # Initialize the weights with "empty"
    # Initialize the biases with "0.0"
    init_const = tf.constant_initializer(0.0)
    init_normal = tf.random_normal_initializer()
    # Format: tf.get_variable(name, shape, datatype, initializer, regularizer...)
    # Weights from Input layer to Hidden Layer => W1
    # W1 matrix is of the form: [Input data samples, number of neurons in Hidden Layer]
    w1 = tf.get_variable('w1', [input.get_shape()[1], 6], initializer=init_normal)

    # Bias is added on the Hidden Layer Neuron.
    # Dimension of Bias Matrix: [number of neurons in hidden layer]
    b1 = tf.get_variable('b1', [6], initializer=init_const)

    # Deeply Connected Layer / Second Hidden Layer
    # Weight Dimension: [num neurons in 1st Hidden Layer, num neurons in 2nd Hidden Layer]
    w2 = tf.get_variable('w2', [6, 5], initializer=init_normal)

    # Bias is added on the Hidden Layer Neuron.
    # Dimension of Bias Matrix: [number of neurons in 2nd Hidden Layer]
    b2 = tf.get_variable('b2', [5], initializer=init_const)

    # Weights from Deeply Connected Layer to Output Neuron / Layer
    # W3 matrix has dimensions: [num neurons in hiddenlayer - 1, num neurons in output layer]
    w3 = tf.get_variable('w3', [5, output_dim], initializer=init_normal)

    # Bias is added on the Output Layer.
    # Dimension of Bias Matrix: [number of neurons in Output Layer]
    b3 = tf.get_variable('b3', [output_dim], initializer=init_const)

    # Activation Function (tanh()) for 1st Hidden Layer
    a1 = tf.nn.tanh(tf.matmul(input, w1) + b1)

    # Activation Function (tanh()) for 2nd Hidden Layer
    a2 = tf.nn.tanh(tf.matmul(a1, w2) + b2)

    # Activation Function (tanh()) for Output Layer
    y_Hat = tf.nn.tanh(tf.matmul(a2, w3) + b3)
    return y_Hat, [w1, b1, w2, b2, w3, b3]

# --------------------------- Train the Neural Network ---------------------------
# Using Momentum Optimizer to decrease the Learning Rate as
# we progress so as to reach the minima and not miss it.
# Useful in case when the input is not a Convex function.
def momentumOptimizer(loss, var_list):
    # Initialize the learning rate to a reasonable value
    baseLearningRate = 0.001
    # Set the decay Rate
    decayRate = 0.95
    # Set the step size for decay
    decaySteps = numTrainIters // 4
    batch = tf.Variable(0)
    learning_rate = tf.train.exponential_decay(
        baseLearningRate,
        batch,
        decaySteps,
        decayRate,
        staircase=True
    )
    optimizer = tf.train.MomentumOptimizer(learning_rate, 0.6).minimize(
        loss,
        global_step=batch,
        var_list=var_list
    )
    return optimizer

# ------------------------ Generative Adversarial Network --------------------------
# Pre-Train the Discriminator.
# Doing this saves time at a later stage and only tuning of Discriminator is required.
with tf.variable_scope("D_pre"):
    # Input Data to the Discriminator
    # N sample values in matrix form
    input_node = tf.placeholder(tf.float32, shape=(M, 1))

    # Training Labels = Number of input samples
    train_labels = tf.placeholder(tf.float32, shape=(M, 1))

    # Input the data to Neural Network and get the Output
    # Discriminator sample (D) = Output (y_Hat)
    # Neural Network[input_node, hidden_dim = 6, output_dim = 1]
    D, theta = MultiLayerPerceptron(input_node, 1)

    # Calculate the loss using Mean Squared Error (MSE)
    loss = tf.reduce_mean(tf.square(D - train_labels))

# Optimize the Discriminator for Minimizing Loss / Cost Function.
optimizer = momentumOptimizer(loss, None)

sess = tf.InteractiveSession()
tf.global_variables_initializer().run()

# ------------------------ Plot Initial Decision Surface -----------------------------
def plot_d0(D, input_node):
    f, ax = plt.subplots(1)
    # p_data
    xs = np.linspace(-5, 5, 1000)
    ax.plot(xs, norm.pdf(xs, loc=mu, scale=sigma), label='p_data')
    # decision boundary
    r = 1000  # resolution (number of points)
    xs = np.linspace(-5, 5, r)
    ds = np.zeros((r, 1))  # decision surface
    # process multiple points in parallel in a minibatch
    for i in range(int(r/M)):
        x = np.reshape(xs[M*i:M*(i + 1)],(M,1))
        ds[M*i:M*(i + 1)] = sess.run(D, {input_node: x})

    ax.plot(xs, ds, label='decision boundary')
    ax.set_ylim(0, 1.1)
    plt.legend()

# plot_d0(D, input_node)
# plt.title('Initial Decision Boundary')
# plt.show()
# sess.close()

# ------------------------------------ Plot Training Loss -----------------------------
lh=np.zeros(1000)
for i in range(1000):
    #d=np.random.normal(mu,sigma,M)
    d=(np.random.random(M)-0.5) * 10.0 # instead of sampling only from gaussian, want the domain to be covered as uniformly as possible
    labels=norm.pdf(d,loc=mu,scale=sigma)
    lh[i],_=sess.run([loss,optimizer], {input_node: np.reshape(d,(M,1)), train_labels: np.reshape(labels,(M,1))})

# plt.plot(lh)
# plt.title('Training Loss')
# plt.show()

# plot_d0(D,input_node)

# Theta gets the weights and biases with lowest cost function for Pre Trained Network D.
# Save it.
learnedWeights = sess.run(theta)
sess.close()

# Generator Network
with tf.variable_scope("G"):
    # M: Input Samples generated (Noise Signal)
    z_node=tf.placeholder(tf.float32, shape=(M,1))
    # Get the output and weights of the Generator using Neural Network
    G,theta_g = MultiLayerPerceptron(z_node,1)
    # Scale by 5 to match with range
    G=tf.multiply(5.0,G)

# Discriminator Network:
# D1 => actual Data input Discriminator
# D2 => Output of Generator is Input to Discriminator
with tf.variable_scope("D") as scope:
    # D1 => Trained on Input Data
    x_node = tf.placeholder(tf.float32, shape=(M,1))
    fc,theta_d = MultiLayerPerceptron(x_node,1)
    D1=tf.maximum(tf.minimum(fc,.99), 0.01)
    # make a copy of D that uses the same variables, but takes in G as input
    scope.reuse_variables()

    # D2 => Takes output of Generator as Input
    # Output of Generator (G) input to Discriminator (D2)
    fc,theta_d = MultiLayerPerceptron(G,1)
    D2 = tf.maximum(tf.minimum(fc,.99), 0.01)

# Calculating the Value of Discriminator "D"
# log(D1(x)) + log(1-D2(x))
# We need to Maximize D1 and Minimize value of D2.
# obj_d: Value receieved after processing Input Data
obj_d = tf.reduce_mean(tf.log(D1)+tf.log(1-D2))

# Calculating value of Generator Function "G"
# log(D2(x))
# We need to maximize log(D2(x)) to successfully fool Discriminator
# obj_g: Value received after processing Generator Data
obj_g = tf.reduce_mean(tf.log(D2))

# set up optimizer for G and D
opt_d = momentumOptimizer(1-obj_d, theta_d)
opt_g = momentumOptimizer(1-obj_g, theta_g)

sess=tf.InteractiveSession()
tf.global_variables_initializer().run()

# Copy the Weights saved in Pre-Training over to New D Network
for i,v in enumerate(theta_d):
    sess.run(v.assign(learnedWeights[i]))

def plot_fig():
    # plots pg, pdata, decision boundary
    f,ax=plt.subplots(1)
    # p_data
    xs=np.linspace(-5,5,1000)
    ax.plot(xs, norm.pdf(xs,loc=mu,scale=sigma), label='p_data')

    # decision boundary
    r=5000 # resolution (number of points)
    xs=np.linspace(-5,5,r)
    ds=np.zeros((r,1)) # decision surface
    # process multiple points in parallel in same minibatch
    for i in range(int(r/M)):
        x=np.reshape(xs[M*i:M*(i+1)],(M,1))
        ds[M*i:M*(i+1)]=sess.run(D1,{x_node: x})

    ax.plot(xs, ds, label='decision boundary')

    # distribution of inverse-mapped points
    zs=np.linspace(-5,5,r)
    gs=np.zeros((r,1)) # generator function
    for i in range(int(r/M)):
        z=np.reshape(zs[M*i:M*(i+1)],(M,1))
        gs[M*i:M*(i+1)]=sess.run(G,{z_node: z})
    histc, edges = np.histogram(gs, bins = 10)
    ax.plot(np.linspace(-5,5,10), histc/float(r), label='p_g')

    ax.set_ylim(0,1.1)
    plt.legend()

plot_fig()
plt.title('Before Training')
plt.show()

# Algorithm 1 of Goodfellow et al 2014
# Instead of optimizing with one pair (x,z) at a time, we update the gradient
# based on the average of M loss gradients computed for M different (x,z) pairs.
# The stochastic gradient estimated from a minibatch is closer to the true gradient
# across the training data.
k=1
histd, histg= np.zeros(numTrainIters), np.zeros(numTrainIters)
for i in range(numTrainIters):
    for j in range(k):
        x= np.random.normal(mu,sigma,M) # sampled m-batch from p_data
        x.sort()
        # Sample Batch Noise Input of Dimension "M"
        # Input Noise signal "Z" streached from "-1 to 1" to "-5 to 5"
        z= np.linspace(-5.0,5.0,M)+np.random.random(M)*0.01
        histd[i],_ = sess.run([obj_d,opt_d], {x_node: np.reshape(x,(M,1)), z_node: np.reshape(z,(M,1))})
    # Sample Noise Prior Signal Input
    z= np.linspace(-5.0,5.0,M)+np.random.random(M)*0.01
    # Update Generator
    histg[i],_ = sess.run([obj_g,opt_g], {z_node: np.reshape(z,(M,1))})
    if i % (numTrainIters//10) == 0:
        print((float(i)/float(numTrainIters))*100)

plt.plot(range(numTrainIters),histd, label='obj_d')
plt.plot(range(numTrainIters), 1-histg, label='obj_g')
plt.legend()

plot_fig()
plt.show()

sess.close()

# ----------------------------------- EOC ----------------------------------------------

max_stars_repo_path: Week 7 code/GAN.py

max_issues_repo_path: Week 7 code/GAN.py

max_forks_repo_path: Week 7 code/GAN.py

reason: import numpy,from scipy


text:
import os, glob, time
import logging
import numpy as np
from collections import Counter
from sklearn.utils import class_weight

class DataLoader(object):

    def __init__(self, split_criterion = 0.3, is_pretrain=False):
        self.logger = logging.getLogger(__name__)
        self.ROOT = os.getcwd()
        self.DATASET_DIR = os.path.join(self.ROOT, "dataset")
        self.PRETRAIN_DIR = os.path.join(self.ROOT, "pretrain_dataset")
        self.split_criterion = split_criterion
        self.is_pretrain = is_pretrain

    def __get_dataset_paths(self, randomize_files=False):
        file_names = []

        for paths in glob.glob(self.DATASET_DIR + "/*.npz"):
            file_names.append(paths.split('/')[-1])

        if randomize_files:
            np.random.shuffle(file_names)
        return file_names

    def split_dataset(self, val_count = 0, randomize_files=False):

        dataset_names = self.__get_dataset_paths(randomize_files=randomize_files)
        no_of_files = len(dataset_names)

        if val_count > 0:
            training_files = dataset_names[val_count:]
            validation_files = dataset_names[0:val_count]

        else:
            validation_file_count = int(no_of_files * self.split_criterion)
            training_file_count = no_of_files - validation_file_count

            training_files = dataset_names[:training_file_count]
            validation_files = dataset_names[training_file_count: no_of_files]

        self.logger.info("Number of Training files = {}, Number of Validation files = {}".format(len(training_files), len(validation_files)))
        return training_files, validation_files

    def get_training_report(self, training_data):

        count = Counter()
        self.logger.info("Training Data Report")
        for npz_files in training_data:
            if self.is_pretrain:
                npz_files = os.path.join(self.PRETRAIN_DIR, npz_files)
            else:
                npz_files = os.path.join(self.DATASET_DIR, npz_files)

            datafile = np.load(npz_files)
            labels = datafile['Labels']
            count.update(labels)
            self.logger.info("Class distribution = {} for file = {}".format(Counter(labels), npz_files))

        total_labels = np.sum(count.values())
        self.logger.info("Total Class distribution = {}".format(count))
        self.logger.info("Total labels = {}".format(total_labels))
        self.logger.info("Light = {}, Deep = {}, REM = {}, Wake = {}".format(count[2], count[1], count[3], count[4]))
        self.logger.info(
            "%Light = {}, %Deep = {}, %REM = {}, %Wake = {}".format((count[2]/float(total_labels))*100,
                                                                          (count[1]/float(total_labels))*100,
                                                                          (count[3]/float(total_labels))*100,
                                                                          (count[4]/float(total_labels))*100
                                                                          ))

    def get_validation_report(self, validation_X, validation_y):
        count_val = Counter(validation_y)
        self.logger.info("Validation Data Report")

        # labels = validation_y
        # count_val.update(labels)
        total_labels = np.sum(count_val.values())
        self.logger.info("Total Class distribution = {}".format(count_val))
        self.logger.info("Total labels = {}".format(total_labels))
        self.logger.info(
            "%Light = {}, %Deep = {}, %REM = {}, %Wake = {}".format((count_val[2] / float(total_labels)) * 100,
                                                                    (count_val[1] / float(total_labels)) * 100,
                                                                    (count_val[3] / float(total_labels)) * 100,
                                                                    (count_val[4] / float(total_labels)) * 100
                                                                    ))

        # self.logger.info("Baseline accuracy = {}".format(np.max((count_val[2] / float(total_labels)) * 100,
        #                                                             (count_val[1] / float(total_labels)) * 100,
        #                                                             (count_val[3] / float(total_labels)) * 100,
        #                                                             (count_val[4] / float(total_labels)) * 100
        #                                                             )))

    def get_dataset_report(self, dataset):

        count = Counter()
        self.logger.info("Dataset Report")
        for npz_files in dataset:
            if self.is_pretrain:
                npz_files = os.path.join(self.PRETRAIN_DIR, npz_files)
            else:
                npz_files = os.path.join(self.DATASET_DIR, npz_files)

            datafile = np.load(npz_files)
            labels = datafile['Labels']
            count.update(labels)
            self.logger.info("Class distribution = {} for file = {}".format(Counter(labels), npz_files))

        total_labels = np.sum(count.values())
        self.logger.info("Total Class distribution = {}".format(count))
        self.logger.info("Total labels = {}".format(total_labels))
        self.logger.info("Light = {}, Deep = {}, REM = {}, Wake = {}".format(count[2], count[1], count[3], count[4]))
        self.logger.info(
            "%Light = {}, %Deep = {}, %REM = {}, %Wake = {}".format((count[2]/float(total_labels))*100,
                                                                          (count[1]/float(total_labels))*100,
                                                                          (count[3]/float(total_labels))*100,
                                                                          (count[4]/float(total_labels))*100
                                                                          ))

    def get_class_weight(self, training_data):

        for npz_files in training_data[0:1]:
            datafile = np.load(npz_files)
            labels = datafile['Labels']

            weights = class_weight.compute_class_weight('balanced', np.unique(labels), labels)
            return weights

    # def get_data(self):
    #     """
    #     Function to view and debug the generators
    #     Returns: Output generated from the generators
    #
    #     """
    #     training_files, validation_files = self.split_dataset()
    #     # training_samples_count = self.get_total_sample_count(training_files)
    #
    #     # self.logger.debug("Total training samples = {}".format(training_samples_count))
    #     return list(SequenceGenerator(training_files))

max_stars_repo_name: shashankpr/DeepSleep

max_issues_repo_name: shashankpr/DeepSleep

max_forks_repo_name: shashankpr/DeepSleep


text:
import warnings

import matplotlib.pyplot as plt
import mmcv
import numpy as np
import torch
from mmcv.ops import RoIPool
from mmcv.parallel import collate, scatter
from mmcv.runner import load_checkpoint

from mmcap.datasets.pipelines import Compose
from mmcap.models import build_caption
from . import init_caption

def extract_encoder_feat(model, tokenizer, img):
    """Extract encoder features of caption model.

    Args:
        model (nn.Module): Image Captioning Model
        tokenizer: For preprocess pipeline
        img (str): img file path

    Returns:
        Extracted feature result.
    """
    cfg = model.cfg
    device = next(model.parameters()).device  # model device
    # prepare data
    # Add dummy caption
    cap_info = dict(caption='',
                    tokenizer=tokenizer)
    data = dict(img_info=dict(filename=img),
                img_prefix=None,
                cap_info=cap_info)
    # build the data pipeline
    test_pipeline = Compose(cfg.data.test.pipeline)

    data = test_pipeline(data)
    data = collate([data], samples_per_gpu=1)
    if next(model.parameters()).is_cuda:
        # scatter to specified GPU
        data = scatter(data, [device])[0]

    # forward the model
    with torch.no_grad():
        # img, img_mask, pos
        result = model.extract_feat(data['img'][0], data['img_mask'][0])

        #result = model(return_loss=False, rescale=True, **data)[0]
    return result


text:
"""
File: safe_pendulum.py
Author: Nathaniel Hamilton modified code from
        https://github.com/openai/gym/blob/master/gym/envs/classic_control/pendulum.py

Description: An update to the inverted pendulum environment that adds a safety constraint and a runtime assurance (RTA)
             strategy to ensure it. The initial conditions and system constraints are the same used in
             https://rcheng805.github.io/files/aaai2019.pdf. The RTA in this environment uses a simplex architecture
             (explained well in this paper http://www.taylortjohnson.com/research/johnson2016tecs.pdf).
             TODO: explain how the RTA works
"""
import gym
from gym import spaces
from gym.utils import seeding
import numpy as np
from os import path

class SafePendulumEnv(gym.Env):
    metadata = {
        'render.modes': ['human', 'rgb_array'],
        'video.frames_per_second': 30
    }

    def __init__(self, g=10.0, unsafe_angle=1.0, reward_style=1):
        """
        TODO: define input variables
        :reward_style: (int) value determining the reward style for training in the
                             environment; 0 is minimizing cost, 1 is maximizing reward,
                             2 is maximize reward and punish SC usage
        """
        self.max_speed = 60.0
        self.max_torque = 15.0
        self.dt = .05
        self.g = g
        self.m = 1.
        self.l = 1.
        self.viewer = None
        self.initial_angle_bound = 0.8  # Pendulum starts +-46 degrees and +-1.0 rad/s
        self.initial_vel_bound = 1.0
        self.unsafe_angle = unsafe_angle
        self.reward_style = reward_style
        self.look_ahead = 100

        high = np.array([1., 1., self.max_speed], dtype=np.float32)
        self.action_space = spaces.Box(
            low=-self.max_torque,
            high=self.max_torque, shape=(1,),
            dtype=np.float32
        )
        self.observation_space = spaces.Box(
            low=-high,
            high=high,
            dtype=np.float32
        )

        self.seed()

    def calculate_reward(self, theta, thetadot, action, intervening=False):
        """
        TODO define inputs
        """
        costs = angle_normalize(theta) ** 2 + .1 * thetadot ** 2 + .001 * (action ** 2)

        # adjust the reward value depending on the reward style
        if self.reward_style == 1:
            reward = 5 - costs
        elif self.reward_style == 2:
            if intervening:
                costs += 1  # an additional cost is applied if the safety controller is used
            reward = 5 - costs
        else:
            reward = -costs

        return reward

    def get_safe_action(self, theta, thetadot):
        """
        TODO: define inputs
        """
        # TODO: Maybe make this better because it is very basic
        # if theta < 0.0:
        #     safe_action = np.array([2])
        # else:
        #     safe_action = np.array([-2])

        u = np.array([((-32.0 / np.pi) * theta)])  # + ((-1.0 / np.pi) * thetadot)])
        safe_action = np.clip(u, -self.max_torque, self.max_torque)[0]
        # print(safe_action)

        return safe_action

    def runtime_assurance(self, action):
        """
        TODO: define variables
        """
        intervening = False
        safe_action = action
        simulated_done = False

        # Simulate one time-step forward with the desired action
        th, thdot = self.state
        next_theta, next_thetadot = self.simulate_one_step(th, thdot, action)
        simulated_next_observation = np.array([np.cos(next_theta), np.sin(next_theta), next_thetadot])
        simulated_reward = self.calculate_reward(next_theta, next_thetadot, action)
        if abs(next_theta) > self.unsafe_angle:
            intervening = True
            safe_action = self.get_safe_action(th, thdot)
            simulated_done = True
        else:
            # Simulate a few time-steps forward with the safety controller to ensure safety property won't be violated
            # in the future
            for i in range(self.look_ahead - 1):
                next_theta, next_thetadot = self.simulate_one_step(next_theta, next_thetadot,
                                                                   self.get_safe_action(next_theta, next_thetadot))
                if abs(next_theta) > self.unsafe_angle:
                    intervening = True
                    safe_action = self.get_safe_action(th, thdot)
                    break

                # The look-ahead can also terminate as safe if the pendulum re-enters the initial conditions
                if abs(next_theta) <= self.initial_angle_bound and abs(next_thetadot) <= self.initial_vel_bound:
                    break

        return intervening, safe_action, simulated_next_observation, simulated_reward, simulated_done

    def seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

    def simulate_one_step(self, th, thdot, u):
        """
        TODO: define inputs
        """
        g = self.g
        m = self.m
        l = self.l
        dt = self.dt

        newthdot = thdot + (-3 * g / (2 * l) * np.sin(th + np.pi) + 3. / (m * l ** 2) * u) * dt
        newth = th + newthdot * dt
        newthdot = np.clip(newthdot, -self.max_speed, self.max_speed)

        return newth, newthdot

    def step(self, u):
        """
        TODO: define variables
        """
        th, thdot = self.state  # th := theta

        g = self.g
        m = self.m
        l = self.l
        dt = self.dt
        done = False

        u = np.clip(u, -self.max_torque, self.max_torque)[0]

        # determine the safety of the desired action
        intervening_bool, safe_u, sim_o2, sim_r, sim_d = self.runtime_assurance(u)

        self.last_u = safe_u  # for rendering

        newthdot = thdot + (-3 * g / (2 * l) * np.sin(th + np.pi) + 3. / (m * l ** 2) * safe_u) * dt
        newth = th + newthdot * dt
        newthdot = np.clip(newthdot, -self.max_speed, self.max_speed)

        self.state = np.array([newth, newthdot])

        # terminate the episode if the safety condition is violated
        if abs(newth) > self.unsafe_angle:
            # print('Safety violated')
            done = True

        reward = self.calculate_reward(newth, newthdot, safe_u, intervening_bool)

        info = {'intervening': intervening_bool, 'action taken': safe_u,
                'sim r': sim_r, 'sim o2': sim_o2, 'sim d': sim_d, 'theta': newth}

        return self._get_obs(), reward, done, info

    def reset(self):
        high = np.array([self.initial_angle_bound, self.initial_vel_bound])
        self.state = self.np_random.uniform(low=-high, high=high)
        self.last_u = None
        return self._get_obs()

    def _get_obs(self):
        theta, thetadot = self.state
        return np.array([np.cos(theta), np.sin(theta), thetadot])

    def render(self, mode='human'):
        if self.viewer is None:
            from gym.envs.classic_control import rendering
            self.viewer = rendering.Viewer(500, 500)
            self.viewer.set_bounds(-2.2, 2.2, -2.2, 2.2)
            rod = rendering.make_capsule(1, .2)
            rod.set_color(.8, .3, .3)
            self.pole_transform = rendering.Transform()
            rod.add_attr(self.pole_transform)
            self.viewer.add_geom(rod)
            axle = rendering.make_circle(.05)
            axle.set_color(0, 0, 0)
            self.viewer.add_geom(axle)
            fname = path.join(path.dirname(__file__), "assets/clockwise.png")
            self.img = rendering.Image(fname, 1., 1.)
            self.imgtrans = rendering.Transform()
            self.img.add_attr(self.imgtrans)

        self.viewer.add_onetime(self.img)
        self.pole_transform.set_rotation(self.state[0] + np.pi / 2)
        if self.last_u:
            self.imgtrans.scale = (-self.last_u / 2, np.abs(self.last_u) / 2)

        return self.viewer.render(return_rgb_array=mode == 'rgb_array')

    def close(self):
        if self.viewer:
            self.viewer.close()
            self.viewer = None

def angle_normalize(x):
    return ((x + np.pi) % (2 * np.pi)) - np.pi

max_stars_repo_name: nphamilton/spinningup

max_issues_repo_name: nphamilton/spinningup

max_forks_repo_name: nphamilton/spinningup


text:
import numpy
import gensim
import re
from safe_pickle import pickle_dump
from dataloader import text_train_test_data_loader
import argparse
import os

"""
Given textual data, this method gets all the word vectors corresponding
to the data and pickles them to a file
"""
def sent_to_w2v_vectors(data, word2vec_model, outfile):

	model = None
	if "GoogleNews" in word2vec_model:
		model = gensim.models.KeyedVectors.load_word2vec_format(word2vec_model, binary=True)
	else:
		model = gensim.models.Word2Vec.load(word2vec_model)

	w2v_data = []
	for row in data:
		words = [word.lower() for word in re.split("\W+", row[1]) if len(word) > 2]
		w2v = []

		usedWords = []
		for word in words:
			if word in model:
				vec = model[word]
				if numpy.isnan(vec).any() or numpy.isinf(vec).any():
					continue
				w2v.append(vec)
				usedWords.append(word)

		if len(w2v) == 0:
			continue

		w2v_data.append([row[0], row[1], w2v, usedWords])

	dirname = os.path.dirname(outfile)
	os.makedirs(dirname, exist_ok=True)
	pickle_dump(w2v_data, outfile)

if __name__ == "__main__":
	parser = argparse.ArgumentParser(description='Sentence to vectors')
	parser.add_argument("train_path", help="train path")
	parser.add_argument("test_path", help="test path")
	parser.add_argument("word2vec_model_path", help="path to the corresponding w2v model")
	parser.add_argument("pickled_train_data_out_path", help="out path for pickled training data")
	parser.add_argument("pickled_test_data_out_path", help="out path for pickled test data")

	args = parser.parse_args()

	train_path = args.train_path
	test_path = args.test_path
	word2vec_model_path = args.word2vec_model_path
	pickled_train_data_out_path = args.pickled_train_data_out_path
	pickled_test_data_out_path = args.pickled_test_data_out_path

	# get text data
	train, test = text_train_test_data_loader(train_path, test_path)

	# pickle training data vectors
	sent_to_w2v_vectors(train, word2vec_model_path, pickled_train_data_out_path)

	# pickle test data vectors
	sent_to_w2v_vectors(test, word2vec_model_path, pickled_test_data_out_path)

max_stars_repo_name: DeepK/hoDMD-experiments

max_issues_repo_name: JIMMY-KSU/hoDMD-experiments

max_forks_repo_name: JIMMY-KSU/hoDMD-experiments


text:
# -*- coding: utf-8 -*-
'''
Module for converting recording files from different data acquiring systems to HDF5 format.
Due to lack of testing data, only data from Blackrock and Plexon were well tested. Data from other acquiring systems
were not tested. If you have these demo files, we are glad to help you test these data.

This work is based on:
    * Bo Zhang, Ji Dai - first version
'''
from .neo import io
import h5py as hp
import numpy as np
import quantities as pq
from . import func_tools as ftl

class transfile():
    '''
    The class for converting different recording files to HDF5 format in a well-defined data structure.
    This class uses the open-source package Neo as data I/O from different data acquiring systems.
    Data structure in the converted HDF5 file:
        spike_channelNumber_unitNumber: contain spike timestamps and waveforms
        analog_channelNumber: contain data values, sampling rate and the start recording time
        neo_events: contain data values and timestamps, which is equal to events in Neo. Further functions, generate_comments and generate_events
                    need to be used to extract comments and events information from neo_events
    Args
        filename (string):
            File name (without extension)
        machine_type (string):
            Which data acquiring system is used for recording the file, like 'blackrock'.
            We are glad to provide other file interface for users. Please contact us through email bozhang23@outlook.com
        replace (Boolean):
            If true, data already existed in the converted file will be overwrited.
        **arg (Dict):
            Extra arguments needed for specific machine_type.
            For 'blackrock':
                nsx_to_load, restriction of .nsx files is needed to be translated.
                channels, restriction of spikes channels is needed to be translated.
                units, restriction of spikes units is needed to be translated.
    Returns
        -
    Examples
        >>> TransFile('myfile','blackrock',True,nsx_to_load=[2,4])
            Translate 'myfile', recorded by 'blackrock' machine, to HDF5 format. Both .ns2
            and .ns4 file will be translated (.nev will always be translated).
    '''
    def __init__(self,filename,machine_type,replace=False,**arg):
        """
        Initialize the TransFile class.
        """
        self.filename = filename
        self.replace = replace
        if machine_type == 'blackrock':
            self.__transBlackrock(**arg)
        if machine_type == 'plexon':
            self.__transPlexon(**arg)
        if machine_type == 'alphaomega':
            self.__transAlphaomega(**arg)

    def __transAlphaomega(self):
        r = io.AlphaOmegaIO(filename = self.filename+'.mpx')
        bl = r.read_block(lazy=False,cascade=True)
        seg = bl.segments[0]

    def __transBlackrock(self,nsx_to_load='none',channels='all',units='all'):
        '''
        Translate files recorded by the Blackrock data acquiring system.
        '''
        r = io.BlackrockIO(self.filename)
        bl = r.read_block(nsx_to_load=nsx_to_load, load_events=True, load_waveforms=True, channels=channels, units=units)
        seg = bl.segments[0]

        # spikes entity
        spikes = dict()
        spikes['spikes'] = dict()
        for spk in seg.spiketrains:
            chan = spk.annotations['ch_idx']
            unit = spk.annotations['unit_id']
            time = np.array(spk.times.rescale(pq.ms))
            waveform = np.array(spk.waveforms.rescale(pq.mV))
            waveform.shape = waveform.shape[0],waveform.shape[-1]
            key = 'spike_'+str(chan)+'_'+str(unit)
            spikes['spikes'][key] = {'times':time,'waveforms':waveform}

        # analogs entity
        analogs = dict()
        analogs['analogs'] = dict()
        for ang in seg.analogsignals:
            if ang.shape[0]>0:
                key = 'analog_'+str(ang.annotations['ch_idx'])
                times = np.array(ang.times.rescale(pq.ms))
                data = np.array(ang.rescale(pq.mV))
                data.shape = data.shape[0]
                sampling_rate = int(ang.sampling_rate.rescale(pq.Hz))
                analogs['analogs'][key] = {'data':data, 'start_time':times[0], 'sampling_rate':sampling_rate}

        # event entity
        events = dict()
        events['neo_events'] = dict()
        for evt in seg.events:
            key = evt.name
            times = np.array(evt.times.rescale(pq.ms))
            labels = np.array(evt.labels)
            events['neo_events'][key] = {'times':times, 'labels':labels}

        trans_file = self.filename + '.h5'
        self.__save_data(spikes, analogs, events, trans_file)

    def __transPlexon(self):
        '''
        Translate files recording by Plexon data acquiring system.
        '''
        r = io.PlexonIO(filename=self.filename+'.plx')
        seg = r.read_segment(lazy=False,cascade=True)

        # spikes entity
        spikes = dict()
        spikes['spikes'] = dict()
        for spk in seg.spiketrains:
            chan = spk.annotations['ch_idx']
            unit = spk.annotations['unit_id']
            time = np.array(spk.times.rescale(pq.ms))
            waveform = np.array(spk.waveforms.rescale(pq.mV))
            waveform.shape = waveform.shape[0],waveform.shape[-1]
            key = 'spike_'+str(chan)+'_'+str(unit)
            spikes['spikes'][key] = {'times':time,'waveforms':waveform}

        # analogs entity
        analogs = dict()
        analogs['analogs'] = dict()
        for ang in seg.analogsignals:
            if ang.shape[0]>0:
                key = 'analog_'+str(ang.annotations['channel_index'])
                times = np.array(ang.times.rescale(pq.ms))
                data = np.array(ang.rescale(pq.mV))
                data.shape = data.shape[0]
                sampling_rate = int(ang.sampling_rate.rescale(pq.Hz))
                analogs['analogs'][key] = {'data':data,'start_time':times[0],'sampling_rate':sampling_rate}

        # event entity
        events = dict()
        events['neo_events'] = dict()
        for evt in seg.events:
            if evt.times.shape[0]>0:
                key = evt.annotations['channel_name']
                times = np.array(evt.times.rescale(pq.ms))
                labels = np.array(evt.labels)
                events['neo_events'][key] = {'times':times, 'labels':labels}

        trans_file = self.filename+'.h5'
        self.__save_data(spikes, analogs, events,trans_file)

    def __save_data(self,spikes,analogs,events, trans_file):
        '''
        Save data to HDF5 file in the well-defined data structure.
        '''
        self.trans_file = trans_file

        spikes = ftl.flatten_dict(spikes)
        analogs = ftl.flatten_dict(analogs)
        events = ftl.flatten_dict(events)

        flat_data = dict()
        flat_data.update(spikes)
        flat_data.update(analogs)
        flat_data.update(events)

        keys_in = list()
        if self.replace is False:
            with hp.File(self.trans_file,'a') as f:
                for key in flat_data.keys():
                    if key in f:
                        keys_in.append(key)

        if len(keys_in) > 0:
            print(keys_in)
            raise ValueError("These data already in file, please set replace = True to update these data")
        else:
            with hp.File(self.trans_file,'a') as f:
                for ite_k,ite_v in flat_data.iteritems():
                    if ite_k in f:
                        del f[ite_k]
                    f[ite_k] = ite_v
                f.flush()
            print("Data are stored now.")

def generate_comments(filename, method, replace=False, **arg):
    '''
    method:
        'move':move data from neo_events to comments
        'map': map data in neo_events to comments
    '''
    if method == 'move':
        with hp.File(filename,'a') as f:
            comments_exist = 'comments' in f
            if (replace is False) and comments_exist:
                raise ValueError("comments already exists in the file, please set replace = True to replace it")
            if (replace is True) and comments_exist:
                del f['comments']

            # read labels and times of comments
            labels = f['neo_events'][arg['key']]['labels'].value
            times = f['neo_events'][arg['key']]['times'].value

            # write labels and times to comments
            f['comments/labels'] = labels
            f['comments/times'] = times

    elif method == 'map':
        with hp.File(filename,'a') as f:
            comments_exist = 'comments' in f
            if (replace is False) and comments_exist:
                raise ValueError("comments already exists in the file, please set replace = True to replace it")
            if (replace is True) and comments_exist:
                del f['comments']

            if len(arg['mapping'])>0:
                comments_times = []
                comments_labels = []
                for map_key, to_value in arg['mapping'].iteritems():
                    if isinstance(to_value,str):
                        evt_key, evt_value = map_key.split('/')
                        mask = f['neo_events'][evt_key]['labels'].value == evt_value
                        evt_times = f['neo_events'][evt_key]['times'].value[mask]
                        evt_labels = np.full(evt_times.shape,to_value,dtype='|S%s'%(len(to_value)))
                        comments_times.append(evt_times)
                        comments_labels.append(evt_labels)
                    elif isinstance(to_value,list):
                        evt_key, evt_value = map_key.split('/')
                        mask = f['neo_events'][evt_key]['labels'].value == evt_value
                        evt_times = f['neo_events'][evt_key]['times'].value[mask]
                        for temp_value in to_value:
                            evt_labels = np.full(evt_times.shape,temp_value,dtype='|S%s'%(len(temp_value)))
                            comments_times.append(evt_times)
                            comments_labels.append(evt_labels)

                comments_times = np.concatenate(comments_times,axis=0)
                comments_labels = np.concatenate(comments_labels,axis=0)
                sorted_index = np.argsort(comments_times)
                f['comments/labels'] = comments_labels[sorted_index]
                f['comments/times'] = comments_times[sorted_index]

def generate_events(filename, method, replace=False, **arg):
    '''
    method:
        'move':move data from events to events
        'map': map data in events to events
    '''
    if method == 'move':
        with hp.File(filename,'a') as f:
            events_exist = 'events' in f
            if (replace is False) and events_exist:
                raise ValueError("events already exists in the file, please set replace = True to replace it")
            if (replace is True) and events_exist:
                del f['events']

            # read labels and times of comments
            labels = f['neo_events'][arg['key']]['labels'].value
            times = f['neo_events'][arg['key']]['times'].value

            # write labels and times to comments
            f['events/labels'] = labels
            f['events/times'] = times
    elif method == 'map':
        with hp.File(filename,'a') as f:
            events_exist = 'events' in f
            if (replace is False) and events_exist:
                raise ValueError("events already exists in the file, please set replace = True to replace it")
            if (replace is True) and events_exist:
                del f['events']

            if len(arg['mapping'])>0:
                events_times = []
                events_labels = []
                for map_key,to_value in arg['mapping'].iteritems():
                    evt_key,evt_value = map_key.split('/')
                    mask = f['neo_events'][evt_key]['labels'].value == evt_value
                    evt_times = f['neo_events'][evt_key]['times'].value[mask]
                    evt_labels = np.full(evt_times.shape,to_value,dtype='|S%s'%(len(to_value)))
                    events_times.append(evt_times)
                    events_labels.append(evt_labels)
                events_times = np.concatenate(events_times,axis=0)
                events_labels = np.concatenate(events_labels,axis=0)
                sorted_index = np.argsort(events_times)
                f['events/labels'] = events_labels[sorted_index]
                f['events/times'] = events_times[sorted_index]

max_stars_repo_name: Research-lab-KUMS/NeoAnalysis

max_issues_repo_name: Research-lab-KUMS/NeoAnalysis

max_forks_repo_name: neoanalysis/NeoAnalysis


text:
#coding:utf-8

import pyaudio
import threading
import numpy as np
from pathlib import Path
from take_picture import AutoTakePictures
#from matplotlib import pyplot as plt

chunk = 1024*2
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 16000
RECORD_SECONDS = 1

STOCK_FILE_NUM = 20

p = pyaudio.PyAudio()
picture = AutoTakePictures()

stream = p.open(
    format = FORMAT,
    channels = CHANNELS,
    rate = RATE,
    input = True,
    output = True,
    frames_per_buffer = chunk,
)

max_data = []
def audio_trans(input):
    frames = (np.frombuffer(input,dtype="int16"))
    max_data.append(max(frames))
    return max_data

# callback per term
def sendlog():
    global max_data

    if len(max_data) != 0:
        #print(max_data)
        mic_mean = int(sum(max_data)/len(max_data))
        max_data = []
        print(mic_mean)
        # over level 15000
        if mic_mean > 15000:
            picture.capture()
        file_numbers = list(Path(picture.save_dir).glob("*.jpg")).__len__()
        if file_numbers >= STOCK_FILE_NUM:
            picture.sendServer(server="18.191.254.247",user="ec2-user",key="jphack2018-01.pem")
            picture.delete()

    # thread per 1
    t = threading.Timer(1,sendlog)
    t.start()

# thread
t = threading.Thread(target = sendlog)
t.start()

while stream.is_active():
    input = stream.read(chunk)
    input = audio_trans(input)

stream.stop_stream()
stream.close()
p.terminate()


text:
from multiprocessing import Pool, TimeoutError
import numpy as np
import time
import os

"""
    Runs the Cartpole Talbe based SARSA lambda experiments
"""

def experiment(run, episodes, sigmas, lambda_parameter):
    """
    Runs a single experiment for each sigma value of SARSA lambda on Cartpole
    :param run_n: The run number, used in the filename of the experiment
    :param episodes: Number of epsiodes to run
    :param sigmas: Values of sigma (noise standard deviation)
    :param lambda_parameter: The lambda value for this experiment
    :return: The filename of the output file
    """
    import numpy as np
    from agents.sarsalambda import SarsaLambda
    from environments.cartpole import NoisyCartPole

    from experiment_util import Logger

    filename = ("../results/cartpole_sarsalambda_lambda_%1.2f_%d.h5" % (lambda_parameter, run))
    l = Logger(filename=filename)

    def cartpole_discretization(x):
        """
        The discretization function used for this experiment
        :param x: The state object
        :return: A tuple representing the discrete state
        """
        s = x.state
        if s.shape != (4,):
            raise ValueError("Expected array of shape (4,). Instead got: %s" % (str(x.shape)))

        out = np.zeros((4,))

        # cart position range [-2.4 2.4]
        out[0] = np.round(5 * (s[0] + 2.4) / 4.8, 0)  # normalized to [0 1]

        # cart velocity [-inf +inf]
        out[1] = np.round(5 * np.sqrt(np.abs(s[1])) * np.sign(s[1]), 0)
        # pole angle : [-.26rad .26rad]
        out[2] = np.round(10 * (s[2] + .26) / .52, 0)  # normalized to [0 1] then multiplied by 10
        # pole velocity at tip
        out[3] = np.round(7 * np.sqrt(np.abs(s[3])) * np.sign(s[3]), 0)
        return tuple(out)

    def transform_state(s):
        # This is a different discretization used in earlier stages, similar to cartpole_discretization
        s = s.state
        s *= np.array([1, 1, 10, 1])
        s *= 2
        s = np.round(s)
        return str(s)

    for sigma in sigmas:
        env = NoisyCartPole(std=sigma, render=False)
        actions = env.valid_actions()

        sl = SarsaLambda(env,
                         lam=lambda_parameter,
                         gamma=1.0,
                         epsilon=0.7,
                         epsilon_step_factor=0.99998,
                         epsilon_min=0.0,
                         fex=cartpole_discretization
                         )

        c = sl.get_configuration()
        experiment = l.start_experiment(c)
        pi = sl.learn(num_iter=episodes, result_handler=experiment.log)
        experiment.save_attribute("pi", pi)
        print("%s finished sigma=%1.2f, run=%i" % (filename, sigma, run))
    return filename

if __name__ == "__main__":
    # Experiment parameters:
    runs = 5
    episodes = 4000
    sigmas = np.array([0, 10 ** -2, 10 ** -1, 10 ** -0])
    lambdas = np.array([0, 0.5, 0.75, 0.9, 1])

    # If more CPUs are available, change the 4 here to the number of desired threads:
    with Pool(processes=32) as pool:
        for i in pool.starmap(experiment, [(r, episodes, sigmas, l) for l in lambdas for r in range(runs)]):
            print("Finished %s" % i)