text:
import imageio
import threading

from .client import Client, run_in_background
from .pygtk import _take_gtk_screen_size, _grab_gtk_pb, _convert_pixbuf_to_numpy, is_gdk
from .utils import _norm_path, resize_array, is_pygtk
from mss import mss
from PIL import Image, ImageSequence
from robot.utils import is_truthy

try:
    import cv2
    import numpy as np
except ImportError:
    raise ImportError('Importing cv2 failed. Make sure you have opencv-python installed.')

class GifClient(Client):

    def __init__(self, screenshot_module, screenshot_directory):
        Client.__init__(self)
        self.screenshot_module = screenshot_module
        self._given_screenshot_dir = _norm_path(screenshot_directory)
        self.optimize = None

    def start_gif_recording(self, name, size_percentage,
                            embed, embed_width, monitor, optimize):
        self.name = name
        self.embed = embed
        self.embed_width = embed_width
        self.optimize = optimize
        self.path = self._save_screenshot_path(basename=self.name, format='gif')
        self.futures = self.grab_frames(size_percentage, self._stop_condition, int(monitor))
        self.clear_thread_queues()

    def stop_gif_recording(self):
        self._stop_thread()
        if is_truthy(self.embed):
            self._embed_screenshot(self.path, self.embed_width)
        if is_truthy(self.optimize):
            frames = []
            for frame in ImageSequence.Iterator(Image.open(self.path)):
                frame = frame.copy()
                frames.append(frame)
            frames[0].save(self.path, save_all=True, append_images=frames[1:], optimize=True)
        return self.path

    @run_in_background
    def grab_frames(self, size_percentage, stop, monitor):
        if is_pygtk(self.screenshot_module):
            self._grab_frames_gtk(size_percentage, stop, monitor)
        else:
            self._grab_frames_mss(size_percentage, stop, monitor)

    def _grab_frames_gtk(self, size_percentage, stop, monitor):
        w, h = _take_gtk_screen_size(monitor)
        width = int(w * size_percentage)
        height = int(h * size_percentage)
        with imageio.get_writer(self.path, mode='I') as writer:
            while not stop.isSet():
                pb = _grab_gtk_pb(monitor)
                if is_gdk():
                    numpy_array = _convert_pixbuf_to_numpy(pb)
                else:
                    numpy_array = pb.get_pixels_array()
                resized_array = resize_array(width, height, numpy_array, size_percentage)
                frame = cv2.cvtColor(resized_array, cv2.COLOR_RGBA2RGB)
                writer.append_data(frame)

    def _grab_frames_mss(self, size_percentage, stop, monitor):
        with mss() as sct:
            mon = sct.monitors[monitor]
            width = int(sct.grab(mon).width * size_percentage)
            height = int(sct.grab(mon).height * size_percentage)
            with imageio.get_writer(self.path, mode='I') as writer:
                while not stop.isSet():
                    sct_img = sct.grab(mon)
                    resized_array = resize_array(width, height, np.array(sct_img), size_percentage)
                    frame = cv2.cvtColor(resized_array, cv2.COLOR_RGB2BGR)
                    writer.append_data(frame)

max_stars_repo_name: simonmeggle/ScreenCapLibrary

max_issues_repo_name: simonmeggle/ScreenCapLibrary

max_forks_repo_name: simonmeggle/ScreenCapLibrary


text:
"""
Demonstration of the usage of arus.core.Pipeline (2)
====================================================

The pipeline uses one annotation stream and one sensor stream as input.
"""

from arus.core.pipeline import Pipeline
import arus
import datetime as dt
import pandas as pd
import numpy as np

def _master_pipeline_processor(chunk_list, **kwargs):
    import pandas as pd
    import numpy as np
    result = {'HEADER_TIME_STAMP': [], 'START_TIME': [],
              'STOP_TIME': [], 'VALUE': [], 'ANNOTATION': []}
    for data, st, et, prev_st, prev_et, name in chunk_list:
        if len(result['START_TIME']) == 0 or result['START_TIME'][-1] != st:
            result['HEADER_TIME_STAMP'].append(st)
            result['START_TIME'].append(st)
            result['STOP_TIME'].append(et)
        if name == 'annotation-stream':
            result['ANNOTATION'].append('-'.join(data.iloc[:, 3].values))
        elif name == 'feature-pipeline':
            result['VALUE'].append(data.iloc[0, 3])
    result = pd.DataFrame.from_dict(result)

    return result

def _feature_pipeline_processor(chunk_list, **kwargs):
    import pandas as pd
    import numpy as np
    import arus
    result = {'HEADER_TIME_STAMP': [],
              'START_TIME': [], 'STOP_TIME': [], 'VALUE': []}
    for data, st, et, prev_st, prev_et, name in chunk_list:
        result['HEADER_TIME_STAMP'].append(st)
        result['START_TIME'].append(st)
        result['STOP_TIME'].append(et)
        vm_values = arus.ext.numpy.vector_magnitude(data.iloc[:, 1:].values)
        values, name = arus.accel.mean(vm_values)
        result['VALUE'].append(values[0, 0])
    result = pd.DataFrame.from_dict(result)
    return result

if __name__ == "__main__":
    # test on one annotation stream and one sensor stream + feature pipeline

    window_size = 12.8
    sr = 80
    start_time = dt.datetime.now()
    gr1 = arus.generator.RandomAccelDataGenerator(
        sr=sr, grange=8, sigma=1, buffer_size=100)
    seg1 = arus.segmentor.SlidingWindowSegmentor(window_size)
    stream1 = arus.Stream(gr1, seg1, name='sensor-stream')

    gr2 = arus.generator.RandomAnnotationDataGenerator(
        labels=["Sitting", 'Standing', 'Lying', 'Walking', 'Running'], duration_mu=8, duration_sigma=2, num_mu=3, buffer_size=100)
    seg2 = arus.segmentor.SlidingWindowSegmentor(
        window_size, st_col=1, et_col=2)
    stream2 = arus.Stream(gr2, seg2, name='annotation-stream')

    feat_pipeline = Pipeline(
        max_processes=2, scheduler='processes', name='feature-pipeline')

    feat_pipeline.add_stream(stream1)
    feat_pipeline.set_processor(_feature_pipeline_processor)
    master_pipeline = Pipeline(
        max_processes=2, scheduler='processes', name='master-pipeline')
    master_pipeline.add_stream(stream2)
    master_pipeline.add_stream(feat_pipeline)
    master_pipeline.set_processor(_master_pipeline_processor)
    master_pipeline.start(start_time=start_time)
    results = []
    for result, st, et, prev_st, prev_et, name in master_pipeline.get_iterator():
        results.append(result)
        if len(results) == 10:
            break
        print(len(results))
    print(master_pipeline.stop())
    print(pd.concat(results, axis=0, sort=False))

ext: py

max_stars_repo_name: qutang/arus

max_issues_repo_name: qutang/arus

max_forks_repo_name: qutang/arus

reason: import numpy


text:
import numpy as np
import sys, os, pdb
import pandas as pd
import argparse
from matplotlib import pyplot as plt
import argparse
from datetime import datetime
import plotly.graph_objects as go
import os
from sklearn.metrics import confusion_matrix
import seaborn as sns
# Set style
sns.set(style="whitegrid")#, color_codes=True)

np.random.seed(0)

parser = argparse.ArgumentParser(description='Plot Rejection Figure')
parser.add_argument('--file', default="datasets/results_JIND.csv", type=str,
					help='path to train data frame with labels')

def main():
	args = parser.parse_args()

	data = pd.read_csv(args.file, )
	df = data.melt('Dataset', var_name='Method',  value_name='% Rejected')
	print(data)
	# plt.figure(figsize=(20,5))
	# ax = sns.catplot(x="% Rejected", y="Method", hue='Dataset', data=df, s=5, height=3, aspect=1.5)
	# ax._legend.remove()
	# plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.25),fancybox=True, shadow=True, ncol=3)
	# plt.tight_layout()

	# path = os.path.dirname(args.file)
	# plt.savefig("{}/{}_Rejection.pdf".format(path, os.path.splitext(os.path.basename(args.file))[0]))

	plt.figure(figsize=(8,7))
	ax = sns.barplot(x="% Rejected", y="Dataset", hue='Method', data=df, edgecolor=(0.2,0.2,0.2))

	ax.set_xlabel(ax.get_xlabel(), fontsize=14)
	ax.set_ylabel("", fontsize=12)
	# print(list(ax.get_xticks()))
	ax.set_xticklabels(ax.get_xticks(), fontsize=12)
	ax.set_yticklabels(ax.get_yticklabels(), fontsize=12, rotation=45, horizontalalignment='right')
	hatches = ['-', '+', 'x', '\\', '*', 'o']

	# for patch in ax.patches:
	# 	clr = patch.get_facecolor()
	# 	patch.set_edgecolor((0,0,0))

	# Loop over the bars
	ind = -1
	for i,thisbar in enumerate(ax.patches):
		# Set a different hatch for each bar
		if i % (len(data.columns) - 1) == 0:
			ind += 1
		thisbar.set_hatch(hatches[ind])

	ax.grid(b=True, which='major', color=(0.7, 0.7, 0.7), linewidth=2.0)

	# ax._legend.remove()
	plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.12),fancybox=True, shadow=True, ncol=3, prop={"size":12})
	plt.tight_layout()

	path = os.path.dirname(args.file)
	plt.savefig("{}/{}_RejectionBar.pdf".format(path, os.path.splitext(os.path.basename(args.file))[0]))

	# plt.figure(figsize=(6,8))
	# ax = sns.barplot(x="Method", y="% Rejected", hue='Dataset', data=df)
	# ax.set_xticklabels(ax.get_xticklabels(), rotation=0, horizontalalignment='center')
	# # ax._legend.remove()
	# plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.1),fancybox=True, shadow=True, ncol=2)
	# plt.tight_layout()

	# path = os.path.dirname(args.file)
	# plt.savefig("{}/{}_RejectionBarVertical.pdf".format(path, os.path.splitext(os.path.basename(args.file))[0]))

	# plt.figure(figsize=(10, 6))
	# ax = sns.lineplot(x="Dataset", y="% Rejected", hue='Method', data=df, markers=True, style="Method")
	# # ax._legend.remove()
	# # ax.set_xticklabels(list(data['Dataset']), rotation=0, horizontalalignment='center')
	# plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.1),fancybox=True, shadow=True, ncol=3)
	# plt.tight_layout()

	# plt.savefig("{}/{}_RejectionLine.pdf".format(path, os.path.splitext(os.path.basename(args.file))[0]))

if __name__ == "__main__":
	main()


text:
#!/usr/bin/env python

"""Example 2 Sin Approximation using the taylor series

Options are:

simulate      - native python simulation
simulate_vhdl - simulate using ghdl cosimulation

Thing to try:

vary p        - the total number of bits in the approximation process
vary q        - the number of fraction bits in the fixed point representation
vary r        - the order of the power series, use odd values starting with 3"""

import sys
from math import pi
from chips import *

#fixed point routines
p = 26
q = 6
r = 9

def to_fixed(x):
    return int(round(x*(2**q)))

def from_fixed(x):
    return x*(2**-q)

#create taylor series
def taylor(angle):

    sin_out = Output()
    approximation = Variable(0)
    i = Variable(0)
    power = Variable(0)
    fact = Variable(0)
    count = Variable(0)
    angle_var = Variable(0)
    sign = Variable(0)

    Process(p,
        Loop(
            angle.read(angle_var),
            approximation.set(angle_var),
            i.set(3),
            sign.set(-1),
            While( i <= r,

                #calculate x**i
                count.set(0),
                power.set(to_fixed(1.0)),
                While(count<i, power.set((power*angle_var)>>q), count.set(count+1)),

                #calculate x!
                count.set(1),
                fact.set(1),
                While(count<=i, fact.set(fact*count), count.set(count+1)),

                #calculate x**i/i!
                approximation.set(approximation + (sign*(power//fact))),
                sign.set(0-sign),

                i.set(i+2),
            ),
            sin_out.write(approximation),
        )
    )
    return sin_out

if "simulate" in sys.argv:
    import numpy as n
    from matplotlib import pyplot as pl

    x=n.linspace(to_fixed(0), to_fixed(pi), 100)
    response=Response(taylor(Sequence(*x)))
    chip = Chip(response)
    chip.reset()
    chip.execute(100000)
    sin_x=[from_fixed(i) for i in response.get_simulation_data()]
    pl.plot(sin_x[:100], 'b-', label="Taylor series approximation")
    pl.plot(n.sin(n.linspace(0,pi,100)), 'r-', label="sin(x)")
    pl.title("Sin Wave Approximation")
    pl.legend()
    pl.show()

elif "simulate_vhdl" in sys.argv:
    import numpy as n
    from matplotlib import pyplot as pl
    from chips.VHDL_plugin import Plugin
    x=n.linspace(to_fixed(0), to_fixed(pi), 100)
    response=Response(taylor(Sequence(*x)))
    chip = Chip(response)
    plugin = Plugin()
    chip.write_code(plugin)
    plugin.ghdl_test("test taylor series", stop_cycles=300000)
    sin_x=[from_fixed(i) for i in response.get_simulation_data(plugin)]
    pl.plot(sin_x[:100], label="Taylor series approximation")
    pl.plot(n.sin(n.linspace(0,pi,100)), label="sin(x)")
    pl.title("Sin Wave Approximation")
    pl.legend()
    pl.show()

elif "simulate_cpp" in sys.argv:
    import numpy as n
    from matplotlib import pyplot as pl
    from chips.cpp_plugin import Plugin
    x=n.linspace(to_fixed(0), to_fixed(pi), 100)
    response=Response(taylor(Sequence(*x)))
    chip = Chip(response)
    plugin = Plugin()
    chip.write_code(plugin)
    plugin.test("test taylor series", stop_cycles=100000)
    sin_x=[from_fixed(i) for i in response.get_simulation_data(plugin)]
    pl.plot(sin_x[:100], label="Taylor series approximation")
    pl.plot(n.sin(n.linspace(0,pi,100)), label="sin(x)")
    pl.title("Sin Wave Approximation")
    pl.legend()
    pl.show()


text:
import numpy as np
from ch05.ex10_twolayer import TwoLayerNetwork
from dataset.mnist import load_mnist
from sklearn.utils import shuffle
import matplotlib.pyplot as plt

if __name__ == '__main__':
    np.random.seed(106)
    # MNIST 데이터 로드
    (X_train, Y_train), (X_test, Y_test) = load_mnist(one_hot_label=True)
    # 2층 신경망 생성
    neural_net = TwoLayerNetwork(input_size=784, hidden_size=32, output_size=10)

    # 100번 반복
    epochs = 30
    batch_size = 128  # 한번에 학습시키는 입력 데이터 개수
    learning_rate = 0.2

    iter_size = max(X_train.shape[0] // batch_size, 1)
    print(iter_size)

    # 전체 데이터 학습을 30회(epochs) 반복
    try_number = []
    loss_record = []
    accuracy_record = []
    for j in range(epochs):
        for i in range(iter_size):
            # forward -> backward -> gradient : gradient 함수 안에 모두 포함
            # batch_size만큼 데이터를 입력해서 gradient 계산
            X_data, Y_label = X_train[i * batch_size:(i + 1) * batch_size], Y_train[i * batch_size:(i + 1) * batch_size]
            gradients = neural_net.gradient(X_data, Y_label)
            loss = neural_net.loss(X_data, Y_label)
            accuracy = neural_net.accuracy(X_data, Y_label)
            # 가중치/편향 행렬들을 수정
            for key in gradients:
                neural_net.params[key] -= learning_rate*gradients[key]
        # 학습 데이터 세트를 무작위로 섞는 (shuffle) 코드 추가
        X_data, Y_label = shuffle(X_data, Y_label)

        try_number.append(j)
        # loss를 계산해서 출력
        print(f'loss{j}: {loss}')
        loss_record.append(loss)
        # accuracy를 계산해서 출력
        print(f'accuracy{j}: {accuracy}')
        accuracy_record.append(accuracy)

    plt.plot(try_number, loss_record)
    plt.xlabel('x')
    plt.ylabel('loss')
    plt.show()

    plt.plot(try_number, accuracy_record)
    plt.xlabel('x')
    plt.ylabel('accuracy')
    plt.show()


text:
import os
import unittest

import numpy as np
import pandas as pd
import xarray as xr

from xcube_server.im import TileGrid
from xcube_server.mldataset import BaseMultiLevelDataset, ComputedMultiLevelDataset

class BaseMultiLevelDatasetTest(unittest.TestCase):
    def test_it(self):
        ds = _get_test_dataset()

        ml_ds = BaseMultiLevelDataset(ds)
        self.assertEqual(3, ml_ds.num_levels)
        self.assertEqual(TileGrid(3, 2, 1, 180, 180, (-180, -90, 180, 90), inv_y=False),
                         ml_ds.tile_grid)

        ds0 = ml_ds.get_dataset(0)
        self.assertIs(ds, ds0)

        ds1 = ml_ds.get_dataset(1)
        self.assertIsNot(ds, ds1)
        self.assertEqual({'time': 14, 'lat': 360, 'lon': 720}, ds1.dims)

        ds2 = ml_ds.get_dataset(2)
        self.assertIsNot(ds, ds2)
        self.assertEqual({'time': 14, 'lat': 180, 'lon': 360}, ds2.dims)

        ml_ds.close()

class ComputedMultiLevelDatasetTest(unittest.TestCase):
    def test_it(self):
        ds = _get_test_dataset()

        ml_ds1 = BaseMultiLevelDataset(ds)

        def input_ml_dataset_getter(ds_id):
            if ds_id == "ml_ds1":
                return ml_ds1
            self.fail(f"unexpected ds_id={ds_id!r}")

        ml_ds2 = ComputedMultiLevelDataset("ml_ds2",
                                           os.path.join(os.path.dirname(__file__), "res", "test", "script.py"),
                                           "compute_dataset",
                                           ["ml_ds1"],
                                           input_ml_dataset_getter,
                                           input_parameters=dict(period='1W'))
        self.assertEqual(3, ml_ds2.num_levels)
        self.assertEqual(TileGrid(3, 2, 1, 180, 180, (-180, -90, 180, 90), inv_y=False),
                         ml_ds2.tile_grid)

        ds0 = ml_ds2.get_dataset(0)
        self.assertEqual({'time': 3, 'lat': 720, 'lon': 1440, 'bnds': 2}, ds0.dims)

        ds1 = ml_ds2.get_dataset(1)
        self.assertEqual({'time': 3, 'lat': 360, 'lon': 720}, ds1.dims)

        ds2 = ml_ds2.get_dataset(2)
        self.assertEqual({'time': 3, 'lat': 180, 'lon': 360}, ds2.dims)

        ml_ds1.close()
        ml_ds2.close()

def _get_test_dataset():
    w = 1440
    h = 720
    p = 14

    x1 = -180.
    y1 = -90.
    x2 = +180.
    y2 = +90.
    dx = (x2 - x1) / w
    dy = (y2 - y1) / h

    data_vars = dict(noise=(("time", "lat", "lon"), np.random.rand(p, h, w)))

    lat_bnds = np.array(list(zip(np.linspace(y2, y1 + dy, num=h),
                                 np.linspace(y2 - dy, y1, num=h))))
    lon_bnds = np.array(list(zip(np.linspace(x1, x2 - dx, num=w),
                                 np.linspace(x1 + x2, x2, num=w))))
    coords = dict(time=(("time",),
                        np.array(pd.date_range(start="2019-01-01T12:00:00Z", periods=p, freq="1D"),
                                 dtype="datetime64[ns]")),
                  lat=(("lat",),
                       np.linspace(y2 - .5 * dy, y1 + .5 * dy, num=h)),
                  lon=(("lon",),
                       np.linspace(x1 + .5 * dx, x2 - .5 * dx, num=w)),
                  lat_bnds=(("lat", "bnds"), lat_bnds),
                  lon_bnds=(("lon", "bnds"), lon_bnds))

    return xr.Dataset(coords=coords, data_vars=data_vars)


text:
import os
import h5py
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

def _combine_features(first_feature, second_feature, type_name):
    if type_name == 'multiply':
        return first_feature * second_feature
    elif type_name == 'maximum':
        return np.maximum(first_feature, second_feature)
    elif type_name == 'minimum':
        return np.minimum(first_feature, second_feature)
    elif type_name == 'squaresum':
        return np.square(first_feature + second_feature)
    elif type_name == 'sign_modify':
        return first_feature * np.sign(second_feature)
    elif type_name == 'cossum':
        return np.cos(first_feature + second_feature)
    elif type_name == 'tanhsum':
        return np.tanh(first_feature + second_feature)
    else:
        raise ValueError('Unsupported type {}'.format(type_name))

def all_interaction_indices(n):
    return np.stack(np.triu_indices(n, k=1), axis=1)

def ablate_interactions(x, interactions, spec_df, k, using_random_draw=True):
    x = x.copy()
    new_y = np.zeros(x.shape[0])

    first_indices, second_indices = np.triu_indices(x.shape[-1], k=1)
    upper_triangular_interactions = interactions[:, first_indices, second_indices]
    interaction_max_indices = np.argsort(np.abs(upper_triangular_interactions),
                                         axis=-1)[:, ::-1][:, :k]
    interaction_pairs = [[(first_indices[i], second_indices[i]) for i in select_index] \
                         for select_index in interaction_max_indices]

    mean_abs_coeff = np.max(np.abs(spec_df['coefficient']))

    for index, row in spec_df.iterrows():
        coeff = row['coefficient']
        type_name = row['interaction_type']
        i  = row['first_feature']
        j  = row['second_feature']
        pair = (i, j)

        first_feature  = x[:, pair[0]]
        second_feature = x[:, pair[1]]

        # This is the key to our remove and retrain experiments.
        # Since the interactions are known, we can regenerate the data
        # and ablate known interactions by redrawing from the distribution
        mask_use_random = [pair in sample_pairs for sample_pairs in interaction_pairs]
        mask_use_random = np.array(mask_use_random)

        if using_random_draw:
            interaction_value = _combine_features(first_feature,
                                                  second_feature,
                                                  type_name)
            random_draw = np.random.randn(x.shape[0])
            interaction_value[mask_use_random] = interaction_value[mask_use_random] * random_draw[mask_use_random]
            coeff = mean_abs_coeff
        else:
            first_feature[mask_use_random]  = np.random.randn(np.sum(mask_use_random))
            second_feature[mask_use_random] = np.random.randn(np.sum(mask_use_random))

            interaction_value = _combine_features(first_feature,
                                                  second_feature,
                                                  type_name)

        new_y += coeff * interaction_value
    return new_y

def generate_data(num_samples,
                  num_features,
                  num_interactions,
                  dataset_name,
                  interaction_types=['multiply', 'maximum', 'minimum', 'squaresum'],
                  equal_coeffs=False):
    x = np.random.randn(num_samples, num_features).astype(np.float32)

    all_possible_indices = all_interaction_indices(num_features)
    choice_indices = np.random.choice(all_possible_indices.shape[0],
                                      size=num_interactions,
                                      replace=False)
    chosen_interaction_pairs = all_possible_indices[choice_indices]
    chosen_types = np.random.choice(len(interaction_types),
                                    size=num_interactions,
                                    replace=True)
    chosen_type_names = [interaction_types[i] for i in chosen_types]
    if equal_coeffs:
        relative_interaction_coefficients = np.full(shape=num_interactions,
                                                    fill_value=1.0)
    else:
        relative_interaction_coefficients = np.random.randn(num_interactions)

    y = np.zeros(num_samples)
    for i, pair in enumerate(chosen_interaction_pairs):
        coeff = relative_interaction_coefficients[i]
        type_name = chosen_type_names[i]

        first_feature  = x[:, pair[0]]
        second_feature = x[:, pair[1]]

        interaction_value = _combine_features(first_feature,
                                              second_feature,
                                              type_name)
        y += coeff * interaction_value

    label_deviation = np.std(y)
    relative_interaction_coefficients /= label_deviation
    y /= label_deviation

    df = pd.DataFrame({
        'first_feature': chosen_interaction_pairs[:, 0],
        'second_feature': chosen_interaction_pairs[:, 1],
        'interaction_type': chosen_type_names,
        'coefficient': relative_interaction_coefficients
    })

    df.to_csv('data/{}_spec.csv'.format(dataset_name),
              index=False)

    x_train, x_test, y_train, y_test = train_test_split(x,
                                                        y,
                                                        test_size=0.2,
                                                        random_state=0)
    np.savez('data/{}.npz'.format(dataset_name),
             x_train=x_train,
             x_test=x_test,
             y_train=y_train,
             y_test=y_test)

def main(argv=None):
    num_samples = 5000
    num_features = 10
    num_interactions = 20

    generate_data(num_samples=num_samples,
                  num_features=num_features,
                  num_interactions=num_interactions,
                  dataset_name='simulated_multiply',
                  interaction_types=['multiply'])

    generate_data(num_samples=num_samples,
                  num_features=num_features,
                  num_interactions=num_interactions,
                  dataset_name='simulated_maximum',
                  interaction_types=['maximum'])

    generate_data(num_samples=num_samples,
                  num_features=num_features,
                  num_interactions=num_interactions,
                  dataset_name='simulated_minimum',
                  interaction_types=['minimum'])

    generate_data(num_samples=num_samples,
                  num_features=num_features,
                  num_interactions=num_interactions,
                  dataset_name='simulated_cossum',
                  interaction_types=['cossum'])

    generate_data(num_samples=num_samples,
                  num_features=num_features,
                  num_interactions=num_interactions,
                  dataset_name='simulated_tanhsum',
                  interaction_types=['tanhsum'])

if __name__ == '__main__':
    main()


text:
# Copyright (c) 2020 by Fraunhofer Institute for Energy Economics
# and Energy System Technology (IEE), Kassel. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

from numpy import linalg
import numpy as np

def calc_lambda(v, eta, rho, d, k, gas_mode, friction_model, dummy):
    """
    Function calculates the friction factor of a pipe. Turbulence is calculated based on
    Nikuradse. If v equals 0, a value of 0.001 is used in order to avoid division by zero.
    This should not be a problem as the pressure loss term will equal zero (lambda * u^2).

    :param v:
    :type v:
    :param eta:
    :type eta:
    :param rho:
    :type rho:
    :param d:
    :type d:
    :param k:
    :type k:
    :param gas_mode:
    :type gas_mode:
    :param friction_model:
    :type friction_model:
    :param dummy:
    :type dummy:
    :return:
    :rtype:
    """
    v_corr = np.where(np.abs(v) < 1e-6, 1e-6 * np.sign(v), v)
    lambda_laminar = np.zeros_like(v)

    re = np.abs(rho * v_corr * d / eta)
    lambda_laminar[v != 0] = 64 / re[v != 0]

    if gas_mode:
        lambda_nikuradse = 1 / (2 * np.log10(d / k) + 1.14) ** 2
    else:
        lambda_nikuradse = 1 / (-2 * np.log10(k / (3.71 * d))) ** 2

    if friction_model == "colebrook":
        lambda_colebrook = colebrook(re, d, k, lambda_nikuradse, dummy)
        return lambda_colebrook, re
    elif friction_model == "swamee-jain":
        lambda_swamee_jain = 0.25 / ((np.log10(k/(3.7*d) + 5.74/(re**0.9)))**2)
        return lambda_swamee_jain, re
    else:
        # lambda_tot = np.where(re > 2300, lambda_laminar + lambda_nikuradse, lambda_laminar)
        lambda_tot = lambda_laminar + lambda_nikuradse
        return lambda_tot, re

def calc_der_lambda(v, eta, rho, d, k, friction_model, lambda_pipe):
    """
    Function calculates the derivative of lambda with respect to v. Turbulence is calculated based
    on Nikuradse. This should not be a problem as the pressure loss term will equal zero
    (lambda * u^2).

    :param v:
    :type v:
    :param eta:
    :type eta:
    :param rho:
    :type rho:
    :param d:
    :type d:
    :param k:
    :type k:
    :param friction_model:
    :type friction_model:
    :param lambda_pipe:
    :type lambda_pipe:
    :return:
    :rtype:
    """

    v_corr = np.where(v == 0, 0.00001, v)

    lambda_laminar_der = -(64 * eta) / (rho * v_corr ** 2 * d)

    if friction_model == "colebrook":
        b_term = 2.51 * eta / (rho * d * np.sqrt(lambda_pipe) * v_corr) + k / (3.71 * d)

        df_dv = -2 * (2.51 * eta / (rho * np.sqrt(lambda_pipe) * v_corr ** 2)) \
                / (np.log(10) * b_term)

        df_dlambda = -0.5 * lambda_pipe ** (-3 / 2) - ((2.51 * eta) / (rho * d * v_corr)) \
                     * lambda_pipe ** (-3 / 2) * 1 / (np.log(10) * b_term)

        lambda_colebrook_der = df_dv / df_dlambda

        return lambda_colebrook_der
    elif friction_model == "swamee-jain":
        param = k/(3.7*d) + 5.74 * (np.abs(eta))**0.9 / ((np.abs(rho*v_corr*d))**0.9)
        lambda_swamee_jain_der = 0.5/np.log(10) * 1/(np.log(param) ** 3) * 1/param * 5.166 \
                                 * (np.abs(eta))**0.9/(((np.abs((rho*d)))**0.9) * (np.abs(v_corr))**1.9)
        return lambda_swamee_jain_der
    else:
        return lambda_laminar_der

def colebrook(re, d, k, lambda_nikuradse, dummy):
    """

    :param re:
    :type re:
    :param d:
    :type d:
    :param k:
    :type k:
    :param lambda_nikuradse:
    :type lambda_nikuradse:
    :param dummy:
    :type dummy:
    :return: lambda_cb
    :rtype:
    """
    lambda_cb = lambda_nikuradse
    converged = False
    error_lambda = []
    niter = 0

    # Inner Newton-loop for calculation of lambda
    while not converged:
        f = lambda_cb ** (-1 / 2) + 2 * np.log10(2.51 / (re * np.sqrt(lambda_cb)) + k / (3.71 * d))

        df_dlambda_cb = (-1 / 2 * lambda_cb ** (-3 / 2)) - (2.51 / re) * lambda_cb ** (-3 / 2) \
                        / (np.log(10) * 2.51 / (re * np.sqrt(lambda_cb) + k / (3.71 * d)))

        x = - f / df_dlambda_cb

        lambda_cb_old = lambda_cb
        lambda_cb = lambda_cb + x

        dx = np.abs(lambda_cb - lambda_cb_old) * dummy
        error_lambda.append(linalg.norm(dx) / (len(dx)))

        if error_lambda[niter] <= 1e-4:
            converged = True

        niter += 1

    return lambda_cb

max_stars_repo_name: nsanina/pandapipes

max_issues_repo_name: nsanina/pandapipes

max_forks_repo_name: nsanina/pandapipes

reason: import numpy,from numpy


text:
"""
SPEECh: Scalable Probabilistic Estimates of EV Charging
Code first published in October 2021.
Developed by Siobhan Powell (siobhan.powell@stanford.edu).
"""

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import pickle
import copy
import os

class DataSetConfigurations(object):
    """Store the information for each data set prepared.

    Attributes:
        ng: number of driver groups
        data_set: name of data set used
        folder: location of the data and model object files

        cluster_reorder_actodend: mapping from cluster labels in agglomerative clustering (ac) to dendrogram (dend)
        cluster_reorder_dendtoac: reverse mapping

        categories: charging segments
        labels: proper names for charging segments
        colours: plotting colours for charging segments
        num_categories: number of charging segments modeled
        rates: charging rate for a session in each segment, in kW

        gmm_names: part of file name for sessions model GMM object
        start_time_scaler: adjustment to start time produced by GMM model
        start_mod: limit when calculating modulo of generated start times

        zkey_weekday, zkey_weekend: naming convention within pz

        timers_dict: dictionary used to remove residential timers
            Includes one entry for each driver group which needs to change.
            For each driver group, a dictionary rebalancing weights among the sessions components.
        timer_cat: which charging segment contains the timers

    Methods:
        original16(): sets up data properties for original data set with 16 clusters. When other datasets are added,
        similar methods should be created for them.

        set_up_dendrogram_cluster_mapping(): where the labels assigned in clustering are ordered differently than how
        clusters are presented in the dendrogram, we reorder them so plotted orders match our expectation from the
        dendrogram.
    """

    def __init__(self, data_set, ng=None):

        self.categories = []
        self.labels = []
        self.colours = {}
        self.num_categories = 0
        self.rates = []
        self.zkey_weekday = ''
        self.zkey_weekend = ''

        self.cluster_reorder_dendtoac = {}
        self.cluster_reorder_actodend = {}

        self.gmm_names = {}
        self.start_time_scaler = 1
        self.start_mod = 1

        self.timers_dict = {}
        self.shift_timers_dict = {}
        self.timer_cat = ''

        if data_set == 'Original16':
            self.data_set = 'Original16'
            self.ng = 16
            self.original16()
        elif data_set == 'NewData':
            self.data_set = 'NewData'
            self.ng = ng
            self.new_data()
        else:
            self.ng = ng
            self.data_set = data_set
            raise Exception('Select set-up method for new data set.')

        self.set_up_dendrogram_cluster_mapping()
        self.folder = 'Data/'+self.data_set+'/'

    def original16(self):
        """Clustering used in first publication."""

        self.categories = ['Home', 'MUD', 'Work', 'Other Slow', 'Other Fast']
        self.labels = ['Residential L2', 'MUD L2', 'Workplace L2', 'Public L2', 'Public DCFC']
        self.colours = {'Residential L2': '#dfc27d', 'MUD L2': '#f6e8c3', 'Workplace L2': '#80cdc1', 'Public L2': '#01665e', 'Public DCFC': '#003c30'}
        self.num_categories = 5
        self.rates = [6.6, 6.6, 6.6, 6.6, 150]
        self.gmm_names = {'Home': 'home', 'Work': 'work', 'Other Slow': 'other_slow', 'MUD': 'mud', 'Other Fast': 'other_fast'}
        self.start_time_scaler = 1/60
        self.zkey_weekday = ' - Fraction of weekdays with session'
        self.zkey_weekend = ' - Fraction of weekenddays with session'
        self.start_mod = 24*3600
        self.timers_dict = {1: {0: 0, 2: 0.40824126 + 0.27438057}, 13: {4: 0, 6: 0, 0: 0.13107678 + (0.13 / (0.13 + 0.25))*(0.03736147 + 0.22645425), 5: 0.25021152 + (0.25 / (0.13 + 0.25))*(0.03736147 + 0.22645425)},
                            3: {0: 0, 1: 0, 4: 0.20294322 + (0.20 / (0.14 + 0.20))*(0.25518963 + 0.03967571), 7: 0.1398116 + (0.14 / (0.14 + 0.20))*(0.25518963 + 0.03967571)}}
        self.shift_timers_dict = {'Components': {1: [0], 13: [4, 6], 3: [0, 1]}, 'Targets': {'PGE': 23, 'SMUD': 0, 'SCE': 21, 'SDGE': 0}}
        self.timer_cat = 'Home'

    def new_data(self):
        """New Clustering. Also note new ng = 9 (replace with true value from implementation on new data."""

        self.categories = ['Home', 'MUD', 'Work', 'Other Slow', 'Other Fast']
        self.labels = ['Residential L2', 'MUD L2', 'Workplace L2', 'Public L2', 'Public DCFC']
        self.colours = {'Residential L2': '#dfc27d', 'MUD L2': '#f6e8c3', 'Workplace L2': '#80cdc1', 'Public L2': '#01665e', 'Public DCFC': '#003c30'}
        self.num_categories = 5
        self.rates = [6.6, 6.6, 6.6, 6.6, 150]
        self.gmm_names = {'Home': 'home', 'Work': 'work', 'Other Slow': 'other_slow', 'MUD': 'mud', 'Other Fast': 'other_fast'}
        self.start_time_scaler = 1/60
        self.zkey_weekday = ' - Fraction of weekdays with session'
        self.zkey_weekend = ' - Fraction of weekenddays with session'
        self.start_mod = 24*3600  # since start time is in seconds
        self.timers_dict = {}
        # Optional: record shifts for removing timers from fitted model
        # self.timers_dict = {group number with timers in it:
        # {gmm segment number with timers in it: 0,
        # other gmm segment number to switch to: current weight + weight from timer segment}}
        self.shift_timers_dict = {}
        # Similarly: self.shift_timers_dict = {'Components': {group number with timers: [list of gmm segments with timers],
        # another group number with timers: [list of gmm segments with timers],},
        # 'Targets': {'PGE': 23, 'SMUD': 0, 'SCE': 21, 'SDGE': 0}}
        self.timer_cat = 'Home'

    def set_up_dendrogram_cluster_mapping(self):

        if self.data_set == 'Original16':
            self.cluster_reorder_actodend = {11: 0, 12: 1, 5: 2, 1: 3, 13: 4, 3: 5, 8: 6, 15: 7, 2: 8, 9: 9, 7: 10, 14: 11, 4: 12, 6: 13, 10: 14, 0: 15}
            self.cluster_reorder_dendtoac = {0: 11, 1: 12, 2: 5, 3: 1, 4: 13, 5: 3, 6: 8, 7: 15, 8: 2, 9: 9, 10: 7, 11: 14, 12: 4, 13: 6, 14: 10, 15: 0}
        else:
            self.cluster_reorder_dendtoac = {i: i for i in range(self.ng)}
            self.cluster_reorder_actodend = {i: i for i in range(self.ng)}

class SPEECh(object):
    """Top-level SPEECh class.

    Attributes:
        data: object from DataSetConfiguration class with set up of data set used.
        pz: segment probabilities,
            e.g. pz['weekday'][0]['Home'] gives the probability a driver in group 0 will charge at home on a weekday
        pg: driver group probabilities

    Methods:
        pg_(): loads the base distribution for pg
        pz_g(): loads the base values for pz
    """

    def __init__(self, data):

        self.data = data

        self.pz = {'weekday': {}, 'weekend': {}}
        self.pg_()
        self.pz_g()

    def pg_(self):
        """Loads the base distribution over driver groups."""
        self.pg = pd.read_csv(self.data.folder+'pg.csv')

    def pz_g(self):
        """Loads the segment probabilities."""
        for i in range(self.data.ng):
            self.pz['weekday'][i] = pd.read_csv(self.data.folder+'pz_weekday_g_'+str(i)+'.csv')
            # if self.weekend_exists:
            self.pz['weekend'][i] = pd.read_csv(self.data.folder+'pz_weekend_g_'+str(i)+'.csv')

class SPEEChGeneralConfiguration(object):
    """General configuration using the speech class to set up the driver groups and segments.

    Attributes:
        speech: object from the SPEECh class
        num_total_drivers: number of drivers in the simulation
        group_configs: sub-configuration for each driver group
        num_drivers: number of drivers assigned to each driver group

        time_step: time step for load profile simulated, in hours
        num_time_steps: number of time steps in the profile
        time_steps_per_hour: number of time steps per hour (inverse of time_step)
        energy_clip: upper bound to energy values generated by GMM

        total_load_dict: dictionary with the total load result
        all_load_dicts: dictionary with the load results for each driver group
        total_load_segments: array with the total load result
        all_load_segments: dictionary of arrays with the load results for each driver group

        remove_timers: option to remove timers from the segment specified in speech

    Methods:
        num_evs: number of electric vehicles (EVs) in simulation and driver groups.
        groups: load configurations for each of the driver groups
        change_ps_zg: can change the distribution over sessions model components, P(s|z, g)
        change_pg: can change the distribution over driver groups, P(g)
        run_all: calculates the load profiles
    """
    def __init__(self, speech, remove_timers=False):

        self.speech = speech
        self.group_configs = {}
        self.num_drivers = np.zeros((self.speech.data.ng, ))
        self.time_step = (1/60)
        self.num_time_steps = 1440
        self.time_steps_per_hour = 60
        self.energy_clip = 100
        self.num_total_drivers = None
        self.total_load_dict = {}
        self.total_load_segments = np.zeros((self.num_time_steps, len(self.speech.data.labels)))
        self.all_load_dicts = {}
        self.all_load_segments = {}

        self.remove_timers = remove_timers
        # self.shift_timers = False

    def num_evs(self, num_total_drivers, col='pg'):
        """Inputs the number of EVs in the simulation. Uses pg to calculate the number assigned to each driver group."""
        self.num_total_drivers = num_total_drivers
        for i in range(self.speech.data.ng):
            self.num_drivers[i] = int(self.num_total_drivers * self.speech.pg.loc[i, col])

    def groups(self):
        """For each driver group, create an object from the class SPEEChGroupConfiguration.
        This method also implements the timer removal option by calling change_ps_zg().
        """
        for i in range(self.speech.data.ng):
            self.group_configs[i] = SPEEChGroupConfiguration(self, i)
            if self.remove_timers:
                if i in self.speech.data.timers_dict.keys():
                    self.change_ps_zg(i, self.speech.data.timer_cat, 'weekday', self.speech.data.timers_dict[i])

    def change_ps_zg(self, g, cat, weekday, new_weights):
        """Change the distribution over sessions model components, P(s|z, g).

        Parameters:
            g: driver group
            cat: charging segment (category)
            weekday: weekday option
            new_weights: dictionary giving new weights for segment cat in driver group g

        The method alters the weights directly in the gmm without saving the original values.
        The input need only specify new weights for a subset of the components;
        the remaining weight will be distributed among the remaining components proportionately.
        """
        if cat in self.group_configs[g].segment_gmms[weekday].keys():

            gmm = self.group_configs[g].segment_gmms[weekday][cat]

            all_inds = np.arange(0, np.shape(gmm.weights_)[0])
            total_rem = 1
            for key, val in new_weights.items():
                gmm.weights_[key] = val
                total_rem -= val
            other_keys = np.delete(all_inds, list(new_weights.keys()))
            gmm.weights_[other_keys] = gmm.weights_[other_keys] * total_rem / sum(gmm.weights_[other_keys])

            self.group_configs[g].segment_gmms[weekday][cat] = gmm

    def change_pg(self, new_weights, new_col='pg', dend=False):
        """Change the distribution over driver groups, P(g).

        Parameters:
            new_weights: dictionary giving the new weights for the driver groups
            new_col: option to keep the original weights in column 'pg' and create a new column with the new weights

        The input need only specify new weights for a subset of the components;
        the remaining weight will be distributed among the remaining components proportionately.
        """

        if dend: # If inputs are given using cluster numbers from the dendrogram
            new_weights_old = copy.deepcopy(new_weights)
            new_weights = {}
            for key, val in new_weights_old.items():
                new_weights[self.speech.data.cluster_reorder_dendtoac[key]] = val

        self.speech.pg[new_col] = self.speech.pg['pg'].copy()
        all_inds = np.arange(0, self.speech.data.ng)
        total_rem = 1
        for key, val in new_weights.items():
            self.speech.pg.loc[key, new_col] = val
            total_rem -= val
        other_keys = np.delete(all_inds, list(new_weights.keys()))
        self.speech.pg.loc[other_keys, new_col] = self.speech.pg.loc[other_keys, new_col] * total_rem / sum(self.speech.pg.loc[other_keys, new_col])

    def run_all(self, verbose=False, weekday='weekday'):
        """Calculates the load profiles for each driver group using class LoadProfile.
        Records the results for the separate groups and for the aggregate.
        """
        self.total_load_dict = {x: np.zeros((self.num_time_steps,)) for x in self.speech.data.labels}
        self.total_load_segments = np.zeros((self.num_time_steps, len(self.speech.data.labels)))
        self.all_load_dicts = {}
        self.all_load_segments = {}
        for g in range(self.speech.data.ng):
            if verbose:
                print('Group '+str(g))
            model = LoadProfile(self, self.group_configs[g], weekday=weekday)
            model.calculate_load()
            self.all_load_dicts[g] = model.load_segments_dict
            self.all_load_segments[g] = model.load_segments_array
            for key, val in model.load_segments_dict.items():
                self.total_load_dict[key] += val
            self.total_load_segments += model.load_segments_array

class SPEEChGroupConfiguration(object):
    """Configuration, sessions counts, and gmms for each individual driver group.

    Attributes:
        speech_config: object from class SPEEChGeneralConfiguration
        g: driver group
        total_drivers: number of drivers in the driver group
        segment_session_numbers: dictionary of the number of sessions to generate from each charging segment
        segment_gmms: the GMM objects needed to generate sessions

    Methods:
        numbers: calculate the values in segment_session_numbers
        load_gmms: load the objects in segment_gmms
    """

    def __init__(self, speech_config, g):

        self.speech_config = speech_config
        self.g = g
        self.total_drivers = speech_config.num_drivers[g]
        self.segment_session_numbers = {'weekday': {}, 'weekend': {}}
        self.segment_gmms = {'weekday': {}, 'weekend': {}}

        self.numbers()
        self.load_gmms()

    def numbers(self, total_drivers=None):
        """Calculates the number of sessions for this driver group in each of the charging segments.

        Some driver groups have a very small number of charging sessions in a certain segment: too small for a sessions
        model to have been trained on. In this method, if the number of sessions in the segment is fewer than 0.1% of
        the number of drivers, we use that as a proxy to catch those cases where the model would throw an error.

        Parameters:
            total_drivers: gives the option to recalculate with a new value of total_drivers
        """

        if total_drivers is not None:
            self.total_drivers = total_drivers

        inds = np.random.choice(range(len(self.speech_config.speech.pz['weekday'][self.g])), int(self.total_drivers), replace=True)
        for cat in self.speech_config.speech.data.categories:
            if os.path.isfile(self.speech_config.speech.data.folder+'GMMs/'+'weekday'+'_'+self.speech_config.speech.data.gmm_names[cat]+'_'+str(self.g)+'.p'):
                self.segment_session_numbers['weekday'][cat] = int(sum(self.speech_config.speech.pz['weekday'][self.g].loc[inds, cat+self.speech_config.speech.data.zkey_weekday]))
            else:
                self.segment_session_numbers['weekday'][cat] = 0
            if os.path.isfile(self.speech_config.speech.data.folder+'GMMs/'+'weekend'+'_'+self.speech_config.speech.data.gmm_names[cat]+'_'+str(self.g)+'.p'):
                self.segment_session_numbers['weekend'][cat] = int(sum(self.speech_config.speech.pz['weekend'][self.g].loc[inds, cat+self.speech_config.speech.data.zkey_weekend]))
            else:
                self.segment_session_numbers['weekend'][cat] = 0

    def load_gmms(self):
        """Loads the GMM model object for each of the segments."""

        for cat in self.speech_config.speech.data.categories:
            weekdaykeys = ['weekday', 'weekend']
            for weekday in weekdaykeys:
                if self.segment_session_numbers[weekday][cat] > 0:
                    key = self.speech_config.speech.data.folder+'GMMs/'+weekday+'_'+self.speech_config.speech.data.gmm_names[cat]+'_'+str(self.g)+'.p'
                    self.segment_gmms[weekday][cat] = pickle.load(open(key, "rb"))

class LoadProfile(object):
    """Calculating the load profile for a given configuration.

    Attributes:
         config: configuration, object from the class SPEEChGeneralConfiguration
         group_config: group configuration, object from the class SPEEChGroupConfiguration
         weekday: weekday option
         load_segments_dict: results, dictionary of load segments
         load_segments_array: results, array of load segments

    Methods:
         calculate_load: calculate the total load for each segment
         end_times_and_load: calculate the uncontrolled load profiles for a set of sessions
    """

    def __init__(self, config, group_config, weekday='weekday'):
        """Other option for weekday: 'weekend'."""

        self.config = config  # speech general configuration
        self.group_config = group_config
        self.weekday = weekday
        self.load_segments_dict = {}
        self.load_segments_array = np.zeros((self.config.num_time_steps, self.config.speech.data.num_categories))

    def calculate_load(self, return_individual_session_parameters=False):
        """For each segment, calculate the total load profile.
        For each segment, the process is as follows:
            Step 1: generate sessions parameters using the segment GMM, stored in full_output
            Step 2: post-process parameters
            Step 3: calculate sessions load profiles, calling end_times_and_load
            Step 4: store the result
        """

        if return_individual_session_parameters:
            individual_session_parameters = {}

        for segment_number in range(self.config.speech.data.num_categories):
            cat = self.config.speech.data.categories[segment_number]
            num_vehicles = self.group_config.segment_session_numbers[self.weekday][cat]
            if num_vehicles > 0:
                gmm = self.group_config.segment_gmms[self.weekday][cat]
                full_output = gmm.sample(num_vehicles)
                output = full_output[0]
                output = output[np.random.choice(np.shape(output)[0], np.shape(output)[0], replace=False), :]
                if self.config.speech.data.start_mod == 1:
                    start_times = (self.config.speech.data.start_time_scaler * np.mod(24*3600*output[:, 0], 24*3600)).astype(int)
                else:
                    start_times = (self.config.speech.data.start_time_scaler * np.mod(output[:, 0], 24*3600)).astype(int)
                energies = np.clip(np.abs(output[:, 1]), 0, self.config.energy_clip)
                end_times, load = self.end_times_and_load(start_times, energies, self.config.speech.data.rates[segment_number])

                if return_individual_session_parameters:
                    individual_session_parameters[cat] = pd.DataFrame({'Start':start_times, 'Energy':np.round(energies,2), 'Duration': (self.config.speech.data.start_time_scaler * np.clip(np.abs(output[:, 2]), 0, 24*3600*7)).astype(int), 'Rate': self.config.speech.data.rates[segment_number] * np.ones(np.shape(start_times))})

            else:
                load = np.zeros((self.config.num_time_steps, ))
            self.load_segments_dict[self.config.speech.data.labels[segment_number]] = load
            self.load_segments_array[:, segment_number] = load

        if return_individual_session_parameters:
            return individual_session_parameters

    def end_times_and_load(self, start_times, energies, rate):
        """Calculate the load profile given data on individual sessions.

        Parameters:
             start_times: set of start time indices
             energies: energy delivered in each session, in kWh
             rate: uncontrolled max charging rate of the session, in kW

         Returns:
             end_times: set of end time indices
             load: total load from the set of sessions, a time series in kW
            """

        time_steps_per_hour = self.config.time_steps_per_hour
        num_time_steps = self.config.num_time_steps
        load = np.zeros((num_time_steps,))
        end_times = np.zeros(np.shape(start_times)).astype(int)

        lengths = (time_steps_per_hour * energies / rate).astype(int)
        extra_charges = energies - lengths * rate / time_steps_per_hour
        inds1 = np.where((start_times + lengths) > num_time_steps)[0]
        inds2 = np.delete(np.arange(0, np.shape(end_times)[0]), inds1)

        end_times[inds1] = (np.minimum(start_times[inds1].astype(int)+lengths[inds1]-num_time_steps, num_time_steps)).astype(int)
        end_times[inds2] = (start_times[inds2] + lengths[inds2]).astype(int)
        inds3 = np.where(end_times >= num_time_steps)[0]
        inds4 = np.delete(np.arange(0, np.shape(end_times)[0]), inds3)

        for i in range(len(inds1)):
            idx = int(inds1[i])
            load[np.arange(int(start_times[idx]), num_time_steps)] += rate * np.ones((num_time_steps - int(start_times[idx]),))
            load[np.arange(0, end_times[idx])] += rate * np.ones((end_times[idx],))
        for i in range(len(inds2)):
            idx = int(inds2[i])
            load[np.arange(int(start_times[idx]), end_times[idx])] += rate * np.ones((lengths[idx],))
        load[0] += np.sum(extra_charges[inds3] * time_steps_per_hour)
        for i in range(len(inds4)):
            load[end_times[int(inds4[i])]] += extra_charges[int(inds4[i])] * time_steps_per_hour

        return end_times, load

class Plotting(object):
    """Plotting class - including final results of load profile, intermediate profiles, and distributions."""

    def __init__(self, speech, config=None, n=5e6):

        self.speech = speech
        if config is None:
            self.config = SPEEChGeneralConfiguration(speech)
            self.config.num_evs(n)
            self.config.groups()
        else:
            self.config = config

    def pg(self, col='pg'):

        vals = np.zeros((self.speech.data.ng, ))
        for i in range(self.speech.data.ng):
            j = self.speech.data.cluster_reorder_dendtoac[i]
            vals[i] = self.speech.pg.loc[j, col]
        plt.figure()
        plt.bar(np.arange(1, self.speech.data.ng+1), vals)
        plt.ylabel('P(G)')
        plt.xlabel('G')
        plt.show()

    def total(self, verbose=False, weekday='weekday', save_str=None):

        self.config.run_all(verbose=verbose, weekday=weekday)
        self.plot_single(self.config.total_load_segments, self.config.total_load_dict, save_str=save_str)

    def groups(self, n=1e5, weekday='weekday', save_string=None):

        nrow = int(np.ceil(np.divide(self.speech.data.ng, 4)))
        fig, axes = plt.subplots(nrow, 4, sharex=True, sharey=True, figsize=(12, int(nrow*3)))
        config = copy.deepcopy(self.config)
        ymax = 0
        for i in range(self.speech.data.ng):
            row = int(np.divide(i, 4))
            col = np.mod(i, 4)
            j = self.speech.data.cluster_reorder_dendtoac[i]
            config.group_configs[j].numbers(total_drivers=n)
            config.group_configs[j].load_gmms()
            model = LoadProfile(config, config.group_configs[j], weekday=weekday)
            model.calculate_load()
            if np.max(np.sum(model.load_segments_array, axis=1)) > ymax:
                ymax = np.max(np.sum(model.load_segments_array, axis=1))
            ylab = False
            if col == 0:
                ylab = True
            axes[row, col] = self.plot(axes[row, col], model.load_segments_array, model.load_segments_dict, title='Group '+str(i), ylab=ylab)
        if (row == nrow) & (col < 3):
            for col_left in np.arange(col, 4):
                axes[row, col_left].set_axis_off()
        for i in range(nrow):
            for j in range(4):
                axes[row, col].set_ylim([0, (1/1000)*ymax])
        plt.tight_layout()
        if save_string is not None:
            plt.savefig(save_string, bbox_inches='tight')
        plt.show()

    def sessions_components(self, g, cat, weekday, n=1e5):

        gmm = self.config.group_configs[g].segment_gmms[weekday][cat]
        output = gmm.sample(n)
        output_values = output[0]
        output_labels = output[1]

        fig, ax = plt.subplots(1,1,figsize=(8,5))
        inds = np.arange(0, np.shape(output_values)[0])
        if self.speech.data.start_mod == 1:
            start_times = (self.speech.data.start_time_scaler * np.mod(24*3600*output_values[inds, 0], 24*3600)).astype(int)
        else:
            start_times = (self.speech.data.start_time_scaler * np.mod(output_values[inds, 0], 24*3600)).astype(int)
        energies = np.clip(np.abs(output_values[inds, 1]), 0, self.config.energy_clip)
        segment_number = np.where(np.array(self.speech.data.categories)==cat)[0][0]
        end_times, load = self.end_times_and_load(start_times, energies, self.speech.data.rates[segment_number])
        load_segments_array = np.zeros((self.config.num_time_steps, self.speech.data.num_categories))
        load_segments_array[:, segment_number] = load
        load_segments_dict = {self.speech.data.labels[segment_number]:load}
        ax = self.plot(ax, load_segments_array, load_segments_dict, 'Total')
        plt.tight_layout()
        plt.show()

        nc = gmm.n_components
        fig, axes = plt.subplots(1, nc, sharex=True, sharey=True, figsize=(3*nc, 3))
        ymax = 0
        for i in range(nc):
            inds = np.where(output_labels == i)[0]
            if self.speech.data.start_mod == 1:
                start_times = (self.speech.data.start_time_scaler * np.mod(24*3600*output_values[inds, 0], 24*3600)).astype(int)
            else:
                start_times = (self.speech.data.start_time_scaler * np.mod(output_values[inds, 0], 24*3600)).astype(int)
            energies = np.clip(np.abs(output_values[inds, 1]), 0, self.config.energy_clip)
            segment_number = np.where(np.array(self.speech.data.categories)==cat)[0][0]
            end_times, load = self.end_times_and_load(start_times, energies, self.speech.data.rates[segment_number])
            load_segments_array = np.zeros((self.config.num_time_steps, self.speech.data.num_categories))
            load_segments_array[:, segment_number] = load
            if np.max(load) > ymax:
                ymax = np.max(load)
            load_segments_dict = {self.speech.data.labels[segment_number]:load}
            axes[i] = self.plot(axes[i], load_segments_array, load_segments_dict, 'Weight: '+str(np.round(gmm.weights_[i], 2)))
        for i in range(nc):
            axes[i].set_ylim([0, (1/1000)*ymax])
        plt.tight_layout()
        plt.show()

    def plot(self, ax, load_segments_array, load_segments_dict, title, ylab=False):

        x = (1/60)*np.arange(0, 1440)
        mark = np.zeros(np.shape(x))
        scaling = 1 / 1000
        unit = 'MW'
        if np.max(scaling * load_segments_array) > 1000:
            scaling = (1 / 1000) * (1 / 1000)
            unit = 'GW'
        for key, val in load_segments_dict.items():
            ax.plot(x, scaling * (mark + val), label=key, color=self.speech.data.colours[key])
            ax.fill_between(x, scaling * mark, scaling * (mark + val), color=self.speech.data.colours[key])
            mark += val
        ax.plot(x, scaling * mark, 'k')
        ax.set_xlim([0, np.max(x)])
        if ylab:
            ax.set_ylabel(unit, fontsize=14)
        ax.set_xlabel('Hour', fontsize=14)
        ax.set_title(title, fontsize=14)
        ax.set_ylim(bottom=0)

        return ax

    def end_times_and_load(self, start_times, energies, rate):

        time_steps_per_hour = self.config.time_steps_per_hour
        num_time_steps = self.config.num_time_steps
        load = np.zeros((num_time_steps,))
        end_times = np.zeros(np.shape(start_times)).astype(int)

        lengths = (time_steps_per_hour * energies / rate).astype(int)
        extra_charges = energies - lengths * rate / time_steps_per_hour
        inds1 = np.where((start_times + lengths) > num_time_steps)[0]
        inds2 = np.delete(np.arange(0, np.shape(end_times)[0]), inds1)

        end_times[inds1] = (np.minimum(start_times[inds1].astype(int)+lengths[inds1]-num_time_steps, num_time_steps)).astype(int)
        end_times[inds2] = (start_times[inds2] + lengths[inds2]).astype(int)
        inds3 = np.where(end_times >= num_time_steps)[0]
        inds4 = np.delete(np.arange(0, np.shape(end_times)[0]), inds3)

        for i in range(len(inds1)):
            idx = int(inds1[i])
            load[np.arange(int(start_times[idx]), num_time_steps)] += rate * np.ones((num_time_steps - int(start_times[idx]),))
            load[np.arange(0, end_times[idx])] += rate * np.ones((end_times[idx],))
        for i in range(len(inds2)):
            idx = int(inds2[i])
            load[np.arange(int(start_times[idx]), end_times[idx])] += rate * np.ones((lengths[idx],))
        load[0] += np.sum(extra_charges[inds3] * time_steps_per_hour)
        for i in range(len(inds4)):
            load[end_times[int(inds4[i])]] += extra_charges[int(inds4[i])] * time_steps_per_hour

        return end_times, load

    def plot_single(self, load_segments_array, load_segments_dict, legend_subset=None, set_ylim=None, save_str=None, title=None):

        x = (1/60)*np.arange(0, 1440)
        mark = np.zeros(np.shape(x))
        scaling = 1 / 1000
        unit = 'MW'
        if np.max(scaling * np.sum(load_segments_array, axis=1)) > 1000:
            scaling = (1 / 1000) * (1 / 1000)
            unit = 'GW'
        plt.figure(figsize=(8, 5))
        for key, val in load_segments_dict.items():
            plt.plot(x, scaling * (mark + val), color=self.speech.data.colours[key])
            if legend_subset is not None:
                if key in legend_subset:
                    plt.fill_between(x, scaling * mark, scaling * (mark + val), label=key, color=self.speech.data.colours[key])
                else:
                    plt.fill_between(x, scaling * mark, scaling * (mark + val), color=self.speech.data.colours[key])
            else:
                plt.fill_between(x, scaling * mark, scaling * (mark + val), label=key, color=self.speech.data.colours[key])
            mark += val
        plt.plot(x, scaling * mark, 'k')
        plt.legend(fontsize=12, loc='upper left')
        plt.xlim([0, np.max(x)])
        if set_ylim is None:
            plt.ylim([0, 1.1 * np.max(scaling * mark)])
        else:
            plt.ylim([0, set_ylim])
        plt.ylabel(unit, fontsize=14)
        plt.xlabel('Hour', fontsize=14)
        plt.xticks(fontsize=14)
        plt.yticks(fontsize=14)
        if title is not None:
            plt.set_title(title)
        if save_str is not None:
            plt.tight_layout()
            plt.savefig(save_str, bbox_inches='tight')
        plt.show()

    def plot_single_above_base(self, load_segments_array, load_segments_dict, base_load, legend_subset=None, set_ylim=None, save_str=None, title=None):

        x = (1/60)*np.arange(0, 1440)
        if np.shape(base_load)[0] == 24:
            xold = np.arange(0, 25)
            yold = np.zeros((25,))
            yold[np.arange(0, 24)] = np.copy(base_load)
            yold[-1] = yold[0]
            f2 = interp1d(xold, yold, kind='cubic')
            base_load = f2(x)
        elif np.shape(base_load)[0] == 288:
            xold = (1/12)*np.arange(0, 289)
            yold = np.zeros((289,))
            yold[np.arange(0, 288)] = np.copy(base_load)
            yold[-1] = yold[0]
            f2 = interp1d(xold, yold, kind='cubic')
            base_load = f2(x)

        # mark = np.zeros(np.shape(x))
        scaling = 1 / 1000
        unit = 'MW'
        if np.max(scaling * (base_load + np.sum(load_segments_array, axis=1))) > 1000:
            scaling = (1 / 1000) * (1 / 1000)
            unit = 'GW'
        plt.figure(figsize=(8, 5))
        plt.plot(x, scaling*base_load, color='k', alpha=0.8)
        plt.fill_between(x, 0, scaling*base_load, color='k', alpha=0.4, label='Base Load')
        mark = base_load
        for key, val in load_segments_dict.items():
            plt.plot(x, scaling * (mark + val), color=self.speech.data.colours[key])
            if legend_subset is not None:
                if key in legend_subset:
                    plt.fill_between(x, scaling * mark, scaling * (mark + val), label=key, color=self.speech.data.colours[key])
                else:
                    plt.fill_between(x, scaling * mark, scaling * (mark + val), color=self.speech.data.colours[key])
            else:
                plt.fill_between(x, scaling * mark, scaling * (mark + val), label=key, color=self.speech.data.colours[key])
            mark += val
        plt.plot(x, scaling * mark, 'k')
        plt.legend(fontsize=12, loc='lower left', ncol=2)
        plt.xlim([0, np.max(x)])
        if set_ylim is None:
            plt.ylim([0, 1.1 * np.max(scaling * mark)])
        else:
            plt.ylim([0, set_ylim])
        plt.ylabel(unit, fontsize=14)
        plt.xlabel('Hour', fontsize=14)
        plt.xticks(fontsize=14)
        plt.yticks(fontsize=14)
        if title is not None:
            plt.set_title(title)
        if save_str is not None:
            plt.tight_layout()
            plt.savefig(save_str, bbox_inches='tight')
        plt.show()

class Scenarios(object):
    """The scenarios for P(G) based on those in the publication. For use by the user interface tool."""

    def __init__(self, scenario_name):

        self.scenario_name = scenario_name
        self.new_weights = {}

        if scenario_name == 'BaseCase':
            self.base_case()
        elif scenario_name == 'Workplace':
            self.workplace()
        elif scenario_name == 'WorkplaceLargeBattery':
            self.workplace_large_batt()
        elif scenario_name == 'HighMUD':
            self.high_mud()
        else:
            self.original_data()

    def base_case(self):

        self.new_weights = {11: 0.05676, 12: 0.02521, 5: 0.04375, 1: 0.24707, 13: 0.14994, 3: 0.206, 8: 0.00727, 15: 0.0077,
                            2: 0.05344, 9: 0.067, 7: 0.01592, 14: 0.00891, 4: 0.0195, 6: 0.00762, 10: 0.0239, 0: 0.06004}

    def workplace(self):

        self.new_weights = {11: 0.086, 12: 0.03819, 5: 0.06628, 1: 0.18438, 13: 0.11189, 3: 0.15373, 8: 0.01102, 15: 0.01167,
                            2: 0.08096, 9: 0.05, 7: 0.02412, 14: 0.0135, 4: 0.02955, 6: 0.01154, 10: 0.03621, 0: 0.09096}

    def workplace_large_batt(self):

        self.new_weights = {11: 0.0, 12: 0.0, 5: 0.19047, 1: 0.18438, 13: 0.11189, 3: 0.15373, 8: 0.02891, 15: 0.03061,
                            2: 0.2124, 9: 0.05, 7: 0.02412, 14: 0.0135, 4: 0.0, 6: 0.0, 10: 0.0, 0: 0.0}

    def high_mud(self):

        self.new_weights = {11: 0.0172, 12: 0.00764, 5: 0.01326, 1: 0.24584, 13: 0.14919, 3: 0.20497, 8: 0.0022, 15: 0.00233,
                            2: 0.01619, 9: 0.3, 7: 0.00482, 14: 0.0027, 4: 0.00591, 6: 0.00231, 10: 0.00724, 0: 0.01819}

    def original_data(self):

        self.new_weights = {11: 0.16336, 12: 0.07255, 5: 0.12591, 1: 0.01981, 13: 0.01202, 3: 0.01652, 8: 0.02093, 15: 0.02216,
                            2: 0.1538, 9: 0.00186, 7: 0.04581, 14: 0.02564, 4: 0.05613, 6: 0.02193, 10: 0.06878, 0: 0.17279}

max_stars_repo_name: SiobhanPowell/speech

max_issues_repo_name: SiobhanPowell/speech

max_forks_repo_name: SiobhanPowell/speech


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Mar 10 05:06:07 2020

@author: gaGzy
"""
'''WORKING GAME ENGINE'''

from tkinter import *
import numpy as np

class TicTacToe:

    #initializing game area
    #The game area is a 3x3 empty matrix filled with nan.

    '''Initialising necessary attributes'''
    def __init__(self):
        self.p1=0
        self.p2=1
        self.draw=False
        self.gameArea=np.empty((3,3))
        self.gameArea.fill(None)
        self.buttonVal={"button1":'00',"button2":'01',"button3":'02',"button4":'10',\
                   "button5":'11',"button6":'12',"button7":'20',"button8":'21',\
                   "button9":'22'}
        self.pos=None
        self.inputChoices=['00','01','02','10','11','12','20','21','22']
        self.biasChoices=['00','02','20','22']

    def screen(self):

        '''goes through either rows or colums or diagonals to see if win'''
        for j in range(0,3):
            if self.gameArea[j,0] == self.gameArea[j,1] and self.gameArea[j,1] == self.gameArea[j,2]:
                if self.gameArea[j,1]==0:
                    return [True,"0"]
                else:
                    return[True,"1"]

        for j in range(0,3):
            if self.gameArea[0,j] == self.gameArea[2,j] and self.gameArea[1,j] == self.gameArea[2,j]:
                if self.gameArea[2,j]==0:
                    return [True,"0"]
                else:
                    return[True,"1"]

        if self.gameArea[0,0] == self.gameArea[2,2] and self.gameArea[1,1] == self.gameArea[0,0]:
                if self.gameArea[0,0]==0:
                    return [True,"0"]
                else:
                    return[True,"1"]

        if self.gameArea[2,0] == self.gameArea[1,1] and self.gameArea[2,0] == self.gameArea[0,2]:
                if self.gameArea[2,0]==0:
                    return [True,"0"]
                else:
                    return[True,"1"]
        return [False,"Draw"]

    def checkWin(self):
        '''checks if the game is won by 8cases anyone or DRAW'''
        decision=self.screen()
#        print(decision)
        if decision[0] ==True:
#            print(decision[1], "won the game")
            return [True,int(decision[1])]

        elif np.isnan(self.gameArea).any() == False:
                #GAME IS DRAW
            self.draw=True
#                print("the Game is draw")
            return [True,'2']
        else:
            return [False,'']
#============================================================================

class playTicTacToe(TicTacToe):

    def __init__(self,*args): #OVERlOADED CONSTRUCTOR
        '''Note:it isn't possible to directly overload constructor in python like Java
        thus this modified approach is used'''
        super().__init__()
        if len(args)>0:
            self.gameArea=args[0]

    '''plays the game'''
    def play(self):
        ''' Loop iterates until game is won/loss/draw
        else keep playing'''
        player=0
        winStatus=self.checkWin()

        while not(winStatus[0]):
            print("the win condition is ",self.checkWin())
            #inputs value into gameArea
            didhe=self.prompt(player)
            print('if he made a valid move:',didhe)
            if didhe==True:
                #change player
                player=int(not(player))
#                if self.checkWin()==True:
        print(self.gameArea)

    '''inputs value 1/0 into the game Area'''
    def prompt(self,player,inputval=None):#new modified to take clicks as well
        '''Asks for a position'''
        if player==0:
            playerValue=self.p1
        else :
            playerValue=self.p2

        if inputval==None :
            self.pos=input(str("{} make a move:".format(int(player)))) #enter 33
#        print(pos,type(pos))
            if(len(self.pos)>2):
                print("enter valid position")
                return False
        else:
            self.pos=inputval

        if np.isnan(self.gameArea[int(self.pos[0]),int(self.pos[1])]):
            self.gameArea[int(self.pos[0]),int(self.pos[1])]=playerValue
            return True

        elif self.gameArea[int(self.pos[0]),int(self.pos[1])] == 1 or self.gameArea[int(self.pos[0]),int(self.pos[1])]==0:
            print('Error:the area is filled already')
            return False

max_stars_repo_name: theLamentingGirl/AI-tictactoe

max_issues_repo_name: theLamentingGirl/AI-tictactoe

max_forks_repo_name: theLamentingGirl/AI-tictactoe


text:
import numpy as np

def sort_states_for_snake_id(state, snake_id):
    '''
    Given states of shape (m, n, s+1) where m and n is the dimension of the map
    and s+1 is the number of snakes + 1 for food, sort the state depth so that the "self" snake
    is at index 1 and other snakes will be at index 2

    Params:
    -------
    state: np.array(m, n, s)
    snake_id: int
        ID of the state to be placed at index 1
    '''
    food_state = state[:, :, 0]
    self_state = state[:, :, snake_id]

    other_states = []
    for i in range(1, state.shape[2]):
        if i == snake_id:
            continue
        other_states.append(state[:, :, i])

    other_states = np.stack(other_states, axis=2)

    output_states = np.zeros(shape=(state.shape[0],
                                    state.shape[1],
                                    3))
    output_states[:, :, 0] = food_state
    output_states[:, :, 1] = self_state

    if other_states[0, 0, 0] == -1: # if states are bordered
        output_states[:, :, 2] = -1

        # Find all values excluding the border, assuming borders are -1
        other_states = np.sum(other_states, axis=2)
        other_states[other_states<0] = -1

        output_states[:, :, 2] = other_states
    else:
        output_states[:, :, 2] = np.sum(other_states, axis=2)

    return output_states

max_issues_repo_name: IronOnet/sagemaker-battlesnake-ai

max_forks_repo_name: IronOnet/sagemaker-battlesnake-ai


text:
"""
get_scattered_chunks.py

Definition of the `get_scattered_chunks()` function, used to generate
a subsample of rows for manual review of the dataset.
"""

import numpy as np
import pandas as pd

def get_scattered_chunks(
    data: pd.DataFrame, n_chunks: int = 5, chunk_size: int = 3
) -> pd.DataFrame:
    """
    Returns a subsample of equally scattered chunks of rows.

    Parameters
    ----------
    data : pd.DataFrame
        Input dataset
    n_chunks : int
        Number of chunks to collect for the subsample
    chunk_size : int
        Number of rows to include in each chunk

    Returns
    -------
    pd.DataFrame
        Subsample data
    """

    endpoint = len(data) - chunk_size
    sample_indices = np.linspace(0, endpoint, n_chunks, dtype=int)
    sample_indices = [
        index for i in sample_indices for index in range(i, i + chunk_size)
    ]
    return data.iloc[sample_indices, :]


text:
# Copyright 2020-2022 OpenDR European Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
import cv2
import time
import pandas as pd
import torch
from typing import Dict
import os
SELECTED_GPUS = [0]
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join([str(gpu_number) for gpu_number in SELECTED_GPUS])

# opendr imports
import argparse
from opendr.perception.facial_expression_recognition import ProgressiveSpatioTemporalBLNLearner
#from opendr.perception.facial_expression_recognition.landmark_based_facial_expression_recognition.\
#    algorithm.datasets.landmark_extractor import landmark_extractor
#from opendr.perception.facial_expression_recognition.landmark_based_facial_expression_recognition.\
#    algorithm.datasets.gen_facial_muscles_data import gen_muscle_data

def tile(a, dim, n_tile):
    a = torch.from_numpy(a)
    init_dim = a.size(dim)
    repeat_idx = [1] * a.dim()
    repeat_idx[dim] = n_tile
    a = a.repeat(*repeat_idx)
    order_index = torch.LongTensor(np.concatenate([init_dim * np.arange(n_tile) + i for i in range(init_dim)]))
    tiled = torch.index_select(a, dim, order_index)
    return tiled.numpy()

def data_normalization(data):
    data = torch.from_numpy(data)
    N, V, C, T, M = data.size()
    data = data.permute(0, 2, 3, 1, 4).contiguous().view(N, C, T, V, M)
    # remove the first 17 points
    data = data[:, :, :, 17:, :]
    N, C, T, V, M = data.size()
    # normalization
    for n in range(N):
        for t in range(T):
            for v in range(V):
                data[n, :, t, v, :] = data[n, :, t, v, :] - data[n, :, t, 16, :]
    return data.numpy()

class VideoReader(object):
    def __init__(self, file_name):
        self.file_name = file_name
        try:  # OpenCV needs int to read from webcam
            self.file_name = int(file_name)
        except ValueError:
            pass

    def __iter__(self):
        self.cap = cv2.VideoCapture(self.file_name)
        if not self.cap.isOpened():
            raise IOError('Video {} cannot be opened'.format(self.file_name))
        return self

    def __next__(self):
        was_read, img = self.cap.read()
        if not was_read:
            raise StopIteration
        return img

def tile(a, dim, n_tile):
    a = torch.from_numpy(a)
    init_dim = a.size(dim)
    repeat_idx = [1] * a.dim()
    repeat_idx[dim] = n_tile
    a = a.repeat(*repeat_idx)
    order_index = torch.LongTensor(np.concatenate([init_dim * np.arange(n_tile) + i for i in range(init_dim)]))
    tiled_a = torch.index_select(a, dim, order_index)
    return tiled_a.numpy()

def lndmrk2numpy(args, num_current_frames, landmarks_list, num_frames):
    C = 3
    T = num_frames
    V = 68
    data_numpy = np.zeros((1, V, 2, num_current_frames, 1))
    seq = np.zeros((1, V, 2, T, 1))
    for t in range(num_current_frames):
        data_numpy[0, :, :, t, 0] = landmarks_list[t][:, :]

    # if we have less than num_frames, repeat frames to reach num_frames
    diff = T - num_current_frames
    if diff == 0:
        seq = data_numpy
    while diff > 0:
        num_tiles = int(diff / num_current_frames)
        if num_tiles > 0:
            data_numpy = tile(data_numpy, 3, num_tiles+1)
            num_current_frames = data_numpy.shape[3]
            diff = T - num_current_frames
        elif num_tiles == 0:
            seq[:, :, :, :num_current_frames, :] = data_numpy
            for j in range(diff):
                seq[:, :, :, num_current_frames+j, :] = data_numpy[:, :, :, -1, :]
            break
    return seq

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--device", help="Device to use (cpu, cuda)", type=str, default="cuda")
    parser.add_argument('--video', default=0,
                        help='path to video file or camera id')
    parser.add_argument('--model_name', type=str, default='pstbln_afew',
                        help='action detection method')
    parser.add_argument('--num_frames', type=int, default=300,
                        help='number of frames to be processed for each action')
    parser.add_argument('--fps', type=int, default=30,
                        help='number of frames per second to be processed by pose estimator and action detector')

    args = parser.parse_args()
    device = args.device
    if args.model_name == 'pstbln_ck+':
        num_point = 303
        num_class = 7
        num_frames = 5
    elif args.model_name == 'pstbln_casia':
        num_point = 309
        num_class = 6
        num_frames = 5
    elif args.model_name == 'pstbln_afew':
        num_point = 312
        num_class = 7
        num_frames = 150

    expression_classifier = ProgressiveSpatioTemporalBLNLearner(device="cuda", dataset_name='afew',
                                                                num_class=num_class,
                                                                num_point=num_point, num_person=1, in_channels=2,
                                                                blocksize=5, topology=[15, 10, 15, 5, 5, 10])

    expression_classifier.init_model()

    dev = f"cuda:0"
    torch.cuda.reset_peak_memory_stats(device=dev)

    '''image_provider = VideoReader('./1.mp4')  # loading a video or get the camera id 0

    counter, avg_fps = 0, 0
    landmarks_list = []
    window = int(30/args.fps)
    f_ind = 0
    for img in image_provider:
        if f_ind % window == 0:
            start_time = time.perf_counter()
            cv2.imwrite(os.path.join('./frames', str(counter) + ".jpg"), img)
            landmarks = landmark_extractor(os.path.join('./frames', str(counter) + ".jpg"), './temp', './shape_predictor_68_face_landmarks.dat')
            counter += 1
            print(counter)
            landmarks_list.append(landmarks)
            if counter > num_frames:
                landmarks_list.pop(0)
                counter = num_frames
            if counter > 0:
                seq = lndmrk2numpy(args, counter, landmarks_list, num_frames)
                norm_data = data_normalization(seq)
            muscle_data = np.zeros((1, 2, num_frames, num_point, 1)) #gen_muscle_data(norm_data, './temp')
            prediction = expression_classifier.infer(muscle_data, monte_carlo_dropout=False)
            end_time = time.perf_counter()
            fps = 1.0 / (end_time - start_time)
            avg_fps = 0.8 * fps + 0.2 * fps
        f_ind += 1
        key = cv2.waitKey(1)
        if key == ord('q'):
            break
    print("Average inference fps: ", avg_fps)
    cv2.destroyAllWindows()'''

    start_time = time.perf_counter()
    muscle_data = np.zeros((1, 2, num_frames, num_point, 1))  # gen_muscle_data(norm_data, './temp')
    prediction = expression_classifier.infer(muscle_data, monte_carlo_dropout=False)
    end_time = time.perf_counter()
    fps = 1.0 / (end_time - start_time)
    avg_fps = 0.8 * fps + 0.2 * fps
    print("Average inference fps: ", avg_fps)

    memory_summary = torch.cuda.memory_summary(device=dev, abbreviated=True)
    stats = {"memory_allocated": torch.cuda.memory_allocated(device=dev), "max_memory_allocated": torch.cuda.max_memory_allocated(device=dev),}
    print("memory_allocated", torch.cuda.memory_allocated(device=dev))
    print("max_memory_allocated", torch.cuda.max_memory_allocated(device=dev))

max_stars_repo_name: negarhdr/opendr

max_issues_repo_name: negarhdr/opendr

max_forks_repo_name: negarhdr/opendr


text:
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from auto_scan_test import PassAutoScanTest, IgnoreReasons
from program_config import TensorConfig, ProgramConfig, OpConfig
import numpy as np
import paddle.inference as paddle_infer
from functools import partial
from typing import Optional, List, Callable, Dict, Any, Set
import unittest

import hypothesis
from hypothesis import given, settings, seed, example, assume, reproduce_failure
import hypothesis.strategies as st

class TestMapMatmulToMulPass(PassAutoScanTest):
    """
     x_var    y_var(persistable)
       \       /
         matmul
    """

    def sample_predictor_configs(self, program_config):
        # cpu
        config = self.create_inference_config(use_gpu=False)
        yield config, ["mul", ], (1e-5, 1e-5)

        # for gpu
        config = self.create_inference_config(use_gpu=True)
        yield config, ["mul", ], (1e-5, 1e-5)

        # TRT
        # config = self.create_trt_inference_config()
        # config.enable_tensorrt_engine(
        #     max_batch_size=10,
        #     workspace_size=10240,
        #     min_subgraph_size=0,
        #     precision_mode=paddle_infer.PrecisionType.Float32,
        #     use_static=False,
        #     use_calib_mode=False)
        # yield config, ["mul", ], (1e-5, 1e-5)

    def add_ignore_pass_case(self):
        # Here we put some skip rules to avoid known bugs
        def teller1(program_config, predictor_config):
            if predictor_config.use_gpu():
                # On 3080, the results of MatMul and Mul are different
                return True

            if predictor_config.tensorrt_engine_enabled():
                # On 3080, the results of MatMul and Mul are different
                return True

                x_shape = list(program_config.inputs["matmul_x"].shape)
                if len(x_shape) > 5:
                    return True
            return False

        self.add_ignore_check_case(
            teller1, IgnoreReasons.PASS_ACCURACY_ERROR,
            "The pass error on TRT while shape of mul_x > 5.")

    def sample_program_config(self, draw):
        # 1. Generate shape and attr of matmul
        x_shape = draw(
            st.lists(
                st.integers(
                    min_value=1, max_value=8), min_size=2, max_size=5))
        y_shape = draw(
            st.lists(
                st.integers(
                    min_value=1, max_value=8), min_size=2, max_size=2))
        y_shape[0] = x_shape[-1]
        alpha = 1.0
        transpose_X = False
        transpose_Y = False

        matmul_op = OpConfig(
            "matmul",
            inputs={"X": ["matmul_x"],
                    "Y": ["matmul_y"]},
            outputs={"Out": ["matmul_out"]},
            alpha=alpha,
            transpose_X=transpose_X,
            transpose_Y=transpose_Y,
            fused_reshape_X=[],
            fused_reshape_Y=[],
            fused_transpose_X=[],
            fused_transpose_Y=[],
            fused_reshape_Out=[],
            fused_transpose_Out=[], )

        ops = [matmul_op, ]
        weights = {"matmul_y": TensorConfig(shape=y_shape), }
        inputs = {"matmul_x": TensorConfig(shape=x_shape), }
        program_config = ProgramConfig(
            ops=ops,
            weights=weights,
            inputs=inputs,
            outputs=ops[-1].outputs["Out"], )
        return program_config

    def test(self):
        self.run_and_statis(
            quant=False,
            max_examples=100,
            passes=["map_matmul_to_mul_pass"],
            max_duration=180)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: zmxdream/Paddle

max_issues_repo_name: zmxdream/Paddle

max_forks_repo_name: zmxdream/Paddle


text:
# Copyright (c) 2011, Roger Lew [see LICENSE.txt]
# This software is funded in part by NIH Grant P20 RR016454.

import unittest
import warnings
import os
import math
from random import shuffle, random
from collections import Counter,OrderedDict
from dictset import DictSet,_rep_generator
from math import isnan, isinf, floor
import numpy as np
from pprint import pprint as pp

from pyvttbl import PyvtTbl
from pyvttbl import DataFrame
from pyvttbl.plotting import *
from pyvttbl.stats import *
from pyvttbl.misc.support import *

class Test_correlation(unittest.TestCase):
    def test0(self):
        R="""Correlation([\
(('t1', 't2'), {'p': 9.699461194116956e-12, 'r': 0.9577922077922078}), \
(('t1', 't3'), {'p': 2.2594982244811486e-09, 'r': -0.924025974025974}), \
(('t2', 't3'), {'p': 6.850166042119375e-08, 'r': -0.8896103896103895})], \
conditions_list=['t1', 't2', 't3'], coefficient='spearman', N=21)"""

        A=[24,61,59,46,43,44,52,43,58,67,62,57,71,49,54,43,53,57,49,56,33]
        B=[42.93472681237495, 78.87307334936268, 75.37292628918023,
           65.49076317291956, 55.55965179772366, 56.777730638998236,
           62.19451880792437, 54.73710611356715, 72.10021832823149,
           85.94377749485642, 78.2087578930983, 72.01681829338037,
           84.27889316830063, 60.20516982367225, 65.6276497088971,
           62.36549856901088, 69.18772114281175, 67.00548667483324,
           59.042687027269466, 71.99214593063917, 45.00831155783992]
        C=[-53.05540625388731, -96.33996451998567, -92.32465861908086,
           -70.90536432779966, -55.953777697739255, -74.12814626217357,
           -75.89188834814621, -64.24093256012688, -89.62208010083313,
           -87.41075066046812, -80.40932820298143, -77.99906284144805,
           -95.31607277596169, -61.672429800914486, -85.26088499198657,
           -63.4402296673869, -74.84950736563589, -85.00433219746624,
           -71.5901436929124, -76.43243666219388, -48.01082320924727]

        cor=Correlation()
        cor.run([A,B,C],['t1','t2','t3'],coefficient='spearman')
        self.assertEqual(repr(cor),R)

    def test1(self):
        R="""\
Bivariate Correlations

                         A           B           C
======================================================
A   spearman                 1       0.958      -0.924
    Sig (2-tailed)           .   9.699e-12   2.259e-09
    N                       21          21          21
------------------------------------------------------
B   spearman             0.958           1      -0.890
    Sig (2-tailed)   9.699e-12           .   6.850e-08
    N                       21          21          21
------------------------------------------------------
C   spearman            -0.924      -0.890           1
    Sig (2-tailed)   2.259e-09   6.850e-08           .
    N                       21          21          21

Larzelere and Mulaik Significance Testing

 Pair     i   Correlation       P       alpha/(k-i+1)   Sig.
============================================================
A vs. B   1         0.958   9.699e-12           0.017   **
A vs. C   2         0.924   2.259e-09           0.025   **
B vs. C   3         0.890   6.850e-08           0.050   **   """
        df=DataFrame()
        df['A']=[24,61,59,46,43,44,52,43,58,67,62,57,71,49,54,43,53,57,49,56,33]
        df['B']=[42.93472681237495, 78.87307334936268, 75.37292628918023,
                 65.49076317291956, 55.55965179772366, 56.777730638998236,
                 62.19451880792437, 54.73710611356715, 72.10021832823149,
                 85.94377749485642, 78.2087578930983, 72.01681829338037,
                 84.27889316830063, 60.20516982367225, 65.6276497088971,
                 62.36549856901088, 69.18772114281175, 67.00548667483324,
                 59.042687027269466, 71.99214593063917, 45.00831155783992]
        df['C']=[-53.05540625388731, -96.33996451998567, -92.32465861908086,
                 -70.90536432779966, -55.953777697739255, -74.12814626217357,
                 -75.89188834814621, -64.24093256012688, -89.62208010083313,
                 -87.41075066046812, -80.40932820298143, -77.99906284144805,
                 -95.31607277596169, -61.672429800914486, -85.26088499198657,
                 -63.4402296673869, -74.84950736563589, -85.00433219746624,
                 -71.5901436929124, -76.43243666219388, -48.01082320924727]

        cor=df.correlation(['A','B','C'],coefficient='spearman')
        print(cor)
        self.assertEqual(str(cor),R)

    def test2(self):
        df = DataFrame()
        df.read_tbl('data/iqbrainsize.txt', delimiter='\t')
        cor = df.correlation(list(df.keys()))

def suite():
    return unittest.TestSuite((
            unittest.makeSuite(Test_correlation)
                              ))

if __name__ == "__main__":
    # run tests
    runner = unittest.TextTestRunner()
    runner.run(suite())

max_stars_repo_name: yk/pyvttbl

max_issues_repo_name: yk/pyvttbl

max_forks_repo_name: yk/pyvttbl


text:
'''
generate_audio_from_annotations.py

This file will iterate over the development dataset, and extract single-channel audio excerpts based on the annotations.
Options:
- write_file (bool): Actually generate output files and folders
- plot (bool): Plot the annotations for each file (not recommended on the whole dataset)
- debug (bool): Print complementary information on the extraction step (again, not recommended on the whole dataset)

- beamforming_mode ('beam' or 'omni')
-- 'beam' will steer a maxRE (aka maximum directivity) beam towards the source
-- 'omni' will just take the audio from the W channel

- overlap_mode ('ov1', 'ov2' or 'all'):
-- 'ov1' extracts only the audio files with ov1
-- 'ov2' extracts only the audio files with ov2
-- 'all' extracts all files regardless of the overlap amount

All files will be created in an output folder located in `params['dataset_dir']`, in a folder for each sound class.
The output folder will have a name according to the structure: 'oracle_mono_signals_[beamforming_mode]_[overlap_mode]
'''

from APRI.localization_detection import parse_annotations
from baseline import parameter
import os
import soundfile as sf
import numpy as np
import matplotlib.pyplot as plt
from baseline.cls_feature_class import create_folder
from APRI.utils import plot_metadata, get_class_name_dict, get_mono_audio_from_event, Event

# %% CONFIG

write_file = False
plot = False
debug = False

beamforming_mode = 'beam' # 'beam' or 'omni'
overlap_mode = 'all' # 'ov1', 'ov2' or 'all'

# %% DEFINITIONS

params = parameter.get_params()
data_folder_path = os.path.join(params['dataset_dir'], 'foa_dev/') # path to audios
gt_folder_path = os.path.join(params['dataset_dir'], 'metadata_dev') # path to annotations
fs = params['fs']
class_name_dict = get_class_name_dict()

# Main output folder for storing data
output_base_name = 'oracle_mono_signals_' + beamforming_mode + '_' + overlap_mode
output_path = os.path.join(params['dataset_dir'], output_base_name)

occurrences_per_class = np.zeros(params['num_classes'], dtype=int)

# %% ANALYSIS

# Iterate over all audio files
audio_files = [f for f in os.listdir(data_folder_path) if not f.startswith('.')]
# audio_files = ['fold1_room1_mix036_ov2.wav']# TODO REMOVE
# for audio_file_name in [audio_files[0]]:
for audio_file_name in audio_files:

    if (overlap_mode is 'all') or (overlap_mode in audio_file_name):

        print('------------------------')
        print(audio_file_name)

        # Open audio file
        b_format, sr = sf.read(os.path.join(data_folder_path, audio_file_name))

        # Get associated metadata file and load content into memory
        metadata_file_name = os.path.splitext(audio_file_name)[0] + '.csv'
        metadata_file_path = os.path.join(gt_folder_path, metadata_file_name)
        csv = np.loadtxt(open(metadata_file_path, "rb"), delimiter=",")

        # ############################################
        # # Delimite events
        event_list = parse_annotations(csv, debug)

        ############################################
        # Prepare folders
        if write_file:
            create_folder(output_path)
            for class_name in class_name_dict.values():
                folder = os.path.join(output_path, class_name)
                create_folder(folder)

        ############################################
        # Get monophonic estimates of the event, and save into files
        for event_idx, event in enumerate(event_list):

            mono_event = get_mono_audio_from_event(b_format, event, beamforming_mode, fs, params['label_hop_len_s'])

            ######################
            event_occurrence_idx = occurrences_per_class[event.get_classID()]
            mono_file_name = str(event_occurrence_idx) + '.wav'
            class_name = class_name_dict[event.get_classID()]

            ######################
            # write file
            if write_file:
                sf.write(os.path.join(output_path, class_name, mono_file_name), mono_event, sr)
            # increment counter
            occurrences_per_class[event.get_classID()] += 1

        if plot:
            plot_metadata(metadata_file_name)

        if debug:
            frames = []
            for e in event_list:
                frames.append(e.get_frames())
            plt.figure()
            plt.grid()
            for f in range(len(frames)):
                plt.plot(frames[f])