text:
import json
import random
import math

import torch
import numpy as np

class DisDataIter(object):
    def __init__(self, real_data_file, fake_data_file, batch_size):
        super(DisDataIter, self).__init__()
        self.batch_size = batch_size
        real_data = self.read_file(real_data_file)
        fake_data = self.read_file(fake_data_file)
        # self.src_data = real_data[0] + fake_data[0]
        # self.tgt_data = real_data[1] + fake_data[1]
        # self.labels = [random.uniform(0.7, 1.2) for _ in range(len(real_data[1]))] \
        #     + [random.uniform(0., .3) for _ in range(len(fake_data[1]))]
        # self.pairs = list(zip(self.src_data, self.tgt_data, self.labels))
        # self.data_num = len(self.pairs)
        # self.indices = range(self.data_num)
        # self.num_batches = int(math.ceil(float(self.data_num)/self.batch_size))
        self.src_real, self.tgt_real = real_data
        self.src_fake, self.tgt_fake = fake_data
        self.real_labels = [1 for _ in range(len(real_data[1]))]
        self.fake_labels = [0 for _ in range(len(fake_data[1]))]
        self.real_pairs = list(zip(self.src_real, self.tgt_real, self.real_labels))
        self.fake_pairs = list(zip(self.src_fake, self.tgt_fake, self.fake_labels))
        self.real_num = len(self.real_pairs)
        self.fake_num = len(self.fake_pairs)
        self.data_num = self.real_num + self.fake_num
        self.real_indices = range(self.real_num)
        self.fake_indices = range(self.fake_num)
        self.num_batches = int(math.ceil(float(self.real_num)/self.batch_size)) + \
            int(math.ceil(float(self.fake_num)/self.batch_size))
        self.real_idx = 0
        self.fake_idx = 0

    def __len__(self):
        return self.num_batches

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def reset(self):
        self.real_idx = 0
        self.fake_idx = 0
        random.shuffle(self.real_pairs)
        random.shuffle(self.fake_pairs)

    def next(self):
        if self.real_idx >= self.real_num and self.fake_idx >= self.fake_num:
            raise StopIteration
        if (random.random() < 0.5 and self.real_idx < self.real_num) \
                or self.fake_idx >= self.fake_num:
            index = self.real_indices[self.real_idx:self.real_idx+self.batch_size]
            pairs = [self.real_pairs[i] for i in index]
            self.real_idx += self.batch_size
        else:
            index = self.fake_indices[self.fake_idx:self.fake_idx+self.batch_size]
            pairs = [self.fake_pairs[i] for i in index]
            self.fake_idx += self.batch_size
        # index = self.indices[self.idx:self.idx+self.batch_size]
        # pairs = [self.pairs[i] for i in index]
        src_data = [p[0] for p in pairs]
        tgt_data = [p[1] for p in pairs]
        label = [p[2] for p in pairs]
        label = torch.LongTensor(np.asarray(label, dtype='int64'))
        # self.idx += self.batch_size
        return src_data, tgt_data, label

    @staticmethod
    def read_file(data_file):
        with open(data_file, "r") as f:
            data = json.load(f)

        """
        [
            {
                "src": List[str]
                "tgt": List[str]
            },
            ...
        ]
        """

        src_data, tgt_data = [], []
        for datum in data:
            src_data.append(datum["src"])
            tgt_data.append(datum["tgt"])

        return src_data, tgt_data

ext: py

max_stars_repo_name: LuxuriaP/editsql

max_issues_repo_name: LuxuriaP/editsql

max_forks_repo_name: LuxuriaP/editsql

reason: import numpy


text:
import numpy as np
import cv2 as cv

from typing import List
from typing_extensions import Literal
from xdog import XDoG
from pathlib import Path

LineArt = List[Literal["xdog", "pencil", "digital", "blend"]]

class BasicProtocol:
    def __init__(self):
        pass

    def __repr__(self):
        return f"{self.__class__.__name__} will be processed."

    def exec(self, thing):
        NotImplementedError

    def __call__(self, thing):
        return self.exec(thing)

class RandomProtocol(BasicProtocol):
    def __init__(self):
        pass

    def __call__(self, img: np.array) -> np.array:
        if np.random.randint(2):
            return self.exec(img)

        else:
            return img

class AddIntensity(RandomProtocol):
    def __init__(self, intensity=1.7):
        self.intensity = intensity

    def exec(self, img):
        const = 255.0 ** (1.0 - self.intensity)
        img = (const * (img ** self.intensity))

        return img

class Morphology(RandomProtocol):
    def __init__(self):
        self.method = ["erode", "dilate"]

    def exec(self, img):
        method = np.random.choice(self.method)

        if method == "dilate":
            img = cv.dilate(img, (5, 5), iterations=1)
        elif method == "erode":
            img = cv.erode(img, (5, 5), iterations=1)

        return img

class ColorVariant(RandomProtocol):
    def __init__(self, max_value=30, thre=200):
        self.mv = max_value
        self.thre = thre

    def exec(self, img):
        value = np.random.randint(self.mv + 1)
        img[img < self.thre] = value

        return img

class LineSelector(BasicProtocol):
    def __init__(self,
                 sketch_path: Path,
                 line_method: LineArt,
                 blend=0.5):
        self.line_method = line_method
        self.blend = 0.5
        self.pre_intensity = AddIntensity(intensity=1.4)
        self.post_intensity = AddIntensity(intensity=(1/1.5))
        self.sketch_path = sketch_path

        self.xdog_process = XDoG()

        self._message(line_method)

    @staticmethod
    def _message(line_method: LineArt):
        print("Considering these line extraction methods...")
        for method in line_method:
            if method == "xdog":
                print("XDoG will be implemented.")
            elif method == "pencil":
                print("SketchKeras will be implemented.")
            elif method == "blend":
                print("XDoG + SketchKeras will be implemented.")
            elif method == "digital":
                print("Sketch Simplification will be implemented.")
        print("\n")

    def _xdog_preprocess(self, path: Path) -> np.array:
        img = self.xdog_process(str(path))
        img = (img * 255.0).reshape(img.shape[0], img.shape[1], 1)
        img = np.tile(img, (1, 1, 3))

        return img

    def _pencil_preprocess(self, path: Path) -> np.array:
        filename = path.name
        line_path = self.sketch_path / Path(filename)
        img = cv.imread(str(line_path))

        return img

    def _blend_preprocess(self, path: Path, blend=0.5) -> np.array:
        xdog_line = self._xdog_preprocess(path)
        penc_line = self._pencil_preprocess(path)
        penc_line = self.pre_intensity.exec(penc_line)

        xdog_blur = cv.GaussianBlur(xdog_line, (5, 5), 1)
        xdog_blur = cv.addWeighted(xdog_blur, 0.75, xdog_line, 0.25, 0)

        blend = cv.addWeighted(xdog_blur, blend, penc_line, (1 - blend), 0)

        return self.post_intensity.exec(blend)

    def exec(self, path: Path) -> np.array:
        method = np.random.choice(self.line_method)

        if method == "xdog":
            img = self._xdog_preprocess(path)
        elif method == "pencil":
            img = self._pencil_preprocess(path)
        elif method == "blend":
            img = self._blend_preprocess(path, self.blend)

        return img

class LineProcessor:
    def __init__(self, sketch_path: Path, line_method: LineArt):
        self.process_list = [
            LineSelector(sketch_path, line_method),
            Morphology(),
            ColorVariant()
        ]

        self._message(self.process_list)

    @staticmethod
    def _message(process_list: List):
        print("Considering these augmentations for line arts...")
        for index, process in enumerate(process_list):
            if index > 0:
                print(process)
        print("\n")

    def __call__(self, x: Path) -> np.array:
        for process in self.process_list:
            x = process(x)

        return x

max_stars_repo_name: Pandinosaurus/Colorization

max_issues_repo_name: Pandinosaurus/Colorization

max_forks_repo_name: Pandinosaurus/Colorization


text:
import numpy as np

#input file
fin = open("times.csv", "rt")
#output file to write the result to
fout = open("loads.csv", "wt")

fout.write('timepoint, date, system_load' + '\n')

#for each line in the input file
for line in fin:
	# read in data like a meathead
	timepoint = line
	year = line[0:4]
	month = line[5:7]
	day = line[8:10]
	hour = line[11:13]
	minute = line[14:16]
	# debug:
	#print(year + month + day + ' ' + hour +':' + minute)

	# random number/load generation
	rng = np.random.default_rng()
	rfloat = rng.random()
	cos_meathead = [0.4,0.4,0.5,0.6,0.5,0.4,0.3,0.5,0.9,1,0.85,0.6]
	time_index = int(hour)
	#print(rfloat)

	# estimate of load characteristics based on min/max of 2020 est. load data
	load = 550 + 700 * rfloat * cos_meathead[int(time_index/2)]

	datepoint_NBC = year + '-' + month + '-' + day
	textline = ','.join(['Oahu',timepoint[:-1],datepoint_NBC,str(load)])

	fout.write(textline)
	fout.write('\n')

	#read replace the string and write to output file

#close input and output files
fin.close()
fout.close()


text:
from __future__ import division, absolute_import

import pytest
from sklearn.datasets import load_iris
import numpy as np

from coresets import *

class TestSensitivity(object):

    @pytest.fixture
    def gen_data(self):
        X, _ = load_iris(return_X_y=True)
        centers = X[np.random.choice(np.arange(X.shape[0]), size=5)]
        return X, centers

    def test_kmeans_sensitivity(self, gen_data):
        # test kmeans sensitivity
        # make sure that the efficient C++ implementation gives the same results as Python
        X, centers = gen_data
        alpha = 1
        sensitivity = kmeans_sensitivity(X, np.ones(X.shape[0]), centers, alpha)

        # calc sensitivity in Python to check Cython impl
        dists = np.zeros((X.shape[0], centers.shape[0]))
        for i, x in enumerate(X):
            for j, c in enumerate(centers):
                dists[i, j] = np.sum((x - c) ** 2)
        assigns = np.argmin(dists, axis=1)
        dists = np.min(dists, axis=1)
        cnts = np.bincount(assigns)
        total = np.mean(dists)
        cluster_tot = np.zeros(centers.shape[0])
        for i, c in enumerate(centers):
            cluster_tot[i] = np.sum(dists[np.where(assigns == i)[0]])
        sensitivity2 = 2 * alpha * dists / total + 4 * alpha / (total * cnts[assigns]) * cluster_tot[assigns] + 4. * \
                       X.shape[0] / cnts[assigns]
        assert np.allclose(sensitivity, sensitivity2)

max_stars_repo_name: zalanborsos/coresets

max_issues_repo_name: zalanborsos/coresets

max_forks_repo_name: zalanborsos/coresets


text:
# gdpyt-analysis: utils: modify
"""
Notes
"""

# imports
import pandas as pd
import numpy as np
from sklearn.neighbors import NearestNeighbors
from utils import bin, io

# scripts

# ---------------------------------   DICTIONARIES (below)  ------------------------------------------------------------

def dficts_rename_key(dficts, new_keys):
    old_keys = list(dficts.keys())

    renamed_dficts = {}
    for ok, nk in zip(old_keys, new_keys):
        renamed_dficts.update({nk: dficts[ok]})

    return renamed_dficts

def dficts_sort(dficts, descending=False):
    keys = list(dficts.keys())

    if descending:
        keys.sort(reverse=True)
    else:
        keys.sort()

    sorted_dficts = {}
    for k in keys:
        sorted_dficts.update({k: dficts[k]})

    return sorted_dficts

def dficts_scale(dficts, columns, multipliers):

    if isinstance(multipliers, (float, int)):
        multipliers = [multipliers]

    if len(multipliers) != len(columns):
        multipliers = np.ones_like(columns, dtype=float) * multipliers

    for name, df in dficts.items():
        for column, multiplier in zip(columns, multipliers):

            if column == 'index':
                df.index = df.index * multiplier
            else:
                df.loc[:, column] *= multiplier

        # update the dictionary
        dficts.update({name: df})

    return dficts

def dficts_new_column(dficts, new_columns, columns, multipliers):

    if isinstance(multipliers, (float, int)):
        multipliers = [multipliers]

    if len(multipliers) != len(columns):
        if len(multipliers) > len(columns):
            iterable_multipliers = iter(multipliers)
        elif len(multipliers) < len(columns):
            multipliers = np.ones_like(columns, dtype=float) * multipliers
            iterable_multipliers = None
        else:
            iterable_multipliers = None
    else:
        iterable_multipliers = None

    for name, df in dficts.items():

        if iterable_multipliers:
            multipliers = [next(iterable_multipliers)]

        for new_column, column, multiplier in zip(new_columns, columns, multipliers):

            # new column == constant
            if column == '':
                df[new_column] = multiplier

            # new column == two columns multiplied together
            elif isinstance(multiplier, str):
                raise ValueError('something may be wrong here')
                df[new_column] = df.loc[:, column] / np.sqrt(df.loc[:, multiplier])

            # new column == column multiplied by constant
            elif column == 'index':
                df[new_column] = df.index * multiplier
            else:
                df[new_column] = df.loc[:, column] * multiplier

        # update the dictionary
        dficts.update({name: df})

    return dficts

def dficts_set_index(dficts, column_to_index):

    for name, df in dficts.items():

        df = df.set_index(keys=column_to_index)

        # update the dictionary
        dficts.update({name: df})

    return dficts

def dficts_shift(dficts, columns, shifts):
    for name, df in dficts.items():
        for column, shift in zip(columns, shifts):

            if column == 'index':
                df.index = df.index + shift
            else:
                df.loc[:, column] += shift

        # update the dictionary
        dficts.update({name: df})

    return dficts

def dficts_flip(dficts, column):
    for name, df in dficts.items():

        if column == 'index':
            df.index = df.index.max() - df.index
        else:
            df.loc[:, column] = df.loc[:, column].max() - df.loc[:, column]
            """df.loc[:, column] *= -1
            df.loc[:, column] += df.loc[:, column].max()"""

        # update the dictionary
        dficts.update({name: df})

    return dficts

def stack_dficts_by_key(dficts, drop_filename=False):
    """
    Merge a dictionary of dataframes into a single dataframe with a new column for dict keys.
    :param dficts:
    :param drop_filename:
    :return:
    """

    dfs = []

    for name, df in dficts.items():

        dfc = df.copy()

        if not drop_filename:
            dfc.insert(0, 'filename', name)

        dfs.append(dfc)

    dfstack = pd.concat(dfs)

    return dfstack

def stack_dficts(dficts, keys):
    """
    Merge a list of dictionaries into a single dictionary with keys 'keys'.

    :param dficts:
    :param keys:
    :return:
    """

    stacked_dficts = {}

    for dfict, key in zip(dficts, keys):

        dfs = []
        for name, df in dfict.items():
            dfs.append(df)

        dfstack = pd.concat(dfs)

        stacked_dficts.update({key: dfstack})

    return stacked_dficts

def split_dficts(dficts, dficts_ground_truth=None, key_to_split=10.0, get_less_than=True, new_keys=None):

    if get_less_than:
        # get dictionary of only gdpyt
        ids = [xk for xk, xv in dficts.items() if xk < key_to_split]
        dfs = [xv for xk, xv in dficts.items() if xk < key_to_split]

        if new_keys:
            dficts_split = {new_keys[i]: dfs[i] for i in range(len(ids))}
        else:
            dficts_split = {ids[i]: dfs[i] for i in range(len(ids))}

        if dficts_ground_truth:
            ids = [xk for xk, xv in dficts_ground_truth.items() if xk < key_to_split]
            dfs = [xv for xk, xv in dficts_ground_truth.items() if xk < key_to_split]

            if new_keys:
                dficts_ground_truth_split = {new_keys[i]: dfs[i] for i in range(len(ids))}
            else:
                dficts_ground_truth_split = {ids[i]: dfs[i] for i in range(len(ids))}

    else:
        # get dictionary of only gdpyt
        ids = [xk for xk, xv in dficts.items() if xk > key_to_split]
        dfs = [xv for xk, xv in dficts.items() if xk > key_to_split]

        if new_keys:
            dficts_split = {new_keys[i]: dfs[i] for i in range(len(ids))}
        else:
            dficts_split = {ids[i]: dfs[i] for i in range(len(ids))}

        if dficts_ground_truth:
            ids = [xk for xk, xv in dficts_ground_truth.items() if xk > key_to_split]
            dfs = [xv for xk, xv in dficts_ground_truth.items() if xk > key_to_split]

            if new_keys:
                dficts_ground_truth_split = {new_keys[i]: dfs[i] for i in range(len(ids))}
            else:
                dficts_ground_truth_split = {ids[i]: dfs[i] for i in range(len(ids))}

    if dficts_ground_truth:
        return dficts_split, dficts_ground_truth_split
    else:
        return dficts_split

def split_dficts_cumulative_series(dficts, dficts_ground_truth, series_parameter='frame', increments=10, key=1):
    """
    Split dficts key='key' into subsets along 'series_parameter' defined by 'increments'.

    :param dficts: dictionary of names and dataframes with test coordinates.
    :param series_parameter: 'frame'
    :param increments: step size to assess convergence.
    :return:
    """
    # assign correct variable names
    if series_parameter == 'frame':
        series_parameter_gt = 'filename'

    # get dataframe
    df = dficts[key]
    df_gt = dficts_ground_truth[key]

    # get length of dataframe along 'series_parameter'
    series_length = len(df[series_parameter].unique())

    # define subset bounds according to 'increments'
    num_increments = series_length / increments
    num_increments = np.ceil(num_increments)
    increment_bounds = np.arange(1, num_increments + 1) * increments

    # split dataframe into subsets along 'series_parameter'
    dfcs = {}
    dfcs_gt = {}

    for bound in increment_bounds:
        name = bound

        # split test coords dataframe
        df_subset = df[df[series_parameter] < bound].copy()
        dfcs.update({name: df_subset})

        # split ground truth dataframe
        df_subset_gt = df_gt[df_gt[series_parameter_gt] < bound].copy()
        dfcs_gt.update({name: df_subset_gt})

    return dfcs, dfcs_gt

# ---------------------------------   DICTIONARIES (above)  ------------------------------------------------------------

# ---------------------------------   DATAFRAMES (below)  --------------------------------------------------------------

def split_df_and_merge_dficts(df, keys, column_to_split, splits, round_to_decimal=0):
    """
    Split a dataframe by column 'column_to_split' along values 'splits' and merge into a single dictionary.

    :param df:
    :param keys:
    :param column_to_split:
    :param splits:
    :param round_to_decimal:
    :return:
    """

    if isinstance(splits, (list, tuple, np.ndarray)):
        dfc = bin.bin_by_list(df, column_to_bin=column_to_split, bins=splits, round_to_decimal=round_to_decimal)
    else:
        raise ValueError("'splits' must be a list of values to split dataframe along column 'column_to_bin'.")

    ks = []
    vs = []
    for key, split in zip(keys, splits):
        df_subset = dfc[dfc['bin'] == split].copy()
        df_subset = df_subset.drop(columns='bin')

        ks.append(key)
        vs.append(df_subset)

    sorted_by_keys = sorted(list(zip(ks, vs)), key=lambda x: x[0])

    ks = [x[0] for x in sorted_by_keys]
    vs = [x[1] for x in sorted_by_keys]

    dficts = dict(zip(ks, vs))

    return dficts

def groupby_stats(df, group_by='id', drop_columns=None):
    """
    Convenience method for calculating dataframe 'group by' quantities.

    :param df:
    :param group_by:
    :param drop_columns:
    :return:
    """
    if 'frame' in df.columns:
        frames = len(df.frame.unique())
    else:
        frames = None

    if drop_columns is not None:
        df = df.copy()
        drop_cols = [d for d in drop_columns if d in df.columns]
        df = df.drop(columns=drop_cols)

    # mean
    df_mean = df.groupby(group_by).mean()

    # standard deviation
    df_std = df.groupby(group_by).std()
    cols_std = {c: c + '_std' for c in df_std.columns}
    df_std = df_std.rename(columns=cols_std)
    # df_std = df_std.multiply(2) - changed on 4/13/22

    # counts
    df_counts = df.groupby(group_by).count()
    df_counts = df_counts.rename(columns={'z': 'z_counts'})
    if frames is None:
        frames = df_counts.z_counts.max()
    df_counts['z_counts_percent'] = df_counts.z_counts / frames * 100
    drop_cols = [d for d in df_counts.columns if d not in ['z_counts', 'z_counts_percent']]
    df_counts = df_counts.drop(columns=drop_cols)

    dfg_stats = pd.concat([df_mean, df_std, df_counts], axis=1, join='inner', sort=False)

    return dfg_stats

def map_values_on_frame_id(dfs, dft):
    """
    Purpose:
        * Problem:
            * columns 'z_est' and 'z_true' in particle similarity curves are not the same as test coords after processing.
            * columns 'frame' and 'id' are identical to test coords.
        * Method:
            1. rename 'z_est' and 'z_true' to 'z_est_raw' and 'z_true_raw'
            2. map 'z_true' and 'z' from test coords to p.s.c. using 'frame' and 'id' columns
    """

    # prepare dfs for mapping
    dfs['z_true_raw'] = dfs['z_true']
    dfs['z_true_map'] = dfs['frame'].astype(str) + '_' + dfs['id'].astype(str)

    dfs['z_est_raw'] = dfs['z_est']
    dfs['z_est_map'] = dfs['frame'].astype(str) + '_' + dfs['id'].astype(str)

    dfs = dfs.astype({'z_true_map': 'str', 'z_est_map': 'str'})

    # prepare dft for mapping
    dft['mapper'] = dft['frame'].astype(str) + '_' + dft['id'].astype(str)
    dft = dft.astype({'mapper': 'str'})

    # drop rows in dfs that aren't in dft
    dfs = dfs[dfs.z_true_map.isin(dft.mapper.values)]

    # create the mapping dictionary
    map_on = dft['mapper'].astype(str).values
    map_z_true_to = dft['z_true'].astype(str).values
    mapping_dict_z_true = {map_on[i]: map_z_true_to[i] for i in range(len(map_on))}

    map_z_to = dft['z'].astype(str).values
    mapping_dict_z = {map_on[i]: map_z_to[i] for i in range(len(map_on))}

    # insert the mapped values
    dfs.loc[:, 'z_true_map'] = dfs.loc[:, 'z_true_map'].map(mapping_dict_z_true)
    dfs.loc[:, 'z_est_map'] = dfs.loc[:, 'z_est_map'].map(mapping_dict_z)

    # change datatype to float
    dfs = dfs.astype({'z_true_map': 'float64', 'z_est_map': 'float64'})
    dfs = dfs.round({'z_true_map': 4, 'z_est_map': 4})
    dfs = dfs.drop(columns=['z_true', 'z_est'])
    dfs = dfs.rename(columns={'z_true_map': 'z_true', 'z_est_map': 'z_est'})

    return dfs

def map_adjacent_z_true(df, df_ground_truth, threshold):
    """
    Map 'z_adj' and 'dz' values to test_coords using NearestNeighbors.
        * 'z_adj' == the z_true value of the adjacent (overlapping) particle.
        * 'dz' == the difference in z (depth) between the two adjacent particles.

    :param df: the raw test_coords output from IDPT analysis.
    :param threshold: for standard dz dataset: threshold = 46
    :return:
    """

    # drop NaNs
    df = df.dropna()

    # initialize list of dataframes
    dfs = []

    # get this frame only
    for fr in df.frame.unique():

        # get this frame only
        dff = df[df['frame'] == fr]

        # test: get ID's and coords
        dff = dff.sort_values('id')
        coords_ids = dff['id'].values
        coords_xy = dff[['x', 'y']].values

        # perform NearestNeighbors
        nneigh = NearestNeighbors(n_neighbors=2, algorithm='ball_tree').fit(coords_xy)
        distances, indices = nneigh.kneighbors(np.array(coords_xy))

        # get only 2nd column
        distances = distances[:, 1]
        indices = indices[:, 1]

        # --- mapping: 2 parts

        # Part 1: map 'z_adj' to all segmented particles (i.e. dx > 7.5)

        # create the mapping of adjacent particle ID's to particle ID's
        mapping_dict = {}
        cids_not_mapped = []

        for distance, idx, cid in zip(distances, indices, coords_ids):
            if distance < threshold:
                mapping_dict.update({cid: dff.id.values[idx.squeeze()]})
            else:
                cids_not_mapped.append([cid, distance, idx])

        # Part 2: map 'z_adj' to non-segmented particles (i.e. dx = 7.5)
        """
        UNDER DEVELOPMENT:

        # ground truth:
        dft = df_ground_truth[df_ground_truth['filename'] == fr]
        dft = dft[dft['x'] < 120]

        for y_pair in dft.y.unique():
            dfty = dft[dft['y'] == y_pair]
            y_pair_z_adj = dfty.iloc[1].z - dfty.iloc[0].z
        true_coords_xy = dft[['x', 'y']].values
        """

        # create column to map to
        dff['z_adj'] = dff['id']
        dff['x_adj'] = dff['id']

        # map adjacent particle ID to particle ID
        dff['z_adj'] = dff['z_adj'].map(mapping_dict)
        dff['x_adj'] = dff['x_adj'].map(mapping_dict)

        # map z_true value to adjacent particle ID
        mapping_dict_z_adjacent = {i: z for (i, z) in zip(dff.id.values, dff.z_true.values)}
        dff['z_adj'] = dff['z_adj'].map(mapping_dict_z_adjacent)

        # map x_adjacent value to adjacent particle ID
        mapping_dict_x_adjacent = {i: x for (i, x) in zip(dff.id.values, dff.x.values)}
        dff['x_adj'] = dff['x_adj'].map(mapping_dict_x_adjacent)

        # append to list
        dfs.append(dff)

    # concat list of dataframes
    df_z_adj = pd.concat(dfs)

    # add 'dz' column
    df_z_adj['dz'] = df_z_adj.z_adj - df_z_adj.z_true

    # add 'dx' column
    df_z_adj['dx'] = df_z_adj.x_adj - df_z_adj.x

    # add 'theta' column
    df_z_adj['theta'] = np.rad2deg(np.arctan(df_z_adj['dz'] / df_z_adj['dx']))

    df_z_adj = df_z_adj.sort_values('frame')

    return df_z_adj

# ---------------------------------   DATAFRAMES (above)  --------------------------------------------------------------

# ----------------------------------------------- HELPER FUNCTIONS -----------------------------------------------------

def merge_calib_pid_defocus_and_correction_coords(path_calib_coords, method, dfs=None):

    if dfs is not None:
        dfxy = dfs[0]
        dfcpid = dfs[1]

        if len(dfxy) > len(dfxy.id.unique()):
            dfxy = dfxy.groupby('id').mean()
    else:
        dfc, dfcpid, dfcpop, dfcstats = io.read_calib_coords(path_calib_coords, method=method)

        # calib coords with (x, y)
        if dfcstats is not None:
            dfxy = dfcstats.groupby('id').mean()
        elif dfc is not None:
            dfxy = dfc.groupby('id').mean()
        else:
            raise ValueError('Either dfcstats or dfc (calibration correction coords) must be readable.')

    # per-particle coords
    dfcpid = dfcpid.set_index('id')

    # merge
    dfcpidxy = pd.concat([dfxy[['x', 'y']], dfcpid], axis=1, join='inner').reset_index()
    dfcpidxy.to_excel(path_calib_coords + '/calib_{}_pid_defocus_stats_xy.xlsx'.format(method), index=False)

    return dfcpidxy

max_stars_repo_name: sean-mackenzie/gdpyt-analysis

max_issues_repo_name: sean-mackenzie/gdpyt-analysis

max_forks_repo_name: sean-mackenzie/gdpyt-analysis


text:
import multiprocessing
import sys
import itertools as it
import warnings

import numpy as np
import mdtraj as md
from progressbar import ProgressBar
from itertools import combinations_with_replacement

from scattering.utils.utils import get_dt, get_unique_atoms
from scattering.utils.constants import get_form_factor

def compute_van_hove(
    trj,
    chunk_length,
    parallel=False,
    water=False,
    r_range=(0, 1.0),
    bin_width=0.005,
    n_bins=None,
    self_correlation=True,
    periodic=True,
    opt=True,
    partial=False,
):
    """Compute the  Van Hove function of a trajectory. Atom pairs
    referenced in partial Van Hove functions are in alphabetical
    order. If specific ordering of atom pairs are needed, user should
    use compute_partial_van_hove then vhf_from_pvhf to compute total
    Van Hove function.

    Parameters
    ----------
    trj : mdtraj.Trajectory
        trajectory on which to compute the Van Hove function
    chunk_length : int
        length of time between restarting averaging
    parallel : bool, default=True
        Use parallel implementation with `multiprocessing`
    water : bool
        use X-ray form factors for water that account for polarization
    r_range : array-like, shape=(2,), optional, default=(0.0, 1.0)
        Minimum and maximum radii.
    bin_width : float, optional, default=0.005
        Width of the bins in nanometers.
    n_bins : int, optional, default=None
        The number of bins. If specified, this will override the `bin_width`
         parameter.
    self_correlation : bool, default=True
        Whether or not to include the self-self correlations
    partial : bool, default = False
        Whether or not to return a dictionary including partial Van Hove function.

    Returns
    -------
    r : numpy.ndarray
        r positions generated by histogram binning
    g_r_t : numpy.ndarray
        Van Hove function at each time and position
    """

    n_physical_atoms = len([a for a in trj.top.atoms if a.element.mass > 0])
    unique_elements = list(
        set([a.element for a in trj.top.atoms if a.element.mass > 0])
    )

    if parallel:
        data = []
        for elem1, elem2 in it.combinations_with_replacement(unique_elements[::-1], 2):
            # Add a bool to check if self-correlations should be analyzed
            self_bool = self_correlation
            if elem1 != elem2:
                self_bool = False
                warnings.warn(
                    "Total VHF calculation: No self-correlations for {} and {}, setting `self_correlation` to `False`.".format(
                        elem1, elem2
                    )
                )

            data.append(
                [
                    trj,
                    chunk_length,
                    "element {}".format(elem1.symbol),
                    "element {}".format(elem2.symbol),
                    r_range,
                    bin_width,
                    n_bins,
                    self_bool,
                    periodic,
                    opt,
                ]
            )

        manager = multiprocessing.Manager()
        partial_dict = manager.dict()
        jobs = []
        version_info = sys.version_info
        for d in data:
            with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as pool:
                if version_info.major == 3 and version_info.minor <= 7:
                    p = pool.Process(target=worker, args=(partial_dict, d))
                elif version_info.major == 3 and version_info.minor >= 8:
                    ctx = multiprocessing.get_context()
                    p = pool.Process(ctx, target=worker, args=(partial_dict, d))
                jobs.append(p)
                p.start()

        for proc in jobs:
            proc.join()

        r = partial_dict["r"]
        del partial_dict["r"]

    else:
        partial_dict = dict()

        for elem1, elem2 in it.combinations_with_replacement(unique_elements[::-1], 2):
            # Add a bool to check if self-correlations should be analyzed
            self_bool = self_correlation
            if elem1 != elem2:
                self_bool = False
                warnings.warn(
                    "Total VHF calculation: No self-correlations for {} and {}, setting `self_correlation` to `False`.".format(
                        elem1, elem2
                    )
                )

            if elem1.symbol > elem2.symbol:
                temp = elem1
                elem1 = elem2
                elem2 = temp
            print("doing {0} and {1} ...".format(elem1, elem2))
            r, g_r_t_partial = compute_partial_van_hove(
                trj=trj,
                chunk_length=chunk_length,
                selection1="element {}".format(elem1.symbol),
                selection2="element {}".format(elem2.symbol),
                r_range=r_range,
                bin_width=bin_width,
                n_bins=n_bins,
                self_correlation=self_bool,
                periodic=periodic,
                opt=opt,
            )
            partial_dict[
                ("element {}".format(elem1.symbol), "element {}".format(elem2.symbol))
            ] = g_r_t_partial

    if partial:
        return partial_dict

    norm = 0
    g_r_t = None

    for key, val in partial_dict.items():
        elem1, elem2 = key
        concentration1 = (
            trj.atom_slice(trj.top.select(elem1)).n_atoms / n_physical_atoms
        )
        concentration2 = (
            trj.atom_slice(trj.top.select(elem2)).n_atoms / n_physical_atoms
        )
        form_factor1 = get_form_factor(element_name=elem1.split()[1], water=water)
        form_factor2 = get_form_factor(element_name=elem2.split()[1], water=water)

        coeff = form_factor1 * concentration1 * form_factor2 * concentration2
        if g_r_t is None:
            g_r_t = np.zeros_like(val)
        g_r_t += val * coeff

        norm += coeff

    # Reshape g_r_t to better represent the discretization in both r and t
    g_r_t_final = np.empty(shape=(chunk_length, len(r)))
    for i in range(chunk_length):
        g_r_t_final[i, :] = np.mean(g_r_t[i::chunk_length], axis=0)

    g_r_t_final /= norm

    t = trj.time[:chunk_length]

    return r, t, g_r_t_final

def worker(return_dict, data):
    key = (data[2], data[3])
    r, g_r_t_partial = compute_partial_van_hove(*data)
    return_dict[key] = g_r_t_partial
    return_dict["r"] = r

def compute_partial_van_hove(
    trj,
    chunk_length=10,
    selection1=None,
    selection2=None,
    r_range=(0, 1.0),
    bin_width=0.005,
    n_bins=200,
    self_correlation=True,
    periodic=True,
    opt=True,
):
    """Compute the partial van Hove function of a trajectory

    Parameters
    ----------
    trj : mdtraj.Trajectory
        trajectory on which to compute the Van Hove function
    chunk_length : int
        length of time between restarting averaging
    selection1 : str
        selection to be considered, in the style of MDTraj atom selection
    selection2 : str
        selection to be considered, in the style of MDTraj atom selection
    r_range : array-like, shape=(2,), optional, default=(0.0, 1.0)
        Minimum and maximum radii.
    bin_width : float, optional, default=0.005
        Width of the bins in nanometers.
    n_bins : int, optional, default=None
        The number of bins. If specified, this will override the `bin_width`
         parameter.
    self_correlation : bool, default=True
        Whether or not to include the self-self correlations

    Returns
    -------
    r : numpy.ndarray
        r positions generated by histogram binning
    g_r_t : numpy.ndarray
        Van Hove function at each time and position
    """
    unique_elements = (
        set([a.element for a in trj.atom_slice(trj.top.select(selection1)).top.atoms]),
        set([a.element for a in trj.atom_slice(trj.top.select(selection2)).top.atoms]),
    )

    if any([len(val) > 1 for val in unique_elements]):
        raise UserWarning(
            "Multiple elements found in a selection(s). Results may not be "
            "direcitly comprable to scattering experiments."
        )

    # Check if pair is monatomic
    # If not, do not calculate self correlations
    if selection1 != selection2 and self_correlation == True:
        warnings.warn(
            "Partial VHF calculation: No self-correlations for {} and {}, setting `self_correlation` to `False`.".format(
                selection1, selection2
            )
        )
        self_correlation = False

    # Don't need to store it, but this serves to check that dt is constant
    dt = get_dt(trj)

    pairs = trj.top.select_pairs(selection1=selection1, selection2=selection2)

    n_chunks = int(trj.n_frames / chunk_length)

    g_r_t = None
    pbar = ProgressBar()

    for i in pbar(range(n_chunks)):
        times = list()
        for j in range(chunk_length):
            times.append([chunk_length * i, chunk_length * i + j])
        r, g_r_t_frame = md.compute_rdf_t(
            traj=trj,
            pairs=pairs,
            times=times,
            r_range=r_range,
            bin_width=bin_width,
            n_bins=n_bins,
            period_length=chunk_length,
            self_correlation=self_correlation,
            periodic=periodic,
            opt=opt,
        )

        if g_r_t is None:
            g_r_t = np.zeros_like(g_r_t_frame)
        g_r_t += g_r_t_frame

    return r, g_r_t

def vhf_from_pvhf(trj, partial_dict, water=False):
    """
    Compute the total Van Hove function from partial Van Hove functions

    Parameters
    ----------
    trj : mdtrj.Trajectory
        trajectory on which partial vhf were calculated form
    partial_dict : dict
        dictionary containing partial vhf as a np.array.
        Key is a tuple of len 2 with 2 atom types

    Return
    -------
    total_grt : numpy.ndarray
        Total Van Hove Function generated from addition of partial Van Hove Functions
    """
    unique_atoms = get_unique_atoms(trj)
    all_atoms = [atom for atom in trj.topology.atoms]

    norm_coeff = 0
    dict_shape = list(partial_dict.values())[0][0].shape
    total_grt = np.zeros(dict_shape)

    for atom_pair in partial_dict.keys():
        # checks if key is a tuple
        if isinstance(atom_pair, tuple) == False:
            raise ValueError("Dictionary key not valid. Must be a tuple.")
        for atom in atom_pair:
            # checks if the atoms in tuple pair are atom types
            if type(atom) != type(unique_atoms[0]):
                raise ValueError(
                    "Dictionary key not valid. Must be type `MDTraj.Atom`."
                )
            # checks if atoms are in the trajectory
            if atom not in all_atoms:
                raise ValueError(
                    f"Dictionary key not valid, `Atom` {atom} not in MDTraj trajectory."
                )

        # checks if key has two atoms
        if len(atom_pair) != 2:
            raise ValueError(
                "Dictionary key not valid. Must only have 2 atoms per pair."
            )

        atom1 = atom_pair[0]
        atom2 = atom_pair[1]
        coeff = (
            get_form_factor(element_name=f"{atom1.element.symbol}", water=False)
            * get_form_factor(element_name=f"{atom2.element.symbol}", water=False)
            * len(trj.topology.select(f"name {atom1.name}"))
            / (trj.n_atoms)
            * len(trj.topology.select(f"name {atom2.name}"))
            / (trj.n_atoms)
        )

        normalized_pvhf = coeff * partial_dict[atom_pair]
        norm_coeff += coeff
        total_grt = np.add(total_grt, normalized_pvhf)

    total_grt /= norm_coeff

    return total_grt

max_stars_repo_name: ramanishsingh/scattering

max_issues_repo_name: ramanishsingh/scattering

max_forks_repo_name: ramanishsingh/scattering


text:
#geom_transform.py
"""
Module: geom_transform - implement geometric transformation
Last updated: 02/02/2020
"""

#import dependencies
import numpy as np
__all__ = ['translation', 'nn', 'linear_interpolate']

"""
translation - function to move the image with alpha and beta spaces
Parameters:
	image	I/P	the image input
	alpha	I/P	the num of spaces to move images horizontally
	beta	I/P	the num of spaces to move images vertically
	new_img	O/P	the moved image
"""
def translation(image, alpha, beta):
	#check alpha and beta in image range
	try:
		if alpha > image.shape[1] or beta > image.shape[0]:
			 raise NameError("Translation Error")
	except NameError:
		print('Alpha of ' + str(alpha) or ' Beta of ' + str(beta) + ' is out of image shape.')

	new_img = np.array([[255] * image.shape[1]] * image.shape[0])

	height = new_img.shape[0]
	width = new_img.shape[1]
	for row in range(height):
		if row + alpha in range(height):
			for col in range(width):
				if col + beta in range(width):
					new_img[row, col] = image[row + alpha, col + beta]

	return new_img
	"""		end of translation		"""

"""
nn - function nearest neigbor to map pixels of scaled images to the original images for image interpolation. By default, assume that scale around (0, 0)
Parameters:
	orgi_img	I/P	original image input
	scaled_img	I/P	scaled image input - 2d array of mapped pixels
	h_ratio		I/P	scaling ratio for height
	w_ratio		I/P	scaling ratio for width
			O/P	color-filled image
"""
def nn(origi_img, scaled_img, h_ratio, w_ratio):
	"""
		nearest neighbor - a linear interpolation to fill pixels of scaled images by picking the neartest pixels
	"""

	#get height and width of scaled_img and origi_img
	height = scaled_img.shape[0]
	h = origi_img.shape[0]
	width = scaled_img.shape[1]
	w = origi_img.shape[1]

	#visit each pixel and map back to original image
	for row in range(height):
		x = round(row / h_ratio)
		if not x in range(h): #if out-of-bound, fill w/ zeros
			continue

		for col in range(width):
			y = round(col / w_ratio)

			#map to the nearest pixels
			if not y in range(w): #if out-of-bound, fill with zero
				continue
			else:
				scaled_img[row, col] = origi_img[x, y]
	return scaled_img
	"""		end of nn		"""

"""
linear_interpolate - function to bilinear interpolatation trasformation
Parameters:
	origi_img	I/P	image input
	scaled_img	I/P	scaled image input - 2d array of mapped pixels
	h_ratio		I/P	scaling ratio for height
	w_ratio		I/P	scaling ratio for width
			O/P	transformed image
"""
def linear_interpolate(origi_img, scaled_img, h_ratio, w_ratio):
	#get height and width of scaled_img and origi_img
	height = scaled_img.shape[0]
	width = scaled_img.shape[1]
	h = origi_img.shape[0]
	w = origi_img.shape[1]

	#visit each pixel and map back to original image
	#extra variables are pre-computed in order to prevent re-computing within for loops
	for row in range(height):
		#get x, floor and ceil of x, compute weight b
		x = row / h_ratio
		fx = np.int(np.floor(x))
		cx = np.int(np.ceil(x))

		#if out-of-bound, fill w/ zero
		if not fx in range(h) or not cx in range(h):
			continue
		#compute weight
		b = x - fx
		nb = 1 - b

		for col in range(width):
			#get y, floor and ceil of y, and compute weight a
			y = col / w_ratio
			fy = np.int(np.floor(y))
			cy = np.int(np.ceil(y))

			#compute transformed pixels
			if fy in range(w) and cy in range(w): #if out-of-bound, fill with zeros
				#compute weight
				a = y - fy
				na = 1 - a
				scaled_img[row, col] = na * nb * origi_img[fx, fy] + na * b * origi_img[fx, cy] + a * nb * origi_img[cx, fy] + a * b * origi_img[cx, cy]
			else:
				pass
	return scaled_img
	"""		linear_interpolate		"""


text:
"""A module containing classes that specify the basic components of a Factor Graph."""

from dataclasses import asdict, dataclass
from typing import Sequence, Tuple, Union

import jax
import jax.numpy as jnp
import numpy as np

from pgmax import utils

@dataclass(frozen=True, eq=False)
class Variable:
    """Base class for variables.
    If desired, this can be sub-classed to add additional concrete
    meta-information

    Args:
        num_states: an int representing the number of states this variable has.
    """

    num_states: int

@jax.tree_util.register_pytree_node_class
@dataclass(frozen=True, eq=False)
class Wiring:
    """Wiring for factors.

    Args:
        edges_num_states: Array of shape (num_edges,)
            Number of states for the variables connected to each edge
        var_states_for_edges: Array of shape (num_edge_states,)
            Global variable state indices for each edge state
    """

    edges_num_states: Union[np.ndarray, jnp.ndarray]
    var_states_for_edges: Union[np.ndarray, jnp.ndarray]

    def __post_init__(self):
        for field in self.__dataclass_fields__:
            if isinstance(getattr(self, field), np.ndarray):
                getattr(self, field).flags.writeable = False

    def tree_flatten(self):
        return jax.tree_util.tree_flatten(asdict(self))

    @classmethod
    def tree_unflatten(cls, aux_data, children):
        return cls(**aux_data.unflatten(children))

@dataclass(frozen=True, eq=False)
class Factor:
    """A factor

    Args:
        variables: List of connected variables

    Raises:
        NotImplementedError: If compile_wiring is not implemented
    """

    variables: Tuple[Variable, ...]
    log_potentials: np.ndarray

    def __post_init__(self):
        if not hasattr(self, "compile_wiring"):
            raise NotImplementedError(
                "Please implement compile_wiring in for your factor"
            )

    @utils.cached_property
    def edges_num_states(self) -> np.ndarray:
        """Number of states for the variables connected to each edge

        Returns:
            Array of shape (num_edges,)
            Number of states for the variables connected to each edge
        """
        edges_num_states = np.array(
            [variable.num_states for variable in self.variables], dtype=int
        )
        return edges_num_states

    @staticmethod
    def concatenate_wirings(wirings: Sequence) -> Wiring:
        """Concatenate a list of Wirings

        Args:
            wirings: A list of Wirings

        Returns:
            Concatenated Wiring
        """
        raise NotImplementedError(
            "Please subclass the Wiring class and override this method."
        )

max_stars_repo_name: StannisZhou/PGMax

max_issues_repo_name: StannisZhou/PGMax

max_forks_repo_name: StannisZhou/PGMax

reason: import numpy,import jax


text:
from __future__ import division
import sys
import numpy as np
from PySide2 import QtCore, QtWidgets
from matplotlib import cm

from sciapp_toolkit.examples.mandelbrot.mandelbrot import mandelbrot_image
from sciapp_toolkit.examples.mandelbrot.ui.ui_main import Ui_MainWindow

class ApplicationWindow(QtWidgets.QMainWindow, Ui_MainWindow):
    """
    Main window for Mandelbrot set visualization application.
    """
    def __init__(self, parent=None):
        # Set up main window
        super(ApplicationWindow, self).__init__(parent)

        # Application attrs
        self._diving = False              # Visualization state variable
        self._dive_timer_interval = 100   # Dive timer increment, in ms
        self._recompute_interval = 100 * self._dive_timer_interval
        self._zoom_frac_per_frame = 0.01  # Zoom-in fraction per frame when
                                          # diving

        # Default params for Mandelbrot computation - lifted directly from the
        # matplotlib example
        self.xmin, self.xmax, self.xn = -2.25, 0.75, 3000/2
        self.ymin, self.ymax, self.yn = -1.25, 1.25, 2500/2
        self.maxiter = 200
        self.horizon = 2.0

        # Set up the GUI
        self.setup_ui(self)

        # Add a timer to initiate zooming of figure
        self.dive_timer = QtCore.QTimer()
        self.compute_timer = QtCore.QTimer()

        # Hook up events to callbacks
        self.dive_control_button.clicked.connect(self.toggle_dive)
        self.reset_button.clicked.connect(self.reset)
        self.dive_timer.timeout.connect(self.increment_zoom)
        self.compute_timer.timeout.connect(self.recompute_mandelbrot)

        # Compute initial mandelbrot set
        self.mandelbrot_ary = mandelbrot_image(self.xmin, self.xmax,
                                               self.ymin, self.ymax,
                                               self.xn, self.yn,
                                               self.maxiter, self.horizon)

        # Set the image
        self.mpl_mandelbrot.image = \
             self.mpl_mandelbrot.axes.imshow(self.mandelbrot_ary,
                                             extent=[self.xmin,
                                                     self.xmax,
                                                     self.ymin,
                                                     self.ymax],
                                             cmap=cm.plasma,
                                             animated=True)

        # Start the application
        self.start()

    def recompute_mandelbrot(self):
        """
        Recompute the mandelbrot set from the current axes limits.
        """
        # Grab iteration number from UI - use default as fallback
        try:
            max_iter = int(self.maxiter_lineedit.text())
            self.maxiter = max_iter
        except ValueError: pass
        # Recompute
        xmin, xmax = self.mpl_mandelbrot.axes.get_xlim()
        ymin, ymax = self.mpl_mandelbrot.axes.get_ylim()
        ary = mandelbrot_image(xmin, xmax, ymin, ymax, self.xn, self.yn,
                               self.maxiter, self.horizon)
        ary = np.flipud(ary)
        # Update image
        self.mpl_mandelbrot.update_image(ary, [xmin, xmax, ymin, ymax])

    def toggle_dive(self):
        if self._diving:
            self.dive_control_button.setText("Start Diving")
            self._diving = False
            self.compute_timer.stop()
        else:
            self.dive_control_button.setText("Pause Diving")
            self._diving = True
            self.compute_timer.start(self._recompute_interval)

    def reset(self):
        """
        Reset Mandelbrot image back to original state.
        """
        # Reset axes
        self.mpl_mandelbrot.axes.set_xlim(self.xmin, self.xmax)
        self.mpl_mandelbrot.axes.set_ylim(self.ymin, self.ymax)
        # Reset image
        self.mpl_mandelbrot.update_image(self.mandelbrot_ary,
                                         [self.xmin, self.xmax,
                                          self.ymin, self.ymax])

    def increment_zoom(self):
        """
        If application is currently in the "diving" state, increment the
        zoom level of the mandelbrot visualization.
        """
        if self._diving:
            self.mpl_mandelbrot.increment_zoom_anchored(self._zoom_frac_per_frame)

    def start(self):
        """
        Start the application.
        """
        self.dive_timer.start(self._dive_timer_interval)

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    win = ApplicationWindow()
    win.show()
    sys.exit(app.exec_())


text:
# This file is part of GridCal.
#
# GridCal is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# GridCal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GridCal.  If not, see <http://www.gnu.org/licenses/>.
import numpy as np
from enum import Enum
import multiprocessing

from GridCal.Engine.basic_structures import Logger
from GridCal.Engine.Simulations.PowerFlow.power_flow_results import PowerFlowResults
from GridCal.Engine.Simulations.Stochastic.stochastic_power_flow_results import StochasticPowerFlowResults
from GridCal.Engine.Simulations.Stochastic.stochastic_power_flow_input import StochasticPowerFlowInput
from GridCal.Engine.Core.time_series_pf_data import TimeCircuit
from GridCal.Engine.Core.multi_circuit import MultiCircuit
from GridCal.Engine.basic_structures import CDF
from GridCal.Engine.Simulations.PowerFlow.power_flow_worker import PowerFlowOptions, single_island_pf, \
                                                                    power_flow_post_process

from GridCal.Engine.Core.time_series_pf_data import compile_time_circuit, BranchImpedanceMode
from GridCal.Engine.Simulations.driver_types import SimulationTypes
from GridCal.Engine.Simulations.driver_template import DriverTemplate

########################################################################################################################
# Monte Carlo classes
########################################################################################################################

class StochasticPowerFlowType(Enum):

    MonteCarlo = 'Monte Carlo'
    LatinHypercube = 'Latin Hypercube'

def make_monte_carlo_input(numerical_input_island: TimeCircuit):
    """
    Generate a monte carlo input instance
    :param numerical_input_island:
    :return:
    """
    n = numerical_input_island.nbus
    Scdf = [None] * n
    Icdf = [None] * n
    Ycdf = [None] * n

    for i in range(n):
        Scdf[i] = CDF(numerical_input_island.Sbus[i, :])
        Icdf[i] = CDF(numerical_input_island.Ibus[i, :])
        Ycdf[i] = CDF(numerical_input_island.Yshunt_from_devices[i, :])

    return StochasticPowerFlowInput(n, Scdf, Icdf, Ycdf)

class StochasticPowerFlowDriver(DriverTemplate):
    name = 'Stochastic Power Flow'
    tpe = SimulationTypes.StochasticPowerFlow

    def __init__(self, grid: MultiCircuit, options: PowerFlowOptions, mc_tol=1e-3, batch_size=100, sampling_points=10000,
                 opf_time_series_results=None,
                 simulation_type: StochasticPowerFlowType = StochasticPowerFlowType.LatinHypercube):
        """
        Monte Carlo simulation constructor
        :param grid: MultiGrid instance
        :param options: Power flow options
        :param mc_tol: monte carlo std.dev tolerance
        :param batch_size: size of the batch
        :param sampling_points: maximum monte carlo iterations in case of not reach the precission
        :param simulation_type: Type of sampling method
        """
        DriverTemplate.__init__(self, grid=grid)

        self.options = options

        self.opf_time_series_results = opf_time_series_results

        self.mc_tol = mc_tol

        self.batch_size = batch_size

        self.sampling_points = sampling_points

        self.simulation_type = simulation_type

        self.results = None

        self.logger = Logger()

        self.pool = None

        self.returned_results = list()

        self.__cancel__ = False

    def get_steps(self):
        """
        Get time steps list of strings
        """
        p = self.results.points_number
        return ['point:' + str(l) for l in range(p)]

    def update_progress_mt(self, res):
        """
        """
        t, _ = res
        progress = (t + 1) / self.sampling_points * 100
        self.progress_signal.emit(progress)
        self.returned_results.append(res)

    def run_single_thread_mc(self):

        self.__cancel__ = False

        # initialize the grid time series results
        # we will append the island results with another function

        # batch_size = self.sampling_points

        self.progress_signal.emit(0.0)
        self.progress_text.emit('Running Monte Carlo Sampling...')

        # compile the multi-circuit
        numerical_circuit = compile_time_circuit(circuit=self.grid,
                                                 apply_temperature=False,
                                                 branch_tolerance_mode=BranchImpedanceMode.Specified,
                                                 opf_results=self.opf_time_series_results)

        # do the topological computation
        calculation_inputs = numerical_circuit.split_into_islands(
            ignore_single_node_islands=self.options.ignore_single_node_islands)

        mc_results = StochasticPowerFlowResults(n=numerical_circuit.nbus,
                                                m=numerical_circuit.nbr,
                                                p=self.sampling_points,
                                                bus_names=numerical_circuit.bus_names,
                                                branch_names=numerical_circuit.branch_names,
                                                bus_types=numerical_circuit.bus_types,
                                                name='Monte Carlo')

        avg_res = PowerFlowResults(n=numerical_circuit.nbus,
                                   m=numerical_circuit.nbr,
                                   n_tr=numerical_circuit.ntr,
                                   n_hvdc=numerical_circuit.nhvdc,
                                   bus_names=numerical_circuit.bus_names,
                                   branch_names=numerical_circuit.branch_names,
                                   transformer_names=numerical_circuit.tr_names,
                                   hvdc_names=numerical_circuit.hvdc_names,
                                   bus_types=numerical_circuit.bus_types)

        variance_sum = 0.0
        v_sum = np.zeros(numerical_circuit.nbus, dtype=complex)

        # For every island, run the time series
        for island_index, numerical_island in enumerate(calculation_inputs):

            # try:
            # set the time series as sampled in the circuit
            # build the inputs
            monte_carlo_input = make_monte_carlo_input(numerical_island)
            mc_time_series = monte_carlo_input(self.sampling_points,
                                               use_latin_hypercube=False)
            Vbus = numerical_island.Vbus[:, 0]

            # short cut the indices
            bus_idx = numerical_island.original_bus_idx
            br_idx = numerical_island.original_branch_idx

            # run the time series
            for t in range(self.sampling_points):

                # set the power values from a Monte carlo point at 't'
                Y, I, S = mc_time_series.get_at(t)

                # Run the set monte carlo point at 't'
                res = single_island_pf(circuit=numerical_island,
                                       Vbus=Vbus,
                                       Sbus=S,
                                       Ibus=I,
                                       branch_rates=numerical_island.branch_rates,
                                       pq=numerical_island.pq,
                                       pv=numerical_island.pv,
                                       vd=numerical_island.vd,
                                       pqpv=numerical_island.pqpv,
                                       options=self.options,
                                       logger=self.logger)

                # Gather the results
                mc_results.S_points[t, bus_idx] = S
                mc_results.V_points[t, bus_idx] = res.voltage
                mc_results.Sbr_points[t, br_idx] = res.Sf
                mc_results.loading_points[t, br_idx] = res.loading
                mc_results.losses_points[t, br_idx] = res.losses

                # determine when to stop
                if t > 1:
                    v_sum += mc_results.get_voltage_sum()
                    v_avg = v_sum / t
                    v_variance = np.abs((np.power(mc_results.V_points - v_avg, 2.0) / (t - 1)).min())

                    # progress
                    variance_sum += v_variance
                    err = variance_sum / t
                    if err == 0:
                        err = 1e-200  # to avoid division by zeros
                    mc_results.error_series.append(err)

                    # emmit the progress signal
                    std_dev_progress = 100 * self.mc_tol / err
                    if std_dev_progress > 100:
                        std_dev_progress = 100
                    self.progress_signal.emit(max((std_dev_progress, t / self.sampling_points * 100)))

                if self.__cancel__:
                    break

            if self.__cancel__:
                break

            # compile MC results
            self.progress_text.emit('Compiling results...')
            mc_results.compile()

            # compute the island branch results
            Sfb, Stb, If, It, Vbranch, loading, \
            losses, Sbus = power_flow_post_process(numerical_island,
                                                                   Sbus=mc_results.S_points.mean(axis=0)[bus_idx],
                                                                   V=mc_results.V_points.mean(axis=0)[bus_idx],
                                                                   branch_rates=numerical_island.branch_rates)

            # apply the island averaged results
            avg_res.Sbus[bus_idx] = Sbus
            avg_res.voltage[bus_idx] = mc_results.voltage[bus_idx]
            avg_res.Sf[br_idx] = Sfb
            avg_res.St[br_idx] = Stb
            avg_res.If[br_idx] = If
            avg_res.It[br_idx] = It
            avg_res.Vbranch[br_idx] = Vbranch
            avg_res.loading[br_idx] = loading
            avg_res.losses[br_idx] = losses

        self.results = mc_results

        # send the finnish signal
        self.progress_signal.emit(0.0)
        self.progress_text.emit('Done!')
        self.done_signal.emit()

        return mc_results

    def run_single_thread_lhs(self):
        """
        Run the monte carlo simulation with Latin Hypercube sampling
        @return:
        """
        self.__cancel__ = False

        # initialize the grid time series results
        # we will append the island results with another function

        # batch_size = self.sampling_points

        self.progress_signal.emit(0.0)
        self.progress_text.emit('Running Latin Hypercube Sampling...')

        # compile the multi-circuit
        numerical_circuit = compile_time_circuit(circuit=self.grid,
                                                 apply_temperature=False,
                                                 branch_tolerance_mode=BranchImpedanceMode.Specified,
                                                 opf_results=self.opf_time_series_results)

        # do the topological computation
        calculation_inputs = numerical_circuit.split_into_islands(ignore_single_node_islands=self.options.ignore_single_node_islands)

        lhs_results = StochasticPowerFlowResults(n=numerical_circuit.nbus,
                                                 m=numerical_circuit.nbr,
                                                 p=self.sampling_points,
                                                 bus_names=numerical_circuit.bus_names,
                                                 branch_names=numerical_circuit.branch_names,
                                                 bus_types=numerical_circuit.bus_types,
                                                 name='Latin Hypercube')

        avg_res = PowerFlowResults(n=numerical_circuit.nbus,
                                   m=numerical_circuit.nbr,
                                   n_tr=numerical_circuit.ntr,
                                   n_hvdc=numerical_circuit.nhvdc,
                                   bus_names=numerical_circuit.bus_names,
                                   branch_names=numerical_circuit.branch_names,
                                   transformer_names=numerical_circuit.tr_names,
                                   hvdc_names=numerical_circuit.hvdc_names,
                                   bus_types=numerical_circuit.bus_types)

        # For every island, run the time series
        for island_index, numerical_island in enumerate(calculation_inputs):

            # try:
            # set the time series as sampled in the circuit
            # build the inputs
            monte_carlo_input = make_monte_carlo_input(numerical_island)
            mc_time_series = monte_carlo_input(self.sampling_points, use_latin_hypercube=True)
            Vbus = numerical_island.Vbus[:, 0]

            # short cut the indices
            bus_idx = numerical_island.original_bus_idx
            br_idx = numerical_island.original_branch_idx

            # run the time series
            for t in range(self.sampling_points):

                # set the power values from a Monte carlo point at 't'
                Y, I, S = mc_time_series.get_at(t)

                # Run the set monte carlo point at 't'
                res = single_island_pf(circuit=numerical_island,
                                       Vbus=Vbus,
                                       Sbus=S,
                                       Ibus=I,
                                       branch_rates=numerical_island.branch_rates,
                                       pq=numerical_island.pq,
                                       pv=numerical_island.pv,
                                       vd=numerical_island.vd,
                                       pqpv=numerical_island.pqpv,
                                       options=self.options,
                                       logger=self.logger)

                # Gather the results
                lhs_results.S_points[t, bus_idx] = S
                lhs_results.V_points[t, bus_idx] = res.voltage
                lhs_results.Sbr_points[t, br_idx] = res.Sf
                lhs_results.loading_points[t, br_idx] = res.loading
                lhs_results.losses_points[t, br_idx] = res.losses

                self.progress_signal.emit(t / self.sampling_points * 100)

                if self.__cancel__:
                    break

            if self.__cancel__:
                break

            # compile MC results
            self.progress_text.emit('Compiling results...')
            lhs_results.compile()

            # compute the island branch results
            Sfb, Stb, If, It, Vbranch, loading, \
            losses, Sbus = power_flow_post_process(numerical_island,
                                                   Sbus=lhs_results.S_points.mean(axis=0)[bus_idx],
                                                   V=lhs_results.V_points.mean(axis=0)[bus_idx],
                                                   branch_rates=numerical_island.branch_rates)

            # apply the island averaged results
            avg_res.Sbus[bus_idx] = Sbus
            avg_res.voltage[bus_idx] = lhs_results.voltage[bus_idx]
            avg_res.Sf[br_idx] = Sfb
            avg_res.St[br_idx] = Stb
            avg_res.If[br_idx] = If
            avg_res.It[br_idx] = It
            avg_res.Vbranch[br_idx] = Vbranch
            avg_res.loading[br_idx] = loading
            avg_res.losses[br_idx] = losses

        self.results = lhs_results

        # send the finnish signal
        self.progress_signal.emit(0.0)
        self.progress_text.emit('Done!')
        self.done_signal.emit()

        return lhs_results

    def run(self):
        """
        Run the monte carlo simulation
        @return:
        """
        # print('LHS run')
        self.__cancel__ = False

        if self.options.multi_thread:
            self.results = self.run_multi_thread()
        else:
            if self.simulation_type == StochasticPowerFlowType.MonteCarlo:
                self.results = self.run_single_thread_mc()
            elif self.simulation_type == StochasticPowerFlowType.LatinHypercube:
                self.results = self.run_single_thread_lhs()

        # send the finnish signal
        self.progress_signal.emit(0.0)
        self.progress_text.emit('Done!')
        self.done_signal.emit()

    def cancel(self):
        """
        Cancel the simulation
        :return:
        """
        self.__cancel__ = True
        self.progress_signal.emit(0.0)
        self.progress_text.emit('Cancelled')
        self.done_signal.emit()


text:
import torch
import numpy

class Config(object):
    """模型参数配置"""

    def __init__(self, base_path, embedding_name):

        self.vocab_path = base_path + '/vocab.pkl'
        self.train_path = base_path + '/train.txt'  # 训练集
        self.dev_path = base_path + '/dev.txt'  # 验证集
        self.test_path = base_path + '/test.txt'
        self.save_path = base_path + '/TextCNN.ckpt'  # 模型训练结果
        class_path = base_path + '/class.txt'
        self.class_list = [x.strip() for x in open(class_path, encoding='utf-8').readlines()]  # 类别名单

        # 预训练词向量
        if embedding_name == 'random':
            self.embedding_pretrained = None
        else:
            # 预训练词向量
            embed_path = base_path + '/' + embedding_name
            self.embedding_pretrained = torch.tensor(numpy.load(embed_path)["embeddings"].astype('float32'))

        self.embedding_size = 300
        if self.embedding_pretrained is not None:
            self.embedding_size = self.embedding_pretrained.size(1)
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

        self.dropout = 0.5  # 随机失活
        self.require_improvement = 1000  # 若超过1000batch效果还没提升，则提前结束训练
        self.num_classes = len(self.class_list)  # 类别数
        self.n_vocab = 0  # 词表大小，在运行时赋值
        self.num_epochs = 20  # epoch数
        self.batch_size = 128  # mini-batch大小
        self.pad_size = 32  # 每句话处理成的长度(短填长切)
        self.learning_rate = 1e-3  # 学习率
        self.filter_sizes = (2, 3, 4)  # 卷积核尺寸

        self.hidden_size = 256
        self.n_gram_vocab = 250499

class Model(torch.nn.Module):

    def __init__(self, config : Config):
        super(Model, self).__init__()
        if config.embedding_pretrained is None:
            self.embedding = torch.nn.Embedding(config.n_vocab, config.embedding_size, padding_idx=config.n_vocab-1)
        else:
            self.embedding = torch.nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
        self.embedding_ngram2 = torch.nn.Embedding(config.n_gram_vocab, config.embedding_size)
        self.embedding_ngram3 = torch.nn.Embedding(config.n_gram_vocab, config.embedding_size)
        self.dropout = torch.nn.Dropout(config.dropout)
        self.full_connect1 = torch.nn.Linear(config.embedding_size * 3, config.hidden_size)
        self.full_connect2 = torch.nn.Linear(config.hidden_size, config.num_classes)

    def forward(self, x):
        outword = self.embedding(x[0])
        out_bigram = self.embedding_ngram2(x[2])
        out_trigram = self.embedding_ngram3(x[3])
        out = torch.cat((outword, out_bigram, out_trigram), -1)

        out = out.mean(dim = 1)
        out = self.dropout(out)
        out = self.full_connect1(out)
        out = torch.nn.functional.relu(out)
        out = self.full_connect2(out)
        return out


text:
from __future__ import division
import os, sys, time
import pickle
from collections import OrderedDict
import yaml
import numpy as np

# ==================== Quaternion tools ====================
# Quaternions are represented as [w, x, y, z].

EPS = np.finfo(float).eps * 4.0

def quaternion_inverse(quaternion):
    q = np.array(quaternion, copy=True)
    np.negative(q[1:], q[1:])
    return q / np.dot(q, q)

def quaternion_matrix(quaternion):
    q = np.array(quaternion, dtype=np.float64, copy=True)
    n = np.dot(q, q)
    if n < EPS:
        return np.identity(4)
    q *= np.sqrt(2.0 / n)
    q = np.outer(q, q)
    return np.array([
        [1.0 - q[2, 2] - q[3, 3], q[1, 2] - q[3, 0], q[1, 3] + q[2, 0]],
        [q[1, 2] + q[3, 0], 1.0 - q[1, 1] - q[3, 3], q[2, 3] - q[1, 0]],
        [q[1, 3] - q[2, 0], q[2, 3] + q[1, 0], 1.0 - q[1, 1] - q[2, 2]]])

def quaternion_multiply(quaternion0, quaternion1):
    # quat0 * quat1 corresponds to R0 * R1
    w0, x0, y0, z0 = quaternion0
    w1, x1, y1, z1 = quaternion1
    return np.array([
        -x1 * x0 - y1 * y0 - z1 * z0 + w1 * w0,
        x1 * w0 + y1 * z0 - z1 * y0 + w1 * x0,
        -x1 * z0 + y1 * w0 + z1 * x0 + w1 * y0,
        x1 * y0 - y1 * x0 + z1 * w0 + w1 * z0], dtype=np.float64)

# ==================== Hyperparameters ====================

class HyperParamHandler(object):

    def __init__(self, params):
        self.params = params
        self.sizes = list(map(len, params))
        self.max_id = 1
        for size in self.sizes:
            assert(size > 0)
            self.max_id *= size

    def get_params_from_id(self, proc_id):
        assert(0 <= proc_id < self.max_id)
        params = []
        for i, size in enumerate(self.sizes):
            params.append(self.params[i][proc_id % size])
            proc_id //= size
        return params

def instantiate_hyperparams(config, id_, header_delim="HEADER"):
    # Instantiate hyperparameters based on the config file and the id, and return the header.
    config["hyperparams"] = config["hyperparams"] if config["hyperparams"] is not None else []

    hyperparams = [config[key] for key in config["hyperparams"]]
    for i in hyperparams:
        assert isinstance(i, list), "Make sure hyperparameters are given as lists!"
    h = HyperParamHandler(hyperparams)
    new_params = h.get_params_from_id(id_ % h.max_id)

    header = [header_delim]
    for key, val in zip(config["hyperparams"], new_params):
        config[key] = val
        header.extend([key, "=", str(val)])
    return ' '.join(header)

# ==================== YAML ordered load/dump ====================

def ordered_load(stream, Loader=yaml.SafeLoader, object_pairs_hook=OrderedDict):
    class OrderedLoader(Loader):
        pass
    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return object_pairs_hook(loader.construct_pairs(node))
    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)
    return yaml.load(stream, OrderedLoader)

def ordered_dump(data, stream=None, Dumper=yaml.SafeDumper, **kwds):
    class OrderedDumper(Dumper):
        pass
    def _dict_representer(dumper, data):
        return dumper.represent_mapping(
            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
            data.items())
    OrderedDumper.add_representer(OrderedDict, _dict_representer)
    return yaml.dump(data, stream, OrderedDumper, **kwds)

# ==================== Parameter scheduling ====================

class ConstSchedule(object):

    def __init__(self, val):
        self.val = val

    def value(self, t):
        return self.val

class LinearSchedule(object):

    def __init__(self, n_steps, initial_val, final_val, update_every):
        self.n_steps = n_steps
        self.final_val = final_val
        self.initial_val = initial_val
        self.update_every = update_every

    def value(self, t):
        step = int(self.n_steps // self.update_every) * self.update_every
        fraction = min(float(t) / self.n_steps, 1.0)
        return self.initial_val + fraction * (self.final_val - self.initial_val)

# ==================== Loading pregrasps ====================

def load_pregrasps(filename, min_phys_score=-1, max_phys_score=1):
    with open(filename, 'rb') as file:
        if sys.version_info >= (3, 0):
            data = pickle.load(file, encoding='bytes', fix_imports=True)
        else:
            data = pickle.load(file)
    # Used to avoid collision due to a potential error in the transform.
    delta = np.array([0, 0, 0.015])
    pregrasps = [np.hstack((el[b'obj_rel_pos'] + delta, el[b'obj_rel_orient']))
                 for el in data if el[b'score'] >= min_phys_score]
    scores = [el[b'score'] for el in data if el[b'score'] >=
              min_phys_score and el[b'score'] <= max_phys_score]

    return pregrasps, scores


text:
#!/usr/bin/env python
# coding: utf-8

# # Euro 2020 (2021) Predictions
#
# <!-- Written report for this analysis can be found [here](../reports/boro_01_market_value.md) -->

# ## 1. Business Understanding
#
# * Determine Busines Objectives
# * Situation Assessment
# * Determine Data Mining Goal
# * Produce Project Plan

# ```
# # 1. Predict results of every match at Euro 2020
# # 2. Make predictions before each round of competition
# # 3. Ideally, at each round, use the predictions to simulate remainder of competition
# # 4. Check against other predictions and actual results
# # 5. Write up process (report/blog)
# ```

# ## 2. Data Understanding
#
# * Collect Initial Data
# * Describe Data
# * Explore Data
# * Verify Data Quality

# In[1]:

import pandas as pd
import os
import numpy as np
import pickle

import matplotlib.pyplot as plt
plt.style.use("seaborn-whitegrid")
import seaborn as sns
sns.set()

import src.utilities as utilities

# In[2]:

match = utilities.get_master("nations_matches")
# match.info()

match = match[['Round', 'Day', 'Date', 'Time', 'Team_1', 'Team_2',
       'Year', 'Goals_1', 'Goals_2',
       'Goal_diff', 'Venue', 'Venue_country', 'Venue_city', 'Home_1',
       'Home_2']]

match["Goal_total"] = match.Goals_1 + match.Goals_2
match["Result"] = None
match.loc[match.Goals_1 == match.Goals_2, "Result"] = "Draw"
match.loc[match.Goals_1 > match.Goals_2, "Result"] = "Win"
match.loc[match.Goals_1 < match.Goals_2, "Result"] = "Loss"

match.describe(include="all").T

# In[3]:

def metric_histograms(df, metrics, discrete=False):
#     df = df_in.dropna(subset=[metric]).fillna("NULL")
    for metric in metrics:
        print("\n{0}\n".format(metric))
#         df[metric].hist()
        sns.histplot(data=df, x=metric, kde=True, discrete=discrete)
        plt.show()
#         sns.boxplot(x=df[metric])
#         plt.show()
        print("\n--------------------")

metric_histograms(match, ["Goals_1", "Goals_2", "Goal_diff", "Goal_total"], discrete=True)

# In[4]:

def group_boxplots(df_in, cols, metric):
    df = df_in.dropna(subset=[metric]).fillna("NULL")
    for col in cols:
        print("\n{0}\n".format(col))
        print(df[col].value_counts())
        sns.boxplot(x=col, y=metric, data=df.sort_values(by=col),
                    showmeans=True, width=0.6)
        plt.xticks(rotation=90)

    #     plt.savefig("../reports/figures/club_01_boxplot_{0}.PNG".format(col))
        plt.show()
        print("\n--------------------")

col_list = ["Round", "Day", "Time", "Year", "Venue_country", "Result"]

group_boxplots(match, col_list, "Goal_total")

# In[ ]:

# In[5]:

summary = utilities.get_master("nations_summaries")
# summary.info()

summary = summary[['Rank Local', 'Rank Global', 'Team', 'Rating',
       'Average Rank', 'Average Rating', '1 Year Change Rank',
       '1 Year Change Rating', 'Matches Total', 'Matches Home', 'Matches Away',
       'Matches Neutral', 'Matches Wins', 'Matches Losses', 'Matches Draws',
       'Goals For', 'Goals Against', 'Year', 'Country',
       'Data Year', 'GDP (PPP)', 'Population']]

summary["GDP (PPP) Per Capita"] = summary['GDP (PPP)'] / summary['Population']

summary.describe(include="all").T

# In[6]:

metric_histograms(summary, ['Rating', 'Average Rank',
       'Average Rating', '1 Year Change Rank', '1 Year Change Rating',
       'Matches Total', 'Matches Home', 'Matches Away', 'Matches Neutral',
       'Matches Wins', 'Matches Losses', 'Matches Draws', 'Goals For',
       'Goals Against', 'GDP (PPP)',
       'Population', 'GDP (PPP) Per Capita'])

# In[ ]:

# ## 3. Data Preperation
#
# * Select Data
# * Clean Data
# * Construct Data
# * Integrate Data
# * Format Data

# In[7]:

data = match.merge(summary, left_on=["Team_1", "Year"], right_on=["Team", "Year"]) #, suffixes=["", "_1"])
data = data.merge(summary, left_on=["Team_2", "Year"], right_on=["Team", "Year"], suffixes=["", " (2)"])
data.sort_values(by=["Date", "Team_1"], inplace=True)
data.reset_index(drop=True, inplace=True)

data["Elo_rating_diff"] = data["Rating"] - data["Rating (2)"]
data["Home_advantage"] = data["Home_1"] - data["Home_2"]
# data["Win_expectency_1"] = ((10**((-(data.Elo_rating_diff + (100 * data.Home_advantage)))/400))+1)**-1
data["Relative_experience"] = data["Matches Total"] / data["Matches Total (2)"]
data["Relative_population"] = data["Population"] / data["Population (2)"]
data["Relative_GDP_per_capita"] = data["GDP (PPP) Per Capita"] / data["GDP (PPP) Per Capita (2)"]
data["Relative_ELO_rating"] = data["Rating"] / data["Rating (2)"]
# data["Relative_ELO_rank_1yr_change"] = data["1 Year Change Rank"] / data["1 Year Change Rank (2)"]
# data["Relative_ELO_rating_1yr_change"] = data["1 Year Change Rating"] / data["1 Year Change Rating (2)"]
# data["Combined_ELO_rating_1yr_change"] = data["1 Year Change Rating"].abs() + data["1 Year Change Rating (2)"].abs()

# model_years = [2000, 2004, 2008, 2012, 2016]
live_years = [2021]
data["Usage"] = "Training"
data.loc[data.Year.isin(live_years), "Usage"] = "Live"

# data = data[["Date", "Year", "Team_1", "Team_2", "Goal_diff", "Goal_total", "Elo_rating_diff", "Home_advantage", "Win_expectency_1",
#              "Relative_experience", "Relative_population", "Relative_GDP_per_capita",
#             "Relative_ELO_rating"]] #, "Relative_ELO_rank_1yr_change", "Relative_ELO_rating_1yr_change"
# ]]

# data.columns
data.describe().T

# In[8]:

skip_cols = [x+" (2)" for x in summary.columns]
# print(skip_cols)
skip_cols = summary.columns.tolist() + skip_cols
# print(skip_cols)
skip_cols = [x for x in skip_cols if x in data.columns ]
# print(skip_cols)

data.drop(columns=skip_cols).corr().style.background_gradient(cmap='coolwarm')

# In[9]:

# data_lim.corr().style.background_gradient(cmap='coolwarm')

# In[ ]:

# ## 4. Modelling
#
# * Select Modelling Technique
# * Generate Test Design
# * Build Model
# * Assess Model

# In[10]:

# from sklearn.dummy import DummyRegressor
# from sklearn.linear_model import LinearRegression
# from sklearn.linear_model import Lasso
# from sklearn.linear_model import Ridge
# # from sklearn.linear_model import BayesianRidge
# from sklearn.ensemble import RandomForestRegressor
# from sklearn.ensemble import GradientBoostingRegressor
# # from sklearn.ensemble import VotingRegressor
# # from sklearn.neighbors import KNeighborsRegressor
# from sklearn.svm import SVR
# # from sklearn.compose import TransformedTargetRegressor
from sklearn.base import BaseEstimator, RegressorMixin

# from sklearn.metrics import median_absolute_error
# from sklearn.metrics import mean_squared_error
# from sklearn.metrics import r2_score

# # from sklearn.utils import resample
# from sklearn.preprocessing import MinMaxScaler
# from sklearn.preprocessing import StandardScaler
# from sklearn.pipeline import Pipeline
# # from sklearn.model_selection import learning_curve
# # from sklearn.model_selection import KFold
# from sklearn.model_selection import train_test_split

# # np.random.seed(1)

# In[11]:

class EloRegressor(BaseEstimator, RegressorMixin):
    def __init__(self): #, yType="Diff", goalWeight=4., goalBoost=16.):
        self.yType = "Diff"
        self.goalWeight = 4.
        self.goalBoost = 1.

    def _show_params(self):
        print("_show_params...")
        print("yType:", self.yType)
        print("goalWeight:", self.goalWeight)
        print("goalBoost:", self.goalBoost)

        return

    def _calc_output(self, X):
        X_tmp = X.copy(deep=True)
        X_tmp["EloRatingDiffWithHomeAdv"] = X_tmp["Elo_rating_diff"] + (100 * X_tmp.Home_advantage)
        X_tmp["WinExpectency1Square"] = (10**((-X_tmp.EloRatingDiffWithHomeAdv)/400))+1
        X_tmp["WinExpectency1"] = X_tmp["WinExpectency1Square"]**-1
        X_tmp["RawGoalDiff"] = (self.goalWeight * (X_tmp.WinExpectency1 - 0.5)).round(0)
        X_tmp["RawGoalDiffAbs"] = X_tmp["RawGoalDiff"].abs()
        X_tmp["EitherWins"] = 0
        X_tmp.loc[X_tmp.RawGoalDiffAbs > 0, "EitherWins"] = 1
#         X_tmp["QualifyGoalsRankAvg"] = (X_tmp["QualifyGoalsRank1"] + X_tmp["QualifyGoalsRank2"]) / 2
        X_tmp["ApplyGoalBoost"] = 0
#         X_tmp.loc[X_tmp.QualifyGoalsRankAvg <= self.goalBoost, "ApplyGoalBoost"] = 1
        X_tmp["Goals1"] = X_tmp["ApplyGoalBoost"]
        X_tmp.loc[X_tmp.RawGoalDiff > 0, "Goals1"] = X_tmp.RawGoalDiff + X_tmp.ApplyGoalBoost
        X_tmp["Goals2"] = X_tmp["ApplyGoalBoost"]
        X_tmp.loc[X_tmp.RawGoalDiff <= 0, "Goals2"] = X_tmp.ApplyGoalBoost - X_tmp.RawGoalDiff
        X_tmp["GoalDiff"] = X_tmp.Goals1 - X_tmp.Goals2
        X_tmp["GoalDiffAbs"] = X_tmp.GoalDiff.abs()
        X_tmp["GoalTotal"] = X_tmp.Goals1 + X_tmp.Goals2

        return X_tmp["Goal"+self.yType].values

    def fit(self, X, y=None):
        if y.name == "Goal_total":
            self.yType = "Total"
#         else:
#             self.yType = "Diff"
        y_tmp = self._calc_output(X).mean()
        y_low = y.quantile(q=0.2)
        y_high = y.quantile(q=0.8)
        while y_tmp < y_low:
            self.goalWeight += 0.05
            y_tmp = self._calc_output(X).mean()
        while y_tmp > y_high:
            self.goalWeight -= 0.05
            y_tmp = self._calc_output(X).mean()
        self._show_params()

        return self

    def predict(self, X, y=None):
        self._show_params()
        return self._calc_output(X)

# ## 5. Evaluation
#
# * Evaluate Results
# * Review Process
# * Determine Next Steps

# ## 6. Deployment
#
# * Plan Deployment
# * Plan Monitoring and Maintenance
# * Produce Final Report
# * Review Project

# In[12]:

pickle_base = "../models/intl_02_{0}.pkl"

print("Un-Pickling files...")

with open(pickle_base.format("gd_model"), "rb") as pkl_1:
    selected_gd_model = pickle.load(pkl_1)
with open(pickle_base.format("gd_features"), "rb") as pkl_2:
    gd_features = pickle.load(pkl_2)
with open(pickle_base.format("gt_model"), "rb") as pkl_3:
    selected_gt_model = pickle.load(pkl_3)
with open(pickle_base.format("gt_features"), "rb") as pkl_4:
    gt_features = pickle.load(pkl_4)

selected_gd_model, gd_features, selected_gt_model, gt_features

# In[13]:

output_prev = pd.read_csv("../data/interim/intl_02_predictions.csv")
output_prev["Year"] = output_prev.Year.astype(str)
output_prev.sort_values(by=["Date", "Team_1"], inplace=True)
output_prev.reset_index(drop=True, inplace=True)
output_prev.info()

# In[14]:

output_new = data.copy(deep=True)[["Date", "Year", "Round", "Team_1", "Team_2", "Usage", "Goals_1", "Goals_2", "Goal_diff", "Goal_total", "Result"]]
output_new.columns = ["Date", "Year", "Round", "Team_1", "Team_2", "Usage", "Actual_score_1", "Actual_score_2", "Actual_goal_diff", "Actual_goal_total", "Actual_result"]
output_new["Year"] = output_new.Year.astype(str)
output_new.sort_values(by=["Date", "Team_1"], inplace=True)
output_new.reset_index(drop=True, inplace=True)
# output.loc[output.index.isin(gd_y_test.index), "Usage"] = "Testing"

gd_pred = selected_gd_model.predict(data[gd_features])
gt_pred = selected_gt_model.predict(data[gt_features])

gd_weight = 1 #.05
gt_weight = 1 #.15

## add weights?
output_new["Predicted_score_1"] = (gd_weight * (gt_pred + gd_pred) / 2).round()
output_new["Predicted_score_2"] = (gt_weight * (gt_pred - gd_pred) / 2).round()

## use earlier predictions where available
output = output_prev.combine_first(output_new)
output = output[output_prev.columns]
# print(output.shape)

output["Predicted_goal_diff"] = output.Predicted_score_1 - output.Predicted_score_2
output["Predicted_goal_total"] = output.Predicted_score_1 + output.Predicted_score_2
output["Predicted_result"] = None
output.loc[output.Predicted_score_1 == output.Predicted_score_2, "Predicted_result"] = "Draw"
output.loc[output.Predicted_score_1 > output.Predicted_score_2, "Predicted_result"] = "Win"
output.loc[output.Predicted_score_1 < output.Predicted_score_2, "Predicted_result"] = "Loss"

output["Correct_result"] = (output.Actual_result == output.Predicted_result).astype(int)
output["Correct_goal_diff"] = (output.Actual_goal_diff == output.Predicted_goal_diff).astype(int)
output["Correct_score"] = ((output.Actual_score_1 == output.Predicted_score_1) & (output.Actual_score_2 == output.Predicted_score_2)).astype(int)
output["Points"] = output.Correct_result + output.Correct_goal_diff + output.Correct_score

for col in ["Correct_result", "Correct_goal_diff", "Correct_score", "Points"]:
    output.loc[pd.isnull(output.Actual_result), col] = np.nan

output.to_csv("../data/interim/intl_02_predictions_live.csv", index=False)
output.describe(include="all").T

# In[15]:

def agg_by_col(df, col, asc=True):
    """
    INPUT:
        df - Match-level output dataframe
        col - Column to aggregate by
        asc - Sort ascending (True) or descending (False)

    OUTPUT:
        agg - Aggregated dataframe
    """
    agg = pd.concat([
        df[pd.notnull(df.Actual_result)][col].value_counts().sort_index(),
        df[pd.notnull(df.Actual_result)].groupby(col)[["Points", "Correct_result", "Correct_goal_diff", "Correct_score",
                                                                  "Predicted_goal_total", "Actual_goal_total"]].mean(),
        df[pd.notnull(df.Actual_result) & (df.Predicted_result != "Draw")][col].value_counts() / df[pd.notnull(df.Actual_result)][col].value_counts(),
        df[pd.notnull(df.Actual_result) & (df.Actual_result != "Draw")][col].value_counts() / df[pd.notnull(df.Actual_result)][col].value_counts(),
    ], axis=1)
    agg.sort_index(ascending=asc, inplace=True)
    agg.columns = ['Matches played', 'Points per game', '% correct result',
       '% correct goal diff', '% correct score', 'Goals per game (predicted)',
       'Goals per game (actual)', '% games won (predicted)',
       '% games won (actual)']
#     print(agg.columns)

    return agg

overall = pd.DataFrame({
    "Matches played": output[pd.notnull(output.Actual_result)].shape[0],
    "Points per game": output[pd.notnull(output.Actual_result)].Points.mean(),
    "% correct result": output[pd.notnull(output.Actual_result)].Correct_result.mean(),
    "% correct goal diff": output[pd.notnull(output.Actual_result)].Correct_goal_diff.mean(),
    "% correct score": output[pd.notnull(output.Actual_result)].Correct_score.mean(),
    "Goals per game (predicted)": output[pd.notnull(output.Actual_result)].Predicted_goal_total.mean(),
    "Goals per game (actual)": output[pd.notnull(output.Actual_result)].Actual_goal_total.mean(),
    "% games won (predicted)": output[pd.notnull(output.Actual_result) & (output.Predicted_result != "Draw")].shape[0] / output[pd.notnull(output.Actual_result)].shape[0],
    "% games won (actual)": output[pd.notnull(output.Actual_result) & (output.Actual_result != "Draw")].shape[0] / output[pd.notnull(output.Actual_result)].shape[0],
}, index=["Overall"])
# print(overall.columns)

summary = pd.concat([agg_by_col(output, "Year"), agg_by_col(output, "Usage", asc=False), overall], axis=0)
summary = summary.round(2)
pct_cols = []
for col in summary.columns:
    if col.startswith("%"):
        pct_cols.append(col)
summary[pct_cols] = (100 * summary[pct_cols]).astype(int).astype(str) + "%"

summary

# In[16]:

output.loc[output.Usage == "Live"].describe().dropna(axis=1, how="any").T

# In[ ]:

max_stars_repo_name: deacona/the-ball-is-round

max_issues_repo_name: deacona/the-ball-is-round

max_forks_repo_name: deacona/the-ball-is-round


text:
# Switching the magnetic field coils and recording the magnetic field change

from nidaqmx import AnalogInputTask, DigitalOutputTask
from time import sleep, strftime
from numpy import savetxt, linspace, column_stack

####################################
## Settings

# Analog Input rate (1/s)
airate = 300000

# Recorded points
numpoints = 30000

####################################

class Relay:

    def __init__(self, channel, value=0):
        ''' Simple relay class

        Input params:
        channel : name of digital channel. E.g. "Dev2/port1/line0"
        value=0 : starting value of the output.
        '''
        self._relay = DigitalOutputTask()
        self._relay.create_channel(channel)
        self.set(value)

    def set(self, value):
        ''' Set the output of the digital channel

        Input params:
        value : set output to this [0, 1]
        '''
        if value in [0, 1]:
            self.value = value
            self._relay.write(self.value,auto_start=True,timeout=1)
        else:
            raise(ValueError)

    def toggle(self):
        ''' Toggle the relay between the two possible valies'''
        self.set(1-self.value)

class Sensors:

    def __init__(self, channels, minv=-10.0, maxv=10.0, srate=1000, snum=1000):
        self._sensor = AnalogInputTask()
        self._sensor.create_voltage_channel(channels, min_val=minv,
            max_val=maxv)
        self._sensor.configure_timing_sample_clock(rate=srate,
            sample_mode='finite', samples_per_channel=snum)

    def start(self):
        self._sensor.start()

    def stop(self):
        self._sensor.stop()

    def wait_until_done(self, timeout = 10):
        self._sensor.wait_until_done(timeout)

    def read(self):
        return self._sensor.read()

# Setup hardware
relay = Relay("Dev2/port1/line0", value = 0)
mag = Sensors("Dev2/ai0:2", minv = -5.0, maxv = 5.0,
    srate = airate, snum = numpoints)

# Method:
# 1) Turn on magnetic field, wait until settles.
# 2) Start recording
# 3) Turn off relay
# 4) Read data

relay.set(value = 1)
sleep(2)

mag.start()
sleep(0.01)
relay.toggle()
mag.wait_until_done(timeout=2)
aidata = mag.read()

tstep = 1.0/airate
tottime = (numpoints - 1) * tstep
timedata = linspace(0, tottime, numpoints)

# Setting up output file
datafile = "gauss_%s.log" %(strftime("%y%m%d_%H%M%S"))
out = file(datafile, 'a')
out.write("#Time(s) Voltage(V):Ch1 / Ch2 / Ch3\n")
savetxt(out, column_stack((timedata, aidata)))
out.close()

del mag
del relay

print "Finished: %s" %(datafile)

max_stars_repo_name: imrehg/labhardware

max_issues_repo_name: imrehg/labhardware

max_forks_repo_name: imrehg/labhardware

reason: from numpy


text:
#!/usr/bin/env python
try:
    from setuptools import setup, Extension
except ImportError :
    raise ImportError("setuptools module required, please go to https://pypi.python.org/pypi/setuptools and follow the instructions for installing setuptools")

class NumpyExtension(Extension):

    def __init__(self, *args, **kwargs):
        Extension.__init__(self, *args, **kwargs)

        self._include_dirs = self.include_dirs
        del self.include_dirs  # restore overwritten property

    # warning: Extension is a classic class so it's not really read-only

    def get_include_dirs(self):
        from numpy import get_include

        return self._include_dirs + [get_include()]

    def set_include_dirs(self, value):
        self._include_dirs = value

    def del_include_dirs(self):
        pass

    include_dirs = property(get_include_dirs,
                            set_include_dirs,
                            del_include_dirs)

setup(maintainer="Forest Gregg",
      version="0.3.2",
      name='dedupe-hcluster',
      packages=['hcluster'],
      maintainer_email="fgregg@datamade.us",
      description="Hierarchical Clustering Algorithms (Information Theory)",
      url="https://github.com/datamade/hcluster",
      license="SciPy License (BSD Style)",
      install_requires=['future', 'numpy'],
      ext_modules=[NumpyExtension('hcluster._hierarchy',
                                  ['hcluster/_hierarchy.c']),

                   NumpyExtension('hcluster._distance_wrap',
                                  ['hcluster/distance_wrap.c'])],
      long_description="""
This library provides Python functions for hierarchical clustering. Its features
include

    * generating hierarchical clusters from distance matrices
    * computing distance matrices from observation vectors
    * computing statistics on clusters
    * cutting linkages to generate flat clusters
    * and visualizing clusters with dendrograms.

The interface is very similar to MATLAB's Statistics Toolbox API to make code
easier to port from MATLAB to Python/Numpy. The core implementation of this
library is in C for efficiency.
""",
      keywords=['dendrogram', 'linkage', 'cluster', 'agglomorative', 'hierarchical', 'hierarchy', 'ward', 'distance'],
      classifiers = ["Topic :: Scientific/Engineering :: Information Analysis",
                     "Topic :: Scientific/Engineering :: Artificial Intelligence",
                     "Topic :: Scientific/Engineering :: Bio-Informatics",
                     "Programming Language :: Python",
                     "Operating System :: OS Independent",
                     "License :: OSI Approved :: BSD License",
                     "Intended Audience :: Science/Research",
                     "Development Status :: 4 - Beta"],
  )