text:
import argparse
import h5py
import multiprocessing as mp
import numpy as np
import os
import sys
import tensorflow as tf
import time

backend = 'Agg' if sys.platform == 'linux' else 'TkAgg'
import matplotlib
matplotlib.use(backend)
import matplotlib.pyplot as plt

from contexttimer import Timer

import hgail.misc.simulation
import hgail.misc.utils

import hyperparams
import utils
from utils import str2bool

def simulate(env, policy, max_steps, render=False, env_kwargs=dict()):
    traj = hgail.misc.simulation.Trajectory()
    x = env.reset(**env_kwargs)
    policy.reset()
    for step in range(max_steps):
        if render: env.render()
        a, a_info = policy.get_action(x)
        nx, r, done, e_info = env.step(a)
        traj.add(
            policy.observation_space.flatten(x),
            a,
            r,
            a_info,
            e_info
        )
        if done: break
        x = nx
    return traj.flatten()

def mutliagent_simulate(
        env,
        policy,
        max_steps,
        render=False,
        env_kwargs=dict()):
    '''
    Description:
        - simulates a vectorized agent in a vectorized environment

    Args:
        - env: env to simulate in, must be vectorized
        - policy: policy, must be vectorized
        - max_steps: max steps in env per episode
        - render: display visual
        - env_kwargs: key word arguments to pass to env

    Returns:
        - a dictionary object with a bit of an unusual format:
            each value in the dictionary is an array with shape
            (timesteps, n_env / n_veh, shape of attribute)
            i.e., first dim corresponds to time
            second dim to the index of the agent
            third dim is for the attribute itself
    '''

    x = env.reset(**env_kwargs)
    n_agents = x.shape[0]
    traj = hgail.misc.simulation.Trajectory()
    dones = [True] * n_agents
    policy.reset(dones)
    for step in range(max_steps):
        if render: env.render()
        a, a_info = policy.get_actions(x)
        nx, r, dones, e_info = env.step(a)
        traj.add(x, a, r, a_info, e_info)
        if any(dones): break
        x = nx
    return traj.flatten()

def collect_trajectories(
        args,
        params,
        egoids,
        starts,
        trajlist,
        pid,
        env_fn,
        policy_fn,
        max_steps,
        use_hgail,
        random_seed):
    env, _, _ = env_fn(args, alpha=0.)
    policy = policy_fn(args, env)
    with tf.Session() as sess:
        # initialize variables
        sess.run(tf.global_variables_initializer())

        # then load parameters
        if use_hgail:
            for i, level in enumerate(policy):
                level.algo.policy.set_param_values(params[i]['policy'])
            policy = policy[0].algo.policy
        else:
            policy.set_param_values(params['policy'])
        normalized_env = hgail.misc.utils.extract_normalizing_env(env)
        if normalized_env is not None:
            normalized_env._obs_mean = params['normalzing']['obs_mean']
            normalized_env._obs_var = params['normalzing']['obs_var']

        # collect trajectories
        nids = len(egoids)
        for i, egoid in enumerate(egoids):
            sys.stdout.write('\rpid: {} traj: {} / {}'.format(pid, i, nids))

            if args.env_multiagent:
                kwargs = dict()
                if random_seed:
                    kwargs = dict(random_seed=random_seed+egoid)
                traj = mutliagent_simulate(
                    env,
                    policy,
                    max_steps=max_steps,
                    env_kwargs=kwargs
                )
                trajlist.append(traj)
            else:
                traj = simulate(
                    env,
                    policy,
                    max_steps=max_steps,
                    env_kwargs=dict(egoid=egoid, start=starts[egoid])
                )
                traj['egoid'] = egoid
                traj['start'] = starts[egoid]
                trajlist.append(traj)

    return trajlist

def parallel_collect_trajectories(
        args,
        params,
        egoids,
        starts,
        n_proc,
        env_fn=utils.build_ngsim_env,
        max_steps=200,
        use_hgail=False,
        random_seed=None):
    # build manager and dictionary mapping ego ids to list of trajectories
    manager = mp.Manager()
    trajlist = manager.list()

    # set policy function
    policy_fn = utils.build_hierarchy if use_hgail else utils.build_policy

    # partition egoids
    proc_egoids = utils.partition_list(egoids, n_proc)

    # pool of processes, each with a set of ego ids
    pool = mp.Pool(processes=n_proc)

    # run collection
    results = []
    for pid in range(n_proc):
        res = pool.apply_async(
            collect_trajectories,
            args=(
                args,
                params,
                proc_egoids[pid],
                starts,
                trajlist,
                pid,
                env_fn,
                policy_fn,
                max_steps,
                use_hgail,
                random_seed
            )
        )
        results.append(res)

    # wait for the processes to finish
    [res.get() for res in results]
    pool.close()
    # let the julia processes finish up
    time.sleep(10)
    return trajlist

def single_process_collect_trajectories(
        args,
        params,
        egoids,
        starts,
        n_proc,
        env_fn=utils.build_ngsim_env,
        max_steps=200,
        use_hgail=False,
        random_seed=None):
    '''
    This function for debugging purposes
    '''
    # build list to be appended to
    trajlist = []

    # set policy function
    policy_fn = utils.build_hierarchy if use_hgail else utils.build_policy
    tf.reset_default_graph()

    # collect trajectories in a single process
    collect_trajectories(
        args,
        params,
        egoids,
        starts,
        trajlist,
        1,
        env_fn,
        policy_fn,
        max_steps,
        use_hgail,
        random_seed
    )
    return trajlist

def collect(
        egoids,
        starts,
        args,
        exp_dir,
        use_hgail,
        params_filename,
        n_proc,
        max_steps=200,
        collect_fn=parallel_collect_trajectories,
        random_seed=None):
    '''
    Description:
        - prepare for running collection in parallel
        - multiagent note: egoids and starts are not currently used when running
            this with args.env_multiagent == True
    '''
    # load information relevant to the experiment
    params_filepath = os.path.join(exp_dir, 'imitate/log/{}'.format(params_filename))
    params = hgail.misc.utils.load_params(params_filepath)

    # validation setup
    validation_dir = os.path.join(exp_dir, 'imitate', 'validation')
    utils.maybe_mkdir(validation_dir)
    output_filepath = os.path.join(validation_dir, '{}_trajectories.npz'.format(
        args.ngsim_filename.split('.')[0]))

    with Timer():
        trajs = collect_fn(
            args,
            params,
            egoids,
            starts,
            n_proc,
            max_steps=max_steps,
            use_hgail=use_hgail,
            random_seed=random_seed
        )

    utils.write_trajectories(output_filepath, trajs)

def load_egoids(filename, args, n_runs_per_ego_id=1, env_fn=utils.build_ngsim_env):
    offset = args.env_H + args.env_primesteps
    basedir = os.path.expanduser('~/.julia/v0.6/NGSIM/data/')
    ids_filename = filename.replace('.txt', '-index-{}-ids.h5'.format(offset))
    ids_filepath = os.path.join(basedir, ids_filename)
    if not os.path.exists(ids_filepath):
        # this should create the ids file
        env_fn(args)
        if not os.path.exists(ids_filepath):
            raise ValueError('file unable to be created, check args')
    ids = np.array(h5py.File(ids_filepath, 'r')['ids'].value)

    # we want to sample start times uniformly from the range of possible values
    # but we also want these start times to be identical for every model we
    # validate. So we sample the start times a single time, and save them.
    # if they exist, we load them in and reuse them
    start_times_filename = filename.replace('.txt', '-index-{}-starts.h5'.format(offset))
    start_times_filepath = os.path.join(basedir, start_times_filename)
    # check if start time filepath exists
    if os.path.exists(start_times_filepath):
        # load them in
        starts = np.array(h5py.File(start_times_filepath, 'r')['starts'].value)
    # otherwise, sample the start times and save them
    else:
        ids_file = h5py.File(ids_filepath, 'r')
        ts = ids_file['ts'].value
        # subtract offset gives valid end points
        te = ids_file['te'].value - offset
        starts = np.array([np.random.randint(s,e+1) for (s,e) in zip(ts,te)])
        # write to file
        starts_file = h5py.File(start_times_filepath, 'w')
        starts_file.create_dataset('starts', data=starts)
        starts_file.close()

    # create a dict from id to start time
    id2starts = dict()
    for (egoid, start) in zip(ids, starts):
        id2starts[egoid] = start

    ids = np.tile(ids, n_runs_per_ego_id)
    return ids, id2starts

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='validation settings')
    parser.add_argument('--n_proc', type=int, default=1)
    parser.add_argument('--exp_dir', type=str, default='../../data/experiments/gail/')
    parser.add_argument('--params_filename', type=str, default='itr_2000.npz')
    parser.add_argument('--n_runs_per_ego_id', type=int, default=1)
    parser.add_argument('--use_hgail', type=str2bool, default=False)
    parser.add_argument('--use_multiagent', type=str2bool, default=False)
    parser.add_argument('--n_multiagent_trajs', type=int, default=10000)
    parser.add_argument('--debug', type=str2bool, default=False)
    parser.add_argument('--random_seed', type=int, default=None)
    parser.add_argument('--n_envs', type=int, default=None)
    parser.add_argument('--remove_ngsim_vehicles', type=str2bool, default=False)

    run_args = parser.parse_args()

    args_filepath = os.path.join(run_args.exp_dir, 'imitate/log/args.npz')
    args = hyperparams.load_args(args_filepath)
    if run_args.use_multiagent:
        args.env_multiagent = True
        args.remove_ngsim_vehicles = run_args.remove_ngsim_vehicles

    if run_args.debug:
        collect_fn = single_process_collect_trajectories
    else:
        collect_fn = parallel_collect_trajectories

    # Raunak commented this out since i80 trajectories not required for validation
#    filenames = [
#        "trajdata_i101_trajectories-0750am-0805am.txt",
#        "trajdata_i101_trajectories-0805am-0820am.txt",
#        "trajdata_i101_trajectories-0820am-0835am.txt",
#        "trajdata_i80_trajectories-0400-0415.txt",
#        "trajdata_i80_trajectories-0500-0515.txt",
#        "trajdata_i80_trajectories-0515-0530.txt"
#    ]

    filenames = [
        "trajdata_i101_trajectories-0750am-0805am.txt",
        "trajdata_i101_trajectories-0805am-0820am.txt",
        "trajdata_i101_trajectories-0820am-0835am.txt"
    ]

    if run_args.n_envs:
        args.n_envs = run_args.n_envs
    sys.stdout.write('{} vehicles with H = {}'.format(args.n_envs, args.env_H))

    for fn in filenames:
        args.ngsim_filename = fn
        if args.env_multiagent:
            # args.n_envs gives the number of simultaneous vehicles
            # so run_args.n_multiagent_trajs / args.n_envs gives the number
            # of simulations to run overall
            egoids = list(range(int(run_args.n_multiagent_trajs / args.n_envs)))
            starts = dict()
        else:
            egoids, starts = load_egoids(fn, args, run_args.n_runs_per_ego_id)
        collect(
            egoids,
            starts,
            args,
            exp_dir=run_args.exp_dir,
            params_filename=run_args.params_filename,
            use_hgail=run_args.use_hgail,
            n_proc=run_args.n_proc,
            collect_fn=collect_fn,
            random_seed=run_args.random_seed
        )


text:
import unittest
import pandas as pd
import category_encoders as ce
import numpy as np

__author__ = 'willmcginnis'

class TestDist(unittest.TestCase):
    """
    """

    def test_dist(self):
        data = np.array([
            ['apple', None],
            ['peach', 'lemon']
        ])
        encoder = ce.OrdinalEncoder(impute_missing=True)
        encoder.fit(data)
        a = encoder.transform(data)
        print(a)
        self.assertEqual(a.values[0, 1], -1)
        self.assertEqual(a.values[1, 1], 0)

        encoder = ce.OrdinalEncoder(impute_missing=False)
        encoder.fit(data)
        a = encoder.transform(data)
        self.assertTrue(np.isnan(a.values[0, 1]))
        self.assertEqual(a.values[1, 1], 0)

ext: py

max_stars_repo_name: amaotone/categorical-encoding

max_issues_repo_name: amaotone/categorical-encoding

max_forks_repo_name: amaotone/categorical-encoding

reason: import numpy


text:
import numpy as np

class Text2Vec:

    def __init__(self, datafile, ndims=0, L1_normalize=0, L2_normalize=0):
        self.datafile = datafile
        self.ndims = ndims
        self.L1_normalize = L1_normalize
        self.L2_normalize = L2_normalize

    def embedding(self, query):
        vec = self.mapping(query)
        if vec is not None:
            vec = np.array(vec)
        return vec

    def do_L1_norm(self, vec):
        L1_norm = np.linalg.norm(vec, 1)
        return 1.0 * np.array(vec) / L1_norm

    def do_L2_norm(self, vec):
        L2_norm = np.linalg.norm(vec, 2)
        return 1.0 * np.array(vec) / L2_norm

# word2vec + average pooling
class AveWord2Vec(Text2Vec):

    # datafile: the path of pre-trained word2vec data
    def __init__(self, datafile, ndims=0, L1_normalize=0, L2_normalize=0):
        Text2Vec.__init__(self, datafile, ndims, L1_normalize, L2_normalize)
        self.word2vec = BigFile(datafile)
        if ndims != 0:
            assert self.word2vec.ndims == self.ndims, "feat dimension is not match %d != %d" % (
                self.word2vec.ndims, self.ndims)
        else:
            self.ndims = self.word2vec.ndims

    def preprocess(self, query, clear):
        if clear:
            words = clean_str(query)
        else:
            words = query.strip().split()
        return words

    def mapping(self, query, clear=True):
        words = self.preprocess(query, clear)

        # print query, '->', words
        renamed, vectors = self.word2vec.read(words)
        renamed2vec = dict(zip(renamed, vectors))

        if len(renamed) != len(words):
            vectors = []
            for word in words:
                if word in renamed2vec:
                    vectors.append(renamed2vec[word])

        if len(vectors) > 0:
            vec = np.array(vectors).mean(axis=0)

            if self.L1_normalize:
                return self.do_L1_norm(vec)
            if self.L2_normalize:
                return self.do_L2_norm(vec)
            return vec
        else:
            return None

# word2vec + average pooling + fliter stop words
class AveWord2VecFilterStop(AveWord2Vec):

    # datafile: the path of pre-trained word2vec data
    def __init__(self, datafile, ndims=0, L1_normalize=0, L2_normalize=0):
        Text2Vec.__init__(self, datafile, ndims, L1_normalize, L2_normalize)
        self.word2vec = BigFile(datafile)
        if ndims != 0:
            assert self.word2vec.ndims == self.ndims, "feat dimension is not match %d != %d" % (
                self.word2vec.ndims, self.ndims)
        else:
            self.ndims = self.word2vec.ndims

    def preprocess(self, query, clear):
        if clear:
            words = clean_str_filter_stop(query)
        else:
            words = query.strip().split()
        return words

    # Bag-of-words

class BoW2Vec(Text2Vec):

    # datafile: the path of bag-of-words vocabulary file
    def __init__(self, datafile, ndims=0, L1_normalize=0, L2_normalize=0):
        Text2Vec.__init__(self, datafile, ndims, L1_normalize, L2_normalize)
        word_vob = map(str.strip, open(datafile).readlines())
        self.word2index = dict(zip(word_vob, range(len(word_vob))))
        if ndims != 0:
            assert len(word_vob) == self.ndims, "feat dimension is not match %d != %d" % (len(word_vob), self.ndims)
        else:
            self.ndims = len(word_vob)
        printStatus(INFO + '.' + self.__class__.__name__, "%d words" % self.ndims)

    def preprocess(self, query):
        return clean_str(query)

    def mapping(self, query):
        words = self.preprocess(query)

        vec = [0.0] * self.ndims
        for word in words:
            if word in self.word2index:
                vec[self.word2index[word]] += 1
            # else:
            #     print word

        if sum(vec) > 0:
            if self.L1_normalize:
                vec = self.do_L1_norm(vec)
            if self.L2_normalize:
                vec = self.do_L2_norm(vec)
            return vec

        ###############################
        # sometimes need to modify
        # else:
        #     return None
        ###############################
        else:
            return vec

# Bag-of-words + fliter stop words
class BoW2VecFilterStop(BoW2Vec):

    def preprocess(self, query):
        return clean_str_filter_stop(query)

max_stars_repo_name: ClaasM/VideoArticleRetrieval

max_issues_repo_name: ClaasM/VideoArticleRetrieval

max_forks_repo_name: ClaasM/VideoArticleRetrieval


text:
from collections import deque
import numpy as np
import argparse
import imutils
import cv2

ap = argparse.ArgumentParser()
ap.add_argument("-v", "--video", help="path to the (optional) video file")
args = vars(ap.parse_args())

greenLower = (29, 86, 6)
greenUpper = (64, 255, 255)

if not args.get("video", False):
    camera = cv2.VideoCapture(0)
else:
    camera = cv2.VideoCapture(args["video"])

while True:
    (grabbed, frame) = camera.read()

    if args.get("video") and not grabbed:
        break

    frame = imutils.resize(frame, width=600)
    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

    mask = cv2.inRange(hsv, greenLower, greenUpper)
    mask = cv2.erode(mask, None, iterations=2)
    mask = cv2.dilate(mask, None, iterations=2)

    cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]
    center = None

    if len(cnts) > 0:
        c = max(cnts, key=cv2.contourArea)
        ((x, y), radius) = cv2.minEnclosingCircle(c)

        if radius > 10:
            cv2.circle(frame, (int(x), int(y)), int(radius), (0, 255, 255), 2)

    cv2.imshow("Frame", frame)
    cv2.imshow("Mask", mask)

    key = cv2.waitKey(1) & 0xFF

    if key == ord("q"):
        break

camera.release()
cv2.destroyAllWindows()


text:
import numpy as np
import pandas as pd
import scipy.stats as st
from plotnine import *
import seaborn as sns
from sklearn.linear_model import LinearRegression
import jenkspy

#from https://stats.stackexchange.com/questions/143974/jenks-natural-breaks-in-python-how-to-find-the-optimum-number-of-breaks
def goodness_of_variance_fit(array, classes):
    # get the break points
    classes = jenkspy.jenks_breaks(array, classes)

    # do the actual classification
    classified = np.array([classify_jenks(i, classes) for i in array])

    # max value of zones
    maxz = max(classified)

    # nested list of zone indices
    zone_indices = [[idx for idx, val in enumerate(classified) if zone + 1 == val] for zone in range(maxz)]

    # sum of squared deviations from array mean
    sdam = np.sum((array - array.mean()) ** 2)

    # sorted polygon stats
    array_sort = [np.array([array[index] for index in zone]) for zone in zone_indices]

    # sum of squared deviations of class means
    sdcm = sum([np.sum((classified - classified.mean()) ** 2) for classified in array_sort])

    # goodness of variance fit
    gvf = (sdam - sdcm) / sdam

    return gvf

def classify_jenks(value, breaks):
    for i in range(1, len(breaks)):
        if value < breaks[i]:
            return i
    return len(breaks) - 1

def calc_qq(df,col):
    sample_qs = [(np.percentile(df[col],i)-np.mean(df[col]))/np.std(df[col]) for i in range(5,100,5)]
    theoretical_qs = [st.norm.ppf(i/100) for i in range(5,100,5)]
    qq = pd.DataFrame({'sample_q':sample_qs,'theoretical_q':theoretical_qs})
    reg = LinearRegression(fit_intercept=False).fit(np.array(qq['theoretical_q'])[:,None],
                                 np.array(qq['sample_q'])[:,None])
    return qq, reg

def boxcox(ser,lamb=0):
    ser+= 1 - ser.min()
    if lamb==0:
        return np.log(ser)
    else:
        return (ser**lamb - 1)/lamb

def boxcox_lamb_df(ser, ls = [i/10 for i in range(-30,31,5)]):
    coefs = []
    for l in ls:
        df = pd.DataFrame.from_dict({'val': boxcox(ser,l)})
        qq, reg = calc_qq(df,'val')
        coefs.append(reg.coef_.squeeze().item())
    return pd.DataFrame({'lamb':ls,'coef':coefs})

def boxcox_lamb(ser, ls = [i/10 for i in range(-30,31,5)]):
    df = boxcox_lamb_df(ser,ls)
    return df.lamb[df.coef.idxmax()]

def boxcox_plot(df, col, ls = [i/10 for i in range(-30,31,5)]):
    lamb_df = boxcox_lamb_df(df[col],ls)
    g = (ggplot(lamb_df, aes(x='lamb',y='coef',group=1)) +
         geom_point() + geom_line())
    return g

max_stars_repo_name: PyThaiNLP/thaimaimeex

max_issues_repo_name: PyThaiNLP/thaimaimeex

max_forks_repo_name: PyThaiNLP/thaimaimeex

reason: import numpy,import scipy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import my_nn as mnn
import excel_reader
from importlib import reload
from math import floor
import numpy as np

excel_reader = reload(excel_reader)

def get_net():
    # training data
    column_titles, x_total, y_total = excel_reader.read_excel(r'../input_file/input_file.xlsx')
    x_total = excel_reader.minmax(x_total)

    # splitting train data
    train_size = floor(len(x_total)*0.9)

    x_train = np.array(x_total[:train_size])
    x_test = np.array(x_total[train_size:])

    y_train = np.array(y_total[:train_size])
    y_test = np.array(y_total[train_size:])

    # setting up different networks with different initial values
    net1, net2, net3 = mnn.Network(), mnn.Network(), mnn.Network()
    net_list = [net1,net2,net3]
    rate_list = [0 for _ in range(len(net_list))]
    index_of_best_net = 0
    max_rate = 0

    for i,net in enumerate(net_list,start=1):

        # network architecture
        net.add(mnn.FCLayer(16, 64))
        net.add(mnn.ActivationLayer(mnn.leaky_relu, mnn.leaky_relu_derivative))
        net.add(mnn.FCLayer(64, 128))
        net.add(mnn.ActivationLayer(mnn.leaky_relu, mnn.leaky_relu_derivative))
        net.add(mnn.FCLayer(128, 16))
        net.add(mnn.ActivationLayer(mnn.leaky_relu, mnn.leaky_relu_derivative))

        # train
        net.use(mnn.mse, mnn.mse_derivative)
        net.fit(x_train, y_train, epochs=1000, learning_rate=0.01)

        # test accuracy
        out = net.predict(x_test)
        rate = 0
        size = 0

        # pick rounded value of largest output
        for j,test in enumerate(out):
            if round(max(test[0])) == y_test[j][0][0]:
                rate += 1
            size += 1
        rate /= size
        rate_list.append(rate)
        if rate > max_rate:
            max_rate = rate
            index_of_best_net = i-1
        print("\n")
        print("prediction rate:")
        print(rate)

    # get best network out of list
    best_net = net_list[index_of_best_net]

    return best_net

if __name__ == "__main__":
    get_net()

max_stars_repo_name: rdgpcampos/meal-machine-learning

max_issues_repo_name: rdgpcampos/meal-machine-learning

max_forks_repo_name: rdgpcampos/meal-machine-learning


text:
from timemachines.skatertools.data.skaterresiduals import random_multivariate_residual
import numpy as np
import pandas as pd

if __name__=='__main__':
    df = random_multivariate_residual(n_obs=1000)
    the_cov = df.cov()
    df_inv = pd.DataFrame(data=np.linalg.pinv(the_cov.values), columns=the_cov.columns, index=the_cov.index)
    print(df_inv[:3].transpose())

max_stars_repo_name: vishalbelsare/timemachines

max_issues_repo_name: vishalbelsare/timemachines

max_forks_repo_name: vishalbelsare/timemachines


text:
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""
python postprocess.py
"""
import os
import numpy as np

from src.config import config
from src.dataset import vox_generator_test

def one_hot(label, num_classes):
    """ one-hot encode """
    label_ = np.zeros([len(label), num_classes])
    label_[np.arange(len(label)), label] = 1
    return label_

def calculate_dice(true_label, pred_label, num_classes):
    """
        calculate dice

        Args:
            true_label: true sparse labels
            pred_label: predict sparse labels
            num_classes: number of classes

        Returns:
            dice evaluation index
    """
    true_label = true_label.astype(int)
    pred_label = pred_label.astype(int)
    true_label = true_label.flatten()
    true_label = one_hot(true_label, num_classes)
    pred_label = pred_label.flatten()
    pred_label = one_hot(pred_label, num_classes)
    intersection = np.sum(true_label * pred_label, axis=0)
    return (2. * intersection) / (np.sum(true_label, axis=0) + np.sum(pred_label, axis=0))

if __name__ == '__main__':
    # test dataset
    test_files = []
    with open(config.test_path) as f:
        for line in f:
            test_files.append(line[:-1])

    data_gen_test = vox_generator_test(config.data_path, test_files, config.correction)

    OFFSET_H = config.offset_height
    OFFSET_W = config.offset_width
    OFFSET_C = config.offset_channel
    HSIZE = config.height_size
    WSIZE = config.width_size
    CSIZE = config.channel_size
    PSIZE = config.pred_size

    OFFSET_PH = (config.height_size - config.pred_size) // 2
    OFFSET_PW = (config.width_size - config.pred_size) // 2
    OFFSET_PC = (config.channel_size - config.pred_size) // 2

    batches_w = int(np.ceil((config.image_width - config.width_size) / float(config.offset_width))) + 1
    batches_h = int(np.ceil((config.image_height - config.height_size) / float(config.offset_height))) + 1
    batches_c = int(np.ceil((config.image_channel - config.channel_size) / float(config.offset_channel))) + 1

    dice_whole, dice_core, dice_et = [], [], []
    data_count = 0
    post_result_path = os.path.join(config.post_result_path)
    for i in range(len(test_files)):
        print('predicting %s' % test_files[i])
        x, x_n, y = data_gen_test.__next__()
        pred = np.zeros([config.image_height, config.image_width, config.image_channel, config.num_classes])
        for hi in range(batches_h):
            offset_h = min(OFFSET_H * hi, config.image_height - HSIZE)
            offset_ph = offset_h + OFFSET_PH
            for wi in range(batches_w):
                offset_w = min(OFFSET_W * wi, config.image_width - WSIZE)
                offset_pw = offset_w + OFFSET_PW
                for ci in range(batches_c):
                    offset_c = min(OFFSET_C * ci, config.image_channel - CSIZE)
                    offset_pc = offset_c + OFFSET_PC
                    data = x[offset_h:offset_h + HSIZE, offset_w:offset_w + WSIZE, offset_c:offset_c + CSIZE, :]
                    if not np.max(data) == 0 and np.min(data) == 0:
                        t1_t1ce_score_path = os.path.join(post_result_path, "flair_t2_node_{}_1.bin".format(data_count))
                        t1_t1ce_score = np.fromfile(t1_t1ce_score_path, dtype=np.float32).reshape(1, 5, 12, 12, 12)
                        t1_t1ce_score = np.transpose(t1_t1ce_score, axes=[0, 2, 3, 4, 1])
                        pred[offset_ph:offset_ph + PSIZE, offset_pw:offset_pw + PSIZE, offset_pc:offset_pc + PSIZE, :] \
                            += np.squeeze(t1_t1ce_score)
                        data_count = data_count + 1
        pred = np.argmax(pred, axis=-1)
        pred = pred.astype(int)
        print('calculating dice...')
        whole_pred = (pred > 0).astype(int)
        whole_gt = (y > 0).astype(int)
        core_pred = (pred == 1).astype(int) + (pred == 4).astype(int)
        core_gt = (y == 1).astype(int) + (y == 4).astype(int)
        et_pred = (pred == 4).astype(int)
        et_gt = (y == 4).astype(int)
        dice_whole_batch = calculate_dice(whole_gt, whole_pred, 2)
        dice_core_batch = calculate_dice(core_gt, core_pred, 2)
        dice_et_batch = calculate_dice(et_gt, et_pred, 2)
        dice_whole.append(dice_whole_batch)
        dice_core.append(dice_core_batch)
        dice_et.append(dice_et_batch)
        print(dice_whole_batch)
        print(dice_core_batch)
        print(dice_et_batch)

    dice_whole = np.array(dice_whole)
    dice_core = np.array(dice_core)
    dice_et = np.array(dice_et)

    print('mean dice whole:')
    print(np.mean(dice_whole, axis=0))
    print('mean dice core:')
    print(np.mean(dice_core, axis=0))
    print('mean dice enhance:')
    print(np.mean(dice_et, axis=0))

max_stars_repo_name: mindspore-ai/models

max_issues_repo_name: mindspore-ai/models

max_forks_repo_name: mindspore-ai/models


text:
import numpy as np
import tensorflow as tf
from encoders_decoders import  decoder_with_fc_only

########################## generator ####################

bnorm_default = True
dropout_default = None # [0.1]

def latent_code_generator_2222(z, out_dim, layer_sizes=[256,256,256, 256 ], b_norm=False, non_linearity=tf.nn.relu, reuse=False, scope=None):
    layer_sizes = layer_sizes + out_dim
    out_signal = decoder_with_fc_only(z, layer_sizes=layer_sizes, b_norm=bnorm_default, dropout_prob=dropout_default, reuse=reuse, scope=scope)
    return out_signal

#################  Discriminator ############################

bnorm_default = True
dropout_default = [0.1]

def latent_code_discriminator_222(in_signal, layer_sizes=[256, 256, 256], b_norm=False, non_linearity=tf.nn.relu, reuse=False, scope=None):
    layer_sizes = layer_sizes + [1]
    d_logit = decoder_with_fc_only(in_signal, layer_sizes=layer_sizes, non_linearity=non_linearity, b_norm=bnorm_default, dropout_prob=dropout_default, reuse=reuse, scope=scope)
    d_prob = tf.nn.sigmoid(d_logit)
    return d_prob, d_logit

max_stars_repo_name: kangxue/LOGAN

max_issues_repo_name: kangxue/LOGAN

max_forks_repo_name: kangxue/LOGAN


text:
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
warnings.simplefilter(action='ignore', category=RuntimeWarning)

import pandas as pd
import math
import numpy as np
import scipy.stats as sci

from statsmodels.stats.proportion import proportions_ztest

from qiskit import execute
from qiskit.circuit import ClassicalRegister

def assertPhase(backend, quantumCircuit, qubits_to_assert, expected_phases, measurements_to_make, alpha):
    ## if "qubits to assert" is just a single value, convert it to a list containing the single value
    if (not isinstance(qubits_to_assert, list)):
        qubits_to_assert = [qubits_to_assert]

    ## if "expected phases" is just a single value, convert it to a list containing the single value
    if (not isinstance(expected_phases, list)):
        expected_phases = [expected_phases]

    ## needs to make at least 2 measurements, one for x axis, one for y axis
    ## realistically we need more for any statistical significance
    if (measurements_to_make < 2):
        raise ValueError("Must make at least 2 measurements")

    ## classical register must be of same length as amount of qubits to assert
    ## if there is no classical register, add one
    if (quantumCircuit.num_clbits == 0):
        quantumCircuit.add_register(ClassicalRegister(len(qubits_to_assert)))
    elif (quantumCircuit.num_clbits != len(qubits_to_assert)):
        raise ValueError("QuantumCircuit classical register length not equal to qubits to assert")

    ## divide measurements to make by 2 as we need to run measurements twice, one for x and one for y
    measurements_to_make = measurements_to_make // 2

    ## copy the circit and set measurement to y axis
    yQuantumCircuit = measure_y(quantumCircuit.copy(), qubits_to_assert)

    ## measure the x axis
    xQuantumCircuit = measure_x(quantumCircuit, qubits_to_assert)

    ## get y axis results
    yJob = execute(yQuantumCircuit, backend, shots=measurements_to_make, memory=True)
    yCounts = yJob.result().get_counts()

    ## get x axis results
    xJob = execute(xQuantumCircuit, backend, shots=measurements_to_make, memory=True)
    xCounts = xJob.result().get_counts()

    ## make a df to keep track of the predicted angles
    resDf = pd.DataFrame(columns=['+','i','-','-i'])

    ## fill the df with the x and y results of each qubit that is being asserted
    classical_qubit_index = 1
    for qubit in qubits_to_assert:
        plus_amount, i_amount, minus_amount, minus_i_amount = 0,0,0,0
        for experiment in xCounts:
            if (experiment[len(qubits_to_assert)-classical_qubit_index] == '0'):
                plus_amount += xCounts[experiment]
            else:
                minus_amount += xCounts[experiment]
        for experiment in yCounts:
            if (experiment[len(qubits_to_assert)-classical_qubit_index] == '0'):
                i_amount += yCounts[experiment]
            else:
                minus_i_amount += yCounts[experiment]
        df = {'+':plus_amount, 'i':i_amount,
              '-':minus_amount, '-i':minus_i_amount}
        resDf = resDf.append(df, ignore_index = True)
        classical_qubit_index+=1

    ## convert the columns to a strict numerical type
    resDf['+'] = resDf['+'].astype(int)
    resDf['i'] = resDf['i'].astype(int)
    resDf['-'] = resDf['-'].astype(int)
    resDf['-i'] = resDf['-i'].astype(int)

    #print(f"full df \n {resDf}")

    xAmount = resDf['-'].tolist()
    yAmount = resDf['-i'].tolist()

    # print(f"x amounts {xAmount}")
    # print(f"y amounts {yAmount}")

    ## make a dataframe that contains p values of chi square tests to analyse results
    ## if x and y counts are both 25/25/25/25, it means that we cannot calculate a phase
    ## we assume that a qubit that is in |0> or |1> position to have 50% chance to fall
    ## either way, like a coin toss: We treat X and Y results like coin tosses
    pValues = pd.DataFrame(columns=['X','Y'])
    pValues['X'] = resDf.apply(lambda row: applyChiSquareX(row, measurements_to_make/2), axis=1)
    pValues['Y'] = resDf.apply(lambda row: applyChiSquareY(row, measurements_to_make/2), axis=1)

    ## check p values on chi square test, we use a low value to be sure that
    ## we only except if we are certain there is an issue with the x, y results
    pValues = pValues > 0.00001

    ## if both pvalues are more than 0.00001, we are pretty certain that the results follow an even distribution
    ## likely that the qubit is not in the fourier basis (very likely in the |0> or |1> state)
    pValues.apply(lambda row: assertIfBothTrue(row), axis=1)

    ## this sequence of operations converts from measured results
    ## into an angle for phase:
    ## with 0   (       0 rad) signifying the |+> state
    ## with 90  (    pi/2 rad) signifying the |i> state
    ## with 180 (      pi rad) signifying the |-> state
    ## with 270 (3 * pi/2 rad) signifying the |-i> state
    resDf = resDf / measurements_to_make
    resDf = resDf * 2
    resDf = resDf - 1
    resDf = np.arccos(resDf)
    resDf = resDf * 180
    resDf = resDf / math.pi

    ## to get a final result for phase on each qubit:
    ## we must get the lowest 2 values for each column
    lowestDf = pd.DataFrame(columns=['lowest','lowest-location','second-lowest','second-lowest-location','estimated-phase'])

    ## store the lowest value as well as what column it is from
    lowestDf['lowest'] = resDf.min(axis=1)
    lowestDf['lowest-location'] = resDf.idxmin(axis=1)

    ## remove the lowest value from the dataframe
    lowestDf = lowestDf.apply(lambda row: setLowestCellToNan(row, resDf), axis=1)

    ## store the second lowest value from the dataframe as well as the column
    lowestDf['second-lowest'] = resDf.min(axis=1)
    lowestDf['second-lowest-location'] = resDf.idxmin(axis=1)

    ## estimate the phase and put it in a new column
    lowestDf['estimated-phase'] =  lowestDf.apply(lambda row: setPhaseEstimate(row), axis=1)

    ## calculate what the expected row would be for the expected phase
    #expectedX, expectedY = expectedPhaseToRow(expected_phases[0],measurements_to_make)
    expectedX = np.zeros(len(expected_phases)).astype(int)
    expectedY = np.zeros(len(expected_phases)).astype(int)

    for idx, phase in enumerate(expected_phases):
        expectedX[idx], expectedY[idx] = expectedPhaseToRow(expected_phases[idx],measurements_to_make)

    #print(type(expectedX[0]))
    # print(f"expected x {expectedX}")
    # print(f"expected y {expectedY}")

    p_values = []

    for i in range(len(qubits_to_assert)):
        ## set observed X values in a table
        observedXtable = [xAmount[i],measurements_to_make-xAmount[i]]
        ## set expected X values in a table
        expectedXtable = [expectedX[i],measurements_to_make-expectedX[i]]

        ## set observed Y values in a table
        observedYtable = [yAmount[i],measurements_to_make-yAmount[i]]
        ## set expected Y values in a table
        expectedYtable = [expectedY[i],measurements_to_make-expectedY[i]]

        xPvalue = sci.chisquare(f_obs=observedXtable, f_exp=expectedXtable)[1]

        yPvalue = sci.chisquare(f_obs=observedYtable, f_exp=expectedYtable)[1]

        #print(observedXtable)
        #print(expectedXtable)
        #print(sci.chisquare(f_obs=observedXtable, f_exp=expectedXtable)[1])

        #print(observedYtable)
        #print(expectedYtable)
        print(sci.chisquare(f_obs=observedYtable, f_exp=expectedYtable)[1])

        p_values.append(xPvalue)
        p_values.append(yPvalue)

        #if (yPvalue != np.NaN and yPvalue <= alpha):
        #    raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in qubit {qubits_to_assert[i]} according to significance level {alpha}"))
        #if (xPvalue != np.NaN and xPvalue <= alpha):
        #    raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in qubit {qubits_to_assert[i]} according to significance level {alpha}"))

    holm_bonferroni_correction(p_values, alpha)

    # ## set observed X values in a table
    # observedX = [xAmount,measurements_to_make-xAmount]
    # ## set expected X values in a table
    # expectedX = [expectedX,measurements_to_make-expectedX]

    # ## set observed Y values in a table
    # observedY = [yAmount,measurements_to_make-yAmount]
    # ## set expected Y values in a table
    # expectedY = [expectedY,measurements_to_make-expectedY]

    # print(observedX)
    # print(expectedX)

    # print(observedY)
    # print(expectedY)

    # print(sci.chisquare(f_obs=observedX, f_exp=expectedX)[1])
    # xPvalue = sci.chisquare(f_obs=observedX, f_exp=expectedX)[1]

    # print(sci.chisquare(f_obs=observedY, f_exp=expectedY)[1])
    # yPvalue = sci.chisquare(f_obs=observedY, f_exp=expectedY)[1]

    # if (yPvalue != np.NaN and yPvalue <= alpha):
    #     raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in the qubits according to significance level {alpha}"))
    # if (xPvalue != np.NaN and xPvalue <= alpha):
    #     raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in the qubits according to significance level {alpha}"))

## assert that 2 qubits are equal
def assertEqual(backend, quantumCircuit, qubit1, qubit2, measurements_to_make, alpha):
    ## needs to make at least 2 measurements, one for x axis, one for y axis
    ## realistically we need more for any statistical significance
    if (measurements_to_make < 2):
        raise ValueError("Must make at least 2 measurements")

    ## classical register must be of same length as amount of qubits to assert
    ## if there is no classical register, add one
    if (quantumCircuit.num_clbits == 0):
        quantumCircuit.add_register(ClassicalRegister(2))
    elif (quantumCircuit.num_clbits != 2):
        raise ValueError("QuantumCircuit classical register must be of length 2")

    ## divide measurements to make by 3 as we need to run measurements twice, one for x and one for y
    measurements_to_make = measurements_to_make // 3

    ## copy the circit and set measurement to y axis
    yQuantumCircuit = measure_y(quantumCircuit.copy(), [qubit1, qubit2])

    ## measure the x axis
    xQuantumCircuit = measure_x(quantumCircuit.copy(), [qubit1, qubit2])

    ## measure the z axis
    zQuantumCircuit = measure_z(quantumCircuit, [qubit1, qubit2])

    ## get y axis results
    yJob = execute(yQuantumCircuit, backend, shots=measurements_to_make, memory=True)
    yMemory = yJob.result().get_memory()
    yCounts = yJob.result().get_counts()

    ## get x axis results
    xJob = execute(xQuantumCircuit, backend, shots=measurements_to_make, memory=True)
    xMemory = xJob.result().get_memory()
    xCounts = xJob.result().get_counts()

    ## get z axis results
    zJob = execute(zQuantumCircuit, backend, shots=measurements_to_make, memory=True)
    zMemory = zJob.result().get_memory()
    zCounts = zJob.result().get_counts()

    # xDf = pd.DataFrame(columns=['q0', 'q1'])
    # yDf = pd.DataFrame(columns=['q0', 'q1'])
    # zDf = pd.DataFrame(columns=['q0', 'q1'])

    # for row in yMemory:
    #     yDf = yDf.append({'q0':row[0], 'q1':row[1]}, ignore_index=True)

    # for row in xMemory:
    #     xDf = xDf.append({'q0':row[0], 'q1':row[1]}, ignore_index=True)

    # for row in zMemory:
    #     zDf = zDf.append({'q0':row[0], 'q1':row[1]}, ignore_index=True)

    # yDf = yDf.astype(int)
    # xDf = xDf.astype(int)
    # zDf = zDf.astype(int)

    #yStat, yPvalue = (sci.ttest_ind(yDf['q0'], yDf['q1']))
    #xStat, xPvalue = (sci.ttest_ind(xDf['q0'], xDf['q1']))
    #zStat, zPvalue = (sci.ttest_ind(zDf['q0'], zDf['q1']))

    print(alpha)

    #print(f"Y p-value {yPvalue}")
    #print(f"X p-value {xPvalue}")
    #print(f"Z p-value {zPvalue}")

    # if (yPvalue != np.NaN and yPvalue <= alpha):
    #     raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in the qubits according to significance level {alpha}"))
    # if (xPvalue != np.NaN and xPvalue <= alpha):
    #     raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in the qubits according to significance level {alpha}"))
    # if (zPvalue != np.NaN and zPvalue <= alpha):
    #     raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in the qubits according to significance level {alpha}"))

    ## make a df to keep track of the predicted angles
    resDf = pd.DataFrame(columns=['0','1','+','i','-','-i'])

    ## fill the df with the x and y results of each qubit that is being asserted
    classical_qubit_index = 1
    for qubit in [qubit1,qubit2]:
        zero_amount, one_amount, plus_amount, i_amount, minus_amount, minus_i_amount = 0,0,0,0,0,0
        for experiment in xCounts:
            if (experiment[2-classical_qubit_index] == '0'):
                plus_amount += xCounts[experiment]
            else:
                minus_amount += xCounts[experiment]
        for experiment in yCounts:
            if (experiment[2-classical_qubit_index] == '0'):
                i_amount += yCounts[experiment]
            else:
                minus_i_amount += yCounts[experiment]
        for experiment in zCounts:
            if (experiment[2-classical_qubit_index] == '0'):
                zero_amount += zCounts[experiment]
            else:
                one_amount += zCounts[experiment]
        df = {'0':zero_amount, '1':one_amount,
              '+':plus_amount, 'i':i_amount,
              '-':minus_amount,'-i':minus_i_amount}
        resDf = resDf.append(df, ignore_index = True)
        classical_qubit_index+=1

    ## convert the columns to a strict numerical type
    resDf['+'] = resDf['+'].astype(int)
    resDf['i'] = resDf['i'].astype(int)
    resDf['-'] = resDf['-'].astype(int)
    resDf['-i'] = resDf['-i'].astype(int)
    resDf['0'] = resDf['0'].astype(int)
    resDf['1'] = resDf['1'].astype(int)

    print(resDf.astype(str))

    print(resDf['1'][0].astype(int))
    print(resDf['1'][1].astype(int))

    print(resDf['-'][0].astype(int))
    print(resDf['-'][1].astype(int))

    print(resDf['-i'][0].astype(int))
    print(resDf['-i'][1].astype(int))

    zStat_z, zPvalue = proportions_ztest(count=[resDf['1'][0],resDf['1'][1]], nobs=[measurements_to_make, measurements_to_make],  alternative='two-sided')
    zStat_x, xPvalue = proportions_ztest(count=[resDf['-'][0],resDf['-'][1]], nobs=[measurements_to_make, measurements_to_make],  alternative='two-sided')
    zStat_y, yPvalue = proportions_ztest(count=[resDf['-i'][0],resDf['-i'][1]], nobs=[measurements_to_make, measurements_to_make],  alternative='two-sided')

    print(zPvalue, zStat_z)
    print(xPvalue, zStat_x)
    print(yPvalue, zStat_y)

    if (yPvalue != np.NaN and yPvalue <= alpha):
        raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in the qubits according to significance level {alpha}"))
    if (xPvalue != np.NaN and xPvalue <= alpha):
        raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in the qubits according to significance level {alpha}"))
    if (zPvalue != np.NaN and zPvalue <= alpha):
        raise(AssertionError(f"Null hypothesis rejected, there is a significant enough difference in the qubits according to significance level {alpha}"))

    # ## apply t test to see if two populations of results on qu  bits are expected to from the same population
    # resDf.apply(lambda row: applyTtest(row), axis=1)

def measure_x(circuit, qubitIndexes):
    cBitIndex = 0
    for index in qubitIndexes:
        circuit.h(index)
        circuit.measure(index, cBitIndex)
        cBitIndex+=1
    return circuit

def measure_y(circuit, qubit_indexes):
    cBitIndex = 0
    for index in qubit_indexes:
        circuit.sdg(index)
        circuit.h(index)
        circuit.measure(index, cBitIndex)
        cBitIndex+=1
    return circuit

def measure_z(circuit, qubit_indexes):
    cBitIndex = 0
    for index in qubit_indexes:
        circuit.measure(index, cBitIndex)
        cBitIndex+=1
    return circuit

def setLowestCellToNan(row, resDf):
    for col in row.index:
        resDf.iloc[row.name][row[col]] = np.nan
    return row

def setPhaseEstimate(row):
    overallPhase = 0
    if(row['lowest-location'] == '+'):
        if(row['second-lowest-location'] == 'i'):
            overallPhase = 0 + row['lowest']
        elif (row['second-lowest-location'] == '-i'):
            overallPhase = 360 - row['lowest']
            if (row['lowest'] == 0):
                overallPhase = 0
    elif(row['lowest-location'] == 'i'):
        if(row['second-lowest-location'] == '+'):
            overallPhase = 90 - row['lowest']
        elif (row['second-lowest-location'] == '-'):
            overallPhase = 90 + row['lowest']
    elif(row['lowest-location'] == '-'):
        if(row['second-lowest-location'] == 'i'):
            overallPhase = 180 - row['lowest']
        elif (row['second-lowest-location'] == '-i'):
            overallPhase = 180 + row['lowest']
    elif(row['lowest-location'] == '-i'):
        if(row['second-lowest-location'] == '+'):
            overallPhase = 270 + row['lowest']
        elif (row['second-lowest-location'] == '-'):
            overallPhase = 270 - row['lowest']
    return overallPhase

def expectedPhaseToRow(expected_phase, number_of_measurements):
    # print(expected_phase)
    # print(number_of_measurements)
    expected_phase_y = expected_phase - 90
    expected_phase_y = expected_phase_y * math.pi
    expected_phase_y = expected_phase_y / 180
    expected_phase_y = np.cos(expected_phase_y)
    expected_phase_y = expected_phase_y + 1
    expected_phase_y = expected_phase_y / 2
    expected_phase_y = expected_phase_y * number_of_measurements
    expected_phase = expected_phase * math.pi
    expected_phase = expected_phase / 180
    expected_phase = np.cos(expected_phase)
    expected_phase = expected_phase + 1
    expected_phase = expected_phase / 2
    expected_phase = expected_phase * number_of_measurements
    # print(f"phase + {expected_phase} ---- phase - {number_of_measurements-expected_phase}")
    # print(f"phase i {expected_phase_y} ---- phase -i {number_of_measurements-expected_phase_y}")
    xRes = int(round(number_of_measurements-expected_phase))
    yRes = int(round(number_of_measurements-expected_phase_y))
    return((xRes, yRes))

def applyChiSquareX(row, expected_amount):
    observed = [row['+'],row['-']]
    expected = [expected_amount,expected_amount]
    return(sci.chisquare(f_obs=observed, f_exp=expected)[1])

def applyChiSquareY(row, expected_amount):
    observed = [row['i'],row['-i']]
    expected = [expected_amount,expected_amount]
    return(sci.chisquare(f_obs=observed, f_exp=expected)[1])

def assertIfBothTrue(row):
    if row.all():
        raise AssertionError("Qubit does not appear to have a phase applied to it!")

def holm_bonferroni_correction(p_values, family_wise_alpha):
    print("holm")
    p_values = [1 if math.isnan(x) else x for x in p_values]
    p_values.sort()
    print(p_values)
    for i in range(len(p_values)):
        if (p_values[i] <= (family_wise_alpha/(len(p_values)-i))):
            print("failed it !")
            raise(AssertionError(f"Null hypothesis rejected"))

# def assertIfExpectedDoNotFitTolerance(row, expected, tolerance):
#     deltaAngle = (row['estimated-phase'] - expected[row.name] + 180 + 360) % 360 - 180
#     print('observed: ' + str(row['estimated-phase']))
#     print('predicted: ' + str(expected[row.name]))
#     print('diff: ' + str(deltaAngle))
#     if (abs(deltaAngle) > tolerance):
#         raise AssertionError(f"The estimated angle ({row['estimated-phase']}) is off the prediction ({expected[row.name]}) +- tolerance value ({tolerance}) specified")

reason: import numpy,import scipy,from statsmodels


text:
import sulplotter as plotter
import numpy as np
from scipy.interpolate import griddata

X = [0.1, 0.2, 0.3, 0.4, 0.5]
Y = [16, 32, 64, 128, 256]

X,Y = np.meshgrid(X,Y)

coord = np.stack([X,Y], axis=-1).reshape([-1,2])
# print(coord.shape)

scr = [[60.1, 57.6, 55.4, 56.3, 58.5],
		[58.8, 54.9, 51.8, 53.1, 55.9],
		[56.1, 52.3, 48.0, 49.1, 51.2],
		[48.2, 45.7, 42.9, 44.2, 46.1],
		[49.1, 46.3, 44.1, 45.8, 48.0]]

# scr = [[47.2, 45.3, 43.1, 44.3, 46.9],
# 		[41.9, 41.1, 40.8, 41.6, 43.8],
# 		[40.8, 39.9, 37.3, 38.6, 40.1],
# 		[39.1, 38.6, 34.1, 36.6, 37.0],
# 		[39.3, 39.1, 34.6, 36.4, 36.8]]
scr = np.float32(scr).reshape([-1])

X_plt = np.arange(0.1, 0.5, 0.01)
Y_plt = np.arange(16, 256, 1)
X_plt,Y_plt = np.meshgrid(X_plt,Y_plt)
grid_z = griddata(coord, scr, (X_plt, Y_plt), method='cubic')
print(grid_z.shape)

from matplotlib import cm
plt = plotter.Surface3D(azim=45)

plt.plot(X_plt,Y_plt, grid_z,cmap=cm.coolwarm,linewidth=0, antialiased=True)
plt.set_label(['Threshold', 'Time', 'Error'])
plt.show(False)

reason: import numpy,from scipy


text:
import os
import PIL
import random
import pathlib
import shutil
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import Conv2D, Dense, MaxPooling2D, Flatten

def download_data():
    dataset_url = "https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz"
    data_dir = tf.keras.utils.get_file('flower_photos', origin=dataset_url, untar=True)
    data_dir = pathlib.Path(data_dir)

    return data_dir

def preprocessing(train_dir, val_dir = None):
    train_datagen = ImageDataGenerator(
        rescale=1. / 255,
        rotation_range=40,
        width_shift_range=0.2,
        height_shift_range=0.2,
        shear_range=0.2,
        zoom_range=0.2,
        fill_mode="nearest",
        horizontal_flip=True
    )
    train_generator = train_datagen.flow_from_directory(
        train_dir,
        target_size=(180, 180),
        batch_size=32,
        class_mode="binary"
    )

    if val_dir is not None:
        validation_datagen = ImageDataGenerator(rescale=1. / 255)

        validation_generator = validation_datagen.flow_from_directory(
            val_dir,
            target_size=(180, 180),
            batch_size=32,
            class_mode="binary"
        )

        return train_generator, validation_generator
    return train_generator

def split_data(SOURCE, TRAINING, TESTING, SPLIT_SIZE):
    x = os.listdir(SOURCE)
    files = random.sample(x, len(x))
    num = int(len(files) * SPLIT_SIZE)
    train = [files[:num], TRAINING]
    test = [files[num:], TESTING]

    for data in [train, test]:
        for img in data[0]:
            shutil.move(f"{SOURCE}/{img}", data[1])

def create_data(data_dir):
    paths = ["flowers/training", "flowers/testing"]
    for path in paths:
        for i in ["tulips", "dandelion", "roses", "sunflowers", "daisy"]:
            if os.path.exists(os.path.join(path, i)):
                pass
            else:
              os.makedirs(os.path.join(path, i))

    train_roses_dir = "flowers/training/roses"
    train_daisy_dir = "flowers/training/daisy"
    train_tulips_dir = "flowers/training/tulips"
    train_dandelion_dir = "flowers/training/dandelion"
    train_sunflowers_dir = "flowers/training/sunflowers"

    test_roses_dir = "flowers/testing/roses"
    test_daisy_dir = "flowers/testing/daisy"
    test_tulips_dir = "flowers/testing/tulips"
    test_dandelion_dir = "flowers/testing/dandelion"
    test_sunflowers_dir = "flowers/testing/sunflowers"

    split_size = .8

    split_data(os.path.join(data_dir, "roses"), train_roses_dir, test_roses_dir, split_size)
    split_data(os.path.join(data_dir, "daisy"), train_daisy_dir, test_daisy_dir, split_size)
    split_data(os.path.join(data_dir, "tulips"), train_tulips_dir, test_tulips_dir, split_size)
    split_data(os.path.join(data_dir, "dandelion"), train_dandelion_dir, test_dandelion_dir, split_size)
    split_data(os.path.join(data_dir, "sunflowers"), train_sunflowers_dir, test_sunflowers_dir, split_size)

def architecture():

    model = Sequential([
        Conv2D(32, 3, activation="relu", input_shape=(180, 180, 3)),
        MaxPooling2D((2, 2)),
        Conv2D(64, 3, activation="relu"),
        MaxPooling2D((2, 2)),
        Conv2D(64, 3, activation="relu"),
        MaxPooling2D((2, 2)),
        Conv2D(128, 3, activation="relu"),
        MaxPooling2D((2, 2)),
        Flatten(),
        Dense(128, activation='relu'),
        Dense(5, activation="softmax")
    ])

    model.compile(optimizer="adam", #RMSprop(lr=0.001),
                  loss='sparse_categorical_crossentropy',
                  metrics=['acc'])

    return model

if __name__ == '__main__':

    data_dir = download_data()
    create_data(data_dir)
    train_dir = "flowers/training"
    val_dir = "flowers/testing"
    train_generator, validation_generator = preprocessing(train_dir, val_dir)
    model = architecture()
    model.summary()

    history = model.fit_generator(train_generator,
                                  epochs=15,
                                  verbose=1,
                                  validation_data=validation_generator)


text:
#!/usr/bin/env python

import time
import numpy as np
from scipy import signal as sig

import arrays as ar
import feature as feat

# ================================================================
# Public functions
# ================================================================

def learnFFfromSeq(seq, Lmin, Lmax, Lfilt=0):
	"""
	Finds the repeating pattern in `seq`.

	Parameters
	----------
	seq : 2D array
		2D array whose rows are time steps and whose columns are dimensions
		of the time series. If there is only one dimension, ensure that it
		is a column vector (so that it is 2D).
	Lmin : int > 0
		The minimum length that an instance of the pattern could be Must be
		> Lmax / 2.
	Lmax : int > 0
		The maximum length that an instance of the pattern could be. Must be
		< 2 * Lmin.
	Lfilt : int, optional
		The width of the hamming filter used to blur the feature matrix. If
		set to < 1, will default to Lmin.

	Returns
	-------
	startIdxs : 1D array
		The times (rows) in seq in which the estimated pattern instances
		begin (inclusive).
	endIdxs : 1D array
		The times (rows) in seq in which the estimated pattern instances
		end (non-inclusive).
	model : 2D array
		A learned model of the pattern. Can be seen as a digital filter
		that, when multiplied with a window of data, yields the log odds of
		that window being an instance of the pattern instead of iid
		Bernoulli random variables (ignoring overlap constraints). In
		practice, this is returned so that it can be plotted to show what
		features are selected.
	X : 2D array
		The feature matrix
	Xblur : 2D array
		The blurred feature matrix
	"""

	Lmin = int(len(seq) * Lmin) if Lmin < 1. else Lmin
	Lmax = int(len(seq) * Lmax) if Lmax < 1. else Lmax
	Lfilt = int(len(seq) * Lfilt) if Lfilt < 1. else Lfilt

	if not Lfilt or Lfilt < 0:
		Lfilt = Lmin

	# extend the first and last values out so that features using
	# longer windows are present for more locations
	padLen = Lmax
	seq = feat.extendSeq(seq, padLen, padLen)

	# build the feature matrix and blurred feature matrix
	X = feat.buildFeatureMat(seq, Lmin, Lmax)
	X, Xblur = feat.preprocessFeatureMat(X, Lfilt)

	# undo padding after constructing feature matrix
	X = X[:, padLen:-padLen]
	Xblur = Xblur[:, padLen:-padLen]
	seq = seq[padLen:-padLen]

	# catch edge case where all nonzeros in a row were in the padding
	keepRowIdxs = ar.nonzeroRows(X, thresh=1.)
	X = X[keepRowIdxs]
	Xblur = Xblur[keepRowIdxs]

	# feature matrices must satisfy these (if you plan on using your own)
	assert(np.min(X) >= 0.)
	assert(np.max(X) <= 1.)
	assert(np.min(Xblur) >= 0.)
	assert(np.max(Xblur) <= 1.)
	assert(np.all(np.sum(X, axis=1) > 0))
	assert(np.all(np.sum(Xblur, axis=1) > 0))

	startIdxs, endIdxs, bsfFilt = _learnFF(seq, X, Xblur, Lmin, Lmax, Lfilt)
	return startIdxs, endIdxs, bsfFilt, X, Xblur

# ================================================================
# Private functions
# ================================================================

def _dotProdsWithAllWindows(x, X):
	"""Slide x along the columns of X and compute the dot product"""
	return sig.correlate2d(X, x, mode='valid').flatten()

def _findAllInstancesFromSeedLoc(X, Xblur, seedStartIdx, seedEndIdx,
	Lmin, Lmax, Lfilt, p0, p0blur, logs_0, bsfScore=0):

	windowLen = seedEndIdx - seedStartIdx # assume end idx not inclusive

	# ================================ candidate location generation

	x0 = Xblur[:, seedStartIdx:seedEndIdx]
	dotProds = _dotProdsWithAllWindows(x0, X)

	# compute best locations to try and then sort them in decreasing order
	bestIdxs = ar.nonOverlappingMaxima(dotProds, Lmin)
	sortIdxs = np.argsort(dotProds[bestIdxs])[::-1]
	idxs = bestIdxs[sortIdxs]

	# ================================ now figure out which idxs should be instances

	# initialize counts
	idx = idxs[0]
	counts = np.copy(X[:, idx:idx+windowLen])
	countsBlur = np.copy(Xblur[:, idx:idx+windowLen])

	bestOdds = -np.inf
	bestFilt = None
	bestLocs = None
	for i, idx in enumerate(idxs[1:]):
		k = i + 2.

		# update counts
		window = X[:, idx:idx+windowLen]
		windowBlur = Xblur[:, idx:idx+windowLen]
		counts += window
		countsBlur += windowBlur

		# pattern odds
		theta_1 = countsBlur / k
		logs_1 = np.log(theta_1)
		logs_1[np.isneginf(logs_1)] = -999 # any non-inf number--will be masked by counts
		logDiffs = (logs_1 - logs_0)
		gains = counts * logDiffs # *must* use this so -999 is masked
		threshMask = gains > 0
		threshMask *= theta_1 > .5
		filt = logDiffs * threshMask
		logOdds = np.sum(counts * filt)

		# nearest enemy odds
		nextWindowOdds = -np.inf
		if k < len(idxs):
			idx = idxs[k]
			nextWindow = X[:, idx:idx+windowLen]
			nextWindowOdds = np.sum(filt * nextWindow) * k

		# subtract nearest enemy or "expected" enemy from noise
		randomOdds = np.sum(filt) * p0blur * k
		penalty = max(randomOdds, nextWindowOdds)
		logOdds -= penalty

		if logOdds > bestOdds:
			bestOdds = logOdds
			bestFilt = np.copy(filt)
			bestLocs = idxs[:k]

	return bestOdds, bestLocs, bestFilt

def _findInstancesUsingSeedLocs(X, Xblur, seedStartIdxs, seedEndIdxs, Lmin,
	Lmax, Lfilt, windowLen=None):

	# precompute feature mat stats so that evaluations of each seed
	# don't have to duplicate work
	p0, p0blur = np.mean(X), np.mean(Xblur)
	featureMeans = np.mean(Xblur, axis=1, keepdims=True)
	theta_0 = np.ones((Xblur.shape[0], windowLen)) * featureMeans
	logs_0 = np.log(theta_0)

	# main loop; evaluate each seed loc and return locs from the best one
	bsfScore = 0
	bsfFilt, bsfLocs = None, None
	for i in range(len(seedStartIdxs)):
		score, locs, filt = _findAllInstancesFromSeedLoc(X, Xblur,
			seedStartIdxs[i], seedEndIdxs[i], Lmin, Lmax, Lfilt,
			p0=p0, p0blur=p0blur, logs_0=logs_0, bsfScore=bsfScore)

		if score > bsfScore:
			bsfScore = score
			bsfFilt = np.copy(filt)
			bsfLocs = np.copy(locs)

	return bsfScore, bsfLocs, bsfFilt

def _extractTrueLocs(X, Xblur, bsfLocs, bsfFilt, windowLen, Lmin, Lmax):

	if bsfFilt is None:
		print "WARNING: _extractTrueLocs(): received None as filter"
		return np.array([0]), np.array([1])

	# compute the total filter weight in each column, ignoring low values
	bsfFiltWindow = np.copy(bsfFilt)
	sums = np.sum(bsfFiltWindow, axis=0)

	# subtract off the amount of weight that we'd expect in each column by chance
	kBest = len(bsfLocs)
	p0 = np.mean(Xblur)
	expectedOnesFrac = np.power(p0, kBest-1)
	expectedOnesPerCol = expectedOnesFrac * X.shape[0]
	sums -= expectedOnesPerCol

	# pick the optimal set of indices to maximize the sum of sequential column sums
	start, end, _ = ar.maxSubarray(sums)

	# ensure we picked at least Lmin points
	sumsLength = len(sums)
	while end - start < Lmin:
		nextStartVal = sums[start-1] if start > 0 else -np.inf
		nextEndVal = sums[end] if end < sumsLength else -np.inf
		if nextStartVal > nextEndVal:
			start -= 1
		else:
			end += 1
	# ensure we picked at most Lmax points
	while end - start > Lmax:
		if sums[start] > sums[end-1]:
			end -= 1
		else:
			start += 1

	locs = np.sort(np.asarray(bsfLocs))
	return locs + start, locs + end

def _computeAllSeedIdxsFromPair(seedIdxs, numShifts, stepLen, maxIdx):
	for idx in seedIdxs[:]:
		for shft in range(numShifts):
			seedIdxs += [idx + stepLen * shft, idx - stepLen * shft]

	seedIdxs = np.sort(np.unique(seedIdxs))
	return seedIdxs[(seedIdxs >= 0) * (seedIdxs <= maxIdx)]

def _learnFF(seq, X, Xblur, Lmin, Lmax, Lfilt, generateSeedsStep=.1):

	padLen = (len(seq) - X.shape[1]) // 2
	X = ar.addZeroCols(X, padLen, prepend=True)
	X = ar.addZeroCols(X, padLen, prepend=False)
	Xblur = ar.addZeroCols(Xblur, padLen, prepend=True)
	Xblur = ar.addZeroCols(Xblur, padLen, prepend=False)

	timeStartSeed = time.clock()

	# find seeds; i.e., candidate instance indices from which to generalize
	numShifts = int(1. / generateSeedsStep) + 1
	stepLen = int(Lmax * generateSeedsStep)
	windowLen = Lmax + stepLen

	# score all subseqs based on how much they don't look like random walks
	# when examined using different sliding window lengths
	scores = np.zeros(len(seq))
	for dim in range(seq.shape[1]):
		# compute these just once, not once per length
		dimData = seq[:, dim].flatten()
		std = np.std(dimData[1:] - dimData[:-1])
		for divideBy in [1, 2, 4, 8]:
			partialScores = feat.windowScoresRandWalk(dimData, Lmin // divideBy, std)
			scores[:len(partialScores)] += partialScores

	# figure out optimal pair based on scores of all subseqs
	bestIdx = np.argmax(scores)
	start = max(0, bestIdx - Lmin)
	end = min(len(scores), start + Lmin)
	scores[start:end] = -1 # disqualify idxs within Lmin of bestIdx
	secondBestIdx = np.argmax(scores)

	# compute all seed idxs from this pair
	seedIdxs = [bestIdx, secondBestIdx]
	maxIdx = X.shape[1] - windowLen - 1
	seedStartIdxs = _computeAllSeedIdxsFromPair(seedIdxs, numShifts, stepLen, maxIdx)
	seedEndIdxs = seedStartIdxs + windowLen

	timeEndSeed = time.clock()

	bsfScore, bsfLocs, bsfFilt = _findInstancesUsingSeedLocs(X, Xblur,
		seedStartIdxs, seedEndIdxs, Lmin, Lmax, Lfilt, windowLen=windowLen)

	startIdxs, endIdxs = _extractTrueLocs(X, Xblur, bsfLocs, bsfFilt,
		windowLen, Lmin, Lmax)

	timeEndFF = time.clock()
	print "learnFF(): seconds to find seeds, regions, total =\n\t{:.3f}\t{:.3f}\t{:.3f}".format(
		timeEndSeed - timeStartSeed, timeEndFF - timeEndSeed, timeEndFF - timeStartSeed)

	return startIdxs, endIdxs, bsfFilt

max_stars_repo_name: dblalock/flock

max_issues_repo_name: dblalock/flock

max_forks_repo_name: dblalock/flock


text:
#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from PIL import Image
import cv2
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import os
import glob
import re
import os
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras import Model
from tensorflow.keras.optimizers import RMSprop
print(tf.__version__)

#Listing all the files present
dirr=os.getcwd()
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

#Checking for availability of GPU
device_name = tf.test.gpu_device_name()
if "GPU" not in device_name:
    print("GPU device not found")
print('Found GPU at: {}'.format(device_name))
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True

# In[ ]:

class augment(object):

    def __init__(self, DATA_DIR, rescale,rotation_range,width_shift_range,height_shift_range,shear_range,zoom_range,horizontal_flip,fill_mode,batch_size,class_mode):
        """

            Input: DATA_DIR: Data Directory
                   Others: Various variable to be used for In-Memory Augmentation

        """

        self.rescale=rescale
        if self.rescale is None:
            self.rescale=1.0/255.

        self.rotation_range=rotation_range
        if self.rotation_range is None:
            self.rotation_range=40

        self.width_shift_range=width_shift_range
        if self.width_shift_range is None:
            self.width_shift_range=0.2

        self.height_shift_range=height_shift_range
        if self.height_shift_range is None:
            self.height_shift_range=0.2

        self.shear_range= shear_range
        if self.shear_range is None:
            self.shear_range=0.2

        self.zoom_range= zoom_range
        if self.zoom_range is None:
            self.zoom_range=0.2

        self.horizontal_flip=horizontal_flip
        if self.horizontal_flip is None:
            self.horizontal_flip=True

        self.fill_mode=fill_mode
        if self.fill_mode is None:
            self.fill_mode='nearest'

        self.batch_size=batch_size
        if self.batch_size is None:
            self.batch_size=20

        self.class_mode= class_mode
        if self.class_mode is None:
            self.class_mode='binary'

        self.train_dir = os.path.join(DATA_DIR,"train")
        if not os.path.isdir(self.train_dir):
            raise IOError(f"{self.train_dir} doesn't exist")

        self.val_dir = os.path.join(DATA_DIR,"validation")
        if not os.path.isdir(self.val_dir):
            raise IOError(f"{self.val_dir} doesn't exist")

    def train_aug(self):

        """ Image Augementation in Training Images"""
        train_datagen=ImageDataGenerator(rescale=self.rescale, rotation_range=self.rotation_range,
                                width_shift_range=self.width_shift_range,
                                height_shift_range=self.height_shift_range,
                                shear_range=self.shear_range,
                                zoom_range=self.zoom_range,
                                horizontal_flip=self.horizontal_flip,
                                fill_mode=self.fill_mode)

        train_generator=train_datagen.flow_from_directory(self.train_dir,target_size=(300,300),batch_size=self.batch_size,
                                                 class_mode=self.class_mode)

        return train_generator

    def val_aug(self):

        """ Image Augementation in Validation Images"""

        val_datagen=ImageDataGenerator(rescale=self.rescale)

        val_generator=val_datagen.flow_from_directory(self.val_dir,target_size=(300,300),batch_size=20,class_mode=self.class_mode)
        return val_generator

# In[ ]:

class Net(object):

    def __init__(self, input_dim,dropout):
        """
        Input:

                input_dim:size of image
                batch_size
        """

        self.input_dim=input_dim
        if self.input_dim is None:
            self.input_dim=(300,300,3)

        self.dropout=dropout
        if self.dropout is None:
            self.dropout=0.2

        #self.batch_size=batch_size
        #if self.batch_size is None:
            #self.batch_size=64

        print(" Model Training Initiated")
        print(" image_size:", self.input_dim)
        print(" dropout:", self.dropout)

    def model_cnn(self):

        """
            a). Model Trained From Scratch With 3 CNN and 3 Max Pool Layers
            b). Model Compliation with RMSProp Algorithm and binary_crossentropy loss function

        """

        from tensorflow.keras.optimizers import RMSprop
        l1_l2=tf.keras.regularizers.l1_l2(l1=0.01, l2=0.01)

        model=tf.keras.models.Sequential([
                                        tf.keras.layers.InputLayer(input_shape=self.input_dim),
                                        tf.keras.layers.Lambda(lambda x : tf.image.resize(x, [100,100])),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Conv2D(16,(1,1),activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Conv2D(32,(1,1),activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Conv2D(64,(1,1),activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Conv2D(128,(2,2),activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.MaxPooling2D(2,2),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Conv2D(256,(2,2),activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.MaxPooling2D(2,2),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Conv2D(512,(2,2),activation='relu'),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Conv2D(512,(2,2),activation='relu'),
                                        tf.keras.layers.Dropout(self.dropout),

                                        tf.keras.layers.Conv2D(1024,(2,2),activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Conv2D(1024,(2,2),activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Conv2D(1024,(2,2),activation='relu'),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Conv2D(2048,(3,3),activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Conv2D(2048,(3,3),activation='relu'),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Conv2D(2048,(3,3),activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.MaxPooling2D(2,2),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Flatten(),
                                        tf.keras.layers.Dense(1024,activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dense(1024,activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Dense(1024,activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dense(512,activation='relu'),
                                        tf.keras.layers.BatchNormalization(),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Dense(512,activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dense(512,activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dense(256,activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dense(256,activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dense(128,activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dropout(self.dropout),
                                        tf.keras.layers.Dense(128,activation='relu'),
                                        tf.keras.layers.BatchNormalization(axis=-1),
                                        tf.keras.layers.Dense(64,activation='relu'),
                                        tf.keras.layers.Dense(32,activation='relu'),
                                        tf.keras.layers.Dense(1,activation='sigmoid')
                ])
        model.summary()

        model.compile(optimizer=RMSprop(0.001),loss='binary_crossentropy', metrics=['acc'])

        return model

# In[ ]:

class myCallback(tf.keras.callbacks.Callback):

    """
        Function: Used to Stop Training When Desired Accuracy is Achieved

    """
    def on_epoch_end(self, epoch, logs={}):
        if(logs.get('val_acc')>0.87):
            print("\nReached 99% accuracy so cancelling training!")
            self.model.stop_training = True

# In[ ]:

class trainer(augment,Net,object):

    def __init__(self,epoch, input_dim,dropout,batch_size,class_mode,DATA_DIR, rescale=1.0/255.,rotation_range=40,width_shift_range=0.2,height_shift_range=0.2,shear_range=0.2,zoom_range=0.2,horizontal_flip=True,fill_mode='nearest'):

        Net.__init__(self,input_dim,dropout)
        augment.__init__(self,DATA_DIR, rescale,rotation_range,width_shift_range,height_shift_range,shear_range,zoom_range,horizontal_flip,fill_mode,batch_size,class_mode)

        self.batch_size=batch_size
        if self.batch_size is None:
            self.batch_size=20

        self.class_mode=class_mode
        if self.class_mode is None:
            self.class_mode='binary'

        self.epochs=epoch
        if self.epochs is None:
            self.epochs=100

    def model_cnn(self):

        """
            Function: Inherits Model from Class Net

        """
        model=Net.model_cnn(self)
        return model

    def train_aug(self):

        """
            Function: Inherits Augmented Function for Training Images from Class Augment

        """
        train_gen= augment.train_aug(self)
        return train_gen

    def val_aug(self):

        """
            Function: Inherits Augmented Function for Validation Images from Class Augment

        """
        val_gen=augment.val_aug(self)
        return val_gen

    def train(self):

        """
            Function: Trains and saves the model

        """

        model=self.model_cnn()
        train_generator=self.train_aug()
        test_generator=self.val_aug()

        callbacks=myCallback()
        device_name = tf.test.gpu_device_name()
        if "GPU" not in device_name:
            print("GPU device not found")
            history = model.fit_generator(train_generator,epochs=self.epochs,validation_data=test_generator,verbose=2,callbacks=[callbacks])
        else:
            print('Found GPU at: {}'.format(device_name))
            config = tf.compat.v1.ConfigProto()
            config.gpu_options.allow_growth = True
            with tf.device('/gpu:0'):
                history = model.fit_generator(train_generator,epochs=self.epochs,validation_data=test_generator,validation_steps = 13,verbose=2,callbacks=[callbacks])

        model.save('final_model_train1.hdf5')
        self.visualize(history)

    def visualize(self,history):
        """
            Function: Outputs Plots of Model's Metric

        """
        acc=history.history['acc']
        val_acc=history.history['val_acc']
        epochs=range(len(acc))
        plt.plot(epochs,acc)
        plt.plot(epochs,val_acc)
        plt.show()

# In[ ]:

if __name__=="__main__":
    path_dir='/kaggle/input/horses-or-humans-dataset/horse-or-human'
    tester=trainer(epoch=200,input_dim=(300,300,3),dropout=0.3,batch_size=20,class_mode='binary',DATA_DIR=path_dir,rescale=1.0/255.,rotation_range=40,width_shift_range=0.2,height_shift_range=0.2,shear_range=0.2,zoom_range=0.2,horizontal_flip=True,fill_mode='nearest')
    tester.train()


text:
import matplotlib
matplotlib.use('Agg')

a = """(0, array([0]))
(0, array([0]))
(0, array([0]))
(0, array([0]))
(0, array([0]))
(0, array([0]))
(0, array([0]))
(0, array([0]))
(1, array([1]))
(1, array([1]))
(1, array([1]))
(1, array([1]))
(1, array([1]))
(1, array([1]))
(1, array([1]))
(1, array([1]))
(2, array([2]))
(2, array([2]))
(2, array([2]))
(2, array([2]))
(2, array([2]))
(2, array([2]))
(2, array([2]))
(3, array([3]))
(3, array([3]))
(3, array([3]))
(3, array([3]))
(3, array([3]))
(3, array([3]))
(3, array([3]))
(3, array([3]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(4, array([4]))
(5, array([5]))
(5, array([5]))
(5, array([5]))
(5, array([5]))
(5, array([5]))
(5, array([5]))
(5, array([5]))
(6, array([6]))
(4, array([6]))
(6, array([6]))
(6, array([6]))
(4, array([6]))
(6, array([6]))
(6, array([6]))
(6, array([6]))
(6, array([6]))
(6, array([6]))
(6, array([6]))
(6, array([6]))
(6, array([6]))
(4, array([6]))
(6, array([6]))
"""

a = a.replace('(', '') \
	.replace(')', '') \
	.replace('array', '') \
	.replace('[', '') \
	.replace(']', '') \
	.split('\n')[:-1]

a = [x.split(',') for x in a]
print(a)

import numpy as np

a = np.array([[int(x[0]), int(x[1])] for x in a])
import matplotlib.pyplot as plt

from sklearn import svm, datasets
from sklearn.model_selection import train_test_split
#from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import confusion_matrix

# import some data to play with

# Split the data into a training set and a test set

# Run classifier, using a model that is too regularized (C too low) to see
# the impact on the results

# Plot non-normalized confusion matrix
titles_options = [("Confusion matrix, without normalization", None),
				  ("Normalized confusion matrix", 'true')]

labels = [
"add_milk",
"add_sugar",
"add_tea_bag",
"add_water",
"nothing",
"stir",
"toggle_on_off"]

def plot_confusion_matrix(cm,
						  target_names,
						  title='Confusion matrix',
						  cmap=None,
						  normalize=True):
	"""
	given a sklearn confusion matrix (cm), make a nice plot

	Arguments
	---------
	cm:           confusion matrix from sklearn.metrics.confusion_matrix

	target_names: given classification classes such as [0, 1, 2]
				  the class names, for example: ['high', 'medium', 'low']

	title:        the text to display at the top of the matrix

	cmap:         the gradient of the values displayed from matplotlib.pyplot.cm
				  see http://matplotlib.org/examples/color/colormaps_reference.html
				  plt.get_cmap('jet') or plt.cm.Blues

	normalize:    If False, plot the raw numbers
				  If True, plot the proportions

	Usage
	-----
	plot_confusion_matrix(cm           = cm,                  # confusion matrix created by
															  # sklearn.metrics.confusion_matrix
						  normalize    = True,                # show proportions
						  target_names = y_labels_vals,       # list of names of the classes
						  title        = best_estimator_name) # title of graph

	Citiation
	---------
	http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html

	"""

	import matplotlib.pyplot as plt
	import numpy as np
	import itertools

	accuracy = np.trace(cm) / float(np.sum(cm))
	misclass = 1 - accuracy

	if cmap is None:
		cmap = plt.get_cmap('Blues')

	plt.figure(figsize=(8, 6))
	plt.imshow(cm, interpolation='nearest', cmap=cmap)
	plt.title(title)
	plt.colorbar()

	if target_names is not None:
		tick_marks = np.arange(len(target_names))
		plt.xticks(tick_marks, target_names, rotation=45)
		plt.yticks(tick_marks, target_names)

	if normalize:
		cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

	thresh = cm.max() / 1.5 if normalize else cm.max() / 2
	for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
		if normalize:
			plt.text(j, i, "{:0.4f}".format(cm[i, j]),
					 horizontalalignment="center",
					 color="white" if cm[i, j] > thresh else "black")
		else:
			plt.text(j, i, "{:,}".format(cm[i, j]),
					 horizontalalignment="center",
					 color="white" if cm[i, j] > thresh else "black")

	plt.tight_layout()
	plt.ylabel('True label')
	plt.xlabel('Predicted label\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))
	plt.savefig("tea_making_cm.png")

print("a shape", a.T.shape)
print("a[0]", a.T[0].shape, a.T[1].shape)
cm = confusion_matrix(y_true=a.T[1], y_pred=a.T[0])

plot_confusion_matrix(cm, target_names=labels, title="I3D Accuracy")

max_stars_repo_name: AssistiveRoboticsUNH/TCG-Classifier

max_issues_repo_name: AssistiveRoboticsUNH/TCG-Classifier

max_forks_repo_name: AssistiveRoboticsUNH/TCG-Classifier


text:
from pyrender import Mesh, Scene, Viewer, Node, OrthographicCamera
from io import BytesIO
import numpy as np
import trimesh, requests, time, math

scene = Scene(bg_color = [0, 0, 0])

mesh_source = "https://github.com/zekihub/3dmodels/raw/master/models/3d/bin/heart.glb"
mesh = trimesh.load(BytesIO(requests.get(mesh_source).content), file_type = 'glb')
meshInstance = Mesh.from_trimesh(list(mesh.geometry.values())[0])
meshNode = Node(mesh = meshInstance, matrix = np.eye(4))
scene.add_node(meshNode)

camera = OrthographicCamera(xmag = 1.0, ymag = 1.0)
cameraNode = Node(camera = camera, matrix = np.eye(4))
scene.add_node(cameraNode)

zoom = 2.5
x = 0.0
y = 0.3
cam_pose = np.array([
    [1.0, 0.0, 0.0, x],
    [0.0, 1.0, 0.0, y],
    [0.0, 0.0, 1.0, zoom],
    [0.0, 0.0, 0.0, 1.0]
])
scene.set_pose(cameraNode, pose = cam_pose)

viewer = Viewer(scene,
                run_in_thread = True,
                record = True,
                rotate = True,
                use_raymond_lighting = True,
                use_direct_lighting = True,
                rotate_rate = math.pi / 2.0,
                refresh_rate = 10,
                lighting_intensity = 5.0,
                rotate_axis = [0, 1, 0])

time.sleep(4)

viewer.close_external()

viewer.save_gif("out.gif")


text:
import csv
import numpy
import tkinter as tk
from tkinter.filedialog import askopenfilename
reacttime = []
corratio = []
Standard_Deviation = []

# GUI
root = tk.Tk()
root.wm_title("4_Nback")
root.geometry("700x200")
path=tk.StringVar()
w0=tk.Label(root, text='4_Nback',width=40)
w0.place(x=190,y=10)
w1=tk.Label(root, text='姓名：').place(x=10, y=60)
name=tk.Entry(root, width=60)
name.place(x=100,y=60)
w2=tk.Label(root, text="时间：").place(x=10, y=90)
time=tk.Entry(root, width=60)
time.place(x=100, y=90)
w3=tk.Label(root, text='文件路径：').place(x=10, y=120)
path1=tk.Entry(root, textvariable=path, width=60)
path1.place(x=100, y=120)
def selectpath():
    path_ = askopenfilename()
    path.set(path_)
def filepathget():
    global filename,time,name
    name=name.get()
    time=time.get()
    filename=path1.get()
    # print(filename)
    root.withdraw()
    root.destroy()
b1 = tk.Button(root, text='确定', font=('Arial', 12), width=10,command=filepathget)
b1.place(x=300, y=150)
b2 = tk.Button(root,text="选择文件",font=('Arial', 10),width=7, height=1, command=selectpath)
b2.place(x=530, y=120)
root.mainloop()

print("文件开始处理。")
# 文件数据读取
f = open(filename, 'r', encoding='utf-8')
filereader = csv.reader(f)
back0_rt_list=[]
back0_rt_list1=[]
back1_corr_list=[]
back2_corr_list=[]
back1_rt_list=[]
back2_rt_list=[]
for i,line in enumerate(filereader):
    if i==0:
        for j in range(len(line)):
            if line[j] == 'key_resp_0back.rt':
                back0_num=j
            if line[j] == 'key_resp_1back.rt':
                back1_rt_num=j
            if line[j] == 'key_resp_1back.corr':
                back1_corr_num=j
            if line[j] == 'key_resp_2back.rt':
                back2_rt_num=j
            if line[j] == 'key_resp_2back.corr':
                back2_corr_num=j
    back0_rt=str(line[back0_num]).strip()
    back1_rt=str(line[back1_rt_num]).strip()
    back1_corr=str(line[back1_corr_num]).strip()
    back2_rt=str(line[back2_rt_num]).strip()
    back2_corr = str(line[back2_corr_num]).strip()
    back0_rt_list1.append(back0_rt)
    if back1_corr == '1' or back1_corr == '0':
        back1_corr_list.append(back1_corr)
    if back2_corr =='1' or back2_corr == '0':
        back2_corr_list.append(back2_corr)
    if back1_corr == '1' :
        back1_rt_list.append(back1_rt)
    if back2_corr== '1' :
        back2_rt_list.append(back2_rt)
for i in back0_rt_list1:
    try  :
        j = eval(i)
        back0_rt_list.append(j)
    except:
        pass
# print(back0_rt_list)
# print(back1_rt_list)
# print(back2_rt_list)
# print(back1_corr_list)
# print(back2_corr_list)

# 求反应时平均值及方差
def aver(data):
    raw_rt=[]
    for i in range(len(data)):
        raw_rt.append(float(data[i]))
    aver=numpy.mean(raw_rt)
    std1=numpy.std(raw_rt)
    return aver,std1

# 求正确率
def correct(data):
    right=0
    total=0
    for i in range(len(data)):
        if  data[i] =='1':
            total += 1
            right += 1
        else :
            total += 1
    ratio=right/total
    ratio=ratio*100
    return  ratio

# 计算结果
total_rt=[back0_rt_list,back1_rt_list,back2_rt_list]
total_corr=[back1_corr_list,back2_corr_list]
for i in range(len(total_rt)):
    aver_con,std=aver(total_rt[i])
    reacttime.append(aver_con)
    Standard_Deviation.append(std)
for i in total_corr:
    corratio_con=correct(i)
    corratio.append(corratio_con)
print(reacttime)
print(Standard_Deviation)
print(corratio)

corratio_print=[100]+corratio
back_rt_list=['0back','1back','2back']
# 数据输出
print("—————————————————————————————")
print("    Nback       准确率        反应时         标准差")
print("—————————————————————————————")
for i in range(0,3):
    print("   {}        {:.2f}%       {:.5f}        {:.5f}".format(back_rt_list[i],corratio_print[i],reacttime[i],Standard_Deviation[i]))
print("—————————————————————————————")

# 绘图
import matplotlib.pyplot as plt
plt.style.use('ggplot')
#解决中文显示问题
plt.rcParams['font.sans-serif'] = ['KaiTi'] # 指定默认字体
plt.rcParams['axes.unicode_minus'] = False # 解决保存图像是负号'-'显示为方块的问题
# 平均反应时
fig=plt.figure()
ax1=fig.add_subplot(1,1,1)
ax1.bar(back_rt_list, reacttime, align='center', color='darkblue',width=0.3)
ax1.errorbar(x=back_rt_list,y=reacttime,yerr=Standard_Deviation,ecolor='grey',capsize=4,color='black',fmt='o')
# ax1.bar(expression,corratio,align='center',color='red')
ax1.xaxis.set_ticks_position("bottom")
ax1.yaxis.set_ticks_position("left")
plt.xlabel('Nback')
plt.ylabel('时间(秒)')
plt.title('图1：平均反应时间（秒）')
plt.savefig('reaction_time.png',dpi=400,bbox_inches='tight')
plt.show()
# 平均准确率
fig1=plt.figure()
ax2=fig1.add_subplot(1,1,1)
ax2.bar(back_rt_list,corratio_print,align='center',color='darkblue',width=0.3)
ax2.xaxis.set_ticks_position("bottom")
ax2.yaxis.set_ticks_position("left")
plt.xlabel('Nback')
plt.ylabel('准确率（%）')
plt.title('图2：平均准确率')
plt.savefig('correction_ratio.png',dpi=400,bbox_inches='tight')
plt.show()

# 绘制pdf文档
from fpdf import FPDF
pdf = FPDF('p','mm','A4')
pdf.add_page()
pdf.add_font('hei','',fname=r'C:\Windows\Fonts\simhei.ttf',uni=True)
pdf.set_font("hei", size=18)
pdf.cell(180, 20, txt="电脑认知测试四：短时记忆能力", ln=1, align="C")
pdf.set_font('hei',size=12)
pdf.cell(180,13,txt='姓名：'+name,ln=1)
pdf.cell(180,13,txt='时间：'+time,ln=1)
pdf.cell(180,7,txt='———————————————————————',ln=1,align="C")
pdf.cell(180,7,txt='Nback       准确率        反应时(秒)        标准差',ln=1,align='C')
pdf.cell(180,7,txt='———————————————————————',ln=1,align='C')
for i in range(0,3):
    text="{}        {:.2f}%        {:.5f}       {:.5f}".format(back_rt_list[i],corratio_print[i],reacttime[i],Standard_Deviation[i])
    pdf.cell(180,7,txt=text,ln=1,align="C")
pdf.cell(180,7,txt='———————————————————————',ln=1,align='C')
pdf.image('reaction_time.png',x=0,y=160,w=100,h=86)
pdf.image('correction_ratio.png',x=110,y=160,w=100,h=86)
pdf.output("4_Nback.pdf")

print("文件处理完成。")


text:
import json
import os
from typing import List

import numpy as np
from tqdm import tqdm

from mmhuman3d.core.conventions.keypoints_mapping import convert_kps
from mmhuman3d.data.data_structures.human_data import HumanData
from mmhuman3d.utils.transforms import rotmat_to_aa
from .base_converter import BaseModeConverter
from .builder import DATA_CONVERTERS

@DATA_CONVERTERS.register_module()
class EftConverter(BaseModeConverter):
    """Eft dataset `Exemplar Fine-Tuning for 3D Human Pose Fitting Towards In-
    the-Wild 3D Human Pose Estimation' 3DV'2021 More details can be found in
    the `paper.

    <https://arxiv.org/pdf/2004.03686.pdf>`__ .

    Args:
        modes (list): 'coco_all', 'coco_part', 'mpii' and/or 'lspet' for
        accepted modes
    """
    ACCEPTED_MODES = ['coco_all', 'coco_part', 'mpii', 'lspet']

    def __init__(self, modes: List = []) -> None:
        super(EftConverter, self).__init__(modes)
        self.json_mapping_dict = {
            'coco_all':
            ['coco_2014_train_fit/COCO2014-All-ver01.json', 'train2014/'],
            'coco_part':
            ['coco_2014_train_fit/COCO2014-Part-ver01.json', 'train2014/'],
            'lspet': ['LSPet_fit/LSPet_ver01.json', ''],
            'mpii': ['MPII_fit/MPII_ver01.json', 'images/']
        }

    @staticmethod
    def center_scale_to_bbox(center: List[float], scale: float) -> List[float]:
        """obtain bbox from center and scale with pixel_std=200."""
        w, h = scale * 200, scale * 200
        x, y = center[0] - w / 2, center[1] - h / 2
        return [x, y, w, h]

    def convert_by_mode(self, dataset_path: str, out_path: str,
                        mode: str) -> dict:
        """
        Args:
            dataset_path (str): Path to directory where raw images and
            annotations are stored.
            out_path (str): Path to directory to save preprocessed npz file
            mode (str): Mode in accepted modes

        Returns:
            dict:
                A dict containing keys image_path, bbox_xywh, keypoints2d,
                keypoints2d_mask, smpl stored in HumanData() format
        """
        # use HumanData to store all data
        human_data = HumanData()
        image_path_, bbox_xywh_, keypoints2d_ = [], [], []
        smpl = {}
        smpl['betas'] = []
        smpl['body_pose'] = []
        smpl['global_orient'] = []

        if mode in self.json_mapping_dict.keys():
            annot_file = os.path.join(dataset_path,
                                      self.json_mapping_dict[mode][0])
            image_prefix = self.json_mapping_dict[mode][1]
        else:
            raise ValueError('provided dataset is not in eft fittings')

        with open(annot_file, 'r') as f:
            eft_data = json.load(f)
        eft_data_all = eft_data['data']

        for data in tqdm(eft_data_all):
            beta = np.array(data['parm_shape']).reshape(10)
            # 3D rotation matrix for 24 joints (24, 3, 3)
            pose_rotmat = np.array(data['parm_pose'])
            pose_rotmat = rotmat_to_aa(pose_rotmat)

            bbox_scale = data['bbox_scale']
            bbox_center = data['bbox_center']
            bbox_xywh = self.center_scale_to_bbox(bbox_center, bbox_scale)

            # (49, 3) keypoints2d in image space according to SPIN format
            gt_keypoint_2d = data['gt_keypoint_2d']

            image_name = data['imageName']

            smpl['body_pose'].append(pose_rotmat[1:].reshape((23, 3)))
            smpl['global_orient'].append(pose_rotmat[0].reshape(-1, 3))
            smpl['betas'].append(beta)

            # store data
            image_path_.append(image_prefix + image_name)
            bbox_xywh_.append(bbox_xywh)
            keypoints2d_.append(gt_keypoint_2d)

        bbox_xywh_ = np.array(bbox_xywh_).reshape((-1, 4))
        bbox_xywh_ = np.hstack([bbox_xywh_, np.ones([bbox_xywh_.shape[0], 1])])
        smpl['body_pose'] = np.array(smpl['body_pose']).reshape((-1, 23, 3))
        smpl['global_orient'] = np.array(smpl['global_orient']).reshape(
            (-1, 3))
        smpl['betas'] = np.array(smpl['betas']).reshape((-1, 10))
        keypoints2d_ = np.array(keypoints2d_).reshape((-1, 49, 3))
        keypoints2d_, mask = convert_kps(keypoints2d_, 'smpl_49', 'human_data')
        human_data['image_path'] = image_path_
        human_data['bbox_xywh'] = bbox_xywh_
        human_data['keypoints2d_mask'] = mask
        human_data['keypoints2d'] = keypoints2d_
        human_data['smpl'] = smpl
        human_data['config'] = 'eft'
        human_data.compress_keypoints_by_mask()

        # store the data struct
        if not os.path.isdir(out_path):
            os.makedirs(out_path)

        out_file = os.path.join(out_path, 'eft_{}.npz'.format(mode))
        human_data.dump(out_file)