text:
import networkx as nx
import unittest
from .dag_util import (binarize_dag, is_binary,
                       unbinarize_dag,
                       remove_edges_via_dijkstra,
                       shrink_by_transitive_closure)
from .test_lst_dag import get_example_6
from .interactions import InteractionsUtil
from nose.tools import assert_equal, assert_true

def _get_example_dag():
    # When create structure
    g = nx.DiGraph()
    g.add_nodes_from(range(1, 13))
    g.add_edges_from([(1, 2), (1, 3), (1, 7),
                      (2, 4), (2, 5), (2, 6),
                      (2, 7), (3, 8), (3, 9),
                      (5, 10), (5, 11), (7, 12),
                      (8, 12), (2, 8)])
    # When add weights
    for n in g.nodes():
        g.node[n][InteractionsUtil.VERTEX_REWARD_KEY] = 1
    g.node[2][InteractionsUtil.VERTEX_REWARD_KEY] = 11  # some special treatment

    for s, t in g.edges():
        g[s][t][InteractionsUtil.EDGE_COST_KEY] = 1
    g[1][2][InteractionsUtil.EDGE_COST_KEY] = 10  # some special treatment

    return g

def test_binarize_dag():
    g = _get_example_dag()

    binary_g = binarize_dag(g,
                            vertex_weight_key=InteractionsUtil.VERTEX_REWARD_KEY,
                            edge_weight_key=InteractionsUtil.EDGE_COST_KEY,
                            dummy_node_name_prefix='d_')
    expected_nodes = range(1, 13) + ['d_{}'.format(i) for i in range(1, 5)]
    assert_equal(
        sorted(expected_nodes),
        sorted(binary_g.nodes())
    )

    expected_edges = [(1, 'd_1'), (1, 7), ('d_1', 2),
                      ('d_1', 3), (3, 8), (3, 9),
                      (2, 'd_2'), (2, 'd_4'),
                      ('d_4', 'd_3'), ('d_4', 7),
                      ('d_2', 4), ('d_2', 8),
                      ('d_3', 6), ('d_3', 5),
                      (7, 12), (8, 12),
                      (5, 10), (5, 11)]
    assert_equal(
        sorted(expected_edges),
        sorted(binary_g.edges())
    )

    node_rewards = [1, 11] + [1] * 10 + [0] * 4
    for r, n in zip(node_rewards, expected_nodes):
        assert_equal(r, binary_g.node[n][InteractionsUtil.VERTEX_REWARD_KEY])

    edge_costs = [0, 1, 10,
                  1, 1, 1,
                  0, 0,
                  0, 1,
                  1, 1,
                  1, 1,
                  1, 1,
                  1, 1]
    for c, (s, t) in zip(edge_costs, expected_edges):
        assert_equal(c, binary_g[s][t][InteractionsUtil.EDGE_COST_KEY])

    assert_true(is_binary(binary_g))

    # check `dummy` attribute is ON for dummy nodes
    for n in g.nodes():
        if isinstance(n, basestring) and n.startswith('d_'):
            assert_true(g.node[n].get('dummy'))

def test_unbinarize_dag():
    g1 = _get_example_dag()
    binary_g = binarize_dag(g1,
                            vertex_weight_key=InteractionsUtil.VERTEX_REWARD_KEY,
                            edge_weight_key=InteractionsUtil.EDGE_COST_KEY,
                            dummy_node_name_prefix='d_')
    g2 = unbinarize_dag(binary_g,
                        edge_weight_key=InteractionsUtil.EDGE_COST_KEY,
    )
    assert_equal(sorted(g1.nodes()), sorted(g2.nodes()))
    assert_equal(sorted(g1.edges()), sorted(g2.edges()))
    match_func = lambda a1, a2: a1.items() == a2.items()
    assert_true(nx.is_isomorphic(g1, g2,
                                 node_match=match_func,
                                 edge_match=match_func))

class EdgeRemovalTest(unittest.TestCase):
    def test_remove_edges_via_dijkstra(self):
        g, _, _ = get_example_6()
        expected_edges = set(g.edges()) - set([(1, 4)])
        g.add_edges_from([(3, 4, {'c': 1}),
                          (4, 5, {'c': 1})])

        t = remove_edges_via_dijkstra(g, source=0)
        assert_equal(expected_edges, set(t.edges()))

class ShrinkTransitiveClosureTestCase(unittest.TestCase):
    def setUp(self):
        g = nx.DiGraph()
        g.add_path(range(5))
        g = nx.transitive_closure(g)
        for s, t in g.edges_iter():
            g[s][t]['c'] = 1
        self.g = g

    # def test_shirnk_by_transitive_closure(self):
    #     new_g = shrink_by_transitive_closure(
    #         self.g,
    #         sorted(self.g.nodes()),
    #         cost_key='c'
    #     )
    #     raise

reason: import networkx


text:
import random
import glob
import matplotlib.pyplot as plt
import numpy as np
from skimage.io import imsave
from skimage.transform import radon, iradon
from sparse_ct.tool import plot_grid
from sparse_ct.data import image_to_sparse_sinogram, ellipses_to_sparse_sinogram
from sparse_ct.reconstructor_2d import IRadonReconstructor

pwd_train = '/external/CT_30_000/train'
pwd_test = '/external/CT_30_000/test'

file_list_train = glob.glob(pwd_train+'/*/*/*.png')
file_list_test = glob.glob(pwd_test+'/*/*/*.png')

# fname = '/external/CT_30_000/train/Images_png_02/Images_png/000089_08_01/038.png'
# '/external/CT_30_000/test/Images_png_56/004457_01_01/027.png'

gt, sinogram, theta, FOCUS = ellipses_to_sparse_sinogram('train', channel=1,
        n_proj=100, size=512, angle1=0.0, angle2=180.0, noise_pow=33.0 )

# for i in range(100):
# fname = random.choice(file_list_train)
# gt, sinogram, theta, FOCUS = image_to_sparse_sinogram(fname, channel=1,
#         n_proj=128, size=512, angle1=0.0, angle2=180.0, noise_pow=12.0 )

# plt.figure()
# plt.imshow(sinogram, cmap='gray')#, vmin=0.0, vmax=1.0)
# plt.figure()
# plt.imshow(gt, cmap='gray', vmin=0.0, vmax=1.0)
# plt.figure()
# plt.imshow(iradon(sinogram, theta=theta), cmap='gray')
# plt.show()

recon_fbp = IRadonReconstructor('FBP')
fbp_img = recon_fbp.calc(sinogram, theta)

imgs = [gt, fbp_img, gt, fbp_img, gt, fbp_img,]

# plot_grid(
#         imgs, FOCUS=None, show=True, number_of_rows=3
# )
# plot_grid(
#         imgs, FOCUS=FOCUS, show=True, number_of_rows=3
# )
plot_grid(
        imgs, ZOOM=FOCUS, show=True, number_of_rows=1, plot1d=50
)

ext: py

max_stars_repo_name: mozanunal/SparseCT

max_issues_repo_name: mozanunal/SparseCT

max_forks_repo_name: mozanunal/SparseCT

reason: import numpy


text:
import random
import json
import secrets
import numpy as np
from lib import static
import random as rd

class User:

    def __init__(self, distributions_manager, datasets_manager):
        self.lastName = str()
        self.firstName = str()
        self.age = int(round(distributions_manager.age_distribution.rvs(1)[0]))
        self.followers = int()
        self.probability = int()
        self.favorite_subjects, self.number_of_favorite_subjects = self.define_favorite_subjects(
            distributions_manager.favorite_subjects_per_user_distribution, datasets_manager.list_of_subjects)
        self.occupation, self.compensation = self.define_occupations_and_compensation(distributions_manager.salary_distribution_per_occupation)
        self.followers = int(round(distributions_manager.followers_distribution.rvs(1)[0]))
        self.id = secrets.token_urlsafe()
        self.country = None
        self.country_code2 = None
        self.continent = None
        self.region = None
        self.location = dict()
        self.city = str()

    def elastic_mapping_user_centric(self, list_message):
        sum_likes = sum([message.likes for message in list_message])
        dic = {
            "salary": self.compensation,
            "average_likes": (sum_likes/len(list_message)) if len(list_message) != 0 else 0,
            "number_of_messages": len(list_message),
            "sum_of_likes": sum_likes,
            "average_time_between": self.message_date_average(list_message)
        }
        dic.update(self.elastic_mapping_message())
        return dic

    def elastic_mapping_message(self):
        dic = {
            "user_id": self.id,
            "last_name": self.lastName,
            "occupation": self.occupation,
            "first_name": self.firstName,
            "age": self.age,
            "followers": self.followers,
            "favorite_subjects": self.favorite_subjects,
            "favorite_subjects_as_string": ' '.join(self.favorite_subjects),
            "number_of_favorite_subjects": self.number_of_favorite_subjects,
            "geo": {
                "country": self.country,
                "continent": self.continent,
                "location": self.location,
                "city": self.city,
                "country_code2": self.country_code2,
                "region": self.region
            }
        }
        return dic

    @staticmethod
    def define_favorite_subjects(number_of_favorite_subjects_distributions, list_of_subjects):
        number_of_subjects = int(round(number_of_favorite_subjects_distributions.rvs(1)[0]))
        subjects = list(np.random.choice(list_of_subjects, number_of_subjects))
        return subjects, len(subjects)

    @staticmethod
    def define_occupations_and_compensation(salary_distribution_per_occupation):
        f = open(static.OCCUPATION)
        data = json.load(f)
        geo_list = []
        for key in data.keys():
            key_list = [key] * data.get(key)
            geo_list += key_list
        occupation = rd.choice(geo_list)
        occupation_normalized = occupation.lower().replace(' ', '_')
        compensation = int(round(salary_distribution_per_occupation[occupation_normalized].rvs(1)[0]))
        modulo = compensation % 100
        compensation = compensation - modulo if compensation < 50 else compensation - modulo + 100
        return occupation, compensation

    def __str__(self):
        return 'firstName: {}, lastName: {}, age: {}, followers: {}'\
            .format(self.firstName, self.lastName, self.age, self.followers)

    @staticmethod
    # Create the users
    def creation_users(yp, distributions_manager, datasets_manager):
        if yp.users is None:
            list_users = []
        else:
            list_users = yp.users
        with open(static.FIRST_NAME) as f:
            first_names = f.read().split('\n')
        with open(static.LAST_NAME) as f:
            last_names = f.read().split('\n')

        peoples = list()
        check = set()

        for i in list_users:
            people = User(distributions_manager, datasets_manager)
            people.firstName, people.lastName = i
            peoples.append(people)
            check.add((people.firstName, people.lastName))
            people.geo_user(yp.localisations)

        for i in range(yp.number_users - len(list_users)):
            people = User(distributions_manager, datasets_manager)
            people.geo_user(yp.localisations)
            people.firstName = first_names[random.randint(0, len(first_names) - 1)]
            people.lastName = last_names[random.randint(0, len(last_names) - 1)]
            while (people.firstName, people.lastName) in check:
                people.lastName = last_names[random.randint(0, len(last_names) - 1)]
            check.add((people.firstName, people.lastName))
            peoples.append(people)

        return peoples

    @staticmethod
    # Create gaussian distribution for the recurrence of messages by user
    def probability_message_user(size):
        s = np.random.normal(static.MU, static.SIGMA, static.SIZE_GAUSSIAN)

        minus = min(s)
        maximum = max(s)

        ratio = (maximum - minus) / size

        probability = [0] * size

        for i in s:
            indices = int(round((maximum - i)/ratio))
            probability[indices - 1] += 1

        total = len(s)

        return [i/total for i in probability]

    def geo_user(self, localisation):
        geo = rd.choice(localisation)
        self.continent = geo[0]
        self.country = geo[1]
        self.region = geo[2]
        self.city = geo[3]
        self.location = {'lon': geo[4], 'lat': geo[5]}
        self.country_code2 = geo[6]

    def message_date_average(self, list_message):
        list_dates = [message.date for message in list_message]
        list_dates.sort(reverse=True)
        delta = list()
        if len(list_dates) <= 1:
            return None
        if len(list_dates) == 2:
            return (list_dates[0] - list_dates[1]).total_seconds()
        for i in range(len(list_dates) - 2):
            delta.append((list_dates[i] - list_dates[i + 1]).total_seconds())

        return sum(delta)/len(delta)

max_stars_repo_name: TweetGenerator/generate-tweet

max_issues_repo_name: MandEProjects/dataset-generator

max_forks_repo_name: MandEProjects/dataset-generator


text:
# Copyright 2021 Peng Cheng Laboratory (http://www.szpclab.com/) and FedLab Authors (smilelab.group)

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from sklearn.datasets import load_svmlight_file
import random
import os
from urllib.request import urlretrieve

from torch.utils.data import Dataset

class Adult(Dataset):
    url = "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/"
    train_file_name = "a9a"
    test_file_name = "a9a.t"
    num_classes = 2
    num_features = 123

    def __init__(self, root, train=True,
                 transform=None,
                 target_transform=None,
                 download=False):
        self.root = root
        self.train = train
        self.transform = transform
        self.target_transform = target_transform

        if not os.path.exists(root):
            os.mkdir(root)

        if self.train:
            self.full_file_path = os.path.join(self.root, self.train_file_name)
        else:
            self.full_file_path = os.path.join(self.root, self.test_file_name)

        if download:
            self.download()

        if not self._local_file_existence():
            raise RuntimeError(
                f"Adult-a9a source data file not found. You can use download=True to "
                f"download it.")

        # now load from source file
        X, y = load_svmlight_file(self.full_file_path)
        X = X.todense()  # transform

        if not self.train:
            X = np.c_[X, np.zeros((len(y), 1))]  # append a zero vector at the end of X_test

        X = np.array(X, dtype=np.float32)
        y = np.array((y + 1) / 2, dtype=np.int32)  # map elements of y from {-1, 1} to {0, 1}
        print(f"Local file {self.full_file_path} loaded.")
        self.data, self.targets = X, y

    def download(self):
        if self._local_file_existence():
            print(f"Source file already downloaded.")
            return

        if self.train:
            download_url = self.url + self.train_file_name
        else:
            download_url = self.url + self.test_file_name

        urlretrieve(download_url, self.full_file_path)

    def _local_file_existence(self):
        return os.path.exists(self.full_file_path)

    def __getitem__(self, index):
        data, label = self.data[index], self.targets[index]

        if self.transform is not None:
            data = self.transform(data)

        if self.target_transform is not None:
            label = self.target_transform(label)

        return data, label

    def __len__(self):
        return len(self.targets)

    def extra_repr(self) -> str:
        return "Split: {}".format("Train" if self.train is True else "Test")

max_stars_repo_name: KarhouTam/FedLab-benchmarks

max_issues_repo_name: KarhouTam/FedLab-benchmarks

max_forks_repo_name: KarhouTam/FedLab-benchmarks


text:
# https://www.pyimagesearch.com/2018/06/04/keras-multiple-outputs-and-multiple-losses/

from keras.models import Sequential
from keras.layers.convolutional import Conv2D
from keras.layers.pooling import MaxPool2D
from keras.optimizers import Adam

from keras.layers.core import Dense, Activation, Dropout, Flatten
from keras.utils import plot_model
from keras.callbacks import TensorBoard

from keras.datasets import cifar10
from keras.utils import np_utils
from sklearn.model_selection import train_test_split

import sys
import os
sys.path.append(os.path.abspath("/Volumes/GoogleDrive/我的云端硬盘/fashion_research/codes"))
from fashionnet2 import FashionNet

import pickle
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import to_categorical
from keras.preprocessing import image
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from tqdm import tqdm
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import to_categorical
from keras.preprocessing import image

from numpy import array
from numpy import argmax
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
#%matplotlib inline

path = "/Volumes/GoogleDrive/我的云端硬盘/fashion_research/data/labels.pkl"

with open(path, 'rb') as handle:
    temp = pickle.load(handle)

labels = pd.DataFrame(temp)

## choose top and bottom features and get one-hot code
### men from 9940.jpg
men_selected = [
    "トップス",
    "トップスcolor",
    "パンツ",
    "パンツcolor"
]
men_labels = labels.loc[9940:, men_selected]
men_labels = men_labels.iloc[:, [0,1,3,4]]

# one hot encode

# define example
data = ['cold', 'cold', 'warm', 'cold', 'hot', 'hot', 'warm', 'cold', 'warm', 'hot']
values = array(data)
print(values)
# integer encode
label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(values)
print(integer_encoded)
# binary encode
onehot_encoder = OneHotEncoder(sparse=False)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
print(onehot_encoded)
# invert first example
inverted = label_encoder.inverse_transform([argmax(onehot_encoded[0, :])])
print(inverted)

## one_code func
def one_code(data):
    values = array(data)
    print(values)
    # integer encode
    label_encoder = LabelEncoder()
    integer_encoded = label_encoder.fit_transform(values)
    print(integer_encoded)
    # binary encode
    onehot_encoder = OneHotEncoder(sparse=False)
    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
    print(onehot_encoded)
    return onehot_encoded
    # invert first example
    #inverted = label_encoder.inverse_transform([argmax(onehot_encoded[0, :])])
    #print(inverted)

men_encoded = pd.DataFrame(index = men_labels.index.values, columns=men_selected)
for i in range(men_labels.shape[1]):
    temp = one_code(men_labels.iloc[:,i])
    for j, value in enumerate(temp):
        men_encoded.iloc[j, i] = value

train_image = []
for i in tqdm(range(men_encoded.shape[0])):
    img = image.load_img('/Volumes/GoogleDrive/我的云端硬盘/fashion_research/image_data/coordinate_images/'+str(men_encoded.index.values[i])+'.jpg',target_size=(150,150,3))
    img = image.img_to_array(img)
    img = img/255
    train_image.append(img)

X = np.array(train_image)

# size=(368,276,3)
image = X
label = men_encoded

X_train, X_test, y_train, y_test = train_test_split(image, label, test_size=0.3, random_state=10)

y_train_tops = y_train.iloc[:,0]
y_train_tops_color = y_train.iloc[:,1]
y_train_pants = y_train.iloc[:,2]
y_train_pants_color = y_train.iloc[:,3]

y_test_tops = y_test.iloc[:,0]
y_test_tops_color = y_test.iloc[:,1]
y_test_pants = y_test.iloc[:,2]
y_test_pants_color = y_test.iloc[:,3]

# initialize our FashionNet multi-output network (pretrained)
EPOCHS = 50
INIT_LR = 1e-3
BS = 32
IMAGE_DIMS = (150,150,3)

model = FashionNet.build(150,150,
	numCategories_top=y_train_tops.iloc[0,].shape[0],
	numColors_top=y_train_tops_color.iloc[0,].shape[0],
	numCategories_bottom=y_train_pants.iloc[0,].shape[0],
	numColors_bottom=y_train_pants_color.iloc[0,].shape[0],
	finalAct="softmax")

# define two dictionaries: one that specifies the loss method for
# each output of the network along with a second dictionary that
# specifies the weight per loss
losses = {
	"category_output": "categorical_crossentropy",
	"color_output": "categorical_crossentropy",
	"category_output2": "categorical_crossentropy",
	"color_output2": "categorical_crossentropy",
}
lossWeights = {"category_output": 1.0, "color_output": 1.0, "category_output2": 1.0, "color_output2": 1.0}

# initialize the optimizer and compile the model
print("[INFO] compiling model...")
opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)
model.compile(optimizer=opt, loss=losses, loss_weights=lossWeights,
	metrics=["accuracy"])

# train the network to perform multi-output classification
H = model.fit(X_train,
	{"category_output": np.array(list(y_train_tops)), "color_output": np.array(list(y_train_tops_color)), "category_output2": np.array(list(y_train_pants)), "color_output2": np.array(list(y_train_pants_color))},
	validation_data=(X_test,
		{"category_output": np.array(list(y_test_tops)), "color_output": np.array(list(y_test_tops_color)), "category_output2": np.array(list(y_test_pants)), "color_output2": np.array(list(y_test_pants_color))}),
	epochs=EPOCHS,
	verbose=1)

# save the model to disk
print("[INFO] serializing network...")
model.save("cnn2.h5")

## Predict

# load model
import tensorflow as tf
from keras.models import load_model
model = load_model('/Volumes/GoogleDrive/我的云端硬盘/fashion_research/cnn2.h5',custom_objects={"tf": tf})
# summarize model.
model.summary()

# example of loading an image with the Keras API
from keras.preprocessing.image import load_img
# load the image
img = load_img('/Volumes/GoogleDrive/我的云端硬盘/fashion_research/image_data/test_img/2.jpg', target_size=(150,150,3))
# report details about the image
#print(type(img))
#print(img.format)
#print(img.mode)
#print(img.size)
# show the image
import matplotlib.pyplot as plt
#plt.imshow(img)

# example of converting an image with the Keras API
from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
from keras.preprocessing.image import array_to_img
import numpy as np
# convert to numpy array
img_array = img_to_array(img)/255
img_array = img_array.reshape((1, 150, 150, 3))
#print(img_array.dtype)
#print(img_array.shape)
# convert back to image
#img_pil = array_to_img(img_array)
#print(type(img))
## predict label
prediction = model.predict(img_array)
idx_top = np.argmax(prediction[0])
idx_top_col = np.argmax(prediction[1])
idx_bottom = np.argmax(prediction[2])
idx_bottom_col = np.argmax(prediction[3])
import pickle

# load the trained convolutional neural network and the label
# binarizer
lb_top = pickle.loads(open("/Volumes/GoogleDrive/我的云端硬盘/fashion_research/men_lb_top", "rb").read())
lb_top_col = pickle.loads(open("/Volumes/GoogleDrive/我的云端硬盘/fashion_research/men_lb_top_col", "rb").read())
lb_bottom = pickle.loads(open("/Volumes/GoogleDrive/我的云端硬盘/fashion_research/men_lb_bottom", "rb").read())
lb_bottom_col = pickle.loads(open("/Volumes/GoogleDrive/我的云端硬盘/fashion_research/men_lb_bottom_col", "rb").read())

print("Top category: %s \nTop color: %s \nBottom category: %s \nBottom color: %s" %(lb_top.classes_[idx_top], lb_top_col.classes_[idx_top_col], lb_bottom.classes_[idx_bottom], lb_bottom_col.classes_[idx_bottom_col]))

reason: import numpy,from numpy


text:
import numpy as np
import matplotlib
import os
import librosa
from dtw import dtw
import shelve
from scipy.io import wavfile
import string
import random
from fastdtw import fastdtw
from scipy.spatial.distance import euclidean

def stretch(inp_array,factor):
        new_array = np.empty(0)
        for i in range(len(inp_array)-1):
            start = inp_array[i]
            end = inp_array[i+1]
            mid = np.linspace(start, end, factor+1)[:-1]*factor
            #print mid
            new_array = np.append(new_array,mid)

        mid = np.linspace(inp_array[len(inp_array)-1], inp_array[-1], factor)*factor
        new_array = np.append(new_array,mid)
        return new_array.reshape(new_array.size)

def align_sound(fileName_inp,fileName_out,sr_long=8000,sr_short=1000):
    sr_factr = sr_long / sr_short;
    x,_ = librosa.load(fileName_inp,sr=sr_long)
    x = x.reshape(-1, 1)
    y,_ = librosa.load(fileName_out,sr=sr_long)
    y = y.reshape(-1, 1)

    x_s,_ = librosa.load(fileName_inp,sr=sr_short)
    x_s = x_s.reshape(-1, 1)
    y_s,_ = librosa.load(fileName_out,sr=sr_short)
    y_s = y_s.reshape(-1, 1)

    apply_fast  = False

    if (apply_fast):
        dist, path = fastdtw(x, y, dist=euclidean)
        path = np.transpose(path)
    else:
        cost, dist, acc, path = dtw(x_s, y_s, dist=lambda x, y: np.linalg.norm(x - y, ord=1))

    path_inp = path[0]
    path_out = path[1]
    long_path_inp = stretch(path_inp, sr_factr)
    long_path_out = stretch(path_out, sr_factr)

    filt = np.ones(500)/(500)
    long_path_inp = np.convolve(long_path_inp, filt, 'same').astype(int)
    long_path_out = np.convolve(long_path_out, filt, 'same').astype(int)

    x_sample = x[long_path_inp]
    y_sample = y[long_path_out]

    name = ''.join(random.choice(string.ascii_uppercase) for _ in range(10))
    print name

    wavfile.write("newfiles/%s_input.wav" %name,sr_long, x_sample)

    wavfile.write("newfiles/%s_output.wav" %name,sr_long, y_sample)
    return (x_sample,y_sample)

if __name__ == "__main__":
    d=shelve.open("/Users/guyknaan/my-stuff/voiceswap/code/data")["train"]
    fileName_x = "/Users/guyknaan/my-stuff/voiceswap/code/%s" %d[10][0]
    fileName_y = "/Users/guyknaan/my-stuff/voiceswap/code/%s" %d[10][1]
    os.system("play %s -r 1000 " %fileName_x)
    os.system("play %s -r 1000 " %fileName_y)
    align_sound(fileName_x,fileName_y)

max_stars_repo_name: guyKN/voiceswap

max_issues_repo_name: guyKN/voiceswap

max_forks_repo_name: guyKN/voiceswap

reason: import numpy,from scipy


text:
import numpy as np
import pytest

import taichi as ti
from taichi import approx

@ti.test()
def test_binop():
    @ti.kernel
    def foo(x: ti.i32, y: ti.i32, a: ti.template()):
        a[0] = x + y
        a[1] = x - y
        a[2] = x * y
        a[3] = ti.ti_float(x) / y
        a[4] = x // y
        a[5] = x % y
        a[6] = x**y
        a[7] = x << y
        a[8] = x >> y
        a[9] = x | y
        a[10] = x ^ y
        a[11] = x & y

    x = 37
    y = 3
    a = ti.field(ti.f32, shape=(12, ))
    b = ti.field(ti.f32, shape=(12, ))

    a[0] = x + y
    a[1] = x - y
    a[2] = x * y
    a[3] = x / y
    a[4] = x // y
    a[5] = x % y
    a[6] = x**y
    a[7] = x << y
    a[8] = x >> y
    a[9] = x | y
    a[10] = x ^ y
    a[11] = x & y

    foo(x, y, b)

    for i in range(12):
        assert a[i] == approx(b[i])

@ti.test()
def test_augassign():
    @ti.kernel
    def foo(x: ti.i32, y: ti.i32, a: ti.template(), b: ti.template()):
        for i in a:
            a[i] = x
        a[0] += y
        a[1] -= y
        a[2] *= y
        a[3] //= y
        a[4] %= y
        a[5] **= y
        a[6] <<= y
        a[7] >>= y
        a[8] |= y
        a[9] ^= y
        a[10] &= y
        b[0] = x
        b[0] /= y

    x = 37
    y = 3
    a = ti.field(ti.i32, shape=(11, ))
    b = ti.field(ti.i32, shape=(11, ))
    c = ti.field(ti.f32, shape=(1, ))
    d = ti.field(ti.f32, shape=(1, ))

    a[0] = x + y
    a[1] = x - y
    a[2] = x * y
    a[3] = x // y
    a[4] = x % y
    a[5] = x**y
    a[6] = x << y
    a[7] = x >> y
    a[8] = x | y
    a[9] = x ^ y
    a[10] = x & y
    c[0] = x / y

    foo(x, y, b, d)

    for i in range(11):
        assert a[i] == b[i]
    assert c[0] == approx(d[0])

@ti.test()
def test_unaryop():
    @ti.kernel
    def foo(x: ti.i32, a: ti.template()):
        a[0] = +x
        a[1] = -x
        a[2] = not x
        a[3] = ~x

    x = 1234
    a = ti.field(ti.i32, shape=(4, ))
    b = ti.field(ti.i32, shape=(4, ))

    a[0] = +x
    a[1] = -x
    a[2] = not x
    a[3] = ~x

    foo(x, b)

    for i in range(4):
        assert a[i] == b[i]

@ti.test()
def test_boolop():
    @ti.kernel
    def foo(a: ti.template()):
        a[0] = 0 and 0
        a[1] = 0 and 1
        a[2] = 1 and 0
        a[3] = 1 and 1
        a[4] = 0 or 0
        a[5] = 0 or 1
        a[6] = 1 or 0
        a[7] = 1 or 1
        a[8] = 1 and 1 and 1 and 1
        a[9] = 1 and 1 and 1 and 0
        a[10] = 0 or 0 or 0 or 0
        a[11] = 0 or 0 or 1 or 0

    a = ti.field(ti.i32, shape=(12, ))
    b = ti.field(ti.i32, shape=(12, ))

    a[0] = 0 and 0
    a[1] = 0 and 1
    a[2] = 1 and 0
    a[3] = 1 and 1
    a[4] = 0 or 0
    a[5] = 0 or 1
    a[6] = 1 or 0
    a[7] = 1 or 1
    a[8] = 1 and 1 and 1 and 1
    a[9] = 1 and 1 and 1 and 0
    a[10] = 0 or 0 or 0 or 0
    a[11] = 0 or 0 or 1 or 0

    foo(b)

    for i in range(12):
        assert a[i] == b[i]

@ti.test()
def test_compare_fail():
    with pytest.raises(ti.TaichiSyntaxError) as e:

        @ti.kernel
        def foo():
            1 in [1]

        foo()

    assert e.value.args[0] == '"In" is not supported in Taichi kernels.'

@ti.test()
def test_single_compare():
    @ti.kernel
    def foo(a: ti.template(), b: ti.template(), c: ti.template()):
        for i in ti.static(range(3)):
            c[i * 6] = a[i] == b[i]
            c[i * 6 + 1] = a[i] != b[i]
            c[i * 6 + 2] = a[i] < b[i]
            c[i * 6 + 3] = a[i] <= b[i]
            c[i * 6 + 4] = a[i] > b[i]
            c[i * 6 + 5] = a[i] >= b[i]

    a = ti.Vector([1, 1, 2])
    b = ti.Vector([2, 1, 1])
    c = ti.field(ti.i32, shape=(18, ))
    d = ti.field(ti.i32, shape=(18, ))

    for i in range(3):
        c[i * 6] = a[i] == b[i]
        c[i * 6 + 1] = a[i] != b[i]
        c[i * 6 + 2] = a[i] < b[i]
        c[i * 6 + 3] = a[i] <= b[i]
        c[i * 6 + 4] = a[i] > b[i]
        c[i * 6 + 5] = a[i] >= b[i]

    foo(a, b, d)
    for i in range(18):
        assert c[i] == d[i]

@ti.test()
def test_chain_compare():
    @ti.kernel
    def foo(a: ti.i32, b: ti.i32, c: ti.template()):
        c[0] = a == b == a
        c[1] = a == b != a
        c[2] = a != b == a
        c[3] = a < b > a
        c[4] = a > b < a
        c[5] = a < b < a
        c[6] = a > b > a
        c[7] = a == a == a == a
        c[8] = a == a == a != a
        c[9] = a < b > a < b
        c[10] = a > b > a < b

    a = 1
    b = 2
    c = ti.field(ti.i32, shape=(11, ))
    d = ti.field(ti.i32, shape=(11, ))

    c[0] = a == b == a
    c[1] = a == b != a
    c[2] = a != b == a
    c[3] = a < b > a
    c[4] = a > b < a
    c[5] = a < b < a
    c[6] = a > b > a
    c[7] = a == a == a == a
    c[8] = a == a == a != a
    c[9] = a < b > a < b
    c[10] = a > b > a < b

    foo(a, b, d)
    for i in range(11):
        assert c[i] == d[i]

@ti.test()
def test_return():
    @ti.kernel
    def foo(x: ti.i32) -> ti.i32:
        return x + 1

    assert foo(1) == 2

@ti.test()
def test_format_print():
    a = ti.field(ti.i32, shape=(10, ))

    @ti.kernel
    def foo():
        a[0] = 1.0
        a[5] = 2.0
        print('Test if the string.format and fstring print works')
        print('string.format: a[0]={}, a[5]={}'.format(a[0], a[5]))
        print(f'fstring: a[0]={a[0]}, a[5]={a[5]}')

@ti.test(print_preprocessed_ir=True)
def test_if():
    @ti.kernel
    def foo(x: ti.i32) -> ti.i32:
        ret = 0
        if x:
            ret = 1
        else:
            ret = 0
        return ret

    assert foo(1)
    assert not foo(0)

@ti.test(print_preprocessed_ir=True)
def test_static_if():
    @ti.kernel
    def foo(x: ti.template()) -> ti.i32:
        ret = 0
        if ti.static(x):
            ret = 1
        else:
            ret = 0
        return ret

    assert foo(1)
    assert not foo(0)

@ti.test(print_preprocessed_ir=True)
def test_struct_for():
    a = ti.field(ti.i32, shape=(10, ))

    @ti.kernel
    def foo(x: ti.i32):
        for i in a:
            a[i] = x

    x = 5
    foo(x)
    for i in range(10):
        assert a[i] == 5

@ti.test(print_preprocessed_ir=True)
def test_grouped_struct_for():
    a = ti.field(ti.i32, shape=(4, 4))

    @ti.kernel
    def foo(x: ti.i32):
        for I in ti.grouped(a):
            a[I] = x

    x = 5
    foo(x)
    for i in range(4):
        for j in range(4):
            assert a[i, j] == 5

@ti.test(print_preprocessed_ir=True)
def test_static_for():
    a = ti.field(ti.i32, shape=(10, ))

    @ti.kernel
    def foo(x: ti.i32):
        for i in ti.static(range(10)):
            a[i] = x

    x = 5
    foo(x)
    for i in range(10):
        assert a[i] == 5

@ti.test(print_preprocessed_ir=True)
def test_static_grouped_for():
    a = ti.field(ti.i32, shape=(4, 4))

    @ti.kernel
    def foo(x: ti.i32):
        for i in ti.static(ti.grouped(ti.ndrange((1, 3), (1, 3)))):
            a[i] = x

    x = 5
    foo(x)
    for i in range(4):
        for j in range(4):
            if 1 <= i < 3 and 1 <= j < 3:
                assert a[i, j] == 5
            else:
                assert a[i, j] == 0

@ti.test(print_preprocessed_ir=True)
def test_range_for_single_argument():
    a = ti.field(ti.i32, shape=(10, ))

    @ti.kernel
    def foo(x: ti.i32):
        for i in range(5):
            a[i] = x

    x = 5
    foo(x)
    for i in range(10):
        if i < 5:
            assert a[i] == 5
        else:
            assert a[i] == 0

@ti.test(print_preprocessed_ir=True)
def test_range_for_two_arguments():
    a = ti.field(ti.i32, shape=(10, ))

    @ti.kernel
    def foo(x: ti.i32):
        for i in range(3, 7):
            a[i] = x

    x = 5
    foo(x)
    for i in range(10):
        if 3 <= i < 7:
            assert a[i] == 5
        else:
            assert a[i] == 0

@ti.test()
def test_range_for_three_arguments():
    a = ti.field(ti.i32, shape=(10, ))

    with pytest.raises(ti.TaichiSyntaxError) as e:

        @ti.kernel
        def foo(x: ti.i32):
            for i in range(3, 7, 2):
                a[i] = x

        x = 5
        foo(x)

    assert e.value.args[0] == "Range should have 1 or 2 arguments, found 3"

@ti.test(print_preprocessed_ir=True)
def test_ndrange_for():
    x = ti.field(ti.f32, shape=(16, 32, 64))

    @ti.kernel
    def func():
        for i, j, k in ti.ndrange((4, 10), (3, 8), 17):
            x[i, j, k] = i + j * 10 + k * 100

    func()
    for i in range(16):
        for j in range(32):
            for k in range(64):
                if 4 <= i < 10 and 3 <= j < 8 and k < 17:
                    assert x[i, j, k] == i + j * 10 + k * 100
                else:
                    assert x[i, j, k] == 0

@ti.test(print_preprocessed_ir=True)
def test_grouped_ndrange_for():
    x = ti.field(ti.i32, shape=(6, 6, 6))
    y = ti.field(ti.i32, shape=(6, 6, 6))

    @ti.kernel
    def func():
        lower = ti.Vector([0, 1, 2])
        upper = ti.Vector([3, 4, 5])
        for I in ti.grouped(
                ti.ndrange((lower[0], upper[0]), (lower[1], upper[1]),
                           (lower[2], upper[2]))):
            x[I] = I[0] + I[1] + I[2]
        for i in range(0, 3):
            for j in range(1, 4):
                for k in range(2, 5):
                    y[i, j, k] = i + j + k

    func()

    for i in range(6):
        for j in range(6):
            for k in range(6):
                assert x[i, j, k] == y[i, j, k]

@ti.test(print_preprocessed_ir=True)
def test_static_for_break():
    n = 10

    @ti.kernel
    def foo(a: ti.template()):
        for i in ti.static(range(n)):
            a[i] = 3
            if ti.static(i >= 5):
                break
                a[i] = 10
            a[i] = 5

    a = ti.field(ti.i32, shape=(n, ))
    foo(a)
    for i in range(n):
        if i < 5:
            assert a[i] == 5
        elif i == 5:
            assert a[i] == 3
        else:
            assert a[i] == 0

@ti.test(print_preprocessed_ir=True)
def test_static_grouped_for_break():
    n = 4

    @ti.kernel
    def foo(a: ti.template()):
        for I in ti.static(ti.grouped(ti.ndrange(n, n))):
            a[I] = 3
            if ti.static(I[0] >= 3):
                break
                a[I] = 10
            a[I] = 5

    a = ti.field(ti.i32, shape=(n, n))
    foo(a)
    for i in range(n):
        for j in range(n):
            if i < 3:
                assert a[i, j] == 5
            elif i == 3 and j == 0:
                assert a[i, j] == 3
            else:
                assert a[i, j] == 0

@ti.test(print_preprocessed_ir=True)
def test_static_for_continue():
    n = 10

    @ti.kernel
    def foo(a: ti.template()):
        for i in ti.static(range(n)):
            a[i] = 3
            if ti.static(i >= 5):
                continue
                a[i] = 10
            a[i] = 5

    a = ti.field(ti.i32, shape=(n, ))
    foo(a)
    for i in range(n):
        if i < 5:
            assert a[i] == 5
        else:
            assert a[i] == 3

@ti.test(print_preprocessed_ir=True)
def test_static_grouped_for_continue():
    n = 4

    @ti.kernel
    def foo(a: ti.template()):
        for I in ti.static(ti.grouped(ti.ndrange(n, n))):
            a[I] = 3
            if ti.static(I[0] >= 3):
                continue
                a[I] = 10
            a[I] = 5

    a = ti.field(ti.i32, shape=(n, n))
    foo(a)
    for i in range(n):
        for j in range(n):
            if i < 3:
                assert a[i, j] == 5
            else:
                assert a[i, j] == 3

@ti.test(print_preprocessed_ir=True)
def test_for_break():
    n = 4

    @ti.kernel
    def foo(a: ti.template()):
        for i in range(n):
            for j in range(n):
                a[i, j] = 3
                if i >= 3:
                    break
                    a[i, j] = 10
                a[i, j] = 5

    a = ti.field(ti.i32, shape=(n, n))
    foo(a)
    for i in range(n):
        for j in range(n):
            if i < 3:
                assert a[i, j] == 5
            elif i == 3 and j == 0:
                assert a[i, j] == 3
            else:
                assert a[i, j] == 0

@ti.test(print_preprocessed_ir=True)
def test_for_continue():
    n = 4

    @ti.kernel
    def foo(a: ti.template()):
        for i in range(n):
            for j in range(n):
                a[i, j] = 3
                if i >= 3:
                    continue
                    a[i, j] = 10
                a[i, j] = 5

    a = ti.field(ti.i32, shape=(n, n))
    foo(a)
    for i in range(n):
        for j in range(n):
            if i < 3:
                assert a[i, j] == 5
            else:
                assert a[i, j] == 3

@ti.test()
def test_while():
    x = ti.field(ti.f32)

    N = 1

    ti.root.dense(ti.i, N).place(x)

    @ti.kernel
    def func():
        i = 0
        s = 0
        while i < 10:
            s += i
            i += 1
        x[0] = s

    func()
    assert x[0] == 45

@ti.test()
def test_while_break():
    ret = ti.field(ti.i32, shape=())

    @ti.kernel
    def func():
        i = 0
        s = 0
        while True:
            s += i
            i += 1
            if i > 10:
                break
        ret[None] = s

    func()
    assert ret[None] == 55

@ti.test()
def test_while_continue():
    ret = ti.field(ti.i32, shape=())

    @ti.kernel
    def func():
        i = 0
        s = 0
        while i < 10:
            i += 1
            if i % 2 == 0:
                continue
            s += i
        ret[None] = s

    func()
    assert ret[None] == 25

@ti.test(print_preprocessed_ir=True)
def test_func():
    @ti.func
    def bar(x):
        return x * x, -x

    a = ti.field(ti.i32, shape=(10, ))
    b = ti.field(ti.i32, shape=(10, ))

    @ti.kernel
    def foo():
        for i in a:
            a[i], b[i] = bar(i)

    foo()
    for i in range(10):
        assert a[i] == i * i
        assert b[i] == -i

@ti.test(print_preprocessed_ir=True)
def test_func_in_python_func():
    @ti.func
    def bar(x: ti.template()):
        if ti.static(x):
            mat = bar(x // 2)
            mat = mat @ mat
            if ti.static(x % 2):
                mat = mat @ ti.Matrix([[1, 1], [1, 0]])
            return mat
        else:
            return ti.Matrix([[1, 0], [0, 1]])

    def fibonacci(x):
        return ti.subscript(bar(x), 1, 0)

    @ti.kernel
    def foo(x: ti.template()) -> ti.i32:
        return fibonacci(x)

    fib = [0, 1, 1, 2, 3, 5, 8, 13, 21, 34]

    for i in range(10):
        assert foo(i) == fib[i]

@ti.test(print_preprocessed_ir=True)
def test_ifexp():
    @ti.kernel
    def foo(x: ti.i32) -> ti.i32:
        return 1 if x else 0

    assert foo(1) == 1
    assert foo(0) == 0

@ti.test(print_preprocessed_ir=True)
def test_static_ifexp():
    @ti.kernel
    def foo(x: ti.template()) -> ti.i32:
        return 1 if ti.static(x) else 0

    assert foo(1) == 1
    assert foo(0) == 0

@ti.test()
def test_static_assign():
    a = ti.field(ti.i32, shape=(1, ))
    b = ti.field(ti.i32, shape=(1, ))

    @ti.kernel
    def foo(xx: ti.template(), yy: ti.template()) -> ti.i32:
        x, y = ti.static(xx, yy)
        x[0] -= 1
        y[0] -= 1
        return x[0] + y[0]

    a[0] = 2
    b[0] = 3
    assert foo(a, b) == 3

@ti.test()
def test_static_assign_element():
    with pytest.raises(ti.TaichiSyntaxError) as e:

        @ti.kernel
        def foo():
            a = ti.static([1, 2, 3])
            a[0] = ti.static(2)

        foo()
    assert e.value.args[
        0] == "Static assign cannot be used on elements in arrays"

@ti.test()
def test_recreate_variable():
    with pytest.raises(ti.TaichiSyntaxError) as e:

        @ti.kernel
        def foo():
            a = 1
            a = ti.static(2)

        foo()
    assert e.value.args[0] == "Recreating variables is not allowed"

@ti.test()
def test_taichi_other_than_ti():
    import taichi as tc

    @tc.func
    def bar(x: tc.template()):
        if tc.static(x):
            mat = bar(x // 2)
            mat = mat @ mat
            if tc.static(x % 2):
                mat = mat @ tc.Matrix([[1, 1], [1, 0]])
            return mat
        else:
            return tc.Matrix([[1, 0], [0, 1]])

    def fibonacci(x):
        return tc.subscript(bar(x), 1, 0)

    @tc.kernel
    def foo(x: tc.template()) -> tc.i32:
        return fibonacci(x)

    fib = [0, 1, 1, 2, 3, 5, 8, 13, 21, 34]

    for i in range(10):
        assert foo(i) == fib[i]

@ti.test(require=ti.extension.assertion, debug=True, gdb_trigger=False)
def test_assert_message():
    @ti.kernel
    def func():
        x = 20
        assert 10 <= x < 20, 'Foo bar'

    with pytest.raises(RuntimeError, match='Foo bar'):
        func()

@ti.test(require=ti.extension.assertion, debug=True, gdb_trigger=False)
def test_assert_message_formatted():
    x = ti.field(dtype=int, shape=16)
    x[10] = 42

    @ti.kernel
    def assert_formatted():
        for i in x:
            assert x[i] == 0, 'x[%d] expect=%d got=%d' % (i, 0, x[i])

    @ti.kernel
    def assert_float():
        y = 0.5
        assert y < 0, 'y = %f' % y

    with pytest.raises(RuntimeError, match=r'x\[10\] expect=0 got=42'):
        assert_formatted()
    # TODO: note that we are not fully polished to be able to recover from
    # assertion failures...
    with pytest.raises(RuntimeError, match=r'y = 0.5'):
        assert_float()

    # success case
    x[10] = 0
    assert_formatted()

@ti.test()
def test_dict():
    @ti.kernel
    def foo(x: ti.template()) -> ti.i32:
        a = {1: 2, 3: 4}
        b = {5: 6, **a}
        return b[x]

    assert foo(1) == 2
    with pytest.raises(KeyError):
        foo(2)

@ti.test()
def test_listcomp():
    @ti.func
    def identity(dt, n: ti.template()):
        return ti.Matrix([[ti.cast(int(i == j), dt) for j in range(n)]
                          for i in range(n)],
                         disable_local_tensor=1)

    @ti.kernel
    def foo(n: ti.template()) -> ti.i32:
        a = identity(ti.i32, n)
        b = [j for i in a for j in i]
        ret = 0
        for i in ti.static(range(n)):
            for j in ti.static(range(n)):
                ret += i * j * b[i * n + j]
        return ret

    assert foo(5) == 1 + 4 + 9 + 16

@ti.test()
def test_dictcomp():
    @ti.kernel
    def foo(n: ti.template()) -> ti.i32:
        a = {i: i * i for i in range(n) if i % 3 if i % 2}
        ret = 0
        for i in ti.static(range(n)):
            if ti.static(i % 3):
                if ti.static(i % 2):
                    ret += a[i]
        return ret

    assert foo(10) == 1 * 1 + 5 * 5 + 7 * 7

@ti.test()
def test_dictcomp_fail():
    @ti.kernel
    def foo(n: ti.template(), m: ti.template()) -> ti.i32:
        a = {i: i * i for i in range(n) if i % 3 if i % 2}
        return a[m]

    with pytest.raises(KeyError):
        foo(5, 2)

    with pytest.raises(KeyError):
        foo(5, 3)

@pytest.mark.skipif(not ti.has_pytorch(), reason='Pytorch not installed.')
@ti.test(exclude=ti.opengl)
def test_ndarray():
    n = 4
    m = 7

    @ti.kernel
    def run(x: ti.any_arr(element_dim=2, layout=ti.Layout.AOS),
            y: ti.any_arr()):
        for i in ti.static(range(n)):
            for j in ti.static(range(m)):
                x[i, j][0, 0] += i + j + y[i, j]

    a = ti.Matrix.ndarray(1, 1, ti.i32, shape=(n, m))
    for i in range(n):
        for j in range(m):
            a[i, j][0, 0] = i * j
    b = np.ones((n, m), dtype=np.int32)
    run(a, b)
    for i in range(n):
        for j in range(m):
            assert a[i, j][0, 0] == i * j + i + j + 1

@ti.test(arch=ti.cpu)
def test_sparse_matrix_builder():
    n = 8
    Abuilder = ti.linalg.SparseMatrixBuilder(n, n, max_num_triplets=100)

    @ti.kernel
    def fill(Abuilder: ti.linalg.sparse_matrix_builder()):
        for i, j in ti.static(ti.ndrange(n, n)):
            Abuilder[i, j] += i + j

    fill(Abuilder)
    A = Abuilder.build()
    for i in range(n):
        for j in range(n):
            assert A[i, j] == i + j

@ti.test()
def test_func_default_value():
    @ti.func
    def bar(s, t=1):
        return s + t

    @ti.kernel
    def foo() -> ti.i32:
        return bar(1)

    assert foo() == 2

@ti.test()
def test_func_default_value_fail():
    with pytest.raises(ti.TaichiSyntaxError):

        @ti.func
        def bar(s, t=1):
            return s + t

        @ti.kernel
        def foo() -> ti.i32:
            return bar(1, 2, 3)

        foo()

@ti.test()
def test_raise():
    dim = 1
    m = ti.Matrix.field(dim, dim, ti.f32)
    ti.root.place(m)

    with pytest.raises(Exception) as e:

        @ti.kernel
        def foo():
            ti.polar_decompose(m, ti.f32)

        foo()
    assert e.value.args[
        0] == "Polar decomposition only supports 2D and 3D matrices."

max_stars_repo_name: ZonePG/taichi

max_issues_repo_name: ZonePG/taichi

max_forks_repo_name: ZonePG/taichi


text:
import numpy as np

from sklearn.metrics import accuracy_score

from keras.layers import Input, Dense, Dropout
from keras.models import Model
import keras
from keras.layers import Input, Dense, BatchNormalization

def create_model_nn(num_hidden_layers, input_layer_width, hidden_layers_width, input_features_num):
    inputs = Input(shape=(input_features_num,))

    x = Dense(input_layer_width, activation='relu')(inputs)
    #x = Dropout(rate=0.2)(x)
    x = BatchNormalization()(x)

    for i in range(num_hidden_layers):
        x = Dense(hidden_layers_width, activation='relu')(x)
        #x = Dropout(rate=0.2)(x)
        x = BatchNormalization()(x)

    predictions = Dense(1, activation='sigmoid')(x)

    model = Model(inputs=inputs, outputs=predictions)
    adam = keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, amsgrad=False)

    model.compile(optimizer=adam,
                  loss='binary_crossentropy',
                  metrics=['accuracy'])

    return model

def clip_at_threshold(y, threshold=0.5):
    y[y >= threshold] = 1
    y[y < threshold] = 0
    return y

def accuracy_from_scores(y_true, y_score, threshold):
    y_pred = np.copy(y_score)
    y_pred[y_pred <= threshold] = 0
    y_pred[y_pred > threshold] = 1
    return accuracy_score(y_true, y_pred)

#### Original source in ./posthoc.
#### Adding here, so importing is easier in this level.
def zero_if_nan(x):
    return 0. if np.isnan(x) else x

def compute_bias(y_pred, y_true, priv, metric):
    gtpr_priv = zero_if_nan(y_pred[priv * y_true == 1].mean())
    gfpr_priv = zero_if_nan(y_pred[priv * (1-y_true) == 1].mean())
    mean_priv = zero_if_nan(y_pred[priv == 1].mean())

    gtpr_unpriv = zero_if_nan(y_pred[(1-priv) * y_true == 1].mean())
    gfpr_unpriv = zero_if_nan(y_pred[(1-priv) * (1-y_true) == 1].mean())
    mean_unpriv = zero_if_nan(y_pred[(1-priv) == 1].mean())

    if metric == "statistical_parity_difference":
        return mean_unpriv - mean_priv
    elif metric == "average_abs_odds_difference":
        return 0.5 * ((gfpr_unpriv - gfpr_priv) + (gtpr_unpriv - gtpr_priv))
    elif metric == "equal_opportunity_difference":
        return gtpr_unpriv - gtpr_priv


text:
import numpy as np
import pickle as pkl
import networkx as nx
import scipy.sparse as sp
from sklearn.datasets import dump_svmlight_file
import sys

# Based on https://github.com/tkipf/gcn/blob/master/gcn/utils.py

def parse_index_file(filename):
    """Parse index file."""
    index = []
    for line in open(filename):
        index.append(int(line.strip()))
    return index

def sample_mask(idx, l):
    """Create mask."""
    mask = np.zeros(l)
    mask[idx] = 1
    return np.array(mask, dtype=np.bool)

def preprocess_features(features):
    """Row-normalize feature matrix and convert to tuple representation"""
    rowsum = np.array(features.sum(1))
    r_inv = np.power(rowsum, -1).flatten()
    r_inv[np.isinf(r_inv)] = 0.
    r_mat_inv = sp.diags(r_inv)
    features = r_mat_inv.dot(features)
    return features

def load_data(dataset_str):
    """
    Loads input data from gcn/data directory

    ind.dataset_str.x => the feature vectors of the training instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.tx => the feature vectors of the test instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.allx => the feature vectors of both labeled and unlabeled training instances
        (a superset of ind.dataset_str.x) as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.y => the one-hot labels of the labeled training instances as numpy.ndarray object;
    ind.dataset_str.ty => the one-hot labels of the test instances as numpy.ndarray object;
    ind.dataset_str.ally => the labels for instances in ind.dataset_str.allx as numpy.ndarray object;
    ind.dataset_str.graph => a dict in the format {index: [index_of_neighbor_nodes]} as collections.defaultdict
        object;
    ind.dataset_str.test.index => the indices of test instances in graph, for the inductive setting as list object.

    All objects above must be saved using python pickle module.

    :param dataset_str: Dataset name
    :return: All data input files loaded (as well the training/test data).
    """
    names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph']
    objects = []
    for i in range(len(names)):
        with open("data/ind.{}.{}".format(dataset_str, names[i]), 'rb') as f:
            if sys.version_info > (3, 0):
                objects.append(pkl.load(f, encoding='latin1'))
            else:
                objects.append(pkl.load(f))

    x, y, tx, ty, allx, ally, graph = tuple(objects)
    test_idx_reorder = parse_index_file("data/ind.{}.test.index".format(dataset_str))
    test_idx_range = np.sort(test_idx_reorder)

    if dataset_str == 'citeseer':
        # Fix citeseer dataset (there are some isolated nodes in the graph)
        # Find isolated nodes, add them as zero-vecs into the right position
        test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder)+1)
        tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
        tx_extended[test_idx_range-min(test_idx_range), :] = tx
        tx = tx_extended
        ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
        ty_extended[test_idx_range-min(test_idx_range), :] = ty
        ty = ty_extended

    features = sp.vstack((allx, tx)).tolil()
    features[test_idx_reorder, :] = features[test_idx_range, :]
    features = preprocess_features(features)

    labels = np.vstack((ally, ty))
    labels[test_idx_reorder, :] = labels[test_idx_range, :]
    idx_test = frozenset(test_idx_range.tolist())
    idx_train = frozenset(range(len(y)))
    idx_val = frozenset(range(len(y), len(y)+500))

    n_nodes = labels.shape[0]
    dump_svmlight_file(features, labels, 'data/{}.svmlight'.format(dataset_str), multilabel=True)
    with open('data/{}.graph'.format(dataset_str), 'w', newline='\n') as f:
        for i in range(n_nodes):
            print(*graph[i], file=f)
    with open('data/{}.split'.format(dataset_str), 'w', newline='\n') as f:
        for i in range(n_nodes):
            split = 0
            if i in idx_train:
                split = 1
            elif i in idx_val:
                split = 2
            elif i in idx_test:
                split = 3
            print(split, file=f)

if __name__ == '__main__':
    for dataset_str in ['pubmed', 'cora', 'citeseer']:
        load_data(dataset_str)

max_stars_repo_name: cai-lw/parallel-gcn

max_issues_repo_name: cai-lw/parallel-gcn

max_forks_repo_name: cai-lw/parallel-gcn

reason: import numpy,import scipy,import networkx


text:
#Float 16bit Multiplication Validation Script
import numpy as np
# a = np.array([-12817,17476,0], dtype=np.int16)
# a = np.array([4369,4396,0], dtype=np.int16)
a = np.array([17767,8738,0,0], dtype=np.int16)
a.dtype = np.float16
a[3]=(a[0]*a[1])+a[2]
a.dtype=np.int16
#list = [hex(i) for i in a]
#print(list)
print(a)


text:
"""Some useful utilities when dealing with neural nets w/ tensorflow and data process.

Parag K. Mital, Jan. 2016, zxc 2018.
"""
import tensorflow as tf
import numpy as np
import pickle
# import matplotlib.pyplot as plt

def montage_batch(images):
    """Draws all filters (n_input * n_output filters) as a
    montage image separated by 1 pixel borders.

    Parameters
    ----------
    batch : numpy.ndarray
        Input array to create montage of.

    Returns
    -------
    m : numpy.ndarray
        Montage image.
    """
    img_h = images.shape[1]
    img_w = images.shape[2]
    n_plots = int(np.ceil(np.sqrt(images.shape[0])))
    m = np.ones(
        (images.shape[1] * n_plots + n_plots + 1,
         images.shape[2] * n_plots + n_plots + 1, 3)) * 0.5

    for i in range(n_plots):
        for j in range(n_plots):
            this_filter = i * n_plots + j
            if this_filter < images.shape[0]:
                this_img = images[this_filter, ...]
                m[1 + i + i * img_h:1 + i + (i + 1) * img_h,
                  1 + j + j * img_w:1 + j + (j + 1) * img_w, :] = this_img
    return m

# %%
def montage(W):
    """Draws all filters (n_input * n_output filters) as a
    montage image separated by 1 pixel borders.

    Parameters
    ----------
    W : numpy.ndarray
        Input array to create montage of.

    Returns
    -------
    m : numpy.ndarray
        Montage image.
    """
    W = np.reshape(W, [W.shape[0], W.shape[1], 1, W.shape[2] * W.shape[3]])
    n_plots = int(np.ceil(np.sqrt(W.shape[-1])))
    m = np.ones(
        (W.shape[0] * n_plots + n_plots + 1,
         W.shape[1] * n_plots + n_plots + 1)) * 0.5
    for i in range(n_plots):
        for j in range(n_plots):
            this_filter = i * n_plots + j
            if this_filter < W.shape[-1]:
                m[1 + i + i * W.shape[0]:1 + i + (i + 1) * W.shape[0],
                  1 + j + j * W.shape[1]:1 + j + (j + 1) * W.shape[1]] = (
                    np.squeeze(W[:, :, :, this_filter]))
    return m

# %%
def corrupt(x, mask=None):
    """Take an input tensor and add uniform masking.

    Parameters
    ----------
    x : Tensor/Placeholder
        Input to corrupt.
    mask: none
    Returns
    -------
    x_corrupted : Tensor
        50 pct of values corrupted.
    """
    import copy
    cor = copy.deepcopy(x)
    if mask is not None:
        for i in range(x.shape[0]):
            cor[i] *= mask[i]
        shuffle_indices = np.random.permutation(np.arange(cor.shape[0]))
        shuffle_data = cor[shuffle_indices]
        x = x[shuffle_indices]
        return shuffle_data, x
    n = np.random.normal(0, 0.1, (x.shape[0], x.shape[1]))
    # y = x + n
    # fig, axs = plt.subplots(2, n_examples, figsize=(10, 2))
    # for example_i in range(n_examples):
    #     axs[0][example_i].imshow(
    #         # np.reshape(test_xs[example_i, :], (28, 28)))
    #         np.reshape(y[example_i, :], (15, 15)))
    #     axs[1][example_i].imshow(
    #         # np.reshape([recon[example_i, :] + mean_img], (28, 28)))
    #         np.reshape([x[example_i, :]], (15, 15)))
    # print ('Plot complete now showing...')
    # fig.show()
    # plt.draw()
    # plt.title("1st function - dataset ones but used our dataset")
    # plt.waitforbuttonpress()
    return x + n
    # return tf.multiply(x, tf.cast(tf.random_uniform(shape=tf.shape(x),
    #                                            minval=0,
    #                                            maxval=2,
    #                                            dtype=tf.int32), tf.float32))

# %%
def weight_variable(shape):
    '''Helper function to create a weight variable initialized with
    a normal distribution

    Parameters
    ----------
    shape : list
        Size of weight variable
    '''
    initial = tf.random_normal(shape, mean=0.0, stddev=0.01)
    return tf.Variable(initial)

# %%
def bias_variable(shape):
    '''Helper function to create a bias variable initialized with
    a constant value.

    Parameters
    ----------
    shape : list
        Size of weight variable
    '''
    initial = tf.random_normal(shape, mean=0.0, stddev=0.01)
    return tf.Variable(initial)
# %%
def loadDataset(batch_size=1000, max = 0, dataset_dir=None):
    """
    load pickle dataset by batches(deprecated)

    Author
    ------
        zxc

    Parameters
    ----------
        batch_size: size of batch
        max: max num of dataset to load. if max == 0, load all(recommend to use load_whole_dataset bellow)
        dataset_dir: dir to dataset
    """
    opendataset = open(dataset_dir,'rb')
    dataset = pickle.load(opendataset)
    dataset = [dataset]
    i = 1
    total = 1
    while True:
        try:
            tmp = pickle.load(opendataset)
            dataset.append(tmp)
            if max != 0 and total == max:
                dataset = np.asarray(dataset)
                yield dataset
                break
            if i == batch_size:
                dataset = np.asarray(dataset)
                yield dataset
                i = 1
                dataset = [pickle.load(opendataset)]
            i += 1
            total += 1
        except Exception as e:
            print(e)
            dataset = np.asarray(dataset)
            yield dataset
            opendataset.close()
            break
    opendataset.close()

def load_whole_dataset(max=0, dataset_dir=None):
    """
    load whole pickle dataset

    Author
    ------
        zxc

    Parameters
    ----------
        max: max num of dataset to load. if max == 0, load all(assume that the size of dataset is no bigger than 5000000, sure you can modify it but mind memory size)
        dataset_dir: dir to dataset
    """
    if max == 0:
        dataset = np.zeros([5000000, 225])
    else:
        dataset = np.zeros([max, 225])
    opendataset = open(dataset_dir,'rb')
    end = 0
    for i in range(dataset.shape[0]):
        try:
            dataset[i] = pickle.load(opendataset)
            end += 1
        except:
            end -= 1
            break
    print(end)
    return dataset[:end]
# %%
def loadlabel(dir=None):
    if dir is None:
        print('dir is expected type of string but get None')
    label_file = open(dir, 'rb')
    label = []
    while True:
        try:
            t = pickle.load(label_file)
            if t is 0:
                label.append(1)
            else:
                label.append(-1)
        except Exception as e:
            print(e)
            break
    return np.asarray(label)


text:
import torch
import numpy as np
from sklearn import metrics
from functools import partial
from collections import deque

from .utils import to_numpy

def get_preds(scores):
  score, preds = torch.max(scores, dim=1)
  return preds

def top_k(scores, k=5, largest=True):
  return torch.topk(scores, k=k, dim=1, largest=largest, sorted=True)

def accuracy(targets, preds):
  if targets.shape != preds.shape:
    preds = get_preds(preds)
  return (targets == preds).sum().float() / len(preds)

def top_k_accuracy(targets, preds, k=1):
  if len(targets.shape) != 1:
    raise ValueError('Multi label targets not supported')
  scores, preds = preds.topk(k, 1, True, True)
  preds = preds.t()
  correct = (preds == targets.view(1, -1).expand_as(preds))

  return correct.sum().float() / len(targets)

top_3_accuracy = partial(top_k_accuracy, k=3)
top_5_accuracy = partial(top_k_accuracy, k=5)
top_10_accuracy = partial(top_k_accuracy, k=10)

def precision_at_k(targets, outputs, k=5):
  scores, top_preds = top_k(outputs, k=k)
  raise NotImplementedError()

def recall_at_k(targets, outputs, k=5):
  scores, top_preds = top_k(outputs, k=k)
  raise NotImplementedError()

def hits_at_k():
  pass

def mean_reciprocal_rank():
  pass

def average_precision(targets, preds, target_threshold=0):
  targets, preds = to_numpy(targets), to_numpy(preds)
  if target_threshold is not None:
    targets = targets > target_threshold
  return metrics.average_precision_score(targets, preds)

def label_ranking_average_precision(targets, preds, target_threshold=0):
  targets, preds = to_numpy(targets), to_numpy(preds)
  if target_threshold is not None:
    targets = targets > target_threshold
  return metrics.label_ranking_average_precision_score(targets, preds)

def coverage_error(targets, preds, target_threshold=0):
  targets, preds = to_numpy(targets), to_numpy(preds)
  if target_threshold is not None:
    targets = targets > target_threshold
  return metrics.coverage_error(targets, preds)

def average_precision_at_k(targets, preds, k=5):
  raise NotImplementedError()

def mean_average_precision_at_k(targets, preds, k=5):
  raise NotImplementedError()

def evaluate_multiclass(
    targets,
    outputs,
    preds=None,
    classes=None
):
  preds = preds or get_preds(outputs)
  targets, outputs, preds = (
    to_numpy(targets),
    to_numpy(outputs),
    to_numpy(preds)
  )
  raise NotImplementedError()

def evaluate_multilabel(
    targets,
    outputs,
    preds=None,
    classes=None,
):
  targets, outputs, preds = (
    to_numpy(targets),
    to_numpy(outputs),
    to_numpy(preds)
  )
  raise NotImplementedError()

class Meter:
  def __init__(self):
    self.reset()

  def reset(self):
    self.max = None
    self.min = None
    self.sum = 0
    self.count = 0

  def update(self, val):
    if self.max is None or val > self.max:
      self.max = val
    if self.min is None or val < self.min:
      self.min = val

    self.sum += val
    self.count += 1

  @property
  def average(self):
    return self.sum / self.count

class WindowMeter:
  def __init__(self, length=10):
    self.length = length
    self.values = deque(maxlen=length)

  def reset(self):
    self.values = deque(maxlen=self.length)

  def update(self, val):
    self.values.append(val)

  @property
  def sum(self):
    return sum(self.values)

  @property
  def max(self):
    return max(self.values)

  @property
  def min(self):
    return min(self.values)

  @property
  def average(self):
    if not self.values: return None
    return sum(self.values) / len(self.values)

def exp_moving_avg(cur, prev=None, alpha=.05, steps=None):
  """exponential moving average"""
  if prev is None:
    return cur
  avg = alpha * cur + prev * (1 - alpha)
  return avg / (1 - alpha ** steps) if steps else avg

def moving_average(data, window=10):
  cumsum = np.cumsum(np.insert(data, 0, 0))
  return (cumsum[window:] - cumsum[:-window]) / float(window)

max_stars_repo_name: michalwols/yann

max_issues_repo_name: michalwols/yann

max_forks_repo_name: michalwols/yann


text:
"""
DFT
"""
import numpy as np
import cv2
from core.component import Component

class Dft(Component):
    """
    Class for calculate discrete fourier transformation
    """
    def __init__(self, options):
        super(Dft, self).__init__(options)
        self.input_observer = Dft.InputObserver(self)

    class InputObserver(Component.InputObserver):
        """
        Class for observers
        """
        def update(self, observable, package):
            self.outer.log_line('calculate discrete fourier transformation')
            package['Dft'] = \
                cv2.dft(np.float32(package['data']), flags=cv2.DFT_COMPLEX_OUTPUT).tostring()

            self.outer.output_notifier.notify_observers(package)

max_stars_repo_name: thomaskuestner/MRIQA

max_issues_repo_name: thomaskuestner/MRIQA

max_forks_repo_name: thomaskuestner/MRIQA


text:
#!/usr/bin/env python3
""" Dumps the snake as a sphere sweep given data points and radii """

__author__ = "Tejaswin Parthasarathy"
__license__ = "GPL"

import numpy as np

def dump_snake_to_povray(t_step, t_pos, t_radius=0.01, t_prefix=""):
    """ Dumps the rod for visualization in povray. Assumes that the
    rod is of unit length almost on the XY plane (normal Z) and translates
    starting from the origin to the positive X axes. If not, you can
    suitably transform your data (as a post-processing step) to fit
    given constraints.

    Parameters
    ----------
    t_step : int
        The current time-step. Useful for sequential/ordered numbering
        of rendered files.
    t_pos : np.array
        The nodal positions of the rod at given t_step, as a (3, *) or
        (*,3) shaped array.
    t_radius: float
        The radius of the snake, as a constant (ignoring the small
        dilation effects)
    t_prefix: str
        Prefix for the file name (can also be a path)

    Returns
    -------
    None

    """

    # 0. Define helper function for pythonic text conversion
    def vectorize(arr, rad):
        """ transforms [a, b, c] into string "<a, b, c>"" """
        n = arr.shape[0]
        temp = [None] * n
        for i in range(n):
            temp[i] = (
                "<%s>" % ",".join([str(e)
                                   for e in arr[i]]) + ",{}".format(rad))
        return ",\n".join(temp)

    # 1. Piece together different parts of the string

    # Arrange data conveniently
    povray_arr = t_pos if t_pos.shape[1] == 3 else t_pos.T
    # Data in z always negative to keep with povray conventions
    povray_arr[:, 2] = -np.abs(povray_arr[:, 2])

    # Define the head string
    HEAD_STR = r"""
#include "scenepovray.inc"

sphere_sweep
    {
        b_spline
        %s,
    """ % "{}".format(povray_arr.shape[0])

    # Define the body string after manipulations

    BODY_STR = vectorize(povray_arr, t_radius)

    # Define the tail string
    TAIL_STR = r"""
        texture
        {
        pigment{ color rgbt<0.45,0.39,1,0.0> }
        finish { phong 1 }
        }
        scale<1,1,1>
        rotate<0,0,0>
        translate<0,0,0>
    }
    """

    # 2. Write to file
    file_name = t_prefix + "snake_{:07d}.pov".format(t_step)

    with open(file_name, "w") as text_file:
        print(HEAD_STR + BODY_STR + TAIL_STR, file=text_file)

def test_dump():
    """ Test object for the dump function"""
    x_vals = np.linspace(0., 1., 50)
    y_vals = 0.1 * np.sin(5 * np.pi * x_vals)
    radius = 0.05
    z_vals = 0.0 * x_vals - radius
    prefix = "./data/"
    data_arr = np.vstack((x_vals, y_vals, z_vals))
    dump_snake_to_povray(0, data_arr, radius, prefix)

    y_vals = 0.1 * np.cos(5 * np.pi * x_vals)
    data_arr = np.vstack((x_vals, y_vals, z_vals))
    dump_snake_to_povray(1000, data_arr, radius, prefix)

if __name__ == "__main__":
    test_dump()


text:
import re
import subprocess
import pandas as pd
import numpy as np
try:
    from StringIO import StringIO as BytesIO
except ImportError:
    from io import BytesIO

TABLE_RE = re.compile("CREATE TABLE \[([^\]]+)\]\s*\(([^\;]*)\)\;",
                      re.MULTILINE | re.DOTALL)

DEF_RE = re.compile("\s*\[(\w+)\]\s*(.*?),")

def list_tables(rdb_file, encoding="latin-1"):
    """
    :param rdb_file: The MS Access database file.
    :param encoding: The content encoding of the output. I assume `latin-1`
        because so many of MS files have that encoding. But, MDBTools may
        actually be UTF-8.
    :return: A list of the tables in a given database.
    """
    tables = subprocess.check_output(['mdb-tables', rdb_file]).decode(encoding)
    return tables.strip().split(" ")

def _extract_dtype(data_type):
    # Note, this list is surely incomplete. But, I only had one .mdb file
    # at the time of creation. If you see a new data-type, patch-pull or just
    # open an issue.
    data_type = data_type.lower()
    if data_type.startswith('double'):
        return np.float_
    elif data_type.startswith('long'):
        return np.int_
    else:
        return None

def _extract_defs(defs_str):
    defs = {}
    lines = defs_str.splitlines()
    for line in lines:
        m = DEF_RE.match(line)
        if m:
            defs[m.group(1)] = m.group(2)
    return defs

def read_schema(rdb_file, encoding='utf8'):
    """
    :param rdb_file: The MS Access database file.
    :param encoding: The schema encoding. I'm almost positive that MDBTools
        spits out UTF-8, exclusively.
    :return: a dictionary of table -> column -> access_data_type
    """
    output = subprocess.check_output(['mdb-schema', rdb_file])
    lines = output.decode(encoding).splitlines()
    schema_ddl = "\n".join(l for l in lines if l and not l.startswith('-'))

    schema = {}
    for table, defs in TABLE_RE.findall(schema_ddl):
        schema[table] = _extract_defs(defs)

    return schema

def to_pandas_schema(schema, implicit_string=True):
    """
    :param schema: the output of `read_schema`
    :param implicit_string: mark strings and unknown dtypes as `np.str_`.
    :return: a dictionary of table -> column -> np.dtype
    """
    pd_schema = {}
    for tbl, defs in schema.items():
        pd_schema[tbl] = None
        sub_schema = {}
        for column, data_type in defs.items():
            dtype = _extract_dtype(data_type)
            if dtype is not None:
                sub_schema[column] = dtype
            elif implicit_string:
                sub_schema[column] = np.str_
        pd_schema[tbl] = sub_schema
    return pd_schema

def read_table(rdb_file, table_name, *args, **kwargs):
    """
    Read a MS Access database as a Pandas DataFrame.

    Unless you set `converters_from_schema=False`, this function assumes you
    want to infer the schema from the Access database's schema. This sets the
    `dtype` argument of `read_csv`, which makes things much faster, in most
    cases. If you set the `dtype` keyword argument also, it overrides
    inferences. The `schema_encoding keyword argument passes through to
    `read_schema`. The `implicit_string` argument passes through to
    `to_pandas_schema`.

    I recommend setting `chunksize=k`, where k is some reasonable number of
    rows. This is a simple interface, that doesn't do basic things like
    counting the number of rows ahead of time. You may inadvertently start
    reading a 100TB file into memory. (Although, being a MS product, I assume
    the Access format breaks after 2^32 bytes -- har, har.)

    :param rdb_file: The MS Access database file.
    :param table_name: The name of the table to process.
    :param args: positional arguments passed to `pd.read_csv`
    :param kwargs: keyword arguments passed to `pd.read_csv`
    :return: a pandas `DataFrame` (or, `TextFileReader` if you set
        `chunksize=k`)
    """
    if kwargs.pop('converters_from_schema', True):
        specified_dtypes = kwargs.pop('dtype', {})
        schema_encoding = kwargs.pop('schema_encoding', 'utf8')
        schemas = to_pandas_schema(read_schema(rdb_file, schema_encoding),
                                   kwargs.pop('implicit_string', True))
        dtypes = schemas[table_name]
        dtypes.update(specified_dtypes)
        if dtypes != {}:
            kwargs['dtype'] = dtypes

    cmd = ['mdb-export', rdb_file, table_name]
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    return pd.read_csv(proc.stdout, *args, **kwargs)


text:
import numpy as np
from phimal_utilities.analysis import load_tensorboard
import pandas as pd
from os import listdir

def collect_runs(ID):
    files = [file for file in listdir(f'{ID}/runs_final/') if file[:len(ID)] == ID]#getting and sorting files
    files.sort()

    df_plot = pd.DataFrame() #df used to store results in
    noise = []
    run = []
    coeffs = []
    ini_coeffs = []
    ini_idx = []
    for file in files:
        df = load_tensorboard(f'{ID}/runs_final/{file}/')
        scaled_coeff_keys = [key for key in df.keys() if key[:6]=='scaled']
        coeffs.append(df.tail(1))
        noise.append(float(file.split('_')[1]))
        run.append(int(file.split('_')[3]))
        ini_sparse_idx = np.any(df[scaled_coeff_keys] == 0, axis=1).idxmax() - 1 # we want the step before
        ini_idx.append(ini_sparse_idx)
        ini_coeffs.append(df[scaled_coeff_keys].iloc[ini_sparse_idx].to_numpy())
        print(f'Done with {file}')

    df_plot['noise'] = noise
    df_plot['run'] = run
    df_plot['first_sparsity'] = ini_idx
    df_coeffs = pd.concat(coeffs).reset_index(drop=True)
    df_ini_coeffs = pd.DataFrame(np.stack(ini_coeffs, axis=0), columns = ['ini_' + key for key in scaled_coeff_keys])
    df_plot = pd.concat([df_plot, df_coeffs, df_ini_coeffs], axis=1)

    return df_plot

# Now run for all
df = collect_runs('threshold')
df.to_pickle('data/threshold_collected.pd')

#df = collect_runs('cluster')
#df.to_pickle('data/cluster_collected.pd')

#df = collect_runs('pdefind')
#df.to_pickle('data/pdefind_collected.pd')


text:
import rospy
import sys
import traceback
import time
import numpy as np

import moveit_commander
import moveit_msgs.msg
import geometry_msgs.msg
import tf.transformations
import tf
import trajectory_msgs.msg

FRAME_ID = "world"  # Reference frame used in all the code below
HOME_POSITION = [0, 0.7, -0.7, 0, -1.52, 0]
TOOL_LINK = "manipulator/tool0"

def set_gripper_posture(posture, set_close=False, set_open=False, effort=0):
    # type: (trajectory_msgs.msg.JointTrajectory, bool, bool) -> None
    """Set the gripper posture to 'open' or 'close' position

    :param posture: posture to set
    :param set_close: Set true to set it to close
    :param set_open: Set true to set it to open
    :param effort: [%] force used in gripping movement, set 0 if the movement
        is a simple move (no grip)
    """
    if set_close and not set_open:
        joint_position = 0.000
    elif set_open and not set_close:
        joint_position = 0.008
    else:  # Not clear what posture to set
        rospy.logwarn("Do not know how to set the gripper posture:"
                      "set_close={}, set_open={}".format(set_close, set_open))
        return
    posture.joint_names.append("eef_joint_finger_1")
    posture.joint_names.append("eef_joint_finger_2")
    posture.points = [trajectory_msgs.msg.JointTrajectoryPoint()]
    posture.points[0].positions.append(joint_position)
    posture.points[0].positions.append(joint_position)
    posture.points[0].time_from_start = rospy.Duration(secs=2, nsecs=0)
    posture.points[0].effort = [effort, effort]
    pass

def is_in_roi(pose):
    x = pose.position.x
    y = pose.position.y
    z = pose.position.z
    return x > -1 and x < 1 and y > 0.05 and y < 2 and z > -.01 and z < 0.5

def add_rnd_box(scene, name):
    # type: (moveit_commander.PlanningSceneInterface) -> None
    size = [0.02, 0.02, 0.02]
    # Set position
    pose = geometry_msgs.msg.PoseStamped()
    pose.header.frame_id = FRAME_ID
    pose.pose.position.x = 0.0 + 1.00 * (np.random.rand() - 0.5)
    pose.pose.position.y = 0.5 + 0.25 * (np.random.rand() - 0.5)
    pose.pose.position.z = 0.0 + 0.15 * np.random.rand()

    quaternion = \
        tf.transformations.quaternion_from_euler(0 + 0.0*(np.random.rand()-0.5),
                                                 0 + 0.0*(np.random.rand()-0.5),
                                                 0.0 + 0.0*(np.random.rand()-0.5))
    pose.pose.orientation.x = quaternion[0]
    pose.pose.orientation.y = quaternion[1]
    pose.pose.orientation.z = quaternion[2]
    pose.pose.orientation.w = quaternion[3]
    # Apply the objects to the scene
    #scene.add_mesh(name, filename=FILENAME, pose=pose, size=(0.001, 0.001, 0.001))
    scene.add_box(name, pose=pose, size=size)
    pass

class PickAndPlace:
    def __init__(self, group, scene):
        # type: (moveit_commander.MoveGroupCommander, moveit_commander.PlanningSceneInterface) -> None
        self.group = group
        self.scene = scene
        self.occupancy = np.zeros(shape=[4, 3, 8], dtype=int)

    def clear_scene(self):
        """Delete all the object from the scene and robot"""
        for object_name in self.scene.get_known_object_names():
            self.scene.remove_world_object(object_name)
        for object_name in self.scene.get_attached_objects():
            self.group.detach_object(object_name)
            self.scene.remove_world_object(object_name)

    def add_collision_objects(self):
        """Create collisions object in the scene"""
        # Add some .stl or objects in your scene here
        #scene.add_mesh(name, filename="C:\\Users\\...", pose=geometry_msgs.msg.PoseStamped(), size=(0.001, 0.001, 0.001))
        pass

    def get_free_place_pose(self):
        # type (None) -> geometrymsgs.msg.PoseStamped
        """Return the a pose to put an object (for pallet purposes)"""
        pose = geometry_msgs.msg.PoseStamped()
        pose.header.frame_id = FRAME_ID

        # Define the initial position of the pallet
        initial_point_x = 0.35
        initial_point_y = 0.00
        initial_point_z = 0.05

        # Find free place in the palletting matrix
        X, Y, Z = self.occupancy.shape
        found = False
        for z in range(Z):
            for y in range(Y):
                for x in range(X):
                    if self.occupancy[x, y, z] == 0:
                        self.occupancy[x, y, z] = 1
                        found = True
                        break
                    if found: break
                if found: break
            if found: break

        current_orientation = self.group.get_current_pose().pose.orientation
        q = [current_orientation.x,
             current_orientation.y,
             current_orientation.z,
             current_orientation.w]
        eul = tf.transformations.euler_from_quaternion(q)
        quaternion = tf.transformations.quaternion_from_euler(
            0,
            0,
            0
        )

        pose.pose.orientation.x = quaternion[0]
        pose.pose.orientation.y = quaternion[1]
        pose.pose.orientation.z = quaternion[2]
        pose.pose.orientation.w = quaternion[3]
        pose.pose.position.x = initial_point_x + 0.03*x
        pose.pose.position.y = initial_point_y - 0.03*y
        pose.pose.position.z = initial_point_z + 0.03*z

        return pose

    def pick(self, object_pose, object_id):
        """Pick up an object on the scene"""
        # Create a grasp position
        grasp = moveit_msgs.msg.Grasp()
        grasp.grasp_pose.header.frame_id = FRAME_ID

        # Create pre_grasp position
        set_gripper_posture(grasp.pre_grasp_posture, set_open=True, effort=0)

        # Set the pre_grasp_approach
        grasp.pre_grasp_approach.direction.header.frame_id = TOOL_LINK
        grasp.pre_grasp_approach.direction.vector.z = 1
        grasp.pre_grasp_approach.min_distance = 0.075
        grasp.pre_grasp_approach.desired_distance = 0.085
        rotation = tf.transformations.quaternion_from_euler(0, -3.14, 0)
        obj_quat = [
            object_pose.orientation.x,
            object_pose.orientation.y,
            object_pose.orientation.z,
            object_pose.orientation.w
        ]
        quaternion = tf.transformations.quaternion_multiply(obj_quat, rotation)
        grasp.grasp_pose.pose.orientation.x = quaternion[0]
        grasp.grasp_pose.pose.orientation.y = quaternion[1]
        grasp.grasp_pose.pose.orientation.z = quaternion[2]
        grasp.grasp_pose.pose.orientation.w = quaternion[3]

        # Set the position of the grasp
        grasp.grasp_pose.pose.position.x = object_pose.position.x
        grasp.grasp_pose.pose.position.y = object_pose.position.y
        grasp.grasp_pose.pose.position.z = object_pose.position.z

        # Set grasp posture
        set_gripper_posture(grasp.grasp_posture, set_close=True, effort=20)

        # Set the post_grasp_approach
        grasp.post_grasp_retreat.direction.header.frame_id = FRAME_ID
        grasp.post_grasp_retreat.direction.vector.z = 1
        grasp.post_grasp_retreat.min_distance = 0.075
        grasp.post_grasp_retreat.desired_distance = 0.085

        self.group.allow_replanning(2)
        self.group.set_max_velocity_scaling_factor(0.01)

        success = self.group.pick(object_name=object_id, grasp=grasp) == 1
        self.group.clear_path_constraints()
        return success

    def place(self, object_id):
        # type: (moveit_commander.MoveGroupCommander) -> None
        """Place a object down on the scene"""
        # Create a place_location position
        place_location = moveit_msgs.msg.PlaceLocation()
        place_location.place_pose.header.frame_id = FRAME_ID
        # Set the pose
        pose = self.get_free_place_pose()
        place_location.place_pose = pose

        # Set pre place approach
        place_location.pre_place_approach.direction.header.frame_id = FRAME_ID
        place_location.pre_place_approach.direction.vector.z = -1
        place_location.pre_place_approach.min_distance = 0.095
        place_location.pre_place_approach.desired_distance = 0.12
        # Post place
        place_location.post_place_retreat.direction.header.frame_id = FRAME_ID
        place_location.post_place_retreat.direction.vector.z = 1
        place_location.post_place_retreat.min_distance = 0.065
        place_location.post_place_retreat.desired_distance = 0.07

        set_gripper_posture(place_location.post_place_posture, set_open=True,
                            effort=0)
        self.group.set_support_surface_name("table")

        self.group.clear_path_constraints()
        success = self.group.place(object_id, place_location)
        return success

if __name__ == '__main__':
    try:
        i = 1
        # Initialize the ROS Node
        import os
        moveit_commander.roscpp_initialize(sys.argv)
        rospy.init_node("pick_and_place_example")
        # Initialize scene and move group commander
        scene = moveit_commander.PlanningSceneInterface()
        group = moveit_commander.MoveGroupCommander("manipulator")
        group.set_planner_id("RRTConnect")
        group.go(wait=True)

        group.set_max_velocity_scaling_factor(0.5)
        group.set_goal_orientation_tolerance(0.5)
        group.set_goal_position_tolerance(0.0001)

        pick_and_place = PickAndPlace(group, scene)
        # Clear the scene from objects of previous session
        pick_and_place.clear_scene()

        # Add scene collision objects (avoid robot to collide with other objects)
        pick_and_place.add_collision_objects()
        done = []
        # Main loop
        while True:
            group.go(HOME_POSITION)
            time.sleep(0.1)

            # Add a box for testing\
            rospy.loginfo("Adding box in a random position in front of the robot.")
            add_rnd_box(scene=scene, name="Box_"+str(i))
            import time
            time.sleep(2)
            # Get the objects in the scene
            objects_names = scene.get_known_object_names()

            for obj_name in objects_names:
                if "Box" not in obj_name:
                    continue
                poses = scene.get_object_poses(object_ids=objects_names)
                pose = poses[obj_name]
                obj = scene.get_objects([obj_name])[obj_name]

                # If the object is in the Region Of Interest, do stuff
                if obj_name not in done:
                    rospy.loginfo("Object '{}' is in ROI!".format(obj_name))
                    # Try to pick the object
                    if pick_and_place.pick(object_pose=pose,
                                           object_id=obj_name):
                        # Try to place the object
                        if not pick_and_place.place(object_id=obj_name):
                            rospy.logerr("Not possible to place the object")
                            raise Exception("Not possible to place the object")
                        i += 1
                        done.append(obj_name)

    except Exception:
        traceback.print_exc()

    moveit_commander.roscpp_shutdown()


text:
"""
This is needed for the MosaicView.
It is similar to the SliceBox but zooms to the cursor position.
"""

from .pyqtgraph_vini.Qt import QtCore, QtGui
import numpy as np
import sys

from .pyqtgraph_vini import *
from .pyqtgraph_vini import functions as fn
from .pyqtgraph_vini import Point
import weakref
from .pyqtgraph_vini import ItemGroup

class WeakList(object):

    def __init__(self):
        self._items = []

    def append(self, obj):
        #Add backwards to iterate backwards (to make iterating more efficient on removal).
        self._items.insert(0, weakref.ref(obj))

    def __iter__(self):
        i = len(self._items)-1
        while i >= 0:
            ref = self._items[i]
            d = ref()
            if d is None:
                del self._items[i]
            else:
                yield d
            i -= 1

class ChildGroup(ItemGroup):

    def __init__(self, parent):
        ItemGroup.__init__(self, parent)

        # Used as callback to inform ViewBox when items are added/removed from
        # the group.
        # Note 1: We would prefer to override itemChange directly on the
        #         ViewBox, but this causes crashes on PySide.
        # Note 2: We might also like to use a signal rather than this callback
        #         mechanism, but this causes a different PySide crash.
        self.itemsChangedListeners = WeakList()

        # excempt from telling view when transform changes
        self._GraphicsObject__inform_view_on_change = False

    def itemChange(self, change, value):
        ret = ItemGroup.itemChange(self, change, value)
        if change == self.ItemChildAddedChange or change == self.ItemChildRemovedChange:
            try:
                itemsChangedListeners = self.itemsChangedListeners
            except AttributeError:
                # It's possible that the attribute was already collected when the itemChange happened
                # (if it was triggered during the gc of the object).
                pass
            else:
                for listener in itemsChangedListeners:
                    listener.itemsChanged()
        return ret

class MosaicSliceBox(ViewBox):
    """
    **Bases:** :class:`GraphicsWidget <pyqtgraph.GraphicsWidget>`

    Box that allows internal scaling/panning of children by mouse drag.
    This class is usually created automatically as part of a :class:`PlotItem <pyqtgraph.PlotItem>` or :class:`Canvas <pyqtgraph.canvas.Canvas>` or with :func:`GraphicsLayout.addViewBox() <pyqtgraph.GraphicsLayout.addViewBox>`.

    Features:

    * Scaling contents by mouse or auto-scale when contents change
    * View linking--multiple views display the same data ranges
    * Configurable by context menu
    * Item coordinate mapping methods

    """

    sigYRangeChanged = QtCore.Signal(object, object)
    sigXRangeChanged = QtCore.Signal(object, object)
    sigRangeChangedManually = QtCore.Signal(object)
    sigRangeChanged = QtCore.Signal(object, object)
    #sigActionPositionChanged = QtCore.Signal(object)
    sigStateChanged = QtCore.Signal(object)
    sigTransformChanged = QtCore.Signal(object)
    sigResized = QtCore.Signal(object)

    center = [0, 0] # coordinates to zoom into.

    ## mouse modes
    PanMode = 3
    RectMode = 1

    ## axes
    XAxis = 0
    YAxis = 1
    XYAxes = 2

    ## for linking views together
    NamedViews = weakref.WeakValueDictionary()   # name: ViewBox
    AllViews = weakref.WeakKeyDictionary()       # ViewBox: None

    def __init__(self, parent=None, border=None, lockAspect=False, enableMouse=True, invertY=False, enableMenu=True, name=None, invertX=False):
        """
        ==============  =============================================================
        **Arguments:**
        *parent*        (QGraphicsWidget) Optional parent widget
        *border*        (QPen) Do draw a border around the view, give any
                        single argument accepted by :func:`mkPen <pyqtgraph.mkPen>`
        *lockAspect*    (False or float) The aspect ratio to lock the view
                        coorinates to. (or False to allow the ratio to change)
        *enableMouse*   (bool) Whether mouse can be used to scale/pan the view
        *invertY*       (bool) See :func:`invertY <pyqtgraph.ViewBox.invertY>`
        *invertX*       (bool) See :func:`invertX <pyqtgraph.ViewBox.invertX>`
        *enableMenu*    (bool) Whether to display a context menu when
                        right-clicking on the ViewBox background.
        *name*          (str) Used to register this ViewBox so that it appears
                        in the "Link axis" dropdown inside other ViewBox
                        context menus. This allows the user to manually link
                        the axes of any other view to this one.
        ==============  =============================================================
        """
        super(MosaicSliceBox, self).__init__(parent=None, border=border, lockAspect=True, enableMouse=True, invertY=False, enableMenu=True, name=None, invertX=False)
        self.zoomCenter = [0, 0]
        self.state['wheelScaleFactor'] = -0.025
        self.useMyMenu()

        """ linkedAxis saves to which axis of the linked view the axis is linked to """
        self.state['linkedAxis']= [None, None]

    def mouseDragEvent(self, ev, axis=None):
        ## if axis is specified, event will only affect that axis.
        ev.accept()  ## we accept all buttons
        pos = ev.pos()

        lastPos = ev.lastPos()
        dif = pos - lastPos
        dif = dif * -1

        ## Ignore axes if mouse is disabled
        mouseEnabled = np.array(self.state['mouseEnabled'], dtype=np.float)
        mask = mouseEnabled.copy()
        if axis is not None:
            mask[1-axis] = 0.0

        ## Scale or translate based on mouse button
        if ev.button() & (QtCore.Qt.RightButton | QtCore.Qt.MidButton):

            if self.state['mouseMode'] == MosaicSliceBox.RectMode:
                if ev.isFinish():  ## This is the final move in the drag; change the view scale now
                    #print "finish"
                    self.rbScaleBox.hide()
                    #ax = QtCore.QRectF(Point(self.pressPos), Point(self.mousePos))
                    ax = QtCore.QRectF(Point(ev.buttonDownPos(ev.button())), Point(pos))
                    ax = self.childGroup.mapRectFromParent(ax)
                    self.showAxRect(ax)
                    self.axHistoryPointer += 1
                    self.axHistory = self.axHistory[:self.axHistoryPointer] + [ax]
                else:
                    ## update shape of scale box
                    self.updateScaleBox(ev.buttonDownPos(), ev.pos())

            else:
                tr = dif*mask
                tr = self.mapToView(tr) - self.mapToView(Point(0,0))
                x = tr.x() if mask[0] == 1 else None
                y = tr.y() if mask[1] == 1 else None

                self._resetTarget()
                if x is not None or y is not None:
                    self.translateBy(x=x, y=y)
                self.sigRangeChangedManually.emit(self.state['mouseEnabled'])

max_stars_repo_name: lipsia-fmri/viff

max_issues_repo_name: lipsia-fmri/vini

max_forks_repo_name: lipsia-fmri/viff