text:
import time
import unittest
from os import pipe
from typing import List, Tuple, cast

import numpy as np
from donkeycar.pipeline.sequence import SizedIterator, TubSequence, Reshaper
from donkeycar.pipeline.types import TubRecord, TubRecordDict

def random_records(size: int = 100) -> List[TubRecord]:
    return [random_record() for _ in range(size)]

def random_record():
    now = int(time.time())
    underlying: TubRecordDict = {
        'cam/image_array': f'/path/to/{now}.txt',
        'user/angle': np.random.uniform(0, 1.),
        'user/throttle': np.random.uniform(0, 1.),
        'user/mode': 'driving',
        'imu/acl_x': None,
        'imu/acl_y': None,
        'imu/acl_z': None,
        'imu/gyr_x': None,
        'imu/gyr_y': None,
        'imu/gyr_z': None
    }
    return TubRecord('/base', None, underlying=underlying)

size = 10

class TestPipeline(unittest.TestCase):

    def setUp(self):
        records = random_records(size=size)
        self.sequence = TubSequence(records=records)

    def test_basic_iteration(self):
        self.assertEqual(len(self.sequence), size)
        count = 0
        for record in self.sequence:
            print(f'Record {record}')
            count += 1

        self.assertEqual(count, size)

    def test_basic_map_operations(self):
        transformed = TubSequence.build_pipeline(
            self.sequence,
            x_transform=lambda record: record.underlying['user/angle'],
            y_transform=lambda record: record.underlying['user/throttle'])

        transformed_2 = TubSequence.build_pipeline(
            self.sequence,
            x_transform=lambda record: record.underlying['user/angle'] * 2,
            y_transform=lambda record: record.underlying['user/throttle'] * 2)

        self.assertEqual(len(transformed), size)
        self.assertEqual(len(transformed_2), size)

        transformed_list = list(transformed)
        transformed_list_2 = list(transformed_2)
        index = np.random.randint(0, 9)

        x1, y1 = transformed_list[index]
        x2, y2 = transformed_list_2[index]

        self.assertAlmostEqual(x1 * 2, x2)
        self.assertAlmostEqual(y1 * 2, y2)

    def test_more_map_operations(self):
        transformed = TubSequence.build_pipeline(
            self.sequence,
            x_transform=lambda record: record.underlying['user/angle'],
            y_transform=lambda record: record.underlying['user/throttle'])

        transformed_2 = TubSequence.build_pipeline(
            self.sequence,
            x_transform=lambda record: record.underlying['user/angle'] * 2,
            y_transform=lambda record: record.underlying['user/throttle'] * 2)

        transformed_3 = TubSequence.map_pipeline(
            x_transform=lambda x: x,
            y_transform=lambda y: y,
            pipeline=transformed_2
        )

        self.assertEqual(len(transformed), size)
        self.assertEqual(len(transformed_2), size)
        self.assertEqual(len(transformed_3), size)

        transformed_list = list(transformed)
        transformed_list_2 = list(transformed_3)
        index = np.random.randint(0, 9)

        x1, y1 = transformed_list[index]
        x2, y2 = transformed_list_2[index]

        self.assertAlmostEqual(x1 * 2, x2)
        self.assertAlmostEqual(y1 * 2, y2)

    def test_map_factory(self):
        transformed = TubSequence.build_pipeline(
            self.sequence,
            x_transform=lambda record: record.underlying['user/angle'],
            y_transform=lambda record: record.underlying['user/throttle'])

        transformed_2 = TubSequence.build_pipeline(
            self.sequence,
            x_transform=lambda record: record.underlying['user/angle'] * 2,
            y_transform=lambda record: record.underlying['user/throttle'] * 2)

        transformed_3 = TubSequence.map_pipeline_factory(
            x_transform=lambda x: x,
            y_transform=lambda y: y,
            factory=lambda: transformed_2
        )

        self.assertEqual(len(transformed), size)
        self.assertEqual(len(transformed_2), size)
        self.assertEqual(len(transformed_3), size)

        transformed_list = list(transformed)
        transformed_list_2 = list(transformed_3)
        index = np.random.randint(0, 9)

        x1, y1 = transformed_list[index]
        x2, y2 = transformed_list_2[index]

        self.assertAlmostEqual(x1 * 2, x2)
        self.assertAlmostEqual(y1 * 2, y2)

    def test_batch_1(self):
        transformed = TubSequence.build_pipeline(
            self.sequence,
            x_transform=lambda record: record.underlying['user/angle'],
            y_transform=lambda record: record.underlying['user/throttle'])

        batch_size = 4
        batched = transformed.batched(batch_size=batch_size)
        self.assertEqual(len(batched), 3)
        for records in batched:
            print(f'Records {records}')
            self.assertLessEqual(len(records), batch_size)

    def test_batch_2(self):
        transformed = TubSequence.build_pipeline(
            self.sequence,
            x_transform=lambda record: record.underlying['user/angle'],
            y_transform=lambda record: record.underlying['user/throttle'])

        batch_size = 4
        batched = transformed.batched(batch_size=batch_size)
        batched = cast(SizedIterator[List[Tuple[float, float]]], batched)
        reshaped = Reshaper(batched=batched)
        self.assertEqual(len(batched), len(reshaped))
        count = 0
        for X, Y in reshaped:
            print(f'X = {X}, Y = {Y}')
            count += 1

        self.assertEqual(count, len(batched))

if __name__ == '__main__':
    unittest.main()


text:
# -*- coding: utf-8 -*-
import os
import argparse
import numpy as np
from tqdm import tqdm
import torch
from torchvision import transforms
from robustness.datasets import ImageNet

from models.autoprotonet import AutoProtoNetEmbedding

from utils import Timer, log

def get_model(options):
    # Choose the embedding network
    is_miniimagenet = options.dataset == 'miniImageNet'
    network = AutoProtoNetEmbedding(activation = options.activation, is_miniimagenet=is_miniimagenet).cuda()
    return network

def get_dataset(options):
    class Lighting(object):
        """
        Lighting noise (see https://git.io/fhBOc)
        """
        def __init__(self, alphastd, eigval, eigvec):
            self.alphastd = alphastd
            self.eigval = eigval
            self.eigvec = eigvec

        def __call__(self, img):
            if self.alphastd == 0:
                return img

            alpha = img.new().resize_(3).normal_(0, self.alphastd)
            rgb = self.eigvec.type_as(img).clone()\
                .mul(alpha.view(1, 3).expand(3, 3))\
                .mul(self.eigval.view(1, 3).expand(3, 3))\
                .sum(1).squeeze()

            return img.add(rgb.view(3, 1, 1).expand_as(img))

    IMAGENET_PCA = {
    'eigval':torch.Tensor([0.2175, 0.0188, 0.0045]),
    'eigvec':torch.Tensor([
        [-0.5675,  0.7192,  0.4009],
        [-0.5808, -0.0045, -0.8140],
        [-0.5836, -0.6948,  0.4203],
    ])
    }

    PRETRAIN_TRANSFORMS_IMAGENET = transforms.Compose([
            transforms.Resize(100),
            transforms.RandomCrop(84),
            transforms.RandomHorizontalFlip(),
            transforms.ColorJitter(
                brightness=0.1,
                contrast=0.1,
                saturation=0.1
            ),
            transforms.ToTensor(),
            Lighting(0.05, IMAGENET_PCA['eigval'],
                        IMAGENET_PCA['eigvec'])
        ])

    VAL_TRANSFORMS_IMAGENET = transforms.Compose([
        transforms.Resize(100),
        transforms.CenterCrop(84),
        transforms.ToTensor(),
    ])

    ds = ImageNet(data_path='/fs/vulcan-datasets/imagenet',
                  transform_train=PRETRAIN_TRANSFORMS_IMAGENET,
                  transform_test=VAL_TRANSFORMS_IMAGENET)

    # Returns (train_loader, test_loader)
    loaders = ds.make_loaders(workers=options.workers, batch_size=options.batch_size)
    return loaders

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--num-epoch', type=int, default=20,
                            help='number of training epochs')
    parser.add_argument('--save-epoch', type=int, default=10,
                            help='frequency of model saving')
    parser.add_argument('--save-path', default='./experiments/exp_1')
    parser.add_argument('--disable_tqdm', action='store_true',
                        help='disables tqdm progress bar')

    parser.add_argument('--activation', type=str, default='LeakyReLU',
                        help='choose which activation function to use. only implemented for R2D2 and ProtoNet')
    parser.add_argument('--dataset', type=str, default='miniImageNet',
                            help='choose which classification head to use. miniImageNet, tieredImageNet, CIFAR_FS, FC100')
    parser.add_argument('--workers', type=int, default=1,
                        help='number of workers for data loaders')
    parser.add_argument('--batch-size', type=int, default=1024,
                        help='number of workers for data loaders')

    opt = parser.parse_args()

    # Use resized ImageNet as pretraining
    (dloader_train, dloader_val) = get_dataset(opt)

    if not os.path.isdir(opt.save_path):
        os.makedirs(opt.save_path+'/', )

    log_file_path = os.path.join(opt.save_path, "train_log.txt")
    if os.path.exists(log_file_path):
        os.remove(log_file_path)
    log(log_file_path, str(vars(opt)))

    embedding_net = get_model(opt)

    optimizer = torch.optim.SGD([{'params': embedding_net.parameters()}],
                                lr=0.1, momentum=0.9, \
                                weight_decay=5e-4, nesterov=True)

    lambda_epoch = lambda e: 1.0 if e < 5 else (0.1 if e < 10 else 0.01 if e < 15 else (0.001))
    lr_scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_epoch, last_epoch=-1)

    min_val_loss = np.inf

    timer = Timer()
    mse_loss = torch.nn.MSELoss()

    for epoch in range(1, opt.num_epoch + 1):
        # Train on the training split

        # Fetch the current epoch's learning rate
        epoch_learning_rate = 0.1
        for param_group in optimizer.param_groups:
            epoch_learning_rate = param_group['lr']

        log(log_file_path, 'Train Epoch: {}\tLearning Rate: {:.4f}'.format(
                            epoch, epoch_learning_rate))

        embedding_net.train()
        train_losses = []
        for i, batch in enumerate(tqdm(dloader_train, disable=opt.disable_tqdm)):
            x, _ = batch
            x = x.cuda()
            recon_x = embedding_net.forward_plus_decoder(x)
            loss = mse_loss(recon_x, x)

            train_losses.append(loss.item())
            if (i % 100 == 0):
                train_loss_avg = np.mean(np.array(train_losses))
                log(log_file_path, 'Train Epoch: {}\tBatch: [{}/{}]\tLoss: {:.4f}\tLoss Avg: {:.2f}'.format(
                            epoch, i, len(dloader_train), loss.item(), train_loss_avg))

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        lr_scheduler.step()

        # Evaluate on the validation split
        embedding_net.eval()
        val_losses = []
        with torch.no_grad():
            for i, batch in enumerate(tqdm(dloader_val, disable=opt.disable_tqdm)):
                x, _ = batch
                x = x.cuda()
                recon_x = embedding_net.forward_plus_decoder(x)
                loss = mse_loss(recon_x, x)

                val_losses.append(loss.item())

            val_loss_avg = np.mean(np.array(val_losses))
            val_loss_ci95 = 1.96 * np.std(np.array(val_losses)) / np.sqrt(len(val_losses))

            if val_loss_avg < min_val_loss:
                min_val_loss = val_loss_avg
                torch.save({'embedding': embedding_net.state_dict()},\
                           os.path.join(opt.save_path, 'best_model.pth'))
                log(log_file_path, 'Validation Epoch: {}\t\t\tLoss: {:.4f} ± {:.2f} (Best)'\
                    .format(epoch, val_loss_avg, val_loss_ci95))
            else:
                log(log_file_path, 'Validation Epoch: {}\t\t\tLoss: {:.4f} ± {:.2f} '\
                    .format(epoch, val_loss_avg, val_loss_ci95))

        torch.save({'embedding': embedding_net.state_dict()}\
                   , os.path.join(opt.save_path, 'last_epoch.pth'))

        if epoch % opt.save_epoch == 0:
            torch.save({'embedding': embedding_net.state_dict()}\
                       , os.path.join(opt.save_path, 'epoch_{}.pth'.format(epoch)))

        log(log_file_path, 'Elapsed Time: {}/{}\n'.format(timer.measure(), timer.measure(epoch / float(opt.num_epoch))))

ext: py

max_stars_repo_name: psandovalsegura/AdversarialQuerying

max_issues_repo_name: psandovalsegura/AdversarialQuerying

max_forks_repo_name: psandovalsegura/AdversarialQuerying

reason: import numpy


text:
import torch
import numpy as np
import torch.nn as nn

from visualization import Presenter

class PVN_Stage2_RLBase(nn.Module):
    def __init__(self, map_channels=1, map_struct_channels=1, map_size=32, crop_size=16, h1=8, structure_h1=8, h2=128, obs_dim=16, name="base"):
        super(PVN_Stage2_RLBase, self).__init__()

        self.map_channels = map_channels
        self.map_structure_channels = map_struct_channels
        self.map_size = map_size
        self.crop_size = crop_size
        self.name = name

        gap = int((map_size - crop_size) / 2)
        self.crop_l = gap
        self.crop_r = map_size - gap

        self.conv1 = nn.Conv2d(map_channels, h1, kernel_size=3, stride=2, padding=1, bias=True)

        self.structconv1 = nn.Conv2d(self.map_structure_channels, structure_h1, kernel_size=3, stride=2, padding=1, bias=True)

        linear_in = int(((self.crop_size / 2) ** 2) * h1) + (8*8*structure_h1) + 2*obs_dim # map channels + coverage channels + observability encoding channels
        print(f"Stage 2 linear input size: {linear_in}")
        self.linear1 = nn.Linear(linear_in, h2)
        self.linear2 = nn.Linear(h2 + linear_in, h2)

        self.goal_in_vec = nn.Parameter(torch.Tensor(obs_dim))
        self.goal_out_vec = nn.Parameter(torch.Tensor(obs_dim))
        self.visit_in_vec = nn.Parameter(torch.Tensor(obs_dim))
        self.visit_out_vec = nn.Parameter(torch.Tensor(obs_dim))

        self.avgpool = nn.AvgPool2d(4)

        self.act = nn.LeakyReLU()
        self.norm1 = nn.InstanceNorm2d(h1)
        self.covnorm1 = nn.InstanceNorm2d(structure_h1)

    def init_weights(self):
        self.goal_in_vec.data.normal_(0, 1.0)
        self.goal_out_vec.data.copy_(-self.goal_in_vec.data)
        self.visit_in_vec.data.normal_(0, 1.0)
        self.visit_out_vec.data.copy_(-self.visit_in_vec.data)

    def backward_hook(self, name, grad):
        if False:
            print(f"Grad stats {self.name}/{name}: {grad.max()} {grad.min()} {grad.mean()}")

    def forward(self, maps_r, map_structure_r):
        maps_r_cropped = maps_r.inner_distribution[:, :, self.crop_l:self.crop_r, self.crop_l:self.crop_r]
        batch_size = maps_r.inner_distribution.shape[0]

        # Create a context vector that encodes goal observability
        # Don't backprop into the embedding vectors - don't risk losing the only input we have
        gin = self.goal_in_vec.detach()[np.newaxis, :].repeat([batch_size, 1])
        gout = self.goal_out_vec.detach()[np.newaxis, :].repeat([batch_size, 1])
        vin = self.visit_in_vec.detach()[np.newaxis, :].repeat([batch_size, 1])
        vout = self.visit_out_vec.detach()[np.newaxis, :].repeat([batch_size, 1])

        p_visit_out = maps_r.outer_prob_mass[:, 0:1].detach()
        p_goal_out = maps_r.outer_prob_mass[:, 1:2].detach()

        g_context_vec = gout * p_goal_out + gin * (1 - p_goal_out)
        v_context_vec = vout * p_visit_out + vin * (1 - p_visit_out)
        obs_context_vec = torch.cat([g_context_vec, v_context_vec], dim=1)

        # 64x64 -> 16x16
        uncov_r_pooled = self.avgpool(map_structure_r)

        if False:
            conv_in_np = conv_in[0].data.cpu().numpy().transpose(1, 2, 0)
            # expand to 0-1 range
            conv_in_np[:, :, 0] /= (np.max(conv_in_np[:, :, 0]) + 1e-10)
            conv_in_np[:, :, 1] /= (np.max(conv_in_np[:, :, 1]) + 1e-10)
            conv_in_np[:, :, 2] /= (np.max(conv_in_np[:, :, 2]) + 1e-10)
            Presenter().show_image(conv_in_np, "rl_conv_in", scale=2)
            #Presenter().show_image(uncov_r_pooled[0], "uncov_pooled", scale=4)

        # From 16x16 down to 8x8
        x = self.act(self.conv1(maps_r_cropped))
        x = self.norm1(x)

        # From 16x16 down to 8x8
        c = self.act(self.structconv1(uncov_r_pooled))
        c = self.covnorm1(c)

        comb_map = torch.cat([x, c], dim=1)
        batch_size = x.shape[0]
        lin_in = comb_map.view(batch_size, -1)
        lin_in = torch.cat([lin_in, obs_context_vec], dim=1)

        x = self.act(self.linear1(lin_in))
        x = torch.cat([lin_in, x], dim=1)
        x = self.act(self.linear2(x))

        return x

max_stars_repo_name: esteng/guiding-multi-step

max_issues_repo_name: esteng/guiding-multi-step

max_forks_repo_name: esteng/guiding-multi-step


text:
# file: numpy_pi.py
"""Calculating pi with Monte Carlo Method and NumPy.
"""

from __future__ import print_function

import numpy                                                   #1

def pi_numpy(total):                                           #2
    """Compute pi.
    """
    x = numpy.random.rand(total)                               #3
    y = numpy.random.rand(total)                               #4
    dist = numpy.sqrt(x * x + y * y)                           #5
    count_inside = len(dist[dist < 1])                         #6
    return 4.0 * count_inside / total

if __name__ == '__main__':

    def test():
        """Time the execution.
        """
        import timeit
        start = timeit.default_timer()
        pi_numpy(int(1e6))
        print('run time', timeit.default_timer() - start)
    test()

max_stars_repo_name: jakerye/notebook

max_issues_repo_name: jakerye/notebook

max_forks_repo_name: jakerye/notebook


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2020 Patrick Lumban Tobing (Nagoya University)
#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)

from __future__ import division

import argparse
import logging
import math
import os
import sys

import numpy as np
import torch
import torch.multiprocessing as mp

from vcneuvoco import GRU_VAE_ENCODER, GRU_SPEC_DECODER, GRU_EXCIT_DECODER, nn_search_batch
from utils import find_files, read_hdf5, read_txt, write_hdf5, check_hdf5

from dtw_c import dtw_c as dtw

import torch.nn.functional as F
import h5py

#np.set_printoptions(threshold=np.inf)

def main():
    parser = argparse.ArgumentParser()
    # decode setting
    parser.add_argument("--feats", required=True,
                        type=str, help="list or directory of source eval feat files")
    parser.add_argument("--spk", required=True,
                        type=str, help="speaker name to be reconstructed")
    parser.add_argument("--model", required=True,
                        type=str, help="model file")
    parser.add_argument("--config", required=True,
                        type=str, help="configure file")
    parser.add_argument("--n_gpus", default=1,
                        type=int, help="number of gpus")
    parser.add_argument("--outdir", required=True,
                        type=str, help="directory to save log")
    parser.add_argument("--string_path", required=True,
                        type=str, help="path of h5 generated feature")
    # other setting
    parser.add_argument("--GPU_device", default=None,
                        type=int, help="selection of GPU device")
    parser.add_argument("--GPU_device_str", default=None,
                        type=str, help="selection of GPU device")
    parser.add_argument("--verbose", default=1,
                        type=int, help="log level")

    args = parser.parse_args()

    if args.GPU_device is not None or args.GPU_device_str is not None:
        os.environ["CUDA_DEVICE_ORDER"]		= "PCI_BUS_ID"
        if args.GPU_device_str is None:
            os.environ["CUDA_VISIBLE_DEVICES"]	= str(args.GPU_device)
        else:
            os.environ["CUDA_VISIBLE_DEVICES"]	= args.GPU_device_str

    # check directory existence
    if not os.path.exists(args.outdir):
        os.makedirs(args.outdir)

    # set log level
    if args.verbose > 0:
        logging.basicConfig(level=logging.INFO,
                            format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S', filemode='w',
                            filename=args.outdir + "/decode.log")
        logging.getLogger().addHandler(logging.StreamHandler())
    elif args.verbose > 1:
        logging.basicConfig(level=logging.DEBUG,
                            format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S', filemode='w',
                            filename=args.outdir + "/decode.log")
        logging.getLogger().addHandler(logging.StreamHandler())
    else:
        logging.basicConfig(level=logging.WARN,
                            format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S', filemode='w',
                            filename=args.outdir + "/decode.log")
        logging.getLogger().addHandler(logging.StreamHandler())
        logging.warn("logging is disabled.")

    # load config
    config = torch.load(args.config)

    # get source feat list
    if os.path.isdir(args.feats):
        feat_list = sorted(find_files(args.feats, "*.h5"))
    elif os.path.isfile(args.feats):
        feat_list = read_txt(args.feats)
    else:
        logging.error("--feats should be directory or list.")
        sys.exit(1)

    # prepare the file list for parallel decoding
    feat_lists = np.array_split(feat_list, args.n_gpus)
    feat_lists = [f_list.tolist() for f_list in feat_lists]
    for i in range(args.n_gpus):
        logging.info('%d: %d' % (i+1, len(feat_lists[i])))

    spk_list = config.spk_list.split('@')
    n_spk = len(spk_list)
    spk_idx = spk_list.index(args.spk)

    stats_list = config.stats_list.split('@')
    assert(n_spk == len(stats_list))

    spk_stat = stats_list[spk_idx]
    gv_mean = read_hdf5(spk_stat, "/gv_range_mean")[1:]

    spk_f0_mean = read_hdf5(spk_stat, "/lf0_range_mean")
    spk_f0_std = read_hdf5(spk_stat, "/lf0_range_std")
    logging.info(spk_f0_mean)
    logging.info(spk_f0_std)

    model_epoch = os.path.basename(args.model).split('.')[0].split('-')[1]
    logging.info('epoch: '+model_epoch)

    model_name = os.path.basename(os.path.dirname(args.model)).split('_')[1]
    logging.info('mdl_name: '+model_name)

    logging.info(config)
    # define gpu decode function
    def gpu_decode(feat_list, gpu, cvlist=None, mcdpow_cvlist=None, mcdpowstd_cvlist=None, mcd_cvlist=None, \
                    mcdstd_cvlist=None, cvlist_dv=None, mcdpow_cvlist_dv=None, mcdpowstd_cvlist_dv=None, \
                    mcd_cvlist_dv=None, mcdstd_cvlist_dv=None, \
                    f0rmse_cvlist=None, f0corr_cvlist=None, caprmse_cvlist=None, \
                    f0rmse_cvlist_dv=None, f0corr_cvlist_dv=None, caprmse_cvlist_dv=None, \
                    cvlist_cyc=None, mcdpow_cvlist_cyc=None, mcdpowstd_cvlist_cyc=None, mcd_cvlist_cyc=None, \
                    mcdstd_cvlist_cyc=None, cvlist_cyc_dv=None, mcdpow_cvlist_cyc_dv=None, mcdpowstd_cvlist_cyc_dv=None, \
                    mcd_cvlist_cyc_dv=None, mcdstd_cvlist_cyc_dv=None, \
                    f0rmse_cvlist_cyc=None, f0corr_cvlist_cyc=None, caprmse_cvlist_cyc=None, \
                    f0rmse_cvlist_cyc_dv=None, f0corr_cvlist_cyc_dv=None, caprmse_cvlist_cyc_dv=None):
        with torch.cuda.device(gpu):
            # define model and load parameters
            with torch.no_grad():
                model_encoder_mcep = GRU_VAE_ENCODER(
                    in_dim=config.mcep_dim+config.excit_dim,
                    n_spk=n_spk,
                    lat_dim=config.lat_dim,
                    hidden_layers=config.hidden_layers_enc,
                    hidden_units=config.hidden_units_enc,
                    kernel_size=config.kernel_size_enc,
                    dilation_size=config.dilation_size_enc,
                    causal_conv=config.causal_conv_enc,
                    bi=config.bi_enc,
                    cont=False,
                    pad_first=True,
                    right_size=config.right_size,
                    ar=config.ar_enc)
                logging.info(model_encoder_mcep)
                model_decoder_mcep = GRU_SPEC_DECODER(
                    feat_dim=config.lat_dim,
                    out_dim=config.mcep_dim,
                    n_spk=n_spk,
                    hidden_layers=config.hidden_layers_dec,
                    hidden_units=config.hidden_units_dec,
                    kernel_size=config.kernel_size_dec,
                    dilation_size=config.dilation_size_dec,
                    causal_conv=config.causal_conv_dec,
                    bi=config.bi_dec,
                    spkidtr_dim=config.spkidtr_dim,
                    pad_first=True,
                    ar=config.ar_dec)
                logging.info(model_decoder_mcep)
                model_encoder_excit = GRU_VAE_ENCODER(
                    in_dim=config.mcep_dim+config.excit_dim,
                    n_spk=n_spk,
                    lat_dim=config.lat_dim_e,
                    hidden_layers=config.hidden_layers_enc,
                    hidden_units=config.hidden_units_enc,
                    kernel_size=config.kernel_size_enc,
                    dilation_size=config.dilation_size_enc,
                    causal_conv=config.causal_conv_enc,
                    bi=config.bi_enc,
                    cont=False,
                    pad_first=True,
                    right_size=config.right_size,
                    ar=config.ar_enc)
                logging.info(model_encoder_excit)
                model_decoder_excit = GRU_EXCIT_DECODER(
                    feat_dim=config.lat_dim_e,
                    cap_dim=config.cap_dim,
                    n_spk=n_spk,
                    hidden_layers=config.hidden_layers_lf0,
                    hidden_units=config.hidden_units_lf0,
                    kernel_size=config.kernel_size_lf0,
                    dilation_size=config.dilation_size_lf0,
                    causal_conv=config.causal_conv_lf0,
                    bi=config.bi_lf0,
                    spkidtr_dim=config.spkidtr_dim,
                    pad_first=True,
                    ar=config.ar_f0)
                logging.info(model_decoder_excit)
                model_vq = torch.nn.Embedding(config.ctr_size, config.lat_dim)
                logging.info(model_vq)
                model_encoder_mcep.load_state_dict(torch.load(args.model)["model_encoder_mcep"])
                model_decoder_mcep.load_state_dict(torch.load(args.model)["model_decoder_mcep"])
                model_encoder_excit.load_state_dict(torch.load(args.model)["model_encoder_excit"])
                model_decoder_excit.load_state_dict(torch.load(args.model)["model_decoder_excit"])
                model_vq.load_state_dict(torch.load(args.model)["model_vq"])
                model_encoder_mcep.cuda()
                model_decoder_mcep.cuda()
                model_encoder_excit.cuda()
                model_decoder_excit.cuda()
                model_vq.cuda()
                model_encoder_mcep.eval()
                model_decoder_mcep.eval()
                model_encoder_excit.eval()
                model_decoder_excit.eval()
                model_vq.eval()
                for param in model_encoder_mcep.parameters():
                    param.requires_grad = False
                for param in model_decoder_mcep.parameters():
                    param.requires_grad = False
                for param in model_encoder_excit.parameters():
                    param.requires_grad = False
                for param in model_decoder_excit.parameters():
                    param.requires_grad = False
                for param in model_vq.parameters():
                    param.requires_grad = False
                if config.ar_enc:
                    yz_in = torch.zeros((1, 1, n_spk+config.lat_dim)).cuda()
                    yz_in_e = torch.zeros((1, 1, n_spk+config.lat_dim_e)).cuda()
                if config.ar_dec or config.ar_f0:
                    mean_stats = torch.FloatTensor(read_hdf5(config.stats, "/mean_"+config.string_path.replace("/","")))
                    scale_stats = torch.FloatTensor(read_hdf5(config.stats, "/scale_"+config.string_path.replace("/","")))
                if config.ar_dec:
                    x_in = ((torch.zeros((1, 1, config.mcep_dim))-mean_stats[config.excit_dim:])/scale_stats[config.excit_dim:]).cuda()
                if config.ar_f0:
                    e_in = torch.cat((torch.zeros(1,1,1), (torch.zeros(1,1,1)-mean_stats[1:2])/scale_stats[1:2], \
                                    torch.zeros(1,1,1), (torch.zeros(1,1,config.cap_dim)-mean_stats[3:config.excit_dim])/scale_stats[3:config.excit_dim]), 2).cuda()
            count = 0
            pad_left = (model_encoder_mcep.pad_left + model_decoder_mcep.pad_left)*2
            pad_right = (model_encoder_mcep.pad_right + model_decoder_mcep.pad_right)*2
            outpad_lefts = [None]*3
            outpad_rights = [None]*3
            outpad_lefts[0] = pad_left-model_encoder_mcep.pad_left
            outpad_rights[0] = pad_right-model_encoder_mcep.pad_right
            outpad_lefts[1] = outpad_lefts[0]-model_decoder_mcep.pad_left
            outpad_rights[1] = outpad_rights[0]-model_decoder_mcep.pad_right
            outpad_lefts[2] = outpad_lefts[1]-model_encoder_mcep.pad_left
            outpad_rights[2] = outpad_rights[1]-model_encoder_mcep.pad_right
            for feat_file in feat_list:
                # convert mcep
                logging.info("recmcep " + feat_file)

                feat_org = read_hdf5(feat_file, "/feat_mceplf0cap")
                logging.info(feat_org.shape)

                with torch.no_grad():
                    feat = F.pad(torch.FloatTensor(feat_org).cuda().unsqueeze(0).transpose(1,2), (pad_left,pad_right), "replicate").transpose(1,2)

                    if config.ar_enc:
                        spk_logits, lat_src, _, _ = model_encoder_mcep(feat, yz_in=yz_in)
                        spk_logits_e, lat_src_e, _, _ = model_encoder_excit(feat, yz_in=yz_in)
                    else:
                        spk_logits, lat_src, _ = model_encoder_mcep(feat)
                        spk_logits_e, lat_src_e, _ = model_encoder_excit(feat)
                    idx_vq = nn_search_batch(lat_src, model_vq.weight)
                    lat_src = model_vq(idx_vq)
                    if outpad_rights[0] > 0:
                        unique, counts = np.unique(idx_vq[:,outpad_lefts[0]:-outpad_rights[0]].cpu().data.numpy(), return_counts=True)
                    else:
                        unique, counts = np.unique(idx_vq[:,outpad_lefts[0]:].cpu().data.numpy(), return_counts=True)
                    logging.info("input vq")
                    logging.info(dict(zip(unique, counts)))
                    idx_vq_e = nn_search_batch(lat_src_e, model_vq.weight)
                    lat_src_e = model_vq(idx_vq_e)
                    if outpad_rights[0] > 0:
                        unique, counts = np.unique(idx_vq_e[:,outpad_lefts[0]:-outpad_rights[0]].cpu().data.numpy(), return_counts=True)
                    else:
                        unique, counts = np.unique(idx_vq_e[:,outpad_lefts[0]:].cpu().data.numpy(), return_counts=True)
                    logging.info("input vq_e")
                    logging.info(dict(zip(unique, counts)))
                    logging.info('input spkpost')
                    if outpad_rights[0] > 0:
                        logging.info(torch.mean(F.softmax(spk_logits[:,outpad_lefts[0]:-outpad_rights[0]], dim=-1), 1))
                    else:
                        logging.info(torch.mean(F.softmax(spk_logits[:,outpad_lefts[0]:], dim=-1), 1))
                    logging.info('input spkpost_e')
                    if outpad_rights[0] > 0:
                        logging.info(torch.mean(F.softmax(spk_logits_e[:,outpad_lefts[0]:-outpad_rights[0]], dim=-1), 1))
                    else:
                        logging.info(torch.mean(F.softmax(spk_logits_e[:,outpad_lefts[0]:], dim=-1), 1))

                    src_code = (torch.ones((1, lat_src.shape[1]))*spk_idx).cuda().long()

                    if config.ar_dec:
                        cvmcep_src, _, _ = model_decoder_mcep(src_code, lat_src, x_in=x_in)
                    else:
                        cvmcep_src, _ = model_decoder_mcep(src_code, lat_src)
                    if config.ar_f0:
                        cvlf0_src, _, _ = model_decoder_excit(src_code, lat_src_e, e_in=e_in)
                    else:
                        cvlf0_src, _ = model_decoder_excit(src_code, lat_src_e)

                    cv_feat = torch.cat((cvlf0_src, cvmcep_src), 2)
                    if config.ar_enc:
                        spk_logits, lat_rec, _, _ = model_encoder_mcep(cv_feat, yz_in=yz_in)
                        spk_logits_e, lat_rec_e, _, _ = model_encoder_excit(cv_feat, yz_in=yz_in)
                    else:
                        spk_logits, lat_rec, _ = model_encoder_mcep(cv_feat)
                        spk_logits_e, lat_rec_e, _ = model_encoder_excit(cv_feat)
                    idx_vq = nn_search_batch(lat_rec, model_vq.weight)
                    lat_rec = model_vq(idx_vq)
                    if outpad_rights[2] > 0:
                        unique, counts = np.unique(idx_vq[:,outpad_lefts[2]:-outpad_rights[2]].cpu().data.numpy(), return_counts=True)
                    else:
                        unique, counts = np.unique(idx_vq[:,outpad_lefts[2]:].cpu().data.numpy(), return_counts=True)
                    logging.info("input vq")
                    logging.info(dict(zip(unique, counts)))
                    idx_vq_e = nn_search_batch(lat_rec_e, model_vq.weight)
                    lat_rec_e = model_vq(idx_vq_e)
                    if outpad_rights[2] > 0:
                        unique, counts = np.unique(idx_vq_e[:,outpad_lefts[2]:-outpad_rights[2]].cpu().data.numpy(), return_counts=True)
                    else:
                        unique, counts = np.unique(idx_vq_e[:,outpad_lefts[2]:].cpu().data.numpy(), return_counts=True)
                    logging.info("input vq_e")
                    logging.info(dict(zip(unique, counts)))
                    logging.info('rec spkpost')
                    if outpad_rights[2] > 0:
                        logging.info(torch.mean(F.softmax(spk_logits[:,outpad_lefts[2]:-outpad_rights[2]], dim=-1), 1))
                    else:
                        logging.info(torch.mean(F.softmax(spk_logits[:,outpad_lefts[2]:], dim=-1), 1))
                    logging.info('rec spkpost_e')
                    if outpad_rights[2] > 0:
                        logging.info(torch.mean(F.softmax(spk_logits_e[:,outpad_lefts[2]:-outpad_rights[2]], dim=-1), 1))
                    else:
                        logging.info(torch.mean(F.softmax(spk_logits_e[:,outpad_lefts[2]:], dim=-1), 1))

                    src_code = (torch.ones((1, lat_rec.shape[1]))*spk_idx).cuda().long()

                    if config.ar_dec:
                        cvmcep_cyc, _, _ = model_decoder_mcep(src_code, lat_rec, x_in=x_in)
                    else:
                        cvmcep_cyc, _ = model_decoder_mcep(src_code, lat_rec)
                    if config.ar_f0:
                        cvlf0_cyc, _, _ = model_decoder_excit(src_code, lat_rec_e, e_in=e_in)
                    else:
                        cvlf0_cyc, _ = model_decoder_excit(src_code, lat_rec_e)

                    if outpad_rights[1] > 0:
                        cvmcep_src = cvmcep_src[:,outpad_lefts[1]:-outpad_rights[1]]
                        cvlf0_src = cvlf0_src[:,outpad_lefts[1]:-outpad_rights[1]]
                    else:
                        cvmcep_src = cvmcep_src[:,outpad_lefts[1]:]
                        cvlf0_src = cvlf0_src[:,outpad_lefts[1]:]

                    feat_rec = torch.cat((torch.round(cvlf0_src[:,:,:1]), cvlf0_src[:,:,1:2], \
                                            torch.round(cvlf0_src[:,:,2:3]), cvlf0_src[:,:,3:], cvmcep_src), \
                                                2)[0].cpu().data.numpy()
                    feat_cyc = torch.cat((torch.round(cvlf0_cyc[:,:,:1]), cvlf0_cyc[:,:,1:2], \
                                            torch.round(cvlf0_cyc[:,:,2:3]), cvlf0_cyc[:,:,3:], cvmcep_cyc), \
                                                2)[0].cpu().data.numpy()

                    cvmcep_src = np.array(cvmcep_src[0].cpu().data.numpy(), dtype=np.float64)
                    cvlf0_src = np.array(cvlf0_src[0].cpu().data.numpy(), dtype=np.float64)

                    cvmcep_cyc = np.array(cvmcep_cyc[0].cpu().data.numpy(), dtype=np.float64)
                    cvlf0_cyc = np.array(cvlf0_cyc[0].cpu().data.numpy(), dtype=np.float64)

                logging.info(cvlf0_src.shape)
                logging.info(cvmcep_src.shape)

                logging.info(cvlf0_cyc.shape)
                logging.info(cvmcep_cyc.shape)

                mcep = np.array(feat_org[:,-model_decoder_mcep.out_dim:])
                f0 = np.array(np.rint(feat_org[:,0])*np.exp(feat_org[:,1]))
                codeap = np.array(np.rint(feat_org[:,2:3])*(-np.exp(feat_org[:,3:feat_org.shape[-1]-model_decoder_mcep.out_dim])))

                cvf0_src = np.array(np.rint(cvlf0_src[:,0])*np.exp(cvlf0_src[:,1]))
                cvcodeap_src = np.array(np.rint(cvlf0_src[:,2:3])*(-np.exp(cvlf0_src[:,3:])))
                f0_rmse = np.sqrt(np.mean((cvf0_src-f0)**2))
                logging.info('F0_rmse_rec: %lf Hz' % (f0_rmse))
                cvf0_src_mean = np.mean(cvf0_src)
                f0_mean = np.mean(f0)
                f0_corr = np.sum((cvf0_src-cvf0_src_mean)*(f0-f0_mean))/\
                            (np.sqrt(np.sum((cvf0_src-cvf0_src_mean)**2))*np.sqrt(np.sum((f0-f0_mean)**2)))
                logging.info('F0_corr_rec: %lf' % (f0_corr))

                codeap_rmse = np.sqrt(np.mean((cvcodeap_src-codeap)**2, axis=0))
                for i in range(codeap_rmse.shape[-1]):
                    logging.info('codeap-%d_rmse_rec: %lf dB' % (i+1, codeap_rmse[i]))

                cvf0_cyc = np.array(np.rint(cvlf0_cyc[:,0])*np.exp(cvlf0_cyc[:,1]))
                cvcodeap_cyc = np.array(np.rint(cvlf0_cyc[:,2:3])*(-np.exp(cvlf0_cyc[:,3:])))
                f0_rmse_cyc = np.sqrt(np.mean((cvf0_cyc-f0)**2))
                logging.info('F0_rmse_cyc: %lf Hz' % (f0_rmse_cyc))
                cvf0_cyc_mean = np.mean(cvf0_cyc)
                f0_mean = np.mean(f0)
                f0_corr_cyc = np.sum((cvf0_cyc-cvf0_cyc_mean)*(f0-f0_mean))/\
                            (np.sqrt(np.sum((cvf0_cyc-cvf0_cyc_mean)**2))*np.sqrt(np.sum((f0-f0_mean)**2)))
                logging.info('F0_corr_cyc: %lf' % (f0_corr_cyc))

                codeap_rmse_cyc = np.sqrt(np.mean((cvcodeap_cyc-codeap)**2, axis=0))
                for i in range(codeap_rmse_cyc.shape[-1]):
                    logging.info('codeap-%d_rmse_cyc: %lf dB' % (i+1, codeap_rmse_cyc[i]))

                spcidx = read_hdf5(feat_file, "/spcidx_range")[0]

                _, _, _, mcdpow_arr = dtw.dtw_org_to_trg(np.array(cvmcep_src[np.array(spcidx),:], \
                                            dtype=np.float64), np.array(mcep[np.array(spcidx),:], dtype=np.float64))
                _, _, _, mcd_arr = dtw.dtw_org_to_trg(np.array(cvmcep_src[np.array(spcidx),1:], \
                                            dtype=np.float64), np.array(mcep[np.array(spcidx),1:], dtype=np.float64))
                mcdpow_mean = np.mean(mcdpow_arr)
                mcdpow_std = np.std(mcdpow_arr)
                mcd_mean = np.mean(mcd_arr)
                mcd_std = np.std(mcd_arr)
                logging.info("mcdpow_rec: %.6f dB +- %.6f" % (mcdpow_mean, mcdpow_std))
                logging.info("mcd_rec: %.6f dB +- %.6f" % (mcd_mean, mcd_std))

                _, _, _, mcdpow_arr = dtw.dtw_org_to_trg(np.array(cvmcep_cyc[np.array(spcidx),:], \
                                            dtype=np.float64), np.array(mcep[np.array(spcidx),:], dtype=np.float64))
                _, _, _, mcd_arr = dtw.dtw_org_to_trg(np.array(cvmcep_cyc[np.array(spcidx),1:], \
                                            dtype=np.float64), np.array(mcep[np.array(spcidx),1:], dtype=np.float64))
                mcdpow_mean_cyc = np.mean(mcdpow_arr)
                mcdpow_std_cyc = np.std(mcdpow_arr)
                mcd_mean_cyc = np.mean(mcd_arr)
                mcd_std_cyc = np.std(mcd_arr)
                logging.info("mcdpow_cyc: %.6f dB +- %.6f" % (mcdpow_mean_cyc, mcdpow_std_cyc))
                logging.info("mcd_cyc: %.6f dB +- %.6f" % (mcd_mean_cyc, mcd_std_cyc))

                logging.info('org f0')
                logging.info(f0[10:15])
                logging.info('rec f0')
                logging.info(cvf0_src[10:15])
                logging.info('cyc f0')
                logging.info(cvf0_cyc[10:15])
                logging.info('org cap')
                logging.info(codeap[10:15])
                logging.info('rec cap')
                logging.info(cvcodeap_src[10:15])
                logging.info('cyc cap')
                logging.info(cvcodeap_cyc[10:15])

                dataset = feat_file.split('/')[1].split('_')[0]
                if 'tr' in dataset:
                    logging.info('trn')
                    f0rmse_cvlist.append(f0_rmse)
                    f0corr_cvlist.append(f0_corr)
                    caprmse_cvlist.append(codeap_rmse)
                    mcdpow_cvlist.append(mcdpow_mean)
                    mcdpow_cvlist.append(mcdpow_mean)
                    mcdpowstd_cvlist.append(mcdpow_std)
                    mcd_cvlist.append(mcd_mean)
                    mcdstd_cvlist.append(mcd_std)
                    cvlist.append(np.var(cvmcep_src[:,1:], axis=0))
                    logging.info(len(cvlist))
                    f0rmse_cvlist_cyc.append(f0_rmse_cyc)
                    f0corr_cvlist_cyc.append(f0_corr_cyc)
                    caprmse_cvlist_cyc.append(codeap_rmse_cyc)
                    mcdpow_cvlist_cyc.append(mcdpow_mean_cyc)
                    mcdpow_cvlist_cyc.append(mcdpow_mean_cyc)
                    mcdpowstd_cvlist_cyc.append(mcdpow_std_cyc)
                    mcd_cvlist_cyc.append(mcd_mean_cyc)
                    mcdstd_cvlist_cyc.append(mcd_std_cyc)
                    cvlist_cyc.append(np.var(cvmcep_cyc[:,1:], axis=0))
                elif 'dv' in dataset:
                    logging.info('dev')
                    f0rmse_cvlist_dv.append(f0_rmse)
                    f0corr_cvlist_dv.append(f0_corr)
                    caprmse_cvlist_dv.append(codeap_rmse)
                    mcdpow_cvlist_dv.append(mcdpow_mean)
                    mcdpowstd_cvlist_dv.append(mcdpow_std)
                    mcd_cvlist_dv.append(mcd_mean)
                    mcdstd_cvlist_dv.append(mcd_std)
                    cvlist_dv.append(np.var(cvmcep_src[:,1:], axis=0))
                    logging.info(len(cvlist_dv))
                    f0rmse_cvlist_cyc_dv.append(f0_rmse_cyc)
                    f0corr_cvlist_cyc_dv.append(f0_corr_cyc)
                    caprmse_cvlist_cyc_dv.append(codeap_rmse_cyc)
                    mcdpow_cvlist_cyc_dv.append(mcdpow_mean_cyc)
                    mcdpow_cvlist_cyc_dv.append(mcdpow_mean_cyc)
                    mcdpowstd_cvlist_cyc_dv.append(mcdpow_std_cyc)
                    mcd_cvlist_cyc_dv.append(mcd_mean_cyc)
                    mcdstd_cvlist_cyc_dv.append(mcd_std_cyc)
                    cvlist_cyc_dv.append(np.var(cvmcep_cyc[:,1:], axis=0))

                logging.info('write rec to h5')
                outh5dir = os.path.join(os.path.dirname(os.path.dirname(feat_file)), args.spk+"-"+args.spk)
                if not os.path.exists(outh5dir):
                    os.makedirs(outh5dir)
                feat_file = os.path.join(outh5dir, os.path.basename(feat_file))
                logging.info(feat_file + ' ' + args.string_path)
                logging.info(feat_rec.shape)
                write_hdf5(feat_file, args.string_path, feat_rec)

                logging.info('write cyc to h5')
                outh5dir = os.path.join(os.path.dirname(os.path.dirname(feat_file)), args.spk+"-"+args.spk+"-"+args.spk)
                if not os.path.exists(outh5dir):
                    os.makedirs(outh5dir)
                feat_file = os.path.join(outh5dir, os.path.basename(feat_file))
                logging.info(feat_file + ' ' + args.string_path)
                logging.info(feat_cyc.shape)
                write_hdf5(feat_file, args.string_path, feat_cyc)

                count += 1
                #if count >= 5:
                #    break

    # parallel decode training
    with mp.Manager() as manager:
        gpu = 0
        processes = []
        cvlist = manager.list()
        mcd_cvlist = manager.list()
        mcdstd_cvlist = manager.list()
        mcdpow_cvlist = manager.list()
        mcdpowstd_cvlist = manager.list()
        f0rmse_cvlist = manager.list()
        f0corr_cvlist = manager.list()
        caprmse_cvlist = manager.list()
        cvlist_dv = manager.list()
        mcd_cvlist_dv = manager.list()
        mcdstd_cvlist_dv = manager.list()
        mcdpow_cvlist_dv = manager.list()
        mcdpowstd_cvlist_dv = manager.list()
        f0rmse_cvlist_dv = manager.list()
        f0corr_cvlist_dv = manager.list()
        caprmse_cvlist_dv = manager.list()
        cvlist_cyc = manager.list()
        mcd_cvlist_cyc = manager.list()
        mcdstd_cvlist_cyc = manager.list()
        mcdpow_cvlist_cyc = manager.list()
        mcdpowstd_cvlist_cyc = manager.list()
        f0rmse_cvlist_cyc = manager.list()
        f0corr_cvlist_cyc = manager.list()
        caprmse_cvlist_cyc = manager.list()
        cvlist_cyc_dv = manager.list()
        mcd_cvlist_cyc_dv = manager.list()
        mcdstd_cvlist_cyc_dv = manager.list()
        mcdpow_cvlist_cyc_dv = manager.list()
        mcdpowstd_cvlist_cyc_dv = manager.list()
        f0rmse_cvlist_cyc_dv = manager.list()
        f0corr_cvlist_cyc_dv = manager.list()
        caprmse_cvlist_cyc_dv = manager.list()
        for i, feat_list in enumerate(feat_lists):
            logging.info(i)
            p = mp.Process(target=gpu_decode, args=(feat_list, gpu, cvlist, mcdpow_cvlist, mcdpowstd_cvlist, \
                                                    mcd_cvlist, mcdstd_cvlist, cvlist_dv, mcdpow_cvlist_dv, \
                                                    mcdpowstd_cvlist_dv, mcd_cvlist_dv, mcdstd_cvlist_dv,\
                                                    f0rmse_cvlist, f0corr_cvlist, caprmse_cvlist,\
                                                    f0rmse_cvlist_dv, f0corr_cvlist_dv, caprmse_cvlist_dv,\
                                                    cvlist_cyc, mcdpow_cvlist_cyc, mcdpowstd_cvlist_cyc, \
                                                    mcd_cvlist_cyc, mcdstd_cvlist_cyc, cvlist_cyc_dv, mcdpow_cvlist_cyc_dv, \
                                                    mcdpowstd_cvlist_cyc_dv, mcd_cvlist_cyc_dv, mcdstd_cvlist_cyc_dv,\
                                                    f0rmse_cvlist_cyc, f0corr_cvlist_cyc, caprmse_cvlist_cyc,\
                                                    f0rmse_cvlist_cyc_dv, f0corr_cvlist_cyc_dv, caprmse_cvlist_cyc_dv,))
            p.start()
            processes.append(p)
            gpu += 1
            if (i + 1) % args.n_gpus == 0:
                gpu = 0
        # wait for all process
        for p in processes:
            p.join()

        # calculate cv_gv statistics
        if len(mcdpow_cvlist) > 0:
            logging.info("mcdpow_rec: %.6f dB (+- %.6f) +- %.6f (+- %.6f)" % (np.mean(np.array(mcdpow_cvlist)), \
                        np.std(np.array(mcdpow_cvlist)),np.mean(np.array(mcdpowstd_cvlist)),\
                        np.std(np.array(mcdpowstd_cvlist))))
            logging.info("mcd_rec: %.6f dB (+- %.6f) +- %.6f (+- %.6f)" % (np.mean(np.array(mcd_cvlist)), \
                        np.std(np.array(mcd_cvlist)),np.mean(np.array(mcdstd_cvlist)),\
                        np.std(np.array(mcdstd_cvlist))))
            cvgv_mean = np.mean(np.array(cvlist), axis=0)
            cvgv_var = np.var(np.array(cvlist), axis=0)
            logging.info("%lf +- %lf" % (np.mean(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean)))), \
                                        np.std(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean))))))
            logging.info("f0rmse_rec: %.6f Hz (+- %.6f)" % (np.mean(np.array(f0rmse_cvlist)),np.std(np.array(f0rmse_cvlist))))
            logging.info("f0corr_rec: %.6f (+- %.6f)" % (np.mean(np.array(f0corr_cvlist)),np.std(np.array(f0corr_cvlist))))
            caprmse_cvlist = np.array(caprmse_cvlist)
            for i in range(caprmse_cvlist.shape[-1]):
                logging.info("caprmse-%d_rec: %.6f dB (+- %.6f)" % (i+1, np.mean(caprmse_cvlist[:,i]),np.std(caprmse_cvlist[:,i])))
            logging.info("mcdpow_cyc: %.6f dB (+- %.6f) +- %.6f (+- %.6f)" % (np.mean(np.array(mcdpow_cvlist_cyc)), \
                        np.std(np.array(mcdpow_cvlist_cyc)),np.mean(np.array(mcdpowstd_cvlist_cyc)),\
                        np.std(np.array(mcdpowstd_cvlist_cyc))))
            logging.info("mcd_cyc: %.6f dB (+- %.6f) +- %.6f (+- %.6f)" % (np.mean(np.array(mcd_cvlist_cyc)), \
                        np.std(np.array(mcd_cvlist_cyc)),np.mean(np.array(mcdstd_cvlist_cyc)),\
                        np.std(np.array(mcdstd_cvlist_cyc))))
            cvgv_mean = np.mean(np.array(cvlist_cyc), axis=0)
            cvgv_var = np.var(np.array(cvlist_cyc), axis=0)
            logging.info("%lf +- %lf" % (np.mean(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean)))), \
                                        np.std(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean))))))
            logging.info("f0rmse_cyc: %.6f Hz (+- %.6f)" % (np.mean(np.array(f0rmse_cvlist_cyc)),np.std(np.array(f0rmse_cvlist_cyc))))
            logging.info("f0corr_cyc: %.6f (+- %.6f)" % (np.mean(np.array(f0corr_cvlist_cyc)),np.std(np.array(f0corr_cvlist_cyc))))
            caprmse_cvlist_cyc = np.array(caprmse_cvlist_cyc)
            for i in range(caprmse_cvlist_cyc.shape[-1]):
                logging.info("caprmse-%d_cyc: %.6f dB (+- %.6f)" % (i+1, np.mean(caprmse_cvlist_cyc[:,i]),np.std(caprmse_cvlist_cyc[:,i])))

            cvgv_mean = np.mean(np.array(np.r_[cvlist,cvlist_cyc]), axis=0)
            cvgv_var = np.var(np.array(np.r_[cvlist,cvlist_cyc]), axis=0)
            logging.info("%lf +- %lf" % (np.mean(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean)))), \
                                        np.std(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean))))))

            string_path = model_name+"-"+str(config.detach)+"-"+str(config.n_half_cyc)+"-"+str(config.lat_dim)+"-"+str(config.ctr_size)\
                            +"-"+str(config.spkidtr_dim)+"-"+str(config.ar_enc)+"-"+str(config.ar_dec)+"-"+str(config.ar_f0)+"-"+model_epoch
            logging.info(string_path)

            string_mean = "/recgv_mean_"+string_path
            string_var = "/recgv_var_"+string_path
            write_hdf5(spk_stat, string_mean, cvgv_mean)
            write_hdf5(spk_stat, string_var, cvgv_var)

        if len(mcdpow_cvlist_dv) > 0:
            logging.info("mcdpow_rec_dv: %.6f dB (+- %.6f) +- %.6f (+- %.6f)" % (np.mean(np.array(mcdpow_cvlist_dv)), \
                        np.std(np.array(mcdpow_cvlist_dv)),np.mean(np.array(mcdpowstd_cvlist_dv)),\
                        np.std(np.array(mcdpowstd_cvlist_dv))))
            logging.info("mcd_rec_dv: %.6f dB (+- %.6f) +- %.6f (+- %.6f)" % (np.mean(np.array(mcd_cvlist_dv)), \
                        np.std(np.array(mcd_cvlist_dv)),np.mean(np.array(mcdstd_cvlist_dv)),\
                        np.std(np.array(mcdstd_cvlist_dv))))
            cvgv_mean = np.mean(np.array(cvlist_dv), axis=0)
            cvgv_var = np.var(np.array(cvlist_dv), axis=0)
            logging.info("%lf +- %lf" % (np.mean(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean)))), \
                                        np.std(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean))))))
            logging.info("f0rmse_rec_dv: %.6f Hz (+- %.6f)" % (np.mean(np.array(f0rmse_cvlist_dv)),np.std(np.array(f0rmse_cvlist_dv))))
            logging.info("f0corr_rec_dv: %.6f (+- %.6f)" % (np.mean(np.array(f0corr_cvlist_dv)),np.std(np.array(f0corr_cvlist_dv))))
            caprmse_cvlist_dv = np.array(caprmse_cvlist_dv)
            for i in range(caprmse_cvlist.shape[-1]):
                logging.info("caprmse-%d_rec_dv: %.6f dB (+- %.6f)" % (i+1, np.mean(caprmse_cvlist_dv[:,i]),np.std(caprmse_cvlist_dv[:,i])))
            logging.info("mcdpow_cyc_dv: %.6f dB (+- %.6f) +- %.6f (+- %.6f)" % (np.mean(np.array(mcdpow_cvlist_cyc_dv)), \
                        np.std(np.array(mcdpow_cvlist_cyc_dv)),np.mean(np.array(mcdpowstd_cvlist_cyc_dv)),\
                        np.std(np.array(mcdpowstd_cvlist_cyc_dv))))
            logging.info("mcd_cyc_dv: %.6f dB (+- %.6f) +- %.6f (+- %.6f)" % (np.mean(np.array(mcd_cvlist_cyc_dv)), \
                        np.std(np.array(mcd_cvlist_cyc_dv)),np.mean(np.array(mcdstd_cvlist_cyc_dv)),\
                        np.std(np.array(mcdstd_cvlist_cyc_dv))))
            cvgv_mean = np.mean(np.array(cvlist_cyc_dv), axis=0)
            cvgv_var = np.var(np.array(cvlist_cyc_dv), axis=0)
            logging.info("%lf +- %lf" % (np.mean(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean)))), \
                                        np.std(np.sqrt(np.square(np.log(cvgv_mean)-np.log(gv_mean))))))
            logging.info("f0rmse_cyc_dv: %.6f Hz (+- %.6f)" % (np.mean(np.array(f0rmse_cvlist_cyc_dv)),np.std(np.array(f0rmse_cvlist_cyc_dv))))
            logging.info("f0corr_cyc_dv: %.6f (+- %.6f)" % (np.mean(np.array(f0corr_cvlist_cyc_dv)),np.std(np.array(f0corr_cvlist_cyc_dv))))
            caprmse_cvlist_cyc_dv = np.array(caprmse_cvlist_cyc_dv)
            for i in range(caprmse_cvlist_cyc_dv.shape[-1]):
                logging.info("caprmse-%d_cyc_dv: %.6f dB (+- %.6f)" % (i+1, np.mean(caprmse_cvlist_cyc_dv[:,i]),np.std(caprmse_cvlist_cyc_dv[:,i])))

if __name__ == "__main__":
    main()

max_stars_repo_name: ml-applications/cyclevae-vc-neuralvoco

max_issues_repo_name: ml-applications/cyclevae-vc-neuralvoco

max_forks_repo_name: ml-applications/cyclevae-vc-neuralvoco


text:
import numpy as np
import matplotlib.pyplot as plt
import reconstruct
import pdb

def padArrays(imglist):
    """
    Pad arrays with a border of 100s, which the reconstructor
    function interprets as NaNs
    """
    outimg = []
    for i in imglist:
        sh = np.shape(i)
        temp = np.zeros((sh[0]+2,sh[1]+2),order='F')+100.
        temp[1:-1,1:-1] = i
        outimg.append(temp)

    return outimg

def southwell(gx,gy,criteria,h,maxiter=10000):
    """
    Take 2D x and y gradient arrays and perform Southwell reconstruction.
    If RMS difference in slope from one iteration to the next falls below
    criteria, then algorithm stops
    gx and gy should be in units of slope (rise/run)
    h is in units of step size (e.g. lenslet width)
    """
    #Handle missing data
    ind = np.logical_or(np.isnan(gx),np.isnan(gy))
    gx[ind] = 100.
    gy[ind] = 100.
    #Create phase object
    phase = np.zeros(np.shape(gx),order='F')
    phase[ind] = 100.
    #Pad arrays
    phase,gx,gy = padArrays([phase,gx,gy])
    #Send through reconstructor
    reconstruct.reconstruct(gx,gy,1e-10,1.,phase,maxiter)
    #Strip off border
    phase = phase[1:-1,1:-1]
    #Handle missing data
    phase[ind] = np.nan
    #Return inverse due to reconstructor sign convention
    return -phase

def example():
    """
    Example of how the reconstructor works
    """
    #Create test image
    xg,yg = np.meshgrid(np.linspace(-1,1,100),np.linspace(-1,1,100))
    img = np.polynomial.legendre.legval2d(xg,yg,[[0,1,0],[0,.5,0],[1,0,0]])
    #Take gradients
    gx,gy = np.gradient(img)
    #Create a boundary region
    rad = np.sqrt(xg**2+yg**2)
    img[rad>1] = np.nan
    gx[rad>1] = np.nan
    gy[rad>1] = np.nan
    #Reconstruct wavefront
    recon = southwell(gx,gy,1e-12,1.)

    #Plot results
    fig = plt.figure()
    fig.add_subplot(1,3,1)
    plt.imshow(img)
    plt.title('Original')
    plt.colorbar()
    fig.add_subplot(1,3,2)
    plt.imshow(recon)
    plt.title('Reconstructed')
    plt.colorbar()
    fig.add_subplot(1,3,3)
    plt.imshow(img-recon)
    plt.title('Residual')
    plt.colorbar()
    return recon

max_stars_repo_name: rallured/PyXFocus

max_issues_repo_name: rallured/PyXFocus

max_forks_repo_name: rallured/PyXFocus


text:
"""
This file may not be shared/redistributed freely. Please read copyright notice in the git repo.
"""
import sys
import collections
import inspect
import types
import pandas as pd
import numpy as np
import shutil
import inspect
import os
import compress_pickle
import itertools
from tqdm import tqdm
from gym.envs.toy_text.discrete import DiscreteEnv
import warnings
from collections import OrderedDict
import glob
import csv
import json
import time
from datetime import datetime

color2num = dict(
    gray=30,
    red=31,
    green=32,
    yellow=33,
    blue=34,
    magenta=35,
    cyan=36,
    white=37,
    crimson=38)

def colorize(string, color, bold=False, highlight=False):
    attr = []
    num = color2num[color]
    if highlight: num += 10
    attr.append(str(num))
    if bold: attr.append('1')
    return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), string)

def plot_data(data, y="accumulated_reward", x="Episode", ci=95, estimator='mean', **kwargs):
    import seaborn as sns
    import matplotlib.pyplot as plt
    if isinstance(data, list): # is this correct even?
        data = pd.concat(data, ignore_index=True,axis=0)
    plt.figure(figsize=(12, 6))
    sns.set(style="darkgrid", font_scale=1.5)
    lp = sns.lineplot(data=data, x=x, y=y, hue="Condition", ci=ci, estimator=estimator, **kwargs)
    plt.legend(loc='best') #.set_draggable(True)

def configure_output_dir(G, d=None):
    """
    Set output directory to d, or to /tmp/somerandomnumber if d is None
    """
    # CDIR = os.path.dirname(os.path.realpath(__file__)).replace('\\', '/')
    G.first_row = True
    G.output_dir = d or "/tmp/experiments/%i" % int(time.time())
    assert not os.path.exists(
        G.output_dir), "Log dir %s already exists! Delete it first or use a different dir" % G.output_dir
    os.makedirs(G.output_dir)
    G.output_file = open(os.path.join(G.output_dir, "log.txt"), 'w')
    print(colorize("Logging data to %s" % G.output_file.name, 'green', bold=True))

class LazyLog(object):
    output_dir = None
    output_file = None
    first_row = True
    log_headers = []
    log_current_row = {}

    def __init__(self, experiment_name, run_name=None, data=None):
        if run_name is None:
            experiment_name += "/"+ datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S.%f")[:-3]
        else:
            experiment_name += "/" + run_name
        configure_output_dir(self, experiment_name)
        if data is not None:
            self.save_params(data)

    def __enter__(self):
        return self

    def save_params(self, data):
        save_params(self, data)

    def dump_tabular(self, verbose=False):
        dump_tabular(self, verbose)

    def log_tabular(self, key, value):
        log_tabular(self, key, value)

    def __exit__(self, type, value, traceback):
        self.output_file.close()

def log_tabular(G, key, val):
    """
    Log a value of some diagnostic
    Call this once for each diagnostic quantity, each iteration
    """
    if G.first_row:
        G.log_headers.append(key)
    else:
        assert key in G.log_headers, "Trying to introduce a new key %s that you didn't include in the first iteration" % key
    assert key not in G.log_current_row, "You already set %s this iteration. Maybe you forgot to call dump_tabular()" % key
    G.log_current_row[key] = val

def dump_tabular(G, verbose=True):
    """
    Write all of the diagnostics from the current iteration
    """
    vals = []
    key_lens = [len(key) for key in G.log_headers]
    max_key_len = max(15, max(key_lens))
    keystr = '%' + '%d' % max_key_len
    fmt = "| " + keystr + "s | %15s |"
    n_slashes = 22 + max_key_len
    print("-" * n_slashes) if verbose else None
    for key in G.log_headers:
        val = G.log_current_row.get(key, "")
        if hasattr(val, "__float__"):
            valstr = "%8.3g" % val
        else:
            valstr = val
        print(fmt % (key, valstr)) if verbose else None
        vals.append(val)
    print("-" * n_slashes) if verbose else None
    if G.output_file is not None:
        if G.first_row:
            G.output_file.write("\t".join(G.log_headers))
            G.output_file.write("\n")
        G.output_file.write("\t".join(map(str, vals)))
        G.output_file.write("\n")
        G.output_file.flush()
    G.log_current_row.clear()
    G.first_row = False

class defaultdict2(collections.defaultdict):
    def __missing__(self, key):
        if self.default_factory is None:
            raise KeyError((key,))

        if isinstance(self.default_factory, types.FunctionType):
            nargs = len(inspect.getfullargspec(self.default_factory).args)
            self[key] = value = self.default_factory(key) if nargs == 1 else self.default_factory()
            return value
        else:
            return super().__missing__(key)

def main_plot(experiments, legends=None, smoothing_window=10, resample_ticks=None,
              x_key="Episode",
              y_key='Accumulated Reward', **kwargs
              ):
    """
    Plot an experiment. To plot invidual lines (i.e. no averaging) use
    > units="Unit", estimator=None,

    """
    ensure_list = lambda x: x if isinstance(x, list) else [x]
    experiments = ensure_list(experiments)

    if legends is None:
        legends = experiments
    legends = ensure_list(legends)

    data = []
    for logdir, legend_title in zip(experiments, legends):
        resample_key = x_key if resample_ticks is not None else None
        data += get_datasets(logdir, x=x_key, condition=legend_title, smoothing_window=smoothing_window, resample_key=resample_key, resample_ticks=resample_ticks)

    plot_data(data, y=y_key, x=x_key, **kwargs)

def get_datasets(fpath, x, condition=None, smoothing_window=None, resample_key=None, resample_ticks=None):
    unit = 0
    if condition is None:
        condition = fpath

    datasets = []
    for root, dir, files in os.walk(fpath):
        if 'log.txt' in files:
            json = os.path.join(root, 'params.json')
            if os.path.exists(json):
                with open(json) as f:
                    param_path = open(json)
                    params = json.load(param_path)
                    # exp_name = params['exp_name']

            log_path = os.path.join(root, 'log.txt')
            if os.stat(log_path).st_size == 0:
                print("Bad plot file", log_path, "size is zero. Skipping")
                continue
            experiment_data = pd.read_table(log_path)
            # raise Exception("Group by ehre.0")
            if smoothing_window:
                ed_x = experiment_data[x]
                experiment_data = experiment_data.rolling(smoothing_window,min_periods=1).mean()
                experiment_data[x] = ed_x

            experiment_data.insert(
                len(experiment_data.columns),
                'Unit',
                unit
            )
            experiment_data.insert(
                len(experiment_data.columns),
                'Condition',
                condition)

            datasets.append(experiment_data)
            # print(experiment_data.columns)
            # if len(experiment_data.columns) > 7:
            #     a = 234
            unit += 1

    nc = f"({unit}x)"+condition[condition.rfind("/")+1:]
    for i, d in enumerate(datasets):
        datasets[i] = d.assign(Condition=lambda x: nc)
        # d.rename(columns={'Condition': nc}, inplace=True)
        # gapminder.rename(columns={'pop': 'population',
        #                           'lifeExp': 'life_exp',
        #                           'gdpPercap': 'gdp_per_cap'},
        #                  inplace=True)

    if resample_key is not None:
        nmax = 0
        vmax = -np.inf
        vmin = np.inf
        for d in datasets:
            nmax = max( d.shape[0], nmax)
            vmax = max(d[resample_key].max(), vmax)
            vmin = min(d[resample_key].min(), vmin)
        if resample_ticks is not None:
            nmax = min(resample_ticks, nmax)

        new_datasets = []
        tnew = np.linspace(vmin + 1e-6, vmax - 1e-6, nmax)
        for d in datasets:
            nd = {}
            cols = d.columns.tolist()
            for c in cols:
                if c == resample_key:
                    y = tnew
                elif d[c].dtype == 'O':
                    # it is an object. cannot interpolate
                    y = [ d[c][0] ] * len(tnew)
                else:
                    y = np.interp(tnew, d[resample_key].tolist(), d[c], left=np.nan, right=np.nan)
                    y = y.astype(d[c].dtype)
                nd[c] = y

            ndata = pd.DataFrame(nd)
            ndata = ndata.dropna()
            new_datasets.append(ndata)
        datasets = new_datasets

    return datasets

def savepdf(pdf):
    '''
    Save command for generating figures.
    '''
    import matplotlib.pyplot as plt
    plt.savefig(pdf)
    # pdf = pdf.strip()
    # pdf = pdf+".pdf" if not pdf.endswith(".pdf") else pdf

    # frame = inspect.stack()[-1]
    # module = inspect.getmodule(frame[0])
    # filename = module.__file__
    # wd = os.path.dirname(filename)
    # pdf_dir = wd +"/pdf"
    # # print(inspect.stack())
    # # print("FILENAME: ", filename)
    # if filename.endswith("_RUN_OUTPUT_CAPTURE.py"):
    #     return
    # if not os.path.isdir(pdf_dir):
    #     os.mkdir(pdf_dir)
    # # print("PDF SAVE> ", wd)
    # if os.path.exists(os.getcwd()+ "/../../../Exercises") and os.path.exists(os.getcwd()+ "/../../../pdf_out"):
    #     # figs = [os.path.join(wd, f"../../../Exercises/ExercisesPython/Exercise{i}/latex/output") for i in range(12)]
    #     lecs = [os.path.join(wd, "../../../shared/output")]
    #     od = lecs+[pdf_dir]
    #     for f in od:
    #         if not os.path.isdir(f):
    #             os.makedirs(f)

    #     on = od[0] + "/" + pdf
    #     plt.savefig(fname=on)
    #     from thtools.slider import convert
    #     convert.pdfcrop(on, fout=on)
    #     for f in od[1:]:
    #         shutil.copy(on, f +"/"+pdf)
    # else:
    #     plt.savefig(fname=wd+"/"+pdf)
    # print(">", pdf)

def log_time_series(experiment, list_obs, max_xticks_to_log=None, run_name=None):
    logdir = f"{experiment}/"

    if max_xticks_to_log is not None and len(list_obs) > max_xticks_to_log:
        I = np.round(np.linspace(0, len(list_obs) - 1, max_xticks_to_log))
        list_obs = [o for i, o in enumerate(list_obs) if i in I.astype(np.int).tolist()]

    with LazyLog(logdir) as logz:
        for n,l in enumerate(list_obs):
            for k,v in l.items():
                logz.log_tabular(k,v)
            if "Steps" not in l:
                logz.log_tabular("Steps", n)
            if "Episode" not in l:
                logz.log_tabular("Episode",n)
            logz.dump_tabular(verbose=False)

def existing_runs(experiment):
    nex = 0
    for root, dir, files in os.walk(experiment):
        if 'log.txt' in files:
            nex += 1
    return nex

def train(env, agent, experiment_name=None, num_episodes=None, verbose=True, reset=True, max_steps=1e10,
          max_runs=None, saveload_model=False):

    if max_runs is not None and existing_runs(experiment_name) >= max_runs:
            return experiment_name, None, True
    stats = []
    steps = 0
    ep_start = 0
    if saveload_model:  # Code for loading/saving models
        did_load = agent.load(os.path.join(experiment_name))
        if did_load:
            stats, recent = load_time_series(experiment_name=experiment_name)
            ep_start, steps = stats[-1]['Episode']+1, stats[-1]['Steps']

    done = False
    with tqdm(total=num_episodes, disable=not verbose) as tq:
        for i_episode in range(num_episodes):
            s = env.reset() if reset else (env.s if hasattr(env, "s") else env.env.s)
            reward = []
            for _ in itertools.count():
                a = agent.pi(s)
                sp, r, done, _ = env.step(a)
                agent.train(s, a, r, sp, done)
                reward.append(r)
                steps += 1
                if done or steps > max_steps:
                    break
                s = sp

            stats.append({"Episode": i_episode + ep_start,
                          "Accumulated Reward": sum(reward),
                          "Average Reward": np.mean(reward),
                          "Length": len(reward),
                          "Steps": steps})
            tq.set_postfix(ordered_dict=OrderedDict(stats[-1]))
            tq.update()
    sys.stderr.flush()
    if saveload_model:
        agent.save(experiment_name)
        if did_load:
            os.rename(recent+"/log.txt", recent+"/log2.txt")  # Shuffle old logs

    if experiment_name is not None:
        log_time_series(experiment=experiment_name, list_obs=stats)
        print(f"Training completed. Logging: '{', '.join( stats[0].keys()) }' to {experiment_name}")
    return experiment_name, stats, done

def cache_write(object, file_name, only_on_professors_computer=False):
    if only_on_professors_computer and not is_this_my_computer():
        """ Probably for your own good :-). """
        return
    # file_name = cn_(file_name) if cache_prefix else file_name
    dn = os.path.dirname(file_name)
    if not os.path.exists(dn):
        os.mkdir(dn)
    print("Writing cache...", file_name)
    with open(file_name, 'wb', ) as f:
        compress_pickle.dump(object, f, compression="lzma")
    print("Done!")

def cache_exists(file_name, cache_prefix=True):
    # file_name = cn_(file_name) if cache_prefix else file_name
    return os.path.exists(file_name)

def cache_read(file_name, cache_prefix=True):
    # file_name = cn_(file_name) if cache_prefix else file_name
    if os.path.exists(file_name):
        with open(file_name, 'rb') as f:
            return compress_pickle.load(f, compression="lzma")
            # return pickle.load(f)
    else:
        return None


text:
#!/usr/bin/env python
"""
madspikes : Spike detection for using a moving median absolute difference filter.

This module was original written by Tino Rau and Matthias Cuntz, and
maintained by Arndt Piayda while at Department of Computational
Hydrosystems, Helmholtz Centre for Environmental Research - UFZ,
Leipzig, Germany, and continued by Matthias Cuntz while at Institut
National de Recherche pour l'Agriculture, l'Alimentation et
l'Environnement (INRAE), Nancy, France.

Copyright (c) 2008-2020 Matthias Cuntz - mc (at) macu (dot) de
Released under the MIT License; see LICENSE file for details.

* Written 2008 by Tino Rau and Matthias Cuntz - mc (at) macu (dot) de
* Maintained by Arndt Piayda since Aug 2014.
* Input can be pandas Dataframe or numpy array(s), Apr 2020, Matthias Cuntz
* Removed iteration, Apr 2020, Matthias Cuntz
* Using numpy docstring format, May 2020, Matthias Cuntz

.. moduleauthor:: Matthias Cuntz, Arndt Piayda, Tino Rau

The following functions are provided

.. autosummary::
   madspikes
"""
from __future__ import division, absolute_import, print_function
import numpy as np
import pandas as pd
try:    # import package
    from .mad import mad
except: # python madspikes.py
    from mad import mad

__all__ = ['madspikes']

def madspikes(dfin, flag=None, isday=None,
              colhead=None, undef=-9999,
              nscan=15*48, nfill=1*48,
              z=7, deriv=2, swthr=10.,
              plot=False):
    """
    Spike detection for using a moving median absolute difference filter.
    Used with Eddy vovariance data in Papale et al. (Biogeosciences, 2006).

    Parameters
    ----------
    dfin : pandas.Dataframe or numpy.array
        time series of data where spike detection with MAD should be applied.

        `dfin` can be a pandas.Dataframe.

        `dfin` can also me a numpy array. In this case `colhead` must be given.
        MAD will be applied along axis=0, i.e. on each column of axis=1.
    flag : pandas.Dataframe or numpy.array, optional
        flag Dataframe or array has the same shape as dfin. Non-zero values in
        `flag` will be treated as missing values in `dfin`.

        If `flag` is numpy array, `df.columns.values` will be used as column heads.
    isday : array_like of bool, optional
        True when it is day, False when night. Must have the same length as dfin.shape[0].

        If `isday` is not given, `dfin` must have a column with head 'SW_IN' or
        starting with 'SW_IN'. `isday` will then be `dfin['SW_IN'] > swthr`.
    colhed : array_like of str, optional
        column names if `dfin` is numpy array.
    undef : float, optional
        values having `undef` value are treated as missing values in `dfin` (default: -9999)

        np.nan is not allowed (working).
    nscan : int, optional
        size of moving window to calculate mad in time steps (default: 15*48)
    nfill : int, optional
        step size of moving window to calculate mad in time steps (default: 1*48)

        mad will be calculated in `nscan` time window. Resulting mask will be applied
        only in `nfill` window in the middle of the `nscan` window. Then `nscan` window
        will be moved by `nfill` time steps.
    z : float, optional
        Input is allowed to deviate maximum `z` standard deviations from the median (default: 7)
    deriv : int, optional
        0: Act on raw input.

        1: Use first derivatives.

        2: Use 2nd derivatives (default).
    swthr : float, optional
        Threshold to determine daytime from incoming shortwave radiation if `isday` not given (default: 10).
    plot : bool, optional
        True: data and spikes are plotted into madspikes.pdf (default: False).

    Returns
    -------
    pandas.Dataframe or numpy array
        flags, 0 everywhere except detected spikes set to 2.

    History
    -------
    Written,    Matthias Cuntz & Tino Rau, 2008
    Maintained, Arndt Piayda,   Aug 2014
    Modified,   Matthias Cuntz, Apr 2020 - input can be pandas Dataframe or numpy array(s)
                                         - removed iteration
                Matthias Cuntz, May 2020 - numpy docstring format
    """
    # numpy or panda
    if isinstance(dfin, (np.ndarray, np.ma.MaskedArray)):
        isnumpy = True
        istrans = False
        assert colhead is not None, 'colhead must be given if input is numpy.ndarray.'
        if dfin.shape[0] == len(colhead):
            istrans = True
            df = pd.DataFrame(dfin.T, columns=colhead)
        elif dfin.shape[1] == len(colhead):
            df = pd.DataFrame(dfin, columns=colhead)
        else:
            raise ValueError('Length of colhead must be number of columns in input array. len(colhead)='+str(len(colhead))+' shape(input)=('+str(dfin.shape[0])+','+str(dfin.shape[1])+').')
    else:
        isnumpy = False
        istrans = False
        assert isinstance(dfin, pd.core.frame.DataFrame), 'Input must be either numpy.ndarray or pandas.DataFrame.'
        df = dfin.copy(deep=True)

    # Incoming flags
    if flag is not None:
        if isinstance(flag, (np.ndarray, np.ma.MaskedArray)):
            fisnumpy = True
            fistrans = False
            if flag.shape[0] == len(df):
                ff = pd.DataFrame(flag, columns=df.columns.values)
            elif flag.shape[1] == len(df):
                fistrans = True
                ff = pd.DataFrame(flag.T, columns=df.columns.values)
            else:
                raise ValueError('flag must have same shape as data array. data: ({:d},{:d}); flag: ({:d},{:d})'.format(dfin.shape[0], dfin.shape[1], flag.shape[0], flag.shape[1]))
            ff = ff.set_index(df.index)
        else:
            fisnumpy = False
            fistrans = False
            assert isinstance(flag, pd.core.frame.DataFrame), 'Flag must be either numpy.ndarray or pandas.DataFrame.'
            ff = flag.copy(deep=True)
    else:
        fisnumpy = isnumpy
        fistrans = istrans
        # flags: 0: good; 1: input flagged; 2: output flagged
        ff              = df.copy(deep=True).astype(int)
        ff[:]           = 0
        ff[df == undef] = 1
        ff[df.isna()]   = 1

    # day or night
    if isday is None:
        sw_id = ''
        for cc in df.columns:
            if cc.startswith('SW_IN'):
                sw_id = cc
                break
        assert sw_id, 'Global radiation with name SW or starting with SW_ must be in input if isday not given.'
        isday = df[sw_id] > swthr # Papale et al. (Biogeosciences, 2006): 20; REddyProc: 10
    if isinstance(isday, (pd.core.series.Series,pd.core.frame.DataFrame)):
        isday = isday.to_numpy()
    isday[isday == undef] = np.nan
    ff[np.isnan(isday)]   = 1

    # parameters
    nrow, ncol = df.shape
    half_scan_win = nscan//2
    half_fill_win = nfill//2

    # calculate dusk and dawn times and separate in day and night
    isdawn         = np.zeros(nrow, dtype=np.bool)
    isdusk         = np.zeros(nrow, dtype=np.bool)
    dis            = isday.astype(int) - np.roll(isday,-1).astype(int) # .astype(bool)
    isdawn[:-1]    = np.where(dis[:-1] == -1, True, False)
    isdusk[:-1]    = np.where(dis[:-1] ==  1, True, False)
    isddday        = isdawn
    tmp            = np.roll(isdusk,1)
    isddday[1:]   += tmp[1:] # start and end of day
    isddnight      = isdusk
    tmp            = np.roll(isdawn,1)
    isddnight[1:] += tmp[1:] # start and end of night

    # iterate over each column of data
    if plot:
        import matplotlib.pyplot as plt
        import matplotlib.backends.backend_pdf as pdf
        pd.plotting.register_matplotlib_converters()
        pp  = pdf.PdfPages('madspikes.pdf')

    cols = list(df.columns)
    for hcol in df.columns:

        if hcol.startswith == 'SW_IN': continue

        data  = df[hcol]
        dflag = ff[hcol]

        # get day and night data
        data_day = data.copy(deep=True)
        data_day[~(isday | isddday) | (dflag != 0) | (data == undef)] = np.nan
        data_night = data.copy(deep=True)
        data_night[~(~isday | isddnight) | (dflag != 0) | (data == undef)] = np.nan

        # iterate over fill window
        for j in range(half_fill_win, nrow-1, 2*half_fill_win):
            j1 = max(j - half_scan_win - 1, 0)
            j2 = min(j + half_scan_win + 1, nrow)
            fill_start = max(j - half_fill_win, 1)
            fill_end   = min(j + half_fill_win, nrow-1)

            dd = data_day[j1:j2].to_numpy()
            day_flag = mad(np.ma.masked_array(data=dd, mask=np.isnan(dd)), z=z, deriv=deriv)
            ff.iloc[fill_start:fill_end, cols.index(hcol)] += np.where(day_flag[fill_start-j1-1:fill_end-j1-1], 2, 0)

            nn = data_night[j1:j2]
            night_flag = mad(np.ma.masked_array(data=nn, mask=np.isnan(nn)), z=z, deriv=deriv)
            ff.iloc[fill_start:fill_end, cols.index(hcol)] += np.where(night_flag[fill_start-j1-1:fill_end-j1-1], 2, 0)

        if plot:
            fig = plt.figure(1)
            sub = fig.add_subplot(111)
            valid = ff[hcol]==0
            l1 = sub.plot(data[valid], 'ob')
            l3 = sub.plot(data[ff[hcol]==2], 'or')
            plt.title(hcol)
            pp.savefig(fig)
            plt.close(fig)

    # Finish

    if plot:
        pp.close()

    if fisnumpy:
        if fistrans:
            return ff.to_numpy().T
        else:
            return ff.to_numpy()
    else:
        return ff

if __name__ == '__main__':
    import doctest
    doctest.testmod()

max_stars_repo_name: mcuntz/hesseflux

max_issues_repo_name: mcuntz/hesseflux

max_forks_repo_name: mcuntz/hesseflux


text:
"""
Vector utility.
"""
import numpy as np

__author__ = "Sergey Vartanov"
__email__ = "me@enzet.ru"

from shapely.geometry import LineString

def compute_angle(vector: np.ndarray) -> float:
    """
    For the given vector compute an angle between it and (1, 0) vector.  The
    result is in [0, 2π].
    """
    if vector[0] == 0.0:
        if vector[1] > 0.0:
            return np.pi / 2.0
        return np.pi + np.pi / 2.0
    if vector[0] < 0.0:
        return np.arctan(vector[1] / vector[0]) + np.pi
    if vector[1] < 0.0:
        return np.arctan(vector[1] / vector[0]) + 2.0 * np.pi
    return np.arctan(vector[1] / vector[0])

def turn_by_angle(vector: np.ndarray, angle: float) -> np.ndarray:
    """Turn vector by an angle."""
    return np.array(
        (
            vector[0] * np.cos(angle) - vector[1] * np.sin(angle),
            vector[0] * np.sin(angle) + vector[1] * np.cos(angle),
        )
    )

def norm(vector: np.ndarray) -> np.ndarray:
    """Compute vector with the same direction and length 1."""
    return vector / np.linalg.norm(vector)

class Polyline:
    """List of connected points."""

    def __init__(self, points: list[np.ndarray]) -> None:
        self.points: list[np.ndarray] = points

    def get_path(self, parallel_offset: float = 0.0) -> str:
        """Construct SVG path commands."""
        points: list[np.ndarray]
        if np.allclose(parallel_offset, 0.0):
            points = self.points
        else:
            try:
                points = (
                    LineString(self.points)
                    .parallel_offset(parallel_offset)
                    .coords
                    if parallel_offset
                    else self.points
                )
            except ValueError:
                points = self.points

        return (
            "M "
            + " L ".join(f"{point[0]},{point[1]}" for point in points)
            + (" Z" if np.allclose(points[0], points[-1]) else "")
        )

    def shorten(self, index: int, length: float) -> None:
        """Make shorten part specified with index."""
        index_2: int = 1 if index == 0 else -2
        diff: np.ndarray = self.points[index_2] - self.points[index]
        self.points[index] = (
            self.points[index] + diff / np.linalg.norm(diff) * length
        )

class Line:
    """Infinity line: Ax + By + C = 0."""

    def __init__(self, start: np.ndarray, end: np.ndarray) -> None:
        # if start.near(end):
        #     util.error("cannot create line by one point")
        self.a: float = start[1] - end[1]
        self.b: float = end[0] - start[0]
        self.c: float = start[0] * end[1] - end[0] * start[1]

    def parallel_shift(self, shift: np.ndarray) -> None:
        """
        Shift current vector according with shift.

        :param shift: shift vector
        """
        self.c -= self.a * shift[0] + self.b * shift[1]

    def is_parallel(self, other: "Line") -> bool:
        """If lines are parallel or equal."""
        return np.allclose(other.a * self.b - self.a * other.b, 0.0)

    def get_intersection_point(self, other: "Line") -> np.ndarray:
        """Get point of intersection current line with other."""
        if other.a * self.b - self.a * other.b == 0.0:
            return np.array((0.0, 0.0))

        x: float = -(self.b * other.c - other.b * self.c) / (
            other.a * self.b - self.a * other.b
        )
        y: float = -(self.a * other.c - other.a * self.c) / (
            other.b * self.a - self.b * other.a
        )
        return np.array((x, y))

    def __repr__(self) -> str:
        return f"{self.a} * x + {self.b} * y + {self.c} == 0"

class Segment:
    """Closed line segment."""

    def __init__(self, point_1: np.ndarray, point_2: np.ndarray) -> None:
        self.point_1: np.ndarray = point_1
        self.point_2: np.ndarray = point_2

        difference: np.ndarray = point_2 - point_1
        vector: np.ndarray = difference / np.linalg.norm(difference)
        self.angle: float = (
            np.arccos(np.dot(vector, np.array((0.0, 1.0)))) / np.pi
        )

    def __lt__(self, other: "Segment") -> bool:
        return (
            ((self.point_1 + self.point_2) / 2.0)[1]
            < ((other.point_1 + other.point_2) / 2.0)[1]
        )  # fmt: skip

max_stars_repo_name: enzet/map-machine

max_issues_repo_name: enzet/Roentgen

max_forks_repo_name: enzet/map-machine


text:
import numpy as np
from numpy import sqrt
from scipy.optimize import fsolve

from biro_constants_bad import *

# Rotation matrix from a Scipy mailing list, March 2009
def rotationMatrix(direction, angle):
    """
    Create a rotation matrix corresponding to the rotation around a general
    axis by a specified angle.

    R = dd^T + cos(a) (I - dd^T) + sin(a) skew(d)

    Parameters:

    angle : float a
    direction : array d
    """
    d = np.array(direction, dtype=np.float64)
    d /= np.linalg.norm(d)

    eye = np.eye(3, dtype=np.float64)
    ddt = np.outer(d, d)
    skew = np.matrix([[    0,  d[2],  -d[1]],
                  [-d[2],     0,   d[0]],
                   [d[1], -d[0],     0]], dtype=np.float64)

    mtx = ddt + np.cos(angle) * (eye - ddt) + np.sin(angle) * skew
    return mtx

def quadraticSolve(E, F, G, qCur):
    t1 = (-F + sqrt(E*E + F*F - G*G))/(G-E)
    t2 = (-F - sqrt(E*E + F*F - G*G))/(G-E)
    q1 = 2*np.arctan(t1)*180/np.pi
    q2 = 2*np.arctan(t2)*180/np.pi
    if q1 < -90:
        q1 += 180
    if q2 < -90:
        q2 += 180
    if ((q1-qCur)**2 < (q2-qCur)**2):
        return q1
#    if q1 < 90: return q1
    else: return q2

def xyz2servo(x, y, z, qA, qB, qC):
    EA = 2*L*(y+a)
    EB = -L*((sqrt(3)*(x+b)) + y + c)
    EC = L*((sqrt(3)*(x-b)) - y - c)

    F = 2*L*z

    GA = x*x + y*y + z*z + a*a + L*L + 2*y*a - l*l
    GB = x*x + y*y + z*z + b*b + c*c + L*L + 2*(b*x + c*y) - l*l
    GC = x*x + y*y + z*z + b*b + c*c + L*L + 2*(c*y - b*x) - l*l

    qA = quadraticSolve(EA, F, GA, qA)
    qB = quadraticSolve(EB, F, GB, qB)
    qC = quadraticSolve(EC, F, GC, qC)
    return (qA, qB, qC)

def servo2xyz(qA, qB, qC):
    # Rotation matrix from frame N to frames of upper links:
    R_A_N = rotationMatrix([1,0,0], qA*np.pi/180)
    R_B_N = rotationMatrix([1,0,0], qB*np.pi/180) * rotationMatrix([0,0,1], 2*np.pi/3)
    R_C_N = rotationMatrix([1,0,0], qC*np.pi/180) * rotationMatrix([0,0,1], 4*np.pi/3)

    # Define frame indexes
    n_x = np.matrix([1,0,0]).T
    n_y = np.matrix([0,1,0]).T
    n_z = np.matrix([0,0,1]).T
    a_x = R_A_N.I*n_x
    a_y = R_A_N.I*n_y
    a_z = R_A_N.I*n_z
    b_x = R_B_N.I*n_x
    b_y = R_B_N.I*n_y
    b_z = R_B_N.I*n_z
    c_x = R_C_N.I*n_x
    c_y = R_C_N.I*n_y
    c_z = R_C_N.I*n_z

    A0 = -sB/2*n_y
    B0 = birobot.rotationMatrix([0,0,1], 2*np.pi/3)*A0
    C0 = birobot.rotationMatrix([0,0,1], 2*np.pi/3)*B0
    A1 = A0 - L*a_y
    B1 = B0 - L*b_y
    C1 = C0 - L*c_y

    # Position vectors
    #r_Pa_N0 =

    return 0

def equations(p):
    x, y, z = p
    eq1 = (x - sP/(2*np.sqrt(3)))

max_stars_repo_path: Python Simulator/birobot.py

max_stars_repo_name: settinger/Birobot

max_issues_repo_path: Python Simulator/birobot.py

max_issues_repo_name: settinger/Birobot

max_forks_repo_path: Python Simulator/birobot.py

max_forks_repo_name: settinger/Birobot

reason: import numpy,from numpy,from scipy