text:
from __future__ import print_function

import argparse
import os

import anndata
import numpy as np
import scanpy as sc
from hyperas import optim
from hyperas.distributions import choice
from hyperopt import Trials, STATUS_OK, tpe
from matplotlib import pyplot as plt
from scipy import sparse

import trvae

def data():
    DATASETS = {
        "CelebA": {"name": 'celeba', "gender": "Male", 'attribute': "Smiling", 'source_key': -1, "target_key": 1,
                   "width": 64, 'height': 64, "n_channels": 3},

    }
    data_key = "CelebA"
    data_dict = DATASETS[data_key]
    data_name = data_dict['name']
    source_key = data_dict.get('source_key', None)
    target_key = data_dict.get('target_key', None)
    img_width = data_dict.get("width", None)
    img_height = data_dict.get("height", None)
    n_channels = data_dict.get("n_channels", None)
    attribute = data_dict.get('attribute', None)
    gender = data_dict.get('gender', None)

    data = trvae.prepare_and_load_celeba(file_path="./data/celeba/img_align_celeba.zip",
                                         attr_path="./data/celeba/list_attr_celeba.txt",
                                         landmark_path="./data/celeba/list_landmarks_align_celeba.txt",
                                         gender=gender,
                                         attribute=attribute,
                                         max_n_images=50000,
                                         img_width=img_width,
                                         img_height=img_height,
                                         restore=True,
                                         save=True)

    if sparse.issparse(data.X):
        data.X = data.X.A

    source_images = data.copy()[data.obs['condition'] == source_key].X
    target_images = data.copy()[data.obs['condition'] == target_key].X

    source_images = np.reshape(source_images, (-1, img_width, img_height, n_channels))
    target_images = np.reshape(target_images, (-1, img_width, img_height, n_channels))

    source_images /= 255.0
    target_images /= 255.0

    source_labels = np.zeros(shape=source_images.shape[0])
    target_labels = np.ones(shape=target_images.shape[0])
    train_labels = np.concatenate([source_labels, target_labels], axis=0)

    train_images = np.concatenate([source_images, target_images], axis=0)
    train_images = np.reshape(train_images, (-1, np.prod(source_images.shape[1:])))

    preprocessed_data = anndata.AnnData(X=train_images)
    preprocessed_data.obs['condition'] = train_labels
    if data.obs.columns.__contains__('labels'):
        preprocessed_data.obs['labels'] = data.obs['condition'].values
    data = preprocessed_data.copy()

    train_size = int(data.shape[0] * 0.85)
    indices = np.arange(data.shape[0])
    np.random.shuffle(indices)
    train_idx = indices[:train_size]
    test_idx = indices[train_size:]

    data_train = data[train_idx, :]
    data_valid = data[test_idx, :]
    print(data_train.shape, data_valid.shape)

    train_data = data_train.copy()[
        ~((data_train.obs['labels'] == -1) & (data_train.obs['condition'] == target_key))]
    valid_data = data_valid.copy()[
        ~((data_valid.obs['labels'] == -1) & (data_valid.obs['condition'] == target_key))]

    return train_data, valid_data, data_name

def create_model(train_data, valid_data, data_name):
    z_dim_choices = {{choice([20, 40, 50, 60, 80, 100])}}
    mmd_dim_choices = {{choice([64, 128, 256])}}

    alpha_choices = {{choice([1.0, 0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001])}}
    beta_choices = {{choice([1, 5, 10, 50, 100, 500, 1000])}}
    # gamma_choices = {{choice([0.001, 0.01, 0.1, 1, 10.0])}}
    batch_size_choices = {{choice([256, 512, 1024])}}
    dropout_rate_choices = {{choice([0.1, 0.2, 0.5, 0.75])}}

    network = trvae.DCtrVAE(x_dimension=(64, 64, 3),
                            z_dimension=z_dim_choices,
                            mmd_dimension=mmd_dim_choices,
                            alpha=alpha_choices,
                            beta=beta_choices,
                            gamma=0,
                            kernel='rbf',
                            arch_style=3,
                            train_with_fake_labels=False,
                            learning_rate=0.001,
                            model_path=f"./models/RCCVAE/hyperopt/{data_name}-{64}x{64}-{True}/{3}-{z_dim_choices}/",
                            gpus=4,
                            dropout_rate=dropout_rate_choices)

    history = network.train(train_data,
                            use_validation=True,
                            valid_adata=valid_data,
                            n_epochs=10000,
                            batch_size=batch_size_choices,
                            verbose=2,
                            early_stop_limit=200,
                            shuffle=True,
                            save=True)

    print(f'Best Reconstruction Loss of model: ({history.history["val_kl_reconstruction_loss"][0]})')
    return {'loss': history.history["val_kl_reconstruction_loss"][0], 'status': STATUS_OK}

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Sample a trained autoencoder.')
    arguments_group = parser.add_argument_group("Parameters")
    arguments_group.add_argument('-n', '--max_evals', type=int, required=True,
                                 help='name of dataset you want to train')

    args = vars(parser.parse_args())
    best_run, best_network = optim.minimize(model=create_model,
                                            data=data,
                                            algo=tpe.suggest,
                                            max_evals=args['max_evals'],
                                            trials=Trials())
    print("All Done!")
    print(best_run)

max_stars_repo_name: gokceneraslan/trVAE

max_issues_repo_name: gokceneraslan/trVAE

max_forks_repo_name: gokceneraslan/trVAE


text:
import matplotlib
import matplotlib.pyplot as plt
import numpy
import math

def hermite_interp(dar,var,i):
    n = math.floor(i)
    if n>=len(dar)-1:
        n=len(dar)-2
    t = i-n

    # Hermite basis functions
    h00 = (2.0 * t**3) - (3.0 * t**2) + 1.0
    h10 = t**3.0  - (2.0 * t**2) + t
    h01 = (-2.0* t**3) + (3.0 * t**2)
    h11 = t**3 - t**2
    # Compute the interpolated value of "y"
    return h00*dar[n] + h10*var[n] + h01*dar[n+1] + h11*var[n+1]

# first derivative of hermite spline
def hermite_interp1(dar,var,i):
    n = math.floor(i)
    if n>=len(dar)-1:
        n=len(dar)-2
    t = i-n

    h00 = (6.0 * t**2) - (6.0 * t)
    h10 = 3.0*t**2  - (4.0 * t) + 1
    h01 = (-6.0* t**2) + (6.0 * t)
    h11 = 3.0*t**2 - 2.0*t
    return h00*dar[n] + h10*var[n] + h01*dar[n+1] + h11*var[n+1]

#second derivative of hermite spline
def hermite_interp2(dar,var,i):
    n = math.floor(i)
    if n>=len(dar)-1:
        n=len(dar)-2
    t = i-n

    # Hermite basis functions
    h00 = (12.0 * t) - 6.0
    h10 = 6.0*t  - 4.0
    h01 = (-12.0* t) + 6.0
    h11 = 6.0*t - 2.0
    # Compute the interpolated value of "y"
    return h00*dar[n] + h10*var[n] + h01*dar[n+1] + h11*var[n+1]

ext: py

max_stars_repo_name: mtimmerm/IPythonNotebooks

max_issues_repo_name: mtimmerm/IPythonNotebooks

max_forks_repo_name: mtimmerm/IPythonNotebooks

reason: import numpy


text:
# source: https://github.com/NVIDIA/apex/tree/master/examples/imagenet
# license: BSD 3-Clause
#
# to install apex:
# pip3 install --user --upgrade -e . --global-option="build_ext" --global-option="--cpp_ext" --global-option="--cuda_ext"
#
# ### Multi-process training with FP16_Optimizer, dynamic loss scaling
#     $ python3 -m torch.distributed.launch --nproc_per_node=2 main_fp16_optimizer.py --fp16 --b 256 --save `git rev-parse --short HEAD` --epochs 300 --dynamic-loss-scale --workers 14 --data /home/costar/datasets/imagenet/
#
# # note that --nproc_per_node is NUM_GPUS.
# # Can add --sync_bn to sync bachnorm values if batch size is "very small" but note this also reduces img/s by ~10%.
#
# Example cifar10 command:
#
#    # TODO(ahundt) verify these are the correct parameters
#    export CUDA_VISIBLE_DEVICES="2" && python3 main_fp16_optimizer.py --autoaugment --auxiliary --cutout --batch_size 128 --epochs 2000 --save sharpDARTS_2k_`git rev-parse --short HEAD`_Cmid32 --arch SHARP_DARTS --mid_channels 32 --init_channels 36 --wd 0.0003 --lr_power_annealing_exponent_order 2 --learning_rate_min 0.0005 --learning_rate 0.05 --dataset cifar10

import argparse
import os
import shutil
import time
import glob
import json
import copy

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import torchvision.models as models
import torchvision

import numpy as np
import random

try:
    from apex.parallel import DistributedDataParallel as DDP
    from apex.fp16_utils import *
except ImportError:
    raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.")

from model import NetworkImageNet as NetworkImageNet
from model import NetworkCIFAR as NetworkCIFAR
from tqdm import tqdm
import dataset
import genotypes
import autoaugment
import operations
import utils
import warmup_scheduler
from cosine_power_annealing import cosine_power_annealing
from train import evaluate
import cifar10_1

model_names = sorted(name for name in models.__dict__
                     if name.islower() and not name.startswith("__")
                     and callable(models.__dict__[name]))

parser = argparse.ArgumentParser(description='PyTorch ImageNet Training')
# parser.add_argument('data', metavar='DIR', help='path to dataset')
parser.add_argument('--data', type=str, default='../data', help='path to dataset', metavar='DIR')
parser.add_argument('--arch', '-a', metavar='ARCH', default='SHARP_DARTS',
                    # choices=model_names,
                    help='model architecture: ' +
                    ' | '.join(model_names) +
                    ' (default: SHARP_DARTS)')
parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',
                    help='number of data loading workers (default: 4)')
parser.add_argument('--epochs', default=300, type=int, metavar='N',
                    help='number of total epochs to run (default: 300)')
parser.add_argument('--start_epoch', default=1, type=int, metavar='N',
                    help='manual epoch number (useful for restarts)')
parser.add_argument('-b', '--batch_size', default=256, type=int,
                    metavar='N', help='mini-batch size per process (default: 256)')
parser.add_argument('--lr', '--learning_rate', dest='learning_rate', default=0.1, type=float,
                    metavar='LR', help='Initial learning rate based on autoaugment https://arxiv.org/pdf/1805.09501.pdf.  Will be scaled by <global batch size>/256: args.learning_rate = args.learning_rate*float(args.batch_size*args.world_size)/256.  A warmup schedule will also be applied over the first 5 epochs.')
parser.add_argument('--learning_rate_min', type=float, default=0.00016, help='min learning rate')
parser.add_argument('--warmup_epochs', default=10, type=int, help='number of epochs for warmup (default: 10)')
parser.add_argument('--warmup_lr_divisor', default=10, type=int, help='factor by which to reduce lr at warmup start (default: 10)')
parser.add_argument('--lr_power_annealing_exponent_order', type=float, default=10,
                    help='Cosine Power Annealing Schedule Base, larger numbers make '
                         'the exponential more dominant, smaller make cosine more dominant.')
parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                    help='momentum')
parser.add_argument('--weight_decay', '--wd', dest='weight_decay', default=1e-4, type=float,
                    metavar='W', help='weight decay (default: 1e-4)')
parser.add_argument('--print_freq', '-p', default=10, type=int,
                    metavar='N', help='print frequency (default: 10)')
parser.add_argument('--resume', default='', type=str, metavar='PATH',
                    help='path to latest checkpoint (default: none)')
parser.add_argument('--restart_lr', action='store_true',
                    help='Used in conjunction with --resume, '
                         'this will restart the lr curve as if it was epoch 1, '
                         'but otherwise retain your current epoch count.')
parser.add_argument('--pretrained', dest='pretrained', action='store_true',
                    help='use pre-trained model')

parser.add_argument('--fp16', action='store_true',
                    help='Run model fp16 mode.')
parser.add_argument('--static-loss-scale', type=float, default=1,
                    help='Static loss scale, positive power of 2 values can improve fp16 convergence.')
parser.add_argument('--dynamic-loss-scale', action='store_true',
                    help='Use dynamic loss scaling.  If supplied, this argument supersedes ' +
                    '--static-loss-scale.')
parser.add_argument('--prof', dest='prof', action='store_true',
                    help='Only run 10 iterations for profiling.')
parser.add_argument('--deterministic', action='store_true', default=False)

parser.add_argument("--local_rank", default=0, type=int)
parser.add_argument('--sync_bn', action='store_true',
                    help='enabling apex sync BN.')

parser.add_argument('--drop_path_prob', type=float, default=0, help='drop path probability')
parser.add_argument('--ops', type=str, default='OPS', help='which operations to use, options are OPS and DARTS_OPS')
parser.add_argument('--primitives', type=str, default='PRIMITIVES',
                    help='which primitive layers to use inside a cell search space,'
                         ' options are PRIMITIVES and DARTS_PRIMITIVES')
parser.add_argument('--save', type=str, default='EXP', help='experiment name')
parser.add_argument('--mid_channels', type=int, default=96, help='C_mid channels in choke SharpSepConv')
parser.add_argument('--layers', type=int, default=14, help='total number of layers')
parser.add_argument('--dataset', type=str, default='imagenet', help='which dataset, only option is imagenet')
parser.add_argument('--init_channels', type=int, default=48, help='num of init channels')
parser.add_argument('--auxiliary', action='store_true', default=False, help='use auxiliary tower')
parser.add_argument('--auxiliary_weight', type=float, default=0.4, help='weight for auxiliary loss')
parser.add_argument('--autoaugment', action='store_true', default=False, help='use cifar10 autoaugment https://arxiv.org/abs/1805.09501')
parser.add_argument('--random_eraser', action='store_true', default=False, help='use random eraser')
parser.add_argument('--cutout', action='store_true', default=False, help='use cutout')
parser.add_argument('--cutout_length', type=int, default=112, help='cutout length')
parser.add_argument('--gpu', type=int, default=0, help='gpu device id')
parser.add_argument('-e', '--evaluate', dest='evaluate', type=str, metavar='PATH', default='',
                    help='evaluate model at specified path on training, test, and validation datasets')
parser.add_argument('--flops', action='store_true', default=False, help='count flops and exit, aka floating point operations.')
parser.add_argument('--load', type=str, default='',  metavar='PATH', help='load weights at specified location')
parser.add_argument('--load_args', type=str, default='',  metavar='PATH',
                    help='load command line args from a json file, this will override '
                         'all currently set args except for --evaluate, and arguments '
                         'that did not exist when the json file was originally saved out.')

cudnn.benchmark = True

best_top1 = 0
args = parser.parse_args()
logger = None
DATASET_CHANNELS = dataset.inp_channel_dict[args.dataset]
DATASET_MEAN = dataset.mean_dict[args.dataset]
DATASET_STD = dataset.std_dict[args.dataset]
# print('>>>>>>>DATASET_CHANNELS: ' + str(DATASET_CHANNELS))

def fast_collate(batch):
    imgs = [img[0] for img in batch]
    targets = torch.tensor([target[1] for target in batch], dtype=torch.int64)
    w = imgs[0].size[0]
    h = imgs[0].size[1]
    tensor = torch.zeros((len(imgs), DATASET_CHANNELS, h, w), dtype=torch.uint8)
    for i, img in enumerate(imgs):
        nump_array = np.asarray(img, dtype=np.uint8)
        # tens = torch.from_numpy(nump_array)
        if(nump_array.ndim < 3):
            nump_array = np.expand_dims(nump_array, axis=-1)
        nump_array = np.rollaxis(nump_array, 2)

        tensor[i] += torch.from_numpy(nump_array)
    return tensor, targets

# CLASSES = 1000

if args.deterministic:
    cudnn.benchmark = False
    cudnn.deterministic = True
    torch.manual_seed(args.local_rank)

def main():
    global best_top1, args, logger

    args.distributed = False
    if 'WORLD_SIZE' in os.environ:
        args.distributed = int(os.environ['WORLD_SIZE']) > 1

    # commented because it is now set as an argparse param.
    # args.gpu = 0
    args.world_size = 1

    if args.distributed:
        args.gpu = args.local_rank % torch.cuda.device_count()
        torch.cuda.set_device(args.gpu)
        torch.distributed.init_process_group(backend='nccl',
                                             init_method='env://')
        args.world_size = torch.distributed.get_world_size()

    # note the gpu is used for directory creation and log files
    # which is needed when run as multiple processes
    args = utils.initialize_files_and_args(args)
    logger = utils.logging_setup(args.log_file_path)

    if args.fp16:
        assert torch.backends.cudnn.enabled, "fp16 mode requires cudnn backend to be enabled."

    if args.static_loss_scale != 1.0:
        if not args.fp16:
            logger.info("Warning:  if --fp16 is not used, static_loss_scale will be ignored.")

    # # load the correct ops dictionary
    op_dict_to_load = "operations.%s" % args.ops
    logger.info('loading op dict: ' + str(op_dict_to_load))
    op_dict = eval(op_dict_to_load)

    # load the correct primitives list
    primitives_to_load = "genotypes.%s" % args.primitives
    logger.info('loading primitives:' + primitives_to_load)
    primitives = eval(primitives_to_load)
    logger.info('primitives: ' + str(primitives))
    # create model
    genotype = eval("genotypes.%s" % args.arch)
    # get the number of output channels
    classes = dataset.class_dict[args.dataset]
    # create the neural network
    if args.dataset == 'imagenet':
        model = NetworkImageNet(args.init_channels, classes, args.layers, args.auxiliary, genotype, op_dict=op_dict, C_mid=args.mid_channels)
        flops_shape = [1, 3, 224, 224]
    else:
        model = NetworkCIFAR(args.init_channels, classes, args.layers, args.auxiliary, genotype, op_dict=op_dict, C_mid=args.mid_channels)
        flops_shape = [1, 3, 32, 32]
    model.drop_path_prob = 0.0
    # if args.pretrained:
    #     logger.info("=> using pre-trained model '{}'".format(args.arch))
    #     model = models.__dict__[args.arch](pretrained=True)
    # else:
    #     logger.info("=> creating model '{}'".format(args.arch))
    #     model = models.__dict__[args.arch]()

    if args.flops:
        model = model.cuda()
        logger.info("param size = %fMB", utils.count_parameters_in_MB(model))
        logger.info("flops_shape = " + str(flops_shape))
        logger.info("flops = " + utils.count_model_flops(model, data_shape=flops_shape))
        return

    if args.sync_bn:
        import apex
        logger.info("using apex synced BN")
        model = apex.parallel.convert_syncbn_model(model)

    model = model.cuda()
    if args.fp16:
        model = network_to_half(model)
    if args.distributed:
        # By default, apex.parallel.DistributedDataParallel overlaps communication with
        # computation in the backward pass.
        # model = DDP(model)
        # delay_allreduce delays all communication to the end of the backward pass.
        model = DDP(model, delay_allreduce=True)

    # define loss function (criterion) and optimizer
    criterion = nn.CrossEntropyLoss().cuda()

    # Scale learning rate based on global batch size
    args.learning_rate = args.learning_rate * float(args.batch_size * args.world_size)/256.
    init_lr = args.learning_rate / args.warmup_lr_divisor
    optimizer = torch.optim.SGD(model.parameters(), init_lr,
                                momentum=args.momentum,
                                weight_decay=args.weight_decay)

    # epoch_count = args.epochs - args.start_epoch
    # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(epoch_count))
    # scheduler = warmup_scheduler.GradualWarmupScheduler(
    #     optimizer, args.warmup_lr_divisor, args.warmup_epochs, scheduler)

    if args.fp16:
        optimizer = FP16_Optimizer(optimizer,
                                   static_loss_scale=args.static_loss_scale,
                                   dynamic_loss_scale=args.dynamic_loss_scale)

    # Optionally resume from a checkpoint
    if args.resume or args.evaluate:
        if args.evaluate:
            args.resume = args.evaluate
        # Use a local scope to avoid dangling references
        def resume():
            if os.path.isfile(args.resume):
                logger.info("=> loading checkpoint '{}'".format(args.resume))
                checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage.cuda(args.gpu))
                args.start_epoch = checkpoint['epoch']
                if 'best_top1' in checkpoint:
                    best_top1 = checkpoint['best_top1']
                model.load_state_dict(checkpoint['state_dict'])
                # An FP16_Optimizer instance's state dict internally stashes the master params.
                optimizer.load_state_dict(checkpoint['optimizer'])
                # TODO(ahundt) make sure scheduler loading isn't broken
                if 'lr_scheduler' in checkpoint:
                    scheduler.load_state_dict(checkpoint['lr_scheduler'])
                elif 'lr_schedule' in checkpoint:
                    lr_schedule = checkpoint['lr_schedule']
                logger.info("=> loaded checkpoint '{}' (epoch {})"
                            .format(args.resume, checkpoint['epoch']))
            else:
                logger.info("=> no checkpoint found at '{}'".format(args.resume))
        resume()

    # # Data loading code
    # traindir = os.path.join(args.data, 'train')
    # valdir = os.path.join(args.data, 'val')

    # if(args.arch == "inception_v3"):
    #     crop_size = 299
    #     val_size = 320 # I chose this value arbitrarily, we can adjust.
    # else:
    #     crop_size = 224
    #     val_size = 256

    # train_dataset = datasets.ImageFolder(
    #     traindir,
    #     transforms.Compose([
    #         transforms.RandomResizedCrop(crop_size),
    #         transforms.RandomHorizontalFlip(),
    #         autoaugment.ImageNetPolicy(),
    #         # transforms.ToTensor(),  # Too slow, moved to data_prefetcher()
    #         # normalize,
    #     ]))
    # val_dataset = datasets.ImageFolder(valdir, transforms.Compose([
    #         transforms.Resize(val_size),
    #         transforms.CenterCrop(crop_size)
    #     ]))

    # train_sampler = None
    # val_sampler = None
    # if args.distributed:
    #     train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
    #     val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset)

    # train_loader = torch.utils.data.DataLoader(
    #     train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),
    #     num_workers=args.workers, pin_memory=True, sampler=train_sampler, collate_fn=fast_collate)

    # val_loader = torch.utils.data.DataLoader(
    #     val_dataset,
    #     batch_size=args.batch_size, shuffle=False,
    #     num_workers=args.workers, pin_memory=True,
    #     sampler=val_sampler,
    #     collate_fn=fast_collate)

    # Get preprocessing functions (i.e. transforms) to apply on data
    # normalize_as_tensor = False because we normalize and convert to a
    # tensor in our custom prefetching function, rather than as part of
    # the transform preprocessing list.
    train_transform, valid_transform = utils.get_data_transforms(args, normalize_as_tensor=False)
    # Get the training queue, select training and validation from training set
    train_loader, val_loader = dataset.get_training_queues(
        args.dataset, train_transform, valid_transform, args.data,
        args.batch_size, train_proportion=1.0,
        collate_fn=fast_collate, distributed=args.distributed,
        num_workers=args.workers)

    if args.evaluate:
        if args.dataset == 'cifar10':
            # evaluate best model weights on cifar 10.1
            # https://github.com/modestyachts/CIFAR-10.1
            train_transform, valid_transform = utils.get_data_transforms(args)
            # Get the training queue, select training and validation from training set
            # Get the training queue, use full training and test set
            train_queue, valid_queue = dataset.get_training_queues(
              args.dataset, train_transform, valid_transform, args.data, args.batch_size,
              train_proportion=1.0, search_architecture=False)
            test_data = cifar10_1.CIFAR10_1(root=args.data, download=True, transform=valid_transform)
            test_queue = torch.utils.data.DataLoader(
                test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.workers)
            eval_stats = evaluate(args, model, criterion, train_queue=train_queue,
                                  valid_queue=valid_queue, test_queue=test_queue)
            with open(args.stats_file, 'w') as f:
                # TODO(ahundt) fix "TypeError: 1869 is not JSON serializable" to include arg info, see train.py
                # arg_dict = vars(args)
                # arg_dict.update(eval_stats)
                # json.dump(arg_dict, f)
                json.dump(eval_stats, f)
            logger.info("flops = " + utils.count_model_flops(model))
            logger.info(utils.dict_to_log_string(eval_stats))
            logger.info('\nEvaluation of Loaded Model Complete! Save dir: ' + str(args.save))
        else:
            validate(val_loader, model, criterion, args)
        return

    lr_schedule = cosine_power_annealing(
        epochs=args.epochs, max_lr=args.learning_rate, min_lr=args.learning_rate_min,
        warmup_epochs=args.warmup_epochs, exponent_order=args.lr_power_annealing_exponent_order,
        restart_lr=args.restart_lr)
    epochs = np.arange(args.epochs) + args.start_epoch

    stats_csv = args.epoch_stats_file
    stats_csv = stats_csv.replace('.json', '.csv')
    with tqdm(epochs, dynamic_ncols=True, disable=args.local_rank != 0, leave=False) as prog_epoch:
        best_stats = {}
        stats = {}
        epoch_stats = []
        best_epoch = 0
        for epoch, learning_rate in zip(prog_epoch, lr_schedule):
            if args.distributed and train_loader.sampler is not None:
                train_loader.sampler.set_epoch(int(epoch))
            # if args.distributed:
                # train_sampler.set_epoch(epoch)
            # update the learning rate
            for param_group in optimizer.param_groups:
                param_group['lr'] = learning_rate
            # scheduler.step()
            model.drop_path_prob = args.drop_path_prob * float(epoch) / float(args.epochs)
            # train for one epoch
            train_stats = train(train_loader, model, criterion, optimizer, int(epoch), args)
            if args.prof:
                break
            # evaluate on validation set
            top1, val_stats = validate(val_loader, model, criterion, args)
            stats.update(train_stats)
            stats.update(val_stats)
            # stats['lr'] = '{0:.5f}'.format(scheduler.get_lr()[0])
            stats['lr'] = '{0:.5f}'.format(learning_rate)
            stats['epoch'] = epoch

            # remember best top1 and save checkpoint
            if args.local_rank == 0:
                is_best = top1 > best_top1
                best_top1 = max(top1, best_top1)
                stats['best_top1'] = '{0:.3f}'.format(best_top1)
                if is_best:
                    best_epoch = epoch
                    best_stats = copy.deepcopy(stats)
                stats['best_epoch'] = best_epoch

                stats_str = utils.dict_to_log_string(stats)
                logger.info(stats_str)
                save_checkpoint({
                    'epoch': epoch,
                    'arch': args.arch,
                    'state_dict': model.state_dict(),
                    'best_top1': best_top1,
                    'optimizer': optimizer.state_dict(),
                    # 'lr_scheduler': scheduler.state_dict()
                    'lr_schedule': lr_schedule,
                    'stats': best_stats
                }, is_best, path=args.save)
                prog_epoch.set_description(
                    'Overview ***** best_epoch: {0} best_valid_top1: {1:.2f} ***** Progress'
                    .format(best_epoch, best_top1))
            epoch_stats += [copy.deepcopy(stats)]
            with open(args.epoch_stats_file, 'w') as f:
                json.dump(epoch_stats, f, cls=utils.NumpyEncoder)
            utils.list_of_dicts_to_csv(stats_csv, epoch_stats)
        stats_str = utils.dict_to_log_string(best_stats, key_prepend='best_')
        logger.info(stats_str)
        with open(args.stats_file, 'w') as f:
            arg_dict = vars(args)
            arg_dict.update(best_stats)
            json.dump(arg_dict, f, cls=utils.NumpyEncoder)
        with open(args.epoch_stats_file, 'w') as f:
            json.dump(epoch_stats, f, cls=utils.NumpyEncoder)
        utils.list_of_dicts_to_csv(stats_csv, epoch_stats)
        logger.info('Training of Final Model Complete! Save dir: ' + str(args.save))

class data_prefetcher():
    def __init__(self, loader, mean=None, std=None, cutout=False, cutout_length=112, cutout_cuts=2):
        self.loader = iter(loader)
        self.stream = torch.cuda.Stream()
        if mean is None:
            mean = [0.485, 0.456, 0.406]
        if std is None:
            std = [0.229, 0.224, 0.225]
        mean = np.array(mean) * 255
        std = np.array(std) * 255
        self.mean = torch.tensor(mean).cuda().view(1,3,1,1)
        self.std = torch.tensor(std).cuda().view(1,3,1,1)
        cutout_dtype = np.float32
        if args.fp16:
            self.mean = self.mean.half()
            self.std = self.std.half()
            cutout_dtype = np.float16
        else:
            self.mean = self.mean.float()
            self.std = self.std.float()

        self.cutout = None
        if cutout:
            self.cutout = utils.BatchCutout(cutout_length, cutout_cuts, dtype=cutout_dtype)
        self.preload()

    def preload(self):
        try:
            self.next_input, self.next_target = next(self.loader)
        except StopIteration:
            self.next_input = None
            self.next_target = None
            return
        with torch.cuda.stream(self.stream):
            self.next_input = self.next_input.cuda(non_blocking=True)
            self.next_target = self.next_target.cuda(non_blocking=True)
            if args.fp16:
                self.next_input = self.next_input.half()
            else:
                self.next_input = self.next_input.float()
            self.next_input = self.next_input.sub_(self.mean).div_(self.std)
            if self.cutout is not None:
                # TODO(ahundt) Fix performance of this cutout call, it makes batch loading time go from 0.001 seconds to 0.05 seconds.
                self.next_input = self.cutout(self.next_input)

    def next(self):
        torch.cuda.current_stream().wait_stream(self.stream)
        input = self.next_input
        target = self.next_target
        self.preload()
        return input, target

def train(train_loader, model, criterion, optimizer, epoch, args):
    loader_len = len(train_loader)
    if loader_len < 2:
        raise ValueError('train_loader only supports 2 or more batches and loader_len: ' + str(loader_len))

    batch_time = AverageMeter()
    data_time = AverageMeter()
    speed = AverageMeter()
    losses = AverageMeter()
    top1m = AverageMeter()
    top5m = AverageMeter()

    # switch to train mode
    model.train()
    end = time.time()
    prefetcher = data_prefetcher(train_loader, mean=args.mean, std=args.std, cutout=args.cutout, cutout_length=args.cutout_length)

    input, target = prefetcher.next()
    i = -1
    if args.local_rank == 0:
        progbar = tqdm(total=len(train_loader), leave=False, dynamic_ncols=True)
    else:
        progbar = None
    while input is not None:
        i += 1
        # scheduler in main now adjusts the lr
        # adjust_learning_rate(optimizer, epoch, i, len(train_loader))

        if args.prof:
            if i > 10:
                break
        # measure data loading time
        data_time.update(time.time() - end)

        # compute output
        # output = model(input)
        # loss = criterion(output, target)

        # note here the term output is equivalent to logits
        output, logits_aux = model(input)
        loss = criterion(output, target)
        if logits_aux is not None and args.auxiliary:
            loss_aux = criterion(logits_aux, target)
            loss += args.auxiliary_weight * loss_aux

        # measure accuracy and record loss
        top1f, top5f = accuracy(output.data, target, topk=(1, 5))

        if args.distributed:
            reduced_loss = reduce_tensor(loss.data)
            top1f = reduce_tensor(top1f)
            top5f = reduce_tensor(top5f)
        else:
            reduced_loss = loss.data

        losses.update(to_python_float(reduced_loss), input.size(0))
        top1m.update(to_python_float(top1f), input.size(0))
        top5m.update(to_python_float(top5f), input.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        if args.fp16:
            optimizer.backward(loss)
        else:
            loss.backward()
        optimizer.step()

        torch.cuda.synchronize()
        # measure elapsed time
        batch_time.update(time.time() - end)

        end = time.time()
        input, target = prefetcher.next()

        if args.local_rank == 0:
            progbar.update()
        if args.local_rank == 0 and i % args.print_freq == 0 and i > 1:
            speed.update(args.world_size * args.batch_size / batch_time.val)
            progbar.set_description(
                #   'Epoch: [{0}][{1}/{2}]\t'
                  'Train (cur/avg)  '
                  'batch_t: {batch_time.val:.3f}/{batch_time.avg:.3f}, '
                  'img/s: {0:.1f}/{1:.1f}  '
                  'load_t: {data_time.val:.3f}/{data_time.avg:.3f}, '
                  'loss: {loss.val:.4f}/{loss.avg:.4f}, '
                  'top1: {top1.val:.2f}/{top1.avg:.2f}, '
                  'top5: {top5.val:.2f}/{top5.avg:.2f}, prog'.format(
                #    epoch, i, len(train_loader),
                   speed.val,
                   speed.avg,
                   batch_time=batch_time,
                   data_time=data_time, loss=losses, top1=top1m, top5=top5m))
    stats = {}
    prefix = 'train_'
    stats = get_stats(progbar, prefix, args, batch_time, data_time, top1m, top5m, losses, speed)
    if progbar is not None:
        progbar.close()
        del progbar
    return stats

def get_stats(progbar, prefix, args, batch_time, data_time, top1, top5, losses, speed):
    stats = {}
    if progbar is not None:
        stats = utils.tqdm_stats(progbar, prefix=prefix)
    stats.update({
        prefix + 'time_step_wall': '{0:.3f}'.format(args.world_size * args.batch_size / batch_time.avg),
        prefix + 'batch_time_one_gpu': '{0:.3f}'.format(batch_time.avg),
        prefix + 'data_time': '{0:.3f}'.format(data_time.avg),
        prefix + 'top1': '{0:.3f}'.format(top1.avg),
        prefix + 'top5': '{0:.3f}'.format(top5.avg),
        prefix + 'loss': '{0:.4f}'.format(losses.avg),
        prefix + 'images_per_second': '{0:.4f}'.format(speed.avg),
    })
    return stats

def validate(val_loader, model, criterion, args):
    loader_len = len(val_loader)
    if loader_len < 2:
        raise ValueError('val_loader only supports 2 or more batches and loader_len: ' + str(loader_len))
    batch_time = AverageMeter()
    data_time = AverageMeter()
    speed = AverageMeter()
    losses = AverageMeter()
    top1m = AverageMeter()
    top5m = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()

    prefetcher = data_prefetcher(val_loader, mean=args.mean, std=args.std)
    input, target = prefetcher.next()
    i = -1
    if args.local_rank == 0:
        progbar = tqdm(total=loader_len)
    else:
        progbar = None
    while input is not None:
        i += 1
        # measure data loading time
        data_time.update(time.time() - end)

        # compute output
        with torch.no_grad():
            # output = model(input)
            # loss = criterion(output, target)
            # note here the term output is equivalent to logits
            output, _ = model(input)
            loss = criterion(output, target)

        # measure accuracy and record loss
        top1f, top5f = accuracy(output.data, target, topk=(1, 5))

        if args.distributed:
            reduced_loss = reduce_tensor(loss.data)
            top1f = reduce_tensor(top1f)
            top5f = reduce_tensor(top5f)
        else:
            reduced_loss = loss.data

        losses.update(to_python_float(reduced_loss), input.size(0))
        top1m.update(to_python_float(top1f), input.size(0))
        top5m.update(to_python_float(top5f), input.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if args.local_rank == 0:
            progbar.update()
        if args.local_rank == 0 and i % args.print_freq == 0:
            speed.update(args.world_size * args.batch_size / batch_time.val)
            progbar.set_description(
                # 'Test: [{0}/{1}]\t'
                  'Valid (cur/avg)  '
                  'batch_t: {batch_time.val:.3f}/{batch_time.avg:.3f}, '
                  'img/s: {0:.1f}/{1:.1f}, '
                  'loss: {loss.val:.4f}/{loss.avg:.4f}, '
                  'top1: {top1.val:.2f}/{top1.avg:.2f}, '
                  'top5: {top5.val:.2f}/{top5.avg:.2f}, prog'.format(
                #    i, len(val_loader),
                   speed.val,
                   speed.avg,
                   batch_time=batch_time, loss=losses,
                   top1=top1m, top5=top5m))

        input, target = prefetcher.next()

    # logger.info(' * top1 {top1.avg:.3f} top5 {top5.avg:.3f}'
    #       .format(top1=top1, top5=top5))
    prefix = 'val_'
    stats = get_stats(progbar, prefix, args, batch_time, data_time, top1m, top5m, losses, speed)
    if progbar is not None:
        progbar.close()
        del progbar
    return top1m.avg, stats

def save_checkpoint(state, is_best, path='', filename='checkpoint.pth.tar', best_filename='model_best.pth.tar'):
    new_filename = os.path.join(path, filename)
    torch.save(state, new_filename)
    if is_best:
        shutil.copyfile(new_filename, os.path.join(path, best_filename))

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def adjust_learning_rate(optimizer, epoch, step, len_epoch):
    """LR schedule that should yield 76% converged accuracy with batch size 256"""
    factor = epoch // 30

    if epoch >= 80:
        factor = factor + 1

    lr = args.learning_rate*(0.1**factor)

    """Warmup"""
    if epoch < 5:
        lr = lr*float(1 + step + epoch*len_epoch)/(5.*len_epoch)

    # if(args.local_rank == 0):
    #     logger.info("epoch = {}, step = {}, lr = {}".format(epoch, step, lr))

    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

def reduce_tensor(tensor):
    rt = tensor.clone()
    dist.all_reduce(rt, op=dist.reduce_op.SUM)
    rt /= args.world_size
    return rt

if __name__ == '__main__':
    main()

max_stars_repo_name: ahundt/sharpDARTS

max_issues_repo_name: ahundt/sharpDARTS

max_forks_repo_name: ahundt/sharpDARTS


text:
"""
Created on Thu Sep 17 18:40:47 2020

@author: twsle
"""

# Imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from matplotlib.lines import Line2D
# import matplotlib.pyplot as plt
import matplotlib.colors

def getNonZeroColumns(data):
    headers = list(data.columns)
    dropnondata = []
    for h in headers:
        if data[h].dtype != np.number:
             dropnondata.append(h)
        elif sum(data[h]) == 0:
            dropnondata.append(h)
    headers = list(set(headers) - set(dropnondata))

    return headers

# Load in the data
data = pd.read_excel(r"C:\ResearchWorkingDirectory\Environmental_PAH_Mutagenicity\Final_Data\mutagenicity_data.xlsx", sheet_name = 'Sheet1')
gramData = pd.read_csv(r"C:\ResearchWorkingDirectory\extrapapersData\grammatica\grammaticaDatabig.csv")
pgData = pd.read_csv(r"C:\ResearchWorkingDirectory\extrapapersData\perezgarrido\perezgarridoDatabig.csv")
haoData = pd.read_csv(r"C:\ResearchWorkingDirectory\extrapapersData\hao\haoDatabig2.csv")
reenuData = pd.read_csv(r"C:\ResearchWorkingDirectory\extrapapersData\Reenu\reenuDatabig2.csv")
debnathData =  pd.read_csv(r"C:\ResearchWorkingDirectory\extrapapersData\debnath\debnath_Databig2.csv")
wangData =  pd.read_csv(r"C:\ResearchWorkingDirectory\extrapapersData\wang\wang_Databig2.csv")
kimData =  pd.read_csv(r"C:\ResearchWorkingDirectory\extrapapersData\kim\kim_Databig2.csv")

headers = set(data.columns)
headershao = set(haoData.columns)
headerspg = set(pgData.columns)
headersG = set(gramData.columns)
headersreenu = set(reenuData.columns)
headersdebnath = set(debnathData.columns)
headerswang = set(wangData.columns)
headerskim = set(kimData.columns)

finalheaders = list(headers.intersection(headersG).
                    intersection(headerspg).
                    intersection(headershao).
                    intersection(headersreenu).
                    intersection(headersdebnath).
                    intersection(headerswang).
                    intersection(headerskim))

data0 = data[finalheaders]
data0['source'] = ['#CC3311']* len(data0) #data from this study

#data from other studies
gramColor = '#808080'
gramData = gramData[finalheaders]
gramData['source'] = [gramColor]* len(gramData)

pgColor = '#808080'
pgData = pgData[finalheaders]
pgData['source'] = [pgColor]* len(pgData)

haoColor = '#808080'
haoData = haoData[finalheaders]
haoData['source'] = [haoColor]* len(haoData)

reenuColor = '#808080'
reenuData = reenuData[finalheaders]
reenuData['source'] = [reenuColor]* len(reenuData)

debnathColor = '#808080'
debnathData= debnathData[finalheaders]
debnathData['source'] = [debnathColor]* len(debnathData)

wangColor = '#808080'
wangData= wangData[finalheaders]
wangData['source'] = [wangColor]* len(wangData)

kimColor = '#808080'
kimData= kimData[finalheaders]
kimData['source'] = [kimColor]* len(kimData)

#put the two bigger studies in front, then the smaller studies, my data at the back.
finaldata = pd.concat([pgData, debnathData, wangData, kimData, haoData,reenuData, gramData,data0])
finaldata = finaldata.dropna(axis = 1)

#Get the colors for the clusters laster
finalheaders.append('source')
colorsSource = finaldata['source']

finaldata = finaldata.drop(columns = ['source'])

headers2 = getNonZeroColumns(finaldata)
X = StandardScaler().fit_transform(finaldata[headers2])

# Create a PCA instance: pca\
pca = PCA(n_components=10)
principalComponents = pca.fit_transform(X)
# Plot the explained variances
features = range(pca.n_components_)
plt.bar(features, pca.explained_variance_ratio_, color='black')
plt.xlabel('PCA features')
plt.ylabel('variance %')
plt.xticks(features)

# Save components to a DataFrame, these are the
# principle components of everything
PCA_components = pd.DataFrame(principalComponents)

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

ax.scatter(PCA_components[0], PCA_components[1], PCA_components[2], alpha=.33, c = colorsSource)
plt.xlabel('PCA 1')
plt.ylabel('PCA 2')

allalpha = 0

fig = plt.figure(figsize=(8,8))
ax = fig.add_subplot(111)

#Perez-Garrido
rgbCol = matplotlib.colors.to_rgb(pgColor)
ax.scatter(PCA_components[1][0:219], PCA_components[2][0:219],
           color = (rgbCol[0],rgbCol[1],rgbCol[2],allalpha),
           edgecolor = (0,0,0,1),linewidth = 0.75, marker = 'v', zorder = 0)

#Debnath
rgbCol = matplotlib.colors.to_rgb(debnathColor)
ax.scatter(PCA_components[1][220:380], PCA_components[2][220:380],
           color = (rgbCol[0],rgbCol[1],rgbCol[2],allalpha),
           edgecolor = (0,0,0,1),linewidth = 0.75,marker = '^', zorder = 0)

#Wang
rgbCol = matplotlib.colors.to_rgb(wangColor)
ax.scatter(PCA_components[1][381:407], PCA_components[2][381:407],
          color = (rgbCol[0],rgbCol[1],rgbCol[2],allalpha),
           edgecolor = (0,0,0,1), linewidth = 0.75,marker = '>', zorder = 2)

#Kim
rgbCol = matplotlib.colors.to_rgb(kimColor)
ax.scatter(PCA_components[1][408:438], PCA_components[2][408:438],
          color = (rgbCol[0],rgbCol[1],rgbCol[2],allalpha),
          edgecolor = (0,0,0,1),linewidth = 0.75, marker = '<', zorder = 3)

#Hao
rgbCol = matplotlib.colors.to_rgb(haoColor)
ax.scatter(PCA_components[1][439:487], PCA_components[2][439:487],
            color = (rgbCol[0],rgbCol[1],rgbCol[2],allalpha),
            edgecolor = (0,0,0,1),linewidth = 0.75, marker = 'P', zorder = 3)

#Reenu
rgbCol = matplotlib.colors.to_rgb(reenuColor)
ax.scatter(PCA_components[1][488:539], PCA_components[2][488:539],
            color = (rgbCol[0],rgbCol[1],rgbCol[2],allalpha),
            edgecolor = (0,0,0,1),linewidth = 0.75, marker = 's', zorder =2)

#Grammatica
rgbCol = matplotlib.colors.to_rgb(gramColor)
ax.scatter(PCA_components[1][540:565], PCA_components[2][540:565],
            color = (rgbCol[0],rgbCol[1],rgbCol[2],allalpha),
            edgecolor = (0,0,0,1),linewidth = 0.75, marker = 'D', zorder = 2)

#plot the current study data on top
rgbCol = matplotlib.colors.to_rgb('#CC3311')
ax.scatter(PCA_components[1][565:], PCA_components[2][565:],
            color = (rgbCol[0],rgbCol[1],rgbCol[2],.9),
            edgecolor = (0,0,0,1),linewidth = 0.75, marker = 'o', zorder = 1)

plt.xlabel('Principal Component 1', fontsize = 16)
plt.ylabel('Principal Component 2', fontsize = 16)

custom_lines = [Line2D([0], [0], color='#CC3311',linewidth = 1, markersize = 10, markeredgecolor = 'black', marker = 'o',lw = 0, alpha = 0.9),  #this study
                Line2D([0], [0], color='#FFFFFF',linewidth = 1, markersize = 10, markeredgecolor = 'black', marker = 'P',lw=0, alpha = 1),  #Hao
                Line2D([0], [0], color='#FFFFFF',linewidth = 1, markersize = 10, markeredgecolor = 'black', marker = 's',lw=0, alpha = 1), #Reenu
                Line2D([0], [0], color='#FFFFFF',linewidth = 1,  markersize = 10, markeredgecolor = 'black', marker = 'v',lw=0, alpha = 1), #Perez-Garrido
                Line2D([0], [0], color='#FFFFFF',linewidth = 1, markersize = 10, markeredgecolor = 'black', marker = '>', lw=0, alpha = 1), #Wang
                 Line2D([0], [0], color='#FFFFFF',linewidth = 1, markersize = 10, markeredgecolor = 'black',  marker = 'D',lw=0, alpha = 1), #Grammatica
                 Line2D([0], [0], color='#FFFFFF',linewidth = 1, markersize = 10, markeredgecolor = 'black', marker = '<',lw=0, alpha = 1), #Kim
                Line2D([0], [0], color='#FFFFFF',linewidth = 1, markersize = 10, markeredgecolor = 'black',marker = '^',lw=0, alpha = 1)]  #Debnath

#put these in chronological order
ax.legend(custom_lines, ( 'This Study',
                         'Hao, 2019',
                         'Reenu, 2014',
                         'Perez-Garrido, 2010',
                         'Wang, 2009',
                         'Grammatica, 2007 ',
                         'Kim, 2006',
                         'Debnath, 1992' ))

ax.xaxis.set_ticks([])
ax.yaxis.set_ticks([])
ax.set_xlim([-21,50])
ax.set_ylim([-21,37])
ax.spines['top'].set_linewidth(2)
ax.spines['bottom'].set_linewidth(2)
ax.spines['left'].set_linewidth(2)
ax.spines['right'].set_linewidth(2)
ax = plt.axes()
ax.grid(False)
plt.show()

fig.savefig('otherQSARS_bw.png', format='png', dpi=1200)


text:
#!/usr/bin/env python3.6
# -*- Coding: UTF-8 -*-
"""
Separete connected moving areas of image in layers.

Example:
--------
>>> import cv2
>>> from blockmatching import *
>>> cap = cv2.VideoCapture('./videos/car.mp4')
>>> started = False
>>> old_frame = None
>>> while cap.isOpened():
>>>    ret, frame = cap.read()
>>>    if ret == True:
>>>        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
>>>        if started is False:
>>>            old_frame = frame
>>>            started = True
>>>        else:
>>>            XP, YP, XD, YD = block_matching(old_frame, frame,
>>>                                            width, height)
>>>            U, V, object_tops, meand = clustering(XD, YD, XP, YP)
>>>            lyrs = layers(frame, object_tops, width, height, sigma=sigma)
>>>            old_frame = frame
>>>    else:
>>>         break
>>>
>>> cap.release()

License
-------
Developed by: E. S. Pereira.
e-mail: pereira.somoza@gmail.com

Copyright [2019] [E. S. Pereira]

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""
from numpy import zeros_like, ones
from scipy.ndimage.filters import gaussian_filter

def layers(frame, object_tops, width, height, sigma=7):
    '''
    Parameter:
    :parameter 2d_array frame: 2d array representing the current  image frame.
    :parameter list object_tops: list of lists with x, y for each graph from clustering
                     algorithm.
    :parameter int width: int - width used in block matching algorithm (size of block)
    :parameter int height: int - height used in block matching algorithm (size of block)
    :parameter int sigma: int - default 7. Used to create a smoothed mask to separete
                     moving areas.
    Return:
    :return list layers: list of 2d array like frame.
    '''
    layers = []
    for clt in object_tops:
        tmp = zeros_like(frame)
        for i, j in clt:
            tmp[i: i + height, j: j + width] = frame[i: i + height,
                                                     j: j + width]
        dst = gaussian_filter(tmp, sigma=sigma)
        tmp = zeros_like(frame)
        tmp[dst != 0] = frame[dst!=0]
        layers.append(tmp)
    return layers

max_stars_repo_name: duducosmos/blockmatching

max_issues_repo_name: duducosmos/blockmatching

max_forks_repo_name: duducosmos/blockmatching

reason: from numpy,from scipy


text:
import networkx as nx
from py2neo import Graph, Node, Relationship
import pandas as pd
import random

graph = Graph("bolt://localhost:7687", auth=("neo4j", "Password"))

def importGexf(gexffilepath, depth = 0):
    '''
    Reads gexf network file from hyphe, update or create all nodes and relationships in neo4j database
    Print . for each 100 nodes/links imported, 1000 for each 1000
    "depth" is used to prefix new properties on node and rel. Value can be 0, 1 or 2
    '''

    # imports or update all nodes / relationships in gexf file from hyphe

    G= nx.read_gexf(gexffilepath, node_type=None, relabel=False, version='1.1draft')
    data = nx.json_graph.node_link_data(G)
    totnbnodes=len(data['nodes'])
    print(totnbnodes," nodes found in gexf")
    i=1

    for node in data['nodes']:
        i=i+1
        nodematch = graph.nodes.match(site_name =node['label']).first()
        if nodematch == None:
            try:
                nodematch = Node('Website', site_name = node['label'])
                nodematch.__primarylabel__ = 'Website'
                nodematch.__primarykey__ = 'site_name'
                graph.merge(nodematch)
            except:
                print("could not import ", node['label'])

        for key in node.keys():
            nodematch["D" + str(depth) + "_" + key] = node[key]
            graph.push(nodematch)
        if i%100 == 0:
            print(".", end=" ")
        if i%1000 ==0:
            print(i,"/",totnbnodes)

    print(i," nodes imported")
    print(len(graph.nodes.match("Website")), "nodes in db after import")

    totnblinks=len(data['links'])
    print(totnblinks," links found in gexf")

    j=0
    for link in data['links']:

        if depth ==0:
            source_n = graph.nodes.match("Website", D0_id = link['source']).first()
            target_n = graph.nodes.match("Website", D0_id = link['target']).first()
        if depth == 1:
            source_n = graph.nodes.match("Website", D1_id = link['source']).first()
            target_n = graph.nodes.match("Website", D1_id = link['target']).first()
        if depth == 2:
            source_n = graph.nodes.match("Website", D2_id = link['source']).first()
            target_n = graph.nodes.match("Website", D2_id = link['target']).first()
        if depth == 3:
            source_n = graph.nodes.match("Website", D3_id = link['source']).first()
            target_n = graph.nodes.match("Website", D3_id = link['target']).first()
        relmatch = graph.relationships.match((source_n,target_n),r_type="LINKS_TO").first()

        try:
            if relmatch == None:
                rel = Relationship(source_n, "LINKS_TO", target_n)
                rel["count_D" + str(depth)]=link['count']
                graph.merge(rel)
            else:
                relmatch["count_D" + str(depth)]=link['count']
                graph.push(relmatch)
            if j%100 == 0:
                print(".", end=" ")
            if j%1000 ==0:
                print(j, "/", totnblinks)
            j=j+1
        except:
            pass
    print(j," links imported")
    print(len(graph.relationships.match()), "links in db after import")

def importGexfLinks(gexffilepath, depth = 0):
    '''
    Reads gexf network file from hyphe, update or create relationships in neo4j database
    Print . for each 100 links imported, 1000 for each 1000
    "depth" is used to prefix new properties on rel. Value can be 0, 1 or 2
    '''

    G= nx.read_gexf(gexffilepath, node_type=None, relabel=False, version='1.1draft')
    data = nx.json_graph.node_link_data(G)
    totnblinks=len(data['links'])
    print(totnblinks," links found in gexf")

    j=1
    for link in data['links']:

        if depth ==0:
            source_n = graph.nodes.match("Website", D0_id = link['source']).first()
            target_n = graph.nodes.match("Website", D0_id = link['target']).first()
        if depth == 1:
            source_n = graph.nodes.match("Website", D1_id = link['source']).first()
            target_n = graph.nodes.match("Website", D1_id = link['target']).first()
        if depth == 2:
            source_n = graph.nodes.match("Website", D2_id = link['source']).first()
            target_n = graph.nodes.match("Website", D2_id = link['target']).first()
        if depth == 3:
            source_n = graph.nodes.match("Website", D3_id = link['source']).first()
            target_n = graph.nodes.match("Website", D3_id = link['target']).first()
        relmatch = graph.relationships.match((source_n,target_n),r_type="LINKS_TO").first()

        try:
            if relmatch == None:
                rel = Relationship(source_n, "LINKS_TO", target_n)
                rel["count_D" + str(depth)]=link['count']
                graph.merge(rel)
            else:
                relmatch["count_D" + str(depth)]=link['count']
                graph.push(relmatch)
            if j%100 == 0:
                print(".", end=" ")
            if j%1000 ==0:
                print(j ,"/",totnblinks)
            j=j+1
        except:
            pass
    print(j," links imported")
    print(len(graph.relationships.match()), "links in db after import")

# This imports all gexf files (takes time)

pathD0IN = "C:\\Users\\Jo\\Documents\\Tech\\Atom_prj\\MyMedia-FillDB\\data\\HypheExport20200520\\202005Websites01_D0_IN.gexf"
importGexf(pathD0IN, 0)

pathD1IN = "C:\\Users\\Jo\\Documents\\Tech\\Atom_prj\\MyMedia-FillDB\\data\\HypheExport20200520\\202005Websites01_D1_IN.gexf"
importGexf(pathD1IN, 1)

# This has not been done entirely
pathD2IN = "C:\\Users\\Jo\\Documents\\Tech\\Atom_prj\\MyMedia-FillDB\\data\\HypheExport20200520\\202005_Websites01_D2_IN.gexf"
importGexf(pathD2IN, 2)

reason: import networkx


text:
"""

Fit the scaling relation in the sims

"""

import os
import sys
import numpy as np
import astropy.table as atpy
from nemo import catalogs, signals, plotSettings, MockSurvey
from astropy.cosmology import FlatLambdaCDM
from scipy import stats
import pyccl as ccl
import pylab as plt
import IPython

#------------------------------------------------------------------------------------------------------------
def calcMass(tab, massOptions, tckQFitDict, fRelWeightsDict, mockSurvey):
    """Calculates masses for cluster data in table. Because why not re-do on the fly when hippo busy?

    """

    count=0
    for row in tab:
        count=count+1
        #print("... %d/%d; %s (%.3f +/- %.3f) ..." % (count, len(tab), row['name'],
                                                              #row['redshift'], row['redshiftErr']))

        tileName=row['tileName']

        # Cuts on z, fixed_y_c for forced photometry mode (invalid objects will be listed but without a mass)
        if row['fixed_y_c'] > 0 and np.isnan(row['redshift']) == False:
            # Corrected for mass function steepness
            massDict=signals.calcMass(row['fixed_y_c']*1e-4, row['fixed_err_y_c']*1e-4,
                                      row['redshift'], row['redshiftErr'],
                                      tenToA0 = massOptions['tenToA0'],
                                      B0 = massOptions['B0'],
                                      Mpivot = massOptions['Mpivot'],
                                      sigma_int = massOptions['sigma_int'],
                                      tckQFit = tckQFitDict[tileName], mockSurvey = mockSurvey,
                                      applyMFDebiasCorrection = True,
                                      applyRelativisticCorrection = True,
                                      fRelWeightsDict = fRelWeightsDict[tileName])
            row['M500c']=massDict['M500c']
            try:
                row['M500c_errPlus']=massDict['M500c_errPlus']
            except:
                IPython.embed()
                sys.exit()
            row['M500c_errMinus']=massDict['M500c_errMinus']

    return tab

#------------------------------------------------------------------------------------------------------------
# Main

# Options - masses are as output by Nemo routines; we compare these to halo catalog (converting as necessary)
massCol="M500c"
#massCol="M200m"

# Websky cosmo - for on-the-fly redone masses
minMass=1e13
areaDeg2=700.   # doesn't matter
zMin=0.0
zMax=2.0
H0, Om0, Ob0, sigma8, ns = 68.0, 0.049+0.261, 0.049, 0.81, 0.965
TCMB=2.72548
cosmoModel=FlatLambdaCDM(H0 = H0, Om0 = Om0, Ob0 = Ob0, Tcmb0 = TCMB)
mockSurvey=MockSurvey.MockSurvey(minMass, areaDeg2, zMin, zMax, H0, Om0, Ob0, sigma8, ns)
massOptions={'tenToA0': 2.65e-05,
             'B0': 0.0,
             'Mpivot': 3.0e+14,
             'sigma_int': 0.2}
tckQFitDict=signals.loadQ("../MFMF_SOSim_3freq_tiles/selFn/QFit.fits")
fRelWeightsDict=signals.loadFRelWeights("../MFMF_SOSim_3freq_tiles/selFn/fRelWeights.fits")

# Make combined table
mergedTabFileName="trueMasses_MFMF_SOSim_3freq_tiles_M500c.fits"
if os.path.exists(mergedTabFileName) == False:
    halos=atpy.Table().read("../halos.fits")
    tab=atpy.Table().read("../MFMF_SOSim_3freq_tiles/MFMF_SOSim_3freq_tiles_mass.fits")
    tab=tab[tab['fixed_SNR'] > 6]

    tab, halos, rDeg=catalogs.crossMatch(tab, halos, radiusArcmin = 1.0)

    zs=halos['z']
    yc=tab['fixed_y_c']
    M200m=halos['M200m']

    M200mDef=ccl.halos.MassDef200m(c_m='Bhattacharya13')
    M500cDef=ccl.halos.MassDef(500, "critical")
    M500c=[]
    count=0
    for m, z in zip(M200m, zs):
        M500c.append(M200mDef.translate_mass(mockSurvey.cosmoModel, m, 1/(1+z), M500cDef))
    M500c=np.array(M500c)

    tab['true_M500c']=M500c/1e14
    tab['true_M200m']=M200m/1e14
    tab['redshift']=zs
    tab.write(mergedTabFileName, overwrite = True)

# Re-do masses on the fly
tab=atpy.Table().read(mergedTabFileName)

# Cut on mass and z to do the fit
MMin=3.0
zBinEdges=[0.2, 0.4, 0.6, 0.8, 1.0]
for i in range(len(zBinEdges)-1):
    zMin=zBinEdges[i]
    zMax=zBinEdges[i+1]

    fitTab=tab[tab['M500c'] > MMin]
    fitTab=fitTab[fitTab['redshift'] > zMin]
    fitTab=fitTab[fitTab['redshift'] < zMax]

    # NOTE: This is done in place anyway
    fitTab=calcMass(fitTab, massOptions, tckQFitDict, fRelWeightsDict, mockSurvey)

    y=fitTab['M500c']
    x=fitTab['true_M500c']
    result=stats.linregress(x, y)
    sumSqRes=np.sum((x-y)**2)
    calibFactor=np.mean(fitTab['true_M500c'])/np.mean(fitTab['M500c'])

    # Scaling relation plot
    plotSettings.update_rcParams()
    plt.figure(figsize=(9.5,9))
    ax=plt.axes([0.1, 0.1, 0.89, 0.89])
    ax.set_aspect('equal')
    plotRange=np.linspace(1.0, 50.0, 100)
    plt.plot(x, y, '.')
    plt.plot(plotRange, plotRange, 'k-')
    plt.xlabel("$M^{\\rm true}_{\\rm 500c}$ (10$^{14}$ $M_{\odot}$)")
    plt.ylabel("$M_{\\rm 500c}$ (10$^{14}$ $M_{\odot}$)")
    plt.xlim(2, 50)
    plt.ylim(2, 50)
    plt.loglog()
    plt.title("%.1f < z < %.1f" % (zMin, zMax))
    plt.savefig("massRecovery_%.1f_%.1f.png" % (zMin, zMax))
    plt.close()

    print("%.1f < z < %.1f:" % (zMin, zMax))
    print("    calibFactor = ", calibFactor)
    print("    sumSqRes = ", sumSqRes)

    #IPython.embed()
    #sys.exit()

max_stars_repo_name: simonsobs/nemo

max_issues_repo_name: simonsobs/nemo

max_forks_repo_name: simonsobs/nemo

reason: import numpy,from scipy,import astropy,from astropy


text:
"""
Trains a SNAIL generative model on CIFAR-10 or Tiny ImageNet data.
Supports using multiple GPUs and machines (the latter using MPI).
"""

def main(args):
  import os
  import sys
  import time
  import json
  from mpi4py import MPI
  import numpy as np
  import tensorflow as tf
  from tqdm import trange

  import pixel_cnn_pp.nn as nn
  import pixel_cnn_pp.plotting as plotting
  from pixel_cnn_pp import model as pxpp_models
  import data.cifar10_data as cifar10_data
  import data.imagenet_data as imagenet_data

  import tf_utils as tfu

  comm = MPI.COMM_WORLD
  num_tasks, task_id = comm.Get_size(), comm.Get_rank()
  save_dir = args.save_dir

  if task_id == 0:
    os.makedirs(save_dir)
    f_log = open(os.path.join(save_dir, 'print.log'), 'w')

  def lprint(*a, **kw):
    if task_id == 0:
      print(*a, **kw)
      print(*a, **kw, file=f_log)

  lprint('input args:\n', json.dumps(vars(args), indent=4,
                                     separators=(',', ':')))  # pretty print args
  # -----------------------------------------------------------------------------
  # fix random seed for reproducibility
  rng = np.random.RandomState(args.seed + task_id)
  tf.set_random_seed(args.seed + task_id)

  # initialize data loaders for train/test splits
  if args.data_set == 'imagenet' and args.class_conditional:
    raise("We currently don't have labels for the small imagenet data set")
  DataLoader = {'cifar': cifar10_data.DataLoader,
                'imagenet': imagenet_data.DataLoader}[args.data_set]
  train_data = DataLoader(args.data_dir, 'train', args.batch_size,
                          rng=rng, shuffle=True, return_labels=args.class_conditional)
  test_data = DataLoader(args.data_dir, 'test', args.batch_size,
                         shuffle=False, return_labels=args.class_conditional)
  obs_shape = train_data.get_observation_size()  # e.g. a tuple (32,32,3)
  assert len(obs_shape) == 3, 'assumed right now'

  if args.nr_gpu is None:
    from tensorflow.python.client import device_lib
    args.nr_gpu = len([d for d in device_lib.list_local_devices()
                       if d.device_type == 'GPU'])

  # data place holders
  x_init = tf.placeholder(tf.float32,
                          shape=(args.init_batch_size,) + obs_shape)
  xs = [tf.placeholder(tf.float32, shape=(args.batch_size, ) + obs_shape)
        for _ in range(args.nr_gpu)]

  def _get_batch(is_training):
    if is_training:
      x = train_data.__next__(args.batch_size)
    else:
      x = test_data.__next__(args.batch_size)
    x = np.cast[np.float32]((x - 127.5) / 127.5)
    return dict(x=x)

  batch_def = dict(x=tfu.vdef(args.batch_size, obs_shape))
  qr = tfu.Struct(
      train=tfu.PyfuncRunner(batch_def, 64, 8, True,
                             _get_batch, is_training=True),
      test=tfu.PyfuncRunner(batch_def, 64, 8, True,
                            _get_batch, is_training=False),
  )
  tf.add_to_collection(tf.GraphKeys.QUEUE_RUNNERS, qr.train)
  tf.add_to_collection(tf.GraphKeys.QUEUE_RUNNERS, qr.test)

  if args.nr_gpu is None:
    from tensorflow.python.client import device_lib
    args.nr_gpu = len([d for d in device_lib.list_local_devices()
                       if d.device_type == 'GPU'])

  sess = tfu.Session(allow_soft_placement=True).__enter__()

  # if the model is class-conditional we'll set up label placeholders +
  # one-hot encodings 'h' to condition on
  if args.class_conditional:
    raise NotImplementedError
    num_labels = train_data.get_num_labels()
    y_init = tf.placeholder(tf.int32, shape=(args.init_batch_size,))
    h_init = tf.one_hot(y_init, num_labels)
    y_sample = np.split(
        np.mod(np.arange(args.batch_size), num_labels), args.nr_gpu)
    h_sample = [tf.one_hot(tf.Variable(y_sample[i], trainable=False), num_labels)
                for i in range(args.nr_gpu)]
    ys = [tf.placeholder(tf.int32, shape=(args.batch_size,))
          for i in range(args.nr_gpu)]
    hs = [tf.one_hot(ys[i], num_labels) for i in range(args.nr_gpu)]
  else:
    h_init = None
    h_sample = [None] * args.nr_gpu
    hs = h_sample

  # create the model
  model_opt = {'nr_resnet': args.nr_resnet, 'nr_filters': args.nr_filters,
               'nr_logistic_mix': args.nr_logistic_mix, 'resnet_nonlinearity': args.resnet_nonlinearity}
  model = tf.make_template('model', getattr(pxpp_models, args.model + "_spec"))

  # run once for data dependent initialization of parameters
  with tf.device('/gpu:0'):
    gen_par = model(x_init, h_init, init=True,
                    dropout_p=args.dropout_p, **model_opt)

  # keep track of moving average
  all_params = tf.trainable_variables()
  lprint('# of Parameters', sum(np.prod(p.get_shape().as_list())
                                for p in all_params))
  ema = tf.train.ExponentialMovingAverage(decay=args.polyak_decay)
  maintain_averages_op = tf.group(ema.apply(all_params))

  loss_gen, loss_gen_test, grads = [], [], []
  for i in range(args.nr_gpu):
    with tf.device('/gpu:%d' % i):
      x = qr.train.batch().x
      gen_par = model(x, hs[i], ema=None,
                      dropout_p=args.dropout_p, **model_opt)
      if isinstance(gen_par, tuple) and len(gen_par) == 3:
        loss_gen.append(nn.discretized_mix_logistic_loss_per_chn(x, *gen_par))
      else:
        loss_gen.append(nn.discretized_mix_logistic_loss(x, gen_par))
      grads.append(tf.gradients(loss_gen[i], all_params))

      x = qr.test.batch().x
      gen_par = model(x, hs[i], ema=ema, dropout_p=0., **model_opt)
      if isinstance(gen_par, tuple) and len(gen_par) == 3:
        loss_gen_test.append(
            nn.discretized_mix_logistic_loss_per_chn(x, *gen_par))
      else:
        loss_gen_test.append(nn.discretized_mix_logistic_loss(x, gen_par))

  # add losses and gradients together and get training updates
  tf_lr = tf.placeholder(tf.float32, shape=[])
  with tf.device('/gpu:0'):
    for i in range(1, args.nr_gpu):
      loss_gen[0] += loss_gen[i]
      loss_gen_test[0] += loss_gen_test[i]
      for j in range(len(grads[0])):
        grads[0][j] += grads[i][j]

  if num_tasks > 1:
    lprint('creating mpi optimizer')
    # If we have multiple mpi processes, average across them.
    flat_grad = tf.concat([tf.reshape(g, (-1,)) for g in grads[0]], axis=0)
    shapes = [g.shape.as_list() for g in grads[0]]
    sizes = [int(np.prod(s)) for s in shapes]
    buf = np.zeros(sum(sizes), np.float32)

    def _gather_grads(my_flat_grad):
      comm.Allreduce(my_flat_grad, buf, op=MPI.SUM)
      np.divide(buf, float(num_tasks), out=buf)
      return buf

    avg_flat_grad = tf.py_func(_gather_grads, [flat_grad], tf.float32)
    avg_flat_grad.set_shape(flat_grad.shape)
    avg_grads = tf.split(avg_flat_grad, sizes, axis=0)
    grads[0] = [tf.reshape(g, v.shape) for g, v in zip(avg_grads, grads[0])]

  # training op
  optimizer = tf.group(nn.adam_updates(
      all_params, grads[0], lr=tf_lr, mom1=0.95, mom2=0.9995, eps=1e-6), maintain_averages_op)

  # convert loss to bits/dim
  total_gpus = sum(comm.allgather(args.nr_gpu))
  lprint('using %d gpus across %d machines' % (total_gpus, num_tasks))
  norm_const = np.log(2.) * np.prod(obs_shape) * args.batch_size
  norm_const *= total_gpus / num_tasks
  bits_per_dim = loss_gen[0] / norm_const
  bits_per_dim_test = loss_gen_test[0] / norm_const

  bits_per_dim = tf.check_numerics(bits_per_dim, 'train loss is nan')
  bits_per_dim_test = tf.check_numerics(bits_per_dim_test, 'test loss is nan')

  new_x_gen = []
  for i in range(args.nr_gpu):
    with tf.device('/gpu:%d' % i):
      gen_par = model(xs[i], hs[i], ema=ema, dropout_p=0, **model_opt)
      new_x_gen.append(
          nn.sample_from_discretized_mix_logistic(gen_par, args.nr_logistic_mix))

  def sample_from_model(sess, n_samples=args.nr_gpu * args.batch_size):
    sample_x = np.zeros((0,) + obs_shape, dtype=np.float32)
    while len(sample_x) < n_samples:
      x_gen = [np.zeros((args.batch_size,) + obs_shape, dtype=np.float32)
               for i in range(args.nr_gpu)]
      for yi in range(obs_shape[0]):
        for xi in range(obs_shape[1]):
          new_x_gen_np = sess.run(new_x_gen,
                                  {xs[i]: x_gen[i] for i in range(args.nr_gpu)})
          for i in range(args.nr_gpu):
            x_gen[i][:, yi, xi, :] = new_x_gen_np[i][:, yi, xi, :]

      sample_x = np.concatenate([sample_x] + x_gen, axis=0)

    img_tile = plotting.img_tile(
        sample_x[:int(np.floor(np.sqrt(n_samples))**2)],
        aspect_ratio=1.0, border_color=1.0, stretch=True)
    img = plotting.plot_img(img_tile, title=args.data_set + ' samples')
    plotting.plt.savefig(
        os.path.join(save_dir, '%s_samples.png' % args.data_set))
    np.save(os.path.join(save_dir, '%s_samples.npy' % args.data_set), sample_x)
    plotting.plt.close('all')

  # init & save
  initializer = tf.global_variables_initializer()
  saver = tf.train.Saver()

  # turn numpy inputs into feed_dict for use with tensorflow
  def make_feed_dict(data, init=False):
    if type(data) is tuple:
      x, y = data
    else:
      x = data
      y = None
    # input to pixelCNN is scaled from uint8 [0,255] to float in range [-1,1]
    x = np.cast[np.float32]((x - 127.5) / 127.5)
    if init:
      feed_dict = {x_init: x}
      if y is not None:
        feed_dict.update({y_init: y})
    else:
      x = np.split(x, args.nr_gpu)
      feed_dict = {xs[i]: x[i] for i in range(args.nr_gpu)}
      if y is not None:
        y = np.split(y, args.nr_gpu)
        feed_dict.update({ys[i]: y[i] for i in range(args.nr_gpu)})
    return feed_dict

  # //////////// perform training //////////////
  lprint('dataset size: %d' % len(train_data.data))
  test_bpd = []
  lr = args.learning_rate

  # manually retrieve exactly init_batch_size examples
  feed_dict = make_feed_dict(train_data.next(args.init_batch_size), init=True)
  train_data.reset()  # rewind the iterator back to 0 to do one full epoch
  lprint('initializing the model...')

  sess.run(initializer, feed_dict)
  if args.load_params:
    # ckpt_file = save_dir + '/params_' + args.data_set + '.ckpt'
    ckpt_file = args.load_params
    lprint('restoring parameters from', ckpt_file)
    saver.restore(sess, ckpt_file)

  # Sync params before starting.
  my_vals = sess.run(all_params)
  vals = [np.zeros_like(v) for v in my_vals]
  [comm.Allreduce(mv, v, op=MPI.SUM) for mv, v in zip(my_vals, vals)]
  assign_ops = [var.assign(val / num_tasks)
                for var, val in zip(all_params, vals)]
  sess.run(assign_ops)

  coord = tfu.start_queue_runners(sess)
  batch_size = args.batch_size * total_gpus
  iters_per_train_epoch = len(train_data.data) // batch_size
  iters_per_test_epoch = len(test_data.data) // batch_size

  lprint('starting training')
  for epoch in range(args.max_epochs):
    begin = time.time()
    # train for one epoch
    train_losses = []

    ti = trange(iters_per_train_epoch)
    for itr in ti:
      if coord.should_stop():
        tfu.stop_queue_runners(coord)

      # forward/backward/update model on each gpu
      lr *= args.lr_decay
      l, _ = sess.run([bits_per_dim, optimizer], {tf_lr: lr})
      train_losses.append(l)
      ti.set_postfix(loss=l, lr=lr)

    train_loss_gen = np.mean(train_losses)

    # compute likelihood over test data
    test_losses = []
    for itr in trange(iters_per_test_epoch):
      if coord.should_stop():
        tfu.stop_queue_runners(coord)

      l = sess.run(bits_per_dim_test)
      test_losses.append(l)

    test_loss_gen = np.mean(test_losses)
    test_bpd.append(test_loss_gen)

    # log progress to console
    stats = dict(epoch=epoch, time=time.time() - begin, lr=lr,
                 train_bpd=train_loss_gen,
                 test_bpd=test_loss_gen)
    all_stats = comm.gather(stats)
    if task_id == 0:
      lprint('-' * 16)
      for k in stats:
        lprint('%s:\t%s' % (k, np.mean([s[k] for s in all_stats])))
      if epoch % args.save_interval == 0:
        path = os.path.join(save_dir, str(epoch))
        os.makedirs(path, exist_ok=True)
        saver.save(sess, os.path.join(path, 'params_%s.ckpt' % args.data_set))

    sample_from_model(sess)

if __name__ == '__main__':
  import argparse
  import datetime
  import dateutil.tz
  import functools
  import os.path as osp

  parser = argparse.ArgumentParser()

  # data I/O
  parser.add_argument('-i', '--data_dir', type=str, default='./data',
                      help='Location for the dataset')
  parser.add_argument('-o', '--save_dir', type=str, default='./data/save',
                      help='Location for parameter checkpoints and samples')
  parser.add_argument('-d', '--data_set', type=str, default='cifar',
                      help='Can be either cifar|imagenet')
  parser.add_argument('-t', '--save_interval', type=int, default=10,
                      help='Every how many epochs to write checkpoint/samples?')
  parser.add_argument('-r', '--load_params', type=str,
                      help='Restore training from previous model checkpoint?')

  # model
  parser.add_argument('--model', type=str, default="h12_noup_smallkey",
                      help='name of the model')
  parser.add_argument('-q', '--nr_resnet', type=int, default=4,
                      help='Number of residual blocks per stage of the model')
  parser.add_argument('-n', '--nr_filters', type=int, default=256,
                      help='Number of filters to use across the model. Higher = larger model.')
  parser.add_argument('-m', '--nr_logistic_mix', type=int, default=10,
                      help='Number of logistic components in the mixture. Higher = more flexible model')
  parser.add_argument('-z', '--resnet_nonlinearity', type=str, default='elu',
                      help='Which nonlinearity to use in the ResNet layers. One of "concat_elu", "elu", "relu" ')
  parser.add_argument('-c', '--class_conditional', dest='class_conditional',
                      action='store_true', help='Condition generative model on labels?')
  # optimization
  parser.add_argument('-l', '--learning_rate', type=float,
                      default=1e-3, help='Base learning rate')
  parser.add_argument('-e', '--lr_decay', type=float, default=0.999998,
                      help='Learning rate decay, applied every step of the optimization')
  parser.add_argument('-b', '--batch_size', type=int, default=8,
                      help='Batch size during training per GPU')
  parser.add_argument('-a', '--init_batch_size', type=int, default=8,
                      help='How much data to use for data-dependent initialization.')
  parser.add_argument('-p', '--dropout_p', type=float, default=0.5,
                      help='Dropout strength (i.e. 1 - keep_prob). 0 = No dropout, higher = more dropout.')
  parser.add_argument('-x', '--max_epochs', type=int, default=5000,
                      help='How many epochs to run in total?')
  parser.add_argument('-g', '--nr_gpu', type=int, default=None,
                      help='How many GPUs to distribute the training across? Defaults to all.')

  # evaluation
  parser.add_argument('--polyak_decay', type=float, default=0.9995,
                      help='Exponential decay rate of the sum of previous model iterates during Polyak averaging')
  # reproducibility
  parser.add_argument('-s', '--seed', type=int, default=42,
                      help='Random seed to use')
  FLAGS = parser.parse_args()

  timestamp = datetime.datetime.now(
      dateutil.tz.tzlocal()).strftime('%Y_%m_%d_%H_%M_%S')
  logdir = 'pixelsnail_%s_%s' % (FLAGS.data_set, timestamp)

  FLAGS.save_dir = osp.join(FLAGS.save_dir, logdir)
  main(FLAGS)

max_stars_repo_name: eyalbetzalel/PixelSNAIL

max_issues_repo_name: eyalbetzalel/PixelSNAIL

max_forks_repo_name: eyalbetzalel/PixelSNAIL


text:
from numpy import ceil

def add_buffer(pipette,
               source_wells,
               dest,
               cols,
               vol,
               source_vol,
               tip=None,
               tip_vol=300,
               remaining=None,
               drop_tip=True,
               pre_mix=None,
               dead_vol=1000 / 8):

    if tip is not None:
        pipette.pick_up_tip(tip)
    else:
        pipette.pick_up_tip()

    source_well = source_wells[0]
    if remaining is None:
        remaining = source_vol

    transfers = int(ceil(vol / (tip_vol - 10)))
    transfer_vol = vol / transfers

    if pre_mix is not None:
        for well in source_wells:
            pipette.mix(pre_mix,
                        tip_vol*.9,
                        well)

    for col in cols:
        for i in range(0, transfers):
            pipette.aspirate(transfer_vol,
                             source_well)
            pipette.air_gap(10)
            pipette.dispense(transfer_vol + 10,
                             dest[col].top())

            remaining -= transfer_vol

            if remaining < transfer_vol + dead_vol:
                source_wells.pop(0)
                try:
                    source_well = source_wells[0]
                except IndexError:
                    print('Ran out of source wells!')
                    raise
                remaining = source_vol

        pipette.blow_out()

    if drop_tip:
        pipette.drop_tip()
    else:
        pipette.return_tip()

    return(remaining, source_wells)

def get_96_from_384_wells(method='interleaved', start=1):
    if method == 'interleaved':
        rows = [chr(64 + x * 2 - (start % 2))
                for x in range(1, 9)]
        cols = [x * 2 - int((start + 1) / 2) % 2
                for x in range(1, 13)]

        for col in cols:
            for row in rows:
                yield('%s%s' % (row, col))

    if method == 'packed':
        for col in range((start - 1) * 6 + 1,
                         (start - 1) * 6 + 7):
            for row in [chr(x + 65) for x in range(0, 16, 2)]:
                yield('%s%s' % (row, col))
        for col in range((start - 1) * 6 + 1,
                         (start - 1) * 6 + 7):
            for row in [chr(x + 65) for x in range(1, 17, 2)]:
                yield('%s%s' % (row, col))

reason: from numpy


text:
import numpy as np

def intersection():
    p = np.array([0, 0])
    r = np.array([1, 1])
    q = np.array([0.1, 0.1])
    s = np.array([.1, .1])

    if np.cross(r, s) == 0 and np.cross((q-p), r) == 0:    # collinear
        # t0 = (q − p) · r / (r · r)
        # t1 = (q + s − p) · r / (r · r) = t0 + s · r / (r · r)
        t0 = np.dot(q-p, r)/np.dot(r, r)
        t1 = t0 + np.dot(s, r)/np.dot(r, r)
        print(t1, t0)
        if ((np.dot(s, r) > 0) and (0 <= t1 - t0 <= 1)) or ((np.dot(s, r) <= 0) and (0 <= t0 - t1 <= 1)):
            print('collinear and overlapping, q_s in p_r')
        else:
            print('collinear and disjoint')
    elif np.cross(r, s) == 0 and np.cross((q-p), r) != 0:  # parallel r × s = 0 and (q − p) × r ≠ 0,
        print('parallel')
    else:
        t = np.cross((q - p), s) / np.cross(r, s)
        u = np.cross((q - p), r) / np.cross(r, s)
        if 0 <= t <= 1 and 0 <= u <= 1:
            # If r × s ≠ 0 and 0 ≤ t ≤ 1 and 0 ≤ u ≤ 1, the two line segments meet at the point p + t r = q + u s
            print('intersection: ', p + t*r)
        else:
            print('not parallel and not intersect')

def point2segment():
    p = np.array([-1, 1])    # coordination of point
    a = np.array([0, 1])    # coordination of line segment end 1
    b = np.array([1, 0])    # coordination of line segment end 2
    ab = b-a    # line ab
    ap = p-a
    distance = np.abs(np.cross(ab, ap)/np.linalg.norm(ab))  # d = (AB x AC)/|AB|
    print(distance)

    # angle  Cos(θ) = A dot B /(|A||B|)
    bp = p-b
    cosTheta1 = np.dot(ap, ab) / (np.linalg.norm(ap) * np.linalg.norm(ab))
    theta1 = np.arccos(cosTheta1)
    cosTheta2 = np.dot(bp, ab) / (np.linalg.norm(bp) * np.linalg.norm(ab))
    theta2 = np.arccos(cosTheta2)
    if np.pi/2 <= (theta1 % (np.pi*2)) <= 3/2 * np.pi:
        print('out of a')
    elif -np.pi/2 <= (theta2 % (np.pi*2)) <= np.pi/2:
        print('out of b')
    else:
        print('between a and b')

if __name__ == '__main__':
    point2segment()
    # intersection()

max_stars_repo_path: ai for simple games/cars/2D_carBackup/collision.py

max_stars_repo_name: gustasvs/AI

max_issues_repo_name: gustasvs/AI

max_forks_repo_name: gustasvs/AI


text:
from itertools import izip

from StringIO import StringIO
from pandas.core.common import adjoin, isnull, _format
from pandas.core.index import MultiIndex, _ensure_index

import pandas.core.common as com
import numpy as np

docstring_to_string = """
    Parameters
    ----------
    frame : DataFrame
        object to render
    buf : StringIO-like, optional
        buffer to write to
    columns : sequence, optional
        the subset of columns to write; default None writes all columns
    col_space : int, optional
        the width of each columns
    header : bool, optional
        whether to print column labels, default True
    index : bool, optional
        whether to print index (row) labels, default True
    na_rep : string, optional
        string representation of NAN to use, default 'NaN'
    formatters : list or dict of one-parameter functions, optional
        formatter functions to apply to columns' elements by position or name,
        default None
    float_format : one-parameter function, optional
        formatter function to apply to columns' elements if they are floats
        default None
    sparsify : bool, optional
        Set to False for a DataFrame with a hierarchical index to print every
        multiindex key at each row, default True
    justify : {'left', 'right'}, default None
        Left or right-justify the column labels. If None uses the option from
        the configuration in pandas.core.common, 'left' out of the box
    index_names : bool, optional
        Prints the names of the indexes, default True """

class SeriesFormatter(object):

    def __init__(self, series, buf=None, header=True, length=True,
                 na_rep='NaN', name=False, float_format=None):
        self.series = series
        self.buf = buf if buf is not None else StringIO()
        self.name = name
        self.na_rep = na_rep
        self.float_format = float_format
        self.length = length
        self.header = header

        def formatter(x, col_width=None):
            return _format(x, self.series.dtype,
                           na_rep=self.na_rep,
                           float_format=self.float_format,
                           col_width=col_width)
        self.formatter = formatter

    def _get_footer(self):
        footer = ''
        if self.name:
            footer += ("Name: %s" % str(self.series.name)
                       if self.series.name else '')

        if self.length:
            if footer:
                footer += ', '
            footer += 'Length: %d' % len(self.series)
        return footer

    def _get_formatted_index(self):
        index = self.series.index
        is_multi = isinstance(index, MultiIndex)
        if is_multi:
            have_header = any(name for name in index.names)
            fmt_index = index.format(names=True)
        else:
            have_header = index.name is not None
            fmt_index = index.format(name=True)
        return fmt_index, have_header

    def _get_formatted_values(self):
        series = self.series
        vals = series.values

        if self.float_format is None:
            float_format = com.print_config.float_format
            if float_format is None:
                float_format = com._float_format_default

        # floating point handling
        if series.dtype == 'O':
            is_float = (series.map(com.is_float) & series.notnull()).values
            leading_space = is_float.any()

            fmt_values = []
            for i, v in enumerate(vals):
                if not is_float[i] and leading_space:
                    fmt_values.append(' %s' % self.formatter(v))
                elif is_float[i]:
                    fmt_values.append(float_format(v))
                else:
                    fmt_values.append(' %s' % self.formatter(v))
        else:
            fmt_values = _format_fixed_width(self.series.values,
                                             self.formatter)
        return fmt_values

    def to_string(self):
        series = self.series

        if len(series) == 0:
            return ''

        fmt_index, have_header = self._get_formatted_index()
        fmt_values = self._get_formatted_values()

        maxlen = max(len(x) for x in fmt_index)
        pad_space = min(maxlen, 60)
        result = ['%s   %s' % (k.ljust(pad_space), v)
                  for (k, v) in izip(fmt_index[1:], fmt_values)]

        if self.header and have_header:
            result.insert(0, fmt_index[0])

        footer = self._get_footer()
        if footer:
            result.append(footer)

        return '\n'.join(result)

class DataFrameFormatter(object):
    """
    Render a DataFrame

    self.to_string() : console-friendly tabular output
    self.to_html()   : html table

    """

    __doc__ += docstring_to_string

    def __init__(self, frame, buf=None, columns=None, col_space=None,
                 header=True, index=True, na_rep='NaN', formatters=None,
                 justify=None, float_format=None, sparsify=True,
                 index_names=True, **kwds):
        self.frame = frame
        self.buf = buf if buf is not None else StringIO()
        self.show_index_names = index_names
        self.sparsify = sparsify
        self.float_format = float_format
        self.formatters = formatters
        self.na_rep = na_rep
        self.col_space = col_space
        self.header = header
        self.index = index

        if justify is None:
            self.justify = com.print_config.colheader_justify
        else:
            self.justify = justify

        self.kwds = kwds

        if columns is not None:
            self.columns = _ensure_index(columns)
        else:
            self.columns = frame.columns

    def to_string(self):
        """
        Render a DataFrame to a console-friendly tabular output.
        """
        frame = self.frame

        to_write = []

        if len(frame.columns) == 0 or len(frame.index) == 0:
            info_line = 'Empty %s\nColumns: %s\nIndex: %s'
            to_write.append(info_line % (type(self.frame).__name__,
                                         repr(frame.columns),
                                         repr(frame.index)))
        else:
            # may include levels names also
            str_index = self._get_formatted_index()
            str_columns = self._get_formatted_column_labels()

            stringified = []

            for i, c in enumerate(self.columns):
                if self.header:
                    fmt_values = self._format_col(c)
                    cheader = str_columns[i]
                    max_len = max(max(len(x) for x in fmt_values),
                                  max(len(x) for x in cheader))
                    if self.justify == 'left':
                        cheader = [x.ljust(max_len) for x in cheader]
                    else:
                        cheader = [x.rjust(max_len) for x in cheader]
                    stringified.append(cheader + fmt_values)
                else:
                    stringified = [self._format_col(c) for c in self.columns]

            if self.index:
                to_write.append(adjoin(1, str_index, *stringified))
            else:
                to_write.append(adjoin(1, *stringified))

        for s in to_write:
            if isinstance(s, unicode):
                to_write = [unicode(s) for s in to_write]
                break

        self.buf.writelines(to_write)

    def _get_col_formatter(self, dtype):
        from pandas.core.common import _format

        def formatter(x, col_width=None):
            return _format(x, dtype, space=self.col_space,
                           na_rep=self.na_rep,
                           float_format=self.float_format,
                           col_width=col_width)
        return formatter

    def _format_col(self, col, i=None):
        if self.formatters is None:
            self.formatters = {}

        if col in self.formatters:
            formatter = self.formatters[col]

            if i is None:
                return [formatter(x) for x in self.frame[col]]
            else:
                return formatter(self.frame[col][i])
        else:
            dtype = self.frame[col].dtype
            formatter = self._get_col_formatter(dtype)

            if i is not None:
                return formatter(self.frame[col][i])
            else:
                return _format_fixed_width(self.frame[col],
                                           formatter)

    def to_html(self):
        """
        Render a DataFrame to a html table.
        """
        def write(buf, s, indent=0):
            buf.write(unicode((' ' * indent) + str(s) + '\n'))

        def write_th(buf, s, indent=0):
            write(buf, '%s	' % str(s), indent)

        def write_td(buf, s, indent=0):
            write(buf, '%s	' % str(s), indent)

        def write_tr(buf, l, indent=0, indent_delta=4, header=False):
            write(buf, '
', indent)
            indent += indent_delta
            if header:
                for s in l:
                    write_th(buf, s, indent)
            else:
                for s in l:
                    write_td(buf, s, indent)
            indent -= indent_delta
            write(buf, '
', indent)

        indent = 0
        indent_delta = 2
        frame = self.frame
        buf = self.buf

        write(buf, '
', indent)

        def _column_header():
            row = [''] * (frame.index.nlevels - 1)

            if isinstance(frame.columns, MultiIndex):
                if self.has_column_names:
                    row.append(single_column_table(frame.columns.names))
                row.extend([single_column_table(c) for c in frame.columns])
            else:
                row.append(frame.columns.name or '')
                row.extend(frame.columns)
            return row

        if len(frame.columns) == 0 or len(frame.index) == 0:
            write(buf, '
', indent  + indent_delta)
            write_tr(buf,
                     [repr(frame.index),
                      'Empty %s' % type(self.frame).__name__],
                     indent + (2 * indent_delta),
                     indent_delta)
            write(buf, '
', indent  + indent_delta)
        else:
            indent += indent_delta

            # header row
            if self.header:
                write(buf, '
', indent)
                row = []

                col_row = _column_header()
                indent += indent_delta
                write_tr(buf, col_row, indent, indent_delta, header=True)
                if self.has_index_names:
                    row = frame.index.names + [''] * len(frame.columns)
                    write_tr(buf, row, indent, indent_delta, header=True)

                write(buf, '
', indent)

            write(buf, '
', indent)

            _bold_row = self.kwds.get('bold_rows', False)
            def _maybe_bold_row(x):
                temp = '%s'
                if _bold_row:
                    return ([temp % y for y in x] if isinstance(x, tuple)
                            else temp % x)
                else:
                    return x

            # write values
            for i in range(len(frame)):
                row = []
                if isinstance(frame.index, MultiIndex):
                    row.extend(_maybe_bold_row(frame.index[i]))
                else:
                    row.append(_maybe_bold_row(frame.index[i]))
                for column in frame.columns:
                    row.append(self._format_col(column, i))
                write_tr(buf, row, indent, indent_delta)
            indent -= indent_delta
            write(buf, '
', indent)
            indent -= indent_delta

        write(buf, '
', indent)

    def _get_formatted_column_labels(self):
        from pandas.core.index import _sparsify

        formatters = self.formatters
        if formatters is None:
            formatters = {}

        def is_numeric_dtype(dtype):
            return issubclass(dtype.type, np.number)

        if isinstance(self.columns, MultiIndex):
            fmt_columns = self.columns.format(sparsify=False, adjoin=False)
            fmt_columns = zip(*fmt_columns)
            dtypes = self.frame.dtypes.values
            need_leadsp = dict(zip(fmt_columns, map(is_numeric_dtype, dtypes)))
            str_columns = zip(*[[' %s' % y
                                if y not in formatters and need_leadsp[x]
                                else str(y) for y in x]
                               for x in fmt_columns])
            if self.sparsify:
                str_columns = _sparsify(str_columns)

            str_columns = [list(x) for x in zip(*str_columns)]
        else:
            fmt_columns = self.columns.format()
            dtypes = self.frame.dtypes
            need_leadsp = dict(zip(fmt_columns, map(is_numeric_dtype, dtypes)))
            str_columns = [[' %s' % x
                            if x not in formatters and need_leadsp[x]
                            else str(x)]
                           for x in fmt_columns]

        if self.show_index_names and self.has_index_names:
            for x in str_columns:
                x.append('')

        return str_columns

    @property
    def has_index_names(self):
        return _has_names(self.frame.index)

    @property
    def has_column_names(self):
        return _has_names(self.frame.columns)

    def _get_formatted_index(self):
        index = self.frame.index
        columns = self.frame.columns

        show_index_names = self.show_index_names and self.has_index_names
        show_col_names = (self.show_index_names and self.has_column_names)

        if isinstance(index, MultiIndex):
            fmt_index = index.format(sparsify=self.sparsify, adjoin=False,
                                     names=show_index_names)
        else:
            fmt_index = [index.format(name=show_index_names)]

        adjoined = adjoin(1, *fmt_index).split('\n')

        # empty space for columns
        if show_col_names:
            col_header = ['%s' % x for x in self._get_column_name_list()]
        else:
            col_header = [''] * columns.nlevels

        if self.header:
            return col_header + adjoined
        else:
            return adjoined

    def _get_column_name_list(self):
        names = []
        columns = self.frame.columns
        if isinstance(columns, MultiIndex):
            names.extend('' if name is None else name
                         for name in columns.names)
        else:
            names.append('' if columns.name is None else columns.name)
        return names

def _format_fixed_width(values, formatter):
    formatted = [formatter(x) for x in values]
    max_len = max(len(x) for x in formatted)
    return [formatter(x, col_width=max_len) for x in values]

def single_column_table(column):
    table = '

'
    for i in column:
        table += ('
%s
' % str(i))
    table += '

'
    return table

def single_row_table(row):  # pragma: no cover
    table = '

'
    for i in row:
        table += ('%s	' % str(i))
    table += '

'
    return table

def _has_names(index):
    if isinstance(index, MultiIndex):
        return any([x is not None for x in index.names])
    else:
        return index.name is not None

max_stars_repo_name: ogrisel/pandas

max_issues_repo_name: ogrisel/pandas

max_forks_repo_name: ogrisel/pandas