text:
import string, copy
# import time
import os, os.path  # , shutil
import sys
import numpy
# from PyQt4.QtCore import *
# from PyQt4.QtGui import *
import operator
import numpy as np
import re

projectroot = os.path.split(os.getcwd())[0]
sys.path.append(projectroot)
sys.path.append(os.path.join(projectroot, 'AuxPrograms'))
sys.path.append(os.path.join(projectroot, 'QtForms'))
sys.path.append(os.path.join(projectroot, 'AuxPrograms'))
sys.path.append(os.path.join(projectroot, 'OtherApps'))
sys.path.append(os.path.join(projectroot, 'BatchProcesses'))
sys.path.append(os.path.join(projectroot, 'AnalysisFunctions'))

# from fcns_math import *
from fcns_io import *
from fcns_ui import *

# from Analysis_Master import Analysis_Master_nointer
# analysismasterclass=Analysis_Master_nointer()

class setup_rcp_and_exp_ssrl():
    def __init__(self, import_path, calib_path, rcpext='.done', expext='.done', anaext='.done',
                 overwrite_runs=False, plate_idstr=None, access='hte',
                 pmidstr=None, sample_no_from_position_index=lambda i: (1 + i),
                 testmode=False):
        """
        the  data should be saved so that once import_path (folder containing spec
        file and a folder "images" with .tif) is given,
        this class knows what to do to copy and name all rcp and exp files
        rcpext and expext should be set to .run for testing and then in standard
        operation can make them .done
        overwrite_runs can be set to True to help with debugging but safer to keep
        as False in case there is a mistkae, really overwrite of a .done run is
        not allowed but this is not checked
        plate_idstr should be auto read but passing the string here overrides that
        access for the data is set here and can be public,hte,tri,muri
        pmidstr will be auto read from the .info file but if for some reason the
        platemap used for xps is different this value can be overridden, which
        is dangerous but the sample_no generated for the .rcp/.exp must
        correspond to the pmidstr
        sample_no_from_position_index should either be aq list of all the
        sample_no in the order they were measured, or a lambda function like
        the default value above when the sample_no were measured in order
        """
        self.access = access
        self.pmidstr = pmidstr
        self.import_path = import_path
        self.calib_path = calib_path
        self.sample_no_from_position_index = sample_no_from_position_index

        # hard coded definitions
        self.compname = 'HTE-SSRL-01'
        self.detectorname = 'Rayonix MarCCD'
        self.pixelsize = '79.0'
        self.datatype = 'ssrl'

        self.plate_idstr = plate_idstr
        iserror = self.parse_shell()
        if iserror:
            return

        self.rcpext = rcpext
        self.expext = expext
        self.anaext = anaext

        iserror = self.setup_folders(overwrite_runs, testmode)
        if iserror:
            return

        self.setup_file_dicts()
        self.add_all_files(testmode)
        self.save_rcp_exp(testmode)

    def setup_folders(self, overwrite_runs, testmode):
        if self.pmidstr is None:
            ans = getplatemappath_plateid(self.plate_idstr, erroruifcn=None, infokey='screening_map_id:',
                                          return_pmidstr=True)
            if ans is None:
                print 'aborting because failed retrieval of platemap id for plate :', self.plate_idstr
                return True
            self.pmidstr = ans[1]

        dropfolder = getdropfolder_exptype(self.datatype)

        if dropfolder is None:
            # messageDialog(None, 'Aborting SAVE because cannot find drop folder').exec_()
            print 'Aborting SAVE because cannot find drop folder'
            return True
        if (not testmode) and not os.path.isdir(dropfolder):
            os.mkdir(dropfolder)

        ellist = getelements_plateidstr(self.plate_idstr)
        rcplab = ''.join(ellist)
        self.rcpmainfoldname = '_'.join([timestampname()[:8], rcplab, get_serial_plate_id(self.plate_idstr)])

        rcpmainfolder = os.path.join(dropfolder, self.rcpmainfoldname)

        if (not testmode) and not os.path.isdir(rcpmainfolder):
            os.mkdir(rcpmainfolder)

        self.runfolderpath = os.path.join(rcpmainfolder, self.data_acquisition_timestamp + self.rcpext)

        if not testmode:
            if os.path.isdir(self.runfolderpath):
                if overwrite_runs:
                    shutil.rmtree(self.runfolderpath)
                else:
                    # messageDialog(None, 'Aborting SAVE because %s folder exists' %rcpmainfolder).exec_()
                    print 'Aborting SAVE because %s folder exists' % rcpmainfolder
                    return True
            os.mkdir(self.runfolderpath)
        return False

    def read_ssrl_calib(self, line_inds=None,
                        delim='='):  # add in the calib values to run parameters, although this calib may need to be manually overridden during the

        with open(self.calib_path, mode='r') as f:
            lines = f.readlines()
        if not line_inds is None:
            print 'lines are {}'.format(lines)
            lines = [lines[i] for i in line_inds]

        for l in lines:
            k, temp, v = l.partition(delim)
            self.run_params_dict[
                l.strip()] = v.strip()  # calib params are string vaslues even for the numbers so convert before sending to functions

        self.run_params_dict['detector_name'] = self.detectorname
        self.run_params_dict['pixel_size_micron'] = self.pixelsize

    def parse_shell(self):
        #unsure what file should be read here?
        #self.lines = [line for line in open(self.import_path)]
        #self.get_block_list()  # get the complete file as a list containing dicts
        #set the plateid
        if self.plate_idstr is None:  # ideally plate_id is read from the filename or spec file because it was entered
            # by user when starting data acquisition, but if it was passed in
            # the class init then ignore
            serial_no_str = os.path.split(self.import_path)[1].partition('_')[0].strip()
            if False in [c.isdigit() for c in serial_no_str]:
                print 'error reading plate_id for ', self.import_path
                return True
            if len(serial_no_str) >= 5:  # expect serial but this fails if plate_id
                self.plate_idstr = serial_no_str[:-1]
            else:
                self.plate_idstr = serial_no_str
        self.run_params_dict = {}

        p_files = os.listdir(self.import_path)
        shellfns = [fn for fn in p_files if not '.' in fn]
        self.speccsvfns = [fn for fn in p_files if  '.csv' in fn]
        self.shellfn = shellfns[0]
        with open(os.path.join(self.import_path, self.shellfn), mode='r') as f:
            lines = f.readlines()
        for l in lines:
            if l.startswith('#D'):
                ts = time.strptime(l[2:].strip(), '%a %b %d %H:%M:%S %Y')
                self.data_acquisition_timestamp = time.strftime('%Y%m%d.%H%M%S', ts)
            if l.startswith('#S'):
                scmd = l.partition('  ')[2].strip()
                if len(scmd) > 0:
                    self.run_params_dict['spec_command'] = scmd

        return False

    def strrep_generic_file_dict_value_sp(self, v):
        return filterchars(str(v), valid_chars="/<>-_.,; ()[]{}/%s%s%s" % (
            string.ascii_letters, string.digits, ''.join(['\\', '%', '&', '^', '!', '#', '*'])))

    def setup_file_dicts(self):
        self.expdict = {}
        self.expdict['experiment_type'] = self.datatype
        self.expdict['exp_version'] = '3'
        self.expdict['description'] = '%s run on plate_id %s' % (self.datatype, self.plate_idstr)
        self.expdict['created_by'] = self.datatype
        self.expdict['access'] = self.access
        runcount = 0
        runk = 'run__%d' % (runcount + 1)
        self.expdict[runk] = {}
        exprund = self.expdict[runk]
        self.rcpdict = {}
        rcpdict = self.rcpdict
        rcpdict['experiment_type'] = self.datatype
        # don't pay attention to this "technique_name" it is an artifact of previous data and does not have the
        # same meaning as e.g. XPSSURVEY
        rcpdict['technique_name'] = self.datatype
        rcpdict['rcp_version'] = '2'

        self.add_run_attr = lambda k, v: [d.update({k: v}) for d in [exprund, rcpdict]]

        self.add_run_attr('screening_map_id', self.pmidstr)

        self.add_run_attr('run_use', 'data')
        self.add_run_attr('plate_id', self.plate_idstr)
        self.add_run_attr('name', self.data_acquisition_timestamp)
        self.add_run_attr('access', self.access)

        self.add_run_attr('computer_name', self.compname)
        exprund['run_path'] = r'/%s/%s/%s/%s' % (
            self.datatype, self.compname.lower(), self.rcpmainfoldname, rcpdict['name'] + self.rcpext)
        exprund['rcp_file'] = rcpdict['name'] + '.rcp'

        rcpdict['parameters'] = {}
        exprund['parameters'] = {}
        self.add_run_param = lambda k, v: [d.update({k: v}) for d in [exprund['parameters'], rcpdict['parameters']]]

        self.add_run_param('plate_id', self.plate_idstr)

        for k, v in self.run_params_dict.iteritems():
            v = self.strrep_generic_file_dict_value_sp(v).strip('[').rstrip(
                ']')  # make lists comma delimited but without the brackets
            self.add_run_param(k, v)

        techdlist = []
        tech = 'SSRL'
        tk = 'files_technique__%s' % tech
        exprund[tk] = {}
        rcpdict[tk] = {}

        exprund[tk]['csv_spec_files'] = {}
        rcpdict[tk]['csv_spec_files'] = {}
        csvtechd = (rcpdict[tk]['csv_spec_files'], exprund[tk]['csv_spec_files'])

        exprund[tk]['meta_spec_files'] = {}
        rcpdict[tk]['meta_spec_files'] = {}
        metatechd = (rcpdict[tk]['meta_spec_files'], exprund[tk]['meta_spec_files'])

        exprund[tk]['image_files'] = {}
        rcpdict[tk]['image_files'] = {}
        tiftechd = (rcpdict[tk]['image_files'], exprund[tk]['image_files'])

        exprund[tk]['calib_files'] = {}
        rcpdict[tk]['calib_files'] = {}
        calibtechd = (rcpdict[tk]['calib_files'], exprund[tk]['calib_files'])

        exprund[tk]['shell_files'] = {}
        rcpdict[tk]['shell_files'] = {}
        shelltechd = (rcpdict[tk]['shell_files'], exprund[tk]['shell_files'])

        # messy fix for : : bug
        self.add_calib_file = lambda fn: [d.update({fn: filed_createflatfiledesc({'file_type': 'ssrl_calib_file'})})
                                          for d in calibtechd]
        self.add_speccsv_file = lambda fn: [
            d.update({fn: filed_createflatfiledesc({'file_type': 'ssrl_spec_csv_file'})})
            for d in csvtechd]
        self.add_metacsv_file = lambda fn, sample_no: [
            d.update({fn: filed_createflatfiledesc({'file_type': 'ssrl_meta_csv_file', 'sample_no': sample_no})})
            for d in metatechd]
        self.add_tif_file = lambda fn, sample_no: [
            d.update({fn: filed_createflatfiledesc({'file_type': 'ssrl_mar_tiff_file', 'sample_no': sample_no})})
            for d in tiftechd]
        self.add_shell_file = lambda fn: [
            d.update({fn: filed_createflatfiledesc({'file_type': 'ssrl_shell_file'})})
            for d in shelltechd]

    def add_all_files(self, testmode):
        # TODO read spec file to get x,y
        self.parse_shell()
        self.d = self.read_ssrl_calib()
        self.calib_fn = os.path.split(self.calib_path)[1]
        self.add_calib_file(self.calib_fn)
        self.add_shell_file(self.shellfn)
        #for now this is a for loop but because this would essentially correspond to dirrefent runs within a folder
        #there should be some better checking of it ... or better data aquisition rules because it is hard to tell a
        # z height adjustment from an actual measurement
        for fn in self.speccsvfns:
            self.add_speccsv_file(fn)

        fns = os.listdir(os.path.join(self.import_path, 'images'))
        self.speccsv_fns, self.tif_fns = [fn for fn in fns if '.txt' in fn], [fn for fn in fns if '.tif' in fn]

        ID_spec = np.array([j.split('_')[1].strip('.txt') for j in self.speccsv_fns]).astype(np.int)
        ID_tif = np.array([j.split('_')[1].strip('.tif') for j in self.tif_fns]).astype(np.int)

        #check if specs and tif are read in the same way so once spec are parsed
        # a simple map can be run on add_tif_file
        if not np.sum(np.argsort(ID_spec)-np.argsort(ID_tif)) == 0:
            argsort_ID_spec = np.argsort(ID_spec)
            self.tif_fns = self.tif_fns[argsort_ID_spec]

        position_index_of_file_index = lambda fi: fi
        sample_no_of_file_index = lambda fi: self.sample_no_from_position_index[
            position_index_of_file_index[position_index_of_file_index(fi)]] if isinstance(
            self.sample_no_from_position_index, list) else self.sample_no_from_position_index(
            position_index_of_file_index(fi))

        for fntif,fncsv in zip(enumerate(self.tif_fns),enumerate(self.speccsv_fns)):
            self.add_tif_file(fntif[1], sample_no_of_file_index(fntif[0]))
            self.add_metacsv_file(fncsv[1], sample_no_of_file_index(fncsv[0]))

    def save_rcp_exp(self, testmode):
        rcpfilestr = strrep_filedict(self.rcpdict)
        p = os.path.join(self.runfolderpath, self.rcpdict['name'] + '.rcp')
        if testmode:
            print 'THIS IS THE RCP FILE THAT WOULD BE SAVED:'
            print rcpfilestr
            print os.path.join(self.import_path, self.shellfn)
            return
        with open(p, mode='w') as f:
            f.write(rcpfilestr)
        # print 'rcp file saved to ', p
        saveexp_txt_dat(self.expdict, saverawdat=False, experiment_type=self.datatype, rundone=self.expext,
                        file_attr_and_existence_check=False, runtodonesavep=self.runfolderpath, savefolder=self.runfolderpath)
        # print 'exp file saved to ', dsavep
        if not testmode:
            shutil.copy(self.calib_path, os.path.join(self.runfolderpath, self.calib_fn))
            shutil.copy(os.path.join(self.import_path, self.shellfn), os.path.join(self.runfolderpath, self.shellfn))
            for fn in self.speccsvfns:
            shutil.copy(os.path.join(self.import_path, fn), os.path.join(self.runfolderpath, fn))

            img_path = os.path.join(self.import_path,'images')
            for fntif, fncsv in zip(self.tif_fns, self.speccsv_fns):
                shutil.copy(os.path.join(img_path, fntif), os.path.join(self.runfolderpath, fntif))
                shutil.copy(os.path.join(img_path, fncsv), os.path.join(self.runfolderpath, fncsv))
        print self.runfolderpath

#REMINDER: this class looks for the shellfile file (no file ending) that contains information
import_path = r'K:\experiments\ssrl\user\2017Jul\43557_CuPdZn_toredo'
calib_path = r'K:\experiments\ssrl\user\SSRLJan2017\calib\version_4.calib'
importclass = setup_rcp_and_exp_ssrl(import_path, calib_path, rcpext='.done', expext='.done', overwrite_runs=True,
                                     plate_idstr=None, access='hte', pmidstr=None,
                                     sample_no_from_position_index=lambda i: (1 + i), testmode=False)

max_stars_repo_name: johnmgregoire/JCAPDataProcess

max_issues_repo_name: johnmgregoire/JCAPDataProcess

max_forks_repo_name: johnmgregoire/JCAPDataProcess


text:
from __future__ import print_function
import argparse
import os
import random
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
from torch.utils.data import DataLoader
from dataset import *
import time
import numpy as np
import torchvision.utils as vutils
from torch.autograd import Variable
from networks import *
from math import log10
import torchvision
import cv2

parser = argparse.ArgumentParser()
parser.add_argument('--test', action='store_true', help='enables test during training')
parser.add_argument('--mse_avg', action='store_true', help='enables mse avg')
parser.add_argument('--num_layers_res', type=int, help='number of the layers in residual block', default=2)
parser.add_argument('--nrow', type=int, help='number of the rows to save images', default=10)

parser.add_argument('--trainfiles', default="path/celeba/train.list", type=str, help='the list of training files')
parser.add_argument('--dataroot', default="path/celeba", type=str, help='path to dataset')

parser.add_argument('--testfiles', default="/disk2/wangxd/WaveletSRNet/data/lfw_landmark.txt", type=str, help='the list of training files')

parser.add_argument('--testroot', default="/nas/wangxudong/lfw/inter_aligned/", type=str, help='path to dataset')

parser.add_argument('--trainsize', type=int, help='number of training data', default=162770)

parser.add_argument('--testsize', type=int, help='number of testing data', default=13233)  # the number of LFW dataset

parser.add_argument('--workers', type=int, help='number of data loading workers', default=2)
parser.add_argument('--batchSize', type=int, default=64, help='input batch size')
parser.add_argument('--test_batchSize', type=int, default=64, help='test batch size')
parser.add_argument('--save_iter', type=int, default=10, help='the interval iterations for saving models')
parser.add_argument('--test_iter', type=int, default=500, help='the interval iterations for testing')
parser.add_argument('--cdim', type=int, default=3, help='the channel-size  of the input image to network')

parser.add_argument('--input_height', type=int, default=112, help='the height  of the input image to network')
parser.add_argument('--input_width', type=int, default=96, help='the width  of the input image to network')

parser.add_argument('--output_height', type=int, default=112, help='the height  of the output image to network')
parser.add_argument('--output_width', type=int, default=96, help='the width  of the output image to network')

parser.add_argument('--crop_height', type=int, default=112, help='the width  of the output image to network')
parser.add_argument('--crop_width', type=int, default=96, help='the width  of the output image to network')

parser.add_argument('--upscale', type=int, default=2, help='the depth of wavelet tranform')
parser.add_argument('--scale_back', action='store_true', help='enables scale_back')
parser.add_argument("--nEpochs", type=int, default=500, help="number of epochs to train for")
parser.add_argument("--start_epoch", default=1, type=int, help="Manual epoch number (useful on restarts)")
parser.add_argument('--lr', type=float, default=0.0002, help='learning rate, default=0.0002')
parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5')
parser.add_argument("--momentum", default=0.9, type=float, help="Momentum, Default: 0.9")
parser.add_argument('--cuda', action='store_true', help='enables cuda')
parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use')
parser.add_argument('--outf', default='./results/', help='folder to output images')
parser.add_argument('--manualSeed', type=int, help='manual seed')
parser.add_argument("--pretrained", default="", type=str, help="path to pretrained model (default: none)")

def main():

    global opt, model
    opt = parser.parse_args()
    print(opt)

    try:
        os.makedirs(opt.outf)
    except OSError:
        pass

    if opt.manualSeed is None:
        opt.manualSeed = random.randint(1, 10000)
    print("Random Seed: ", opt.manualSeed)
    random.seed(opt.manualSeed)
    torch.manual_seed(opt.manualSeed)
    if opt.cuda:
        torch.cuda.manual_seed_all(opt.manualSeed)

    cudnn.benchmark = True

    if torch.cuda.is_available() and not opt.cuda:
        print("WARNING: You have a CUDA device, so you should probably run with --cuda")

    ngpu = int(opt.ngpu)
    nc = opt.cdim
    mag = int(math.pow(2, opt.upscale))
    groups = mag ** 2
    if opt.scale_back:
      is_scale_back = True
    else:
      is_scale_back = False

    #--------------build models--------------------------
    srnet = NetSR(opt.upscale, num_layers_res=opt.num_layers_res)
    if opt.pretrained:
        if os.path.isfile(opt.pretrained):
            print("=> loading model '{}'".format(opt.pretrained))
            weights = torch.load(opt.pretrained)
            pretrained_dict = weights['model'].state_dict()
            model_dict = srnet.state_dict()
            # print(model_dict)
            pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
            model_dict.update(pretrained_dict)
            # 3. load the new state dict
            srnet.load_state_dict(model_dict)
            # srnet.load_state_dict(weights['model'].state_dict())
        else:
            print("=> no model found at '{}'".format(opt.pretrained))
    print(srnet)

    wavelet_dec = WaveletTransform(scale=opt.upscale, dec=True)
    wavelet_rec = WaveletTransform(scale=opt.upscale, dec=False)

    criterion_m = nn.MSELoss(size_average=True)

    if opt.cuda:
      srnet = srnet.cuda()
      wavelet_dec = wavelet_dec.cuda()
      wavelet_rec = wavelet_rec.cuda()
      criterion_m = criterion_m.cuda()

    # optimizer_sr = optim.Adam(srnet.parameters(), lr=opt.lr, betas=(opt.momentum, 0.999), weight_decay=0.0005)

    #-----------------load dataset--------------------------

    # train_list, _ = loadFromFile(opt.trainfiles, opt.trainsize)

    # train_set = ImageDatasetFromFile(train_list, opt.dataroot,
    #           input_height=opt.input_height, input_width=opt.input_width,
    #           output_height=opt.output_height, output_width=opt.output_width,
    #           crop_height=opt.crop_height, crop_width=opt.crop_width,
    #           is_random_crop=True, is_mirror=True, is_gray=False,
    #           upscale=mag, is_scale_back=is_scale_back)

    # train_data_loader = torch.utils.data.DataLoader(train_set, batch_size=opt.batchSize,
    #                                  shuffle=True, num_workers=int(opt.workers))

    test_list, _ = loadFromFile(opt.testfiles, opt.testsize)

    test_set = ImageDatasetFromFile(test_list, opt.testroot,
                  input_height=opt.output_height, input_width=opt.output_width,
                  output_height=opt.output_height, output_width=opt.output_width,
                  crop_height=None, crop_width=None,
                  is_random_crop=False, is_mirror=False, is_gray=False,
                  upscale=mag, is_scale_back=is_scale_back)
    test_data_loader = torch.utils.data.DataLoader(test_set, batch_size=opt.test_batchSize,
                                         shuffle=False, num_workers=int(opt.workers))

    start_time = time.time()

    #--------only Test--------------------
    epoch=0
    iteration=0

    srnet.eval()
    avg_psnr = 0
    for titer, batch in enumerate(test_data_loader,0):
        input, target = Variable(batch[0]), Variable(batch[1])
        if opt.cuda:
            input = input.cuda()
            target = target.cuda()

        wavelets = forward_parallel(srnet, input, opt.ngpu)
        prediction = wavelet_rec(wavelets)
        mse = criterion_m(prediction, target)
        psnr = 10 * log10(1 / (mse.data[0]) )
        avg_psnr += psnr

        save_images(prediction, "Epoch_{:03d}_Iter_{:06d}_{:02d}_o.jpg".format(epoch, iteration, titer),
                                path=opt.outf, nrow=opt.nrow)

    print("===> Avg. PSNR: {:.4f} dB".format(avg_psnr / len(test_data_loader)))
    # srnet.train()
    #----------------Train by epochs--------------------------
    # for epoch in range(opt.start_epoch, opt.nEpochs + 1):
    #     if epoch%opt.save_iter == 0:
    #         save_checkpoint(srnet, epoch, 0, 'sr_')
    #
    #     for iteration, batch in enumerate(train_data_loader, 0):
    #         #--------------test-------------
    #         if iteration % opt.test_iter is 0 and opt.test:
    #             srnet.eval()
    #             avg_psnr = 0
    #             for titer, batch in enumerate(test_data_loader,0):
    #                 input, target = Variable(batch[0]), Variable(batch[1])
    #                 if opt.cuda:
    #                     input = input.cuda()
    #                     target = target.cuda()

    #                 wavelets = forward_parallel(srnet, input, opt.ngpu)
    #                 prediction = wavelet_rec(wavelets)
    #                 mse = criterion_m(prediction, target)
    #                 psnr = 10 * log10(1 / (mse.data[0]) )
    #                 avg_psnr += psnr
    #
    #                 save_images(prediction, "Epoch_{:03d}_Iter_{:06d}_{:02d}_o.jpg".format(epoch, iteration, titer),
    #                             path=opt.outf, nrow=opt.nrow)
    #
    #
    #             print("===> Avg. PSNR: {:.4f} dB".format(avg_psnr / len(test_data_loader)))
    #             srnet.train()
    #
    #         #--------------train------------

    #         input, target = Variable(batch[0]), Variable(batch[1], requires_grad=False)
    #         if opt.cuda:
    #           input = input.cuda()
    #           target = target.cuda()
    #
    #         target_wavelets = wavelet_dec(target)
    #
    #         batch_size = target.size(0)
    #         wavelets_lr = target_wavelets[:,0:3,:,:]
    #         wavelets_sr = target_wavelets[:,3:,:,:]
    #
    #         wavelets_predict = forward_parallel(srnet, input, opt.ngpu)
    #         img_predict = wavelet_rec(wavelets_predict)
    #
    #
    #         loss_lr = loss_MSE(wavelets_predict[:,0:3,:,:], wavelets_lr, opt.mse_avg)
    #         loss_sr = loss_MSE(wavelets_predict[:,3:,:,:], wavelets_sr, opt.mse_avg)
    #         loss_textures = loss_Textures(wavelets_predict[:,3:,:,:], wavelets_sr)
    #         loss_img = loss_MSE(img_predict, target, opt.mse_avg)
    #
    #         loss = loss_sr.mul(0.99) + loss_lr.mul(0.01) + loss_img.mul(0.1) + loss_textures.mul(1)
    #
    #         optimizer_sr.zero_grad()
    #         loss.backward()
    #         optimizer_sr.step()
    #
    #         info = "===> Epoch[{}]({}/{}): time: {:4.4f}:".format(epoch, iteration, len(train_data_loader), time.time()-start_time)
    #         info += "Rec: {:.4f}, {:.4f}, {:.4f}, Texture: {:.4f}".format(loss_lr.data[0], loss_sr.data[0],
    #                             loss_img.data[0], loss_textures.data[0])
    #
    #         print(info)

def forward_parallel(net, input, ngpu):
    if ngpu > 1:
        return nn.parallel.data_parallel(net, input, range(ngpu))
    else:
        return net(input)

def save_checkpoint(model, epoch, iteration, prefix=""):
    model_out_path = "model/" + prefix +"model_epoch_{}_iter_{}.pth".format(epoch, iteration)
    state = {"epoch": epoch ,"model": model}
    if not os.path.exists("model/"):
        os.makedirs("model/")

    torch.save(state, model_out_path)

    print("Checkpoint saved to {}".format(model_out_path))

def save_images(images, name, path, nrow=10):
  #print(images.size())
  img = images.cpu()
  im = img.data.numpy().astype(np.float32)
  #print(im.shape)
  im = im.transpose(0,2,3,1)
  imsave(im, [nrow, int(math.ceil(im.shape[0]/float(nrow)))], os.path.join(path, name) )

def merge(images, size):
  #print(images.shape())
  h, w = images.shape[1], images.shape[2]
  img = np.zeros((h * size[0], w * size[1], 3))
  #print(img)
  for idx, image in enumerate(images):
    image = image * 255
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    i = idx % size[1]
    j = idx // size[1]
    img[j*h:j*h+h, i*w:i*w+w, :] = image
  return img

def imsave(images, size, path):
  img = merge(images, size)
  # print(img)
  return cv2.imwrite(path, img)

if __name__ == "__main__":
    main()

ext: py

reason: import numpy


text:
#!/usr/bin/env python

import sys
import imread
import numpy as np

for f in sys.argv[1:]:
    print(f)
    im = imread.imread(f)[:, :, :3]
    ramp = np.zeros(im.shape)
    ramp[...] = 0.5
    half = ramp.shape[0] / 2
    ramp[:half, :, :] = np.linspace(0.7, 0.5, half).reshape((half, 1, 1))
    new = 1.0 - 2 * (1 - ramp) * (1 - im.astype(float) / 255.0)
    new = new * 255
    new = new.astype(np.uint8)
    imread.imsave("bumped_" + f, new)

max_stars_repo_name: hobeika/world-flags-sprite

max_issues_repo_name: hobeika/world-flags-sprite

max_forks_repo_name: hobeika/world-flags-sprite


text:
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for routing."""

from absl.testing import parameterized
import numpy as np
import tensorflow as tf

from official.nlp.modeling.layers import routing

class TokenImportanceTest(tf.test.TestCase, parameterized.TestCase):

  def test_token_importance(self):
    token_importance_embed = routing.TokenImportanceWithMovingAvg(
        vocab_size=4,
        init_importance=10.0,
        moving_average_beta=0.995)
    importance = token_importance_embed(np.array([[0, 1], [2, 3]]))
    self.assertAllClose(importance, np.array([[10.0, 10.0], [10.0, 10.0]]))
    token_importance_embed.update_token_importance(
        token_ids=np.array([[0, 1]]),
        importance=np.array([[0.0, 0.0]]))
    importance = token_importance_embed(np.array([[0, 1], [2, 3]]))
    self.assertAllClose(importance, np.array([[9.95, 9.95], [10.0, 10.0]]))

class TopKSelectionTest(tf.test.TestCase, parameterized.TestCase):

  def test_top_k_selection(self):
    token_selection = routing.SelectTopK(top_k=2)
    selected, _ = token_selection(np.array([[0, 1, 2, 3], [4, 3, 2, 1]]))
    self.assertAllClose(selected, np.array([[3, 2], [0, 1]]))

  def test_random_k_selection(self):
    token_selection = routing.SelectTopK(random_k=2)
    selected, _ = token_selection(np.array([[0, 1, 2, 3], [4, 3, 2, 1]]))
    self.assertAllClose(selected.shape, (2, 2))

  def test_top_k_random_k(self):
    token_selection = routing.SelectTopK(top_k=1, random_k=1)
    selected, _ = token_selection(np.array([[0, 1, 2, 3], [4, 3, 2, 1]]))
    self.assertAllClose(selected.shape, (2, 2))

if __name__ == "__main__":
  tf.test.main()


text:
import numpy as np
import warnings

#disable annoying warnings
np.warnings.filterwarnings('ignore')
warnings.filterwarnings("ignore", category=DeprecationWarning)

# ignore all warnings
def warn(*args, **kwargs):
    pass
warnings.warn = warn

###########################################
## Define useful preset band combinations
###########################################
RGB = (680.0,550.0,505.0)
"""
Wavelengths for red [680.0], green [550.0] and blue [505.0]- useful for plotting.
Note that we use the upper end of blue as this is the first band of rikola data.
"""

VNIR = (800.0, 550.0, 505.0)
"""Useful preview for VNIR data using (infrared [1972.0], green [644.0], blue [1450.0])."""

SWIR = (2200.0,2250.0,2350.0)
"""Useful preview for SWIR data (2200.0, 2250.0, 2350.0) sensitive to clay, mica, carbonate and amphibole absorbtions."""

BROAD = (1972.0,644.0,1450.0) #useful preview for data that covers visible VNIR and SWIR range
"""Useful preview that covers VNIR and SWIR range (1972.0,644.0,1450.0) ."""

MWIR = (3000., 3400., 3800. )
"""Useful preview for MWIR range (3000., 3400., 3800. )."""

LWIR = TIR = (10101.01, 9174.31, 8547.01)
"""Useful preview for TIR range (10101.01, 9174.31, 8547.01)."""

band_select_threshold = 10.
"""Maximum distance (in nanometers) to use when matching wavelengths with band indices. See HyData.get_band_index(...) for more detail."""

#import basic data classes
from .hyheader import HyHeader
from .hydata import HyData
from .hyimage import HyImage
from .hycloud import HyCloud
from .hylibrary import HyLibrary
from .hyscene import HyScene
from .hyfeature import HyFeature, MultiFeature, MixedFeature


text:
import numpy as np
import argparse

# parse command line inputs
parser = argparse.ArgumentParser(description='This script is a simple example on how to read a NPY binary file.')
parser.add_argument('-f', '--file', required=True, type=str, help='configuration yaml file')
# configuration file
filename = parser.parse_args().file

data = np.load(filename, allow_pickle=True, encoding='latin1', fix_imports=True).flat[0]

print(f'These are the parent keywords: \n\t{sorted(data.keys())}')
events = list(data.keys())
print(f'Inside each of the parent keyword you will find the following keys: \n\t{sorted(data[events[0]].keys())}')
sites = list(data[events[0]].keys())
print(f'Each key is a dictionary: \n\t{data[events[0]][sites[0]].keys()}')

print(f'In example you can access data with a nested cycle:')
for n, event in enumerate(events):
    for site in sites:
        print(f'Template {event} - Site {site} - Visibility Intervals:')
        if type(data[event][site]) == float:
            print(f'\tThis contains NaNs ---> the source is not observable due to daylight or moon.')
        else:
            print(f"\tThis contains the start and stop of each night.")
            for night in data[event][site]:
                print(f"\t{night}")
                print(f"\tstart: {data[event][site][night]['start']}")
                print(f"\tstop: {data[event][site][night]['stop']}")
                print(f"\n\tEach night contains the irfs intervals under the 'irfs' keyword.")
                if type(data[event][site][night]['irfs']['zref']) == float:
                    print(f'\tThis contains NaNs ---> the source is not observable at the site.')
                else:
                    for i in range(len(data[event][site][night]['irfs']['zref'])):
                        print(f'\t\tstart: {data[event][site][night]["irfs"]["start"][i]}')
                        print(f'\t\tstop: {data[event][site][night]["irfs"]["stop"][i]}')
                        print(f'\t\tzref: {data[event][site][night]["irfs"]["zref"][i]}')

    if n > 2:
        print("Let's stop after 3 templates.")
        break

max_stars_repo_name: cta-rta/rta-sci-grb

max_issues_repo_name: cta-rta/rta-sci-grb

max_forks_repo_name: cta-rta/rta-sci-grb


text:
import setuptools

# To compile f90 with f2py 'setup' has to come from numpy.
# This is not the case with cython.
# In addition, the option cmdclass = {'build_ext': build_ext} is not needed
# Check for example:
# https://github.com/perrette/python-fortran-cpp-template/blob/master/setup.py
# https://stackoverflow.com/questions/14453208/mixing-f2py-with-distutils
# https://gist.github.com/johntut/1d8edea1fd0f5f1c057c

from numpy.distutils.core import setup
from numpy.distutils.extension import Extension

ext_math = Extension(
    name = 'vector_normalization.lib.libmath',
    extra_compile_args = [],
    libraries = [],
    language = 'f90',
    sources = ['vector_normalization/lib/libmath.f90'],
)

setup(
    name='vector_normalization',
    version='1.0',
    author='Diego Prada | UIBCDF Lab',
    author_email='uibcdf@gmail.com',
    package_dir={'vector_normalization': 'vector_normalization'},
    packages=setuptools.find_packages(),
    ext_modules=[ext_math],
    url='http://uibcdf.org',
    download_url ='https://github.com/uibcdf/TestConda_withF90Libs',
    license='MIT',
    description="Sample case python package with f90 libs.",
    long_description="Sample case showing how to distribute a python package with fortran f90 libraries in conda.",
)

reason: from numpy


text:
from abc import ABCMeta, abstractmethod
import numpy as np
from cronex import CronExpression
import math
from datetime import datetime
import collections
from itertools import cycle

class TimeRepresentation(metaclass=ABCMeta):
    """
    General abstract class for representing time when the bank operations should occur.
    Actual implementation classes that represent time should implement this class.
    """

    @abstractmethod
    def evaluate(self,datetime): raise NotImplementedError #implemented method should return bool

class Cron(TimeRepresentation):
    """
    Class for representing cron times. This class is meant to be used when we can be certain that on certain days
    or in certain times, the transaction should be executing.
    Note:
        The package we use for cron parsing (cronex) supports using repeaters (for periodicity).
        To use repeater put '%number' in expression.
    """
    def __init__(self,expression):
        """
        Args:
            expression (str) : classic cron expression string
        """
        self.cronjob = CronExpression(expression)

    def evaluate(self,datetime):
        self.last_time = datetime
        return self.cronjob.check_trigger(datetime.timetuple()[:5])

    def evaluate_expression(self):
        return self.cronjob.numerical_tab

    def get_expression(self):
        return self.cronjob.string_tab

    def get_epoch(self):
        return self.cronjob.epoch

    def set_epoch(self,epoch):
        """
        Setting the epoch will probably be required for repeaters to work correctly. The epoch is important since
        it is the start from which the repeaters will measure the period.
        """
        try: # test if the specified parameters are right in terms of a correct datetime
            datetime(*epoch[:5])
        except:
            raise ValueError("Some value in the tuple doesn't match a correct date. Use the format: year, month,day,hour,minute.")
        if len(epoch) == 6:
            self.cronjob.epoch = epoch
        else:
            raise ValueError("Use tuple with 6 values being in this order: year,month,day,hour,minute,utc_offset!")

class TimeDistribution(TimeRepresentation):
    """
    Update as of 23.10.2017: This class shouldn't be used because it doesn't preserve the required probabilities.
        A better implementation is TimeDistribExp that we should be using.
    Class for representing time distributions by using different granularities of time.
    The specified granularities used are : month, day, hour and minute. This class is meant to be used when
    we want to define the time for transaction's execution as a probability on a certain day.
    TODO: Afaik there is no way to specify that the probability of transaction to execute should increase
    with the time getting later and later.. is that okay?
    TODO: Implement weekday granularity.
    TODO: Implement some form of counter? maxcount? for a period of time
    """

    def __init__(self, td_dict):
        """
        Args:
            td_dict (dictionary) : should be a dictionary with keys of any from "month", "day", "hour", "minute".
                Then the values in those keys should be either probability dictionaries (f.e. for hour there
                should be keys from 0 to 59 with their respective probabilities)
        """
        # USING ORDERED DICTs for the probabilities
        self.timedist = td_dict #dict contains distributions for : month,week, day, hour, minute
        # the distributions supplied should be either a histogram of probabilities (in our case list is enough)
        # or a discrete distribution
        # bud histogram alebo binomial distribution
        #for every such supplied list we should check that the probabilities add up to 1
        for k,v in self.timedist.items():
            self.timedist[k]=self.check_probabilities2(v)

    def check_probabilities(self, prob_dict):
        """
        Check that the sum of the probabilities for the specified time granularity is equal to 1.
        Args:
            prob_dict (dict or scipy.stats type) : should be either a dictionary of probabilities or
                a random variable from the scipy.stats module
        """
        #prob_dict doesnt necessarily need to be a dictionary, we can use scipy distribution..
        if TimeDistribution.getname(prob_dict).startswith('scipy.stats'):
            return prob_dict
        if np.sum(prob_dict.values()) == 1:
            return prob_dict
        else:
            values=list(prob_dict.values())
            return collections.OrderedDict(zip(prob_dict.keys(),np.array(values)/np.sum(values)))

    def check_probabilities2(self,prob_dict):
        """
        Check that the probabilities for the specified time granularity are in the range
        of 0<=x<=1.
        """
        if TimeDistribution.getname(prob_dict).startswith('scipy.stats'):
            return prob_dict
        values=list(map(lambda x: x if 0<= x <= 1 else 1 if x > 1 else 0, prob_dict.values()))
        return collections.OrderedDict(zip(prob_dict.keys(),values))

    @classmethod
    def getname(cls,obj):
        """Gets the 'class' name from the object. From my experience it also gives the full module path which is
        very beneficial for us since that's what we need to identify scipy random variables.
        """
        return str(obj.__class__).split("'")[1]

    def evaluate(self,timestamp):
        """
        Check for a certain timestamp whether the operation should execute.
        """
        #TODO: should consider how to evaluate when there is both month and week defined
        datetime_names ={"month":"day", "week":"weekday","day":"hour", "hour":"minute","minute":"second"}
        counter=0
        for name,prob_dict in self.timedist.items():
            if self.prob_eval(prob_dict,getattr(timestamp,datetime_names[name])):
                counter+=1
                continue
            else:
                return (False,counter)
        return (True,counter)

    def prob_eval(self,prob_dict,value):
        #whats the difference betweeen generating a new random number between 0 and 1 for all of the time lengths and just
        # using one generated random number for all of them ? does it matter or not?
        #check if prob_dict really is a dictionary or it could be a scipy random variable
        if TimeDistribution.getname(prob_dict).startswith('scipy.stats') : #this is a scipy random variable
            generated=prob_dict.rvs() #generate a random number from the distribution
            # check if it matches the value
            generated = math.ceil(generated)
            return True if generated == value else False
        else: #it should be a dictionary
            if value not in prob_dict:
                return False
            return True if np.random.random() < prob_dict[value] else False

class PeriodicTime(TimeRepresentation):

    def __init__(self,time_start,time_period,granularity):
        """
        Class for representing periodic times.
        Args:
            time_start (datetime) : the start time of the operation (when the first operation should be executed)
            time_period (timedelta) : the period of the operation (how often should the operation be executed
                from the start point).
            granularity (str) : can be several values - from least specific to most specific
                - should be either 'day','hour','minute'. We don't specify months and years because that kind of period
                can and should be represented in days in this case.
        """
        self.time_start = time_start
        self.time_period = time_period
        self.multiplicator = 0
        if granularity not in ["day","hour","minute"]:
            raise ValueError("Specified granularity for the Periodic Time class is wrong! The value is:", granularity)
        self.granularity = granularity
        self.granul_list = ["year","month","day","hour","minute"]
        index = self.granul_list.index(granularity)
        self.granul_list = self.granul_list[:index+1]

    def total_time(self,delta,timetype):
        """
        Get total time from timedelta object - in days or hours or minutes.
        """
        seconds = delta.total_seconds()
        if timetype == "days":
            return delta.days
        elif timetype == "hours":
            return seconds // 3600
        elif timetype == "minutes":
            return (seconds) // 60
        else:
            raise ValueError("Wrong value for timetype parameter! value is : ", timetype)

    def evaluate(self,timestamp):
        """
        Args:
            timestamp (timedelta) : timestamp that we want to evaluate the operation on"""
        delta = timestamp - self.time_start
        timetype=self.granularity+"s"
        result,remainder = divmod(self.total_time(delta,timetype),self.total_time(self.time_period,timetype))
        return True if remainder == 0 else False

class TimeDistribExp(TimeRepresentation):
    """
    This is a different,better,working version of TimeDistribution class. Class for representing time distributions
        by using different granularities of time.
    The specified granularities used are : month, day, hour and minute. This class is meant to be used when
        we want to define the time for transaction's execution as a probability on a certain day.
    TODO: Implement weekday granularity.
    TODO: we cannot work with scipy random variables using this class, so far we can only use histograms
        or rather dictionaries with (time_value,probability) tuples
    Kedze zo sekvencie dni, mesiacov, atd, je tazke povedat ze kedy chcem mat zaciatok (napr mesiace jan,jun,nov s nenulovou
    pravdepodobnostou - ako viem kedy vygenerovat novy prvok --> uzivatel mohol zamyslat sekvenciu jan,jun,nov alebo
    jun,nov,jan atd atd,preto uzivatel by si mal sam vediet specifikovat kedy jeho obdobie konci.
    Philosophy of this class:
        As input we get some distribution over certain time periods (month,day,hour,etc.). We also get number of transactions
        to generate in this period, and when the sequence of the biggest time period is over (more info to write here!!)
    """
    __months=list(range(1,13))
    __days=list(range(1,32))
    __hours=list(range(0,24))
    __minutes=list(range(0,60))
    __timetypes={"month":__months,"day":__days,
                   "hour":__hours,"minute":__minutes}

    def crange(start, stop, modulo):
        """
        Implements circular range. Useful for when we want to represent start of period and end of period.
            When we know that we are in the interval between the end of period and start of period that is <end;start>
            then we can safely assume we can generate new schedules for transactions.
        """
        result = []
        index = start
        while index != stop:
            result.append(index)
            index = (index + 1) % modulo
        return result

    def get_biggest_timetype(self):
        """
        The biggest timetype will serve as something to keep track of the start and end of period for which we generate
            the schedules of transactions. Depending on what is the specified biggest timetype (f.e. "month","day","hour")
            the schedule will change.
        """
        a={"month":2,"day":1.5,"hour":1,"minute":0.5}
        mx=0 #max value
        mx_tt="second"
        for timetype in self.timedist.keys():
            if a[timetype] > mx:
                mx=a[timetype]
                mx_tt=timetype
        return mx_tt

    def get_tt_reordered(self):
        """
        Moves what should be the last element of the biggest timetype values array to the end of the array.
            This way we have clearly defined start at the start of array and end at the end of the array.
        """
        biggest_tt_arr=list(self.timedist[self.biggest_timetype].keys())
        end_ind=biggest_tt_arr.index(self.last_biggest_timetype_value)
        while biggest_tt_arr.index(self.last_biggest_timetype_value) != (len(biggest_tt_arr)-1):
            biggest_tt_arr += [biggest_tt_arr.pop(0)]
        # mali by sme si vytvorit set v ktorom su vsetky hodnoty od zaciatku obdobia do konca obdobia
        # treba na to mozno vyuzit cycle vtedy ked pojdeme do noveho mesiaca/dna/whatever
        return biggest_tt_arr

    def __init__(self, td_dict,last_biggest_timetype_val,num_transactions):
        """
        Args:
            td_dict (dictionary) : should be a dictionary with keys of any from "month", "day", "hour", "minute".
                Then the values in those keys should be either probability dictionaries (f.e. for hour there
                should be keys from 0 to 59 with their respective probabilities)
        """
        # USING ORDERED DICTs for the probabilities
        self.timedist = td_dict #dict contains distributions for : month,week, day, hour, minute
        # the distributions supplied should be either a histogram of probabilities (in our case list is enough)
        # or a discrete distribution
        # bud histogram alebo binomial distribution
        #for every such supplied list we should check that the probabilities add up to 1
        for k,v in self.timedist.items():
            self.timedist[k]=self.check_probabilities2(v)

        self.gen_next()
        self.biggest_timetype=self.get_biggest_timetype()
        self.should_gen_next=False
        self.last_biggest_timetype_value=last_biggest_timetype_val
        self.num_transactions=num_transactions
        self.cur_tx_counter=0
        self.tx_timestamps_to_execute=[]
        for _ in range(num_transactions):
            self.tx_timestamps_to_execute.append(self.gen_next())
        #self.get_end_of_seq(list(self.timedist[self.biggest_timetype].keys()),self.biggest_timetype)#self.next_timetype[self.biggest_timetype]
        biggest_tt_general=TimeDistribExp.__timetypes[self.biggest_timetype]
        biggest_tt=self.get_tt_reordered()
        start=biggest_tt[0]
        end=biggest_tt[-1]
        self.active_range=crange(start,end+1,max(biggest_tt_general)+1)
#         pool=cycle_with_index(biggest_tt,biggest_tt.index(STARTPOINT))

    def get_end_of_seq(self,timetype):
        pool = cycle(TimeDistribExp.timetypes[timetype])
        return pool

    def check_probabilities2(self,prob_dict):
        """
        Check that the probabilities for the specified time granularity are in the range
        of 0<=x<=1 and also check that they sum up to one.
        """
        if TimeDistribution.getname(prob_dict).startswith('scipy.stats'):
            return prob_dict
        values=list(map(lambda x: x if 0<= x <= 1 else 1 if x > 1 else 0, prob_dict.values()))
        velka_suma=sum(values)
        values=[(val/velka_suma) for val in values]
        return collections.OrderedDict(zip(prob_dict.keys(),values))

    @classmethod
    def getname(cls,obj):
        """Gets the 'class' name from the object. From my experience it also gives the full module path which is
        very beneficial for us since that's what we need to identify scipy random variables.
        """
        return str(obj.__class__).split("'")[1]

    def check_timestamp(self,object_timestamp,sent_timestamp):
        for timetype,value in object_timestamp.items():
            if timetype == "executed":
                continue
            if value != getattr(sent_timestamp,timetype):
                return False
        return True

    def evaluate(self,timestamp):
        """
        Check for a certain timestamp whether the operation should execute.
        """
        #TODO: should consider how to evaluate when there is both month and week defined

        if (self.cur_tx_counter > 0) and getattr(timestamp,self.biggest_timetype) not in self.active_range:
#     and self.biggest_tt_start < getattr(timestamp,self.biggest_timetype) > self.last_biggest_timetype_value:
            print ("Vygenerovanie novych timestampov")
            self.tx_timestamps_to_execute=[]
            for _ in range(self.num_transactions):
                self.tx_timestamps_to_execute.append(self.gen_next())
            self.should_gen_next=False
            self.cur_tx_counter=0

#         else:
            # don't generate new next_timestamp because conditions weren't met
            # the conditions (2) are: the transaction with the last generated timestamp should have been executed,
            # the number of transactions in some period should have been met
            # the period must be over
            # conditions (1) are: we're currently in the period where the transactions should be executing
            # and we haven't yet made the desired amount of transactions
            # there's a clear problem with this implementation: if we randomly generate timestamp at the end of the period
            # as our first timestamp, then we will probably not be able to generate the desired amount of transactions
            # that we wished to
            # solutions : randomly generate the amount of num_transactions at once

        for object_timestamp in self.tx_timestamps_to_execute:
            if object_timestamp["executed"] == True:
                continue
            return_val=self.check_timestamp(object_timestamp,timestamp)
            if return_val == True:
                self.cur_tx_counter+=1
                object_timestamp["executed"]=True
                return True
#                 break

        return False

    def gen_next(self):
        """
        Generate the next timestamp when the transaction should be executed.
        """
        granularity=["month","day","hour","minute"]
        next_timestamp={}
        for timetype in granularity:
            if timetype in self.timedist:
                prob_dct=self.timedist[timetype]
                next_timestamp[timetype]=np.random.choice(list(prob_dct.keys()),p=list(prob_dct.values()))

#         self.next_timestamp = next_timestamp
        next_timestamp["executed"]=False
        return next_timestamp


text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

sns.set(rc={'figure.figsize': [10, 10]}, font_scale=1.3)

# In[2]:

#Read Data
df = pd.read_csv('electricity_prices.csv',low_memory=False)
df

# In[3]:

#dataFrame Information
df.info()

# # Data Analyst

# In[4]:

# Data Convertion to Numircal
numircal = ['ForecastWindProduction','SystemLoadEA','SMPEA','ORKTemperature','ORKWindspeed',
'CO2Intensity','ActualWindProduction','SystemLoadEP2','SMPEP2']
for col in numircal:
    df[col] = pd.to_numeric(df[col] , errors='coerce')

# In[5]:

#data convertion to datetime
df['DateTime'] = pd.to_datetime(df['DateTime'] ,format='%d/%m/%Y %H:%M' ,errors='coerce')

# In[6]:

df['Holiday'].unique()

# In[7]:

df.describe()

# In[8]:

df.groupby('Year').describe()['SMPEP2'].transpose()

# In[9]:

df.groupby('Year').describe()['ForecastWindProduction'].transpose()

# In[10]:

df.corr()[['ForecastWindProduction','SystemLoadEA','SMPEA','ORKTemperature','ORKWindspeed',
'CO2Intensity','ActualWindProduction','SystemLoadEP2','SMPEP2']]

# In[11]:

sns.heatmap(df.corr()[['ForecastWindProduction','SystemLoadEA','SMPEA','ORKTemperature','ORKWindspeed',
'CO2Intensity','ActualWindProduction','SystemLoadEP2','SMPEP2']], annot=True)

# In[12]:

sns.heatmap(df.corr()[['ForecastWindProduction','ActualWindProduction']], annot=True)

# In[13]:

sns.heatmap(df.corr()[['SystemLoadEA','SystemLoadEP2']], annot=True)

# In[14]:

df.groupby('HolidayFlag').describe()['SMPEP2'].transpose()

# In[15]:

df[df['ORKTemperature'] == df['ORKTemperature'].max()]

# In[16]:

df.sort_values(by='ForecastWindProduction',ascending=False,).head()

# In[17]:

df.groupby('Month').describe()['SMPEP2'].transpose()

# In[18]:

df.groupby('Year').describe()['ActualWindProduction'].transpose()

# In[19]:

df.groupby('Year').describe()['ForecastWindProduction'].transpose()

# In[20]:

df[df['ForecastWindProduction'] == df['ActualWindProduction']].head()

# # Data Visualization

# In[21]:

data_Numeric = ['HolidayFlag','DayOfWeek','WeekOfYear','Day','Month','Year','PeriodOfDay','ForecastWindProduction',
'ActualWindProduction','SystemLoadEA','SMPEA','ORKTemperature','ORKWindspeed','CO2Intensity','SystemLoadEP2','SystemLoadEA','SMPEP2']
for col in data_Numeric:
    sns.displot(df[col],height=8,bins=30)

# In[22]:

sns.jointplot(data = df,x = 'SMPEP2' ,y ='Day',color='r')

# In[23]:

sns.jointplot(data = df,x = 'SMPEP2' ,y ='SystemLoadEA',color='m')

# In[24]:

sns.jointplot(data = df,x = 'SMPEP2' ,y ='SMPEA',color='y')

# In[25]:

sns.jointplot(data = df,x = 'SMPEP2' ,y ='ForecastWindProduction',color='g')

# In[26]:

sns.jointplot(data = df,x = 'ActualWindProduction' ,y ='ForecastWindProduction',color='r')

# In[27]:

sns.jointplot(data = df,x = 'SystemLoadEP2' ,y ='SystemLoadEA',color='r')

# In[28]:

sns.jointplot(data = df,x = 'SMPEP2' ,y ='ORKWindspeed',color='m')

# In[29]:

sns.jointplot(data = df,x = 'SMPEP2' ,y ='ORKTemperature',color='m')

# In[30]:

sns.countplot(y='Holiday', data=df, palette='flare')

# In[31]:

sns.countplot(x='Day', data=df, palette='flare')

# In[32]:

sns.countplot(x='DayOfWeek', data=df, palette='flare')

# In[33]:

sns.countplot(x='Month', data=df, palette='flare')

# In[34]:

sns.countplot(x='Year', data=df, palette='flare')

# In[35]:

sns.countplot(x='HolidayFlag', data=df, palette='flare')

# In[36]:

sns.countplot(x='PeriodOfDay', data=df, palette='flare')

# In[37]:

sns.boxplot(x='DayOfWeek', y='ForecastWindProduction', data=df, palette='flare',hue='Year')

# In[38]:

sns.boxplot(x='DayOfWeek', y='ActualWindProduction', data=df, palette='flare',hue='Year')

# In[39]:

sns.boxplot(x='DayOfWeek', y='SystemLoadEA', data=df, palette='flare',hue='Year')

# In[40]:

sns.boxplot(x='DayOfWeek', y='SMPEP2', data=df, palette='flare',hue='Year')

# In[41]:

sns.boxplot(x='DayOfWeek', y='ORKWindspeed', data=df, palette='flare',hue='Year')

# In[42]:

sns.boxplot(x='DayOfWeek', y='ORKTemperature', data=df, palette='flare',hue='Year')

# In[43]:

sns.boxplot(x='Year', y='ORKTemperature', data=df)

# In[44]:

sns.boxplot(x='Year', y='SMPEP2', data=df, palette='flare')

# In[45]:

sns.boxplot(x='Year', y='SMPEA', data=df, palette='flare')

# In[46]:

sns.boxplot(x='Year', y='SystemLoadEA', data=df, palette='flare')

# In[47]:

sns.boxplot(x='Year', y='ORKWindspeed', data=df, palette='flare')

# In[48]:

sns.boxplot(x='Year', y='ActualWindProduction', data=df, palette='flare')

# In[49]:

sns.boxplot(x='Year', y='ForecastWindProduction', data=df, palette='flare')

# In[50]:

sns.boxplot(x='PeriodOfDay', y='SMPEA', data=df, palette='flare')

# In[51]:

sns.boxplot(x='PeriodOfDay', y='SystemLoadEP2', data=df, palette='flare')

# In[52]:

sns.boxplot(x='PeriodOfDay', y='CO2Intensity', data=df, palette='flare')

# In[53]:

sns.pairplot(df,diag_kind='kde')

# In[54]:

plt.scatter(x= df['SMPEP2'],y=df['ORKWindspeed'])
plt.xlabel('SMPEP2')
plt.ylabel('ORKWindspeed')
plt.show()

# In[55]:

plt.scatter(x= df['SMPEP2'],y=df['CO2Intensity'])
plt.xlabel('SMPEP2')
plt.ylabel('CO2Intensity')
plt.show()

# In[56]:

plt.scatter(x= df['SMPEP2'],y=df['ActualWindProduction'])
plt.xlabel('SMPEP2')
plt.ylabel('ActualWindProduction')
plt.show()

# In[57]:

plt.scatter(x= df['SMPEP2'],y=df['SystemLoadEP2'],c='y')
plt.xlabel('SMPEP2')
plt.ylabel('SystemLoadEP2')
plt.show()

# In[58]:

plt.scatter(x= df['ORKWindspeed'],y=df['ForecastWindProduction'],c='g')
plt.xlabel('ORKWindspeed')
plt.ylabel('ForecastWindProduction')
plt.show()

# # Data preprocessing

# In[59]:

df.info()

# In[60]:

df[df['Holiday'] =='None']

# In[61]:

df.drop('Holiday',axis = 1 ,inplace=True)

# In[62]:

# outfiter & Missing Data
df.isna().sum()

# In[63]:

from sklearn.impute import KNNImputer
imputer = KNNImputer()
for col in ['ForecastWindProduction','SystemLoadEA','SMPEA','ORKTemperature','ORKWindspeed',
'CO2Intensity','ActualWindProduction','SystemLoadEP2','SMPEP2']:
    df[col]=imputer.fit_transform(df[[col]])

# In[64]:

df.columns

# In[65]:

df = df[['HolidayFlag', 'DayOfWeek', 'WeekOfYear', 'Day', 'Month',
       'Year', 'PeriodOfDay', 'ForecastWindProduction', 'SystemLoadEA',
       'SMPEA', 'ORKTemperature', 'ORKWindspeed', 'CO2Intensity',
       'ActualWindProduction', 'SystemLoadEP2', 'SMPEP2']]

# In[66]:

df

# In[67]:

# Split data to Train and Test Sets
x = df.drop('SMPEP2',axis=1)
y = df['SMPEP2']
print(x.shape)
print(y.shape)

# In[68]:

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x,y,test_size = 0.25)

# In[69]:

print(x_train.shape)
print(x_test.shape)
print(y_train.shape)
print(y_test.shape)

# In[70]:

# Feature Scaling
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(x_train)
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)

# # Machine Learning

# In[71]:

from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error

# In[72]:

models = {
    "LR": LinearRegression(),
    "KNNR" : KNeighborsRegressor(),
    "SVR": SVR(),
    "DT": DecisionTreeRegressor(),
    "RF": RandomForestRegressor(),
    "XGBR": XGBRegressor()
}

# In[73]:

for name, model in models.items():
    print(f'Using model: {name}')
    model.fit(x_train, y_train)
    print(f'Training Score: {model.score(x_train, y_train)}')
    print(f'Test Score: {model.score(x_test, y_test)}')
    y_pred = model.predict(x_test)
    print(f'RMSE: {np.sqrt(mean_squared_error(y_test, y_pred))}')
    print('-----------------------------------------------------------')

# In[83]:

model = RandomForestRegressor()
model.fit(x_train, y_train)

# In[84]:

y_pred = model.predict(x_test)
y_pred

# In[85]:

y_test

# In[86]:

from sklearn.metrics import r2_score
r2_score(y_test, y_pred)

# In[87]:

plt.plot(y_test,'o')


text:
from sklearn.preprocessing import QuantileTransformer, StandardScaler
import pandas as pd
import numpy as np
import torch

class MyPreprocessor(object):
    '''
    It implements a quantile transformation on subset of featrues.

    It supports inverse transfrom on pytorch tensor, unlike sklearn coltransformer
    '''

    def __init__(self, cols_norm, way='quantile', random_state=1377,
                 n_quantiles=1000, quantile_noise=1e-3):
        assert way in ['quantile', 'standard']

        self.way = way
        self.random_state = random_state
        self.cols_norm = cols_norm
        self.n_quantiles = n_quantiles
        self.quantile_noise = quantile_noise

        self.feature_names = None
        self.num_features = None
        self.cols_norm_idxes = None
        self.transformer = None
        self.X_shape = None
        self.torch_device = None
        self.in_cols = None

    def fit(self, X):
        assert isinstance(X, pd.DataFrame), 'X is not a dataframe! %s' % type(X)
        self.feature_names = X.columns
        self.num_features = len(self.feature_names)
        self.cols_norm_idxes = [idx for idx, f in enumerate(self.feature_names) if f in self.cols_norm]

        quantile_train = X.copy()
        quantile_train = quantile_train[self.cols_norm]

        if self.quantile_noise:
            r = np.random.RandomState(self.random_state)
            stds = np.std(quantile_train.values, axis=0, keepdims=True)
            noise_std = self.quantile_noise / np.maximum(stds, self.quantile_noise)
            quantile_train += noise_std * r.randn(*quantile_train.shape)

        if self.way == 'quantile':
            self.transformer = QuantileTransformer(random_state=self.random_state,
                                          n_quantiles=self.n_quantiles,
                                          output_distribution='normal',
                                          copy=False)
        else:
            self.transformer = StandardScaler(copy=False)

        self.transformer.fit(quantile_train)

    def subset_transform(self, X, columns):
        '''
        Given a dataframe X that's a subset of original X, only transform those columns
        '''
        X, datatype = self.to_numpy(X)

        # First create a large numpy of original size
        X_temp = np.zeros((X.shape[0], len(self.cols_norm)))

        common_cols = [c for c in columns if c in self.cols_norm]
        idxes_map = {c: idx for idx, c in enumerate(self.cols_norm)}
        idxes_map2 = {c: idx for idx, c in enumerate(columns)}

        X_temp[:, [idxes_map[c] for c in common_cols]] = X[:, [idxes_map2[c] for c in common_cols]]
        X_temp = self.transformer.transform(X_temp)

        X_ret = X.copy()
        X_ret[:, [idxes_map2[c] for c in common_cols]] = X_temp[:, [idxes_map[c] for c in common_cols]]

        X_ret = self.to_type(X_ret, datatype)
        return X_ret

    def transform(self, X):
        assert self.transformer is not None, 'Not call fit() before'

        X, datatype = self.to_numpy(X)
        assert X.shape[-1] == self.num_features, f'Passed in {X.shape[-1]} != {self.num_features}'

        X_subset = X[:, self.cols_norm_idxes]
        X_subset = self.transformer.transform(X_subset)

        X = X.copy()

        X[:, self.cols_norm_idxes] = X_subset

        X = self.to_type(X, datatype)
        return X

    def inverse_transform(self, X):
        assert self.transformer is not None, 'Not call fit() before'

        X, datatype = self.to_numpy(X)
        assert X.shape[-1] == self.num_features, f'Passed in {X.shape[-1]} != {self.num_features}'

        X_subset = X[:, self.cols_norm_idxes]
        X_subset = self.transformer.inverse_transform(X_subset)

        X = X.copy()
        X[:, self.cols_norm_idxes] = X_subset

        X = self.to_type(X, datatype)
        return X

    def to_numpy(self, X):
        datatype = 'numpy'
        if isinstance(X, pd.DataFrame):
            self.in_cols = X.columns
            X = X.values
            datatype = 'pandas'
        elif isinstance(X, torch.Tensor):
            datatype = 'torch'
            self.torch_device = X.device
            with torch.no_grad():
                X = X.cpu().numpy()

        if len(X.shape) > 2: # 3d tensor in time-series
            self.X_shape = X.shape
            X = X.reshape((-1, self.num_features))

        return X, datatype

    def to_type(self, X, datatype):
        if datatype == 'pandas':
            return pd.DataFrame(X, columns=self.in_cols)
        if datatype == 'torch':
            X = torch.from_numpy(X)
            X = X.to(self.torch_device)
            self.torch_device = None

            if self.X_shape is not None:
                X = X.reshape(*self.X_shape)
                self.X_shape = None
            return X

        # numpy
        if self.X_shape is not None:
            X = X.reshape(self.X_shape)
            self.X_shape = None
        return X


text:
# coding=utf-8

"""
Module to perform predictions on data using
command line interface
"""

import argparse
import logging
import os
import shutil

import numpy as np
import tensorflow as tf

import deepreg.model.layer_util as layer_util
import deepreg.model.optimizer as opt
import deepreg.parser as config_parser
from deepreg.dataset.loader.util import normalize_array
from deepreg.model.network.build import build_model
from deepreg.util import (
    build_dataset,
    build_log_dir,
    calculate_metrics,
    save_array,
    save_metric_dict,
)

EPS = 1.0e-6

def build_pair_output_path(indices: list, save_dir: str) -> (str, str):
    """
    Create directory for saving the paired data

    :param indices: indices of the pair, the last one is for label
    :param save_dir: directory of output
    :return: - save_dir, str, directory for saving the moving/fixed image
             - label_dir, str, directory for saving the rest outputs
    """

    # cast indices to string and init directory name
    pair_index = "pair_" + "_".join([str(x) for x in indices[:-1]])
    pair_dir = os.path.join(save_dir, pair_index)
    os.makedirs(pair_dir, exist_ok=True)

    if indices[-1] >= 0:
        label_index = f"label_{indices[-1]}"
        label_dir = os.path.join(pair_dir, label_index)
        os.makedirs(label_dir, exist_ok=True)
    else:
        label_dir = pair_dir

    return pair_dir, label_dir

def predict_on_dataset(
    dataset: tf.data.Dataset,
    fixed_grid_ref: tf.Tensor,
    model: tf.keras.Model,
    model_method: str,
    save_dir: str,
    save_nifti: bool,
    save_png: bool,
):
    """
    Function to predict results from a dataset from some model

    :param dataset: where data is stored
    :param fixed_grid_ref: shape=(1, f_dim1, f_dim2, f_dim3, 3)
    :param model: model to be used for prediction
    :param model_method: str, ddf / dvf / affine / conditional
    :param save_dir: str, path to store dir
    :param save_nifti: if true, outputs will be saved in nifti format
    :param save_png: if true, outputs will be saved in png format
    """
    # remove the save_dir in case it exists
    if os.path.exists(save_dir):
        shutil.rmtree(save_dir)

    sample_index_strs = []
    metric_lists = []
    for _, inputs_dict in enumerate(dataset):
        outputs_dict = model.predict(x=inputs_dict)

        # moving image/label
        # (batch, m_dim1, m_dim2, m_dim3)
        moving_image = inputs_dict["moving_image"]
        moving_label = inputs_dict.get("moving_label", None)
        # fixed image/labelimage_index
        # (batch, f_dim1, f_dim2, f_dim3)
        fixed_image = inputs_dict["fixed_image"]
        fixed_label = inputs_dict.get("fixed_label", None)

        # indices to identify the pair
        # (batch, num_indices) last indice is for label, -1 means unlabeled data
        indices = inputs_dict.get("indices")
        # ddf / dvf
        # (batch, f_dim1, f_dim2, f_dim3, 3)
        ddf = outputs_dict.get("ddf", None)
        dvf = outputs_dict.get("dvf", None)
        affine = outputs_dict.get("affine", None)  # (batch, 4, 3)

        # prediction
        # (batch, f_dim1, f_dim2, f_dim3)
        pred_fixed_label = outputs_dict.get("pred_fixed_label", None)
        pred_fixed_image = (
            layer_util.resample(vol=moving_image, loc=fixed_grid_ref + ddf)
            if ddf is not None
            else None
        )

        # save images of inputs and outputs
        for sample_index in range(moving_image.shape[0]):
            # save moving/fixed image under pair_dir
            # save moving/fixed label, pred fixed image/label, ddf/dvf under label dir
            # if labeled, label dir is a sub dir of pair_dir, otherwise = pair_dir

            # init output path
            indices_i = indices[sample_index, :].numpy().astype(int).tolist()
            pair_dir, label_dir = build_pair_output_path(
                indices=indices_i, save_dir=save_dir
            )

            # save image/label
            # if model is conditional, the pred_fixed_image depends on the input label
            conditional = model_method == "conditional"
            arr_save_dirs = [
                pair_dir,
                pair_dir,
                label_dir if conditional else pair_dir,
                label_dir,
                label_dir,
                label_dir,
            ]
            arrs = [
                moving_image,
                fixed_image,
                pred_fixed_image,
                moving_label,
                fixed_label,
                pred_fixed_label,
            ]
            names = [
                "moving_image",
                "fixed_image",
                "pred_fixed_image",  # or warped moving image
                "moving_label",
                "fixed_label",
                "pred_fixed_label",  # or warped moving label
            ]
            for arr_save_dir, arr, name in zip(arr_save_dirs, arrs, names):
                if arr is not None:
                    # for files under pair_dir, do not overwrite
                    save_array(
                        save_dir=arr_save_dir,
                        arr=arr[sample_index, :, :, :],
                        name=name,
                        gray=True,
                        save_nifti=save_nifti,
                        save_png=save_png,
                        overwrite=arr_save_dir == label_dir,
                    )

            # save ddf / dvf
            arrs = [ddf, dvf]
            names = ["ddf", "dvf"]
            for arr, name in zip(arrs, names):
                if arr is not None:
                    arr = normalize_array(arr=arr[sample_index, :, :, :])
                    save_array(
                        save_dir=label_dir if conditional else pair_dir,
                        arr=arr,
                        name=name,
                        gray=False,
                        save_nifti=save_nifti,
                        save_png=save_png,
                    )

            # save affine
            if affine is not None:
                np.savetxt(
                    fname=os.path.join(
                        label_dir if conditional else pair_dir, "affine.txt"
                    ),
                    x=affine[sample_index, :, :].numpy(),
                    delimiter=",",
                )

            # calculate metric
            sample_index_str = "_".join([str(x) for x in indices_i])
            if sample_index_str in sample_index_strs:
                raise ValueError(
                    "Sample is repeated, maybe the dataset has been repeated."
                )
            sample_index_strs.append(sample_index_str)

            metric = calculate_metrics(
                fixed_image=fixed_image,
                fixed_label=fixed_label,
                pred_fixed_image=pred_fixed_image,
                pred_fixed_label=pred_fixed_label,
                fixed_grid_ref=fixed_grid_ref,
                sample_index=sample_index,
            )
            metric["pair_index"] = indices_i[:-1]
            metric["label_index"] = indices_i[-1]
            metric_lists.append(metric)

    # save metric
    save_metric_dict(save_dir=save_dir, metrics=metric_lists)

def build_config(config_path: (str, list), log_dir: str, ckpt_path: str) -> [dict, str]:
    """
    Function to create new directory to log directory to store results.

    :param config_path: string or list of strings, path of configuration files
    :param log_dir: string, path to store logs.
    :param ckpt_path: str, path where model is stored.
    :return: - config, configuration dictionary
             - log_dir, path of the directory for saving outputs
    """
    # check ckpt_path
    if not ckpt_path.endswith(".ckpt"):
        raise ValueError(
            "checkpoint path should end with .ckpt, got {}".format(ckpt_path)
        )

    # init log directory
    log_dir = build_log_dir(log_dir)

    # load config
    if config_path == "":
        # use default config, which should be provided in the log folder
        config = config_parser.load_configs(
            "/".join(ckpt_path.split("/")[:-2]) + "/config.yaml"
        )
    else:
        # use customized config
        logging.warning(
            "Using customized configuration."
            "The code might break if the config of the model doesn't match the saved model."
        )
        config = config_parser.load_configs(config_path)
    return config, log_dir

def predict(
    gpu: str,
    gpu_allow_growth: bool,
    ckpt_path: str,
    mode: str,
    batch_size: int,
    log_dir: str,
    sample_label: str,
    config_path: (str, list),
    save_nifti: bool = True,
    save_png: bool = True,
):
    """
    Function to predict some metrics from the saved model and logging results.

    :param gpu: str, which env gpu to use.
    :param gpu_allow_growth: bool, whether to allow gpu growth or not
    :param ckpt_path: str, where model is stored, should be like log_folder/save/xxx.ckpt
    :param mode: train / valid / test, to define which split of dataset to be evaluated
    :param batch_size: int, batch size to perform predictions in
    :param log_dir: str, path to store logs
    :param sample_label: sample/all, not used
    :param save_nifti: if true, outputs will be saved in nifti format
    :param save_png: if true, outputs will be saved in png format
    :param config_path: to overwrite the default config
    """
    # TODO support custom sample_label
    logging.warning(
        "sample_label is not used in predict. It is True if and only if mode == 'train'."
    )

    # env vars
    os.environ["CUDA_VISIBLE_DEVICES"] = gpu
    os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "false" if gpu_allow_growth else "true"

    # load config
    config, log_dir = build_config(
        config_path=config_path, log_dir=log_dir, ckpt_path=ckpt_path
    )
    preprocess_config = config["train"]["preprocess"]
    preprocess_config["batch_size"] = batch_size

    # data
    data_loader, dataset, _ = build_dataset(
        dataset_config=config["dataset"],
        preprocess_config=preprocess_config,
        mode=mode,
        training=False,
        repeat=False,
    )

    # optimizer
    optimizer = opt.build_optimizer(optimizer_config=config["train"]["optimizer"])

    # model
    model = build_model(
        moving_image_size=data_loader.moving_image_shape,
        fixed_image_size=data_loader.fixed_image_shape,
        index_size=data_loader.num_indices,
        labeled=config["dataset"]["labeled"],
        batch_size=preprocess_config["batch_size"],
        model_config=config["train"]["model"],
        loss_config=config["train"]["loss"],
    )

    # metrics
    model.compile(optimizer=optimizer)

    # load weights
    # https://stackoverflow.com/questions/58289342/tf2-0-translation-model-error-when-restoring-the-saved-model-unresolved-objec
    model.load_weights(ckpt_path).expect_partial()

    # predict
    fixed_grid_ref = tf.expand_dims(
        layer_util.get_reference_grid(grid_size=data_loader.fixed_image_shape), axis=0
    )  # shape = (1, f_dim1, f_dim2, f_dim3, 3)
    predict_on_dataset(
        dataset=dataset,
        fixed_grid_ref=fixed_grid_ref,
        model=model,
        model_method=config["train"]["model"]["method"],
        save_dir=log_dir + "/test",
        save_nifti=save_nifti,
        save_png=save_png,
    )

    # close the opened files in data loaders
    data_loader.close()

def main(args=None):
    """
    Function to run in command line with argparse to predict results on data
    for a given model
    """
    parser = argparse.ArgumentParser(
        description="predict", formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )

    parser.add_argument(
        "--gpu",
        "-g",
        help="GPU index for training."
        '-g "" for using CPU'
        '-g "0" for using GPU 0'
        '-g "0,1" for using GPU 0 and 1.',
        type=str,
        required=True,
    )

    parser.add_argument(
        "--gpu_allow_growth",
        "-gr",
        help="Prevent TensorFlow from reserving all available GPU memory",
        default=False,
    )

    parser.add_argument(
        "--ckpt_path",
        "-k",
        help="Path of checkpointed model to load",
        default="",
        type=str,
        required=True,
    )

    parser.add_argument(
        "--mode",
        "-m",
        help="Define the split of data to be used for prediction. One of train / valid / test",
        type=str,
        default="test",
        required=True,
    )

    parser.add_argument(
        "--batch_size", "-b", help="Batch size for predictions", default=1, type=int
    )

    parser.add_argument(
        "--log_dir", "-l", help="Path of log directory", default="", type=str
    )

    # TODO use this argument
    parser.add_argument(
        "--sample_label",
        "-s",
        help="Method of sampling labels",
        default="all",
        type=str,
    )

    parser.add_argument("--save_nifti", dest="nifti", action="store_true")
    parser.add_argument("--no_nifti", dest="nifti", action="store_false")
    parser.set_defaults(nifti=True)

    parser.add_argument("--save_png", dest="png", action="store_true")
    parser.add_argument("--no_png", dest="png", action="store_false")
    parser.set_defaults(png=False)

    parser.add_argument(
        "--config_path",
        "-c",
        help="Path of config, must end with .yaml. Can pass multiple paths.",
        type=str,
        nargs="*",
        default="",
    )

    args = parser.parse_args(args)

    predict(
        args.gpu,
        args.gpu_allow_growth,
        args.ckpt_path,
        args.mode,
        args.batch_size,
        args.log_dir,
        args.sample_label,
        args.config_path,
        args.nifti,
        args.png,
    )

if __name__ == "__main__":
    main()

max_stars_repo_name: zlinzju/DeepReg

max_issues_repo_name: zlinzju/DeepReg

max_forks_repo_name: zlinzju/DeepReg


text:
import pandas as pd
import numpy as np
import plotly.graph_objs as go

raw_data = imp.load_source('info', 'plots/perfil_tempo_tramitacao/get_raw_data.py').output['raw_data']

org = ['Administrativa', 'Comissões Temáticas',
       'Congresso Nacional', 'Constituição e Justiça',
       'Mesa', 'Plenário', 'Temporárias', 'Tributação e Finanças']

bins = np.linspace(0, 300, 31)

def plot(input, raw_data, filtered_data):
    status = input['situacao-perfil']
    periodo = input['tempo-perfil']

    return prepare_plot(raw_data, filtered_data, status, org, bins, periodo)

def draw_plot(x, y, z, hm_col):
    trace = [go.Heatmap(z=z,
                        y=y,
                        x=x,
                        # text=hovertext,
                        # hoverinfo='text',
                        colorscale=[

                            [0, hm_col[6]],
                            [1 / 7, hm_col[6]],

                            # Let values between 10-20% of the min and max of z
                            # have color rgb(20, 20, 20)
                            [1 / 7, hm_col[5]],
                            [2 / 7, hm_col[5]],

                            # Values between 20-30% of the min and max of z
                            # have color rgb(40, 40, 40)
                            [2 / 7, hm_col[4]],
                            [3 / 7, hm_col[4]],

                            [3 / 7, hm_col[3]],
                            [4 / 7, hm_col[3]],

                            [4 / 7, hm_col[2]],
                            [5 / 7, hm_col[2]],

                            [5 / 7, hm_col[1]],
                            [6 / 7, hm_col[1]],

                            [6 / 7, hm_col[0]],
                            [1.0, hm_col[0]]],
                        zmin=0,
                        zmax=70,
                        colorbar=go.ColorBar(title='Tempo no órgão (%)', titleside='top'),
                        showscale=True)]

    layout = dict(
            margin=dict(l=150, r=50, b=50, t=100, pad=4),
            xaxis=dict(title='Tempo de tramitação na Câmara (meses)'))

    figure = dict(data=trace, layout=layout)
    return figure

def prepare_data(raw_data, filtered_data, status=None, periodo=None):
    if status:
        # df = raw_data[raw_data['situacao_tipo'] == status]
        # df['dataInicio'] = pd.to_datetime(df['dataInicio'])
        # df = df[df['dataInicio'] >= str(periodo[0])]
        # df = df[df['dataInicio'] <= str(periodo[1])]
        df = filtered_data
        print('FILTEDER!!!!!!')
        print(filtered_data)
    else:
        df = raw_data

    df1 = df.groupby(['idProposicao', 'tipoOrgao']).sum()['tempo'].reset_index()
    df1 = df1.pivot(index='idProposicao', columns='tipoOrgao', values='tempo') / 30
    df1['Sum'] = df1.sum(1)
    df1 = df1.sort_values(by='Sum').reset_index()

    org_ext = list(set(list(df1.columns)).intersection(org))

    org_sum = org_ext + ['Sum']

    a = df1.fillna(0)[org_sum]

    b = a.values
    s = a['Sum'].values

    tempo_prep = pd.DataFrame(np.divide(b.T, s).T * 100, columns=org_sum)
    groups = tempo_prep.groupby(np.digitize(df1['Sum'], bins))

    temp = groups.mean()[org_ext]

    temp = temp.fillna(0)

    return temp

# Passar para dentro de draw_plot_1
def prepare_plot(raw_data, filtered_data, status, org, bins, periodo):
    temp = prepare_data(raw_data, filtered_data, status, periodo)

    x = bins
    y = org
    z = temp.values.T

    # hovertext = list()
    # for yi, yy in enumerate(y):
    #    hovertext.append(list())
    #    for xi, xx in enumerate(x):
    #        if xi < len(z1[yi]):
    #            xx = int(xx)
    #            hovertext[-1].append(
    #                'Dos PLs tramitados de {} a {} meses na Câmara,<br>'.format(xx, xx + 10) + ' {0:.3f}% '.format(
    #                    z1[yi][xi]) + 'do tempo foi em {}'.format(yy))
    #        else:
    #            hovertext[-1].append('Não houveram PLs tramitados na Câmara por {} meses'.format(xx))

    hm_col = ['#450A5C', '#423C81', '#34608C', '#2583A5', '#61B7C7', '#75D2DA', '#C1E1EA']

    return draw_plot(x, y, z, hm_col)

output = {'plot': plot}

if __name__ == '__main__':
    pass

max_stars_repo_name: JoaoCarabetta/viz-parallel

max_issues_repo_name: JoaoCarabetta/viz-parallel

max_forks_repo_name: JoaoCarabetta/viz-parallel


text:
# -*- coding: utf-8 -*-
# **********************************
# Author: Lizi
# Contact:  alisonbrielee@gmail.com
# **********************************

import numpy as np
import nibabel as nib
import scipy.io as sio

image_path_table = np.load('/root/data/registration/MultiPropReg/data/test_data_with_mask.npz')['arr_0']
good_labels = sio.loadmat('/root/data/registration/MultiPropReg/data/labels.mat')['labels'][0][:15]
index = list(range(len(good_labels)))

def convert(seg):
    labels = (np.unique(seg))
    extra_labels = list(set(labels).difference(set(good_labels)))

    output = np.copy(seg)
    for i in extra_labels:
        output[seg == i] = 15
    for k, v in zip(good_labels, index):
        output[seg == k ] = v
    return output

max_stars_repo_name: Alison-brie/AutoReg

max_issues_repo_name: Alison-brie/AutoReg

max_forks_repo_name: Alison-brie/AutoReg

reason: import numpy,import scipy


text:
# %% import

import os
import pathlib
import requests
import shutil
import sys
import numpy as np
import pandas as pd
from datetime import datetime

# sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
import dummy_data
import fibermorph

# %% functions

def create_results_cache():
    relpath = "fibermorph/demo"
    datadir = pathlib.Path.cwd().joinpath(relpath)
    cache = fibermorph.make_subdirectory(datadir, "results_cache")

    # Designate where fibermorph should make the directory with all your results - this location must exist!
    os.makedirs(cache, exist_ok=True)
    output_directory = os.path.abspath(cache)

    return output_directory

def delete_results_cache():
    cache = pathlib.Path.cwd().joinpath("fibermorph/demo/results_cache")

    print("Deleting {}".format(os.path.abspath(cache)))
    shutil.rmtree(cache)

    return True

def url_files(im_type):

    if im_type == "curv":

        demo_url = [
            "https://github.com/tinalasisi/fibermorph_DemoData/raw/master/test_input/curv/004_demo_curv.tiff",
            "https://github.com/tinalasisi/fibermorph_DemoData/raw/master/test_input/curv/027_demo_nocurv.tiff"]

        return demo_url

    elif im_type == "section":

        demo_url = [
            "https://github.com/tinalasisi/fibermorph_DemoData/raw/master/test_input/section/140918_demo_section.tiff",
            "https://github.com/tinalasisi/fibermorph_DemoData/raw/master/test_input/section/140918_demo_section2.tiff"]

        return demo_url

def download_im(tmpdir, demo_url):

    for u in demo_url:
        r = requests.get(u, allow_redirects=True)
        open(str(tmpdir.joinpath(pathlib.Path(u).name)), "wb").write(r.content)

    return True

def get_data(im_type="all"):
    relpath = "fibermorph/demo/data"
    datadir = pathlib.Path.cwd().joinpath(relpath)
    datadir = fibermorph.make_subdirectory(datadir, "tmpdata")

    if im_type == "curv" or im_type == "section":
        tmpdir = fibermorph.make_subdirectory(datadir, im_type)
        urllist = url_files(im_type)

        download_im(tmpdir, urllist)
        return tmpdir

    else:
        typelist = ["curv", "section"]
        for im_type in typelist:
            tmpdir = fibermorph.make_subdirectory(datadir, im_type)
            urllist = url_files(im_type)

            download_im(tmpdir, urllist)

            return True

def teardown_data(append=""):
    datadir = pathlib.Path.cwd().joinpath("fibermorph/demo/data/tmpdata/"+append)

    print("Deleting {}".format(str(datadir.resolve())))

    try:
        shutil.rmtree(datadir)
    except FileNotFoundError:
        print("The file doesn't exist. Nothing has been deleted")

    return True

def validation_curv(output_location, repeats=3):
    jetzt = datetime.now()
    timestamp = jetzt.strftime("%b%d_%H%M_")
    testname = str(timestamp + "ValidationTest_Curv")

    main_output_path = fibermorph.make_subdirectory(output_location, append_name=testname)

    dummy_dir = fibermorph.make_subdirectory(main_output_path, append_name="ValidationData")
    shape_list = ["arc", "line"]

    replist = [el for el in shape_list for i in range(repeats)]

    output_path = fibermorph.make_subdirectory(main_output_path, append_name="ValidationAnalysis")

    for shape in replist:
        print(shape)
        df, img, im_path, df_path = dummy_data.dummy_data_gen(
            output_directory=dummy_dir,
            shape=shape,
            min_elem=10,
            max_elem=20,
            im_width=5200,
            im_height=3900,
            width=10)

        valid_df = pd.DataFrame(df).sort_values(by=['ref_length'], ignore_index=True).reset_index(drop=True)

        test_df = fibermorph.curvature_seq(im_path, output_path, resolution=1, window_size_mm=10, save_img=False, test=True)

        test_df2 = pd.DataFrame(test_df).sort_values(by=['length'], ignore_index=True).reset_index(drop=True)

        col_list = ['error_length']

        if shape == "arc":
            valid_df['index1'] = valid_df['ref_length'] * valid_df['ref_radius']
            valid_df = pd.DataFrame(valid_df).sort_values(by=['index1'], ignore_index=True).reset_index(drop=True)
            test_df2['radius'] = 1 / test_df2['curv_median']
            test_df2['index2'] = test_df2['length'] * test_df2['radius']
            test_df2 = pd.DataFrame(test_df2).sort_values(by=['index2'], ignore_index=True).reset_index(drop=True)
            test_df2['error_radius'] = abs(valid_df['ref_radius'] - test_df2['radius']) / valid_df['ref_radius']
            test_df2['error_curvature'] = abs(valid_df['ref_curvature'] - test_df2['curv_median']) / valid_df[
                'ref_curvature']

            col_list = ['error_radius', 'error_curvature', 'error_length']

        test_df2['error_length'] = abs(valid_df['ref_length'] - test_df2['length']) / valid_df['ref_length']

        valid_df2 = valid_df.join(test_df2)

        error_df = valid_df2
        # error_df = valid_df2[col_list]

        im_name = im_path.stem
        df_path = pathlib.Path(output_path).joinpath(str(im_name) + "_errordata.csv")
        error_df.to_csv(df_path)

        print("Results saved as:\n")
        print(df_path)

    shutil.rmtree(pathlib.Path(output_path).joinpath("analysis"))

    return main_output_path

def validation_section(output_location, repeats=12):
    jetzt = datetime.now()
    timestamp = jetzt.strftime("%b%d_%H%M_")
    testname = str(timestamp + "ValidationTest_Section")

    main_output_path = fibermorph.make_subdirectory(output_location, append_name=testname)

    dummy_dir = fibermorph.make_subdirectory(main_output_path, append_name="ValidationData")
    shape_list = ["circle", "ellipse"]

    replist = [el for el in shape_list for i in range(repeats)]

    output_path = fibermorph.make_subdirectory(main_output_path, append_name="ValidationAnalysis")

    for shape in replist:
        print(shape)
        df, img, im_path, df_path = dummy_data.dummy_data_gen(
            output_directory=dummy_dir,
            shape=shape,
            min_elem=1,
            max_elem=1,
            im_width=5200,
            im_height=3900,
            width=1)

        valid_df = pd.DataFrame(df).sort_values(by=[0], axis=1)
        min_ax = np.asarray(valid_df)[0][0]
        max_ax = np.asarray(valid_df)[0][1]
        valid_df['ref_min'] = min_ax
        valid_df['ref_max'] = max_ax
        valid_df['ref_eccentricity'] = np.sqrt(1 - (min_ax ** 2) / (max_ax ** 2))
        valid_df.drop(columns=['ref_height', 'ref_width'])

        test_df = fibermorph.analyze_section(im_path, output_path, minsize=0, maxsize=3900, resolution=1.0)

        test_df['error_min'] = abs(valid_df['ref_min'] - test_df['min']) / valid_df['ref_min']
        test_df['error_max'] = abs(valid_df['ref_max'] - test_df['max']) / valid_df['ref_max']

        test_df['error_area'] = abs(valid_df['ref_area'] - test_df['area']) / valid_df['ref_area']
        test_df['error_eccentricity'] = np.nan_to_num(
            abs(valid_df['ref_eccentricity'] - test_df['eccentricity']) / valid_df['ref_eccentricity'], posinf=0)

        valid_df2 = valid_df.join(test_df)

        col_list = ['error_min', 'error_max', 'error_area', 'error_eccentricity']

        error_df = valid_df2
        # error_df = valid_df2[col_list]

        im_name = im_path.stem
        df_path = pathlib.Path(output_path).joinpath(str(im_name) + "_errordata.csv")
        error_df.to_csv(df_path)

        print("Results saved as:\n")
        print(df_path)

    return main_output_path

# %% Main modules

def real_curv():
    """Downloads curvature data and runs fibermorph_curv analysis.

    Returns
    -------
    bool
        True.

    """
    input_directory = get_data("curv")
    jetzt = datetime.now()
    timestamp = jetzt.strftime("%b%d_%H%M_")
    testname = str(timestamp + "DemoTest_Curv")

    output_location = fibermorph.make_subdirectory(create_results_cache(), append_name=testname)

    fibermorph.curvature(input_directory, output_location, jobs=1, resolution=132, window_size_mm=0.5, save_img=True)

    return True

def real_section():
    """Downloads section data and runs fibermorph_section analysis.

    Returns
    -------
    bool
        True.

    """
    input_directory = get_data("section")

    jetzt = datetime.now()
    timestamp = jetzt.strftime("%b%d_%H%M_")
    testname = str(timestamp + "DemoTest_Section")

    output_dir = fibermorph.make_subdirectory(create_results_cache(), append_name=testname)

    fibermorph.section(input_directory, output_dir, jobs=4, resolution=1.06)

    return True

def dummy_curv():
    """Creates dummy data, runs curvature analysis and provides error data for this analysis compared to known values from the dummy data.

    Returns
    -------
    bool
        True.

    """
    output_dir = validation_curv(create_results_cache(), repeats=1)
    print("Validation data and error analyses are saved in:\n")
    print(output_dir)

    return True

def dummy_section():
    """Creates dummy data, runs section analysis and provides error data for this analysis compared to known values from the dummy data.

    Returns
    -------
    bool
        True.

    """
    output_dir = validation_section(create_results_cache(), repeats=2)
    print("Validation data and error analyses are saved in:\n")
    print(output_dir)

    return True


text:
from django.db import models
from . import cgan as cg
from time import time

import tensorflow as tf
import numpy as np
import sys

sys.path.append('../')
from utils import image_utils as iu

SEED = 1337
is_model_loaded = False

dirs = {
    'output': './cgan/gen_img/',
    'model_path': './cgan/cgan_model/',
}

np.random.seed(SEED)

# Global TF obj
tf_sess = None
cgan_model = None

class CGANInference:

    @staticmethod
    def load_model(verbose=True):
        global is_model_loaded
        global tf_sess, cgan_model

        if not is_model_loaded:
            # GPU configure
            config = tf.ConfigProto()
            config.gpu_options.allow_growth = True

            tf_sess = tf.Session(config=config)
            cgan_model = cg.CGAN(tf_sess)  # CGAN Model
            if verbose:
                print("[+] CGAN Model loaded!")

            # initializing
            tf_sess.run(tf.global_variables_initializer())

            # Load model & Graph & Weights
            ckpt = tf.train.get_checkpoint_state(dirs['model_path'])
            if ckpt and ckpt.model_checkpoint_path:
                # Restores from checkpoint
                cgan_model.saver.restore(tf_sess, ckpt.model_checkpoint_path)

                saved_global_step = int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
                if verbose:
                    print("[+] global step : %d" % saved_global_step, " successfully loaded")

                is_model_loaded = True
            else:
                if verbose:
                    print('[-] No checkpoint file found :( pre-trained model is needed!')

                is_model_loaded = False

    @staticmethod
    def gen_image(input_label=1, verbose=True):
        global is_model_loaded
        global tf_sess, cgan_model

        num_label = np.eye(10)[input_label]  # one-hot encoding
        num_label = num_label.reshape(-1, 10)

        # Training G model with sample image and noise
        sample_z = np.random.uniform(-1., 1., [1, cgan_model.z_dim]).astype(np.float32)
        samples = tf_sess.run(cgan_model.g,
                              feed_dict={
                                  cgan_model.c: num_label,
                                  cgan_model.z: sample_z,
                                  cgan_model.do_rate: 0.0,
                              })

        samples = np.reshape(samples, [-1, 28, 28, 1])
        if verbose:
            print("[*] samples shape : ", samples.shape)

        # Export image generated by model G
        sample_image_height = 1
        sample_image_width = 1
        sample_dir = dirs['output'] + "gen.png"  # .format(input_label)

        # Generated image save
        iu.save_images(samples,
                       size=[sample_image_height, sample_image_width],
                       image_path=sample_dir)
        if verbose:
            print("[*] generated image (%d) is saved" % input_label)

    def __init__(self, input_label=0, verbose=True):
        self.success = False
        self.verbose = verbose

        self.load_model(verbose=self.verbose)  # Loading CGAN Model
        if is_model_loaded:
            self.start_time = time()
            self.gen_image(input_label, verbose=self.verbose)
            self.end_time = time()

            if self.verbose:
                print("[*] Total {:.4f}s is taken".format(self.end_time - self.start_time))

            self.success = True
        else:
            self.success = False

    def __str__(self):
        return "CGAN Model"