text:
#!/usr/bin/env python
# coding: utf-8

import os
import numpy as np
import random
import math
import pydicom
import pandas as pd
import shutil
import tensorflow as tf
import xml.etree.ElementTree as ET
from functools import partial, update_wrapper

os.environ["CUDA_VISIBLE_DEVICES"] = "0"
os.environ['TF_ENABLE_AUTO_MIXED_PRECISION'] = '1'

# Retrieving blocks of a numpy array
from skimage.util import view_as_blocks
# Retrieving blocks of a numpy array with given stride sizes
from skimage.util.shape import view_as_windows
from random import randint

from tqdm import tqdm
from random import randint
import matplotlib.pyplot as plt

from keras.models import Model, load_model
from keras.layers import Input, Flatten, Dense, concatenate, Conv2D, Conv3D, MaxPooling2D, MaxPooling3D, \
    Conv2DTranspose, Conv3DTranspose
from keras.layers import Activation, add, multiply, Lambda
from keras.layers import AveragePooling2D, AveragePooling3D, average, UpSampling2D, UpSampling3D, Dropout
from keras.optimizers import Adam, SGD, RMSprop
from keras.initializers import glorot_normal, random_normal, random_uniform
from keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping
from tensorflow.keras.callbacks import *
from keras import backend as K
from keras.layers.normalization import BatchNormalization
from keras.losses import binary_crossentropy
import numpy as np

from hyperopt import fmin, hp, tpe, Trials, space_eval
from hyperopt.pyll import scope as ho_scope
from hyperopt.pyll.stochastic import sample as ho_sample

class plaquetypes:
    limits = pd.DataFrame(
        [['DenseCalcium', 351, 10000], ['Fibrous', 131, 350], ['FibrousFatty', 76, 130], ['NecroticCore', -30, 75],
         ['NonCalcified', -1000, 350]], columns=['type', 'lower', 'upper'])

# ,['MedisDenseCalcium',351,10000],['MedisFibrous',151,350],['MedisFibrousFatty',31,150],['MedisNecroticCore',-100,30]

def wrapped_partial(func, *args, **kwargs):
    partial_func = partial(func, *args, **kwargs)
    update_wrapper(partial_func, func)
    return partial_func

def get_pairs(vesel_list):
    return_list = []
    for ID in vesel_list:
        if ID.endswith('NoContour.dcm'):
            return_list.append([ID, ID[:-13] + "Contour1.dcm", ID[:-13] + "Contour2.dcm", ID[:8]])
    return return_list

def find_included_segment(xml_file):
    try:
        doc = ET.parse(xml_file)
        root = doc.getroot()

        info = 0
        included = False
        while not (included):
            info += 1
            if (root[2][info][25].attrib['value'] == 'SEGMENT_TYPE_NORMAL'):
                breakpoints = [root[2][info][24].attrib['value'], root[2][info][1].attrib['value']]
                included = True
    except:
        doc = ET.parse(xml_file)
        root = doc.getroot()

        info = 0
        included = False
        while not (included):
            info += 1
            if (root[3][info][25].attrib['value'] == 'SEGMENT_TYPE_NORMAL'):
                breakpoints = [root[3][info][24].attrib['value'], root[3][info][1].attrib['value']]
                included = True
    return breakpoints

def get_xml(basic_path2, dicom):
    dicom = dicom.replace('_', '/')
    dicom = dicom[:-13] + 'data_model.xml'
    xml_file = os.path.join(basic_path2, dicom)
    return xml_file

def get_image_mask_for_ID_tuple(ID_tuple, basic_path, basic_path2, dir_to_save, plaques_only):
    image_path = os.path.join(basic_path, ID_tuple[0])
    mask_1_path = os.path.join(basic_path, ID_tuple[1])
    mask_2_path = os.path.join(basic_path, ID_tuple[2])
    image = pydicom.dcmread(image_path)
    mask_1 = pydicom.dcmread(mask_1_path)
    mask_2 = pydicom.dcmread(mask_2_path)
    if plaques_only==True:
        xml_file = get_xml(basic_path2, ID_tuple[0])
        breakpoints = np.array(find_included_segment(xml_file))
        breakpoints = breakpoints.astype(float)
        breakpoints = breakpoints / image.SpacingBetweenSlices
        if 'Plakk' in dir_to_save:
            image_array = (image.pixel_array)[int(breakpoints[0]):int(breakpoints[1]), :, :]
            mask_1_array = (mask_1.pixel_array)[int(breakpoints[0]):int(breakpoints[1]), :, :]
            mask_2_array = (mask_2.pixel_array)[int(breakpoints[0]):int(breakpoints[1]), :, :]
        else:
            image_array = (image.pixel_array)[int(breakpoints[0]):, :, :]
            mask_1_array = (mask_1.pixel_array)[int(breakpoints[0]):, :, :]
            mask_2_array = (mask_2.pixel_array)[int(breakpoints[0]):, :, :]
    else:
        image_array = (image.pixel_array)
        mask_1_array = (mask_1.pixel_array)
        mask_2_array = (mask_2.pixel_array)

    return [image_array, mask_1_array, mask_2_array]

def apply_breakpoints(image_array, breakpoints, dir_to_save):
    if 'Plakk' in dir_to_save:
        image_array = (image_array)[int(breakpoints[0]):int(breakpoints[1]), :, :]
    else:
        image_array = (image_array)[int(breakpoints[0]):, :, :]
    return image_array

def osszefuz(x):
    x.append(x[0][:7])
    x.append(x[0][16:18])
    return x

def save_all_patch_for_image_mask_pair(ID_tuple,
                                       dir_to_save,
                                       patch_shape,
                                       stride_size,
                                       train_val_test,
                                       basic_path,
                                       basic_path2,
                                       truncate=True,
                                       plaques_only=False
                                       ):
    """Saves all 3 dimensional patches

    Arguments
    -----------
    ID_tuple

    dir_to_save : string
        Folder to save the patches.

    train_val_test : string
        possible values: 'train', 'val', or 'test'.
        Subfolders for dataset split.

    Outputs
    -----------
    None
    """

    image_array, mask_1, mask_2 = get_image_mask_for_ID_tuple(ID_tuple, basic_path, basic_path2, dir_to_save,plaques_only)
    mask_1 = np.where(mask_1 > 0, 1, 0)
    mask_2 = np.where(mask_2 > 0, 1, 0)
    dif_array = np.where(mask_2 - mask_1 == 1, 1, 0)
    # dif_array = get_all_plaques(dif_array)
    # Count saved patches
    total_patches_for_ID = 0
    image_to_pad = image_array
    nodule_to_pad = mask_1
    lung_to_pad = mask_2
    dif_to_pad = dif_array
    # Order of the saved patch, appended to filename
    patch_count = 0

    # Shape of original images
    size_X = image_to_pad.shape[2]
    size_Y = image_to_pad.shape[1]
    size_Z = image_to_pad.shape[0]

    image_to_block = np.zeros((size_Z + patch_shape[2],
                               size_Y,
                               size_X))
    image_to_block[:size_Z, :size_Y, :size_X] = image_to_pad

    nodule_to_block = np.zeros((size_Z + patch_shape[2],
                                size_Y,
                                size_X))
    nodule_to_block[:size_Z, :size_Y, :size_X] = nodule_to_pad

    lung_to_block = np.zeros((size_Z + patch_shape[2],
                              size_Y,
                              size_X))
    lung_to_block[:size_Z, :size_Y, :size_X] = lung_to_pad

    dif_to_block = np.zeros((size_Z + patch_shape[2],
                             size_Y,
                             size_X))
    dif_to_block[:size_Z, :size_Y, :size_X] = dif_to_pad

    # patch_shape is originally in order XYZ, however for view as window we need it in ZYX
    patch_shape_ZYX = [patch_shape[2], patch_shape[1], patch_shape[0]]
    # Same as patch_shape
    stride_size_ZYX = [stride_size[2], stride_size[1], stride_size[0]]
    # Create blocks of the numpy arrays using view_as_blocks from skimage.util
    image_patches = view_as_windows(image_to_block, window_shape=patch_shape_ZYX, step=stride_size_ZYX)
    nodule_patches = view_as_windows(nodule_to_block, window_shape=patch_shape_ZYX, step=stride_size_ZYX)
    lung_patches = view_as_windows(lung_to_block, window_shape=patch_shape_ZYX, step=stride_size_ZYX)
    dif_patches = view_as_windows(dif_to_block, window_shape=patch_shape_ZYX, step=stride_size_ZYX)

    # view_as_windows creates 6 dimensional numpy arrays:
    # first 3 dimensions encode the position of the patch, last 3 dimensions encode patch shape.
    # We will iterate through the number of patches
    number_of_patches = image_patches.shape[0] * image_patches.shape[1] * image_patches.shape[2]
    for counter in range(number_of_patches):
        patch_coor_1 = int(counter // (image_patches.shape[1] * image_patches.shape[2]))

        patch_coor_2 = int(((counter - patch_coor_1 * image_patches.shape[1] * image_patches.shape[2])
                            // image_patches.shape[2]))

        patch_coor_3 = int(counter - patch_coor_1 * image_patches.shape[1] * image_patches.shape[2]
                           - patch_coor_2 * image_patches.shape[2])

        image_patch = image_patches[patch_coor_1][patch_coor_2][patch_coor_3]
        nodule_patch = nodule_patches[patch_coor_1][patch_coor_2][patch_coor_3]
        lung_patch = lung_patches[patch_coor_1][patch_coor_2][patch_coor_3]
        dif_patch = dif_patches[patch_coor_1][patch_coor_2][patch_coor_3]
        if truncate == True:
            # vedd ki a 16:48
            image_patch = image_patch[:, 16:48, 16:48]
            nodule_patch = nodule_patch[:, 16:48, 16:48]
            lung_patch = lung_patch[:, 16:48, 16:48]
            dif_patch = dif_patch[:, 16:48, 16:48]
        if plaques_only and np.count_nonzero(dif_patch) > 0:
            image_patch_file = os.path.join(dir_to_save, train_val_test, "images",
                                            ID_tuple[0][:-13] + str(patch_count) + '.npy')
            np.save(image_patch_file, image_patch.astype(np.float32))

            nodule_patch_file = os.path.join(dir_to_save, train_val_test, "masks_1",
                                             ID_tuple[0][:-13] + str(patch_count) + '.npy')

            np.save(nodule_patch_file, nodule_patch.astype(np.uint8))

            lung_patch_file = os.path.join(dir_to_save, train_val_test, "masks_2",
                                           ID_tuple[0][:-13] + str(patch_count) + '.npy')

            np.save(lung_patch_file, lung_patch.astype(np.uint8))

            plaque_patch_file = os.path.join(dir_to_save, train_val_test, "plaques",
                                             ID_tuple[0][:-13] + str(patch_count) + '.npy')

            np.save(plaque_patch_file, dif_patch.astype(np.uint8))
            patch_count += 1
            total_patches_for_ID += 1

        if plaques_only == False:
            image_patch_file = os.path.join(dir_to_save, train_val_test, "images",
                                            ID_tuple[0][:-13] + str(patch_count) + '.npy')
            np.save(image_patch_file, image_patch.astype(np.float32))

            nodule_patch_file = os.path.join(dir_to_save, train_val_test, "masks_1",
                                             ID_tuple[0][:-13] + str(patch_count) + '.npy')

            np.save(nodule_patch_file, nodule_patch.astype(np.uint8))

            lung_patch_file = os.path.join(dir_to_save, train_val_test, "masks_2",
                                           ID_tuple[0][:-13] + str(patch_count) + '.npy')

            np.save(lung_patch_file, lung_patch.astype(np.uint8))

            plaque_patch_file = os.path.join(dir_to_save, train_val_test, "plaques",
                                             ID_tuple[0][:-13] + str(patch_count) + '.npy')

            np.save(plaque_patch_file, dif_patch.astype(np.uint8))
            patch_count += 1
            total_patches_for_ID += 1

def save_all_patch(ID_tuple_list,
                   dir_to_save,
                   patch_shape,
                   stride_size,
                   basic_path,
                   basic_path2,
                   truncate=True,
                   train_val_test_split=[0.8, 0.2, 0.0],
                   plaques_only=True,
                   val_patients=None,
                   test_patients=None):
    # First delete the directory, where we would like to save the patches to avoid naming collisions
    if os.path.exists(dir_to_save):
        shutil.rmtree(dir_to_save)
    # Create parent directory
    os.mkdir(dir_to_save)
    os.mkdir(os.path.join(dir_to_save, "file_logs"))
    # Then create folders train, test, val  containing images and masks folders.
    train_dir, test_dir, val_dir = [os.path.join(dir_to_save, "train"),
                                    os.path.join(dir_to_save, "test"),
                                    os.path.join(dir_to_save, "val")]
    # Create train_dir
    os.mkdir(train_dir)
    os.mkdir(os.path.join(train_dir, "images"))
    os.mkdir(os.path.join(train_dir, "plaques"))
    os.mkdir(os.path.join(train_dir, "masks_1"))
    os.mkdir(os.path.join(train_dir, "masks_2"))

    # Create test_dir
    os.mkdir(test_dir)
    os.mkdir(os.path.join(test_dir, "images"))
    os.mkdir(os.path.join(test_dir, "plaques"))
    os.mkdir(os.path.join(test_dir, "masks_1"))
    os.mkdir(os.path.join(test_dir, "masks_2"))

    # Create val_dir
    os.mkdir(val_dir)
    os.mkdir(os.path.join(val_dir, "images"))
    os.mkdir(os.path.join(val_dir, "plaques"))
    os.mkdir(os.path.join(val_dir, "masks_1"))
    os.mkdir(os.path.join(val_dir, "masks_2"))

    total_number_of_IDs = len(ID_tuple_list)
    # Create thresholds for train-val-test split
    number_of_IDs_train = int(train_val_test_split[0] * total_number_of_IDs)
    number_of_IDs_val = int(train_val_test_split[1] * total_number_of_IDs)
    number_of_IDs_test = int(train_val_test_split[2] * total_number_of_IDs)

    patients = []
    for counter, ID_tuple in tqdm(enumerate(ID_tuple_list)):
        patients.append(ID_tuple[3])
    patients = np.unique(patients)

    random.seed(42)
    if test_patients==None:
        test_patients = random.sample(set(patients).difference(set(val_patients)),
                                  int(len(patients) * train_val_test_split[2]))
    if val_patients==None:
        val_patients = random.sample(set(patients), int(len(patients) * train_val_test_split[1]))

    # Save images to the corresponding subfolders using the functions above.
    for counter, ID_tuple in tqdm(enumerate(ID_tuple_list)):
        if ID_tuple[3].rstrip('_') in val_patients:
            train_val_test = "val"
        elif ID_tuple[3].rstrip('_') in test_patients:
            train_val_test = "test"
        else:
            train_val_test = "train"
        save_all_patch_for_image_mask_pair(ID_tuple,
                                           patch_shape=patch_shape,
                                           stride_size=stride_size,
                                           truncate=truncate,
                                           dir_to_save=dir_to_save,
                                           train_val_test=train_val_test,
                                           basic_path=basic_path,
                                           basic_path2=basic_path2,
                                           plaques_only=plaques_only)

    return val_patients

epsilon = 1e-5
smooth = 1

def dsc(y_true, y_pred, args):
    smooth = args.smooth
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
    return score

def dice_loss(y_true, y_pred, args):
    loss = 1 - dsc(y_true, y_pred, args)
    return loss

def tp(y_true, y_pred, args):
    smooth = args.smooth
    y_pred_pos = K.round(K.clip(y_pred, 0, 1))
    y_pos = K.round(K.clip(y_true, 0, 1))
    tp = (K.sum(y_pos * y_pred_pos) + smooth) / (K.sum(y_pos) + smooth)
    return tp

def tn(y_true, y_pred, args):
    smooth = args.smooth
    y_pred_pos = K.round(K.clip(y_pred, 0, 1))
    y_pred_neg = 1 - y_pred_pos
    y_pos = K.round(K.clip(y_true, 0, 1))
    y_neg = 1 - y_pos
    tn = (K.sum(y_neg * y_pred_neg) + smooth) / (K.sum(y_neg) + smooth)
    return tn

def fp(y_true, y_pred, args):
    smooth = args.smooth
    y_pred_pos = K.round(K.clip(y_pred, 0, 1))
    y_pred_neg = 1 - y_pred_pos
    y_pos = K.round(K.clip(y_true, 0, 1))
    y_neg = 1 - y_pos
    fp = (K.sum(y_neg * y_pred_pos) + smooth) / (K.sum(y_pred) + smooth)
    return fp

def fn(y_true, y_pred, args):
    smooth = args.smooth
    y_pred_pos = K.round(K.clip(y_pred, 0, 1))
    y_pred_neg = 1 - y_pred_pos
    y_pos = K.round(K.clip(y_true, 0, 1))
    y_neg = 1 - y_pos
    fn = (K.sum(y_pos * y_pred_neg) + smooth) / (K.sum(y_pred_neg) + smooth)
    return fn

def tversky(y_true, y_pred, args):
    smooth = args.smooth
    y_true_pos = K.flatten(y_true)
    y_pred_pos = K.flatten(y_pred)
    true_pos = K.sum(y_true_pos * y_pred_pos)
    false_neg = K.sum(y_true_pos * (1 - y_pred_pos))
    false_pos = K.sum((1 - y_true_pos) * y_pred_pos)
    alpha = args.alpha
    return (true_pos + smooth) / (true_pos + alpha * false_neg + (1 - alpha) * false_pos + smooth)

def tversky_loss(y_true, y_pred, args):
    return 1 - tversky(y_true, y_pred, args)

def focal_tversky(y_true, y_pred, args):
    gamma = args.gamma = 0.75
    pt_1 = tversky(y_true, y_pred, args)
    return K.pow((1 - pt_1), gamma)

def multiloss(y_true, y_pred, args):
    if args.loss_type == 1:
        return focal_tversky(y_true, y_pred, args)
    else:
        return dice_loss(y_true, y_pred, args)

# Visaulization functions

def gray_to_colored(im):
    colored = np.repeat(np.expand_dims(im, axis=-1), 3, axis=-1).astype(float)
    colored = 1 * (colored - np.amin(colored)) / (np.amax(colored) - np.amin(colored))
    return colored

def superimpose_mask(image_array, mask_array, opacity=0.8):
    superimposed = gray_to_colored(image_array)
    reds = np.zeros(mask_array.shape + (3,)).astype(np.bool)
    reds[:, :, 0] = mask_array == 1
    superimposed[reds] = opacity * 1 + (1 - opacity) * superimposed[reds]
    return superimposed

def visualize_slice_mask_pair(image_array, mask_1_array, mask_2_array, plaque_array, opacity=0.8, name=""):
    ax, plots = plt.subplots(2, 4, figsize=(25, 10))
    ax.suptitle(name)
    plots[0, 0].axis('off')
    plots[0, 0].imshow(mask_2_array - mask_1_array, cmap=plt.cm.bone)
    plots[0, 1].axis('off')
    plots[0, 1].imshow(mask_1_array, cmap=plt.cm.bone)
    plots[0, 2].axis('off')
    plots[0, 2].imshow(mask_2_array, cmap=plt.cm.bone)
    plots[0, 3].axis('off')
    plots[0, 3].imshow(plaque_array, cmap=plt.cm.bone)

    plots[1, 0].axis('off')
    plots[1, 0].imshow(superimpose_mask(image_array, mask_2_array - mask_1_array, opacity=opacity))
    plots[1, 1].axis('off')
    plots[1, 1].imshow(superimpose_mask(image_array, mask_1_array, opacity=opacity))
    plots[1, 2].axis('off')
    plots[1, 2].imshow(superimpose_mask(image_array, mask_2_array, opacity=opacity))
    plots[1, 3].axis('off')
    plots[1, 3].imshow(superimpose_mask(image_array, plaque_array, opacity=opacity))
    plt.show()

def expand_as_3d(tensor, rep, name):
    my_repeat = Lambda(lambda x, repnum: K.repeat_elements(x, repnum, axis=4), arguments={'repnum': rep},
                       name='psi_up' + name)(tensor)
    return my_repeat

def AttnGatingBlock3D(x, g, inter_shape, name):
    '''
    Analogous implementation of the 3D attention gate used in the Attention U-Net 3D.
    '''
    shape_x = K.int_shape(x)  # 32
    shape_g = K.int_shape(g)  # 16

    theta_x = Conv3D(inter_shape, (2, 2, 2), strides=(2, 2, 2), padding='same', name='xl' + name)(x)  # 16
    shape_theta_x = K.int_shape(theta_x)

    phi_g = Conv3D(inter_shape, (1, 1, 1), padding='same')(g)
    upsample_g = Conv3DTranspose(inter_shape, (3, 3, 3), strides=(
    shape_theta_x[1] // shape_g[1], shape_theta_x[2] // shape_g[2], shape_theta_x[3] // shape_g[3]), padding='same',
                                 name='g_up' + name)(phi_g)  # 16

    concat_xg = add([upsample_g, theta_x])
    act_xg = Activation('relu')(concat_xg)
    psi = Conv3D(1, (1, 1, 1), padding='same', name='psi' + name)(act_xg)
    sigmoid_xg = Activation('sigmoid')(psi)
    shape_sigmoid = K.int_shape(sigmoid_xg)
    upsample_psi = UpSampling3D(
        size=(shape_x[1] // shape_sigmoid[1], shape_x[2] // shape_sigmoid[2], shape_x[3] // shape_sigmoid[3]))(
        sigmoid_xg)  # 32

    upsample_psi = expand_as_3d(upsample_psi, shape_x[4], name)
    y = multiply([upsample_psi, x], name='q_attn' + name)

    result = Conv3D(shape_x[4], (1, 1, 1), padding='same', name='q_attn_conv' + name)(y)
    result_bn = BatchNormalization(name='q_attn_bn' + name)(result)
    return result_bn

def UnetConv3D(input, outdim, is_batchnorm, name):
    '''
    Analogous implementation of the pair of convolutional layers used by the U-Net 3D.
    '''
    x = Conv3D(outdim, (3, 3, 3), strides=(1, 1, 1), kernel_initializer="glorot_normal", padding="same",
               name=name + '_1')(input)
    if is_batchnorm:
        x = BatchNormalization(name=name + '_1_bn')(x)
    x = Activation('relu', name=name + '_1_act')(x)

    x = Conv3D(outdim, (3, 3, 3), strides=(1, 1, 1), kernel_initializer="glorot_normal", padding="same",
               name=name + '_2')(x)
    if is_batchnorm:
        x = BatchNormalization(name=name + '_2_bn')(x)
    x = Activation('relu', name=name + '_2_act')(x)
    return x

def UnetGatingSignal3D(input, is_batchnorm, name):
    '''
    Implementation of the gating signal appearing in the upsampling branch of the Attention U-Net 3D:
    simply a 1x1 convolution followed by batch normalization and ReLU.
    '''
    shape = K.int_shape(input)
    x = Conv3D(shape[4] * 1, (1, 1, 1), strides=(1, 1, 1), padding="same", kernel_initializer="glorot_normal",
               name=name + '_conv')(input)
    if is_batchnorm:
        x = BatchNormalization(name=name + '_bn')(x)
    x = Activation('relu', name=name + '_act')(x)
    return x

def tiny_attn_unet3D(opt, input_size, args):
    '''
    Analogue of the above defined attn_unet3D with less layers, resulting in a smaller U shape.
    '''
    inputs = Input(shape=input_size)
    conv1 = UnetConv3D(inputs, 8*args.kernel_power, is_batchnorm=True, name='conv1')
    pool1 = MaxPooling3D(pool_size=(2, 2, 2))(conv1)
    pool1 = Dropout(args.dropout)(pool1)

    conv2 = UnetConv3D(pool1, 8*args.kernel_power, is_batchnorm=True, name='conv2')
    pool2 = MaxPooling3D(pool_size=(2, 2, 2))(conv2)
    pool2 = Dropout(args.dropout)(pool2)

    center = UnetConv3D(pool2, 16*args.kernel_power, is_batchnorm=True, name='center')

    g3 = UnetGatingSignal3D(center, is_batchnorm=True, name='g3')
    attn3 = AttnGatingBlock3D(conv2, g3, 8*args.kernel_power, '_3')
    up3 = concatenate([Conv3DTranspose(8*args.kernel_power, (3, 3, 3), strides=(2, 2, 2), padding='same', activation='relu',
                                       kernel_initializer="glorot_normal")(center), attn3], name='up3')

    up4 = concatenate([Conv3DTranspose(8*args.kernel_power, (3, 3, 3), strides=(2, 2, 2), padding='same', activation='relu',
                                       kernel_initializer="glorot_normal")(up3), conv1], name='up4')

    mask_1 = Conv3D(1, (1, 1, 1), activation='sigmoid', name='mask_1')(up4)
    mask_2 = Conv3D(1, (1, 1, 1), activation='sigmoid', name='mask_2')(up4)
    dif = Conv3D(1, (1, 1, 1), activation='sigmoid', name='dif')(up4)

    model = Model(inputs=[inputs], outputs=[mask_1, mask_2, dif])
    model.compile(optimizer=opt,
                  loss=[wrapped_partial(dice_loss, args=args), wrapped_partial(dice_loss, args=args),
                        wrapped_partial(multiloss, args=args)],
                  loss_weights=[0.1, 0.1, 0.8],
                  metrics=[[wrapped_partial(dsc, args=args)], [wrapped_partial(dsc, args=args)],
                           [wrapped_partial(dsc, args=args), wrapped_partial(tp, args=args),
                            wrapped_partial(tn, args=args)]])
    return model

def full_ct_model_evaluation(image, model, z_stride, which_prediction):
    # Shape of original images
    size_X = image.shape[2]
    size_Y = image.shape[1]
    size_Z = image.shape[0]

    image_paded = np.zeros((size_Z + 24,
                            size_Y,
                            size_X))

    image_paded[:size_Z, :size_Y, :size_X] = image / 512

    prediction_array = np.zeros((size_Z + 24,
                                 size_Y,
                                 size_X))

    coverage_array = np.zeros((size_Z + 24,
                               size_Y,
                               size_X))

    # Containers for batch predictions
    patch_boundaries_list = []
    counter = 0

    # Stride along Z axis:
    for z_start in range(0, prediction_array.shape[2], z_stride):
        z_end = z_start + 24
        if (np.count_nonzero(image[z_start:z_end, :, :]) > 1):
            patch_boundaries_list.append([z_start, z_end])
    for patch_index in range(0, len(patch_boundaries_list)):
        # patch_boundaries in current batch
        temporal_boundaries = patch_boundaries_list[patch_index]
        temp_patches = []
        # Extracting patches for prediction
        current_patch = image_paded[temporal_boundaries[0]:temporal_boundaries[1],
                        16:48,
                        16:48]
        current_patch = np.expand_dims(current_patch, axis=0)
        # Updating prediction_array and coverage_array
        predicted_patch = model.predict(np.expand_dims(current_patch, axis=-1))

        # 0 belső maszk 1 külső maszk 2 differencia

        prediction = predicted_patch[which_prediction]

        prediction = np.reshape(prediction, [24, 32, 32])

        prediction_array[temporal_boundaries[0]:temporal_boundaries[1],
        16:48,
        16:48] += prediction

        # print(prediction_array[32, 32, 32])

        coverage_array[temporal_boundaries[0]:temporal_boundaries[1],
        :,
        :] += 1

    coverage_array = np.maximum(coverage_array, np.ones(coverage_array.shape))
    # Taking the average prediction value for the pixels
    prediction_array = np.divide(prediction_array, coverage_array)
    # print(prediction_array[32,32,32])
    # Removing the prediction values outside of the CT
    prediction_array = prediction_array[0:size_Z, 0:size_Y, 0:size_X]

    # The average prediction value is continuous between 0 and 1,
    # so for the segmentation we have to threshold it
    prediction_array = (prediction_array > 1 / 2) * 1

    return prediction_array

def mask_from_dicom(contour_file_name):
    ds = pydicom.dcmread(contour_file_name)
    pixels = np.array(ds.pixel_array)
    pixels = np.where(pixels > 0, 1, 0)
    return pixels

def read_in(file_path, patient, vessel):
    ds = pydicom.dcmread(os.path.join(file_path, patient + '_' + vessel + '_NoContour.dcm'))
    pixelSpacing = ds.PixelSpacing
    sliceSpacing = ds.SpacingBetweenSlices
    Image = np.array(ds.pixel_array)
    Mask_1 = get_all_plaques(mask_from_dicom(os.path.join(file_path, patient + '_' + vessel + '_Contour1.dcm')))
    Mask_2 = get_all_plaques(mask_from_dicom(os.path.join(file_path, patient + '_' + vessel + '_Contour2.dcm')))
    Diff = Mask_2 - Mask_1
    originalds = pydicom.dcmread(os.path.join(file_path, patient, vessel, 'volume.dcm'))
    oripixelSpacing = originalds.PixelSpacing
    orisliceSpacing = originalds.SpacingBetweenSlices
    return Image, Diff, Mask_1, Mask_2, pixelSpacing, sliceSpacing, oripixelSpacing, orisliceSpacing

def xml_parse(root):
    itemlist = []
    for child in root:
        itemlist.append(np.array(child[1].text.split(' ')).reshape(4, 4))
        return itemlist

def xml_load(file):
    root = ET.parse(file).getroot()
    return root

def xml_parse_path(root):
    itemlist = []
    for child in root:
        itemlist.append(np.array(child))
        return itemlist

def dice_coefficient(mask1, mask2):
    # print(np.sum(mask1))
    return ((2.0 * np.sum(mask1 * mask2)) / (np.sum(mask1) + np.sum(mask2)))

def filter_supp2(filename):
    if '.dcm' in filename:
        return True
    else:
        return False

def get_pairs2(vesel_list):
    return_list = []
    for ID in vesel_list:
        if ID.endswith('NoContour.dcm'):
            return_list.append([ID[:-14]])
    return return_list

class CyclicLR(Callback):
    """Copied from https://github.com/bckenstler/CLR/
    This callback implements a cyclical learning rate policy (CLR).
    The method cycles the learning rate between two boundaries with
    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).
    The amplitude of the cycle can be scaled on a per-iteration or
    per-cycle basis.
    This class has three built-in policies, as put forth in the paper.
    "triangular":
        A basic triangular cycle w/ no amplitude scaling.
    "triangular2":
        A basic triangular cycle that scales initial amplitude by half each cycle.
    "exp_range":
        A cycle that scales initial amplitude by gamma**(cycle iterations) at each
        cycle iteration.
    For more detail, please see paper.

    # Example
        ```python
            clr = CyclicLR(base_lr=0.001, max_lr=0.006,
                                step_size=2000., mode='triangular')
            model.fit(X_train, Y_train, callbacks=[clr])
        ```

    Class also supports custom scaling functions:
        ```python
            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))
            clr = CyclicLR(base_lr=0.001, max_lr=0.006,
                                step_size=2000., scale_fn=clr_fn,
                                scale_mode='cycle')
            model.fit(X_train, Y_train, callbacks=[clr])
        ```
    # Arguments
        base_lr: initial learning rate which is the
            lower boundary in the cycle.
        max_lr: upper boundary in the cycle. Functionally,
            it defines the cycle amplitude (max_lr - base_lr).
            The lr at any cycle is the sum of base_lr
            and some scaling of the amplitude; therefore
            max_lr may not actually be reached depending on
            scaling function.
        step_size: number of training iterations per
            half cycle. Authors suggest setting step_size
            2-8 x training iterations in epoch.
        mode: one of {triangular, triangular2, exp_range}.
            Default 'triangular'.
            Values correspond to policies detailed above.
            If scale_fn is not None, this argument is ignored.
        gamma: constant in 'exp_range' scaling function:
            gamma**(cycle iterations)
        scale_fn: Custom scaling policy defined by a single
            argument lambda function, where
            0 <= scale_fn(x) <= 1 for all x >= 0.
            mode paramater is ignored
        scale_mode: {'cycle', 'iterations'}.
            Defines whether scale_fn is evaluated on
            cycle number or cycle iterations (training
            iterations since start of cycle). Default is 'cycle'.
    """

    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
                 gamma=1., scale_fn=None, scale_mode='cycle'):
        super(CyclicLR, self).__init__()

        self.base_lr = base_lr
        self.max_lr = max_lr
        self.step_size = step_size
        self.mode = mode
        self.gamma = gamma
        if scale_fn == None:
            if self.mode == 'triangular':
                self.scale_fn = lambda x: 1.
                self.scale_mode = 'cycle'
            elif self.mode == 'triangular2':
                self.scale_fn = lambda x: 1/(2.**(x-1))
                self.scale_mode = 'cycle'
            elif self.mode == 'exp_range':
                self.scale_fn = lambda x: gamma**(x)
                self.scale_mode = 'iterations'
        else:
            self.scale_fn = scale_fn
            self.scale_mode = scale_mode
        self.clr_iterations = 0.
        self.trn_iterations = 0.
        self.history = {}

        self._reset()

    def _reset(self, new_base_lr=None, new_max_lr=None,
               new_step_size=None):
        """Resets cycle iterations.
        Optional boundary/step size adjustment.
        """
        if new_base_lr != None:
            self.base_lr = new_base_lr
        if new_max_lr != None:
            self.max_lr = new_max_lr
        if new_step_size != None:
            self.step_size = new_step_size
        self.clr_iterations = 0.

    def clr(self):
        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
        if self.scale_mode == 'cycle':
            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)
        else:
            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)

    def on_train_begin(self, logs={}):
        logs = logs or {}

        if self.clr_iterations == 0:
            K.set_value(self.model.optimizer.lr, self.base_lr)
        else:
            K.set_value(self.model.optimizer.lr, self.clr())

    def on_batch_end(self, epoch, logs=None):

        logs = logs or {}
        self.trn_iterations += 1
        self.clr_iterations += 1

        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))
        self.history.setdefault('iterations', []).append(self.trn_iterations)

        for k, v in logs.items():
            self.history.setdefault(k, []).append(v)

        K.set_value(self.model.optimizer.lr, self.clr())


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Mar 12 14:43:25 2019

@author: riccardo
"""

import funcs as H
# import Helper2 as H2
import numpy as np
import torch
import torchvision
from MUNet import MUnet, PARAMS
import time
import tensorboardX as tbX
import os
from torch.optim import Adam

def MonoLoss(Ytrue,Ypred):
    return -torch.div( torch.sum(torch.mul(torch.mul(Ytrue,Ypred),2)), torch.sum(torch.mul(Ypred,Ypred)) + torch.sum(torch.mul(Ytrue,Ytrue)) )

def CateLoss(Ytrue,Ypred):
    return -torch.div( torch.mul(2, torch.sum( torch.mul(Ytrue,Ypred)  )  ), torch.sum(torch.mul(Ypred,Ypred)) + torch.sum(torch.mul(Ytrue,Ytrue)) )

class Loss():
    def __init__(self,categories):
        self.categories=categories
    def __call__(self,Ytrue,Ypred):

        Mask=Ytrue.narrow(1,0,1)
        PredMask=Ypred[0]
        Labels=Ytrue.narrow(1,1,self.categories)
        LabelsPred=Ypred[1]

        return MonoLoss(Mask,PredMask) + CateLoss(Labels,LabelsPred)

folds=5
decay=0.00
maxtime=12 *60*60 # hours

Network=MUnet

PAR=PARAMS

name='RunName'
SemW=0
DiceW=1
BBalance=False
crop=False
CCEW=0
Lossf=Loss(PAR['Categories'])

folder='dataset/path'
saveprogress='path/save.tar'

CurrentFold=1

while CurrentFold <= folds:
    scaler = H.Rescale(1.01,0.95,0.5)
    rotator = H.RotCoronal(5,0.5)
    normalizator = H.Normalizer()#'/mnt/90FACB14FACAF60E/U-Net/Data/CRL0616_01_DATA_ANALYSIS')
    tensorize = H.ToTensor()
    transforms = torchvision.transforms.Compose([rotator, scaler, normalizator, tensorize])
    transforms_eval = torchvision.transforms.Compose([normalizator, tensorize])
    training_set=H.RodentDatasetCR(folder,fold=CurrentFold,FoldType='Train',transform=transforms)
    test_set=H.RodentDatasetCR(folder,fold=CurrentFold,FoldType='Test',transform=transforms_eval)

    dataloader = torch.utils.data.DataLoader(training_set, batch_size=1, shuffle=True)
    testloader = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=True)

    run_name='Fold'+str(CurrentFold)+name
    runfolder='runs/'+run_name
    savefile='saves/'+run_name+'/bestbyloss.tar'
    # 'Fold'+str(CurrentFold)+'_3D_Vanilla_Skip'

    #Lossf=Loss(BoundaryWeight,DiceWeight,W)
    step=0
    os.mkdir('saves/'+run_name)

    UN=Network().cuda()
    TotalTime=0
    epoch=0
    BestStep=0
    bestlossstep=0
    bestloss=0
    bestdice=0
    optimizer=Adam(UN.parameters(),weight_decay=decay)
    savedict={'dices':[],
              'losses':[],
              'mean_dices':[],
              'mean_loss':[],
              'step':[]}

    UN.train()

    writer=tbX.SummaryWriter(runfolder)

    time_check=time.time()

    while TotalTime < maxtime and (step - BestStep < 3000):
        dice=[]
        print('Epoch '+str(epoch))

        for i, sample in enumerate(dataloader):
            step+=1
            torch.cuda.empty_cache()
            optimizer.zero_grad()

            Labels = UN(sample['MRI'])
            loss = Lossf(sample['labels'],Labels)
            loss.backward()
            optimizer.step()

            dice.append(H.DiceScores(sample['labels'],Labels))

        print('Training set mean dices: ')
        print(np.mean(dice))

        epoch+=1
        UN.eval()
        dices=[]
        losses=[]
        TotalTime+=time.time()-time_check
        time_check=time.time()

        for i, sample in enumerate(testloader):
            with torch.no_grad():
                torch.cuda.empty_cache()
                result=UN(sample['MRI'])
                dices.append(H.DiceScores(sample['labels'],result))
                losses.append( float(Lossf(sample['labels'],result)))
                specify='_'+sample['timepoint'][0]+'_'+sample['rodent_numer'][0]

        inftime=(time.time()-time_check)/len(testloader)
        print('Inference time: '+str(inftime)+' per volume')

        dices=np.array(dices)
        savedict['dices'].append(dices)
        losses=np.array(losses)
        candidate_loss=np.mean(losses)
        dices=np.mean(dices,axis=0)
        print('Dev set dices:')
        print(dices)
        savedict['losses'].append(losses)
        savedict['mean_dices'].append(dices)
        savedict['mean_loss'].append(candidate_loss)
        savedict['step'].append(step)

        writer.add_scalar('Val Loss',float(candidate_loss), global_step=step)
        writer.add_scalar('Val Dice Mask',float(dices[0]), global_step=step)
        writer.add_scalar('Val Dice Cortex',float(dices[1]), global_step=step)
        writer.add_scalar('Val Dice Hippocampus',float(dices[2]), global_step=step)
        writer.add_scalar('Val Dice Ventricles',float(dices[3]), global_step=step)
        writer.add_scalar('Val Dice Striatum',float(dices[4]), global_step=step)
        writer.add_scalar('Val Mean Dice',float(np.mean(dices)),global_step=step)
        torch.save({
                'epoch': epoch,
                'model_state_dict': UN.state_dict(),
                'optimizer_state_dict': optimizer.state_dict(),
                'loss': loss,
                'step': step,
                'bestloss':bestloss,
                'CurrentFold':CurrentFold,
                'bestdice':bestdice,
                'TotalTime':TotalTime,
                'bestlossstep':bestlossstep,
                'savedict':savedict
                }, saveprogress)
        UN.train()

        if candidate_loss<bestloss:
            bestloss=candidate_loss
            bestlossstep=step
            BestStep=step
            bestdice_loss=dices
            torch.save({
                    'epoch': epoch,
                    'model_state_dict': UN.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                    'loss': loss,
                    'step': step,
                    'bestloss':bestloss,
                    'bestdice':bestdice,
                    'dice_scores':dices,
                    'CurrentFold':CurrentFold,
                    'bestlossstep':bestlossstep,
                    'TotalTime':TotalTime,
                    'savedict':savedict
                    }, savefile)

    writer.add_graph(UN, [sample['MRI']])
    writer.close()

    del UN
    torch.cuda.empty_cache()
    optimizer.zero_grad()
    del optimizer
    CurrentFold+=1

ext: py

max_stars_repo_name: Hierakonpolis/MU-Net

reason: import numpy


text:
import numpy as np
import math
import os

from .tflite import Model
from .tflite.BuiltinOperator import BuiltinOperator

# binary size for the TF-Lite Micro runtime, update this to match your setting
TFMicro_CodeSize = 96217

# Parse tflite to our model format
class TFMicroMemSimulator(object):
    def __init__(self, filepath):
        self.TFSize = 0
        self.Memory = {}

        # path to the tflite file
        self.filepath = filepath
        self.model = self.loadTFmodel(filepath)
        self.subgraph = self.model.Subgraphs(0)
        self.builtin_op_code = self.__build_str_map(BuiltinOperator())

    # public functions
    def loadTFmodel(self, filepath):
        buf = open(filepath, 'rb').read()
        self.TFSize = os.stat(filepath).st_size
        return Model.Model.GetRootAsModel(buf, 0)

    def getFlash(self):
        return self.TFSize + TFMicro_CodeSize

    # store the size of required memory of each data structure on microcontrollers, need to update/use another table if the platform (ISA) change
    SizeTable = {
        'CreateInPlaceSimpleMemoryAllocator': 24,
        'tensor': 64,
        'TfLiteAffineQuantization': 12,
        'IntBytes': 4,
        'FloatBytes': 4,
        'ArrayBytes': 4,
        'Node': 40,
        # OpData
        'CONV_2D': 24,
        'ADD': 4,
        'DEPTHWISE_CONV_2D': 28,
        'AVERAGE_POOL_2D': 40,
        'SOFTMAX': 4,
        'PAD': 0,
        'FULLY_CONNECTED': 12,
        'RESHAPE': 36,
        # This is just temporary memory, which does not count for peak memory
        'allocation_info': 20

    }

    def simulateMemoryAllocation(self):
        self.Memory['CreateInPlaceSimpleMemoryAllocator'] = self.SizeTable['CreateInPlaceSimpleMemoryAllocator']
        self.Memory['tensors'] = self.SizeTable['tensor'] * self.subgraph.TensorsLength()
        runtimeTensor = 0
        for i in range(self.subgraph.TensorsLength()):
            tensor = self.subgraph.Tensors(i)
            q = tensor.Quantization()
            # for micro_allocator to copy the quantization information from the serialized data.

            if q.ScaleLength() > 0:
                runtimeTensor += self.SizeTable['TfLiteAffineQuantization']
                # zero_point
                runtimeTensor += self.SizeTable['IntBytes'] * q.ScaleLength() + self.SizeTable['ArrayBytes'] #Int array
                # scale
                runtimeTensor += self.SizeTable['FloatBytes'] * q.ScaleLength() + self.SizeTable['ArrayBytes'] #Float array

        self.Memory['runtimeTensor'] = runtimeTensor

        # AllocateNodeAndRegistrations
        # Node
        self.Memory['Node'] = self.SizeTable['Node'] * self.subgraph.OperatorsLength()
        # OpData
        OpData = 0
        for i in range(self.subgraph.OperatorsLength()):
            op = self.subgraph.Operators(i)
            op_code_str = self.__getOpCodeStr(op)
            # Need to include all possiblely used layer Opdata in the SizeTable
            alignment = 8 # Memory alignment
            OpSize = self.SizeTable[op_code_str]
            if (sum(self.Memory.values()) + OpData) % alignment :
                OpData += math.ceil(OpSize / alignment) * alignment
            else:
                OpData += OpSize

        self.Memory['OpData'] = OpData
        # FinishTensorAllocation
        # allocation_info
        self.Memory['allocation_info'] = self.SizeTable['allocation_info'] * self.subgraph.TensorsLength()

        # Activation
        layerAc = []
        layerOutIdx = []
        operators_len = self.subgraph.OperatorsLength()
        for i in range(operators_len):
            op = self.subgraph.Operators(i)
            op_code_str = self.__getOpCodeStr(op)
            input_tensors = self.__get_input_tensors(op)
            output_tensors = self.__get_output_tensors(op)
            act = 0
            input_size = input_tensors[0].tensor.ShapeAsNumpy().prod()
            output_size = output_tensors[0].tensor.ShapeAsNumpy().prod()
            act += input_size + output_size

            # residual, has two inputs and need to trace back
            output_idx = output_tensors[0].tensor_idx
            layerOutIdx.append(output_idx)
            if(op_code_str == "ADD"):
                input2_size = input_tensors[1].tensor.ShapeAsNumpy().prod()
                input2_idx = input_tensors[1].tensor_idx
                act += input2_size
                for j in range(i-1, 0, -1):
                    # find the layer of which output is the second input
                    if(input2_idx == layerOutIdx[j]):
                        break
                    layerAc[j] += input2_size

            layerAc.append(act)

        self.Memory['Activation'] = max(layerAc)

    def getActivationMemory(self):
        return sum(self.Memory.values())

    def __getOpCodeStr(self, op):
        op_code_list_idx = op.OpcodeIndex()
        op_code_id = self.model.OperatorCodes(
            op_code_list_idx).DeprecatedBuiltinCode()
        return self.builtin_op_code[op_code_id]

    def __build_str_map(self, obj):
        ret = {}
        for field_name in dir(obj):
            if not field_name.startswith("_"):
                field_value = getattr(obj, field_name)
                if isinstance(field_value, int):
                    ret[field_value] = field_name
        return ret

    def __getOpCodeStr(self, op):
        op_code_list_idx = op.OpcodeIndex()
        op_code_id = self.model.OperatorCodes(
            op_code_list_idx).DeprecatedBuiltinCode()
        return self.builtin_op_code[op_code_id]

    def __get_input_tensors(self, op):
        operator_inputs = op.InputsAsNumpy()
        return self.__get_tensors(operator_inputs)

    def __get_output_tensors(self, op):
        operator_outputs = op.OutputsAsNumpy()
        return self.__get_tensors(operator_outputs)

    def __get_tensors(self, tensors_idx_list):
        """Get tensor wrapper list from given TFLite tensor index list"""
        return_list = list()
        for tensor_idx in tensors_idx_list:
            if tensor_idx < 0:
                return_list.append(TensorWrapper(tensor_idx, 0, 0))
                continue

            tensor = self.subgraph.Tensors(tensor_idx)
            buffer_idx = tensor.Buffer()
            buffer = self.model.Buffers(buffer_idx)

            # Check if the tensors are quantized. Parse if yes.
            qnn_params = None
            tflite_qnn_params = tensor.Quantization()
            if tflite_qnn_params is not None:
                # TFLite supports both per-tensor and per-axis (aka channel) quantization.  For
                # per-tensor quantization, scale and zero points are scalar values.  For per-axis
                # quantization, scale and zero points for the weights are tensors (activations are
                # per-tensor quantized). However, the TFLite quantization spec puts restrictions on
                # zero points for per-axis quantization.  Specifically, the zero point is a tensor
                # but all values are 0. More information can be found here -
                # https://www.tensorflow.org/lite/performance/quantization_spec

                tflite_scale = tflite_qnn_params.ScaleAsNumpy()
                tflite_zero_point = tflite_qnn_params.ZeroPointAsNumpy()
                is_qnn_params_valid = True

                # Handle Per-axis and per-tensor cases
                if isinstance(tflite_scale, np.ndarray):
                    assert isinstance(tflite_zero_point, np.ndarray)

                    # Tensor - Per-axis quantization
                    if tflite_scale.size != 1 and tflite_zero_point.size != 1:
                        scale = tflite_scale
                        zero_point = tflite_zero_point
                        zero_point = int(zero_point[0])

                    # Scalar - Per-tensor quantization
                    elif tflite_scale.size == 1 and tflite_zero_point.size == 1:
                        scale = float(tflite_scale[0])
                        zero_point = int(tflite_zero_point[0])

                    else:
                        raise NotImplementedError(
                            "Quantized type {} (scale) and  {} (zero point) not supported".format(
                                type(tflite_scale), type(tflite_zero_point)
                            )
                        )
                elif tflite_scale == 0 and tflite_zero_point == 0:
                    # Handle corner case for ops like quantized reshape whose second operand (shape)
                    # has zero scale and zero zero point. This is not used.
                    is_qnn_params_valid = False
                else:
                    raise NotImplementedError(
                        "Quantized type {} not supported".format(
                            type(tflite_scale))
                    )

                # Check that the scale and zero points are valid.
                if is_qnn_params_valid:
                    qnn_params = dict()
                    qnn_params["scale"] = scale
                    qnn_params["zero_point"] = zero_point
            return_list.append(TensorWrapper(
                tensor_idx, tensor, buffer, qnn_params))
        return return_list

# from TVM
class TensorWrapper(object):
    """Tensor wrapper for TFLite Tensor"""

    def __init__(self, tensor_idx, tensor, buffer, qnn_params=None):
        self.tensor_idx = tensor_idx
        self.tensor = tensor
        self.buffer = buffer
        self.qnn_params = qnn_params

max_stars_repo_name: ali-hassan/mcunet


text:
"""
Starter code for SSH app

Misc installation:

Open-cv
https://www.pyimagesearch.com/2018/09/19/pip-install-opencv/

"""

# Determine if moving (if no change in sensor
# Look down until sensor starts to change

# Think about computer vision, what are some easy ones? Equipment challenge time?
import os
TEMP_WORKING_DIR = "/tmp/pycharm_project_465"
os.getcwd()
os.chdir(TEMP_WORKING_DIR)
os.system("ls")
import sys
sys.path = [TEMP_WORKING_DIR] + sys.path
import numpy as np
from app.src.rover import Rover
from time import time
from time import sleep

# rover = Rover()
#
# rover.forward()
# rover.stop()
# rover.forward_left()
#
# rover.read_distance()
# rover.stop()
#
# rover.backward()
# rover.read_distance()
# rover.stop()
# rover.speed = 200

# Move until an obstacle is close (better safe than sorry)
"""
Obstacle avoid algorithm
1. Capture starting sensor value (inches)
    a. If within 24 inches, rotate right for short time (move ~25 degrees)
    b. Continue until have moved in 360 or sensor returns >=36 inches
2. Drive forward
    a. If sensor shows > 36 feet increase speed every second
    b. If sensor shows < 36 feet set speed to 80
3. Avoid obstacles
    a. If sensor show <= 24 inches stop
    b. Rotate to right 25 degrees
    c. Continue until sensor returns >= 36 inches
"""

# Initialize rover
rover = Rover()

# Get initial distance

MIN_SENSOR_VALUE = 24
MID_SENSOR_VALUE = 36
MAX_ROVER_SPEED = 250
SPEED_INCREMENT = 10
STARTING_SPEED = 80

def rotate_25_degrees():
    rover.spin_right()
    sleep(.75)
    rover.stop()

"""
Simple obstacle avoidance
"""
while True:

    sensor_value = rover.read_distance()
    if sensor_value <= MIN_SENSOR_VALUE:
        rotate_25_degrees()
    elif sensor_value > MID_SENSOR_VALUE:
        rover.speed = min(MAX_ROVER_SPEED, rover.speed + SPEED_INCREMENT)
        rover.forward()
    else:
        rover.speed = STARTING_SPEED
        rover.forward()
    sleep(.5)

# Good start above
# By only being able to turn 25 degrees to the right, the robot can
#  get stuck in some situations

# Detecting the angle of the surface in front of the robot
# could inform which is best to turn.
# Pan to the left, if sensor reads further, then left is better
# Pan to the right, if sensor reads further, then right is better
# If straight on to an obstacle, then perhaps back up or turn 180 degrees

"""
More dynamic obstacle avoidance
"""

# Estimate angle of obstacle ahead
# Capture starting distance
# Pan left, measure
# Pan right, measure
# Determine of A-C and C-B angle
from math import acos, degrees

# https://stackoverflow.com/questions/18583214/calculate-angle-of-triangle-python
# https://www.mathsisfun.com/algebra/trig-solving-sss-triangles.html#:~:text=%22SSS%22%20is%20when%20we%20know,again%20to%20find%20another%20angle
# Tilt slightly to ensure accurate distance reading
# TODO Start here
# Function for determining angle of obstacle
side_c_midpoint = np.min([rover.read_distance() for _ in range(3)])
# Pan left

for _ in range(10):
    rover.pan_left()
    sleep(.15)
# Capture distance again

start = time()
side_a = np.min([rover.read_distance() for _ in range(3)])
print("%.2f" % (time() - start))
# Pan right
for _ in range(20):
    rover.pan_right()
    sleep(.15)
# Capture distance again
side_b = np.min([rover.read_distance() for _ in range(3)])
# Restraighten
for _ in range(10):
    rover.pan_left()
    sleep(.15)

"""
# Steps to compute angle of wall
"""
# Measure distance from sensor to point on wall directly in front (using heading) (side D)
# Tilt slightly up and down to get a good read and avoid obstructions

# Measure distance using to obstacle ahead
# Tilt up
# Measure
# Recenter vertical
# Tilt down
# Measure
# Re-center vertical

# If computing angle from heading to the left
# Pan left
# Tilt up
# Measure
# Recenter vertical
# Tilt down
# Measure
# Re-center vertical and horizontal

# If computing angle from heading to the right
# Pan right
# Tilt up
# Measure
# Recenter vertical
# Tilt down
# Measure
# Re-center vertical and horizontal

# Pan to the left (or right) and again capture distance (side E)
# Perhaps while panning, ensure wall does not end using sensor
# If it ends then use the last known value prior to dropping off
# Calculate length of wall using pathagorem theorem (side F)
# Compute angle of the angle to the relative right of the robot (between sides D and F)

# TO use this output, this can work as a sensor reading for localization,
#  but can also be used for simple
# navigation to move about the room
# Using minimum distance from a sample seems to make sense when avoiding obstacles

"""
Function to calibrate motor speed to robot velocity
"""
# Compute the robot velocity and keep mapping to corresponding motor speed (ranges 0-255)
# Could standardize the motor speed options and calibrate each one
# May need to recalibrate depending on battery level and type of surface
# Perhaps this could also be a mechanism for measuring battery level
# Or battery needs to be determined and monitored for better localization

# Algorithm
# Align with a wall, adjust so heading is directly towards wall (90 degrees)
# Measure distance to wall (back up to allow 36 inches at least)
# Measure distance (A), ensure >= 36 inches
# Using desired calibration speed, drive towards wall for .5 seconds and stop
# Measure angle, and ensure still at 90 degrees (if not, start over? perhaps just once)
# Measure distance (B)
# Calculate velocity as inches traveled per second (A-B) / .5

"""
How does motor speed translate to rotation speed?
"""

# Measure the distance from wheel to wheel and inform robot
# Will that even help? Maybe it'll help with an estimate
# Perhaps with a quicker rotation I can compute the angle against the
#  same wall and get a more reliable reading.

import picamera.array
import picamera
import time

cam_resolution = (200, 200)
camera = picamera.PiCamera(resolution=cam_resolution)
time.sleep(2)
# Flip picture
camera.hflip = True
# camera.vflip = True
camera.rotation = 90

start = time.time()
with picamera.array.PiRGBArray(camera) as stream:
    camera.capture(stream, format='rgb')
    output_arr = stream.array
print("%.2f" % (time.time() - start))

from matplotlib import pyplot as plt

image = plt.imshow(output_arr)
plt.show()

output_arr.shape

"""

"""

max_stars_repo_name: ChrisJFarr/WifiRover

max_issues_repo_name: ChrisJFarr/WifiRover

max_forks_repo_name: ChrisJFarr/WifiRover


text:
from qfengine.data.price.price_source import MySQLPriceDataSource as SQLTable
from qfengine.asset import assetClasses

import pandas as pd
from typing import Union, List, Dict
import os
import numpy as np
import logging
import functools
from qfengine import settings
import concurrent.futures

logger = logging.getLogger(__name__)

# todo: MOVE THESE TO RESPECTIVE DIR MODULE
class DataVendorMySQL(SQLTable):
    create_schema = (
      '''
      CREATE TABLE `%s` (
        `id` int NOT NULL AUTO_INCREMENT,
        `name` varchar(64) NOT NULL,
        `website_url` varchar(255) NULL,
        `api_endpoint_url` varchar(255) NULL,
        `api_key_id` varchar(255) NULL,
        `api_key` varchar(255) NULL,
        `created_date` datetime NULL DEFAULT CURRENT_TIMESTAMP(),
        `last_updated_date` datetime NULL DEFAULT CURRENT_TIMESTAMP() ON UPDATE CURRENT_TIMESTAMP(),
        PRIMARY KEY (`id`)
      ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8
      '''
    )
    def __init__(
                 self,
                 db_credentials:Dict=None,
                 name:str = 'data_vendor',
                 **kwargs
    ):
      super().__init__(
              (db_credentials if db_credentials is not None
              else settings.MYSQL_CREDENTIALS),
                  name,
                  (DataVendorMySQL.create_schema %name),
                  **kwargs
                       )

      from qfengine.data import vendor_api
      import pandas as pd
      #---| init all availables
      _vendorAPIs = {api:getattr(vendor_api,api) for api in vendor_api.__all__}
      currentInfo = []
      _to_pop = []
      for api_name, API in _vendorAPIs.items():
        _i = {'name':api_name}
        for f in self.all_accepted_columns:
            try:
              f_dat = getattr(API,f)
            except:
              pass
            else:
              if callable(f_dat):
                f_dat = f_dat()
              _i[f] = f_dat
        missing_required = [c for c in self.required_columns if c not in _i]
        if len(missing_required) != 0:
          _to_pop.append(api_name)
        else:
          currentInfo.append(_i)
      currentInfo = pd.DataFrame.from_dict(currentInfo)
      currentInfo = currentInfo.where(pd.notnull(currentInfo),None)
      upserted = self.upsertDF(currentInfo,["name"])
      for p in _to_pop:
        _vendorAPIs.pop(p)

      self._APIs = {API.name: API for _,API in _vendorAPIs.items()}

      #---| check all init for essential funcs

    def get_vendor_API(self,vendor:str):
      assert vendor in self.List
      return self._APIs[vendor]()

    @property
    def DF(self):
      return self._fullDF().set_index("name").reindex(list(self._APIs.keys()))

    @property
    def List(self):
      return list(self._APIs.keys())

class ExchangeMySQL(SQLTable):
    create_schema = (
      '''
      CREATE TABLE `%s` (
        `id` int NOT NULL AUTO_INCREMENT,
        `ref_id` varchar(32) NOT NULL,
        `name` varchar(255) NOT NULL,
        `currency` varchar(64) NULL,
        `region` varchar(255) NULL,
        `created_date` datetime NULL DEFAULT CURRENT_TIMESTAMP(),
        `last_updated_date` datetime NULL DEFAULT CURRENT_TIMESTAMP() ON UPDATE CURRENT_TIMESTAMP(),
        PRIMARY KEY (`id`)
      ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8
      '''
    )
    def __init__(
                 self,
                 db_credentials:Dict = None,
                 vendors:DataVendorMySQL = None,
                 name:str = 'exchange',
                 **kwargs
    ):
      super().__init__(
              (db_credentials if db_credentials is not None
              else settings.MYSQL_CREDENTIALS),
                  name,
                    (ExchangeMySQL.create_schema %name),
                    **kwargs
                       )

      self.vendors = vendors or DataVendorMySQL(db_credentials,mdb_conn = self._conn)

    def UPDATE(self, vendor:str = 'IEX', DF:pd.DataFrame=None):
      if DF is None:
        DF = self.vendors.get_vendor_API(vendor).exchangesDF()
      upserted = self.upsertDF(DF,['ref_id'])
      for i,_count in upserted.items():
        if _count > 0:
          logger.warning("%s %s Exchanges" %(str(i).upper(), str(_count)))

    @property
    def DF(self):
      return self._fullDF().set_index("ref_id")

    @property
    def List(self):
      return [e[0] for e in self.executeSQL("select ref_id from %s" %self._name)]

class SecurityMySQL(SQLTable):
    create_schema = (
      '''
      CREATE TABLE `%s` (
        `id` int NOT NULL AUTO_INCREMENT,
        `exchange_id` int NOT NULL,
        `symbol` varchar(10) NOT NULL,
        `type` varchar(10) NULL,
        `name` varchar(255) NULL,
        `sector` varchar(255) NULL,
        `industry` varchar(255) NULL,
        `currency` varchar(32) NULL,
        `region` varchar(32) NULL,
        `figi` varchar(255) NULL,
        `created_date` datetime NULL DEFAULT CURRENT_TIMESTAMP(),
        `last_updated_date` datetime NULL DEFAULT CURRENT_TIMESTAMP() ON UPDATE CURRENT_TIMESTAMP(),
        PRIMARY KEY (`id`),
        KEY `exchange_id` (`exchange_id` ASC),
        KEY `symbol`  (`symbol` ASC),
        CONSTRAINT `fk_exchange_id`
          FOREIGN KEY (`exchange_id`)
          REFERENCES `exchange` (`id`)
          ON DELETE NO ACTION
          ON UPDATE NO ACTION
      ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8
      '''
    )
    def __init__(
                 self,
                 db_credentials:Dict = None,
                 exchanges:ExchangeMySQL=None,
                 vendors:DataVendorMySQL=None,
                 name:str = 'security',
                 **kwargs
    ):
      super().__init__(
              (db_credentials if db_credentials is not None
              else settings.MYSQL_CREDENTIALS),
              name,
              (SecurityMySQL.create_schema %name),
              **kwargs
                       )

      self.exchanges = exchanges or ExchangeMySQL(db_credentials, mdb_conn = self._conn)
      self.vendors = vendors or self.exchanges.vendors

    def UPDATE(self, vendor:str = 'IEX', DF:pd.DataFrame = None,):
      if DF is None:
        DF = self.vendors.get_vendor_API(vendor).symbolsDF()
      if 'exchange_id' not in DF.columns:
        potential_exch_cols = [c for c in DF.columns if 'exchange' in c.lower()]
        db_exch_id = self.exchanges.DF['id']
        for c in potential_exch_cols:
          db_exch = db_exch_id.reindex(DF[c].unique())
          if len(db_exch.dropna()) == len(db_exch):
            db_exch = db_exch.where(pd.notnull(db_exch),None)
            DF['exchange_id'] = [db_exch[i] for i in DF[c].values]
            break
        assert 'exchange_id' in DF.columns, ("unidentified exchange(s) in DF")
        DF = DF.drop(columns=potential_exch_cols)
      if not isinstance(DF.index, pd.RangeIndex):
        DF.reset_index(inplace=True)

      assert set(DF.columns).issubset(set(self.all_accepted_columns)), "Unrecognized column(s): %s" %str([c for c in DF.columns if c not in self.all_accepted_columns])
      upserted = self.upsertDF(DF,['symbol','exchange_id'])
      for i,_count in upserted.items():
        if _count > 0:
          logger.warning("%s %s Security Symbols" %(str(i).upper(), str(_count)))

    @property
    def DF(self):
      return self._fullDF().set_index("symbol")

    @property
    def List(self):
      return [e[0] for e in self.executeSQL("select symbol from %s" %self._name)]

class DailyPriceMySQL(SQLTable):
    create_schema = (
      '''
      CREATE TABLE `%s` (
        `id` int NOT NULL AUTO_INCREMENT,
        `data_vendor_id` int NOT NULL,
        `symbol_id` int NOT NULL,
        `price_date` date NOT NULL,
        `open` decimal(19,4) NULL,
        `high` decimal(19,4) NULL,
        `low` decimal(19,4) NULL,
        `close` decimal(19,4) NULL,
        `volume` bigint NULL,
        `created_date` datetime NULL DEFAULT CURRENT_TIMESTAMP(),
        `last_updated_date` datetime NULL DEFAULT CURRENT_TIMESTAMP() ON UPDATE CURRENT_TIMESTAMP(),
        PRIMARY KEY (`id`),
        KEY `price_date` (`price_date` ASC),
        KEY `data_vendor_id` (`data_vendor_id`),
        KEY `symbol_id` (`symbol_id`),
        CONSTRAINT `fk_symbol_id `
          FOREIGN KEY (`symbol_id`)
          REFERENCES `security` (`id`)
          ON DELETE NO ACTION
          ON UPDATE NO ACTION,
        CONSTRAINT `fk_data_vendor_id`
          FOREIGN KEY (`data_vendor_id`)
          REFERENCES `data_vendor` (`id`)
          ON DELETE NO ACTION
          ON UPDATE NO ACTION
      ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8
      '''
                    )

    def __init__(
                 self,
                 asset_type:assetClasses,
                  db_credentials:Dict=None,
                  symbols:SecurityMySQL=None,
                  vendors:DataVendorMySQL=None,
                  name:str = 'daily_price',
                      by_vendor:str = None,
                      symbols_list:List[str] = None,
                      **kwargs
    ):
      super().__init__(
              db_credentials = (db_credentials if db_credentials is not None
                                else settings.MYSQL_CREDENTIALS),
                  price_table_name = name,
                    price_table_schema = (DailyPriceMySQL.create_schema %name),
                    **kwargs
                       )
      self.symbols = symbols or SecurityMySQL(db_credentials, mdb_conn = self._conn)
      self.vendors = vendors or self.symbols.vendors

      self.asset_type = asset_type
      self.symbols_list = symbols_list
      self.by_vendor = by_vendor

      if self.by_vendor:
        assert self.by_vendor in self.vendors.List
        self.symbols_list = self._query_available_symbols_in_database_by_vendor(self.by_vendor)
        if settings.PRINT_EVENTS:
          print("Initialized DailyPriceMySQL DataSource From Vendor '%s' | Available Symbols Count = %s" %(str(self.by_vendor), str(len(self.symbols_list))))
      self._cached_copies = []

   #!----------| ABSTRACTED METHODS OF A PRICE DATA SOURCE |------------#
    #---| Self-Copy
    def create_price_source_copy(self,
                                 cache_copy:bool = False,
    ):
      copy = DailyPriceMySQL(
                        asset_type = self.asset_type,
                        db_credentials = self._db_credentials.copy(),
                        name = self._full_credentials['table_name'],
                        by_vendor = None,
                            )
      copy.by_vendor = self.by_vendor #--| skip SQL vetting
      copy.symbols_list = (self.symbols_list.copy() if self.symbols_list else self.symbols_list)
      if cache_copy:
        self._cached_copies.append(copy)
      return copy
    #---------------------------|

    #------| Assets/Universe
    def assetsDF(self,
                 **kwargs
    )->pd.DataFrame:
        df = self.symbols.DF.astype(str)
        if self.symbols_list:
          df = df.reindex(self.symbols_list)

        if 'sector' in kwargs:
          assert isinstance(kwargs['sector'],str)

          df = df[df.sector == kwargs['sector']]

        return df

    def assetsList(self,
                   **kwargs
    )->list:
      return list(self.assetsDF(**kwargs).index.values)

    @property
    def sectorsList(self)->list:
      return self.symbols.DF.sector.dropna().unique()
    #---------------------------|

    #-----------| Price
    def get_assets_bid_ask_dfs(self,
                               asset:str,
                               *assets:str,
                                  start_dt=None,
                                  end_dt=None,
    )->pd.DataFrame:
      return self._price_dfs_to_bid_ask_dfs(
                      self.get_assets_historical_price_dfs(asset,
                                                            *assets,
                                                            start_dt = start_dt,
                                                            end_dt = end_dt
                                                        )
      )

    def get_assets_historical_price_dfs(self,
                                        asset:str,
                                        *assets:str,
                                            price:str = None,
                                            start_dt = None,
                                            end_dt = None,
                                              adjusted = None,
                                              **kwargs
    )->pd.DataFrame:
        if price:
            assert price in [
                  "open", "high", "low",
                  "close","volume"
                          ]
        #--| parallelizing queries for performance
        symbols = [asset] + [s for s in assets]
        result = self._assets_daily_price_DF(*symbols)
        if price:
          result = result[
                  [col for col in result.columns if price in col]
                      ]
          result.columns = result.columns.get_level_values('symbols')

        if start_dt:
          result = result[result.index >= self._format_dt(start_dt)]
        if end_dt:
          result = result[result.index <= self._format_dt(end_dt)]

        return result
    #---------------------------|

    #----| Price Date Ranges
    @functools.lru_cache(maxsize = 1024 * 1024)
    def get_assets_price_date_ranges_df(self,
                                        asset:str,
                                        *assets:str,
    )->pd.DataFrame:

        symbols = [asset] + [s for s in assets]
        if self.symbols_list:
          assert set(symbols).issubset(self.symbols_list)

        def _get_result(source, symbol, vendor):
          return {
              'symbol': symbol,
              'start_dt': self._format_dt(source._asset_symbol_min_price_date_by_vendor(symbol, vendor)),
              'end_dt': self._format_dt(source._asset_symbol_max_price_date_by_vendor(symbol,vendor)),
                }
        final_df = pd.DataFrame()
        for vendor in self.vendorsList:
          result = pd.DataFrame.from_dict(
                                list(
                                  concurrent.futures.ThreadPoolExecutor().map(
                                                                          _get_result,
                                                                          *zip(*(
                                                                            (
                                                                              self.create_price_source_copy(cache_copy = True),
                                                                              symbol,
                                                                              vendor,
                                                                            ) for symbol in symbols
                                                                                ))
                                                                              )
                                    )
                                          ).set_index('symbol').dropna()
          self._close_cached_copies()
          final_df = final_df.append(result)
          symbols = [s for s in symbols if s not in final_df.index]
          if len(symbols) == 0:
            break
        return final_df

    def get_assets_minimum_start_dt(self,
                                        asset:str,
                                        *assets:str,
    )->pd.Timestamp:
        return self._format_dt(max(
          self.get_assets_price_date_ranges_df(
                            asset, *assets
                                            ).start_dt.values
                 ))

    def get_assets_maximum_end_dt(self,
                                      asset:str,
                                      *assets:str,
    )->pd.Timestamp:
        return self._format_dt(min(
          self.get_assets_price_date_ranges_df(
                            asset, *assets
                                            ).end_dt.values
                 ))
    #---------------------------|

   #!----------------------------------------------------------------#

    def update_assets_daily_price(self,
              vendor:str,
              batch_size:int=100,
              symbols_to_update:List[str] = None,
              skip_empty_update:bool = True,
              DF:pd.DataFrame=None,
    ):
      assert vendor in self.vendors.List
      vendor_id = self.vendors.DF.id[vendor]
      symbols_id = self.symbols.DF['id']

      inserted_count = 0
      updated_count = 0
      t0 = pd.Timestamp.now()

      if DF is not None: #---| UPSERT ONCE WITH GIVEN DATAFRAME (NO API CALLS WILL BE MADE)
        self._upsert_daily_price_DF(DF,['price_date', 'symbol_id', 'data_vendor_id'])

      else: #---| PERFORM UPSERT IN BATCHES OF SYMBOLS BY MAKING API CALLS
        if symbols_to_update is not None:
          omitted = [s for s in symbols_to_update if s not in symbols_id.index]
          if len(omitted) > 0:
            logger.warning("Omitting %s given symbols that are not in database universe" %str(len(omitted)))
            logger.warning(str(omitted))
          symbols_id = symbols_id.reindex([s for s in symbols_to_update if s not in omitted])
        symbols_id = dict(symbols_id)
        assert len(symbols_id) != 0, "No symbols in Symbol table, code out schematic or manually prepare it (symbol.csv)"

        logger.warning("----------------------------------------------------------------")
        logger.warning("---------------| Updating Equities Daily Prices |---------------")
        logger.warning("----------------------------------------------------------------")

        logger.warning("Checking database for latest available price_date from '%s' of %s symbols..." %(vendor,str(len(symbols_id))))
        #!---| Query for all symbols to get their max price_dates from SQL Database
        symbols_by_max_dates = {}
        for s in symbols_id:
          d = self._asset_symbol_max_price_date_by_vendor(s,vendor)
          if d not in symbols_by_max_dates:
            symbols_by_max_dates[d] = []
          if s not in symbols_by_max_dates[d]:
            symbols_by_max_dates[d].append(s)

        #!---| Download latest daily price for symbols with their respective start dates & process for upsert to Database
        logger.warning("Performing Update in batches of %s" %str(batch_size))
        for start_date,symbols in symbols_by_max_dates.items():
          logger.warning("For %s symbols with '%s' as start_date:" %(str(len(symbols)),str(start_date)))

          i = 0
          batch_number = 1
          while True:
            symbols_batch = symbols[i:i+batch_size]
            batch_data = self.vendors.get_vendor_API(vendor).get_barset(symbols_batch,"1D",start_date)
            try:
              batch_data = self._transform_DF_for_daily_price_upsert(batch_data, vendor = vendor, start_date=start_date)
            except Exception:
              logger.warning("Cannot transform, skipping this batch of symbols: %s" %str(symbols_batch))
              pass
            else:
              upserted = self.upsertDF(batch_data,no_filter=True)
              for a,_count in upserted.items():
                if _count > 0:
                  if a == 'inserted':
                    logger.warning("  Batch #%s : %s New Data Points Inserted" %(str(batch_number),_count))
                    inserted_count += _count
                  elif a == 'updated':
                    logger.warning("  Batch #%s : %s Existing Data Points Updated" %(str(batch_number),_count))
                    updated_count += _count
              if (upserted['inserted'] == upserted['updated'] == 0) and skip_empty_update:
                logger.warning("  No New Data Upserted. Skipping remaining symbols (set skip_empty_update=False for otherwise)")
                break

            #---| Loop Breaker
            if symbols_batch[-1] == symbols[-1]:
              break
            else:
              i += batch_size
              batch_number += 1

      print("Update Completed:")
      print("--Total Data Points Inserted: %s" %(str(inserted_count)))
      print("--Total Data Points Updated: %s" %(str(updated_count)))
      print("--Total Time Elapsed: %s" %(str(pd.Timestamp.now() - t0)))

    @property
    def vendorsDF(self,)->pd.DataFrame:
      return self.vendors.DF.reindex(self.vendorsList)

    @property
    def vendorsList(self,)->list:
      return self.vendors.List if self.by_vendor is None else [self.by_vendor]

   #!---| BACKEND
    def _transform_DF_for_daily_price_upsert(self,
                                    upsert_df:pd.DataFrame,
                                    **kwargs
    ):
        df = upsert_df.copy()
        #---| Case I: Basic DF with no MultiIndex columns
        if not isinstance(df.columns,pd.MultiIndex):
          assert 'symbol_id' in df.columns
          if 'price_date' not in df.columns:
            assert not isinstance(df.index, pd.RangeIndex)
            df.index = pd.DatetimeIndex(df.index)
            df['price_date'] = df.index.values
            df.index = range(len(df))
          if 'data_vendor_id' not in df.columns:
            vendor_id = int(self.vendors.DF.id[kwargs['vendor']]) if 'vendor' in kwargs else (
                                        int(kwargs['vendor_id']) if 'vendor_id' in kwargs else None
                                  )
            assert isinstance(vendor_id, int)
            df['data_vendor_id'] = vendor_id
          return df.where(pd.notnull(df), None)

        #---| Case II: MultiIndex Columns of (symbols, columns)
        else:
          assert not isinstance(df.index, pd.RangeIndex)
          df.index = pd.DatetimeIndex(df.index)
          symbols_id = dict(kwargs['symbols_id']) if 'symbols_id' in kwargs else dict(self.symbols.DF['id'])
          vendor_id = int(self.vendors.DF.id[kwargs['vendor']]) if 'vendor' in kwargs else (
                            int(kwargs['vendor_id']) if 'vendor_id' in kwargs else None
                                          )
          assert isinstance(vendor_id, int)
          assert isinstance(symbols_id, dict)

          try:
            df_symbols = list(df.columns.get_level_values('symbols').unique())
          except KeyError:
            if settings.PRINT_EVENTS:
              print("Daily Price columns does not contain 'symbols' as name. "
                    "Attempting to grab the first index locations..."
                    )
            df_symbols = list(pd.Series([i[0] for i in df.columns]).unique())

          assert set(df_symbols).issubset(set(symbols_id.keys())), (
                          "Daily Price data contains unidentified symbol(s) without id(s): %s" %(
                                                                        str([
                                                                                s for s in df_symbols if s not in symbols_id
                                                                                ])
                                                                                )
                                                              )

          start_date = pd.Timestamp(kwargs['start_date']) if ('start_date' in kwargs) else None
          transformed_df = pd.DataFrame()
          for s in df_symbols:
            _df = df[s]
            _df.index = pd.DatetimeIndex(_df.index)
            if start_date:
              _df = _df[_df.index > start_date]
            _df['symbol_id'] = symbols_id[s]
            _df['price_date'] = _df.index.values
            _df.index = range(len(_df))
            transformed_df = transformed_df.append(_df)
          transformed_df['data_vendor_id'] = vendor_id
          transformed_df.index = range(len(transformed_df))
          transformed_df = transformed_df.where(pd.notnull(transformed_df), None)
          return transformed_df

    def _upsert_daily_price_DF(self,
                            DF:pd.DataFrame,
                            filter_columns:List[str] = None,
                            **kwargs
    ):
      if settings.PRINT_EVENTS:
        print("Upserting Equities Daily Price with given DF.")
      upserted = self.upsertDF(self._transform_DF_for_daily_price_upsert(DF, **kwargs), filter_columns)
      for a,_count in upserted.items():
        if _count > 0:
          if a == 'inserted':
            logger.warning("%s New Data Points Inserted" %(str(_count)))
          elif a == 'updated':
            logger.warning("%s Existing Data Points Updated" %(str(_count)))
      if (upserted['inserted'] == upserted['updated'] == 0):
        logger.warning("No New Data Upserted from DF given.")

    @functools.lru_cache(maxsize = 1024 * 1024)
    def _assets_daily_price_DF(self,
                                asset:str,
                                *assets:str
    ):
        symbols = [asset] + [s for s in assets]
        if self.symbols_list:
          assert set(symbols).issubset(self.symbols_list)
        result = pd.DataFrame()
        for vendor in self.vendorsList:
          all_dfs = concurrent.futures.ThreadPoolExecutor().map(
                                          self.__class__._query_asset_symbol_daily_price_DF,
                                          *zip(*(
                                            (
                                              self.create_price_source_copy(cache_copy = True),
                                              symbol,
                                              vendor,
                                            ) for symbol in symbols
                                                ))
                                                          )
          final_df = pd.concat([
                  d.where(pd.notna(d), np.nan) for d in all_dfs if (
                                                  not d.where(pd.notna(d), np.nan).dropna().empty
                                                                  )
                                ], axis=1)
          self._close_cached_copies()
          final_df.columns.names = ('symbols','columns')
          final_df = final_df.set_index(final_df.index.tz_localize(settings.TIMEZONE))
          result = pd.concat([result,final_df], axis=1)
          symbols = [s for s in symbols if s not in result.columns.get_level_values('symbols')]
          if len(symbols) == 0:
            break

        if len(symbols) > 0:
          if settings.PRINT_EVENTS:
            print("Warning: Queried Daily Prices DataFrame is missing %s symbols:" %len(symbols))
            print(symbols)

        return result

    @functools.lru_cache(maxsize = 1024 * 1024)
    def _asset_symbol_max_price_date_by_vendor(self,
                                               asset:str,
                                               vendor:str
    ):
      dat = self.executeSQL(
        '''
        SELECT max(price_date)
        FROM %s dp
        INNER JOIN %s sym
        INNER JOIN %s vendor
        ON
          dp.symbol_id = sym.id AND
          dp.data_vendor_id = vendor.id
        WHERE
          sym.symbol = '%s' AND
          vendor.name = '%s' AND
          dp.close IS NOT NULL AND
          dp.open IS NOT NULL
        ''' %(
              self._name,
              self.symbols._name,
              self.vendors._name,
              asset,
              vendor
            )
      )
      return None if len(dat) == 0 else dat[0][0]

    @functools.lru_cache(maxsize = 1024 * 1024)
    def _asset_symbol_min_price_date_by_vendor(self,
                                               asset:str,
                                               vendor:str
    ):
      dat = self.executeSQL(
        '''
        SELECT min(price_date)
        FROM %s dp
        INNER JOIN %s sym
        INNER JOIN %s vendor
        ON
          dp.symbol_id = sym.id AND
          dp.data_vendor_id = vendor.id
        WHERE
          sym.symbol = '%s' AND
          vendor.name = '%s' AND
          dp.close IS NOT NULL AND
          dp.open IS NOT NULL
        ''' %(
              self._name,
              self.symbols._name,
              self.vendors._name,
              asset,
              vendor
            )
      )
      return None if len(dat) == 0 else dat[0][0]

    @functools.lru_cache(maxsize = 1024 * 1024)
    def _query_available_symbols_in_database_by_vendor(self, vendor:str):
      assert vendor in self.vendors.List
      sqlDF = self.assetsDF().reset_index().set_index('id')
      sqlDF.index = sqlDF.index.astype(int)
      df = sqlDF.reindex(
            [
            int(i[0]) for i in self.executeSQL(
              'select distinct symbol_id from %s where data_vendor_id = %s' %(
                                                                              self._name,
                                                                              self.vendors.DF.id[vendor]
                                                                             )
                                              )
            ]
                        )
      df = df.reindex(df[['symbol']].dropna().index)
      return [] if df.empty else list(df.set_index('symbol').sort_index().index.values)

    def _convert_dt_to_date(self, dt):
        return pd.Timestamp(
                self._format_dt(dt).date(),
                tz = settings.TIMEZONE
                        )

    def _format_dt(self, dt):
      try:
        return pd.Timestamp(dt).tz_convert(settings.TIMEZONE)
      except TypeError:
        try:
          return pd.Timestamp(dt).tz_localize(settings.TIMEZONE)
        except:
          raise

    def _close_cached_copies(self):
      for source in self._cached_copies:
        source._conn.close()
      self._cached_copies = []

    def _query_asset_symbol_daily_price_DF(self,
                                      asset:str,
                                      vendor:str,
    )->pd.DataFrame:
      df_cols = [
        "open", "high", "low",
                "close","volume"
                          ]
      select_str = "dp.price_date, " + str(
                                    [
                                      ".".join(["dp",p]) for p in df_cols
                                    ]
                                          )[1:-1].replace("'","")

      cond_str = "vendor.name = '%s' AND sym.symbol = '%s'" %(vendor, asset)
      sql = (
            '''
            SELECT %s
            FROM %s as sym
            INNER JOIN %s AS dp
            INNER JOIN %s AS vendor
            ON
                dp.symbol_id = sym.id AND
                dp.data_vendor_id = vendor.id
            WHERE
                %s
            ORDER BY
                dp.price_date ASC
            '''%(
                select_str,
                self.symbols._name,
                self._name,
                self.vendors._name,
                cond_str
                )
              )
      dat = np.array(self.executeSQL(sql))

      cols = [
              np.array([asset for _ in df_cols]),
              np.array(df_cols)
             ]

      if len(dat) == 0:
        return pd.DataFrame(columns = cols)
      else:
        df = pd.DataFrame(
                      data = dat[:,1:],
                      index = pd.DatetimeIndex(dat[:,0]),
                      columns = cols
                        )
        for c in df.columns:
          df[c] = pd.to_numeric(df[c])
        return df

    def _price_dfs_to_bid_ask_dfs(self,
                                  price_df:pd.DataFrame
    ):
        def _symbol_price_to_bid_ask(bar_df, symbol):
          cols = [
                np.array([symbol, symbol]),
                np.array(['bid', 'ask'])
                ]
          if bar_df.dropna().empty:
            return pd.DataFrame(columns = cols)
          bar_df = bar_df.sort_index()
          oc_df = bar_df.loc[:, ['open', 'close']]
          oc_df['pre_market'] = oc_df['close'].shift(1)
          oc_df['post_market'] = oc_df['close']
          oc_df = oc_df.dropna()
          # Convert bars into separate rows for open/close prices
          # appropriately timestamped
          seq_oc_df = oc_df.T.unstack(level=0).reset_index()
          seq_oc_df.columns = ['datetime', 'market', 'price']

          seq_oc_df.loc[seq_oc_df['market'] == 'open', 'datetime'] += pd.Timedelta(hours=9, minutes=30)
          seq_oc_df.loc[seq_oc_df['market'] == 'close', 'datetime'] += pd.Timedelta(hours=16, minutes=00)
          seq_oc_df.loc[seq_oc_df['market'] == 'pre_market', 'datetime'] += pd.Timedelta(hours=0, minutes=00)
          seq_oc_df.loc[seq_oc_df['market'] == 'post_market', 'datetime'] += pd.Timedelta(hours=23, minutes=59)

          # TODO: Unable to distinguish between Bid/Ask, implement later
          dp_df = seq_oc_df[['datetime', 'price']]
          dp_df['bid'] = dp_df['price']
          dp_df['ask'] = dp_df['price']
          dp_df = dp_df.loc[:, ['datetime', 'bid', 'ask']].fillna(method='ffill').set_index('datetime').sort_index()
          dp_df.columns = cols
          return dp_df

        bid_ask_df = pd.concat(
                      [
                        _symbol_price_to_bid_ask(price_df[symbol], symbol)
                          for symbol in price_df.columns.get_level_values('symbols').unique()
                      ], axis = 1
                    )
        bid_ask_df.columns.names = ('symbols', 'columns')

        return bid_ask_df

max_stars_repo_name: jp-quant/qfengine

max_issues_repo_name: jp-quant/qfengine

max_forks_repo_name: jp-quant/qfengine


text:
from PIL import Image, ImageDraw, ImageFont
from matplotlib import pyplot as plt
from errorHandler import ErrorHandler

import os
import sys
import argparse
import json
import math
import numpy as np
import seaborn as sns; sns.set()
import tiles

DEBUG = False

mu_channel = 0
sig_channel = 1
hits_channel = 2

coord_dtype = float

parser = argparse.ArgumentParser(description='Retrieve required segment from overlay.')
parser.add_argument('x_min', type=coord_dtype, nargs=1, help='Minimum X coordinate of bounding box')
parser.add_argument('y_min', type=coord_dtype, nargs=1, help='Minimum Y coordinate of bounding box')
parser.add_argument('x_max', type=coord_dtype, nargs=1, help='Maximum X coordinate of bounding box')
parser.add_argument('y_max', type=coord_dtype, nargs=1, help='Maximum Y coordinate of bounding box')
parser.add_argument('--errors_file', type=str, nargs=1, help='Path to errors JSON file')
parser.add_argument('--DEBUG', dest='DEBUG', action='store_const', const=True, default=False)

if DEBUG: print("Args: " + str(sys.argv))

try:
    args = parser.parse_args(sys.argv[1:])
except argparse.ArgumentTypeError:
    exit(99)

err = ErrorHandler(args.errors_file[0])

DEBUG = DEBUG or args.DEBUG

# Get values from args
req_x_min = args.x_min[0]
req_x_max = args.x_max[0]
req_y_min = args.y_min[0]
req_y_max = args.y_max[0]

try:
    overlay_canvas = tiles.load_tiles (
        req_x_min,
        req_y_min,
        req_x_max,
        req_y_max,
        DEBUG=DEBUG
    )
except MemoryError:
    err.exitOnError("MemoryError")

if req_x_min < -180 or req_x_max > 180 or req_y_max > 90 or req_y_min < -90:
    raise err.exitOnError("LimitsError")

if req_x_min > req_x_max  or req_y_max < req_y_min:
    raise err.exitOnError("InvertedValues")

if DEBUG: print("Done!")

# Convert sigma-mu to gradient
if DEBUG:
    if (np.max(overlay_canvas[:, :, hits_channel]) == 0):
        print("None active!")
    else:
        print("Found active!")
    print("Mu range: {} <> {}".format(np.min(overlay_canvas[:, :, mu_channel]), np.max(overlay_canvas[:, :, mu_channel])))
    print("Hits range: {} <> {}".format(np.min(overlay_canvas[:, :, hits_channel]), np.max(overlay_canvas[:, :, hits_channel])))

try:
    _mu = overlay_canvas[:, :, mu_channel]
except MemoryError:
    err.exitOnError("MemoryError")

fig = plt.figure(1, figsize=(overlay_canvas.shape[1], overlay_canvas.shape[0]), dpi=1)
ax = sns.heatmap(np.array(_mu), xticklabels=False, yticklabels=False, cbar=False, cmap=sns.diverging_palette(10, 150, sep=80), vmin=0.4, vmax=0.6)
fig.tight_layout(pad=0)
fig.canvas.draw()
data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
aux_canvas = np.concatenate(
    (data.reshape(fig.canvas.get_width_height()[::-1] + (3, )),
    np.where(overlay_canvas[:, :, mu_channel] != 0.5, 255, 0).astype(np.uint8)[:,:,np.newaxis]), # FIXME should be where overlay_canvas[:, :, hits_channel] > 0
    axis=2)

assert aux_canvas.shape[:2] == overlay_canvas.shape[:2], "Expected {}, got {}".format(str(overlay_canvas.shape[:2]), str(aux_canvas.shape[:2]))

overlay_canvas_img = Image.fromarray(aux_canvas, mode="RGBA")

if DEBUG: overlay_canvas_img.show()

(left, lower, right, upper) = tiles.bounding_box(req_x_min, req_y_min, req_x_max, req_y_max)

if DEBUG: print("Cropping {}, {}, {}, {}".format(left, upper, right, lower))

if DEBUG:
    d = ImageDraw.Draw(overlay_canvas_img)

    ellipse_rad = 10

    d.ellipse([left-ellipse_rad, upper-ellipse_rad, left+ellipse_rad, upper+ellipse_rad], fill=(255, 0, 0))
    d.ellipse([right-ellipse_rad, lower-ellipse_rad, right+ellipse_rad, lower+ellipse_rad], fill=(0, 0, 255))

    overlay_canvas_img.show()

overlay_canvas_img = overlay_canvas_img.crop((left, upper, right, lower))

if DEBUG: print(overlay_canvas_img.size)

if DEBUG: overlay_canvas_img.show()

nonce = os.urandom(32).hex()

with open("./tmp/"+nonce+".png", "wb") as f:
    overlay_canvas_img.save(f, format='PNG')

print(nonce, end="")