text:
"""
This is a draft of machine learning code, so that we can test how to do the machine learning algorithm of the gravitational lenses.
"""
# IMPORTS
import os
import numpy as np
import matplotlib.pyplot as plt
import makeExcelTable
from astropy.utils.data import get_pkg_data_filename
from astropy.io import fits
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import accuracy_score
from sklearn import model_selection
from sklearn.utils import shuffle

# FUNCTIONS
def getPositiveSimulatedTrain(base_dir='Training/Positive'):
    """
    This gets the g, r, and i of the 10 000 positively simulated images from the
    PositiveWithDESSky, as well as returning the positively simulate array.

    Args:
        base_dir (string):      This the root file path of the positively simulated images.
    Returns:
        data_pos(numpy array):   This is the array of positively simulated images.
    """

    folders = {}
    for root, dirs, files in os.walk(base_dir):
        for folder in dirs:
            key = folder
            value = os.path.join(root, folder)
            folders[key] = value

    # number of Positive DataPoints
    num_data_targets = len(folders)

    data_pos = np.zeros([num_data_targets, 3, 100, 100])

    # key is name of folder number
    # value is the number of the folder to be added to the file name

    counter = 0
    for key, value in folders.items():
        g_name = get_pkg_data_filename(value + '/' + str(key) + '_g_norm.fits')
        r_name = get_pkg_data_filename(value + '/' + str(key) + '_r_norm.fits')
        i_name = get_pkg_data_filename(value + '/' + str(key) + '_i_norm.fits')

        # g_name = get_pkg_data_filename(value + '/' + str(key) + '_posSky_g.fits')
        # r_name = get_pkg_data_filename(value + '/' + str(key) + '_posSky_r.fits')
        # i_name = get_pkg_data_filename(value + '/' + str(key) + '_posSky_i.fits')

        g = fits.open(g_name)[0].data[0:100, 0:100]
        r = fits.open(r_name)[0].data[0:100, 0:100]
        i = fits.open(i_name)[0].data[0:100, 0:100]

        data_pos[counter] = [g, r, i]
        counter += 1
        # just to run, and use less things
        # if counter > 1500:
        #     break
    print("GOT POSITIVE TRAINING DATA")
    return data_pos

def getNegativeDESTrain(base_dir='Training/Negative'):
    """
    This gets the g, r, and i  10 000 negative images from the
    DES/DES_Processed folder, as well as returning the
    negative array,

    Args:
        base_dir (string):      This the root file path of the negative images.
    Returns:
        data_neg (numpy array):  This is the array of negative images.
    """
    folders_neg = []
    for root, dirs, files in os.walk(base_dir):
        for folder in dirs:
            folders_neg.append(os.path.join(root, folder))
    num_data_targets = len(folders_neg)
    data_neg = np.zeros([num_data_targets, 3, 100, 100])

    for var in range(len(folders_neg)):
        # g_name = get_pkg_data_filename(folders_neg[var] + '/g_WCSClipped.fits')
        # r_name = get_pkg_data_filename(folders_neg[var] + '/r_WCSClipped.fits')
        # i_name = get_pkg_data_filename(folders_neg[var] + '/i_WCSClipped.fits')

        g_name = get_pkg_data_filename(folders_neg[var] + '/g_norm.fits')
        r_name = get_pkg_data_filename(folders_neg[var] + '/r_norm.fits')
        i_name = get_pkg_data_filename(folders_neg[var] + '/i_norm.fits')

        g = fits.open(g_name)[0].data[0:100, 0:100]
        r = fits.open(r_name)[0].data[0:100, 0:100]
        i = fits.open(i_name)[0].data[0:100, 0:100]

        data_neg[var] = [g, r, i]
        # just to run, and use less things
        # if var > 1500:
        #     break
    print("DONE 2")
    return data_neg

def getPositiveSimulatedTest(base_dir='Testing/Positive'):
    folders = {}
    for root, dirs, files in os.walk(base_dir):
        for folder in dirs:
            key = folder
            value = os.path.join(root, folder)
            folders[key] = value

    # number of Positive DataPoints
    num_data_target = len(folders)

    positive_test = np.zeros([num_data_target, 3, 100, 100])

    # key is name of folder number
    # value is the number of the folder to be added to the file name

    counter = 0
    for key, value in folders.items():
        g_name = get_pkg_data_filename(value + '/' + str(key) + '_g_norm.fits')
        r_name = get_pkg_data_filename(value + '/' + str(key) + '_r_norm.fits')
        i_name = get_pkg_data_filename(value + '/' + str(key) + '_i_norm.fits')

        # g_name = get_pkg_data_filename(value + '/' + str(key) + '_posSky_g.fits')
        # r_name = get_pkg_data_filename(value + '/' + str(key) + '_posSky_r.fits')
        # i_name = get_pkg_data_filename(value + '/' + str(key) + '_posSky_i.fits')

        g = fits.open(g_name)[0].data[0:100, 0:100]
        r = fits.open(r_name)[0].data[0:100, 0:100]
        i = fits.open(i_name)[0].data[0:100, 0:100]

        positive_test[counter] = [g, r, i]
        counter += 1
        # just to run, and use less things
        # if counter > 1500:
        #     break
    print("GOT POSITIVE TESTING DATA")
    return positive_test

def getNegativeDESTest(base_dir='Testing/Negative'):
    """
    This gets the g, r, and i  10 000 negative images from the
    DES/DES_Processed folder, as well as returning the
    negative array,

    Args:
        base_dir (string):      This the root file path of the negative images.
    Returns:
        negative_test (numpy array):  This is the array of negative images.
    """
    folders_neg = []
    for root, dirs, files in os.walk(base_dir):
        for folder in dirs:
            folders_neg.append(os.path.join(root, folder))
    num_data_target = len(folders_neg)
    negative_test = np.zeros([num_data_target, 3, 100, 100])

    for var in range(len(folders_neg)):
        # if var > 1500:
        # break
        # g_name = get_pkg_data_filename(folders_neg[var]+'/g_WCSClipped.fits')
        # r_name = get_pkg_data_filename(folders_neg[var]+'/r_WCSClipped.fits')
        # i_name = get_pkg_data_filename(folders_neg[var]+'/i_WCSClipped.fits')

        g_name = get_pkg_data_filename(folders_neg[var] + '/g_norm.fits')
        r_name = get_pkg_data_filename(folders_neg[var] + '/r_norm.fits')
        i_name = get_pkg_data_filename(folders_neg[var] + '/i_norm.fits')

        g = fits.open(g_name)[0].data[0:100, 0:100]
        r = fits.open(r_name)[0].data[0:100, 0:100]
        i = fits.open(i_name)[0].data[0:100, 0:100]

        negative_test[var] = [g, r, i]
        # just to run, and use less things
        # if var >= 1000:
        #     break
    print("GOT NEGATIVE TESTING DATA")
    return negative_test

def loadImage(positive_array, negative_array):
    """
    This loads the positive and negative arrays, and makes an image dataset
    numpy array that is made through adding the images of the positive and
    negative arrays. This also makes a label dataset, by adding the appropriate
    labels for the positive and negative arrays.

    Args:
        positive_array (numpy array):    This is the positively simulated array of gravitational lenses.
        negative_array (numpy array):    This is the negative array of from DES.
    Returns:
        image_train (numpy array):      This is the numpy array of the positive and negative arrays added
                                        together to make a single array.
        image_labels (numpy array):     This is the numpy array of the labels for the positive and negative
                                        arrays added together to make a single array.
    """

    positive_data = []
    negative_data = []
    positive_label = []
    negative_label = []
    image_train = []
    image_labels = []

    for num in range(0, len(positive_array)):
        image_train.append(positive_array[num])
        # label_pos = 'Gravitational Lensing'
        label_pos = 1  # assign 1 for gravitational lensing
        image_labels.append(label_pos)

    for num in range(0, len(negative_array)):
        image_train.append(negative_array[num])
        # label_neg = 'No Gravitational Lensing'
        label_neg = 0  # assign 0 for no lensing
        image_labels.append(label_neg)
    print("DONE 3")
    return np.array(image_train), np.array(image_labels)

def getUnseenNegative(num, base_dir='UnseenData'):
    """
    This gets the unseen g, r, and i unknown/negative images, according to the number specified.
    This is so that the correct number of unknown images, is retrieved, according to the
    unseen known lenses.

    Args:
        num (integer):              This is the number specified, which equates to how many unseen known
                                    gravitational lenses.
                                    This indicates how many unseen negative images is to be retrieved.
        base_dir (string):          This is the root directory file path of where the unknown lenses are situated in.
    Returns:
        data_unknown (numpy array):  This is the numpy array of the unknown dataset.
    """
    path_unknown = '%s' % base_dir
    if num == 47:
        path_unknown = '%s/Negative47' % path_unknown
    elif num == 84:
        path_unknown = '%s/Negative84' % path_unknown
    elif num == 131:
        path_unknown = '%s/Negative131' % path_unknown

    folders_unknown = []
    for root, dirs, files in os.walk(path_unknown):
        for folder in dirs:
            folders_unknown.append(os.path.join(root, folder))

    num_data_target = len(folders_unknown)
    data_unknown = np.zeros([num_data_target, 3, 100, 100])

    for var in range(len(folders_unknown)):
        # gName = get_pkg_data_filename(folders_unknown[var]+'/g_WCSClipped.fits')
        # rName = get_pkg_data_filename(folders_unknown[var]+'/r_WCSClipped.fits')
        # iName = get_pkg_data_filename(folders_unknown[var]+'/i_WCSClipped.fits')

        gName = get_pkg_data_filename(folders_unknown[var] + '/g_norm.fits')
        rName = get_pkg_data_filename(folders_unknown[var] + '/r_norm.fits')
        iName = get_pkg_data_filename(folders_unknown[var] + '/i_norm.fits')

        g = fits.open(gName)[0].data[0:100, 0:100]
        r = fits.open(rName)[0].data[0:100, 0:100]
        i = fits.open(iName)[0].data[0:100, 0:100]

        data_unknown[var] = [g, r, i]
    return data_unknown

def getTestSet():
    positive_test = getPositiveSimulatedTest()
    negative_test = getNegativeDESTest()
    images_test, labels_test = loadImage(positive_test, negative_test)
    return images_test, labels_test

def getDES2017(base_dir='KnownLenses/DES2017/'):
    """
    This is used to get g, r, and i images of the DES2017 array, which contains 47 unseen known lenses.
    Args:
        base_dir (string):          This is the root directory of the DES2017 folder.
    Returns:
        data_known_des (numpy array): This is the numpy array of the the DES2017 dataset.
    """

    known_des2017 = []
    for root, dirs, files in os.walk(base_dir):
        for folder in dirs:
            known_des2017.append(os.path.join(root, folder))

    num_data_targets = len(known_des2017)
    data_known_des = np.zeros([num_data_targets, 3, 100, 100])

    for var in range(len(known_des2017)):
        # g_name = get_pkg_data_filename(known_des2017[var]+'/g_WCSClipped.fits')
        # r_name = get_pkg_data_filename(known_des2017[var]+'/r_WCSClipped.fits')
        # i_name = get_pkg_data_filename(known_des2017[var]+'/i_WCSClipped.fits')

        g_name = get_pkg_data_filename(known_des2017[var] + '/g_norm.fits')
        r_name = get_pkg_data_filename(known_des2017[var] + '/r_norm.fits')
        i_name = get_pkg_data_filename(known_des2017[var] + '/i_norm.fits')

        g = fits.open(g_name)[0].data[0:100, 0:100]
        r = fits.open(r_name)[0].data[0:100, 0:100]
        i = fits.open(i_name)[0].data[0:100, 0:100]

        data_known_des[var] = [g, r, i]

    return data_known_des

def getJacobs(base_dir='KnownLenses/Jacobs_KnownLenses/'):
    """
    This is used to get g, r, and i images of the known Jacobs dataset, which contains 84 unseen known lenses.

    Args:
        base_dir (string):          This is the root directory of the DES2017 folder.
    Returns:
        dataKnownDES (numpy array): This is the numpy array of the the DES2017 dataset.
    """

    known_jacobs = []
    for root, dirs, files in os.walk(base_dir):
        for folder in dirs:
            known_jacobs.append(os.path.join(root, folder))
    num_data_targets = len(known_jacobs)
    data_known_jacobs = np.zeros([num_data_targets, 3, 100, 100])

    for var in range(len(known_jacobs)):
        # g_name = get_pkg_data_filename(known_jacobs[var] + '/g_WCSClipped.fits')
        # r_name = get_pkg_data_filename(known_jacobs[var] + '/r_WCSClipped.fits')
        # i_name = get_pkg_data_filename(known_jacobs[var] + '/i_WCSClipped.fits')

        g_name = get_pkg_data_filename(known_jacobs[var] + '/g_norm.fits')
        r_name = get_pkg_data_filename(known_jacobs[var] + '/r_norm.fits')
        i_name = get_pkg_data_filename(known_jacobs[var] + '/i_norm.fits')

        g = fits.open(g_name)[0].data[0:100, 0:100]
        r = fits.open(r_name)[0].data[0:100, 0:100]
        i = fits.open(i_name)[0].data[0:100, 0:100]

        data_known_jacobs[var] = [g, r, i]
    return data_known_jacobs

def getUnknown(num, base_dir='KnownLenses'):
    """
    This gets the unseen g, r, and i unknown/negative images, according to the number specified.
    This is so that the correct number of unknown images, is retrieved, according to the
    unseen known lenses.

    Args:
        num (integer):              This is the number specified, which equates to how many unseen known
                                    gravitational lenses.
                                    This indicates how many unseen negative images is to be retrieved.
        base_dir (string):          This is the root directory file path of where the unknown lenses are situated in.
    Returns:
        data_unknown (numpy array):  This is the numpy array of the unknown dataset.
    """

    if num == 47:
        path_unknown = '%s/Unknown_Processed_47' % base_dir
    elif num == 84:
        path_unknown = '%s/Unknown_Processed_84' % base_dir
    elif num == 131:
        path_unknown = '%s/Unknown_Processed_131' % base_dir
    elif num == 1000:
        path_unknown = '%s/Unknown_Processed_1000' % base_dir

    folders_unknown = []
    for root, dirs, files in os.walk(path_unknown):
        for folder in dirs:
            folders_unknown.append(os.path.join(root, folder))

    num_data_targets = len(folders_unknown)
    data_unknown = np.zeros([num_data_targets, 3, 100, 100])

    for var in range(len(folders_unknown)):
        # g_name = get_pkg_data_filename(folders_unknown[var]+'/g_WCSClipped.fits')
        # r_name = get_pkg_data_filename(folders_unknown[var]+'/r_WCSClipped.fits')
        # i_name = get_pkg_data_filename(folders_unknown[var]+'/i_WCSClipped.fits')

        g_name = get_pkg_data_filename(folders_unknown[var] + '/g_norm.fits')
        r_name = get_pkg_data_filename(folders_unknown[var] + '/r_norm.fits')
        i_name = get_pkg_data_filename(folders_unknown[var] + '/i_norm.fits')

        g = fits.open(g_name)[0].data[0:100, 0:100]
        r = fits.open(r_name)[0].data[0:100, 0:100]
        i = fits.open(i_name)[0].data[0:100, 0:100]

        data_unknown[var] = [g, r, i]

    return data_unknown

def getPositiveSimulated1000(base_dir='NewLenses/PositiveWithDESSky'):
    folders = {}
    for root, dirs, files in os.walk(base_dir):
        for folder in dirs:
            key = folder
            value = os.path.join(root, folder)
            folders[key] = value

    # number of Positive DataPoints
    num_data_target = len(folders)

    data_pos_1000 = np.zeros([num_data_target, 3, 100, 100])

    # key is name of folder number
    # value is the number of the folder to be added to the file name

    counter = 0
    for key, value in folders.items():
        g_name = get_pkg_data_filename(value + '/' + str(key) + '_g_norm.fits')
        r_name = get_pkg_data_filename(value + '/' + str(key) + '_r_norm.fits')
        i_name = get_pkg_data_filename(value + '/' + str(key) + '_i_norm.fits')

        # g_name = get_pkg_data_filename(value + '/' + str(key) + '_posSky_g.fits')
        # r_name = get_pkg_data_filename(value + '/' + str(key) + '_posSky_r.fits')
        # i_name = get_pkg_data_filename(value + '/' + str(key) + '_posSky_i.fits')

        g = fits.open(g_name)[0].data[0:100, 0:100]
        r = fits.open(r_name)[0].data[0:100, 0:100]
        i = fits.open(i_name)[0].data[0:100, 0:100]

        data_pos_1000[counter] = [g, r, i]
        counter += 1
        # just to run, and use less things
        # if counter > 1500:
        #     break
    return data_pos_1000

def getTestSet():
    data_des_2017 = getDES2017()
    negative_47 = getUnknown(47)

    # data_jacobs = getJacobs()
    # data_known_131 = np.vstack((data_des_2017,data_jacobs))
    # negative_131 = getUnknown(131)

    images, labels = loadImage(data_des_2017, negative_47)

    # data_pos_1000 = getPositiveSimulated1000()
    # unknown_1000 = getUnknown(1000)
    # images, labels = loadImage(data_pos_1000, unknown_1000)

    return images, labels

def testDES2017():
    """
    This tests the unseen DES2017 images and unknown 47 images, to get the accuracy rate
    of these unseen images that aren't used in training.

    Returns:
        known_des2017_array (numpy array):    This is the numpy array of the known DES2017 images.
        accuracy_score_47 (float):           This is the accuracy score of the 47 unseen unknown images and of the
                                            47 images from DES2017, being tested on the already learnt set.
        k_fold_accuracy_47(float):            This is the accuracy score of the 47 unseen unknown images and of the 47
                                            images from DES2017 after k fold cross validation, being
                                            tested on the already learnt set.

    """

    known_des2017_array = getDES2017()

    num = 47
    unknown_array = getUnknown(num)

    image_test, labels_test = loadImage(known_des2017_array, unknown_array)
    x_image_test = image_test.reshape(image_test.shape[0], image_test.shape[1] * image_test.shape[2] * image_test.shape[
        3])  # batch size, height*width*3channels

    encoder = LabelEncoder()
    y_image_labels = encoder.fit_transform(labels_test)

    y_pred = clf_images.predict(x_image_test)
    accuracy_score_47 = (accuracy_score(y_image_labels, y_pred)) * 100.0

    results = model_selection.cross_val_score(clf_images, x_image_test, y_image_labels, cv=k_fold)
    k_fold_accuracy_47 = (results.mean()) * 100.0
    k_fold_std_47 = results.std()

    return known_des2017_array, accuracy_score_47, k_fold_accuracy_47, k_fold_std_47

def testJacobs():
    """
    This tests the unseen Jacobs images and unknown 84 images, to get the accuracy rate
    of these unseen images that aren't used in training.

    Returns:
        known_jacobs_array (numpy array):     This is the numpy array of the known Jacobs images.
        accuracy_score_84 (float):           This is the accuracy score of the 84 unseen unknown images and of the
                                            84 images from Jacobs, being tested on the already learnt set.
        k_fold_accuracy_84(float):            This is the accuracy score of the 84 unseen unknown images and of the 84
                                            images from Jacobs after k fold cross validation, being
                                            tested on the already learnt set.

    """

    known_jacobs_array = getJacobs()

    num = 84
    unknown_array = getUnknown(num)

    image_jacobs_test, labels_jacobs_test = loadImage(known_jacobs_array, unknown_array)
    x_image_test = image_jacobs_test.reshape(image_jacobs_test.shape[0],
                                             image_jacobs_test.shape[1] * image_jacobs_test.shape[2] *
                                             image_jacobs_test.shape[
                                                 3])  # batchsize, height*width*3channels

    encoder = LabelEncoder()
    y_image_labels = encoder.fit_transform(labels_jacobs_test)

    y_pred = clf_images.predict(x_image_test)
    accuracy_score_84 = (accuracy_score(y_image_labels, y_pred)) * 100

    results = model_selection.cross_val_score(clf_images, x_image_test, y_image_labels, cv=k_fold)
    k_fold_accuracy_84 = (results.mean()) * 100
    k_fold_std_84 = results.std()

    return known_jacobs_array, accuracy_score_84, k_fold_accuracy_84, k_fold_std_84

def testDES2017AndJacobs(known_des2017_array, known_jacobs_array):
    """
    This tests the unseen DES2017 and Jacobs images together with the unknown 131 images, to get the accuracy rate
    of these unseen images that aren't used in training.

    Args:
        known_des2017_array (numpy array):    This is the dataset of the unseen known DES2017 images.
        known_jacobs_array (numpy array):     This is the dataset of the unseen known Jacobs images.
    Returns:
        accuracy_score_131 (float):          This is the accuracy score of the 131 unseen unknown images and of the
                                            131 images from DES2017 and Jacobs, being tested on the already learnt set.
        k_fold_accuracy_131(float):           This is the accuracy score of the 131 unseen unknown images and of the 131
                                            images from DES2017 and Jacobs after k fold cross validation, being
                                            tested on the already learnt set.
    """

    all_known_array = np.vstack((known_des2017_array, known_jacobs_array))

    num = 131
    unknown_array = getUnknown(num)

    image_known_test, labels_known_test = loadImage(all_known_array, unknown_array)
    x_image_test = image_known_test.reshape(image_known_test.shape[0],
                                            image_known_test.shape[1] * image_known_test.shape[2] *
                                            image_known_test.shape[
                                                3])  # batchsize, height*width*3channels

    encoder = LabelEncoder()
    y_image_labels = encoder.fit_transform(labels_known_test)

    y_pred = clf_images.predict(x_image_test)

    accuracy_score_131 = (accuracy_score(y_image_labels, y_pred)) * 100

    results = model_selection.cross_val_score(clf_images, x_image_test, y_image_labels, cv=k_fold)
    k_fold_accuracy_131 = (results.mean()) * 100
    k_fold_std_131 = results.std()

    return accuracy_score_131, k_fold_accuracy_131, k_fold_std_131

def makeTrainTest(positive_array, negative_array):
    """
    This makes the training and testing data sets that are to be made. This creates
    a training image data set with the positive and negative images together. This
    also creates a training label data set with the positive and negative images together.

    Args:
        positive_array (numpy array):    This is the positively simulated dataset images.
        negative_array (numpy array):    This is the negative DES images.
    Returns:
        x_train (numpy array):          This is the array of the training set of the training images,
                                        which is 80% of the image training set.
        x_test (numpy array):           This is the array of the testing set of the training images, which
                                        is the 20% of the training images.
        y_train (numpy array):          This is the array of the labels of the training labels, which is 80%
                                        of the training labels.
        y_test (numpy array):           This is the array of the labels of the testing labels, which is 20%
                                        of the training labels.
        train_percent (float):          This is the percentage of data used for training (1- test_percent).
        test_percent (float):           This is the percentage of date used for testing.
        image_train_std (float):         This is the standard deviation of the entire training set, all 20000 images.
        image_train_mean (float):        This is the mean of the entire training set, all 20000 images.
        image_train_shape (list):        This is the shape of the entire training set, all 20000 images.
        image_labels_shape (list):       This is the shape of the entire training sets' labels, all 20000 labels.
        x_train_shape (list):            This is the shape of the training set of the images.
        x_test_shape (list):             This is the shape of the testing set of the images.
        y_train_shape (list):            This is the shape of the training set of the labels.
        y_test_shape (list):             This is the shape of the testing set of the labels.
    """

    image_train, image_labels = loadImage(positive_array, negative_array)
    image_train_std = image_train.std()
    image_train_mean = image_train.mean()
    image_train_shape = image_train.shape
    print("image_train shape: " + str(image_train_shape))
    image_labels_shape = image_labels.shape

    image_new_test, labels_new_test = getTestSet()

    # reshape x
    image_train_reshaped = image_train.reshape(image_train.shape[0], image_train.shape[1] * image_train.shape[2] *
                                               image_train.shape[3])  # batch size, height*width*3channels
    # print("x shape: " + str(x.shape))

    image_new_test_reshaped = image_new_test.reshape(image_new_test.shape[0], image_new_test.shape[1] *
                                                     image_new_test.shape[2] * image_new_test.shape[3])
    # Encoding y now
    encoder = LabelEncoder()
    y_labels_train = encoder.fit_transform(image_labels)
    y_labels_new = encoder.fit_transform(labels_new_test)

    # Doing a train-test split with sklearn, to train the data, where 20% of the training data is used for the test data
    test_percent = 0.2
    # x_train, x_test, y_train, y_test = train_test_split(x, y, shuffle=True, test_size=test_percent)

    x_train = shuffle(image_train_reshaped)
    y_train = shuffle(y_labels_train)
    x_test = shuffle(image_new_test_reshaped)
    y_test = shuffle(y_labels_new)

    x_train_shape = x_train.shape
    x_test_shape = x_test.shape
    y_train_shape = y_train.shape
    y_test_shape = y_test.shape
    print("x_train: " + str(x_train_shape))
    print("y_train: " + str(y_train.shape))
    print("x_test shape: " + str(x_test_shape))
    print("y_test shape: " + str(y_train.shape))

    train_percent = (1 - test_percent)

    return (x_train, x_test, y_train, y_test, train_percent, test_percent, image_train_std, image_train_mean,
            image_train_shape, image_labels_shape, x_train_shape, x_test_shape, y_train_shape, y_test_shape)

# _____________________________________________________________________________________________________________________________
# MAIN

positive_array = getPositiveSimulatedTrain()
negative_array = getNegativeDES()

x_train, x_test, y_train, yTest, trainPercent, test_percent, image_train_std, image_train_mean, image_train_shape, \
image_labels_shape, x_train_shape, x_test_shape, y_train_shape, y_test_shape = makeTrainTest(
    positive_array, negative_array)

# Trianing the data with MLPClassifier, from scikit learn
clf_images = MLPClassifier(activation='relu',
                           hidden_layer_sizes=(100, 100, 100),  # 3 layers of 100 neurons each
                           solver='adam',
                           verbose=True,
                           max_iter=100,
                           batch_size=200,
                           early_stopping=True)  # batch size = 200 default

description = str(clf_images)

# Getting training loss
clf_images.fit(x_train, y_train)
train_loss = clf_images.loss_curve_

# Accuracy Testing
y_pred = clf_images.predict(x_test)
model_accuracy_score = (accuracy_score(yTest, y_pred)) * 100

# Testing number of ones and zeroes
y_test_index = np.round(y_pred)
Ones = np.count_nonzero(y_test_index == 1)
Zeroes = np.count_nonzero(y_test_index == 0)
print("Ones: %s / 1000" % (Ones))
print("Zeroes: %s / 1000" % (Zeroes))

# Getting validation loss
clf_images.fit(x_test, yTest)
val_loss = clf_images.loss_curve_

epochs = range(1, 30)
plt.plot(train_loss, label='Training Loss')
plt.plot(val_loss, label='Validation Loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.savefig('../Results/TrainingvsValidationLoss_Sklearn.png')

# Cross Validation
n_splits = 10
random_state = 100
k_fold = model_selection.KFold(n_splits=n_splits, random_state=random_state)
score = model_selection.cross_val_score(clf_images, x_test, yTest, scoring='accuracy', cv=k_fold)
k_fold_accuracy = (score.mean()) * 100
k_fold_std = score.std()

fig4 = plt.figure()
plt.plot(score, label='Scores')
plt.legend()
fig4.savefig('../Results/SkLearnKFold_Scores.png')

# ______________________________________________________________________________________________________________________
known_des2017, accuracy_score_47, k_fold_accuracy_47, k_fold_std_47 = testDES2017()
known_jacobs, accuracy_score_84, k_fold_accuracy_84, k_fold_std_84 = testJacobs()
accuracy_score_131, k_fold_accuracy_131, k_fold_std_131 = testDES2017AndJacobs(known_des2017, known_jacobs)

print("Accuracy Score: " + str(model_accuracy_score))
print("Accuracy Type: " + str(type(model_accuracy_score)))
print("K Fold Accuracy: "+str(k_fold_accuracy))
print("K Fold Std: " + str(k_fold_std))
print("Accuracy_47: " + str(accuracy_score_47))
print("K Fold Accuracy_47: " + str(k_fold_accuracy_47))
print("K Fold Std_47: " + str(k_fold_std_47))
print("Accuracy_84: " + str(accuracy_score_84))
print("K Fold Accuracy_84: " + str(k_fold_accuracy_84))
print("K Fold Std_84: " + str(k_fold_std_84))
print("Accuracy_131: " + str(accuracy_score_131))
print("K Fold Accuracy_131: " + str(k_fold_accuracy_131))
print("K Fold Std_131: " + str(k_fold_std_131))

# write to ml_Lenses_results.xlsx
# makeExcelTable.makeInitialTable()
elementList = makeExcelTable.getElementList(description, image_train_std, image_train_mean, image_train_shape,
                                            image_labels_shape, trainPercent, test_percent, x_train_shape, x_test_shape,
                                            y_train_shape, y_test_shape, n_splits, random_state, model_accuracy_score,
                                            k_fold_accuracy,
                                            k_fold_std, accuracy_score_47, k_fold_accuracy_47, k_fold_std_47,
                                            accuracy_score_84, k_fold_accuracy_84, k_fold_std_84, accuracy_score_131,
                                            k_fold_accuracy_131, k_fold_std_131)
filename = '../Results/ml_Lenses_results.csv'
makeExcelTable.appendRowAsList(filename, elementList)

max_stars_repo_name: Annarien/GravitationalLenses

max_issues_repo_name: Annarien/GravitationalLenses

max_forks_repo_name: Annarien/GravitationalLenses

reason: import numpy,from astropy


text:
import numpy as np
from pyV2DL3.eventdisplay.IrfExtractor import find_closest_az
from pyV2DL3.eventdisplay.IrfExtractor import find_nearest

def test_find_nearest():
    az_centers = np.array(
        [
            187.5,
            206.25,
            232.5,
            255.0,
            277.5,
            300.0,
            322.5,
            345.0,
            7.5,
            30.0,
            52.5,
            75.0,
            97.5,
            120.0,
            142.5,
            161.25,
        ]
    )
    az_50 = find_nearest(az_centers, 50.0)
    az_188 = find_nearest(az_centers, 188.0)

    assert az_50 == 10 and az_188 == 0

def test_find_closest_az():
    azMins = np.array(
        [
            -1000.0,
            -180.0,
            -157.5,
            -135.0,
            -112.5,
            -90.0,
            -67.5,
            -45.0,
            -22.5,
            0.0,
            22.5,
            45.0,
            67.5,
            90.0,
            112.5,
            135.0,
            150.0,
        ]
    )
    azMaxs = np.array(
        [
            -165.0,
            -150.0,
            -120.0,
            -97.5,
            -75.0,
            -52.5,
            -30.0,
            -7.5,
            15.0,
            37.5,
            60.0,
            82.5,
            105.0,
            127.5,
            150.0,
            172.5,
            1000.0,
        ]
    )
    az_bin_to_store_1 = find_closest_az(146.20, azMins, azMaxs)
    az_bin_to_store_2 = find_closest_az(-180.0, azMins, azMaxs)
    az_bin_to_store_3 = find_closest_az(320.0, azMins, azMaxs)
    assert az_bin_to_store_1 == 14 and az_bin_to_store_2 == 8 and az_bin_to_store_3 == 6

if __name__ == "__main__":
    test_find_nearest()
    test_find_closest_az()

ext: py

reason: import numpy


text:
#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import requests
import re
import numpy as np
from bs4 import BeautifulSoup

# In[ ]:

root = "http://gyakorikerdesek.hu"
page = requests.get(root)
soup = BeautifulSoup(page.content, 'html.parser')
html = list(soup.children)[2]

menus = []
for menu in soup.find_all(class_='menu'):
    menus.append(menu.a.get("href"))

page = requests.get(root + menus[1])
allatok = BeautifulSoup(page.content, 'html.parser')

# In[ ]:

def get_valasz(page):
    valasz_hasznos = re.compile("A válasz (\d+)%-ban hasznosnak tűnik.")
    iro_hasznos = re.compile("A válaszíró (\d+)%-ban hasznos válaszokat ad.")
    valasz_page = page.find_all("td", class_="valaszok vtop")
    return_valasz = None
    if valasz_page:
        valaszok = []
        scoreok = []
        for val in valasz_page:
            val_text = val.get_text().strip("\n")
            if val_text:
                valasz_m = valasz_hasznos.findall(val_text)
                iro_m = iro_hasznos.findall(val_text)
                if valasz_m and iro_m:
                    avg = (int(valasz_m[0]) + int(iro_m[0])) / 2
                    valasz = valasz_hasznos.search(val_text)
                    valasz = val_text.split(valasz.group())
                    valaszok.append(valasz[0])
                    scoreok.append(avg)
        if valaszok and scoreok:
            return_valasz = valaszok[np.argmax(scoreok)]
    return return_valasz

# In[ ]:

corp = {}
for i,kerdes in enumerate(allatok.find_all(class_="kerdes_lista")[1].find_all("tr")):
    try:
        kerdes_url = kerdes.find("a").get("href")
        subpage = requests.get(root + kerdes_url)
        allatok_kerdes = BeautifulSoup(subpage.content, 'html.parser')
        hosszu_kerdes = allatok_kerdes.find(class_="kerdes").find_all("td")[2].get_text()
        rovid_kerdes = allatok_kerdes.find(class_="kerdes").find_all("td")[2].h1.get_text()
        kategoriak = []
        for kategoria in allatok_kerdes.find("select").find_all("option")[1:]:
            kategoriak.append(kategoria.get_text())
        valasz = get_valasz(allatok_kerdes)
        if valasz:
            corp[i] = {"rovid_kerdes": rovid_kerdes, "hosszu_kerdes": hosszu_kerdes, "kategoriak": kategoriak, "valasz": valasz}
        print(i)
    except Exception as e:
        print(e)
        continue

# In[ ]:

for k in range(1,1000):
    page = requests.get(root + "/allatok__oldal-"+str(k))
    allatok = BeautifulSoup(page.content, 'html.parser')
    kerdesek_raw = allatok.find_all(class_="kerdes_lista")
    if len(kerdesek_raw) == 1:
        kerdesek = kerdesek_raw[0].find_all("tr")
    elif len(kerdesek_raw) == 2:
        kerdesek = kerdesek_raw[1].find_all("tr")
    for kerdes in kerdesek:
        try:
            kerdes_url = kerdes.find("a").get("href")
            subpage = requests.get(root + kerdes_url)
            allatok_kerdes = BeautifulSoup(subpage.content, 'html.parser')
            hosszu_kerdes = allatok_kerdes.find(class_="kerdes").find_all("td")[2].get_text()
            rovid_kerdes = allatok_kerdes.find(class_="kerdes").find_all("td")[2].h1.get_text()
            kategoriak = []
            for kategoria in allatok_kerdes.find("select").find_all("option")[1:]:
                kategoriak.append(kategoria.get_text())
            valasz = get_valasz(allatok_kerdes)
            if valasz:
                corp[i] = {"rovid_kerdes": rovid_kerdes, "hosszu_kerdes": hosszu_kerdes, "kategoriak": kategoriak, "valasz": valasz}
            i += 1
            print(i)
        except Exception as e:
            print(e)
            continue
    print("page: " + str(k))

felhasznalo_kerdese = re.compile("(^\w+) nevű felhasználó kérdése:")
felhasznalo_valasza = re.compile("(^\w+) nevű felhasználó válasza:")
inds = []
for c in corp:
    hosszu_kerdes = corp[c]["hosszu_kerdes"].strip("\n").replace("(adsbygoogle = window.adsbygoogle || []).push({});","").split("\n")
    hosszu_kerdes = " ".join(hosszu_kerdes)
    valasz_unfiltered = corp[c]["valasz"]
    m = felhasznalo_valasza.search(valasz_unfiltered)
    if m:
        valasz_unfiltered = valasz_unfiltered.replace(m.group(), "")
    hosszu_valasz = valasz_unfiltered.strip("\n").replace("(adsbygoogle = window.adsbygoogle || []).push({});","").replace("\n", " ").strip()
    corp[c]["valasz"] = hosszu_valasz
    if "Keress kérdéseket hasonló témákban:" in hosszu_kerdes:
        kerdes = hosszu_kerdes.split("Keress kérdéseket hasonló témákban:")
        kerdes_text = kerdes[0].strip().replace("\n"," ")
        m = felhasznalo_kerdese.search(kerdes_text)
        if m:
            kerdes_text = kerdes_text.replace(m.group(), "")
        corp[c]["hosszu_kerdes"] = kerdes_text
        corp[c]["keywords"] = [i.strip() for i in kerdes[1].split(",")]
        kat = corp[c]["kategoriak"]
        kat_filt = [i.replace("» ", "") for i in kat]
        corp[c]["kategoriak"] = kat_filt
        inds.append(c)

filtered_corp = [corp[i] for i in corp if i in inds]
unique_corp = []
rovid_kerdesek = []
for i in filtered_corp:
    if i["rovid_kerdes"] not in rovid_kerdesek:
        unique_corp.append(i)
        rovid_kerdesek.append(i["rovid_kerdes"])

import json
with open("gyakori_corpus_allatok", "w+") as f:
    s = json.dumps(unique_corp)
    f.write(s)


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import argparse
import numpy as np
import shlex
import subprocess
import sys
import wave
import ffmpeg
import json
import os
from pathlib import Path

from deepspeech import Model, version
from timeit import default_timer as timer

try:
    from shhlex import quote
except ImportError:
    from pipes import quote

export_json = False
debug_mode = False
def print_debug(text):
    if debug_mode:
        print(text)

def token_to_string(token):
    return "Text: " + (token.text if token.text else "\" \"") + ", start_time: "+str(token.start_time)+", timestep: "+str(token.timestep)

print("Executed client.py")
def convert_samplerate(audio_path, desired_sample_rate):
    prev_path = os.getcwd()
    mono_path = get_original_name(audio_path)
    os.chdir("./sox")
    sox_cmd = "\'"+str(os.getcwd()).replace("/", "\\")+'\sox.exe\' {} --type raw --bits 16 --channels 1 --rate {} --encoding signed-integer --endian little --compression 0.0 --no-dither - '.format(quote(audio_path.replace("/", "\\")), desired_sample_rate)
    try:
        output = subprocess.check_output(shlex.split(sox_cmd), stderr=subprocess.PIPE)
    except subprocess.CalledProcessError as e:
        raise RuntimeError('SoX returned non-zero status: {}'.format(e.stderr))
    except OSError as e:
        raise OSError(e.errno, 'SoX not found, use {}hz files or install it: {}'.format(desired_sample_rate, e.strerror))

    os.chdir(prev_path)
    return desired_sample_rate, np.frombuffer(output, np.int16)

def metadata_to_string(metadata):
    return ''.join(token.text for token in metadata.tokens)

def words_from_candidate_transcript(metadata):
    word = ""
    word_list = []
    word_start_time = 0
    # Loop through each character
    for i, token in enumerate(metadata.tokens):
        # Append character to word if it's not a space
        if token.text != " ":
            if len(word) == 0:
                # Log the start time of the new word
                word_start_time = token.start_time

            word = word + token.text
        # Word boundary is either a space or the last character in the array
        if token.text == " " or i == len(metadata.tokens) - 1:
            word_duration = token.start_time - word_start_time

            if word_duration < 0:
                word_duration = 0

            each_word = dict()
            each_word["word"] = word
            each_word["start_time"] = round(word_start_time, 4)
            each_word["duration"] = round(word_duration, 4)

            word_list.append(each_word)
            # Reset
            word = ""
            word_start_time = 0

    return word_list

def metadata_json_output(metadata):
    json_result = dict()
    json_result["transcripts"] = [{
        "confidence": transcript.confidence,
        "words": words_from_candidate_transcript(transcript),
    } for transcript in metadata.transcripts]
    return json.dumps(json_result, indent=2)

class VersionAction(argparse.Action):
    def __init__(self, *args, **kwargs):
        super(VersionAction, self).__init__(nargs=0, *args, **kwargs)

    def __call__(self, *args, **kwargs):
        print('DeepSpeech ', version())
        exit(0)

def main():
    parser = argparse.ArgumentParser(description='Running DeepSpeech inference.')
    parser.add_argument('--model', required=True,
                        help='Path to the model (protocol buffer binary file)')
    parser.add_argument('--scorer', required=False,
                        help='Path to the external scorer file')
    parser.add_argument('--audio', required=True,
                        help='Path to the audio file to run (WAV format)')
    parser.add_argument('--beam_width', type=int,
                        help='Beam width for the CTC decoder')
    parser.add_argument('--lm_alpha', type=float,
                        help='Language model weight (lm_alpha). If not specified, use default from the scorer package.')
    parser.add_argument('--lm_beta', type=float,
                        help='Word insertion bonus (lm_beta). If not specified, use default from the scorer package.')
    parser.add_argument('--version', action=VersionAction,
                        help='Print version and exits')
    parser.add_argument('--extended', required=False, action='store_true',
                        help='Output string from extended metadata')
    parser.add_argument('--json', required=False, action='store_true',
                        help='Output json from metadata with timestamp of each word')
    parser.add_argument('--candidate_transcripts', type=int, default=3,
                        help='Number of candidate transcripts to include in JSON output')
    parser.add_argument('--hot_words', type=str,
                        help='Hot-words and their boosts.')
    parser.add_argument('--output', type=str,
                        help='Output folder path where the srt file will be saved.')
    parser.add_argument('--srt_name', type=str,
                        help='Name the output srt file will have.')
    parser.add_argument('--fake_video', type=str,
                        help='It will create a video using subprocess and an image.')
    parser.add_argument('--video_to_audio', type=str,
                        help='It will create a video using subprocess and an image.')
    args = parser.parse_args()

    print('Loading model from file {}'.format(args.model), file=sys.stderr)
    model_load_start = timer()
    # sphinx-doc: python_ref_model_start
    ds = Model(args.model)
    # sphinx-doc: python_ref_model_stop
    model_load_end = timer() - model_load_start
    print('Loaded model in {:.3}s.'.format(model_load_end), file=sys.stderr)

    if args.beam_width:
        ds.setBeamWidth(args.beam_width)

    desired_sample_rate = ds.sampleRate()

    if args.scorer:
        print('Loading scorer from files {}'.format(args.scorer), file=sys.stderr)
        scorer_load_start = timer()
        ds.enableExternalScorer(args.scorer)
        scorer_load_end = timer() - scorer_load_start
        print('Loaded scorer in {:.3}s.'.format(scorer_load_end), file=sys.stderr)

        if args.lm_alpha and args.lm_beta:
            ds.setScorerAlphaBeta(args.lm_alpha, args.lm_beta)

    if args.hot_words:
        print('Adding hot-words', file=sys.stderr)
        for word_boost in args.hot_words.split(','):
            word,boost = word_boost.split(':')
            ds.addHotWord(word,float(boost))

    if args.audio[-3:] != "wav":
        print("Video detectect, extracting audio file")
        args.audio = get_audio_from_file(args.audio)

    fin = wave.open(args.audio, 'rb')
    fs_orig = fin.getframerate()
    is_mono = fin.getnchannels() == 1

    if fs_orig != desired_sample_rate or not is_mono:
        print('Warning: original sample rate ({}) is different than {}hz. Resampling might produce erratic speech recognition.'.format(fs_orig, desired_sample_rate), file=sys.stderr)
        fs_new, audio = convert_samplerate(args.audio, desired_sample_rate)
    else:
        audio = np.frombuffer(fin.readframes(fin.getnframes()), np.int16)

    audio_length = fin.getnframes() * (1/fs_orig)
    fin.close()

    print('Running inference.', file=sys.stderr)
    inference_start = timer()
    # sphinx-doc: python_ref_inference_start
    if args.extended:
        #metadata = ds.sttWithMetadata(audio, 1)
        #print("metadata")
        #print(metadata)
        #for id, transcript in metadata.transcripts:
        #    print(i)
        #    print(transcript.confidence)
        #    print(words_from_candidate_transcript(transcript))
        #metadata = metadata.transcripts[0].tokens
        obj = ds.sttWithMetadata(audio, 1)
        metadata = obj.transcripts[0].tokens
        #################################################
        if len(metadata) == 0:
            sys.exit("Sorry, this audio seems unintelligible or too short")

        res = process_captions(metadata)
        print_debug("Finished reading, the final result:")
        print_debug(res)
        #f = open("output.srt", "w")

        path_segmented = get_original_name(args.audio)
        path_to_save = generate_path(path_segmented, args.output, args.srt_name)

        if os.path.exists(path_to_save):
            os.remove(path_to_save)
        f = open(path_to_save, "w")
        f.write(res)
        f.close()
        print("Successfully saved in "+path_to_save)
        if export_json:
            jsonfile = open("results.json", "w")
            json.dumps(data, jsonfile)
            jsonfile.close()

        #print(metadata)
        #print(metadata_to_string(ds.sttWithMetadata(audio, 1).transcripts[0]))
    elif args.json:
        print(metadata_json_output(ds.sttWithMetadata(audio, args.candidate_transcripts)))
    else:
        print(ds.stt(audio))
    # sphinx-doc: python_ref_inference_stop
    inference_end = timer() - inference_start
    print('Inference took %0.3fs for %0.3fs audio file.' % (inference_end, audio_length), file=sys.stderr)

def process_captions(metadata):
    index = 1
    threshold_soft = 0.05 # if exceed and possible, next line
    threshold_hard = 0.7  # if exceed, next block
    offset = 2
    limit_characters_line = 35
    #s = ""
    word = ""
    line = ["", ""]
    res = ""
    #new_block = True
    new_word = True
    prev = metadata[0].start_time
    block_start_time = metadata[0].start_time
    block_end_time = 0
    before_read_time = metadata[0].start_time
    after_read_time = 0
    line_count = 0 # Each block has a maximum of 2 lines
    data = []
    for id, t in enumerate(metadata):
        if export_json:
            data.append({"text":t.text, "start_time": t.start_time, "timestep": t.timestep})
        #block_end_time = t.start_time
        #if new_block:
        #    before_read_time = t.start_time
        #    new_block = False
        if new_word:
            print_debug("\nNew word started at "+str(t.start_time))
            before_read_time = t.start_time
            new_word = False

        #if id > 23 and id < 90: print(t.text, t.start_time)

        # As we didn't found an space nor the space is long, we're still with the same word
        if t.text != " " and t.start_time - prev < threshold_hard: # and t.start_time - prev < threshold_hard:
            print_debug("Added letter to the word " + token_to_string(t))
            prev = t.start_time
            word += t.text
        # Word change
        else:
            if t.start_time - prev >= threshold_hard: # Silence greater than threshold
                print_debug("There was a great silence: "+str(t.start_time - prev)+", it shouldn't be greater than "+str(threshold_hard)+" to keep with the previous LINE")
                line[line_count] += word
                res += str(index) + "\n" + time_format(block_start_time) +" --> "+ time_format(prev) + "\n" + line[0] + "\n" + line[1]+"\n\n"
                print_debug("We save the already stated word with its timestamp\n")

                prev = t.start_time
                block_start_time = t.start_time
                line_count = 0
                line = ["", ""]
                word = t.text
                #new_block = True
                index += 1
            else: # word read
                #after_read_time = t.start_time
                print_debug("Complete read word: " + word)
                #print("id; ", id, "len meta: ", len(metadata))
                if len(line[line_count]) + len(word) > limit_characters_line: # Characters per line reached (new line or new block required)
                    if line_count == 0 : # and metadata[id+1].start_time - t.start_time < threshold_hard
                        print_debug("Line reached its limit, Saving the word in the next LINE")
                        line_count += 1
                        line[line_count] += word + " "
                        word = ""
                        print_debug("Lines result so far is:\n\tLinea 1: " +line[0]+ "\nLinea 2: "+line[1]+"\n")
                    else: # We're on the second line, therefore, we need a new block
                        print_debug("Line reached its limit, but we're at the second line, so we need a new BLOCK")
                        print_debug("Lines result so far is:\n\tLinea 1: " +str(line[0])+ "\nLinea 2: " + line[1])
                        print_debug("We save both line to the final result\n")
                        line_count = 0
                        res += str(index) + "\n" + time_format(block_start_time) +" --> "+ time_format(after_read_time) + "\n" + line[0] + "\n" + line[1]+"\n\n"
                        block_start_time = before_read_time
                        line = ["", ""]
                        line[line_count] = word + " "
                        word = ""
                        #new_block = True
                        index += 1

                else: # trivial case: we can save the word and continue reading.
                    print_debug("Finished reading word: " + word)
                    line[line_count] += word + " "
                    word = ""
                    new_word = True
                    after_read_time = t.start_time
    line[line_count] += word
    res += str(index) + "\n" + time_format(block_start_time) +" --> "+ time_format(metadata[len(metadata)-1].start_time) + "\n" + line[0] + "\n" + line[1]+"\n\n"
    return res

def time_format(seconds):
    mon, sec = divmod(seconds, 60)
    hr, mon = divmod(mon, 60)
    return ("{0:02.0f}:{1:02.0f}:{2:0>6.3f}".format(hr, mon, sec)).replace(".", ",")

def generate_path(path_segmented, output_path, output_name):
    if output_path:
        path_to_save = output_path
    else:
        path_to_save = path_segmented[0]

    if output_name:
        path_to_save = path_to_save + "/" + (output_name if output_name[-3:] == "srt" else output_name + ".srt")
    else:
        path_to_save = path_to_save + "/" + path_segmented[1] + ".srt"
    return path_to_save

def get_original_name(original_path):
    pathFragms = original_path.split("/")
    fileName = pathFragms[len(pathFragms)-1].split(".")[0]
    pathFragms.pop()
    return ["/".join(pathFragms), fileName]

def get_audio_from_file(audio_path):
    original_path = get_original_name(audio_path)
    new_path = original_path[0] + "/" + original_path[1] + ".wav"
    stream = ffmpeg.input(audio_path)
    stream = ffmpeg.output(stream.audio, new_path, ar=16000, ac=1)
    stream = ffmpeg.overwrite_output(stream)
    ffmpeg.run(stream)
    return new_path

def prueba():
    print("esteeee  "+str(os.getcwd()))

    print("esteeee2  "+str(os.getcwd()))

    stream = ffmpeg.input('test.mp4')
    stream = ffmpeg.output(stream.audio, 'test.wav', ar=16000)
    stream = ffmpeg.overwrite_output(stream)
    ffmpeg.run(stream)

if __name__ == '__main__':
    main()

max_stars_repo_name: JaviAibar/DeepSpeechUnity

max_issues_repo_name: JaviAibar/DeepSpeechUnity

max_forks_repo_name: JaviAibar/DeepSpeechUnity


text:
import torch
import torch.nn.functional as F
import numpy as np
import torch.nn as nn
import matplotlib.pyplot as plt
import cv2 as cv
import os
from torch.autograd import Variable
import colorsys
from skimage.measure import find_contours
from matplotlib.patches import Polygon
import random
import imageio

class FocalLoss(nn.Module):
    def __init__(self, gamma=0, alpha=None, size_average=True):
        super(FocalLoss, self).__init__()
        self.gamma = gamma
        self.alpha = alpha
        if isinstance(alpha,(float,int)): self.alpha = torch.Tensor([alpha,1-alpha])
        if isinstance(alpha,list): self.alpha = torch.Tensor(alpha)
        self.size_average = size_average

    def forward(self, input, target):
        if input.dim()>2:
            input = input.view(input.size(0),input.size(1),-1)  # N,C,H,W => N,C,H*W
            input = input.transpose(1,2)    # N,C,H*W => N,H*W,C
            input = input.contiguous().view(-1,input.size(2))   # N,H*W,C => N*H*W,C
        target = target.view(-1,1)

        logpt = F.log_softmax(input)
        logpt = logpt.gather(1,target)
        logpt = logpt.view(-1)
        pt = Variable(logpt.data.exp())

        if self.alpha is not None:
            if self.alpha.type()!=input.data.type():
                self.alpha = self.alpha.type_as(input.data)
            at = self.alpha.gather(0,target.data.view(-1))
            logpt = logpt * Variable(at)

        loss = -1 * (1-pt)**self.gamma * logpt
        if self.size_average: return loss.mean()
        else: return loss.sum()

def mask_cross_entropy(y_true, y_pred, alpha, mask=None):
    '''
    calculate the croos-entropy loss function with mask
    :param y_true: tensor, size=N*D*H*W
    :param y_pred: tensor, size=N*class_n*D*H*W
    :param mask: tensor, size=weights on each voxel  N*D*H*W
    :return: voxel weighted cross entropy loss
    '''
    log_prob = F.log_softmax(y_pred, dim=1)
    prob = F.softmax(y_pred, dim=1)
    shape = y_pred.shape
    y_true_tensor = onehot(y_true, shape[1])
    # loss = torch.cuda.FloatTensor([0])
    loss = 0
    for i in range(shape[1]):
        y_task = y_true_tensor[:,i,...]
        y_prob = log_prob[:, i, ...]
        if torch.is_tensor(mask):
            loss += torch.mean(-y_task * y_prob * mask)*alpha[i]
        else:
            loss += torch.mean(-y_task * y_prob)*alpha[i]
    return loss

def mask_cross_entropy_3d(y_true, y_pred, alpha, mask=None):
    '''
    calculate the croos-entropy loss function with mask
    :param y_true: tensor, size=N*slice*H*W
    :param y_pred: tensor, size=N*slice*H*W
    :param mask: tensor, size=weights on each voxel  N*D*H*W
    :return: voxel weighted cross entropy loss
    '''
    y_true_0 = y_true[:,0, ...]
    y_true_1 = y_true[:, 1, ...]
    y_true_2 = y_true[:, 2, ...]

    y_pred_0 = y_pred[:,0:2, ...]
    y_pred_1 = y_pred[:, 2:4, ...]
    y_pred_2 = y_pred[:, 4:6, ...]

    loss0 = mask_cross_entropy(y_true_0, y_pred_0, alpha)
    loss1 = mask_cross_entropy(y_true_1, y_pred_1, alpha)
    loss2 = mask_cross_entropy(y_true_2, y_pred_2, alpha)

    loss = loss1+loss2+loss0

    return loss

def mask_focal_loss(y_true, y_pred, alpha, gamma=0, mask=None):
    '''
    calculate the croos-entropy loss function with mask
    :param y_true: tensor, size=N*D*H*W
    :param y_pred: tensor, size=N*class_n*D*H*W
    :param mask: tensor, size=weights on each voxel  N*D*H*W
    :return: voxel weighted cross entropy loss
    '''
    log_prob = F.log_softmax(y_pred, dim=1)
    prob = F.softmax(y_pred, dim=1)
    shape = y_pred.shape
    y_true_tensor = onehot(y_true, shape[1])
    # loss = torch.cuda.FloatTensor([0])
    loss = 0
    assert isinstance(alpha, list)
    alpha = torch.Tensor(alpha)
    # gamma = torch.Tensor(gamma)
    for i in range(shape[1]):
        y_task = y_true_tensor[:,i,...]
        y_prob = log_prob[:, i, ...]
        focal_weight = (1-prob)**gamma
        if torch.is_tensor(mask):
            loss += torch.mean(-y_task * y_prob * mask*focal_weight)*alpha[i]
        else:
            loss += torch.mean(-y_task * y_prob*focal_weight)*alpha[i]
    return loss

def onehot(input, class_n):
    '''
    onehot for pytorch
    :param input: N*H*W
    :param class_n:
    :return:N*n_class*H*W
    '''
    shape = input.shape
    onehot = torch.zeros((class_n,)+shape).cuda()
    for i in range(class_n):
        onehot[i, ...] = (input == i)
    onehot_trans = onehot.permute(1,0,2,3)
    return onehot_trans

def onehot3d(input, class_n):
    '''
    onehot for pytorch
    :param input: N*H*W*D
    :param class_n:
    :return:N*n_class*H*W*D
    '''
    shape = input.shape
    onehot = torch.zeros((class_n,)+shape).cuda()
    for i in range(class_n):
        onehot[i, ...] = (input == i)
    onehot_trans = onehot.permute(1,0,2,3,4)
    return onehot_trans

# one-hot encoding method(efficient)
def onehot_encoding(array, class_num):
    '''
    the function turn a regular array to a one-hot representation form
    :param array: input array
    :param class_num: number of classes
    :return: one-hot encoding of array
    '''
    label_one_hot = np.zeros(array.shape + (class_num,), dtype="int16")
    for k in range(class_num):
        label_one_hot[..., k] = (array == k)
    return label_one_hot

def filter3D(image, kernel):
    '''
    do 3D convolution to an input ndarray
    :param image: 3d volume data W*H*D
    :param kernel: 2d filter
    :return: convolved result
    '''
    shape = image.shape
    convled_list = []
    for i in range(shape[0]):
        convolve = cv.filter2D(image[i, ...], -1, kernel)
        convled_list.append(convolve)
    out = np.array(convled_list)
    return out

def Canny3D(image, thresh1, thresh2):
    '''
    apply canny algorithm in a 3D fashion
    :param image:
    :param thresh1:
    :param thresh2:
    :return:
    '''
    shape = image.shape
    convled_list = []
    for i in range(shape[0]):
        convolve = cv.Canny(image[i, ...], threshold1=thresh1, threshold2=thresh2)
        convled_list.append(convolve)
    out = np.array(convled_list)
    out = out/255
    return out

def normalize(data):
    max = np.max(data)
    min = np.min(data)
    normalized = (data-min)/(max-min+0.00001)
    return normalized

def initialize_weights(*models):
    for model in models:
        for module in model.modules():
            if isinstance(module, nn.Conv2d) or isinstance(module, nn.Linear):
                nn.init.kaiming_normal(module.weight)
                if module.bias is not None:
                    module.bias.data.zero_()
            elif isinstance(module, nn.BatchNorm2d):
                module.weight.data.fill_(1)
                module.bias.data.zero_()

class Evaluation(object):
    def __init__(self):
        pass

    # save 3d volume as slices
    def save_slice_img(self, volume_path, output_path):
        file_name = os.path.basename(volume_path)
        output_dir  = os.path.join(output_path, file_name)
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        else:
            pass
        input_volume = nib.load(volume_path).get_data()
        # mapping to 0-1
        vol_max = np.max(input_volume)
        vol_min = np.min(input_volume)
        input_unit = (input_volume-vol_min)/(vol_max - vol_min)
        width, height, depth= input_unit.shape
        for i in range(0, depth):
            slice_path = os.path.join(output_dir, str(i)+'.png')
            img_i = input_unit[:, :, i]
            # normalize to 0-255
            img_i = (img_i*255).astype('uint8')
            # cv.imwrite(slice_path, img_i)
        return input_unit

    def save_slice_img_label(self, img_volume, pre_volume, gt_volume,
                             output_path, file_name, show_mask=False, show_gt = False):
        assert img_volume.shape == pre_volume.shape
        if show_gt:
            assert img_volume.shape == gt_volume.shape
        width, height, depth = img_volume.shape
        # gray value mapping   from MRI value to pixel value(0-255)
        volume_max = np.max(img_volume)
        volume_min = np.min(img_volume)
        volum_mapped = (img_volume-volume_min)/(volume_max-volume_min)
        volum_mapped = (255*volum_mapped).astype('uint8')
        # construct a directory for each volume to save slices
        dir_volume = os.path.join(output_path, file_name)
        if not os.path.exists(dir_volume):
            os.makedirs(dir_volume)
        else:
            pass
        for i in range(depth):
            img_slice = volum_mapped[:, :, i]
            pre_slice = pre_volume[:, :, i]
            if show_gt:
                gt_slice = gt_volume[:, :, i]
            else:
                gt_slice = []
            self.save_contour_label(img=img_slice, pre=pre_slice, gt=gt_slice,
                                    save_path=dir_volume, file_name=i,show_mask=show_mask,show_gt=show_gt)

    def apply_mask(self, image, mask, color, alpha=0.5):
        """Apply the given mask to the image.
        """
        for c in range(image.shape[-1]):
            image[:, :, c] = np.where(mask == 1,
                                      image[:, :, c] *
                                      (1 - alpha) + alpha * color[c] * 255,
                                      image[:, :, c])
        return image

    def random_colors(self, N, bright=True):
        """
        Generate random colors.
        To get visually distinct colors, generate them in HSV space then
        convert to RGB.
        """
        brightness = 1.0 if bright else 0.7
        hsv = [(i / N, 1, brightness) for i in range(N)]
        colors = list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv))
        random.shuffle(colors)
        return colors

    def save_contour_label(self, img, pre, save_path='',color="red", file_name=None, show_mask=False):
        # single channel to multi-channel
        img = np.expand_dims(img, axis=-1)
        img = np.tile(img, (1, 1, 3))
        height, width = img.shape[:2]
        _, ax = plt.subplots(1, figsize=(height, width))

        # Generate random colors
        # colors = self.random_colors(4)
        # Prediction result is illustrated as red and the groundtruth is illustrated as blue
        colors = [[1.0, 0, 0], [0, 0, 1.0]]
        if color == "red":
            color_used = colors[0]
        elif color == "blue":
            color_used = colors[1]
        else:
            raise Exception("unkown color")
        # Show area outside image boundaries.

        # ax.set_ylim(height + 10, -10)
        # ax.set_xlim(-10, width + 10)
        ax.set_ylim(height + 0, 0)
        ax.set_xlim(0, width + 0)
        ax.axis('off')
        # ax.set_title("volume mask")
        masked_image = img.astype(np.uint32).copy()

        if show_mask:
            masked_image = self.apply_mask(masked_image, pre, color_used)
        # Mask Polygon
        # Pad to ensure proper polygons for masks that touch image edges.
        padded_mask_pre = np.zeros(
            (pre.shape[0] + 2, pre.shape[1] + 2), dtype=np.uint8)
        padded_mask_pre[1:-1, 1:-1] = pre
        contours = find_contours(padded_mask_pre, 0.5)
        for verts in contours:
            # reduce padding and  flipping from (y, x) to (x, y)
            verts = np.fliplr(verts) - 1
            p = Polygon(verts, facecolor="none", edgecolor=color_used, linewidth=1)
            ax.add_patch(p)

        # reduce the blank part generated by plt and keep the original resolution
        fig = plt.gcf()
        fig.set_size_inches(height/37.5, width/37.5)
        plt.gca().xaxis.set_major_locator(plt.NullLocator())
        plt.gca().yaxis.set_major_locator(plt.NullLocator())
        plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0)
        plt.margins(0, 0)

        ax.imshow(masked_image.astype(np.uint8))
        # plt.show()
        fig.savefig('{}/{}.png'.format(save_path, file_name))
        # clear the image after saving
        plt.cla()
        plt.close(fig)

def make_one_hot(input, num_classes):
    """Convert class index tensor to one hot encoding tensor.

    Args:
         input: A tensor of shape [N, 1, *]
         num_classes: An int of number of class
    Returns:
        A tensor of shape [N, num_classes, *]
    """
    shape = np.array(input.shape)
    shape[1] = num_classes
    shape = tuple(shape)
    result = torch.zeros(shape)
    result = result.scatter_(1, input.cpu(), 1)

    return result

class BinaryDiceLoss(nn.Module):
    """Dice loss of binary class
    Args:
        smooth: A float number to smooth loss, and avoid NaN error, default: 1
        p: Denominator value: \sum{x^p} + \sum{y^p}, default: 2
        predict: A tensor of shape [N, *]
        target: A tensor of shape same with predict
        reduction: Reduction method to apply, return mean over batch if 'mean',
            return sum if 'sum', return a tensor of shape [N,] if 'none'
    Returns:
        Loss tensor according to arg reduction
    Raise:
        Exception if unexpected reduction
    """
    def __init__(self, smooth=1, p=2, reduction='mean'):
        super(BinaryDiceLoss, self).__init__()
        self.smooth = smooth
        self.p = p
        self.reduction = reduction

    def forward(self, predict, target):
        assert predict.shape[0] == target.shape[0], "predict & target batch size don't match"
        predict = predict.contiguous().view(predict.shape[0], -1)
        target = target.contiguous().view(target.shape[0], -1)

        num = torch.sum(torch.mul(predict, target), dim=1) + self.smooth
        den = torch.sum(predict.pow(self.p) + target.pow(self.p), dim=1) + self.smooth

        loss = 1 - num / den

        if self.reduction == 'mean':
            return loss.mean()
        elif self.reduction == 'sum':
            return loss.sum()
        elif self.reduction == 'none':
            return loss
        else:
            raise Exception('Unexpected reduction {}'.format(self.reduction))

class DiceLoss(nn.Module):
    """Dice loss, need one hot encode input
    Args:
        weight: An array of shape [num_classes,]
        ignore_index: class index to ignore
        predict: A tensor of shape [N, C, *]
        target: A tensor of same shape with predict
        other args pass to BinaryDiceLoss
    Return:
        same as BinaryDiceLoss
    """
    def __init__(self, weight=None, ignore_index=None, **kwargs):
        super(DiceLoss, self).__init__()
        self.kwargs = kwargs
        self.weight = weight
        self.ignore_index = ignore_index

    def forward(self, predict, target):
        assert predict.shape == target.shape, 'predict & target shape do not match'
        dice = BinaryDiceLoss(**self.kwargs)
        total_loss = 0
        predict = F.softmax(predict, dim=1)

        for i in range(target.shape[1]):
            if i != self.ignore_index:
                dice_loss = dice(predict[:, i], target[:, i])
                if self.weight is not None:
                    assert self.weight.shape[0] == target.shape[1], \
                        'Expect weight shape [{}], get[{}]'.format(target.shape[1], self.weight.shape[0])
                    dice_loss *= self.weights[i]
                total_loss += dice_loss

        return total_loss/target.shape[1]

def output_gif(input_path, output_path, gif_name):
    '''
    generate gif
    :param input_path: path of the input inages series
    :param output_path: gif output path
    :param gif_name: name of the gif
    :return:
    '''

    outfilename = os.path.join(output_path, '{}.gif'.format(gif_name))
    frames = []
    paths = os.listdir(input_path)
    paths_sort = sorted(paths, key=lambda x: int((os.path.splitext(x))[0]))
    for path in paths_sort:
        fullpath = os.path.join(input_path, path)
        im = imageio.imread(fullpath)
        frames.append(im)
    imageio.mimsave(outfilename, frames, 'GIF', duration=0.5)

if __name__ == "__main__":

    test_path = './data_val'
    subject_id = 9
    subject_name = 'subject-%d-' % subject_id

max_stars_repo_name: JohnleeHIT/SEX-Net

max_issues_repo_name: JohnleeHIT/SEX-Net

max_forks_repo_name: JohnleeHIT/SEX-Net


text:
from copy import deepcopy

import numpy as np

from .modify_curves import (_replace_by_beziers, _split_seg)
from .camera import Camera
from .defs import LinearGradient, RadialGradient, Gourad
from .element import Element
from .group import Group
from .light import Light
from .svgpathtools import Path as SVGPath
from .svgpathtools import Arc, CubicBezier, Line, parse_path
from .utils import unit_vector

class Path(Element):
    def __init__(self, data=None, shading_type=None):
        self._data_3d = []
        self._index_map = dict()
        self._shading_type = shading_type
        self._bbox = None
        self._transformed_points = None
        self._vertex_avg = None
        if data is not None:
            self.set_segments(data)
        Element.__init__(self)

    def set_segments(self, data):
        self._data_3d = []
        self._index_map = dict()
        self._transformed_points = None
        self._vertex_avg = None
        self._data_2d = self._get_svg_path(data)
        self._convert_2d_to_3d()
        self._create_bbox()
        return self

    def add_segments(self, data, index=None):
        data = self._get_svg_path(data)
        if index:
            for comp in reversed(data):
                self._data_2d.insert(index, comp)
        else:
            self._data_2d.extend(data)

        self._convert_2d_to_3d()
        self._create_bbox()
        return self

    def remove_segments(self, indices=[-1]):
        for i in indices:
            self._data_2d.pop(i)

        self._convert_2d_to_3d()
        self._create_bbox()
        return self

    def modify_segments(self, indicies, modified_segments):
        assert len(indicies) == len(modified_segments)
        for i in range(len(indicies)):
            self._data_2d.pop(indicies[i])
            self._data_2d.insert(indicies[i], modified_segments[i])

        self._convert_2d_to_3d()
        self._create_bbox()
        return self

    def shading_type(self, shading_type):
        self._shading_type = shading_type
        return self

    def get_z_index(self):
        self._get_vertex_avg()
        if self._z_index is None:
            return self._vertex_avg[2]
        else:
            return self._z_index

    def union(self, path):
        union_path = SVGPath()
        this_path_copy = deepcopy(self._data_2d)
        other_path_copy = deepcopy(path._data_2d)
        Path._partial_union_path(0, this_path_copy, path._data_2d, union_path)
        Path._partial_union_path(0, other_path_copy, self._data_2d, union_path)
        Path._path_joiner(union_path, 0)
        self.set_segments(union_path)
        return self

    def intersection(self, path):
        intersection_path = SVGPath()
        this_path_copy = deepcopy(self._data_2d)
        other_path_copy = deepcopy(path._data_2d)
        Path._partial_intersection_path(0, this_path_copy, path._data_2d, intersection_path)
        Path._partial_intersection_path(0, other_path_copy, self._data_2d, intersection_path)
        Path._path_joiner(intersection_path, 0)
        self.set_segments(intersection_path)
        return self

    def difference(self, path):
        difference_path = SVGPath()
        this_path_copy = deepcopy(self._data_2d)
        other_path_copy = deepcopy(path._data_2d)
        Path._partial_union_path(0, this_path_copy, path._data_2d, difference_path)
        Path._partial_intersection_path(0, other_path_copy, self._data_2d, difference_path)
        Path._path_joiner(difference_path, 0)
        self.set_segments(difference_path)
        return self

    def _create_bbox(self):
        x_min = self._data_3d[:, 0].min()
        y_min = self._data_3d[:, 1].min()
        z_min = self._data_3d[:, 2].min()
        x_max = self._data_3d[:, 0].max()
        y_max = self._data_3d[:, 1].max()
        z_max = self._data_3d[:, 2].max()
        self._bbox = np.array([
            [x_min, y_min, z_min, 1.0],    # 0, 0, 0
            [x_min, y_min, z_max, 1.0],    # 0, 0, 1
            [x_min, y_max, z_min, 1.0],    # 0, 1, 0
            [x_min, y_max, z_max, 1.0],    # 0, 1, 1
            [x_max, y_min, z_min, 1.0],    # 1, 0, 0
            [x_max, y_min, z_max, 1.0],    # 1, 0, 1
            [x_max, y_max, z_min, 1.0],    # 1, 1, 0
            [x_max, y_max, z_max, 1.0],    # 1, 1, 1
        ])

    def _get_transformed_points(self):
        self._transformed_points = self.transform_3d().dot(self._data_3d.T).T

    def _get_vertex_avg(self):
        self._get_transformed_points()
        self._vertex_avg = sum(self._transformed_points[:-1]) / (len(self._transformed_points) - 1)

    def _border_length(self):
        self._convert_3d_to_2d()
        return self._data_2d.length()

    def _str_fill(self):
        if self._shading_type:
            if isinstance(self._fill, np.ndarray):
                if self._shading_type == 1:
                    self._get_vertex_avg()
                    face_to_light = unit_vector(Light.get_position() - self._vertex_avg)
                    face_normal = unit_vector(self.transform_3d().dot(np.array([0,0,1,0])))
                    ambient_coeff = 0.33
                    diffuse_coeff = 0.67 * (abs(face_normal[:3].dot(face_to_light[:3])))
                    color = self._fill * (ambient_coeff + diffuse_coeff)
                    return f'fill="rgb{tuple(color)}"'
                elif self._shading_type == 2:
                    face_normal = unit_vector(self.transform_3d().dot(np.array([0, 0, 1, 0])))
                    colors = []
                    ambient_coeff = 0.33
                    for i in range(len(self._data_3d)):
                        corner_to_light = unit_vector(Light.get_position() - self._data_3d[i])
                        diffuse_coeff = 0.67 * (abs(face_normal[:3].dot(corner_to_light[:3])))
                        colors.append(self._fill * (ambient_coeff + diffuse_coeff))
                    self.filter(Gourad(self, colors))
                    return f'fill="none"'
        if isinstance(self._fill, LinearGradient):
            return f'fill="url(#{self._fill.id})"'
        elif isinstance(self._fill, RadialGradient):
            return f'fill="url(#{self._fill.id})"'
        return f'fill="{self._str_color(self._fill)}"'

    def _draw(self):
        self._convert_3d_to_2d()
        s = f'<path d="{self._data_2d.d()}" '
        s += super()._draw()
        s += "></path>"
        return s

    def _convert_2d_to_3d(self):
        def c2v(z):
            return np.array([z.real, z.imag, 0, 1])

        temp_map = dict()
        index = 0
        comp_index = 0
        for comp in self._data_2d:
            point_index = 0
            for point in comp:
                temp_index = temp_map.get(point, -1)
                if temp_index + 1:
                    self._index_map[(comp_index, point_index)] = temp_index
                else:
                    temp_map[point] = index
                    self._data_3d.append(c2v(point))
                    self._index_map[(comp_index, point_index)] = index
                    index += 1
                point_index += 1
            comp_index += 1
        self._data_3d = np.array(self._data_3d)

    def _convert_3d_to_2d(self):
        def v2c(v):
            return complex(round(v[0], 1), round(v[1], 1))

        self._get_transformed_points()
        camera_transformed_points = Camera._camera_view(self._transformed_points)
        for comp_index, point_index in self._index_map:
            p = v2c(camera_transformed_points[self._index_map[(comp_index, point_index)]])
            if point_index == 0:
                self._data_2d[comp_index].start = p
            if len(self._data_2d[comp_index]) == 2:
                if point_index == 1:
                    self._data_2d[comp_index].end = p
            else:
                if point_index == 1:
                    self._data_2d[comp_index].control1 = p
                elif point_index == 2:
                    self._data_2d[comp_index].control2 = p
                elif point_index == 3:
                    self._data_2d[comp_index].end = p

    def _apply_static_transform(self):
        def v2c(v):
            return complex(v[0], v[1])

        self._data_3d = self.static.dot(self._data_3d.T).T
        for comp_index, point_index in self._index_map:
            p = v2c(self._data_3d[self._index_map[(comp_index, point_index)]])
            if point_index == 0:
                self._data_2d[comp_index].start = p
            if len(self._data_2d[comp_index]) == 2:
                if point_index == 1:
                    self._data_2d[comp_index].end = p
            else:
                if point_index == 1:
                    self._data_2d[comp_index].control1 = p
                elif point_index == 2:
                    self._data_2d[comp_index].control2 = p
                elif point_index == 3:
                    self._data_2d[comp_index].end = p

        self._create_bbox()
        self.static = np.eye(4, dtype=np.float)
        return self

    def _get_svg_path(self, data):
        if type(data) == str:
            data = SVGPath(*_replace_by_beziers(parse_path(data)))
        elif type(data) == SVGPath:
            data = SVGPath(*_replace_by_beziers(data))
        else:
            data = SVGPath(*data)
        return data

    @staticmethod
    def _path_joiner(path, index):
        if index != len(path):
            for i in range(index + 1, len(path)):
                if abs(path[index].end - path[i].start) < 0.01:
                    path.insert(index + 1, path.pop(i))
                    break
                if abs(path[index].end - path[i].end) < 0.01:
                    temp = path[i].start
                    path[i].start = path[i].end
                    path[i].end = temp
                    if isinstance(path[i], CubicBezier):
                        temp = path[i].control1
                        path[i].control1 = path[i].control2
                        path[i].control2 = temp
                    path.insert(index + 1, path.pop(i))
                    break

            Path._path_joiner(path, index + 1)

    @staticmethod
    def _partial_union_path(index, this_path, other_path, union_path):
        if index != len(this_path):
            intersections = SVGPath(this_path[index]).intersect(other_path)
            for i in reversed(range(len(intersections))):
                if not 0.0001 < intersections[i][0][2] < 0.9999:
                    intersections.pop(i)

            if intersections:
                this_t = intersections[0][0][2]
                segs = _split_seg(this_path[index], this_t)
                if len(segs) > 1:
                    this_path.pop(index)
                    this_path.insert(index, segs[1])
                    this_path.insert(index, segs[0])
                Path._partial_union_path(index, this_path, other_path, union_path)

            else:
                pt = this_path[index].point(0.5)
                is_inside = len(other_path.intersect(SVGPath(Line(pt, pt.real - 10000j)))) % 2
                if not is_inside:
                    union_path.append(this_path[index])
                Path._partial_union_path(index + 1, this_path, other_path, union_path)

    @staticmethod
    def _partial_intersection_path(index, this_path, other_path, intersection_path):
        if index != len(this_path):
            intersections = SVGPath(this_path[index]).intersect(other_path)
            for i in reversed(range(len(intersections))):
                if not 0.0001 < intersections[i][0][2] < 0.9999:
                    intersections.pop(i)
            if intersections:
                this_t = intersections[0][0][2]
                segs = _split_seg(this_path[index], this_t)
                if len(segs) > 1:
                    this_path.pop(index)
                    this_path.insert(index, segs[1])
                    this_path.insert(index, segs[0])
                Path._partial_intersection_path(index, this_path, other_path, intersection_path)
            else:
                pt = this_path[index].point(0.5)
                is_inside = len(other_path.intersect(SVGPath(Line(pt, pt.real - 10000j)))) % 2
                if is_inside:
                    intersection_path.append(this_path[index])
                Path._partial_intersection_path(index + 1, this_path, other_path, intersection_path)

    # def _format_path_3d(self, data_3d):
    #     temp_map = dict()
    #     index = 0
    #     comp_index = 0
    #     identifier = 0
    #     for comp in data_3d:
    #         point_index = 0
    #         for point in comp[1:]:
    #             self._data_3d.append(point)

    #             if comp[0] == "l":
    #                 identifier = 11 if point_index == 0 else 12
    #             elif comp[0] == "p":
    #                 identifier = 21 if point_index == 0 else\
    #                              22 if point_index == 1 else\
    #                              23 if point_index == 2 else 24

    #             self._index_map[(identifier, comp_index, point_index)] = index
    #             point_index += 1
    #             index += 1
    #         comp_index += 1
    #     self._data_3d = np.array(self._data_3d)

    # def _c2v(self, z):
    #     return np.array([z.real, z.imag, 1])

    # def _v2c(self, v):
    #     return complex(v[0], v[1])

    # def abs_transform(self):
    #     matrix = self.transformation()
    #     if (matrix == np.eye(3)).all():
    #         return self
    #     for i in range(len(self)):
    #         self[i].start = self._v2c(matrix.dot(self._c2v(self[i].start)))
    #         self[i].end = self._v2c(matrix.dot(self._c2v(self[i].end)))
    #         if type(self[i]) == CubicBezier:
    #             self[i].control1 = self._v2c(
    #                 matrix.dot(self._c2v(self[i].control1)))
    #             self[i].control2 = self._v2c(
    #                 matrix.dot(self._c2v(self[i].control2)))
    #     self._create_bbox = matrix.dot(self._create_bbox.T).T
    #     self.dynamic_reset()
    #     self.reset()
    #     return self

    # def shift_index(self, index):
    #     for i in range(index - 1):
    #         self.append(self.pop(0))
    #     return self

    # def path2bezier(self, points_needed=None):
    #     for i in range(len(self.data)):
    #         start = self.data[i].start
    #         end = self.data[i].end
    #         if type(self.data[i]) == Line:
    #             self.data[i] = CubicBezier(start, start, end, end)
    #         elif type(self.data[i]) == QuadraticBezier:
    #             control = self.data[i].control
    #             self.data[i] = CubicBezier(start, (start + 2 * control) / 3,
    #                                        (2 * control + end) / 3, end)
    #     if points_needed:
    #         comp_len = []
    #         for i in range(len(self.data)):
    #             comp_len.append([i, self.data[i].length()])
    #         points_needed = points_needed - len(self.data)
    #         for i in range(points_needed):
    #             comp_len.sort(key=lambda x: x[0], reverse=True)
    #             index = comp_len[0][0]
    #             self.data[index:index + 1] = _split_bezier(self.data[index])
    #             _update_comp_len_list(comp_len, index)

    #     return self

    # def path2lines(self, points_needed):
    #     pts = []
    #     data = SVGPath()
    #     if points_needed > 0:
    #         for i in range(points_needed + 1):
    #             pts.append(self.data.point(i / points_needed))
    #         for start, end in zip(pts[:-1], pts[1:]):
    #             data.append(Line(start, end))
    #     state = self.__dict__.copy()
    #     state.pop("data", 0)
    #     return Path(data, **state)

max_stars_repo_name: partha-ghosh/pyeffects

max_issues_repo_name: partha-ghosh/pyeffects

max_forks_repo_name: partha-ghosh/pyeffects


text:
#!/usr/bin/env python3

""" A script containing the basic principles of the extraction primitive inner
workings"""

from __future__ import division, print_function
from ghostdr import polyfit
import numpy as pn

# Firstly, let's find all the needed files

fitsdir='/Users/mireland/data/ghost/cal_frames/'
fitsdir='/Users/marc/Documents/ghost/mulga/cals/'

#Define the files in use (NB xmod.txt and wavemod.txt should be correct)
arc_file  = fitsdir+"arc95_std_blue.fits"

# Where is the default location for the model? By default it is a parameter
# in the ghost class. If this needs to be overwritten, go ahead.
# This is the xmod file. Wherever it is saved from the flat reduction.
xmodel_file=fitsdir+'GHOST_1_1_blue_std_xmodPolyfit.fits'

# All the other models... which are currently in the "test" directory.
wmodel_file=test_files_dir+'wparams_blue_std.fits'
spatmod_file=test_files_dir+'spatmod.fits'
specmod_file=test_files_dir+'specmod.fits'
rotmod_file=test_files_dir+'rotmod2.fits'

#Input the slit arrays.
image_array = pyfits.getdata(fitsdir + 'arc95_std_SLIT.fits').astype(float)

#Get the data
arc_data = pyfits.getdata(arc_file)

#instantiate the ghostsim arm
arm = polyfit.GhostArm('blue',mode='std')

#Get the initial default model from the lookup location
xpars=pyfits.getdata(xmodel_file)
wpars=pyfits.getdata(wmodel_file)
spatpars=pyfits.getdata(spatmod_file)
specpars=pyfits.getdata(specmod_file)
rotpars=pyfits.getdata(rotmod_file)

slitview = polyfit.SlitView(image_array, flat_image_array, mode='std')
arm.spectral_format_with_matrix(xpars,wpars,spatpars,specpars,rotpars)

#!!! These lines below actually go after the wmodel_file tweaking !!!

# The extractor is given the polyfit "arm" object, and a slitview object which has
# been instantiated with the slit viewer data.
extractor = polyfit.Extractor(arm, slitview)
# Now extract. The option to correct for sky depends on the type of file.
arc_flux, arc_var = extractor.one_d_extract(arc_data, correct_for_sky=False)

#Now write the output to a file, in whatever format suits the recipe system best.
pyfits.writeto('outputs.fits',[arc_flux,arc_var])

max_stars_repo_name: GeminiDRSoftware/GHOSTDR

max_issues_repo_name: GeminiDRSoftware/GHOSTDR

max_forks_repo_name: GeminiDRSoftware/GHOSTDR