text:
# @author Simon Stepputtis <sstepput@asu.edu>, Interactive Robotics Lab, Arizona State University

import tensorflow as tf
import numpy as np

class BasisModel(tf.keras.layers.Layer):
    def __init__(self, dimensions, nfunctions, scale, **kwarg):
        super(BasisModel, self).__init__(name="attention", **kwarg)
        self._degree = nfunctions
        self.scale   = scale

    def build(self, input_shape):
        self.centers = np.linspace(0.0, 1.01, self._degree, dtype = np.float32)
        self.centers = tf.convert_to_tensor(self.centers)

    @tf.function
    def call(self, inputs, training=None):
        weights     = tf.transpose(inputs[0], perm=[0,2,1])
        weights_std = inputs[1]
        positions   = inputs[2]
        basis_funcs = self.compute_basis_values(positions)
        result      = tf.linalg.matmul(basis_funcs, weights)

        return result, tf.zeros_like(result)

    def get_config(self):
        config = super(TopDownAttention, self).get_config()
        config.update({'units': self.units})
        return config

    def compute_basis_values(self, x):
        centers = tf.tile(tf.expand_dims(self.centers,0), [tf.shape(x)[1], 1])
        x       = tf.expand_dims(x, 2)
        funcs   = tf.exp(-( tf.math.pow((x - centers), 2) / (2.0 * self.scale) ))
        return funcs


text:
'''
  File name: findDerivatives.py
  Author: Tarmily Wen
  Date created: Dec. 8, 2019
'''

import numpy as np
from scipy import signal
import cv2

'''
  File clarification:
    Compute gradient put ginformation of the inrayscale image
    - Input I_gray: H x W matrix as image
    - Output Mag: H x W matrix represents the magnitude of derivatives
    - Output Magx: H x W matrix represents the derivatives along x-axis
    - Output Magy: H x W matrix represents the derivatives along y-axis
    - Output Ori: H x W matrix represents the orientation of derivatives
'''

def findDerivatives(I_gray):
    # smoothing kernels
    gaussian = np.array(
        [[2, 4, 5, 4, 2], [4, 9, 12, 9, 4], [5, 12, 15, 12, 5], [4, 9, 12, 9, 4], [2, 4, 5, 4, 2]]) / 159.0

    # kernel for x and y gradient
    dx = np.asarray([[-1.0, 0.0, 1.0], [-2.0, 0.0, 2.0], [-1.0, 0.0, 1.0]])
    dy = np.asarray([[1.0, 2.0, 1.0], [0.0, 0.0, 0.0], [-1.0, -2.0, -1.0]])

    ###############################################################################
    # Your code here: calculate the gradient magnitude and orientation
    ###############################################################################

    Gx= signal.convolve2d(gaussian, np.rot90(dx, 2), 'same')
    Gy= signal.convolve2d(gaussian, np.rot90(dy, 2), 'same')

    Magx = signal.convolve2d(I_gray, np.rot90(Gx, 2), 'same')
    Magy = signal.convolve2d(I_gray, np.rot90(Gy, 2), 'same')

    Mag = np.sqrt(Magx*Magx+Magy*Magy)

    Ori = np.arctan2(Magy, Magx)/np.pi*180

    return Mag, Magx, Magy, Ori

if __name__ == '__main__':
    I_g = np.arange(0, 81, 1)
    I_g = I_g.reshape(9, 9)
    findDerivatives(I_g)

ext: py

max_stars_repo_name: magicwenli/morpher

max_issues_repo_name: magicwenli/morpher

max_forks_repo_name: magicwenli/morpher

reason: import numpy,from scipy


text:
import numpy as np
import pandas as pd
import requests
# from bs4 import BeautifulSoup
import re
import gzip
import xml.etree.ElementTree as ET
from random import sample
import math
import subprocess
import random
from collections import defaultdict
import glob, os, sys
import networkx as nx
from sklearn import svm
from sklearn.model_selection import train_test_split
import scipy
import json
from sklearn.metrics import confusion_matrix
from sklearn.metrics import fbeta_score
from sklearn.manifold import TSNE
import seaborn as sns
import matplotlib.pyplot as plt

def generate_id(start_index, data_list, name_of_file, result_dir):
    """
    generate unique id for each element in the list and write into txt file. And the index starts from the specified
    start_index.
    """
    indexes = start_index + np.arange(len(data_list))
    with open(result_dir + name_of_file + '.txt', 'w', encoding = 'utf-8') as myfile:
        for i in indexes:
            myfile.write(str(i) + " " + data_list[i-start_index] + "\n")
    myfile.close()
    return indexes[-1]

def mapping(matrix, rows, columns, file_name):
    """
    Map the matrix into the sample txt file from net_dbis.zip
    """
    # Get row and col indexes of non zero elements in the sparse matrix
    row_col_indexes = matrix.nonzero()

    with open(file_name + ".txt", 'w', encoding = 'utf-8') as myfile:
        for i in range(len(row_col_indexes[0])):
            row_num = row_col_indexes[0][i]
            col_num = row_col_indexes[1][i]
            myfile.write(rows[row_num]+" "+columns[col_num]+"\n")
    myfile.close()

def generate_graph_pairs():
    result_dir = "./node_pairs_result/"
    app_ids = [x.split(' ')[0] for x in open(result_dir+"id_app_320.txt", encoding = 'utf-8').readlines()]
    api_ids = [x.split(' ')[0] for x in open(result_dir+"id_api_320.txt", encoding = 'utf-8').readlines()]
    mapping(matrix_A_train.transpose(), api_ids, app_ids, result_dir+"api_app_320")
    mapping(matrix_B_train, api_ids, api_ids, result_dir+"same_block_api_320")
    mapping(matrix_P_train, api_ids, api_ids, result_dir+"same_package_api_320")

if __name__ == "__main__":
    with open('../../config/data-params.json') as fh:
        data_cfg = json.load(fh)
    matrix_A_train = scipy.sparse.load_npz('../' + data_cfg['matrix_A_position'])
    matrix_B_train = scipy.sparse.load_npz('../' + data_cfg['matrix_B_position'])
    matrix_P_train = scipy.sparse.load_npz('../' + data_cfg['matrix_P_position'])
    generate_graph_pairs()

reason: import numpy,import scipy,import networkx


text:
from sys import argv
# import sys
# import gsum as gm
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
# import scipy.stats as stats
# from scipy.interpolate import interp1d
# import os
# import h5py
# from sklearn.gaussian_process.kernels import RBF, WhiteKernel
# import math
# import re
import gc
from ChEFT_GP_imports import joint_plot, offset_xlabel, m_p, m_n, hbarc, E_to_p, Q_approx, \
deg_to_qcm, deg_to_qcm2, softmax_mom, GPHyperparameters, FileNaming, PosteriorBounds, \
OrderInfo, versatile_train_test_split, InputSpaceBunch, \
ObservableBunch, Interpolation, TrainTestSplit, ScaleSchemeBunch, LengthScale, \
GSUMDiagnostics
import copy as cp
# import urllib
# import tables

# import warnings
# warnings.filterwarnings("error")

# See: https://ianstormtaylor.com/design-tip-never-use-black/
# softblack = '#262626'
softblack = 'k'  # Looks better when printed on tex file
gray = '0.7'

mpl.rcParams['figure.dpi'] = 180
mpl.rcParams['font.size'] = 9
mpl.rcParams['text.usetex'] = True
mpl.rcParams['font.family'] = 'serif'

mpl.rcParams['axes.labelsize'] = 9
mpl.rcParams['axes.edgecolor'] = softblack
mpl.rcParams['axes.xmargin'] = 0
mpl.rcParams['axes.labelcolor'] = softblack
mpl.rcParams['axes.linewidth']

mpl.rcParams['ytick.direction'] = 'in'
mpl.rcParams['xtick.direction'] = 'in'
mpl.rcParams['xtick.labelsize'] = 9
mpl.rcParams['ytick.labelsize'] = 9
mpl.rcParams['xtick.color'] = softblack
mpl.rcParams['ytick.color'] = softblack
mpl.rcParams['xtick.minor.size'] = 2.4
mpl.rcParams['ytick.minor.size'] = 2.4

mpl.rcParams['legend.title_fontsize'] = 9
mpl.rcParams['legend.fontsize'] = 9
mpl.rcParams['legend.edgecolor'] = 'inherit'  # inherits from axes.edgecolor, to match
mpl.rcParams['legend.facecolor'] = (1, 1, 1, 0.6)  # Set facecolor with its own alpha, so edgecolor is unaffected
mpl.rcParams['legend.fancybox'] = True
mpl.rcParams['legend.borderaxespad'] = 0.8
mpl.rcParams['legend.framealpha'] = None  # Do not set overall alpha (affects edgecolor). Handled by facecolor above
mpl.rcParams['patch.linewidth'] = 0.8  # This is for legend edgewidth, since it does not have its own option

text_bbox = dict(boxstyle='round', fc=(1, 1, 1, 0.6), ec=softblack, lw=0.8)
mpl.rc('savefig', transparent=False, bbox='tight', pad_inches=0.05, dpi=300, format='pdf')

edgewidth = 0.6

# # We get the NN data from a separate place in our github respository.
# nn_online_pot = 'pwa93'
# nn_online_url = 'https://github.com/buqeye/buqeyebox/blob/master/nn_scattering/NN-online-Observables.h5?raw=true'
# nno_response = urllib.request.urlopen(nn_online_url)
# nn_online_file = tables.open_file("nn_online_example.h5", driver="H5FD_CORE",
#                           driver_core_image=nno_response.read(),
#                           driver_core_backing_store=0)
# SGT_nn_online = nn_online_file.get_node('/' + nn_online_pot + '/SGT').read()
# DSG_nn_online = nn_online_file.get_node('/' + nn_online_pot + '/DSG').read()[:, :-1]
# AY_nn_online = nn_online_file.get_node('/' + nn_online_pot + '/PB').read()[:, :-1]
# A_nn_online = nn_online_file.get_node('/' + nn_online_pot + '/A').read()[:, :-1]
# D_nn_online = nn_online_file.get_node('/' + nn_online_pot + '/D').read()[:, :-1]
# AXX_nn_online = nn_online_file.get_node('/' + nn_online_pot + '/AXX').read()[:, :-1]
# AYY_nn_online = nn_online_file.get_node('/' + nn_online_pot + '/AYY').read()[:, :-1]

# for each choice of scale and scheme, sets the total possible orders and nomenclature
EKM0p8fm = ScaleSchemeBunch('scattering_observables_EKM_R-0p8fm.h5', \
            np.array([0, 2, 3, 4, 5]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds']], \
            "EKM", "0p8fm")
EKM0p9fm = ScaleSchemeBunch('scattering_observables_EKM_R-0p9fm.h5', \
            np.array([0, 2, 3, 4, 5]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds']], \
            "EKM", "0p9fm")
EKM1p0fm = ScaleSchemeBunch('scattering_observables_EKM_R-1p0fm.h5', \
            np.array([0, 2, 3, 4, 5]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds']], \
            "EKM", "1p0fm")
EKM1p1fm = ScaleSchemeBunch('scattering_observables_EKM_R-1p1fm.h5', \
            np.array([0, 2, 3, 4, 5]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds']], \
            "EKM", "1p1fm")
EKM1p2fm = ScaleSchemeBunch('scattering_observables_EKM_R-1p2fm.h5', \
            np.array([0, 2, 3, 4, 5]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds']], \
            "EKM", "1p2fm")

RKE400MeV = ScaleSchemeBunch('scattering_observables_RKE_L-400MeV.h5', \
            np.array([0, 2, 3, 4, 5, 6]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds', 'Purples']], \
            "RKE", "400MeV")
RKE450MeV = ScaleSchemeBunch('scattering_observables_RKE_L-450MeV.h5', \
            np.array([0, 2, 3, 4, 5, 6]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds', 'Purples']], \
            "RKE", "450MeV")
RKE500MeV = ScaleSchemeBunch('scattering_observables_RKE_L-500MeV.h5', \
            np.array([0, 2, 3, 4, 5, 6]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds', 'Purples']], \
            "RKE", "500MeV")
RKE550MeV = ScaleSchemeBunch('scattering_observables_RKE_L-550MeV.h5', \
            np.array([0, 2, 3, 4, 5, 6]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds', 'Purples']], \
            "RKE", "550MeV")

EMN450MeV = ScaleSchemeBunch('scattering_observables_EM-450MeV.h5', \
            np.array([0, 2, 3, 4, 5]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds']], \
            "EMN", "450MeV")
EMN500MeV = ScaleSchemeBunch('scattering_observables_EM-500MeV.h5', \
            np.array([0, 2, 3, 4, 5]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds']], \
            "EMN", "500MeV")
EMN550MeV = ScaleSchemeBunch('scattering_observables_EM-550MeV.h5', \
            np.array([0, 2, 3, 4, 5]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens', 'Blues', 'Reds']], \
            "EMN", "550MeV")

GT0p9fm = ScaleSchemeBunch('scattering_observables_Gezerlis-0p9fm.h5', \
            np.array([0, 2, 3]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens']], \
            "GT", "0p9fm")
GT1p0fm = ScaleSchemeBunch('scattering_observables_Gezerlis-1p0fm.h5', \
            np.array([0, 2, 3]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens']], \
            "GT", "1p0fm")
GT1p1fm = ScaleSchemeBunch('scattering_observables_Gezerlis-1p1fm.h5', \
            np.array([0, 2, 3]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens']], \
            "GT", "1p1fm")
GT1p2fm = ScaleSchemeBunch('scattering_observables_Gezerlis-1p2fm.h5', \
            np.array([0, 2, 3]), \
            [plt.get_cmap(name) for name in ['Oranges', 'Greens']], \
            "GT", "1p2fm")

# scale_scheme_bunch_array = [EKM0p8fm, EKM0p9fm, EKM1p0fm, EKM1p1fm, EKM1p2fm, \
#             RKE400MeV, RKE450MeV, RKE500MeV, RKE550MeV, EMN450MeV, EMN500MeV, \
#             EMN550MeV, GT0p9fm, GT1p0fm, GT1p1fm, GT1p2fm]

# # based on user input, allows only certain scale/scheme combinations through
# if argv[1] == "EKM0p8fm" or argv[1] == "EKM0p9fm" or argv[1] == "EKM1p0fm" \
#     or argv[1] == "EKM1p1fm" or argv[1] == "EKM1p2fm" or argv[1] == "EMN450MeV" \
#     or argv[1] == "EMN500MeV" or argv[1] == "EMN550MeV" or argv[1] == "RKE400MeV" \
#     or argv[1] == "RKE450MeV" or argv[1] == "RKE500MeV" or argv[1] == "RKE550MeV" \
#     or argv[1] == "GT0p9fm" or argv[1] == "GT1p0fm" or argv[1] == "GT1p1fm" \
#     or argv[1] == "GT1p2fm":
#     # print(argv[1])
#     scale_scheme_bunch_array = [b for b in scale_scheme_bunch_array \
#             if (b.potential_string + b.cutoff_string) == argv[1]]
#     # for b in scale_scheme_bunch_array: print(b.potential_string + b.cutoff_string)
# elif argv[1] == "all_potentials":
#     pass
# else:
#     raise NameError("Valid scheme/scale combinations: EKM0p8fm, EKM0p9fm, "+
#             "EKM1p0fm, EKM1p1fm, EKM1p2fm, EMN450MeV, EMN500MeV, "+
#             "EMN550MeV, RKE400MeV, RKE450MeV, RKE500MeV, RKE550MeV, "+
#             "GT0p9fm, GT1p0fm, GT1p1fm, GT1p2fm, all_potentials")

# # gets energies for evaluating observables from user
# energies_input = argv[3]
# E_input_array = [int(x) for x in list(energies_input.split(","))]

# # sets the method for parametrizing the dimensionless parameter Q
# Q_param_method_array = ["poly", "max", "sum"]

# # based on user input, allows only certain parametrizations through
# if argv[5] == "poly" or argv[5] == "max" or argv[5] == "sum":
#     # print(argv[5])
#     Q_param_method_array = [b for b in Q_param_method_array \
#             if b == argv[5]]
#     # print(Q_param_method_array)
# elif argv[5] == "all_Q":
#     pass
# else:
#     raise NameError("Valid Q parametrizations: poly, max, sum, all_Q")

# creates the training and testing masks for observables plotted against angle
Fullspaceanglessplit = TrainTestSplit("allangles", 6, 3, \
                                    xmin_train_factor = 0, xmax_train_factor = 1)
Forwardanglessplit = TrainTestSplit("forwardangles", 6, 3, \
                                    xmin_train_factor = 0, xmax_train_factor = 5/6)
Backwardanglessplit = TrainTestSplit("backwardangles", 6, 3, \
                                    xmin_train_factor = 1/6, xmax_train_factor = 1)
traintestsplit_vsangle_array = [Fullspaceanglessplit, Forwardanglessplit, Backwardanglessplit]

# creates the training and testing masks for observables plotted against energy
Nolowenergysplit = TrainTestSplit("nolowenergy", 3, 4, \
                            offset_train_min_factor = 100/350, xmin_train_factor = 100/350, \
                            offset_test_min_factor = 100/350, xmin_test_factor = 100/350, \
                            offset_train_max_factor = -50/350, offset_test_max_factor = -50/350)
Yeslowenergysplit = TrainTestSplit("yeslowenergy", 4, 4, \
                            offset_train_min_factor = 0, xmin_train_factor = 0.01, \
                            offset_test_min_factor = 0, xmin_test_factor = 0, \
                            offset_train_max_factor = -50/350, offset_test_max_factor = -50/350)
traintestsplit_vsenergy_array = [Nolowenergysplit, Yeslowenergysplit]

# # based on user input, allows only certain train/test splits through
# if argv[6] == "allangles" or argv[6] == "forwardangles" \
#     or argv[6] == "backwardangles":
#     # print(argv[6])
#     train_test_split_array = [b for b in traintestsplit_vsangle_array \
#             if b.name == argv[6]]
#     # print(train_test_split_array)
# elif argv[6] == "nolowenergy" or argv[6] == "yeslowenergy":
#     # print(argv[6])
#     train_test_split_array = [b for b in traintestsplit_vsenergy_array \
#             if b.name == argv[6]]
#     # print(train_test_split_array)
# elif argv[6] == "all_vsangle_splits":
#     train_test_split_array = traintestsplit_vsangle_array
# elif argv[6] == "all_vsenergy_splits":
#     train_test_split_array = traintestsplit_vsenergy_array
# else:
#     raise NameError("Valid train/test splits: allangles, forwardangles, "+
#             "backwardangles, nolowenergy, yeslowenergy, all_vsangle_splits"+
#             "all_vsenergy_splits")

# # bounds for 2D posterior PDF figures
# PosteriorBounds_deg = PosteriorBounds(1e-6, 100, 100, 300, 900, 300)
# PosteriorBounds_cos = PosteriorBounds(1e-6, 2, 100, 300, 900, 300)

# # gets orders for evaluation from user
# orders_input = argv[7]

# def README():
#     print("Arguments for GPAnalysis():\n")
#     print("scale_scheme_bunch_array (ScaleSchemeBunch list): EKM0p8fm, EKM0p9fm, EKM1p0fm, EKM1p1fm, EKM1p2fm, RKE400MeV, RKE450MeV, RKE500MeV, RKE550MeV, EMN450MeV, EMN500MeV, EMN550MeV, GT0p9fm, GT1p0fm, GT1p1fm, GT1p2fm")
#     print("Default: [EKM0p9fm]\n")
#     print("observable_input (str list): \"SGT\", \"DSG\", \"AY\", \"A\", \"D\", \"AXX\", \"AYY\"")
#     print("Default: [\"DSG\"]\n")
#     print("E_input_array (int list): any integer x such that 1 <= x <= 350")
#     print("Must be [0] for SGT")
#     print("Default: [150]\n")
#     print("Q_param_method_array (str list): \"poly\", \"max\", \"cos\"")
#     print("Default: [\"poly\"]\n")
#     print("input_space_input (str list): \"Elab\", \"prel\" for SGT")
#     print("input_space_input (str list): \"deg\", \"cos\", \"qcm\", \"qcm2\" for all other observables")
#     print("Default: [\"cos\"]\n")
#     print("train_test_split_array (TrainTestSplit list): Nolowenergysplit, Yeslowenergysplit for SGT")
#     print("train_test_split_array (TrainTestSplit list): Fullspaceanglessplit, Forwardanglessplit, Backwardanglessplit for all other observables")
#     print("Default: Fullspaceanglessplit\n")
#     print("orders_input (int list, or str): 2, 3, 4, 5, 6")
#     print("May be \"all\" to evaluate all orders for all potentials")
#     print("Default: \"all\"\n")
#     print("length_scale_input (LengthScale)")
#     print("Default: LengthScale(0.25, 0.25, 4, whether_fit = True)\n")
#     print("fixed_sd (float): any positive float")
#     print("Default: None\n")

def GPAnalysis(scale_scheme_bunch_array = [EKM0p9fm],
               observable_input = ["DSG"],
               E_input_array = [150],
               Q_param_method_array = ["poly"],
               input_space_input = ["cos"],
               train_test_split_array = [Fullspaceanglessplit],
               orders_input = "all",
               length_scale_input = LengthScale("1/16-1_fitted", 0.25, 0.25, 4, whether_fit = True),
               fixed_sd = None,
               print_all_classes = False,
               savefile_type = 'pdf',
               plot_coeffs_bool = True,
               plot_md_bool = True,
               plot_pc_bool = True,
               plot_ci_bool = True,
               plot_plotzilla_bool = True,
               save_coeffs_bool = True,
               save_md_bool = True,
               save_pc_bool = True,
               save_ci_bool = True,
               save_plotzilla_bool = True,
               filename_addendum = ""):
    """
    scale_scheme_bunch_array (ScaleSchemeBunch list): potential/cutoff
        combinations for evaluation.
    Built-in options:
        EKM0p8fm, EKM0p9fm, EKM1p0fm, EKM1p1fm, EKM1p2fm
        ( https://doi.org/10.1140/epja/i2015-15053-8 );
        RKE400MeV, RKE450MeV, RKE500MeV, RKE550MeV
        ( https://doi.org/10.1140/epja/i2018-12516-4 );
        EMN450MeV, EMN500MeV, EMN550MeV
        ( https://doi.org/10.1103/PhysRevC.96.024004 );
        GT0p9fm, GT1p0fm, GT1p1fm, GT1p2fm
        ( https://doi.org/10.1103/PhysRevC.90.054323 )
    Default: [EKM0p9fm]

    observable_input (str list): observables for evaluation. Note that SGT
        should not be in the same list as other observables.
    Built-in options: "SGT", "DSG", "AY", "A", "D", "AXX", "AYY"
    Default: ["DSG"]

    E_input_array (int list): energies for evaluation. Note that SGT must be
        treated differently since it is not evaluated at one energy at a time.
    May be any integer x such that 1 <= x <= 350
    Must be [0] for SGT
    Default: [150]

    Q_param_method_array (str list): methods of parametrizing the dimensionless
        expansion parameter Q for evaluation.
    Built-in options: "poly", "max", "cos"
    Default: ["poly"]

    input_space_input (str list): input spaces for evaluation. Note that SGT
        must be treated differently since it is not evaluated at one energy at
        a time.
    Built-in options: "Elab", "prel" for SGT
    Built-in options: "deg", "cos", "qcm", "qcm2" for all other observables
    Default: ["cos"]

    train_test_split_array (TrainTestSplit list): splits of training and
        testing points for evaluation. Note that SGT must be treated
        differently since it is not evaluated at one energy at a time.
    Built-in options: Nolowenergysplit, Yeslowenergysplit for SGT
    Built-in options: Fullspaceanglessplit, Forwardanglessplit,
        Backwardanglessplit for all other observables
    Default: Fullspaceanglessplit

    orders_input (int list, or str if "all"): orders for evaluation. May be
        any list containing 2, 3, 4, 5, 6 in any order. May be "all" to
        evaluate all orders for all potentials.
    Built-in options: [0, 2, 3, 4, 5] for EKM; [0, 2, 3, 4, 5, 6] for RKE;
        [0, 2, 3, 4, 5] for EMN; [0, 2, 3] for GT+.
    Default: "all"

    length_scale_input (LengthScale): initial guess for the correlation
        length in the kernel (as a factor of the total size of the input space)
        plus boundaries of the fit procedure as factors of the initial guess
        for the correlation length. Fitting may be bypassed when whether_fit =
        False.
    Default: LengthScale(0.25, 0.25, 4, whether_fit = True)

    fixed_sd (float): fixed standard deviation for the Gaussian process fit.
        May be any positive float. If None, then there is no fixed standard
        deviation and it is calculated by the fitting procedure.
    Default: None

    savefile_type (str): string for specifying the type of file to be saved.
    Default: 'png'

    filename_addendum (str): string for distinguishing otherwise similarly named
        files.
    Default: ''
    """
    mpl.rc('savefig', transparent=False, bbox='tight', pad_inches=0.05, dpi=300,
           format=savefile_type)

    try:
        # runs through the potentials
        for o, scalescheme in enumerate(scale_scheme_bunch_array):
            # gets observable data from a local file
            # default location is the same as this program's
            try:
                SGT = scalescheme.get_data('SGT')
                DSG = scalescheme.get_data('DSG')
                AY = scalescheme.get_data('PB')
                A = scalescheme.get_data('A')
                D = scalescheme.get_data('D')
                AXX = scalescheme.get_data('AXX')
                AYY = scalescheme.get_data('AYY')
                t_lab = scalescheme.get_data('t_lab')
                degrees = scalescheme.get_data('degrees')
            except:
                raise Exception("Data could not be found in the location specified.")

            # creates the bunch for each observable to be plotted against angle
            SGTBunch = ObservableBunch("SGT", SGT, E_input_array, '\sigma_{\mathrm{tot}}', \
                                        "dimensionful")
            DSGBunch = ObservableBunch("DSG", DSG, E_input_array, '\sigma', \
                                        "dimensionful")
            AYBunch = ObservableBunch("AY", AY, E_input_array, 'A_{y}', \
                                        "dimensionless")
            ABunch = ObservableBunch("A", A, E_input_array, 'A', \
                                        "dimensionless")
            DBunch = ObservableBunch("D", D, E_input_array, 'D', \
                                        "dimensionless")
            AXXBunch = ObservableBunch("AXX", AXX, E_input_array, 'A_{xx}', \
                                        "dimensionless")
            AYYBunch = ObservableBunch("AYY", AYY, E_input_array, 'A_{yy}', \
                                        "dimensionless")

            observable_array = [SGTBunch, DSGBunch, AYBunch, ABunch, DBunch, AXXBunch, AYYBunch]

            observable_array = [b for b in observable_array if b.name in observable_input]

            # # filters out observables based upon user input
            # if argv[2] == "SGT" or argv[2] == "DSG" or argv[2] == "AY" or argv[2] == "A" \
            #     or argv[2] == "D" or argv[2] == "AXX" or argv[2] == "AYY":
            #     # print(argv[2])
            #     observable_array = [b for b in observable_array if b.name == argv[2]]
            #     # print(vsangle_observable_array)
            # elif argv[2] == "our_obs":
            #     observable_array = [SGTBunch, DSGBunch, ABunch]
            # elif argv[2] == "all_obs":
            #     pass
            # else:
            #     raise NameError("Valid observables: SGT, DSG, AY, A, D, AXX, AYY, our_obs, "+
            #             "all_obs")

            # # turns the string argument for orders into an array for orders
            # if orders_input == "all":
            #     orders_input_array = scalescheme.orders_full
            # else:
            #     orders_input_array = [int(x) for x in list(orders_input.split(","))]
            #     # print(orders_input_array)

            # turns the string argument for orders into an array for orders
            if orders_input == "all":
                orders_input_array = scalescheme.orders_full
            else:
                orders_input_array = orders_input.copy()
                orders_input_array.sort()

            # turns the array for orders into an array for colors
            colors_index_array = orders_input_array.copy()
            for i, o in enumerate(colors_index_array):
                colors_index_array[i] = o - 2
            # print(colors_index_array)

            # adds a 0 as the first entry in the array for orders if one is not already there
            if orders_input_array[0] != 0:
                orders_input_array = [0] + orders_input_array
            # print(orders_input_array)

            # creates a mask for orders and colors
            mask_orders = np.zeros(len(scalescheme.cmaps), dtype = bool)
            for i, o in enumerate(colors_index_array):
                mask_orders[o] = True
            # print(mask_orders)

            # This ensures we only analyze the non-trivial information at
            # O(Q^2), O(Q^3), O(Q^4), and O(Q^5)
            excluded = [0]
            mask_full = ~ np.isin(scalescheme.orders_full, excluded)
            Lambdab = 600

            # runs through the observables
            for m, observable in enumerate(observable_array):
                # runs through the energies at which to evaluate the observables
                for j, E_lab in enumerate(observable.energies):
                    # creates the bunches for the input spaces
                    DegBunch = InputSpaceBunch("deg", \
                                        degrees, \
                                        E_to_p(E_lab, "np"), \
                                        r'$\theta$ (deg)', \
                                        [r'$', observable.title, r'(\theta, E_{\mathrm{lab}}= ', E_lab, \
                                          '\,\mathrm{MeV})$'])

                    # note that the input space here is -cos(theta), not cos(theta) (as it once was)
                    CosBunch = InputSpaceBunch("cos", \
                                        -np.cos(np.radians(degrees)), \
                                        np.array( [softmax_mom(E_to_p(E_lab, "np"), q) \
                                                  for q in deg_to_qcm(E_to_p(E_lab, "np"), degrees)] ), \
                                        r'$-\mathrm{cos}(\theta)$', \
                                        [r'$', observable.title, r'(-\mathrm{cos}(\theta), E_{\mathrm{lab}}= ', \
                                          E_lab, '\,\mathrm{MeV})$'])

                    QcmBunch = InputSpaceBunch("qcm", \
                                        deg_to_qcm(E_to_p(E_lab, "np"), degrees), \
                                        np.array( [softmax_mom(E_to_p(E_lab, "np"), q) \
                                                  for q in deg_to_qcm(E_to_p(E_lab, "np"), degrees)] ), \
                                        r'$q_{\mathrm{cm}}$ (MeV)', \
                                        [r'$', observable.title, r'(q_{\mathrm{cm}}, E_{\mathrm{lab}}= ', \
                                          E_lab, '\,\mathrm{MeV})$'])

                    Qcm2Bunch = InputSpaceBunch("qcm2", \
                                        deg_to_qcm2(E_to_p(E_lab, "np"), degrees), \
                                        np.array( [softmax_mom(E_to_p(E_lab, "np"), q) \
                                                  for q in deg_to_qcm(E_to_p(E_lab, "np"), degrees)] ), \
                                        r'$q_{\mathrm{cm}}^{2}$ (MeV$^{2}$)', \
                                        [r'$', observable.title, r'(q_{\mathrm{cm}}^{2}, E_{\mathrm{lab}}= ', \
                                          E_lab, '\,\mathrm{MeV})$'])

                    ElabBunch = InputSpaceBunch("Elab", \
                                        t_lab, \
                                        E_to_p(t_lab, "np"), \
                                        r'$E_{\mathrm{lab}}$ (MeV)', \
                                        [r'$', observable.title, r'(E_{\mathrm{lab}})$'])

                    PrelBunch = InputSpaceBunch("prel", \
                                        E_to_p(t_lab, "np"), \
                                        E_to_p(t_lab, "np"), \
                                        r'$p_{\mathrm{rel}}$ (MeV)', \
                                        [r'$', observable.title, r'(p_{\mathrm{rel}})$'])

                    vsquantity_array = [DegBunch, CosBunch, QcmBunch, Qcm2Bunch, ElabBunch, PrelBunch]
                    vsquantity_array = [b for b in vsquantity_array if b.name in input_space_input]

                    # # filters out input spaces based upon user input
                    # if argv[4] == "deg" or argv[4] == "cos" or argv[4] == "qcm" \
                    #     or argv[4] == "qcm2" or argv[4] == "Elab" or argv[4] == "prel":
                    #     # print(argv[4])
                    #     vsquantity_array = [b for b in vsquantity_array if b.name == argv[4]]
                    #     # print(vsangle_quantity_array)
                    # elif argv[4] == "all_vsangle_inputs":
                    #     vsquantity_array = [DegBunch, CosBunch, QcmBunch, Qcm2Bunch]
                    # elif argv[4] == "all_vsenergy_inputs":
                    #     vsquantity_array = [ElabBunch, PrelBunch]
                    # else:
                    #     raise NameError("Valid input spaces: deg, cos, qcm, qcm2, "+
                    #             "Elab, prel, all_inputs")

                    # creates each input space bunch's title
                    for bunch in vsquantity_array: bunch.make_title()

                    # runs through the parametrization methods
                    for k, Q_param_method in enumerate(Q_param_method_array):
                        # runs through the input spaces
                        for i, vs_quantity in enumerate(vsquantity_array):
                            # runs through the training and testing masks
                            for l, traintestsplit in enumerate(train_test_split_array):
                                # conforms the training and testing masks to each input space
                                traintestsplit.make_masks(vs_quantity.input_space, observable.data)
                                # print("data = " + str(observable.data))

                                # chooses a starting guess for the GP length scale optimization procedure
                                LengthScaleGuess = length_scale_input
                                LengthScaleGuess.make_guess(vs_quantity.input_space)

                                # creates the GP with all its hyperparameters
                                ratio_dsg = Q_approx(vs_quantity.mom, Q_param_method, Lambda_b = Lambdab)
                                # print("ratio = " + str(ratio_dsg))
                                center = 0
                                df = 1
                                disp = 0
                                std_scale = 1
                                GPHyper_DSG = GPHyperparameters(LengthScaleGuess, center, ratio_dsg, \
                                                df = df, disp = disp, scale = std_scale, seed = 4,
                                                sd = fixed_sd)

                                # information for naming the savefiles
                                FileName_DSG = FileNaming(scalescheme.potential_string, \
                                                scalescheme.cutoff_string, Q_param_method,
                                                filename_addendum = filename_addendum)

                                # information on the orders for each potential
                                Orders_DSG = OrderInfo(scalescheme.orders_full, mask_full, \
                                                scalescheme.colors, scalescheme.light_colors, \
                                                orders_restricted = orders_input_array, \
                                                mask_restricted = mask_orders)

                                # creates the object used to generate and plot statistical diagnostics
                                MyPlot = GSUMDiagnostics(observable, Lambdab, vs_quantity, \
                                        traintestsplit, GPHyper_DSG, Orders_DSG, FileName_DSG, \
                                        E_lab = E_lab, E_lab_x = t_lab, constrained = False)

                                # plots figures
                                if plot_coeffs_bool:
                                    MyPlot.PlotCoefficients(whether_save = save_coeffs_bool)
                                if plot_md_bool:
                                    MyPlot.PlotMD(whether_save = save_md_bool)
                                if plot_pc_bool:
                                    MyPlot.PlotPC(whether_save = save_pc_bool)
                                if plot_ci_bool:
                                    MyPlot.PlotCredibleIntervals(whether_save = save_ci_bool)
                                # if vs_quantity.name == "deg":
                                #     MyPlot.PlotPosteriorPDF(PosteriorBounds_deg)
                                # elif vs_quantity.name == "cos":
                                #     MyPlot.PlotPosteriorPDF(PosteriorBounds_cos)
                                if plot_plotzilla_bool:
                                    MyPlot.Plotzilla(whether_save = save_plotzilla_bool)
    except:
        print("Error encountered in running loop.")

    # prints all instances of the classes relevant for the arguments of
    # GPAnalysis()
    if print_all_classes:
        scalescheme_current_list = []
        observable_current_list = []
        inputspace_current_list = []
        traintest_current_list = []
        lengthscale_current_list = []

        for obj in gc.get_objects():
            if isinstance(obj, ScaleSchemeBunch):
                scalescheme_current_list.append(obj.name)
            elif isinstance(obj, ObservableBunch):
                observable_current_list.append(obj.name)
            elif isinstance(obj, InputSpaceBunch):
                inputspace_current_list.append(obj.name)
            elif isinstance(obj, TrainTestSplit):
                traintest_current_list.append(obj.name)
            elif isinstance(obj, LengthScale):
                lengthscale_current_list.append(obj.name)

        print("\n\n************************************")
        print("Available potentials: " + str(scalescheme_current_list))
        print("Available observables: " + str(observable_current_list))
        print("Available Q parametrizations: ['poly', 'max', 'sum']")
        print("Available input spaces: " + str(inputspace_current_list))
        print("Available train/test splits: " + str(traintest_current_list))
        print("Available length scales: " + str(lengthscale_current_list))
        print("************************************")

max_stars_repo_name: buqeye/buqeyebox

max_issues_repo_name: buqeye/buqeyebox

max_forks_repo_name: buqeye/buqeyebox

reason: import numpy,import scipy,from scipy


text:
# Authors: Florian Hofer
#
# License: BSD (3-clause)

"""Read datasets containing ECG data and sleep stage annotations."""

import csv
import datetime
from dataclasses import dataclass
from enum import IntEnum
from pathlib import Path
from typing import Iterator, NamedTuple, Optional, Union
from xml.etree import ElementTree

import numpy as np

from ..config import get_config
from ..heartbeats import detect_heartbeats
from .nsrr import _download_nsrr_file, _get_nsrr_url, _list_nsrr, download_nsrr
from .physionet import _list_physionet, download_physionet

class SleepStage(IntEnum):
    """
    Mapping of AASM sleep stages to integers.

    To facilitate hypnogram plotting, values increase with wakefulness.
    """

    WAKE = 5
    REM = 4
    N1 = 3
    N2 = 2
    N3 = 1
    UNDEFINED = -1

class Gender(IntEnum):
    """Mapping of gender to integers."""

    FEMALE = 0
    MALE = 1

@dataclass
class SubjectData:
    """
    Store data about a single subject.

    Attributes
    ----------
    gender : int, optional
        The subject's gender, stored as an integer as defined by `Gender`,
        by default `None`.
    age : int, optional
        The subject's age in years, by default `None`.
    weight : float, optional
        The subject's weight in kg, by default `None`.
    """

    gender: Optional[int] = None
    age: Optional[int] = None
    weight: Optional[float] = None

@dataclass
class SleepRecord:
    """
    Store a single sleep record.

    Attributes
    ----------
    sleep_stages : np.ndarray, optional
        Sleep stages according to AASM guidelines, stored as integers as
        defined by :class:`SleepStage`, by default `None`.
    sleep_stage_duration : int, optional
        Duration of each sleep stage in seconds, by default `None`.
    id : str, optional
        The record's ID, by default `None`.
    recording_start_time : datetime.time, optional
        Time at which the recording was started, by default `None`.
    heartbeat_times : np.ndarray, optional
        Times of heartbeats relative to recording start in seconds, by
        default `None`.
    subject_data : SubjectData, optional
        Dataclass containing subject data, such as gender or age, by
        default `None`.
    """

    sleep_stages: Optional[np.ndarray] = None
    sleep_stage_duration: Optional[int] = None
    id: Optional[str] = None
    recording_start_time: Optional[datetime.time] = None
    heartbeat_times: Optional[np.ndarray] = None
    subject_data: Optional[SubjectData] = None

class _ParseNsrrXmlResult(NamedTuple):
    sleep_stages: np.ndarray
    sleep_stage_duration: int
    recording_start_time: datetime.time

def _parse_nsrr_xml(xml_filepath: Path) -> _ParseNsrrXmlResult:
    """
    Parse NSRR XML sleep stage annotation file.

    Parameters
    ----------
    xml_filepath : pathlib.Path
        Path of the annotation file to read.

    Returns
    -------
    sleep_stages : np.ndarray
        Sleep stages according to AASM guidelines, stored as integers as
        defined by :class:`SleepStage`.
    sleep_stage_duration : int
        Duration of each sleep stage in seconds.
    recording_start_time : datetime.time
        Time at which the recording was started.

    """
    STAGE_MAPPING = {
        'Wake|0': SleepStage.WAKE,
        'Stage 1 sleep|1': SleepStage.N1,
        'Stage 2 sleep|2': SleepStage.N2,
        'Stage 3 sleep|3': SleepStage.N3,
        'Stage 4 sleep|4': SleepStage.N3,
        'REM sleep|5': SleepStage.REM,
        'Unscored|9': SleepStage.UNDEFINED,
    }

    root = ElementTree.parse(xml_filepath).getroot()

    epoch_length = root.findtext('EpochLength')
    if epoch_length is None:
        raise RuntimeError(f'EpochLength not found in {xml_filepath}.')
    epoch_length = int(epoch_length)

    start_time = None
    annot_stages = []

    for event in root.find('ScoredEvents'):
        if event.find('EventConcept').text == 'Recording Start Time':
            start_time = event.find('ClockTime').text.split()[1]
            start_time = datetime.datetime.strptime(start_time, '%H.%M.%S').time()

        if event.find('EventType').text == 'Stages|Stages':
            epoch_duration = int(float(event.findtext('Duration')))
            stage = STAGE_MAPPING[event.findtext('EventConcept')]
            annot_stages.extend([stage] * int(epoch_duration / epoch_length))

    if start_time is None:
        raise RuntimeError(f'"Recording Start Time" not found in {xml_filepath}.')

    return _ParseNsrrXmlResult(
        np.array(annot_stages, dtype=np.int8),
        epoch_length,
        start_time,
    )

def read_mesa(
    records_pattern: str = '*',
    heartbeats_source: str = 'annotation',
    offline: bool = False,
    keep_edfs: bool = False,
    data_dir: Optional[Union[str, Path]] = None,
) -> Iterator[SleepRecord]:
    """
    Lazily read records from MESA (https://sleepdata.org/datasets/mesa).

    Each MESA record consists of an `.edf` file containing raw
    polysomnography data and an `.xml` file containing annotated events.
    Since the entire MESA dataset requires about 385 GB of disk space,
    `.edf` files can be deleted after heartbeat times have been extracted.
    Heartbeat times are cached in an `.npy` file in
    `<data_dir>/mesa/preprocessed/heartbeats`.

    Parameters
    ----------
    records_pattern : str, optional
         Glob-like pattern to select record IDs, by default `'*'`.
    heartbeats_source : {'annotation', 'cached', 'ecg'}, optional
        If `'annotation'` (default), get heartbeat times from
        `polysomnography/annotations-rpoints/<record_id>-rpoints.csv` (not
        available for all records). If `'ecg'`, use
        `sleepecg.detect_heartbeats` on the ECG contained in
        `polysomnography/edfs/<record_id>.edf` and cache the result to
        `preprocessed/heartbeats/<record_id>.npy`. If `'cached'`, get the
        cached heartbeats.
    offline : bool, optional
        If `True`, search for local files only instead of using the NSRR
        API, by default `False`.
    keep_edfs : bool, optional
        If `False`, remove `.edf` after heartbeat detection, by default
        `False`.
    data_dir : str | pathlib.Path, optional
        Directory where all datasets are stored. If `None` (default), the
        value will be taken from the configuration.

    Yields
    ------
    SleepRecord
        Each element in the generator is a :class:`SleepRecord`.
    """
    from mne.io import read_raw_edf

    DB_SLUG = 'mesa'
    ANNOTATION_DIRNAME = 'polysomnography/annotations-events-nsrr'
    EDF_DIRNAME = 'polysomnography/edfs'
    HEARTBEATS_DIRNAME = 'preprocessed/heartbeats'
    RPOINTS_DIRNAME = 'polysomnography/annotations-rpoints'

    GENDER_MAPPING = {0: Gender.FEMALE, 1: Gender.MALE}

    heartbeats_source_options = {'annotation', 'cached', 'ecg'}
    if heartbeats_source not in heartbeats_source_options:
        raise ValueError(
            f'Invalid value for parameter `heartbeats_source`: {heartbeats_source}, '
            f'possible options: {heartbeats_source_options}',
        )

    if data_dir is None:
        data_dir = get_config('data_dir')

    db_dir = Path(data_dir).expanduser() / DB_SLUG
    annotations_dir = db_dir / ANNOTATION_DIRNAME
    edf_dir = db_dir / EDF_DIRNAME
    heartbeats_dir = db_dir / HEARTBEATS_DIRNAME

    for directory in (annotations_dir, edf_dir, heartbeats_dir):
        directory.mkdir(parents=True, exist_ok=True)

    if not offline:
        download_url = _get_nsrr_url(DB_SLUG)

        subject_data_filename, subject_data_checksum = _list_nsrr(
            'mesa',
            'datasets',
            'mesa-sleep-dataset-*.csv',
            shallow=True,
        )[0]
        subject_data_filepath = db_dir / subject_data_filename
        _download_nsrr_file(
            download_url + subject_data_filename,
            target_filepath=subject_data_filepath,
            checksum=subject_data_checksum,
        )

        checksums = {}
        xml_files = _list_nsrr(
            DB_SLUG,
            ANNOTATION_DIRNAME,
            f'mesa-sleep-{records_pattern}-nsrr.xml',
            shallow=True,
        )
        checksums.update(xml_files)
        requested_records = [Path(file).stem[:-5] for file, _ in xml_files]

        edf_files = _list_nsrr(
            DB_SLUG,
            EDF_DIRNAME,
            f'mesa-sleep-{records_pattern}.edf',
            shallow=True,
        )
        checksums.update(edf_files)

        rpoints_files = _list_nsrr(
            DB_SLUG,
            RPOINTS_DIRNAME,
            f'mesa-sleep-{records_pattern}-rpoint.csv',
            shallow=True,
        )
        checksums.update(rpoints_files)
    else:
        subject_data_filepath = next((db_dir / 'datasets').glob('mesa-sleep-dataset-*.csv'))
        xml_files = sorted(annotations_dir.glob(f'mesa-sleep-{records_pattern}-nsrr.xml'))
        requested_records = [file.stem[:-5] for file in xml_files]

    subject_data_array = np.loadtxt(
        subject_data_filepath,
        delimiter=',',
        skiprows=1,
        usecols=[0, 3, 5],  # [mesaid, gender, age]
        dtype=int,
    )

    subject_data = {}
    for mesaid, gender, age in subject_data_array:
        subject_data[f'mesa-sleep-{mesaid:04}'] = SubjectData(
            gender=GENDER_MAPPING[gender],
            age=age,
        )

    for record_id in requested_records:
        heartbeats_file = heartbeats_dir / f'{record_id}.npy'
        if heartbeats_source == 'annotation':
            rpoints_filename = f'{RPOINTS_DIRNAME}/{record_id}-rpoint.csv'
            rpoints_filepath = db_dir / rpoints_filename
            if not rpoints_filepath.is_file():
                if not offline and rpoints_filename in checksums:
                    _download_nsrr_file(
                        download_url + rpoints_filename,
                        rpoints_filepath,
                        checksums[rpoints_filename],
                    )
                else:
                    print(f'Skipping {record_id} due to missing heartbeat annotations.')
                    continue

            heartbeat_times = np.loadtxt(
                rpoints_filepath,
                delimiter=',',
                skiprows=1,
                usecols=18,  # column 18 ('seconds') contains the annotated heartbeat times
            )
            # for some reason some (39) records have unsorted annotations
            heartbeat_times.sort()
        elif heartbeats_source == 'cached':
            if not heartbeats_file.is_file():
                print(f'Skipping {record_id} due to missing cached heartbeats.')
                continue
            heartbeat_times = np.load(heartbeats_file)
        elif heartbeats_source == 'ecg':
            edf_filename = EDF_DIRNAME + f'/{record_id}.edf'
            edf_filepath = db_dir / edf_filename
            edf_was_available = edf_filepath.is_file()
            if not offline:
                _download_nsrr_file(
                    download_url + edf_filename,
                    edf_filepath,
                    checksums[edf_filename],
                )

            rec = read_raw_edf(edf_filepath, verbose=False)
            ecg = rec.get_data('EKG').ravel()
            fs = rec.info['sfreq']
            heartbeat_indices = detect_heartbeats(ecg, fs)
            heartbeat_times = heartbeat_indices / fs
            np.save(heartbeats_file, heartbeat_times)

            if not edf_was_available and not keep_edfs:
                edf_filepath.unlink()

        xml_filename = ANNOTATION_DIRNAME + f'/{record_id}-nsrr.xml'
        xml_filepath = db_dir / xml_filename
        if not offline:
            _download_nsrr_file(
                download_url + xml_filename,
                xml_filepath,
                checksums[xml_filename],
            )

        parsed_xml = _parse_nsrr_xml(xml_filepath)

        yield SleepRecord(
            sleep_stages=parsed_xml.sleep_stages,
            sleep_stage_duration=parsed_xml.sleep_stage_duration,
            id=record_id,
            recording_start_time=parsed_xml.recording_start_time,
            heartbeat_times=heartbeat_times,
            subject_data=subject_data[record_id],
        )

def read_slpdb(
    records_pattern: str = '*',
    offline: bool = False,
    data_dir: Optional[Union[str, Path]] = None,
) -> Iterator[SleepRecord]:
    """
    Lazily read records from SLPDB (https://physionet.org/content/slpdb).

    Required files are downloaded from PhysioNet to `<data_dir>/slpdb`.

    Parameters
    ----------
    records_pattern : str, optional
         Glob-like pattern to select record IDs, by default `'*'`.
    offline : bool, optional
        If `True`, search for local files only instead of downloading from
        PhysioNet, by default `False`.
    data_dir : str | pathlib.Path, optional
        Directory where all datasets are stored. If `None` (default), the
        value will be taken from the configuration.

    Yields
    ------
    SleepRecord
        Each element in the generator is a :class:`SleepRecord`.
    """
    # https://physionet.org/content/slpdb/1.0.0/
    import wfdb

    DB_SLUG = 'slpdb'

    STAGE_MAPPING = {
        'W': SleepStage.WAKE,
        'R': SleepStage.REM,
        '1': SleepStage.N1,
        '2': SleepStage.N2,
        '3': SleepStage.N3,
        '4': SleepStage.N3,
    }

    if data_dir is None:
        data_dir = get_config('data_dir')

    data_dir = Path(data_dir).expanduser()
    db_dir = data_dir / DB_SLUG

    requested_records = _list_physionet(
        data_dir=data_dir,
        db_slug=DB_SLUG,
        pattern=records_pattern,
    )

    if not offline:
        download_physionet(
            data_dir=data_dir,
            db_slug=DB_SLUG,
            requested_records=requested_records,
            extensions=['.hea', '.dat', '.st'],
        )

    for record_id in requested_records:
        record_file = str(db_dir / record_id)

        record = wfdb.rdrecord(record_file)
        start_time = record.base_time
        ecg = np.asarray(record.p_signal[:, record.sig_name.index('ECG')])
        fs = record.fs

        heartbeat_indices = detect_heartbeats(ecg, fs)
        heartbeat_times = heartbeat_indices / fs

        annot_st = wfdb.rdann(record_file, 'st')

        # Some 30 second windows don't have a sleep stage annotation, so
        # the annotation array is initialized with `SleepStage.UNDEFINED`
        # for every 30 second window.
        for sample_time, annotation in zip(annot_st.sample[::-1], annot_st.aux_note[::-1]):
            if annotation[0] in STAGE_MAPPING:
                number_of_sleep_stages = sample_time // (30 * fs) + 1
                break

        sleep_stages = np.full(number_of_sleep_stages, SleepStage.UNDEFINED)

        # Most annotations are at sample indices which are multiples of
        # 30*fs. However, annotations which would be at sample index 0, are
        # at sample index 1. Integer divison is used when calculating the
        # stage index to move these annotations to sample index 0.
        for sample_time, annotation in zip(annot_st.sample, annot_st.aux_note):
            if annotation[0] in STAGE_MAPPING:
                sleep_stages[sample_time // (30 * fs)] = STAGE_MAPPING[annotation[0]]

        # Age and weight are given in the last line of the header file,
        # which is contained in record.comments[0] and looks like this:
        # '44 M 89 32-01-89' ('<age> <gender> <weight> <unspecified>')
        # For some records, age/weight is given as 'x'.
        age, _, weight, _ = record.comments[0].split()
        subject_data = SubjectData(
            gender=Gender.MALE,  # all slpdb subjects were male
            age=None if age == 'x' else int(age),
            weight=None if weight == 'x' else int(weight),
        )

        yield SleepRecord(
            sleep_stages=sleep_stages,
            sleep_stage_duration=30,
            id=record_id,
            recording_start_time=start_time,
            heartbeat_times=heartbeat_times,
            subject_data=subject_data,
        )

def read_shhs(
    records_pattern: str = '*',
    heartbeats_source: str = 'annotation',
    offline: bool = False,
    keep_edfs: bool = False,
    data_dir: Optional[Union[str, Path]] = None,
) -> Iterator[SleepRecord]:
    """
    Lazily read records from SHHS (https://sleepdata.org/datasets/shhs).

    Each SHHS record consists of an `.edf` file containing raw
    polysomnography data and an `.xml` file containing annotated events.
    Since the entire SHHS dataset requires about 356 GB of disk space,
    `.edf` files can be deleted after heartbeat times have been extracted.
    Heartbeat times are cached in an `.npy` file in
    `<data_dir>/shhs/preprocessed/heartbeats`.

    Parameters
    ----------
    records_pattern : str, optional
         Glob-like pattern to select record IDs, by default `'*'`.
    heartbeats_source : {'annotation', 'cached', 'ecg'}, optional
        If `'annotation'` (default), get heartbeat times from
        `polysomnography/annotations-rpoints/shhsX/<record_id>-rpoints.csv`
        (not available for all records). If `'ecg'`, use
        `sleepecg.detect_heartbeats` on the ECG contained in
        `polysomnography/edfs/shhsX/<record_id>.edf` and cache the result
        to `preprocessed/heartbeats/shhsX/<record_id>.npy`. If `'cached'`,
        get the cached heartbeats.
    offline : bool, optional
        If `True`, search for local files only instead of using the NSRR
        API, by default `False`.
    keep_edfs : bool, optional
        If `False`, remove `.edf` after heartbeat detection, by default
        `False`.
    data_dir : str | pathlib.Path, optional
        Directory where all datasets are stored. If `None` (default), the
        value will be taken from the configuration.

    Yields
    ------
    SleepRecord
        Each element in the generator is a :class:`SleepRecord`.
    """
    from mne.io import read_raw_edf

    DB_SLUG = 'shhs'
    ANNOTATION_DIRNAME = 'polysomnography/annotations-events-nsrr'
    EDF_DIRNAME = 'polysomnography/edfs'
    HEARTBEATS_DIRNAME = 'preprocessed/heartbeats'
    RPOINTS_DIRNAME = 'polysomnography/annotations-rpoints'

    # see shhs/datasets/shhs-data-dictionary-0.16.0-domains.csv lines 91+92
    GENDER_MAPPING = {'2': Gender.FEMALE, '1': Gender.MALE}

    heartbeats_source_options = {'annotation', 'cached', 'ecg'}
    if heartbeats_source not in heartbeats_source_options:
        raise ValueError(
            f'Invalid value for parameter `heartbeats_source`: {heartbeats_source}, '
            f'possible options: {heartbeats_source_options}',
        )

    if data_dir is None:
        data_dir = get_config('data_dir')

    data_dir = Path(data_dir).expanduser()
    db_dir = data_dir / DB_SLUG
    annotations_dir = db_dir / ANNOTATION_DIRNAME
    edf_dir = db_dir / EDF_DIRNAME
    heartbeats_dir = db_dir / HEARTBEATS_DIRNAME

    for directory in (annotations_dir, edf_dir, heartbeats_dir):
        directory.mkdir(parents=True, exist_ok=True)

    if not offline:
        download_url = _get_nsrr_url(DB_SLUG)

        download_nsrr(
            DB_SLUG,
            'datasets',
            'shhs?-dataset-*.csv',
            shallow=True,
            data_dir=data_dir,
        )

        checksums = {}
        xml_files = _list_nsrr(
            DB_SLUG,
            ANNOTATION_DIRNAME,
            f'{records_pattern}-nsrr.xml',
            shallow=False,
        )
        checksums.update(xml_files)
        requested_records = [file[-27:-9] for file, _ in xml_files]

        edf_files = _list_nsrr(
            DB_SLUG,
            EDF_DIRNAME,
            f'{records_pattern}.edf',
            shallow=False,
        )
        checksums.update(edf_files)

        rpoints_files = _list_nsrr(
            DB_SLUG,
            RPOINTS_DIRNAME,
            f'{records_pattern}-rpoint.csv',
            shallow=False,
        )
        checksums.update(rpoints_files)
    else:
        xml_files = sorted(annotations_dir.rglob(f'{records_pattern}-nsrr.xml'))
        requested_records = [str(file)[-27:-9] for file in xml_files]

    subject_data = {}

    if any(r.startswith('shhs1') for r in requested_records):
        subject_data_file_shhs1 = next((db_dir / 'datasets').glob('shhs1-dataset-*.csv'))
        with open(subject_data_file_shhs1, newline='') as csvfile:
            reader = csv.DictReader(csvfile)
            for row in reader:
                record_id = f'shhs1-{row["nsrrid"]}'
                subject_data[record_id] = SubjectData(
                    gender=GENDER_MAPPING[row['gender']],
                    age=int(row['age_s1']),
                    weight=float(row['weight']) if row['weight'] else None,
                )
    if any(r.startswith('shhs2') for r in requested_records):
        subject_data_file_shhs2 = next((db_dir / 'datasets').glob('shhs2-dataset-*.csv'))
        with open(subject_data_file_shhs2, newline='') as csvfile:
            reader = csv.DictReader(csvfile)
            for row in reader:
                record_id = f'shhs2-{row["nsrrid"]}'
                subject_data[record_id] = SubjectData(
                    gender=GENDER_MAPPING[row['gender']],
                    age=int(row['age_s2']),
                    weight=None,  # subject weight was not recorded in shhs2
                )

    for record_id in requested_records:
        heartbeats_file = heartbeats_dir / f'{record_id}.npy'
        if heartbeats_source == 'annotation':
            rpoints_filename = f'{RPOINTS_DIRNAME}/{record_id}-rpoint.csv'
            rpoints_filepath = db_dir / rpoints_filename
            if not rpoints_filepath.is_file():
                if not offline and rpoints_filename in checksums:
                    _download_nsrr_file(
                        download_url + rpoints_filename,
                        rpoints_filepath,
                        checksums[rpoints_filename],
                    )
                else:
                    print(f'Skipping {record_id} due to missing heartbeat annotations.')
                    continue
            heartbeat_times = np.loadtxt(
                rpoints_filepath,
                delimiter=',',
                skiprows=1,
                usecols=19,  # column 19 ('seconds') contains the annotated heartbeat times
            )
        elif heartbeats_source == 'cached':
            if not heartbeats_file.is_file():
                print(f'Skipping {record_id} due to missing cached heartbeats.')
                continue
        elif heartbeats_source == 'ecg':
            edf_filename = EDF_DIRNAME + f'/{record_id}.edf'
            edf_filepath = db_dir / edf_filename
            edf_was_available = edf_filepath.is_file()
            if not offline:
                _download_nsrr_file(
                    download_url + edf_filename,
                    edf_filepath,
                    checksums[edf_filename],
                )

            rec = read_raw_edf(edf_filepath, verbose=False)
            ecg = rec.get_data('ECG').ravel()
            fs = rec.info['sfreq']
            heartbeat_indices = detect_heartbeats(ecg, fs)
            heartbeat_times = heartbeat_indices / fs

            heartbeats_file.parent.mkdir(parents=True, exist_ok=True)
            np.save(heartbeats_file, heartbeat_times)

            if not edf_was_available and not keep_edfs:
                edf_filepath.unlink()

        xml_filename = ANNOTATION_DIRNAME + f'/{record_id}-nsrr.xml'
        xml_filepath = db_dir / xml_filename
        if not offline:
            _download_nsrr_file(
                download_url + xml_filename,
                xml_filepath,
                checksums[xml_filename],
            )

        parsed_xml = _parse_nsrr_xml(xml_filepath)

        yield SleepRecord(
            sleep_stages=parsed_xml.sleep_stages,
            sleep_stage_duration=parsed_xml.sleep_stage_duration,
            id=record_id[6:],  # remove subdirectory
            recording_start_time=parsed_xml.recording_start_time,
            heartbeat_times=heartbeat_times,
            subject_data=subject_data[record_id[6:]],  # remove subdirectory]
        )

max_stars_repo_name: hofaflo/sleepecg

max_issues_repo_name: hofaflo/sleepecg

max_forks_repo_name: hofaflo/sleepecg

reason: import numpy


text:
#!/usr/bin/env python
__author__ = "etseng@pacb.com"

"""
Reporter for junction summary for one or more samples.
Recommended to run before scrubbing sample GFFs prior to chaining.

Suggested process is:
1. run collapse to get GFF for each sample
2. run this report script on all sample GFFs
3. run scrubber on all sample GFFs
4. run chaining using scrubbed (cleaned) sample GFFs
"""
import sys
from collections import defaultdict
from csv import DictWriter
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union

import numpy as np
import typer
from Bio import SeqIO
from sklearn.cluster import Birch

import cupcake.sequence.GFF as GFF
from cupcake import version_callback
from cupcake import cupcake_logger as logger

app = typer.Typer(name="cupcake.tofu.counting.summarize_sample_GFF_junctions")

def sanity_check(
    sample_dirs: List[Dict[str, Union[str, Path]]],
    gff_filename: Union[str, Path],
    genome_filename: Optional[Union[str, Path]] = None,
    junction_filename: Optional[Union[str, Path]] = None,
) -> None:
    for d in sample_dirs.values():
        file = Path(d, gff_filename)
        if not file.exists():
            logger.error(f"Expected GFF file {file} does not exist. Abort!")
            sys.exit(-1)

    if genome_filename is not None and not Path(genome_filename).exists():
        logger.error(f"Genome file {genome_filename} given but does not exist. Abort!")
        sys.exit(-1)

    if junction_filename is not None and not Path(junction_filename).exists():
        logger.error(
            f"Junction file {junction_filename} given but does not exist. Abort!"
        )
        sys.exit(-1)

def read_config(filename: Path) -> Tuple[Dict[str, Path], List[str], Path, Path, Path]:
    """
    SAMPLE=<name>;<path>

    must also have
    GFF_FILENAME=

    optional:
    GENOME_FILENAME=
    JUNCTION_FILENAME=
    GROUP_FILENAME=

    Everything else will be ignored (so you can re-use sample.config for chain_samples.py)
    """
    sample_dirs: Dict[str, Path] = {}
    sample_names: List[str] = []
    gff_filename: Optional[Union[str, Path]] = None
    genome_filename: Optional[Union[str, Path]] = None
    junction_filename: Optional[Union[str, Path]] = None

    if not filename.exists():
        raise FileNotFoundError(f"The config file {filename} could not be found!")

    with open(filename) as f:
        for line in f:
            if line.startswith("tmpSAMPLE="):
                logger.error(
                    "Please only use SAMPLE=, not tmpSAMPLE= for junction reports!"
                )
                sys.exit(-1)
            elif line.startswith("SAMPLE="):
                name, path = line.strip()[len("SAMPLE=") :].split(";")
                if name.startswith("tmp_"):
                    logger.error(
                        f"Sample names are not allowed to start with tmp_! Please change {name} to something else."
                    )
                    sys.exit(-1)
                sample_dirs[name] = Path(path).resolve()
                sample_names.append(name)
            elif line.startswith("GFF_FILENAME="):
                gff_filename = Path(line.strip()[len("GFF_FILENAME=") :])
            elif line.startswith("GENOME_FILENAME="):
                genome_filename = Path(line.strip()[len("GENOME_FILENAME=") :])
            elif line.startswith("JUNCTION_FILENAME="):
                junction_filename = Path(line.strip()[len("JUNCTION_FILENAME=") :])

    if gff_filename is None:
        raise Exception(
            f"Expected GFF_FILENAME= but not in config file {filename}! Abort."
        )

    if len(sample_names) == 0:
        logger.error("No samples given. Exit.")
        sys.exit(-1)

    return sample_dirs, gff_filename, genome_filename, junction_filename

def read_annotation_junction_bed(junction_filename: Union[str, Path]) -> defaultdict:
    """
    junction.bed is in format:

    seqname, left (0-based), right (0-based), +/-

    following junction.bed format from TopHat
    http://ccb.jhu.edu/software/tophat/manual.shtml
    """
    junction = defaultdict(dict)  # (seqname, strand) --> (start, end)
    for line in open(junction_filename):
        chrom, left, right, strand = line.strip().split("\t")
        junction[chrom, strand][(int(left), int(right))] = 1
    return junction

def summarize_junctions(
    sample_dirs: Dict[str, Path],
    # sample_names: List[str],
    gff_filename: Union[str, Path],
    output_prefix: Union[str, Path],
    genome_d: Optional[Union[str, Path]] = None,
    junction_known: Optional[Union[str, Path]] = None,
) -> defaultdict:
    """
    1. for each sample, read all the GFF, store the junction information (both 0-based)

    """
    junc_by_chr_strand = defaultdict(
        lambda: defaultdict(list)
    )  # (seqname,strand) --> (donor,acceptor) --> samples it show up in (more than once possible)

    for sample_name, d in sample_dirs.items():
        for r in GFF.collapseGFFReader(Path(d, gff_filename)):
            n = len(r.ref_exons)
            if n == 1:
                continue  # ignore single exon transcripts
            for i in range(n - 1):
                donor = r.ref_exons[i].end - 1  # make it 0-based
                accep = r.ref_exons[i + 1].start  # start is already 0-based
                junc_by_chr_strand[r.seqname, r.strand][donor, accep].append(
                    sample_name
                )

    # write junction report
    with open(f"{output_prefix}.junction.bed", "w") as f1, open(
        f"{output_prefix}.junction_detail.txt", "w"
    ) as f:
        f1.write(f'track name=junctions description="{output_prefix}" useScore=1\n')

        JUNC_DETAIL_FIELDS = [
            "seqname",
            "left",
            "right",
            "strand",
            "num_transcript",
            "num_sample",
            "genome",
            "annotation",
            "label",
        ]

        writer = DictWriter(f, JUNC_DETAIL_FIELDS, delimiter="\t")
        writer.writeheader()
        keys = list(junc_by_chr_strand)
        keys.sort()
        for _seqname, _strand in keys:
            v = junc_by_chr_strand[_seqname, _strand]
            v_keys = list(v)
            v_keys.sort()
            labels = cluster_junctions(v_keys)
            for i, (_donor, _accep) in enumerate(v_keys):
                rec = {
                    "seqname": _seqname,
                    "left": _donor,
                    "right": _accep,
                    "strand": _strand,
                    "num_transcript": len(v[_donor, _accep]),
                    "num_sample": len(set(v[_donor, _accep])),
                }
                # f.write("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t".format(_chr, _donor, _accep, _strand, len(v[_donor,_accep]), len(set(v[_donor,_accep]))))
                f1.write(
                    f"{_seqname}\t{_donor}\t{_accep + 1}\t{output_prefix}\t{len(v[_donor, _accep])}\t{_strand}\n"
                )
                # if genome is given, write acceptor-donor site
                if genome_d is None or _seqname not in genome_d:
                    rec["genome"] = "NA"
                    # f.write("NA\t")
                else:
                    up, down = (
                        genome_d[_seqname][(_donor + 1) : (_donor + 3)],
                        genome_d[_seqname][(_accep - 2) : _accep],
                    )
                    if _strand == "+":
                        rec["genome"] = f"{str(up.seq).upper()}-{str(down.seq).upper()}"
                        # f.write("{0}-{1}\t".format(str(up.seq).upper(), str(down.seq).upper()))
                    else:
                        rec[
                            "genome"
                        ] = f"{str(down.reverse_complement().seq).upper()}-{str(up.reverse_complement().seq).upper()}"
                        # f.write("{0}-{1}\t".format(str(down.reverse_complement().seq).upper(), str(up.reverse_complement().seq).upper()))
                # if annotation is given, check if matches with annotation
                if junction_known is None:
                    rec["annotation"] = "NA"
                    # f.write("NA\n")
                else:
                    if (_seqname, _strand) in junction_known and (
                        _donor,
                        _accep,
                    ) in junction_known[_seqname, _strand]:
                        rec["annotation"] = "Y"
                        # f.write("Y\t")
                    else:
                        rec["annotation"] = "N"
                        # f.write("N\t")
                rec["label"] = f"{_seqname}_{_strand}_{labels[i]}"
                writer.writerow(rec)
            # f.write("{c}_{s}_{lab}\n".format(c=_seqname, s=_strand, lab=labels[i]))

    return junc_by_chr_strand

def cluster_junctions(juncs: List[int]) -> np.ndarray:
    birch_model = Birch(threshold=3, n_clusters=None)
    X = np.array(juncs)
    birch_model.fit(X)

    return birch_model.labels_

@app.command(name="")
def main(
    config: Union[str, Path] = typer.Argument(..., help="Config filename"),
    output_prefix: str = typer.Argument(..., help="Output prefix"),
    version: bool = typer.Option(
        None,
        "--version",
        callback=version_callback,
        is_eager=True,
        help="Prints the version of the SQANTI3 package.",
    ),
):

    try:
        (
            sample_dirs,
            gff_filename,
            genome_filename,
            junction_filename,
        ) = read_config(config)
    except FileNotFoundError as error:
        logger.error(error)

    sanity_check(sample_dirs, gff_filename, genome_filename, junction_filename)

    if genome_filename is not None:
        logger.info(f"Reading genome file {genome_filename}...")
        genome_d = SeqIO.to_dict(SeqIO.parse(open(genome_filename), "fasta"))
    else:
        logger.info("No genome file given. Ignore.")
        genome_d = None

    if junction_filename is not None:
        logger.info(f"Reading junction file {junction_filename}....")
        junction_bed = read_annotation_junction_bed(junction_filename)
    else:
        logger.info("No junction file given. Ignore.")
        junction_bed = None

    summarize_junctions(
        sample_dirs,
        gff_filename,
        output_prefix,
        genome_d,
        junction_bed,
    )

if __name__ == "__main__":
    typer.run(main)


text:
import numpy as np
import cv2

cv2.namedWindow("frame",1)

#get from webcam
cap = cv2.VideoCapture(0)

#find params for corner detection via Shi-Tomasi
shi_params = dict( maxCorners = 3000,
                       qualityLevel = 0.05, #determines amount of relevant points determined
                       minDistance = 7,
                       blockSize = 7 )

#find params for Lucas-Kanade optical flow
lk_params = dict( winSize  = (12,12), #larger window means better tracking, but slower computation (more cost)
                  maxLevel = 0,
                  criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))

#random color gen!!! -- color[i].tolist()
#color = np.random.randint(0,255,(1000000,3))

#take first frame
ret, old_frame = cap.read()

#grayscale convert it
old_gray = cv2.cvtColor(old_frame, cv2.COLOR_RGB2GRAY)

while(1):
    #we need to continuously find good params to track
    p_old = cv2.goodFeaturesToTrack(old_gray, mask = None, **shi_params)

    #read curr frame and grayscale it
    ret,frame = cap.read()
    fgray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)

    #init p_new in case we wanted to use flags
    p_new = np.zeros_like(p_old)

    #use Lucas-Kanade algorithm to find optical flow
    p_new, status, error = cv2.calcOpticalFlowPyrLK(old_gray, fgray, p_old, p_new, **lk_params)

    #find the best matching points
    good_new = p_new[status == 1] #status is 1 for a match
    good_old = p_old[status == 1]

    mask = np.zeros_like(frame);

    #draw the overlaying tracking img
    for i,(new,old) in enumerate(zip(good_new,good_old)):
        a,b = new.ravel() #tmp new value
        c,d = old.ravel() #tmp old value -- necessary for drawing line tracking

        #draws a line connecting the old point with the new point
        cv2.line(mask,(a,b),(c,d),(0,255,0),2)

        #draws the new dot
        cv2.circle(frame,(int(a),int(b)),4,(255,0,0),-1)

    #this is if we want to add lines into the tracking to follow the path
    img = cv2.add(frame,mask)

    #show on window
    cv2.imshow("frame",img)

    #update the previous frame and previous points
    old_gray = fgray.copy()

    #to exit window
    if cv2.waitKey(10) == 27:
        break

#clean up
cv2.destroyAllWindows()
cap.release()


text:
from torch.utils.data import Dataset
import numpy as np
import torch
from . import functions

class TokensDataset(Dataset):
    def __init__(self, X, Y):
        self.X = self.encode_x(X)
        self.y = Y

    @staticmethod
    def encode_x(x: list) -> list:
        max_len = len(max(x, key=lambda i: len(i)))
        encoded = []
        for i in x:
            encoded.append(np.array(functions.encode(i, max_len)))
        return encoded

    @staticmethod
    def collate_fn(objs: list) -> (torch.LongTensor, torch.Tensor):
        data = ([i[0] for i in objs])
        labels = ([i[1] for i in objs])
        data = torch.LongTensor(data)
        labels = torch.tensor(labels)
        return data, labels

    def __len__(self):
        return len(self.y)

    def __getitem__(self, idx):
        return self.X[idx], self.y[idx]

max_stars_repo_name: GroupLe/grouple-face-tagger

max_issues_repo_name: GroupLe/grouple-face-tagger

max_forks_repo_name: GroupLe/grouple-face-tagger