text:
"""
Use this scoring program in other programs as:

    import ScoreNeuroPsych
    ScoreNeuroPsych.ScoreAll()

    This could be added to the end of the GUI program, so that when the GUI is closed
    the data os scored and updated.

To do:
    Also add something so that it collapses across multiple runs of the DMS, VSTM and NBack tasks

"""
import os
import sys
import fnmatch
import shutil
import pandas as pd
import numpy as np
import glob
import datetime
import collections

# What folder is this file in?
dir_path = os.path.dirname(os.path.realpath(__file__))
dir_path =  '/Users/jasonsteffener/Documents/GitHub/CognitiveTasks'
# This will load the config file containing the location of the data folder
# If there is an error it means that the GUI program has not been run.
# The GUI checks to see if thie config file exists. If it does not then it is created.
print(dir_path)
# This is expecting this repo to sit next to the repo with the task code
sys.path.append(os.path.join(dir_path, '..','CognitiveTasks','ConfigFiles'))
sys.path.append(os.path.join(dir_path, 'code'))

import ProcessNeuroPsychFunctions
import ProcessBehavioralFunctions
import NeuropsychDataFolder
# Load up the data location as a global variable
AllInDataFolder = NeuropsychDataFolder.NeuropsychDataFolder
# Where to put the summary data
AllOutDataFolder = os.path.join(os.path.split(AllInDataFolder)[0], 'SummaryData')

def main():
    # Cycle over all data folders and load them up
    NewData = CycleOverDataFolders()
    # find the name of the existing results file
    ExistingDataFileName = LocateOutDataFile()
    print(ExistingDataFileName)
    # Load the existing results file
    if os.path.exists(ExistingDataFileName):
        # Found the existing data file
        OldData = LoadOutDataFile(ExistingDataFileName)
        # created an updated results datafram, respectivein the "locked down"
        # data rows
        UpdatedData = CreateUpdatedDataFrameOfResults(NewData, OldData)
        # Create an updated output file name
        UpdatedDataFileName = CreateOutFileName()
        # write out the updated data and move the old data file
        WriteOutNewdataMoveOldData(UpdatedData, UpdatedDataFileName, ExistingDataFileName)
    else:
        # There is no old data file
        OldData = []
        NewData.to_csv(ExistingDataFileName, index = False, float_format='%.3f')
    return NewData

def CycleOverDataFolders():
    # Take as input the folder that contains folders of data
    #cycle over folders
    # Enter each folder and identify the visit folders in them.
    # These start with the letter V
    df = pd.DataFrame()
    ListOfDict = []
    # get all sub dirs
    subdirs = glob.glob(os.path.join(AllInDataFolder,'*/'))

    for subdir in subdirs:
        print(subdir)
        # check subdir based on some criteria
        CurDir = os.path.split(subdir)[0]
        CurDir = os.path.split(CurDir)[-1]
        if CurDir.isdigit():

            print("FOUND FOLDER: %s"%(CurDir))
            #enter the directory and find visit folders

            VisitFolders = glob.glob(os.path.join(subdir,'*/'))
            # What if there are no visit folders?
            # Then define teh visit as V001 and use a filename to idetify the visitid
            # Check to see if there is a visiti folder in the subject folder and if there is
            # no visit folder, check to see if there are data files
            if (len(VisitFolders) > 0):
                for visFold in VisitFolders:
                    CurVis = os.path.split(visFold)[0]
                    CurVis = os.path.split(CurVis)[-1]
                    if CurVis[-4:-2] == 'V0':
                        # From the directory structre extract the subject ID and the visit ID
                        subid = CurDir
                        Visid = CurVis
                        print('====================================')
                        print('%s, %s'%(subid, Visid))
                        # Load up the raw data from the files in the visit folder
                        Results = LoadRawData(os.path.join(AllInDataFolder, subid, Visid),subid)
                        # Results = LoadRawDataSHORT(os.path.join(AllOutDataFolder, subid, Visid),subid)
                        FlatResults = ProcessBehavioralFunctions.FlattenDict(Results)
                        # add subid and visitid
                        FlatResults['AAsubid'] = subid
                        FlatResults['AAVisid'] = Visid
                        FlatResults['AAChecked'] = 0
                        ListOfDict.append(FlatResults)
            elif len(os.listdir(subdir)) > 0:
                # This seems to be data in a subject folder with no visit folder
                # Create the visit ID
                subid = CurDir
                Visid = FindVisitIDFromFileNames(subdir)

                # Load up the raw data from the files in the visit folder
                Results = LoadRawData(os.path.join(AllInDataFolder, subid),subid)
                FlatResults = ProcessBehavioralFunctions.FlattenDict(Results)
                # add subid and visitid
                FlatResults['AAsubid'] = subid
                FlatResults['AAVisid'] = Visid
                FlatResults['AAChecked'] = 0
                ListOfDict.append(FlatResults)

    df = pd.DataFrame(ListOfDict)

    # Move the last three columns to the beginning of the data frame
    # Make list of column names
    ColNameList = []
    for col in df:
        ColNameList.append(col)
    # Now move the last three columns to the beginning
    ItemsToMove = ['AAsubid', 'AAVisid', 'AAChecked']
    count = 0
    for j in ItemsToMove:
        # Find the location of the item
        index = ColNameList.index(j)
        ColNameList.insert(count,ColNameList.pop(index))
        count += 1
    # Now apply these rearranged columns to the dataframe
    df = df[ColNameList]
    return df

def FindVisitIDFromFileNames(subdir):
    # Make a list of the files in the folder
    ListOfFiles = glob.glob(os.path.join(subdir,'*.csv'))
    # Find the date and time stamps encoded in teh file names
    for filePath in ListOfFiles:
        Visid = ProcessBehavioralFunctions.ParseFileNamesForDateTime(filePath)
        break
    return Visid

def LoadRawData(VisitFolder, subid):
    # Given a visit folder, check for the existance of specific files
    # read the file and process teh results
    # This function looks for very specific files

    print('working on %s'%(subid))
    Results = collections.OrderedDict()
    # Stroop
    Data = ReadFile(VisitFolder, subid, 'Stroop_Color_')
    Results['StrpC'] = ProcessNeuroPsychFunctions.ProcessStroopColor(Data)
    print('\tStroop Color loaded')

    Data = ReadFile(VisitFolder, subid, 'Stroop_Word_')
    Results['StrpW'] = ProcessNeuroPsychFunctions.ProcessStroopWord(Data)
    print('\tStroop Word loaded')

    Data = ReadFile(VisitFolder, subid, 'Stroop_ColorWord')
    Results['StrpCW'] = ProcessNeuroPsychFunctions.ProcessStroopColorWord(Data)
    print('\tStroop Color/Word loaded')

    # Wisconsin Card Sort
    Data = ReadFile(VisitFolder, subid, 'WCST')
    Results['WCST'] = ProcessNeuroPsychFunctions.ProcessWCST(Data)
    print('\t WCST loaded')

    # Antonyms
    Data = ReadFile(VisitFolder, subid, 'Vocab_Antonyms')
    Results['Ant'] = ProcessNeuroPsychFunctions.ProcessAntonym(Data)
    print('\tAntonym loaded')

    # NART
    Data = ReadFile(VisitFolder, subid, 'Vocab_NART')
    Results['NART'] = ProcessNeuroPsychFunctions.ProcessNART(Data)
    print('\tNART loaded')

    # Digit Span
    # Forward
    Data = ReadFile(VisitFolder, subid, 'DigitSpan_Forward')
    Dir = 'Forward'
    Results['DSFor'] = ProcessNeuroPsychFunctions.ProcessDigitSpan(Data, Dir)
    print('\tDS Forwad loaded')

    # Backward
    Data = ReadFile(VisitFolder, subid, 'DigitSpan_Backward')
    Dir = 'Backward'
    Results['DSBack'] = ProcessNeuroPsychFunctions.ProcessDigitSpan(Data, Dir)
    print('\tDS Backward loaed')

    # Pattern Comparison
    Data = ReadFile(VisitFolder, subid, 'Speed_PatternComp')
    Results['PComp'] = ProcessNeuroPsychFunctions.ProcessPattComp(Data)
    print('\tPattern Comparison loaded')

    # Matrics
    Data = ReadFile(VisitFolder, subid, 'Matrices_Main')
    Results['Matr'] = ProcessNeuroPsychFunctions.ProcessMatrices(Data)
    print('\tMatrices loaded')

    # DMS
    Data = ReadFile(VisitFolder, subid, 'DMS_Block_BehRun1')
    CapacityData = ReadFile(VisitFolder, subid, 'DMS_CAPACITY')
    Data = ProcessNeuroPsychFunctions.CheckDMSDataFrameForLoad(Data)
    tempResults = ProcessNeuroPsychFunctions.ProcessDMSBlockv2(Data, CapacityData)
    Results['DMSBeh1'] = Reorder_DMS_VSTM_Results(tempResults, 'DMS')
    print('\tDMS loaded')

    # VSTM
    Data = ReadFile(VisitFolder, subid, 'VSTM_Block_BehRun1')
    CapacityData = ReadFile(VisitFolder, subid, 'VSTM_CAPACITY')
    tempResults = ProcessNeuroPsychFunctions.ProcessVSTMBlockv2(Data, CapacityData)
    Results['VSTMBeh1'] = Reorder_DMS_VSTM_Results(tempResults, 'VSTM')
    print('\tVSTM loaded')

    # SRT
    Data = ReadFile(VisitFolder, subid, 'SRT_ImmRecall')
    Results['SRTImm'] = ProcessNeuroPsychFunctions.ProcessSRTImm(Data)
    print('\tSRT Imm recall loaded')

    Data = ReadFile(VisitFolder, subid, 'SRT_Recog')
    Results['SRTRecog'] = ProcessNeuroPsychFunctions.ProcessSRTRecog(Data)
    print('\tSRT Recognition loaded')

    Data = ReadFile(VisitFolder, subid, 'SRT_DelRecall')
    Results['SRTDel'] = ProcessNeuroPsychFunctions.ProcessSRTDelay(Data)
    print('\tSRT Delayed recall loaded')

    # N-Back
    # Load data files for both N-Back runs
    Data1 = ReadFile(VisitFolder, subid, 'NBack*BehRun*1_20')
    Data2 = ReadFile(VisitFolder, subid, 'NBack*BehRun*2_20')

    tempResults1 = ProcessNeuroPsychFunctions.ProcessNBack(Data1)
    tempResults2 = ProcessNeuroPsychFunctions.ProcessNBack(Data2)
    #Results['NBack'] = Reorder_NBack_Results(tempResults)
    if len(tempResults1) > 0 and len(tempResults2) > 0:
        AllData = Data1.append(Data2)
        if len(AllData) > 0:
            tempResultsAll = ProcessNeuroPsychFunctions.ProcessNBack(AllData)
            Results['NBack'] = Reorder_NBack_Results(tempResultsAll)
    elif len(tempResults1) > 0:
        Results['NBack'] = Reorder_NBack_Results(tempResults1)
    else:
        pass
    return Results

def LoadRawDataSHORT(VisitFolder, subid):
    # Given a visit folder, check for the existance of specific files
    # read the file and process teh results
    # This function looks for very specific files

    print('working on %s'%(subid))
    Results = {}

    # DMS
    Data = ReadFile(VisitFolder, subid, 'DMS_Block_BehRun1')
    Data = ProcessNeuroPsychFunctions.CheckDMSDataFrameForLoad(Data)
    tempResults = ProcessNeuroPsychFunctions.ProcessDMSBlockv2(Data)
    Results['DMSBeh1'] = Reorder_DMS_VSTM_Results(tempResults, 'DMS')
    print('\tDMS loaded')

    return Results

def ReadFile(VisitFolder, subid, TaskTag):
    # Find the file that matches the TaskTag
    # If multiple CSV files are found then the user is prompted to pick one.
    # Un selected files are renamed with XXX at their beginning.
    # The next time this program is run on this folder there will now only be one file
    # available and the user will not be prompted again
    Data = []
    # List all files in the visit folder
    ll = os.listdir(VisitFolder)
    # create the string you are looking for which is a combo of the subid and the task name
    SearchString = subid + '_' + TaskTag
    matching = fnmatch.filter(ll,SearchString+'*.csv')
    if len(matching) == 0:
        # Check to see if the file is a TXT file
        matching = fnmatch.filter(ll,SearchString+'*.txt')
    # It is possible that there are multipel files with similar names.
    # The following asks the user for the correct one and then renames the others
    count = 1
    if len(matching) > 1:
        # There are more than one file!
        print('There are multiple files found for %s in folder: %s'%(SearchString, VisitFolder))
        for i in matching:
            # print the name and size of files
            SizeOfFile = np.round(os.stat(os.path.join(VisitFolder,matching[0])).st_size/1048)
            print('\t%d) %s, size = %0.0f kB'%(count, i,SizeOfFile))
            count += 1
        sel = input('Which one should be kept?  (Press return to skip) ')
        if len(sel) > 0:
            SelectedFile = matching[int(sel)-1]
            # Rename the unselected files so they will hopefully not be selected the next time!
            count = 1
            for i in matching:
                if not count == int(sel):
                    OutName = 'XXX_' + i
                    print(OutName)
                    shutil.move(os.path.join(VisitFolder,i), os.path.join(VisitFolder, OutName))
                count += 1
        else:
            # If none of teh files are selected, rename them so they are not viewed again
            count = 1
            for i in matching:
                OutName = 'XXX_' + i
                print(OutName)
                shutil.move(os.path.join(VisitFolder,i), os.path.join(VisitFolder, OutName))
            SelectedFile = False
    elif len(matching) == 1:
        SelectedFile= matching[0]
    else:
        SelectedFile = False
        print('\t%s >> Did not find any files!!!'%(TaskTag))
    if SelectedFile != False:
        # Now open the file
        InputFile = os.path.join(VisitFolder, SelectedFile)
        # Read whole file into a dataframe
        # Note, in order for the data to be read as a dataframe all columns need to have headings.
        # If not an error is thrown
        Data = pd.read_csv(InputFile)
        # If the data is to be read into a big list use this:
        #    fid = open(InputFile, 'r')
        #    Data = csv.reader(fid)
        #    Data = list(Data)
    return Data

def PutDataIntoOutputFile():
    # There will be a single output resultsvfile
    # it will have these columns:
    #   partID
    #   visitID, which will often be 1,2,3
    #   data checked, this cannot be changed by the program but only by a human
    #   data completeness flag
    #
    #
    # First, the part id and visit id are read from the folder names.
    # Then the output data is checked to find this person. If found the data checked flag is TRUE
    # if yes, check to see if data is complete in out file.
    # if not, then load all data and see if the missing data is now available
    df2 = pd.DataFrame.from_dict(FlatResults, orient='index')
    pass

def LoadOutDataFile(OutDataFilePathName):
    # Make a data frame from CSV file
    OutDF = pd.read_csv(OutDataFilePathName)
    return OutDF

def CreateOutFileName():
    # Create a file to hold processed data using the time and date
    # to indicate when it was made
    BaseFileName = 'NCM_Master_NP'
    now = datetime.datetime.now()
    NowString = now.strftime("_updated_%b-%d-%Y_%H-%M.csv")
    NewOutFileName = os.path.join(AllOutDataFolder, BaseFileName + NowString)
    return NewOutFileName

def LocateOutDataFile():
    # Locate an existing processed data file and if it does not exist, then make it.
    BaseFileName = 'NCM_Master_NP'
    # What files exist with this name?
    Files = glob.glob(os.path.join(AllOutDataFolder, BaseFileName + '*.csv'))
    now = datetime.datetime.now()
    NowString = now.strftime("_updated_%b-%d-%Y_%H-%M.csv")
    NewOutFileName = BaseFileName + NowString
    if len(Files) == 0:
        FileName = os.path.join(AllOutDataFolder, NewOutFileName)
    else:
        # this will open an existing file
        FileName = Files[-1]
    return FileName

def CreateUpdatedDataFrameOfResults(NewData, OldData):
    # Extract the columns
    ColName = []
    for c in OldData.columns:
        ColName.append(c)

    # Create a dataframe to hold teh updated data
    OutDataFrame = pd.DataFrame(columns = ColName)
    # Now cycle over each row and compare
    for index, NewRow in NewData.iterrows():
        # Add the new data

        NewDataSubId = NewRow['AAsubid']
        NewDataVisitId = NewRow['AAVisid']
#        print(NewDataSubId)
#        print(NewDataVisitId)
        # for each subid and visit found in the new data look for it in the old data
        # If the same subid/visitid is found in both check the Old data column
        # labeled AAChecked to see if it is 1. If so then skip this data line in the new data
        # and go onto the next one.

        # If the value is 0 in the old data file, then the data line has NOT been checked
        # by a human and it is OK for this program to overwrite it.
        # Data is written out as follows:
        OldRow = OldData.loc[(OldData['AAsubid'] == int(NewDataSubId)) & (OldData['AAVisid'] == NewDataVisitId)]

        if len(OldRow) > 0:
            # found this data in the exisiting data file
            # Check to see if the data has been checked by a human
            if int(OldRow['AAChecked']) == 1:
                # It has been checked
                # write temp to the out data file
                OutDataRow = OldRow
            else:
                # Data is in the out file but it has not been checked
                OutDataRow = NewRow
        else:
            # Did not find the new data in the old data file
            OutDataRow = NewRow
        # Add OutDataRow to the updated out dataframe
        OutDataFrame = OutDataFrame.append(OutDataRow)
    return OutDataFrame

def Reorder_DMS_VSTM_Results(Results, TaskTag):
    # When the results are calculated it is easier to code the scoring based on load
    # but this is order hard to read at the output.
    # This code reorders results based on the measure instead of the load
    # What measures to cycle over
    MeasureList = ['RT', 'Acc','NResp']
    # what type of measures to cycle over
    TypeList = ['Rel', 'Abs']
    # create an empty ordered dictionary
    Res = collections.OrderedDict()
    # Now add the capacity back in
    CapStr = TaskTag + '_Cap'
    Res[CapStr] = Results[CapStr]
    for Type in TypeList:
        for Tag in MeasureList:
            for k in range(1,11):
                for i in Results:
                    if (i.find(Type) >= 0) and (i.find(Tag) >= 0) and (i.find('Load'+str(k).zfill(2)) >= 0):
                        Res[i] = Results[i]
    return Res

def Reorder_NBack_Results(Results):
    # When the results are calculated it is easier to code the scoring based on load
    # but this is order hard to read at the output.
    # This code reorders results based on the measure instead of the load
    # What measures to cycle over
    MeasureList = ['HIT', 'HitRT','FA', 'FaRT','N']
    # create an empty ordered dictionary
    Res = collections.OrderedDict()
    for Tag in MeasureList:
        for k in range(0,4):
            for i in Results:
                if (i.find(Tag) >= 0) and (i.find('Load'+str(k).zfill(2)) >= 0):
                    Res[i] = Results[i]
    return Res

def WriteOutNewdataMoveOldData(UpdatedData, UpdatedDataFileName, ExistingDataFileName):
    # Move the old file
    OldDataFolder = os.path.join(AllOutDataFolder, 'OldResultFiles')
    # if the folder for old data does not exist, then make it
    if not os.path.exists(OldDataFolder):
        os.mkdir(OldDataFolder)
    # change the name of the results file so it is not confused with current data
    MovedDataFile = os.path.join(OldDataFolder, 'X_'+os.path.basename(ExistingDataFileName))
    shutil.move(ExistingDataFileName, MovedDataFile)
    # Now that the old data is moved, write out the updated data
    UpdatedData.to_csv(UpdatedDataFileName, index = False, float_format='%.3f')
# #
if __name__ == "__main__":
    main()

max_stars_repo_name: NCMlab/ScoringCognitiveTasks

max_issues_repo_name: NCMlab/ScoringCognitiveTasks

max_forks_repo_name: NCMlab/ScoringCognitiveTasks


text:
###############################################################################
# Code from
# https://github.com/pytorch/vision/blob/master/torchvision/datasets/folder.py
# Modified the original code so that it also loads images from the current
# directory as well as the subdirectories
###############################################################################

import torch.utils.data as data

import segyio
from PIL import Image
import os
import os.path
import numpy as np

SEIS_EXTENSIONS = [
    '.segy', '.SEGY','sgy', "SGY"
]

def random_int_with_mute(total,mute,size=1000):
    #total: should be the total range
    #mute: should be the interval to mute
    lista=np.arange(total)
    lista_mutada=np.concatenate((lista[:mute[0]],lista[mute[1]:]))
    return np.random.choice(lista_mutada,size)

def is_seismic_file(filename):
    return any(filename.endswith(extension) for extension in SEIS_EXTENSIONS)

def make_imgs(lines,nimg,mode,size=256, mute=[264,401]):

    l=len(lines)
    xl=len(lines[0])-size
    d=len(lines.T)-size
    rl=nimg//l
    imgs=[]

    if mode=='random':
        #samples=[np.random.randint(0,high=l,size=nimg),np.random.randint(0,high=xl,size=nimg),np.random.randint(0,high=d,size=nimg)]
        samples=[random_int_with_mute(l, mute,size=nimg),np.random.randint(0,high=xl,size=nimg),np.random.randint(0,high=d,size=nimg)]

        for i in range(nimg):
            imgs.append(lines[samples[0][i]][samples[1][i]:samples[1][i]+size].T[samples[2][i]:samples[2][i]+size])

    if mode=='sequential':

        samples = [np.tile(np.arange(0,l,step=1,dtype='int'),rl+1)[0:nimg],
                   np.repeat(np.arange(0,xl,step=xl//(rl+1),dtype='int'),l)[0:nimg],
                   np.repeat(np.arange(0,d,step=d//(rl+1),dtype='int'),l)[0:nimg]]

        for i in range(nimg):
            imgs.append(lines[samples[0][i]][samples[1][i]:samples[1][i]+size].T[samples[2][i]:samples[2][i]+size])

    return imgs,samples

def make_dataset(dir,nimg,nlines ,mute, phase, mode):
    ### must have only one seismic file in dir###
    seismic = []
    assert os.path.isdir(dir), '%s is not a valid directory' % dir

    for root, _, fnames in sorted(os.walk(dir)):
        for fname in fnames:
            if is_seismic_file(fname):
                path = os.path.join(root, fname)
                seismic.append(path)

    print(seismic[0])
    with segyio.open(seismic[0], iline=193, xline=197) as f:
        f.mmap()
        seis = segyio.tools.collect(f.iline[:])
        tseis = seis[nlines+1:]
        seis = seis[:nlines+1]
        if phase == 'test':
            imgs , samples = make_imgs(tseis,nimg,mode=mode)
        else:
            imgs , samples = make_imgs(seis,nimg,mode=mode)

    return imgs, samples, seis.mean(), seis.max()

def default_loader(path):
    return Image.open(path).convert('RGB')

class SeismicFolder(data.Dataset):

    def __init__(self, root, transform=None, return_samples=False):
        imgs, samples = make_dataset(root)
        if len(imgs) == 0:
            raise(RuntimeError("Found 0 seismic in: " + root + "\n"
                               "Supported seismic extensions are: " +
                               ",".join(SEIS_EXTENSIONS)))

        self.root = root
        self.imgs = imgs
        self.samples = samples
        self.transform = transform
        self.return_samples = return_samples

    def __getitem__(self, index):
        img = self.imgs[index]
        sample = [item[index] for item in self.samples]
        if self.transform is not None:
            img = self.transform(img)
        if self.return_samples:
            return img, sample
        else:
            return img
    def __len__(self):
        return len(self.imgs)

ext: py

reason: import numpy


text:
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from scipy.stats import gaussian_kde

from RNA_model import Gillespie_kinetics, get_mRNA_distr_properties

Nsimulations = 200
recompute_mRNA = False
plot_separate = False

def main():

    alpha_list = [{'alpha_on':1,'alpha_off':1},{'alpha_on':10,'alpha_off':1},{'alpha_on':1,'alpha_off':10},{'alpha_on':10,'alpha_off':10}]
    #alpha_list = [4]

    r_deg = 1
    r_on = 0.8
    r_off = 3
    r_prod = 100

    """
    #Nanog:0.0019 0.007 0.80 0.0022
    k_deg = 0.0022
    k_on = 0.0019
    k_off = 0.007
    k_prod = 0.80
    r_on = k_on/k_deg
    r_off = k_off/k_deg
    r_prod = k_prod/k_deg
    """

    final_rates = np.array([r_on, r_off, r_prod, r_deg])
    rmin = np.min(final_rates)
    Tend = 7/rmin

    if recompute_mRNA:
        mRNA_output = []
        for alpha_params in alpha_list:
            param_list = dict()
            param_list['r_deg'] = 1
            param_list['r_on'] = r_on
            param_list['r_off'] = r_off
            param_list['r_prod'] = r_prod
            param_list['Tend'] = Tend

            param_list['alpha_on'] = alpha_params['alpha_on']
            param_list['alpha_off'] = alpha_params['alpha_off']
            param_list['alpha_prod'] = 1

            mean_sim, CV, Entropy, Fano, mRNA_population = Gillespie_kinetics(param_list, plot = True, N_simulations = Nsimulations)
            mRNA_output.append(mRNA_population)

            # https://thirdorderscientist.org/homoclinic-orbit/2013/10/24/kernel-density-estimation-for-random-variables-with-bounded-support-mdash-the-transformation-trick

        np.save('mRNA_output.npy', mRNA_output)

    else:
        mRNA_output = np.load('mRNA_output.npy')

    plt.rcParams.update({'font.size': 18})
    plt.figure(figsize=(8,5))
    for ai, alpha_params in enumerate(alpha_list):
        data_ = mRNA_output[ai]

        if plot_separate:
            print(alpha_params)
            mean, CV, Entropy, Fano = get_mRNA_distr_properties(data_, pop_is_steady = True, plot = False)
            print('   mean %.2f' % mean)
            print('   CV %.2f' % CV)
            print('   Fano %.2f' % Fano)
            print('   Entropy %.2f' % Entropy)
            print()

        #transform data with log function
        data = np.log((data_ + 1))

        density = gaussian_kde(data)
        density.covariance_factor = lambda : 0.35 #Smoothing parameter (higher = smoother)
        density._compute_covariance()
        x_vals = np.linspace(np.min(data),np.max(data),200)
        y_vals = density(x_vals)

        x_vals2 = (np.exp(x_vals) - 1)
        y_vals2 = y_vals/(x_vals2 + 1)

        #plt.plot(x_vals,y_vals,color = sns.color_palette()[ai+2], lw=2, label = 'alpha = %s' % alpha)
        plt.plot(x_vals2,y_vals2,color = sns.color_palette()[ai+2], lw=3, label = str(alpha_params))

        if plot_separate:
            bins = np.arange(0,np.nanmax(data_)+1)
            plt.hist(data_, bins = bins, edgecolor='black', color = 'green', alpha = 0.3, label = 'Simulated', density = True)
            plt.show()

    plt.ylabel('PDF')
    plt.xlabel('mRNA counts')
    plt.legend(prop = {'size':14})
    plt.ylim(0,0.05)
    plt.axvline(x=0, color = 'black', lw=2)
    plt.show()

"""
    t.k.e <- function(x)
    {
        1) log transform the obs
        y = log(x)

        2) estimate the density of the pseudo obs with KDE
        k = approxfun(density(y))

        3) estimate the density of the original obs
        seq=seq(min(x),max(x),length.out=5e2)
        t.density = as.numeric(vector(length=length(seq)))

        for (i in 1:length(seq))
        {
            xx=seq[i]
            t.density[i] = k(log(xx))/xx
        }
        output=list(x=seq,y=t.density)
    }

"""

def simBetaPoisson(kon,koff,kprod, num = 1000):
    from scipy.stats import beta, poisson
    rvs = []
    p_array = beta.rvs(kon, koff, size=num)
    for p in p_array:
        n = poisson.rvs(kprod*p, size=1)[0]
        rvs.append(n)

    #plt.hist(rvs)
    #plt.show()
    return np.array(rvs)

if __name__ == "__main__":
    plt.rcParams.update({'font.size': 16})
    main()

max_stars_repo_path: REGIR/Biochemical_applications/3_RNA_transcription (Fig.4)/3_RNA_shape_parameter (Fig. 4E).py

max_stars_repo_name: Aurelien-Pelissier/REGIR

max_issues_repo_name: Aurelien-Pelissier/REGIR

max_forks_repo_name: Aurelien-Pelissier/REGIR

reason: import numpy,from scipy


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import properties
import numpy as np
import warnings

from . import Maps
from . import Utils

class SphinxProp(object):
    """
    Update the auto-documenter from properties
    https://github.com/3ptscience/properties/issues/153
    """
    def sphinx_class(self):
        return ':class:`{cls} <{ref}>`'.format(
            cls=self.__class__.__name__,
            ref='SimPEG.Props.{}'.format(self.__class__.__name__)
        )

class Array(SphinxProp, properties.Array):

    class_info = 'a numpy, Zero or Identity array'

    def validate(self, instance, value):
        if isinstance(value, (Utils.Zero, Utils.Identity)):
            return value
        return super(Array, self).validate(instance, value)

class Float(SphinxProp, properties.Float):

    class_info = 'a float, Zero or Identity'

    def validate(self, instance, value):
        if isinstance(value, (Utils.Zero, Utils.Identity)):
            return value
        return super(Float, self).validate(instance, value)

class Integer(SphinxProp, properties.Integer):

    class_info = 'an Integer or *'

    def validate(self, instance, value):
        if isinstance(value, str):
            assert value == '*', 'value must be an integer or *, not {}'.format(
                value
            )
            return value
        return super(Integer, self).validate(instance, value)

class Model(SphinxProp, properties.Array):

    class_info = 'a numpy array'
    _required = False
    _shape = {('*',), ('*', '*')}

class Mapping(SphinxProp, properties.Property):

    class_info = 'a SimPEG Map'
    _required = False

    @property
    def prop(self):
        return getattr(self, '_prop', None)

    @prop.setter
    def prop(self, value):
        assert isinstance(value, PhysicalProperty)
        value._mapping = self  # Skip the setter
        self._prop = value

    @property
    def reciprocal(self):
        if self.prop and self.prop.reciprocal:
            return self.prop.reciprocal.mapping

    @property
    def reciprocal_prop(self):
        if self.prop and self.prop.reciprocal:
            return self.prop.reciprocal

    def clear_props(self, instance):
        for prop in (self.prop, self.reciprocal_prop, self.reciprocal):
            if prop is not None:
                if prop.name in instance._props:
                    delattr(instance, prop.name)
                else:
                    setattr(instance, prop.name, None)

    def validate(self, instance, value):
        if not isinstance(value, Maps.IdentityMap):
            self.error(instance, value)
        return value

    def get_property(self):

        scope = self

        def fget(self):
            value = self._get(scope.name)
            if value is not None:
                return value
            if scope.reciprocal is None:
                return None
            reciprocal = self._get(scope.reciprocal.name)
            if reciprocal is None:
                return None
            return Maps.ReciprocalMap() * reciprocal

        def fset(self, value):
            if value is not properties.utils.undefined:
                value = scope.validate(self, value)
            self._set(scope.name, value)
            if value is not properties.utils.undefined:
                scope.clear_props(self)

        def fdel(self):
            self._set(scope.name, properties.utils.undefined)

        return property(fget=fget, fset=fset, fdel=fdel, doc=scope.doc)

    def as_pickle(self, instance):
        return instance._get(self.name)

class PhysicalProperty(SphinxProp, properties.Property):

    class_info = 'a physical property'
    reciprocal = None
    _required = False

    @property
    def mapping(self):
        return getattr(self, '_mapping', None)

    @mapping.setter
    def mapping(self, value):
        assert isinstance(value, Mapping)
        value._prop = self  # Skip the setter
        self._mapping = value

    def clear_mappings(self, instance):
        if self.mapping is not None:
            if self.mapping.name in instance._props:
                delattr(instance, self.mapping.name)
            else:
                setattr(instance, self.mapping.name, None)
        if self.reciprocal is not None:
            if self.reciprocal.mapping is not None:
                if self.reciprocal.mapping.name in instance._props:
                    delattr(instance, self.reciprocal.mapping.name)
                else:
                    setattr(instance, self.reciprocal.mapping.name, None)

    def validate(self, instance, value):
        assert isinstance(value, (np.ndarray, float)), (
            "Physical properties must be numpy arrays or floats."
        )
        return value

    def get_property(self):

        scope = self

        def fget(self):

            default = self._get(scope.name)
            if default is not None:
                return default
            if scope.reciprocal:
                default = self._get(scope.reciprocal.name)
                if default is not None:
                    # set by default reciprocal
                    return 1.0 / default
            if scope.mapping is None and scope.reciprocal is None:
                return None
            if scope.mapping is None:
                # look to the reciprocal
                if scope.reciprocal.mapping is None:
                    # there is no reciprocal mapping
                    reciprocal_val = self._get(scope.reciprocal.name)
                    if reciprocal_val is None:
                        raise AttributeError(
                            'A default for {}/{} has not been set'.format(
                                scope.name, scope.reciprocal.name
                            )
                        )
                # Set by mapped reciprocal
                return 1.0 / getattr(self, scope.reciprocal.name)

            mapping = getattr(self, scope.mapping.name)
            if mapping is None:
                raise AttributeError(
                    'A default `{}` or mapping `{}` has not been set.'.format(
                        scope.name,
                        scope.mapping.name
                    )
                )
            if self.model is None:
                raise AttributeError(
                    'A `model` is required for physical property {}'.format(
                        scope.name
                    )
                )
            return mapping * self.model

        def fset(self, value):
            if value is not properties.utils.undefined:
                value = scope.validate(self, value)
                if scope.reciprocal:
                    delattr(self, scope.reciprocal.name)
            self._set(scope.name, value)
            if value is not properties.utils.undefined:
                scope.clear_mappings(self)

        def fdel(self):
            self._set(scope.name, properties.utils.undefined)

        return property(fget=fget, fset=fset, fdel=fdel, doc=scope.doc)

    def as_pickle(self, instance):
        return instance._get(self.name)

    def summary(self, instance):
        default = instance._get(self.name)
        if default is not None:
            return '[*] {}: set by default value'.format(
                self.name
            )
        if self.reciprocal:
            default = instance._get(self.reciprocal.name)
            if default is not None:
                return '[*] {}: set by default reciprocal: 1.0 / {}'.format(
                    self.name,
                    self.reciprocal.name
                )
        if self.mapping is None and self.reciprocal is None:
            return '[ ] {}: property not set'.format(
                self.name,
                self.reciprocal.name
            )
        if self.mapping is None:
            if self.reciprocal.mapping is None:
                # there is no reciprocal mapping
                reciprocal_val = instance._get(self.reciprocal.name)
                if reciprocal_val is None:
                    return '[ ] {}: default for {}/{} not set'.format(
                        self.name, self.name, self.reciprocal.name
                    )
            return '[*] {}: set by mapped reciprocal 1.0 / ({} * {})'.format(
                self.name, self.reciprocal.mapping.name, self.reciprocal.name
            )

        mapping = getattr(instance, self.mapping.name)

        if mapping is None:
            return '[ ] {}: default `{}` or mapping `{}` not set'.format(
                self.name, self.name, self.mapping.name
            )
        if instance.model is None:
            return (
                '[ ] {}: model({}) required, from active `{}` mapping: {}'
            ).format(
                self.name,
                mapping.shape[1],
                self.mapping.name,
                str(mapping)
            )

        correct_shape = (
            mapping.shape[1] == '*' or
            mapping.shape[1] == len(instance.model)
        )

        if correct_shape:
            return '[*] {}: set by the `{}` mapping: {} * model({})'.format(
                self.name, self.mapping.name, str(mapping), len(instance.model)
            )

        return '[ ] {}: incorrect mapping/model shape: {} * model({})'.format(
            self.name, str(mapping), len(instance.model)
        )

class Derivative(SphinxProp, properties.GettableProperty):

    physical_property = None

    @property
    def mapping(self):
        """The mapping looks through to the physical property map."""
        if self.physical_property is None:
            return None
        return self.physical_property.mapping

    def get_property(self):

        scope = self

        def fget(self):
            if scope.physical_property is None:
                return Utils.Zero()
            if scope.mapping is None:
                return Utils.Zero()
            mapping = getattr(self, scope.mapping.name)
            if mapping is None:
                return Utils.Zero()
            if self.model is None:
                return Utils.Zero()

            return mapping.deriv(self.model)

        return property(fget=fget, doc=scope.doc)

def Invertible(help, default=None):

    mapping = Mapping(
        "Mapping of {} to the inversion model.".format(help)
    )

    physical_property = PhysicalProperty(
        help,
        mapping=mapping
    )
    if default is not None:
        physical_property.default = default

    property_derivative = Derivative(
        "Derivative of {} wrt the model.".format(help),
        physical_property=physical_property
    )

    return physical_property, mapping, property_derivative

def Reciprocal(prop1, prop2):
    prop1.reciprocal = prop2
    prop2.reciprocal = prop1

class BaseSimPEG(properties.HasProperties):
    """"""

class HasModel(BaseSimPEG):

    model = Model("Inversion model.")

    @property
    def _all_map_names(self):
        """Returns all Mapping properties"""
        return sorted([
            k for k in self._props
            if isinstance(self._props[k], Mapping)
        ])

    @property
    def _act_map_names(self):
        """Returns all active Mapping properties"""
        return sorted([
            k for k in self._all_map_names
            if getattr(self, k) is not None
        ])

    @property
    def needs_model(self):
        """True if a model is necessary"""
        return len(self._act_map_names) > 0

    @property
    def _has_nested_models(self):
        for k in self._props:
            if (
                    isinstance(self._props[k], properties.Instance) and
                    issubclass(self._props[k].instance_class, HasModel)
               ):
                return True
        return False

    @properties.validator('model')
    def _check_model_valid(self, change):
        """Checks the model length and necessity"""
        if change['value'] is properties.utils.undefined:
            return True

        if not self.needs_model and not self._has_nested_models:
            warnings.warn(
                "Cannot add model as there are no active mappings"
                ", choose from: ['{}']".format(
                    "', '".join(self._all_map_names)
                )
            )
            return

        errors = []

        for name in self._act_map_names:
            mapping = getattr(self, name)
            correct_shape = (
                mapping.shape[1] == '*' or
                mapping.shape[1] == len(change['value'])
            )
            if not correct_shape:
                errors += [
                    '{}: expected model of len({}) for {}'.format(
                        name,
                        mapping.shape[1],
                        str(mapping)
                    )
                ]
        if len(errors) == 0:
            return True

        warnings.warn(
            'Model of len({}) incorrect shape for mappings: \n    {}'.format(
                len(change['value']),
                '\n    '.join(errors)
            )
        )

    @properties.validator
    def _check_valid(self):
        errors = []

        # Check if the model is necessary
        if self.needs_model and self.model is None:
            errors += ['model must not be None']
        if not self.needs_model and self.model is not None:
            errors += ['there are no active maps, but a model is provided']

        # Check each map is the same size
        shapes = []
        for name in self._act_map_names:
            shape = getattr(self, name).shape[1]
            if shape == '*':
                continue
            shapes += [shape]
        if not all(x == shapes[0] for x in shapes):
            errors += ['the mappings are not the same shape']

        # Check that the model is the same size as the mappings
        if len(shapes) > 0 and self.model is not None:
            if not len(self.model) == shapes[0]:
                errors += ['the model must be len({})'.format(shapes[0])]

        # Check each physical property
        check_boxes = sorted([
            self._props[k].summary(self) for k in self._props
            if isinstance(self._props[k], PhysicalProperty)
        ])
        for line in check_boxes:
            if line[:3] == '[ ]':
                errors += [line[4:]]

        if len(errors) == 0:
            return True

        raise ValueError(
            'The {} instance has the following errors: \n - {}'.format(
                self.__class__.__name__,
                '\n - '.join(errors)
            )
        )

    def summary(self):
        prop_names = sorted([
            k for k in self._props
            if isinstance(self._props[k], PhysicalProperty)
        ])

        out = ['Physical Properties:']

        # Grab the physical property summaries
        for prop in prop_names:
            out += [' ' + self._props[prop].summary(self)]

        # Grab the validation errors
        try:
            self.validate()
            out += ['', 'All checks pass!']
        except ValueError as e:
            out += ['', str(e)]

        return '\n'.join(out)

class LocationVector(properties.Array):

    class_info = ''

    def validate(self, instance, value):
        """
        coerce the shape to a 1D vector if an array is input. See
        https://github.com/seequent/properties/issues/250
        """
        if len(value.shape) > 1:
            value = value.flatten()

        if value.shape not in self.shape:
        # if len(value) != self.shape[0]:
            raise Exception(
                'loc must be length {}, the provided input is length {}'.format(
                    self.shape, len(value)
                )
            )

        return super(properties.Array, self).validate(instance, value)

max_stars_repo_name: kimjaed/simpeg

max_issues_repo_name: kimjaed/simpeg

max_forks_repo_name: kimjaed/simpeg


text:
import scipy.sparse as sps
import numpy as np
from utils.ensembler import ensembler
from utils.post_processing import  eurm_to_recommendation_list_submission
from utils.post_processing import  eurm_to_recommendation_list
from utils.post_processing import  eurm_remove_seed
from utils.submitter import Submitter
from utils.evaluator import Evaluator
from utils.datareader import Datareader
from utils.definitions import *

mode = "offline"
type = "unique"

w = []
print("[ Loading weights ]")
for i in range(1, 11):
    arg = np.load("weight/cat" + str(i) + ".npy")
    print(arg[- 1])
    w.append(list(arg[:len(arg) - 1].astype(np.float)))

print("[ Loading matrix name ]")
if mode == "offline":
    matrix_dict = load_obj("matrix_dict", path="")
    dir = "offline/"

if mode == "online":
    matrix_dict = load_obj("matrix_dict_online", path="")
    dir = "online/"

if type == "unique":
    print("[ Loading cat 1 ]")
    cat = 1
    a = sps.load_npz(dir+matrix_dict["nlp"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["top_pop_cat1"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["nlp_rp3beta"]+"-cat" + str(cat) + ".npz")
    matrix = [[a, b, c]]

    print("[ Loading cat 2 ]")
    cat = 2
    a = sps.load_npz(dir+matrix_dict["cbf_item_album"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["cbf_item_artist"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["nlp"]+"-cat" + str(cat) + ".npz")
    d = sps.load_npz(dir+matrix_dict["rp3beta"]+"-cat" + str(cat) + ".npz")
    e = sps.load_npz(dir+matrix_dict["cf_user"]+"-cat" + str(cat) + ".npz")
    f = sps.load_npz(dir+matrix_dict["slim"]+"-cat" + str(cat) + ".npz")
    g = sps.load_npz(dir+matrix_dict["cbf_user_artist"]+"-cat" + str(cat) + ".npz")
    h = sps.load_npz(dir+matrix_dict["pers_top_pop"]+"-cat" + str(cat) + ".npz")
    i = sps.load_npz(dir+matrix_dict["cbf_user_album"]+"-cat" + str(cat) + ".npz")
    l = sps.load_npz(dir+matrix_dict["pers_top_pop_2"]+"-cat" + str(cat) + ".npz")
    matrix.append([a, b, c, d, e, f, g, h, i, l])

    print("[ Loading cat 3 ]")
    cat = 3
    a = sps.load_npz(dir+matrix_dict["cbf_item_album"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["cbf_item_artist"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["nlp"]+"-cat" + str(cat) + ".npz")
    d = sps.load_npz(dir+matrix_dict["rp3beta"]+"-cat" + str(cat) + ".npz")
    e = sps.load_npz(dir+matrix_dict["cf_user"]+"-cat" + str(cat) + ".npz")
    f = sps.load_npz(dir+matrix_dict["slim"]+"-cat" + str(cat) + ".npz")
    g = sps.load_npz(dir+matrix_dict["cbf_user_artist"]+"-cat" + str(cat) + ".npz")
    h = sps.load_npz(dir+matrix_dict["cbf_user_album"]+"-cat" + str(cat) + ".npz")

    matrix.append([a, b, c, d, e, f, g, h])

    print("[ Loading cat 4 ]")
    cat = 4
    a = sps.load_npz(dir+matrix_dict["cbf_item_album"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["cbf_item_artist"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["rp3beta"]+"-cat" + str(cat) + ".npz")
    d = sps.load_npz(dir+matrix_dict["cf_user"]+"-cat" + str(cat) + ".npz")
    e = sps.load_npz(dir+matrix_dict["slim"]+"-cat" + str(cat) + ".npz")
    f = sps.load_npz(dir+matrix_dict["cbf_user_artist"]+"-cat" + str(cat) + ".npz")
    g = sps.load_npz(dir+matrix_dict["cbf_user_album"]+"-cat" + str(cat) + ".npz")
    matrix.append([a, b, c, d, e, f, g])

    print("[ Loading cat 5 ]")
    cat = 5
    a = sps.load_npz(dir+matrix_dict["cbf_item_album"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["cbf_item_artist"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["nlp"]+"-cat" + str(cat) + ".npz")
    d = sps.load_npz(dir+matrix_dict["rp3beta"]+"-cat" + str(cat) + ".npz")
    e = sps.load_npz(dir+matrix_dict["cf_user"]+"-cat" + str(cat) + ".npz")
    f = sps.load_npz(dir+matrix_dict["slim"]+"-cat" + str(cat) + ".npz")
    g = sps.load_npz(dir+matrix_dict["cbf_user_artist"]+"-cat" + str(cat) + ".npz")
    h = sps.load_npz(dir+matrix_dict["cbf_user_album"]+"-cat" + str(cat) + ".npz")
    matrix.append([a, b, c, d, e, f, g, h])

    print("[ Loading cat 6 ]")
    cat = 6
    a = sps.load_npz(dir+matrix_dict["cbf_item_album"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["cbf_item_artist"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["rp3beta"]+"-cat" + str(cat) + ".npz")
    d = sps.load_npz(dir+matrix_dict["cf_user"]+"-cat" + str(cat) + ".npz")
    e = sps.load_npz(dir+matrix_dict["slim"]+"-cat" + str(cat) + ".npz")
    f = sps.load_npz(dir+matrix_dict["cbf_user_artist"]+"-cat" + str(cat) + ".npz")
    g = sps.load_npz(dir+matrix_dict["cbf_user_album"]+"-cat" + str(cat) + ".npz")

    matrix.append([a, b, c, d, e, f, g])

    print("[ Loading cat 7 ]")
    cat = 7
    a = sps.load_npz(dir+matrix_dict["cbf_item_album"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["cbf_item_artist"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["nlp"]+"-cat" + str(cat) + ".npz")
    d = sps.load_npz(dir+matrix_dict["rp3beta"]+"-cat" + str(cat) + ".npz")
    e = sps.load_npz(dir+matrix_dict["cf_user"]+"-cat" + str(cat) + ".npz")
    f = sps.load_npz(dir+matrix_dict["slim"]+"-cat" + str(cat) + ".npz")
    g = sps.load_npz(dir+matrix_dict["cbf_user_artist"]+"-cat" + str(cat) + ".npz")
    h = sps.load_npz(dir+matrix_dict["cbf_user_album"]+"-cat" + str(cat) + ".npz")
    matrix.append([a, b, c, d, e, f, g, h])

    print("[ Loading cat 8 ]")
    cat = 8
    a = sps.load_npz(dir+matrix_dict["cbf_item_album"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["cbf_item_artist"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["nlp"]+"-cat" + str(cat) + ".npz")
    d = sps.load_npz(dir+matrix_dict["rp3beta"]+"-cat" + str(cat) + ".npz")
    e = sps.load_npz(dir+matrix_dict["cf_user"]+"-cat" + str(cat) + ".npz")
    f = sps.load_npz(dir+matrix_dict["slim"]+"-cat" + str(cat) + ".npz")
    g = sps.load_npz(dir+matrix_dict["cbf_user_artist"]+"-cat" + str(cat) + ".npz")
    h = sps.load_npz(dir+matrix_dict["cbf_user_album"]+"-cat" + str(cat) + ".npz")
    matrix.append([a, b, c, d, e, f, g, h])

    print("[ Loading cat 9 ]")
    cat = 9
    a = sps.load_npz(dir+matrix_dict["cbf_item_album"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["cbf_item_artist"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["nlp"]+"-cat" + str(cat) + ".npz")
    d = sps.load_npz(dir+matrix_dict["rp3beta"]+"-cat" + str(cat) + ".npz")
    e = sps.load_npz(dir+matrix_dict["cf_user"]+"-cat" + str(cat) + ".npz")
    f = sps.load_npz(dir+matrix_dict["slim"]+"-cat" + str(cat) + ".npz")
    g = sps.load_npz(dir+matrix_dict["cbf_user_artist"]+"-cat" + str(cat) + ".npz")
    h = sps.load_npz(dir+matrix_dict["rp3beta_cat9"]+"-cat" + str(cat) + ".npz")
    i = sps.load_npz(dir+matrix_dict["cbf_user_album"]+"-cat" + str(cat) + ".npz")
    matrix.append([a, b, c, d, e, f, g, h, i])

    print("[ Loading cat 10 ]")
    cat = 10
    a = sps.load_npz(dir+matrix_dict["cbf_item_album"]+"-cat" + str(cat) + ".npz")
    b = sps.load_npz(dir+matrix_dict["cbf_item_artist"]+"-cat" + str(cat) + ".npz")
    c = sps.load_npz(dir+matrix_dict["nlp"]+"-cat" + str(cat) + ".npz")
    d = sps.load_npz(dir+matrix_dict["rp3beta"]+"-cat" + str(cat) + ".npz")
    e = sps.load_npz(dir+matrix_dict["cf_user"]+"-cat" + str(cat) + ".npz")
    f = sps.load_npz(dir+matrix_dict["slim"]+"-cat" + str(cat) + ".npz")
    g = sps.load_npz(dir+matrix_dict["cbf_user_artist"]+"-cat" + str(cat) + ".npz")
    h = sps.load_npz(dir+matrix_dict["cbf_user_album"]+"-cat" + str(cat) + ".npz")
    matrix.append([a, b, c, d, e, f, g, h])

    rprec = []
    for i in range(0, 10):
        print("[ Ensembling cat", i+1, "]")
        rprec.append(ensembler(matrix[i], w[i], normalization_type="max"))
    res = sps.vstack(rprec).tocsr()

    import time
    name = "ensemble-"+mode+"-data-"+time.strftime("%x")+"-"+time.strftime("%X")
    name = name.replace("/", "_")
    sps.save_npz("results/"+name+".npz", res)

    print("[ Initizalizing Datereader ]")
    dr = Datareader(verbose=False, mode=mode, only_load="False")

    res = eurm_to_recommendation_list(res, datareader=dr)

    if mode == "offline":
        print("[ Initizalizing Evaluator ]")
        ev = Evaluator(dr)
        ev.evaluate(res, name="ens")

    if mode == "online":
        print("[ Initizalizing Submitter ]")
        sb = Submitter(dr)
        sb.submit(recommendation_list=res, name=name, track="main", verify=True,
                  gzipped=False)

#
#
#
# if type == "splitted":
#     mode = "offline"
#
#     print("[ Loading weights ]")
#     w_rprec = []
#     tmp = 0
#     for i in range(1, 11):
#         arg = np.load("rprec/cat" + str(i) + ".npy")
#         tmp += -float(arg[-1])
#         w_rprec.append(list(arg[:len(arg)-1].astype(np.float)))
#
#     w_ndcg = []
#     tmp = 0
#     for i in range(1, 11):
#         arg = np.load("rprec/cat" + str(i) + ".npy")
#         tmp += -float(arg[-1])
#         w_ndcg.append(list(arg[:len(arg)-1].astype(np.float)))
#
#
#     if mode == "offline":
#         cat = 1
#         print("[ Loading cat 1 ]")
#         a = sps.load_npz("offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-top_pop-mean=28-perc=0.1-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/offline-nlp_into_rp3beta-cat" + str(cat) + ".npz")
#
#
#         matrix = [[a, b, c]]
#
#
#         print("[ Loading cat 2 ]")
#         cat = 2
#
#         a = sps.load_npz("offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("offline/p_top_pop_offline-cat" + str(cat) + ".npz")
#         i = sps.load_npz("offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#         l = sps.load_npz("offline/top_pop_2_album_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h, i ,l])
#
#
#
#         print("[ Loading cat 3 ]")
#         cat = 3
#
#         a = sps.load_npz("offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#         matrix.append([a, b, c, d, e, f, g, h])
#
#
#         print("[ Loading cat 4 ]")
#         cat = 4
#
#         a = sps.load_npz("offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         d = sps.load_npz("offline/offline-cfuser-cat" + str(cat) + ".npz")
#         e = sps.load_npz("offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         f = sps.load_npz("offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         g = sps.load_npz("offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#
#
#         matrix.append([a, b, c, d, e, f, g])
#
#
#         print("[ Loading cat 5 ]")
#         cat = 5
#
#         a = sps.load_npz("offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h])
#
#
#         print("[ Loading cat 6 ]")
#         cat = 6
#
#         a = sps.load_npz("offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         d = sps.load_npz("offline/offline-cfuser-cat" + str(cat) + ".npz")
#         e = sps.load_npz("offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         f = sps.load_npz("offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         g = sps.load_npz("offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g])
#
#
#         print("[ Loading cat 7 ]")
#         cat = 7
#
#         a = sps.load_npz("offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h])
#
#
#         print("[ Loading cat 8 ]")
#         cat = 8
#
#         a = sps.load_npz("offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h])
#
#
#         print("[ Loading cat 9 ]")
#         cat = 9
#
#         a = sps.load_npz("offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("offline/offline_CAT9rp3b_cut=25_knn=850_topk=750_shrink250_2018-05-28_15-19_LAST-cat" + str(cat) + ".npz")
#         i = sps.load_npz("offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h, i])
#
#
#         print("[ Loading cat 10 ]")
#         cat = 10
#
#         a = sps.load_npz("offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#
#
#         matrix.append([a, b, c, d, e, f, g, h])
#
#         rprec = []
#         for i in range(0, 10):
#             print("[ RPrec - Ensembling cat", i, "]")
#             rprec.append(ensembler(matrix[i], w_rprec[i], normalization_type="max"))
#         rprec = sps.vstack(rprec).tocsr()
#
#
#         sps.save_npz("rprec_offline.npz", rprec)
#
#     if mode == "offline":
#
#
#
#         cat = 1
#         print("[ Loading cat 1 ]")
#         a = sps.load_npz("ndcg_offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-top_pop-mean=28-perc=0.1-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/offline-nlp_into_rp3beta-cat" + str(cat) + ".npz")
#
#
#         matrix = [[a, b, c]]
#
#
#         print("[ Loading cat 2 ]")
#         cat = 2
#
#         a = sps.load_npz("ndcg_offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("ndcg_offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("ndcg_offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("ndcg_offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("ndcg_offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("ndcg_offline/p_top_pop_offline-cat" + str(cat) + ".npz")
#         i = sps.load_npz("ndcg_offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#         l = sps.load_npz("ndcg_offline/top_pop_2_album_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h, i ,l])
#
#
#
#         print("[ Loading cat 3 ]")
#         cat = 3
#
#         a = sps.load_npz("ndcg_offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("ndcg_offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("ndcg_offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("ndcg_offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("ndcg_offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("ndcg_offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#         matrix.append([a, b, c, d, e, f, g, h])
#
#
#         print("[ Loading cat 4 ]")
#         cat = 4
#
#         a = sps.load_npz("ndcg_offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         d = sps.load_npz("ndcg_offline/offline-cfuser-cat" + str(cat) + ".npz")
#         e = sps.load_npz("ndcg_offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         f = sps.load_npz("ndcg_offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         g = sps.load_npz("ndcg_offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#
#
#         matrix.append([a, b, c, d, e, f, g])
#
#
#         print("[ Loading cat 5 ]")
#         cat = 5
#
#         a = sps.load_npz("ndcg_offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("ndcg_offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("ndcg_offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("ndcg_offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("ndcg_offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("ndcg_offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h])
#
#
#         print("[ Loading cat 6 ]")
#         cat = 6
#
#         a = sps.load_npz("ndcg_offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         d = sps.load_npz("ndcg_offline/offline-cfuser-cat" + str(cat) + ".npz")
#         e = sps.load_npz("ndcg_offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         f = sps.load_npz("ndcg_offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         g = sps.load_npz("ndcg_offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g])
#
#
#         print("[ Loading cat 7 ]")
#         cat = 7
#
#         a = sps.load_npz("ndcg_offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("ndcg_offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("ndcg_offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("ndcg_offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("ndcg_offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("ndcg_offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h])
#
#
#         print("[ Loading cat 8 ]")
#         cat = 8
#
#         a = sps.load_npz("ndcg_offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("ndcg_offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("ndcg_offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("ndcg_offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("ndcg_offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("ndcg_offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h])
#
#
#         print("[ Loading cat 9 ]")
#         cat = 9
#
#         a = sps.load_npz("ndcg_offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("ndcg_offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("ndcg_offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("ndcg_offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("ndcg_offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("ndcg_offline/offline_CAT9rp3b_cut=25_knn=850_topk=750_shrink250_2018-05-28_15-19_LAST-cat" + str(cat) + ".npz")
#         i = sps.load_npz("ndcg_offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#         matrix.append([a, b, c, d, e, f, g, h, i])
#
#
#         print("[ Loading cat 10 ]")
#         cat = 10
#
#         a = sps.load_npz("ndcg_offline/offline-cbf_item_album-cat" + str(cat) + ".npz")
#         b = sps.load_npz("ndcg_offline/offline-cbf_item_artist-cat" + str(cat) + ".npz")
#         c = sps.load_npz("ndcg_offline/eurm_nlp_fusion_offline-cat" + str(cat) + ".npz")
#         d = sps.load_npz("ndcg_offline/offline-rp3beta-cat" + str(cat) + ".npz")
#         e = sps.load_npz("ndcg_offline/offline-cfuser-cat" + str(cat) + ".npz")
#         f = sps.load_npz("ndcg_offline/slim_bpr_completo_test1-cat" + str(cat) + ".npz")
#         g = sps.load_npz("ndcg_offline/eurm_cbfu_artists_offline-cat" + str(cat) + ".npz")
#         h = sps.load_npz("ndcg_offline/eurm_cbfuser_albums_offline-cat" + str(cat) + ".npz")
#
#
#
#         matrix.append([a, b, c, d, e, f, g, h])
#
#         ndcg = []
#         for i in range(0, 10):
#             print("[ RPrec - Ensembling cat", i, "]")
#             ndcg.append(ensembler(matrix[i], w_ndcg[i], normalization_type="max"))
#         ndcg = sps.vstack(ndcg).tocsr()
#
#         es = []
#         sps.save_npz("ndcg_offline.npz", ndcg)
#
#
#         print("[ Initizalizing Datereader ]")
#         dr = Datareader(verbose=False, mode=mode, only_load="False")
#
#
#
#         from utils.post_processing import append_rec_list
#         from utils.post_processing import remove_rec_list_after_cutoff
#         res = append_rec_list(remove_rec_list_after_cutoff(eurm_to_recommendation_list(rprec, datareader=dr), dr), eurm_to_recommendation_list(ndcg, datareader=dr))
#
#
#         if mode == "offline":
#             print("[ Initizalizing Evaluator ]")
#             ev = Evaluator(dr)
#             ev.evaluate(res, name="ens")
#
#         # sps.save_npz("ensemble_per_cat_" + mode + "test_blocchi.npz", res)
#
#
#
#
#
#     # if mode == "online":
#     #     print("[ Loading cat 1 ]")
#     #     a = sps.load_npz("online/online_nlp_knn100_bm25-cat1.npz")
#     #     b = sps.load_npz("online/online-top_pop-mean=28-perc=0.1.npz")[0:1000]
#     #
#     #     matrix = [[a, b]]
#     #
#     #
#     #     print("[ Loading cat 2 ]")
#     #     cat = 2
#     #
#     #     a = sps.load_npz("online/online-cbf_item-album-850-cat"+str(cat)+".npz")
#     #     b = sps.load_npz("online/online-cbf_item-artist-850-cat"+str(cat)+".npz")
#     #     c = sps.load_npz("online/online_nlp_knn100_bm25-cat"+str(cat)+".npz")
#     #     d = sps.load_npz("online/online-rp3beta-cat"+str(cat)+".npz")
#     #     e = sps.load_npz("online/online-cf_user-850-cat"+str(cat)+".npz")
#     #     f = sps.load_npz("online/slim_online-cat"+str(cat)+".npz")
#     #     g = sps.load_npz("online/eurm_cbu_artists_online-cat"+str(cat)+".npz")
#     #
#     #     matrix.append([a, b, c, d, e, f, g])
#     #
#     #
#     #
#     #     print("[ Loading cat 3 ]")
#     #     cat = 3
#     #
#     #     a = sps.load_npz("online/online-cbf_item-album-850-cat" + str(cat) + ".npz")
#     #     b = sps.load_npz("online/online-cbf_item-artist-850-cat" + str(cat) + ".npz")
#     #     c = sps.load_npz("online/online_nlp_knn100_bm25-cat" + str(cat) + ".npz")
#     #     d = sps.load_npz("online/online-rp3beta-cat" + str(cat) + ".npz")
#     #     e = sps.load_npz("online/online-cf_user-850-cat" + str(cat) + ".npz")
#     #     f = sps.load_npz("online/slim_online-cat" + str(cat) + ".npz")
#     #     g = sps.load_npz("online/eurm_cbu_artists_online-cat" + str(cat) + ".npz")
#     #
#     #     matrix.append([a, b, c, d, e, f, g])
#     #
#     #
#     #     print("[ Loading cat 4 ]")
#     #     cat = 4
#     #
#     #     a = sps.load_npz("online/online-cbf_item-album-850-cat" + str(cat) + ".npz")
#     #     b = sps.load_npz("online/online-cbf_item-artist-850-cat" + str(cat) + ".npz")
#     #     c = sps.load_npz("online/online-rp3beta-cat" + str(cat) + ".npz")
#     #     d = sps.load_npz("online/online-cf_user-850-cat" + str(cat) + ".npz")
#     #     e = sps.load_npz("online/slim_online-cat" + str(cat) + ".npz")
#     #     f = sps.load_npz("online/eurm_cbu_artists_online-cat" + str(cat) + ".npz")
#     #
#     #     matrix.append([a, b, c, d, e, f])
#     #
#     #
#     #     print("[ Loading cat 5 ]")
#     #     cat = 5
#     #
#     #     a = sps.load_npz("online/online-cbf_item-album-850-cat" + str(cat) + ".npz")
#     #     b = sps.load_npz("online/online-cbf_item-artist-850-cat" + str(cat) + ".npz")
#     #     c = sps.load_npz("online/online_nlp_knn100_bm25-cat" + str(cat) + ".npz")
#     #     d = sps.load_npz("online/online-rp3beta-cat" + str(cat) + ".npz")
#     #     e = sps.load_npz("online/online-cf_user-850-cat" + str(cat) + ".npz")
#     #     f = sps.load_npz("online/slim_online-cat" + str(cat) + ".npz")
#     #     g = sps.load_npz("online/eurm_cbu_artists_online-cat" + str(cat) + ".npz")
#     #
#     #     matrix.append([a, b, c, d, e, f, g])
#     #
#     #
#     #     print("[ Loading cat 6 ]")
#     #     cat = 6
#     #
#     #     a = sps.load_npz("online/online-cbf_item-album-850-cat" + str(cat) + ".npz")
#     #     b = sps.load_npz("online/online-cbf_item-artist-850-cat" + str(cat) + ".npz")
#     #     c = sps.load_npz("online/online-rp3beta-cat" + str(cat) + ".npz")
#     #     d = sps.load_npz("online/online-cf_user-850-cat" + str(cat) + ".npz")
#     #     e = sps.load_npz("online/slim_online-cat" + str(cat) + ".npz")
#     #     f = sps.load_npz("online/eurm_cbu_artists_online-cat" + str(cat) + ".npz")
#     #
#     #     matrix.append([a, b, c, d, e, f])
#     #
#     #
#     #     print("[ Loading cat 7 ]")
#     #     cat = 7
#     #
#     #     a = sps.load_npz("online/online-cbf_item-album-850-cat" + str(cat) + ".npz")
#     #     b = sps.load_npz("online/online-cbf_item-artist-850-cat" + str(cat) + ".npz")
#     #     c = sps.load_npz("online/online_nlp_knn100_bm25-cat" + str(cat) + ".npz")
#     #     d = sps.load_npz("online/online-rp3beta-cat" + str(cat) + ".npz")
#     #     e = sps.load_npz("online/online-cf_user-850-cat" + str(cat) + ".npz")
#     #     f = sps.load_npz("online/slim_online-cat" + str(cat) + ".npz")
#     #     g = sps.load_npz("online/eurm_cbu_artists_online-cat" + str(cat) + ".npz")
#     #
#     #     matrix.append([a, b, c, d, e, f, g])
#     #
#     #
#     #     print("[ Loading cat 8 ]")
#     #     cat = 8
#     #
#     #     a = sps.load_npz("online/online-cbf_item-album-850-cat" + str(cat) + ".npz")
#     #     b = sps.load_npz("online/online-cbf_item-artist-850-cat" + str(cat) + ".npz")
#     #     c = sps.load_npz("online/online_nlp_knn100_bm25-cat" + str(cat) + ".npz")
#     #     d = sps.load_npz("online/online-rp3beta-cat" + str(cat) + ".npz")
#     #     e = sps.load_npz("online/online-cf_user-850-cat" + str(cat) + ".npz")
#     #     f = sps.load_npz("online/slim_online-cat" + str(cat) + ".npz")
#     #     g = sps.load_npz("online/eurm_cbu_artists_online-cat" + str(cat) + ".npz")
#     #
#     #     matrix.append([a, b, c, d, e, f, g])
#     #
#     #
#     #     print("[ Loading cat 9 ]")
#     #     cat = 9
#     #
#     #     a = sps.load_npz("online/online-cbf_item-album-850-cat" + str(cat) + ".npz")
#     #     b = sps.load_npz("online/online-cbf_item-artist-850-cat" + str(cat) + ".npz")
#     #     c = sps.load_npz("online/online_nlp_knn100_bm25-cat" + str(cat) + ".npz")
#     #     d = sps.load_npz("online/online-rp3beta-cat" + str(cat) + ".npz")
#     #     e = sps.load_npz("online/online-cf_user-850-cat" + str(cat) + ".npz")
#     #     f = sps.load_npz("online/slim_online-cat" + str(cat) + ".npz")
#     #     g = sps.load_npz("online/eurm_cbu_artists_online-cat" + str(cat) + ".npz")
#     #     h = sps.load_npz("online/online_CAT9rp3b_cut=25_knn=850_topk=750_2018-05-29_11-17_LAST.npz")[8000:9000]
#     #
#     #     matrix.append([a, b, c, d, e, f, g, h])
#     #
#     #
#     #     print("[ Loading cat 10 ]")
#     #     cat = 10
#     #
#     #     a = sps.load_npz("online/online-cbf_item-album-850-cat" + str(cat) + ".npz")
#     #     b = sps.load_npz("online/online-cbf_item-artist-850-cat" + str(cat) + ".npz")
#     #     c = sps.load_npz("online/online_nlp_knn100_bm25-cat" + str(cat) + ".npz")
#     #     d = sps.load_npz("online/online-rp3beta-cat" + str(cat) + ".npz")
#     #     e = sps.load_npz("online/online-cf_user-850-cat" + str(cat) + ".npz")
#     #     f = sps.load_npz("online/slim_online-cat" + str(cat) + ".npz")
#     #     g = sps.load_npz("online/eurm_cbu_artists_online-cat" + str(cat) + ".npz")
#     #
#     #
#     #
#     #
#     #     matrix.append([a, b, c, d, e, f, g])
#
#     #
#     #
#     # res = []
#     # for i in range(0, 10):
#     #     print("[ Ensembling cat",i,"]")
#     #     res.append(ensembler(matrix[i], w[i], normalization_type="max"))
#     #
#     # print("[ Initizalizing Datereader ]")
#     # dr = Datareader(verbose=False, mode=mode, only_load="False")
#     #
#     # ret = eurm_remove_seed(sps.vstack(res).tocsr(), datareader=dr)
#     #
#     # if mode == "offline":
#     #     print("[ Initizalizing Evaluator ]")
#     #     ev = Evaluator(dr)
#     #     from utils.post_processing import shift_rec_list_cutoff
#     #
#     #     ev.evaluate(shift_rec_list_cutoff(eurm_to_recommendation_list(ret), dr), name="ens")
#     #
#     # sps.save_npz("ensemble_per_cat_" + mode + "4_giugno.npz", ret)
#     #
#     # if mode == "online":
#     #     sb = Submitter(dr)
#     #     sb.submit(recommendation_list=eurm_to_recommendation_list_submission(eurm_remove_seed(ret, dr)), name="best_test", track="main",
#     #               verify=True, gzipped=False)

max_stars_repo_name: edervishaj/spotify-recsys-challenge

max_issues_repo_name: kiminh/spotify-recsys-challenge

max_forks_repo_name: kiminh/spotify-recsys-challenge

reason: import numpy,import scipy


text:
"""
Selection of similarity variables.
"""
import numpy as np
import pandas as pd
import pickle
import itertools
import copy
import Utils as util
from itertools import islice
from random import randint
import random

def partition (list_in, n):
    list_in = copy.deepcopy(list_in)
    random.shuffle(list_in)
    return [list_in[i::n] for i in range(n)]

# load data. Data has <s, s', a, r> samples.
feat_to_select = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

# TODO: use post-session-5 data in the end
data  = pd.read_csv('data_samples_post_sess_2.csv', converters={'s0': eval, 's1': eval})
data = data.values.tolist()

# TODO: choose features chosen after session 5 in the end
with open('Level_3_G_algorithm_chosen_features', 'rb') as f:
    feat_sel = pickle.load(f)

with open('IDs', 'rb') as f:
    user_ids = pickle.load(f)
num_people = len(user_ids)

# Create folds based on user IDs
# TODO: use 10 folds in the end
num_folds = 2
id_folds = partition(user_ids, num_folds)

num_act = 4
num_feat = len(feat_to_select)
num_samples = len(data)

# TODO: use all traits in the end
num_traits = 6 # Personality (5 dim.) and TTM-phase for PA

# Settings for calculation of Q-values
q_num_iter = 100000 * num_samples # num_samples = num people after session 2
discount_factor = 0.85
alpha = 0.01

# TODO: to be replaced with actual data from Qualtricx
traits_orig = np.ones((num_samples, num_traits))
traits_orig[0, 0] = 5
traits_names = ['TTM_PA', 'P0', 'P1', 'P2', 'P3', 'P4']
df_traits = pd.DataFrame(traits_orig, columns=traits_names)

# Scale columns of traits to [0, 1]
for c in traits_names:
    df_traits[c] -= df_traits[c].min()
    df_traits[c] /= df_traits[c].max()

improv_threshold = 0.001
improv = 1

trait_sel = []

while improv > improv_threshold: # while adding more traits leads to improvement

    # not yet selected features
    trait_not_sel = [i for i in traits_names if not i in trait_sel]

    for trait in trait_not_sel: # for each not yet selected feature

        traits = df_traits[trait_sel+[trait]].to_numpy() # selected + this not yet selected feature

        for fold in range(num_folds): # for each fold of user IDs

            fold_test_people = id_folds[fold]
            num_test_people = len(fold_test_people)
            fold_train_people = [i for i in user_ids if not i in fold_test_people]
            num_train_people = len(fold_train_people)

            # for each test person
            for p1 in fold_test_people:

                data_p = copy.deepcopy(data)

                d_E = np.zeros(num_train_people)

                # compute Euclidean distances based on traits for each sample
                for p2 in fold_train_people:

                    d_E[p2] = np.linalg.norm(traits[p1] - traits[p2])

                sum_E = sum(d_E) # sum of Euclidean distances

                # increase frequency of samples based on Euclidean distances
                for p2 in range(num_samples):
                    weight = int(round(d_E[p2]/sum_E, 3) * 1000) # increase sample size by factor of 1000

                    for w in range(weight):
                        data_p.append(data[p2])

                num_samples_p = len(data_p)

                abstract_states = [list(i) for i in itertools.product([0, 1], repeat = num_feat)]

                # Compute transition function and reward function
                trans_func = np.zeros((int(2 ** num_feat), num_act, int(2 ** num_feat)))
                reward_func = np.zeros((int(2 ** num_feat), num_act))
                reward_func_count = np.zeros((int(2 ** num_feat), num_act))
                for s_ind, s in enumerate(abstract_states):
                    for data_index in range(num_samples):
                        if list(np.take(np.array(data[data_index][0]), feat_sel)) == s:
                            trans_func[s_ind, data[data_index][2], abstract_states.index(list(np.take(data[data_index][1], feat_sel)))] += 1
                            r = int(data[data_index][3])
                            if r == 0:
                                r = -1
                            reward_func[s_ind, data[data_index][2]] += r
                            reward_func_count[s_ind, data[data_index][2]] += 1

                    # Normalize
                    for a in range(num_act):
                        summed = sum(trans_func[s_ind, a])
                        if summed > 0:
                            trans_func[s_ind, a] /= summed
                        if reward_func_count[s_ind, a] > 0:
                            reward_func[s_ind, a] /= reward_func_count[s_ind, a]

                # Value iteration
                q_values_exact, _ = util.get_Q_values_opt_policy(discount_factor, trans_func, reward_func)
                opt_policy = [[[[a for a in range(num_act) if q_values_exact[abstract_states.index([i, j, k])][a] == max(q_values_exact[abstract_states.index([i, j, k])])] for k in range(2)] for j in range(2)] for i in range(2)]