text:
#Antonin Vobecky

import numpy as np
import glob
import os
import imageio as imio
import pickle
from sklearn.neighbors import KNeighborsClassifier
from sklearn import svm
from math import ceil,floor
import shutil

def findThreshold(pos, neg):
    pos = [(x, 0) for x in pos]
    neg = [(x, 1) for x in neg]
    allAvg = pos + neg
    allAvg = sorted(allAvg, key=lambda tup: tup[1])

    bestIdx = 0
    bestVal = 0
    i = 0
    for thr in allAvg:
        correct = 0
        for avg in allAvg:
            if avg < thr and avg[1] == 0:
                correct += 1
            if avg > thr and avg[1] == 1:
                correct += 1
        if correct > bestVal:
            bestVal = correct
            bestIdx = i
        i += 1
    print('most correctly classified: ', bestVal, ' (', bestVal / len(allAvg), '%)')
    print('index of best threshold: ', bestIdx)
    print(allAvg[bestIdx])
    return allAvg[bestIdx][0]

def computeRes(set):
    from keras.models import model_from_json
    print('compute res')
    model_weights = "Network/model_1out_BN_DO-07_TF_1-improvement-05-0.68.hdf5"
    model_json = "Network/model_tensorflow_1out_BN_DO-07_TF_1.json"
    # load json and create model
    json_file = open(model_json, 'r')
    loaded_model_json = json_file.read()
    json_file.close()
    model = model_from_json(loaded_model_json)
    # load weights into new model
    model.load_weights(model_weights)
    print("Loaded model from disk - val acc 0.68 after 5 epochs")

    # evaluate loaded model on test data
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

    if set=='train':
        print('TRAINING SET')
        posBaseDir = '/storage/plzen1/home/vobecant/dataset_new/train/1/'
        negBaseDir = '/storage/plzen1/home/vobecant/dataset_new/train/0/'
    elif set=='val':
        print('VALIDATION SET')
        posBaseDir = '/storage/plzen1/home/vobecant/dataset_new/val/1/'
        negBaseDir = '/storage/plzen1/home/vobecant/dataset_new/val/0/'
    elif set == 'test':
        print('TEST SET')
        posBaseDir = 'Bags/'
        negBaseDir = 'Nothing/'

    if not os.path.exists('Nothing'):
        os.makedirs('Nothing')

    posFolders = next(os.walk(posBaseDir))[1]  # took help - not confident
    negFolders = next(os.walk(negBaseDir))[1]  # did not take help - confident

    i = 1
    posRes = []
    posImgs = 0
    posFoldersUsed = []

    for posFolder in posFolders:
        filelist = glob.glob(posBaseDir + str(posFolder) + '/*.png')
        filelist.sort()
        if len(filelist) < 30:
            continue
        posFoldersUsed.append(posFolder)
        print(i-1,posFolder)
        posImgs = posImgs + len(filelist)
        imgs = np.array([np.array(imio.imread(fname)) for fname in filelist])
        imgs = imgs[..., np.newaxis]
        res = model.predict_proba(imgs, batch_size=1, verbose=0)
        posRes.append(res[:, 0])
        print(i, "/", len(posFolders),' folder ',posFolder," with",
              len(filelist)," pictures")
        i = i + 1
    nPosCompleted = i
    print('completed ',i,' folders with pictures')
    print()

    i = 1
    negRes = []
    negImgs = 0
    for negFolder in negFolders:
        #if nPosCompleted == i:
        #    print('reached equal number of positive and negative bags')
        #    break
        filelist = glob.glob(negBaseDir + str(negFolder) + '/*.png')
        filelist.sort()
        if len(filelist) < 30:
            continue
        print(i-1+nPosCompleted, negFolder)
        negImgs = negImgs + len(filelist)
        imgs = np.array([np.array(imio.imread(fname)) for fname in filelist])
        imgs = imgs[..., np.newaxis]
        res = model.predict_proba(imgs, batch_size=1, verbose=0)
        negRes.append(res[:,0])
        print(str(i) + "/" + str(len(negFolders)) + '' + ' negative folder ' + str(negFolder) + " with " + str(
            len(filelist)) + " pictures")
        i = i + 1
    print('completed ', i, ' folders with pictures')
    print()

    res = {'pos': posRes, 'neg': negRes}

    bins = {'pos': posRes, 'neg': negRes}
    if set=='train':
        pickle.dump(bins, open('/storage/plzen1/home/vobecant/res_train126.p', 'wb'), protocol=2)
    elif set=='val':
        pickle.dump(bins, open('/storage/plzen1/home/vobecant/res_val126.p', 'wb'), protocol=2)
    elif set == 'test':
        pickle.dump(bins, open('./Network/res_test3.p', 'wb'), protocol=2)
        pickle.dump(posFoldersUsed, open('./Network/res_test3_folders.p', 'wb'), protocol=2)

    return res

def makeAllHists(pRes,nRes,nbins):
    posHist = np.array([])
    for p in pRes:
        posHist = np.append(posHist,makeHist(p,nbins))
    posHist = np.reshape(posHist,[-1,nbins]).tolist()
    y_pos = np.ones([len(posHist),])

    negHist = np.array([])
    for n in nRes:
        negHist = np.append(negHist,makeHist(n,nbins))
    negHist = np.reshape(negHist,[-1,nbins]).tolist()
    y_neg = np.zeros([len(negHist),])

    x = posHist+negHist
    y = np.append(np.ones([len(posHist), 1]), np.zeros([len(negHist), 1]))

    return x,y

def makeHist(res, nbins):
    bins = np.zeros([nbins,])
    for a in res:
        bins[findBin(a,nbins)]+=1
    bins = normalizeBin(bins)
    return bins

def findBin(value, nbins):
    step = 1.0/nbins
    cur_threshold = step
    for i in range(0,nbins):
        if value<cur_threshold:
            return i
        cur_threshold+=step

def normalizeBin(bins):
    nSamples = np.sum(bins)
    for i in range(0,len(bins)):
        bins[i] = bins[i] / nSamples
    return bins

def classif_kNN(x_train, y_train, x_test, y_test, n, ls):
    nbrs = KNeighborsClassifier(n_neighbors=n, algorithm='auto',leaf_size=ls).fit(x_train,y_train)
    classif = nbrs.predict(x_test)
    diffs = np.sum(abs(np.subtract(classif,y_test)))
    err = diffs/len(y_test)
    print('         n=',n,', error=',err)
    return err

def classif_svm(x_train, y_train, x_test, y_test,kernel,c,test):
    #svc = svm.LinearSVC(C=c)
    svc = svm.SVC(C=c,kernel=kernel,probability=True)
    svc.fit(x_train,y_train)
    classif = svc.predict(x_test)
    #probs = svc.predict_proba(x_test)
    dist = svc.decision_function(X=x_test)
    diffs = np.sum(abs(np.subtract(classif, y_test)))
    correct = np.where(np.subtract(classif, y_test)==0)
    correct = correct[0]
    distCorrect = dist[correct]
    distCorrect = np.abs(distCorrect)
    bestId = distCorrect.argsort()[-5:][::-1]
    bestDist = distCorrect[bestId]
    err = diffs / len(y_test)
    if test:
        print('         c=',c,kernel+' SVM error=', err)
    return err, bestId, bestDist

def genXY(bins):
    posBins = np.reshape(bins['pos'],[-1,10]).tolist()
    negBins = np.reshape(bins['neg'],[-1,10]).tolist()
    x = posBins+negBins
    y = np.append(np.ones([len(posBins),1]),np.zeros([len(negBins),1]))
    return x,y

def res2vec(res,poolType):
    '''
    Converts vecotor of results 'res' of variable length to vector of length K
    :param res: results
    :param k: length of output vector
    :return: vector of length k
    '''
    bins = [20,15,10]
    vec=[]
    for bin in bins:
        l = len(res)
        win = np.ceil(l/bin) # window
        win = int(win)
        str = np.floor(l/bin) # stride
        str = int(str)
        startId = 0
        endId = win
        if poolType=='max':
            while endId<l:
                vec.append(max(res[startId:endId]))
                startId+=str
                endId+=str
        elif poolType=='mean':
            while endId<l:
                vec.append(np.mean(res[startId:endId]))
                startId+=str
                endId+=str
        elif poolType=='median':
            while endId<l:
                vec.append(np.median(res[startId:endId]))
                startId+=str
                endId+=str
        else:
            print('Unsupported pooling type.')
    return vec

def spp(res, nbins, poolType):
    '''
    Converts vecotor of results 'res' of variable length to vector of length K
    :param res: results
    :param k: length of output vector
    :return: vector of length k
    '''
    vec=[]
    l = len(res)
    win = np.floor(l/nbins) + (l % nbins)# window
    win = int(win)
    str = np.floor(l/nbins) # stride
    str = int(str)
    startId = 0
    endId = win
    if poolType=='max':
        while endId<=l:
            vec.append(max(res[startId:endId]))
            startId+=str
            endId+=str
    elif poolType=='mean':
        while endId<=l:
            vec.append(np.mean(res[startId:endId]))
            startId+=str
            endId+=str
    elif poolType=='median':
        while endId<=l:
            vec.append(np.median(res[startId:endId]))
            startId+=str
            endId+=str
    else:
        print('Unsupported pooling type.')
    return vec

def test_spp(x_train, y_train, x_val, y_val, x_test, y_test):
    kernels = ['linear', 'poly', 'rbf', 'sigmoid']
    #kernels = ['linear']
    cs = [0.001, 0.01, 0.025, 0.05, 0.1, 0.2, 0.3, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 5, 7, 10]
    best_kernel = ''
    best_c = 0
    lowest_err = 1
    bestId = []

    for kernel in kernels:
        for c in cs:
            err, bestId, bestDist = classif_svm(x_train=x_train, y_train=y_train, x_test=x_val, y_test=y_val, kernel=kernel, c=c,test=False)
            if err < lowest_err:
                lowest_err = err
                best_kernel=kernel
                best_c=c
    #print('lowest val error:',lowest_err)

    tst_err, bestId, bestDist = classif_svm(x_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, kernel=best_kernel, c=best_c, test=True)
    #print('test error of configuration with lowest val error:',tst_err)
    return tst_err, best_kernel, best_c, bestId, bestDist

def createConstOutput(res,outSize,poolType):
    pos_res = res['pos']
    neg_res = res['neg']
    x = []
    for r in pos_res:
        x.append(spp(r, nbins=outSize, poolType=poolType))
    for r in neg_res:
        x.append(spp(r, nbins=outSize, poolType=poolType))
    y = np.append(np.ones([len(pos_res), 1]), np.zeros([len(neg_res), 1]))
    return x,y

def showBestId(bestId):
    print(bestId)
    #posBaseDir = '/storage/plzen1/home/vobecant/dataset_new/TESTOVANI/1/'
    posBaseDir = 'Bags/'
    #negBaseDir = '/storage/plzen1/home/vobecant/dataset_new/TESTOVANI/0/'
    negBaseDir = 'Nothing/'

    posFolders = next(os.walk(posBaseDir))[1]  # took help - not confident
    negFolders = next(os.walk(negBaseDir))[1]  # did not take help - confident

    allFolders = np.array(posFolders+negFolders)
    allFoldersCorrect = []

    for posFolder in posFolders:
        if posFolder in bestId:
            filelist = glob.glob(posBaseDir + str(posFolder) + '/*.png')
            filelist.sort()
            if len(filelist) < 30:
                continue
            posFolder = posBaseDir + posFolder
            allFoldersCorrect.append(posFolder)
    '''
    nPos = len(allFoldersCorrect)

    for negFolder in negFolders:
        filelist = glob.glob(negBaseDir + str(negFolder) + '/*.png')
        if len(filelist) < 30:
            continue
        negFolder = negBaseDir + negFolder
        allFoldersCorrect.append(negFolder)

    allFoldersCorrect = np.array(allFoldersCorrect)
    bestFolders = allFoldersCorrect[bestId]
    '''
    import matplotlib.pyplot as plt
    from matplotlib import gridspec
    i=0
    for folder in allFoldersCorrect:

        filelist = glob.glob(str(folder) + '/*.png')
        tit = 'uncertain'
        filelist.sort()
        i+=1
        imgs = np.array([np.array(imio.imread(fname)) for fname in filelist])
        nrow = 7
        ncol = ceil((imgs.shape[0])/7)
        #print(nrow,ncol)
        fig = plt.figure(figsize=(ncol + 1, nrow + 1))
        plt.title(tit)
        gs = gridspec.GridSpec(nrow, ncol,
                               wspace=0.0, hspace=0.3,
                               top=1. - 0.5 / (nrow + 1), bottom=0.5 / (nrow + 1),
                               left=0.5 / (ncol + 1), right=1 - 0.5 / (ncol + 1))
        for j in range(1,imgs.shape[0]):
            im = imgs[j, :, :]
            #print(floor((i+1)/ncol),floor((i+1)/nrow))
            ax = plt.subplot(nrow,ncol,j)
            ax.imshow(im, cmap='gray')
            plt.axis('off')
            #plt.title(res[i, 0])
            ax.set_xticklabels([])
            ax.set_yticklabels([])
        #fig.title(tit)
        plt.show()

# Find sections with decision ambiguity
def TestWholeNewFolder(sequenceObtained, parametersFound, showPictures):
    if sequenceObtained == False:
        computeRes('test')
    best_err = 1
    best_size=0
    best_kernel = ''
    best_c = 0
    best_pool = ''
    bestId = []
    bestDist = []
    outSizeMax = 30
    poolTypes = ['mean','max']
    train_res = np.load('Network/res_train.p',allow_pickle=True)
    test_res = np.load('Network/res_test3.p',allow_pickle=True)
    folders = np.load('./Network/res_test3_folders.p',allow_pickle=True)

    best_pool = 'max'
    best_size = 13
    if parametersFound == False:
        for size in range(1,outSizeMax+1):
            for pool in poolTypes:
                print('size:',size,'pooling type:',pool)
                x_train, y_train = createConstOutput(train_res,size,pool)
                x_val, y_val = createConstOutput(val_res,size,pool)
                x_test, y_test = createConstOutput(test_res,size,pool)
                tst_err, tst_kernel, tst_c, id, dist = test_spp(x_train,y_train,x_val,y_val,x_test,y_test)
                if tst_err<best_err:
                    bestId = id
                    best_err=tst_err
                    best_size = size
                    best_kernel=tst_kernel
                    best_c=tst_c
                    best_pool = pool
                    best_dist = dist
        print('best error:',best_err,'best size:',best_size,'best pool:',best_pool,'best kernel:',best_kernel,'best c:',best_c)
        print(bestId) # [18 29  7 27 22] for best error: 0.244444444444 best size: 13 best pool: max best kernel: poly best c: 0.3

    x_train, y_train = createConstOutput(train_res,poolType=best_pool, outSize=best_size)
    x_test, y_test = createConstOutput(test_res, poolType=best_pool, outSize=best_size)
    svc = svm.SVC(C=0.3, kernel='poly', probability=True)
    svc.fit(x_train, y_train)
    classif = svc.predict(x_test)
    dist = svc.decision_function(X=x_test)

    print(classif)
    print(folders)
    bestId = []
    noDecisionAmbiguity = []
    cnt = 0
    for clas in classif:
        if classif[cnt] >= 1:
            bestId.append(folders[cnt])
        else:
            noDecisionAmbiguity.append(folders[cnt])
        cnt += 1

    #Save classified facial images for output video

    array = []
    with open('Network/locations.p', 'rb') as f:
        array = pickle.load(f)
    rectangles = []
    rectanglesNoAmbiguity = []
    for folder in bestId:
        images = os.listdir('Bags/'+folder)
        for image in images:
            tmpNumber = image.split('.')
            tmpNumber = int(tmpNumber[0])
            for location in array[tmpNumber]:
                if location != [] and str(location[0]) == str(folder):
                    rectangles.append([tmpNumber,location[2],location[3],location[4],location[5],str(folder)])

    for folder in noDecisionAmbiguity:
        images = os.listdir('Bags/'+folder)
        for image in images:
            tmpNumber = image.split('.')
            tmpNumber = int(tmpNumber[0])
            for location in array[tmpNumber]:
                if location != [] and str(location[0]) == str(folder):
                    rectanglesNoAmbiguity.append([tmpNumber,location[2],location[3],location[4],location[5],str(folder)])

    if showPictures == True:
        showBestId(bestId)

    print('Done locating sections with decision ambiguity')
    with open('Network/rectangles.p', 'wb') as f:
        pickle.dump(rectangles,f)

    with open('Network/rectanglesNoDecisionAmbiguity.p', 'wb') as f:
        pickle.dump(rectanglesNoAmbiguity,f)

    if os.path.exists('Nothing'):
        shutil.rmtree('Nothing', ignore_errors=False, onerror=None)

    if os.path.exists('Bags'):
        shutil.rmtree('Bags', ignore_errors=False, onerror=None)

    return rectangles, rectanglesNoAmbiguity

def ShowOnlySurvey(showPictures):
    from src.ShowHistory import load_data_survey

    best_pool = 'max'
    best_size = 13
    train_res = np.load('res_train.p',allow_pickle=True)
    test_res = np.load('res_test20.p',allow_pickle=True)
    x_train, y_train = createConstOutput(train_res, poolType=best_pool, outSize=best_size)
    x_test, y_test = createConstOutput(test_res, poolType=best_pool, outSize=best_size)
    test = {'x': x_test, 'y': y_test}
    import pickle
    with open('Network/test_survey.pickle', 'wb') as handle:
        pickle.dump(test, handle, protocol=pickle.HIGHEST_PROTOCOL)
    print('survey saved')
    svc = svm.SVC(C=0.3, kernel='poly', probability=True)
    svc.fit(x_train, y_train)
    classif = svc.predict(x_test)
    # probs = svc.predict_proba(x_test)
    dist = svc.decision_function(X=x_test)
    diffs = np.sum(abs(np.subtract(classif, y_test)))
    correct = np.where(np.subtract(classif, y_test) == 0)
    correct = correct[0]
    distCorrect = dist[correct]
    distCorrect = np.abs(distCorrect)
    bestId = distCorrect.argsort()[-5:][::-1]
    bestDist = distCorrect[bestId]
    err = diffs / len(y_test)

    print('bestIDs:',bestId)
    print('err:', err)
    if showPictures == True:
        bestId = np.array([13, 11, 14, 10,  8])
        showBestId(bestId)

max_stars_repo_name: JahodaPaul/DecisionAmbiguityRecognition

max_issues_repo_name: qparis/DecisionAmbiguityRecognition

max_forks_repo_name: qparis/DecisionAmbiguityRecognition


text:
import scipy as SP

class covariance:
    """
    abstract super class for all implementations of covariance functions
    """
    def __init__(self,P):
        self.P = P
        self._calcNumberParams()
        self._initParams()

    def setParams(self,params):
        """
        set hyperparameters
        """
        self.params = params
        self.params_have_changed=True

    def getParams(self):
        """
        get hyperparameters
        """
        return self.params

    def getNumberParams(self):
        """
        return the number of hyperparameters
        """
        return self.n_params

    def K(self):
        """
        evaluates the kernel for given hyperparameters theta and inputs X
        """
        LG.critical("implement K")
        print("%s: Function K not yet implemented"%(self.__class__))
        return None

    def Kgrad_param(self,i):
        """
        partial derivative with repspect to the i-th hyperparamter theta[i]
        """
        LG.critical("implement Kgrad_theta")
        print("%s: Function K not yet implemented"%(self.__class__))
        return None

    def _calcNumberParams(self):
        """
        calculates the number of parameters
        """
        pass

    def _initParams(self):
        """
        initialize paramters to vector of zeros
        """
        params = SP.zeros(self.n_params)
        self.setParams(params)

    def Kgrad_param_num(self,i,h=1e-4):
        """
        check discrepancies between numerical and analytical gradients
        """
        params  = self.params.copy()
        e = SP.zeros_like(params); e[i] = 1
        self.setParams(params-h*e)
        C_L = self.K()
        self.setParams(params+h*e)
        C_R = self.K()
        self.setParams(params)
        RV = (C_R-C_L)/(2*h)
        return RV

ext: py

max_stars_repo_name: PMBio/GNetLMM

max_issues_repo_name: PMBio/GNetLMM

max_forks_repo_name: PMBio/GNetLMM

reason: import scipy


text:
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import struct

from model import model

def binary(num):
    return "".join(f"{c:0>8b}" for c in struct.pack('!f', num))

def floatbin(num):
    binstring = binary(num)
    return binstring[0], binstring[1:9], binstring[9:21], binstring[21:]

def tofixedfloat(x):
    N_total_bits = 18
    N_integer_bits = 5
    sign, expbin, f1, f2 = floatbin(x)
    frac = f1 + f2
    expdec = int(expbin, base=2) - 127
    fracdec = "1"+frac
    ret = "0"*(N_integer_bits - 1 - expdec) + fracdec
    ret = ret[:N_total_bits]
    if sign == "1":
        ret = "".join([("1" if c == "0" else "0") for c in ret])
        negativeret = int(ret, base=2) + 1
        ret = f"{negativeret:b}"
        ret = ret[-N_total_bits:]
    return ret

def tolistexpr(x):
    s = tofixedfloat(x)
    s = "".join(reversed(s))
    ret = "(" + " ".join(s) + ")"
    return ret

def writemat(A, filename):
    with open(filename, "wt") as f:
        f.write("")
    with open(filename, "at") as f:
        f.write("(\n")
        for i in range(A.shape[0]):
            f.write("(")
            for j in range(A.shape[1]):
                f.write(tolistexpr(A[i,j]))
            f.write(")\n")
        f.write(")")

if __name__ == "__main__":
    model.load_weights("./params.h5")

    A_1, B_1 = [x.numpy() for x in model.layers[1].weights]
    A_2, B_2 = [x.numpy() for x in model.layers[3].weights]

    print(np.max(A_1), np.min(A_1),
          np.max(A_2), np.min(A_2),
          np.max(B_1), np.min(B_1),
          np.max(B_2), np.min(B_2))

    x = np.array(
        [
        [0,1,1],
        [0,0,1],
        [0,0,1],
        [0,1,0],
        [1,0,0]
        ], dtype=np.float64)
    x = x.reshape(-1)
    x

    def matpredict(x):
        return np.clip(x @ A_1 + B_1, 0, None) @ A_2 + B_2

    print(matpredict(x))

    print(np.min(x @ A_1), np.max(x @ A_1))
    print(np.min(x @ A_1 + B_1), np.max(x @ A_1 + B_1))
    print(np.min(np.clip(x @ A_1 + B_1, 0, None) @ A_2), np.max(np.clip(x @ A_1 + B_1, 0, None) @ A_2))

    writemat(A_1.transpose(), "A_1_T.lisp")
    writemat(B_1.reshape((1,-1)), "B_1.lisp")
    writemat(A_2.transpose(), "A_2_T.lisp")
    writemat(B_2.reshape((1,-1)), "B_2.lisp")

max_stars_repo_name: woodrush/sectorlisp-nn

max_issues_repo_name: woodrush/sectorlisp-nn

max_forks_repo_name: woodrush/sectorlisp-nn

reason: import numpy


text:
"""Precision for ranking."""
import numpy as np

from matchzoo.engine.base_metric import BaseMetric
from sklearn.metrics import  f1_score

class F1(BaseMetric):
    """Precision metric."""

    ALIAS = 'f1'

    def __init__(self, k: int = 1, threshold: float = 0.5):
        """
        :class:`PrecisionMetric` constructor.

        :param k: Number of results to consider.
        :param threshold: the label threshold of relevance degree.
        """
        self.k = k
        self.threshold = threshold

    def __repr__(self) -> str:
        """:return: Formated string representation of the metric."""
        return f"{self.ALIAS}"

    def __call__(self, y_true: np.array, y_pred: np.array) -> float:

        y_pred[y_pred >= self.threshold] = 1.0
        y_pred[y_pred <  self.threshold] = 0.0

        return f1_score(y_true=y_true, y_pred=y_pred)

        # best, best_t = self.search_f1(y_true, y_pred)
        #
        # return best

    def search_f1(self, y_true, y_pred):
        best = 0
        best_t = 0
        for i in range(30, 60):
            tres = i / 100
            y_pred_bin = (y_pred > tres).astype(int)
            score = f1_score(y_true, y_pred_bin)
            if score > best:
                best = score
                best_t = tres

        # y_pred_bin = (y_pred > best_t).astype(int)
        # best = f1_score(y_true, y_pred_bin)

        return best, best_t

max_stars_repo_name: AlfredWGA/House-Property-QA

max_issues_repo_name: AlfredWGA/House-Property-QA

max_forks_repo_name: AlfredWGA/House-Property-QA


text:
import os
import sys
import shutil
import warnings
warnings.simplefilter('ignore')

from helper import extract_info

sys.path.append(os.getcwd())

from fp.traindata_samplers import CompleteData
from fp.missingvalue_handlers import CompleteCaseAnalysis, ModeImputer, DataWigSimpleImputer
from fp.dataset_experiments import AdultDatasetWhiteExperiment
from fp.scalers import NamedStandardScaler, NamedMinMaxScaler
from fp.learners import NonTunedLogisticRegression, LogisticRegression, DecisionTree, NonTunedLogisticRegression, NonTunedDecisionTree
from fp.post_processors import NoPostProcessing, RejectOptionPostProcessing, EqualOddsPostProcessing, CalibratedEqualOddsPostProcessing
from fp.pre_processors import NoPreProcessing, DIRemover, Reweighing

import numpy as np
import itertools
import matplotlib.pyplot as plt

#creating list of parameters that we will alter to observe variations
seeds = [0xbeef, 0xcafe, 0xdead, 0xdeadcafe, 0xdeadbeef, 0xbeefcafe, 0xcafebeef, 50, 583, 5278, 100000, 0xefac,0xfeeb, 0xdaed, 0xefacdaed, 0xfeebdead]
learners = [NonTunedLogisticRegression(), LogisticRegression(), NonTunedDecisionTree(), DecisionTree()]
processors = [(NoPreProcessing(), NoPostProcessing()), (DIRemover(1.0), NoPostProcessing()), (DIRemover(0.5), NoPostProcessing()), (Reweighing(), NoPostProcessing()),
              (NoPreProcessing(), RejectOptionPostProcessing()), (NoPreProcessing(), CalibratedEqualOddsPostProcessing())]
impute_column_list = ['workclass', 'occupation', 'native-country']
label_column = 'income-per-year'
datawig_imputer = DataWigSimpleImputer(impute_column_list, label_column,out='out')
missing_value_imputers = [CompleteCaseAnalysis(),  ModeImputer(impute_column_list), datawig_imputer]

def calculate_metrics(seed, learner, missing_value_imputer,pre_processor,post_processor):
    '''
    Experiment function to run the experiments
    '''
    exp = AdultDatasetWhiteExperiment(
        fixed_random_seed=seed,
        train_data_sampler=CompleteData(),
        missing_value_handler=missing_value_imputer,
        numeric_attribute_scaler=NamedStandardScaler(),
        learners=[learner],
        pre_processors=[pre_processor],
        post_processors=[post_processor],
        optimal_validation_strategy=[])
    exp.run()

def run_exp(seeds, learners, processors):
    '''
    This is the main driver function that calls the calculate_metrics to give metrices on combinations of various learners, pre and post processing techniques.
    '''
    accuracy, disp_imp, fnr, fpr = [], [], [], []
    for learner in learners:
        for processor in processors:
            for imputer in missing_value_imputers:
                imputer_acc, imputer_di, imputer_fnr, imputer_fpr = [], [], [], []
                for seed in seeds:
                    calculate_metrics(seed, learner, imputer, pre_processor=processor[0], post_processor=processor[1])
                    extract_info(imputer_acc, imputer_di, imputer_fnr, imputer_fpr)
                accuracy.append(imputer_acc)
                disp_imp.append(imputer_di)
                fnr.append(imputer_fnr)
                fpr.append(imputer_fpr)
    return accuracy, disp_imp, fnr, fpr

accuracy, disp_imp, fnr, fpr  = run_exp(seeds, learners, processors)

def plotter(x, y, x_ticks, x_label, main_title):
    '''
    Function to plot various comparison plots.
    '''
    learner_list = ['NonTunedLogistic','TunedLogistic', 'NonTunedDecisionTree', 'TunedDecisionTree']
    processor_list = ['NoPreProcessing', 'DIRemover(1.0)', 'DIRemover(0.5)', 'Reweighing', 'Reject Option', 'Caliberated Equal Odds']
    title_list = list(itertools.product(learner_list,processor_list))
    label_list = [('CompleteCase',  'ModeImputer', 'datawig_simple')]
    fig, axs = plt.subplots(len(learner_list), len(processor_list), figsize=((10,20)))
    axs = axs.flatten()
    for i in range(0, len(y), 3):
        loc = i//3
        axs[loc].scatter([float(valuei) for valuei in x[i]], [float(valuei) for valuei in y[i]], c='b', marker='o')
        axs[loc].scatter([float(valuei) for valuei in x[i + 1]], [float(valuei) for valuei in y[i + 1]], c='r', marker='o')
        axs[loc].scatter([float(valuei) for valuei in x[i + 2]], [float(valuei) for valuei in y[i + 2]], c='g', marker='o')
        axs[loc].set_xticks(x_ticks)
        axs[loc].set_yticks(np.arange(0.5, 1, 0.1))
        axs[loc].set_title(title_list[i//3],fontsize=8)

        axs[loc].grid(True)
        axs[loc].set_xlabel(x_label)
        axs[loc].set_ylabel('Accuracy')
        axs[loc].legend(label_list[int(i%3/2)])
    fig.suptitle(main_title)
    plt.subplots_adjust(wspace=0.3, hspace=0.43)
    fig.savefig('examples/' + main_title + '.png')
    plt.show()

plotter(x=disp_imp, y=accuracy, x_ticks=[0.5, 1, 1.5], x_label='DI', main_title='missing_data_accuracy_vs_di')
plotter(x=fnr, y=accuracy, x_ticks=[-0.4, 0, 0.4], x_label='FNR', main_title='missing_data_accuracy_vs_fnr')
plotter(x=fpr, y=accuracy, x_ticks=[-0.4, 0, 0.4], x_label='FPR', main_title='missing_data_accuracy_vs_fpr')

max_stars_repo_name: DataResponsibly/FairPrep

max_issues_repo_name: DataResponsibly/FairPrep

max_forks_repo_name: DataResponsibly/FairPrep


text:
"""
@author: Salvatore Calderaro
@author: Simone Contini
"""

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import GridSearchCV
from sklearn.decomposition import LatentDirichletAllocation as LDA
from textblob import TextBlob
from wordcloud import WordCloud
from deep_translator import GoogleTranslator
import os
import configobj
import numpy as np
import pyLDAvis.sklearn
import pickle
import matplotlib.pyplot as plt
from PIL import Image

config = configobj.ConfigObj('env.b')

vec_path = config['MODEL_VEC_PATH']
model_path = config['MODEL_PATH']
wordCloud_path = config['WORDCLOUD_PATH']
topics_per_period_path = config['TOPICS_PER_PERIOD_PATH']
emotions_path = config['EMOTION_PATH']
emotions_per_topic_path = config['EMOTION_PER_TOPIC_PATH']
epidemic_path = config['EPIDEMIC_PATH']

cv = pickle.load(open(vec_path, 'rb')) # CountVectorizer
model = pickle.load(open(model_path, 'rb'))

"""
Funzione che applica l'algoritmo LDA (Latent Dirichlet Allocation) ai documenti
della lista in input.
"""
def apply_LDA(documents):
    count_vectorizer = CountVectorizer(min_df=10, max_df=0.95, ngram_range=(1,1))
    feature_matrix = count_vectorizer.fit_transform(documents)
    search_params = {'n_components': [5], 'learning_decay': [.5, .7, .9]}
    lda = LDA()
    model = GridSearchCV(lda, search_params)
    model.fit(feature_matrix)
    best_lda_model = model.best_estimator_
    lda_output = best_lda_model.transform(feature_matrix)
    prob_topic = np.round(lda_output, 2)
    topic_indexes = np.argmax(prob_topic, axis=1)
    n_topics = len(np.unique(topic_indexes))

    # Calcolo Perplexity (minore è, meglio è)
    #print('Perplexity: ', best_lda_model.perplexity(feature_matrix))

    # Calcolo Log likelihood (maggiore è, meglio è)
    #print('Log likelihood: ', best_lda_model.score(feature_matrix))

    return count_vectorizer, best_lda_model, topic_indexes, n_topics

"""
Funzione che stampa, per ogni topic, le top 10 word
"""
def show_top10words_topics(vectorizer, lda_model, n_words):
    keywords = np.array(vectorizer.get_feature_names())
    topic_keywords = []

    for topic_weights in lda_model.components_:
        top_keyword_locs = (-topic_weights).argsort()[:n_words]
        topic_keywords.append(list(keywords.take(top_keyword_locs)))

    print(topic_keywords)

"""
Funzione che crea una word cloud per i topic presi in input
"""
def create_wordCloud(topic_indexes, tweets, n_topics, country, lang, period):
    list_phrases_topics = [[] for _ in range(n_topics)]
    i = 0
    for topic in topic_indexes:
        list_phrases_topics[topic].append(tweets[i])
        i += 1

    for j in range(len(list_phrases_topics)):
        text_cloud = ''

        for feedback in list_phrases_topics[j]:
            text_cloud += ' ' + feedback

        cloud = WordCloud(max_font_size = 40).generate(text_cloud)

        path = wordCloud_path + country + '/' + country + '_' + str(period) + '_' + 'topic' + str(j) + '.png'
        cloud.to_file(path)

"""
Funzione che, per il periodo preso in input, effettua l'emotion analysis di tutti
i topic.
"""
def detect_topic_emotion(topic_indexes, tweets, n_topics, country, lang, period):
    list_phrases_topics = [[] for _ in range(n_topics)]
    i = 0
    for topic in topic_indexes:
        list_phrases_topics[topic].append(tweets[i])
        i += 1

    for j in range(len(list_phrases_topics)):
        freq = {"joy": 0, "sadness": 0, "fear": 0, "anger": 0, "surprise": 0, "neutral": 0, "disgust": 0, "shame": 0}
        for topic_tweets in list_phrases_topics[j]:
            if lang != 'en':
                topic_tweets = GoogleTranslator(source='auto', target='en').translate(topic_tweets)

            aus = []
            aus.append(topic_tweets)
            tweet_vec = cv.transform(aus).toarray()
            res = model.predict(tweet_vec)
            freq[res[0]] += 1

        labels = freq.keys()
        x_pos = np.arange(len(labels))
        val = freq.values()

        fig_title = country + ' ' + str(period) + ' ' + 'Topic' + ' ' + str(j+1) + ' ' + 'Emotion Analysis'
        fig_emo_topic = plt.figure()
        plt.title(fig_title, fontsize=12)
        plt.pie(val)
        plt.legend(labels, loc="best")
        plt.axis('equal')
        plt.tight_layout()
        plt.savefig(emotions_per_topic_path + country + '/' + str(period) + '/' + country + '_' + str(j+1) + '.png', format='png', dpi=1200)
        plt.close(fig_emo_topic)

"""
Funzione che, per il periodo preso in input, effettua l'emotion analysis (attraverso
l'uso di algoritmi di machine learning) dei tweet
"""
def detect_emotion_periods(tweets, lang, country, period):
    freq = {"joy": 0, "sadness": 0, "fear": 0, "anger": 0, "surprise": 0, "neutral": 0, "disgust": 0, "shame": 0}

    for tweet in tweets:
        if lang != 'en':
            text = GoogleTranslator(source='auto', target='en').translate(tweet)
        else:
            text = tweet

        aus = []
        aus.append(text)
        text_vec = cv.transform(aus).toarray()

        res = model.predict(text_vec)
        freq[res[0]] += 1

    if period == 1:
        per = '01/01/2020 - 31/05/2020'
    elif period == 2:
        per = '01/06/2020 - 30/09/2020'
    elif period == 3:
        per = '01/10/2020 - 31/12/2020'
    else:
        per = '01/01/2021 - 09/06/2021'

    labels = freq.keys()
    x_pos = np.arange(len(labels))
    val = freq.values()

    fig_title = country + ' ' + per + ' ' + 'Emotion Analysis'
    fig_emo_topic2 = plt.figure()
    plt.title(fig_title, fontsize=12)
    plt.pie(val)
    plt.legend(labels, loc="best")
    plt.axis('equal')
    plt.tight_layout()
    plt.savefig(emotions_path + country + '/' + country + '_' + str(period) + '.png', format='png', dpi=1200)
    plt.show()

"""
Plot grafici
"""
def plot_analysis(country, period):
    freq = {"joy": 0, "sadness": 0, "fear": 0, "anger": 0, "surprise": 0, "neutral": 0, "disgust": 0, "shame": 0}
    labels = freq.keys()

    wc_image_path = wordCloud_path + country + '/'
    emo_per_topic_path = emotions_per_topic_path + country + '/' + str(period) + '/'
    emo_image_path = emotions_path + country + '/' + country + '_' + str(period) + '.png'
    epidemic_image_path = epidemic_path + country + '.png'

    if period == 1:
        per = '01/01/2020 - 31/05/2020'
    elif period == 2:
        per = '01/06/2020 - 30/09/2020'
    elif period == 3:
        per = '01/10/2020 - 31/12/2020'
    else:
        per = '01/01/2021 - 09/06/2021'

    all_files_wc = os.listdir(wc_image_path)
    all_files_pie = os.listdir(emo_per_topic_path)
    cur_wc_files = []
    cur_pie_files = []

    for file1 in (all_files_wc):
        if('_' + str(period) in file1):
            cur_wc_files.append(file1)

    cur_wc_files.sort(key=lambda x: x.split('_topic')[1])
    all_files_pie.sort(key=lambda x: x.split('_')[1])

    list_wordClouds_images = []
    for f in cur_wc_files:
        f = wc_image_path + f
        list_wordClouds_images.append(Image.open(f,'r'))

    list_piePerTopic_images = []
    for f in all_files_pie:
        f = emo_per_topic_path + f
        list_piePerTopic_images.append(Image.open(f,'r'))

    fig = plt.figure()
    gs1 = fig.add_gridspec(nrows=2,ncols=3)
    ax1 = fig.add_subplot(gs1[0,0])
    ax2 = fig.add_subplot(gs1[0,1])
    ax3 = fig.add_subplot(gs1[0,2])
    ax4 = fig.add_subplot(gs1[1,0])
    ax5 = fig.add_subplot(gs1[1,1])
    ax6 = fig.add_subplot(gs1[1,2])

    ax1.imshow(list_wordClouds_images[0])
    ax1.set_title("Topic 1")
    ax1.axis('off')

    ax2.imshow(list_wordClouds_images[1])
    ax2.set_title("Topic 2")
    ax2.axis('off')

    ax3.imshow(list_wordClouds_images[2])
    ax3.set_title("Topic 3")
    ax3.axis('off')

    ax4.imshow(list_piePerTopic_images[0])
    ax4.set_title("Emotion Pie Topic 1")
    ax4.axis('off')

    ax5.imshow(list_piePerTopic_images[1])
    ax5.set_title("Emotion Pie Topic 2")
    ax5.axis('off')

    ax6.imshow(list_piePerTopic_images[2])
    ax6.set_title("Emotion Pie Topic 3")
    ax6.axis('off')

    sub_title = 'List of topics ' + per + ' - ' + country
    fig.suptitle(sub_title)
    plt.savefig(topics_per_period_path + country + '/' + country + '_' + str(period) + '1.png', format='png', dpi=1200)
    plt.show()

    fig2 = plt.figure()
    gs2 = fig2.add_gridspec(nrows=2,ncols=2)
    ax1 = fig2.add_subplot(gs2[0,0])
    ax2 = fig2.add_subplot(gs2[0,1])
    ax3 = fig2.add_subplot(gs2[1,0])
    ax4 = fig2.add_subplot(gs2[1,1])

    ax1.imshow(list_wordClouds_images[3])
    ax1.set_title("Topic 4")
    ax1.axis('off')

    ax2.imshow(list_wordClouds_images[4])
    ax2.set_title("Topic 5")
    ax2.axis('off')

    ax3.imshow(list_piePerTopic_images[3])
    ax3.set_title("Emotion Pie Topic 4")
    ax3.axis('off')

    ax4.imshow(list_piePerTopic_images[4])
    ax4.set_title("Emotion Pie Topic 5")
    ax4.axis('off')

    sub_title = 'List of topics ' + per + ' - ' + country
    fig2.suptitle(sub_title)
    plt.savefig(topics_per_period_path + country + '/' + country + '_' + str(period) + '2.png', format='png', dpi=1200)
    plt.show()


text:
from tqdm import tqdm
import json
import numpy as np
import random
import math
from matplotlib import pyplot as plt

from sklearn.manifold import TSNE

plt.style.use('ggplot')
NLI_LABELS = ["non_entailment_0", "entailment", "non_entailment_1"]
NLI_DATASETS = ["HANS", "MNLI_TRAIN", "MNLI_DEV", "MNLI_OVERLAP"]

def visualize_predictions(all_probs: np.ndarray, all_labels: np.ndarray, conf_plot_file: str):

    width = 0.1
    num_bins = 10
    all_bins = [[] for x in range(num_bins)]
    all_bins_stat = [[0, 0, 0.] for _ in range(num_bins)] # cor_num, num, conf_accum

    pred_probs = all_probs.max(axis=1)
    pred_labels = all_probs.argmax(axis=1)
    # correct_probs = []
    for pred_prob, pred_label, true_label in zip(pred_probs, pred_labels, all_labels):
        which_bins = math.floor(pred_prob / width)
        all_bins[which_bins].append((pred_prob, pred_label, true_label))

        all_bins_stat[which_bins][1] += 1
        all_bins_stat[which_bins][2] += pred_prob

        if pred_label == true_label:
            all_bins_stat[which_bins][0] += 1

    all_bins_acc = []
    for bin in all_bins_stat:
        if bin[1] == 0:
            all_bins_acc.append(0.)
        else:
            all_bins_acc.append(float(bin[0])/ bin[1])
    all_bins_conf = []
    for bin in all_bins_stat:
        if bin[1] == 0:
            all_bins_conf.append(0.)
        else:
            all_bins_conf.append(bin[2] / bin[1])

    all_nums = [x[1] for x in all_bins_stat]
    ECE_bin = [(all_nums[x]/sum(all_nums)) * abs(all_bins_acc[x] - all_bins_conf[x]) for x in range(len(all_bins_acc))]

    fracts = [x/sum(all_nums) for x in all_nums]
    acc_fracts = [x*y for x, y in zip(fracts, all_bins_acc)]

    objects = ('0.', '0.1', '0.2', '0.3', '0.4', '0.5', '0.6', '0.7', '0.8', '0.9')
    y_pos = np.arange(len(objects))

    plt.bar(y_pos, fracts, align='center', alpha=0.5, label="dataset fraction")
    plt.bar(y_pos, acc_fracts, align='center', alpha=0.5, label="correctly predicted")
    plt.xticks(y_pos, objects)
    plt.ylabel('percentage of dataset')
    plt.title('predictions confidence plot')
    plt.legend(loc="upper right", fontsize=12)

    plt.savefig(conf_plot_file)
    plt.show()
    plt.close()

    return sum(ECE_bin), all_bins_acc, all_bins_conf, all_nums

def visualize_penultimate(all_labels: np.ndarray, all_is_correct: np.ndarray,
                          all_penultimates: np.ndarray, all_dataset_ids: np.ndarray,
                          plot_file: str=None, sample=-1):
    if sample > 0:
        rand_idx = np.random.randint(all_penultimates.shape[0], size=sample)
        all_labels = all_labels[rand_idx]
        all_is_correct = all_is_correct[rand_idx]
        all_penultimates = all_penultimates[rand_idx, :]

    tsne = TSNE(n_components=2, random_state=555, perplexity=50, n_iter=5000)
    print("computing the tsne reduction...")
    all_penultimates = tsne.fit_transform(all_penultimates)

    plt.figure(figsize=(12, 10))

    ax = plt.gca()

    marker_types = ["D", "s", "o", "^"]
    color_types = [next(ax._get_lines.prop_cycler)['color'],
                   next(ax._get_lines.prop_cycler)['color'],
                   next(ax._get_lines.prop_cycler)['color']]
    for dat_ix in np.unique(all_dataset_ids):
        for label in np.unique(all_labels):
            cur_penultimates = []
            cur_incor_penultimates = []
            for cur_label, cur_is_correct, cur_dataset_id, cur_penultimate in zip(all_labels, all_is_correct,
                                                                                all_dataset_ids, all_penultimates):
                # if dat_ix == 0 and label == 1 and cur_dataset_id == 0:
                #     import pdb; pdb.set_trace()
                if cur_label == label and cur_dataset_id == dat_ix:
                    cur_penultimates.append(np.expand_dims(cur_penultimate, 0))
                    if cur_is_correct == 0:
                        cur_incor_penultimates.append(np.expand_dims(cur_penultimate, 0))

            # if dat_ix == 0:
            #     continue

            if len(cur_penultimates) == 0:
                # import pdb; pdb.set_trace()
                continue

            cur_penultimates = np.concatenate(cur_penultimates, axis=0)

            label_str = NLI_LABELS[label]
            dataset_str = NLI_DATASETS[dat_ix]

            size = plt.rcParams['lines.markersize']**2
            if dat_ix == 3:
                size = size * 2

            if len(cur_incor_penultimates) > 0:
                cur_incor_penultimates = np.concatenate(cur_incor_penultimates, axis=0)
                if dat_ix == 0:
                    plt.scatter(cur_incor_penultimates[:, 0], cur_incor_penultimates[:, 1],
                                alpha=0.8, marker='x', color=color_types[label], s=size)
                else:
                    plt.scatter(cur_incor_penultimates[:, 0], cur_incor_penultimates[:, 1],
                                alpha=0.8, marker='*', color=color_types[label], s=size)

            plt.scatter(cur_penultimates[:, 0], cur_penultimates[:, 1],
                        label=dataset_str+"_"+label_str, alpha=0.5, marker=marker_types[dat_ix],
                        color=color_types[label], s=size)

    plt.legend()

    plt.savefig(plot_file)
    plt.show()
    plt.close()

    return all_penultimates


text:
import numpy as np
from windsaloft.jetstream import jet_streams

def test_raster_base_coordinates_ne():
    w = 30
    Y, X = np.mgrid[-w:w:91j, -w:w:180j]
    u = -1 - X**2 + Y
    v = 1 + X - Y**2

    feature_collection = jet_streams(
        u, v, pixel_dist=30, min_value=10, smooth=0, zigzag_degrees=45)
    x, y = feature_collection.features[0].geometry.coordinates[0][0]
    assert x == -178.0
    assert y == 87.830286

max_stars_repo_name: olehz/windsaloft

max_issues_repo_name: olehz/windsaloft

max_forks_repo_name: olehz/windsaloft


text:
import sys
import os
import subprocess
import re
import math
from typing import Sequence, Dict, Mapping, List
from enum import Enum, auto
import numpy as np

cryst_patt = re.compile(r"^ +(-?\d+\.\d+){6} *$")
chargespin_patt = re.compile(r"^ *(-?\d+) +(-?\d+) *$")
hartree = 627.5094736
gauss_polar_convert = 0.52917721067 ** 3

def eprint(*args, kwargs: dict = None):
    """Prints to stderr; effectively wraps the print() function with kwargs['file'] = sys.stderr, and kwargs['flush'] =
    True unless already defined in kwargs."""
    if kwargs is None:
        kwargs = dict()
    kwargs['file'] = sys.stderr
    if 'flush' not in kwargs:
        kwargs['flush'] = True
    print(*args, **kwargs)

def verbose_call(args: Sequence[str], kwargs = None, to_stderr: bool = True):
    out_str = "Calling:"
    for a in args:
        out_str += f" {a}"
    if to_stderr:
        eprint(out_str)
    else:
        print(out_str)
    if kwargs is None:
        kwargs = dict()
    subprocess.run(args, **kwargs)

def version_file(fname: str) -> str:
    if os.path.exists(fname):
        for i in range(1000):
            trial_fname = f"{fname}_{i}"
            if not os.path.exists(trial_fname):
                return trial_fname
        raise RuntimeError(f"Could not version {fname} to {fname}_x for 1<=x<=1000!")
    else:
        return fname

def name_to_atom(xyzf: str, name: str) -> (int, str, float, float, float, int, Sequence[int]):
    with open(xyzf, 'r') as f:
        f.readline()
        line = f.readline()
        if (cryst_patt.match(line)):
            line = f.readline()
        while line != '':
            toks = line.split()
            test_str = toks[1] + toks[0]
            if test_str == name:
                bonds = [int(t) for t in toks[6:]]
                return int(toks[0]), toks[1], float(toks[2]), float(toks[3]), float(toks[4]), int(toks[5]), bonds
            line = f.readline()
    eprint(f"No match for {name} found in {xyzf}!")
    return None

def audit_files(files: Mapping[str, float] = None) -> (Mapping[str, float], Mapping[str, float]):
    # Files may be None if attempting to generate the first mapping.
    if files is None:
        files = dict()

    modified = dict()
    added = dict()
    for fi in os.scandir():
        if fi.is_file():
            fi_name = fi.name
            if fi_name in files:
                mtime = fi.stat().st_mtime
                if mtime != files[fi_name]:
                    modified[fi_name] = mtime
            else:
                added[fi_name] = fi.stat().st_mtime
    return modified, added

def log_audit_files(files: Dict[str, float], about: str):
    modified, added = audit_files(files)
    log_audit_files.n_log_audit += 1
    eprint(f"AUDIT {log_audit_files.n_log_audit:>3d}: ", kwargs={'end': ''})

    if len(modified.keys()) == 0 and len(added.keys()) == 0:
        eprint(f"No effect from {about}")
        return

    if len(modified) > 0:
        eprint(f"Modified since {about}:")
        for m, t in modified.items():
            eprint(f"Updated timestamp on {m}: {files[m]:.6f} -> {t:.6f}")
    if len(added) > 0:
        eprint(f"Added since {about}:")
        for a, t in added.items():
            eprint(f"Added file {a} with timestamp {t:.6f}")

    # Now, update the mapping.
    for m, t in modified.items():
        files[m] = t
    for a, t in added.items():
        files[a] = t
    eprint("\n")
log_audit_files.n_log_audit = 0

def dist2(coord1: np.ndarray, coord2: np.ndarray) -> float:
    return np.square(coord1 - coord2).sum()

def get_probe_dirs(base_dir: str = ".") -> List[str]:
    return [f.path for f in os.scandir(base_dir) if (f.is_dir() and os.path.exists(f"{f.path}{os.sep}QM_PR.xyz"))]

class QMProgram(Enum):
    GAUSSIAN = tuple("GAUSS_SCRDIR")
    PSI4 = tuple("PSI4_SCRATCH")

    def get_scrdir_name(self) -> str:
        return self.value[0]

    def get_scrdir(self, fallback_scrdir: str = None) -> str:
        own_varname = self.get_scrdir_name()
        scrdir = os.environ.get(own_varname)
        if scrdir is None:
            for qmp in QMProgram:
                qmp_varname = qmp.get_scrdir_name()
                scrdir = os.environ.get(qmp_varname)
                if scrdir is not None:
                    eprint(f"WARNING: Failed to find environment variable {own_varname}; falling back to {qmp_varname} "
                           f"with value {scrdir}")
                    return scrdir
        else:
            return scrdir

        if scrdir is None:
            if fallback_scrdir is None:
                raise ValueError(f"ERROR: Could not find environment variable {own_varname} nor any fallback "
                                 f"environment variables.")
            else:
                eprint(f"WARNING: Could not find environment variable {own_varname} nor any fallback environment "
                       f"variables: falling back to provided scratch directory {fallback_scrdir}")
                return fallback_scrdir

def parse_qm_program(parsed: str, default_program: QMProgram = QMProgram.PSI4) -> QMProgram:
    parsed = parsed.upper()
    if parsed.startswith("GAUSS"):
        return QMProgram.GAUSSIAN
    elif parsed == "PSI4":
        return QMProgram.PSI4
    else:
        return default_program

def extract_molspec(qm_fi: str, program: QMProgram) -> (int, int, np.ndarray, np.ndarray):
    atoms = []
    coords = []
    charge = None
    spin = None
    if program == QMProgram.PSI4:
        with open(qm_fi, 'r') as r:
            line = r.readline()
            in_molspec = False
            while line != "":
                if in_molspec:
                    if "}" in line:
                        in_molspec = False
                    toks = line.strip().rstrip("}").rstrip().split()
                    if len(toks) > 3:
                        assert len(toks) < 6
                        atoms.append(toks[0])
                        coords.append(toks[1:4])
                    else:
                        assert not in_molspec
                elif line.startswith("molecule"):
                    if not line.rstrip().endswith("{"):
                        assert r.readline().strip() == "{"
                    toks = r.readline().strip().split()
                    assert len(toks) == 2
                    charge = int(toks[0])
                    spin = int(toks[1])
                    in_molspec = True
                line = r.readline()
    elif program == QMProgram.GAUSSIAN:
        with open(qm_fi, 'r') as r:
            line = r.readline()
            in_molspec = False
            after_routing = False
            while line != "":
                if in_molspec:
                    if line == "\n":
                        in_molspec = False
                        after_routing = False
                    else:
                        toks = line.strip().split()
                        assert len(toks) == 4
                        atoms.append(toks[0])
                        coords.append(toks[1:4])
                elif line.startswith("#"):
                    after_routing = True
                elif after_routing:
                    m = chargespin_patt.match(line)
                    if m:
                        toks = line.strip().split()
                        in_molspec = True
                        charge = int(toks[0])
                        spin = int(toks[1])
                line = r.readline()
    else:
        raise ValueError(f"Unrecognized QM program {program}")

    coords = np.array(coords, dtype=np.float64)
    assert charge is not None
    assert spin is not None
    assert len(atoms) > 0
    assert len(atoms) == coords.shape[0]
    assert coords.shape[1] == 3
    return charge, spin, np.array(atoms), coords

def list2_to_arr(list2: Sequence[list]) -> np.ndarray:
    return np.array([item for sublist in list2 for item in sublist])

gauss_float_patt = re.compile(r'-?0\.(\d+)D([+\-]\d{2})')

def convert_gauss_float(g_num: str) -> str:
    m = gauss_float_patt.match(g_num)
    if not m:
        raise ValueError(f"String {g_num} is not a Gaussian/Tinker style floating-point number!")
    neg_str = ""
    if g_num[0] == "-":
        neg_str = "-"

    nums = m.group(1)
    exp = int(m.group(2)) - 1
    return f"{neg_str}{nums[0]}.{nums[1:]}e{exp:d}"

def parse_gauss_float(g_num: str) -> float:
    return float(convert_gauss_float(g_num))

def symlink_nofail(source: str, dest: str) -> bool:
    """Creates a symbolic link if none already exists. Intended to avoid raising an error when the symlink already
    exists."""
    if os.path.exists(dest):
        return False
    os.symlink(source, dest)
    return True

singleton_patt = re.compile(r'^[0-9]+$')
range_patt = re.compile(r'^([0-9]+)-([0-9]+)$')

def parse_jml_range(range_str: str) -> Sequence[int]:
    """Adaptation of my comma-separated, 1-indexed, inclusive ranges from FFX."""
    toks = range_str.split(",")
    indices = set()
    for tok in toks:
        if singleton_patt.match(tok):
            indices.add(int(tok) - 1)
        else:
            m = range_patt.match(tok)
            if m:
                lb = int(m.group(1))
                ub = int(m.group(2))
                if (lb > ub):
                    eprint(f"Warning: specified range {tok} has lb > ub!")
                    temp = lb
                    lb = ub
                    ub = temp
                for i in range(lb - 1, ub, 1):
                    indices.add(i)
            else:
                eprint(f"Warning: unable to parse range {tok}")
    return sorted(indices)

def run_potential_3(potential: str, file_base: str):
    verbose_call([potential, "3", f"{file_base}.xyz", "Y"])

def parse_truth(val: str) -> bool:
    false_vals = ['false', '0', 'disabled', 'no']
    if val.lower() in false_vals:
        return False
    return bool(val)

max_stars_repo_name: jacob-litman/fittingScripts

max_issues_repo_name: jacob-litman/fittingScripts

max_forks_repo_name: jacob-litman/fittingScripts


text:
import numpy as np
import skimage.exposure as exposure
from .util import AttributeDict

from sklearn.decomposition import PCA

def extract_features(image, measurements, debug=False):
	"""
	Given an image as a Numpy array and a set of measurement objects
	implementing a compute method returning a feature vector, return a combined
	feature vector.
	"""

	# TODO(liam): parallelize multiple measurements on an image by using Celery

	def trace(m, x):
		"""if the debug flag is set to true, log parameters of each measurement"""
		if debug:
			print("{}: len(x) = {}".format(type(m).__name__, len(x)))

		return x
	return np.hstack([trace(m, m.compute(image)) for m in measurements])

def normalize_features(X):
	# -- recenter features and normalize over the dataset
	X -= np.mean(X, axis=0)

	# normalize where nonzero
	nz = X != 0
	X[nz] /= np.linalg.norm(X[nz], axis=0)

	# normalize for each record
	X /= np.vstack(np.linalg.norm(X, axis=1))

	return X

def feature_postprocessing(X, options):
	_options = AttributeDict({'normalize': True, 'fill_nans': False, 'pca': None})
	_options.update(options or {}); options = _options

	# make sure everything is a float64
	X = X.astype('float64')

	if options.fill_nans:
		X = np.nan_to_num(X)

	if options.normalize:
		X = normalize_features(X)

	if options.pca:
		pca = PCA(n_components=options.pca['components'])
		X = pca.fit_transform(X)

	return X

def image_preprocessing(im, options):
	_options = AttributeDict({'normalize': True, 'equalize': None})
	_options.update(options or {}); options = _options

	if options.normalize:
		im = exposure.rescale_intensity(im)

	if options.equalize:
		if options.equalize['method'] == "histogram":
			im = exposure.equalize_hist(im)
		elif options.equalize['method'] == "stretch":
			pmin, pmax = np.percentile(im,
				(options.equalize['saturation'], 100-options.equalize['saturation']))
			im = exposure.rescale_intensity(im, in_range=(pmin, pmax))

	return im


text:
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from __future__ import print_function

import cv2
import dlib
import fire
import json
import math
import numpy as np
import os
import skvideo.io
import skvideo.datasets
import sys

from skimage import io
from base64 import b64encode

stdout = getattr(sys.stdout, 'buffer', sys.stdout)
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('./shape_predictor_68_face_landmarks.dat')

def triangle_center(p1, p2, p3):
    x = (p1[0] + p2[0] + p3[0]) / 3
    y = (p1[1] + p2[1] + p3[1]) / 3
    return (x, y)

def triangle_area(p1, p2, p3):
    return ((p2[0] - p1[0]) * (p3[1] - p1[1]) - (p2[1] - p1[1]) * (p3[0] - p1[0]))

def polygon_center(count, points):
    s = 0.0
    gx = 0.0
    gy = 0.0
    for i in range(2, count):
        s1 = triangle_area(points[0], points[i - 1], points[i])
        pt = triangle_center(points[0], points[i - 1], points[i])
        gx += s1 * pt[0]
        gy += s1 * pt[1]
        s += s1
    if s == 0.0:
        x = sum([p[0] for p in points]) / float(count)
        y = sum([p[1] for p in points]) / float(count)
        return (x, y)
    else:
        return (gx / s, gy / s)

def imgcat_for_iTerm2(filename):
    with open(filename, 'rb') as f:
        data = f.read()
        if os.environ['TERM'].startswith('screen'):
            osc = b'\033Ptmux;\033\033]1337;File='
            st = b'\a\033\\\n'
        else:
            osc = b'\033]1337;File='
            st = b'\a\n'
        stdout.write(b'%ssize=%d;inline=1:%s%s' %
                     (osc, len(data), b64encode(data), st))

def detect_pix_format_of_video(filename):
    metadata = skvideo.io.ffprobe(filename)
    pix_fmt = metadata["video"]["@pix_fmt"]
    return pix_fmt

def metadata_of_video(filename):
    return skvideo.io.ffprobe(filename)["video"]

face_index = 0
frame_count = 0
yuv_fmt_list = ['yuv420p']

def collect_faces_from_video(filename, output, confidence, resize, prefix, zerofill, rotate, expansion, frame_skip, webcam, imgcat):
    global face_index
    global frame_count

    if frame_skip == 0:
        frame_skip = 1

    resize_flag = False
    sar = '1:1'
    pix_fmt = 'yuv420p'

    if webcam != '':
        print("video: built-in camera or webcam")
        cap = cv2.VideoCapture(webcam)
    else:
        print("video:", filename)
        metadata = metadata_of_video(filename)
        print("codec:",metadata["@codec_long_name"])
        sar = metadata["@sample_aspect_ratio"]
        print("SAR (sample_aspect_ratio):", sar)
        pix_fmt = detect_pix_format_of_video(filename)
        pix_fmt = metadata["@pix_fmt"]
        print("pix_fmt:", pix_fmt)
        print("duration:", metadata["@duration"])
        print("nb_frames:", metadata["@nb_frames"])

        # Calculate frame resize ratio
        # 1440x1080 and SAR4:3 -> 4/3=1.333.. x 1440 = 1920
        resize_rate_width, resize_rate_height = float(sar.split(':')[0]), float(sar.split(':')[1])
        if (resize_rate_width > resize_rate_height):
            base_resize_rate = resize_rate_height
        else:
            base_resize_rate = resize_rate_width
        resize_rate_width = resize_rate_width / base_resize_rate
        resize_rate_height = resize_rate_height / base_resize_rate

        # Check resize is necessary
        if (resize_rate_width != 1.0) or (resize_rate_height != 1.0):
            resize_flag = True

        cap = cv2.VideoCapture(filename)

    if (not cap.isOpened()):
        print("Unable to connect to camera or video")
        sys.exit()

    while(cap.isOpened()):
        ret, frame = cap.read()
        if ret == False:
            break

        height, width = frame.shape[:2]

        if frame_count % frame_skip == 0:
            # print("frame:", frame_count)
            if resize_flag:
                # Resize according to SAR
                width = int(width * resize_rate_width)
                height = int(height * resize_rate_height)
                frame = cv2.resize(frame, (width, height), interpolation = cv2.INTER_CUBIC)

            # Check YUV or RGB
            if pix_fmt in yuv_fmt_list:
                dframe = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB)
            else:
                dframe = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

            dets, scores, _idx = detector.run(dframe, 1)

            for (face, score) in zip(dets, scores):
                if (score > confidence):
                    collect_faces_from_frame(face, frame, width, height, output, confidence, resize, prefix, zerofill, rotate, expansion, webcam, imgcat)

        frame_count = frame_count + 1

    cap.release()

def collect_faces_from_image(filename, output, confidence, resize, prefix, zerofill, rotate, expansion, frame_skip, webcam, imgcat):
    global face_index

    print("image:", filename)
    img = io.imread(filename)
    try:
        # rgb
        if img.shape[2] == 3:
            frame = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    except IndexError:
        # gray
        frame = img
    height, width = frame.shape[:2]

    dets, scores, _idx = detector.run(img, 1)

    for (face, score) in zip(dets, scores):
        if (score > confidence):
            collect_faces_from_frame(face, frame, width, height, output, confidence, resize, prefix, zerofill, rotate, expansion, webcam, imgcat)

def collect_faces_from_frame(face_rect, frame, width, height, output, confidence, resize, prefix, zerofill, rotate, expansion, webcam, imgcat):
    left, top  = face_rect.left(), face_rect.top()
    right, bottom = face_rect.right(), face_rect.bottom()

    if not top < 0 and left < 0 and bottom > height and right > width:
        # ignore irregular rect
        return
    else:
        shape = predictor(frame, face_rect)
        left_eye_points = []
        right_eye_points = []
        for i in range(36, 42):
            left_eye_points.append((shape.part(i).x, shape.part(i).y))
        for i in range(42, 48):
            right_eye_points.append((shape.part(i).x, shape.part(i).y))
        left_eye = polygon_center(6, left_eye_points)
        right_eye = polygon_center(6, right_eye_points)

        eyes_center = ( (left_eye[0] + right_eye[0]) / 2, (left_eye[1] + right_eye[1]) / 2 )
        pupil_distance = math.sqrt(
            (right_eye[1] - left_eye[1])**2 + (right_eye[0] - left_eye[0])**2)

        expansion_rate = expansion - 1.0
        pd_int = int(round(pupil_distance * expansion_rate))

        start_x = max(0, left - pd_int)
        start_y = max(0, top - pd_int)
        end_x = min(width, right + pd_int)
        end_y = min(height, bottom + pd_int)

        # ignore when not square
        w = end_x - start_x
        h = end_y - start_y
        if w != h:
            return

        if rotate:
            # rotate face
            radian = math.atan2(
                left_eye[1] - right_eye[1], right_eye[0] - left_eye[0])
            rot = cv2.getRotationMatrix2D(eyes_center, -np.rad2deg(radian), 1.0)
            im_affine = cv2.warpAffine(frame, rot, frame.shape[:2][::-1], flags=cv2.INTER_LANCZOS4)
            dst_img = im_affine[start_y:end_y, start_x:end_x]
        else:
            dst_img = frame[start_y:end_y, start_x:end_x]

        save_face(dst_img, output, resize, prefix, zerofill, imgcat)

def save_face(face_img, output, resize, prefix, zerofill, imgcat):
    global face_index

    if resize:
        resize = int(resize)
        face_img = cv2.resize(face_img, (resize, resize), interpolation = cv2.INTER_LANCZOS4)

    number_padded = str(face_index).zfill(zerofill)
    filename = prefix + number_padded + ".png"
    filepath = os.path.join(output, filename)

    if cv2.imwrite(filepath, face_img):
        print("Saved:", filepath)
    else:
        print("Error: Failed to save", filepath)
        sys.exit(1)

    face_index = face_index + 1

    if imgcat:
        imgcat_for_iTerm2(filepath)

def find_all_files(target_dir):
    for root, dirs, files in os.walk(target_dir):
        for filename in files:
            yield os.path.join(root, filename)

img_file_ext = ['.png', '.PNG',
                '.jpg', '.JPG', '.jpeg', '.JPEG',
                '.gif', '.GIF']
video_file_ext = ['.mp4', '.MP4']

def collect(target='', output='faces', confidence=0.6, resize='', prefix='face', zerofill=5, rotate=False, expansion=1.5, frame_skip=30, webcam='', imgcat=False):
    # check target is specified
    if target == '':
        # check webcam option
        if webcam == '':
            print('Error: Please specify target.')
            sys.exit(1)
        else:
            collect_faces_from_video(target, output, confidence, resize, prefix, zerofill, rotate, expansion, frame_skip, webcam, imgcat)
    else:
        if webcam != '':
            print('`--target` option is ignored.')
            collect_faces_from_video(target, output, confidence, resize, prefix, zerofill, rotate, expansion, frame_skip, webcam, imgcat)

    if expansion < 1.0:
        print('Error: expansion value must be at least 1.0')
        sys.exit(1)

    # Check file or directory
    if (os.path.isdir(target)):
        for filename in find_all_files(target):
            collect(filename, output, confidence, resize, prefix, zerofill, rotate, expansion, frame_skip, webcam, imgcat)
    else:
        _filename, extension = os.path.splitext(target)
        if extension in img_file_ext:
            collect_faces_from_image(target, output, confidence, resize, prefix, zerofill, rotate, expansion, frame_skip, webcam, imgcat)
        elif extension in video_file_ext:
            collect_faces_from_video(target, output, confidence, resize, prefix, zerofill, rotate, expansion, frame_skip, webcam, imgcat)
        else:
            print("Skip:", target)

if __name__ == '__main__':
    fire.Fire(collect)

max_stars_repo_name: knjcode/face-collector

max_issues_repo_name: knjcode/face-collector

max_forks_repo_name: knjcode/face-collector


text:
import torch
import os
import pdb
from nltk.tokenize import sent_tokenize
from transformers import AutoTokenizer
import random
import argparse
from multiprocessing import Pool
import numpy as np

from led_summ.utils.compute_scores import (
    compute_scores,
    get_entities,
)
from tqdm import tqdm
from timeit import default_timer as timer
from rouge_score import rouge_scorer
import re
import spacy

def get_src_tgt_with_mask(
    mask_indices,
    truncated_doc,
    tokenizer,
    max_length_input,
    max_length_output,
    non_mask_ratio=0.5,
    unchanged_mask_indices=None,
):
    tgt = []
    cur_idx = 0
    # print(mask_indices, int(len(mask_indices) * non_mask_ratio))
    if unchanged_mask_indices is None:
        non_mask_indices = random.sample(
            list(mask_indices), int(len(mask_indices) * non_mask_ratio)
        )
    else:
        non_mask_indices = unchanged_mask_indices

    for i_d in range(len(truncated_doc)):
        for i_s in range(len(truncated_doc[i_d])):
            if cur_idx in mask_indices:
                tgt.append(truncated_doc[i_d][i_s])
                if cur_idx not in non_mask_indices:
                    truncated_doc[i_d][i_s] = tokenizer.mask_token
            cur_idx += 1

    src = []
    truncated_doc = " %s " % (tokenizer.convert_tokens_to_ids("<doc-sep>")).join(
        [" ".join(d) for d in truncated_doc]
    )
    truncated_doc += " %s" % (tokenizer.convert_tokens_to_ids("<doc-sep>"))
    src = tokenizer.encode(truncated_doc, max_length=max_length_input, truncation=True)
    tgt = " ".join(tgt)
    tgt = tokenizer.encode(tgt, max_length=max_length_output, truncation=True)
    assert len(src) < max_length_input or len(src) == max_length_input
    assert len(tgt) < max_length_output or len(tgt) == max_length_output
    return src, tgt

def select_sent_with_entities(entities, all_docs, num_sent, metric):
    sent_plain = [s for doc_list in all_docs for s in doc_list]
    entity_info = {
        k: {"max_score": 0, "indice": -1} for k in entities.keys() if entities[k] > 1
    }
    for i, sent in enumerate(sent_plain):
        for e in entity_info.keys():
            if e.lower() in sent["text"].lower():
                if sent[metric] > entity_info[e]["max_score"]:
                    entity_info[e]["max_score"] = sent[metric]
                    entity_info[e]["indice"] = i
    sorted_entities = [
        k for k, v in sorted(entities.items(), key=lambda item: item[1]) if v > 1
    ][::-1]
    selected_sent = set()
    for k in sorted_entities:
        selected_sent.add(entity_info[k]["indice"])
        # stop when desired number of sentences are selected
        if len(selected_sent) >= num_sent:
            break
    # if entities are not enough
    if len(selected_sent) < num_sent:
        scores_flat = [s[metric] for s in sent_plain]
        greedy_indices = np.argsort(scores_flat)[::-1]
        for s in greedy_indices:
            if s not in selected_sent:
                selected_sent.add(s)
                if len(selected_sent) >= num_sent:
                    break
    return sorted(list(selected_sent))

def preprocessing_entities(entities):
    new_entities = {}
    for ent in entities.keys():
        new_ent = ent.lower().replace("the", "").strip()
        new_ent = new_ent.replace("'s", "").strip()
        new_entities[new_ent] = new_entities.get(new_ent, 0) + entities[ent]
    return new_entities

def truncate_doc(all_docs, max_length_input, mask_ratio, non_mask_ratio):
    # Truncate the documents to desired
    truncated_doc = []
    truncated_scores = []
    for doc in all_docs:
        cur_idx = 0
        all_sent_text = []
        all_scores = []
        for s in doc:
            cur_idx += len(s["text"].split())
            # if add the new sentence exceeds the expected length limit, don't add it
            if cur_idx >= (
                (max_length_input * (1 + mask_ratio * (1 - non_mask_ratio)))
                // len(all_docs)
            ):
                break
            all_sent_text.append(s["text"] + " .")
            all_scores.append(s)
        truncated_doc.append(all_sent_text)
        truncated_scores.append(all_scores)
    return truncated_doc, truncated_scores

def process_single_data_with_scores(
    all_docs,
    tokenizer,
    max_length_input,
    max_length_output,
    mask_ratio,
    metric="pegasus_score",
    strategy="greedy",
    non_mask_ratio=0.5,
):
    """
    all docs: dictionary for a single cluster, like
        {
         'entities_pyramid': {'ent1':1,'ent2':2,...}
         'entities': {'ent1':1,'ent2':3,...}
         'data':[[{'text': 'I like apple',
                    'pegasus_score': 0.3,
                    'pyramid_rouge': 0.5},
                  {'text': 'And I also like banana',
                    'pegasus_score': 0.5,
                    'pyramid_rouge': 0.4}],
                  ...],
                 [{'text': ...,
                    'pegasus_score': 0.4,
                    'pyramid_rouge': 0.6},
                 ...],
                ...]
        }

    """
    # if with entity

    entities = all_docs["entities_pyramid"]
    all_docs = all_docs["data"]
    # Truncate the documents to desired
    truncated_doc, scores = truncate_doc(
        all_docs, max_length_input, mask_ratio, non_mask_ratio
    )
    total_num_sentences = sum([len(d) for d in truncated_doc])
    mask_sent_num = int(total_num_sentences * mask_ratio)
    if strategy == "greedy":  # can only greedy on one metric
        scores_flat = [s for s_list in scores for s in s_list[metric]]
        # sorted_indices = np.argsort(scores_flat)[::-1]
        mask_indices = np.argsort(scores_flat)[::-1][:mask_sent_num]
    elif strategy == "greedy_entity_pyramid":
        mask_indices = select_sent_with_entities(
            entities, scores, mask_sent_num, metric
        )

    src, tgt = get_src_tgt_with_mask(
        mask_indices,
        truncated_doc,
        tokenizer,
        max_length_input,
        max_length_output,
        non_mask_ratio,
    )
    return {"src": src, "tgt": tgt}

def compute_all_scores_single_data(all_docs, i):
    cluster = [
        [
            s.strip()
            for p in re.split("\n+", doc)
            # for s in re.split(r"\.|\?|!", p)
            for s in sent_tokenize(p)
            if s.strip() != ""
        ]
        for doc in all_docs
    ]
    # take much time process very long sequence.
    if sum([len(d) for d in cluster]) > 600:
        return [
            [{"text": s, "pegasus_score": 0, "pyramid_rouge": 0} for s in d]
            for d in cluster
        ]
    scorer = rouge_scorer.RougeScorer(
        ["rouge1", "rouge2", "rougeL", "rougeLsum"], use_stemmer=True
    )
    data_with_scores = compute_scores(cluster, scorer)

    # update entities
    result = update_entities(data_with_scores)

    return result

def update_entities(single_data_with_scores):
    nlp = spacy.load("en_core_web_sm")
    entity_pyramid, entity = get_entities(nlp, single_data_with_scores)
    result = {
        "data": single_data_with_scores,
        "entities": entity,
        "entities_pyramid": entity_pyramid,
    }
    return result

def process_all_newshead(
    input_dir,
    tokenizer,
    max_length_input,
    max_length_output,
    mask_ratio,
    output_dir,
    data_splits,
    num_workers,
    mode="compute_all_scores",
    metric="pegasus_score",
    strategy="greedy",
    non_mask_ratio=0.5,
):
    """
    Generate the pretraining data for wikisum dataset
    """
    if isinstance(data_splits, str):
        data_splits = [data_splits]
    for data_type in data_splits:
        all_files = [
            f
            for f in os.listdir(os.path.join(input_dir, data_type))
            if f.endswith(".pt")
        ]
        all_files = all_files[::-1]
        if mode == "compute_all_scores":
            out_path = os.path.join(output_dir, data_type)
        elif mode == "pretraining_data_with_score":
            out_path = os.path.join(
                output_dir,
                "%s_%s_%.1f_%.1f" % (strategy, metric, mask_ratio, non_mask_ratio),
                data_type,
            )
        if not os.path.exists(out_path):
            os.makedirs(out_path)
        print(out_path)
        for i_f, f in enumerate(tqdm(all_files)):
            if os.path.exists(os.path.join(out_path, f)):
                continue
            time_start = timer()
            all_data = torch.load(os.path.join(input_dir, data_type, f))
            if num_workers > 1:
                with Pool(num_workers) as processor:
                    if mode == "compute_all_scores":
                        inputs = [
                            ([doc["text"] for doc in d["articles"]], i)
                            for i, d in enumerate(all_data)
                        ]
                        new_data = processor.starmap(
                            compute_all_scores_single_data, inputs
                        )

                    elif mode == "pretraining_data_with_score":
                        inputs = [
                            (
                                d,
                                tokenizer,
                                max_length_input,
                                max_length_output,
                                mask_ratio,
                                metric,
                                strategy,
                                non_mask_ratio,
                            )
                            for i, d in enumerate(all_data)
                        ]
                        new_data = processor.starmap(
                            process_single_data_with_scores, inputs
                        )
            else:
                new_data = []
                for i, d in enumerate(all_data):
                    if mode == "compute_all_scores":
                        new_data.append(
                            compute_all_scores_single_data(
                                [doc["text"] for doc in d["articles"]], i
                            )
                        )

                    elif mode == "pretraining_data_with_score":
                        new_data.append(
                            process_single_data_with_scores(
                                d,
                                tokenizer,
                                max_length_input,
                                max_length_output,
                                mask_ratio,
                                metric,
                                strategy,
                                non_mask_ratio,
                            )
                        )

            torch.save(new_data, os.path.join(out_path, f))
            time_end = timer()
            print("%s saved! %d/%d finish!" % (f, i_f + 1, len(all_files)))
            print("finish one file, time is %f" % (time_end - time_start))
            # break

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    # Gneral
    parser.add_argument("--max_length_input", default=4096, type=int)
    parser.add_argument("--max_length_output", default=1024, type=int)
    parser.add_argument("--mask_ratio", default=0.3, type=float)
    parser.add_argument("--non_mask_ratio", default=0.5, type=float)
    parser.add_argument("--primer_path", default="./PRIMER", type=str)

    parser.add_argument(
        "--output_dir",
        type=str,
        default="./data_with_scores/",
    )
    parser.add_argument(
        "--input_dir",
        type=str,
        default="./dataset/newshead/",
    )
    parser.add_argument(
        "--data_splits",
        default="train",
        type=str,
        nargs="+",
        choices=["train", "valid", "test"],
    )
    parser.add_argument("--num_worker", default=1, type=int)
    parser.add_argument(
        "--mode",
        choices=[
            "compute_all_scores",
            "pretraining_data_with_score",
        ],
        default="pretraining_data",
        type=str,
    )

    parser.add_argument(
        "--metric",
        choices=["pegasus_score", "pyramid_rouge"],
        default="pegasus_score",
        type=str,
    )
    parser.add_argument(
        "--strategy",
        choices=[
            "greedy",
            "greedy_entity_pyramid",
        ],
        default="greedy",
        type=str,
    )

    args = parser.parse_args()
    print(args)
    tokenizer = AutoTokenizer.from_pretrained(args.primer_path)

    output_dir = os.path.join(args.output_dir, "newshead")
    process_all_newshead(
        args.input_dir,
        tokenizer,
        args.max_length_input,
        args.max_length_output,
        args.mask_ratio,
        output_dir,
        args.data_splits,
        args.num_worker,
        args.mode,
        args.metric,
        args.strategy,
        args.non_mask_ratio,
    )

max_stars_repo_name: rubenwol/PRIMER

max_issues_repo_name: rubenwol/PRIMER

max_forks_repo_name: rubenwol/PRIMER


text:
import profilopy as pp
import matplotlib.pyplot as plt
from scipy.interpolate import UnivariateSpline
from scipy.ndimage.filters import median_filter

import numpy as np

run6 = {"startStep" : 4400,
        "endStep" : 4800,
       "windStatus" : "UPWIND",
       "force" : 15,
       "speed" : 50,
       'notes' : "Smoke, smoke, smoke !!!"}

run_angle = {"startStep" : 9960-30,
        "endStep" : 10080+30,
       "windStatus" : "DOWNWIND",
       "force" : 40,   #in T
       "speed" : 10,    #km/h
       'notes' : "angles"}

run13 = {"startStep" : 10500-120,
        "endStep" : 10560+30,
       "windStatus" : "UPWIND",
       "force" : 40,   #in T
       "speed" : 20,    #km/h
       'notes' : "smoke (medium)"}

analyzer_morning = pp.Analyzer(json_path="data/morning_select.json")

analyzer = pp.Analyzer(json_path='data/run6.json') #XXX
t0, tf = run6["startStep"], run6["endStep"] #XXX

keys = list(analyzer.data[0].keys())
keys = np.array([float(key) for key in keys])

keys = keys[(keys >= t0) & (keys <= tf)]

analyzer.preprocessor = pp.functions.preprocessing.preprocessor_1

analyzer.processor = pp.functions.processing.get_depth
values_depth = analyzer.get_depth_list(keys)

x,y =[], []
for key, value in zip(keys, values_depth):
       if not np.isnan(value):
              x.append(key)
              y.append(value)

y = median_filter(y, size=3)
spl = UnivariateSpline(x, y, s=6)
# print(spl(keys))

plt.figure(figsize=(12,6))
plt.scatter(keys, values_depth,label="Groove depth", s=2.5)
plt.plot(keys, spl(keys), label="Interpolating spline")
plt.xlabel("Time (s)", size=15)
plt.ylabel("Groove depth (mm)", size=15)
# plt.ylim(0, 2.5) #XXX
plt.xlim(t0, tf)
plt.grid(True)
plt.legend()

plt.savefig('results/run6_run.png')

max_stars_repo_name: nsarrazin/profilometer-analysis

max_issues_repo_name: nsarrazin/profilometer-analysis

max_forks_repo_name: nsarrazin/profilometer-analysis

reason: import numpy,from scipy


text:
import ITlib
import string
import sys
import numpy
import matplotlib.pyplot as plt
from scipy import signal

x = [0.1, 0.15, 0.28, 0.45, 0.75, 0.82, 0.88, 0.81, 0.55, 0.3, 0.14, -0.2, -0.5, -0.9, -0.8, -0.76, -0.75, -0.72, -0.5, -0.4, -0.33, -0.1, 0.1, 0.12, 0.22, 0.34, 0.35, 0.37]
x = numpy.array(x)
x = signal.resample(x, 102)
t = numpy.arange(0,x.shape[0])

step = 5;
x2 = x[0:x.shape[0]+step+step*2:step]
t2 = t[0:x.shape[0]+step+step*2:step]
plt.plot(t, x)
plt.stem(t2, x2, 'r')
plt.axis('off')

plt.show()


text:
import tensorflow as tf
import numpy as np
import torch

from tqdm import tqdm

from .post_model import PostModel

class PostModelTf():
    def __init__(self):
        self.post_model = PostModel()
        self.image_size = 32
        self.num_channels = 3
        self.num_labels = 10

    def predict(self, x):
        # print("===========================================")
        # print(type(x))
        # # print(x)
        #
        # x = tf.reshape(x, [-1, 32 * 32 * 3])
        # y = tf.reduce_max(x, axis=1)
        y = tf.py_function(func=self.py_predict, inp=[x], Tout=tf.float32)
        return y

    def py_predict(self, x):
        if not isinstance(x, np.ndarray):
            x = x.numpy()

        x = x.astype(np.float32)

        y_list = []
        batch_size = 64
        for i in range(0, len(x), batch_size):
            x_batch = x[i:i+batch_size, :, :, :]
            x_batch = torch.from_numpy(x_batch).cuda()
            # x_batch = x_batch.unsqueeze(0)
            # B x W x H x C -> B x C x W x H
            x_batch = x_batch.permute(0, 3, 1, 2)
            # if i == 0:
            #     self.post_model.update_post_model(x_batch)
            y_batch = self.post_model.forward(x_batch)
            y_list.append(y_batch.detach())
        y = torch.cat(y_list)
        # y = self.post_model.forward(x)
        y = y.detach().cpu().numpy()
        return y

max_stars_repo_name: JokerYan/ZOO-Attack

max_issues_repo_name: JokerYan/ZOO-Attack

max_forks_repo_name: JokerYan/ZOO-Attack


text:
"""
Author: Liangliang ZHENGT
Time: 2020/02/27
file: logging_helper.py

logging helper function for logging epochs and loss when training and testing
"""
import numpy as np
import time

def logging_helper(model, logfile, loss, char_er, epochs = 0, delta_time = 0, mode='train'):
    if mode != 'train' and mode != 'test' and mode != 'config' and mode != 'dev':
        raise TypeError('mode should be train or test or config.')

    if mode == 'config':
        with open(logfile, "a") as savefile:
            savefile.write(str(model.config)+'\n')
    elif mode == 'train':
        with open(logfile, "a") as savefile:
            savefile.write(str(time.strftime("%X %x %Z"))+'\n')
            savefile.write("Epoch:" +str(epochs + 1)+ "training error is "+ str(loss)+'\n')
            savefile.write("Epoch:" +str(epochs + 1)+ "training char error is "+ str(char_er)+'\n')
            savefile.write("Epoch:" +str(epochs + 1)+ "training time "+ str(delta_time)+' s\n')
    elif mode == 'test':
        logfile = 'TEST_'+logfile
        with open(logfile, "a") as savefile:
            savefile.write(str(model.config)+'\n')
            savefile.write(str(time.strftime("%X %x %Z"))+'\n')
            savefile.write("testing erro is :"+str(loss)+'\n')
            savefile.write("testing char erro is :"+str(char_er)+'\n')


text:
import time
import requests
import numpy as np
import pandas as pd

from tradingfeatures import apiBase

class binanceBase(apiBase):

    def __init__(self):
        super(binanceBase, self).__init__(
            name = 'binance',
            per_step = 1000,
            sleep = 0.7,
        )

        self.base_address = 'https://api.binance.com'
        self.address = '/api/v3/klines'
        self.start = 1484778000     # 1364778000
        self.limit = 1000
        self.default_columns = ['open', 'high', 'low', 'close', 'volume', 'quote_asset_vol',
                            'number_of_trades', 'taker_base_asset_vol', 'taker_sell_base_asset_vol',
                            'taker_quote_asset_vol']

        self.symbol_dict = {
            'btcusd': 'BTCUSDT',
            'ethusd': 'ETHUSDT',
            'ethbtc': 'ETHBTC',
            'ltcusd': 'LTCUSDT',
            'bchusd': 'BCHUSDT',
            'eosusd': 'EOSUSDT',
            'xrpusd': 'XRPUSDT',
        }

    def get(self,
            limit: int = None,
            symbol: str = None,
            address: str = None,
            query: dict = None,
            start: int = None,
            end: int = None,
            interval: str = '1h',
            columns: list = None,
            return_r: bool = False,
            ):

        address = address or self.address
        address = self.base_address + address
        symbol = symbol or 'btcusd'

        start, end, out_of_range = self.calc_start(limit, start, end, interval)
        if out_of_range:
            return self.get_hist(symbol=symbol, start=start, end=end, interval=interval)

        symbol = self.symbol_check(symbol)  # had to give raw symbol above, this has to be after

        if query is None:
            limit = self.limit if limit is None else limit
            start, end = self.ts_to_mts(start), self.ts_to_mts(end)

            query = {'symbol': symbol, 'interval': interval, 'startTime': start, 'endTime': end, 'limit': limit}

        r = requests.get(address, params=query)

        result = r.json()
        if len(result) == 0:
            return None

        df = pd.DataFrame(result, columns=['open_time', 'open', 'high', 'low', 'close', 'volume', 'close_time', 'quote_asset_vol',
                                    'number_of_trades', 'taker_base_asset_vol', 'taker_quote_asset_vol', 'ignore'])

        df = df.astype(float)
        df['taker_sell_base_asset_vol'] = df['volume'] - df['taker_base_asset_vol']
        df['timestamp'] = df['open_time'].div(1000).astype(int)
        df.pop('open_time')
        df.pop('close_time')
        df.pop('ignore')

        df = df.set_index('timestamp')
        df.index = df.index.astype(int)
        df = df.astype(float)

        if columns is not None:
            return df[columns]
        return df

    def get_hist(self, *args, **kwargs):
        # start = 1500000000
        return super(binanceBase, self).get_hist(
            *args, **kwargs
        )

    def _start_check(self, address, symbol):
        raise Exception('Cant sort oldest item')


text:
# Plotting tools

from riptide import TimeSeries
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import matplotlib.gridspec as gridspec
# Use non-interactive backend for plotting.
mpl.use('Agg')
# Enable use of LaTeX labels in plots.
plt.rcParams.update({"text.usetex": True, "text.latex.preamble": r"\usepackage{amsmath}"})
##########################################################################
# Scatter plot of periodicity candidates in the radio frequency vs. period diagram
def scatterplot_period_radiofreq(cand_periods, cand_freqs, cand_snrs, cand_flags, basename, min_period, max_period, min_freq, max_freq, plot_formats=['.png']):
    """
    Produce a scatter plot of periodicity detections in the radio frequency vs. trial period diagram.
    Candidate S/N values are shown on a color scale.
    Fundamental frequencies are displayed with a circular marker, whereas harmonics are indicated by a plus symbol.

    Parameters
    ----------
    cand_periods: 1D Numpy array
        Trial periods (s) of detected signals

    cand_freqs: 1D Numpy array
        Radio frequencies (MHz) corresponding to candidate detections

    cand_snrs: 1D Numpy array
        Matched filtering S/N values of detected candidates

    cand_flags: 1D Numpy array
        Harmonic flags ('F', 'S', or 'H') assigned to each candidate

    basename: string
        Output plot basename, including output path

    min_period: float
        Minimum trial period (s) shown on x-axis of plot

    max_period: float
        Maximum trial period (s) on x-axiss of plot

    min_freq: float
        Minimum radio frequency (MHz) limit on y-axis of plot

    max_freq: float
        Maximum radio frequency (MHz) shown on y-axis of plot

    plot_formats: list
        List of file formats for saving plot to disk
    """
    # Set up color map.
    cmap = plt.cm.cividis
    norm = mpl.colors.Normalize(vmin = np.min(cand_snrs), vmax = np.max(cand_snrs))
    # Begin plotting.
    fig1 = plt.figure(1, figsize=(7,6))
    marker_count = 0 # Keep track of the number of distinct marker shapes to plot.
    # Plot fundamental frequencies.
    f_idx = np.where(cand_flags=='F')[0]
    if len(f_idx) > 0:
        plt.scatter(x=cand_periods[f_idx], y=cand_freqs[f_idx], c=cand_snrs[f_idx], s=38, marker='o', cmap=cmap, norm=norm, label='Fundamental')
        marker_count += 1
    # Plot sub-harmonics.
    subharm_idx = np.where(cand_flags=='S')[0]
    if len(subharm_idx) > 0:
        plt.scatter(x=cand_periods[subharm_idx], y=cand_freqs[subharm_idx], c=cand_snrs[subharm_idx], s=38, marker='+', cmap=cmap, norm=norm, label='Subharmonic')
        marker_count += 1
    # Plot harmonics.
    harm_idx = np.where(cand_flags=='H')[0]
    if len(harm_idx) > 0:
        plt.scatter(x=cand_periods[harm_idx], y=cand_freqs[harm_idx], c=cand_snrs[harm_idx], s=38, marker='x', cmap=cmap, norm=norm, label='Harmonic')
        marker_count += 1
    # Set up colorbar.
    cbar = plt.colorbar(plt.cm.ScalarMappable(cmap = cmap, norm = norm))
    cbar.set_label('Matched filtering S/N', fontsize=16)
    # Set axes labels.
    plt.xlabel('Trial folding period (s)', fontsize=16)
    plt.ylabel('Radio frequency (MHz)', fontsize=16)
    # Set tick properties.
    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)
    # Set axes limits.
    plt.gca().set_xlim( (min_period, max_period) )
    plt.gca().set_ylim( (min_freq, max_freq) )
    # Show legend.
    if marker_count>1:
        leg = plt.legend(bbox_to_anchor=(0.0, -0.22, 1.1, 0.1), loc='center', ncol=3, fancybox=True, frameon=True,
                         borderpad=0.3, fontsize=14, handletextpad=0.2, columnspacing=2.0)
        leg.get_frame().set_edgecolor('silver')
    # Set tight layout for figure.
    plt.tight_layout()
    # Save plot to disk.
    for format in plot_formats:
        plt.savefig(basename + format)
    plt.close()
##########################################################################
# Candidate verification plot including periodogram, average pulse profile, and pulse stacks in phase-time plane.
def candverf_plot(period, bins, detrended_ts, periodograms, annotations, start_mjds, max_snr, periodaxis_log, plot_name, output_formats):
    '''
    Each row corresponds to a different data file.
    Left subplot in each row = Periodogram. Vertical dashed red line denotes folding period.
    Top right subplot in each row = Average pulse profile
    Bottom right subplot in each row = Phase-time diagram with flux density on the grayscale

    Parameters
    ----------
    period: float
        Candidate folding period (s)

    bins: integer
       No. of bins desired across folded profile

    detrended_ts: list
        List of riptide TimeSeries objects containing detrended, normalized time series from different data files

    periodograms: list
        List of riptide Periodogram objects precomputed for different data files

    annotations: list
        List of custom plot annotations. For example, indicate beam and ON/OFF labels

    start_mjds: list
        List of start MJDs (UTC) of data sets

    max_snr: float
        Maximum S/N limit to be shown on y-scale of periodogram plot

    periodaxis_log: Boolean
        Do you want to use a log scale for the period axis of the periodogram? (True/False)

    plot_name: string
        Name (including path but excluding extension) of output plot

    output_formats: list
        List of file extensions that specify output plot formats
    '''
    N_datafiles = len(detrended_ts) # No. of data files
    fig = plt.figure(figsize=(12,14))
    # Make outer gridspec.
    outer = gridspec.GridSpec(N_datafiles, 2, figure=fig, height_ratios=list(np.ones(N_datafiles)))
    # 1 row per data file
    for j in range(N_datafiles):
        # Left column = Periodogram
        gs1 = gridspec.GridSpecFromSubplotSpec(1, 1, subplot_spec = outer[j,0])
        ax1 = plt.subplot(gs1[0])
        ax1.plot(periodograms[j].periods, periodograms[j].snrs.max(axis=1), '-k')
        ax1.annotate(annotations[j], xy=(0.8,0.82), xycoords='axes fraction', fontsize=14)
        ax1.annotate('Start MJD (UTC) = %.4f'%  (start_mjds[j]), xy=(0.03,0.82), xycoords='axes fraction', fontsize=14)
        # Indicate candidate period with a light red vertical line.
        ax1.axvline(x=period, ymin=0.0, ymax=0.8, color='salmon', linestyle='--')
        # Set common S/N axes limits for all plots in the left column.
        ax1.set_ylim((-0.1, max_snr))
        # Use log-spaced period axis if specified.
        if periodaxis_log:
            ax1.set_xscale('log')
            # Show x-axis labels in decimal form.
            ax1.xaxis.set_major_formatter(ticker.FuncFormatter(lambda x,pos: ('${{:.{:1d}f}}$'.format(int(np.maximum(-np.log10(x),0)))).format(x)))
        # Set font size of axes labels.
        ax1.tick_params(axis='x', labelsize=14)
        ax1.tick_params(axis='y', labelsize=14)

        # Right column = Phase-time diagram and average profile
        phase_time = detrended_ts[j].fold(period, bins, subints=None) # Shape = (No. of full periods, No. of bins)
        profile = detrended_ts[j].fold(period, bins, subints=1)
        # Normalize profile to unit maximum.
        profile /= np.max(profile)
        phasebin_centers = (0.5 + np.arange(0, bins))/bins
        gs2 = gridspec.GridSpecFromSubplotSpec(2, 1, subplot_spec = outer[j,1], height_ratios = [1, 1], hspace = 0)
        # Average pulse profile
        ax20 = plt.subplot(gs2[0])
        ax20.plot(phasebin_centers, profile, '-k')
        ax20.set_ylabel(r'$\overline{S}$ (a.u.)', fontsize=16)
        ax20.tick_params(axis='y', labelsize=14)
        # Phase-time diagram
        ax21 = plt.subplot(gs2[1],sharex=ax20)
        ax21.imshow(phase_time, origin='lower', interpolation='nearest', aspect='auto',cmap='Greys',
                    extent=[phasebin_centers[0], phasebin_centers[-1], 0.0, len(phase_time)*period])
        ax21.set_ylabel(r'$t$ (s)', fontsize=16)
        ax21.tick_params(axis='x', labelsize=14)
        ax21.tick_params(axis='y', labelsize=14)
        ax21.set_xlim((0., 1.))
        # Hide xticks for all but the bottom row.
        if j!=(N_datafiles-1):
            ax1.set_xticks([])
            ax20.set_xticks([])
            ax21.set_xticks([])
        else:
            ax1.set_xlabel('Trial period (s)', fontsize=16)
            ax20.xaxis.set_visible(False)
            ax21.set_xlabel('Phase relative to start MJD', fontsize=16)
    #  Figure boundary formatting
    fig.subplots_adjust(left=0.07, right=0.98, bottom=0.07, top=0.98, hspace=0.1, wspace=0.2)
    fig.text(0.013, 0.55, 'Matched filtering S/N', va='center', rotation='vertical', fontsize=16)
    for format in output_formats:
        plt.savefig(plot_name+format)
    plt.close()
##########################################################################
# Plot phase-resolved dynamic spectrum.
def plot_phaseds(phaseresolved_ds, freqs_MHz, period, start_MJD, plot_name, plot_formats):
    """
    Outputs a grayscale imshow plot of a phase-resolved spectrum.

    Parameters
    ----------
    phaseresolved_ds: 2D Numpy array
        Data array of shape (Nchans, Nbins)

    freqs_MHz: 1D Numpy array
        Radio frequencies (MHz) corresponding to different channels

    period: float
        Folding period (s)

    start_MJD: float
        Start MJD (UTC) of observation

    plot_name: string
        Output plot basename, including output path

    plot_formats: list
        List of file formats for saving plot to disk
    """
    bins  = len(phaseresolved_ds[0]) # No. of phase bins
    phasebin_centers = (0.5 + np.arange(0, bins))/bins # Central phase of different bins
    # Set up figure
    fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(8,7), constrained_layout=True,
                             gridspec_kw={'height_ratios':[1,3], 'width_ratios':[3,1]})
    # Hide axes boundaries of top right plot.
    axes[0,1].axis('off')
    # Top left plot = Frequency-averaged profile
    axes[0,0].plot(phasebin_centers, np.mean(phaseresolved_ds, axis=0), '-k')
    axes[0,0].set_ylabel(r'$\overline{S}$ (arb. units)', fontsize=16)
    axes[0,0].tick_params(axis='y', labelsize=14)
    axes[0,0].set_xticklabels([])
    axes[0,0].set_xlim(0.0, 1.0)
    # Bottom left plot = Phase-resolved spectrum
    axes[1,0].imshow(phaseresolved_ds, aspect='auto', origin='lower', interpolation='nearest', cmap='Greys',
                     extent=[phasebin_centers[0], phasebin_centers[-1], freqs_MHz[0], freqs_MHz[-1]])
    axes[1,0].set_xlabel('Phase relative to MJD %.4f UTC'% (start_MJD), fontsize=16)
    axes[1,0].set_ylabel('Radio frequency (MHz)', fontsize=16)
    axes[1,0].tick_params(axis='x', labelsize=14)
    axes[1,0].tick_params(axis='y', labelsize=14)
    axes[1,0].set_xlim((0.0, 1.0))
    axes[1,0].set_ylim((freqs_MHz[0], freqs_MHz[-1]))
    # Bottom right plot = Profile spectrum
    axes[1,1].plot(np.mean(phaseresolved_ds,axis=1), freqs_MHz, '-k')
    axes[1,1].set_xlabel(r'$\langle S_{\nu} \rangle$ (arb. units)', fontsize=16)
    axes[1,1].set_yticklabels([])
    axes[1,1].set_ylim((freqs_MHz[0], freqs_MHz[-1]))
    axes[1,1].tick_params(axis='x', labelsize=14)
    # Overall figure settings
    fig.text(axes[0,1].get_position().x0+axes[0,1].margins()[0]+0.01, axes[0,1].get_position().y0+axes[0,1].margins()[1]+0.06,
             '$P = %.5f$ s'% (period), fontsize=16)
    fig.text(axes[0,1].get_position().x0+axes[0,1].margins()[0]+0.01, axes[0,1].get_position().y0+axes[0,1].margins()[1]+0.01,
             '$N_{\mathrm{bins}} = %d$'% (bins), fontsize=16)
    # Save plot to disk.
    for format in plot_formats:
        plt.savefig(plot_name+format)
    plt.close()

max_stars_repo_name: UCBerkeleySETI/blipss

max_issues_repo_name: UCBerkeleySETI/blipss

max_forks_repo_name: UCBerkeleySETI/blipss