text:
import torch
import torch.nn as nn
import cv2
import os
from darknet import DarkNet
from util import *
import numpy as np
import argparse
import time
from torch.autograd import Variable
import pickle
import pandas
import random

def cmd_line():
    cmd = argparse.ArgumentParser(description="YOLO Detector")

    cmd.add_argument('--images', dest='imgs', default='../../data/imgs', type=str)
    cmd.add_argument('--det', dest='det', default='../../data/det', type=str)
    cmd.add_argument('--bs', dest='bs', default=1)
    cmd.add_argument('--nms', dest='nms', default=.4)
    cmd.add_argument('--conf', dest='conf', default=.5)
    cmd.add_argument('--cfg', dest='cfg', default='cfg/yolov3.cfg', type=str)
    #cmd.add_argument('--cfg', dest='cfg', default='src/detector/cfg/yolov3.cfg', type=str)
    cmd.add_argument('--reso', dest='reso', default="416", type=str)
    cmd.add_argument('--weights', dest='weights', default='cfg/yolov3.weights', type=str)
    #cmd.add_argument('--weights', dest='weights', default='src/detector/cfg/yolov3.weights', type=str)

    return cmd.parse_args()

def draw_boxes(x, results):

    br_corner = tuple(x[3:5].int())
    tl_corner = tuple(x[1:3].int())
    cl = int(x[-1])
    image = results[int(x[0])]
    label = '{}'.format(classes[cl])
    colour = random.choice(box_colours)
    cv2.rectangle(image, tl_corner, br_corner, colour, 1)
    text_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_DUPLEX, 1, 1)[0]
    br_corner = text_size[0] + tl_corner[0] + 3, text_size[1] + tl_corner[1] + 4
    cv2.rectangle(image, tl_corner, br_corner, colour, -1)
    cv2.putText(image, label, (tl_corner[0], text_size[1] + tl_corner[1] + 4), cv2.FONT_HERSHEY_DUPLEX, 1, [255, 255, 255], 1)

    return image

try:
    classes = load_classes('../../data/coco.names')
except FileNotFoundError:
    classes = load_classes('data/coco.names')

no_classes = len(classes)

params = cmd_line()
imgs = params.imgs
nms = float(params.nms)
batch_size = int(params.bs)
conf = float(params.conf)
gpu = torch.cuda.is_available()

model = DarkNet(params.cfg) #setup
model.load_weights(params.weights)
print("network loaded")
if gpu:
    model.cuda()

model.network_info['height'] = params.reso
input_dim = int(model.network_info['height'])
if ((input_dim % 32 != 0) or (input_dim <= 32)):
    raise Exception('invalid image size')

model.eval() #eval mode
read_s = time.time() #need to note time

try: #get image locations in list
    image_paths = [os.path.join(os.path.realpath('.'), imgs, img) for img in os.listdir(imgs)]
except NotADirectoryError:
    image_paths = []
    image_paths.append(os.path.join(os.path.realpath('.'), imgs))
except FileNotFoundError:
    print('no file/dir with name: ', imgs)
    exit()

if not os.path.exists(params.det): #create if doesn't exist
    os.makedirs(params.det)

load_s = time.time()
loaded_imgs = [cv2.imread(im) for im in image_paths] #load as numpy arr
# keep list of original image dims
image_batches = list(map(convert_image_to_input, loaded_imgs, [input_dim for im in range(len(image_paths))])) #pytorch vars for imgs
image_dims = [(im.shape[1], im.shape[0]) for im in loaded_imgs]
image_dims = torch.FloatTensor(image_dims).repeat(1,2)
if gpu:
    image_dims = image_dims.cuda()

left = 0
if (len(image_dims) % batch_size):
    left = 1
if batch_size != 1:
    no_batches = len(image_paths)// batch_size + left
    image_batches = [torch.cat((image_batches[i*batch_size: min((i+1)*batch_size, len(image_batches))])) for i in range(no_batches)]

det_loop_s = time.time()
output_init = False

for i, batch in enumerate(image_batches):
    load_start = time.time()
    if gpu:
        batch = batch.cuda()
    with torch.no_grad():
        pred = model(Variable(batch), gpu)
    pred = results(pred, no_classes, conf, nms)
    load_end = time.time()

    if type(pred) == int:
        for img_no, img in enumerate(image_paths[i*batch_size : min((i+1)*batch_size, len(image_paths))]):
            img_id = img_no + batch_size*i
            print("time taken to detect objects in: {:15s} : {:5.2f} s".format(img.split("/")[-1], (load_end - load_start)/batch_size))
            print("{:15s} {:s} \n".format("objects detected: ", " "))
        continue

    pred[:,0] = pred[:,0] + i*batch_size
    if not output_init:
        out = pred
        output_init = True
    else:
        out = torch.cat((out, pred))

    for img_no, img in enumerate(image_paths[batch_size*i : min((i+1)*batch_size, len(image_paths))]):
        img_id = img_no+ batch_size*img_no
        objects = [classes[int(x[-1])] for x in out if int(x[0]) == img_id]
        print("time taken to detect objects in: {:15s} : {:5.2f} s".format(img.split("/")[-1], (load_end - load_start)/batch_size))
        print("{:15s} {:s} \n".format("objects detected: ", " ".join(objects)))

    if gpu:
        torch.cuda.synchronize() #prevent async. calling

try:
    out
except NameError:
    print("no detections")
    exit()

image_dims = torch.index_select(image_dims, 0, out[:,0].long())

scale_factor = torch.min(input_dim/image_dims, 1)[0].view(-1, 1) #make coords of bbox conform to image on padded area
out[:,[1,3]] -= (input_dim - scale_factor * image_dims[:,0].view(-1, 1))/2
out[:,[2,4]] -= (input_dim - scale_factor * image_dims[:,1].view(-1, 1))/2
out[:,1:5] /= scale_factor #undo resize image scaling

for coord in range (out.shape[0]):
    out[coord, [1,3]] = torch.clamp(out[coord, [1,3]], 0, image_dims[coord,0])
    out[coord, [2,4]] = torch.clamp(out[coord, [2,4]], 0, image_dims[coord,1])

class_load = time.time()

try:
    box_colours = pickle.load(open('colours.p', 'rb'))
except FileNotFoundError:
    box_colours = pickle.load(open('src/detector/colours.p', 'rb'))
except:
    print('colours not found')
    exit()

draw_time = time.time()

list(map(lambda x : draw_boxes(x, loaded_imgs), out))
det_names = pandas.Series(image_paths).apply(lambda x: "{}/detection_{}".format(params.det, x.split('/')[-1]))

list(map(cv2.imwrite, det_names, loaded_imgs))
end = time.time()

print("{:20s}: {:3.3f}".format("Address read time", load_s - read_s))
print("{:20s}: {:3.3f}".format("Batch loading time", det_loop_s - load_s))
print("{:40s}".format("Detected " + str(len(image_paths)) + " images"))
print("{:20s}: {:3.3f}".format("", end - draw_time))

torch.cuda.empty_cache()


text:
import functools
import operator

def sample_mean(x):
    total = functools.reduce(operator.add,x)
    # this
    # for item in x:
    #     total+=item
    ave = total/len(x)
    return ave

def sample_variance(x):
    ave = sample_mean(x)
    total = 0
    for item in x:
        total += (item-ave)*(item-ave)
    variance = total/len(x)
    return variance

def mat_add(A,B):
    try:
        C=[]
        if len(B)!=len(A):# row # comparison
            raise
        i_row = 0
        for row_A in A:
            row_B = B[i_row]
            if len(row_B)!=len(row_A):# col # comparison
                raise
            C.append(list(map(operator.add, row_A, row_B)))
            i_row += 1
        return C
    except Exception as e:
        return None

def fisher_info(mean, var):
    ele00=1/(var*var)
    ele11=1/(2*var**4)
    # print(f"[[{ele00} 0]")
    # print(f"[0 {ele11}]]")
    return [[ele00, 0],[0, ele11]]

if __name__=="__main__":
    x = [.2, -.1, -1.9, -.4, -1.8]
    # print(sample_mean(x))
    # print(sample_variance(x))
    # fisher_info(0,1)
    # print(mat_add([[1.4, 0],[0, 1.3]],[[1, 0],[0, 1]]))
    import numpy as np
    I=np.array(fisher_info(0,1))
    A=[sample_mean(x), sample_variance(x)-1]
    At=np.transpose(A)
    print(len(x)*At.dot(I).dot(A))
    print(5*(.64+.114*.228))

ext: py

max_stars_repo_name: easai/stat

max_issues_repo_name: easai/stat

max_forks_repo_name: easai/stat

reason: import numpy


text:
#!/usr/bin/python3

"""
This file including the operation of MBA expression by the string-related operation.
"""

import numpy as np
import re
import sys
import traceback
import z3

def postfix(itemString):
    """transform infixExpre into postfixExpre
    Algorithm:
        step1: if operator, stack in;
        step2: if "(", stack in.
        step3: if variable, pop out the all continued unary operator until other operator or "("
        step4: if ")", pop out all operators until "(", then pop all continued unary operator.
        step5: goto step1.
    Arg:
        itemString: bitwise expression string persented in infix.
    Return:
        itemStr: expression string persented in postfix.
    """
    itemStr = ""
    boperatorList = ["&", "|", "^"]
    uoperator = "~"
    opeList = []

    for (idx, char) in enumerate(itemString):
         #open parenthesis, stack it
        if char == "(":
            opeList.append(char)
        #binary operatork, stack it
        elif char in boperatorList:
            opeList.append(char)
        #unary operator
        elif uoperator in char:
            opeList.append(char)
        #closed parenthesis, pop out the operator to string
        elif char == ")":
            while(opeList and opeList[-1] != "("):
                itemStr += opeList[-1]
                opeList.pop()
            if opeList and opeList[-1] != "(":
                print("error!")
                sys.exit(0)
            #open parenthesis found
            opeList.pop()
            #unary operator found before open parenthesis
            while(opeList and opeList[-1] == "~"):
                itemStr += opeList[-1]
                opeList.pop()
        #variable name found
        else:
            itemStr += char
            #top of stack is unary operator
            while(opeList and opeList[-1] in uoperator):
                itemStr += opeList[-1]
                opeList.pop()

    if len(opeList) > 1:
        print("error in function postfix!")
        sys.exit(0)
    #have one operator without parenthesis
    elif len(opeList):
        itemStr += opeList[0]

    return itemStr

def postfix_cal(itemString, vnumber=0):
    """calculate the result of the expression string.
    Args:
        itemString: bitwise expression string persented in postfix.
        vnumber: the number of variables.
    Returns:
        result: the result list of the expression string.
    Raises:
        through out SystemExit exception.
    """
    if vnumber == 1:
        x = [0,1]
    elif vnumber == 2:
        y = [0, 1] * 2
        x = [0, 0, 1, 1]
    elif vnumber == 3:
        y = [0, 1] * 4
        x = [0, 0, 1, 1] * 2
        z = [0] * 4 + [1] * 4
    elif vnumber == 4:
        y = [0, 1] * 8
        x = [0, 0, 1, 1] * 4
        z = [0] * 4 + [1] * 4 + [0] * 4 + [1] * 4
        t = [0] * 8 + [1] * 8
    else:
        print("please passing correct argument of the number of variable:1, 2, 3, 4")
        traceback.print_stack()
        sys.exit(0)
    variableList = ["x", "y", "z", "t"]
    binaryOperator = ["&", "|", "^"]
    unaryOperator = ["~"]

    stack = []
    for c in itemString:
        if c in variableList:
            stack.insert(0, eval(c))
        elif c in unaryOperator:
            operation = stack.pop(0)
            res = np.invert(operation)
            res = res % 2
            stack.insert(0, res)
        elif c in binaryOperator:
            operation1 = stack.pop(0)
            operation2 = stack.pop(0)
            if c == "&":
                res = np.bitwise_and(operation1, operation2)
                stack.insert(0, res)
            elif c == "|":
                res = np.bitwise_or(operation1, operation2)
                stack.insert(0, res)
            elif c == "^":
                res = np.bitwise_xor(operation1, operation2)
                stack.insert(0, res)

    if len(stack) > 1:
        print("error in function of postfix_cal!")
    else:
        result = stack[0]
        if isinstance(result, list):
            pass
        else:
            result = result.tolist()

        return result

def verify_mba_unsat(leftExpre, rightExpre, bitnumber=2):
    """check the relaion whether the left expression is euqal to the right expression.
    Args:
        leftExpre: left expression.
        rightExpre: right expression.
        bitnumber: the number of the bits of the variable.
    Returns:
        True: equation.
        False: unequal.
    Raises:
        None.
    """
    x,y,z,t,a,b,c,d,e,f = z3.BitVecs("x y z t a b c d e f", bitnumber)

    leftEval = eval(leftExpre)
    rightEval = eval(rightExpre)

    solver = z3.Solver()
    solver.add(leftEval != rightEval)
    result = solver.check()

    if str(result) != "unsat":
        return False
    else:
        return True

def truthtable_term_list(termList, vnumber=0):
    """obtain the result vector based on the term list on mba expression.
    Args:
        termList: mba expression string splited by the + or - operator.
        vnumber: the number of variable.
    Returns:
        result: the result vector presenting by np.array format of the mba expression.
    Raises:
        through out SystemExit exception.
    """
    #function call error
    if not vnumber:
        print("please passing argument on the number of variable")
        traceback.print_stack()
        sys.exit(0)
    #empty expression
    if not termList:
        return [0] * 2**vnumber
    result = np.zeros((2**vnumber,), dtype=np.int)
    for item in termList:
        itemList = re.split("\*", item)
        #only bitwise expression or constant
        if len(itemList) == 1:
            bitwiseExpre = itemList[0]
            if bool(re.search("\d", bitwiseExpre)):
                #constant
                result += np.array([int(bitwiseExpre) * -1] * 2**vnumber)
            else:
                #only bitwise expression
                if bitwiseExpre[0] == "+":
                    result += np.array(truthtable_bitwise(bitwiseExpre[1:], vnumber))
                elif bitwiseExpre[0] == "-":
                    result -= np.array(truthtable_bitwise(bitwiseExpre[1:], vnumber))
                else:
                    result += np.array(truthtable_bitwise(bitwiseExpre, vnumber))
        #coefficient and bitwise
        elif len(itemList) == 2:
            coefficient = int(itemList[0])
            bitwiseExpre = itemList[1]
            result += coefficient * np.array(truthtable_bitwise(bitwiseExpre, vnumber))

    return result.tolist()

def truthtable_bitwise(bitExpre, vnumber):
    """generate the truth table of a bitwise expression.
    Args:
        bitExpre: a bitwise expression.
        vnumber: the number of variables in a expression.
    Returns:
        truthList: a truth table on a format of list.
    """
    truthList = postfix_cal(postfix(bitExpre), vnumber)

    return truthList

def truthtable_expression(expreStr, vnumber):
    """generate the truth table on a linear MBA expression.
    Args:
        expreStr: a linear MBA expression.
        vnumber: the number of variables in a expression.
    Returns:
        truthtalbeList: a truth table on a format of list.
    """
    termList = expression_2_term(expreStr)

    #truth table of bitwise expression
    truthtableList = truthtable_term_list(termList, vnumber)

    return truthtableList

def expression_2_term(expreStr):
    """generate the truth table on a linear MBA expression.
    Args:
        expreStr: a linear MBA expression.
    Returns:
        termList: a list of terms on bitwise expression.
        constantList: a list of constant term of the linear MBA expression.
    """
    itemList = re.split("([\+-])", expreStr)
    item0 = itemList[0]
    termList = []
    constantList = []
    if item0 != "":
        itemList.insert(0, "")
    for (idx, item) in enumerate(itemList):
        if item == "+" or item == "-" or item == "":
            continue
        #bitwise term
        elif re.search("\w+", item):
            term = itemList[idx - 1] + itemList[idx]
            termList.append(term)
        #constant term
        elif re.search("\d+", item):
            term = itemList[idx - 1] + itemList[idx]
            constantList.append(term)
        else:
            print("This is something wrong in mba expression.")
            traceback.print_stack()
            sys.exit(0)

    return termList +  constantList

def generate_coe_bit(mbatermList):
    """split the one term into pair, [coe, bit]
    Arg:
        mbatermList: one list of terms.
    Return:
        coeBitList: one list of pair [coe, bit]
    """
    coeBitList = []
    for term in mbatermList:
        itemList = re.split("\*", term)
        maycoe= itemList[0]
        #not coefficient
        if not bool(re.search("\d", maycoe)):
            bit = itemList[0]
            bit = bit.replace("+", "")
            if "-" not in term:
                coe = 1
            else:
                coe = -1
                bit = bit.replace("-", "")
            coeBitList.append([coe, bit])
        #multi bitwise expression
        elif len(itemList) >= 2:
            #1 is for the first time synbol
            bit = term[len(maycoe)+1:]
            coeBitList.append([maycoe, bit])
        #only constant
        elif len(itemList) == 1 and bool(re.search("\d", maycoe)):
            coeValue = int(maycoe) * -1
            coeBitList.append([str(coeValue), "~(x&~x)"])

        else:
            print("error in function of generate_coe_bit")
            exit(0)

    return coeBitList

def combine_term(mbaExpre):
    """combining like terms of the mba expression
    Args:
        mbaExpre: the mba expression.
    Return:
        newmbaExpre: new mba expression have combined like terms.
    """
    #get pair of coefficient and bitwise on the mba expression
    termList = expression_2_term(mbaExpre)
    coeBitList = generate_coe_bit(termList)

    #firstly sort the list
    coeBitList = sorted(coeBitList, key=lambda x: x[1])
    #walk the list and combine like terms
    newcoeBitList = []
    itemList = coeBitList[0]
    for item in coeBitList[1:]:
        if item[1] != itemList[1]:
            #delete 0 term
            if itemList[0] != "0":
                newcoeBitList.append(itemList)
            itemList = item
        else:
            coe1 = itemList[0]
            coe2 = item[0]
            coe = eval(coe1) + eval(coe2)
            itemList[0] = str(coe)
    #handle the last one term
    if newcoeBitList[-1][1] != itemList[1]:
        newcoeBitList.append(itemList)
    #coefficent and bitwise expression to terms
    newtermList = []
    for item in newcoeBitList:
        coe = item[0]
        bit = item[1]
        if coe[0] == "+" or coe[0] == "-":
            term = "{coe}*{bit}".format(coe=coe, bit=bit)
        else:
            coe = "+" + coe
            term = "{coe}*{bit}".format(coe=coe, bit=bit)
        newtermList.append(term)
    #terms to expression
    newmbaExpre = "".join(newtermList)
    if newmbaExpre[0] == "+":
        newmbaExpre = newmbaExpre[1:]
    #verification
    z3res = verify_mba_unsat(mbaExpre, newmbaExpre)
    if not z3res:
        print("error in merge_bitwise!")
        sys.exit(0)

    return newmbaExpre

def addMBA(mbaExpre1, mbaExpre2):
    """two mba expression addition.
    Args:
        mbaExpre1: one MBA expression.
        mbaExpre2: another one MBA expression.
    Return:
        mbaExpre: the mba expression by addition of mbaExpre1 and mbaExpre2
    """
    #get the coefficient and related bitwise
    mbaterm1List = expression_2_term(mbaExpre1)
    coeBitList1 = generate_coe_bit(mbaterm1List)
    coeBitDict1 = {}
    for item in coeBitList1:
        coeBitDict1[item[1]] = item[0]
    mbaterm2List = expression_2_term(mbaExpre2)
    coeBitList2 = generate_coe_bit(mbaterm2List)
    coeBitDict2 = {}
    for item in coeBitList2:
        coeBitDict2[item[1]] = item[0]

    commonKey = set(coeBitDict1.keys()) & set(coeBitDict2.keys())
    for key in commonKey:
        coe1 = coeBitDict1[key]
        coe2 = coeBitDict2[key]
        res = eval(coe1) + eval(coe2)
        if not res:
            coeBitDict1.pop(key)
        else:
            coeBitDict1[key] = str(res)
        coeBitDict2.pop(key)
    #the last pair of coefficient and bitwise
    coeBitDict = coeBitDict1
    coeBitDict.update(coeBitDict2)
    termList = []
    for item in coeBitDict.items():
        term = item[1] + "*" + item[0]
        if term[0] == "-" or term[0] == "+":
            termList.append(term)
        else:
            term = "+" + term
            termList.append(term)
    #construct the mba expression
    mbaExpre = "".join(termList)
    if mbaExpre[0] == "+":
        mbaExpre = mbaExpre[1:]

    #verification
    oriExpre = ""
    if mbaExpre2[0] == "+" or mbaExpre2[0] == "-":
        oriExpre = mbaExpre1 + mbaExpre2
    else:
        oriExpre = mbaExpre1 + "+" + mbaExpre2
    z3res = verify_mba_unsat(oriExpre, mbaExpre)
    if not z3res:
        print("error in addMBA!")
        sys.exit(0)

    return mbaExpre

def variable_list(expreStr):
    """get the set of variable of expression.
    Args:
        expreStr: the mba expression string.
    Return:
        variableList: the list of variables.
    """
    varSet = set(expreStr)
    variableList = []
    for i in varSet:
        #the variable name
        if i in ["x", "y", "z", "t", "a", "b", "c", "d", "e", "f"]:
            variableList.append(i)

    return variableList


text:
import numpy as np
import pandas as pd

from sklearn.feature_extraction.text import CountVectorizer
from IPython.display import display, Markdown, HTML

def ambiguous_examples_analysis(workspace_pd, lang_util, threshold=0.7):
    """
    Analyze the test workspace and find out similar utterances that belongs to different intent
    :param workspace_pd: pandas dataframe in format of [utterance,label]
    :param threshold: cut off for similarity score
    :return: pands dataframe in format of ['Intent1', 'Utterance1', 'Intent2', 'Utterance2',
                                           'similarity score']
    """
    # first create the feature matrix
    vectorizer = CountVectorizer(ngram_range=(1, 2), tokenizer=lang_util.tokenize)
    workspace_bow = vectorizer.fit_transform(workspace_pd["utterance"]).todense()
    cos_sim_score_matrix = _calculate_cosine_similarity(workspace_bow)

    # remove the lower triangle of the matrix and apply threshold
    similar_utterance_index = np.argwhere(
        (cos_sim_score_matrix - np.tril(cos_sim_score_matrix)) > threshold
    )
    similar_utterance_pd = pd.DataFrame(
        columns=["Intent1", "Utterance1", "Intent2", "Utterance2", "similarity score"]
    )

    for index in similar_utterance_index:
        if (
            workspace_pd["intent"].iloc[index[0]]
            != workspace_pd["intent"].iloc[index[1]]
        ):
            intent1 = workspace_pd["intent"].iloc[index[0]]
            utterance1 = workspace_pd["utterance"].iloc[index[0]]
            intent2 = workspace_pd["intent"].iloc[index[1]]
            utterance2 = workspace_pd["utterance"].iloc[index[1]]
            score = cos_sim_score_matrix[index[0], index[1]]
            temp_pd = pd.DataFrame(
                {
                    "Intent1": [intent1],
                    "Utterance1": [utterance1],
                    "Intent2": [intent2],
                    "Utterance2": [utterance2],
                    "similarity score": [score],
                }
            )
            similar_utterance_pd = similar_utterance_pd.append(
                temp_pd, ignore_index=True
            )

    if not similar_utterance_pd.empty:
        with pd.option_context("max_colwidth", 250):
            display(
                HTML(
                    similar_utterance_pd.sort_values(
                        by=["similarity score"], ascending=False
                    ).to_html(index=False)
                )
            )
    else:
        display(Markdown("### There are no similar utterances within different Intent"))

    return similar_utterance_pd

def _calculate_cosine_similarity(workspace_bow):
    """
    Given bow representation of the workspace utterance, calculate cosine similarity score
    :param workspace_bow: dense representation of BOW of workspace utterances
    :return: cosine_similarity_matrix
    """
    # normalized and calculate cosine similarity
    workspace_bow = workspace_bow / np.linalg.norm(workspace_bow, axis=1, keepdims=True)
    cosine_similarity_matrix = workspace_bow.dot(np.transpose(workspace_bow))
    return cosine_similarity_matrix


text:
# 导入库
import jieba.posseg
import jieba.analyse
from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer

# 正则包
# 自然语言处理包
import jieba.analyse

# html 包
import html

# 导入库
import pandas as pd
import xlrd
import re  # 正则表达式库
import collections  # 词频统计库
import numpy as np  # numpy数据处理库
import jieba  # 结巴分词
import wordcloud  # 词云展示库
from PIL import Image  # 图像处理库
import matplotlib.pyplot as plt  # 图像展示库
import datetime
import sys, codecs
import jieba.posseg

print("my gf is xiaocong YU!")

def extract_keyword(content):  # 提取关键词
    # 正则过滤 html 标签
    re_exp = re.compile(r"(<style>.*?</style>)|(<[^>]+>)", re.S)
    content = re_exp.sub(" ", content)
    # html 转义符实体化
    content = html.unescape(content)
    # 切割
    seg = [i for i in jieba.cut(content, cut_all=True) if i != ""]
    # 提取关键词
    keywords = jieba.analyse.extract_tags("|".join(seg), topK=200, withWeight=False)
    return keywords

excelFile1 = r"/Users/19723/Desktop/今日话题 - 雪球.xlsx"
# excelFile2 = r'/Users/19723/Desktop/财联社A股24小时电报-上市公司动态-今日股市行情报道.xlsx'
# excelFile3 = r'/Users/19723/Desktop/7_24小时全球财经直播_同花顺财经.xlsx'
df1 = pd.DataFrame(pd.read_excel(excelFile1))
# df2 = pd.DataFrame(pd.read_excel(excelFile2))
# df3 = pd.DataFrame(pd.read_excel(excelFile3))
# df4 = pd.concat([df1,df2,df3],axis=0)
df5 = df1.sort_values(
    ["字段"], axis=0, ascending=True, inplace=False, kind="quicksort", na_position="last"
)
df5.to_excel(datetime.datetime.now().strftime("%m-%d") + "每日财经.xls")
df5.to_csv(datetime.datetime.now().strftime("%m-%d") + "每日财经.csv")

data = xlrd.open_workbook(datetime.datetime.now().strftime("%m-%d") + "每日财经.xls")
st = data.sheets()[0]  # 读取第一个表
st1 = st.col_values(3)  # 读取第二列

t = open(datetime.datetime.now().strftime("%m-%d") + "每日财经.txt", "w", encoding="UTF-8")
for i in st1:
    t.write(str(i) + "\n")
t.close()

# 读取文件
fn = open(
    datetime.datetime.now().strftime("%m-%d") + "每日财经.txt", encoding="UTF-8"
)  # 打开文件
string_data = fn.read()  # 读出整个文件
fn.close()  # 关闭文件

# 文本预处理
pattern = re.compile(u'\t|\n|\.|-|:|;|\)|\(|\?|"')  # 定义正则表达式匹配模式
string_data = re.sub(pattern, "", string_data)  # 将符合模式的字符去除

# 文本分词
seg_list_exact = jieba.cut(string_data, cut_all=False)  # 精确模式分词
object_list = []
remove_words = [
    u"的",
    u"，",
    u"和",
    u"是",
    u"随着",
    u"对于",
    u"对",
    u"等",
    u"能",
    u"都",
    u"。",
    u" ",
    u"、",
    u"中",
    u"在",
    u"了",
    u"通常",
    u"如果",
    u"我们",
    u"需要",
    u"财联社",
    u"日讯",
    u"'",
    u",",
    u"【",
    u"】",
    u"联社",
    u":",
    u"“",
    u"”",
    u"1",
    u"2",
    u"3",
    u"4",
    u"5",
    u"6",
    u"7",
    u"8",
    u"9",
    u"（",
    u"）",
    u"《",
    u"》",
    u"、",
    u"\\",
    u";",
    u"并",
    u"至",
    u"称",
    u"于",
    u"要求",
    u"已",
    u"企业",
    u"行业",
    u"预计",
    u"产品",
    u"其",
    u"较",
    u"未来",
    u"2020",
    u"有",
    u"公司",
    u"表示",
    u"进行",
    u"：",
    u"·",
    u"/",
    u"据",
    u"相关",
    u"与",
    u"将",
    u"为",
    u"月",
    u"目前",
    u"日",
    u"后来",
    u"·",
    u"；",
    u"/",
    u"但",
    u"个",
    u"上",
    u"及",
    u"下",
    u"部分",
    u"后",
    u"还",
    u"一些",
    u"从",
    u"也",
    u"或",
    u"已经",
    u"这",
    u"就",
    u"元",
    u"其中",
    u"可",
    u"向",
    u"超",
    u"报",
    u"年",
    u"正在",
    u"该",
    u"只",
    u"财",
    u"还",
    u"前",
    u"内",
    u"时",
    u"以及",
    u"例",
    u"可能",
    u"近日",
    u"不",
    u"内",
    u"自",
    u"被",
    u"均",
    u"㎡",
    u"型",
    u"要",
    u"均",
    u"以",
    u"报道",
    u"今日",
    u"累计",
    u"此前",
    u"做",
    u"要",
    u"而",
    u"亿美元",
    u"亿元",
    u"会",
    u"以来",
    u"涉及",
    u"成为",
    u"不再",
    u"到",
    u"收到",
    u"截止",
    u"正",
    u"没有",
    u"经",
    u"点",
    u"万亿",
    u"n",
    u"来",
    u"当前",
    u"消息",
    u"仍",
    u"10",
    u"11",
    u"12",
    u"13",
    u"14",
    u"15",
    u"16",
    u"17",
    u"18",
    u"19",
    u"万亿元",
    u"20",
    u"21",
    u"22",
    u"23",
    u"24",
    u"25",
    u"26",
    u"27",
    u"28",
    u"29",
    u"30",
    u"31",
    u"指出",
    u"更",
    u"天",
    u"占",
    u"通过",
    u"认为",
    u"发现",
    u"同时",
    u"认为",
    u"家",
    u"桶",
    u"—",
    u"次",
    u"且",
    u"因",
    u"本",
    u"同花顺",
    u"雪",
    u"球网",
    u"市场",
    u"同比",
    u"2021",
    "*",
]
# 自定义去除词库

for word in seg_list_exact:  # 循环读出每个分词
    if word not in remove_words:  # 如果不在去除词库中
        object_list.append(word)  # 分词追加到列表

# 词频统计
word_counts = collections.Counter(object_list)  # 对分词做词频统计
word_counts_top200 = word_counts.most_common(100)  # 获取前10最高频的词
print(word_counts_top200)  # 输出检查

# 词频展示
mask = np.array(Image.open("/Users/19723/Desktop/Everyday财经/SOURCE/gold.png"))  # 定义词频背景
wc = wordcloud.WordCloud(
    background_color="white",  # 设置背景颜色
    font_path="/Users/19723/Desktop/Everyday财经/SOURCE/Hiragino Sans GB.ttc",  # 设置字体格式
    mask=mask,  # 设置背景图
    max_words=100,  # 最多显示词数
    max_font_size=100,  # 字体最大值
    scale=50,  # 调整图片清晰度，值越大越清楚
)

wc.generate_from_frequencies(word_counts)  # 从字典生成词云
image_colors = wordcloud.ImageColorGenerator(mask)  # 从背景图建立颜色方案
wc.recolor(color_func=image_colors)  # 将词云颜色设置为背景图方案
wc.to_file("词频.png")  # 将图片输出为文件
plt.imshow(wc)  # 显示词云
plt.axis("off")  # 关闭坐标轴
plt.show()  # 显示图像

df = pd.read_excel("/Users/19723/Desktop/今日话题 - 雪球.xlsx", header=0)
df.columns = ["id", "title", "abstract"]
df.to_excel("/Users/19723/Desktop/今日话题 - 雪球.xlsx")

# !/usr/bin/python
# coding=utf-8
# 采用TF-IDF方法提取文本关键词
# http://scikit-learn.org/stable/modules/feature_extraction.html#tfidf-term-weighting

"""
       TF-IDF权重：
           1、CountVectorizer 构建词频矩阵
           2、TfidfTransformer 构建tfidf权值计算
           3、文本的关键字
           4、对应的tfidf矩阵
"""

# 数据预处理操作：分词，去停用词，词性筛选
def dataPrepos(text, stopkey):
    l = []
    pos = ["n", "nz", "v", "vd", "vn", "l", "a", "d"]  # 定义选取的词性
    seg = jieba.posseg.cut(text)  # 分词
    for i in seg:
        if i.word not in stopkey and i.flag in pos:  # 去停用词 + 词性筛选
            l.append(i.word)
    return l

# tf-idf获取文本top10关键词
def getKeywords_tfidf(data, stopkey, topK):
    idList, titleList, abstractList = data["id"], data["title"], data["abstract"]
    corpus = []  # 将所有文档输出到一个list中，一行就是一个文档
    for index in range(len(idList)):
        text = "%s。%s" % (titleList[index], abstractList[index])  # 拼接标题和摘要
        text = dataPrepos(text, stopkey)  # 文本预处理
        text = " ".join(text)  # 连接成字符串，空格分隔
        corpus.append(text)

    # 1、构建词频矩阵，将文本中的词语转换成词频矩阵
    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(corpus)  # 词频矩阵,a[i][j]:表示j词在第i个文本中的词频
    # 2、统计每个词的tf-idf权值
    transformer = TfidfTransformer()
    tfidf = transformer.fit_transform(X)
    # 3、获取词袋模型中的关键词
    word = vectorizer.get_feature_names()
    # 4、获取tf-idf矩阵，a[i][j]表示j词在i篇文本中的tf-idf权重
    weight = tfidf.toarray()
    # 5、打印词语权重
    ids, titles, keys = [], [], []
    for i in range(len(weight)):
        print(u"-------这里输出第", i + 1, u"行文本的词语tf-idf------")
        ids.append(idList[i])
        titles.append(titleList[i])
        df_word, df_weight = [], []  # 当前文章的所有词汇列表、词汇对应权重列表
        for j in range(len(word)):
            print(word[j], weight[i][j])
            df_word.append(word[j])
            df_weight.append(weight[i][j])
        df_word = pd.DataFrame(df_word, columns=["word"])
        df_weight = pd.DataFrame(df_weight, columns=["weight"])
        word_weight = pd.concat([df_word, df_weight], axis=1)  # 拼接词汇列表和权重列表
        word_weight = word_weight.sort_values(by="weight", ascending=False)  # 按照权重值降序排列
        keyword = np.array(word_weight["word"])  # 选择词汇列并转成数组格式
        word_split = [keyword[x] for x in range(0, topK)]  # 抽取前topK个词汇作为关键词
        word_split = " ".join(word_split)
        keys.append(word_split)

    result = pd.DataFrame(
        {"id": ids, "title": titles, "key": keys}, columns=["id", "title", "key"]
    )
    #         result = pd.DataFrame({"id": ids, "key": keys},columns=['id', 'key'])
    return result

def main():
    # 读取数据集
    dataFile = r"/Users/19723/Desktop/今日话题 - 雪球.xlsx"
    data = pd.read_excel(dataFile)
    # 停用词表
    stopkey = [
        w.strip()
        for w in codecs.open(
            "/Users/19723/Desktop/Everyday财经/SOURCE/chinese_stopword.txt",
            "r",
            encoding="UTF-8",
        ).readlines()
    ]
    # tf-idf关键词抽取
    result = getKeywords_tfidf(data, stopkey, 7)
    result.to_csv(
        "/Users/19723/Desktop/Everyday财经/SOURCE/keys_TFIDF.csv",
        index=False,
        encoding="UTF-8",
    )
    outcome = pd.DataFrame(result)

if __name__ == "__main__":
    main()

# 替代掉换行符，筛选一波没用的词
# 词频分类新闻
process = pd.read_csv("/Users/19723/Desktop/Everyday财经/SOURCE/keys_TFIDF.csv")
origin = pd.read_excel("/Users/19723/Desktop/今日话题 - 雪球.xlsx")
origin = origin.assign(keys=process["key"])
origin.to_csv("/Users/19723//Desktop/Everyday财经/SOURCE/tmp.csv")
species = pd.read_excel("/Users/19723/Desktop/Everyday财经/SOURCE/种类.xlsx")

maxmi = []

# 对于每条新闻进行分类，并保存jaccard数据至列表
a = 0
for row in origin.itertuples():
    keywords_x = getattr(row, "keys")
    locals()["list" + str(int(a))] = []
    for row in species.itertuples():
        keywords_y = getattr(row, "species")  # 输出每一行
        intersection = len(list(set(keywords_x).intersection(set(keywords_y))))
        union = len(list(set(keywords_x).union(set(keywords_y))))
        # 除零处理
        sim = float(intersection) / union if union != 0 else 0
        locals()["list" + str(int(a))].append(float(sim))
    temp = locals()["list" + str(int(a))].index(
        max(locals()["list" + str(int(a))])
    )  # 返回最大值所在的索引值
    maxmi.append(temp)
    a += 1

# 对maxmi这个rank进行更高级的处理
maxmii = []
YIQING = 0
YIJISHICHANG = 0
GUONEIGUSHI = 0
HONGGUAN = 0
SHEHUIXINWEN = 0
QIHUO = 0
WAIHUI = 0
FIS = 0
GONGNONG = 0
YILIAO = 0
XIAOFEI = 0
JIAOYU = 0
KEJI = 0
XINNENGYUAN = 0
JIJIAN = 0
SHUZIHUOBI = 0
ZIRANQIHOU = 0
for i in maxmi:
    if i == 0:
        maxmii.append("疫情")
        YIQING += 1
    elif i == 1:
        maxmii.append("一级市场")
        YIJISHICHANG += 1
    elif i == 2:
        maxmii.append("国内股市")
        GUONEIGUSHI += 1
    elif i == 3:
        maxmii.append("宏观")
        HONGGUAN += 1
    elif i == 4:
        maxmii.append("社会新闻")
        SHEHUIXINWEN += 1
    elif i == 5:
        maxmii.append("期货")
        QIHUO += 1
    elif i == 6:
        maxmii.append("外汇")
        WAIHUI += 1
    elif i == 7:
        maxmii.append("FIs")
        FIS += 1
    elif i == 8:
        maxmii.append("工农实业")
        GONGNONG += 1
    elif i == 9:
        maxmii.append("医疗")
        YILIAO += 1
    elif i == 10:
        maxmii.append("消费")
        XIAOFEI += 1
    elif i == 11:
        maxmii.append("教育")
        JIAOYU += 1
    elif i == 12:
        maxmii.append("科技")
        KEJI += 1
    elif i == 13:
        maxmii.append("新能源汽车")
        XINNENGYUAN += 1
    elif i == 14:
        maxmii.append("基建")
        JIJIAN += 1
    elif i == 15:
        maxmii.append("数字货币")
        SHUZIHUOBI += 1
    else:
        maxmii.append("自然气候")
        ZIRANQIHOU += 1

# rank 进行联立
temp = pd.DataFrame(maxmii)

# for i in range(0, 6):
#     b2 = str2.find('，报')
#     b3 = str2.find('点。')
#     str2 = str2[:b2] + '；' + str2[b3+2:] #替换点数

# 去掉括号
origin = origin.assign(rank=temp[0])
origin1 = origin["abstract"].str.extract(r"】(.*)", expand=False)
origin1 = pd.DataFrame(origin1)
origin1.fillna(origin, inplace=True)
origin1 = origin1.assign(rank=origin["rank"])
origin1.to_excel("/Users/19723/Desktop/最终文件.xlsx")

print(
    "yiqing",
    YIQING,
    "yijishichang",
    YIJISHICHANG,
    "GUONEIGUSHI",
    GUONEIGUSHI,
    "HONGGUAN",
    HONGGUAN,
    "SHEHUIXINWEN",
    SHEHUIXINWEN,
    "QIHUO",
    QIHUO,
    "WAIHUI",
    WAIHUI,
    "FIS",
    FIS,
    "GONGNONG",
    GONGNONG,
    "YILIAO",
    YILIAO,
    "XIAOFEI",
    XIAOFEI,
    "JIAOYU",
    JIAOYU,
    "KEJI",
    KEJI,
    "XINNENGYUAN",
    XINNENGYUAN,
    "JIJIAN",
    JIJIAN,
    "SHUZIHUOBI",
    SHUZIHUOBI,
    "ZIRANQIHOU",
    ZIRANQIHOU,
)

max_stars_repo_name: IlIIIIIIlI/NewsUtilization

max_issues_repo_name: IlIIIIIIlI/NewsUtilization

max_forks_repo_name: IlIIIIIIlI/NewsUtilization


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import cv2
import copy
import numpy as np

from imgaug import augmenters as iaa

class Augmentation():

    def __init__(self, config):

        self.config = config

        ### augmentors by https://github.com/aleju/imgaug
        sometimes = lambda aug: iaa.Sometimes(0.5, aug)

        # Define our sequence of augmentation steps that will be applied to every image
        # All augmenters with per_channel=0.5 will sample one value _per image_
        # in 50% of all cases. In all other cases they will sample new values
        # _per channel_.
        self.aug_pipe = iaa.Sequential(
            [
                # apply the following augmenters to most images
                #iaa.Fliplr(0.5), # horizontally flip 50% of all images
                #iaa.Flipud(0.2), # vertically flip 20% of all images
                #sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width
                sometimes(iaa.Affine(
                    #scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
                    #translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
                    #rotate=(-5, 5), # rotate by -45 to +45 degrees
                    #shear=(-5, 5), # shear by -16 to +16 degrees
                    #order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
                    #cval=(0, 255), # if mode is constant, use a cval between 0 and 255
                    #mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
                )),
                # execute 0 to 5 of the following (less important) augmenters per image
                # don't execute all of them, as that would often be way too strong
                iaa.SomeOf((0, 5),
                    [
                        #sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
                        iaa.OneOf([
                            iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0
                            iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7
                            iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7
                        ]),
                        iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
                        #iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
                        # search either for all edges or for directed edges
                        #sometimes(iaa.OneOf([
                        #    iaa.EdgeDetect(alpha=(0, 0.7)),
                        #    iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),
                        #])),
                        iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images
                        iaa.OneOf([
                            iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels
                            #iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
                        ]),
                        #iaa.Invert(0.05, per_channel=True), # invert color channels
                        iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)
                        iaa.Multiply((0.5, 1.5), per_channel=0.5), # change brightness of images (50-150% of original value)
                        iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast
                        #iaa.Grayscale(alpha=(0.0, 1.0)),
                        #sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
                        #sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around
                    ],
                    random_order=True
                )
            ],
            random_order=True
        )

    def aug_image(self, train_instance, jitter):
        image_name = train_instance['filename']
        image = cv2.imread(image_name)

        if image is None: print('Cannot find ', image_name)

        h, w, c = image.shape
        all_objs = copy.deepcopy(train_instance['object'])

        if jitter:
            ### scale the image
            scale = np.random.uniform() / 10. + 1.
            image = cv2.resize(image, (0,0), fx = scale, fy = scale)

            ### translate the image
            max_offx = (scale-1.) * w
            max_offy = (scale-1.) * h
            offx = int(np.random.uniform() * max_offx)
            offy = int(np.random.uniform() * max_offy)

            image = image[offy : (offy + h), offx : (offx + w)]

            ### flip the image
            flip = np.random.binomial(1, .5)
            if flip > 0.5: image = cv2.flip(image, 1)

            image = self.aug_pipe.augment_image(image)

        # resize the image to standard size
        image = cv2.resize(image, (self.config['IMAGE_H'], self.config['IMAGE_W']))
        image = image[:,:,::-1]

        # fix object's position and size
        for obj in all_objs:
            for attr in ['xmin', 'xmax']:
                if jitter: obj[attr] = int(obj[attr] * scale - offx)

                obj[attr] = int(obj[attr] * float(self.config['IMAGE_W']) / w)
                obj[attr] = max(min(obj[attr], self.config['IMAGE_W']), 0)

            for attr in ['ymin', 'ymax']:
                if jitter: obj[attr] = int(obj[attr] * scale - offy)

                obj[attr] = int(obj[attr] * float(self.config['IMAGE_H']) / h)
                obj[attr] = max(min(obj[attr], self.config['IMAGE_H']), 0)

            if jitter and flip > 0.5:
                xmin = obj['xmin']
                obj['xmin'] = self.config['IMAGE_W'] - obj['xmax']
                obj['xmax'] = self.config['IMAGE_W'] - xmin

        return image, all_objs

max_stars_repo_name: skyimager/aqvs

max_issues_repo_name: skyimager/aqvs

max_forks_repo_name: skyimager/aqvs


text:
import os
import sys
import pickle
import traceback
import numpy as np
import pandas as pd
import pyqtgraph as pg
from glob import glob

from PyQt5 import QtCore, QtWidgets

from Code.Data import plots
import Code.Data.calculations as c
from Code.Data.TxtFile import TxtFile
from Code.Data.SDYFile import SDYFile
from Code.UI.layout_label import Ui_MainWindow

if QtCore.QT_VERSION >= 0x50501:
    def excepthook(type_, value, traceback_):
        traceback.print_exception(type_, value, traceback_)
        QtCore.qFatal('')

    sys.excepthook = excepthook

SAMPLE_FREQ = 200
PD_OFFSET_TIME = 0.05
PD_OFFSET_POINT = int(PD_OFFSET_TIME * SAMPLE_FREQ)
PLOT_PEAKS = True

class LabelledLinearRegionItem(pg.LinearRegionItem):
    def __init__(self, values, movable, label):
        super(LabelledLinearRegionItem, self).__init__(values=values, movable=movable)
        self.label = pg.InfLineLabel(self.lines[1], label, position=0.3, rotateAxis=(1, 0), anchor=(1, 1))

class LabelUI(Ui_MainWindow):
    def __init__(self, mainwindow):
        super(LabelUI, self).__init__()
        self.setupUi(mainwindow)
        self.actionLoad_study.triggered.connect(self.load_study_folder)
        self.comboBox_txtsdyFiles.activated.connect(self.load_txtsdyFile)
        self.checkBox_Pa.clicked.connect(self.toggle_pa)
        self.GraphicsLayout = pg.GraphicsLayout()
        self.graphicsView_.setCentralItem(self.GraphicsLayout)
        self.graphicsView_.show()
        self.verticalLayout_Buttons.setAlignment(QtCore.Qt.AlignTop)
        self.pushButton_ExportStudy.clicked.connect(self.export_study)
        self.pushButton_ExportAll.clicked.connect(self.export_all)

        self.studyFolderPath = None
        self.studyData = dict()
        self.TxtSdyFile = None
        self.calculations = dict()

        with open("./information.txt", 'r') as f:
            self.textBrowser.setHtml("\n".join(f.readlines()))

        self.plot_pressure, self.plot_flow, self.plot_pressure_ratios = None, None, None
        self.plot_resistances, self.plot_ensemble_rest, self.plot_ensemble_hyp = None, None, None
        self.slider_group_rest, self.slider_group_hyp = None, None
        self.ensemble_data_rest, self.ensemble_data_hyp = None, None
        self.button_rest, self.button_hyp = None, None
        self.button_notch_rest, self.button_enddiastole_rest = None, None
        self.button_notch_hyp, self.button_enddiastole_hyp = None, None
        self.slider_notch_rest, self.slider_notch_hyp = None, None
        self.slider_enddiastole_rest, self.slider_enddiastole_hyp = None, None

    def refresh_ui(self):
        if self.studyFolderPath:
            self.comboBox_txtsdyFiles.setEnabled(True)
            self.label_PatientID.setText(f"Patient ID: {self.studyData.get('id', 'NA')}")

            # Get rid of the old files list and add the new ones
            self.comboBox_txtsdyFiles.clear()
            self.comboBox_txtsdyFiles.addItem("Please select a file")

            txt_file_paths = glob(os.path.join(self.studyFolderPath, "*.txt"))
            sdy_file_paths = glob(os.path.join(self.studyFolderPath, "*.sdy"))
            for txt_file_path in txt_file_paths:
                labels = self.load_cph(f"{txt_file_path}.cph")
                labels.pop('pa', None)  # Remove the PA key so doesn't count as a label
                self.comboBox_txtsdyFiles.addItem("{} - {} labels".format(txt_file_path, len(labels)))
            for sdy_file_path in sdy_file_paths:
                labels = self.load_cph(f"{sdy_file_path}.cph")
                labels.pop('pa', None)  # Remove the PA key so doesn't count as a label
                self.comboBox_txtsdyFiles.addItem("{} - {} labels".format(sdy_file_path, len(labels)))

            # Clear the plot window
            self.GraphicsLayout = pg.GraphicsLayout()
            self.graphicsView_.setCentralItem(self.GraphicsLayout)
            self.graphicsView_.show()
        else:  # If a study folder path isn't set, the file box shouldn't be clickable
            self.comboBox_txtsdyFiles.setEnabled(False)

    def load_study_folder(self):
        self.studyFolderPath = QtWidgets.QFileDialog.getExistingDirectory(None, "Select a folder", "./data/",
                                                                          QtWidgets.QFileDialog.ShowDirsOnly)
        self.refresh_ui()

    def toggle_pa(self):
        self.save_cph()
        self.load_txtsdyFile()

    def load_txtsdyFile(self):
        self.studyFolderPath = None
        self.studyData = dict()
        self.TxtSdyFile = None
        self.calculations = dict()

        self.plot_pressure, self.plot_flow, self.plot_pressure_ratios = None, None, None
        self.plot_resistances, self.plot_ensemble_rest, self.plot_ensemble_hyp = None, None, None
        self.slider_group_rest, self.slider_group_hyp = None, None
        self.ensemble_data_rest, self.ensemble_data_hyp = None, None
        self.slider_notch_rest, self.slider_notch_hyp = None, None
        self.slider_enddiastole_rest, self.slider_enddiastole_hyp = None, None

        study_path = self.comboBox_txtsdyFiles.currentText().rsplit(' ', 3)[0]
        try:
            if os.path.splitext(study_path)[-1] == ".txt":
                self.TxtSdyFile = TxtFile(studypath=study_path, pd_offset=PD_OFFSET_POINT)
                self.checkBox_Pa.setEnabled(False)
            elif os.path.splitext(study_path)[-1] == ".sdy":
                pa_channel = 'pa_physio' if self.checkBox_Pa.isChecked() else 'pa_trans'
                self.TxtSdyFile = SDYFile(filepath=study_path, pa_channel=pa_channel)
                self.checkBox_Pa.setEnabled(True)
            self.label_PatientID.setText(f"Patient ID:\t{self.TxtSdyFile.patient_id}")
            self.label_StudyDate.setText(f"Study date:\t{self.TxtSdyFile.study_date}")
            self.label_ExportDate.setText(f"Export date:\t{self.TxtSdyFile.export_date}")
        except FileNotFoundError:
            print(f"!!!UNABLE TO FIND FILE {study_path}!!!")

        self.plot_txtsdyFile()
        self.draw_buttons()
        self.load_saved_labels()
        self.perform_calculations()

    def plot_txtsdyFile(self):
        pg.setConfigOptions(antialias=True)

        self.GraphicsLayout = pg.GraphicsLayout()
        self.GraphicsLayout.setSpacing(0)
        self.GraphicsLayout.setContentsMargins(0., 0., 0., 0.)
        self.graphicsView_.setCentralItem(self.GraphicsLayout)
        self.graphicsView_.show()

        # Data
        # data_pa = self.clip_wave(np.array(self.TxtSdyFile.df['pa'], dtype=np.float))
        # data_pd = self.clip_wave(np.array(self.TxtSdyFile.df['pd'], dtype=np.float),
        #                          ref_wave=np.array(self.TxtSdyFile.df['pa'], dtype=np.float))
        data_pa = np.array(self.TxtSdyFile.df['pa'])
        data_pd = np.array(self.TxtSdyFile.df['pd'])
        data_time = np.array(self.TxtSdyFile.df['time'])
        data_flow = np.array(self.TxtSdyFile.df['flow'])
        data_pdpa = plots.pdpa(self, peaks=self.TxtSdyFile.peaks)

        data_pdpa_filtered = plots.pdpa_filtered(self, pdpa=data_pdpa)
        data_microvascular_resistance = plots.microvascular_resistance(self, peaks=self.TxtSdyFile.peaks)
        data_microvascular_resistance_filtered = plots.filtered_resistance(data_microvascular_resistance)
        data_stenosis_resistance = plots.stenosis_resistance(self, peaks=self.TxtSdyFile.peaks)
        data_stenosis_resistance_filtered = plots.filtered_resistance(data_stenosis_resistance)

        # Plots
        self.plot_pressure = self.GraphicsLayout.addPlot(row=0, col=0, colspan=2, title='Pressure')
        self.plot_flow = self.GraphicsLayout.addPlot(row=1, col=0, colspan=2, title='Flow')
        self.plot_pressure_ratios = self.GraphicsLayout.addPlot(row=2, col=0, colspan=2,
                                                                title='Distal:Proximal Pressure')
        self.plot_resistances = self.GraphicsLayout.addPlot(row=3, col=0, colspan=2, title='Resistances')
        self.plot_ensemble_rest = self.GraphicsLayout.addPlot(row=4, col=0, colspan=1, title='Resting Ensemble')
        self.plot_ensemble_hyp = self.GraphicsLayout.addPlot(row=4, col=1, colspan=1, title='Hyperaemic Ensemble')
        for p in (self.plot_pressure, self.plot_flow, self.plot_pressure_ratios, self.plot_resistances):
            p.addLegend()
        for p in (self.plot_flow, self.plot_pressure_ratios, self.plot_resistances):
            p.setXLink(self.plot_pressure)

        # Lines
        self.plot_pressure.plot(x=data_time, y=data_pa, name='Pa', pen='r')
        self.plot_pressure.plot(x=data_time, y=data_pd, name='Pd', pen='y')
        self.plot_flow.plot(x=data_time, y=data_flow, name='Flow', pen='g')
        self.plot_pressure_ratios.plot(x=data_pdpa['x'], y=data_pdpa['y'], name='PdPa (beat-wise)',
                                       pen=(255, 255, 0, 100))
        self.plot_pressure_ratios.plot(x=data_pdpa_filtered['x'], y=data_pdpa_filtered['y'], name='PdPa (filtered)',
                                       pen=(255, 255, 0, 200))
        self.plot_resistances.plot(x=data_microvascular_resistance['x'],
                                   y=data_microvascular_resistance['y'],
                                   name='Microvascular (beat-wise)',
                                   pen=(0, 255, 255, 100))
        self.plot_resistances.plot(x=data_microvascular_resistance_filtered['x'],
                                   y=data_microvascular_resistance_filtered['y'],
                                   name='Microvascular (filtered)',
                                   pen=(0, 255, 255, 200))
        self.plot_resistances.plot(x=data_stenosis_resistance['x'],
                                   y=data_stenosis_resistance['y'],
                                   name='Stenosis (beat-wise)',
                                   pen=(255, 0, 255, 100))
        self.plot_resistances.plot(x=data_stenosis_resistance_filtered['x'],
                                   y=data_stenosis_resistance_filtered['y'],
                                   name='Stenosis (filtered)',
                                   pen=(255, 0, 255, 200))

        # ECG gating indicators
        if PLOT_PEAKS:
            peak_times = np.array([data_time[peak] for peak in self.TxtSdyFile.peaks])
            self.plot_pressure.plot(x=peak_times, y=np.repeat(max(data_pd), len(self.TxtSdyFile.peaks)),
                                    pen=(200, 200, 200),
                                    symbolBrush=(255, 0, 0),
                                    symbolPen='w')

    def click_button(self, btn):
        if btn.slider_active:
            btn.slider_active = False
            btn.setStyleSheet(f"background-color: 'red'")
        else:
            btn.slider_active = True
            btn.setStyleSheet(f"background-color: 'green'")
        if btn is self.button_rest:
            self.create_slider_group('rest') if btn.slider_active else self.destroy_slider_group('rest')
        elif btn is self.button_hyp:
            self.create_slider_group('hyp') if btn.slider_active else self.destroy_slider_group('hyp')
        elif btn is self.button_notch_rest:
            self.create_marker('notch_rest') if btn.slider_active else self.destroy_marker('notch_rest')
        elif btn is self.button_notch_hyp:
            self.create_marker('notch_hyp') if btn.slider_active else self.destroy_marker('notch_hyp')
        elif btn is self.button_enddiastole_rest:
            self.create_marker('enddiastole_rest') if btn.slider_active else self.destroy_marker('enddiastole_rest')
        elif btn is self.button_enddiastole_hyp:
            self.create_marker('enddiastole_hyp') if btn.slider_active else self.destroy_marker('enddiastole_hyp')
        else:
            print(f"Unknown button clicked!")

        self.perform_calculations(save=True)  # Save in case something removed; if added OK as not placed yet

    def create_marker(self, marker_type, value=None):
        slider = pg.InfiniteLine(pos=0.2, movable=True, label='',
                                 labelOpts={'position': 0.5, 'rotateAxis': (1, 0), 'anchor': (1, 1)})
        slider.sigPositionChangeFinished.connect(lambda: self.perform_calculations(save=True))
        if value:
            slider.setValue(value)
        if marker_type == 'notch_rest':
            slider.label = pg.InfLineLabel(slider, text="Dicrotic notch")
            self.plot_ensemble_rest.addItem(slider)
            self.slider_notch_rest = slider
        elif marker_type == 'notch_hyp':
            slider.label = pg.InfLineLabel(slider, text="Dicrotic notch")
            self.plot_ensemble_hyp.addItem(slider)
            self.slider_notch_hyp = slider
        elif marker_type == 'enddiastole_rest':
            slider.label = pg.InfLineLabel(slider, text="End diastole")
            self.plot_ensemble_rest.addItem(slider)
            self.slider_enddiastole_rest = slider
        elif marker_type == 'enddiastole_hyp':
            slider.label = pg.InfLineLabel(slider, text="End diastole")
            self.plot_ensemble_hyp.addItem(slider)
            self.slider_enddiastole_hyp = slider
        else:
            raise ValueError(f"Unknown button type pressed: {marker_type}")

    def destroy_marker(self, marker_type):
        if marker_type == 'notch_rest':
            self.plot_ensemble_rest.removeItem(self.slider_notch_rest)
            self.slider_notch_rest = None
        elif marker_type == 'notch_hyp':
            self.plot_ensemble_hyp.removeItem(self.slider_notch_hyp)
            self.slider_notch_hyp = None
        elif marker_type == 'enddiastole_rest':
            self.plot_ensemble_rest.removeItem(self.slider_enddiastole_rest)
            self.slider_enddiastole_rest = None
        elif marker_type == 'enddiastole_hyp':
            self.plot_ensemble_hyp.removeItem(self.slider_enddiastole_hyp)
            self.slider_enddiastole_hyp = None
        else:
            raise ValueError(f"Unknown button type pressed: {marker_type}")

    def create_slider_group(self, rest_or_hyp, range_from=None, range_to=None):
        if range_from is None or range_to is None:
            x_lower, x_upper = self.plot_pressure.getAxis('bottom').range
            range_from = ((x_upper - x_lower) * 0.20) + x_lower
            range_to = ((x_upper - x_lower) * 0.25) + x_lower
        if rest_or_hyp == 'rest':
            self.slider_group_rest = []
            for p in (self.plot_pressure, self.plot_flow, self.plot_pressure_ratios, self.plot_resistances):
                slider = LabelledLinearRegionItem(values=(range_from, range_to), movable=True, label='Rest')
                slider.sigRegionChangeFinished.connect(
                    lambda s=slider, group='rest': self.adjust_all_sliders_in_group(s, group))
                p.addItem(slider, ignoreBounds=True)
                self.slider_group_rest.append(slider)
        elif rest_or_hyp == 'hyp':
            self.slider_group_hyp = []
            for p in (self.plot_pressure, self.plot_flow, self.plot_pressure_ratios, self.plot_resistances):
                slider = LabelledLinearRegionItem(values=(range_from, range_to), movable=True, label='Hyperaemia')
                slider.sigRegionChangeFinished.connect(
                    lambda s=slider, group='hyp': self.adjust_all_sliders_in_group(s, group))
                p.addItem(slider, ignoreBounds=True)
                self.slider_group_hyp.append(slider)

    def destroy_slider_group(self, rest_or_hyp):
        if rest_or_hyp == 'rest':
            slider_group = self.slider_group_rest
        elif rest_or_hyp == 'hyp':
            slider_group = self.slider_group_hyp
        else:
            raise ValueError(f"Unknown value for rest_or_hyp: {rest_or_hyp}")
        for p in (self.plot_pressure, self.plot_flow, self.plot_pressure_ratios, self.plot_resistances):
            for slider in slider_group:  # Try to remove each slider from plot, as a list
                p.removeItem(slider)
        if rest_or_hyp == 'rest':
            self.slider_group_rest = []
        elif rest_or_hyp == 'hyp':
            self.slider_group_hyp = []
        self.perform_calculations()

    def draw_buttons(self):
        def clear_layout(layout):
            for i in reversed(range(layout.count())):
                try:
                    layout.itemAt(i).widget().setParent(None)
                except AttributeError:
                    pass

        def initialise_button(labelui, text):
            btn = QtWidgets.QPushButton(labelui.centralwidget)
            btn.setText(text)
            btn.slider_active = False
            btn.clicked.connect(lambda state, button=btn: labelui.click_button(button))
            btn.setStyleSheet(f"background-color: 'red'")
            self.verticalLayout_Buttons.addWidget(btn)
            return btn

        clear_layout(self.verticalLayout_Buttons)

        self.button_rest = initialise_button(self, 'Rest')
        self.button_hyp = initialise_button(self, 'Hyperaemia')
        self.button_notch_rest = initialise_button(self, 'Dicrotic notch (rest)')
        self.button_enddiastole_rest = initialise_button(self, 'End diastole (rest)')
        self.button_notch_hyp = initialise_button(self, 'Dicrotic notch (hyperaemia)')
        self.button_enddiastole_hyp = initialise_button(self, 'End diastole (hyperaemia)')

    def adjust_all_sliders_in_group(self, slider, slider_group):
        slider_min, slider_max = slider.getRegion()
        if slider_group == 'rest':
            slider_group = self.slider_group_rest
        elif slider_group == 'hyp':
            slider_group = self.slider_group_hyp
        else:
            raise ValueError(f"Unknown slider group")
        for s in slider_group:
            if s.getRegion() != (slider_min, slider_max):  # Only move if needed
                s.setRegion((slider_min, slider_max))
        self.perform_calculations(save=True)

    def calculate_ensemble(self, rest_or_hyp):
        if rest_or_hyp == 'rest':
            plot = self.plot_ensemble_rest
            slider_notch = self.slider_notch_rest
            slider_enddiastole = self.slider_enddiastole_rest
        elif rest_or_hyp == 'hyp':
            plot = self.plot_ensemble_hyp
            slider_notch = self.slider_notch_hyp
            slider_enddiastole = self.slider_enddiastole_hyp
        else:
            raise ValueError(f"Unknown rest_or_hyp value {rest_or_hyp}")
        ensemble_data, n_rejected = c.ensemble_beats(self, rest_or_hyp)
        plot.clear()
        if ensemble_data:
            t0 = ensemble_data[0]['time']
            for beat in ensemble_data:
                plot.plot(x=t0, y=beat['pa'], pen=(192, 192, 192, 100))
            mean_pa = c.average_beats_from_beat_list(ensemble_data, measure='pa')
            plot.plot(x=t0, y=mean_pa, pen='g', width=100)
        plot.setTitle(f"Resting Ensemble ({len(ensemble_data)} beats; {n_rejected} rejected)")
        if slider_notch:
            plot.addItem(slider_notch)
        if slider_enddiastole:
            plot.addItem(slider_enddiastole)
        return ensemble_data

    def calculate(self):
        self.calculations['pressures'] = []
        self.calculations['pressure_ratios'] = []
        self.calculations['flows'] = []
        self.calculations['flow_ratios'] = []
        self.calculations['resistances'] = []
        if self.ensemble_data_rest:
            self.calculations['pressures'].append({'name': 'WC Pa',
                                                   'state': 'rest',
                                                   'phase': 'mean',
                                                   'value': c.wholecycle_measure(self, 'rest', 'pa', 'mean')})
            self.calculations['pressures'].append({'name': 'WC Pd',
                                                   'state': 'rest',
                                                   'phase': 'mean',
                                                   'value': c.wholecycle_measure(self, 'rest', 'pd', 'mean')})
            self.calculations['pressures'].append({'name': 'WC Pa',
                                                   'state': 'rest',
                                                   'phase': 'peak',
                                                   'value': c.wholecycle_measure(self, 'rest', 'pa', 'peak')})
            self.calculations['pressures'].append({'name': 'WC Pd',
                                                   'state': 'rest',
                                                   'phase': 'peak',
                                                   'value': c.wholecycle_measure(self, 'rest', 'pd', 'peak')})

            pdpa = c.wholecycle_measure(self, 'rest', 'pd', 'mean') / c.wholecycle_measure(self, 'rest', 'pa', 'mean')
            self.calculations['pressure_ratios'].append({'name': 'WC PdPa', 'value': pdpa})

            dfr = c.dpr_measure(self, 'rest', 'pd', 'mean') / c.dpr_measure(self, 'rest', 'pa', 'mean')
            self.calculations['pressure_ratios'].append({'name': 'dPR', 'value': dfr})

            self.calculations['pressure_ratios'].append({'name': 'RFR', 'value': c.rfr(self)})

            self.calculations['flows'].append({'name': 'WC Flow',
                                               'state': 'rest',
                                               'phase': 'mean',
                                               'value': c.wholecycle_measure(self, 'rest', 'flow', 'mean')})
            self.calculations['flows'].append({'name': 'WC Flow',
                                               'state': 'rest',
                                               'phase': 'peak',
                                               'value': c.wholecycle_measure(self, 'rest', 'flow', 'peak')})

            p_delta = c.wholecycle_measure(self, 'rest', 'pa', 'mean') - c.wholecycle_measure(self, 'rest', 'pd',
                                                                                'mean')
            bsr_mean = p_delta / c.wholecycle_measure(self, 'rest', 'flow', 'mean')
            bsr_peak = p_delta / c.wholecycle_measure(self, 'rest', 'flow', 'peak')
            self.calculations['resistances'].append({'name': 'BSR',
                                                     'phase': 'mean',
                                                     'value': bsr_mean})
            self.calculations['resistances'].append({'name': 'BSR',
                                                     'phase': 'peak',
                                                     'value': bsr_peak})

            bmr_mean = c.wholecycle_measure(self, 'rest', 'pd', 'mean') / c.wholecycle_measure(self, 'rest', 'flow',
                                                                                'mean')
            bmr_peak = c.wholecycle_measure(self, 'rest', 'pd', 'mean') / c.wholecycle_measure(self, 'rest', 'flow',
                                                                                'peak')
            self.calculations['resistances'].append({'name': 'BMR',
                                                     'phase': 'mean',
                                                     'value': bmr_mean})
            self.calculations['resistances'].append({'name': 'BMR',
                                                     'phase': 'peak',
                                                     'value': bmr_peak})

        if self.ensemble_data_hyp:
            self.calculations['pressures'].append({'name': 'WC Pa',
                                                   'state': 'hyp',
                                                   'phase': 'mean',
                                                   'value': c.wholecycle_measure(self, 'hyp', 'pa', 'mean')})
            self.calculations['pressures'].append({'name': 'WC Pd',
                                                   'state': 'hyp',
                                                   'phase': 'mean',
                                                   'value': c.wholecycle_measure(self, 'hyp', 'pd', 'mean')})
            self.calculations['pressures'].append({'name': 'WC Pa',
                                                   'state': 'hyp',
                                                   'phase': 'peak',
                                                   'value': c.wholecycle_measure(self, 'hyp', 'pa', 'peak')})
            self.calculations['pressures'].append({'name': 'WC Pd',
                                                   'state': 'hyp',
                                                   'phase': 'peak',
                                                   'value': c.wholecycle_measure(self, 'hyp', 'pd', 'peak')})

            ffr = c.wholecycle_measure(self, 'hyp', 'pd', 'mean') / c.wholecycle_measure(self, 'hyp', 'pa', 'mean')
            self.calculations['pressure_ratios'].append({'name': 'FFR', 'value': ffr})

            self.calculations['flows'].append({'name': 'WC Flow',
                                               'state': 'hyp',
                                               'phase': 'mean',
                                               'value': c.wholecycle_measure(self, 'hyp', 'flow', 'mean')})
            self.calculations['flows'].append({'name': 'WC Flow',
                                               'state': 'hyp',
                                               'phase': 'peak',
                                               'value': c.wholecycle_measure(self, 'hyp', 'flow', 'peak')})

            p_delta = c.wholecycle_measure(self, 'hyp', 'pa', 'mean') - c.wholecycle_measure(self, 'hyp', 'pd', 'mean')
            hsr_mean = p_delta / c.wholecycle_measure(self, 'hyp', 'flow', 'mean')
            hsr_peak = p_delta / c.wholecycle_measure(self, 'hyp', 'flow', 'peak')
            self.calculations['resistances'].append({'name': 'HSR',
                                                     'phase': 'mean',
                                                     'value': hsr_mean})
            self.calculations['resistances'].append({'name': 'HSR',
                                                     'phase': 'peak',
                                                     'value': hsr_peak})

            hmr_mean = c.wholecycle_measure(self, 'hyp', 'pd', 'mean') / c.wholecycle_measure(self, 'hyp', 'flow',
                                                                                'mean')
            hmr_peak = c.wholecycle_measure(self, 'hyp', 'pd', 'mean') / c.wholecycle_measure(self, 'hyp', 'flow',
                                                                                'peak')
            self.calculations['resistances'].append({'name': 'HMR',
                                                     'phase': 'mean',
                                                     'value': hmr_mean})
            self.calculations['resistances'].append({'name': 'HMR',
                                                     'phase': 'peak',
                                                     'value': hmr_peak})

        if self.ensemble_data_rest and self.ensemble_data_hyp:
            cfr_mean = c.wholecycle_measure(self, 'hyp', 'flow', 'mean') / c.wholecycle_measure(self, 'rest', 'flow',
                                                                                'mean')
            cfr_peak = c.wholecycle_measure(self, 'hyp', 'flow', 'peak') / c.wholecycle_measure(self, 'rest', 'flow',
                                                                                'peak')
            self.calculations['flow_ratios'].append({'name': 'CFR',
                                                     'phase': 'mean',
                                                     'value': cfr_mean})
            self.calculations['flow_ratios'].append({'name': 'CFR',
                                                     'phase': 'peak',
                                                     'value': cfr_peak})

        if self.slider_group_rest and self.slider_notch_rest and self.slider_enddiastole_rest:
            self.calculations['pressures'].append({'name': 'Sysolic Pa',
                                                   'state': 'rest',
                                                   'phase': 'mean',
                                                   'value': c.systolic_measure(self, 'rest', 'pa', 'mean')})
            self.calculations['pressures'].append({'name': 'Sysolic Pd',
                                                   'state': 'rest',
                                                   'phase': 'mean',
                                                   'value': c.systolic_measure(self, 'rest', 'pd', 'mean')})
            self.calculations['pressures'].append({'name': 'Wavefree Pa',
                                                   'state': 'rest',
                                                   'phase': 'mean',
                                                   'value': c.wavefree_measure(self, 'rest', 'pa', 'mean')})
            self.calculations['pressures'].append({'name': 'Wavefree Pd',
                                                   'state': 'rest',
                                                   'phase': 'mean',
                                                   'value': c.wavefree_measure(self, 'rest', 'pd', 'mean')})
            self.calculations['pressures'].append({'name': 'Wavefree Pa',
                                                   'state': 'rest',
                                                   'phase': 'peak',
                                                   'value': c.wavefree_measure(self, 'rest', 'pa', 'peak')})
            self.calculations['pressures'].append({'name': 'Wavefree Pd',
                                                   'state': 'rest',
                                                   'phase': 'peak',
                                                   'value': c.wavefree_measure(self, 'rest', 'pd', 'peak')})

            ifr = c.wavefree_measure(self, 'rest', 'pd', 'mean') / c.wavefree_measure(self, 'rest', 'pa', 'mean')
            self.calculations['pressure_ratios'].append({'name': 'iFR', 'value': ifr})

            self.calculations['flows'].append({'name': 'Wavefree flow',
                                               'state': 'rest',
                                               'phase': 'mean',
                                               'value': c.wavefree_measure(self, 'rest', 'flow', 'mean')})
            self.calculations['flows'].append({'name': 'Wavefree flow',
                                               'state': 'rest',
                                               'phase': 'peak',
                                               'value': c.wavefree_measure(self, 'rest', 'flow', 'peak')})

        if self.slider_group_hyp and self.slider_notch_hyp and self.slider_enddiastole_hyp:
            self.calculations['pressures'].append({'name': 'Sysolic Pa',
                                                   'state': 'hyp',
                                                   'phase': 'mean',
                                                   'value': c.systolic_measure(self, 'hyp', 'pa', 'mean')})
            self.calculations['pressures'].append({'name': 'Sysolic Pd',
                                                   'state': 'hyp',
                                                   'phase': 'mean',
                                                   'value': c.systolic_measure(self, 'hyp', 'pd', 'mean')})
            self.calculations['pressures'].append({'name': 'Wavefree Pa',
                                                   'state': 'hyp',
                                                   'phase': 'mean',
                                                   'value': c.wavefree_measure(self, 'hyp', 'pa', 'mean')})
            self.calculations['pressures'].append({'name': 'Wavefree Pd',
                                                   'state': 'hyp',
                                                   'phase': 'mean',
                                                   'value': c.wavefree_measure(self, 'hyp', 'pd', 'mean')})
            self.calculations['pressures'].append({'name': 'Wavefree Pa',
                                                   'state': 'hyp',
                                                   'phase': 'peak',
                                                   'value': c.wavefree_measure(self, 'hyp', 'pa', 'peak')})
            self.calculations['pressures'].append({'name': 'Wavefree Pd',
                                                   'state': 'hyp',
                                                   'phase': 'peak',
                                                   'value': c.wavefree_measure(self, 'hyp', 'pd', 'peak')})

            ifrh = c.wavefree_measure(self, 'hyp', 'pd', 'mean') / c.wavefree_measure(self, 'hyp', 'pa', 'mean')
            self.calculations['pressure_ratios'].append({'name': 'iFRa', 'value': ifrh})

            self.calculations['flows'].append({'name': 'Wavefree flow',
                                               'state': 'hyp',
                                               'phase': 'mean',
                                               'value': c.wavefree_measure(self, 'hyp', 'flow', 'mean')})
            self.calculations['flows'].append({'name': 'Wavefree flow',
                                               'state': 'hyp',
                                               'phase': 'peak',
                                               'value': c.wavefree_measure(self, 'hyp', 'flow', 'peak')})

        if self.slider_group_rest and self.slider_group_hyp and self.slider_notch_rest and self.slider_notch_hyp and \
                self.slider_enddiastole_rest and self.slider_enddiastole_hyp:
            sys_cfr_mean = c.systolic_measure(self, 'hyp', 'flow', 'mean') / c.systolic_measure(self, 'rest', 'flow',
                                                                                'mean')
            sys_cfr_peak = c.systolic_measure(self, 'hyp', 'flow', 'peak') / c.systolic_measure(self, 'rest', 'flow',
                                                                                'peak')
            self.calculations['flow_ratios'].append({'name': 'Systolic CFR',
                                                     'phase': 'mean',
                                                     'value': sys_cfr_mean})
            self.calculations['flow_ratios'].append({'name': 'Systolic CFR',
                                                     'phase': 'peak',
                                                     'value': sys_cfr_peak})

            wf_cfr_mean = c.wavefree_measure(self, 'hyp', 'flow', 'mean') / c.wavefree_measure(self, 'rest', 'flow',
                                                                                'mean')
            try:
                wf_cfr_peak = c.wavefree_measure(self, 'hyp', 'flow', 'peak') / c.wavefree_measure(self, 'rest', 'flow',
                                                                                'peak')
            except ZeroDivisionError:
                wf_cfr_peak = 0
            self.calculations['flow_ratios'].append({'name': 'Wavefree CFR',
                                                     'phase': 'mean',
                                                     'value': wf_cfr_mean})
            self.calculations['flow_ratios'].append({'name': 'Wavefree CFR',
                                                     'phase': 'peak',
                                                     'value': wf_cfr_peak})

    def display_calculations(self):
        tables = [self.tableWidget_Pressures, self.tableWidget_Flows]
        calc_dicts = [self.calculations['pressures'], self.calculations['flows']]
        for table, calcs in zip(tables, calc_dicts):
            table.setRowCount(len(calcs))
            table.setColumnCount(4)
            for i in range(table.columnCount()):
                table.horizontalHeader().setSectionResizeMode(i, QtWidgets.QHeaderView.Stretch)
            table.setHorizontalHeaderLabels(("Parameter", "State", "Phase", "Value"))
            for i_calc, calc_dict in enumerate(calcs):
                try:
                    table.setItem(i_calc, 0, QtWidgets.QTableWidgetItem(str(calc_dict['name'])))
                    table.setItem(i_calc, 1, QtWidgets.QTableWidgetItem(str(calc_dict['state'])))
                    table.setItem(i_calc, 2, QtWidgets.QTableWidgetItem(str(calc_dict['phase'])))
                    table.setItem(i_calc, 3, QtWidgets.QTableWidgetItem(str(round(calc_dict['value'], 2))))
                except KeyError as e:
                    print(f"Failed to find {e} in {calc_dict}")

        tables = [self.tableWidget_FlowRatios, self.tableWidget_Resistances]
        calc_dicts = [self.calculations['flow_ratios'], self.calculations['resistances']]
        for table, calcs in zip(tables, calc_dicts):
            table.setRowCount(len(calcs))
            table.setColumnCount(3)
            for i in range(table.columnCount()):
                table.horizontalHeader().setSectionResizeMode(i, QtWidgets.QHeaderView.Stretch)
            table.setHorizontalHeaderLabels(("Parameter", "Phase", "Value"))
            for i_calc, calc_dict in enumerate(calcs):
                table.setItem(i_calc, 0, QtWidgets.QTableWidgetItem(str(calc_dict['name'])))
                table.setItem(i_calc, 1, QtWidgets.QTableWidgetItem(str(calc_dict['phase'])))
                table.setItem(i_calc, 2, QtWidgets.QTableWidgetItem(str(round(calc_dict['value'], 2))))

        tables = [self.tableWidget_PressureRatios]
        calc_dicts = [self.calculations['pressure_ratios']]
        for table, calcs in zip(tables, calc_dicts):
            table.setRowCount(len(calcs))
            table.setColumnCount(2)
            for i in range(table.columnCount()):
                table.horizontalHeader().setSectionResizeMode(i, QtWidgets.QHeaderView.Stretch)
            table.setHorizontalHeaderLabels(("Parameter", "Value"))
            for i_calc, calc_dict in enumerate(calcs):
                table.setItem(i_calc, 0, QtWidgets.QTableWidgetItem(str(calc_dict['name'])))
                table.setItem(i_calc, 1, QtWidgets.QTableWidgetItem(str(round(calc_dict['value'], 2))))

    def save_cph(self):
        save_dict = {}
        save_dict['pa'] = self.checkBox_Pa.isChecked()
        try:
            save_dict['range_rest'] = self.slider_group_rest[0].getRegion()
        except (TypeError, IndexError):
            pass
        try:
            save_dict['range_hyp'] = self.slider_group_hyp[0].getRegion()
        except (TypeError, IndexError):
            pass
        try:
            save_dict['notch_rest'] = self.slider_notch_rest.value()
        except AttributeError:
            pass
        try:
            save_dict['notch_hyp'] = self.slider_notch_hyp.value()
        except AttributeError:
            pass
        try:
            save_dict['enddiastole_rest'] = self.slider_enddiastole_rest.value()
        except AttributeError:
            pass
        try:
            save_dict['enddiastole_hyp'] = self.slider_enddiastole_hyp.value()
        except AttributeError:
            pass
        with open(f"{self.TxtSdyFile.studypath}.cph", 'wb') as f:
            pickle.dump(save_dict, f)
        print("Saved")

    def load_saved_labels(self):
        cph_path = f"{self.TxtSdyFile.studypath}.cph"
        saved_cph = self.load_cph(cph_path)
        pa = saved_cph.get('pa', True)
        range_rest = saved_cph.get('range_rest', None)
        range_hyp = saved_cph.get('range_hyp', None)
        notch_rest = saved_cph.get('notch_rest', None)
        notch_hyp = saved_cph.get('notch_hyp', None)
        enddiastole_rest = saved_cph.get('enddiastole_rest', None)
        enddiastole_hyp = saved_cph.get('enddiastole_hyp', None)
        self.checkBox_Pa.setChecked(pa)
        if type(self.TxtSdyFile) == SDYFile:
            pa_channel = 'pa_physio' if pa else 'pa_trans'
            if self.TxtSdyFile.pa_channel != pa_channel:
                self.TxtSdyFile.pa_channel = pa_channel
                self.TxtSdyFile.parse_data()
                self.plot_txtsdyFile()
        if range_rest:
            self.create_slider_group(rest_or_hyp='rest', range_from=range_rest[0], range_to=range_rest[1])
            self.button_rest.slider_active = True
            self.button_rest.setStyleSheet(f"background-color: 'green'")
        if range_hyp:
            self.create_slider_group(rest_or_hyp='hyp', range_from=range_hyp[0], range_to=range_hyp[1])
            self.button_hyp.slider_active = True
            self.button_hyp.setStyleSheet(f"background-color: 'green'")
        if notch_rest:
            self.create_marker(marker_type='notch_rest', value=notch_rest)
            self.button_notch_rest.slider_active = True
            self.button_notch_rest.setStyleSheet(f"background-color: 'green'")
        if notch_hyp:
            self.create_marker(marker_type='notch_hyp', value=notch_hyp)
            self.button_notch_hyp.slider_active = True
            self.button_notch_hyp.setStyleSheet(f"background-color: 'green'")
        if enddiastole_rest:
            self.create_marker(marker_type='enddiastole_rest', value=enddiastole_rest)
            self.button_enddiastole_rest.slider_active = True
            self.button_enddiastole_rest.setStyleSheet(f"background-color: 'green'")
        if enddiastole_hyp:
            self.create_marker(marker_type='enddiastole_hyp', value=enddiastole_hyp)
            self.button_enddiastole_hyp.slider_active = True
            self.button_enddiastole_hyp.setStyleSheet(f"background-color: 'green'")

    @staticmethod
    def load_cph(filename):
        if os.path.exists(filename):
            with open(filename, 'rb') as f:
                return pickle.load(f)
        else:
            # Save file not found
            return {}

    def perform_calculations(self, save=False):
        self.ensemble_data_rest = self.calculate_ensemble(rest_or_hyp='rest')
        self.ensemble_data_hyp = self.calculate_ensemble(rest_or_hyp='hyp')
        self.calculate()
        self.display_calculations()
        if save:
            self.save_cph()

    def export_study(self):
        study_dict = {}
        for table in [self.tableWidget_Pressures, self.tableWidget_PressureRatios, self.tableWidget_Flows, self.tableWidget_FlowRatios, self.tableWidget_Resistances]:
            for i_row in range(table.rowCount()):
                dict_key, dict_val = [], None
                for i_col in range(table.columnCount()):
                    if i_col != table.columnCount()-1:
                        dict_key.append(table.item(i_row, i_col).text())
                    else:
                        dict_val = table.item(i_row, i_col).text()
                study_dict["_".join(dict_key)] = dict_val
        df = pd.DataFrame([study_dict])
        df.to_csv(self.TxtSdyFile.studypath+".csv")

    def export_all(self):
        print(f"WARNING: Support for exporting all studies in a folder not implemented")

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = LabelUI(MainWindow)
    MainWindow.show()
    print('Showing')
    app.exec_()

max_stars_repo_name: jphdotam/Cophy

max_issues_repo_name: jphdotam/Cophy

max_forks_repo_name: jphdotam/Cophy


text:
from tensorflow.keras.layers import Dense,Flatten, InputLayer
from tensorflow.keras.models import Sequential
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.optimizers import Adam
import numpy as np
from pandas.core.window.indexers import BaseIndexer
import constants

class Model():
    fileNameTrainedNet = "trainednet"
    countInputNeurons = 0
    countOutputNeurons = 3  # neuron0 = drive straight, neuron1 = drive left, neuron2 drive right

    def __init__(self, countInputNeurons):
        self.countInputNeurons = countInputNeurons
        self.model = Sequential([
            #Dense(128, input_shape=(4,), activation='sigmoid'),
            Dense(128, input_shape=(countInputNeurons,), activation='relu'),
            #Dense(16, activation='relu') doesnt work
            #Dense(16, activation='sigmoid')
            Dense(self.countOutputNeurons, activation='softmax')
        ])

        #print(self.model.output_shape)
        print(self.model.summary())

        self.model.compile(
                loss=categorical_crossentropy,
                optimizer=Adam(),
                metrics=['accuracy'])

        print("init neuronal net")
        self.getModelOutput(np.ones((1, self.countInputNeurons)))

    def getFileName(self):
        return self.fileNameTrainedNet + "-" + str(self.countInputNeurons)

    def getFileNameImageDataNP(self):
        return self.fileNameTrainedNet + "-" + str(self.countInputNeurons) + "-" + "ImagesNP.csv"

    def getFileNameTrainingVectorDataNP(self):
        return self.fileNameTrainedNet + "-" + str(self.countInputNeurons) + "-" + "TrainingVectorNP.csv"

    def loadNumpyArrayImageData(self):
        try:
            print("loadNumpyArrayImageData")
            self.dataImagesNA = np.loadtxt(self.getFileNameImageDataNP(), delimiter=',')
            print("self.dataImagesNA.shape")
            print(self.dataImagesNA.shape)
        except:
            print("loading from file " + self.getFileNameImageDataNP() + " not possible" )

    def loadNumpyArrayTrainingVectorData(self):
        try:
            print("loadNumpyArrayTrainingVectorData")
            self.dataTrainingVectorNA = np.loadtxt(self.getFileNameTrainingVectorDataNP(), delimiter=',')
            print("self.dataTrainingVectorNA.shape")
            print(self.dataTrainingVectorNA.shape)
        except:
            print("loading from file " + self.getFileNameTrainingVectorDataNP() + " not possible" )

    def loadModel(self, algoWithFails):
        try:
            print("loadModel weights")
            self.model.load_weights(self.getFileName())
        except:
            print("loading model from file " + self.getFileName() + " not possible" )

        if not algoWithFails:
            self.loadNumpyArrayImageData()
            self.loadNumpyArrayTrainingVectorData()

    def saveNumpyArrayImageData(self):
        try:
            print("saveNumpyArrayImageData")
            if self.dataImagesNA.shape[0] <= 0: return
            np.savetxt(self.getFileNameImageDataNP(), self.dataImagesNA, delimiter=',')
        except:
            print("Saving file " + self.getFileNameImageDataNP() + " not possible" )

    def saveNumpyArrayTrainingVectorData(self):

        try:
            print("saveNumpyArrayTrainingVectorData")
            if self.dataTrainingVectorNA.shape[0] <= 0: return
            np.savetxt(self.getFileNameTrainingVectorDataNP(), self.dataTrainingVectorNA, delimiter=',')
        except:
            print("Saving file " + self.getFileNameTrainingVectorDataNP() + " not possible" )

    def saveModel(self, algoWithFails):
        try:
            print("saveModel weights")
            self.model.save_weights(self.getFileName())
        except:
            print("Saving model from file " + self.getFileName() + " not possible" )

        if not algoWithFails:
            pass
            self.saveNumpyArrayImageData()
            self.saveNumpyArrayTrainingVectorData()

    def getModelOutput(self, inputPara):
        # input = np.zeros((0, self.countInputNeurons))
        output = self.model.predict(inputPara)
        return output

    def trainingWithFails(self, imageData, learningVectorData):
        # mnist example mnist-sacred.train_fc.py
        #x_train.shape
        #(60000, 28, 28, 1)
        #y_train.shape
        # (60000, 10)

        # ai.other.trainRL2on2.py
        #x_train.shape
        #(1, 4)
        #y_train.shape
        #(1, 16)

        print("imageData.shape")
        print(imageData.shape)
        print("learningVectorData.shape")
        print(learningVectorData.shape)

        #imageData.shape
        #(4, 49)
        #learningVectorData.shape
        #(4, 3)

        self.model.train_on_batch(imageData, learningVectorData)
        self.dataImagesNA = imageData
        self.dataTrainingVectorNA = learningVectorData

        # score = model.evaluate(x_test, y_test, verbose=0)

        #print("imageData.shape")
        #print(imageData.shape)
        #ix0 = imageData.shape[0] # number of images
        #ix1 = imageData.shape[1] # x
        #ix2 = imageData.shape[2] # y
        #print(ix0)
        #print(ix1)
        #print(ix2)

        #print("learningVectorData.shape")
        #print(learningVectorData.shape)
        #ax0 = learningVectorData.shape[0] # number of images
        #ax1 = learningVectorData.shape[1] # x
        #ax2 = learningVectorData.shape[2] # y
        #print(ax0)
        #print(ax1)
        #print(ax2)

        #print(learningVectorData[0])
        #print(learningVectorData[1])

        #for img in range(0, ax0):
        #    self.printNP(imageData[img])
        #    print(learningVectorData[img])

    def imageEquals(self, image1, image2):
        comparison = image1 == image2
        return comparison.all()

    def imageExistInImageData(self, image):
        rows = self.dataImagesNA.shape[0]

        for row in range(0, rows):
            if self.imageEquals(image, self.dataImagesNA[row]):
                #print("found duplicate")
                return True

        #print("found no duplicate")
        return False

    def appendWithoutDuplicates(self, imageData, learningVectorData):

        rows = imageData.shape[0]
        for row in range(0, rows):
            image = imageData[row]
            learnVector = learningVectorData[row]

            imageDataList = []
            learningVectorList = []

            #print("learnVector.shape")
            #print(learnVector.shape)

            if not self.imageExistInImageData(image):
                imageDataList.append(image)
                learningVectorList.append(learnVector)

                imageDataAppend = np.array(imageDataList)
                learningVectorAppend = np.array(learningVectorList)

                self.dataImagesNA = np.append(self.dataImagesNA, imageDataAppend, axis=0)
                self.dataTrainingVectorNA = np.append(self.dataTrainingVectorNA, learningVectorAppend, axis=0)

    def appendTrainingData(self, imageData, imageDataSize, learningVectorData, learningVectorSize):
        if hasattr(self, 'dataTrainingVectorNA'):
            #print("self.dataTrainingVectorNA exist")
            pass
        else:
            #print("self.dataTrainingVectorNA doesnt exist")
            self.dataImagesNA         = np.empty((0, imageDataSize), float)
            self.dataTrainingVectorNA = np.empty((0, learningVectorSize), float)

        rows = self.dataImagesNA.shape[0]
        print("count rows training data before adding: " + str(rows))

        self.appendWithoutDuplicates(imageData, learningVectorData)

        rows = self.dataImagesNA.shape[0]
        print("count rows training data after adding:  " + str(rows))

    def training(self, imageData, imageDataSize, learningVectorData, learningVectorSize, algoWithFails):
        verbose = False
        if algoWithFails:
            if verbose:
                rows = learningVectorData.shape[0]
                for row in range(0, rows):
                    print(" " + str(constants.imageSumVector(imageData[row])) + " ", end="")
                    print(learningVectorData[row])

            self.model.fit(imageData, learningVectorData, epochs=10, verbose=False)
            #loss, accuracy = self.model.train_on_batch(imageData, learningVectorData)

            loss, accuracy = self.model.evaluate(imageData, learningVectorData)
            print("accuracy: " + str(accuracy))
        else:
            self.appendTrainingData(imageData, imageDataSize, learningVectorData, learningVectorSize)
            if verbose:
                rows = self.dataImagesNA.shape[0]
                for row in range(0, rows):
                    print(" " + str(constants.imageSumVector(self.dataImagesNA[row])) + " ", end="")
                    print(self.dataTrainingVectorNA[row])

            self.model.fit(self.dataImagesNA, self.dataTrainingVectorNA, epochs=10, verbose=False)

            loss, accuracy = self.model.evaluate(self.dataImagesNA, self.dataTrainingVectorNA)
            print("accuracy: " + str(accuracy))