text:
import numpy as np

SUB_SHARE = 0.5

TRAIN_DE = '../data/MTTT/tok/ted_train_en-de.tok.clean.de.txt'
TRAIN_EN = '../data/MTTT/tok/ted_train_en-de.tok.clean.en.txt'
DEV_DE = '../data/MTTT/tok/ted_dev_en-de.tok.de.txt'
DEV_EN = '../data/MTTT/tok/ted_dev_en-de.tok.en.txt'
TEST_DE = '../data/MTTT/tok/ted_test1_en-de.tok.de.txt'
TEST_EN = '../data/MTTT/tok/ted_test1_en-de.tok.en.txt'

SUB_TRAIN_DE = '../data/MTTT/tok/sub_ted_train_en-de.tok.clean.de.txt'
SUB_TRAIN_EN = '../data/MTTT/tok/sub_ted_train_en-de.tok.clean.en.txt'
SUB_DEV_DE = '../data/MTTT/tok/sub_ted_dev_en-de.tok.de.txt'
SUB_DEV_EN = '../data/MTTT/tok/sub_ted_dev_en-de.tok.en.txt'
SUB_TEST_DE = '../data/MTTT/tok/sub_ted_test1_en-de.tok.de.txt'
SUB_TEST_EN = '../data/MTTT/tok/sub_ted_test1_en-de.tok.en.txt'

np.random.seed(0)

#Split func

def split(src_de,src_en,dest_de,dest_en,share=SUB_SHARE):

    print(src_de)
    print(src_en)

    with open(src_de) as f:
        lines_de = f.readlines()

    with open(src_en) as f:
        lines_en = f.readlines()

    N = len(lines_de)
    assert(N==len(lines_en))
    print(N)
    print(share)

    lens = []
    for i in range(N):
        lens.append(0.5*len(lines_de[i])+0.5*len(lines_en[i]))

    lens = np.array(lens)
    pct = np.percentile(lens,[0,25,50,75,100])
    print(pct)

    # obtain indices per quantile
    idxs_per_quantile = {j: [] for j in range(4)}

    for i in range(N):
        for j in range(len(idxs_per_quantile)):
            #print(j,pct[j],pct[j+1])
            if lens[i] >= pct[j] and lens[i] < pct[j+1]+0.5:
                idxs_per_quantile[j].append(i)

    #print(idxs_per_quantile)
    sub_idx = []

    for j in range(4):
        np.random.shuffle(idxs_per_quantile[j])
        subN = int(share * len(idxs_per_quantile[j]))
        print(subN)
        sub_idx.extend(idxs_per_quantile[j][:subN])

    print(len(sub_idx))

    with open(dest_de, "w") as f:
        for i in range(N):
            if i in sub_idx:
                f.write(lines_de[i])

    with open(dest_en, "w") as f:
        for i in range(N):
            if i in sub_idx:
                f.write(lines_en[i])

# Process files
split(TRAIN_DE,TRAIN_EN,SUB_TRAIN_DE,SUB_TRAIN_EN)
split(DEV_DE,DEV_EN,SUB_DEV_DE,SUB_DEV_EN)
split(TEST_DE,TEST_EN,SUB_TEST_DE,SUB_TEST_EN)

max_stars_repo_name: searchivarius/OpenNMT-py

max_issues_repo_name: searchivarius/OpenNMT-py

max_forks_repo_name: searchivarius/OpenNMT-py


text:
"""
Embedded Python Blocks:

Each time this file is saved, GRC will instantiate the first class it finds
to get ports and parameters of your block. The arguments to __init__  will
be the parameters. All of them are required to have default values!
"""

import numpy as np
import pylab
from gnuradio import gr
import pmt

class msg_block(gr.basic_block):  # other base classes are basic_block, decim_block, interp_block
    """This block converts strings from the QT GUI Message Edit Box to uint8 vectors"""

    def __init__(self):  # only default arguments here
        """arguments to this function show up as parameters in GRC"""
        gr.basic_block.__init__(
            self,
            name='Embedded Python Block',   # will show up in GRC
            in_sig=None,
            out_sig=None
        )
        self.message_port_register_out(pmt.intern('msg_out'))
        self.message_port_register_in(pmt.intern('msg_in'))
        self.set_msg_handler(pmt.intern('msg_in'), self.handle_msg)

    def handle_msg(self, msg):
        nvec = pmt.to_python(msg)
        self.message_port_pub(pmt.intern('msg_out'), pmt.cons(pmt.make_dict(), pmt.pmt_to_python.numpy_to_uvector(np.array([ord(c) for c in nvec], np.uint8))))

    def work(self, input_items, output_items):
        pass

ext: py

reason: import numpy


text:
"""
Purpose: To compute all pairs Wasserstein on 1D distributions of periodicity
scores for each cell for each gray level.
Also clustering on this
"""
import numpy as np
from scipy import sparse
import scipy.io as sio
import matplotlib.pyplot as plt
from ripser import ripser
from persim import plot_diagrams as plot_dgms
from SlidingWindow import *
from Periodicity import *
from scipy.cluster.hierarchy import dendrogram, linkage, fcluster
from scipy.stats import wasserstein_distance

def getWasserstein(X, dim = 30):
    """
    Compute all pairs of Wasserstein distances between
    distributions of a list of values, assuming that
    all of the lists have the same number of values so that
    the CDF is easily invertible
    Parameters
    ----------
    X: ndarray(N, M)
        N lists, each with M unordered samples
    Returns
    -------
    D: ndarray(N, N)
        All pairs 1D Wasserstein distances between lists in X
    """
    bins = np.linspace(0, 1, dim)
    N = X.shape[0]
    D = np.zeros((N, N))
    for i in range(N):
        if i%250 == 0:
            print("%i of %i"%(i, N))
        histi, _ = np.histogram(X[i, :], bins=bins)
        for j in range(i+1, N):
            histj, _ = np.histogram(X[j, :], bins=bins)
            D[i, j] = wasserstein_distance(histi, histj)
    D += D.T
    return D

def getWassersteinSw1PerSBlocks(scoretype="pitch"):
    """
    Compute the Wasserstein distances between all pairs of
    distributions of block scores for each time series over
    all gray levels
    Parameters
    ----------
    scoretype: string
        The type of scores we're looking at
        "pitch" for autocorrelation or "tda" for Sw1PerS
    Returns
    -------
    D: ndarray(N, N)
        All pairs Wasserstein distances between time series
    grays: ndarray(N)
        The gray levels of each row/column of D
    X: ndarray(N, NBlocks)
        All scores for all blocks
    allndivs: ndarray(N)
        The number of divisions of each row/column of D
    """
    X = np.array([])
    grays = np.array([])
    allndivs = np.array([])
    for t in [0, 2, 4, 10]:
        scores = np.loadtxt("data/p53_%iGy_%s_scores.txt"%(t, scoretype))
        divs = np.loadtxt("data/div_%iGy.txt"%t)
        ndivs = np.sum(divs, 1)
        valid = np.sum(np.isnan(scores), 1) == 0
        scores = scores[valid == 1, :]
        ndivs = ndivs[valid == 1]
        idx = np.argsort(ndivs)
        ndivs = ndivs[idx]
        scores = scores[idx, :]
        if X.size == 0:
            X = scores
        else:
            X = np.concatenate((X, scores), 0)
        grays = np.concatenate((grays, np.array([t]*scores.shape[0])))
        allndivs = np.concatenate((allndivs, ndivs))
    D = getWasserstein(X)
    return D, X, grays, allndivs

def getWassersteinSw1PerSBlocks_July2019(scoretype="pitch"):
    """
    Compute the Wasserstein distances between all pairs of
    distributions of block scores for each time series over
    all gray levels, for the July 2019 dataset
    Parameters
    ----------
    scoretype: string
        The type of scores we're looking at
        "pitch" for autocorrelation or "tda" for Sw1PerS
    Returns
    -------
    D: ndarray(N, N)
        All pairs Wasserstein distances between time series
    grays: ndarray(N)
        The gray levels of each row/column of D
    X: ndarray(N, NBlocks)
        All scores for all blocks
    allndivs: ndarray(N)
        The number of divisions of each row/column of D
    """
    X = np.array([])
    grays = np.array([])
    allndivs = np.array([])
    X = np.loadtxt("data_july_2019/p53_%s_scores.txt"%scoretype)
    divs = np.loadtxt("data_july_2019/divisions.txt")
    divs = divs[:, 0:240]
    allndivs = np.sum(divs, 1)
    idx = np.argsort(allndivs)
    allndivs = allndivs[idx]
    X = X[idx, :]
    fin = open("data_july_2019/p53_indices.txt")
    grayidxs = [int(s) for s in fin.readlines()[0].split()]
    fin.close()
    grays = np.zeros(X.shape[0])
    for i, v in enumerate([0, 0.5, 1, 2, 4, 8]):
        grays[grayidxs[i]:grayidxs[i+1]] = v
    grays = grays[idx]
    D = getWasserstein(X)
    return D, X, grays, allndivs

def getEuclideanAutocorrelation():
    """
    Compute the Euclidean distances and autocorrelation
    between all pairs of time series
    Returns
    -------
    D: ndarray(N, N)
        All pairs Wasserstein distances between time series
    grays: ndarray(N)
        The gray levels of each row/column of D
    allndivs: ndarray(N)
        The number of divisions of each row/column of D
    """
    X = np.array([])
    grays = np.array([])
    allndivs = np.array([])
    for t in [0, 2, 4, 10]:
        Xt = np.loadtxt("data/p53_%iGy.txt"%t)
        divs = np.loadtxt("data/div_%iGy.txt"%t)
        ndivs = np.sum(divs, 1)
        valid = np.sum(Xt == -1, 1) == 0
        Xt = Xt[valid == 1, :]
        ndivs = ndivs[valid == 1]
        idx = np.argsort(ndivs)
        ndivs = ndivs[idx]
        Xt = Xt[idx, :]
        if X.size == 0:
            X = Xt
        else:
            X = np.concatenate((X, Xt), 0)
        grays = np.concatenate((grays, np.array([t]*Xt.shape[0])))
        allndivs = np.concatenate((allndivs, ndivs))
    N = X.shape[0]
    Win = 11
    XA = []
    for i in range(N):
        xk = X[i, :]
        xk = detrend_timeseries(xk, Win)
        X[i, :] = xk
        xk = xk - np.mean(xk)
        y = np.correlate(xk, xk, 'full')
        y = y[y.size//2:]
        y /= y[0]
        XA.append(y)
    XA = np.array(XA)
    D = getSSM(X)
    DA = getSSM(XA)
    return D, DA, X, grays, allndivs

def make_clusterplots(D, X, grays, allndivs, nclusters = 4, euclidean=False, filename="Clusters.png"):
    """
    Make Caroline's cluster plots and save them to a file "Clusters.png"
    Parameters
    ----------
    D: ndarray(N, N)
        A distance matrix between all distributions
    X: ndarray(N, K)
        A set of K samples in each distribution for each time series
    grays: ndarray(N)
        Gray levels for each time series
    allndivs: ndarray(N)
        Number of divisions for each time series
    nclusters: int
        Number of clusters to perform
    eucildean: boolean
        Whether to treat the rows of the distance matrix as
        Euclidean coordinates or whether to use the raw distance matrix
    filename: string
        Name of file to which to save plot
    """
    N = D.shape[0]
    if euclidean:
        Z = linkage(D, 'ward')
    else:
        Z = linkage(D[np.triu_indices(N, 1)], 'ward')
    cluster_idxs = fcluster(Z, nclusters, criterion='maxclust')
    allcounts = []
    labels = []
    allidxs = [] # Indices of time series in each gray/divs group
    grays = grays.flatten()
    allndivs = allndivs.flatten()
    for gray in np.unique(grays):
        for ndivs in np.unique(allndivs):
            idxs = cluster_idxs[(grays == gray)*(allndivs == ndivs)]
            if len(idxs) == 0:
                continue
            allidxs.append(idxs)
            counts = [np.sum(idxs == c) for c in range(1, nclusters+1)]
            allcounts.append(counts)
            labels.append("%g/%g"%(gray, ndivs))
    allcounts = np.array(allcounts, dtype=float).T
    idx = np.argmax(allcounts, 0)
    allcountsmax = np.zeros(allcounts.shape, dtype=int)
    allcountsmax[idx, np.arange(allcounts.shape[1])] = 1
    allcounts /= np.sum(allcounts, 0)[None, :]
    # Compute means for each type
    means_per_type = []
    for i, l in enumerate(labels):
        gray, ndivs = [float(s) for s in l.split("/")]
        x = X[(grays == gray)*(allndivs == ndivs), :]
        means_per_type.append(np.mean(x))
    # Compute means for each cluster
    means_per_cluster = np.zeros(allcountsmax.shape[0])
    for i in range(allcountsmax.shape[0]):
        x = np.array([])
        for j in range(allcountsmax.shape[1]):
            if allcountsmax[i, j] > 0:
                gray, ndivs = [float(s) for s in labels[j].split("/")]
                xj = X[(grays == gray)*(allndivs == ndivs)]
                if x.size == 0:
                    x = xj
                else:
                    x = np.concatenate((x, xj), 0)
        means_per_cluster[i] = np.mean(x.flatten())
    # Sort clusters in ascending order of mean
    idx = np.argsort(means_per_cluster)
    means_per_cluster = means_per_cluster[idx]
    allcounts = allcounts[idx, :]
    allcountsmax = allcountsmax[idx, :]
    # Plot the results
    plt.figure(figsize=(20, 6))
    plt.subplot(211)
    plt.imshow(allcounts, vmin=0, vmax=1)
    plt.xticks(np.arange(allcounts.shape[1]), labels)
    plt.subplot(212)
    plt.imshow(allcountsmax)
    plt.xticks(np.arange(allcounts.shape[1]), ["%s\n%.3g"%(l, m) for (l, m) in zip(labels, means_per_type)])
    plt.yticks(np.arange(len(means_per_cluster)), ["  %.3g  "%m for m in means_per_cluster])
    plt.tight_layout()
    plt.savefig(filename, bbox_inches='tight')

def exclude_gray_div_types(D, X, grays, allndivs, types):
    """
    Exclude some types of gray/division combinations from the data
    Parameters
    ----------
    D: ndarray(N, N)
        A distance matrix between all distributions
    X: ndarray(N, K)
        A set of K samples in each distribution for each time series
    grays: ndarray(N)
        Gray levels for each time series
    allndivs: ndarray(N)
        Number of divisions for each time series
    types: list of tuple(gray, div)
        The types to exlude
    Returns
    -------
    D, X, grays, allndivs
        The above variables without gray/div types
    """
    grays = grays.flatten()
    allndivs = allndivs.flatten()
    for (gray, div) in types:
        idx = np.arange(D.shape[0])
        idx = idx[(grays == gray)*(allndivs == div) == 0]
        D = D[idx, :]
        D = D[:, idx]
        X = X[idx, :]
        grays = grays[idx]
        allndivs = allndivs[idx]
    return D, X, grays, allndivs

def testEuclideanAutocorrelation():
    D, DA, X, grays, allndivs = getEuclideanAutocorrelation()
    make_clusterplots(D, X, grays, allndivs, nclusters=3, filename="EuclideanClusters.png")
    make_clusterplots(DA, X, grays, allndivs, nclusters=3, filename="AutocorrelationClusters.png")

if __name__ == '__main__':
    D, X, grays, allndivs = getWassersteinSw1PerSBlocks("tda")
    sio.savemat("tda.mat", {"D":D, "X":X, "grays":grays, "allndivs":allndivs})
    """
    res = sio.loadmat("pitch.mat")
    D, X, grays, allndivs = res["D"], res["X"], res["grays"], res["allndivs"]
    make_clusterplots(D, X, grays, allndivs, nclusters=3, filename="ClustersPitch.png")
    """

if __name__ == '__main__2':
    #D, X, grays, allndivs = getWassersteinSw1PerSBlocks_July2019("tda")
    #sio.savemat("tda_july_2019.mat", {"D":D, "X":X, "grays":grays, "allndivs":allndivs})
    res = sio.loadmat("tda_july_2019.mat")
    D, X, grays, allndivs = res["D"], res["X"], res["grays"], res["allndivs"]
    #D, X, grays, allndivs = exclude_gray_div_types(D, X, grays, allndivs, [(4, 3), (2, 7)])
    nclusters = 4
    make_clusterplots(D, X, grays, allndivs, nclusters=nclusters, filename="ClustersPitch_July_2019_%iClusters.png"%nclusters)
    nclusters = 6
    make_clusterplots(D, X, grays, allndivs, nclusters=nclusters, filename="ClustersPitch_July_2019_%iClusters.png"%nclusters)

reason: import numpy,import scipy,from scipy


text:
import pygame as pg
from pygame.locals import *
from settings import *
from snake import Snake
import random
import sys
import train
import numpy as np
import json
from tempfile import TemporaryFile

train_ = False
if not train_:
    with open('stats.json', 'r') as fp:
        stats = json.load(fp)
        if "generation" in stats and "max_score" in stats:
            generation = stats["generation"]
            max_score = stats["max_score"]
        else:
            generation = 0
            max_score = 0
else:
    generation = 0
    max_score = 0

class Game:
    def __init__(self):

        self.money = pg.image.load("images/money.png")
        self.swiss = pg.image.load("images/swiss.png")
        self.moneyrect = self.money.get_rect()
        self.swissrect = self.swiss.get_rect()

        pg.init()
        self.FOOD = []
        self.DISPLAY = pg.display.set_mode(SIZE)
        self.FONT = pg.font.Font("fonts/RobotoMono-Medium.ttf", 20)

        pg.time.set_timer(USEREVENT + 1, 1) # move snake
        pg.time.set_timer(USEREVENT + 2, 15) # Create Food
        pg.time.set_timer(USEREVENT + 3, 20000) # Save weigths

        self.Player1 = Snake(color = COLORS["white"], up = pg.K_UP, down = pg.K_DOWN, right = pg.K_RIGHT, left = K_LEFT, display = self.DISPLAY, game_over = self.game_over)
        self.createFood()

        global max_score, generation
        while True:
            self.DISPLAY.fill(COLORS['black'])
            for event in pg.event.get():
                if event.type == QUIT:
                    pg.quit()
                    sys.exit()
                if event.type == pg.KEYDOWN:
                    self.Player1.newEvent(event)
                if event.type == USEREVENT + 1:
                    pass
                if event.type == USEREVENT + 2:
                    self.createFood()
                if event.type == USEREVENT + 3:
                    if not train_:
                        with open('storage.json', 'w') as fp:
                            json.dump(train.storage, fp)
                        with open('stats.json', 'w') as fp:
                            json.dump({'max_score': max_score, 'generation': generation}, fp)
                        np.savetxt("Q.txt", train.Q)

            action = train.chooseAction(self.Player1, self.FOOD)
            self.Player1.act(action)
            self.Player1.moveSnake()

            self.Player1.update(self.FOOD, self.DISPLAY, self.swiss, self.swissrect)
            if not self.FOOD:
                self.createFood()

            for pos in self.FOOD:
                self.moneyrect.x = pos[0] * TILE_SIZE
                self.moneyrect.y = pos[1] * TILE_SIZE
                self.DISPLAY.blit(self.money, self.moneyrect)
                #self.drawFood(pos[0] * TILE_SIZE, pos[1] * TILE_SIZE)

            if not train_:
                score = self.FONT.render('Score: ' + str(self.Player1.SCORE), True, COLORS["text"])
                self.DISPLAY.blit(score, (10, 5))

                # global max_score, generation
                maxScore = self.FONT.render('Max score: ' + str(max_score), True, COLORS["text"])
                self.DISPLAY.blit(maxScore, (10, 30))

                epoch = self.FONT.render('Generation: ' + str(generation), True, COLORS["text"])
                self.DISPLAY.blit(epoch, (10, 55))

            pg.display.update()

    def drawFood(self, x,y):
        pg.draw.rect(self.DISPLAY,COLORS['red'],(x,y,TILE_SIZE,TILE_SIZE))

    def createFood(self):
        if len(self.FOOD) == 0:
            newPos = self.Player1.POSITIONS[0]
            while newPos in self.Player1.POSITIONS:
                newPos = (random.randint(1,(WIDTH//TILE_SIZE) -1),random.randint(1,(HEIGHT//TILE_SIZE) -1))

            self.FOOD.append(newPos)

    def getScore(self):
        return self.Player1.SCORE

    def game_over(self):
        global max_score
        if self.Player1.SCORE > max_score:
            max_score = self.Player1.SCORE
        global generation
        generation += 1
        self.Player1 = Snake(color = COLORS["white"], up = pg.K_UP, down = pg.K_DOWN, right = pg.K_RIGHT, left = K_LEFT, display = self.DISPLAY, game_over = self.game_over)

if __name__== "__main__":
    GAME = Game()

max_stars_repo_name: bernatixer/BeatThePython

max_issues_repo_name: bernatixer/BeatThePython

max_forks_repo_name: bernatixer/BeatThePython


text:
import torch
import sys
from torch.utils import data # 获取迭代数据
from torch.autograd import Variable # 获取变量
import torchvision
from torchvision.datasets import mnist # 获取数据集
import matplotlib.pyplot as plt
import numpy as np
data_tf = torchvision.transforms.Compose(
    [
        torchvision.transforms.ToTensor(),
        torchvision.transforms.Normalize([0.5],[0.5])
    ]
)

data_path = r'C:\Users\15247\PycharmProjects\Task2'
# 获取数据集
train_data = mnist.MNIST(data_path,train=True,transform=data_tf,download=False)
test_data = mnist.MNIST(data_path,train=False,transform=data_tf,download=False)

train_loader = data.DataLoader(train_data,batch_size=128,shuffle=True)
test_loader = data.DataLoader(test_data,batch_size=100,shuffle=True)

class CNNnet(torch.nn.Module):
    def __init__(self):
        super(CNNnet,self).__init__()
        self.conv1 = torch.nn.Sequential(
            torch.nn.Conv2d(in_channels=1,
                            out_channels=16,
                            kernel_size=3,
                            stride=2,
                            padding=1),
            torch.nn.BatchNorm2d(16),
            torch.nn.ReLU()
        )
        self.conv2 = torch.nn.Sequential(
            torch.nn.Conv2d(16,32,3,2,1),
            torch.nn.BatchNorm2d(32),
            torch.nn.ReLU()
        )
        self.conv3 = torch.nn.Sequential(
            torch.nn.Conv2d(32,64,3,2,1),
            torch.nn.BatchNorm2d(64),
            torch.nn.ReLU()
        )
        self.conv4 = torch.nn.Sequential(
            torch.nn.Conv2d(64,64,2,2,0),
            torch.nn.BatchNorm2d(64),
            torch.nn.ReLU()
        )
        self.mlp1 = torch.nn.Linear(2*2*64,100)
        self.mlp2 = torch.nn.Linear(100,10)
    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        x = self.conv4(x)
        x = self.mlp1(x.view(x.size(0),-1))
        x = self.mlp2(x)
        return x
model = CNNnet()
print(model)
loss_func = torch.nn.CrossEntropyLoss()
opt = torch.optim.Adam(model.parameters(),lr=0.001)
loss_count = []
for epoch in range(2):
    for i,(x,y) in enumerate(train_loader):
        batch_x = Variable(x) # torch.Size([128, 1, 28, 28])
        batch_y = Variable(y) # torch.Size([128])
        # 获取最后输出
        out = model(batch_x) # torch.Size([128,10])
        # 获取损失
        loss = loss_func(out,batch_y)
        # 使用优化器优化损失
        opt.zero_grad()  # 清空上一步残余更新参数值
        loss.backward() # 误差反向传播，计算参数更新值
        opt.step() # 将参数更新值施加到net的parmeters上
        if i%20 == 0:
            loss_count.append(loss.item())
            print('{}:\t'.format(i), loss.item())
            torch.save(model,r'C:\Users\15247\PycharmProjects\Task2\log_CNN')
        if i % 100 == 0:
            for a,b in test_loader:
                test_x = Variable(a)
                test_y = Variable(b)
                out = model(test_x)
                # print('test_out:\t',torch.max(out,1)[1])
                # print('test_y:\t',test_y)
                accuracy = torch.max(out,1)[1].numpy() == test_y.numpy()
                print('accuracy:\t',accuracy.mean())
                break
plt.figure('PyTorch_CNN_Loss')
plt.plot(np.array(loss_count),label='Loss')
plt.legend()
plt.show()
def one_hot(data):
    hot = np.zeros([10])
    hot[data] = 1
    return hot
# 并且在计算损失前，加入下面一行代码，将真实值转为one-hot形式
y = [one_hot(i) for i in y]
model = torch.load(r'C:\Users\15247\PycharmProjects\Task2\log_CNN')
accuracy_sum = []
for i,(test_x,test_y) in enumerate(test_loader):
    test_x = Variable(test_x)
    test_y = Variable(test_y)
    out = model(test_x)
    # print('test_out:\t',torch.max(out,1)[1])
    # print('test_y:\t',test_y)
    accuracy = torch.max(out,1)[1].numpy() == test_y.numpy()
    accuracy_sum.append(accuracy.mean())
    print('accuracy:\t',accuracy.mean())

print('总准确率：\t',sum(accuracy_sum)/len(accuracy_sum))
# 精确率图
print('总准确率：\t',sum(accuracy_sum)/len(accuracy_sum))
plt.figure('Accuracy')
plt.plot(accuracy_sum,'o',label='accuracy')
plt.title('Pytorch_CNN_Accuracy')
plt.legend()
plt.show()


text:
import tensorflow as tf
import numpy as np

class W2V_LEX_CNN_MC(object):
    """
    A CNN for text classification. - multi channel version
    Uses an embedding layer, followed by a convolutional, max-pooling and softmax layer.
    """
    def __init__(
            self, sequence_length, num_classes,
            embedding_size, filter_sizes, num_filters, l2_reg_lambda=0.0, l1_reg_lambda=0.0):

        # Placeholders for input, output and dropout
        self.input_x = tf.placeholder(tf.float32, [None, sequence_length, embedding_size, 2], name="input_x")
        self.input_y = tf.placeholder(tf.float32, [None, num_classes], name="input_y")
        self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")

        # Keeping track of l2 regularization loss (optional)
        l2_loss = tf.constant(0.0)
        l1_loss = tf.constant(0.0)

        # Embedding layer
        with tf.device('/cpu:0'), tf.name_scope("embedding"):
            self.embedded_chars = self.input_x
            # self.embedded_chars_expanded = self.embedded_chars
            # self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1)
            # print '[self.embedded_chars_expanded]', self.embedded_chars_expanded

        # Create a convolution + maxpool layer for each filter size
        pooled_outputs = []
        for i, filter_size in enumerate(filter_sizes):
            with tf.name_scope("conv-maxpool-%s" % filter_size):
                # Convolution Layer
                filter_shape = [filter_size, embedding_size, 2, num_filters]
                W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
                b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b")

                # l2_loss += tf.nn.l2_loss(W)/1000
                # l2_loss += tf.nn.l2_loss(b)/1000

                conv = tf.nn.conv2d(
                    self.embedded_chars,
                    W,
                    strides=[1, 1, 1, 1],
                    padding="VALID",
                    name="conv")
                # Apply nonlinearity
                h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
                # Maxpooling over the outputs
                pooled = tf.nn.max_pool(
                    h,
                    ksize=[1, sequence_length - filter_size + 1, 1, 1],
                    strides=[1, 1, 1, 1],
                    padding='VALID',
                    name="pool")
                pooled_outputs.append(pooled)

        # Combine all the pooled features
        num_filters_total = num_filters * len(filter_sizes)
        self.h_pool = tf.concat(3, pooled_outputs)
        self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])

        # Add dropout
        with tf.name_scope("dropout"):
            self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)

        # Final (unnormalized) scores and predictions
        with tf.name_scope("output"):
            W = tf.get_variable(
                "W",
                shape=[num_filters_total, num_classes],
                initializer=tf.contrib.layers.xavier_initializer())
            b = tf.Variable(tf.constant(0.1, shape=[num_classes]), name="b")
            l2_loss += tf.nn.l2_loss(W)/30
            l2_loss += tf.nn.l2_loss(b)/30
            l1_loss += tf.reduce_sum(tf.abs(W))
            l1_loss += tf.reduce_sum(tf.abs(b))
            self._b = b
            self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores")
            self.predictions = tf.argmax(self.scores, 1, name="predictions")

        # CalculateMean cross-entropy loss
        with tf.name_scope("loss"):
            losses = tf.nn.softmax_cross_entropy_with_logits(self.scores, self.input_y)
            self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss + l1_reg_lambda*l1_loss

        # Accuracy
        with tf.name_scope("accuracy"):
            self.golds = tf.argmax(self.input_y, 1, name="golds")
            correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")

        with tf.name_scope("avg_f1"):
            self.golds = tf.argmax(self.input_y, 1, name="golds")
            self.preds = self.predictions

            # positive recall
            pos_gold_sel = tf.equal(self.golds, 2)  # positive_gold
            posg_golds = tf.boolean_mask(self.golds, pos_gold_sel)
            posg_preds = tf.boolean_mask(self.preds, pos_gold_sel)
            correct_predictions_pr = tf.equal(posg_golds, posg_preds)
            pos_r = tf.reduce_mean(tf.cast(correct_predictions_pr, "float"), name="pos_recall")

            # positive precision
            pos_pred_sel = tf.equal(self.preds, 2)  # positive_pred
            posp_golds = tf.boolean_mask(self.golds, pos_pred_sel)
            posp_preds = tf.boolean_mask(self.preds, pos_pred_sel)
            correct_predictions_pp = tf.equal(posp_golds, posp_preds)
            pos_p = tf.reduce_mean(tf.cast(correct_predictions_pp, "float"), name="pos_precision")

            # negative recall
            neg_gold_sel = tf.equal(self.golds, 0)  # positive_gold
            negg_golds = tf.boolean_mask(self.golds, neg_gold_sel)
            negg_preds = tf.boolean_mask(self.preds, neg_gold_sel)
            correct_predictions_nr = tf.equal(negg_golds, negg_preds)
            self.neg_r = tf.reduce_mean(tf.cast(correct_predictions_nr, "float"), name="neg_recall")

            # negative precision
            neg_pred_sel = tf.equal(self.preds, 0)  # positive_pred
            negp_golds = tf.boolean_mask(self.golds, neg_pred_sel)
            negp_preds = tf.boolean_mask(self.preds, neg_pred_sel)
            correct_predictions_np = tf.equal(negp_golds, negp_preds)
            self.neg_p = tf.reduce_mean(tf.cast(correct_predictions_np, "float"), name="neg_precision")

            self.f1_neg = 2 * self.neg_p * self.neg_r / (self.neg_p + self.neg_r) * 100
            self.f1_pos = 2 * pos_p * pos_r / (pos_p + pos_r) * 100

            self.avg_f1 = (self.f1_neg + self.f1_pos) / 2

class W2V_LEX_CNN_MC_A2V(object):
    """
    A CNN for text classification. - multi channel version
    Uses an embedding layer, followed by a convolutional, max-pooling and softmax layer.
    """
    def __init__(
            self, sequence_length, num_classes,
            embedding_size, filter_sizes, num_filters, embedding_size_lex,
            attention_depth_w2v, attention_depth_lex,
            l2_reg_lambda=0.0, l1_reg_lambda=0.0):

        # Placeholders for input, output and dropout
        self.input_x_2c = tf.placeholder(tf.float32, [None, sequence_length, embedding_size, 2], name="input_x_2c")
        self.input_x = tf.placeholder(tf.float32, [None, sequence_length, embedding_size], name="input_x")
        self.input_y = tf.placeholder(tf.float32, [None, num_classes], name="input_y")
        self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")

        self.input_x_lexicon = tf.placeholder(tf.float32, [None, sequence_length, embedding_size_lex],
                                              name="input_x_lexicon")

        # Keeping track of l2 regularization loss (optional)
        l2_loss = tf.constant(0.0)
        l1_loss = tf.constant(0.0)

        # Embedding layer
        with tf.device('/cpu:0'), tf.name_scope("embedding"):
            self.embedded_chars = self.input_x
            self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1)
            print self.embedded_chars_expanded

            # lexicon embedding
            self.embedded_chars_lexicon = self.input_x_lexicon
            self.embedded_chars_expanded_lexicon = tf.expand_dims(self.embedded_chars_lexicon, -1)

            print '[self.embedded_chars]', self.embedded_chars
            print '[self.embedded_chars_expanded]', self.embedded_chars_expanded

            print '[self.embedded_chars_lexicon]', self.embedded_chars_lexicon
            print '[self.embedded_chars_expanded_lexicon]', self.embedded_chars_expanded_lexicon

        attention_outputs = []
        with tf.name_scope("pre-attention"):
            U_shape = [embedding_size, attention_depth_w2v]  # (400, 60)
            self.U_w2v = tf.Variable(tf.truncated_normal(U_shape, stddev=0.1), name="U_w2v")
            U_shape = [embedding_size_lex, attention_depth_lex]  # (15, 60)
            self.U_lex = tf.Variable(tf.truncated_normal(U_shape, stddev=0.1), name="U_lex")

            self.embedded_chars_tr = tf.batch_matrix_transpose(self.embedded_chars)
            self.embedded_chars_lexicon_tr = tf.batch_matrix_transpose(self.embedded_chars_lexicon)
            print '[self.embedded_chars_lexicon_tr]', self.embedded_chars_lexicon_tr

            def fn_matmul_w2v(previous_output, current_input):
                print(current_input.get_shape())
                current_ouput = tf.matmul(current_input, self.U_w2v)
                print 'previous_output', previous_output
                print 'current_ouput', current_ouput
                return current_ouput

            def fn_matmul_lex(previous_output, current_input):
                print(current_input.get_shape())
                current_ouput = tf.matmul(current_input, self.U_lex)
                print 'previous_output', previous_output
                print 'current_ouput', current_ouput
                return current_ouput

            initializer = tf.constant(np.zeros([sequence_length, attention_depth_w2v]), dtype=tf.float32)
            WU_w2v = tf.scan(fn_matmul_w2v, self.embedded_chars, initializer=initializer)
            print '[WU_w2v]', WU_w2v

            initializer = tf.constant(np.zeros([sequence_length, attention_depth_lex]), dtype=tf.float32)
            LU_lex = tf.scan(fn_matmul_lex, self.embedded_chars_lexicon, initializer=initializer)
            print '[LU_lex]', LU_lex

            WU_w2v_expanded = tf.expand_dims(WU_w2v, -1)
            print '[WU_w2v_expanded]', WU_w2v_expanded  # (?, 60(seq_len), 60(depth), 1)

            w2v_pool = tf.nn.max_pool(
                WU_w2v_expanded,
                ksize=[1, 1, attention_depth_w2v, 1],
                strides=[1, 1, 1, 1],
                padding='VALID',
                name="w2v_pool")

            print '[w2v_pool]', w2v_pool  # (?, 60(seq_len), 1, 1) #select attention for w2v

            LU_lex_expanded = tf.expand_dims(LU_lex, -1)
            print '[LU_lex_expanded]', LU_lex_expanded  # (?, 60(seq_len), 60(depth), 1)

            lex_pool = tf.nn.max_pool(
                LU_lex_expanded,
                ksize=[1, 1, attention_depth_lex, 1],
                strides=[1, 1, 1, 1],
                padding='VALID',
                name="lex_pool")

            print '[lex_pool]', lex_pool  # (?, 60(seq_len), 1, 1) #select attention for lex

            w2v_pool_sq = tf.expand_dims(tf.squeeze(w2v_pool, squeeze_dims=[2, 3]), -1)  # (?, 60, 1)
            print '[w2v_pool_sq]', w2v_pool_sq

            lex_pool_sq = tf.expand_dims(tf.squeeze(lex_pool, squeeze_dims=[2, 3]), -1)  # (?, 60, 1)
            print '[lex_pool_sq]', lex_pool_sq

            attentioned_w2v = tf.batch_matmul(self.embedded_chars_tr, w2v_pool_sq)
            attentioned_lex = tf.batch_matmul(self.embedded_chars_lexicon_tr, lex_pool_sq)

            attentioned_w2v_sq = tf.squeeze(attentioned_w2v, squeeze_dims=[2])
            attentioned_lex_sq = tf.squeeze(attentioned_lex, squeeze_dims=[2])

            print '[attentioned_w2v]', attentioned_w2v_sq
            print '[attentioned_lex]', attentioned_lex_sq
            attention_outputs.append(attentioned_w2v_sq)
            attention_outputs.append(attentioned_lex_sq)

        # Create a convolution + maxpool layer for each filter size
        pooled_outputs = []
        for i, filter_size in enumerate(filter_sizes):
            with tf.name_scope("conv-maxpool-%s" % filter_size):
                # Convolution Layer
                filter_shape = [filter_size, embedding_size, 2, num_filters]
                W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
                b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b")

                # l2_loss += tf.nn.l2_loss(W)/1000
                # l2_loss += tf.nn.l2_loss(b)/1000

                conv = tf.nn.conv2d(
                    self.input_x_2c,
                    W,
                    strides=[1, 1, 1, 1],
                    padding="VALID",
                    name="conv")
                # Apply nonlinearity
                h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
                # Maxpooling over the outputs
                pooled = tf.nn.max_pool(
                    h,
                    ksize=[1, sequence_length - filter_size + 1, 1, 1],
                    strides=[1, 1, 1, 1],
                    padding='VALID',
                    name="pool")
                pooled_outputs.append(pooled)

        # Combine all the pooled features
        num_filters_total = num_filters * len(filter_sizes)
        self.h_pool = tf.concat(3, pooled_outputs)
        self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])

        self.appended_pool = tf.concat(1, [self.h_pool_flat, attention_outputs[0], attention_outputs[1]])
        print '[self.appended_pool]', self.appended_pool
        num_filters_total = num_filters_total + embedding_size + embedding_size_lex

        # Add dropout
        with tf.name_scope("dropout"):
            self.h_drop = tf.nn.dropout(self.appended_pool, self.dropout_keep_prob)

        # Final (unnormalized) scores and predictions
        with tf.name_scope("output"):
            W = tf.get_variable(
                "W",
                shape=[num_filters_total, num_classes],
                initializer=tf.contrib.layers.xavier_initializer())
            b = tf.Variable(tf.constant(0.1, shape=[num_classes]), name="b")
            l2_loss += tf.nn.l2_loss(W)/30
            l2_loss += tf.nn.l2_loss(b)/30
            l1_loss += tf.reduce_sum(tf.abs(W))
            l1_loss += tf.reduce_sum(tf.abs(b))
            self._b = b
            self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores")
            self.predictions = tf.argmax(self.scores, 1, name="predictions")

        # CalculateMean cross-entropy loss
        with tf.name_scope("loss"):
            losses = tf.nn.softmax_cross_entropy_with_logits(self.scores, self.input_y)
            self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss + l1_reg_lambda*l1_loss

        # Accuracy
        with tf.name_scope("accuracy"):
            self.golds = tf.argmax(self.input_y, 1, name="golds")
            correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")

        with tf.name_scope("avg_f1"):
            self.golds = tf.argmax(self.input_y, 1, name="golds")
            self.preds = self.predictions

            # positive recall
            pos_gold_sel = tf.equal(self.golds, 2)  # positive_gold
            posg_golds = tf.boolean_mask(self.golds, pos_gold_sel)
            posg_preds = tf.boolean_mask(self.preds, pos_gold_sel)
            correct_predictions_pr = tf.equal(posg_golds, posg_preds)
            pos_r = tf.reduce_mean(tf.cast(correct_predictions_pr, "float"), name="pos_recall")

            # positive precision
            pos_pred_sel = tf.equal(self.preds, 2)  # positive_pred
            posp_golds = tf.boolean_mask(self.golds, pos_pred_sel)
            posp_preds = tf.boolean_mask(self.preds, pos_pred_sel)
            correct_predictions_pp = tf.equal(posp_golds, posp_preds)
            pos_p = tf.reduce_mean(tf.cast(correct_predictions_pp, "float"), name="pos_precision")

            # negative recall
            neg_gold_sel = tf.equal(self.golds, 0)  # positive_gold
            negg_golds = tf.boolean_mask(self.golds, neg_gold_sel)
            negg_preds = tf.boolean_mask(self.preds, neg_gold_sel)
            correct_predictions_nr = tf.equal(negg_golds, negg_preds)
            self.neg_r = tf.reduce_mean(tf.cast(correct_predictions_nr, "float"), name="neg_recall")

            # negative precision
            neg_pred_sel = tf.equal(self.preds, 0)  # positive_pred
            negp_golds = tf.boolean_mask(self.golds, neg_pred_sel)
            negp_preds = tf.boolean_mask(self.preds, neg_pred_sel)
            correct_predictions_np = tf.equal(negp_golds, negp_preds)
            self.neg_p = tf.reduce_mean(tf.cast(correct_predictions_np, "float"), name="neg_precision")

            self.f1_neg = 2 * self.neg_p * self.neg_r / (self.neg_p + self.neg_r) * 100
            self.f1_pos = 2 * pos_p * pos_r / (pos_p + pos_r) * 100

            self.avg_f1 = (self.f1_neg + self.f1_pos) / 2

max_stars_repo_name: bgshin/doc-classify-multi-gpu

max_issues_repo_name: bgshin/doc-classify-multi-gpu

max_forks_repo_name: bgshin/doc-classify-multi-gpu


text:
import numpy as np
from quadrature import lobatto_quad
import matplotlib.pyplot as plt
from functools import lru_cache

def derivative_poly_nodes(p, nodes):
    """For computation of the derivative at the nodes a more efficient and accurate formula can
       be used, see [1]:

                 | \frac{c_{k}}{c_{j}}\frac{1}{x_{k}-x_{j}},          k \neq j
                 |
       d_{kj} = <
                 | \sum_{l=1,l\neq k}^{p+1}\frac{1}{x_{k}-x_{l}},     k = j
                 |

                 with
        c_{k} = \prod_{l=1,l\neq k}^{p+1} (x_{k}-x_{l}).

    Args:
        p (int) = degree of polynomial
        type_poly (string) = 'Lobatto', 'Gauss', 'Extended Gauss'
    [1] Costa, B., Don, W. S.: On the computation of high order
      pseudospectral derivatives, Applied Numerical Mathematics, vol.33
       (1-4), pp. 151-159
        """
    # compute distances between the nodes
    xi_xj = nodes.reshape(p + 1, 1) - nodes.reshape(1, p + 1)
    # diagonals to one
    xi_xj[np.diag_indices(p + 1)] = 1
    # compute (ci's)
    c_i = np.prod(xi_xj, axis=1)
    # compute ci/cj = ci_cj(i,j)
    c_i_div_cj = np.transpose(c_i.reshape(1, p + 1) / c_i.reshape(p + 1, 1))
    # result formula
    derivative = c_i_div_cj / xi_xj
    # put the diagonals equal to zeros
    derivative[np.diag_indices(p + 1)] = 0
    # compute the diagonal values enforning sum over rows = 0
    derivative[np.diag_indices(p + 1)] = -np.sum(derivative, axis=1)
    return derivative

def lagrange_basis(nodes, x=None):
    if x is None:
        x = nodes
    p = np.size(nodes)
    basis = np.ones((p, np.size(x)))
    # lagrange basis functions
    for i in range(p):
        for j in range(p):
            if i != j:
                basis[i, :] *= (x - nodes[j]) / (nodes[i] - nodes[j])
    return basis

def derivative_poly(p, nodes, x):
    """Return the derivatives of the polynomials in the domain x."""
    nodal_derivative = derivative_poly_nodes(p, nodes)
    polynomials = lagrange_basis(nodes, x)
    return np.transpose(nodal_derivative) @ polynomials

def edge_basis(nodes, x=None):
    """Return the edge polynomials."""
    if x is None:
        x = nodes
    p = np.size(nodes) - 1
    derivatives_poly = derivative_poly(p, nodes, x)
    edge_poly = np.zeros((p, np.size(x)))
    for i in range(p):
        for j in range(i + 1):
            edge_poly[i] -= derivatives_poly[j, :]
    return edge_poly

# if __name__ == '__main__':
#     p = 4
#     nodes, weights = lobatto_quad(p)
#     x = np.linspace(-1, 1, 50)
#
#     l_basis = lagrange_basis(nodes, x)
#     e_basis = edge_basis(nodes, x)
#
#     plt.plot(x, np.transpose(e_basis))
#     plt.show()


text:
import numpy as np
import librosa

class ToMelSpectrogram(object):

    def __init__(self, sampling_rate=22050, nr_mels=256, nr_fft=2048, hop_size=512, power=2, fmin=40, fmax=11025):
        self.sampling_rate = sampling_rate
        self.nr_mels = nr_mels
        self.power = power
        self.fmin = fmin
        self.fmax = fmax
        self.nr_fft = nr_fft
        self.hop_size = hop_size

        self.melW = librosa.filters.mel(
            sr=sampling_rate,
            n_fft=nr_fft,
            n_mels=nr_mels,
            fmin=fmin,
            fmax=fmax
        ).T

    def __call__(self, x):

        samples = x['x']

        if len(samples.shape) == 1:
            samples = samples.reshape((1, -1))

        channels = []

        for sample in samples:

            stft_matrix = librosa.core.stft(
                y=sample,
                n_fft=self.nr_fft,
                hop_length=self.hop_size,
                window=np.hanning(self.nr_fft),
                center=True,
                dtype=np.complex64,
                pad_mode='reflect'
            ).T

            mel_spectrogram = np.dot(np.abs(stft_matrix) ** self.power, self.melW)

            channels.append(mel_spectrogram.T)

        x['x'] = np.stack(channels, axis=0)
        return x

class ToDB(object):

    def __call__(self, sample):

        sample['x'] = librosa.core.power_to_db(
            sample['x'],
            ref=1.0,
            amin=1e-10,
            top_db=None
        )

        return sample


text:
""" PyGlassTools python API
"""
import sys;
import ctypes;
import os;
from tqdm import tqdm
import numpy as np

#In order for our MPI wrapper to work , we need to set dlopen flag to need to RTLD_GLOBAL
flags = sys.getdlopenflags();
sys.setdlopenflags(flags | ctypes.RTLD_GLOBAL);

from pyglasstools import _pyglasstools;

#Initialize a single communicator during module call
comm = _pyglasstools.Communicator()
rank = comm.getRank()
size = comm.getSizeGlobal()

#Save mode
savemode = "unordered"
def set_savemode(name):
    savemode = name

## \internal
_default_excepthook = sys.excepthook;

# \brief Override pythons except hook to abort MPI runs
def _pyglasstools_sys_excepthook(type, value, traceback):
    _default_excepthook(type, value, traceback);
    sys.stdout.flush();
    sys.stderr.flush();
    comm.abort(1);

sys.excepthook = _pyglasstools_sys_excepthook

#The module should save more than one logger, which analyzes various observables
loggers_list = [];
solvers_list = [];
globalsysdata = None
globalpotential = None

def set_sysdata(sysdata):
    global globalsysdata
    globalsysdata = sysdata

def update_sysdata(frame_num):
    global globalsysdata
    globalsysdata.update(frame_num)

def get_sysdata():
    global globalsysdata
    return globalsysdata

def set_potential(potential):
    global globalpotential
    globalpotential = potential

def get_potential():
    global globalpotential
    return globalpotential

import atexit
#This is an important function for anyone who wants to tinker with how the analysis is run and also for "cleanly" exiting the module
def reset():
    global loggers_list, solvers_list
    loggers_list.clear()
    solvers_list.clear()
atexit.register(reset)

import time
def wait_until(timeout, period=0.25):
    global ioflag
    mustend = time.time() + timeout
    while time.time() < mustend:
        if ioflag:
            return True
        time.sleep(period)
    return False

import signal
import logging
import logging.handlers
log = logging.getLogger()

# class based on: http://stackoverflow.com/a/21919644/487556
class DelayedInterrupt(object):
    def __enter__(self):
        self.signal_received = False
        self.old_handler = {}
        self.old_handler[signal.SIGINT] = signal.signal(signal.SIGINT, self.handler)
        self.old_handler[signal.SIGTERM] = signal.signal(signal.SIGTERM, self.handler)

    def handler(self, sig, frame):
        self.signal_received = (sig, frame)
        print(f'Signal {sig} received by Process [{rank}]. Delaying KeyboardInterrupt.')
    def __exit__(self, type, value, traceback):
        for sig in [signal.SIGINT, signal.SIGTERM]:
            signal.signal(sig, self.old_handler[sig])
            if self.signal_received:
                self.old_handler[sig](*self.signal_received)

from pyglasstools import utils;
from random import randint, uniform

def analyze(frame_list):
    global globalsysdata
    totallength = len(frame_list)
    frame_list = globalsysdata.setup_checkpoint(frame_list)
    if not frame_list:
        if rank == 0:
            print("No more frames to analyze. Check your logfiles/output")
    else:
        if rank == 0:
            progressbar = tqdm(total=totallength,file=sys.stdout,leave=False,initial=frame_list[0])
            print("")

        for frame_num in frame_list:
            for solver in solvers_list:
                solver.update(frame_num);
                solver.run();
            with DelayedInterrupt():
                for logger in loggers_list:
                    logger.save(frame_num);
                #It's sufficient to go use at least one solver to do the checkpointing
                if rank == 0 and globalsysdata.checkpointfile is not None:
                    with open(globalsysdata.checkpointfile,"w") as f:
                        f.write("Frame {}".format(frame_num+1))
                        f.close()
                if rank == 0:
                    progressbar.update(1)
                    print("")
                #Barrier for consistent update
                comm.barrier()
                pass
        if rank == 0:
            progressbar.close()

max_stars_repo_name: muhammadhasyim/pyglasstools

max_issues_repo_name: muhammadhasyim/pyglasstools

max_forks_repo_name: muhammadhasyim/pyglasstools


text:
"""
    Main function to run the XOR inference algorithm on a given single-layer directed network. The subcases SpringRank
    and MultiTensor are included for baseline comparison.

    Is it possible to use cross-validation, i.e.
        - Hold-out part of the dataset (pairs of edges labeled by unordered pairs (i,j));
        - Infer parameters on the training set;
        - Calculate performance measures in the test set.
    and grid search for optimizing the number of communities and the beta parameter (inverse temperature parameter for
    the hierarchical structure inference task).
"""

import sys

sys.path.append('../modules')

import os
import re
import yaml
import time
import numpy as np
from argparse import ArgumentParser
from tools import import_data, shuffle_indices_all_matrix, extract_mask_kfold

import XOR as XOR
import SpringRank as SR
import MultiTensor as MT

def main(args):

    # setting to run the algorithm
    if args['settings'] is None:
        settings_path = '../setting_' + args['algorithm'] + '.yaml'
    else:
        settings_path = args['settings']
    with open(settings_path) as f:
        conf = yaml.load(f, Loader = yaml.FullLoader)
    # create output folder
    if not os.path.exists(conf['out_folder']):
        os.makedirs(conf['out_folder'], exist_ok = True)

    # set data label depending on data generation type
    if conf['gt']:
        conf['label'] = re.sub('syn_', '', args['adj'][:-4])  # synthetic data
    else:
        conf['label'] = args['adj'][:-4]  # real data

    '''
    Model parameters
    '''
    network = args['in_folder'] + args['adj']  # network complete path
    algorithm = args['algorithm']  # algorithm to use to generate the samples

    '''
    Import data
    '''
    A, B = import_data(network, header = 0, force_dense = True)
    nodes = A[0].nodes()
    # reconstruct path to corresponding ground truth
    conf['in_folder'] = args['in_folder'] + 'results_' + conf['label']

    assert isinstance(B, np.ndarray)

    '''
    Run main routine for the chosen inference model.
    '''
    if algorithm in {'SR', 'MT'}:
        conf['classification'] = False

    if conf['cv']:
        # create subfolder for CV
        conf['out_folder'] += '{}-fold_cv/'.format(args['NFold'])
        if not os.path.exists(conf['out_folder']):
            os.makedirs(conf['out_folder'], exist_ok = True)

        # name of the network without extension
        adjacency = args['adj'].split('.dat')[0]
        # run inference
        run_inference_cv(B, nodes, algorithm, adjacency, args, conf)
    else:
        # run inference
        run_inference(B, nodes, algorithm, args, conf)

def run_inference(B, nodes, algorithm, args, conf):
    """

    Parameters
    ----------
    B
    nodes
    algorithm
    args
    conf

    """

    # Get dimensions
    L = B.shape[0]
    N = B.shape[-1]
    K = args['K']
    beta0 = args['beta']

    if isinstance(K, list) and len(K) == 1:
        K = K[0]
    elif isinstance(K, int):
        pass
    else:
        raise ValueError('For testing the algorithm with different number of communities K, please use the Cross '
                         'Validation option (yalm configuration file). ')

    print(f'\n--- Run inference with {algorithm} model ---', end = '\n\n')

    time_start = time.time()
    if algorithm == 'XOR':
        model = XOR.EitherOr(N = N, L = L, K = K, beta0 = beta0, **conf)
    elif algorithm == 'SR':
        model = SR.SpringRank(N = N, L = L, **conf)
    elif algorithm == 'MT':
        model = MT.MultiTensor(N = N, L = L, K = K, **conf)
    else:
        raise ValueError('Selected model not in the list of available models. Please use SR, MT or XOR!')

    # Run inference
    if algorithm != 'SR':
        out = model.fit(data = B, nodes = nodes)
        if conf['out_inference']:
            # Output latent variables and stats
            model.save_results(out, **conf)
    else:
        _ = model.fit(data = B)

    print(f'\nTime elapsed: {np.round(time.time() - time_start, 2)} seconds.')

def run_inference_cv(B, nodes, algorithm, adjacency, args, conf):
    """

    Parameters
    ----------
    B
    nodes
    algorithm
    adjacency
    args
    conf

    """
    # Set seed random number generator
    prng = np.random.RandomState(seed = 42)
    # Get dimensions
    L = B.shape[0]
    N = B.shape[-1]
    K = args['K']

    if not isinstance(K, list):
        K = [K]

    print(f'\n---  Run inference with CV procedure, {algorithm} model ---')

    time_start = time.time()

    rseed = prng.randint(1000)
    indices = shuffle_indices_all_matrix(N, L, rseed = rseed)
    init_end_file = conf['label']

    for fold in range(args['NFold']):
        print('\nFOLD ', fold)

        mask = extract_mask_kfold(indices, N, fold = fold, NFold = args['NFold'])
        if args['out_mask']:
            outmask = conf['out_folder'] + 'mask_f' + str(fold) + '_' + adjacency + '_inf' + algorithm + '.npz'
            np.savez_compressed(outmask, mask = mask)
            print(f'Mask saved at: {outmask}')

        '''
        Set up training dataset
        '''
        B_train = B.copy()
        B_train[mask > 0] = 0

        '''
        Run on the training for every K
        '''
        tic = time.time()

        conf['label'] = init_end_file + '_' + str(fold)
        fit_cv(B_train, nodes, N, L, K, args['beta'], algorithm, mask = np.logical_not(mask), **conf)

        print(f'Time elapsed for fold {fold}: {np.round(time.time() - tic, 2)} seconds.')

    print(f'Total time elapsed: {np.round(time.time() - time_start, 2)} seconds.')

def fit_cv(B, nodes, N, L, K, b, alg, mask=None, **conf):
    """
        Model directed networks by using a probabilistic generative model that assume community
        and/or hierarchical structure. The inference is performed via EM algorithm.

        Parameters
        ----------
        B : ndarray
            Graph adjacency tensor.
        nodes : list
                List of nodes IDs.
        N : int
            Number of nodes.
        L : int
            Number of layers.
        K : list int
            List of number of communities.
        b: float
           Inverse temperature parameter.
        alg : str
                    Configuration to use (MM, XOR, SR, MT).
        mask : ndarray
               Mask for cv.

    """

    label = conf['label']

    if alg == 'SR':
        model = SR.SpringRank(N = N, L = L, **conf)
        _ = model.fit(data = B, mask = mask)

    else:
        for k in K:
            print(f'Using K = {k}.')
            conf['label'] = label + f'_K{k}'
            if alg == 'MT':
                model = MT.MultiTensor(N = N, L = L, K = k, **conf)
            elif alg == 'XOR':
                model = XOR.EitherOr(N = N, L = L, K = k, beta0 = b, **conf)
            else:
                raise ValueError('Selected model not in the list of available models. Please use SR, MT or XOR!')

            # Run inference
            out = model.fit(data = B, nodes = nodes, mask = mask)
            if conf['out_inference']:
                # Output latent variables and stats
                model.save_results(out, mask = mask, **conf)
            conf['rseed'] += 100000

if __name__ == '__main__':

    p = ArgumentParser()
    p.add_argument('-a', '--algorithm', type = str, choices = ['XOR', 'SR', 'MT'], default = 'XOR',
                   help = 'Code for the desired model, among XOR, SpringRank, MultiTensor. '
                          'Default = "XOR"')
    p.add_argument('-K', '--K', type = int, nargs = '+', default = 3, help = 'Number of communities to infer.')
    p.add_argument('-b', '--beta', type = float, default = 5, help = 'Inverse temperature parameter.')
    p.add_argument('-A', '--adj', type = str, default = 'syn_test_r0.8_XOR_112.dat', help = 'Name of the network file.')
    p.add_argument('-f', '--in_folder', type = str, default = '../../data/input/', help = 'Folder path of the input '
                                                                                'network.')
    p.add_argument('-F', '--NFold', type = int, default = 5, help = 'Number of folds to perform cross-validation')
    p.add_argument('-m', '--out_mask', type = bool, default = False, help = 'Flag to output the masks')
    p.add_argument('-s', '--settings', type = str, help = 'Path of the settings file. If None, it is inferred using the'
                                                          ' selected algorithm, i.e. "../setting_<algorithm>.yaml"')
    args = p.parse_args()

    main(vars(args))

max_stars_repo_name: liacov/XOR-rankcom

max_issues_repo_name: liacov/XOR-rankcom

max_forks_repo_name: liacov/XOR-rankcom


text:
""" """

import json
import gensim
import couchdb
import configparser
import numpy as np
from nltk import RegexpTokenizer
from nltk.corpus import stopwords
from os import listdir
from os.path import isfile, join
from db_handler import *
from open_files import OpenFile
from fuzzywuzzy import fuzz
from fuzzywuzzy import process

from sklearn.preprocessing import StandardScaler
from sklearn.cluster import AffinityPropagation
from sklearn.cluster import DBSCAN

from sklearn import metrics
from sklearn.decomposition import PCA
from scipy.cluster.hierarchy import dendrogram, linkage, ward
#from sklearn.datasets.samples_generator import make_blobs

config = configparser.ConfigParser()
config.read('settings.ini')
dataset_path = config["DEFAULT"]["dataset_path"]

tokenizer = RegexpTokenizer(r'\w+')
stopword_set = set(stopwords.words('turkish'))
# This function does all cleaning of data using two objects above

def nlp_clean(data):
    new_data = []
    for d in data:
        new_str = d.lower()
        dlist = tokenizer.tokenize(new_str)
        dlist = list(set(dlist).difference(stopword_set))
        new_data.append(dlist)
    return new_data

class LabeledLineSentence(object):

    def __init__(self, doc_list, labels_list):

        self.labels_list = labels_list
        self.doc_list = doc_list

    def __iter__(self):

        for idx, doc in enumerate(self.doc_list):
              # yield gensim.models.doc2vec.LabeledSentence(doc, [self.labels_list[idx]])
            yield gensim.models.doc2vec.TaggedDocument(doc, [self.labels_list[idx]])

class SimilarityRatio():
    def __init__(self, files, file_format, method=None):

        self.files = files
        self.files_opened = []
        for f in self.files:
            self.files_opened.append(OpenFile(f))
        self.docLabels = []
        self.db_server = db_handler()
        for doc in self.files_opened:
            self.docLabels.append(doc.location)

        # create a list data that stores the content of all text files in order of their names in docLabels
        data = []
        if file_format == "docx" or file_format == "pptx":
            for doc in self.files_opened:
                #data.append(open(doc, encoding='latin-1').read())
                db = db_ds
                data.append(doc.text)
        elif file_format == "xlsx":
            for i, doc in enumerate(self.files_opened):
                #data.append(open(doc, encoding='latin-1').read())
                db = db_xs
                try:
                    data.append(json.dumps(doc.tables, skipkeys=True))
                except:
                    print("error parsing document {}".format(
                        self.docLabels[i]))
                    data.append("")

        data = nlp_clean(data)
        if method == "fuzzywuzzy":
            for i, f1 in enumerate(data):
                for f2 in data[i+1:]:
                    # print(self.docLabels[i],self.docLabels[i+1])
                    x = fuzz.ratio(f1, f2)
                    y = fuzz.partial_ratio(f1, f2)
                    print(
                        "overall similarity ration: {} %\npartial similarity ration: {}".format(x, y))
                    db_data = {'dok_id': {'dok_1': self.docLabels[i], 'dok_2': self.docLabels[i+1]},
                               'kullanici': user_default, 'overall similarity ratio': x, 'partial similarity ratio': y}
                    self.db_server.save(
                        db, db_data, doc_id=self.docLabels[i]+"_"+self.docLabels[i+1])

        elif method == "inference":
            #res = self.db_server.query(db_gensim,["_attachments"],query_key="_id", query_value=file_format)

            #model_loc ="{}gensim_models/docx/models/doc2vec_{}.model".format(server_default,file_format)
            model_loc = "models/doc2vec_{}.model".format(file_format)
            # loading the model
            d2v_model = gensim.models.doc2vec.Doc2Vec.load(model_loc)
            # d2v_model.init_sims(replace=False)

            # infer_vector is non-deterministic; i.e. the resulting vector is different each time, but it should be similar enough with a good model
            infervec = d2v_model.infer_vector(
                data[0], alpha=0.025, min_alpha=0.025, steps=300)
            similar_doc = d2v_model.docvecs.most_similar([infervec])
            most_similar = similar_doc[0][0]
            print(type(most_similar))
            print("most similar: {}".format(most_similar))

            #db_res = self.db_server.query(db_dc,["_id","docs"])
            db_res = self.db_server.query(
                db_dc, ["docs", "clusters"], query_key="_id", query_value=file_format)
            print(db_res)
            db_res_a = []
            db_res_b = []
            for row in db_res:
                # db_res_a.append(row)
                for a in row.key[0]:
                    db_res_a.append(a)
                for b in row.key[1]:
                    db_res_b.append(b)
            # print(db_res_a)
            # print(db_res_b)
            most_similar_class = db_res_b[db_res_a.index(most_similar)]
            print("most likely class: {}".format(most_similar_class))
            print("other documents in same category")
            for i in range(len(db_res_b)):
                if db_res_b[i] == most_similar_class:
                    print(db_res_a[i])

        else:
            # iterator returned over all documents
            it = LabeledLineSentence(data, self.docLabels)
            model = gensim.models.Doc2Vec(
                vector_size=300, min_count=0, alpha=0.025, min_alpha=0.025)
            model.build_vocab(it)
            # training of model
            for epoch in range(100):
                #print ('iteration '+str(epoch+1))
                model.train(it, total_examples=model.corpus_count, epochs=3)
                model.alpha -= 0.002
                model.min_alpha = model.alpha

            model.save('models/doc2vec_{}.model'.format(file_format))

            db_g = db_gensim
            db_data = {"time": "time", "path": dataset_path}
            self.db_server.save(db_g, db_data, doc_id=file_format,
                                attachment='models/doc2vec_{}.model'.format(file_format))

            print("model saved")

            # loading the model
            d2v_model = gensim.models.doc2vec.Doc2Vec.load(
                'models/doc2vec_{}.model'.format(file_format))

            # start testing
            X = []
            # printing the vector of documents in docLabels
            for i, _ in enumerate(self.docLabels):
                docvec = d2v_model.docvecs[i]
                # print(docvec)
                X.append(docvec)
            X = np.array(X)
            #docvec = d2v_model.docvecs[0]
            #print (docvec)
            #docvec = d2v_model.docvecs[1]
            #print (docvec)

            # to get most similar document with similarity scores using document-index
            #similar_doc = d2v_model.docvecs.most_similar(0)
            # print(similar_doc)

            # for doc in similar_doc:
            #    db_data = {'dok_id' : {'dok_1' : self.docLabels[0],'dok_2' : doc[0]}, 'kullanici': user_default, 'benzerlik orani': str(doc[1])}
            #    self.db_server.save(db, db_data)
            #similar_doc = d2v_model.docvecs.most_similar(1)
            # print(similar_doc)

            # printing the vector of the file using its name
            # docvec = d2v_model.docvecs['shakespeare-hamlet.txt'] #if string tag used in training
            # print(docvec)
            # to get most similar document with similarity scores using document- name
            #sims = d2v_model.docvecs.most_similar('shakespeare-hamlet.txt')
            # print(sims)

            # #############################################################################
            # Compute Affinity

            if True:
                af = AffinityPropagation(preference=-50).fit(X)
                cluster_centers_indices = af.cluster_centers_indices_
                n_clusters_ = len(cluster_centers_indices)
                labels = af.labels_
            else: #trying DBScan instead
                X = StandardScaler().fit_transform(X)
                af = DBSCAN(eps=3, min_samples=2).fit(X)
                core_samples_mask = np.zeros_like(af.labels_, dtype=bool)
                core_samples_mask[af.core_sample_indices_] = True

                labels = af.labels_
                unique_labels = set(labels)
                n_clusters_ = len(unique_labels)

            #labels2 = []
            # for i, lb in enumerate(labels):
            #    labels2.append(self.files[i].split('/')[-1])
            #print("labels: {}".format(labels))
            #print("labels2: {}".format(labels2))

            print("number of clusters: {}".format(n_clusters_))
            dic = {i: np.where(labels == i)[0] for i in range(n_clusters_)}
            dic2 = {}
            # print(dic)

            for key, value in dic.items():
                print("cluster {}:".format(key))
                for e in value:
                    print("{} : {}".format(e, self.files[e].split('/')[-1]))
                    dic2[self.docLabels[e]] = key

            print(dic2)

            # print('Estimated number of clusters: %d' % n_clusters_)
            # print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels_true, labels))
            # print("Completeness: %0.3f" % metrics.completeness_score(labels_true, labels))
            # print("V-measure: %0.3f" % metrics.v_measure_score(labels_true, labels))
            # print("Adjusted Rand Index: %0.3f"
            #     % metrics.adjusted_rand_score(labels_true, labels))
            # print("Adjusted Mutual Information: %0.3f"
            #     % metrics.adjusted_mutual_info_score(labels_true, labels))
            #print("Silhouette Coefficient: %0.3f"
            #      % metrics.silhouette_score(X, labels, metric='sqeuclidean'))

            # #############################################################################
            # Plot result
            import matplotlib.pyplot as plt
            from mpl_toolkits.mplot3d import Axes3D
            from itertools import cycle

            plt.close('all')
            plt.figure(figsize=(25, 10))
            plt.clf()

            # reduce dimensions
            # pca = PCA(n_components=2)
            # reduced = pca.fit_transform(X)
            # X = reduced

            if True:
                colors = cycle('bgrcmykbgrcmykbgrcmykbgrcmyk')
                for k, col in zip(range(n_clusters_), colors):
                    class_members = labels == k
                    cluster_center = X[cluster_centers_indices[k]]
                    plt.plot(X[class_members, 0], X[class_members, 1], col + '.')
                    plt.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
                            markeredgecolor='k', markersize=5)
                    for x in X[class_members]:
                        plt.plot([cluster_center[0], x[0]], [
                                cluster_center[1], x[1]], col)

                plt.title(
                    'Clustering with Affinity Propagation | Estimated number of clusters: %d' % n_clusters_)
                plt.savefig(
                    'models/{}_affinity_clusters.png'.format(file_format), dpi=300)
            else:
                colors = [plt.cm.Spectral(each) for each in np.linspace(0, 1, len(unique_labels))]
                for k, col in zip(unique_labels, colors):
                    if k == -1:
                        # Black used for noise.
                        col = [0, 0, 0, 1]

                    class_member_mask = (labels == k)

                    xy = X[class_member_mask & core_samples_mask]
                    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
                            markeredgecolor='k', markersize=14)

                    xy = X[class_member_mask & ~core_samples_mask]
                    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
                            markeredgecolor='k', markersize=6)

                plt.title(
                    'Clustering with DBScan | Estimated number of clusters: %d' % n_clusters_)
                plt.savefig(
                    'models/{}_dbscan_clusters.png'.format(file_format), dpi=300)

            plt.show()

            #db = db_dc
            db_data = dic2
            db_data["docs"] = self.docLabels
            db_data["clusters"] = labels.tolist()
            self.db_server.save(db_dc, db_data, doc_id=file_format,
                                attachment='models/{}_affinity_clusters.png'.format(file_format))

            # #########################
            # hierarchical

            linkage_matrix = []
            #linkage_matrix.append(linkage(X, method='single', metric='euclidean'))
            linkage_matrix.append(
                linkage(X, method='average', metric='euclidean'))
            #linkage_matrix.append(linkage(X, method='complete', metric='euclidean'))
            #linkage_matrix.append(linkage(X, method='ward', metric='euclidean'))

            #linkage_matrix.append(linkage(X, method='single', metric='seuclidean'))
            # linkage_matrix.append(linkage(X, method='average', metric='seuclidean'))
            #linkage_matrix.append(linkage(X, method='complete', metric='seuclidean'))

            for n, l in enumerate(linkage_matrix):
                # calculate full dendrogram
                plt.figure(figsize=(25, 10))
                plt.title('Hierarchical Clustering Dendrogram')
                plt.ylabel('word')
                plt.xlabel('distance')

                dendrogram(
                    l,
                    leaf_rotation=0.,  # rotates the x axis labels
                    leaf_font_size=16.,  # font size for the x axis labels
                    orientation='left',
                    leaf_label_func=lambda v: str(self.files[v].split('/')[-1])
                )
                # plt.savefig('clusters_{}.png'.format(n), dpi=200) #save figure as ward_clusters
                plt.savefig(
                    'models/{}_hierarchical_clusters.png'.format(file_format), dpi=300)
                plt.show()

                db_data = {}
                self.db_server.save(db_dc, db_data, doc_id=file_format,
                                    attachment='models/{}_hierarchical_clusters.png'.format(file_format))

reason: import numpy,from scipy


text:
## Written and copyright by Mohit Agarwal
## Georgia Institute of Technology
## Email: magarwal37@gatech.edu

import os
import numpy as np
from scipy.signal import *
import csv
import matplotlib.pyplot as plt

#Options to read: 'EEG-IO', 'EEG-VV', 'EEG-VR', 'EEG-MB'
data_folder = 'EEG-VR'

# Parameters and bandpass filtering
fs = 250.0
def lowpass(sig, fc, fs, butter_filt_order):
    B,A = butter(butter_filt_order, np.array(fc)/(fs/2), btype='low')
    return lfilter(B, A, sig, axis=0)

# function to read stimulations
def decode_stim(data_path, file_stim):
    interval_corrupt = []
    blinks = []
    n_corrupt = 0
    with open(os.path.join(data_path,file_stim)) as csvfile:
        readCSV = csv.reader(csvfile, delimiter=',')
        for row in readCSV:
            if row[0]=="corrupt":
                n_corrupt = int(row[1])
            elif n_corrupt > 0:
                if float(row[1]) == -1:
                    t_end = data_sig[-1,0]
                else:
                    t_end = float(row[1])
                interval_corrupt.append([float(row[0]), t_end])
                n_corrupt = n_corrupt - 1
            elif row[0]=="blinks":
                #check that n_corrupt is 0
                if not n_corrupt==0:
                    print("!Error in parsing")
            else:
                blinks.append([float(row[0]), int(row[1])])
    blinks = np.array(blinks)

    return interval_corrupt, blinks

# Reading data files
file_idx = 0
list_of_files = [f for f in os.listdir(data_folder) if os.path.isfile(os.path.join(data_folder, f)) and '_data' in f]
file_sig = list_of_files[file_idx]
file_stim = list_of_files[file_idx].replace('_data','_labels')
print("Reading: ", file_sig, file_stim)

# Loading data
if data_folder is 'EEG-IO' or data_folder is 'EEG-MB':
	data_sig = np.loadtxt(open(os.path.join(data_folder,file_sig), "rb"), delimiter=";", skiprows=1, usecols=(0,1,2))
elif data_folder is 'EEG-VR' or data_folder is 'EEG-VV':
	data_sig = np.loadtxt(open(os.path.join(data_folder,file_sig), "rb"), delimiter=",", skiprows=5, usecols=(0,1,2))
	data_sig = data_sig[0:(int(200*fs)+1),:]
	data_sig = data_sig[:,0:3]
	data_sig[:,0] = np.array(range(0,len(data_sig)))/fs

# Loading Stimulations
interval_corrupt, groundtruth_blinks = decode_stim(data_folder, file_stim)

# FIltering
data_sig[:,1] = lowpass(data_sig[:,1], 10, fs, 4)
data_sig[:,2] = lowpass(data_sig[:,2], 10, fs, 4)

# Visualizing data and ground truth
chan_id = 1
fig, ax1 = plt.subplots()
ax1.plot(data_sig[:,0], data_sig[:,chan_id])
for i in interval_corrupt:
    ax1.axvspan(i[0], i[1], alpha=0.5, color='red')
for d in groundtruth_blinks:
    if d[1] < 2:
        plt.axvline(x=d[0], color='green')
    elif d[1] == 2:
        plt.axvline(x=d[0], color='black')
plt.show()


text:
# Copyright 2022 Google.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""An extended model for prompt tuning with a "wayward prompts" regularizer.

We have an extra term in our loss function that regularizes the learned prompt
towards the embedded representation of a discrete prompt from
Khashabi, et al. (2021) https://arxiv.org/abs/2112.08348
"""

from typing import Any, Callable, List, Mapping, Optional, Sequence

from flax import linen as nn
from flax import optim
from flax import traverse_util
from flax.core import unfreeze
import jax.numpy as jnp
import seqio
from t5x import decoding
from t5x import models

from flaxformer.types import Array

def length(x: Any) -> int:
  """A length function we can bind to in gin."""
  return len(x)

def encode_string(s: str, vocab: seqio.SentencePieceVocabulary) -> List[str]:
  """Break a string into sentence pieces."""
  return vocab.tokenizer.EncodeAsPieces(s)

def execute_initializer(init: Callable[[Array, Sequence[int]], Array],
                        rng: Array, shape: Sequence[int]) -> Array:
  """A function to execute a flax initializer outside of the .init method."""
  return init(rng, shape)

def squared_l2(x: Array) -> float:
  """Calculate the squared l2 norms of a sequence of arrays.

  Note:
    We use the squared l2 norm as things like the ranking will the same
    without needing to do the expensive sqrt.

  Args:
    x: The sequence of arrays to calculate the norm of. [T, H]

  Returns:
    The norm over the hidden dimension of the sequence of arrays. [T]
  """
  return jnp.sum(jnp.square(x), axis=1)

def squared_l2_distance(x: Array, y: Array) -> Array:
  """Calculate the squared l2 dist between arrays, normalized by length."""
  l, _ = x.shape
  return jnp.sum(squared_l2(x - y)) / l

class WaywardPromptEncoderDecoderModel(models.EncoderDecoderModel):
  """Regularize a prompt towards a discrete representation a la (Khashabi, et al., 2021)."""

  def __init__(
      self,
      module: nn.Module,
      input_vocabulary: seqio.Vocabulary,
      output_vocabulary: seqio.Vocabulary,
      optimizer_def: optim.OptimizerDef,
      decode_fn: models.DecodeFnCallable = decoding.beam_search,
      feature_converter_cls: Optional[Callable[...,
                                               seqio.FeatureConverter]] = None,
      label_smoothing: float = 0.0,
      z_loss: float = 0.0,
      loss_normalizing_factor: Optional[float] = None,
      gamma: float = 0.01,
      distance: Callable[[Array, Array], Array] = squared_l2_distance,
      discrete_prompt: Array = None,
      prompt_path: str = "encoder/prompt/prompt/prompt",
  ):
    super().__init__(
        module=module,
        input_vocabulary=input_vocabulary,
        output_vocabulary=output_vocabulary,
        optimizer_def=optimizer_def,
        decode_fn=decode_fn,
        feature_converter_cls=feature_converter_cls,
        label_smoothing=label_smoothing,
        z_loss=z_loss,
        loss_normalizing_factor=loss_normalizing_factor,
    )
    self.gamma = gamma
    self.distance = distance
    self.discrete_prompt = discrete_prompt
    self.prompt_path = prompt_path

  def loss_fn(
      self,
      params: models.PyTreeDef,
      batch: Mapping[str, jnp.ndarray],
      dropout_rng: Optional[jnp.ndarray],
  ):
    loss, metrics = super().loss_fn(
        params=params, batch=batch, dropout_rng=dropout_rng)

    flat_params = {
        "/".join(k): v
        for k, v in traverse_util.flatten_dict(unfreeze(params)).items()
    }
    continuous_prompt = flat_params[self.prompt_path]
    wayward_loss = self.distance(continuous_prompt, self.discrete_prompt)
    return loss + self.gamma * wayward_loss, metrics

max_stars_repo_name: dumpmemory/prompt-tuning

max_issues_repo_name: dumpmemory/prompt-tuning

max_forks_repo_name: dumpmemory/prompt-tuning

reason: import jax