text:
# -*- coding: utf-8 -*-

if __name__ == '__main__':
    import numpy as np
    from scipy import signal

    from matplotlib import pyplot as plt

    # https://blog.csdn.net/itnerd/article/details/108362607
    # 二次曲线叠加正弦余弦-------------------------------------------------------
    N = 200
    t = np.linspace(0, 1, N)
    y = 6*t*t + np.cos(10*2*np.pi*t*t) + np.sin(6*2*np.pi*t)

    peaks, _ = signal.find_peaks(y, distance=1) #distance表示极大值点两两之间的距离至少大于等于5个水平单位

    print(peaks)
    print(len(peaks))  # the number of peaks

    plt.figure(figsize=(10,5))
    plt.plot(y)
    for i in range(len(peaks)):
        plt.plot(peaks[i], y[peaks[i]],'*',markersize=10)
    plt.show()

    # https://blog.csdn.net/chehec2010/article/details/117336967
    import matplotlib.pyplot as plt
    from scipy.misc import electrocardiogram
    from scipy.signal import find_peaks
    import numpy as np
    x = electrocardiogram()[2000:4000]
    peaks, _ = find_peaks(x, height=0)
    plt.figure(figsize=(10,8))
    plt.plot(x)
    plt.plot(peaks, x[peaks], "x")
    plt.plot(np.zeros_like(x), "--", color="gray")
    plt.show()

    import matplotlib.pyplot as plt
    from scipy.misc import electrocardiogram
    from scipy.signal import find_peaks
    import numpy as np
    #选择大于0的
    # x = electrocardiogram()[2000:4000]
    # peaks, _ = find_peaks(x, height=0)
    # plt.plot(x)
    # plt.plot(peaks, x[peaks], "x")
    # plt.plot(np.zeros_like(x), "--", color="gray")
    # plt.show()

    #小于0一下的
    x = electrocardiogram()[2000:4000]
    border = np.sin(np.linspace(0, 3 * np.pi, x.size))
    peaks, _ = find_peaks(x, height=(-border, border))
    plt.figure(figsize=(10,8))
    plt.plot(x)
    plt.plot(-border, "--", color="gray")
    plt.plot(border, ":", color="gray")
    plt.plot(peaks, x[peaks], "x")
    plt.show()

    import matplotlib.pyplot as plt
    from scipy.misc import electrocardiogram
    from scipy.signal import find_peaks
    import numpy as np

    #我们可以通过要求至少150个样本的距离来轻松选择心电图(ECG)中QRS络合物的位置
    x = electrocardiogram()[2000:4000]
    peaks, _ = find_peaks(x, distance=150)
    plt.figure(figsize=(10,8))
    plt.plot(x)
    plt.plot(peaks, x[peaks], "x")
    plt.show()

    import matplotlib.pyplot as plt
    from scipy.misc import electrocardiogram
    from scipy.signal import find_peaks
    import numpy as np

    #通过将允许的突出限制为0.6
    x = electrocardiogram()[2000:4000]
    peaks, properties = find_peaks(x, prominence=(None, 0.6))
    plt.figure(figsize=(10,8))
    plt.plot(x)
    plt.plot(peaks, x[peaks], "x")
    plt.show()

    import matplotlib.pyplot as plt
    from scipy.misc import electrocardiogram
    from scipy.signal import find_peaks
    import numpy as np

    #要仅选择非典型心跳，我们将两个条件结合起来：最小突出1和至少20个样本的宽度。
    x = electrocardiogram()[17000:18000]
    peaks, properties = find_peaks(x, prominence=1, width=20)

    plt.figure(figsize=(10,8))
    plt.plot(x)
    plt.plot(peaks, x[peaks], "x")
    plt.vlines(x=peaks, ymin=x[peaks] - properties["prominences"],
               ymax = x[peaks], color = "C1")
    plt.hlines(y=properties["width_heights"], xmin=properties["left_ips"],
              xmax=properties["right_ips"], color = "C1")
    plt.show()

max_stars_repo_name: Genlovy-Hoo/dramkit

max_issues_repo_name: Genlovy-Hoo/dramkit

max_forks_repo_name: Genlovy-Hoo/dramkit


text:
import numpy as np
from autograd import Tensor

class Criterion(object):
    def __call__(self, preds, targets):
        return self.apply(preds, targets)

    def apply(self, *args, **kwargs):
        raise NotImplementedError(
        f'user defined Criterion object has not implemented apply method')

class NLLLoss(Criterion):
    """implements the SparseCategoricalCrossEntroyLoss
    by getting predictions that have been logsoftmaxed
    prior to calling the loss function. targets should be
    indices of the target class.
    """
    def apply(self, logits, targets):
        n_classes = logits.shape[-1]
        y = np.zeros((targets.shape[0], n_classes)).astype(np.float32)
        y[range(y.shape[0]), targets] = -1.0 * n_classes
        targets = Tensor(y)
        return logits.mul(targets).mean()

ext: py

max_stars_repo_name: willeagren/autograd

max_issues_repo_name: willeagren/autograd

max_forks_repo_name: willeagren/autograd

reason: import numpy


text:
#!/usr/bin/env python
# coding: utf-8

# In[2]:

import numpy as np
import pandas as pd
import xlrd
import glob
import cv2

def getImName(image_path,folder_path):
    """
    get image name by clean up image path
    Parameters:
        -image_path: path of image in the folder
        -folder_path: path of folder that contain images
    Returns:
        -name: name of the image
    """
    name = image_path.replace(folder_path.replace("/*.jpg","")+"\\","")
    print(name)
    return name

def getCenterROI(image_name,df):
    """
    get center ROI from image name and dataframe
    Parameters:
        -image_name: name of image that match with name in dataframe
        -df: dataframe that contain image name and ROI center coordinate
    Returns:
        -x,y: coordinate of ROI center
    """
    x = df.loc[image_name][0]
    y = df.loc[image_name][1]
    return x,y

def getROI(img,x,y,ROI_size):
    """
    crop square ROI from image with center x, y coordinate
    Parameters:
        -img: input image
        -x,y: center coordinate of ROI
        -ROI_size: size of ROI in pixel
    Returns:
        -imCrop: image of ROI
        -x1,y1: coordinate of top left corner of the ROI
        -x2,y2: coordinate of botton right corner of the ROI
    """
    x1 = int(x-(ROI_size/2))
    y1 = int(y-(ROI_size/2))
    x2 = int(x+(ROI_size/2))
    y2 = int(y+(ROI_size/2))
    imCrop = img[y1:y2,x1:x2]
    return imCrop,x1,y1,x2,y2

def readImgFolder(path):
    """
    read image form folder and create a list of images path in side the folder
    Parameters:
        -path: folder path (ex.'RAW_FUNDUS_INPUT/*.jpg')
    Returns:
        -image_list: list of images path in the folder
    """
    image_list = []
    for filename in glob.glob(path):
        image_list.append(filename)
    return image_list

def ShowResizedIm(img,windowname,scale):
    """
    opencv imshow resized image on a new window
    Parameters:
        -img: image
        -window: window name
        -scale: size of the display image will be divided by this value(ex. scale=2 will make image 2 time smaller)
    """
    cv2.namedWindow(windowname, cv2.WINDOW_NORMAL)        # Create window with freedom of dimensions
    height, width = img.shape[:2]   #get image dimension
    cv2.resizeWindow(windowname,int(width/scale) ,int(height/scale))                    # Resize image
    cv2.imshow(windowname, img)                            # Show image
'''
example
#=========USER START================
#folder path
img_folder_path = 'RAW_FUNDUS_INPUT/*.jpg'
ROI_path = 'ROI.xlsx'
ROI_size = 500
#=========USER END================

#create dataframe of ROI.xlsx
df = pd.read_excel(ROI_path, index_col=0)
#crate list of image path in the folder
img_path_list = readImgFolder(img_folder_path)
for img_path in img_path_list:
    #read only name
    img_name = getImName(img_path,img_folder_path)
    #read x,y coordinate from ROI.xlsx
    x_ROI,y_ROI = getCenterROI(img_name,df)
    #read image
    img = cv2.imread(img_path)
    #get ROI image, x1, y1, x2, and y2
    img_ROI,x1_ROI,y1_ROI,x2_ROI,y2_ROI = getROI(img,x_ROI,y_ROI,ROI_size)
    #show imCrop
    ShowResizedIm(img_ROI,"image ROI",2)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
    #=====================PUT YOUR CODE HERE===========================
    #
    #==================================================================
'''


text:
import numpy as np
import matplotlib.pyplot as plt

def solver(m, H0, L0, dt, a, b, c, d, t0):
    """Solve the difference equations for H and L over m years
    with time step dt (measured in years."""

    num_intervals = int(m/float(dt))
    t = np.linspace(t0, t0 + m, num_intervals+1)
    H = np.zeros(t.size)
    L = np.zeros(t.size)

    print 'Init:', H0, L0, dt
    H[0] = H0
    L[0] = L0

    for n in range(0, len(t)-1):
        H[n+1] = H[n] + a*dt*H[n] - b*dt*H[n]*L[n]
        L[n+1] = L[n] + d*dt*H[n]*L[n] - c*dt*L[n]
    return H, L, t

# Load in data file
data = np.loadtxt('Hudson_Bay.csv', delimiter=',', skiprows=1)
# Make arrays containing x-axis and hares and lynx populations
t_e = data[:,0]
H_e = data[:,1]
L_e = data[:,2]

# Simulate using the model
H, L, t = solver(m=20, H0=34.91, L0=3.857, dt=0.1,
                 a=0.4807, b=0.02482, c=0.9272, d=0.02756,
                 t0=1900)

# Visualize simulations and data
plt.plot(t_e, H_e, 'b-+', t_e, L_e, 'r-o', t, H, 'm--', t, L, 'k--')
plt.xlabel('Year')
plt.ylabel('Numbers of hares and lynx')
plt.axis([1900, 1920, 0, 140])
plt.title(r'Population of hares and lynx 1900-1920 (x1000)')
plt.legend(('H_e', 'L_e', 'H', 'L'), loc='upper left')
plt.savefig('Hudson_Bay_sim.pdf')
plt.savefig('Hudson_Bay_sim.png')
plt.show()

max_stars_repo_name: anacost/MachineLearning

max_issues_repo_name: bernharl/MachineLearning

max_forks_repo_name: bernharl/MachineLearning


text:
import numpy as np
import pandas as pd
#通过numpy创建Series
result = pd.Series(np.arange(4,10))
#通过python数组创建Series
print(result)
result = pd.Series([11,12,14],index=['北京','上海','深圳'])
print(result)
#通过字典创建Series
result = pd.Series({'北京':11, '上海':12, '深圳':14})
print(result)

#Series 允许索引重复
result = pd.Series([11,12,14],index=['北京','上海','上海'])
print(result)
print(result['上海'])

data_3_4 = pd.DataFrame(np.arange(10,22).reshape(3,4))
print(data_3_4)
# 打印第一行数据
print(data_3_4[:2])
# 打印第一列数据
print(data_3_4[:][1])
print(data_3_4.shape)
print(data_3_4.dtypes)
print(data_3_4.ndim)
print(data_3_4.index)
print(data_3_4.columns)
print(data_3_4.values)

# 打印前5个
print("-->前2个:")
print(data_3_4.head(2))
# 打印后5个
print("-->后2个:")
print(data_3_4.tail(2))
# 打印描述信息(实验中好用)
print("-->描述信息:")
print(data_3_4.describe())


text:
from copy import copy
from os.path import exists
from shutil import rmtree

from math import sqrt

import matplotlib.pyplot as plt
import networkx as nx
import pyro
import torch
from torch.utils.tensorboard import SummaryWriter
from pyro.optim import Adam
from pyro.infer import SVI
from torch_geometric.data import Data
from torch_geometric.utils import k_hop_subgraph, to_networkx
from tqdm.autonotebook import tqdm

from samplers.BaseSampler import BaseSampler

class BayesExplainer:
    def __init__(self, model: torch.nn.Module, sampler: BaseSampler, node_idx: int, k: int,
                 x: torch.Tensor, y: torch.Tensor, edge_index: torch.Tensor):
        device = torch.device('cpu')
        self.model = model.to(device)
        self.x = x.to(device)
        self.y_true = y.to(device)
        self.edge_index = edge_index.to(device)
        self.node_idx = node_idx
        self.k = k
        self.subset, self.edge_index_adj, self.mapping, self.edge_mask_hard = k_hop_subgraph(
            self.node_idx, k, self.edge_index, relabel_nodes=True)
        self.x_adj = self.x[self.subset]
        self.device = device

        with torch.no_grad():
            self.preds = model(self.x, self.edge_index_adj).exp()

        self.N = self.edge_index_adj.size(1)

        self.sampler = sampler

    def ma(self, l, window):
        cumsum, moving_aves = [0], []

        for i, x in enumerate(l, 1):
            cumsum.append(cumsum[i - 1] + x)
            if i >= window:
                moving_ave = (cumsum[i] - cumsum[i - window]) / window
                moving_aves.append(moving_ave)
        return moving_aves

    def train(self, epochs: int = 3000, lr: float = 0.005, window: int = 1000, log: bool = True, base: str = "."):
        pyro.get_param_store().clear()
        adam_params = {"lr": lr, "betas": (0.95, 0.999)}
        optimizer = Adam(adam_params)
        # setup the inference algorithm
        svi = SVI(self.sampler.sample_model, self.sampler.sample_guide, optimizer,
                  loss=self.sampler.loss_fn)

        n_steps = epochs
        # do gradient steps
        if log:
            pbar = tqdm(range(n_steps))
            name = self.sampler.run_name()
            path = f"{base}/runs/individual/{name}"
            if exists(path):
                rmtree(path)
            writer = SummaryWriter(path)
        else:
            pbar = range(1, n_steps+1)
        elbos = []
        for step in pbar:
            elbo = svi.step(self.x_adj, self.preds[self.mapping[0]], self)
            elbos.append(elbo)
            avgs = self.ma(elbos, window)
            if step >= window:
                disp = avgs[-1]
            else:
                disp = elbo
            if log:
                pbar.set_description("Loss -> %.4f" % disp)
                if step % 100 == 0:
                    writer.add_scalar("ELBO", elbo, step)
                    if step >= window:
                        writer.add_scalar("Loss", disp, step - window)
                    ax, _ = self.visualize_subgraph()
                    writer.add_figure("Importance Graph", ax.get_figure(), step)
                    writer.add_histogram("Importances", self.sampler.edge_mask(self), step)
        return avgs

    def edge_mask(self):
        return self.sampler.edge_mask(self)

    def visualize_subgraph(self, edge_mask=None, threshold=None, **kwargs):
        # Only operate on a k-hop subgraph around `node_idx`.
        subset, edge_index, _, _ = k_hop_subgraph(
            self.node_idx, self.k, self.edge_index, relabel_nodes=True)

        if edge_mask is None:
            edge_mask = self.sampler.edge_mask(self)

        if threshold is not None:
            edge_mask = (edge_mask >= threshold).to(torch.float)

        if self.y_true is None:
            self.y = torch.zeros(edge_index.max().item() + 1,
                            device=edge_index.device)
        else:
            self.y = self.y_true[subset].to(torch.float) / self.y_true.max().item()

        data = Data(edge_index=edge_index, att=edge_mask, y=self.y,
                    num_nodes=self.y.size(0)).to('cpu')
        G = to_networkx(data, node_attrs=['y'], edge_attrs=['att'])
        mapping = {k: i for k, i in enumerate(subset.tolist())}
        G = nx.relabel_nodes(G, mapping)

        node_kwargs = copy(kwargs)
        node_kwargs['node_size'] = kwargs.get('node_size') or 800
        node_kwargs['cmap'] = kwargs.get('cmap') or 'Accent'

        label_kwargs = copy(kwargs)
        label_kwargs['font_size'] = kwargs.get('font_size') or 10

        pos = nx.spring_layout(G)
        ax = plt.gca()
        ax.axis('off')
        for source, target, data in G.edges(data=True):
            ax.annotate(
                '', xy=pos[target], xycoords='data', xytext=pos[source],
                textcoords='data', arrowprops=dict(
                    arrowstyle="->",
                    alpha=max(data['att'], 0.05),
                    shrinkA=sqrt(node_kwargs['node_size']) / 2.0,
                    shrinkB=sqrt(node_kwargs['node_size']) / 2.0,
                    connectionstyle="arc3,rad=0.1",
                ))
        nx.draw_networkx_nodes(G, pos, node_color=self.y.tolist(), **node_kwargs)
        nx.draw_networkx_labels(G, pos, **label_kwargs)

        return ax, G

reason: import networkx


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Nov 24 22:29:44 2016

@author: Admin
"""
from __future__ import division
import numpy as np

AU = 149597871000
Ms = 1.989e30
Me = 5.972e24
Mm = 7.342e22

"Defining Variables"
N = 2
t_max = 3.1556e7; t = 0
dt_max = t_max/5000

v =  (2*np.pi*AU)/t_max
m = 384399000; vm = v + 1022

mass = np.array([Ms,Me])

pos = np.zeros((N,3))
vel = np.zeros((N,3))

pos[1] = np.array([0,AU,0])
#pos[2] = np.array([0,AU + m,0])
vel[1] = np.array([v,0,0])
#vel[2] = np.array([vm,0,0])

e = 0.0005*AU; n = 0.1

a = []; Ta = []
b = []; Tb = []
c = []; Tc = []

Tsum = []

T = []; dT = []

max_stars_repo_name: Gezerj/Python

max_issues_repo_name: Gezerj/Python

max_forks_repo_name: Gezerj/Python


text:
#!/usr/bin/env python3

"""
Project 5: Handwritten Letters Recognition with Neural Network
Author: Stephen Xie <***@andrew.cmu.edu>

This project trains a neural network with a single hidden layer to recognize
handwritten letters.

Data Assumptions:

1. The dataset is a csv file in which the first column be the indices of the
   actual letter (0: 'a', 1: 'e', 2: 'g', 3: 'i', 4: 'l', 5: 'n', 6: 'o',
   7: 'r', 8: 't', 9: 'u'), and the remaining 128 columns be the pixel values
   that can be reshaped into a 16 x 8 OCR grayscale (even though the given
   data is binary (B/W), we assume arbitrary values in range [0, 1]) image for
   each row.
2. The dataset always has the same number of features (128) and same output
   label space ({0, 1, ..., 9}) for each sample.

Notes:

1. This implementation does not have any forms of regularization (l1 / l2).
2. The implementation of Stochastic Gradient Descent (SGD) does not shuffle
   the training data, i.e. it goes through the training dataset in its original
   order during each iteration. This may not be "stochastic" but it allows
   producing deterministic results for graders.
3. Learning rate is fixed throughout the training process (set by command flag)
   for the benefit of the graders.
4. This implementation follows module-based automatic differentiation (AD)
   convention for easier gradient debugging (faster to locate the layer in
   which the gradient goes wrong) and easier model mutation in the future.
"""

import sys
import numpy as np

class NeuralNet:

    # a map between unique indices and their corresponding labels that matches
    # the dataset convention
    label_dict = {0: 'a', 1: 'e', 2: 'g', 3: 'i', 4: 'l', 5: 'n', 6: 'o',
                  7: 'r', 8: 't', 9: 'u'}

    def __init__(self, train_input, validation_input, train_out,
                 validation_out, metrics_out, num_epoch, hidden_units,
                 init_flag, learning_rate):
        t_labels, t_data = self.load(train_input)
        v_labels, v_data = self.load(validation_input)
        # number of inputs values excluding bias per sample; here it's just the
        # number of pixels per picture
        self.input_size = 16 * 8

        with open(metrics_out, mode='w') as f:
            # train model
            self.alpha, self.beta = self.train(t_labels, t_data, v_labels,
                                               v_data, num_epoch, hidden_units,
                                               init_flag, learning_rate, f)
            # print('alpha =\n', self.alpha, '\nbeta =\n', self.beta)

            # make predictions
            f.write('error(train): %.2f\n' % self._predict(t_labels, t_data,
                                                           train_out))
            f.write('error(validation): %.2f\n' % self._predict(v_labels, v_data,
                                                                validation_out))

    def load(self, input_file):
        """
        Load data into memory.
        Returns: labels list, data table
        """
        raw_data = np.genfromtxt(input_file, delimiter=',', dtype=np.int,
                                 autostrip=True)
        # in numpy, arrays slices are just views on the original array
        data = raw_data[:, 1:]
        # convert labels to one-hot representation
        # trick used here: pick the correct 1-hot representation for each label
        # from a matrix of pre-generated 1-hot representations of unique labels
        labels = np.eye(len(self.label_dict))[raw_data[:, 0]]

        # TODO: visualize dataset (comment out before submission)
        # self._visualize_dataset(raw_data[:, 0], data)

        return labels, data

    def train(self, t_labels, t_data, v_labels, v_data, num_epoch,
              hidden_units, init_flag, learning_rate, metrics_f):
        """
        Train the neural net using Stochastic Gradient Descent (SGD).
        """
        # initialize parameters
        alpha, beta = self._init_params(init_flag, hidden_units)
        # instead of checking for convergence, specify exact number of times
        # SGD loops through all of the training data.
        for e in range(num_epoch):
            for y, x in zip(t_labels, t_data):
                # convert x, y to column vectors, and add a bias term to x
                x = self._add_bias_vec(x.reshape(-1, 1))
                y = y.reshape(-1, 1)

                # forward pass
                a, z, b, y_hat, loss = self.nn_forward(x, y, alpha, beta)

                # backward pass
                djd_alpha, djd_beta = self.nn_backward(x, y, y_hat, alpha, beta, a, z, b)

                # update parameters
                alpha -= learning_rate * djd_alpha
                beta -= learning_rate * djd_beta

            # report cross entropy with the new alpha & beta
            metrics_f.write('epoch=%d crossentropy(train): %.11f\n' %
                            (e + 1, self.mean_cross_entropy(t_labels, t_data,
                             alpha, beta)))
            metrics_f.write('epoch=%d crossentropy(validation): %.11f\n' %
                            (e + 1, self.mean_cross_entropy(v_labels, v_data,
                             alpha, beta)))
            # metrics_f.write(
            #     '%d,%.11f,%.11f\n' %
            #     (e + 1,
            #      self.mean_cross_entropy(t_labels, t_data, alpha, beta),
            #      self.mean_cross_entropy(v_labels, v_data, alpha, beta)))

        return alpha, beta

    def _predict(self, labels, data, output_file):
        """
        Perform a forward pass through the neural net with the trained
        parameters on the given dataset. Output predicted labels as well as
        error metrics.
        """
        total_errors = 0
        with open(output_file, mode='w') as f:
            for y, x in zip(labels, data):
                # convert x, y to column vectors, and add a bias term to x
                x = self._add_bias_vec(x.reshape(-1, 1))
                y = y.reshape(-1, 1)
                a, z, b, y_hat, loss = self.nn_forward(x, y, self.alpha, self.beta)
                predicted = np.argmax(y_hat)
                f.write('%d\n' % predicted)
                if predicted != np.nonzero(y)[0][0]:
                    total_errors += 1

        return total_errors / len(labels)

    def _init_params(self, init_flag, hidden_units):
        """
        Initialize parameters according to the init_flag: 1 for random
        initializations (a uniform distribution from -0.1 to 0.1) and 2 for
        zero initializations. Bias term will always be initialized to zero.

        hidden_units: number of hidden units specified by user
        """
        if init_flag == 2:
            alpha, beta = np.zeros((self.input_size + 1, hidden_units)), \
                          np.zeros((hidden_units + 1, len(self.label_dict)))
        else:
            alpha, beta = np.random.uniform(-0.1, 0.1,
                                            (self.input_size + 1, hidden_units)), \
                          np.random.uniform(-0.1, 0.1,
                                            (hidden_units + 1, len(self.label_dict)))
            alpha[0], beta[0] = 0, 0  # set bias terms to 0

        return alpha, beta

    def nn_forward(self, x, y, alpha, beta):
        """
        Perform a forward pass on the neural network.
        """
        a = self._linear_forward(x, alpha)
        z = self._sigmoid_forward(a)
        b = self._linear_forward(z, beta)
        loss, y_hat = self._output_layers_forward(b, y)

        return a, z, b, y_hat, loss

    def nn_backward(self, x, y, y_hat, alpha, beta, a, z, b):
        """
        Perform a backward pass on the neural network.
        """
        djdb = self._output_layers_backward(y_hat, y)
        # assert self._test_grad(djdb, b, (self._output_layers_forward, y))
        djd_beta, djdz = self._linear_backward(djdb, beta, z)
        # assert self._test_grad(djdz, z, (self._linear_forward, beta),
        #                        (self._output_layers_forward, y))
        djda = self._sigmoid_backward(djdz, z)
        # assert self._test_grad(djda, a, self._sigmoid_forward,
        #                        (self._linear_forward, beta),
        #                        (self._output_layers_forward, y))
        djd_alpha, djdx = self._linear_backward(djda, alpha, x)
        # assert self._test_grad(djdx, x, (self._linear_forward, alpha),
        #                        self._sigmoid_forward,
        #                        (self._linear_forward, beta),
        #                        (self._output_layers_forward, y))

        return djd_alpha, djd_beta

    def mean_cross_entropy(self, labels, data, alpha, beta):
        """
        Perform a forward pass through the neural net with the given dataset
        and parameters to calculate mean cross entropy.
        """
        total_loss = 0
        for y, x in zip(labels, data):
            # convert x, y to column vectors, and add a bias term to x
            x = self._add_bias_vec(x.reshape(-1, 1))
            y = y.reshape(-1, 1)
            a, z, b, y_hat, loss = self.nn_forward(x, y, alpha, beta)
            total_loss += loss

        return total_loss / len(labels)

    # sigmoid module

    def _sigmoid_forward(self, x):
        """
        Applies forward function of the sigmoid layer to each element and adds
        a bias term to the front.
        """
        return self._add_bias_vec(self._sigmoid(x))

    def _sigmoid_backward(self, deriv, z):
        """
        Applies forward function of the sigmoid layer to each element.
        """
        # remove derivative for bias term first
        return deriv[1:] * self._d_sigmoid(z[1:])

    # linear module

    def _linear_forward(self, x, param):
        """
        Applies forward function of the linear layer to each element.
        """
        return np.dot(param.T, x)

    def _linear_backward(self, deriv, param, z):
        """
        Returns dJd\\param, dJdz.
        """
        return np.dot(z, deriv.T), np.dot(param, deriv)

    # softmax + cross entropy module

    def _output_layers_forward(self, x, y):
        """
        Applies forward function of the last two layers combined (softmax +
        cross entropy).
        """
        y_hat = self._softmax(x)
        loss = self._cross_entropy(y_hat, y)
        return loss, y_hat

    def _output_layers_backward(self, y_hat, y):
        """
        Applies backward function of the last two layers combined (softmax +
        cross entropy).
        """
        return y_hat - y

    # helper functions

    def _add_bias_vec(self, col_vec):
        """
        Returns a new column vector with a bias term 1 added to the top.
        """
        return np.concatenate(([[1]], col_vec))

    def _sigmoid(self, x):
        """
        Performs sigmoid function calculation and returns the result.
        """
        return 1 / (1 + np.exp(-x))

    def _softmax(self, x):
        """
        Performs probability calculation for multinomial logistic regression
        and returns the result.
        """
        exp = np.exp(x)
        return exp / np.sum(exp)

    def _cross_entropy(self, y_hat, y):
        """
        Calculates and returns cross entropy loss.

        y: the actual y value from dataset (either 0 or 1)
        y_hat: the predicted probabilities corresponding to y
        """
        return -np.sum(y * np.log(y_hat))

    def _d_sigmoid(self, y):
        """
        Calculates and returns derivatives of a sigmoid layer.
        """
        return y * (1 - y)

    def _test_grad(self, grad, x, *func_bundles):
        """
        A helper utility used for testing gradients computed by backpropagation.
        Returns false if the provided gradient doesn't roughly equal to the
        result calculated by finite difference approximation.

        grad: the gradient to be tested
        x: the input vector for the target function
        func_bundles: a list of tuples of functions and their additional parameters
                    (the first function should not include the input vector),
                    disassembled from the original nested function; ordered by
                    how close the current function is to the input vector x.

        Usage Example:
        Say you have a nested function sigmoid(linear(x), theta), call _test_grad
        like this: _test_grad(grad, x, linear, (sigmoid, theta))
        """
        epsilon = 1e-5
        # the parameters accepted by the nearest function defines the vector length
        # of the final gradient
        grad_ref = np.zeros((len(x), 1))
        for i in range(len(x)):
            # only 1 input is modified with epsilon at a time
            # declare type explicitly, since x could be of integer type
            y1, y2 = np.array(x, copy=True, dtype=np.float), \
                     np.array(x, copy=True, dtype=np.float)
            y1[i], y2[i] = y1[i] + epsilon, y2[i] - epsilon
            for bundle in func_bundles:
                if callable(bundle):
                    # the current bundle contains only the function
                    y1 = bundle(y1)
                    y2 = bundle(y2)
                else:
                    # must pass in additional parameters as well
                    # asterisk before a tuple unwraps it into parameters
                    y1 = bundle[0](y1, *bundle[1:])
                    if isinstance(y1, tuple):
                        y1 = y1[0]
                    y2 = bundle[0](y2, *bundle[1:])
                    if isinstance(y2, tuple):
                        y2 = y2[0]

            grad_ref[i] = (y1 - y2) / (2 * epsilon)

        # print(func_bundles)
        # print('grad', grad)
        # print('grad_ref', grad_ref)

        # test whether all gradients are a close match
        return np.all(np.absolute(grad_ref - grad) < epsilon)

    def _visualize_dataset(self, labels, data):
        """
        A test utility used to visualize a random subset of the given labels
        and corresponding 16x8 B/W pixel data. Note that the provided data must
        have at least 36 records.
        """
        import matplotlib.pyplot as plt

        fig = plt.figure()
        inds = np.random.randint(len(labels), size=36)
        for i, ind in enumerate(inds):
            # setting xticks & yticks to empty removes the ticks but not the
            # entire axis, unlike ax.axis('off'); in this way, each image still
            # keeps a 1-pixel black border around it
            ax = fig.add_subplot(6, 6, i + 1, xticks=[], yticks=[])
            pixels = data[ind].reshape(16, 8)
            ax.imshow(pixels, cmap='gray')
            ax.text(-6, 15, self.label_dict[labels[ind]], color='black',
                    fontsize=12)

if __name__ == '__main__':
    train_input = sys.argv[1]
    validation_input = sys.argv[2]
    train_out = sys.argv[3]
    validation_out = sys.argv[4]
    metrics_out = sys.argv[5]
    num_epoch = int(sys.argv[6])
    hidden_units = int(sys.argv[7])
    init_flag = int(sys.argv[8])
    learning_rate = float(sys.argv[9])

    model = NeuralNet(train_input, validation_input, train_out, validation_out,
                      metrics_out, num_epoch, hidden_units, init_flag,
                      learning_rate)

    # model = NeuralNet('largeTrain.csv', 'largeValidation.csv', 'train_out.txt',
    #                   'validation_out.txt', 'metrics_out.txt', 2, 4, 2, 0.1)

max_stars_repo_path: 3- Neural Network/neuralnet.py

max_stars_repo_name: Stephen-X/machine-learning-algorithms

max_issues_repo_path: 3- Neural Network/neuralnet.py

max_issues_repo_name: Stephen-X/machine-learning-algorithms

max_forks_repo_path: 3- Neural Network/neuralnet.py

max_forks_repo_name: Stephen-X/machine-learning-algorithms


text:
# Copyright (c) 2020, embedINN
# All rights reserved.

# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.

# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import iotSDR.at86rf215_driver as radio
import iotSDR.at86rf215_defs as defs
from pynq import MMIO

import numpy as np

class Transceiver(object):
    """This class is used for the radio/pl/ps settings"""

    def __init__(self,channel='ch1',overlay=None):

        self.channel = channel
        self.overlay  = overlay
        self._radio_init(self.channel)
        self._pl_cores_init(self.channel)

        #self.rxtx_lvds_reset(self.channel)

    def _radio_init(self,channel):
        """channel: at86rf215 chip selection"""

        self.spidev = 2 if channel == 'ch2' else 1

        self.radio = radio.At86rf215(0,0,self.spidev)
        self.radio.spi_init()
        self.radio.radio_reset()

        #Configure RFCHIP as IQ samples
        """FIXME:Sample Rate is fixed in PL"""
        """FIXME:lvds ddr delay caliberation"""
        sample_rate = 2
        ddrDelay = 1
        self.radio.radio_write_spi(defs.RG_RF_IQIFC1,0x1 << 4 | ddrDelay)
        self.radio.radio_write_spi(defs.RG_RF09_RXDFE,sample_rate)
        self.radio.radio_write_spi(defs.RG_RF_IQIFC0,0x34)
        self.radio.radio_trx_enable()
        self.radio.radio_trx_24_enable()

        self.radio_ddr_delay(3)

    def setTxSampleRate(self,rate,chan):
        """
        0x1 fS = 4000kHz
        0x2 fS = 2000kHz
        0x3 fS = (4000/3)kHz
        0x4 fS = 1000kHz
        0x5 fS = 800kHz
        0x6 fS = (2000/3)kHz
        0x8 fS = 500kHz
        0xA fS = 400kHz
        """
        #FIXME:Sample rate support in Pl
        defaultRate = rate
        if chan == "subGHz":
            self.radio.radio_write_spi(defs.RG_RF09_TXDFE,defaultRate)

        if chan == "24GHz":
            self.radio.radio_write_spi(defs.RG_RF24_TXDFE,defaultRate)

    def setRxSampleRate(self,rate,chan):
        """
        0x1 fS = 4000kHz
        0x2 fS = 2000kHz
        0x3 fS = (4000/3)kHz
        0x4 fS = 1000kHz
        0x5 fS = 800kHz
        0x6 fS = (2000/3)kHz
        0x8 fS = 500kHz
        0xA fS = 400kHz
        """
        #FIXME:Sample rate support in Pl
        defaultRate = rate
        if chan == "subGHz":
            self.radio.radio_write_spi(defs.RG_RF09_RXDFE,defaultRate)

        if chan == "24GHz":
            self.radio.radio_write_spi(defs.RG_RF24_RXDFE,defaultRate)

    def radio_ddr_delay(self,delay):
        """Delay = 0,1,2,3"""
        self.radio.radio_write_spi(defs.RG_RF_IQIFC1,0x1 << 4 | delay)

    def radio_start_reception(self,chan):
        if chan == "subGHz":
            self.radio.radio_trx_enable()
            self.radio.radio_rx_now()
        if chan == "24GHz":
            self.radio.radio_trx_24_enable()
            self.radio.radio_rx_24_now()

    def radio_start_transmission(self,chan):
        if chan == "subGHz":
            self.radio.radio_trx_enable()
            self.radio.radio_tx_now()
        if chan == "24GHz":
            self.radio.radio_trx_24_enable()
            self.radio.radio_tx_24_now()

    def radio_prep_mode(self):
        self.radio.radio_trx_enable()

    def radio_loopback(self):
        self.radio.radio_write_spi(defs.RG_RF_IQIFC0,0x34)

    def radio_tx_power(self,val=28):
        self.radio.radio_write_spi(defs.RG_RF09_PAC,val)

    def radio_tx_power_24g(self,val=28):
        self.radio.radio_write_spi(defs.RG_RF24_PAC,val)

    def txrx_frequency(self,
        channel_spacing_kHz = 25,
        frequency_0_kHz = 850000,
        channel = 11
        ):

        channel_set_up = (channel_spacing_kHz, frequency_0_kHz, channel)
        self.radio.radio_set_frequency(channel_set_up)

    def txrx_frequency_24g(self,
        channel_spacing_kHz = 5000,
        frequency_0_kHz = 850000,
        channel = 11
        ):

        channel_set_up = (channel_spacing_kHz, frequency_0_kHz, channel)
        self.radio.radio_set_frequency_2_4ghz(channel_set_up)

    """PL Side Methods"""

    def _pl_cores_init(self,channel):

        self.ADDRESS_RANGE  = 0x1000
        self.ADDRESS_OFFSET = 0x00

        #RX IP bit offset settings
        self.IQENABLE_OFFSET    = 8
        self.FIFORESET_OFFSET   = 4
        self.LVDSRESET_OFFSET   = 12
        self.IQSTREAMSEL_OFFSET = 0

        #TX IP bit offset settings
        self.BRAM_ADDR_RANGE       = 4096
        self.EN_BRAM_STREAM_OFFSET = 0
        self.ADDRESS_RESET_OFFSET  = 1
        self.TX_SAMPLE_RATE_OFFSET = 4

        """PL IP cores Addresses"""
        if (self.channel == 'ch1'):
            RECEIVER_SETTINGS_ADDRESS     = 0x41200000
            SAMPLESPERBLOCK_ADDRESS       = 0x41210000
            TRANSMITTER_IQ_BRAM_ADDRESS   = 0x42000000
            TRANSMITTER_SETTINGS_ADDRESS  = 0x41230000
            self.TxSettings   = MMIO(TRANSMITTER_SETTINGS_ADDRESS, 0x4)
            self.IQStreamBram = MMIO(TRANSMITTER_IQ_BRAM_ADDRESS, self.BRAM_ADDR_RANGE)
            self.RxSettings   = MMIO(RECEIVER_SETTINGS_ADDRESS, 0x4)
            self.BlockSizeReg = MMIO(SAMPLESPERBLOCK_ADDRESS, self.ADDRESS_RANGE)
            self.pl_dma_txrx  = self.overlay.TRX_CH1.axi_dma_txrx_09
        else:
            RECEIVER_SETTINGS_ADDRESS     = 0x41240000
            SAMPLESPERBLOCK_ADDRESS       = 0x41220000
            TRANSMITTER_IQ_BRAM_ADDRESS   = 0x40000000
            TRANSMITTER_SETTINGS_ADDRESS  = 0x41250000
            self.TxSettings   = MMIO(TRANSMITTER_SETTINGS_ADDRESS, 0x4)
            self.IQStreamBram = MMIO(TRANSMITTER_IQ_BRAM_ADDRESS, self.BRAM_ADDR_RANGE)
            self.RxSettings   = MMIO(RECEIVER_SETTINGS_ADDRESS, 0x4)
            self.BlockSizeReg = MMIO(SAMPLESPERBLOCK_ADDRESS, self.ADDRESS_RANGE)
            self.pl_dma_txrx  = self.overlay.TRX_CH2.axi_dma_txrx_09

    def rx_samples_select(self,source=2):
        """
        source =
        enable_tx_loopback = 0x0
        eanble_rf_09       = 0x1
        eanble_rf_24       = 0x2
        """
        #bit 0-1
        mask = self.RxSettings.read(0) & 0xFFFFFFFC
        self.RxSettings.write(0, source | mask)

    def enable_pl_rx_samples(self,val):
        #bit 8
        mask = self.RxSettings.read(0) & 0xFFFFFEFF
        self.RxSettings.write(0, val  << 8 | mask)

    def tx_samples_select(self,source=1):
        # bit 0
        mask = self.TxSettings.read(0) & 0xFFFFFFFE
        self.TxSettings.write(0x0, source | mask )

    def set_rx_block_size(self,samples_size=1024):
        self.BlockSizeReg.write(0, samples_size)

    def rx_fifo_reset(self):
        # bit 4
        mask = self.RxSettings.read(0) & 0xFFFFFFF7
        self.RxSettings.write(0, 0x1 << 4 | mask)
        self.RxSettings.write(0, 0x0 << 4 | mask)

    def rxtx_lvds_reset(self):
        # bit 12
        mask = self.RxSettings.read(0) & 0xFFFFFFF7
        self.RxSettings.write(0, 0x1 << 12 | mask)
        self.RxSettings.write(0, 0x0 << 12 | mask)

    def enable_buffer_tx_samples(self,val):
        #FIXME: Seperate enable and source  sel
        # bit 0
        mask = self.TxSettings.read(0) & 0xFFFFFFFE
        self.TxSettings.write(0x0, val | mask )

    def set_tx_LUT(self,IQsamples,chan):

        self.Isamples  = np.array(IQsamples.real*(2**13-1),dtype=np.int16)
        self.Qsamples  = np.array(IQsamples.imag*(2**13-1),dtype=np.int16)
        #print("Core:",(IQsamples))
        j = 0
        for bramAdress in range(0,len(self.Isamples)*4,4):
                y = ((int(self.Isamples[j]) << 16) & 0xFFFFFFFF | int(self.Qsamples[j]) & 0xFFFF) &  0xFFFFFFFF
                self.IQStreamBram.write(bramAdress,y)
                j+=1

        self.enable_buffer_tx_samples(1)
        self.radio_start_transmission(chan)

    def set_tx_LUT_remote(self,IQsamples):

        #self.radio.radio_reset()

        self.Isamples  = np.array(IQsamples.real)#*(2**13-1),dtype=np.int16)
        self.Qsamples  = np.array(IQsamples.imag)#*(2**13-1),dtype=np.int16)
        #print("Core:",(IQsamples))
        j = 0
        for bramAdress in range(0,len(self.Isamples)*4,4):
                y = ((int(self.Isamples[j]) << 16) & 0xFFFFFFFF | int(self.Qsamples[j]) & 0xFFFF) &  0xFFFFFFFF
                self.IQStreamBram.write(bramAdress,y)
                j+=1

        self.enable_buffer_tx_samples(1)
        self.radio_start_transmission(chan)

max_stars_repo_name: mfkiwl/iotSDR

max_issues_repo_name: mfkiwl/iotSDR

max_forks_repo_name: mfkiwl/iotSDR


text:
from pymatch.ReinforcementLearning.callback import UncertaintyUpdater
from pymatch.ReinforcementLearning.learner import ReinforcementLearner
import numpy as np
import torch

class MockLearner(ReinforcementLearner):
    def __init__(self, memory):
        self.train_loader = memory
        self.train_dict = {'epochs_run': 0}
        self.device = 'cpu'

    def __call__(self, X):
        return X

class MockMemory:
    def __init__(self, memory: dict):
        self.memory = memory

    def sample_loader(self, shuffle):
        return [[[0], torch.tensor(self.memory['state']), [0], [0], [0]]]

class MockHat:
    def __call__(self, X, *args, **kwargs):
        std = torch.zeros(size=(len(X), 4))
        std[:, 0] = torch.tensor(np.arange(10, 20))
        return X, std

updater = UncertaintyUpdater(head=MockHat())
learner = MockLearner(memory=MockMemory({'state': np.arange(0, 10).reshape(-1, 1, 1)}))

updater(learner)

learner.train_loader.memory

torch.load

max_stars_repo_name: raharth/PyMatch

max_issues_repo_name: raharth/PyMatch

max_forks_repo_name: raharth/PyMatch


text:
import pyopencl as cl
import numpy as np

class OpenClRuntime:
    def __init__(self):
        self.context = cl.create_some_context()
        self.queue = cl.CommandQueue(self.context)

        self.array1 = np.eye(100, dtype=np.int32)
        self.array2 = np.eye(100, dtype=np.int32)

        self.load_program("multidimensionalAddition.cl")

    def load_program(self, filename):
        f = open(filename, 'r')
        fstr = "".join(f.readlines())
        self.program = cl.Program(self.context, fstr).build()

    def execute(self):
        array1Buf = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=self.array1)
        array2Buf = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=self.array1)

        output = np.zeros_like(self.array1, dtype=np.int32)
        outputBuf = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=output)

        self.program.add(self.queue, self.array1.shape, None, array1Buf, array2Buf, outputBuf)
        self.result = np.zeros_like(self.array1, dtype=np.int32)

        cl.enqueue_read_buffer(self.queue, outputBuf, self.result).wait();

    def print_out(self):
        for i in range(len(self.result)):
            output = "";
            for f in range(len(self.result)):
                output = output + " " + str(self.result[i][f]);

            print(output);

if __name__ == "__main__":
    runtime = OpenClRuntime();
    runtime.execute();
    runtime.print_out();
    print("End");

max_stars_repo_name: duffleit/pythonOpenCL

max_issues_repo_name: duffleit/pythonOpenCL

max_forks_repo_name: duffleit/pythonOpenCL


text:
import numpy as np
import os

def load_embedding(word2idx, dim, file_path):

    v_len = len(word2idx)
    # word2idx is included with UNK, EOS, SOS
    weights = np.random.uniform(low=-0.05, high=0.05,size=(v_len, dim))
    # index 0 is for PAD
    weights= np.concatenate((np.zeros((1,dim)), weights))
    token_in_we = 0
    if not os.path.exists(file_path):
        raise FileNotFound
    with open(file_path, mode='r') as f:
        for line in f:
            values = line.split()
            word = values[0]
            idx = word2idx.get(word)
            if idx is not None:
                coef = np.asarray(values[1:], dtype='float32')
                weights[idx] = coef
                token_in_we +=1

    no_we_words = v_len - token_in_we
    print(no_we_words, " tokens does not exist in the word embedding")

    return weights

def data2idx(data, HP, tokenizer):

    data_idxs = []
    sent_lengths = []
    docs_lengths = []
    for doc in data:
        doc_idx, sent_lens = tokenizer.text2idx(doc, pad_length=HP.MAX_WORD, add_eos=True)
        data_idxs.append(doc_idx)
        sent_lengths.append(sent_lens)
        docs_lengths.append(len(doc))

    return data_idxs, sent_lengths, docs_lengths

def batchizer(docs_idx, sent_lengths, docs_lengths, labels, batch_size, max_sent):

#     # shuffle data
    m = len(labels)

    per = np.arange(m)
    np.random.shuffle(per)
    docs_idx_shuffled = [docs_idx[index] for index in per]
    sent_lengths_shuffled = [sent_lengths[index] for index in per]
    docs_lengths_shuffled = [docs_lengths[index] for index in per]
    labels_shuffled = [labels[index] for index in per]

    n_batch = int(m / batch_size)

    for i in range(n_batch):
        # slice the list
        batch_docs_idx = docs_idx_shuffled[i*batch_size:(i+1)*batch_size]
        batch_sent_length = sent_lengths_shuffled[i*batch_size:(i+1)*batch_size]
        # pad sentences to max_sent length
        batch_sent_length = [k[:max_sent] + [0]*np.max(max_sent - len(k),0) for k in batch_sent_length]

        batch_docs_length = docs_lengths_shuffled[i*batch_size:(i+1)*batch_size]
        # trim docs_length greater than 20
        batch_docs_length = [min(k, max_sent) for k in batch_docs_length]
        batch_labels = labels_shuffled[i*batch_size:(i+1)*batch_size]
        batch_max_words = np.max(np.max(batch_sent_length))
        batch_x = np.zeros((batch_size, max_sent, batch_max_words), dtype=np.int32)
        for d, doc in enumerate(batch_docs_idx):
            for s, sent in enumerate(doc):
                if s == max_sent:
                    break
                batch_x[d,s, 0:len(sent)] = sent[:]
        yield batch_x, batch_sent_length, batch_docs_length, batch_labels


text:
# File slightly modified from OANet
import torch
import numpy as np
import os
import cv2
from six.moves import xrange
from oan_loss import MatchLoss
from oan_evaluation import eval_nondecompose, eval_decompose
from oan_utils import tocuda, get_pool_result
import utils

def test_sample(args):
    _xs, _dR, _dt, _e_hat, _y_hat, _y_gt, config, = args
    _xs = _xs.reshape(-1, 4).astype('float64')
    _dR, _dt = _dR.astype('float64').reshape(3,3), _dt.astype('float64')
    _y_hat_out = _y_hat.flatten().astype('float64')
    e_hat_out = _e_hat.flatten().astype('float64')

    _x1 = _xs[:, :2]
    _x2 = _xs[:, 2:]
    # current validity from network
    _valid = _y_hat_out
    # choose top ones (get validity threshold)
    _valid_th = np.sort(_valid)[::-1][config.obj_top_k]
    _mask_before = _valid >= max(0, _valid_th)

    if not config.use_ransac:
        _err_q, _err_t, _, _, _num_inlier, _mask_updated, _R_hat, _t_hat = \
            eval_nondecompose(_x1, _x2, e_hat_out, _dR, _dt, _y_hat_out)
    else:
        # actually not use prob here since probs is None
        _err_q, _err_t, _, _, _num_inlier, _mask_updated, _R_hat, _t_hat = \
            eval_decompose(_x1, _x2, _dR, _dt, mask=_mask_before, method=cv2.RANSAC, \
            probs=None, weighted=False, use_prob=True)
    if _R_hat is None:
        _R_hat = np.random.randn(3,3)
        _t_hat = np.random.randn(3,1)
    return [float(_err_q), float(_err_t), float(_num_inlier), _R_hat.reshape(1,-1), _t_hat.reshape(1,-1)]

def dump_res(measure_list, res_path, eval_res, tag):
    # dump test results
    for sub_tag in measure_list:
        # For median error
        ofn = os.path.join(res_path, "median_{}_{}.txt".format(sub_tag, tag))
        with open(ofn, "w") as ofp:
            ofp.write("{}\n".format(np.median(eval_res[sub_tag])))

    ths = np.arange(7) * 5
    cur_err_q = np.array(eval_res["err_q"]) * 180.0 / np.pi
    cur_err_t = np.array(eval_res["err_t"]) * 180.0 / np.pi
    # Get histogram
    q_acc_hist, _ = np.histogram(cur_err_q, ths)
    t_acc_hist, _ = np.histogram(cur_err_t, ths)
    qt_acc_hist, _ = np.histogram(np.maximum(cur_err_q, cur_err_t), ths)
    num_pair = float(len(cur_err_q))
    q_acc_hist = q_acc_hist.astype(float) / num_pair
    t_acc_hist = t_acc_hist.astype(float) / num_pair
    qt_acc_hist = qt_acc_hist.astype(float) / num_pair
    q_acc = np.cumsum(q_acc_hist)
    t_acc = np.cumsum(t_acc_hist)
    qt_acc = np.cumsum(qt_acc_hist)
    # Store return val
    for _idx_th in xrange(1, len(ths)):
        ofn = os.path.join(res_path, "acc_q_auc{}_{}.txt".format(ths[_idx_th], tag))
        with open(ofn, "w") as ofp:
            ofp.write("{}\n".format(np.mean(q_acc[:_idx_th])))
        ofn = os.path.join(res_path, "acc_t_auc{}_{}.txt".format(ths[_idx_th], tag))
        with open(ofn, "w") as ofp:
            ofp.write("{}\n".format(np.mean(t_acc[:_idx_th])))
        ofn = os.path.join(res_path, "acc_qt_auc{}_{}.txt".format(ths[_idx_th], tag))
        with open(ofn, "w") as ofp:
            ofp.write("{}\n".format(np.mean(qt_acc[:_idx_th])))

    ofn = os.path.join(res_path, "all_acc_qt_auc20_{}.txt".format(tag))
    np.savetxt(ofn, np.maximum(cur_err_q, cur_err_t))
    ofn = os.path.join(res_path, "all_acc_q_auc20_{}.txt".format(tag))
    np.savetxt(ofn, cur_err_q)
    ofn = os.path.join(res_path, "all_acc_t_auc20_{}.txt".format(tag))
    np.savetxt(ofn, cur_err_t)

    # Return qt_auc20
    ret_val = np.mean(qt_acc[:4])  # 1 == 5
    return ret_val

def denorm(x, T):
    x = (x - np.array([T[0,2], T[1,2]])) / np.asarray([T[0,0], T[1,1]])
    return x

def test_process(mode, model, cur_global_step, data_loader, config):
    model.eval()
    match_loss = MatchLoss(config)
    loader_iter = iter(data_loader)

    # save info given by the network
    network_infor_list = ["geo_losses", "cla_losses", "l2_losses", 'precisions', 'recalls', 'f_scores']
    network_info = {info:[] for info in network_infor_list}

    results, pool_arg = [], []
    eval_step, eval_step_i, num_processor = 100, 0, 8
    with torch.no_grad():
        for test_data in loader_iter:
            test_data = tocuda(test_data)
            # res_logits, res_e_hat = model(test_data)

            # stupid inconsistency between ours and OANet forces this:
            test_data["Rs"] = test_data["R"]
            test_data["ts"] = test_data["t"]

            res_angles, res_ys = model(test_data["xs"])

            res_e_hat = utils.normalize_E_batch(
                utils.essential_matrix_from_angles(res_angles)
            )
            e_hat = torch.reshape(res_e_hat, (-1, 9))

            y_hat, _ = res_ys.max(dim=1)  # Note that these are not actually logits,
            # so the outputted precision, recall, f-scores do not make sense.

            loss, geo_loss, cla_loss, l2_loss, prec, rec = match_loss.run(cur_global_step, test_data, y_hat, e_hat)
            info = [geo_loss, cla_loss, l2_loss, prec, rec, 2*prec*rec/(prec+rec+1e-15)]
            for info_idx, value in enumerate(info):
                network_info[network_infor_list[info_idx]].append(value)

            if config.use_fundamental:
                # unnorm F
                e_hat = torch.matmul(torch.matmul(test_data['T2s'].transpose(1,2), e_hat.reshape(-1,3,3)),test_data['T1s'])
                # get essential matrix from fundamental matrix
                e_hat = torch.matmul(torch.matmul(test_data['K2s'].transpose(1,2), e_hat.reshape(-1,3,3)),test_data['K1s']).reshape(-1,9)
                e_hat = e_hat / torch.norm(e_hat, dim=1, keepdim=True)

            for batch_idx in range(e_hat.shape[0]):
                test_xs = test_data['xs'][batch_idx].detach().cpu().numpy()
                if config.use_fundamental: # back to original
                    x1, x2 = test_xs[0,:,:2], test_xs[0,:,2:4]
                    T1, T2 = test_data['T1s'][batch_idx].cpu().numpy(), test_data['T2s'][batch_idx].cpu().numpy()
                    x1, x2 = denorm(x1, T1), denorm(x2, T2) # denormalize coordinate
                    K1, K2 = test_data['K1s'][batch_idx].cpu().numpy(), test_data['K2s'][batch_idx].cpu().numpy()
                    x1, x2 = denorm(x1, K1), denorm(x2, K2) # normalize coordiante with intrinsic
                    test_xs = np.concatenate([x1,x2],axis=-1).reshape(1,-1,4)

                pool_arg += [(test_xs, test_data['Rs'][batch_idx].detach().cpu().numpy(), \
                              test_data['ts'][batch_idx].detach().cpu().numpy(), e_hat[batch_idx].detach().cpu().numpy(), \
                              y_hat[batch_idx].detach().cpu().numpy(),  \
                              test_data['ys'][batch_idx,:,0].detach().cpu().numpy(), config)]

                eval_step_i += 1
                if eval_step_i % eval_step == 0:
                    results += get_pool_result(num_processor, test_sample, pool_arg)
                    pool_arg = []
        if len(pool_arg) > 0:
            results += get_pool_result(num_processor, test_sample, pool_arg)

    measure_list = ["err_q", "err_t", "num", 'R_hat', 't_hat']
    eval_res = {}
    for measure_idx, measure in enumerate(measure_list):
        eval_res[measure] =  np.asarray([result[measure_idx] for result in results])

    if config.res_path == '':
        config.res_path = os.path.join(config.log_path[:-5], mode)
    tag = "ours" if not config.use_ransac else "ours_ransac"
    ret_val = dump_res(measure_list, config.res_path, eval_res, tag)
    return [ret_val, np.mean(np.asarray(network_info['geo_losses'])), np.mean(np.asarray(network_info['cla_losses'])), \
        np.mean(np.asarray(network_info['l2_losses'])), np.mean(np.asarray(network_info['precisions'])), \
        np.mean(np.asarray(network_info['recalls'])), np.mean(np.asarray(network_info['f_scores']))]

def test(data_loader, model, config):
    save_file_best = config.save_file_best
    if not os.path.exists(save_file_best):
        print("Model File {} does not exist! Quiting".format(save_file_best))
        exit(1)
    # Restore model
    checkpoint = torch.load(save_file_best)
    start_epoch = checkpoint['epoch']
    model.load_state_dict(checkpoint['state_dict'])
    model.cuda()
    print("Restoring from " + str(save_file_best) + ', ' + str(start_epoch) + "epoch...\n")
    if config.res_path == '':
        config.res_path = config.model_path[:-5]+'test'
    print('save result to '+config.res_path)
    va_res = test_process("test", model, 0, data_loader, config)
    print('WARNING: only the two first values below are meaningful')
    print('test result '+str(va_res))
    return va_res
def valid(data_loader, model, step, config):
    config.use_ransac = False
    return test_process("valid", model, step, data_loader, config)

max_stars_repo_name: georg-bn/zz-net

max_issues_repo_name: georg-bn/zz-net

max_forks_repo_name: georg-bn/zz-net


text:
from flask import Flask,jsonify
from flask_restplus import Api, fields, Resource
from pathlib import Path
from Gender_Classifier import retrainModel, normalize, name_encoding

import tensorflow as tf
import numpy as np
import csv

app = Flask(__name__)

api = Api(
    app,
    version='1.X',
    title='Gender Classifer API',
    description='This Application Programming Interface is used to predict the Gender of a Person given the name of the person')

ns = api.namespace('api')

parser = api.parser()
parser.add_argument(
    'Name',
    required=True,
    type= str,
    help='Give the Name of a person like Ajay',
    location='form',
    action='append')

modelParser = api.parser()
modelParser.add_argument(
    'Name',
    required=True,
    type= str,
    help='Give the Name of a person like Ajay',
    location='form')
modelParser.add_argument(
    'Gender',
    required=True,
    type= str,
    help='M/F',
    location='form')

@ns.route('/classifyGender')
class ClassifiyGender(Resource):
    @api.doc(parser=parser)
    def post(self):
        args = parser.parse_args()
        nameList = args['Name']
        resultList = []
        for name in nameList:
            if(name.isalpha()):
                result = self.get_result(name)
                resultList.append(result)
            else:
                return app.response_class(response="Error",status=404)

        response = jsonify(resultList)
        response.status_code=200
        return response

    def get_result(self, name):
        model_dir = Path("Gender_Classifier/gender_model.h5")
        model = tf.keras.models.load_model(model_dir)
        nameList = [name]
        prediction = model.predict(np.asarray([np.asarray(name_encoding(normalize(name))) for name in nameList]))
        return {
            'Name':name,
            'Male':(prediction.tolist())[0][0],
            'Female':(prediction.tolist())[0][1]
        }

@ns.route('/retrainModel')
class RetrainModel(Resource):
    @api.doc(parser=modelParser)
    def post(self):
        args = modelParser.parse_args()
        dataset_dir = Path("Gender_Classifier/name_gender.csv")
        with open(dataset_dir,'a') as f:
            writer = csv.writer(f)
            writer.writerow([args['Name'],args['Gender'],1])

        retrainModel()
        return app.response_class(response="Success",status=200)

if __name__ == '__main__':
    app.run(debug=True)

max_stars_repo_name: HarshCasper/Ling-API

max_issues_repo_name: HarshCasper/Ling-API

max_forks_repo_name: HarshCasper/Ling-API


text:
import os
from shutil import copyfile

import numpy as np

from dataset_specific.kits import utils

root_path = '/cache/suhita/data/kits/'
perc = 1.0
# training

fold_num = 1
labelled_path = '/data/suhita/temporal/kits/preprocessed_labeled_train'
labelled_fold_num = '/data/suhita/temporal/kits/Folds/train_fold' + str(fold_num) + '.npy'
un_labelled_path = '/data/suhita/temporal/kits/output/UL_' + str(perc) + '_PP/'
labelled_train_num = np.load(labelled_fold_num).shape[0]

labelled_num = np.load(labelled_fold_num).shape[0]
labelled_files_lst = np.load(labelled_fold_num)
un_labelled_files_lst = os.listdir(un_labelled_path)

train_fold = np.load('/data/suhita/temporal/kits/Folds/train_fold' + str(fold_num) + '.npy')
print(train_fold[0:10])
nr_samples = train_fold.shape[0]

# np.random.seed(5)
np.random.seed(5)
np.random.shuffle(train_fold)
print(train_fold[0:10])

labelled_num_considrd = train_fold[:int(nr_samples * perc)]
remaining_labelled = set(train_fold).difference(set(labelled_num_considrd))

counter = 0

data_path = '/cache/suhita/data/kits/fold_' + str(fold_num) + '_P' + str(perc)
utils.makedir(data_path)
for i in labelled_num_considrd:
    # name = labelled_files_lst[i]
    print(i, counter)
    fold_name = 'case_' + str(counter)
    utils.makedir(os.path.join(data_path, fold_name))
    copyfile(os.path.join(labelled_path, i, 'img_left.npy'), os.path.join(data_path, fold_name, 'img_left.npy'))
    copyfile(os.path.join(labelled_path, i, 'img_right.npy'),
             os.path.join(data_path, fold_name, 'img_right.npy'))
    copyfile(os.path.join(labelled_path, i, 'segm_left.npy'),
             os.path.join(data_path, fold_name, 'segm_left.npy'))
    copyfile(os.path.join(labelled_path, i, 'segm_right.npy'),
             os.path.join(data_path, fold_name, 'segm_right.npy'))
    counter = counter + 1

print('remianing labelled')
for i in remaining_labelled:
    # name = labelled_files_lst[i]
    print(i, counter)
    fold_name = 'case_' + str(counter)
    utils.makedir(os.path.join(data_path, fold_name))
    copyfile(os.path.join(labelled_path, i, 'img_left.npy'), os.path.join(data_path, fold_name, 'img_left.npy'))
    copyfile(os.path.join(labelled_path, i, 'img_right.npy'),
             os.path.join(data_path, fold_name, 'img_right.npy'))
    copyfile(os.path.join(labelled_path, i, 'segm_left.npy'),
             os.path.join(data_path, fold_name, 'segm_left.npy'))
    copyfile(os.path.join(labelled_path, i, 'segm_right.npy'),
             os.path.join(data_path, fold_name, 'segm_right.npy'))
    counter = counter + 1

print('unlabelled start...')

for i in np.arange(len(un_labelled_files_lst)):
    name = un_labelled_files_lst[i]
    print(name, counter)
    fold_name = 'case_' + str(counter)
    utils.makedir(os.path.join(data_path, fold_name))
    copyfile(os.path.join(un_labelled_path, name, 'img_left.npy'),
             os.path.join(data_path, fold_name, 'img_left.npy'))
    copyfile(os.path.join(un_labelled_path, name, 'img_right.npy'),
             os.path.join(data_path, fold_name, 'img_right.npy'))
    copyfile(os.path.join(un_labelled_path, name, 'segm_left.npy'),
             os.path.join(data_path, fold_name, 'segm_left.npy'))
    copyfile(os.path.join(un_labelled_path, name, 'segm_right.npy'),
             os.path.join(data_path, fold_name, 'segm_right.npy'))
    counter = counter + 1


text:
# encoding: utf-8
"""
@author : zhirui zhou
@contact: evilpsycho42@gmail.com
@time   : 2020/5/27 14:44
"""
# encoding: utf-8
"""
@author : zhirui zhou
@contact: evilpsycho42@gmail.com
@time   : 2020/5/12 16:33
"""
from deepseries.model import Wave2Wave, RNN2RNN
from deepseries.train import Learner
from deepseries.data import Value, create_seq2seq_data_loader, forward_split
from deepseries.nn import RMSE, MSE
import deepseries.functional as F
import numpy as np
import torch

batch_size = 16
enc_len = 36
dec_len = 12
series_len = 1000

epoch = 100
lr = 0.001

valid_size = 12
test_size = 12

series = np.sin(np.arange(0, series_len)) + np.random.normal(0, 0.1, series_len) + np.log2(np.arange(1, series_len+1))
series = series.reshape(1, 1, -1)

train_idx, valid_idx = forward_split(np.arange(series_len), enc_len=enc_len, valid_size=valid_size+test_size)
valid_idx, test_idx = forward_split(valid_idx, enc_len, test_size)

# mask test, will not be used for calculating mean/std.
mask = np.zeros_like(series).astype(bool)
mask[:, :, test_idx] = False
series, mu, std = F.normalize(series, axis=2, fillna=True, mask=mask)

# wave2wave train
train_dl = create_seq2seq_data_loader(series[:, :, train_idx], enc_len, dec_len, sampling_rate=0.1,
                                      batch_size=batch_size, seq_last=True, device='cuda')
valid_dl = create_seq2seq_data_loader(series[:, :, valid_idx], enc_len, dec_len,
                                      batch_size=batch_size, seq_last=True, device='cuda')

wave = Wave2Wave(target_size=1, num_layers=6, num_blocks=1, dropout=0.1, loss_fn=RMSE())
wave.cuda()
opt = torch.optim.Adam(wave.parameters(), lr=lr)
wave_learner = Learner(wave, opt, root_dir="./wave", )
wave_learner.fit(max_epochs=epoch, train_dl=train_dl, valid_dl=valid_dl, early_stopping=True, patient=16)
wave_learner.load(wave_learner.best_epoch)

# rnn2rnn train
train_dl = create_seq2seq_data_loader(series[:, :, train_idx], enc_len, dec_len, sampling_rate=0.1,
                                      batch_size=batch_size, seq_last=False, device='cuda')
valid_dl = create_seq2seq_data_loader(series[:, :, valid_idx], enc_len, dec_len,
                                      batch_size=batch_size, seq_last=False, device='cuda')
rnn = RNN2RNN(series_size=1, hidden_size=64, compress_size=32, residual=True, attn_heads=None,
              attn_size=None, debug=False, loss_fn=RMSE())
rnn.cuda()
rnn_learner = Learner(rnn, torch.optim.Adam(rnn.parameters(), lr=lr), root_dir="./rnn")
rnn_learner.fit(max_epochs=epoch, train_dl=train_dl, valid_dl=valid_dl, early_stopping=True, patient=16)
rnn_learner.load(rnn_learner.best_epoch)

# rnn2rnn attention train
train_dl = create_seq2seq_data_loader(series[:, :, train_idx], enc_len, dec_len, sampling_rate=0.1,
                                      batch_size=batch_size, seq_last=False, device='cuda')
valid_dl = create_seq2seq_data_loader(series[:, :, valid_idx], enc_len, dec_len,
                                      batch_size=batch_size, seq_last=False, device='cuda')
rnn_attn = RNN2RNN(series_size=1, hidden_size=256, compress_size=128, residual=True, attn_heads=1,
              attn_size=128, debug=False, loss_fn=RMSE())
rnn_attn.cuda()
rnn_attn_learner = Learner(rnn_attn, torch.optim.Adam(rnn_attn.parameters(), lr=lr), root_dir="./rnn")
rnn_attn_learner.fit(max_epochs=epoch, train_dl=train_dl, valid_dl=valid_dl, early_stopping=True, patient=16)
rnn_attn_learner.load(rnn_attn_learner.best_epoch)

# metric
wave_preds = wave_learner.model.predict(torch.tensor(series[:, :, test_idx[:-12]]).float().cuda(), 12).cpu().numpy().reshape(-1)
rnn_preds = rnn_learner.model.predict(torch.tensor(series[:, :, test_idx[:-12]]).float().cuda().transpose(2, 1), 12)[0].cpu().numpy().reshape(-1)
rnn_attn_preds = rnn_attn_learner.model.predict(torch.tensor(series[:, :, test_idx[:-12]]).float().cuda().transpose(2, 1), 12)[0].cpu().numpy().reshape(-1)

import matplotlib.pyplot as plt

plt.plot(wave_preds, label="wave2wave preds")
plt.plot(rnn_preds, label="rnn2rnn preds")
plt.plot(rnn_attn_preds, label="rnn2rnn attention preds")
plt.plot(series[:, :, test_idx[-12:]].reshape(-1), label="target")
plt.legend()

max_stars_repo_name: EvilPsyCHo/Deep-Time-Series-Prediction

max_issues_repo_name: luxixiang/Deep-Time-Series-Prediction

max_forks_repo_name: luxixiang/Deep-Time-Series-Prediction


text:
import numpy as np

def frank_correction_factors(ghi, dni_extra, times, solar_elevation):
    """Applies the proposed transmissivity-based irradiance corrections to COSMO
    data based on Frank et al.

    TODO: Add citation to Frank!
    """
    transmissivity = ghi / dni_extra
    sigmoid = 1 / (1 + np.exp(-(transmissivity - 0.5) / 0.03))

    # Adjust cloudy regime
    months = times.month
    cloudyFactors = np.empty(months.shape)

    cloudyFactors[months == 1] = 0.7776553729824053
    cloudyFactors[months == 2] = 0.7897164461247639
    cloudyFactors[months == 3] = 0.8176553729824052
    cloudyFactors[months == 4] = 0.8406805293005672
    cloudyFactors[months == 5] = 0.8761808928311765
    cloudyFactors[months == 6] = 0.9094139886578452
    cloudyFactors[months == 7] = 0.9350856478115459
    cloudyFactors[months == 8] = 0.9191682419659737
    cloudyFactors[months == 9] = 0.912703795259561
    cloudyFactors[months == 10] = 0.8775035625999711
    cloudyFactors[months == 11] = 0.8283158353933402
    cloudyFactors[months == 12] = 0.7651417769376183
    cloudyFactors = np.broadcast_to(
        cloudyFactors.reshape((cloudyFactors.size, 1)), ghi.shape)

    cloudyFactors = cloudyFactors * (1 - sigmoid)

    # Adjust clearsky regime
    e = solar_elevation
    clearSkyFactors = np.ones(e.shape)
    clearSkyFactors[np.where((e >= 10) & (e < 20))] = 1.17612920884004
    clearSkyFactors[np.where((e >= 20) & (e < 30))] = 1.1384180020822825
    clearSkyFactors[np.where((e >= 30) & (e < 40))] = 1.1022951259566156
    clearSkyFactors[np.where((e >= 40) & (e < 50))] = 1.0856852748290704
    clearSkyFactors[np.where((e >= 50) & (e < 60))] = 1.0779254457050245
    clearSkyFactors[np.where(e >= 60)] = 1.0715262914980628

    clearSkyFactors *= sigmoid

    # Apply to ghi
    totalCorrectionFactor = clearSkyFactors + cloudyFactors

    del clearSkyFactors, cloudyFactors, e, months, sigmoid, transmissivity

    return totalCorrectionFactor

max_stars_repo_name: OfficialCodexplosive/RESKit

max_issues_repo_name: OfficialCodexplosive/RESKit

max_forks_repo_name: OfficialCodexplosive/RESKit


text:
import os
import sys
sys.path.append('/export/u1/homes/weichao/Workspace/disentangling-vae/')

from disvae.utils.modelIO import save_model, load_model, load_metadata
from disvae.models.losses import LOSSES, RECON_DIST
from disvae.models.vae import MODELS
from utils.datasets import get_dataloaders, get_img_size, DATASETS, get_background
from utils.mnist_classifier import Net as MNIST_Net

import six
import abc
from scipy.optimize import approx_fprime
from skimage.io import imread
import torch
import torchvision
import torchvision.transforms.functional as TF
import numpy as np

from datetime import datetime
from PIL import Image

BASE = '/export/u1/homes/weichao/Workspace/disentangling-vae/'
ROOT_DIR = BASE + "exp_adv/"
RES_DIR = ROOT_DIR + "/TRPO/results/"

class VAE_Dynamics():
	def __init__(self, vae):
		""" Load VAE models """
		exp_dir = os.path.join(ROOT_DIR, vae)
		self.model = load_model(exp_dir, is_gpu = False)
		self.device = 'cpu'
		self.model.eval()
		self.cur_data = None

	def _state_size(self):
		return self.model.img_size

	def _action_size(self):
		return self.model.latent_dim

	def _reset(self, data):
		""" Load data """
		self.cur_data = data

	def _step(self, action):
		with torch.no_grad():
			means, logvars = self.model.encoder(self.cur_data.unsqueeze(0).cpu())
			latent_var = means[0]
			latent_var += torch.tensor(action.flatten()).cpu()
			nxt_data = self.model.decoder(latent_var.unsqueeze(0)).cpu()[0]
		return nxt_data

class Basic_Dynamics():
	def __init__(self, vae):
		self.cur_data = None

	def _state_size(self):
		return torch.empty(1, 32, 32).size()

	def _action_size(self):
		return torch.empty(1, 32, 32).size()

	def _reset(self, data):
		""" Load data """
		self.cur_data = data

	def _step(self, action):
		nxt_data = self.cur_data.float() + torch.tensor(action).float()
		nxt_data = nxt_data.clamp(0., 1.)
		return nxt_data

class Dynamics(Basic_Dynamics):
	def __init__(self, dataset, vae, cls, target = 10):
		super(Dynamics, self).__init__(vae)
		self.dataset = dataset
		self.classifier = MNIST_Net().float()
		self.classifier.load_state_dict(\
			torch.load(open(os.path.join(ROOT_DIR, cls, 'model.pt'), 'rb')), strict = False)

		""" Target logits """
		self.target_logits = np.zeros([10])
		self.initial_data = None
		self.initial_state = None
		self.initial_target = None
		#self.initial_logits = None
		self.cur_data = None
		self.initial_state = None
		self.cur_step = None

		timestampStr = datetime.now().strftime("%d-%b-%Y-%H-%M-%S-%f")
		self.result_dir = os.path.join(RES_DIR, timestampStr)
		os.mkdir(self.result_dir)

		data_loader = get_dataloaders(self.dataset, batch_size=1)
		if True:
			(initial_data, initial_target) = next(iter(data_loader))
			self.initial_data = initial_data.squeeze(0)
			self.initial_state = self.initial_data.flatten()
			self.initial_logits = self.classifier(initial_data).detach().cpu().numpy()
			self.initial_target = initial_target.squeeze(0)

			if target is not None:
				if target > 9 or self.initial_target == target:
					self.target_logits[self.initial_target] = 1.0
					self.target_logits = 1./9. * (1.0 - self.target_logits)
			else:
				self.target_logits[target] = 1.0

		self._reset(self.initial_data)
		self.cur_state = self.cur_data.flatten()
		cur_image = self.render()
		cur_image.save(os.path.join(self.result_dir, '0.png'))

	def action_size(self):
		return self._action_size()

	@property
	def action_space(self):
		return [1 * 32 * 32]

	def state_size(self):
		return self._state_size()

	@property
	def state_space(self):
		return [1 * 32 * 32]

	def reset(self):
		self.cur_step = 0
		self._reset(self.initial_data)
		self.cur_state = self.cur_data.flatten().cpu().numpy()
		return self.cur_state

	def render(self):
		cur_data = torch.zeros_like(self.cur_data)
		cur_data.copy_(self.cur_data)
		img = cur_data.mul_(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to('cpu', torch.uint8).numpy()
		toImage = torchvision.transforms.ToPILImage()
		img = toImage(img)
		return img

	def reward(self, action = None):
		rew_logits = 0.
		rew_action = 0
		with torch.no_grad():
			pred_logits = self.classifier(self.cur_data.unsqueeze(0)).squeeze(0).cpu().numpy()

		if self.target_logits is not None:
			#rew_logits = - np.linalg.norm(self.target_logits - np.exp(pred_logits), ord = 2)
			rew_logits = np.sum(pred_logits * self.target_logits + \
							np.log(1 - np.exp(pred_logits)) * (1 - self.target_logits))
			if np.argmax(self.target_logits) == np.argmax(pred_logits):
				rew_logits += 100.0
				done = True
			else:
				rew_logits += 0.0
				done = False
		else:
			rew_logits = np.linalg.norm(self.initial_logits - pred_logits, ord = 2)
			if np.argmax(self.initial_logits) != np.argmax(pred_logits):
				rew_logits += 100.0
				done = True
			else:
				rew_logits += 0.0
				done = False

		rew_action = - np.linalg.norm(action.flatten())
		return rew_logits + rew_action, done

	def step(self, action):
		self.cur_step += 1

		cur_action = action.reshape(self.cur_data.size())
		self.cur_data = self._step(cur_action)
		self.cur_state = self.cur_data.flatten().cpu().numpy()

		reward, done = self.reward(cur_action)
		if done:
			cur_image = self.render()
			cur_image.save(os.path.join(self.result_dir, 'Done_' + str(self.cur_step) + '.png'))

		return self.cur_state, reward, done, None

reason: import numpy,from scipy


text:
import sys
import math
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import scipy.cluster.hierarchy as sch

def plotcnv(cnvArr, chrArr, title, norm=None, norm_vals=None, norm_inds=None,
            cmap=plt.cm.bwr, minv=None, maxv=None, save_name=None, collapse_to_zero=None,
            collapse_max_to=None, collapse_min_to=None, dendrogram=True, linkage_method='ward'):
    fig = plt.figure(figsize=(16,8))
    if dendrogram:
        ax1 = fig.add_axes([0.01,0.1,0.1,0.6])
    if not isinstance(cnvArr, pd.DataFrame):
        cnvdf=pd.DataFrame(cnvArr)
    else:
        cnvdf = cnvArr
    newcnvdf = cnvdf.copy()
    if norm == 'mean':
        if isinstance(norm_inds, np.ndarray):
            newcnvdf = newcnvdf.subtract(newcnvdf.loc[norm_inds].mean(axis=0),axis=1).as_matrix()
        elif isinstance(norm_vals, np.ndarray):
            newcnvdf = newcnvdf
        else:
            newcnvdf=newcnvdf.subtract(newcnvdf.mean(axis=0),axis=1).as_matrix()
    elif norm == 'mode':
        newcnvdf = newcnvdf.subtract(newcnvdf.mode(axis=0),axis=1).as_matrix()
    else:
        newcnvdf = newcnvdf.as_matrix()

    if dendrogram:
        Y = sch.linkage(newcnvdf, method=linkage_method)
        Z1 = sch.dendrogram(Y, truncate_mode='level', orientation='right')
        ax1.set_xticks([])
        ax1.set_yticks([])

    # identify maximum value as 95 percentile
    #allv=cnvdf.values.flatten()

    allv=np.ravel(newcnvdf)
    allv=allv[np.nonzero(allv)]
    if maxv == None and collapse_max_to == None:
        maxv=np.percentile(allv,99)
    if minv == None and collapse_min_to == None:
        minv=np.percentile(allv,1)
    if maxv == None:
        maxv = collapse_max_to
    if minv == None:
        minv = collapse_min_to

    if norm != 'none':
        if -minv>maxv:
            maxv=-minv
        if maxv>-minv:
            minv=-maxv

    if collapse_min_to != None:
        inds = np.where(newcnvdf < collapse_min_to)
        newcnvdf[inds] = np.full(len(inds[0]), collapse_min_to)
    if collapse_max_to != None:
        inds = np.where(newcnvdf > collapse_max_to)
        newcnvdf[inds] = np.full(len(inds[0]), collapse_max_to)
    if collapse_to_zero != None:
        inds = np.where((newcnvdf >= -collapse_to_zero) & (newcnvdf <= collapse_to_zero))
        newcnvdf[inds] = np.zeros(len(inds[0]))
        inds = np.where(newcnvdf > collapse_to_zero)
        newcnvdf[inds] = np.subtract(newcnvdf[inds], collapse_to_zero)
        inds = np.where(newcnvdf < -collapse_to_zero)
        newcnvdf[inds] = np.add(newcnvdf[inds], collapse_to_zero)

    # Plot distance matrix.
    axmatrix = fig.add_axes([0.12,0.1,0.7,0.6])
    if dendrogram:
        idx1 = np.array(sch.leaves_list(Y))
        newcnvdf = newcnvdf[idx1,:]
    im = axmatrix.matshow(newcnvdf, aspect='auto', origin='lower',
						  cmap=cmap,vmin=minv,vmax=maxv)

    xticks=[]
    xcords=[]
    xticklabels=[]
    for i in range(0,len(chrArr)):
        if (i==len(chrArr)-1) or (chrArr[i]!=chrArr[i+1]):
            xcords.append(i)
            plt.axvline(x=i,color='black')
            if int(chrArr[i])==23:
                xticklabels.append('X')
            elif int(chrArr[i])==24:
                xticklabels.append('Y')
            else:
                xticklabels.append(chrArr[i])
    for i in range(len(xcords)):
        if i==0:
            xticks.append(xcords[i]/2.0)
        else:
            xticks.append(xcords[i-1]+(xcords[i]-xcords[i-1])/2.0)
    plt.xticks(xticks,xticklabels)
    axmatrix.set_yticks([])
    axmatrix.set_title(title)

    # Plot colorbar.
    axcolor = fig.add_axes([0.83,0.1,0.02,0.6])
    cbar = plt.colorbar(im, cax=axcolor)
    if collapse_to_zero != None:
        ticks = cbar.ax.yaxis.get_ticklabels()
        new_ticks = []
        for i in range(len(ticks)):
            curTick = ticks[i].get_text()
            if i < np.floor(len(ticks)/2):
                new_ticks.append(-1*(float(curTick[1:]) + collapse_to_zero))
            elif i > np.floor(len(ticks)/2):
                new_ticks.append(float(curTick) + collapse_to_zero)
            else:
                new_ticks.append(0.0)
        cbar.ax.set_yticklabels(new_ticks)
    # plt.show()

    if save_name != None:
        plt.savefig(save_name, bbox_inches='tight')
# plot 2 cnv arrays together with reference one refCnvArr on top

"""
def plot2cnv(cnvArr, refCnvArr, chrArr, title, refTitle, minv=0.0,maxv=0.5):
    cnvdf=pd.DataFrame(cnvArr)
    refcnvdf=pd.DataFrame(refCnvArr)
    #refCnvArr=refcnvdf.subtract(cnvdf.mean(axis=0),axis=1).as_matrix()
    #cnvArr=cnvdf.subtract(cnvdf.mean(axis=0),axis=1).as_matrix()
    refCnvArr=refcnvdf.subtract(refcnvdf.mean(axis=0),axis=1).as_matrix()
    cnvArr=cnvdf.subtract(cnvdf.mean(axis=0),axis=1).as_matrix()

    fig = plt.figure(figsize=(16,8))
    ax1 = fig.add_axes([0.01,0.1,0.1,0.6])
    Y = sch.linkage(cnvArr, method='ward')
    Z1 = sch.dendrogram(Y, orientation='right')
    Y2 = sch.linkage(refCnvArr, method='ward')
    Z2 = sch.dendrogram(Y2, orientation='right')
    ax1.set_xticks([])
    ax1.set_yticks([])

    allv=np.concatenate((np.ravel(cnvArr),np.ravel(refCnvArr)),axis=0)
    allv=allv[np.nonzero(allv)]
    maxv=np.percentile(allv,98)
    minv=np.percentile(allv,2)

    # computing x ticks
    xticks=[]
    xcords=[]
    xticklabels=[]
    for i in range(0,len(chrArr)):
        if (i==len(chrArr)-1) or (chrArr[i]!=chrArr[i+1]):
            xcords.append(i)
            #plt.axvline(x=i,color='black')
            if int(chrArr[i])==23:
                xticklabels.append('X')
            elif int(chrArr[i])==24:
                xticklabels.append('Y')
            else:
                xticklabels.append(chrArr[i])
    for i in range(len(xcords)):
        if i==0:
            xticks.append(xcords[i]/2.0)
        else:
            xticks.append(xcords[i-1]+(xcords[i]-xcords[i-1])/2.0)

    axmatrix = fig.add_axes([0.12,0.1,0.7,0.6])
    idx1 = Z1['leaves']
    cnvArr = cnvArr[idx1,:]
    im = axmatrix.matshow(cnvArr, aspect='auto', origin='lower', cmap=plt.cm.bwr,vmin=minv,vmax=maxv)
    for i in range(0,len(chrArr)):
        if (i==len(chrArr)-1) or (chrArr[i]!=chrArr[i+1]):
            plt.axvline(x=i,color='black')
    plt.xticks(xticks,xticklabels)
    axmatrix.set_yticks([])
    axmatrix.set_title(title)

    axrefmatrix = fig.add_axes([0.12,0.8,0.7,0.6])
    idx2 = Z2['leaves']
    refCnvArr = refCnvArr[idx2,:]
    im = axrefmatrix.matshow(refCnvArr, aspect='auto', origin='lower', cmap=plt.cm.bwr,vmin=minv,vmax=maxv)
    for i in range(0,len(chrArr)):
        if (i==len(chrArr)-1) or (chrArr[i]!=chrArr[i+1]):
            plt.axvline(x=i,color='black')
    plt.xticks(xticks,xticklabels)
    axrefmatrix.set_yticks([])
    axrefmatrix.set_title(refTitle)

    # Plot colorbar.
    axcolor = fig.add_axes([0.83,0.1,0.02,0.6])
    plt.colorbar(im, cax=axcolor)
"""

reason: import numpy,import scipy


text:
import os

import numpy as np
import re
import csv

# 絶対パスにする
BASE_PATH = os.path.dirname(__file__)
TRAIN_PATH = os.path.join(BASE_PATH, 'dataset/train.tsv')
TEST_PATH = os.path.join(BASE_PATH, 'dataset/test.tsv')
DEV_PATH = os.path.join(BASE_PATH, 'dataset/dev.tsv')

def load_data(num_words=14817, is_id=True):
    """sst-2の学習用とテスト用(devファイル)データを出力する
    最大語彙数は14817

    sst-2の学習用データの組を出力する
    ラベルの説明 0 bad, 1 good

    Args:
        bool: numpyとして出力するか

    Returns:
        ([string], [int]), ([string], [int]): (学習用データ, ラベル),  (テスト用データ, ラベル), vocab, word2id, id2word
    """

    train_text_data = []
    train_label_data = []
    with open(TRAIN_PATH, 'r', encoding='utf-8') as f:
        texts = csv.reader(f, delimiter='\t')
        texts.__next__()
        for row in texts:
            train_text_data.append(row[0].lower())
            train_label_data.append(int(row[1]))

    test_text_data = []
    test_label_data = []
    with open(DEV_PATH, 'r', encoding='utf-8') as f:
        texts = csv.reader(f, delimiter='\t')
        texts.__next__()
        for row in texts:
            test_text_data.append(row[0].lower())
            test_label_data.append(int(row[1]))

    if not is_id:
        return (np.array(train_text_data), np.array(train_label_data)), (np.array(test_text_data), np.array(test_label_data))

    tokens = []
    for text in train_text_data:
        tokens.extend(text.split())

    vocab, word2id, index = {}, {}, 1
    vocab['<pad>'] = 0
    word2id['<pad>'] = 0
    for token in tokens:
        if token not in vocab:
            if index < num_words:
                vocab[token] = index
                word2id[token] = index
                index += 1
            else:
                word2id[token] = 0

    train_data = [[word2id[word] for word in sentence.split()] for sentence in train_text_data]

    id2word = {index: word for index, word in enumerate(word2id)}

    tokens = []
    for text in test_text_data:
        tokens.extend(text.split())

    for token in tokens:
        if token not in vocab:
            word2id[token] = 0

    test_data = [[word2id[word] for word in sentence.split()] for sentence in test_text_data]

    return (np.array(train_data), np.array(train_label_data)), (np.array(test_data), np.array(test_label_data)), vocab, word2id, id2word

if __name__=='__main__':
    (x_train, y_train), (x_test, y_test), vocab, word2id, id2word= load_data(num_words=100000)

    for i in range(10):
        print("train: {} => label: {}".format(x_train[i], y_train[i]))
        print("test: {} => label: {}".format(x_test[i], y_test[i]))

    print(len(vocab))

max_stars_repo_name: saicrips/only-my-datasets

max_issues_repo_name: saicrips/only-my-datasets

max_forks_repo_name: saicrips/only-my-datasets


text:
"""Convert clouds to clusters.

Usage:
    cloud_to_clusters.py path <path> [options]
    cloud_to_clusters.py dir <dir> [options]

Options:
    --out=<out>     Directory containing output [default: ./data/clusters]
"""

from collections import defaultdict

import docopt
import glob
import numpy as np
import os.path

def write_clouds_dir_to_clusters(clouds_dir: str, out_dir: str):
    """Write all clouds contained in subdirectories from cloud_dir/.

    Hardcoded to use the directory structure in KITTI_raw at Aspire.

    cloud_dir/<drive>/seg/data/*.npy
    """
    for directory in sorted(os.listdir(clouds_dir)):
        clouds_path = os.path.join(clouds_dir, directory, 'seg', 'data', '*.npy')
        new_out_dir = os.path.join(out_dir, directory)
        write_clouds_to_clusters(clouds_path, new_out_dir)

def write_clouds_to_clusters(clouds_path: str, out_dir: str):
    """Write all clouds to cluster numpy files.

    This function assumes that all clouds are formatted as (64, 1024, 13),
    where the last dimension is the number of channels, and 64/1024 are various
    sets of points.
    """
    for cloud_path in glob.iglob(clouds_path):
        cloud = np.load(cloud_path).reshape(-1, 13)
        clusters = cloud_to_clusters(cloud)
        cluster_dir = get_cluster_dir(cloud_path, out_dir)
        write_clusters(clusters, cluster_dir)
        print(' * [INFO] Finished processing cloud',
              os.path.basename(cloud_path).replace('.npy', ''),
              '(saved to ', cluster_dir, ')')

def cloud_to_clusters(cloud: np.array) -> np.array:
    """Converts provided cloud into a set of clusters."""
    clusters = defaultdict(lambda: [])
    for vertex in cloud:
        center = tuple(vertex[6:9])
        if center != (0, 0, 0):
            clusters[center].append(vertex)

    print(' * [INFO]', len(list(clusters.keys())), 'clusters found')
    for key in clusters:
        clusters[key] = process_cluster(np.vstack(clusters[key]))
    return clusters

def process_cluster(cluster: np.array, scale: float=3.0) -> np.array:
    """Pre-process the cluster.

    We first demean and then scale up the object.
    """
    demeaned = cluster.copy()
    demeaned[:, :3] -= cluster[:, 6:9]
    demeaned[:, :3] *= scale
    return demeaned

def get_cluster_dir(cloud_path: str, out_dir: str) -> str:
    """Creates new path for cluster, using cloud path and output root dir."""
    cloud_name = os.path.basename(cloud_path).replace('.npy', '')
    cluster_dir = os.path.join(out_dir, cloud_name)
    os.makedirs(cluster_dir, exist_ok=True)
    return cluster_dir

def write_clusters(clusters: np.array, cluster_dir: str):
    """Writes all clusters to individual numpy files."""
    for center, vertices in clusters.items():
        out_path = os.path.join(cluster_dir, '%f_%f_%f' % center)
        out_data = np.vstack(vertices)
        assert out_data.shape[1] > 3
        np.save(out_path, out_data)

def main():
    arguments = docopt.docopt(__doc__)
    if arguments['path']:
        write_clouds_to_clusters(arguments['<path>'], arguments['--out'])
    else:
        write_clouds_dir_to_clusters(arguments['<dir>'], arguments['--out'])

if __name__ == '__main__':
    main()

max_stars_repo_name: alvinwan/pcmatch

max_issues_repo_name: alvinwan/pcmatch

max_forks_repo_name: alvinwan/pcmatch


text:
# -*- coding: utf-8 -*-

import random as rn
rn.seed(2)
from numpy.random import seed
seed(2)
from tensorflow import set_random_seed
set_random_seed(2)
import tensorflow as tf
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
from keras import backend as K
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)

import os
import numpy as np

def load_data(name='freq', get_all_special_data=True):
    if name == 'freq':
        path = 'D:/Python/TAR/dataset/metadata/bags of words'
    elif name == 'chi2':
        path = 'D:/Python/TAR/chi2_scores/metadata/bags_of_words'
    elif name == 'tfidf':
        path = 'D:/Python/TAR/tf-idf_scores/metadata/bags_of_words'
    else:
        raise ValueError

    train_negative = path + '/negative'
    train_positive = path + '/positive'
    test_negative = path + '/negative_test'
    test_positive = path + '/positive_test'

    special_path = 'D:/Python/TAR/special-data/bags of words'
    special_train_negative = special_path + '/negative/'
    special_train_positive = special_path + '/positive/'
    special_test_negative = special_path + '/negative_test/'
    special_test_positive = special_path + '/positive_test/'

    #
    # load train data
    #
    train = []
    train_X = []
    train_S = []
    train_y = []
    os.chdir(train_negative)
    negative_files = os.listdir()
    #print('negative train files:', len(negative_files))
    for txtfile in negative_files:
        with open(txtfile, 'r', encoding='utf8') as file:
            vector = file.readlines()
            vector = [int(token[:-1]) for token in vector]  # remove '\n', convert values to int
            special_vector = []
            with open(special_train_negative + txtfile, 'r', encoding='utf-8') as sf:
                special_vector = sf.readlines()
                special_vector = [float(token[:-1]) for token in special_vector]
                if get_all_special_data == False:
                    special_vector = [special_vector[1], special_vector[4], special_vector[5], special_vector[8]]
            train.append([np.array(vector), np.array(special_vector), np.array([1, 0])])
    os.chdir(train_positive)
    positive_files = os.listdir()
    #print('positive train files:', len(positive_files))
    for txtfile in positive_files:
        with open(txtfile, 'r', encoding='utf8') as file:
            vector = file.readlines()
            vector = [int(token[:-1]) for token in vector]  # remove '\n', convert values to int
            special_vector = []
            with open(special_train_positive + txtfile, 'r', encoding='utf-8') as sf:
                special_vector = sf.readlines()
                special_vector = [float(token[:-1]) for token in special_vector]
                if get_all_special_data == False:
                    special_vector = [special_vector[1], special_vector[4], special_vector[5], special_vector[8]]
            train.append([np.array(vector), np.array(special_vector), np.array([0, 1])])
            train.append([np.array(vector), np.array(special_vector), np.array([0, 1])])
            train.append([np.array(vector), np.array(special_vector), np.array([0, 1])])
            train.append([np.array(vector), np.array(special_vector), np.array([0, 1])])
            train.append([np.array(vector), np.array(special_vector), np.array([0, 1])])

    train = np.array(train)
    #np.random.shuffle(train)
    # don't shuffle here, shuffle data controlably when necessary
    for sample in train:
        train_X.append(sample[0])
        train_S.append(sample[1])
        train_y.append(sample[2])

    #
    # load test data
    #
    test = []
    test_X = []
    test_S = []
    test_y = []
    os.chdir(test_negative)
    negative_files = os.listdir()
    #print('negative test files:', len(negative_files))
    for txtfile in negative_files:
        with open(txtfile, 'r', encoding='utf8') as file:
            vector = file.readlines()
            vector = [int(token[:-1]) for token in vector]  # remove '\n', convert values to int
            special_vector = []
            with open(special_test_negative + txtfile, 'r', encoding='utf-8') as sf:
                special_vector = sf.readlines()
                special_vector = [float(token[:-1]) for token in special_vector]
                if get_all_special_data == False:
                    special_vector = [special_vector[1], special_vector[4], special_vector[5], special_vector[8]]
            test.append([np.array(vector), np.array(special_vector), np.array([1, 0])])
    os.chdir(test_positive)
    positive_files = os.listdir()
    #print('positive test files:', len(positive_files))
    for txtfile in positive_files:
        with open(txtfile, 'r', encoding='utf8') as file:
            vector = file.readlines()
            vector = [int(token[:-1]) for token in vector]  # remove '\n', convert values to int
            special_vector = []
            with open(special_test_positive + txtfile, 'r', encoding='utf-8') as sf:
                special_vector = sf.readlines()
                special_vector = [float(token[:-1]) for token in special_vector]
                if get_all_special_data == False:
                    special_vector = [special_vector[1], special_vector[4], special_vector[5], special_vector[8]]
            test.append([np.array(vector), np.array(special_vector), np.array([0, 1])])

    test = np.array(test)
    #np.random.shuffle(test)
    for sample in test:
        test_X.append(sample[0])
        test_S.append(sample[1])
        test_y.append(sample[2])
    #print('len(test_y) =', len(test_y))
    return np.array(train_X), np.array(train_S), np.array(train_y), np.array(test_X), np.array(test_S), np.array(test_y)

def get_nn_predict_values():
    os.chdir('D:/Python/TAR/system')

    #
    # train data
    #
    freq_train = []
    with open('freq_train.txt', 'r', encoding='utf-8') as f:
        lines = f.readlines()
        vector = [float(x[:-1]) for x in lines]
    freq_train.append(np.array(vector))
    freq_train = np.array(freq_train)

    chi2_train = []
    with open('chi2_train.txt', 'r', encoding='utf-8') as f:
        lines = f.readlines()
        vector = [float(x[:-1]) for x in lines]
    chi2_train.append(np.array(vector))
    chi2_train = np.array(chi2_train)

    tfidf_train = []
    with open('tfidf_train.txt', 'r', encoding='utf-8') as f:
        lines = f.readlines()
        vector = [float(x[:-1]) for x in lines]
    tfidf_train.append(np.array(vector))
    tfidf_train = np.array(tfidf_train)

    #
    # test data
    #
    freq_test = []
    with open('freq_test.txt', 'r', encoding='utf-8') as f:
        lines = f.readlines()
        vector = [float(x[:-1]) for x in lines]
    freq_test.append(np.array(vector))
    freq_test = np.array(freq_test)

    chi2_test = []
    with open('chi2_test.txt', 'r', encoding='utf-8') as f:
        lines = f.readlines()
        vector = [float(x[:-1]) for x in lines]
    chi2_test.append(np.array(vector))
    chi2_test = np.array(chi2_test)

    tfidf_test = []
    with open('tfidf_test.txt', 'r', encoding='utf-8') as f:
        lines = f.readlines()
        vector = [float(x[:-1]) for x in lines]
    tfidf_test.append(np.array(vector))
    tfidf_test = np.array(tfidf_test)

    return freq_train, chi2_train, tfidf_train, freq_test, chi2_test, tfidf_test

max_stars_repo_name: TvrtkoSternak/TAR-project

max_issues_repo_name: TvrtkoSternak/TAR-project

max_forks_repo_name: TvrtkoSternak/TAR-project

reason: import numpy,from numpy


text:
#!/usr/bin/env python3

""" Use TT-SVD to approximate a large square SVD """

__authors__ = ['Melven Roehrig-Zoellner <Melven.Roehrig-Zoellner@DLR.de>']
__date__ = '2020-12-16'

import numpy as np
import pitts_py
import functools
import timeit
import argparse

def timer(func):
    """measure runtime of the decorated function"""
    @functools.wraps(func)
    def wrapper_fun(*args, **kwargs):
        wtime = timeit.default_timer()
        value = func(*args, **kwargs)
        wtime = timeit.default_timer() - wtime
        print(func.__name__, "wtime:", wtime)
        return value
    return wrapper_fun

@timer
def random_square_low_rank_matrix(n, r):
    sigma = np.linspace(1, 0, r, endpoint=False, dtype=np.float64)
    U, _ = np.linalg.qr(np.random.rand(n,r).astype(dtype=np.float64))
    Vt, _ = np.linalg.qr(np.random.rand(n,r).astype(dtype=np.float64))
    M = U @ np.diag(sigma) @ Vt.T
    return M, U, sigma, Vt

@timer
def numpy_svd(M):
    U, S, Vt = np.linalg.svd(M)
    return U, S, Vt

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='approximate truncated square SVD of a 2^n x 2^n matrix by TT-SVD')
    parser.add_argument('d', type=int)
    parser.add_argument('r', type=int)
    parser.add_argument('rtt', type=int)
    parser.add_argument('nIter', type=int)

    args = parser.parse_args()

    pitts_py.initialize(True)

    # generate square random matrix of given rank
    n = 2**args.d
    M, U_ref, sigma_ref, Vt_ref = random_square_low_rank_matrix(n, args.r)
    print('Singular values:', sigma_ref)

    U, sigma, Vt = numpy_svd(M)
    print('Computed singular values:', sigma[:args.r])
    err = np.linalg.norm(sigma[:args.r] - sigma_ref, ord=np.inf)
    print('Error:', err)

    Xm = pitts_py.MultiVector_double(n*n//2, 2)
    Xm_view = np.array(Xm, copy=False)
    Xm_view[...] = M.reshape((n*n//2, 2), order='F')
    Xm_view = None
    dims = [2,]*args.d*2
    work = pitts_py.MultiVector_double()
    Xtt = pitts_py.fromDense(Xm, work, dims, rankTolerance=0.0001, maxRank=args.rtt)
    cores = [Xtt.getSubTensor(i) for i in range(len(dims))]
    for i in range(args.d):
        r1 = cores[i].shape[0]
        # 2 = cores[i].shape[1]
        r2 = cores[i].shape[2]
        coreU, coreS, coreVt = np.linalg.svd(cores[i].reshape((r1*2,r2), order='F'), full_matrices=False)
        cores[i] = coreU.reshape((r1,2,r2), order='F')
        r3 = cores[i+1].shape[2]
        tmp = cores[i+1].reshape((r2,2*r3), order='F')
        if i+1 < args.d:
            tmp = np.diag(coreS) @ coreVt.T @ tmp
        else:
            sigmaTT = coreS
            tmp = coreVt.T @ tmp
        cores[i+1] = tmp.reshape((r2,2,r3), order='F')
    print('TT-SVD computed singular values:', sigmaTT[:args.r])
    err = np.linalg.norm(sigmaTT[:args.r] - sigma_ref, ord=np.inf)
    print('Error:', err)
    UTT = np.ones((1,1), dtype=np.float64)
    for i in range(args.d):
        r1 = cores[i].shape[0]
        # 2 = cores[i].shape[1]
        r2 = cores[i].shape[2]
        UTT = UTT @ cores[i].reshape((r1,2*r2), order='F')
        UTT = UTT.reshape((UTT.size//r2, r2), order='F')
    print('UTT', UTT[:,:args.r].T @ U_ref)
    VTT = np.eye(len(sigmaTT), dtype=np.float64)
    for i in range(args.d, 2*args.d):
        r1 = cores[i].shape[0]
        # 2 = cores[i].shape[1]
        r2 = cores[i].shape[2]
        VTT = VTT @ cores[i].reshape((r1,2*r2), order='F')
        VTT = VTT.reshape((VTT.size//r2, r2), order='F')
    VTT = VTT.reshape((len(sigmaTT),n), order='F')
    print('VTT', VTT[:args.r,:] @ Vt_ref)

    pitts_py.finalize(True)

max_stars_repo_name: melven/pitts

max_issues_repo_name: melven/pitts

max_forks_repo_name: melven/pitts


text:
import pandas as pd
import numpy as np
import plotly
import plotly.express as px
import os

# ridge_test_results = r'static/ridge_test_results.csv'

ridge_test_results  = os.path.join("static" , "ridge_test_results.csv")

df = pd.read_csv(ridge_test_results, low_memory=False)
fig = px.scatter(df, x="Predicted Rating", y="Actual Rating", trendline="ols", hover_data=['Accuracy','Title', 'Genre' , 'Duration (min)' , 'Budget (mm)' , 'MPAA Rating' ])
fig.show()