text:
from __future__ import division
from __future__ import division
from __future__ import division

from itertools import combinations_with_replacement

# ==========================================DataManipulation============================================================
def shuffle_data(X, y, seed=None):
    """ Random shuffle of the samples in X and y """
    if seed:
        np.random.seed(seed)
    idx = np.arange(X.shape[0])
    np.random.shuffle(idx)
    return X[idx], y[idx]

def batch_iterator(X, y=None, batch_size=64):
    """ Simple batch generator """
    n_samples = X.shape[0]
    for i in np.arange(0, n_samples, batch_size):
        begin, end = i, min(i + batch_size, n_samples)
        if y is not None:
            yield X[begin:end], y[begin:end]
        else:
            yield X[begin:end]

def divide_on_feature(X, feature_i, threshold):
    """ Divide dataset based on if sample value on feature index is larger than
        the given threshold """
    split_func = None
    if isinstance(threshold, int) or isinstance(threshold, float):
        split_func = lambda sample: sample[feature_i] >= threshold
    else:
        split_func = lambda sample: sample[feature_i] == threshold

    X_1 = np.array([sample for sample in X if split_func(sample)])
    X_2 = np.array([sample for sample in X if not split_func(sample)])

    return np.array([X_1, X_2])

def polynomial_features(X, degree):
    n_samples, n_features = np.shape(X)

    def index_combinations():
        combs = [combinations_with_replacement(range(n_features), i) for i in range(0, degree + 1)]
        flat_combs = [item for sublist in combs for item in sublist]
        return flat_combs

    combinations = index_combinations()
    n_output_features = len(combinations)
    X_new = np.empty((n_samples, n_output_features))

    for i, index_combs in enumerate(combinations):
        X_new[:, i] = np.prod(X[:, index_combs], axis=1)

    return X_new

def get_random_subsets(X, y, n_subsets, replacements=True):
    """ Return random subsets (with replacements) of the data """
    n_samples = np.shape(X)[0]
    # Concatenate x and y and do a random shuffle
    X_y = np.concatenate((X, y.reshape((1, len(y))).T), axis=1)
    np.random.shuffle(X_y)
    subsets = []

    # Uses 50% of training samples without replacements
    subsample_size = int(n_samples // 2)
    if replacements:
        subsample_size = n_samples  # 100% with replacements

    for _ in range(n_subsets):
        idx = np.random.choice(
            range(n_samples),
            size=np.shape(range(subsample_size)),
            replace=replacements)
        X = X_y[idx][:, :-1]
        y = X_y[idx][:, -1]
        subsets.append([X, y])
    return subsets

def normalize(X, axis=-1, order=2):
    """ Normalize the dataset X """
    l2 = np.atleast_1d(np.linalg.norm(X, order, axis))
    l2[l2 == 0] = 1
    return X / np.expand_dims(l2, axis)

def standardize(X):
    """ Standardize the dataset X """
    X_std = X
    mean = X.mean(axis=0)
    std = X.std(axis=0)
    for col in range(np.shape(X)[1]):
        if std[col]:
            X_std[:, col] = (X_std[:, col] - mean[col]) / std[col]
    # X_std = (X - X.mean(axis=0)) / X.std(axis=0)
    return X_std

def train_test_split(X, y, test_size=0.5, shuffle=True, seed=None):
    """ Split the data into train and test sets """
    if shuffle:
        X, y = shuffle_data(X, y, seed)
    # Split the training data from test data in the ratio specified in
    # test_size
    split_i = len(y) - int(len(y) // (1 / test_size))
    X_train, X_test = X[:split_i], X[split_i:]
    y_train, y_test = y[:split_i], y[split_i:]

    return X_train, X_test, y_train, y_test

def k_fold_cross_validation_sets(X, y, k, shuffle=True):
    """ Split the data into k sets of training / test data """
    if shuffle:
        X, y = shuffle_data(X, y)

    n_samples = len(y)
    left_overs = {}
    n_left_overs = (n_samples % k)
    if n_left_overs != 0:
        left_overs["X"] = X[-n_left_overs:]
        left_overs["y"] = y[-n_left_overs:]
        X = X[:-n_left_overs]
        y = y[:-n_left_overs]

    X_split = np.split(X, k)
    y_split = np.split(y, k)
    sets = []
    for i in range(k):
        X_test, y_test = X_split[i], y_split[i]
        X_train = np.concatenate(X_split[:i] + X_split[i + 1:], axis=0)
        y_train = np.concatenate(y_split[:i] + y_split[i + 1:], axis=0)
        sets.append([X_train, X_test, y_train, y_test])

    # Add left over samples to last set as training samples
    if n_left_overs != 0:
        np.append(sets[-1][0], left_overs["X"], axis=0)
        np.append(sets[-1][2], left_overs["y"], axis=0)

    return np.array(sets)

def to_categorical(x, n_col=None):
    """ One-hot encoding of nominal values """
    if not n_col:
        n_col = np.amax(x) + 1
    one_hot = np.zeros((x.shape[0], n_col))
    one_hot[np.arange(x.shape[0]), x] = 1
    return one_hot

def to_nominal(x):
    """ Conversion from one-hot encoding to nominal """
    return np.argmax(x, axis=1)

def make_diagonal(x):
    """ Converts a vector into an diagonal matrix """
    m = np.zeros((len(x), len(x)))
    for i in range(len(m[0])):
        m[i, i] = x[i]
    return m

# ===========================================DataOperation==============================================================
import math

def calculate_entropy(y):
    """ Calculate the entropy of label array y """
    log2 = lambda x: math.log(x) / math.log(2)
    unique_labels = np.unique(y)
    entropy = 0
    for label in unique_labels:
        count = len(y[y == label])
        p = count / len(y)
        entropy += -p * log2(p)
    return entropy

def mean_squared_error(y_true, y_pred):
    """ Returns the mean squared error between y_true and y_pred """
    mse = np.mean(np.power(y_true - y_pred, 2))
    return mse

def calculate_variance(X):
    """ Return the variance of the features in dataset X """
    mean = np.ones(np.shape(X)) * X.mean(0)
    n_samples = np.shape(X)[0]
    variance = (1 / n_samples) * np.diag((X - mean).T.dot(X - mean))

    return variance

def calculate_std_dev(X):
    """ Calculate the standard deviations of the features in dataset X """
    std_dev = np.sqrt(calculate_variance(X))
    return std_dev

def euclidean_distance(x1, x2):
    """ Calculates the l2 distance between two vectors """
    distance = 0
    # Squared distance between each coordinate
    for i in range(len(x1)):
        distance += pow((x1[i] - x2[i]), 2)
    return math.sqrt(distance)

def accuracy_score(y_true, y_pred):
    """ Compare y_true to y_pred and return the accuracy """
    accuracy = np.sum(y_true == y_pred, axis=0) / len(y_true)
    return accuracy

def calculate_covariance_matrix(X, Y=None):
    """ Calculate the covariance matrix for the dataset X """
    if Y is None:
        Y = X
    n_samples = np.shape(X)[0]
    covariance_matrix = (1 / (n_samples - 1)) * (X - X.mean(axis=0)).T.dot(Y - Y.mean(axis=0))

    return np.array(covariance_matrix, dtype=float)

def calculate_correlation_matrix(X, Y=None):
    """ Calculate the correlation matrix for the dataset X """
    if Y is None:
        Y = X
    n_samples = np.shape(X)[0]
    covariance = (1 / n_samples) * (X - X.mean(0)).T.dot(Y - Y.mean(0))
    std_dev_X = np.expand_dims(calculate_std_dev(X), 1)
    std_dev_y = np.expand_dims(calculate_std_dev(Y), 1)
    correlation_matrix = np.divide(covariance, std_dev_X.dot(std_dev_y.T))

    return np.array(correlation_matrix, dtype=float)

# =========================================Kernels======================================================================

def linear_kernel(**kwargs):
    def f(x1, x2):
        return np.inner(x1, x2)

    return f

def polynomial_kernel(power, coef, **kwargs):
    def f(x1, x2):
        return (np.inner(x1, x2) + coef) ** power

    return f

def rbf_kernel(gamma, **kwargs):
    def f(x1, x2):
        distance = np.linalg.norm(x1 - x2) ** 2
        return np.exp(-gamma * distance)

    return f

# ==========================================LossFunction================================================================

class Loss(object):
    def loss(self, y_true, y_pred):
        return NotImplementedError()

    def gradient(self, y, y_pred):
        raise NotImplementedError()

    def acc(self, y, y_pred):
        return 0

class SquareLoss(Loss):
    def __init__(self): pass

    def loss(self, y, y_pred):
        return 0.5 * np.power((y - y_pred), 2)

    def gradient(self, y, y_pred):
        return -(y - y_pred)

class CrossEntropy(Loss):
    def __init__(self): pass

    def loss(self, y, p):
        # Avoid division by zero
        p = np.clip(p, 1e-15, 1 - 1e-15)
        return - y * np.log(p) - (1 - y) * np.log(1 - p)

    def acc(self, y, p):
        return accuracy_score(np.argmax(y, axis=1), np.argmax(p, axis=1))

    def gradient(self, y, p):
        # Avoid division by zero
        p = np.clip(p, 1e-15, 1 - 1e-15)
        return - (y / p) + (1 - y) / (1 - p)

class SotfMaxLoss(Loss):
    def gradient(self, y, p):
        return y - p

# ==============================================Plot====================================================================
import progressbar
import matplotlib.pyplot as plt
import numpy as np

bar_widgets = [
    'Training: ', progressbar.Percentage(), ' ', progressbar.Bar(marker="-", left="[", right="]"),
    ' ', progressbar.ETA()
]

class Plot():
    def __init__(self):
        self.cmap = plt.get_cmap('viridis')

    def _transform(self, X, dim):
        covariance = calculate_covariance_matrix(X)
        eigenvalues, eigenvectors = np.linalg.eig(covariance)
        # Sort eigenvalues and eigenvector by largest eigenvalues
        idx = eigenvalues.argsort()[::-1]
        eigenvalues = eigenvalues[idx][:dim]
        eigenvectors = np.atleast_1d(eigenvectors[:, idx])[:, :dim]
        # Project the data onto principal components
        X_transformed = X.dot(eigenvectors)

        return X_transformed

    def plot_regression(self, lines, title, axis_labels=None, mse=None, scatter=None,
                        legend={"type": "lines", "loc": "lower right"}):

        global s, scatter_plots, scatter_labels
        if scatter:
            scatter_plots = scatter_labels = []
            for s in scatter:
                scatter_plots += [plt.scatter(s["x"], s["y"], color=s["color"], s=s["size"])]
                scatter_labels += [s["label"]]
            scatter_plots = tuple(scatter_plots)
            scatter_labels = tuple(scatter_labels)

        for l in lines:
            li = plt.plot(l["x"], l["y"], color=s["color"], linewidth=l["width"], label=l["label"])

        if mse:
            plt.suptitle(title)
            plt.title("MSE: %.2f" % mse, fontsize=10)
        else:
            plt.title(title)

        if axis_labels:
            plt.xlabel(axis_labels["x"])
            plt.ylabel(axis_labels["y"])

        if legend["type"] == "lines":
            plt.legend(loc="lower_left")
        elif legend["type"] == "scatter" and scatter:
            plt.legend(scatter_plots, scatter_labels, loc=legend["loc"])

        plt.show()

    # Plot the dataset X and the corresponding labels y in 2D using PCA.
    def plot_in_2d(self, X, y=None, title=None, accuracy=None, legend_labels=None):
        X_transformed = self._transform(X, dim=2)
        x1 = X_transformed[:, 0]
        x2 = X_transformed[:, 1]
        class_distr = []

        y = np.array(y).astype(int)

        colors = [self.cmap(i) for i in np.linspace(0, 1, len(np.unique(y)))]

        # Plot the different class distributions
        for i, l in enumerate(np.unique(y)):
            _x1 = x1[y == l]
            _x2 = x2[y == l]
            _y = y[y == l]
            class_distr.append(plt.scatter(_x1, _x2, color=colors[i]))

        # Plot legend
        if not legend_labels is None:
            plt.legend(class_distr, legend_labels, loc=1)

        # Plot title
        if title:
            if accuracy:
                perc = 100 * accuracy
                plt.suptitle(title)
                plt.title("Accuracy: %.1f%%" % perc, fontsize=10)
            else:
                plt.title(title)

        # Axis labels
        plt.xlabel('Principal Component 1')
        plt.ylabel('Principal Component 2')

        plt.show()

    # Plot the dataset X and the corresponding labels y in 3D using PCA.
    def plot_in_3d(self, X, y=None):
        X_transformed = self._transform(X, dim=3)
        x1 = X_transformed[:, 0]
        x2 = X_transformed[:, 1]
        x3 = X_transformed[:, 2]
        fig = plt.figure()
        ax = fig.add_subplot(111, projection='3d')
        ax.scatter(x1, x2, x3, c=y)
        plt.show()

max_stars_repo_name: datamonday/ML-Algorithm-Source-Code

max_issues_repo_name: datamonday/ML-Algorithm-Source-Code

max_forks_repo_name: datamonday/ML-Algorithm-Source-Code


text:
import numpy as np
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.axes3d as p3
import matplotlib.animation as animation
import pandas as pd
from sys import exit
def update_lines(num, data, line):
    # NOTE: there is no .set_data() for 3 dim data...
    line.set_data(data[0:2, :num])
    line.set_3d_properties(data[2, :num])
    return line

# Attaching 3D axis to the figure
fig = plt.figure()
ax = p3.Axes3D(fig)

# Reading the data from a CSV file using pandas
repo = pd.read_csv('data.csv',sep=',',header=0)
data = np.array((repo['x'].values, repo['y'].values, repo['z'].values))
print (data.shape[1])
#exit()

# Creating fifty line objects.
# NOTE: Can't pass empty arrays into 3d version of plot()
line = ax.plot(data[0, 0:1], data[1, 0:1], data[2, 0:1])[0]

# Setting the axes properties
ax.set_xlim3d([-2.0, 2.0])
ax.set_xlabel('X')

ax.set_ylim3d([-2.0, 2.0])
ax.set_ylabel('Y')

ax.set_zlim3d([-2.0, 2.0])
ax.set_zlabel('Z')

ax.set_title('3D Test')

# Creating the Animation object
line_ani = animation.FuncAnimation(fig, update_lines, data.shape[1], fargs=(data, line), interval=50, blit=False)

plt.show()

ext: py

max_stars_repo_name: kingslair/MatPlotLib

max_issues_repo_name: kingslair/MatPlotLib

max_forks_repo_name: kingslair/MatPlotLib

reason: import numpy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
This script provide functions which encode pre-processed NOAA-GridSat-B1
dataset into vectors of length 2048.

The autoencoder is developed and teset with Tensorflow(2.5.0),
CUDA 11.1/11.3 with cuDNN 8.1.0/8.2.0.
'''
import numpy as np
import pandas as pd
import os, argparse, logging, csv, h5py
import tensorflow as tf
import tensorflow_hub as hub

__author__ = "Ting-Shuo Yo"
__copyright__ = "Copyright 2020~2022, DataQualia Lab Co. Ltd."
__credits__ = ["Ting-Shuo Yo"]
__license__ = "Apache License 2.0"
__version__ = "0.1.0"
__maintainer__ = "Ting-Shuo Yo"
__email__ = "tingyo@dataqualia.com"
__status__ = "development"
__date__ = '2021-06-15'

# Parameters
NY = 256
NX = 256
NC = 1

# Utility functions
def list_preprocessed_gridsatb1_files(dir, suffix='.npy', to_remove=['.npy']):
    ''' To scan through the sapecified dir and get the corresponding file with suffix. '''
    import os
    import pandas as pd
    xfiles = []
    for root, dirs, files in os.walk(dir, followlinks=True):  # Loop through the directory
        for fn in files:
            if fn.endswith(suffix):         # Filter files with suffix
                timestamp = fn
                for s in to_remove:         # Removing prefix and suffix to get time-stamp
                    timestamp = timestamp.replace(s,'')
                xfiles.append({'timestamp':timestamp, 'xuri':os.path.join(root, fn)})
    return(pd.DataFrame(xfiles).sort_values('timestamp').reset_index(drop=True))

# Binary reader
def read_preprocessed_gridsatb1(furi):
    import numpy as np
    tmp = np.load(furi)
    image = np.stack([tmp,tmp,tmp], axis=2)
    return(image)

def read_multiple_preprocessed_noaagridsatb1(flist):
    ''' This method reads in a list of NOAA-GridSat-B1 images and returns a numpy array. '''
    import numpy as np
    data = []
    for f in flist:
        tmp = read_preprocessed_gridsatb1(f)
        data.append(tmp)
    return(np.array(data))

def encode_preprocessed_noaagridsatb1(model, flist, batch_size):
    ''' Encode the data with pre-trained-model. '''
    nSample = len(flist)
    # Prepare for results
    results =None
    # Read in and encode data by batch
    batch_start = 0
    batch_end = batch_size
    while batch_start < nSample:
        limit = min(batch_end, nSample)
        logging.info('Encoding batch: '+str(batch_start)+' - '+str(limit)+' of '+str(nSample)+'.')
        X = read_multiple_preprocessed_noaagridsatb1(flist['xuri'].iloc[batch_start:limit])
        features = model.signatures['default'](tf.convert_to_tensor(X))
        if results is None:
            results = features['default'].numpy()
        else:
            results = np.vstack([results, features['default'].numpy()])
        # Increment the loop
        batch_start += batch_size
        batch_end += batch_size
    # Return results as an numpy array
    return(results)

#-----------------------------------------------------------------------
def main():
    # Configure Argument Parser
    parser = argparse.ArgumentParser(description='Building convolutional autoencoder .')
    parser.add_argument('--datapath', '-i', help='the directory containing NOAA-GridSat-B1 data in numpy array.')
    parser.add_argument('--model', '-m', help='the pre-trained model directory.')
    parser.add_argument('--output', '-o', help='the prefix of output files.')
    parser.add_argument('--logfile', '-l', default=None, help='the log file.')
    parser.add_argument('--batch_size', '-b', default=64, type=int, help='the batch size.')

    args = parser.parse_args()
    # Set up logging
    if not args.logfile is None:
        logging.basicConfig(level=logging.DEBUG, filename=args.logfile, filemode='w')
    else:
        logging.basicConfig(level=logging.DEBUG)
    logging.debug(args)
    # Get data files
    logging.info('Scanning data files.')
    datainfo = list_preprocessed_gridsatb1_files(args.datapath)
    # Load model
    logging.info("Load pre-trained model from " + str(args.model))
    model = hub.load(args.model)
    # Debug info
    nSample = datainfo.shape[0]
    logging.info("Ecnoding total data size: "+str(nSample))
    # Encode data with the autoencoder
    features = encode_preprocessed_noaagridsatb1(model, datainfo, args.batch_size)
    # Prepare output
    pd.DataFrame(features, index=datainfo['timestamp']).to_csv(args.output+'_features.csv')
    np.save(args.output+'_features.npy', features)
    # done
    return(0)

#==========
# Script
#==========
if __name__=="__main__":
    main()

max_stars_repo_name: tingsyo/dlgridsat

max_issues_repo_name: tingsyo/dlgridsat

max_forks_repo_name: tingsyo/dlgridsat


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: oesteban
# @Date:   2015-11-19 16:44:27
# @Last Modified by:   oesteban
# @Last Modified time: 2017-03-08 13:52:24

"""
===================
Data handler module
===================

Reads in and writes CSV files with the IQMs

"""
from __future__ import absolute_import, division, print_function, unicode_literals

import numpy as np
import pandas as pd
from builtins import str

from mriqc import logging
from mriqc.utils.misc import BIDS_COMP
LOG = logging.getLogger('mriqc.classifier')

def read_iqms(feat_file):
    """ Reads in the features """

    bids_comps = list(BIDS_COMP.keys())
    x_df = pd.read_csv(feat_file, index_col=False,
                       dtype={col: str for col in bids_comps})

    # Find present bids bits and sort by them
    bids_comps_present = list(set(x_df.columns.ravel().tolist()) & set(bids_comps))
    x_df = x_df.sort_values(by=bids_comps_present)

    # Remove sub- prefix in subject_id
    x_df.subject_id = x_df.subject_id.str.lstrip('sub-')

    # Remove columns that are not IQMs
    feat_names = list(x_df._get_numeric_data().columns.ravel())
    for col in bids_comps:
        try:
            feat_names.remove(col)
        except ValueError:
            pass

    for col in feat_names:
        if col.startswith(('size_', 'spacing_', 'Unnamed')):
            feat_names.remove(col)

    return x_df, feat_names, bids_comps_present

def read_labels(label_file, rate_label='rate', binarize=True):
    """ Reads in the labels """
    # Massage labels table to have the appropriate format

    bids_comps = list(BIDS_COMP.keys())

    y_df = pd.read_csv(label_file, index_col=False,
                       dtype={col: str for col in bids_comps})

    # Find present bids bits and sort by them
    bids_comps_present = list(set(y_df.columns.ravel().tolist()) & set(bids_comps))
    y_df = y_df.sort_values(by=bids_comps_present)
    y_df.subject_id = y_df.subject_id.str.lstrip('sub-')

    # Convert string labels to ints
    try:
        y_df.loc[y_df[rate_label].str.contains('fail', case=False, na=False), rate_label] = -1
        y_df.loc[y_df[rate_label].str.contains('exclude', case=False, na=False), rate_label] = -1
        y_df.loc[y_df[rate_label].str.contains('maybe', case=False, na=False), rate_label] = 0
        y_df.loc[y_df[rate_label].str.contains('may be', case=False, na=False), rate_label] = 0
        y_df.loc[y_df[rate_label].str.contains('ok', case=False, na=False), rate_label] = 1
        y_df.loc[y_df[rate_label].str.contains('good', case=False, na=False), rate_label] = 1
    except AttributeError:
        pass

    y_df[[rate_label]] = y_df[[rate_label]].apply(pd.to_numeric, errors='raise')

    if binarize:
        y_df.loc[y_df[rate_label] >= 0, rate_label] = 0
        y_df.loc[y_df[rate_label] < 0, rate_label] = 1

    return y_df[bids_comps_present + ['site', rate_label]]

def read_dataset(feat_file, label_file, rate_label='rate', merged_name=None,
                 binarize=True):
    """ Reads in the features and labels """

    x_df, feat_names, _ = read_iqms(feat_file)
    y_df = read_labels(label_file, rate_label, binarize)

    # Remove failed cases from Y, append new columns to X
    y_df = y_df[y_df['subject_id'].isin(list(x_df.subject_id.values.ravel()))]

    # Merge Y dataframe into X
    x_df = pd.merge(x_df, y_df, on='subject_id', how='left')

    if merged_name is not None:
        x_df.to_csv(merged_name, index=False)

    # Drop samples with invalid rating
    nan_labels = x_df[x_df[rate_label].isnull()].index.ravel().tolist()
    if nan_labels:
        LOG.info('Dropping %d samples for having non-numerical '
                 'labels', len(nan_labels))
        x_df = x_df.drop(nan_labels)

    # Print out some info
    nsamples = len(x_df)
    LOG.info('Created dataset X="%s", Y="%s" (N=%d valid samples)',
             feat_file, label_file, nsamples)

    nfails = int(x_df[rate_label].sum())
    LOG.info('Ratings distribution: "fail"=%d / "ok"=%d (%f%% failed)',
             nfails, nsamples - nfails, nfails * 100 / nsamples)

    return x_df, feat_names

def zscore_dataset(dataframe, excl_columns=None, by='site',
                   njobs=-1):
    """ Returns a dataset zscored by the column given as argument """
    from multiprocessing import Pool, cpu_count

    LOG.info('z-scoring dataset ...')

    if njobs <= 0:
        njobs = cpu_count()

    sites = list(set(dataframe[[by]].values.ravel().tolist()))
    columns = list(dataframe._get_numeric_data().columns.ravel())

    if excl_columns is None:
        excl_columns = []

    for col in columns:
        if not np.isfinite(np.sum(dataframe[[col]].values.ravel())):
            excl_columns.append(col)

    if excl_columns:
        for col in excl_columns:
            try:
                columns.remove(col)
            except ValueError:
                pass

    zs_df = dataframe.copy()

    pool = Pool(njobs)
    args = [(zs_df, columns, s) for s in sites]
    results = pool.map(zscore_site, args)
    for site, res in zip(sites, results):
        zs_df.loc[zs_df.site == site, columns] = res

    zs_df.replace([np.inf, -np.inf], np.nan)
    nan_columns = zs_df.columns[zs_df.isnull().any()].tolist()

    if nan_columns:
        LOG.warn('Columns %s contain NaNs after z-scoring.', ", ".join(nan_columns))
        zs_df[nan_columns] = dataframe[nan_columns].values

    return zs_df

def zscore_site(args):
    """ z-scores only one site """
    from scipy.stats import zscore
    dataframe, columns, site = args
    return zscore(dataframe.loc[dataframe.site == site, columns].values,
                  ddof=1, axis=0)

max_stars_repo_name: effigies/mriqc

max_issues_repo_name: effigies/mriqc

max_forks_repo_name: effigies/mriqc

reason: import numpy,from scipy


text:
# Copyright (c) 2018 CEA
# Author: Yann Leprince <y.leprince@fz-juelich.de>
#
# This software is made available under the MIT licence, see LICENCE.txt.

import gzip
import io

import numpy as np

from neuroglancer_scripts.mesh import (
    save_mesh_as_precomputed,
    read_precomputed_mesh,
    save_mesh_as_neuroglancer_vtk,
)

def dummy_mesh(num_vertices=4, num_triangles=3):
    vertices = np.reshape(
        np.arange(3 * num_vertices, dtype=np.float32),
        (num_vertices, 3)
    )
    triangles = np.reshape(
        np.arange(3 * num_triangles, dtype=np.uint32),
        (num_triangles, 3)
    ) % num_vertices
    return vertices, triangles

def test_precomputed_mesh_roundtrip():
    vertices, triangles = dummy_mesh()
    file = io.BytesIO()
    save_mesh_as_precomputed(file, vertices, triangles)
    file.seek(0)
    vertices2, triangles2 = read_precomputed_mesh(file)
    assert np.array_equal(vertices, vertices2)
    assert np.array_equal(triangles, triangles2)

def test_precomputed_mesh_gzip_file_roundtrip():
    vertices, triangles = dummy_mesh()
    bytes_io = io.BytesIO()
    with gzip.GzipFile(fileobj=bytes_io, mode="wb") as f:
        save_mesh_as_precomputed(f, vertices, triangles)
    buf = bytes_io.getvalue()
    with gzip.GzipFile(fileobj=io.BytesIO(buf), mode="rb") as f:
        vertices2, triangles2 = read_precomputed_mesh(f)
    assert np.array_equal(vertices, vertices2)
    assert np.array_equal(triangles, triangles2)

def test_write_vtk_mesh():
    vertices, triangles = dummy_mesh()
    file = io.StringIO()
    save_mesh_as_neuroglancer_vtk(
        file, vertices, triangles,
        vertex_attributes=[{
            "name": "dummy_attribute",
            "values": np.arange(vertices.shape[0])
        }],
        title="dummy mesh"
    )

max_stars_repo_name: neurodata/neuroglancer-scripts

max_issues_repo_name: stephen-zhouyang/neuroglancer-scripts

max_forks_repo_name: stephen-zhouyang/neuroglancer-scripts


text:
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import os

def compute_stats_ar(results, ar_params, verbose=False):
    weights = results["weights"]
    error = results["predicted"] - results["actual"]
    stats = {}

    abs_error = np.abs(weights - ar_params)

    symmetric_abs_coeff = np.abs(weights) + np.abs(ar_params)
    stats["sMAPE (AR-coefficients)"] = 100 * np.mean(abs_error / (10e-9 + symmetric_abs_coeff))

    sTPE = 100 * np.sum(abs_error) / (10e-9 + np.sum(symmetric_abs_coeff))
    stats["sTPE (AR-coefficients)"] = sTPE

    # predictions error
    stats["MSE"] = np.mean(error ** 2)

    if verbose:
        print("MSE: {}".format(stats["MSE"]))
        print("sMAPE (AR-coefficients): {:6.3f}".format(stats["sMAPE (AR-coefficients)"]))
        print("sTPE (AR-coefficients): {:6.3f}".format(stats["sTPE (AR-coefficients)"]))
        # print("Relative error: {:6.3f}".format(stats["TP (AR-coefficients)"]))
        # print("Mean relative error: {:6.3f}".format(mean_rel_error))

        print("AR params: ")
        print(ar_params)

        print("Weights: ")
        print(weights)
    return stats

def plot_loss_curve(losses, test_loss=None, epoch_losses=None, show=False, save=False):
    fig = plt.figure()
    fig.set_size_inches(12, 6)
    ax = plt.axes()
    ax.set_xlabel("Iteration")
    ax.set_ylabel("Loss")
    x_loss = list(range(len(losses)))
    plt.plot(x_loss, losses, 'b', alpha=0.3)
    if epoch_losses is not None:
        iter_per_epoch = int(len(losses) / len(epoch_losses))
        epoch_ends = int(iter_per_epoch/2) + iter_per_epoch*np.arange(len(epoch_losses))
        plt.plot(epoch_ends, epoch_losses, 'b')
    if test_loss is not None:
        plt.hlines(test_loss, xmin=x_loss[0], xmax=x_loss[-1])
    if save:
        if not os.path.exists('results'):
            os.makedirs('results')
        figname = 'results/loss_DAR.png'
        plt.savefig(figname, dpi=600, bbox_inches='tight')
    plt.show()
    # plt.close()

def plot_prediction_sample(predicted, actual, num_obs=100, model_name="AR-Net", save=False):
    fig2 = plt.figure()
    fig2.set_size_inches(10, 6)
    plt.plot(actual[0:num_obs])
    plt.plot(predicted[0:num_obs])
    plt.legend(["Actual Time-Series", "{}-Prediction".format(model_name)])
    if save:
        if not os.path.exists('results'):
            os.makedirs('results')
        figname = 'results/prediction_{}.png'.format(model_name)
        plt.savefig(figname, dpi=600, bbox_inches='tight')
    plt.show()

def plot_error_scatter(predicted, actual, model_name="AR-Net", save=False):
    # error = predicted - actual
    fig3 = plt.figure()
    fig3.set_size_inches(6, 6)
    plt.scatter(actual, predicted - actual, marker='o', s=10, alpha=0.3)
    plt.legend(["{}-Error".format(model_name)])
    if save:
        if not os.path.exists('results'):
            os.makedirs('results')
        figname = 'results/scatter_{}.png'.format(model_name)
        plt.savefig(figname, dpi=600, bbox_inches='tight')
    plt.show()

def plot_weights(ar_val, weights, ar, model_name="AR-Net", save=False):
    df = pd.DataFrame(
        zip(
            list(range(1, ar_val + 1)) * 2,
            ["AR-Process (True)"] * ar_val + [model_name] * ar_val,
            list(ar) + list(weights)
        ),
        columns=["AR-coefficient (lag number)", "model", "value (weight)"]
    )
    plt.figure(figsize=(10, 6))
    palette = {"Classic-AR": "C0", "AR-Net": "C1", "AR-Process (True)": "k"}
    sns.barplot(x="AR-coefficient (lag number)", hue="model", y="value (weight)", data=df)
    if save:
        if not os.path.exists('results'):
            os.makedirs('results')
        figname = 'results/weights_{}_{}.png'.format(ar_val, model_name, palette=palette)
        plt.savefig(figname, dpi=600, bbox_inches='tight')

    plt.show()

def plot_results(results, model_name="MODEL", save=False):
    plot_prediction_sample(results["predicted"], results["actual"], num_obs=100, model_name=model_name, save=save)
    plot_error_scatter(results["predicted"], results["actual"], model_name=model_name, save=save)

def jsonize(results):
    for key, value in results.items():
        if type(value) is list:
            if type(value[0]) is list:
                results[key] = [["{:8.5f}".format(xy) for xy in x] for x in value]
            else:
                results[key] = ["{:8.5f}".format(x) for x in value]
        else:
            results[key] = "{:8.5f}".format(value)
    return results

def list_of_dicts_2_dict_of_lists(sources):
    keys = sources[0].keys()
    res = {}
    for key in keys:
        res[key] = [d[key] for d in sources]
    return res

def list_of_dicts_2_dict_of_means(sources):
    keys = sources[0].keys()
    res = {}
    for key in keys:
        res[key] = np.mean([d[key] for d in sources])
    return res

def list_of_dicts_2_dict_of_means_minmax(sources):
    keys = sources[0].keys()
    res = {}
    for key in keys:
        values = [d[key] for d in sources]
        res[key] = (np.mean(values), min(values), max(values))
    return res

def get_json_filenames(values, subdir=None):
    ar_filename = get_json_filenames_type("AR", values, subdir)
    dar_filename = get_json_filenames_type("DAR", values, subdir)
    return ar_filename, dar_filename

def get_json_filenames_type(model_type, values, subdir=None):
    filename = 'results/{}{}_{}.json'.format(
        subdir + "/" if subdir is not None else "",
        model_type,
        "-".join([str(x) for x in values])
    )
    return filename

def intelligent_regularization(sparsity):
    if sparsity is not None:
        # best:
        # lam = 0.01 * (1.0 / sparsity - 1.0)
        lam = 0.02 * (1.0 / sparsity - 1.0)
        # lam = 0.05 * (1.0 / sparsity - 1.0)

        # alternatives
        # l1 = 0.02 * (np.log(2) / np.log(1 + sparsity) - 1.0)
        # l1 = 0.1 * (1.0 / np.sqrt(sparsity) - 1.0)
    else:
        lam = 0.0
    return lam

max_stars_repo_name: MateusGheorghe/AR-Net

max_issues_repo_name: MateusGheorghe/AR-Net

max_forks_repo_name: MateusGheorghe/AR-Net


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import numpy as np
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), "../../"))
from split_data.input_file import InputFile
from os import path
APP_ROOT = path.dirname(path.abspath(__file__))

class ClassSummary():
    """Class Summary class.

    This class is the summary class

    """

    def __init__(self, file_name, wiki_vector, input_path, output_path=""):
        """
        Args:
            word_list(list): you set the word list.
            wiki_vector: you set the wiki vector dictionary
            class_vector: you set the word class dictionary
        """
        self.__file_name = file_name
        self.input_path = input_path
        self.output_path = output_path
        self.output_flag = False
        if output_path != "":
            self.output_flag = True
        self.extension = "_vector.txt"
        self.input_module = InputFile(APP_ROOT + input_path + file_name)
        if file_name != "":
            self.input_module.input_special_format_file()
        self.__word_list = self.input_module.get_file_data()
        self.__wiki_vector = wiki_vector
        self.__class_word_vector = {}
        self.__word_vector = {}
        self.__average_vector = {}

    def summary_class(self):
        for word in self.__word_list:
            word = re.sub("\]|\[|\'|\s", "", word.strip())
            self.__add_wiki_vector(word.split(","))

    def summary_vector(self):
        self.__add_wiki_vector(self.__word_list)

    def summary_vector_word_list(self, word_list):
        self.__add_wiki_vector(word_list)

    def __add_wiki_vector(self, word_list):
        for word in word_list:
            if word in self.__wiki_vector:
                if word not in self.__word_vector:
                    self.__word_vector.update({word: self.__wiki_vector[word]})
        self.__calculate_average_vector()

    def __calculate_average_vector(self):
        average_list = []
        for word, vector in self.__word_vector.items():
            average_list.append(vector)
        average_vector = np.array(average_list)
        if average_vector != []:
            with open(APP_ROOT + self.output_path + self.__file_name.strip() + self.extension, 'w', encoding='utf-8') as fo:
                 self.__average_vector = np.mean(average_vector, axis=0).tolist()
                 if self.output_flag:
                     fo.write(str(self.__average_vector))
                     fo.write("\n")

    def get_average_vector(self):
        return self.__average_vector

max_stars_repo_licenses: Unlicense MIT

max_issues_repo_licenses: Unlicense MIT

max_forks_repo_licenses: Unlicense MIT


text:
import os
import argparse
import torch
import numpy as np
import torchvision
from torch_geometric.data import Data, InMemoryDataset, download_url, DataLoader
from torch_geometric.utils import dense_to_sparse
import networkx as nx
import matplotlib.pyplot as plt
import torchvision.transforms as transforms
import scipy.sparse as sp
import h5py

def get_image_opt(opt):
  opt['im_dataset'] = 'MNIST'  #datasets = ['MNIST','CIFAR']

  opt['input_dropout'] = 0.5
  opt['dropout'] = 0
  opt['optimizer'] = 'rmsprop'
  opt['lr'] = 0.0047
  opt['decay'] = 5e-4
  opt['self_loop_weight'] = 0.555  #### 0?
  opt['alpha'] = 0.918
  opt['time'] = 12.1
  opt['augment'] = False #True   #False need to view image
  opt['attention_dropout'] = 0
  opt['adjoint'] = False

  opt['epoch'] = 4 #3 #10#20 #400
  opt['batch_size'] = 64 #64  # doing batch size for mnist
  opt['train_size'] = 512 #128#512 #4096 #2048 #512 #2047:#4095:#511:#5119:#1279:#
  opt['test_size'] = 64 #2559:#1279:
  assert (opt['train_size']) % opt['batch_size'] == 0, "train_size needs to be multiple of batch_size"
  assert (opt['test_size']) % opt['batch_size'] == 0, "test_size needs to be multiple of batch_size"

  if opt['im_dataset'] == 'MNIST':
    opt['im_width'] = 28
    opt['im_height'] = 28
    opt['im_chan'] = 1
    opt['hidden_dim'] = 1 #16    #### 1 or 3 rgb?
    opt['num_feature'] = 1  # 1433   #### 1 or 3 rgb?
    opt['num_class'] = 10  # 7  #### mnist digits

  elif opt['im_dataset'] == 'CIFAR':
    opt['im_width'] = 32
    opt['im_height'] = 32
    opt['im_chan'] = 3
    # ????
    opt['hidden_dim'] = 3 #16    #### 1 or 3 rgb?
    opt['num_feature'] = 3  # 1433   #### 1 or 3 rgb?
    opt['num_class'] = 10  # 7  #### mnist digits

  opt['num_nodes'] = opt['im_height'] * opt['im_width'] * opt['im_chan'] #2708  ###pixels
  opt['simple'] = False #True
  opt['diags'] = True
  opt['ode'] = 'ode' #'att' don't think att is implmented properly on this codebase?
  opt['linear_attention'] = True
  opt['batched'] = True
  return opt

def edge_index_calc(im_height, im_width, im_chan, diags = False):
    edge_list = []
    def oneD():
        for i in range(im_height * im_width):
            #corners
            if i in [0, im_width-1, im_height * im_width - im_width, im_height * im_width - 1]:
                if i  == 0:
                    edge_list.append([i,1])
                    edge_list.append([i,im_width])
                    edge_list.append([i,im_width + 1]) if diags == True else 0
                elif i == im_width - 1:
                    edge_list.append([i, im_width - 2])
                    edge_list.append([i, 2 * im_width - 1])
                    edge_list.append([i, 2 * im_width - 2]) if diags == True else 0
                elif i == im_height * im_width - im_width:
                    edge_list.append([i, im_height * im_width - 2*im_width])
                    edge_list.append([i, im_height * im_width - im_width+1])
                    edge_list.append([i, im_height * im_width - 2*im_width+1]) if diags == True else 0
                elif i == im_height * im_width - 1:
                    edge_list.append([i, im_height * im_width - 2])
                    edge_list.append([i, im_height * im_width - 1 - im_width])
                    edge_list.append([i, im_height * im_width - im_width - 2]) if diags == True else 0
            # top edge
            elif i in range(1,im_width-1):
                edge_list.append([i,i-1])
                edge_list.append([i,i+1])
                edge_list.append([i,i+im_width])
                if diags:
                    edge_list.append([i, i + im_width -1])
                    edge_list.append([i, i + im_width + 1])
            # bottom edge
            elif i in range(im_height * im_width - im_width, im_height * im_width):
                edge_list.append([i,i-1])
                edge_list.append([i,i+1])
                edge_list.append([i,i-im_width])
                if diags:
                    edge_list.append([i, i - im_width -1])
                    edge_list.append([i, i - im_width + 1])
            # middle
            else:
                if i % im_width == 0: # left edge
                    edge_list.append([i,i+1])
                    edge_list.append([i,i-im_width])
                    edge_list.append([i,i+im_width])
                    if diags:
                        edge_list.append([i, i - im_width + 1])
                        edge_list.append([i, i + im_width + 1])
                elif (i + 1) % im_width == 0: # right edge
                    edge_list.append([i,i-1])
                    edge_list.append([i,i-im_width])
                    edge_list.append([i,i+im_width])
                    if diags:
                        edge_list.append([i, i - im_width - 1])
                        edge_list.append([i, i + im_width - 1])
                else:
                    edge_list.append([i,i-1])
                    edge_list.append([i,i+1])
                    edge_list.append([i,i-im_width])
                    edge_list.append([i,i+im_width])
                    if diags:
                        edge_list.append([i, i - im_width - 1])
                        edge_list.append([i, i - im_width + 1])
                        edge_list.append([i, i + im_width - 1])
                        edge_list.append([i, i + im_width + 1])
        return edge_list

    edge_list = oneD()
    ret_edge_tensor = torch.tensor(edge_list).T

    #this is wrong need to put colour channels as featurs not extra nodes, saving code in case come back to
    # if im_chan == 1:
    #     edge_list = oneD()
    #     ret_edge_tensor = torch.tensor(edge_list).T
    # else:
    #     edge_list = oneD()
    #     edge_tensor = torch.tensor(edge_list).T
    #     edge_list_3D = []
    #     for i in range(im_chan):
    #         chan_edge_tensor = edge_tensor + i * im_height * im_width
    #         edge_list_3D.append(chan_edge_tensor)
    #     ret_edge_tensor = torch.cat(edge_list_3D,dim=1)
    if diags:
        assert ret_edge_tensor.shape[1] == (8*(im_width-2)*(im_height-2)\
                                    + 2*5*(im_width-2) + 2*5*(im_height-2)\
                                    + 4*3) ,"Wrong number of fixed grid edges (inc diags)"
    else:
        assert ret_edge_tensor.shape[1] == (4*(im_width-2)*(im_height-2) \
                                    + 2*3*(im_width-2) + 2*3*(im_height-2)\
                                    + 4*2) ,"Wrong number of fixed grid edges (exc diags)"
    return ret_edge_tensor

def create_in_memory_dataset(opt, type, data_loader, edge_index, im_height, im_width, im_chan, root, processed_file_name=None):
    class IMAGE_IN_MEM(InMemoryDataset):
        def __init__(self, root, transform=None, pre_transform=None, pre_filter=None):
            super(IMAGE_IN_MEM, self).__init__(root, transform, pre_transform, pre_filter)
            self.data, self.slices = torch.load(self.processed_paths[0])

        @property
        def raw_file_names(self):
           return []
        @property
        def processed_file_names(self):
            return [processed_file_name]
        def download(self):
            pass #download_url(self.url, self.raw_dir)

        # @property
        # def num_classes(self):
        #     r"""The number of classes in the dataset."""
        #     y = self.data.y
        #     return y.max().item() + 1 if y.dim() == 1 else y.size(1)
        def process(self):
            graph_list = []
            for batch_idx, (data, target) in enumerate(data_loader):
                if type == "GNN":
                    if batch_idx > 0:
                        break
                    self.tensor = torch.tensor(opt['num_class'] - 1)
                    y = self.tensor  #<- hack the datset num_classes property (code above)
                elif type == "Train":
                    if opt['testing_code'] == True and batch_idx > opt['train_size'] - 1:
                        break
                    y = target
                elif type == "Test":
                    if opt['testing_code'] == True and batch_idx > opt['test_size'] - 1:
                        break
                    y = target
                x = data.view(im_chan, im_width * im_height)#, -1)
                x = x.T

                graph = Data(x=x, y=y.unsqueeze(dim=0), edge_index=edge_index)
                graph_list.append(graph)

            self.data, self.slices = self.collate(graph_list)
            torch.save((self.data, self.slices), self.processed_paths[0])

    return IMAGE_IN_MEM(root)

def load_data(opt):
  im_height = opt['im_height']
  im_width = opt['im_width']
  im_chan = opt['im_chan']
  exdataset = opt['im_dataset']

  if opt['im_dataset'] == 'MNIST':
    transform = transforms.Compose([transforms.ToTensor(),
                                    transforms.Normalize((0.1307,), (0.3081,))])
    data_train = torchvision.datasets.MNIST('../data/'+exdataset + '/', train=True, download=True,
                                            transform=transform)
    data_test = torchvision.datasets.MNIST('../data/'+exdataset+'/', train=False, download=True,
                                           transform=transform)
  elif opt['im_dataset'] == 'CIFAR':
    transform = transforms.Compose([transforms.ToTensor(),
                                    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
    data_train = torchvision.datasets.CIFAR10('../data/' + exdataset + '/', train=True, download=True,
                                            transform=transform)
    data_test = torchvision.datasets.CIFAR10('../data/' + exdataset + '/', train=False, download=True,
                                           transform=transform)

  train_loader = torch.utils.data.DataLoader(data_train, batch_size=1, shuffle=True)
  test_loader = torch.utils.data.DataLoader(data_test, batch_size=1, shuffle=True)

  edge_index = edge_index_calc(im_height, im_width, im_chan, diags=opt['diags'])
  print("creating in_memory_datasets")
  if opt['testing_code'] == True:
      Graph_GNN = create_in_memory_dataset(opt, "GNN", train_loader, edge_index, im_height, im_width, im_chan,
                                             root='../data/PyG'+exdataset+'GNN/', processed_file_name='Graph'+exdataset+'GNN.pt')
      Graph_train = create_in_memory_dataset(opt, "Train", train_loader, edge_index, im_height, im_width, im_chan,
                                             root='../data/PyG'+exdataset+ str(opt['train_size'])+'Train/', processed_file_name='Graph'+exdataset+str(opt['train_size']) +'Train.pt')
      Graph_test = create_in_memory_dataset(opt, "Test", test_loader, edge_index, im_height, im_width, im_chan,
                                             root='../data/PyG'+exdataset+ str(opt['test_size'])+'Test/', processed_file_name='Graph'+exdataset+str(opt['test_size']) +'Test.pt')
  else:
      Graph_GNN = create_in_memory_dataset(opt, "GNN", train_loader, edge_index, im_height, im_width, im_chan,
                                             root='../data/PyG'+exdataset+'GNN/', processed_file_name='PyG'+exdataset+'GNN.pt')
      Graph_train = create_in_memory_dataset(opt, "Train", train_loader, edge_index, im_height, im_width, im_chan,
                                             root='../data/PyG'+exdataset+'Train/', processed_file_name='PyG'+exdataset+'Train.pt')
      Graph_test = create_in_memory_dataset(opt, "Test", test_loader, edge_index, im_height, im_width, im_chan,
                                             root='../data/PyG'+exdataset+'Test/', processed_file_name='PyG'+exdataset+'Test.pt')
  return Graph_GNN, Graph_train, Graph_test

# from SuperPixData import load_matlab_file, stack_matrices
def create_Superpix75(opt, type, root, processed_file_name=None):
    class IMAGE_IN_MEM(InMemoryDataset):
        def __init__(self, root, transform=None, pre_transform=None, pre_filter=None):
            super(IMAGE_IN_MEM, self).__init__(root, transform, pre_transform, pre_filter)
            self.data, self.slices = torch.load(self.processed_paths[0])
        @property
        def raw_file_names(self):
           return []
        @property
        def processed_file_names(self):
            return [processed_file_name]
        def download(self):
            pass #download_url(self.url, self.raw_dir)

        def load_labels(self, fname):
            tmp = load_matlab_file(fname, 'labels')
            tmp = tmp.astype(np.int32)
            return tmp.flatten()

        def process(self):
            graph_list = []
            # load
            n_supPix = 75
            path_main = '../data/SuperMNIST/MNIST/'

            if type == "GNN":
                y = torch.tensor(opt['num_class']-1)
            elif type == "Train":
                path_train_vals = os.path.join(path_main,'datasets/mnist_superpixels_data_%d/train_vals.mat' % n_supPix)
                path_coords_train = os.path.join(path_main,'datasets/mnist_superpixels_data_%d/train_patch_coords.mat' % n_supPix)
                path_train_labels = os.path.join(path_main, 'datasets/MNIST_preproc_train_labels/MNIST_labels.mat')
                path_train_centroids = os.path.join(path_main,'datasets/mnist_superpixels_data_%d/train_centroids.mat' % n_supPix)

                vals_train = load_matlab_file(path_train_vals, 'vals')
                tmp = load_matlab_file(path_coords_train, 'patch_coords')
                coords_train = stack_matrices(tmp, n_supPix)

                # compute the adjacency matrix
                adj_mat_train = np.zeros(
                    (coords_train.shape[0], coords_train.shape[1], coords_train.shape[2]))
                for k in range(coords_train.shape[0]):
                    adj_mat_train[k, :, :] = np.isfinite(coords_train[k, :, :, 1])

                train_labels = self.load_labels(path_train_labels)
                batch_size = opt['batch_size']
                for i in range(opt['train_size'] // batch_size):
                    x = vals_train[i*batch_size:(i+1)*batch_size+1,:]
                    y = train_labels[i]
                    pos = coords_train

                    edge_index , edge_attr = dense_to_sparse(adj_mat_train[i,:,:].squeeze())

                    graph_list.append(Data(x=x, edge_index=edge_index, edge_attr=edge_attr, y=y, pos=pos))

            elif type == "Test":
                path_test_vals = os.path.join(path_main,'datasets/mnist_superpixels_data_%d/test_vals.mat' % n_supPix),
                path_coords_test = os.path.join(path_main,'datasets/mnist_superpixels_data_%d/test_patch_coords.mat' % n_supPix),
                path_test_labels = os.path.join(path_main, 'datasets/MNIST_preproc_test_labels/MNIST_labels.mat'),
                path_test_centroids = os.path.join(path_main,'datasets/mnist_superpixels_data_%d/test_centroids.mat' % n_supPix)

                vals_test = load_matlab_file(path_test_vals, 'vals')
                tmp = load_matlab_file(path_coords_test, 'patch_coords')
                coords_test = stack_matrices(tmp, n_supPix)

                # compute the adjacency matrix
                adj_mat_test = np.zeros(
                    (coords_test.shape[0], coords_test.shape[1], coords_test.shape[2]))
                for k in range(coords_test.shape[0]):
                    adj_mat_test[k, :, :] = np.isfinite(coords_test[k, :, :, 1])

                test_labels = self.load_labels(path_test_labels)

            self.data, self.slices = self.collate(graph_list)
            torch.save((self.data, self.slices), self.processed_paths[0])
    return IMAGE_IN_MEM(root)

def load_Superpix75Mat(opt): #, path='../data/SuperMNIST/MNIST/datasets/mnist_superpixels_data_75/'):

    # 'path_train_vals' : os.path.join(path_main, 'datasets/mnist_superpixels_data_%d/train_vals.mat' % n_supPix),
    # 'path_coords_train' : os.path.join(path_main, 'datasets/mnist_superpixels_data_%d/train_patch_coords.mat' % n_supPix),
    # 'path_train_labels' : os.path.join(path_main, 'datasets/MNIST_preproc_train_labels/MNIST_labels.mat'),
    # 'path_train_centroids' : os.path.join(path_main, 'datasets/mnist_superpixels_data_%d/train_centroids.mat' % n_supPix)},
    # 'test':{
    # 'path_test_vals' : os.path.join(path_main, 'datasets/mnist_superpixels_data_%d/test_vals.mat' % n_supPix),
    # 'path_coords_test' : os.path.join(path_main, 'datasets/mnist_superpixels_data_%d/test_patch_coords.mat' % n_supPix),
    # 'path_test_labels' : os.path.join(path_main, 'datasets/MNIST_preproc_test_labels/MNIST_labels.mat'),
    # 'path_test_centroids' : os.path.join(path_main, 'datasets/mnist_superpixels_data_%d/test_centroids.mat' % n_supPix)}}

    print("creating in_memory_datasets")
    # type = "GNN"
    # Graph_GNN = create_Superpix75(opt, type,
    #             root='../data/SuperPix75'+'_'+type+'/', processed_file_name='GraphSuperPix75'+type+'.pt')
    type = "Train"
    Graph_train = create_Superpix75(opt, type,
                root='../data/SuperPix75'+ str(opt['train_size']) +type+'/', processed_file_name='GraphSuperPix75'+ str(opt['train_size'])+type+'.pt')
    type = "Test"
    Graph_test = create_Superpix75(opt, type,
                root='../data/SuperPix75'+ str(opt['test_size']) +type+'/', processed_file_name='GraphSuperPix75'+ str(opt['test_size'])+type+'.pt')

    return Graph_GNN, Graph_train, Graph_test

def imshow(img):
    img = img / 2 + 0.5     # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--use_image_defaults', default='MNIST',
                        help='#Image version# Whether to run with best params for cora. Overrides the choice of dataset')
    # parser.add_argument('--use_image_defaults', action='store_true',
    #                     help='Whether to run with best params for cora. Overrides the choice of dataset')
    # parser.add_argument('--dataset', type=str, default='Cora',
    #                     help='Cora, Citeseer, Pubmed, Computers, Photo, CoauthorCS')
    parser.add_argument('--hidden_dim', type=int, default=16, help='Hidden dimension.')  ######## NEED
    parser.add_argument('--input_dropout', type=float, default=0.5, help='Input dropout rate.')
    parser.add_argument('--dropout', type=float, default=0.0, help='Dropout rate.')
    parser.add_argument('--optimizer', type=str, default='adam', help='One from sgd, rmsprop, adam, adagrad, adamax.')
    parser.add_argument('--lr', type=float, default=0.01, help='Learning rate.')
    parser.add_argument('--decay', type=float, default=5e-4, help='Weight decay for optimization')
    parser.add_argument('--self_loop_weight', type=float, default=1.0, help='Weight of self-loops.')
    parser.add_argument('--epoch', type=int, default=10, help='Number of training epochs per iteration.')
    parser.add_argument('--alpha', type=float, default=1.0, help='Factor in front matrix A.')
    parser.add_argument('--time', type=float, default=1.0, help='End time of ODE integrator.')
    parser.add_argument('--augment', action='store_true',
                        help='double the length of the feature vector by appending zeros to stabilist ODE learning')
    parser.add_argument('--alpha_dim', type=str, default='sc', help='choose either scalar (sc) or vector (vc) alpha')
    parser.add_argument('--alpha_sigmoid', type=bool, default=True, help='apply sigmoid before multiplying by alpha')
    parser.add_argument('--beta_dim', type=str, default='sc', help='choose either scalar (sc) or vector (vc) beta')
    # ODE args
    parser.add_argument('--method', type=str, default='dopri5',
                        help="set the numerical solver: dopri5, euler, rk4, midpoint")  ######## NEED
    parser.add_argument('--ode', type=str, default='ode',
                        help="set ode block. Either 'ode', 'att', 'sde'")  ######## NEED
    parser.add_argument('--adjoint', default=False, help='use the adjoint ODE method to reduce memory footprint')
    parser.add_argument('--tol_scale', type=float, default=1., help='multiplier for atol and rtol')
    parser.add_argument('--ode_blocks', type=int, default=1, help='number of ode blocks to run')
    parser.add_argument('--simple', type=bool, default=False,
                        help='If try get rid of alpha param and the beta*x0 source term')
    # SDE args
    parser.add_argument('--dt_min', type=float, default=1e-5, help='minimum timestep for the SDE solver')
    parser.add_argument('--dt', type=float, default=1e-3, help='fixed step size')
    parser.add_argument('--adaptive', type=bool, default=False, help='use adaptive step sizes')
    # Attention args
    parser.add_argument('--leaky_relu_slope', type=float, default=0.2,
                        help='slope of the negative part of the leaky relu used in attention')
    parser.add_argument('--attention_dropout', type=float, default=0., help='dropout of attention weights')
    parser.add_argument('--heads', type=int, default=1, help='number of attention heads')
    parser.add_argument('--attention_norm_idx', type=int, default=0, help='0 = normalise rows, 1 = normalise cols')
    parser.add_argument('--attention_dim', type=int, default=64,
                        help='the size to project x to before calculating att scores')
    parser.add_argument('--mix_features', type=bool, default=False,
                        help='apply a feature transformation xW to the ODE')
    parser.add_argument('--linear_attention', type=bool, default=False,
                        help='learn the adjacency using attention at the start of each epoch, but do not update inside the ode')
    parser.add_argument('--mixed_block', type=bool, default=False,
                        help='learn the adjacency using a mix of attention and the Laplacian at the start of each epoch, but do not update inside the ode')

    # visualisation args
    parser.add_argument('--batch_size', type=int, default=64, help='Batch size')
    parser.add_argument('--batched', type=bool, default=True,
                        help='Batching')
    parser.add_argument('--im_width', type=int, default=28, help='im_width')
    parser.add_argument('--im_height', type=int, default=28, help='im_height')
    parser.add_argument('--diags', type=bool, default=False,
                        help='Edge index include diagonal diffusion')
    parser.add_argument('--im_dataset', type=str, default='MNIST',
                        help='MNIST, CIFAR')
    args = parser.parse_args()
    opt = vars(args)
    opt = get_image_opt(opt)

    # load_Superpix75Mat(opt)

    # Cora = get_dataset('Cora', '../data', False)
    # gnn = GNN(self.opt, dataset, device=self.device)
    # odeblock = gnn.odeblock
    # func = odeblock.odefunc

    # img_size = 32#28
    # im_width = img_size
    # im_height = img_size
    # im_chan = 3 #1
    # exdataset = 'CIFAR' #'MNIST'

    # train_loader = torch.utils.data.DataLoader(
    #   torchvision.datasets.MNIST('data/' + exdataset + '/', train=True, download=True,
    #                              transform=torchvision.transforms.Compose([
    #                                torchvision.transforms.ToTensor(),
    #                                torchvision.transforms.Normalize(
    #                                  (0.1307,), (0.3081,))
    #                              ])),
    #                                 batch_size=1, shuffle=True)

    # transform = transforms.Compose(
    #     [transforms.ToTensor(),
    #      transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
    #
    # train_loader = torch.utils.data.DataLoader(
    #   torchvision.datasets.CIFAR10('data/' + exdataset + '/', train=True, download=True,
    #                              transform=transform),
    #                                 batch_size=1, shuffle=True)
    #
    #
    # edge_index = edge_index_calc(im_height, im_width)
    #
    # opt = get_image_opt({})
    # Graph = create_in_memory_dataset(opt, "Train", train_loader, edge_index, im_height, im_width, im_chan,
    #                                                                         root='./data/Graph' + exdataset + 'GNN/',
    #                                                                         processed_file_name='Graph' + exdataset + 'GNN2.pt')
    #
    # fig = plt.figure(figsize=(32,62))
    # # for i in range(6):
    # #     plt.subplot(2, 3, i + 1)
    #
    # for i in range(20):
    #     plt.subplot(5, 4, i + 1)
    #     plt.tight_layout()
    #     digit = Graph[i]
    #     plt.title("Ground Truth: {}".format(digit.y.item()))
    #     plt.xticks([])
    #     plt.yticks([])
    #     A = digit.x#.view(im_height, im_width, im_chan)
    #     A = A.numpy()
    #     A = np.reshape(A, (im_height, im_width, im_chan), order='F')
    #     A = A / 2 + 0.5  # unnormalize
    #     plt.imshow(np.transpose(A, (1, 0, 2)))
    # # plt.show()
    # plt.savefig("GraphImages.png", format="PNG")

max_stars_repo_name: jeongwhanchoi/graph-neural-pde

max_issues_repo_name: jeongwhanchoi/graph-neural-pde

max_forks_repo_name: jeongwhanchoi/graph-neural-pde

reason: import numpy,import scipy,import networkx


text:
import sys
import pickle
import argparse
import numpy as np
from pitf import PITF

def read_data(filepath):
    return np.genfromtxt(filepath, delimiter=' ', dtype=int)

p = argparse.ArgumentParser()
p.add_argument("-i", "--infile", help="input data file", type=argparse.FileType('r'), required=True)
p.add_argument("-m", "--modelfile", help="input model file", type=argparse.FileType('r'), required=True)
p.add_argument("-o", "--outfile", help="output file (default=STDOUT)", type=argparse.FileType('w'), nargs='?', default=sys.stdout)
args = p.parse_args()

data = read_data(args.infile)
model = pickle.load(args.modelfile)

predicted = model.predict2(data)
np.savetxt(args.outfile, predicted, fmt='%.0f')

max_stars_repo_name: SamHaoYuan/pitf

max_issues_repo_name: SamHaoYuan/pitf

max_forks_repo_name: SamHaoYuan/pitf


text:
'''checking functions for register class'''

# pylint: disable=E1101, W1401

import numpy
from . import qubit

def register_init_check(function):
    """Decorator to check the arguments of initialization function in register class.

    Arguments:
        function {} -- The tested function
    """

    def wrapper(self, qubit_list):
        """Method to initialize an instance of the register class. The input is a list of elements
        in Qubit or Random_Qubit class. Also this list must contain at least 2 elements.

        Arguments:
            qubit_list {list} -- List of objects from Qubit or Random_Qubit class

        Raises:
            ValueError, TypeError

        Examples:
            >>> import qvantum
            >>>
            >>> q1 = qvantum.Random_Qubit()
            >>> q2 = qvantum.Random_Qubit()
            >>>
            >>> r = qvantum.Register([q1, q2])
            >>> r.show()
            '|Ψ> = (-0.4171-0.2953i)|00> + (-0.4106-0.7059i)|01> + (-0.1120-0.0875i)|10> + (-0.1049-0.2015i)|11>'
        """

        if isinstance(qubit_list, list) \
            and all(isinstance(elem, (qubit.Qubit, qubit.Random_Qubit)) for elem in qubit_list):
            if len(qubit_list) >= 2:
                return function(self, qubit_list)

            else:
                raise ValueError('Invalid input! Qubit list must contain at least 2 qubit ' +\
                    'object.')

        else:
            raise TypeError('Invalid input! Argument must be a list of qubit objects.')

    return wrapper

def get_states_check(function):
    """Decorator to check the arguments of getting states function in register class.

    Arguments:
        function {} -- The tested function
    """

    def wrapper(self, nth=None):
        """Method to return with the n-th possible state for the regsiter if the parameter is
        definit. If it isn’t then the return value is the list of all possible states.

        Keyword Arguments:
            nth {int, None} -- Number of n-th possible state (default: {None})

        Raises:
            TypeError

        Examples:
            >>> import qvantum
            >>>
            >>> q1 = qvantum.Random_Qubit()
            >>> q2 = qvantum.Random_Qubit()
            >>>
            >>> r = qvantum.Register([q1, q2])
            >>> r.show()
            '|Ψ> = (0.0184-0.0130i)|00> + (0.0597+0.1181i)|01> + (-0.1003-0.1326i)|10> + (0.8560-0.4708i)|11>'
            >>> r.get_states()
            ['00', '01', '10', '11']
            >>> r.get_states(2)
            '10'
        """

        if isinstance(nth, int) or nth is None:
            return function(self, nth)

        else:
            raise TypeError('Invalid input! Argument must be integer.')

    return wrapper

def get_amplitudes_check(function):
    """Decorator to check the arguments of getting amplitudes function in register class.

    Arguments:
        function {} -- The tested function
    """

    def wrapper(self, nth=None):
        """Method to return with the coefficient of the n-th possible state for the regsiter
        if the parameter is definit. If it isn’t, then the return value is the list of the
        coefficients of all possible states.

        Keyword Arguments:
            nth {int, None} -- Number of n-th possible amplitude (default: {None})

        Raises:
            TypeError

        Examples:
            >>> import qvantum
            >>>
            >>> q1 = qvantum.Random_Qubit()
            >>> q2 = qvantum.Random_Qubit()
            >>>
            >>> r = qvantum.Register([q1, q2])
            >>> r.show()
            '|Ψ> = (0.1075+0.7037i)|00> + (0.6331-0.0247i)|01> + (0.2171-0.0638i)|10> + (-0.0347-0.1983i)|11>'
            >>> r.get_amplitudes(2)
            (-0.10034614628094177-0.1325886060571926j)
        """

        if isinstance(nth, int) or nth is None:
            return function(self, nth)

        else:
            raise TypeError('Invalid input! Argument must be integer or None type.')

    return wrapper

def set_amplitudes_check(function):
    """Decorator to check the arguments of setting amplitudes function in register class.

    Arguments:
        function {} -- The tested function
    """

    def wrapper(self, amp_list):
        """Method to set new coefficients for the possible states of the register. The input
        parameter is a list of real or complex number and their squared sum must be equal to 1.
        Number of elements in the least must be equal with the number of possible states.

        Arguments:
            amp_list {list} -- List of int, float or complex objects

        Raises:
            ValueError, TypeError

        Examples:
            >>> import qvantum
            >>>
            >>> q1 = qvantum.Random_Qubit()
            >>> q2 = qvantum.Random_Qubit()
            >>> q3 = qvantum.Random_Qubit()
            >>>
            >>> r = qvantum.Register([q1, q2, q3])
            >>> r.show()
            '|Ψ> = (0.0385+0.1339i)|000> + (0.3000+0.0702i)|001> + (0.1090+0.3519i)|010> + (0.7969+0.1696i)|011> + (-0.0195-0.0380i)|100> + (-0.0945-0.0033i)|101> + (-0.0537-0.0995i)|110> + (-0.2500-0.0038i)|111>'
            >>> r.set_amplitudes([0, 0, 1, 0, 0, 0, 0, 0])
            '|Ψ> = (0.0000+0.0000i)|000> + (0.0000+0.0000i)|001> + (1.0000+0.0000i)|010> + (0.0000+0.0000i)|011> + (0.0000+0.0000i)|100> + (0.0000+0.0000i)|101> + (0.0000+0.0000i)|110> + (0.0000+0.0000i)|111>'
        """

        if isinstance(amp_list, list) \
            and all(isinstance(elem, (int, float, complex)) for elem in amp_list):
            if round(numpy.sum(numpy.square(numpy.absolute(amp_list))) - 1, 10) == 0:
                return function(self, amp_list)

            else:
                raise ValueError('Invalid input! The square sum of absolute value of ' +\
                    'amplitudes must be equal to 1.')

        else:
            raise TypeError('Invalid input! Argument must be a list of integer, float or ' +\
                'complex numbers.')

    return wrapper

def measure_nth_qubit_check(function):
    """Decorator to check the arguments of measuring qubit function in register class.

    Arguments:
        function {} -- The tested function
    """

    def wrapper(self, nth):
        """Method to perform a measurement on the n-th qubit in the register and return the final
        state of the register after the process. This final state is randomized regarding to the
        amplitudes of the register. The input parameter must be an integer corresponding to the
        number of qubits in the register.

        Arguments:
            nth {int} -- Number of n-th possible qubit

        Raises:
            TypeError

        Examples:
            >>> import qvantum
            >>>
            >>> q1 = qvantum.Random_Qubit()
            >>> q2 = qvantum.Random_Qubit()
            >>> q3 = qvantum.Random_Qubit()
            >>>
            >>> r = qvantum.Register([q1, q2, q3])
            >>> r.show()
            '|Ψ> = (-0.5585-0.0451i)|000> + (0.2304+0.3353i)|001> + (0.0120+0.0257i)|010> + (0.0090-0.0185i)|011> + (-0.3614+0.4567i)|100> + (0.4228-0.0056i)|101> + (0.0291+0.0045i)|110> + (-0.0108-0.0185i)|111>'
            >>> r.measure_nth_qubit(2)
            1
            >>> r.show()
            '|Ψ> = (0.0000+0.0000i)|000> + (0.3921+0.5707i)|001> + (0.0000+0.0000i)|010> + (0.0153-0.0315i)|011> + (0.0000+0.0000i)|100> + (0.7196-0.0095i)|101> + (0.0000+0.0000i)|110> + (-0.0184-0.0314i)|111>'
        """

        if isinstance(nth, int):
            return function(self, nth)

        else:
            raise TypeError('Invalid input! Argument must be integer.')

    return wrapper

def delete_qubit_check(function):
    """Decorator to check the arguments of deleting qubit function in register class.

    Arguments:
        function {} -- The tested function
    """

    def wrapper(self, nth):
        """Method to delete the n-th qubit from the regsiter. This method has some drawback which
        is discussed later. The input parameter must be an integer corresponding to the number of
        qubits in the register.

        Arguments:
            nth {int} -- Number of n-th possible qubit

        Raises:
            ValueError, TypeError

        Examples:
            >>> import qvantum
            >>>
            >>> q1 = qvantum.Random_Qubit()
            >>> q2 = qvantum.Random_Qubit()
            >>> q3 = qvantum.Random_Qubit()
            >>>
            >>> r = qvantum.Register([q1, q2, q3])
            >>> r.show()
            '|Ψ> = (-0.0414-0.7876i)|000> + (-0.3986-0.4355i)|001> + (-0.0142+0.0084i)|010> + (-0.0042+0.0116i)|011> + (0.1003-0.0924i)|100> + (0.0140-0.1011i)|101> + (-0.0027-0.0010i)|110> + (-0.0020+0.0007i)|111>'
            >>> r.delete_qubit(2)
            >>> r.show()
            '|Ψ> = (0.4927-0.6160i)|00> + (-0.0083-0.5904i)|01> + (0.1364-0.0024i)|10> + (0.0777-0.0663i)|11>'
        """

        if isinstance(nth, int):
            return function(self, nth)

        else:
            raise TypeError('Invalid input! Argument must be integer.')

    return wrapper

def insert_qubit_check(function):
    """Decorator to check the arguments of inserting qubit function in register class.

    Arguments:
        function {} -- The tested function
    """

    def wrapper(self, q, nth):
        """Method to insert a given qubit into a register. The input parameter must be an integer
        corresponding to the number of qubits in the register.

        Arguments:
            q {qubit} -- The qubit to be inserted
            nth {int} -- The index where the qubit is inserted

        Raises:
            ValueError, TypeError

        Examples:
            >>> import qvantum
            >>>
            >>> q1 = qvantum.Random_Qubit()
            >>> q2 = qvantum.Random_Qubit()
            >>>
            >>> r = qvantum.Register([q1, q2])
            >>> r.show()
            '|Ψ> = (0.2206+0.0465i)|00> + (0.1222-0.4511i)|01> + (0.1796+0.3251i)|10> + (0.6939-0.3335i)|11>'
            >>> q3 = qvantum.Random_Qubit()
            >>> r.insert_qubit(q3, 2)
            >>> r.show()
            '|Ψ> = (-0.0764+0.1775i)|000> + (-0.0002-0.1162i)|001> + (0.3584+0.1790i)|010> + (-0.2404-0.0133i)|011> + (-0.3035+0.0960i)|100> + (0.1446-0.1253i)|101> + (0.1629+0.6394i)|110> + (-0.2424-0.3140i)|111>'
            >>> r.insert_qubit(q3, 2)
            >>> r.show()
            '|Ψ> = (-0.1362-0.0942i)|0000> + (0.0976+0.0195i)|0001> + (0.0976+0.0195i)|0010> + (-0.0585+0.0125i)|0011> + (-0.2109+0.2709i)|0100> + (0.0518-0.1998i)|0101> + (0.0518-0.1998i)|0110> + (0.0190+0.1226i)|0111> + (-0.0293-0.2712i)|1000> + (0.0809+0.1427i)|1001> + (0.0809+0.1427i)|1010> + (-0.0786-0.0595i)|1011> + (-0.5648+0.0288i)|1100> + (0.3048-0.1506i)|1101> + (0.3048-0.1506i)|1110> + (-0.1323+0.1558i)|1111>'
        """

        if isinstance(q, (qubit.Qubit, qubit.Random_Qubit)) and isinstance(nth, int):
            return function(self, q, nth)

        else:
            raise TypeError('Invalid input! Argument must a pair of qubit object and integer.')

    return wrapper

max_stars_repo_name: vorpex/qvantum

max_issues_repo_name: vorpex/qvantum

max_forks_repo_name: vorpex/qvantum


text:
import numpy as np
import tensorflow as tf
from styx_msgs.msg import TrafficLight

class TLClassifier(object):
    def __init__(self):
        # load classifier
        self.graph = tf.Graph()
        with self.graph.as_default():
            graph_def = tf.GraphDef()
            with tf.gfile.GFile("./light_classification/ssd_mobilenet.pb", 'rb') as f:
                frozen_graph = f.read()
                graph_def.ParseFromString(frozen_graph)
                tf.import_graph_def(graph_def, name='')

        self.image = self.graph.get_tensor_by_name('image_tensor:0')
        self.scores = self.graph.get_tensor_by_name('detection_scores:0')
        self.classes = self.graph.get_tensor_by_name('detection_classes:0')

        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        self.sess = tf.Session(graph=self.graph, config=config)

    def get_classification(self, image):
        """Determines the color of the traffic light in the image

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """
        # implement light color prediction
        with self.graph.as_default():
            unsqueezed_image = np.expand_dims(image, axis=0)

            (scores, classes) = self.sess.run([self.scores, self.classes],
              feed_dict={self.image: unsqueezed_image})

        scores = np.squeeze(scores)
        classes = np.squeeze(classes).astype(np.int32)

        state = TrafficLight.UNKNOWN
        if scores[0] > 0.5:
            if classes[0] == 1:
                state = TrafficLight.GREEN
            elif classes[0] == 2:
                state = TrafficLight.RED
            elif classes[0] == 3:
                state = TrafficLight.YELLOW
        return state


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

import logging
import multiprocessing
import numpy
import random
import sys

from core.qlearning.workers import QLearningEntrenarWorker, QLearningRecorrerWorker
from core.gridworld.gridworld import GridWorld
from core.tecnicas.tecnica import QLTecnica

class QLearning(object):
    u"""Algoritmo QLearning"""
    def __init__(self, gridworld, gamma, tecnica, episodes, iterations_pack,
                 init_value_fn, matriz_diff_pack, cant_max_iter_gral_pack=(sys.maxint, 1),
                 excluir_tipos_vecinos=None):
        """
        Inicializador de QLearning.

        :param gridworld: GridWorld sobre el cual se aplicará el algoritmo.
        :param gamma: Parámetro Gamma de QLearning.
        :param tecnica: Técnica a utilizar.
        :param episodes: Cantidad de veces que se puede alcanzar el estado final.
        :param init_value: Valor con que se inicializa cada estado de la matriz.
        """
        super(QLearning, self).__init__()

        self._logger = logging.getLogger()

        self._gridworld = gridworld
        self._gamma = gamma
        self._tecnica_pack = tecnica
        self._episodes = episodes
        self._excluir_tipos_vecinos = excluir_tipos_vecinos
        self._init_value_fn = init_value_fn
        self._iterations_pack = iterations_pack
        self._mat_diff_pack = matriz_diff_pack
        self._cant_max_iter_gral_pack = cant_max_iter_gral_pack

    def _generar_estado_aleatorio(self):
        u"""
        Devuelve una tupla conteniendo las coordenadas X e Y aleatorias.
        """
        # Fila
        x = random.randint(1, self._gridworld.ancho)
        # Columna
        y = random.randint(1, self._gridworld.alto)
        return (x, y)

    def entrenar(self, out_queue, error_queue):
        u"""
        Ejecuta el algoritmo de aprendizaje en otro hilo/proceso. Devuelve una
        referencia al hilo/proceso ejecutado.

        :param out_queue: Cola de salida.
        :param error_q: Cola de errores (salida)
        """
        # Cola de entrada
        inp_queue = multiprocessing.Queue()

        # Encolar Matriz R, Matriz Q, Número de episodios, Parámetro Gamma
        inp_queue.put((self._gridworld.estados,
                       self._gridworld.coordenadas,
                       self._gamma,
                       self._episodes,
                       self._iterations_pack,
                       self._tecnica_pack,
                       (self._gridworld.alto, self._gridworld.ancho),
                       False,
                       self._gridworld.tipos_vecinos_excluidos,
                       self._init_value_fn,
                       self._mat_diff_pack,
                       self._cant_max_iter_gral_pack
                       ))

        qlearning_entrenar_worker = None

        try:
            # Crear worker de Entrenamiento
            qlearning_entrenar_worker = QLearningEntrenarWorker(inp_queue,
                                                                out_queue,
                                                                error_queue
                                                                )
            qlearning_entrenar_worker.daemon = True
            qlearning_entrenar_worker.start()
        except multiprocessing.ProcessError as pe:
            self._logger.debug(pe)
            raise multiprocessing.ProcessError
        finally:
            pass

        return qlearning_entrenar_worker

    def recorrer(self, matriz_q, mat_est_acc, estado_inicial, out_queue, error_queue):
        u"""
        Ejecuta el algoritmo de recorrido en otro hilo/proceso. Devuelve una
        referencia al hilo/proceso ejecutado.

        :param out_queue: Cola de salida.
        :param error_q: Cola de errores (salida)
        """
        # Cola de entrada
        inp_queue = multiprocessing.Queue()
        inp_queue.put((matriz_q, mat_est_acc, estado_inicial))
        qlearning_recorrer_worker = None

        try:
            # Crear worker de Explotación
            qlearning_recorrer_worker = QLearningRecorrerWorker(inp_queue,
                                                                out_queue,
                                                                error_queue
                                                                )
            qlearning_recorrer_worker.daemon = True
            qlearning_recorrer_worker.start()
        except multiprocessing.ProcessError as pe:
            self._logger.debug(pe)
            raise multiprocessing.ProcessError
        finally:
            pass

        return qlearning_recorrer_worker

    def get_gamma(self):
        return self._gamma

    def set_gamma(self, valor):
        self._gamma = valor

    def get_tecnica(self):
        return self._tecnica_pack

    def set_tecnica(self, valor):
        if isinstance(valor, QLTecnica):
            self._tecnica_pack = valor
        else:
            raise TypeError(u"El parámetro debe ser del tipo QLTécnica")

    def get_gridworld(self):
        return self._gridworld

    def set_gridworld(self, valor):
        u"""
        Asigna un GridWorld dado a Q-Learning.

        :param valor: Arreglo de estados conteniendo un GridWorld.
        """
        if isinstance(valor, GridWorld):
            self._gridworld = valor
        else:
            raise TypeError(u"El parámetro debe ser del tipo GridWorld")

    def get_estado(self, x, y):
        """
        Devuelve la referencia al estado determinado por las coordenadas X e Y.

        :param x: Coordenada X del estado (fila)
        :param y: Coordenada Y del estado (columna)
        """
        return self._matriz_q[x - 1][y - 1]

    def set_estado(self, x, y, estado):
        """
        Almacena un estado dado en una posición determinada por las coordenadas
        X e Y.

        :param x: Coordenada X de destino (fila)
        :param y: Coordenada Y de destino (columna)
        :param estado: Estado a almacenar en la matriz.
        """
        self._matriz_q[x - 1][y - 1] = estado

    def get_episodes(self):
        return self._episodes

    def set_episodes(self, value):
        self._episodes = value

    def get_coordenadas(self):
        return self._coordenadas

    def get_iterations_pack(self):
        return self._iterations_pack

    def set_iterations_pack(self, valor):
        u"""
        Establece un conjunto de opciones referidas a las iteraciones de Q-Learning.

        :param valor: Tupla con el formato: (Activar Limitador (Booleano), Cantidad Iteraciones Máxima (Entero))
        """
        self._iterations_pack = valor

    def get_max_iterations(self):
        self._iterations_pack[1]

    def get_matriz_q(self, include_vecinos=False):
        u"""
        Genera y devuelve la matriz de vecinos, matriz R y matriz Q.
        """
        # Verificar si hay tipos de vecinos a excluir de la matriz R
        if self.tipos_vec_excluidos is None:
            self.tipos_vec_excluidos = []

        # Cachear acceso a métodos y atributos
        get_vecinos_estado = self.get_vecinos_estado
        get_estado = self.get_estado
        tipos_vec_excluidos = self.tipos_vec_excluidos
        q_init_value_fn = self.q_init_value_fn
        ancho = self.ancho
        alto = self.alto
        dimension = ancho * alto

        # Matriz de vecinos
        matriz_estados = numpy.empty((alto, ancho), numpy.int)

        # Matriz Q
        matriz_q = numpy.empty((dimension, dimension), numpy.float)
        matriz_q.fill(numpy.nan)

        for i in xrange(alto):
            fila = []
            fappend = fila.append

            for j in xrange(ancho):
                # Obtener estado actual y su ID de tipo
                estado = get_estado(i + 1, j + 1)
                estado_ide = estado.tipo.ide
                # Obtener los estados vecinos del estado actual (i, j)
                vecinos = get_vecinos_estado(i + 1, j + 1, True)

                # Calcular posición en eje Y
                y_coord = (i * alto) + j

                if include_vecinos:
                    evecinos = []
                    evappend = evecinos.append

                for x, y in vecinos:
                    est_vec = get_estado(x, y)

                    if est_vec.tipo.ide not in tipos_vec_excluidos:
                        # Calcular posición en eje X
                        x_coord = ((x - 1) * ancho) + (y - 1)

                        # Establecer valor Q inicial en matriz
                        matriz_q[y_coord][x_coord] = q_init_value_fn

                        if include_vecinos:
                            # Agregar coordenadas de vecino
                            evappend((y_coord, x_coord))

                if include_vecinos:
                    # Agregar columna a la fila
                    fappend((estado_ide, evecinos))
                else:
                    fappend(estado_ide)

            # Agregar fila a matriz de vecinos
            matriz_estados[i] = fila

        return matriz_q

    def get_vecinos_estado(self, x, y, iterate=False):
        u"""
        Devuelve los estados adyacentes en función de un estado dado.
        Fuente: http://stackoverflow.com/questions/2373306/pythonic-and-efficient-way-of-finding-adjacent-cells-in-grid

        :param x: Fila del estado
        :param y: Columna del estado
        """
        coordenadas = self.coordenadas
        vecinos = []
        vappend = vecinos.append

        for fila, columna in ((x + i, y + j)
                              for i in (-1, 0, 1) for j in (-1, 0, 1)
                              if i != 0 or j != 0):
            if (fila, columna) in coordenadas:
                vappend((fila, columna))
        if iterate:
            return iter(vecinos)
        else:
            return vecinos

    def matriz_q_to_string(self):
        u"""
        Devuelve un string representando la matriz Q en una estructura tabular.
        """
        matriz_q = self.get_matriz_q()
        return "\n".join(["| {0} |".format(" | ".join(j))
                          for j in [[str(i) for i in f]
                                    for f in matriz_q]])

    gamma = property(get_gamma, set_gamma, None, u"Propiedad Gamma de QLearning")
    tecnica = property(get_tecnica, set_tecnica, None, u"Propiedad Técnica de QLearning")
    matriz_q = property(get_matriz_q, None, None, "Propiedad Matriz Q")
    gridworld = property(get_gridworld, set_gridworld, None, "Propiedad GridWorld")
    coordenadas = property(get_coordenadas, None, None, "Propiedad Coordenadas")
    episodes = property(get_episodes, set_episodes, None, "Propiedad Episodios")


text:
import numpy as np
import pandas as pd
import sys
from tabulate import tabulate

def isValidPrior(data):
    if (data.values < 0).any():
        return False
    elif data.sum(axis=1)[0] != 1.0:
        return False
    return True

def isValidChannel(data):
    if (data.values < 0).any():
        return False
    elif (data.sum(axis=1) != 1.0).any():
        return False
    return True

def calculateJoint(filename):
    prior = getPrior(filename)
    channel = getChannel(filename)
    if prior is None or channel is None:
        return
    prior = pd.DataFrame(np.tile(prior,(len(channel.columns),1)))
    joint = channel
    joint = joint * prior.T
    return joint

def calculatePY(data):
    df = pd.DataFrame(data.sum())
    print("P(x|Y)")
    print(df.T)
    return df.T

def calculatePosterior(joint, py):
    py = 1.0 / py
    print("COEF")
    print(py)
    py = pd.DataFrame(np.tile(py,(len(joint.index),1)))
    posterior = joint * py
    return posterior

def calculateHyperMatrix(data):
    toDrop = []
    indexes = range(1,(len(data.columns)+1))
    titles = [("y" + str(i)) for i in indexes]
    visited = {}
    for t in indexes:
        visited[t] = False

    for name, values in data.iteritems():
        for i in range((int(name)+1),len(data.columns)):
            comparison = np.where(data[name] == data[i], 1, 0).sum()
            if visited[i] == True:
                continue
            if comparison == len(data.index):
                titles[name] = titles[name] + " or " + titles[i]
                toDrop.append(titles[i])
                visited[i] = True

    data.columns = titles
    data = data.drop(toDrop, axis=1)
    data.add_prefix('P(X|')
    data.add_suffix(')')
    return data

def calculateShannonEntropyPrior(data):
    logs = data.apply(lambda x: x * math.log(x))
    logs = (logs.T.sum() * -1)
    return logs

def calculateGuessingEntropyPrior(data):
    data = data.T.sort_values(by=[0], ascending=False).T
    n = len(data.columns)
    entropy = 0.0
    for i in range(0,n):
        entropy = entropy + ((i+1) * data.T.iloc[i][0])
    return entropy

def calculateBayesVulnerabilityPrior(data):
    return data.T.max()

def calculateProbNTriesPrior(n, prior):
    numRows = n.iloc[0][0]
    end = len(prior.columns)
    index = end - numRows

    ordered = prior.T.sort_values(by=[0])
    ordered = ordered.iloc[index:]
    return ordered.sum()

def calculateGVulnerabilityPrior(gfunc, prior):
    ncols = len(prior.columns)
    nrows = len(gfunc.index)

    maxim = 0.0
    for i in range(0, nrows):
        value = 0.0
        for j in range(0,ncols):
            value = value + gfunc.iloc[i, j] * prior.iloc[0, j]
        if maxim < value:
            maxim = value

    return maxim

def calculateOuter(data):
    df = pd.DataFrame(data.sum())
    return df.T

def applyShannon(val):
    if val == 0.0:
        return 0.0
    return val * math.log(val)

def calculateShannonEntropyPosterior(data, outer):
    logs = data.applymap(applyShannon)
    logs = logs.sum()
    entropy = logs * outer
    entropy = (entropy.T.sum() * -1)
    return entropy

def calculateGuessingEntropyByColumn(data, index):
    data = data.sort_values(by=[index], ascending=False)
    data.index = range(0,len(data.index))
    n = len(data.index)
    entropy = 0.0
    for i in range(0,n):
        entropy = entropy + ((i+1) * data.T.iloc[index][i])
    return entropy

def calculateGuessingEntropyPosterior(data, outer):
    ncols = len(data.columns)
    nrwos = len(data.index)

    entropy = 0.0
    for i in range(0,ncols):
        guessing = calculateGuessingEntropyByColumn(data, i)
        entropy = entropy + (guessing * outer[i])

    return entropy

def calculateBayesEntropyPosterior(data, outer):
    ncols = len(data.columns)
    nrwos = len(data.index)

    entropy = 0.0
    for i in range(0,ncols):
        bayes = data.T.iloc[i].max()
        entropy = entropy + (bayes * outer[i])

    return entropy

def calculateProbNTries(n, posterior, colID):
    numRows = n.iloc[0][0]
    end = len(posterior.columns)
    index = end - numRows

    ordered = posterior.T.sort_values(by=[colID])
    ordered.index = range(0, len(posterior.columns))
    ordered = ordered.iloc[index:]
    return ordered[colID].sum()

def calculateNTriesPosterior(posterior, outer, n):
    ncols = len(posterior.columns)

    entropy = 0.0
    for i in range(0, ncols):
        pNTries = calculateProbNTries(n, posterior.T, i)
        entropy = entropy + (pNTries * outer[i])

    return entropy

def calculateGVulnerabilityPosterior(posterior, outer, gainFunc):
    nColPost = len(posterior.columns)
    nRowPost = len(posterior.index)
    nColGain = len(gainFunc.columns)
    nColOuter = len(outer.columns)

    vulnerability = 0.0
    for i in range(0, nColPost):
        best = 0.0
        for k in range(0, nColGain):
            vuln = 0.0
            for j in range(0, nRowPost):
                vuln = vuln + posterior.iloc[j, i] * gainFunc.iloc[k, j]
            if vuln > best:
                best = vuln
        vulnerability = vulnerability + best * outer.iloc[0, i]

    return vulnerability

def calculateAdditiveLeakage(Gvprior, Gvpost):
    leak = Gvpost - Gvprior
    return leak

def calculateMultiplicativeLeakage(Gvprior, Gvpost):
    if Gvprior == 0.0:
        return 99999999.99999999
    leak = Gvpost / Gvprior
    return leak


text:
# -*- coding: utf-8 -*-
"""
Barycenters
===========

This example shows three methods to compute barycenters of time series.
For an overview over the available methods see the :mod:`tslearn.barycenters`
module.

*tslearn* provides three methods for calculating barycenters for a given set of
time series:

* *Euclidean barycenter* is simply the arithmetic mean for
  each individual point in time, minimizing the summed euclidean distance
  for each of them. As can be seen below, it is very different from the
  DTW-based methods and may often be inappropriate. However, it is the
  fastest of the methods shown.
* *DTW Barycenter Averaging (DBA)* is an iteratively refined barycenter,
  starting out with a (potentially) bad candidate and improving it
  until convergence criteria are met. The optimization can be accomplished
  with (a) expectation-maximization [1] and (b) stochastic subgradient
  descent [2]. Empirically, the latter "is [often] more stable and finds better
  solutions in shorter time" [2].
* *Soft-DTW barycenter* uses a differentiable loss function to iteratively
  find a barycenter [3]. The method itself and the parameter
  :math:`\\gamma=1.0` is described in more detail in the section on
  :ref:`DTW<dtw>`. There is also a dedicated
  :ref:`example<sphx_glr_auto_examples_plot_barycenter_interpolate.py>`
  available.

[1] F. Petitjean, A. Ketterlin & P. Gancarski. A global averaging method for
dynamic time warping, with applications to clustering. Pattern Recognition,
Elsevier, 2011, Vol. 44, Num. 3, pp. 678-693.

[2] D. Schultz & B. Jain. Nonsmooth Analysis and Subgradient Methods for
Averaging in Dynamic Time Warping Spaces. Pattern Recognition, 74, 340-358.

[3] M. Cuturi & M. Blondel. Soft-DTW: a Differentiable Loss Function for
Time-Series. ICML 2017.
"""

# Author: Romain Tavenard, Felix Divo
# License: BSD 3 clause

import numpy
import matplotlib.pyplot as plt

from tslearn.barycenters import \
    euclidean_barycenter, \
    dtw_barycenter_averaging, \
    dtw_barycenter_averaging_subgradient, \
    softdtw_barycenter
from tslearn.datasets import CachedDatasets

# fetch the example data set
numpy.random.seed(0)
X_train, y_train, _, _ = CachedDatasets().load_dataset("Trace")
X = X_train[y_train == 2]
length_of_sequence = X.shape[1]

def plot_helper(barycenter):
    # plot all points of the data set
    for series in X:
        plt.plot(series.ravel(), "k-", alpha=.2)
    # plot the given barycenter of them
    plt.plot(barycenter.ravel(), "r-", linewidth=2)

# plot the four variants with the same number of iterations and a tolerance of
# 1e-3 where applicable
ax1 = plt.subplot(4, 1, 1)
plt.title("Euclidean barycenter")
plot_helper(euclidean_barycenter(X))

plt.subplot(4, 1, 2, sharex=ax1)
plt.title("DBA (vectorized version of Petitjean's EM)")
plot_helper(dtw_barycenter_averaging(X, max_iter=50, tol=1e-3))

plt.subplot(4, 1, 3, sharex=ax1)
plt.title("DBA (subgradient descent approach)")
plot_helper(dtw_barycenter_averaging_subgradient(X, max_iter=50, tol=1e-3))

plt.subplot(4, 1, 4, sharex=ax1)
plt.title("Soft-DTW barycenter ($\gamma$=1.0)")
plot_helper(softdtw_barycenter(X, gamma=1., max_iter=50, tol=1e-3))

# clip the axes for better readability
ax1.set_xlim([0, length_of_sequence])

# show the plot(s)
plt.tight_layout()
plt.show()

max_stars_repo_name: johannfaouzi/tslearn

max_issues_repo_name: johannfaouzi/tslearn

max_forks_repo_name: johannfaouzi/tslearn


text:
import matplotlib.pyplot as plt
import numpy as np

data = np.genfromtxt('totalRewardsInEpochs.csv', delimiter='\t', skip_header=1, dtype=None)

rewards = data[:,0]
episodes = data[:,1]

plt.plot(episodes, rewards)
plt.title("Total Rewards Over Time")
plt.xlim(left=0, right=len(data))
plt.xlabel("Training Episodes")
plt.ylabel("Total Reward per Episode")
plt.show()


text:
# -*- coding: utf-8 -*-
"""
Unit tests for ConicSection

:author: Adrian Schlatter
"""

import unittest
from pyConic import ConicSection
import numpy as np

class Test_UnitCircle(unittest.TestCase):
    """Test unit-circle"""

    def setUp(self):
        self.A33 = [[1, 0], [0, 1]]
        self.v = [[0], [0]]
        self.F = -1
        self.cs = ConicSection(self.A33, self.v, self.F)

    def test_proper(self):
        """A circle is proper (non-degenerate)"""

        self.assertTrue(self.cs.isProper)
        self.assertFalse(self.cs.isDegenerate)

    def test_center(self):
        """This circle is centered at (0, 0)"""

        self.assertTrue((self.cs.center == np.array([[0], [0]])).all())

    def test_coefficients(self):
        """Test correctnes of unit-circle coefficients"""

        coefs = self.cs.coefficients

        self.assertEqual(coefs, (1, 0, 1, 0, 0, -1))

    def test_A33(self):
        """Test A33"""
        self.assertTrue((np.array(self.A33) == self.cs.A33).all())

    def test_v(self):
        """Test v"""
        self.assertTrue((np.array(self.v) == self.cs.v).all())

    def test_F(self):
        """Test F"""
        self.assertEqual(np.array(self.F), self.cs.F)

    def test_conicType(self):
        """Test that circle has conic-type 'circle'"""
        self.assertEqual('circle', self.cs.conicType)

    def test_central(self):
        """Unit circle is centered"""
        self.assertTrue(self.cs.isCentral)

    def test_standardForm(self):
        """Test correct standard form of unit circle"""

        a, b, t, M = self.cs.standardForm
        self.assertEqual((a, b, t[0, 0], t[1, 0]), (1., 1., 0., 0.))

    def test_centeredEquation(self):
        """Test centered equation of unit circle"""

        A33, K = self.cs.centeredEquation
        self.assertTrue((self.A33 == A33).all())
        self.assertEqual(K, 1.)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: adrianschlatter/pyConic

max_issues_repo_name: adrianschlatter/pyConic

max_forks_repo_name: adrianschlatter/pyConic