text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 12 12:35:01 2018

@author: abhijay
"""

import numpy as np
import pandas as pd

from sklearn import preprocessing
from sklearn import tree

from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import BaggingClassifier

import matplotlib.pyplot as plt
import matplotlib

font = {'size'   : 14}
matplotlib.rc('font', **font)

#import os
#os.chdir('/home/abhijay/Documents/ML/hw_3/Q_6/')

##### For making the target variable binary #####
def make_target_variable(data):
    data['salary-bracket'] = data['salary-bracket'].apply(lambda y: 0 if y==" <=50K" else 1)
    return data

#### Sepatate categorical values from continuous for label encoding of the categorical data #####
def find_categorical_continuous_features(data):
    categorical_features = [data.columns[col] for col, col_type in enumerate(data.dtypes) if col_type == np.dtype('O') ]
    continuous_features = list(set(data.columns) - set(categorical_features))
    return categorical_features, continuous_features

def plot_results(train_accuracy, dev_accuracy, test_accuracy, x_label, experiment_name, saveAs):
    w = 0.27
    ind = np.arange(len(train_accuracy))
    fig, ax = plt.subplots(figsize=(20, 8))
    dat = np.array([train_accuracy,dev_accuracy,test_accuracy]).T
    ax.bar(ind, dat[:,0], width=w, label = 'Train Accuracy')
    ax.bar(ind+w, dat[:,1], width=w, label = 'Dev Accuracy')
    ax.bar(ind+w*2, dat[:,2], width=w, label = 'Test Accuracy')
    ax.set_ylabel( 'Accuracy', fontsize=15)
    ax.set_xlabel( x_label, fontsize=15)
    ax.set_xticks(ind+w)
    ax.set_xticklabels(experiment_name)
    ax.legend(loc='upper right')
    ax.set_ylim(0, 110)
    ax.set_title(saveAs)
    plt.xticks(rotation=70)
    plt.gcf().subplots_adjust(bottom=0.2)
    fig.savefig('plots/'+saveAs)

if __name__ == "__main__":

    print ("\n=============== Ans 6 ===============\n")

    col_names = ["age","workclass","education","marital-status","occupation","race","gender","hours-per-week","native-country","salary-bracket"]

    ##### Load data #####
    train_data = pd.read_csv("income-data/income.train.txt", names = col_names)
    dev_data = pd.read_csv("income-data/income.dev.txt", names = col_names)
    test_data = pd.read_csv("income-data/income.test.txt", names = col_names)

    train_data = make_target_variable(train_data)
    test_data = make_target_variable(test_data)
    dev_data = make_target_variable(dev_data)

    categorical_features_, continuous_features_ = find_categorical_continuous_features(train_data.iloc[:,0:-1])

    categorical_features = [train_data.columns.get_loc(c) for c in categorical_features_]

    continuous_features = [train_data.columns.get_loc(c) for c in continuous_features_]

    ##### Encoding categorical values to labels #####
    le = preprocessing.LabelEncoder()
    all_df = pd.concat([train_data,test_data,dev_data])
    for feature in categorical_features_:
        le.fit(all_df[feature])
        train_data[feature] = le.transform(train_data[feature])
        test_data[feature] = le.transform(test_data[feature])
        dev_data[feature] = le.transform(dev_data[feature])

    featuresUniqueValues = [train_data[col].unique() for col in col_names]

    ##### Convert pandas dataframe to numpy array #####
    x = train_data.iloc[:,0:train_data.shape[1]-1].values
    y = (train_data.values)[:,-1]

    x_test = test_data.iloc[:,0:test_data.shape[1]-1].values
    y_test = (test_data.values)[:,-1]

    x_dev = dev_data.iloc[:,0:dev_data.shape[1]-1].values
    y_dev = (dev_data.values)[:,-1]

    ##### Bagging #####
    print ("\n=============== Bagging ===============\n")
    train_accuracy = []
    dev_accuracy = []
    test_accuracy = []
    experiment_name = []

    for maxDepthOfTree in [1,2,3,5,10]:
        for noOfTrees in [10, 20, 40, 60, 80, 100]:
            print ("\n\n---------------")
            print ("maxDepthOfTree: ",maxDepthOfTree)
            print ("noOfTrees: ",noOfTrees)
            print ("---------------")
            clf = BaggingClassifier(tree.DecisionTreeClassifier(max_depth=maxDepthOfTree),n_estimators=noOfTrees)

            clf = clf.fit(x, y)

            train_accuracy_ = round(100*np.sum(y == clf.predict(x))/x.shape[0],2)
            dev_accuracy_ = round(100*np.sum(y_dev == clf.predict(x_dev))/x_dev.shape[0],2)
            test_accuracy_ = round(100*np.sum(y_test == clf.predict(x_test))/x_test.shape[0],2)

            train_accuracy.append(train_accuracy_)
            dev_accuracy.append(dev_accuracy_)
            test_accuracy.append(test_accuracy_)
            experiment_name.append("("+str(maxDepthOfTree)+','+str(noOfTrees)+")")

            print ("\nTraining Accuracy: "+str( train_accuracy_)+"%")
            print ("\nDev Accuracy: "+str( dev_accuracy_)+"%")
            print ("\nTesting Accuracy: "+str( test_accuracy_)+"%")

    saveAs = 'Bagging - Accuracy Plot'
    xlabel = '(Max depth of tree, No. of boosting iterations)' ##### No. of boosting iterations is the same as n_estimators or number of trees
    plot_results(train_accuracy, dev_accuracy, test_accuracy, xlabel, experiment_name, saveAs)

    ##### Boosting #####
    print ("\n=============== Boosting ===============\n")
    train_accuracy = []
    dev_accuracy = []
    test_accuracy = []
    experiment_name = []

    for maxDepthOfTree in [1,2,3]:
        for noOfTrees in [10, 20, 40, 60, 80, 100]:
            ##### Boosting iteration is the same as

            print ("\n\n---------------")
            print ("maxDepthOfTree: ",maxDepthOfTree)
            print ("noOfTrees: ",noOfTrees)
            print ("---------------")
            clf = AdaBoostClassifier(tree.DecisionTreeClassifier(max_depth=maxDepthOfTree),n_estimators=noOfTrees)

            clf = clf.fit(x, y)

            train_accuracy_ = round(100*np.sum(y == clf.predict(x))/x.shape[0],2)
            dev_accuracy_ = round(100*np.sum(y_dev == clf.predict(x_dev))/x_dev.shape[0],2)
            test_accuracy_ = round(100*np.sum(y_test == clf.predict(x_test))/x_test.shape[0],2)

            train_accuracy.append(train_accuracy_)
            dev_accuracy.append(dev_accuracy_)
            test_accuracy.append(test_accuracy_)
            experiment_name.append("("+str(maxDepthOfTree)+','+str(noOfTrees)+")")

            print ("\nTraining Accuracy: "+str( train_accuracy_)+"%")
            print ("\nDev Accuracy: "+str( dev_accuracy_)+"%")
            print ("\nTesting Accuracy: "+str( test_accuracy_)+"%")

    saveAs = 'Boosting - Accuracy Plot'
    xlabel = '(Max depth of tree, No. of trees)'
    plot_results(train_accuracy, dev_accuracy, test_accuracy, xlabel, experiment_name, saveAs)

max_stars_repo_name: abhijayghildyal/BaggingBoosting-scikit

max_issues_repo_name: abhijayghildyal/BaggingBoosting-scikit

max_forks_repo_name: abhijayghildyal/BaggingBoosting-scikit


text:
import chainer
import numpy as np
from chainer.dataset import dataset_mixin

class STL10Dataset(dataset_mixin.DatasetMixin):
    def __init__(self, test=False):
        from argparser import args
        images = np.load(args.data_dir)
        labels = np.zeros([images.shape[0], 1])  # no use,
        self.dset = (images, labels)
        print("load stl-10.  shape: ", len(self.dset[0]))

    def __len__(self):
        return len(self.dset[0])

    def get_example(self, i):
        image = np.asarray(self.dset[0][i] / 128. - 1., np.float32)
        image += np.random.uniform(size=image.shape, low=0., high=1. / 128)
        return image, self.dset[1][i]

ext: py

reason: import numpy


text:
#These imports are used for the dataset because torch has a better implementation of getting the data
from torch.utils.data import dataloader, TensorDataset
from torchvision import transforms, datasets
import torch

#These imports are for the network itself
import tensorflow as tf
from tensorflow import nn, layers
import numpy as np
#to show test images
import matplotlib.pyplot as plt
#personal files
from ops import *

def getCifarDataset():
    compose = transforms.Compose([
        transforms.Resize(64),
        transforms.ToTensor(), #converts an image from 0 -> 255 to 0 -> 1 and into a floatTensor of channels x height x width
        transforms.Normalize((.5, .5, .5), (.5, .5, .5))])#normalizes between -1 and 1 because that what generator outputs
        #works because subtracts each rgb channel by .5 and divides by .5 so least is -1 and most is 1
    return datasets.CIFAR10(root="./dataset", train=True, transform=compose, download=True)

def getCustomDataset(file):
    return TensorDataset(torch.from_numpy(loadImages(file)))

"""Load and Prepare Data"""
dataset = getCustomDataset("ImagesX64.npy")
batchSize = 256
dataLoader = dataloader.DataLoader(dataset, batch_size=batchSize, shuffle=True)
numBatches = len(dataLoader)
imageShape = (64, 64, 3)
noiseLength = 100
numEpochs = 200
#normalize randomness
tf.set_random_seed(7)

"""Models"""
#takes image x and ouputs a value between 0 and 1 where 0 is fake and 1 is real
def discriminator(x):#might be too powerful, already lowered learning rate, but might need to add dropout also
    with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE):
        with tf.variable_scope("convolutional_layer_1"):
            x = convolutLayer(x, 128)#3x64x64 -> 128x32x32
            #don't batch normalize in first layer of discriminator
            x = nn.leaky_relu(x, alpha=0.2)
        with tf.variable_scope("convolutional_layer_2"):
            x = convolutLayer(x, 256)#128x32x32 -> 256x16x16
            x = layers.batch_normalization(x, training=True)
            x = nn.leaky_relu(x, alpha=0.2)
        with tf.variable_scope("convolutional_layer_3"):
            x = convolutLayer(x, 512)#256x16x16 -> 512x8x8
            x = layers.batch_normalization(x, training=True)
            x = nn.leaky_relu(x, alpha=0.2)
        with tf.variable_scope("convolution_layer_4"):
            x = convolutLayer(x, 1024)#512x8x8 -> 1024x4x4
            x = layers.batch_normalization(x, training=True)
            x = nn.leaky_relu(x, alpha=0.2)
        with tf.variable_scope("flatten"):
            x = tf.reshape(x, (-1, 4, 4, 1024))#from 3d object 1024x4x4
            logits = layers.dense(x, units=1, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer())#1024x4x4 to a shape of 1 to sigmoid
        with tf.variable_scope("output"):
            out = nn.sigmoid(x)
    return out, logits #needed for loss function otherwise you double sigmoid

def generator(z):
    with tf.variable_scope("generator"):
        with tf.variable_scope("reshape_and_flatten"):
            z = layers.dense(inputs=z, units=4*4*1024)#flatten
            z = tf.reshape(z, (-1, 4, 4, 1024))#reshape noise
        with tf.variable_scope("deconvolutional_layer_1"):
            z = deconvolutLayer(z, 512)#1024x4x4 -> 512x8x8
            z = layers.batch_normalization(z, training=True)
            z = nn.relu(z)
        with tf.variable_scope("deconvolutional_layer_2"):
            z = deconvolutLayer(z, 256)#512x8x8 -> 256x16x16
            z = layers.batch_normalization(z, training=True)
            z = nn.relu(z)
        with tf.variable_scope("deconvolutional_layer_3"):
            z = deconvolutLayer(z, 128)#256x16x16 -> 128x32x32
            z = layers.batch_normalization(z, training=True)
            z = nn.relu(z)
        with tf.variable_scope("deconvolutional_layer_4"):
            z = deconvolutLayer(z, 3)#128x32x32 -> 3x64x64
            #no batch normalization in last layer of generatror
            #don't use relu for output
        with tf.variable_scope("output"):
            out = nn.tanh(z)
            #output to show it off
            tf.summary.image("Generated Images", out, max_outputs=16)
    return out

#Placeholders
#makes input into discriminator a 4d array where it is array of 3d arrays to represent images
x = tf.placeholder(dtype=tf.float32, shape=(None, ) + imageShape, name="Images")
#noise unkown length for unkown number of images to be made
z = tf.placeholder(dtype=tf.float32, shape=(None, noiseLength), name="Noise")

#Make models and set to use gpu
#test if gpu is available
device = ""
if tf.test.is_gpu_available():
    device = "/gpu:0"
else:
    device = "/cpu:0"
with tf.device(device):
    generatorSamples = generator(z)#make generator with z (noise) placeholder
    discriminatorReal, discriminatorRealLogits = discriminator(x)#make discriminator that takes data from the real
    discriminatorFake, discriminatorFakeLogits = discriminator(generatorSamples)#make discriminator that takes fake data

discriminatorLossReal = tf.reduce_mean(
                           nn.sigmoid_cross_entropy_with_logits(
                               logits=discriminatorRealLogits, labels=tf.ones_like(discriminatorRealLogits),
                               name="discriminator_loss_real"
                               #takes real input and makes the labels 1 or real because it wants to identify real data as real
                           )
                        )

discriminatorLossFake = tf.reduce_mean(
                           nn.sigmoid_cross_entropy_with_logits(
                               logits=discriminatorFakeLogits, labels=tf.zeros_like(discriminatorFakeLogits),
                               name="discriminator_loss_fake"
                               #takes fake input and makes the labels 0 or fake because it wants to identify fake data as fake
                           )
                        )

generatorLoss = tf.reduce_mean(
                           nn.sigmoid_cross_entropy_with_logits(
                               logits=discriminatorFakeLogits, labels=tf.ones_like(discriminatorFakeLogits),
                               name="generator_loss"
                               #takes fake input and makes the labels 1 or real because generator wants to make its fake dat seem more real
                           )
                        )

discriminatorTotalLoss = discriminatorLossReal + discriminatorLossFake

#Optimzer setup
trainableVariables = tf.trainable_variables()
#seperate trainable variables into ones for discriminator and generator
dTrainableVariables = [var for var in trainableVariables if "discriminator" in var.name]
gTrainableVariables = [var for var in trainableVariables if "generator" in var.name]

#build adam optimizers. paper said to use .0002. discriminator a tad strong so used .0001
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
    discriminatorOptimizer = tf.train.AdamOptimizer(0.00015).minimize(discriminatorTotalLoss, var_list=dTrainableVariables)
    generatorOptimizer = tf.train.AdamOptimizer(0.0002).minimize(generatorLoss, var_list=gTrainableVariables)

#config for session with multithreading, but limit to 3 of my 4 CPUs (tensor uses all by default: https://stackoverflow.com/questions/38836269/does-tensorflow-view-all-cpus-of-one-machine-as-one-device)
config = tf.ConfigProto(intra_op_parallelism_threads=2, inter_op_parallelism_threads=1, allow_soft_placement=True, log_device_placement=True)

#Test noise
testNoise = noise(16)

#Saver for when stuff goes wrong
saver = tf.train.Saver()

with tf.Session(config=config) as sess:
    sess.run(tf.global_variables_initializer())
    for epoch in range(numEpochs):
        for numBatch, realData in enumerate(dataLoader):

            realData = realData[0].numpy()#convert to numpy array

            #Train Discriminator and Generator. The values are none, the loss, an array of all real prediction, an array of all fake predictions
            #none, and generator loss
            _, dLoss, dRealPred, dFakePred, _, gLoss = sess.run([discriminatorOptimizer, discriminatorTotalLoss,
                                                                discriminatorReal, discriminatorFake, generatorOptimizer,
                                                                generatorLoss],
                                                                feed_dict={ x : realData, z : noise(batchSize) })
            #print losses
            print('Epoch: [{}/{}], Batch Num: [{}/{}]'.format(epoch, numEpochs, numBatch, numBatches))
            print('Discriminator Loss: {:.4f}, Generator Loss: {:.4f}'.format(dLoss, gLoss))
            print('D(x): {:.4f}, D(G(z)): {:.4f}'.format(dRealPred.mean(), dFakePred.mean()))

            #Save model and data and log predictions
            if numBatch % 100 == 0:
                saver.save(sess, "./model/DCGAN_Epoch_%s_Batch_%s.ckpt" % (epoch, numBatch))
                #show most recent with model save move out of this if you want constant checking
                testImages = sess.run(generatorSamples, feed_dict={ z : testNoise })
                #images are height, width, rgb value
                testImages = denormalize(testImages)
                for i in range(1, len(testImages)+1):
                    plt.subplot(4, 4, i)
                    plt.imshow(testImages[i-1])
                plt.pause(1)

max_stars_repo_name: esslushy/AbstractArtGenerator

max_issues_repo_name: esslushy/AbstractArtGenerator

max_forks_repo_name: esslushy/AbstractArtGenerator


text:
import logging
from typing import Dict, List

import networkx as nx

class LineageGraph:
    def __init__(
        self,
        nodes: List[Dict[str, str]],
        edges: List[Dict[str, str]],
        name: str = "Lineage",
    ):
        self.name = name
        self._graph = nx.DiGraph()
        for node in nodes:
            node_id = node["id"]
            node_attributes = {"name": node["name"], "type": node["type"]}
            logging.debug("Add Node: {}, {}".format(node_id, node_attributes))
            self._graph.add_node(node_id, **node_attributes)

        for edge in edges:
            logging.debug("Edge: <{}>, <{}>".format(edge["source"], edge["target"]))
            self._graph.add_edge(edge["source"], edge["target"])

    @property
    def graph(self):
        return self._graph

    @graph.setter
    def graph(self, new_graph):
        self._graph = new_graph

max_stars_repo_name: vrajat/data-lineage

max_issues_repo_name: vrajat/data-lineage

max_forks_repo_name: vrajat/data-lineage

reason: import networkx


text:
import pandas as pd
import numpy as np

INPUT_PATH = "./output" # Do I have to change to "./output" if called from feature_extraction.sh -> YES
OUTPUT_PATH = "./output" # Run as "../output" when running as an individual script, not as part of feature_extraction.sh
INPUT_CSV = "standardised_features.csv"
OUTPUT_CSV = "one_hot_encoded_features.csv"
OUTPUT_NUMPY = "one_hot_encoded"

class OneHotEncoding:

    def __init__(self, input_path, input_csv):
        self.input_path = input_path
        self.input_csv = input_csv

        df_standardised_features = pd.read_csv(self.input_path + '/' + self.input_csv)

        # One hot encode CID column
        df = pd.get_dummies(df_standardised_features['CID'])

        df_join = pd.concat([df, df_standardised_features], axis=1, join='inner') # Join the databases
        df_join.drop('CID', axis=1, inplace=True)

        df_join.to_csv(OUTPUT_PATH + '/' + OUTPUT_CSV, encoding='utf-8', index=False, header=False) # Save joined data base to a csv file
        one_hot_encoded = df_join.to_numpy() # Convert pandas to numpy array
        np.save(OUTPUT_PATH + '/' + OUTPUT_NUMPY, one_hot_encoded) # Save numpy array to disk

def main():
    OneHotEncoding(INPUT_PATH, INPUT_CSV)

if __name__ == '__main__':
    main()


text:
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

import math
from collections import defaultdict
from typing import Any, Dict

import networkx as nx

def _modularity_component(
    intra_community_degree: float,
    total_community_degree: float,
    network_degree_sum: float,
    resolution: float,
) -> float:
    community_degree_ratio = math.pow(total_community_degree, 2.0) / (
        2.0 * network_degree_sum
    )
    return (intra_community_degree - resolution * community_degree_ratio) / (
        2.0 * network_degree_sum
    )

def _assertions(
    graph: nx.Graph,
    partitions: Dict[Any, int],
    weight_attribute: str,
    resolution: float,
):
    if not isinstance(graph, nx.Graph):
        raise TypeError("graph must be a networkx undirected graph")
    if graph.is_directed():
        raise ValueError("The graph must be an undirected graph")
    if graph.is_multigraph():
        raise ValueError(
            "Multigraphs must be provided in the form of a non multigraph."
        )
    if not nx.is_weighted(graph, weight=weight_attribute):
        raise ValueError(
            f"weight_attribute {weight_attribute} not found on every edge in the provided graph"
        )
    if not isinstance(partitions, dict):
        raise TypeError("partitions must be a dictionary")
    if not isinstance(resolution, float):
        raise TypeError("resolution must be a float")

def modularity(
    graph: nx.Graph,
    partitions: Dict[Any, int],
    weight_attribute: str = "weight",
    resolution: float = 1.0,
) -> float:
    """
    Given an undirected graph and a dictionary of vertices to community ids, calculate
    the modularity.

    Parameters
    ----------
    graph : nx.Graph
        An undirected graph
    partitions : Dict[Any, int]
        A dictionary representing a community partitioning scheme with the keys being
        the vertex and the value being a community id.
    weight_attribute : str
        The edge data attribute on the graph that contains a float weight for the edge.
    resolution : float
        The resolution to use when calculating the modularity.

    Returns
    -------
    float
                The sum of the modularity of each of the communities.

    Raises
    ------
    TypeError
        If ``graph`` is not a networkx Graph or
        If ``partitions`` is not a dictionary or
        If ``resolution`` is not a float
    ValueError
        If ``graph`` is unweighted
        If ``graph`` is directed
        If ``graph`` is a multigraph

    References
    ----------
    .. [1] https://en.wikipedia.org/wiki/Modularity_(networks)
    """
    _assertions(graph, partitions, weight_attribute, resolution)

    components = modularity_components(graph, partitions, weight_attribute, resolution)

    return sum(components.values())

def modularity_components(
    graph: nx.Graph,
    partitions: Dict[Any, int],
    weight_attribute: str = "weight",
    resolution: float = 1.0,
) -> Dict[int, float]:
    """
    Given an undirected, weighted graph and a community partition dictionary,
    calculates a modularity quantum for each community ID. The sum of these quanta
    is the modularity of the graph and partitions provided.

    Parameters
    ----------
    graph : nx.Graph
        An undirected graph
    partitions : Dict[Any, int]
        A dictionary representing a community partitioning scheme with the keys being
        the vertex and the value being a community id.
    weight_attribute : str
        The edge data attribute on the graph that contains a float weight for the edge.
    resolution : float
        The resolution to use when calculating the modularity.

    Returns
    -------
    Dict[int, float]
        A dictionary of the community id to the modularity component of that community

    Raises
    ------
    TypeError
        If ``graph`` is not a networkx Graph or
        If ``partitions`` is not a dictionary or
        If ``resolution`` is not a float
    ValueError
        If ``graph`` is unweighted
        If ``graph`` is directed
        If ``graph`` is a multigraph
    """
    _assertions(graph, partitions, weight_attribute, resolution)

    total_edge_weight = 0.0

    communities = set(partitions.values())

    degree_sums_within_community: Dict[int, float] = defaultdict(lambda: 0.0)
    degree_sums_for_community: Dict[int, float] = defaultdict(lambda: 0.0)
    for vertex, neighbor_vertex, weight in graph.edges(data=weight_attribute):
        vertex_community = partitions[vertex]
        neighbor_community = partitions[neighbor_vertex]
        if vertex_community == neighbor_community:
            if vertex == neighbor_vertex:
                degree_sums_within_community[vertex_community] += weight
            else:
                degree_sums_within_community[vertex_community] += weight * 2.0
        degree_sums_for_community[vertex_community] += weight
        degree_sums_for_community[neighbor_community] += weight
        total_edge_weight += weight

    return {
        comm: _modularity_component(
            degree_sums_within_community[comm],
            degree_sums_for_community[comm],
            total_edge_weight,
            resolution,
        )
        for comm in communities
    }

max_stars_repo_name: daxpryce/graspologic

max_issues_repo_name: daxpryce/graspologic

max_forks_repo_name: daxpryce/graspologic


text:
import numpy as np
import bokeh.plotting
import bokeh.layouts
import bokeh.models
figure = bokeh.plotting.figure()
btn = bokeh.models.Button()

def pixels(N, M):
    # image (i, j) maps to (y, x) in real space
    container = np.empty((M, N), dtype=np.uint32)
    view = container.view(dtype=np.uint8).reshape(M, N, 4)
    alpha = 255
    for i in range(M):
        if i % 2 == 0:
            red = 158
        else:
            red = 0
        for j in range(N):
            if (i*M + j) % 2 == 0:
                green = int((i / N) * 255)
                blue = int((j / M) * 255)
            else:
                green = int((j / M) * 255)
                blue = int((i / N) * 255)
            view[i, j, 0] = red
            view[i, j, 1] = green
            view[i, j, 2] = blue
            view[i, j, 3] = alpha
    return container

N, M = 1800, 900
rgba = pixels(N, M)
print("finished generating pixels: {} x {}".format(N, M))
source = bokeh.models.ColumnDataSource({
    "x": [-19],
    "y": [-13],
    "dw": [72],
    "dh": [36],
    "image": [rgba],
})
figure.image_rgba(
        x="x",
        y="y",
        dw="dw",
        dh="dh",
        image="image",
        source=source)
document = bokeh.plotting.curdoc()
document.add_root(bokeh.layouts.column(figure, btn))

max_stars_repo_name: andrewgryan/bokeh-playground

max_issues_repo_name: andrewgryan/bokeh-playground

max_forks_repo_name: andrewgryan/bokeh-playground


text:
#*******************************************************************************
# Copyright 2020 Intel Corporation
# All Rights Reserved.
#
# This software is licensed under the Apache License, Version 2.0 (the
# "License"), the following terms apply:
#
# You may not use this file except in compliance with the License.  You may
# obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#*******************************************************************************

# daal4py SVM example for shared memory systems

import daal4py as d4p
import numpy as np
import os
from daal4py.oneapi import sycl_buffer

# let's try to use pandas' fast csv reader
try:
    import pandas
    read_csv = lambda f, c, t=np.float64: pandas.read_csv(f, usecols=c, delimiter=',', header=None, dtype=t)
except:
    # fall back to numpy loadtxt
    read_csv = lambda f, c, t=np.float64: np.loadtxt(f, usecols=c, delimiter=',', ndmin=2)

try:
    from dpctx import device_context, device_type
    with device_context(device_type.gpu, 0):
        gpu_available=True
except:
    try:
        from daal4py.oneapi import sycl_context
        with sycl_context('gpu'):
            gpu_available=True
    except:
        gpu_available=False

# Commone code for both CPU and GPU computations
def compute(train_indep_data, train_dep_data, test_indep_data, method='defaultDense'):
    # Configure a SVM object to use linear kernel
    kernel_function = d4p.kernel_function_linear(method='defaultDense', k=1.0, b=0.0)
    train_algo = d4p.svm_training(method=method, kernel=kernel_function, C=1.0, accuracyThreshold=1e-3, tau=1e-8, cacheSize=600000000)

    train_result = train_algo.compute(train_indep_data, train_dep_data)

    # Create an algorithm object and call compute
    predict_algo = d4p.svm_prediction(kernel=kernel_function)
    predict_result = predict_algo.compute(test_indep_data, train_result.model)
    decision_result = predict_result.prediction
    predict_labels = np.where(decision_result >=0, 1, -1)
    return predict_labels, decision_result

# At this moment with sycl we are working only with numpy arrays
def to_numpy(data):
    try:
        from pandas import DataFrame
        if isinstance(data, DataFrame):
            return np.ascontiguousarray(data.values)
    except:
        pass
    try:
        from scipy.sparse import csr_matrix
        if isinstance(data, csr_matrix):
            return data.toarray()
    except:
        pass
    return data

def main(readcsv=read_csv):
    # input data file
    train_file = os.path.join('..', 'data', 'batch', 'svm_two_class_train_dense.csv')
    predict_file = os.path.join('..', 'data', 'batch', 'svm_two_class_test_dense.csv')

    nFeatures = 20
    train_data = readcsv(train_file, range(nFeatures))
    train_labels = readcsv(train_file, range(nFeatures, nFeatures + 1))
    predict_data = readcsv(predict_file, range(nFeatures))
    predict_labels = readcsv(predict_file, range(nFeatures, nFeatures + 1))

    predict_result_classic, decision_function_classic = compute(train_data, train_labels, predict_data, 'boser')

    train_data = to_numpy(train_data)
    train_labels = to_numpy(train_labels)
    predict_data = to_numpy(predict_data)

    try:
        from dpctx import device_context, device_type
        gpu_context = lambda: device_context(device_type.gpu, 0)
    except:
        from daal4py.oneapi import sycl_context
        gpu_context = lambda: sycl_context('gpu')

    # It is possible to specify to make the computations on GPU
    if gpu_available:
        with gpu_context():
            sycl_train_data = sycl_buffer(train_data)
            sycl_train_labels = sycl_buffer(train_labels)
            sycl_predict_data = sycl_buffer(predict_data)

            predict_result_gpu, decision_function_gpu = compute(sycl_train_data, sycl_train_labels, sycl_predict_data, 'thunder')
            assert np.allclose(predict_result_gpu, predict_result_classic)

    return predict_labels, predict_result_classic, decision_function_classic

if __name__ == "__main__":
    predict_labels, predict_result, decision_function = main()
    np.set_printoptions(precision=0)
    print("\nSVM classification decision function (first 10 observations):\n", decision_function[0:10])
    print("\nSVM classification predict result (first 10 observations):\n", predict_result[0:10])
    print("\nGround truth (first 10 observations):\n", predict_labels[0:10])
    print('All looks good!')

reason: import numpy,from scipy


text:
import numpy as np
import cudamat as cm
import pickle
import random
import pdb
import logging

class SGD:

    def __init__(self,model,maxBatch,alpha=1e-2,optimizer='nesterov',
                 momentum=0.9, maxGradNorm=1500):
        self.model = model
        self.maxBatch = maxBatch
        self.it = 0
        self.momentum = momentum # momentum
        self.alpha = alpha # learning rate
        self.optimizer = optimizer
        self.maxGNorm = maxGradNorm # gradient clip norm value

        if self.optimizer == 'nesterov':
            self.velocity = [[cm.CUDAMatrix(np.zeros(w.shape)),
                              cm.CUDAMatrix(np.zeros(b.shape))]
                              for w,b in self.model.stack]
        elif self.optimizer == 'adagrad':
            # TODO Do toFile and fromFile for adagrad
            assert False
            epsilon = 0.0
            self.gradt = [[cm.CUDAMatrix(epsilon+np.zeros(w.shape)),
                           cm.CUDAMatrix(epsilon+np.zeros(b.shape))]
                           for w,b in self.model.stack]

        self.costt = []
        self.expcost = []
        self.regcost = []

    def toFile(self, fid):
        stack = []
        for w, b in self.velocity:
            w.copy_to_host()
            b.copy_to_host()
            stack.append([w.numpy_array, b.numpy_array])
        pickle.dump([self.it, self.costt, self.expcost, stack], fid)

    def fromFile(self, fid):
        # TODO it, costt, expcost, velocity
        params = pickle.load(fid)
        it, costt, expcost, stack = params
        self.it = it
        self.costt = costt
        self.expcost = expcost
        for (w, b), (wi, bi) in zip(self.velocity, stack):
            w.numpy_array[:] = wi[:]
            b.numpy_array[:] = bi[:]
            w.copy_to_device()
            b.copy_to_device()

    def run(self,data_dict,alis,keys,sizes):
        """
        Runs stochastic gradient descent with nesterov acceleration.
        Model is objective.
        """

        # momentum setup
        momIncrease = 10
        mom = 0.5

        # randomly select minibatch
       	random.shuffle(keys)

        for k in keys:
            self.it += 1

            if self.it > momIncrease:
                mom = self.momentum

            mb_data = data_dict[k]
            if mb_data.shape[1] > self.maxBatch:
                logging.info("SKIPPING utt exceeds batch length\
                        (Utterance length %d)." % mb_data.shape[1])
                continue

            mb_labels = np.array(alis[k],dtype=np.int32)

            if mb_data.shape[1] < mb_labels.shape[0]:
                logging.info("SKIPPING utt frames less than label length "
                       "(Utterance length %d, Num Labels %d)."
                       ""%(mb_data.shape[1],mb_labels.shape[0]))
                continue

            if self.optimizer == 'nesterov':
                # w = w+mom*velocity (evaluate gradient at future point)
                self.model.updateParams(mom,self.velocity)

            cost,grad,skip = self.model.costAndGrad(mb_data,mb_labels)

            if self.optimizer == 'nesterov':
                # undo update
                # w = w-mom*velocity
                self.model.updateParams(-mom,self.velocity)

            # Compute norm of all parameters as one vector
            gnorm = 0.0
            for dw,db in grad:
                gnorm += dw.euclid_norm()**2
                gnorm += db.euclid_norm()**2
            gnorm = np.sqrt(gnorm)

	    if skip:
		logging.info("SKIPPING: Key=%s, Cost=%f, SeqLen=%d, NumFrames=%d."%(k, cost,mb_labels.shape[0],mb_data.shape[1]))
		continue

            if np.isfinite(cost):
                # compute exponentially weighted cost
                if self.it > 1:
                    self.expcost.append(.01*cost + .99*self.expcost[-1])
                else:
                    self.expcost.append(cost)
                self.costt.append(cost)

                if self.model.reg > 0.0:
                    rc = self.model.regcost
                    if len(self.regcost) > 0:
                        self.regcost.append(0.01*rc + 0.99*self.regcost[-1])
                    else:
                        self.regcost.append(rc)

            # velocity = mom*velocity - alpha*grad
            if self.optimizer == 'nesterov':
                alph = self.alpha
                if gnorm > self.maxGNorm:
                    alph *= (self.maxGNorm/gnorm)
                for vs,gs in zip(self.velocity,grad):
                    vw,vb = vs
                    dw,db = gs
                    vw.mult(mom)
                    vb.mult(mom)
                    vw.add_mult(dw,alpha=-alph)
                    vb.add_mult(db,alpha=-alph)
                update = self.velocity
                scale = 1.0

            elif self.optimizer == 'adagrad':
                delta = 1e-10
                for gts,gs in zip(self.gradt,grad):
                    dwt,dbt = gts
                    dw,db = gs
                    gamma = 1. - 1./(1e-2*self.it+1)
                    cm.add_pow(dwt,dw,2,alpha=gamma,target=dwt)
                    cm.add_pow(dbt,db,2,alpha=gamma,target=dbt)
                    dwt.add(delta,target=dwt)
                    dbt.add(delta,target=dbt)
                    cm.mult_pow(dw,dwt,-0.5,target=dw)
                    cm.mult_pow(db,dbt,-0.5,target=db)
                    dwt.add(-delta,target=dwt)
                    dbt.add(-delta,target=dbt)
                update = grad
                scale = -self.alpha

	    # update params
	    self.model.updateParams(scale,update)

            if self.it%1 == 0:
                print ("Iter %d : Cost=%.4f, ExpCost=%.4f, GradNorm=%.4f, "
                       "SeqLen=%d, NumFrames=%d.")%(self.it,cost,
                       self.expcost[-1],gnorm,mb_labels.shape[0],
                       mb_data.shape[1])

max_stars_repo_name: SrikarSiddarth/stanford-ctc

max_issues_repo_name: guker/stanford-ctc

max_forks_repo_name: guker/stanford-ctc


text:
from absl.testing import absltest
from absl.testing import parameterized

import numpy as onp
from numpy import testing

from fax import converge
from fax import loop
from fax import test_util

import jax
import jax.numpy as np
import jax.test_util
from jax.config import config
config.update("jax_enable_x64", True)

class LoopTest(jax.test_util.JaxTestCase):

    @parameterized.parameters(
        {"unroll": True, "jit": False},
        {"unroll": False, "jit": False},
        {"unroll": True, "jit": True},
        {"unroll": False, "jit": True},
    )
    def testFixedPointIteration(self, unroll, jit):
        mat_size = 5
        rtol = atol = 1e-10
        max_steps = 500

        matrix = test_util.generate_stable_matrix(mat_size, eps=1e-1)
        offset = onp.random.rand(5)
        init_x = np.zeros_like(offset)

        def convergence_test(x_new, x_old):
            return converge.max_diff_test(x_new, x_old, rtol, atol)

        def step(x_old):
            return test_util.ax_plus_b(x_old, matrix, offset)

        def solve(x):
            return loop.fixed_point_iteration(
                init_x=x,
                func=step,
                convergence_test=convergence_test,
                max_iter=max_steps,
                batched_iter_size=1,
                unroll=unroll,
            )

        if jit:
            solve = jax.jit(solve)

        sol = solve(init_x)

        true_sol = test_util.solve_ax_b(matrix, offset)

        testing.assert_allclose(sol.value, true_sol, rtol=1e-5, atol=1e-5)
        self.assertTrue(sol.converged)
        self.assertLessEqual(sol.iterations, max_steps)
        testing.assert_allclose(sol.value, sol.previous_value, rtol=1e-5,
                                atol=1e-5)

    @parameterized.parameters(
        {"unroll": True},
        {"unroll": False},
    )
    def testFixedPointDiverge(self, unroll):
        rtol = atol = 1e-10
        max_steps = 10

        init_x = np.zeros(())

        def convergence_test(x_new, x_old):
            return converge.max_diff_test(x_new, x_old, rtol, atol)

        def step(x_old):
            return x_old + 1

        sol = loop.fixed_point_iteration(
                init_x=init_x,
                func=step,
                convergence_test=convergence_test,
                max_iter=max_steps,
                batched_iter_size=1,
                unroll=unroll,
            )
        self.assertFalse(sol.converged)
        self.assertEqual(sol.iterations, max_steps)

    @parameterized.parameters(
        {"unroll": True},
        {"unroll": False},
    )
    def testBatchedLoop(self, unroll):
        max_steps = 10

        def step(x):
            return x - 1

        init_x = np.zeros(())
        batched_sol = loop.fixed_point_iteration(
            init_x=init_x,
            func=step,
            convergence_test=lambda *args: False,
            max_iter=max_steps,
            batched_iter_size=5,
            unroll=unroll,
        )

        loop_sol = loop.fixed_point_iteration(
            init_x=init_x,
            func=step,
            convergence_test=lambda *args: False,
            max_iter=max_steps,
            batched_iter_size=1,
            unroll=unroll,
        )

        testing.assert_array_equal(batched_sol, loop_sol)

    @parameterized.parameters(
        {"unroll": True},
        {"unroll": False},
    )
    def testTermination(self, unroll):
        if unroll:
            self.skipTest((
                "Can't terminate early when unrolling until `jax.lax.cond` "
                "supports differentiation."))
        max_steps = 10

        def step(x):
            return x - 1

        init_x = np.zeros(())
        term_sol = loop.fixed_point_iteration(
            init_x=init_x,
            func=step,
            convergence_test=lambda x, *args: x <= -5,
            max_iter=max_steps,
            batched_iter_size=1,
            unroll=unroll,
        )

        last_i, fixed_value = loop.unrolled(
            0,
            init_x=init_x,
            func=step,
            num_iter=5,
            return_last_two=False,
        )

        self.assertEqual(fixed_value, term_sol.value)
        self.assertEqual(last_i, term_sol.iterations)

    def testUnrollFixedpointLoop(self):
        max_steps = 10

        def step(x):
            return x - 1

        init_x = np.zeros(())
        unroll_sol = loop.fixed_point_iteration(
            init_x=init_x,
            func=step,
            convergence_test=lambda *args: False,
            max_iter=max_steps,
            batched_iter_size=1,
            unroll=True,
        )

        loop_sol = loop.fixed_point_iteration(
            init_x=init_x,
            func=step,
            convergence_test=lambda *args: False,
            max_iter=max_steps,
            batched_iter_size=1,
            unroll=False,
        )

        testing.assert_array_equal(unroll_sol, loop_sol)

    def testJITUnrollFixedpointLoop(self):
        max_steps = 10

        def step(x):
            return x - 1

        init_x = np.zeros(())

        @jax.jit
        def run_unrolled(x):
            return loop.fixed_point_iteration(
                init_x=x,
                func=step,
                convergence_test=lambda *args: False,
                max_iter=max_steps,
                batched_iter_size=1,
                unroll=True,
            )

        unroll_sol = run_unrolled(init_x)

        @jax.jit
        def run_loop(x):
            return loop.fixed_point_iteration(
                init_x=x,
                func=step,
                convergence_test=lambda *args: False,
                max_iter=max_steps,
                batched_iter_size=1,
                unroll=False,
            )
        loop_sol = run_loop(init_x)

        testing.assert_array_equal(unroll_sol, loop_sol)

        jax.grad(lambda x: run_unrolled(x).value)(init_x)

    @parameterized.parameters(
        {"jit": False},
        {"jit": True},
    )
    def testUnrollGrad(self, jit):
        max_steps = 10

        def step(x):
            return x*0.1

        def converge_test(x_new, x_old):
            return np.max(x_new - x_old) < 1e-3

        init_x = np.ones(())

        def run_unrolled(x):
            return loop.fixed_point_iteration(
                init_x=x,
                func=step,
                convergence_test=converge_test,
                max_iter=max_steps,
                batched_iter_size=1,
                unroll=True,
            ).value

        grad_fun = jax.grad(run_unrolled)
        if jit:
            grad_fun = jax.jit(grad_fun)
        grad_fun(init_x)

    def testBatchedRaise(self):
        max_steps = 10

        def step(x):
            return x - 1

        def neg_batch():
            init_x = np.zeros(())
            return loop.fixed_point_iteration(
                init_x=init_x,
                func=step,
                convergence_test=lambda *args: False,
                max_iter=max_steps,
                batched_iter_size=0,
            )

        def oversize_batch():
            init_x = np.zeros(())
            return loop.fixed_point_iteration(
                init_x=init_x,
                func=step,
                convergence_test=lambda *args: False,
                max_iter=max_steps,
                batched_iter_size=max_steps + 1,
            )

        def none_divisible_batch():
            init_x = np.zeros(())
            return loop.fixed_point_iteration(
                init_x=init_x,
                func=step,
                convergence_test=lambda *args: False,
                max_iter=max_steps,
                batched_iter_size=3,
            )

        self.assertRaises(ValueError, neg_batch)
        self.assertRaises(ValueError, oversize_batch)
        self.assertWarns(UserWarning, none_divisible_batch)

    def testNoneMaxIter(self):
        max_steps = None

        def step(x):
            return x + 1

        init_x = np.zeros(())
        loop_sol = loop.fixed_point_iteration(
            init_x=init_x,
            func=step,
            convergence_test=lambda i, *args: i > 10,
            max_iter=max_steps,
            batched_iter_size=3,
        )
        self.assertIsNotNone(loop_sol)
        loop_sol.value.block_until_ready()

    def testUnrolledLoop(self):
        max_steps = 11

        def step(x):
            return x - 1

        init_x = np.zeros(())

        batched_i, batched_x, batched_x_old = loop.unrolled(
            0,
            init_x=init_x,
            func=step,
            num_iter=max_steps,
            return_last_two=True,
        )

        single_i, single_batched_x = loop.unrolled(
            0,
            init_x=init_x,
            func=step,
            num_iter=max_steps,
            return_last_two=False,
        )

        loop_x, loop_x_old = (0., 0.)
        for i in range(max_steps):
            loop_x, loop_x_old = step(loop_x), loop_x

        testing.assert_array_equal(batched_x, single_batched_x)
        testing.assert_array_equal(batched_x, loop_x)
        testing.assert_array_equal(batched_x_old, loop_x_old)

        self.assertEqual(i + 1, single_i)
        self.assertEqual(i + 1, batched_i)

def _fixedpoint_iteration_solver(unroll,
                                 param_func,
                                 default_rtol=1e-10,
                                 default_atol=1e-10,
                                 default_max_iter=200,
                                 default_batched_iter_size=1):

        def fixed_point_iteration_solver(init_x, params):
            dtype = converge.tree_smallest_float_dtype(init_x)
            rtol, atol = converge.adjust_tol_for_dtype(default_rtol,
                                                       default_atol,
                                                       dtype)

            def convergence_test(x_new, x_old):
                return converge.max_diff_test(x_new, x_old, rtol, atol)

            func = param_func(params)
            sol = loop.fixed_point_iteration(
                init_x=init_x,
                func=func,
                convergence_test=convergence_test,
                max_iter=default_max_iter,
                batched_iter_size=default_batched_iter_size,
                unroll=unroll,
            )

            return sol.value
        return fixed_point_iteration_solver

class UnrolledFixedPointIterationTest(test_util.FixedPointTestCase):

    def make_solver(self, param_func):
        return _fixedpoint_iteration_solver(unroll=True, param_func=param_func)

if __name__ == "__main__":
    absltest.main()

max_stars_repo_name: evgenii-nikishin/fax

max_issues_repo_name: evgenii-nikishin/fax

max_forks_repo_name: evgenii-nikishin/fax

reason: import numpy,from numpy,import jax,from jax


text:
import sys
import numpy as np
import torch
from torch import nn
from torch.autograd import Variable
from util.holder import *
from util.util import *

# loss on unique core argument
class UniqueRoleLoss(torch.nn.Module):
	def __init__(self, opt, shared):
		super(UniqueRoleLoss, self).__init__()
		self.opt = opt
		self.shared = shared

		self.one = torch.ones(1).float()
		if opt.gpuid != -1:
			self.one = to_device(self.one, self.opt.gpuid)

		self.violation_cnt = 0
		self.num_prop = 0
		self.num_ex = 0

		self.core_labels = []
		self.core_label_idx = []
		for idx, l in self.opt.label_map_inv.items():
			if 'B-A' in l and l[-1].isnumeric():	# only applies to B-A*
				self.core_labels.append(l)
				self.core_label_idx.append(idx)
		print('unique role constraint applies to: ', self.core_labels)

	def forward(self, log_pa, score, v_label, v_l, role_label, roleset_id, extra):
		orig_l = self.shared.orig_seq_l
		batch_l, source_l, _, _ = log_pa.shape

		assert(self.opt.use_gold_predicate == 1)
		loss = torch.zeros(1)
		if self.opt.gpuid != -1:
			loss = to_device(loss, self.opt.gpuid)

		#self._gold_role = role_label

		num_prop = 0
		for i in range(batch_l):
			v_i = v_label[i, :v_l[i]]

			if self.opt.use_gold_predicate == 1:
				log_core = log_pa[i, v_i, :orig_l[i]][:, :, self.core_label_idx]	# num_v, orig_l, num_core_label
				log_core_ext = log_core.unsqueeze(1).expand(-1, orig_l[i], -1, -1)	# num_v, orig_l, orig_l, num_core_label

				mask = torch.eye(orig_l[i]).to(log_core)*10000
				log_neg_core_ext = (self.one - log_core_ext.exp()).clamp(min=1e-6).log()	# num_v, orig_l, orig_l, num_core_label
				log_neg_core_ext = log_neg_core_ext + mask.unsqueeze(0).unsqueeze(-1)

				loss_i = torch.relu(log_core - log_neg_core_ext.min(2)[0]).sum()

			loss = loss + loss_i
			num_prop += v_l[i]

		# stats
		if hasattr(self.shared, 'viterbi_pred'):
			pred_idx = self.shared.viterbi_pred
		else:
			pred_idx = log_pa.argmax(-1)
		self.analyze(pred_idx, v_label, v_l)

		# stats
		self.num_prop += int(num_prop)
		self.num_ex += batch_l

		# # average over number of predicates or num_ex
		normalizer = float(num_prop) if self.opt.use_gold_predicate == 1 else sum([orig_l[i] for i in range(batch_l)])
		#print('framrrole', loss / normalizer)
		return loss / normalizer, None

	def analyze(self, pred_idx, v_label, v_l):
		batch_l, source_l, _ = pred_idx.shape
		for i in range(batch_l):
			v_i = v_label[i, :v_l[i]]
			pred_idx_i = pred_idx[i, v_i, :]	# (num_v, source_l)
			#pred_idx_i = self._gold_role[i, :v_l[i]]
			acc = None
			for k, l in enumerate(self.core_label_idx):
				dup_core_cnt = ((pred_idx_i == l).sum(-1) > 1).int()	# (num_v)
				acc = dup_core_cnt if k == 0 else acc + dup_core_cnt
			self.violation_cnt += (acc > 0).sum().item()

	# return a string of stats
	def print_cur_stats(self):
		rho = float(self.violation_cnt) / self.num_prop
		return "unique rho: {0:.3f}".format(rho)
		#return ""

	# get training metric (scalar metric, extra metric)
	def get_epoch_metric(self):
		rho = float(self.violation_cnt) / self.num_prop
		return rho, [rho]
		#return None, []

	def begin_pass(self):
		self.violation_cnt = 0
		self.num_prop = 0
		self.num_ex = 0

	def end_pass(self):
		pass


text:
"""
This file creates a GUI that provides a front-end for the MatPy program.
"""

"""Ignores all warnings (Remove this to debug)"""
import warnings
warnings.filterwarnings("ignore")

"""Tkinter for GUI"""
import Tkinter as tk
from Tkinter import Entry
import tkFileDialog
from Tkinter import N, S, E, W

"""Library for getting data"""
import parser

"""Must explicitly enable this for GUI to work"""
import matplotlib
matplotlib.use("TkAgg")

"""Library for plotting data"""
import graph_suite as plot

"""Data handlers"""
import material_analytics
import irreversible_stressstrain
from irreversible_stressstrain import StressStrain as strainmodel

"""Optimization tools for model training"""
import optimization_suite
from scipy.optimize import basinhopping
from scipy.optimize import brute
from pybrain.optimization import GA

def getfile():
    """Returns the model selected by the user (contained in a data file)"""

    """Opens file browser and gets selection"""
    name = tkFileDialog.askopenfilename()

    """Creates model for data"""
    if 'xml' in name:
        model = strainmodel(name, type='xml')

    else:
        model = strainmodel(name)

    return model

def display():
    """Displays experimental data from the requested file"""
    plot.plot2D(getfile().get_experimental_data(), marker = 'o', title = 'File', xtitle = 'Strain ($\epsilon$)', ytitle= 'Stress ($\sigma$)')

def display_with_yield(disp = True):
    """Display the plot of the file with its yield point and returns the model and that point."""

    """Opens file browser and gets selection"""
    model = getfile()
    data = model.get_experimental_data()

    yieldpoint = None

    """Gets radio button input as to which method to use to optimize"""
    if yield_method.get() == 1:
        yieldpoint = material_analytics.yield_stress_classic_unfitted(data)

    elif yield_method.get() == 2:
        yieldpoint = material_analytics.yield_stress_classic_fitted(data)

    elif yield_method.get() == 3:
        yieldpoint = material_analytics.yield_stress(data)

    else:
        elastic, plastic = material_analytics.kmeanssplit(data)
        yieldpoint = plastic[0][None,]

    if disp:
        """Displays the found yield stress"""
        plot.plotmult2D(data, yieldpoint, title = 'File', marker1 = 'o', xtitle = 'Strain ($\epsilon$)', ytitle= 'Stress ($\sigma$)')

    return model, yieldpoint

def statistical_model():
    """Display the plot of the file fitted to a statistical model"""

    """Displays the currently selected yield point"""
    model, yieldpoint = display_with_yield(disp=False)
    data = model.get_experimental_data()

    """This is for the logarithmic model"""
    if fit_method.get() == 1:
        model = material_analytics.log_approx(data)

    """This is for the custom, slope-based model"""
    if fit_method.get() == 2:
        model = material_analytics.stress_model(data, yieldpoint)

    """Take some points from the model to plot them"""
    fittedpts = material_analytics.samplepoints(model,[0.,max(data[:,0])],10000)
    plot.plotmult2D(data,fittedpts,marker1 = 'o', marker2 = '-',title = 'Stress v Strain',xtitle ='Strain ($\epsilon$)',ytitle='Stress ($\sigma$)')

def constitutive_model():
    """Display the plot of the file fitted to a constitutive model"""

    """Displays the currently selected yield point"""
    model, yieldpoint = display_with_yield(disp=False)
    data = model.get_experimental_data()

    """Will need to be set to user-input guess"""
    guess = [-150,1]

    """[0,1] is the first row, second column, which is the stress values"""
    SS_stress = yieldpoint[0,1]

    model_training_methods = ['Nelder-Mead','Powell','CG','Newton-CG','BFGS','L-BFGS-B','SLSQP','COBYLA','TNC','Basinhopping','Brute Force','Genetic Algorithm']

    """Because basinhopping, brute force, and GA are all in separate libraries, they are handled as separate cases."""
    optmethod = optimization_method.get()-1
    if optmethod<9:
        model_params = optimization_suite.minimize_suite(model.mcfunc, methods=[model_training_methods[optmethod],], guess = guess ,SS_stress=SS_stress)

    elif optmethod==9:
        model_params = optimization_suite.minimize_suite(model.mcfunc, methods=[basinhopping,], guess = guess ,SS_stress=SS_stress)

    elif optmethod==10:
        model_params = optimization_suite.minimize_suite(model.mcfunc, methods=[brute,], guess = guess ,SS_stress=SS_stress)

    else:
        model_params = optimization_suite.GA_minimize(model.mcfunc, guess)

    """Plots the data versus the fitted irreversible model data"""
    plot.plotmult2D(data, model.irreversible_model(model_params,SS_stress), title = 'Fitted Thermodynamics', xtitle = 'Strain ($\epsilon$)', ytitle= 'Stress ($\sigma$)')

"""Setting window frame"""
root = tk.Tk()
root.resizable(width=True, height=True)
root.wm_title('Select your data file')
root.geometry('800x280')

"""Placing user event buttons"""
"""(Setting up the plotting experimental data button)"""
display_data = tk.Button(root, text='Plot your experimental data',command=display)
display_data.grid(row=13,column = 0, sticky=N+S+E+W)

"""(Plotting with the predicted yield point)"""
display_yield = tk.Button(root, text='Plot with yield',command=display_with_yield)
display_yield.grid(row=13,column = 1, sticky=N+S+E+W)

"""(Plotting the constitutive model)"""
constitutive = tk.Button(root, text='Plot constitutive model',command=constitutive_model)
constitutive.grid(row=13,column = 2, sticky=N+S+E+W)

"""(Plotting the statistical model)"""
statistical = tk.Button(root, text='Plot statistical model',command=statistical_model)
statistical.grid(row=13,column = 3, sticky=N+S+E+W)

"""Yield Prediction radio buttons"""
"""(Selection Variable)"""
lbl1 = tk.Label(root, text='Yield point selection')
lbl1.grid(row=0, column=0, sticky='W')
yield_method = tk.IntVar()

"""(Buttons)"""
b1 = tk.Radiobutton(root, text='Classic Unfitted', variable=yield_method, value=1)
b2 = tk.Radiobutton(root, text='Classic Fitted', variable=yield_method, value=2)
b3 = tk.Radiobutton(root, text='Custom', variable=yield_method, value=3)
b4 = tk.Radiobutton(root, text='Clustering', variable=yield_method, value=4)

"""(Button Placement)"""
b1.grid(row=1, column=0, sticky='W')
b2.grid(row=2, column=0, sticky='W')
b3.grid(row=3, column=0, sticky='W')
b4.grid(row=4, column=0, sticky='W')

"""Statistical Model Fitting"""
"""(Fitting Variable)"""
lbl_fit = tk.Label(root, text='Statistical Fit Method')
lbl_fit.grid(row=0, column=1)
fit_method = tk.IntVar()

"""(Buttons)"""
s1 = tk.Radiobutton(root, text='Logarithmic Fit', variable=fit_method, value=1)
s2 = tk.Radiobutton(root, text='Custom Fit', variable=fit_method, value=2)

"""(Button Placement)"""
s1.grid(row=1, column=1)
s2.grid(row=2, column=1)

"""Model fitting method radio buttons"""
"""(Selection Variable)"""
lbl2 = tk.Label(root, text='Optimization Technique')
lbl2.grid(row=0, column=5, sticky='E')
optimization_method = tk.IntVar()

"""(Buttons)"""
m1 = tk.Radiobutton(root, text='Nelder-Mead', variable=optimization_method, value=1)
m2 = tk.Radiobutton(root, text='Powell', variable=optimization_method, value=2)
m3 = tk.Radiobutton(root, text='CG', variable=optimization_method, value=3)
m4 = tk.Radiobutton(root, text='Newton-CG', variable=optimization_method, value=4)
m5 = tk.Radiobutton(root, text='BFGS', variable=optimization_method, value=5)
m6 = tk.Radiobutton(root, text='L-BFGS-B', variable=optimization_method, value=6)
m7 = tk.Radiobutton(root, text='SLSQP', variable=optimization_method, value=7)
m8 = tk.Radiobutton(root, text='COBYLA', variable=optimization_method, value=8)
m9 = tk.Radiobutton(root, text='TNC', variable=optimization_method, value=9)
m10 = tk.Radiobutton(root, text='Basinhopping', variable=optimization_method, value=10)
m11 = tk.Radiobutton(root, text='Brute Force', variable=optimization_method, value=11)
m12 = tk.Radiobutton(root, text='Genetic Algorithm', variable=optimization_method, value=12)

"""(Button Placement)"""
m1.grid(row=1, column=5, sticky='E')
m2.grid(row=2, column=5, sticky='E')
m3.grid(row=3, column=5, sticky='E')
m4.grid(row=4, column=5, sticky='E')
m5.grid(row=5, column=5, sticky='E')
m6.grid(row=6, column=5, sticky='E')
m7.grid(row=7, column=5, sticky='E')
m8.grid(row=8, column=5, sticky='E')
m9.grid(row=9, column=5, sticky='E')
m10.grid(row=10, column=5, sticky='E')
m11.grid(row=11, column=5, sticky='E')
m12.grid(row=12, column=5, sticky='E')

"""Start the program"""
root.mainloop()

max_stars_repo_name: snesnehne/MatPy

max_issues_repo_name: snesnehne/MatPy

max_forks_repo_name: snesnehne/MatPy

reason: from scipy


text:
import os

import matplotlib as mpl
import numpy as np
from scipy.io import loadmat

mpl.use('Agg')
import matplotlib.pyplot as plt
from utils.load_features import LoadData

class LAGFeatureExtraction(object):
    def __init__(self, lag=2, compute_lag=False):
        self.dir_path = os.path.dirname(os.path.realpath(__file__))
        self.subjects, self.channels, self.time_stamps = 32, 32, 8064
        self.subject_1_path = os.path.abspath(os.path.join(self.dir_path, '', "DEAP_s/s_1.mat"))
        self.folder = 'LAG'
        self.lag = lag
        np.random.seed(31415)
        if not os.listdir(self.folder) or compute_lag:
            print("extracting lag features:")
            self.extract_lag_features()

    def view_s1(self):
        s1 = loadmat(self.subject_1_path)
        s1_label = s1['label']
        s1_data = s1['data']
        print("s1.label:{}, s1.data:{}".format(s1_label.shape, s1_data.shape))
        s1_obs1_chan1 = s1_data[1, 1, :]
        label_obs1 = s1_label[1, :]
        s1_obs9_chan1 = s1_data[9, 1, :]
        label_obs9 = s1_label[9, :]
        print("s1_obs1_chan1:{}, label:{}".format(s1_obs1_chan1.shape, label_obs1))
        print("s1_obs9_chan1:{}, label:{}".format(s1_obs1_chan1.shape, label_obs9))
        self.plot_single_channel(signal=s1_obs1_chan1, name='s1_obs1_chan_1')
        self.plot_single_channel(signal=s1_obs9_chan1, name='s1_obs9_chan1')

        s1_obs1_chan1_lag2 = self.lag_map(s1_obs1_chan1)
        s1_obs9_chan1_lag2 = self.lag_map(s1_obs9_chan1)
        print("s1_obs1_chan1_lag2:{}".format(s1_obs1_chan1_lag2.shape))
        print("s1_obs9_chan1_lag2:{}".format(s1_obs9_chan1_lag2.shape))
        self.plot_lag(s1_obs1_chan1_lag2, name='s1_obs1_chan1_lag2')
        self.plot_lag(s1_obs9_chan1_lag2, name='s1_obs9_chan1_lag2')

    @staticmethod
    def plot_single_channel(signal, name):
        plt.figure()
        plt.plot(signal)
        plt.xlabel('time')
        plt.ylabel('signal')
        title = 'signal_{}'.format(name)
        plt.title(title)
        plt.savefig(title)

    def lag_map(self, signal):
        idx = np.arange(int(self.time_stamps))
        lag_output = []
        for i in idx:
            max_index = i + self.lag
            if max_index >= self.time_stamps:
                break
            else:
                lag_indexes = np.arange(start=i, stop=max_index, step=1)
            lag_trans = np.array(signal[lag_indexes])
            # print("max_index:{}, lag_indexes:{} lag_trans:{}".format(max_index, lag_indexes, lag_trans.shape))
            lag_output.append(lag_trans)
        return np.array(lag_output)

    @staticmethod
    def plot_lag(signal, name):
        plt.figure()
        plt.scatter(signal[:, 0], signal[:, 1])
        plt.xlabel('pt1')
        plt.ylabel('pt2')
        title = 'lag_2_{}'.format(name)
        plt.title(title)
        plt.savefig(title)

    def extract_lag_features(self):
        for subj in np.arange(start=1, stop=self.subjects + 1, step=1):
            print("subject:{}".format(subj))
            file = "DEAP_s/s_{}.mat".format(subj)
            path = os.path.abspath(os.path.join(self.dir_path, '', file))
            s = loadmat(path)
            s_label = s['label']
            s_data = s['data']
            print("data:{}, label:{}".format(s_data.shape, s_label.shape))
            subject_data = []
            subject_label = []
            for obs in np.arange(s_data.shape[0]):
                channels_lag2 = []
                s_label_obs = s_label[obs, :]
                for channel in np.arange(self.channels):
                    lag_map = self.lag_map(s_data[obs, channel, :])
                    channels_lag2.append(lag_map)
                subject_data.append(channels_lag2)
                subject_label.append(s_label_obs)
            subject_data = np.array(subject_data)
            subject_label = np.array(subject_label)
            print("subject_obs:{}, subject_label{}".format(subject_data.shape, subject_label.shape))
            data_file = os.path.abspath(
                os.path.join(self.dir_path, '.', '{}/{}_data'.format(self.folder, 's_{}'.format(subj))))
            label_file = os.path.abspath(
                os.path.join(self.dir_path, '.', '{}/{}_label'.format(self.folder, 's_{}'.format(subj))))
            np.save(data_file, subject_data)
            np.save(label_file, subject_label)
        print('LAG feature extraction complete')

    def load_features(self, valid_idx, test_idx):
        load = LoadData(folder=self.folder)
        return load.load_features(test_idx=test_idx, valid_idx=valid_idx)

if __name__ == '__main__':
    np.random.seed(31415)
    lag_feature = LAGFeatureExtraction()
    lag_feature.view_s1()
    # lag.extract_lag_features()


text:
from utils import *
import numpy as np
import pandas as pd
import os

# =============================================================================
# data csv source dir
# =============================================================================

source_dir = '/ds_csv'

# =============================================================================
# load csv files
# =============================================================================

df_in_tb = pd.read_csv(os.path.join(source_dir,'in_tb.csv'))
df_sh_tb = pd.read_csv(os.path.join(source_dir,'sh_tb.csv'))
df_mc_tb = pd.read_csv(os.path.join(source_dir,'mc_tb.csv'))
df_niaid_tb = pd.read_csv(os.path.join(source_dir,'niaid_tb.csv'))

df_in_normal = pd.read_csv(os.path.join(source_dir,'in_normal.csv'))
df_sh_normal = pd.read_csv(os.path.join(source_dir,'sh_normal.csv'))
df_mc_normal = pd.read_csv(os.path.join(source_dir,'mc_normal.csv'))
df_niaid_normal = pd.read_csv(os.path.join(source_dir,'niaid_normal.csv'))

# =============================================================================
# split csv into 80% train, 10% validation and 10% holdout test
# =============================================================================

in_tb_train, in_tb_val, in_tb_test = train_validate_test_split(df_in_tb)
sh_tb_train, sh_tb_val, sh_tb_test = train_validate_test_split(df_sh_tb)
mc_tb_train, mc_tb_val, mc_tb_test = train_validate_test_split(df_mc_tb)
niaid_tb_train, niaid_tb_val, niaid_tb_test = train_validate_test_split(df_niaid_tb)

in_normal_train, in_normal_val, in_normal_test = train_validate_test_split(df_in_normal)
sh_normal_train, sh_normal_val, sh_normal_test = train_validate_test_split(df_sh_normal)
mc_normal_train, mc_normal_val, mc_normal_test = train_validate_test_split(df_mc_normal)
niaid_normal_train, niaid_normal_val, niaid_normal_test = train_validate_test_split(df_niaid_normal)

# =============================================================================
# save csv file to source dir
# =============================================================================

in_tb_train.to_csv(os.path.join(source_dir,'in_tb_train.csv'))
in_tb_val.to_csv(os.path.join(source_dir,'in_tb_val.csv'))
in_tb_test.to_csv(os.path.join(source_dir,'in_tb_test.csv'))

sh_tb_train.to_csv(os.path.join(source_dir,'sh_tb_train.csv'))
sh_tb_val.to_csv(os.path.join(source_dir,'sh_tb_val.csv'))
sh_tb_test.to_csv(os.path.join(source_dir,'sh_tb_test.csv'))

mc_tb_train.to_csv(os.path.join(source_dir,'mc_tb_train.csv'))
mc_tb_val.to_csv(os.path.join(source_dir,'mc_tb_val.csv'))
mc_tb_test.to_csv(os.path.join(source_dir,'mc_tb_test.csv'))

niaid_tb_train.to_csv(os.path.join(source_dir,'niaid_tb_train.csv'))
niaid_tb_val.to_csv(os.path.join(source_dir,'niaid_tb_val.csv'))
niaid_tb_test.to_csv(os.path.join(source_dir,'niaid_tb_test.csv'))

in_normal_train.to_csv(os.path.join(source_dir,'in_normal_train.csv'))
in_normal_val.to_csv(os.path.join(source_dir,'in_normal_val.csv'))
in_normal_test.to_csv(os.path.join(source_dir,'in_normal_test.csv'))

sh_normal_train.to_csv(os.path.join(source_dir,'sh_normal_train.csv'))
sh_normal_val.to_csv(os.path.join(source_dir,'sh_normal_val.csv'))
sh_normal_test.to_csv(os.path.join(source_dir,'sh_normal_test.csv'))

mc_normal_train.to_csv(os.path.join(source_dir,'mc_normal_train.csv'))
mc_normal_val.to_csv(os.path.join(source_dir,'mc_normal_val.csv'))
mc_normal_test.to_csv(os.path.join(source_dir,'mc_normal_test.csv'))

niaid_normal_train.to_csv(os.path.join(source_dir,'niaid_normal_train.csv'))
niaid_normal_val.to_csv(os.path.join(source_dir,'niaid_normal_val.csv'))
niaid_normal_test.to_csv(os.path.join(source_dir,'niaid_normal_test.csv'))


text:
import pytest
import numpy as np
from bluesky.plans import scan
from ophyd.sim import motor, det, SynGauss
from bluesky.callbacks.fitting import PeakStats
from scipy.special import erf

def get_ps(x, y, shift=0.5):
    """ peak status calculation from CHX algorithm.
    """
    lmfit = pytest.importorskip('lmfit')
    ps = {}
    x = np.array(x)
    y = np.array(y)

    COM = np.sum(x * y) / np.sum(y)
    ps['com'] = COM

    # from Maksim: assume this is a peak profile:
    def is_positive(num):
        return True if num > 0 else False

    # Normalize values first:
    ym = (y - np.min(y)) / (np.max(y) - np.min(y)) - shift  # roots are at Y=0

    positive = is_positive(ym[0])
    list_of_roots = []
    for i in range(len(y)):
        current_positive = is_positive(ym[i])
        if current_positive != positive:
            list_of_roots.append(x[i - 1] + (x[i] - x[i - 1]) / (abs(ym[i]) + abs(ym[i - 1])) * abs(ym[i - 1]))
            positive = not positive
    if len(list_of_roots) >= 2:
        FWHM = abs(list_of_roots[-1] - list_of_roots[0])
        CEN = list_of_roots[0] + 0.5 * (list_of_roots[1] - list_of_roots[0])

        ps['fwhm'] = FWHM
        ps['cen'] = CEN

    else:    # ok, maybe it's a step function..
        print('no peak...trying step function...')
        ym = ym + shift

        def err_func(x, x0, k=2, A=1,  base=0):  # erf fit from Yugang
            return base - A * erf(k * (x - x0))

        mod = lmfit.Model(err_func)
        # estimate starting values:
        x0 = np.mean(x)
        # k=0.1*(np.max(x)-np.min(x))
        pars = mod.make_params(x0=x0, k=2, A=1., base=0.)
        result = mod.fit(ym, pars, x=x)
        CEN = result.best_values['x0']
        FWHM = result.best_values['k']
        ps['fwhm'] = FWHM
        ps['cen'] = CEN

    return ps

def test_peak_statistics(RE):
    """peak statistics calculation on simple gaussian function
    """
    x = 'motor'
    y = 'det'
    ps = PeakStats(x, y)
    RE.subscribe(ps)
    RE(scan([det], motor, -5, 5, 100))

    fields = ["x", "y", "min", "max", "com", "cen", "crossings", "fwhm", "lin_bkg"]
    for field in fields:
        assert hasattr(ps, field), f"{field} is not an attribute of ps"

    np.allclose(ps.cen, 0, atol=1e-6)
    np.allclose(ps.com, 0, atol=1e-6)
    fwhm_gauss = 2 * np.sqrt(2 * np.log(2))  # theoretical value with std=1
    assert np.allclose(ps.fwhm, fwhm_gauss, atol=1e-2)

def test_peak_statistics_compare_chx(RE):
    """This test focuses on gaussian function with noise.
    """
    s = np.random.RandomState(1)
    noisy_det_fix = SynGauss('noisy_det_fix', motor, 'motor', center=0, Imax=1,
                             noise='uniform', sigma=1, noise_multiplier=0.1, random_state=s)
    x = 'motor'
    y = 'noisy_det_fix'
    ps = PeakStats(x, y)
    RE.subscribe(ps)

    RE(scan([noisy_det_fix], motor, -5, 5, 100))
    ps_chx = get_ps(ps.x_data, ps.y_data)

    assert np.allclose(ps.cen, ps_chx['cen'], atol=1e-6)
    assert np.allclose(ps.com, ps_chx['com'], atol=1e-6)
    assert np.allclose(ps.fwhm, ps_chx['fwhm'], atol=1e-6)

def test_peak_statistics_with_derivatives(RE):
    """peak statistics calculation on simple gaussian function with derivatives
    """
    x = "motor"
    y = "det"
    num_points = 100
    ps = PeakStats(x, y, calc_derivative_and_stats=True)
    RE.subscribe(ps)
    RE(scan([det], motor, -5, 5, num_points))

    assert hasattr(ps, "derivative_stats")
    der_fields = ["x", "y", "min", "max", "com", "cen", "crossings", "fwhm", "lin_bkg"]
    for field in der_fields:
        assert hasattr(ps.derivative_stats, field), f"{field} is not an attribute of ps.der"

    assert type(ps.derivative_stats.x) is np.ndarray
    assert type(ps.derivative_stats.y) is np.ndarray
    assert type(ps.derivative_stats.min) is tuple
    assert type(ps.derivative_stats.max) is tuple
    assert type(ps.derivative_stats.com) is np.float64
    assert type(ps.derivative_stats.cen) is np.float64
    assert type(ps.derivative_stats.crossings) is np.ndarray
    if len(ps.derivative_stats.crossings) >= 2:
        assert type(ps.derivative_stats.fwhm) is float
    else:
        assert ps.derivative_stats.fwhm is None
    assert len(ps.derivative_stats.x) == num_points - 1
    assert len(ps.derivative_stats.y) == num_points - 1
    assert np.allclose(np.diff(ps.y_data), ps.derivative_stats.y, atol=1e-10)

max_stars_repo_name: AbbyGi/bluesky

max_issues_repo_name: AbbyGi/bluesky

max_forks_repo_name: AbbyGi/bluesky