text:
# -*- coding: utf-8 -*-
"""
Created on Tue Apr  6 15:23:30 2021

@author: Jerem
"""

# -*- coding: utf-8 -*-
"""
Created on Tue Apr  6 14:00:58 2021

@author: Jerem
"""

import os
import numpy as np
import json
from PIL import Image
from matplotlib import patches
from matplotlib import pyplot
import matplotlib.image as mpimg

def detect_red_light(I):
    '''
    This function takes a numpy array <I> and returns a list <bounding_boxes>.
    The list <bounding_boxes> should have one element for each red light in the
    image. Each element of <bounding_boxes> should itself be a list, containing
    four integers that specify a bounding box: the row and column index of the
    top left corner and the row and column index of the bottom right corner (in
    that order). See the code below for an example.

    Note that PIL loads images in RGB order, so:
    I[:,:,0] is the red channel
    I[:,:,1] is the green channel
    I[:,:,2] is the blue channel
    '''

    bounding_boxes = [] # This should be a list of lists, each of length 4. See format example below.
    scores = []
    top_boxes = []
    '''
    BEGIN YOUR CODE
    '''

    '''
    As an example, here's code that generates between 1 and 5 random boxes
    of fixed size and returns the results in the proper format.
    '''
    file_num = -1
    for file in anotated_files:
        file_num += 1
        red_light =  np.asarray(Image.open(os.path.join(anotated_path, file)))
        box_height, box_width, box_channels = np.shape(red_light)
        top_boxes = []
       # num_boxes = np.random.randint(1,5)
        (n_rows,n_cols,n_channels) = np.shape(I)
        num_boxes = int((n_rows*n_cols)/(box_height*box_width))

        for i in range(num_boxes*10):
            tl_row = np.random.randint(n_rows - box_height)
            tl_col = np.random.randint(n_cols - box_width)
            br_row = tl_row + box_height
            br_col = tl_col + box_width

            top_boxes.append([tl_row,tl_col,br_row,br_col])

        light_vector = red_light.reshape((-1,))
        #print(np.shape(light_vector))
        for box in top_boxes:
            sub_image = I[box[0]:box[2],box[1]:box[3]]

            score = (np.dot(sub_image.reshape((-1,)), light_vector))
            scores.append(score)
        bounding_boxes += top_boxes
        '''
        END YOUR CODE
        '''

    for i in range(len(bounding_boxes)):
        assert len(bounding_boxes[i]) == 4

    return bounding_boxes, scores
##################################

# This code is used to generate the examples for Q5
# Continued at the end
###################################################
def show_box(key, boxs):
    index = file_names.index(key)
    I = Image.open(os.path.join(data_path,file_names[index]))
    I = np.asarray(I)
    img = mpimg.imread(data_path+'/'+key)

    for box in boxs:
        box_height = box[2] - box[0]
        box_width = box[3] - box[1]
        figure, ax = pyplot.subplots(1)
        rect = patches.Rectangle((box[0],box[1]),box_width,box_height, edgecolor='r', facecolor="none")
        ax.imshow(img)
        ax.add_patch(rect)
        #img = Image.fromarray(I[box[0]:box[2],box[1]:box[3]])

        #img.show()
 ####################################### #
##########################################
# set the path to the downloaded data:
data_path = "C:/Users/Jerem/OneDrive - California Institute of Technology/Caltech-/Classes/Spring 2021/EE 148/HW1/RedLights2011_Medium/RedLights2011_Medium"
anotated_path = "C:/Users/Jerem/OneDrive - California Institute of Technology/Caltech-/Classes/Spring 2021/EE 148/HW1/RedLights2011_Medium/Anotated"
# set a path for saving predictions:
preds_path = 'C:/Users/Jerem/OneDrive - California Institute of Technology/Caltech-/Classes/Spring 2021/EE 148/HW1/predictions/'
os.makedirs(preds_path,exist_ok=True) # create directory if needed

# get sorted list of files:
file_names = sorted(os.listdir(data_path))
anotated_files = sorted(os.listdir(anotated_path))
anotated_files = [f for f in anotated_files if '.jpg' in f]
# remove any non-JPEG files:
file_names = [f for f in file_names if '.jpg' in f]

preds = {}
scores = {}
all_scores = []
mean = 0
for i in range(len(file_names)):

    # read image using PIL:
    I = Image.open(os.path.join(data_path,file_names[i]))

    # convert to numpy array:
    I = np.asarray(I)
    mean += np.mean(I)
print(i)
mean/i

for i in range(len(file_names)):

    # read image using PIL:
    I = Image.open(os.path.join(data_path,file_names[i]))

    # convert to numpy array:
    I = np.asarray(I)

    preds[file_names[i]], scores[file_names[i]] = detect_red_light(I)
    all_scores.append(scores[file_names[i]])

#red_light =  np.asarray(Image.open(os.path.join(anotated_path, anotated_files[2])))
#box_height, box_width, box_channels = np.shape(red_light)
cutoff = np.percentile(np.reshape(all_scores,(-1,)), 99.2)
final_preds= {}
high_scores = []
for key in preds:
    #for i in range(len(preds[key])):
    best_guess = scores[key].index(np.max(scores[key]))
    #best_guess = np.sort(scores[keys)[-1]]
    if scores[key][best_guess] > cutoff:
        high_scores.append(preds[key][best_guess])
    final_preds[key] = high_scores
    high_scores = []

 #   preds[file_names[i]] = detect_red_light(I)
# save preds (overwrites any previous predictions!)
with open(os.path.join(preds_path,'preds.json'),'w') as f:
    json.dump(preds,f)

red_light =  np.asarray(Image.open(os.path.join(anotated_path, anotated_files[0])))
img = Image.fromarray(red_light)
img.show()

######################################
# Here is the rest of the code to generate examples for Q5
######################################
i = 0
for p in final_preds:
    i+=1
    show_box(p, final_preds[p])
    if i > 100:
        break


text:
import numpy as np
from sklearn.decomposition import PCA
import h5py

def pca(enc):
    """
    Uses the PCA algorithm to sort the latent vectors by importance of each feature.
    Input: filename - The encoded dataset file name
    """
    dataset = h5py.File(enc, 'r')
    data = dataset['data'][:]
    dataset.close()

    normalized = data - np.mean(data, axis=0)
    pca = PCA(n_components=normalized.shape[1])
    pca.fit(normalized)
    values = np.sqrt(pca.explained_variance_)
    vectors = pca.components_
    np.save('eigenvalues.npy', values)
    np.save('eigenvectors.npy', vectors)

ext: py

max_stars_repo_name: danielMonas/VAEriation

max_issues_repo_name: danielMonas/VAEriation

max_forks_repo_name: danielMonas/VAEriation

reason: import numpy


text:
from ror.Relation import INDIFFERENCE, PREFERENCE
from ror.data_loader import RORParameter, read_dataset_from_txt
import unittest
from ror.Dataset import Dataset, RORDataset
import numpy as np

class TestTxtDatasetReader(unittest.TestCase):
    def test_reading_dataset_from_txt(self):
        loading_result = read_dataset_from_txt("tests/datasets/example.txt")
        data = loading_result.dataset

        self.assertIs(type(data), RORDataset)
        self.assertEqual(len(data.criteria), 2)
        self.assertEqual(data.criteria[0][0], "MaxSpeed")
        self.assertEqual(data.criteria[0][1], "g")
        self.assertEqual(data.criteria[1][0], "FuelCons")
        self.assertEqual(data.criteria[1][1], "c")

        self.assertEqual(len(data.alternatives), 5)
        self.assertEqual(data.alternatives[0], "b01")
        self.assertEqual(data.alternatives[4], "b05")

        self.assertIs(type(data.matrix[0, 0]), np.float64)
        self.assertEqual(data.matrix[0, 0], 90)
        self.assertIs(type(data.matrix[4, 0]), np.float64)
        self.assertEqual(data.matrix[4, 0], 83)

        # cost type criteria are reversed
        # (multiplied by -1 so we can treat them as gain type criteria)
        self.assertIs(type(data.matrix[4, 1]), np.float64)
        self.assertEqual(data.matrix[4, 1], -26)
        self.assertIs(type(data.matrix[0, 1]), np.float64)
        self.assertEqual(data.matrix[0, 1], -27)

    def test_reading_dataset_from_txt(self):
        loading_result = read_dataset_from_txt("tests/datasets/example.txt")
        data = loading_result.dataset

        self.assertIs(type(data), RORDataset)
        self.assertEqual(len(data.criteria), 2)
        self.assertEqual(data.criteria[0][0], "MaxSpeed")
        self.assertEqual(data.criteria[0][1], "g")
        self.assertEqual(data.criteria[1][0], "FuelCons")
        self.assertEqual(data.criteria[1][1], "c")

        self.assertEqual(len(data.alternatives), 5)
        self.assertEqual(data.alternatives[0], "b01")
        self.assertEqual(data.alternatives[4], "b05")

        self.assertIs(type(data.matrix[0, 0]), np.float64)
        self.assertEqual(data.matrix[0, 0], 90)
        self.assertIs(type(data.matrix[4, 0]), np.float64)
        self.assertEqual(data.matrix[4, 0], 83)

        # cost type criteria are reversed
        # (multiplied by -1 so we can treat them as gain type criteria)
        self.assertIs(type(data.matrix[4, 1]), np.float64)
        self.assertEqual(data.matrix[4, 1], -26)
        self.assertIs(type(data.matrix[0, 1]), np.float64)
        self.assertEqual(data.matrix[0, 1], -27)

    def test_reading_dataset_from_txt_with_preferences(self):
        loading_result = read_dataset_from_txt("tests/datasets/ror_dataset.txt")
        data = loading_result.dataset

        self.assertIs(type(data), RORDataset)
        self.assertEqual(len(data.criteria), 2)
        self.assertEqual(data.criteria[0][0], "MaxSpeed")
        self.assertEqual(data.criteria[0][1], "g")
        self.assertEqual(data.criteria[1][0], "FuelCons")
        self.assertEqual(data.criteria[1][1], "c")

        self.assertEqual(len(data.alternatives), 14)
        self.assertEqual(data.alternatives[0], "b01")
        self.assertEqual(data.alternatives[4], "b05")

        self.assertIs(type(data.matrix[0, 0]), np.float64)
        self.assertEqual(data.matrix[0, 0], 90)
        self.assertIs(type(data.matrix[4, 0]), np.float64)
        self.assertEqual(data.matrix[4, 0], 83)

        self.assertEqual(len(data.preferenceRelations), 3)
        self.assertEqual(data.preferenceRelations[0].relation, INDIFFERENCE)
        self.assertEqual(data.preferenceRelations[0].alternative_1, "b01")
        self.assertEqual(data.preferenceRelations[0].alternative_2, "b02")
        self.assertEqual(data.preferenceRelations[1].relation, PREFERENCE)
        self.assertEqual(data.preferenceRelations[1].alternative_1, "b06")
        self.assertEqual(data.preferenceRelations[1].alternative_2, "b03")
        self.assertEqual(data.preferenceRelations[2].relation, PREFERENCE)
        self.assertEqual(data.preferenceRelations[2].alternative_1, "b08")
        self.assertEqual(data.preferenceRelations[2].alternative_2, "b07")

        self.assertEqual(len(data.intensityRelations), 1)
        self.assertEqual(data.intensityRelations[0].relation, PREFERENCE)
        self.assertEqual(data.intensityRelations[0].alternative_1, "b04")
        self.assertEqual(data.intensityRelations[0].alternative_2, "b08")
        self.assertEqual(data.intensityRelations[0].alternative_3, "b07")
        self.assertEqual(data.intensityRelations[0].alternative_4, "b06")

    def test_reading_with_preferences(self):
        loading_result = read_dataset_from_txt("tests/datasets/ror_dataset_with_parameters.txt")
        data = loading_result.dataset
        parameters = loading_result.parameters

        self.assertIs(type(data), RORDataset)
        self.assertEqual(len(data.criteria), 2)
        self.assertEqual(len(data.alternatives), 14)
        self.assertEqual(len(data.preferenceRelations), 3)
        self.assertEqual(len(data.intensityRelations), 1)

        self.assertAlmostEqual(parameters[RORParameter.EPS], 2e-11)
        self.assertAlmostEqual(parameters[RORParameter.INITIAL_ALPHA], 0.1)

max_stars_repo_name: jakub-tomczak/ror

max_issues_repo_name: jakub-tomczak/ror

max_forks_repo_name: jakub-tomczak/ror


text:
"""
PlacesCNN to predict the scene by category, attribute, and class activation map in a single pass.

by Bolei Zhou, sep 2, 2017
last modified date: Dec. 27, 2017, migrating everything to python36 and latest pytorch and torchvision
"""
import logging
import os

import numpy as np
import torch
from PIL import Image
from torch.autograd import Variable as V
from torch.nn import functional as F
from torchvision import transforms as trn

from .wideresnet import resnet18

_LOGGER = logging.getLogger(__name__)
_LOGGER.setLevel(logging.DEBUG)

torch.nn.Module.dump_patches = True

dir_places365_model = os.path.join(os.path.dirname(__file__), "model")

def load_labels():
    """Prepare all labels read from 'categories_places365.txt'."""
    # scene category relevant
    file_path_category = os.path.join(dir_places365_model, "categories_places365.txt")
    classes = list()
    with open(file_path_category) as class_file:
        for line in class_file:
            classes.append(line.strip().split(" ")[0][3:])
    classes = tuple(classes)

    # indoor and outdoor relevant
    file_path_IO = os.path.join(dir_places365_model, "IO_places365.txt")
    with open(file_path_IO) as f:
        lines = f.readlines()
        labels_IO = []
        for line in lines:
            items = line.rstrip().split()
            labels_IO.append(int(items[-1]) - 1)  # 0 is indoor, 1 is outdoor
    labels_IO = np.array(labels_IO)

    # scene attribute relevant
    file_path_attribute = os.path.join(dir_places365_model, "labels_sunattribute.txt")
    with open(file_path_attribute) as f:
        lines = f.readlines()
        labels_attribute = [item.rstrip() for item in lines]

    file_path_W = os.path.join(dir_places365_model, "W_sceneattribute_wideresnet18.npy")
    W_attribute = np.load(file_path_W)

    return classes, labels_IO, labels_attribute, W_attribute

def returnTF():
    """Load the image transformer."""
    tf = trn.Compose(
        [
            trn.Resize((224, 224)),
            trn.ToTensor(),
            trn.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
        ]
    )
    return tf

def remove_nonspace_separators(text):
    """Remove non space separators."""
    return " ".join(" ".join(" ".join(text.split("_")).split("/")).split("-"))

# load the labels
classes, labels_IO, labels_attribute, W_attribute = load_labels()

def inference_places365(img_path) -> dict:
    """Inference with places365."""
    features_blobs = []

    def hook_feature(module, input, output):
        features_blobs.append(np.squeeze(output.data.cpu().numpy()))

    def load_model():
        # this model has a last conv feature map as 14x14
        # model_file = os.path.join(dir_places365_model,'whole_wideresnet18_places365_python36.pth.tar')
        model_file = os.path.join(dir_places365_model, "wideresnet18_places365.pth.tar")

        model = resnet18(num_classes=365)
        checkpoint = torch.load(model_file, map_location=lambda storage, loc: storage)
        state_dict = {
            str.replace(k, "module.", ""): v
            for k, v in checkpoint["state_dict"].items()
        }
        model.load_state_dict(state_dict)
        model.eval()
        # hook the feature extractor
        features_names = [
            "layer4",
            "avgpool",
        ]  # this is the last conv layer of the resnet
        for name in features_names:
            model._modules.get(name).register_forward_hook(hook_feature)
        return model

    # load the model
    model = load_model()

    # load the transformer
    tf = returnTF()  # image transformer

    # get the softmax weight
    params = list(model.parameters())
    weight_softmax = params[-2].data.numpy()
    weight_softmax[weight_softmax < 0] = 0

    # load the test image
    # img_url = 'http://places2.csail.mit.edu/imgs/12.jpg'
    # os.system('wget %s -q -O test.jpg' % img_url)
    img = Image.open(img_path)
    input_img = V(tf(img).unsqueeze(0))

    # forward pass
    logit = model.forward(input_img)
    h_x = F.softmax(logit, 1).data.squeeze()
    probs, idx = h_x.sort(0, True)
    probs = probs.numpy()
    idx = idx.numpy()

    res = {}

    # output the IO prediction
    io_image = np.mean(labels_IO[idx[:10]])  # vote for the indoor or outdoor
    if io_image < 0.5:
        res["environment"] = "indoor"
    else:
        res["environment"] = "outdoor"

    # output the prediction of scene category
    res["categories"] = []
    for i in range(0, 5):
        res["categories"].append(remove_nonspace_separators(classes[idx[i]]))

    # output the scene attributes
    responses_attribute = W_attribute.dot(features_blobs[1])
    idx_a = np.argsort(responses_attribute)
    res["attributes"] = [
        remove_nonspace_separators(labels_attribute[idx_a[i]])
        for i in range(-1, -10, -1)
    ]

    return res

max_stars_repo_name: Sick-codes/core

max_issues_repo_name: Sick-codes/core

max_forks_repo_name: Sick-codes/core


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import pydotplus
from sklearn.ensemble import BaggingRegressor
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.metrics import classification_report
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import r2_score
from sklearn import datasets
from mlxtend.plotting import plot_decision_regions

#Il Bagging utilizza un classificatore per il suo miglioramento,
#Quindi fornisce alla fine una media dei risultati di classificatori

#Importiamo il dataset
iris = datasets.load_iris()
#Creiamo il test e train data
X = iris.data[:, 0:4]
y = iris.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0)
#Definiamo l'ensemble model
bagging = BaggingRegressor(LinearRegression(), n_jobs=1, n_estimators=1000, max_features=0.8)
#Otteniamo lo score di validazione
scores = cross_val_score( bagging, X_train, y_train)
print(scores.mean())
#Eseguiamo il training
bagging.fit(X_train, y_train)
#Effettuiamo una predizione
pred = bagging.predict(X_test)
#Otteniamo i risultati
print("R2 SCORE: ", r2_score(y_test, pred))


text:
# -*- coding: utf-8 -*-
"""
Merge multiple data files and summarize in excel
"""

__author__ = "Mike Shuser"

import glob
import pandas as pd
import numpy as np
import xlwings as xl

def compute_weighted_scores(df):
    drivers = [
        'instock',
        'fast',
        'friendly',
        'clean',
        'price',
        'quality',
    ]
    all_data = df[drivers].copy()
    all_data['friendly'].replace(99, np.nan, inplace=True)

    df_wFriendly = all_data.loc[all_data['friendly'].notnull(), :].copy()
    df_noFriendly = all_data.loc[all_data['friendly'].isnull(), :].copy()
    df_noFriendly.friendly.replace(np.nan, 0, inplace=True)

    dsat_wFriendly = [0.18, 0.21, 0.21, 0.13, 0.16, 0.11]
    cf4_wFriendly = [0.21, 0.24, 0.37, 0.18, 0, 0]
    weights_wFriendly = np.array([dsat_wFriendly, cf4_wFriendly]).T

    dsat_noFriendly = [0.2278481, 0.2658228, 0, 0.1645570, 0.2025316, 0.1392405]
    cf4_noFriendly = [0.3333333, 0.3809524, 0, 0.2857143, 0, 0]
    weights_noFriendly = np.array([dsat_noFriendly, cf4_noFriendly]).T

    scores_wFriendly = df_wFriendly.to_numpy() @ weights_wFriendly
    scores_noFriendly = df_noFriendly.to_numpy() @ weights_noFriendly

    scores = np.vstack([scores_wFriendly, scores_noFriendly])
    ndx = np.hstack([df_wFriendly.index, df_noFriendly.index])

    final = pd.DataFrame(
        index=ndx,
        data=scores,
        columns=['derived_sat', 'cf4']
    )

    return final

if __name__ == '__main__':

    path = "C:/CSAT_data/"

    column_layout = pd.read_excel(f"{path}/layout.xlsx")['var'].tolist()
    meta = glob.glob(f'{path}/*Alignment*.csv')[0]
    meta = pd.read_csv(meta, index_col=0)
    meta_cols = ['Province', 'Region', 'Market']

    fids = list(range(1,12))
    merged_col_ndx = ['derived_sat', 'cf4'] + column_layout
    merged = pd.DataFrame(columns=merged_col_ndx)

    #merge single files
    for fid in fids:
        print(fid)
        query = glob.glob(f"{path}/raw_monthly/CSAT_m{fid}_*.csv")
        assert len(query) == 1, "multiple files found, review source directory"

        df = pd.read_csv(query[0])

        derived = compute_weighted_scores(df)
        df = df.merge(derived, left_index=True, right_index=True)

        df = df[merged_col_ndx]
        merged = pd.concat([merged, df], ignore_index=True, sort=False)

    means = merged.groupby('store_num').mean()
    counts = merged.groupby('store_num').count()

    for col in meta_cols:
        means[col] = means.index.map(meta[col])
        counts[col] = counts.index.map(meta[col])

    wb = xl.Book()
    ws = wb.sheets[0]
    ws.range(1, 1).options(index=True).value = means
    ws.range(1, len(merged_col_ndx) + 3).options(index=True).value = counts

max_stars_repo_name: mikeshuser/data-prep-and-pull

max_issues_repo_name: mikeshuser/data-prep-and-pull

max_forks_repo_name: mikeshuser/data-prep-and-pull


text:
from numpy import loadtxt, where, zeros, e, array, log, ones, append, linspace
from pylab import scatter, show, legend, xlabel, ylabel, contour, title
from scipy.optimize import fmin_bfgs

def sigmoid(X):
    '''Compute the sigmoid function '''
    #d = zeros(shape=(X.shape))

    den = 1.0 + e ** (-1.0 * X)

    d = 1.0 / den

    return d

def cost_function_reg(theta, X, y, l):
    '''Compute the cost and partial derivatives as grads
    '''

    h = sigmoid(X.dot(theta))

    thetaR = theta[1:, 0]

    J = (1.0 / m) * ((-y.T.dot(log(h))) - ((1 - y.T).dot(log(1.0 - h)))) \
            + (l / (2.0 * m)) * (thetaR.T.dot(thetaR))

    delta = h - y
    sumdelta = delta.T.dot(X[:, 1])
    grad1 = (1.0 / m) * sumdelta

    XR = X[:, 1:X.shape[1]]
    sumdelta = delta.T.dot(XR)

    grad = (1.0 / m) * (sumdelta + l * thetaR)

    out = zeros(shape=(grad.shape[0], grad.shape[1] + 1))

    out[:, 0] = grad1
    out[:, 1:] = grad

    return J.flatten(), out.T.flatten()

def map_feature(x1, x2):
    '''
    Maps the two input features to quadratic features.

    Returns a new feature array with more features, comprising of
    X1, X2, X1 ** 2, X2 ** 2, X1*X2, X1*X2 ** 2, etc...

    Inputs X1, X2 must be the same size
    '''
    x1.shape = (x1.size, 1)
    x2.shape = (x2.size, 1)
    degree = 6
    out = ones(shape=(x1[:, 0].size, 1))

    m, n = out.shape

    for i in range(1, degree + 1):
        for j in range(i + 1):
            r = (x1 ** (i - j)) * (x2 ** j)
            out = append(out, r, axis=1)

    return out

#load the dataset
data = loadtxt('ex2data2.txt', delimiter=',')

X = data[:, 0:2]
y = data[:, 2]

pos = where(y == 1)
neg = where(y == 0)
scatter(X[pos, 0], X[pos, 1], marker='o', c='b')
scatter(X[neg, 0], X[neg, 1], marker='x', c='r')
xlabel('Microchip Test 1')
ylabel('Microchip Test 2')
legend(['y = 1', 'y = 0'])
#show()

m, n = X.shape

y.shape = (m, 1)

it = map_feature(X[:, 0], X[:, 1])

#Initialize theta parameters
initial_theta = zeros(shape=(it.shape[1], 1))

#Set regularization parameter lambda to 1
l = 1

# Compute and display initial cost and gradient for regularized logistic
# regression
cost, grad = cost_function_reg(initial_theta, it, y, l)

def decorated_cost(theta):
    return cost_function_reg(theta, it, y, l)

print fmin_bfgs(decorated_cost, initial_theta, maxfun=400)

theta = [
 1.273005,
 0.624876,
 1.177376,
 -2.020142,
 -0.912616,
 -1.429907,
 0.125668,
 -0.368551,
 -0.360033,
 -0.171068,
 -1.460894,
 -0.052499,
 -0.618889,
 -0.273745,
 -1.192301,
 -0.240993,
 -0.207934,
 -0.047224,
 -0.278327,
 -0.296602,
 -0.453957,
 -1.045511,
 0.026463,
 -0.294330,
 0.014381,
 -0.328703,
 -0.143796,
 -0.924883,
]

#Plot Boundary
u = linspace(-1, 1.5, 50)
v = linspace(-1, 1.5, 50)
z = zeros(shape=(len(u), len(v)))
for i in range(len(u)):
    for j in range(len(v)):
        z[i, j] = (map_feature(array(u[i]), array(v[j])).dot(array(theta)))

z = z.T
contour(u, v, z)
title('lambda = %f' % l)
xlabel('Microchip Test 1')
ylabel('Microchip Test 2')
legend(['y = 1', 'y = 0', 'Decision boundary'])
show()

def predict(theta, X):
    '''Predict whether the label
    is 0 or 1 using learned logistic
    regression parameters '''
    m, n = X.shape
    p = zeros(shape=(m, 1))

    h = sigmoid(X.dot(theta.T))

    for it in range(0, h.shape[0]):
        if h[it] > 0.5:
            p[it, 0] = 1
        else:
            p[it, 0] = 0

    return p

#% Compute accuracy on our training set
p = predict(array(theta), it)
print 'Train Accuracy: %f' % ((y[where(p == y)].size / float(y.size)) * 100.0)

reason: from numpy,from scipy


text:
"""
Data manager script with helper classes for neural system identification
"""

import pickle
import os
import numpy as np

DATA_PATH = os.path.join("/data/data_binned_responses")
FILE = "cadena_ploscb_data.pkl"

class Dataset:
    @staticmethod
    def load_data(file_name=FILE):
        """
        Load dataset pickle and return the contained dict.
        """
        file_path = os.path.join(DATA_PATH, file_name)
        with open(file_path, "rb") as g:
            return pickle.load(g)

    @staticmethod
    def manage_repeats(data_dict):
        """
        This function takes the sessions with only two repeats and duplicates them
        to match those with four repeats replacing the nans in the dict.
        """
        responses = data_dict["responses"].copy()
        idx_two_reps = data_dict["repetitions"] == 2
        responses[2:, :, idx_two_reps] = responses[:2, :, idx_two_reps]
        data_dict["responses"] = responses
        return data_dict

    @staticmethod
    def preprocess_nans(data_dict):
        """
        This function replaces nan responses by zeros and creates
        an array to identify location of original nans.
        """
        responses = data_dict["responses"].copy()
        is_realresponse = ~np.isnan(responses)
        responses[np.isnan(responses)] = 0
        data_dict["responses"] = responses
        data_dict["is_realresponse"] = is_realresponse
        return data_dict

    @staticmethod
    def add_train_test_types(data_dict, types_train="all", types_test="all"):
        """
        This function adds 'train_types' and 'test_types' fields in the
        data dictionary to restrict the stimulus types for training and testing.
        There are five types of stimulus: 'original', 'conv1', 'conv2', 'conv3', 'conv4'
        Specify 'all' to use all of types for training/testing
        """
        main_types = ["original", "conv1", "conv2", "conv3", "conv4"]

        def _process_types(types):
            if types == "all":
                types = main_types
            else:
                if not (isinstance(types, (list,))):
                    types = [types]
            return types

        data_dict["types_train"] = _process_types(types_train)
        data_dict["types_test"] = _process_types(types_test)

        return data_dict

    @staticmethod
    def get_clean_data(file_name=FILE, *args, **kwargs):
        """
        Wrapper of the functions above to get clean data dictionary for preprocessing
        """
        data_dict = Dataset.load_data(file_name)
        data_dict = Dataset.manage_repeats(data_dict)
        data_dict = Dataset.preprocess_nans(data_dict)
        data_dict = Dataset.add_train_test_types(data_dict, *args, **kwargs)
        return data_dict

class MonkeySubDataset:
    def __init__(self, data, seed=None, train_frac=0.8, subsample=1, crop=0):
        """Prepare data to be processed by model.

        Args:
            data (dict): Data loaded by Dataset.get_clean_data()
            seed (int, optional): Seed to split data into train, validation and test set. Defaults to None.
            train_frac (float, optional): Percentage of dataset to be used as train set. Defaults to 0.8.
            subsample (int, optional): Downsampling factor. Defaults to 1.
            crop (int, optional): Crop stimulus by that value from all sides before downsampling. Defaults to 0.
        """
        if crop == 0:
            images = data["images"][:, 0::subsample, 0::subsample, None]
        else:
            images = data["images"][:, crop:-crop:subsample, crop:-crop:subsample, None]
        responses = data["responses"].astype(np.float32)
        real_resps = data["is_realresponse"].astype(np.float32)
        types_train = data["types_train"]
        types_test = data["types_test"]

        # Dimensions
        num_reps, num_images, num_neurons = responses.shape
        num_train_images = int(num_images * train_frac)

        # Normalize images
        image_ids = data["image_ids"].flatten()
        image_types = data["image_types"].flatten()
        imgs_shift = np.mean(images)
        print("Subtracting mean:", imgs_shift)

        imgs_sd = np.std(images)
        images = (images - imgs_shift) / imgs_sd

        self.num_reps = num_reps

        self.images = np.tile(images[:num_train_images, :, :, :], [num_reps, 1, 1, 1])
        self.images_test = images[num_train_images:, :, :, :]

        self.responses = responses[:, :num_train_images, :].reshape([num_train_images * num_reps, num_neurons])
        self.responses_test = responses[:, num_train_images:, :]

        self.real_resps = real_resps[:, :num_train_images, :].reshape([num_train_images * num_reps, num_neurons])
        self.real_resps_test = real_resps[:, num_train_images:, :]

        self.image_ids = np.tile(image_ids[:num_train_images], [num_reps])
        self.image_ids_test = image_ids[num_train_images:]

        self.types = np.tile(image_types[:num_train_images], [num_reps])
        self.types_test = image_types[num_train_images:]

        # Select indices of image types
        idx_trn = np.array([True if x in types_train else False for x in self.types])
        idx_tst = np.array([True if x in types_test else False for x in self.types_test])

        self.images = self.images[idx_trn]
        self.images_test = self.images_test[idx_tst]

        self.responses = self.responses[idx_trn]
        self.responses_test = self.responses_test[:, idx_tst, :]

        self.real_resps = self.real_resps[idx_trn]
        self.real_resps_test = self.real_resps_test[:, idx_tst, :]

        self.image_ids = self.image_ids[idx_trn]
        self.image_ids_test = self.image_ids_test[idx_tst]

        self.types = self.types[idx_trn]
        self.types_test = self.types_test[idx_tst]

        self.num_neurons = num_neurons
        self.num_images = num_images
        self.num_train_images = int(self.images.shape[0] / num_reps)
        self.px_x = images.shape[1]
        self.px_y = images.shape[2]
        if seed:
            np.random.seed(seed)
        perm = np.random.permutation(self.num_train_images)
        train_idx = np.sort(perm[: round(self.num_train_images * train_frac)])
        self.train_idx = np.hstack([train_idx + i * self.num_train_images for i in range(num_reps)])
        val_idx = np.sort(perm[round(self.num_train_images * train_frac) :])
        self.val_idx = np.hstack([val_idx + i * self.num_train_images for i in range(num_reps)])
        self.num_train_samples = self.train_idx.size
        self.next_epoch()

        self.subject_id = data["subject_id"]
        self.repetitions = data["repetitions"]

    def nanarray(self, real_resps, resps):
        """Inserts nan at every position in resps where corresponding positions in real_resps are 0.
        Hence, leading to an array of resps where non-real responses have value nan.
        """

        return np.where(real_resps, resps, np.nan)

    def minibatch(self, batch_size):
        """Returns a minibatch of training images

        Args:
            batch_size (int): Number of samples in minibatch

        Returns:
            np.array, np.array, np.array: Returns test images, responses and real responses
                (if response of a neuron was not recorded for an image, the according real responses value is 0)
        """

        im = self.images[self.train_idx]
        res = self.responses[self.train_idx]
        isreal = self.real_resps[self.train_idx]
        if self.minibatch_idx + batch_size > len(self.train_perm):
            self.next_epoch()
        idx = self.train_perm[self.minibatch_idx + np.arange(0, batch_size)]
        self.minibatch_idx += batch_size
        return im[idx, :, :], res[idx, :], isreal[idx, :]

    def next_epoch(self):
        self.minibatch_idx = 0
        self.train_perm = np.random.permutation(self.num_train_samples)

    def train(self):
        """Get training set.

        Returns:
            np.array, np.array, np.array: Returns training images, responses and real responses
                (if response of a neuron was not recorded for an image, the according real responses value is 0)
        """

        return (
            self.images[self.train_idx],
            self.responses[self.train_idx],
            self.real_resps[self.train_idx],
        )

    def val(self):
        """Get validation set.

        Returns:
            np.array, np.array, np.array: Returns validation images, responses and real responses
                (if response of a neuron was not recorded for an image, the according real responses value is 0)
        """

        return (
            self.images[self.val_idx],
            self.responses[self.val_idx],
            self.real_resps[self.val_idx],
        )

    def test(self):
        """Get test set.

        Returns:
            np.array, np.array, np.array: Returns test images, responses and real responses
                (if response of a neuron was not recorded for an image, the according real responses value is 0)
        """

        return self.images_test, self.responses_test, self.real_resps_test


text:
"""Collection of similar classifiers and operations on them."""

import common_python.constants as cn
from common_python.util import util

import collections
import copy
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

CrossValidationResult = collections.namedtuple(
    "CrossValidationResult",
    "mean std collection"
    )

class ClassifierCollection(object):

  def __init__(self, clfs=None, features=None,
      classes=None, scores=None):
    """
    :param list-Classifier clfs: methods-fit, predict, score
    :param list-object features:
    :param list-object classes:
    :param list-float scores:
    """
    self.clfs = util.setList(clfs)
    self.features = util.setList(features)
    self.classes = util.setList(classes)
    # Score for each classifier if present
    self.scores = util.setList(scores)

  def update(self, other):
    """
    Updates the values of the collection with those in other.
    :param ClassifierCollection other:
    """
    self.clfs = copy.deepcopy(other.clfs)
    self.features = list(other.features)
    self.classes = list(other.classes)
    self.scores = list(other.scores)

  @classmethod
  def _partitionIndices(cls, container, all_indices, test_indices):
    """
    Partitions data indices into test and training instances based on
        specified test indicies.
    :param pd.DataFrame/pd.Series container:
    :param list all_indices:
    :param list test_indices:
    :return pd.DataFrame/pd.Series, pd.DataFrame/pd.Series:
    """
    train_indices = list(set(all_indices).difference(test_indices))
    if isinstance(container, pd.DataFrame):
      container_test = container.loc[test_indices, :]
      container_train = container.loc[train_indices, :]
    else:
      container_test = container.loc[test_indices]
      container_train = container.loc[train_indices]
    return container_train, container_test

  @classmethod
  def _make(cls, clf, df_X, ser_y, count, selTestIndicesFunc):
    """
    Creates a collection of fitted models using the provided
    function to select test indices.
    :param Classifier clf: untrained clf with fit, score methods
    :param pd.DataFrame df_X: columns of features, rows of instances
    :param pd.Series ser_y: state values
    :param int count: number of models to create
    :param function selTestIndicesFunc:
         Selects test holdout indices from randomly order indices
         input is pd.Series for class; output is list of indices
    :param dict kwargs: optional arguments passed to classifier
    :return ClassifierCollection:
    Notes
        1. df_X, ser_y must have the same index
    """
    def copyContainers(*containers):
      """
      Creates copies of the containers, randomizing the order or rows.
      :param list-pd.DataFrame/pd.Series containers:
      :return list-container:
      """
      new_containers = []
      indices = np.random.permutation(containers[0].index.tolist())
      for container in containers:
        container.index = indices
        new_container = container.sort_index()
        new_containers.append(new_container)
      return new_containers
    # Initializations
    clfs = []
    scores = []
    classes = list(set(ser_y.unique()))
    dff_X = df_X.copy()
    serr_y = ser_y.copy()
    indices = dff_X.index.tolist()
    # Construct the fitted classifiers
    for _ in range(count):
      # Construct test set
      new_clf = copy.deepcopy(clf)
      dff_X, serr_y = copyContainers(dff_X, serr_y)
      test_indices = selTestIndicesFunc(serr_y)
      df_X_train, df_X_test = cls._partitionIndices(
          dff_X, indices, test_indices)
      ser_y_train, ser_y_test = cls._partitionIndices(
          serr_y, indices, test_indices)
      # Train the clf and evaluate the clf
      new_clf.fit(df_X_train, ser_y_train)
      clfs.append(new_clf)
      scores.append(new_clf.score(df_X_test, ser_y_test))
    return cls(clfs, df_X.columns.tolist(),
        classes=classes, scores=scores)

  @classmethod
  def makeByRandomHoldout(cls, clf, df_X, ser_y, count, holdouts=1):
    """
    Creates a collection of fitted models with randomly chosen holdouts.
    :param Classifier clf: untrained clf with fit, score methods
    :param pd.DataFrame df_X: columns of features, rows of instances
    :param pd.Series ser_y: state values
    :param int count: number of models to create
    :param int holdouts: number of instances per state in test data
    :return ClassifierCollection:
    Notes
      1. df_X, ser_y must have the same index
    """
    def selTestIndices(ser):
      """
      :param pd.Series ser:
      :return list-indices: test indices
      """
      return ser.index.tolist()[0:holdouts]
    #
    return cls._make(clf, df_X, ser_y, count, selTestIndices)

  @classmethod
  def makeByRandomStateHoldout(cls, clf, df_X, ser_y, count, holdouts=1):
    """
    Creates a collection of fitted models with randomly
    chosen holdouts.
    :param Classifier clf: untrained clf
    :param pd.DataFrame df_X:
        columns of features, rows of instances
    :param pd.Series ser_y: state values
    :param int count: number of models to create
    :param int holdouts: number of instances
        per state in test data
    :return ClassifierCollection:
    Notes
      1. df_X, ser_y must have the same index
    """
    def selTestIndices(ser):
      """
      :param pd.Series ser:
      :return list-indices: test indices
      """
      classes = ser.unique().tolist()
      classes.sort()
      test_indices = []
      for cl in classes:
        serr = ser[ser == cl]
        if len(serr) <= holdouts:
          raise ValueError("Class %s has fewer than %d holdouts" %
              (cl, holdouts))
        idx = serr.index[0:holdouts].tolist()
        test_indices.extend(idx)
      return test_indices
    #
    return cls._make(clf, df_X, ser_y, count, selTestIndices)

  def crossValidate(self):
    """
    Computes cross validation statistics for a collection
    :return float, float:
    """
    return np.mean(self.scores), np.std(self.scores)

  @classmethod
  def crossValidateByState(cls, clf, df_X, ser_y,
      num_clfs):
    """
    Does cross validation for a classification class
    that supports the fit and score methods.
    Folds are chosen to equalize the distribution
    of states in the test set.
    :param Classifier clf: Instantiated classifier
    :param pd.DataFrame df_X: feature matrix
    :param pd.DataFrame ser_y: classes
    :param int num_clfs: Number of classifiers to create
        This is the number of iterations in the
        cross validation.
    :return CrossValidationResult:
    """
    collection = cls.makeByRandomStateHoldout(clf, df_X,
        ser_y, num_clfs)
    clf_mean, clf_std = collection.crossValidate()
    return CrossValidationResult(mean=clf_mean,
        std=clf_std, collection=collection)

max_issues_repo_name: ScienceStacks/PythonCommon

max_forks_repo_name: ScienceStacks/PythonCommon


text:
#!/home/zhihou/anaconda3/envs/tf/bin/python

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from pickle import UnpicklingError

import _init_paths
import tensorflow as tf
import numpy as np
import argparse
import pickle
import json
import ipdb
import os

from ult.config import cfg
from ult.Generate_HICO_detection import Generate_HICO_detection

def parse_args():
    parser = argparse.ArgumentParser(description='Test VCL on HICO')
    parser.add_argument('--num_iteration', dest='iteration',
            help='Specify which weight to load',
            default=1800000, type=int)
    parser.add_argument('--iter_start', dest='iter_start',
                        help='Specify which weight to load',
                        default=20000, type=int)
    parser.add_argument('--model', dest='model',
            help='Select model',
            default='VCL_ResNet50_HICO', type=str)
    parser.add_argument('--forever', dest='forever', action='store_true', default=False)
    parser.add_argument('--fuse_type', dest='fuse_type', default='spv')
    # parser.add_argument('--object_thres', dest='object_thres',
    #                     help='Object threshold',
    #                     default=0.8, type=float)
    # parser.add_argument('--human_thres', dest='human_thres',
    #                     help='Human threshold',
    #                     default=0.6, type=float)

    args = parser.parse_args()
    return args

import signal, psutil

def kill_child_processes(parent_pid, sig=signal.SIGTERM):
    try:
        parent = psutil.Process(parent_pid)
    except psutil.NoSuchProcess:
        return

    children = parent.children(recursive=True)
    # print('kill', children)
    for process in children:
        os.system('kill -9 '+str(process.pid))
        # process.send_signal(signal.SIGKILL)

if __name__ == '__main__':
    print("""I use Octave to evaluate the performace cause I did not install matlab in my machine""")
    # DATA_DIR = '/run/user/1000/gvfs/ftp:host=172.26.1.56/'
    DATA_DIR = cfg.LOCAL_DATA
    max_iteration = 3000001
    args = parse_args()
    from multiprocessing import Pool
    process_num = 2 if args.fuse_type == 'spv' else 2
    if not os.path.exists(cfg.LOCAL_DATA + '/csv'):
        os.makedirs(cfg.LOCAL_DATA + '/csv')
    # global pool
    # if pool is None:
    # pool = Pool(processes=process_num)
    pool = None
    # print(args)
    iteration = args.iteration
    iter_list = []
    stride = 5000
    if args.iter_start < 0:
        iter_list = [iteration]
    else:
        iter_list = range(args.iter_start, max_iteration, stride)

    import re
    if args.model.__contains__('*'):
        # print('line', args.model)

        r = re.compile(args.model)

        import glob
        tmp = glob.glob(DATA_DIR + '/Weights/*')
        tmp.sort(key=os.path.getmtime)
        tmp = tmp[-20:]
        # tmp = list(set(os.listdir(DATA_DIR + '/Weights/')))
        tmp = [item.split('/')[-1] for item in tmp]
        model_arr = list(filter(r.match, tmp))
        model_arr = sorted(model_arr)
        model_arr = [item.strip() for item in model_arr]
        # print([item for item in enumerate(model_arr)])
    else:
        model_arr = args.model.split(',')

    filesss = os.listdir(cfg.LOCAL_DATA + '/Results/')
    r = re.compile(args.model)
    filesss = list(filter(r.match, filesss))
    filesss = sorted(filesss)

    iteration = args.iteration
    fuse_type = args.fuse_type
    model = args.model
    for suffix in [ '_tin']:

        if model.startswith('VCL_V_') and args.fuse_type != 'v':
            continue
        cur_list = []
        result_file_name = DATA_DIR + '/csv/' + model + '_' + str(fuse_type) + suffix+'.csv'
        weight = cfg.ROOT_DIR + '/Weights/' + model + '/HOI_iter_' + str(iteration) + '.ckpt'

        import os
        output_file = cfg.LOCAL_DATA + '/Results/' + str(iteration) + '_' + model + suffix + '.pkl'
        if not os.path.exists(output_file) or os.path.getsize(output_file) < 1000:
            print("not exists", output_file)
            continue
        f = open(result_file_name, 'a')
        f.write('%6d ' % iteration)
        f.close()
        import time
        time.sleep(20)
        print ('iter = ' + str(iteration) + ', path = ' + weight  + ', fuse_type = ' + str(fuse_type))

        HICO_dir = cfg.ROOT_DIR + '/Results/HICO/' + str(iteration) + '_' + model + '_'+str(fuse_type) +'/'
        if not os.path.exists(os.environ['HOME'] + '/Results/HICO/'):
            os.makedirs(os.environ['HOME'] + '/Results/HICO/')
        # print(output_file, HICO_dir)
        from oct2py import Oct2Py

        oct = Oct2Py()
        try:
            Generate_HICO_detection(output_file, HICO_dir, fuse_type, pool)
        except EOFError as e:
            print('%s EOF' % (output_file))
            break
        except UnpicklingError as e:
            print("%s UnpicklingError" %(output_file))
            f = open(result_file_name, 'r')
            lines = f.readlines()
            w = open(result_file_name, 'a')
            for item in lines[:-1]:
                w.write(item)
            w.close()
            break
        print ('iter = ' + str(iteration) + ', path = ' + weight  + ', fuse_type = ' + str(fuse_type))
        # import os
        prefix = model  +'_'+fuse_type
        dst = 'output/ho_1_s/hico_det_test2015/rcnn_caffenet_pconv_ip_'+prefix + '_iter_' + str(iteration)
        oct.eval('addPaths')

        oct.push('dst', dst)
        oct.push('dect_dir', HICO_dir)
        oct.eval('OctGenerate_detection')

        oct.push('exp_name', 'rcnn_caffenet_ho_pconv_ip1_s_'+prefix + '_iter_' + str(iteration))
        oct.push('exp_dir', 'ho_1_s')
        oct.push('prefix', 'rcnn_caffenet_pconv_ip_'+prefix)
        oct.push('result_file', result_file_name)
        oct.push('format', 'obj')
        oct.push('score_blob', 'n/a')
        oct.push('image_set', 'test2015')
        oct.push('iter', iteration)

        oct.push('eval_mode', 'def')
        oct.eval('oct_eval_one')

        f = open(result_file_name, 'a')
        f.write('\n')
        f.close()

        import os
        print("Remove temp files", HICO_dir)
        filelist = [f for f in os.listdir(HICO_dir)]
        for f in filelist:
            if os.path.exists(os.path.join(HICO_dir, f)):
                os.remove(os.path.join(HICO_dir, f))
        os.removedirs(HICO_dir)

        remove_dir = dst
        print(dst)
        filelist = [f for f in os.listdir(remove_dir)]
        for f in filelist:
            if os.path.exists(os.path.join(remove_dir, f)):
                os.remove(os.path.join(remove_dir, f))
        os.removedirs(remove_dir)
        # kill_child_processes(os.getpid())

max_stars_repo_name: abreza/HOI-CL

max_issues_repo_name: abreza/HOI-CL

max_forks_repo_name: abreza/HOI-CL


text:
#This file obtaines calibrated predictions (Platt scaling)
import sparsechem as sc
import scipy.io
import scipy.special
import numpy as np
import pandas as pd
import torch
import sys
import argparse
from scipy.sparse import csr_matrix
from sklearn.linear_model import LogisticRegression as LR
from torch.utils.data import DataLoader
from scipy.special import expit

# argparse
parser = argparse.ArgumentParser(description="Using trained model to make predictions.")
parser.add_argument("--y_hat_test", help='Sparse pattern file with predictions for test fold.', type=str, required=True)
parser.add_argument("--y_hat_validation", help='Sparse pattern file with predictions for validation fold.', type=str, required=True)
parser.add_argument("--y_class", "--y", "--y_classification", help="Sparse pattern file for classification, optional. If provided returns predictions for given locations only (matrix market, .npy or .npz)", type=str, default=None)
parser.add_argument("--folding", help="Folds for rows of y, optional. Needed if only one fold should be predicted.", type=str, required=False)
parser.add_argument("--TargetID", type=int, required=True)
parser.add_argument("--fold_va", help="Validation fold number", type=int, default=None)
args=parser.parse_args()

#load data
TargetID=args.TargetID
y_class = sc.load_sparse(args.y_class)
y_hat_va=sc.load_sparse(args.y_hat_validation)
y_hat_te=sc.load_sparse(args.y_hat_test)

#select rows in y_class that are included in validation fold
folding = np.load(args.folding) if args.folding else None
keep    = np.isin(folding, args.fold_va)
keep_2 = np.isin(folding, 0)
y_class_va = sc.keep_row_data(y_class, keep)
y_class_te = sc.keep_row_data(y_class, keep_2)

#select Column by Target ID
y_class_te_TargetID=y_class_te[:, TargetID]
y_class_va_TargetID=y_class_va[:, TargetID]
y_hat_va_TargetID=y_hat_va[:, TargetID]
y_hat_te_TargetID=y_hat_te[:, TargetID]

#keep rows with predictions
y_class_te_selected=y_class_te_TargetID[np.nonzero(y_hat_te_TargetID)]
y_class_va_selected=y_class_va_TargetID[np.nonzero(y_hat_va_TargetID)]
y_hat_va_selected=y_hat_va_TargetID[np.nonzero(y_hat_va_TargetID)]
y_hat_te_selected=y_hat_te_TargetID[np.nonzero(y_hat_te_TargetID)]

#prepare data for logistic regression
y_class_te_selected=y_class_te_selected.A.flatten()
y_class_va_selected=y_class_va_selected.A.flatten()
y_hat_va_selected=y_hat_va_selected.A.T
y_hat_te_selected=y_hat_te_selected.A.T

# inverse sigmoid of test and validation dataset
y_hat_va_withoutSigmoid=scipy.special.logit(y_hat_va_selected)
y_hat_te_withoutSigmoid=scipy.special.logit(y_hat_te_selected)

# train model
lr=LR()
lr.fit(y_hat_va_withoutSigmoid,y_class_va_selected)

#predict values with LM model for test dataset
y_hat_calibrated=lr.predict_proba(y_hat_te_withoutSigmoid)
y_hat_platt=y_hat_calibrated[:,np.nonzero(lr.classes_==1)].flatten()
y_hat_platt=np.reshape(y_hat_platt, (-1,y_hat_platt.shape[0]))
print(lr.score(y_hat_va_withoutSigmoid,y_class_va_selected))
print(lr.score(y_hat_te_withoutSigmoid,y_class_te_selected))
#obtain csr_matrix for platt predictiond
y_hat_platt=csr_matrix(y_hat_platt)

#save prediction as np-file in folder of y_hat_test
outprefix=args.y_hat_test[:-4]
np.save(str(outprefix)+'plattScaling_TargetID'+str(TargetID), y_hat_platt)

#outprefix=args.y_hat_test[19:-4]
#np.save('predictions/repeats/'+str(TargetID)+'/'+str(outprefix)+'plattScaling_TargetID'+str(TargetID), y_hat_platt)

max_stars_repo_name: hannahrosafriesacher/SparseChem

max_issues_repo_name: hannahrosafriesacher/SparseChem

max_forks_repo_name: hannahrosafriesacher/SparseChem

reason: import numpy,import scipy,from scipy


text:
#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Stream dataset.
"""

import os
from os.path import join, exists
import numpy as np

from pgl.utils.data.dataloader import Dataloader
from pgl.utils.data.dataset import StreamDataset as PglStreamDataset

from pahelix.utils.data_utils import save_data_list_to_npz, load_npz_to_data_list

__all__ = ['StreamDataset']

class StreamDataset(object):
    """tbd"""
    def __init__(self,
            data_generator=None,
            npz_data_path=None):
        super(StreamDataset, self).__init__()
        assert (data_generator is None) ^ (npz_data_path is None), \
                "Only data_generator or npz_data_path should be set."
        self.data_generator = data_generator
        self.npz_data_path = npz_data_path

        if not npz_data_path is None:
            self.data_generator = self._load_npz_data(npz_data_path)

    def _load_npz_data(self, data_path):
        files = [file for file in os.listdir(data_path) if file.endswith('.npz')]
        for file in files:
            data_list = load_npz_to_data_list(join(data_path, file))
            for data in data_list:
                yield data

    def _save_npz_data(self, data_list, data_path, max_num_per_file=10000):
        if not exists(data_path):
            os.makedirs(data_path)

        sub_data_list = []
        count = 0
        for data in self.data_generator:
            sub_data_list.append(data)
            if len(sub_data_list) == 0:
                file = 'part-%05d.npz' % count
                save_data_list_to_npz(join(data_path, file), sub_data_list)
                sub_data_list = []
                count += 1
        if len(sub_data_list) > 0:
            file = 'part-%05d.npz' % count
            save_data_list_to_npz(join(data_path, file), sub_data_list)

    def save_data(self, data_path):
        """tbd"""
        self._save_npz_data(self.data_generator, data_path)

    def iter_batch(self, batch_size, num_workers=4, shuffle_size=1000, collate_fn=None):
        """tbd"""
        class _TempDataset(PglStreamDataset):
            def __init__(self, data_generator):
                self.data_generator = data_generator
            def __iter__(self):
                for data in self.data_generator:
                    yield data

        return Dataloader(_TempDataset(self.data_generator),
                batch_size=batch_size,
                num_workers=num_workers,
                stream_shuffle_size=shuffle_size,
                collate_fn=collate_fn)

max_stars_repo_name: WorldEditors/PaddleHelix

max_issues_repo_name: WorldEditors/PaddleHelix

max_forks_repo_name: WorldEditors/PaddleHelix


text:
from __future__ import annotations

from math import sqrt
from typing import Any, Callable, Union, Optional, Tuple, Sequence, List, TypeVar, Mapping, Dict, cast, MutableSequence
from abc import ABC, abstractmethod
import ujson as json
import numpy as np

import torch
import torch.nn as nn

import numbers
import string
from collections import defaultdict

ALL_CHARACTERS = string.printable
N_CHARACTERS = len(ALL_CHARACTERS)

JSONType = Union[str, numbers.Number, Dict[str, Any], List[Any]]
TensorPair = Tuple[torch.Tensor, torch.Tensor]
PathSeq = Tuple[Union[str, numbers.Number], ...]

def registration_name(path: PathSeq):
    return "_".join(str(p) for p in path)

def empty_embedding(mem_dim, device):
    return torch.zeros(1, mem_dim, requires_grad=True, device=device)

class DeviceModuleDict(nn.ModuleDict):
    def add_module(self, name: str, module):
        module.to(self.device)
        super().add_module(name, module)

class KeyDefaultDict(defaultdict):
    default_factory: Optional[Callable[[str], Any]]

    def __missing__(self, key: str) -> Any:
        if self.default_factory is None:
            raise KeyError(key)
        else:
            ret = self[key] = self.default_factory(key)
            return ret

# module for childsumtreelstm
class JSONNN(nn.Module, ABC):
    def __init__(
            self,
            mem_dim: int = 128,
            decode_json: bool = False,
            path_length: int = 1,
            tie_weights_primitives: bool = False,
            tie_weights_containers: bool = False,
            homogeneous_types: bool = False,
    ):
        super(JSONNN, self).__init__()

        self.mem_dim = mem_dim
        self.device = None
        self.decode_json = decode_json
        self.path_length = path_length

        self.tie_primitives = tie_weights_primitives if path_length > 0 else True
        self.tie_containers = tie_weights_containers if path_length > 0 else True
        self.homogeneous_types = homogeneous_types

    def add_module(self, name: str, module):
        module.to(self.device)
        super().add_module(name, module)

    def forward(self, node_as_json_str: str):
        if not self.decode_json:
            node_as_json_value = json.loads(node_as_json_str)
        else:
            node_as_json_value = node_as_json_str

        return self.embed_node(node_as_json_value, path=("___root___", ))
        # return self.embedNode(node_as_json_str, path_str=["___root___"])[0]

    def embed_node(self, node_as_json_value: JSONType, path: PathSeq = None) -> Optional[TensorPair]:
        path = self._canonical(path) or []

        if isinstance(node_as_json_value, dict):
            path = path if not self.tie_containers else ('___default___', )
            child_states = [child_state for child_name, child in node_as_json_value.items()
                            if (child_state := self.embed_node(child, path + (child_name, ))) is not None]
            if not child_states:
                return None
            return self.embed_object(child_states, path)

        if isinstance(node_as_json_value, list):
            path = path if not self.tie_containers else ('___default___', )
            child_states = [child_state for i, child in enumerate(node_as_json_value)
                             if (child_state := self.embed_node(child, path + (i, )))]

            if not child_states:
                return None
            return self.embed_array(child_states, path)

        elif isinstance(node_as_json_value, str):  # STRING
            path = path if not self.tie_primitives else ('___default___', )
            return self.embed_string(node_as_json_value, path)

        elif isinstance(node_as_json_value, numbers.Number):  # NUMBER
            path = path if not self.tie_primitives else ('___default___', )
            return self.embed_number(node_as_json_value, path)

        else:
            # not impl error
            return None

    @abstractmethod
    def embed_array(self, child_states: Sequence[TensorPair], path: PathSeq) -> Optional[TensorPair]:
        """Embeds JSON arrays"""

    @abstractmethod
    def embed_object(self, child_states: Sequence[TensorPair], path: PathSeq) -> Optional[TensorPair]:
        """Embeds JSON objects"""

    @abstractmethod
    def embed_string(self, node: str, path: PathSeq) -> Optional[TensorPair]:
        """Embeds JSON strings"""

    @abstractmethod
    def embed_number(self, node: numbers.Number, path: PathSeq) -> Optional[TensorPair]:
        """Embeds JSON numbers"""

    def _canonical(self, path: PathSeq) -> Tuple[str, ...]:
        # Restrict to the last `self.path_length` elements and replace ints with placeholder '___list___'
        return tuple(
                p if not isinstance(p, numbers.Number) else '___list___'
                for p in path[-self.path_length:]
        )

class ElementEmbedder(nn.Module):
    def __init__(self):
        super().__init__()

        self.paths = set()

class ObjectEmbedder(nn.Module):
    def __init__(self, mem_dim: int):
        super().__init__()

        self.mem_dim = mem_dim
        self.iouh = nn.Linear(mem_dim, 3 * mem_dim)
        self.fh = DeviceModuleDict()
        self.lout = DeviceModuleDict()
        self.paths = set()

    def add_path(self, path_str: str):
        self.paths.add(path_str)
        self.fh.add_module(path_str, nn.Linear(self.mem_dim, self.mem_dim))
        self.lout.add_module(path_str, nn.Linear(self.mem_dim, self.mem_dim))

    def forward(self, child_states: Sequence[TensorPair], path: PathSeq) -> Optional[TensorPair]:
        path_str = registration_name(path)
        if path_str not in self.paths:
            self.add_path(path_str)

        hs = []
        fcs = []
        for child_h, child_c in child_states:
            hs.append(child_h)

            f = self.fh[path_str](child_h) # Eq. 4.
            fc = torch.mul(torch.sigmoid(f), child_c) # Eq. 7.
            fcs.append(fc)

        if not hs:
            return None

        hs_bar = torch.sum(torch.cat(hs), dim=0, keepdim=True) # Eq. (2)
        iou = self.iouh(hs_bar) # Eqs. (3), (5), (6)
        i, o, u = torch.split(iou, iou.size(1) // 3, dim=1) # Eqs. (3), (5), (6)
        i, o, u = torch.sigmoid(i), torch.sigmoid(o), torch.tanh(u) # Eqs. (3), (5), (6)

        c = torch.mul(i, u) + torch.sum(torch.cat(fcs), dim=0, keepdim=True) # Eq. 7
        h = torch.mul(o, torch.tanh(c)) # Eq. (8)

        h_hat = self.lout[path_str](h)

        return h_hat, c

class ArrayEmbedder(nn.Module):
    def __init__(self, mem_dim: int):
        super().__init__()

        self.mem_dim = mem_dim
        self.lstm_cell = nn.LSTMCell(input_size=self.mem_dim * 2, hidden_size=self.mem_dim)
        self.lout = DeviceModuleDict()
        self.paths = set()

    def add_path(self, path_str: str):
        self.paths.add(path_str)
        self.lout.add_module(path_str, nn.Linear(self.mem_dim, self.mem_dim))

    def forward(self, child_states: Sequence[TensorPair], path: PathSeq) -> Optional[TensorPair]:
        path_str = registration_name(path)
        if path_str not in self.paths:
            self.add_path(path_str)

        h = empty_embedding(self.mem_dim, self.device)
        c = empty_embedding(self.mem_dim, self.device)
        for child_h, child_c in child_states:
            child_ch = torch.cat([child_h, child_c], dim=1)
            h, c = self.lstm_cell(child_ch, (h, c))
        h_hat = self.lout[path_str](h)
        return h_hat, c

class StringEmbedder(nn.Module):
    def __init__(self, mem_dim: int):
        super().__init__()

        self.mem_dim = mem_dim
        self.string_encoder = nn.Embedding(N_CHARACTERS, self.mem_dim)
        self.rnn = DeviceModuleDict()
        self.paths = set()

    def add_path(self, path_str: str):
        self.paths.add(path_str)
        self.rnn.add_module(path_str, nn.LSTM(self.mem_dim, self.mem_dim, 1))

    def forward(self, string: str, path: PathSeq) -> TensorPair:
        path_str = registration_name(path)
        if path_str not in self.paths:
            self.add_path(path_str)

        tensor_input = torch.tensor(
                [ALL_CHARACTERS.index(char) for char in string if char in ALL_CHARACTERS],
                device=self.device,
        ).long()

        encoded_input = self.string_encoder(tensor_input.view(1, -1)).view(-1, 1, self.mem_dim)
        output, (hidden, cell) = self.rnn[path_str](encoded_input)
        return hidden.mean(dim=1), empty_embedding(self.mem_dim, self.device)

class NumberEmbedder(nn.Module):
    def __init__(self, mem_dim: int, alpha: float):
        super().__init__()

        self.mem_dim = mem_dim
        self.alpha = alpha
        self.embeddings = DeviceModuleDict()
        self.statistics = DeviceModuleDict()
        self.paths = set()

    def add_path(self, path_str: str):
        self.paths.add(path_str)
        self.embeddings.add_module(path_str, nn.Linear(1, self.mem_dim))
        self.statistics.add_module(path_str, nn.Identity())
        self.statistics[path_str].register_buffer('mean', torch.zeros(1, 1, requires_grad=False))
        self.statistics[path_str].register_buffer('sqmean', torch.ones(1, 1, requires_grad=False))

    def forward(self, number: numbers.Number, path: PathSeq) -> TensorPair:
        path_str = registration_name(path)
        if path_str not in self.paths:
            self.add_path(path_str)

        encoded = self.embeddings[path_str](
                self.normalize_number(number, path_str).clone().to(dtype=torch.float32, device=self.device)
        )

        return encoded, empty_embedding(self.mem_dim, self.device)

    def normalize_number(self, number: numbers.Number, path_str: str) -> torch.Tensor:
        mean = self.statistics[path_str].get_buffer('mean')
        sqmean = self.statistics[path_str].get_buffer('sqmean')

        # If in training mode, update the mean and square mean
        # TODO: Update this only once per batch
        if self.training:
            mean.mul_(self.alpha).add_((1.0 - self.alpha) * number)
            sqmean.mul_(self.alpha).add_((1.0 - self.alpha) * number ** 2)

        return (number - mean) / torch.sqrt(sqmean - mean ** 2)

class JSONTreeLSTM(JSONNN):
    def __init__(self,
                 mem_dim: int = 128,
                 decode_json: bool = False,
                 path_length: int = 1,
                 tie_weights_containers: bool = False,
                 tie_weights_primitives: bool = False,
                 homogeneous_types: bool = False,
                 number_average_fraction: float = 0.999,
        ):
        super(JSONTreeLSTM, self).__init__(
                mem_dim,
                decode_json,
                path_length,
                tie_weights_primitives,
                tie_weights_containers,
                homogeneous_types,
        )

        self.object_embedder = ObjectEmbedder(self.mem_dim)
        self.array_embedder = ArrayEmbedder(self.mem_dim)
        self.string_embedder = StringEmbedder(self.mem_dim)
        self.number_embedder = NumberEmbedder(self.mem_dim, number_average_fraction)

        if self.homogeneous_types:
            self.embed_array = self.embed_object
            self.embed_number = lambda num, path: self.embed_string(str(num), path)

    def forward(self, node_as_json_str: str) -> torch.Tensor:
        result = super().forward(node_as_json_str)
        if result is None: return torch.cat([empty_embedding(self.mem_dim, self.device)] * 2, 1)
        return torch.cat(result, 1)

    def embed_object(self, child_states: Sequence[TensorPair], path: PathSeq) -> Optional[TensorPair]:
        return self.object_embedder(child_states, path)

    def embed_array(self, child_states: Sequence[TensorPair], path: PathSeq) -> Optional[TensorPair]:
        return self.array_embedder(child_states, path)

    def embed_string(self, string: str, path: PathSeq) -> TensorPair:
        return self.string_embedder(string, path)

    def embed_number(self, num: numbers.Number, path: PathSeq) -> TensorPair:
        return self.number_embedder(num, path)

class SetJSONNN(JSONNN):
    def __init__(self, mem_dim=128,
                 tie_weights_containers=False,
                 tie_weights_primitives=False,
                 homogeneous_types=False):
        super(SetJSONNN, self).__init__(mem_dim)

        self.mem_dim = mem_dim
        self.embedder = KeyDefaultDict(self._new_embedder)
        self.out = KeyDefaultDict(self._new_out)

        self.string_encoder = nn.Embedding(N_CHARACTERS, self.mem_dim)
        self.string_rnn = KeyDefaultDict(self._new_string_rnn)

        self.number_embeddings = KeyDefaultDict(self._new_number)
        self.number_stats = defaultdict(lambda: [])

        self.tie_primitives = tie_weights_primitives
        self.tie_containers = tie_weights_containers
        self.homogeneous_types = homogeneous_types

    def forward(self, node_as_json_str):
        if isinstance(node_as_json_str, str):
            try:
                node_as_json_str = json.loads(node_as_json_str)
            except:
                raise Exception(node_as_json_str)

        node_as_json_str = self.embed_node(node_as_json_str, path=("___root___", ))
        if node_as_json_str is None: return self._init_c()
        return node_as_json_str

    def _init_c(self):
        return torch.empty(1, self.mem_dim).fill_(0.).requires_grad_()

    def _new_embedder(self, key):
        l1 = torch.nn.Linear(self.mem_dim, self.mem_dim)
        l2 = torch.nn.Linear(self.mem_dim, self.mem_dim)
        self.add_module(str(key) + "_l1", l1)
        self.add_module(str(key) + "_l2", l2)
        model = torch.nn.Sequential(l1, torch.nn.ReLU(), l2)
        return model

    def _new_out(self, key):
        l1 = torch.nn.Linear(self.mem_dim, self.mem_dim)
        self.add_module(str(key) + "_out_l1", l1)
        # l2 = torch.nn.Linear(self.mem_dim, self.mem_dim)
        # self.add_module(str(key)+"_out_l2", l2)
        # l3 = torch.nn.Linear(self.mem_dim, self.mem_dim)
        # self.add_module(str(key)+"_out_l3", l3)
        model = torch.nn.Sequential(torch.nn.ReLU(),
                                    l1, torch.nn.ReLU(), )
        # l2, torch.nn.ReLU(),
        # l3, torch.nn.ReLU() )
        return model

    def embed_object(self, child_states, path):
        canon_path = self._canonical(path) if not self.tie_containers else "___default___"

        c_ts = []
        for c in child_states:
            c_t = self.embedder["___default___"](c)
            c_ts.append(c_t)

        if not c_ts:
            return None

        return self.out[canon_path](torch.max(torch.cat(c_ts),
                                              dim=0, keepdim=True)[0])

    def embed_array(self, child_states, path):
        canon_path = self._canonical(path) if not self.tie_containers else "___default___"

        c_ts = []
        for c in child_states:
            c_t = self.embedder[canon_path](c)
            c_ts.append(c_t)

        if not c_ts:
            return None

        return self.out[canon_path](torch.max(torch.cat(c_ts),
                                              dim=0, keepdim=True)[0])

    def _new_string_rnn(self, key):
        lstm = nn.LSTM(self.mem_dim, self.mem_dim, 1)
        self.add_module(str(key) + "_StringLSTM", lstm)
        return lstm

    def embed_string(self, string, path):
        canon_path = self._canonical(path) if not self.tie_primitives else "___default___"
        if string == "":
            return self._init_c()

        tensor_input = torch.zeros(len(string)).long()
        for c in range(len(string)):
            try:
                tensor_input[c] = ALL_CHARACTERS.index(string[c])
            except:
                continue

        encoded_input = self.string_encoder(tensor_input.view(1, -1)).view(-1, 1, self.mem_dim)
        output, hidden = self.string_rnn[canon_path](encoded_input)
        return hidden[0].mean(dim=1)

    def _new_number(self, key):
        layer = nn.Linear(1, self.mem_dim)
        self.add_module(str(key), layer)
        return layer

    def embed_number(self, num, path):
        if self.homogeneous_types:
            return self.embed_string(str(num), path)

        canon_path = self._canonical(path) if not self.tie_primitives else "___default___"

        if len(self.number_stats[canon_path]) < 100:
            self.number_stats[canon_path].append(num)

        num -= np.mean(self.number_stats[canon_path])

        if len(self.number_stats[canon_path]) > 3:
            std = np.std(self.number_stats[canon_path])
            if not np.allclose(std, 0.0):
                num /= std

        encoded = self.number_embeddings[canon_path](torch.Tensor([[num]]))
        return encoded

class FlatJSONNN(JSONNN):
    def __init__(self, mem_dim=128):
        super(FlatJSONNN, self).__init__(mem_dim)

        self.string_encoder = nn.Embedding(N_CHARACTERS, self.mem_dim)
        self.string_rnn = KeyDefaultDict(self._new_string_rnn)

        self.numberEmbeddings = KeyDefaultDict(self._new_number)
        self.numberStats = defaultdict(lambda: [])

    def forward(self, node_as_json_str):
        return self.embed_node(node_as_json_str, path=["___root___"])

    def embed_object(self, child_states, path):
        return torch.sum(torch.cat(child_states), dim=0, keepdim=True)

    def embed_array(self, child_states, path):
        return torch.sum(torch.cat(child_states), dim=0, keepdim=True)

    def _new_string_rnn(self, key):
        lstm = nn.LSTM(self.mem_dim, self.mem_dim, 1)
        self.add_module(str(key) + "_StringLSTM", lstm)
        return lstm

    def embed_string(self, string, path):
        if string == "":
            return torch.empty(1, self.mem_dim).fill_(0.).requires_grad_()

        tensor_input = torch.zeros(len(string)).long()
        for c in range(len(string)):
            try:
                tensor_input[c] = ALL_CHARACTERS.index(string[c])
            except:
                continue
        encoded_input = self.string_encoder(tensor_input.view(1, -1)).view(-1, 1, self.mem_dim)
        output, hidden = self.string_rnn[self._canonical(path)](encoded_input)
        return hidden[0][:, -1, :]

    def _new_number(self, key):
        layer = nn.Linear(1, self.mem_dim)
        self.add_module(str(key), layer)
        return layer

    def embed_number(self, num, path):
        if len(self.numberStats[self._canonical(path)]) < 100:
            self.numberStats[self._canonical(path)].append(num)

        num -= np.mean(self.numberStats[self._canonical(path)])

        if len(self.numberStats[self._canonical(path)]) > 3:
            std = np.std(self.numberStats[self._canonical(path)])
            if not np.allclose(std, 0.0):
                num /= std

        encoded = self.numberEmbeddings[self._canonical(path)](torch.Tensor([[num]]))
        return encoded

if __name__ == "__main__":

    SOME_JSON = """
    {"menu": {
        "header": "SVG Viewer",
        "id": 23.5,
        "items": [
            {"id": "Open"},
            {"id": "OpenNew", "label": "Open New"},
            null,
            {"id": "ZoomIn", "label": "Zoom In"},
            {"id": "ZoomOut", "label": "Zoom Out"},
            {"id": "OriginalView", "label": "Original View"},
            null,
            {"id": "Quality"},
            {"id": "Pause"},
            {"id": "Mute"},
            null,
            {"id": "Find", "label": "Find..."},
            {"id": "FindAgain", "label": "Find Again"},
            {"id": "Copy"},
            {"id": "CopyAgain", "label": "Copy Again"},
            {"id": "CopySVG", "label": "Copy SVG"},
            {"id": "ViewSVG", "label": "View SVG"},
            {"id": "ViewSource", "label": "View Source"},
            {"id": "SaveAs", "label": "Save As"},
            null,
            {"id": "Help"},
            {"id": "About", "label": "About Adobe CVG Viewer..."}
        ]
    }}
    """

    # json_decoded = json.loads(SOME_JSON)
    # print(" \n\n\n ")
    # print(json_decoded)
    # print("\n\n\nStarting Json decoding... ")

    # model = JSONTreeLSTM()
    # x = model(json_decoded, "some json")

    # print("Representation:")
    # print(x.forward())

    import tqdm

    HIDDEN_SIZE = 128
    embedder = JSONTreeLSTM(HIDDEN_SIZE)
    output_layer = torch.nn.Linear(HIDDEN_SIZE * 2, 2)
    model = torch.nn.Sequential(embedder, torch.nn.ReLU(), output_layer)
    criterion = torch.nn.CrossEntropyLoss()

    BATCH_SIZE = 1
    LEARNING_RATE = 0.001

    # need to run a forward pass to initialise parameters for optimizer :(
    with open("../../Datasets/dota_matches/match_{0:06d}.json".format(0)) as f:
        json_decoded = json.load(f)
    model(json_decoded)
    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)

    running_loss = 0.0
    running_acc = 0.0

    for i in tqdm.trange(50000):

        if i % BATCH_SIZE == 0:
            optimizer.zero_grad()

        with open("../../Datasets/dota_matches/match_{0:06d}.json".format(i)) as f:
            json_decoded = json.load(f)

        labs = torch.LongTensor([json_decoded['radiant_win']], )
        json_decoded['radiant_win'] = None

        # json_decoded['teamfights'] = None
        # json_decoded['players'] = None
        json_decoded['objectives'] = None
        json_decoded['radiant_gold_adv'] = None
        json_decoded['radiant_xp_adv'] = None
        json_decoded['chat'] = None
        json_decoded['tower_status_radiant'] = None
        json_decoded['tower_status_dire'] = None
        json_decoded['barracks_status_radiant'] = None
        json_decoded['barracks_status_dire'] = None
        inputs = json_decoded['teamfights']
        # print(json_decoded)

        # forward + backward + optimize
        outputs = model(inputs).view(1, -1)
        loss = criterion(outputs, labs) / BATCH_SIZE
        loss.backward()

        if (i + 1) % BATCH_SIZE == 0:
            optimizer.step()

        # print statistics
        _, predicted = torch.max(outputs, 1)
        acc = (predicted == labs)
        running_acc += acc.item()

        running_loss += loss.item() * BATCH_SIZE

        if (i + 1) % 100 == 0:  # print every 1000 mini-batches
            tqdm.tqdm.write('[%d, %5d] loss: %.3f, acc: %.3f' %
                            (1, i + 1, running_loss / 100, running_acc))
            running_loss = 0.0
            running_acc = 0.0


text:
import numpy as np
import math
from scipy.interpolate import interp1d
from src.thymio.Thymio import Thymio

# Sensor measurements
sensor_distances = np.array([i for i in range(0, 21)])
sensor_measurements = np.array([5120, 4996, 4964, 4935, 4554, 4018, 3624, 3292, 2987,
                                2800, 2580, 2307, 2039, 1575, 1127, 833, 512, 358, 157, 52, 0])
# sensor_distances = np.array([0, 30])
# sensor_measurements = np.array([5120, 0])

# Thymio outline
center_offset = np.array([5.5, 5.5])
thymio_coords = np.array([[0, 0], [11, 0], [11, 8.5], [10.2, 9.3],
                          [8, 10.4], [5.5, 11], [3.1, 10.5],
                          [0.9, 9.4], [0, 8.5], [0, 0]]) - center_offset

# Sensor positions and orientations
sensor_pos_from_center = np.array(
    [[0.9, 9.4], [3.1, 10.5], [5.5, 11.0], [8.0, 10.4], [10.2, 9.3], [8.5, 0], [2.5, 0]]) - center_offset
sensor_angles = np.array([120, 105, 90, 75, 60, -90, -90]) * math.pi / 180

def sensor_val_to_cm_dist(val: int) -> int:
    """
    Interpolation from sensor values to distances in cm

    :param val: the sensor value that you want to convert to a distance
    :return:    corresponding distance in cm
    """
    if val == 0:
        return np.inf

    f = interp1d(sensor_measurements, sensor_distances)
    return np.asscalar(f(val))

def obstacles_pos_from_sensor_val(sensor_val):
    """
    Returns a list containing the position of the obstacles
    w.r.t the center of the Thymio robot.

    :param sensor_val:     sensor values provided clockwise starting from the top left sensor.
    :return: numpy.array()  that contains the position of the different obstacles
    """
    dist_to_sensor = [sensor_val_to_cm_dist(x) for x in sensor_val]
    dx_from_sensor = [d * math.cos(alpha) for (d, alpha) in zip(dist_to_sensor, sensor_angles)]
    dy_from_sensor = [d * math.sin(alpha) for (d, alpha) in zip(dist_to_sensor, sensor_angles)]
    obstacles_pos = [[x[0] + dx, x[1] + dy] for (x, dx, dy) in
                     zip(sensor_pos_from_center, dx_from_sensor, dy_from_sensor)]
    return np.array(obstacles_pos)

class SensorHandler:
    """
    Get the data from every sensors

    :param thymio:      class of the robot to refer to
    """

    def __init__(self, thymio: Thymio):
        self.thymio = thymio

    def ground(self):
        return {"ground": self.thymio["prox.ground.reflected"]}

    def speed(self):
        return {"left_speed": self.thymio["motor.left.speed"],
                "right_speed": self.thymio["motor.right.speed"]}

    def sensor_raw(self):
        return {"sensor": self.thymio["prox.horizontal"]}

    def all_raw(self):
        """
        Fetch the all the data data of thymio

        :return: return the data of thymio of the time interval
        """
        return {"sensor": self.thymio["prox.horizontal"],
                "ground": self.thymio["prox.ground.reflected"],
                "left_speed": self.thymio["motor.left.speed"],
                "right_speed": self.thymio["motor.right.speed"]}

    def sensor_cm(self):
        val = {"sensor": self.thymio["prox.horizontal"]}
        return obstacles_pos_from_sensor_val(val['sensor'])

    def all_cm(self):
        """
        Fetch the all the data data of thymio and converts sensor to cm

        :return: return the data of thymio of the time interval
        """
        return {"sensor": obstacles_pos_from_sensor_val(self.thymio["prox.horizontal"]),
                "ground": self.thymio["prox.ground.reflected"],
                "left_speed": self.thymio["motor.left.speed"],
                "right_speed": self.thymio["motor.right.speed"]}

max_stars_repo_name: KookaS/Basics-mobile-robotics

max_issues_repo_name: KookaS/Basics-mobile-robotics

max_forks_repo_name: KookaS/basics-mobile-robotics

reason: import numpy,from scipy


text:
import streamlit as st

from pymatreader import read_mat

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.decomposition import PCA
from sklearn import metrics

st.title('Hyperspectral Image Classification')
Dataset = st.sidebar.selectbox('select Dataset',('Indian pines','pavia university','salinas'))
classify = st.sidebar.selectbox('select classifier',('SVM','decisiontree','KNeighborsClassifier','RandomForestClassifier'))
st.balloons()

def main():
    if Dataset == 'Indian pines':
        x = read_mat('Indian_pines_corrected.mat')['indian_pines_corrected']
        y = read_mat('Indian_pines_gt.mat')['indian_pines_gt']
        ys = y.shape
        names = ['Alfalfa', 'Corn-notill', 'Corn-mintill', 'Corn'
                        ,'Grass-pasture', 'Grass-trees', 'Grass-pasture-mowed',
                        'Hay-windrowed', 'Oats', 'Soybean-notill', 'Soybean-mintill',
                        'Soybean-clean', 'Wheat', 'Woods', 'Buildings-Grass-Trees-Drives',
                        'Stone-Steel-Towers']
    elif Dataset == 'pavia university':
        x = read_mat('PaviaU.mat')['paviaU']
        y = read_mat('PaviaU_gt.mat')['paviaU_gt']
        ys = y.shape
        names = ['Asphalt', 'Meadows', 'Gravel', 'Trees', 'Painted metal sheets', 'Bare Soil', 'Bitumen',
                 'Self-Blocking Bricks', 'Shadows']
    elif Dataset == 'salinas':
        x = read_mat('Salinas_corrected.mat')['salinas_corrected']
        y = read_mat('Salinas_gt.mat')['salinas_gt']
        ys = y.shape
        names = ['Brocoli_green_weeds_1','Brocoli_green_weeds_2','Fallow','Fallow_rough_plow','Fallow_smooth',
                        'Stubble','Celery','Grapes_untrained','Soil_vinyard_develop','Corn_senesced_green_weeds',
                        'Lettuce_romaine_4wk','Lettuce_romaine_5wk','Lettuce_romaine_6wk','Lettuce_romaine_7wk',
                        'Vinyard_untrained','Vinyard_vertical_trellis']
    fig = plt.figure(figsize=(12, 6))
    q = np.random.randint(x.shape[2])
    plt.imshow(x[:, :, q], cmap='nipy_spectral')
    plt.axis('off')
    plt.title(f'Band - {q}')
    st.pyplot()
    plt.figure(figsize=(12, 6))
    plt.imshow(y, cmap='jet')
    plt.colorbar()
    plt.axis('off')
    #plt.show()
    st.title('ground_truth')
    st.pyplot()
    q = x.reshape(-1, x.shape[2])
    df = pd.DataFrame(data=q)
    df = pd.concat([df, pd.DataFrame(data=y.ravel())], axis=1)
    df.columns = [f'band{i}' for i in range(1, 1 + x.shape[2])] + ['class']
    df.to_csv('hssi.csv')
    df2 = pd.read_csv('hssi.csv')
    del df2['Unnamed: 0']
    pca = PCA(n_components=40)
    dt = pca.fit_transform(df2.iloc[:, :-1].values)
    r = pd.concat([pd.DataFrame(data=dt), pd.DataFrame(data=y.ravel())], axis=1)
    r.columns = [f'PC-{i}' for i in range(1, 41)] + ['class']
    r.head()
    r.to_csv('hssi_after_pca.csv')
    fig = plt.figure(figsize=(12, 6))
    q = np.random.randint(x.shape[2])
    plt.imshow(x[:, :, q], cmap='nipy_spectral')
    plt.axis('off')
    plt.title(f'Band - {q}')
    f = r[r['class'] != 0]
    X = f.iloc[:, :-1].values
    y = f.loc[:, 'class'].values
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=11, stratify=y)
    if classify == 'SVM':
        svm = SVC(C=100, kernel='rbf', cache_size=10 * 1024)
        svm.fit(X_train, y_train)
        ypred = svm.predict(X_test)
        data = confusion_matrix(y_test, ypred)
        df_cm = pd.DataFrame(data, columns=np.unique(names), index=np.unique(names))
        df_cm.index.name = 'Actual'
        df_cm.columns.name = 'Predicted'
        plt.figure(figsize=(12, 6))
        sns.set(font_scale=1.4)
        st.title('Confusion Matrix')
        sns.heatmap(df_cm, cmap="Reds", annot=True, annot_kws={"size": 16}, fmt='d')
        st.write('Confusion Matrix')
        st.pyplot()
        accuracy = classification_report(y_test, ypred, target_names=names)
        st.write(accuracy)
        l = []
        for i in range(r.shape[0]):
            if r.iloc[i, -1] == 0:
                l.append(0)
            else:
                l.append(svm.predict(r.iloc[i, :-1].values.reshape(1, -1)))
        clmap = np.array(l).reshape(ys[0],ys[1], ).astype('float')
        plt.figure(figsize=(12, 6))
        plt.imshow(clmap, cmap='nipy_spectral')
        plt.colorbar()
        plt.axis('off')
        st.title('Classification Map')
        st.pyplot()
        plt.figure(figsize=(12, 6))
        pixel_no = np.random.randint(df2.shape[0])
        plt.plot(range(1, 201), df2.iloc[pixel_no, :-1].values.tolist(), 'b--', label=f'Class - {df2.iloc[pixel_no, -1]}')
        plt.legend()
        plt.title(f'Pixel({pixel_no}) signature', fontsize=14)
        plt.xlabel('Band Number', fontsize=14)
        plt.ylabel('Pixel Intensity', fontsize=14)
        st.pyplot()
    elif classify == 'decisiontree':
        model = DecisionTreeClassifier()
        model = model.fit(X_train, y_train)
        y_pred = model.predict(X_test)
        data = confusion_matrix(y_test, y_pred)
        df_cm = pd.DataFrame(data, columns=np.unique(names), index=np.unique(names))
        df_cm.index.name = 'Actual'
        df_cm.columns.name = 'Predicted'
        plt.figure(figsize=(12, 6))
        sns.set(font_scale=1.4)
        st.title('Confusion Matrix')
        sns.heatmap(df_cm, cmap="Reds", annot=True, annot_kws={"size": 16}, fmt='d')
        st.pyplot()
        st.write("Accuracy:", metrics.accuracy_score(y_test, y_pred))
        l2 = []
        for i in range(r.shape[0]):
            if r.iloc[i, -1] == 0:
                l2.append(0)
            else:
               l2.append(model.predict(r.iloc[i, :-1].values.reshape(1, -1)))
        cl2map = np.array(l2).reshape(ys[0],ys[1], ).astype('float')
        plt.figure(figsize=(12, 6))
        plt.imshow(cl2map, cmap='nipy_spectral')
        plt.colorbar()
        plt.axis('off')
        st.title('Classification Map')
        st.pyplot()
        accuracy = classification_report(y_test, y_pred, target_names=names)
        st.write(accuracy)
    elif classify == 'KNeighborsClassifier':
        model = KNeighborsClassifier(n_neighbors=5)
        model = model.fit(X_train, y_train)
        y_pred = model.predict(X_test)
        st.write("Accuracy:", metrics.accuracy_score(y_test, y_pred))
        data = confusion_matrix(y_test, y_pred)
        df_cm = pd.DataFrame(data, columns=np.unique(names), index=np.unique(names))
        df_cm.index.name = 'Actual'
        df_cm.columns.name = 'Predicted'
        plt.figure(figsize=(12, 6))
        sns.set(font_scale=1.4)
        st.title('Confusion Matrix')
        sns.heatmap(df_cm, cmap="Reds", annot=True, annot_kws={"size": 16}, fmt='d')
        st.pyplot()
        l3 = []
        for i in range(r.shape[0]):
            if r.iloc[i, -1] == 0:
                l3.append(0)
            else:
                l3.append(model.predict(r.iloc[i, :-1].values.reshape(1, -1)))
        cl3map = np.array(l3).reshape(ys[0],ys[1], ).astype('float')
        plt.figure(figsize=(12, 6))
        plt.imshow(cl3map, cmap='nipy_spectral')
        plt.colorbar()
        plt.axis('off')
        st.title('Classification Map')
        st.pyplot()
        accuracy = classification_report(y_test, y_pred, target_names=names)
        st.write(accuracy)
    elif classify == 'RandomForestClassifier':
        model3 = RandomForestClassifier(n_estimators=100)
        model3.fit(X_train, y_train)
        y_pred = model3.predict(X_test)
        st.write("Accuracy:", metrics.accuracy_score(y_test, y_pred))
        data = confusion_matrix(y_test, y_pred)
        df_cm = pd.DataFrame(data, columns=np.unique(names), index=np.unique(names))
        df_cm.index.name = 'Actual'
        df_cm.columns.name = 'Predicted'
        plt.figure(figsize=(12, 6))
        sns.set(font_scale=1.4)
        st.title('Confusion Matrix')
        sns.heatmap(df_cm, cmap="Reds", annot=True, annot_kws={"size": 16}, fmt='d')
        st.pyplot()
        l4 = []
        for i in range(r.shape[0]):
            if r.iloc[i, -1] == 0:
                l4.append(0)
            else:
                l4.append(model3.predict(r.iloc[i, :-1].values.reshape(1, -1)))
        cl4map = np.array(l4).reshape(ys[0],ys[1],).astype('float')
        plt.figure(figsize=(12, 6))
        plt.imshow(cl4map, cmap='nipy_spectral')
        plt.colorbar()
        plt.axis('off')
        st.title('Classification Map')
        st.pyplot()
        st.write(classification_report(y_test, y_pred, target_names=names))

if st.sidebar.button('classify'):
        main()


text:
from Reward import Reward
import numpy as np

class LinearReward(Reward):
	def __init__(self, arg_dict={}):
		Reward.__init__(self, arg_dict)
		# for key in reward_dict:
		# 	setattr(self, key, reward_dict[key])

	def getTheta(self, user):
		return user.theta

	def getReward(self, user, pickedArticle):
		return np.dot(user.theta, pickedArticle.featureVector)

	def get_user_features(self, user):
		return user.theta

max_stars_repo_name: cyrilli/BanditLib

max_issues_repo_name: cyrilli/BanditLib

max_forks_repo_name: cyrilli/BanditLib


text:
"""
Load a CSV file into a record array and edit it in a gtk treeview
"""

import gtk
import numpy as np
import matplotlib.mlab as mlab
import matplotlib.cbook as cbook
import mpl_toolkits.gtktools as gtktools

datafile = cbook.get_sample_data('demodata.csv', asfileobj=False)
r = mlab.csv2rec(datafile, converterd={'weekdays': str})

liststore, treeview, win = gtktools.edit_recarray(r)
win.set_title('click to edit')
win.connect('delete-event', lambda *args: gtk.main_quit())
gtk.main()

max_stars_repo_name: jbbrokaw/matplotlib

max_issues_repo_name: jbbrokaw/matplotlib

max_forks_repo_name: jbbrokaw/matplotlib


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    In this module we define a helper object for training.
    It records most of training and validation metrics at each epoch.
    It defines an early stopping criteria.
"""

import os
from os.path import isfile
import numpy as np

import pandas as pd

def fill_table_line(data, index, dic):
    """
    Fills one line in specific table with two list
    values and their corresponding names.
    """
    for name, val in dic.items():
        if not np.isscalar(val):
            val = "NOT SCALAR"
        data.loc[index, name] = val
    return data

def any_bigger_in_array(data, tracked_var, lag, eps, increase=True):
    """
    Checks on a pandas table if to early stop with respect to the variable_name
    """
    data = data.fillna(0)
    stop = False
    tracked_values = np.array(data[tracked_var])
    if not increase:
        tracked_values = tracked_values * -1
    if len(tracked_values) > lag - 1:
        lagged_value = tracked_values[-lag]
        after_lagged_values = tracked_values[-(lag-1):]
        time_to_stop = ((lagged_value - after_lagged_values) > -1*eps).all()
        stop = time_to_stop
    return stop

class ScoreRecorder(object):
    """
    Useful object to store training and test metrics
    during training.
    It can also perform early stopping.
    """
    def __init__(self, saver, session, log,
                 stop_early=True, lag=10, eps=10**-4):

        self.train_file_name = os.path.join(log, 'train_scores.csv')
        self.test_file_name = os.path.join(log, 'test_scores.csv')

        if isfile(self.train_file_name):
            self.data_train = pd.read_csv(self.train_file_name, index_col=0)
        else:
            self.data_train = pd.DataFrame()

        if isfile(self.test_file_name):
            self.data_test = pd.read_csv(self.test_file_name, index_col=0)
        else:
            self.data_test = pd.DataFrame()

        self.steps_before_allowed_to_check = lag
        self.track = None
        self.eps = eps
        self.lag = lag
        self.log = log
        self.stop_early = stop_early
        self.saver = saver
        self.sess = session
#        self.saver._max_to_keep = lag + 1

    def all_tables(self):
        """
        return all_tables
        """
        return {'train': self.data_train, 'test': self.data_test}

    def diggest(self, epoch_number, training_dic, train=True):
        """
        Collects and fills a table for two given list (names and values)
        Do be used in the for loop training
        """
        if train:
            self.data_train = fill_table_line(self.data_train, epoch_number,
                                              training_dic)
            self.data_train.to_csv(self.train_file_name)
        else:
            self.data_test = fill_table_line(self.data_test, epoch_number,
                                             training_dic)
            self.data_test.to_csv(self.test_file_name)
    def find_best_epoch(self, tracking_variable, increase=True, train_set=True):
        """
        Find best epoch for model test set
        """
        if train_set:
            data = self.data_train.fillna(0.)
        else:
            data = self.data_test.fillna(0.)
        if increase:
            lagged_value = data.tail(self.lag + 1)[tracking_variable].idxmax()
        else:
            lagged_value = data.tail(self.lag + 1)[tracking_variable].idxmin()
        return lagged_value

    def save_best(self, tracking_variable, save_weights=True, train_set=True):
        """
        Save best model
        """
        increase = True
        if tracking_variable == "loss":
            increase = False
        best_epoch = self.find_best_epoch(tracking_variable, increase, train_set)
        self.saver.restore(self.sess, "{}/model.ckpt-{}".format(self.log, best_epoch + 1))
        last_step = self.data_test.index.max()
        if best_epoch != last_step:
            # only does on the test set
            if train_set:
                self.data_train.loc[last_step + 1] = self.data_train.loc[best_epoch]
            else:
                self.data_test.loc[last_step + 1] = self.data_test.loc[best_epoch]
            if save_weights:
                self.saver.save(self.sess, self.log + '/' + "model.ckpt", last_step + 2)

    def stop(self, tracking_variable, train_set=True):
        """
        If or not to perform early stopping for a given
        tracking variable on the test set.
        """
        self.steps_before_allowed_to_check = max(self.steps_before_allowed_to_check - 1, 0)
        if self.steps_before_allowed_to_check == 0:
            if self.stop_early:
                increase = True
                if tracking_variable == "loss":
                    increase = False
                if train_set:
                    stop = any_bigger_in_array(self.data_train, tracking_variable,
                                               self.lag, self.eps, increase)
                else:
                    stop = any_bigger_in_array(self.data_test, tracking_variable,
                                               self.lag, self.eps, increase)
            else:
                stop = False
        else:
            stop = False
        return stop