text:
# Credits for some snippets of code to https://github.com/dandrewmyers/numerical/blob/master/mc_pi.py

import matplotlib.pyplot as plt # for visual representations of the data
import numpy as np
import xlsxwriter as xlsx # to export data to excel workbooks
import time as time  # to show time to calculate

# input total number of random points
total_random_points = 100000  # int(input("\nNumber of random points for Monte Carlo estimate of Pi?\n>"))

# start time of calculation
start_time = time.time()

# name for new workbook. Convention will be (Trial length)Pi(Trial Number)
name = str(total_random_points) + 'Pi' + str(10) + '.xlsx'
# setting up an Excel workbook for the project
workbook = xlsx.Workbook(name)
worksheet = workbook.add_worksheet()

# number of random points inside unit circle and total random points
inside_circle = 0

# create empty x and y arrays for eventual scatter plot of generated random points
x_plot_array = np.empty(shape=(1, total_random_points))
y_plot_array = np.empty(shape=(1, total_random_points))

# create specific X and Y arrays for color rules
x_inside_plot_array = np.empty(shape=(1, total_random_points))
x_outside_plot_array = np.empty(shape=(1, total_random_points))

y_inside_plot_array = np.empty(shape=(1, total_random_points))
y_outside_plot_array = np.empty(shape=(1, total_random_points))

# generate random points and count points inside unit circle
# top right quadrant of unit circle only
for i in range(0, total_random_points):
    # generate random x, y in range [0, 1]
    x = np.random.rand()
    x_plot_array = np.append(x_plot_array, [x])
    y = np.random.rand()
    y_plot_array = np.append(y_plot_array, [y])
    # calc x^2 and y^2 values
    x_squared = x ** 2
    y_squared = y ** 2
    # count if inside unit circle, top right quadrant only
    if np.sqrt(x_squared + y_squared) < 1.0:
        inside_circle += 1
        x_inside_plot_array = np.append(x_inside_plot_array, [x])
        y_inside_plot_array = np.append(y_inside_plot_array, [y])
    else:
        x_outside_plot_array = np.append(x_outside_plot_array, [x])
        y_outside_plot_array = np.append(y_outside_plot_array, [y])
    # calc approximate pi value
    pi_approx = inside_circle / (i + 1) * 4

# final numeric output for pi estimate. Outputted to terminal window.
print("\n--------------")
print(f"\nApproximate value for pi: {pi_approx}")
print(f"Difference to exact value of pi: {pi_approx-np.pi}")
print(f"Percent Error: (approx-exact)/exact*100: {(pi_approx-np.pi)/np.pi*100}%")
print(f"Execution Time: {time.time() - start_time} seconds\n")

# setting up the excel export

worksheet.write_string(0, 0, 'X values')
worksheet.write_string(0, 1, 'Y values')

# creating new arrays to deal with the nonsense data
length_array = len(x_plot_array)
X_Values = x_plot_array[total_random_points:length_array]
Y_Values = y_plot_array[total_random_points:length_array]

# creating a 2d array to make it easier to export to excel
combined = np.vstack((X_Values, Y_Values))

# excel export
row = 1
for col, data in enumerate(combined):
    worksheet.write_column(row, col, data)

worksheet.write(0, 5, 'Approximate value for pi:')
worksheet.write(0, 6, pi_approx)
worksheet.write(1, 5, 'Difference to exact value of pi:')
worksheet.write(1, 6, pi_approx - np.pi)
worksheet.write(2, 5, 'Percent Error: (approx-exact)/exact*100:')
worksheet.write(2, 6, (pi_approx - np.pi) / np.pi * 100)
worksheet.write(3, 5, 'Execution Time:')
worksheet.write(3, 6, str(time.time() - start_time) + ' seconds')
workbook.close()

# plot output of random points and circle.
inside_circle_plot = plt.scatter(x_inside_plot_array, y_inside_plot_array, color='blue', s=.1)
outside_circle_plot = plt.scatter(x_outside_plot_array, y_outside_plot_array, color='red', s=.1)
circle_plot = plt.Circle((0, 0), 1, color='black', linewidth=2, fill=False)

ax = plt.gca()
ax.cla()

ax.add_artist(inside_circle_plot)
ax.add_artist(outside_circle_plot)
ax.add_artist(circle_plot)

plt.show()
print('test')

max_stars_repo_name: IkeKap/StatsProject

max_issues_repo_name: IkeKap/StatsProject

max_forks_repo_name: IkeKap/StatsProject


text:
# /*******************************************************************************
# *  Author : Xiao Wang
# *  Email  : wang3702@purdue.edu xiaowang20140001@gmail.com
# *******************************************************************************/
import torch
from torch.autograd import Variable
import time
import os
import torch.nn.functional as F
import sys
from ops.utils import AverageMeter
from ops.utils import Calculate_top_accuracy
from Data_Processing.data_prefetcher import data_prefetcher
import numpy as np
def Val_AETAll_fast(valloader, model, criterion, epoch,
                 use_cuda, aet_model,use_aet,aet_criterion,
                 params,Val_logger,writer,Type_Label):
    model.eval()
    if use_aet:
        aet_model1 = aet_model[0]
        aet_model2 = aet_model[1]
        aet_model3 = aet_model[2]
        aet_model4 = aet_model[3]
        aet_model5 = aet_model[4]
        aet_model1.eval()
        aet_model2.eval()
        aet_model3.eval()
        aet_model4.eval()
        aet_model5.eval()
    batch_time = AverageMeter()
    AET_loss1 = AverageMeter()
    AET_loss2 = AverageMeter()
    AET_loss3 = AverageMeter()
    AET_loss4 = AverageMeter()
    AET_loss5 = AverageMeter()
    Entropy_Loss=AverageMeter()
    Closs = AverageMeter()  # loss_x in mixmatch
    accuracies = AverageMeter()
    avg_accuracies = AverageMeter()
    top5 = AverageMeter()
    end= time.time()
    data_time = AverageMeter()
    iteration=len(valloader)
    prefetcher = data_prefetcher(valloader)
    with torch.no_grad():
        for batch_idx in range(iteration):
            read_data=prefetcher.next()
            (img1, img1_u2), img2, img3, img4, img5,img6, img7,aff_para, transform_matrix,oper_params, target=read_data
            data_time.update(time.time() - end)
            if use_cuda:
                img1, target = img1.cuda(), target.cuda(non_blocking=True)
                img2 = img2.cuda()
                img3 = img3.cuda()
                img4 = img4.cuda()
                img5 = img5.cuda()
                img6 = img6.cuda()
                img7 = img7.cuda()
                if use_aet:
                    aff_para=aff_para.cuda()
                    transform_matrix = transform_matrix.cuda()
                    oper_params=oper_params.cuda()
            outputs = model(img1)
            loss = criterion(outputs, target)
            prec1, prec5 = Calculate_top_accuracy(outputs.data, target.data, topk=(1, 5))
            Closs.update(loss.item(), img1.size(0))
            accuracies.update(prec1.item(), img1.size(0))
            top5.update(prec5.item(), img1.size(0))
            outputs_u2 = model(img1_u2)
            outputs_u3 = model(img2)
            outputs_u4 = model(img3)
            outputs_u5 = model(img4)
            outputs_u6 = model(img5)
            outputs_u7 = model(img6)
            outputs_u8 = model(img7)
            p = (torch.softmax(outputs, dim=1) + torch.softmax(outputs_u2, dim=1) ) / 2  # mean the softmax maatrix
            pt = p ** (1 / params['T'])
            logits_u = pt / pt.sum(dim=1, keepdim=True)
            p_1 = F.softmax(logits_u, dim=1)
            loss_ent=0

            p = (torch.softmax(outputs, dim=1) + torch.softmax(outputs_u2, dim=1)+torch.softmax(outputs_u3, dim=1)
                 +torch.softmax(outputs_u4, dim=1)+torch.softmax(outputs_u5, dim=1)+torch.softmax(outputs_u6, dim=1)
            +torch.softmax(outputs_u7, dim=1)+torch.softmax(outputs_u8, dim=1)) / 8  # mean the softmax maatrix
            pt = p ** (1 / params['T'])
            logits_u = pt / pt.sum(dim=1, keepdim=True)
            prec1, prec5 = Calculate_top_accuracy(logits_u.data, target.data, topk=(1, 5))
            avg_accuracies.update(prec1.item(),p_1.size(0))
            #also report the sharpened prediction(Averaged prediction)
            if use_aet:
                use_key = ['Attention']
                #projective transformation
                feature_map1,atten1 = model(img1, use_key)
                feature_map2,atten2 = model(img2, use_key)
                pred_transform = aet_model1(feature_map1, feature_map2)
                transform_matrix = transform_matrix.view(-1, 8)
                aet_loss = aet_criterion(pred_transform, transform_matrix)
                AET_loss1.update(aet_loss.item(),img1.size(0))
                #affine transformation
                feature_map2, atten2 = model(img3, use_key)
                pred_transform = aet_model2(feature_map1, feature_map2)
                aet_loss = aet_criterion(pred_transform, aff_para)
                AET_loss2.update(aet_loss.item(), img1.size(0))
                # similarity transformation
                feature_map2, atten2 = model(img4, use_key)
                pred_transform = aet_model3(feature_map1, feature_map2)
                aet_loss = aet_criterion(pred_transform, aff_para[:,:5])
                AET_loss3.update(aet_loss.item(), img1.size(0))
                # euclidean transformation
                feature_map2, atten2 = model(img5, use_key)
                pred_transform = aet_model4(feature_map1, feature_map2)
                aet_loss = aet_criterion(pred_transform, aff_para[:, :4])
                AET_loss4.update(aet_loss.item(), img1.size(0))
                # Color Contrast brightness sharpeness transformation
                feature_map2, atten2 = model(img6, use_key)
                pred_transform = aet_model5(feature_map1, feature_map2)
                aet_loss = aet_criterion(pred_transform, oper_params)
                AET_loss5.update(aet_loss.item(), img1.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()
            if use_aet:
                print('Epoch: [{0}][{1}/{2}]\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                      'AET1 {AET1.val:.4f} ({AET1.avg:.4f})\t'
                      'AET2 {AET2.val:.4f} ({AET2.avg:.4f})\t'
                      'AET3 {AET3.val:.4f} ({AET3.avg:.4f})\t'
                      'AET4 {AET4.val:.4f} ({AET4.avg:.4f})\t'
                      'AET4 {AET5.val:.4f} ({AET5.avg:.4f})\t'
                      'Closs {Closs.val:.4f} ({Closs.avg:.4f})\t'

                      'Top1 {acc.val:.3f} ({acc.avg:.3f})\t'
                      'Top5 {acc1.val:.3f} ({acc1.avg:.3f})\t'
                    'AvgTop1 {acc2.val:.3f} ({acc2.avg:.3f})\t'
                    .format(
                    epoch,
                    batch_idx + 1,
                    len(valloader),
                    batch_time=batch_time,
                    data_time=data_time,
                    AET1=AET_loss1,
                    AET2=AET_loss2,
                    AET3=AET_loss3,
                    AET4=AET_loss4,
                    AET5=AET_loss5,
                    Closs=Closs,
                    acc=accuracies,
                    acc1=top5,acc2=avg_accuracies))

    if use_aet:
        Val_logger.log({
        'epoch': epoch,
        'AET1': AET_loss1.avg,
        'AET2': AET_loss2.avg,
        'AET3': AET_loss3.avg,
        'AET4': AET_loss4.avg,
        'AET5': AET_loss5.avg,
        'Closs': Closs.avg,
        'top1': accuracies.avg,
        'top5': top5.avg,
        'top1avg':avg_accuracies.avg,
          #  'Entropy':Entropy_Loss.avg,
        })
        if writer!=None:
            writer.add_scalars('Data/AET_Loss1', {Type_Label+'_loss': AET_loss1.avg}, epoch)
            writer.add_scalars('Data/AET_Loss2', {Type_Label + '_loss': AET_loss2.avg}, epoch)
            writer.add_scalars('Data/AET_Loss3', {Type_Label + '_loss': AET_loss3.avg}, epoch)
            writer.add_scalars('Data/AET_Loss4', {Type_Label + '_loss': AET_loss4.avg}, epoch)
            writer.add_scalars('Data/AET_Loss5', {Type_Label + '_loss': AET_loss5.avg}, epoch)
            writer.add_scalars('Data/CLoss', {Type_Label+'_loss': Closs.avg}, epoch)
            #writer.add_scalars('Data/Entropy', {Type_Label + '_loss': Entropy_Loss.avg}, epoch)
            writer.add_scalars('Accuracy/top1', {Type_Label+'_accu': accuracies.avg}, epoch)
            writer.add_scalars('Accuracy/top5', {Type_Label+'_accu': top5.avg}, epoch)
            writer.add_scalars('Accuracy/top1avg', {Type_Label + '_accu': avg_accuracies.avg}, epoch)

    return Closs.avg,accuracies.avg

ext: py

max_stars_repo_name: dcohenYoobic/EnAET

max_issues_repo_name: dcohenYoobic/EnAET

max_forks_repo_name: dcohenYoobic/EnAET

reason: import numpy


text:
import os
import cv2
import numpy as np
from keras.models import model_from_json
from keras.preprocessing import image

#load model
model = model_from_json(open("fer.json", "r").read())
#load weights
model.load_weights('fer.h5')

face_haar_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

cap=cv2.VideoCapture(0)

while True:
    ret,test_img=cap.read()# captures frame and returns boolean value and captured image
    if not ret:
        continue
    gray_img= cv2.cvtColor(test_img, cv2.COLOR_BGR2GRAY)

    faces_detected = face_haar_cascade.detectMultiScale(gray_img, 1.32, 5)

    for (x,y,w,h) in faces_detected:
        cv2.rectangle(test_img,(x,y),(x+w,y+h),(255,0,0),thickness=7)
        roi_gray=gray_img[y:y+w,x:x+h]#cropping region of interest i.e. face area from  image
        roi_gray=cv2.resize(roi_gray,(48,48))
        img_pixels = image.img_to_array(roi_gray)
        img_pixels = np.expand_dims(img_pixels, axis = 0)
        img_pixels /= 255

        predictions = model.predict(img_pixels)

        #find max indexed array
        max_index = np.argmax(predictions[0])

        emotions = ('angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral')
        predicted_emotion = emotions[max_index]

        cv2.putText(test_img, predicted_emotion, (int(x), int(y)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)

    resized_img = cv2.resize(test_img, (1000, 700))
    cv2.imshow('Facial emotion analysis ',resized_img)

    if cv2.waitKey(10) == ord('q'):#wait until 'q' key is pressed
        break

cap.release()
cv2.destroyAllWindows


text:
import numpy as np
import pandas as pd
import os,sys,random
import json

def input_synergy_data(dataset):
    '''
    Load synergy datasets. load all data in the specified dataset and revise into the same format.

    param:
        dataset: str
    '''

    function_mapping = {'NCI_ALMANAC':'process_almanac', 'ONeil':'process_oneil'}

    def process_almanac():
        data = pd.read_csv("../data/synergy_data/NCI_ALMANAC/ComboDrugGrowth_Nov2017.csv")
        # summarize 3*3 or 5*3 data into one by calculating the mean score
        summary_data = data.groupby(['SCREENER','NSC1','NSC2','CELLNAME']).agg({"SCORE":'mean',"STUDY":'count'}).reset_index().rename(columns={'SCORE':'MEAN_SCORE',
                                                                                'STUDY':'count'}).astype({'NSC1':'int32','NSC2':'int32'})
        # some experiments may fail and get NA values, drop these experiments
        summary_data = summary_data.dropna()

        # remove data from 1A center (11513 data points)
        summary_data = summary_data[summary_data['SCREENER']!="1A"]

        summary_data = summary_data[['NSC1','NSC2','CELLNAME','MEAN_SCORE']].rename(columns={'NSC1':'drug1','NSC2':'drug2','CELLNAME':'cell','MEAN_SCORE':'score'})

        # replace 'MDA-MB-231/ATCC' to 'MDA-MB-231'
        summary_data = summary_data.replace('MDA-MB-231/ATCC', 'MDA-MB-231')

        return summary_data

    def process_oneil():
        pass

    # use locals() to run the specified function
    data = locals()[function_mapping[dataset]]()
    return data

def input_cellline_data(dataset):
    '''
    Load cell line features. load all data in the specified dataset and revise into the same format.
    Store all kinds of cell lines features in a dictionary.

    param:
        dataset: str
    '''

    function_mapping = {'NCI_60':'process_NCI60', 'CCLE':'process_CCLE'}

    def process_NCI60():

        def load_file(postfix):
            data = pd.read_table("../data/cell_line_data/NCI-60/data_NCI-60_%s.txt" % postfix)
            if postfix != 'mir':
                data.drop(columns=['Probe id','Gene name','Chromosome','Start','End','Cytoband','RefSeq(mRNA)','RefSeq(protein)'], inplace=True)
            else:
                data.drop(columns=['Probe id','Gene name','Chromosome','Start','End',
                                        'Cytoband','RefSeq(mRNA)','RefSeq(protein)','MirBase Name',"Entrez gene id"], inplace=True)
            # use entrez gene id as index
            if postfix != "mir":
                data.set_index("Entrez gene id", inplace=True)
            else:
                data.set_index('miRNA Accession #', inplace=True)
            # remove the prefix of the column names: "BR:MDA-MB-231" to "MDA-MB-231"
            data.columns = list(map(lambda x:x.split(":")[1], list(data.columns)))
            return data

        data_dicts = {}
        # load all cell line features
        for file_type in ['exp', 'cop', 'mut', 'met', 'pro', 'mir']:
            data_dicts[ file_type ] = load_file(file_type)

        # load RNAi features

        return data_dicts

    def process_CCLE():
        pass

    data = locals()[function_mapping[dataset]]()
    return data

def input_drug_data():
    data_dicts = {}

    # load fingerprint data
    with open("../data/drug_data/drug_fingerprint_morgan_3_256.json") as f:
        fingerprint = json.load(f)
    fingerprint = pd.DataFrame(fingerprint)
    fingerprint.columns = fingerprint.columns.astype(int)
    data_dicts['morgan_fingerprint'] = fingerprint

    # load drug target (bag of words encoding)
    drug_targets = pd.read_csv("../data/drug_data/drug_targets.csv")
    drug_mapping = dict(zip(drug_targets['nsc_id'].unique().tolist(), range(len(drug_targets['nsc_id'].unique()))))
    gene_mapping = dict(zip(drug_targets['GeneSymbol'].unique().tolist(), range(len(drug_targets['GeneSymbol'].unique()))))
    encoding = np.zeros((len(drug_targets['nsc_id'].unique()), len(drug_targets['GeneSymbol'].unique())))
    for _, row in drug_targets.iterrows():
        encoding[drug_mapping[row['nsc_id']], gene_mapping[row['GeneSymbol']]] = 1
    target_feats = dict()
    for drug, row_id in drug_mapping.items():
        target_feats[int(drug)] = encoding[row_id].tolist()
    data_dicts['drug_target'] = pd.DataFrame(target_feats)

    # load monotherapy data
    with open("../data/synergy_data/NCI_ALMANAC/monotherapy_dict.json") as f:
        monotherapy = json.load(f)
    monotherapy = pd.DataFrame(monotherapy)
    monotherapy.columns = monotherapy.columns.astype(int)
    data_dicts['monotherapy'] = monotherapy

    return data_dicts

def mapping_ids():
    pass


text:
from ..supervised.adaboost import AdaBoost
from ..helpers.utils.data_utils import data_helper
from ..helpers.utils.display import plot_dimensioner

import numpy as np
from sklearn import datasets

def adaboost():
    """
        Classifies dataset samples using Adaboost
    """
    data = datasets.load_digits()
    y = data.target
    digit_one, digit_two = 1, 8

    idx = np.append(
        np.where(y == digit_one)[0],
        np.where(y == digit_two)[0])

    y = data.target[idx]
    # Change labels [-1, 1]
    y[y == digit_one] = -1
    y[y == digit_two] = 1
    X = data.data[idx]

    X_train, X_test, y_train, y_test = data_helper.split_train_test(
        X, y, test_size=.50)

    clf = AdaBoost(n_classifiers=5)
    clf.fit(X_train, y_train)
    y_pred, acc = clf.predict(X_test, y_test=y_test)
    print('Accuracy: ', acc)

    plot_dimensioner.plot_in_two_d(
        X_test, y_pred, title='AdaBoost Classifier',
        accuracy=acc)

max_stars_repo_name: mugoh/ML

max_issues_repo_name: hogum/ML

max_forks_repo_name: hogum/ML


text:
"""Training scripts for GraphDTA backbone."""

import rdkit
import torch
import sklearn
import numpy as np
import pandas as pd
import sys, os
import os.path
from os import path
import random
from random import shuffle
from time import time
from rdkit import Chem
import torch.nn as nn
from argparse import ArgumentParser

from models.gat import GATNet
from models.gat_gcn import GAT_GCN
from models.gcn import GCNNet
from models.ginconv import GINConvNet
from utils import *
from processing import process_data
from get_len import get_kiba_len

# Set ranodm seed
seed = 1
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True

# Set loss function
loss_fn = nn.MSELoss()

# Basic settings
LOG_INTERVAL = 20

# Training script
def train(model, device, train_loader, optimizer, epoch):
    """Training script for GraphDTA backbone model.

    Args:
        model: DeepDTA backbone model.
        device: Device.
        train_loader: Dataloader of training set.
        optimizer: Optimizer.
        epoch: Epoch.

    Returns:
        loss: Ouput training loss.
    """
    print('Training on {} samples...'.format(len(train_loader.dataset)))
    model.train()
    for batch_idx, data in enumerate(train_loader):
        data = data.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = loss_fn(output, data.y.view(-1, 1).float().to(device))
        loss.backward()
        optimizer.step()
        if batch_idx % LOG_INTERVAL == 0:
            print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch,
                                                                           batch_idx * len(data.x),
                                                                           len(train_loader.dataset),
                                                                           100. * batch_idx / len(train_loader),
                                                                           loss.item()))
    return loss.item()

def predicting(model, device, loader):
    """Predicting script for GraphDTA backbone model.

    Args:
        model: GraphDTA backbone model.
        device: Device.
        loader: Dataloader of validation/testing set.

    Returns:
        res_label: Output ground truth label.
        res_pred: Output prediction.
    """
    model.eval()
    total_preds = torch.Tensor()
    total_labels = torch.Tensor()
    print('Make prediction for {} samples...'.format(len(loader.dataset)))
    with torch.no_grad():
        for data in loader:
            data = data.to(device)
            output = model(data)
            res_pred = torch.cat((total_preds, output.cpu()), 0)
            res_label = torch.cat((total_labels, data.y.view(-1, 1).cpu()), 0)
    return res_label.numpy().flatten(),res_pred.numpy().flatten()

def cal_len(path):
    """Calculate length of each group."""
    lines = open(path,'r').readlines()
    li = []
    for line in lines:
        li.append(int(line.strip()))
    lens = np.sum(li)
    return li, lens

def main(args):
    """Main function."""
    # Basic settings
    best_ci = 0
    best_epoch = 0
    best_train_loss = 10000
    rounds = args.rounds

    # Set CUDA device
    cuda_name = "cuda:" + str(args.cudanum)
    device = torch.device(cuda_name if torch.cuda.is_available() else "cpu")

    # Modeling...
    modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][args.model]
    model_st = modeling.__name__
    print(model_st)
    model = modeling().to(device)

    # Optimizer
    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)    # Adam

    # Load data
    train_data = pd.read_csv("../../Data/KIBA/CV"+str(rounds)+"/CV"+str(rounds)+"_KIBA_unseenP_seenD_train.csv")
    val_data = pd.read_csv("../../Data/KIBA/CV"+str(rounds)+"/CV"+str(rounds)+"_KIBA_unseenP_seenD_val.csv")
    test_data = pd.read_csv("../../Data/KIBA/test_KIBA_unseenP_seenD.csv")

    train_set = process_data(train_data, 'train')
    val_set = process_data(val_data, 'val')
    test_set = process_data(test_data, 'test')

    train_generator = TestbedDataset(root = 'dataset', dataset = 'KIBA_train' + str(rounds), xd = train_set[0],
                                xt = train_set[1], y = train_set[2], smile_graph = train_set[3])
    val_generator = TestbedDataset(root = 'dataset', dataset = 'KIBA_val'  + str(rounds), xd = val_set[0],
                                xt = val_set[1], y = val_set[2], smile_graph = val_set[3])
    test_generator = TestbedDataset(root = 'dataset', dataset = 'KIBA_test', xd = test_set[0],
                                xt = test_set[1], y = test_set[2], smile_graph = test_set[3])

    # Make mini-batch processing
    train_loader = DataLoader(train_generator, batch_size = args.batchsize, shuffle = True)
    val_loader = DataLoader(val_generator, batch_size = args.batchsize, shuffle = False)
    test_loader = DataLoader(test_generator, batch_size = args.batchsize, shuffle = False)

    # Training...
    print("Training.....")
    for epoch in range(args.epochs):
        print("===============Go for Training===============")
        train_loss = train(model, device, train_loader, optimizer, epoch+1)

        # Validation...
        G, P = predicting(model, device, val_loader)
        val_ci = ci(G, P)

        val_path = "../../Data/KIBA/CV"+str(rounds)+"/CV"+str(rounds)+"_val.txt"
        # Check if kiba len file exists
        if(path.exists(val_path) == False):
            get_kiba_len()

        # Calculate Weighted CI, Average CI of validation set
        li,lens = cal_len(val_path)
        s = 0
        w_ci,a_ci = [],[]
        for l in li:
            try:
                w_ci.append(l*ci(G[s:s+l],P[s:s+l]))
                a_ci.append(ci(G[s:s+l],P[s:s+l]))
            except:
                pass
            s += l
        weight_ci, average_ci = np.sum(w_ci)/np.sum(li), np.mean(a_ci)

        print("===============Go for Validation===============")
        print("Weighted CI:",weight_ci)
        print("Average CI:",average_ci)
        print("Overall CI:",val_ci)

        files = open("bestResult/GraphDTA_"+model_st+"_kiba_result"+str(args.rounds)+".txt",'a')
        files.write("val_averageCI: "+str(average_ci)+", val_weightedCI: "+str(weight_ci)+", val_overallCI: "+str(val_ci)+", train_loss: "+str(train_loss)+'\n')
        model_name = "bestModel/GraphDTA_"+model_st+"_kiba_"+str(rounds)+".model"

        # Save the best result
        if average_ci > best_ci:
            best_ci = average_ci
            best_epoch = epoch
            best_train_loss = train_loss
            # Save best model
            print("Saving the best model...")
            torch.save(model.state_dict(), model_name)

    print("===============Go for Testing===============")
    # Load the model
    model.load_state_dict(torch.load(model_name))

    # Testing...
    test_G, test_P = predicting(model, device, test_loader)
    test_CI, test_MSE = ci(test_G,test_P), mse(test_G,test_P)

    test_path = "../../Data/KIBA/kiba_len.txt"
    # Check if kiba len file exists
    if(path.exists(test_path) == False):
        get_kiba_len()
    # Calculate Weighted CI, Average CI of testing set
    t_li ,t_lens = cal_len(test_path)
    s = 0
    w_ci,a_ci = [],[]
    for l in t_li:
        try:
            w_ci.append(l*concordance_index(G[s:s+l],P[s:s+l]))
            a_ci.append(concordance_index(G[s:s+l],P[s:s+l]))
        except:
            pass
        s += l
    test_weight_ci, test_average_ci = np.sum(w_ci)/np.sum(t_li), np.mean(a_ci)

    # Save the testing result
    files.write("test_MSE:" + str(test_MSE) + ", test_averageCI:" + str(test_average_ci) +
                ", test_weightedCI:" + str(test_weight_ci) + ", test_overallCI:" + str(test_CI) + "\n")
    files.write("best_epoch:" + str(best_epoch + 1) + ", best_train_loss:" + str(best_train_loss) + "\n")

if __name__ == "__main__":
    parser = ArgumentParser(description='Starting...')

    parser.add_argument('--batchsize', default=512, type=int, metavar='N', help='Batch size')
    parser.add_argument('--epochs', default=200, type=int, metavar='N', help='Number of total epochs')
    parser.add_argument('--rounds', default=1, type=int, metavar='N', help='The Nth round')
    parser.add_argument('--lr', default=5e-4, type=float, metavar='LR', help='Initial learning rate', dest='lr')
    parser.add_argument('--cudanum', default=0, type=int, metavar='N', help='The Nth CUDA device')
    parser.add_argument('--model', default=0, type=int, metavar='N', help='Select from GINConvNet, GATNet, GAT_GCN, GCNNet')

    args = parser.parse_args()

    beginT = time()
    print("Starting Time: {}".format(beginT))
    main(args)
    endT = time()
    print("Ending Time: {}".format(endT))
    print("Duration is: {}".format(endT - beginT))

max_issues_repo_name: xueeinstein/PaddleHelix

max_forks_repo_name: xueeinstein/PaddleHelix


text:
from setuptools import setup, Extension
from Cython.Build import cythonize

import numpy

numpy_inc = numpy.get_include()

extensions = [
	Extension("cytohmm.markov_chain",
			["src/cytohmm/markov_chain.pyx"],
		include_dirs = [numpy_inc],
		),
	Extension("cytohmm.likelihood",
			["src/cytohmm/likelihood.pyx"],
		include_dirs = [numpy_inc],
		),
#	Extension("cytohmm.sequence_segmenter",
#			["src/sequence_segmenter.py"],
#		),
	]

setup(
	package_dir={'cytohmm': 'src/cytohmm'},
	ext_modules = cythonize(extensions),
	setup_requires=['numpy','cython'],
	packages=['cytohmm',],
	zip_safe=False,
)

max_stars_repo_name: harry-zuzan/CytoHMM

max_issues_repo_name: harry-zuzan/CytoHMM

max_forks_repo_name: harry-zuzan/CytoHMM


text:
import abc

import cv2
import numpy as np

import utils

class ContactDetectionPipeline(abc.ABC):
    """ abstract class for any type of contact detector """
    def __init__(self, debug=False):
        self.contact_detected_frame = None
        self.is_debug = debug

    @abc.abstractmethod
    def detect(self, frame):
        """
        gets a depth map video frame and detects contact on the canvas interface
        """
        raise NotImplementedError

class AdaptiveThresholdDetection(ContactDetectionPipeline):
    def __init__(self, flip=None, debug=False):
        super().__init__(debug)
        self.kernel_3 = np.ones((3,3), np.uint8)
        self.kernel_5 = np.ones((5,5), np.uint8)
        self.flip = flip
        self.kinect_width = 640
        self.kinect_height = 480
        self.detection_padding = 10

    def detect(self, frame):
        # desaturate if necessary
        if len(frame.shape) > 2 and frame.shape[2] == 3:
            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

        if self.flip is not None:
            frame = cv2.flip(frame, self.flip)

        # create a more separate view of the not canvas objects and the canvas
        _, otsu_mask = cv2.threshold(frame, 0, 255, cv2.THRESH_OTSU)
        frame_inv = cv2.bitwise_not(frame)
        frame_objectnoise_zeroed = cv2.bitwise_and(frame_inv, otsu_mask) # object (mostly) noise zeroed, canvas is white

        if frame_objectnoise_zeroed.sum() < 100: # hard coded value
            frame_objectnoise_zeroed = frame_inv # there was no noise to be zeroed

        # first adaptive threshold shows the object noise and the contacts
        # second adaptive threshold shows only the object noise
        # subtracting the two from eachother gives the contacts
        contact_highlighted_with_noise = cv2.adaptiveThreshold(frame_objectnoise_zeroed, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 53, -3)
        possible_noise = cv2.adaptiveThreshold(frame_objectnoise_zeroed, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 53, -20)
        possible_noise_extrapolated = cv2.dilate(possible_noise, self.kernel_3, iterations=20)
        contact_highlighthed_noise_eliminated = cv2.subtract(contact_highlighted_with_noise, possible_noise_extrapolated)

        # eliminate small noise in the contacts binary image
        # and close the contact components with kernel 5 dilation 3 times
        small_noise_eliminated = cv2.erode(contact_highlighthed_noise_eliminated, self.kernel_5, iterations=1)
        connected_close_components = cv2.dilate(small_noise_eliminated, self.kernel_5, iterations=3)

        # find contours and filter those that has smaller area than 1% of the total frame area
        contours, _ = cv2.findContours(connected_close_components.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
        contact_contours = list(filter(lambda contour: cv2.contourArea(contour) < frame.shape[0]*frame.shape[1]*0.01, contours))

        if self.is_debug:
            contact_detected_frame = np.full((*frame.shape[:2], 3), 0, dtype=np.uint8)

            for contour in contact_contours:
                (x,y), radius = cv2.minEnclosingCircle(contour)
                cv2.circle(contact_detected_frame, (int(x),int(y)), 7, (0,255,0), -1)

            self.contact_detected_frame = contact_detected_frame

        # calculate contact centers and propagate
        contact_centers = []
        for contact_contour in contact_contours:
            M = cv2.moments(contact_contour)
            cX = int(M["m10"] / M["m00"])
            cY = int(M["m01"] / M["m00"])
            if cX > self.detection_padding and cX < self.kinect_width - self.detection_padding:
                if cY > self.detection_padding and cY < self.kinect_height - self.detection_padding:
                    contact_centers.append((cX,cY))

        return contact_centers

max_stars_repo_name: gaborpelesz/cubenect

max_issues_repo_name: gaborpelesz/cubenect

max_forks_repo_name: gaborpelesz/cubenect


text:
#!/usr/bin/env python
from __future__ import print_function
import os
import Queue
import yaml
import rospy
import std_msgs.msg
import numpy as np

from food_region import FoodRegion
from em3242_angle_sensor_ros.msg import EM3242_AngleSensorData

from flywheel_ros.msg import FlyWheelData
from flywheel_ros.msg import FoodRegionData

class FixedFoodSourcesNode(object):

    Default_Param_File = 'fixed_food_params.yaml'

    def __init__(self):
        self.start_position = None

        rospy.init_node('fixed_food_source')
        self.flywheel_data_pub = rospy.Publisher('/flywheel_data', FlyWheelData, queue_size=10)

        self.get_param()
        self.create_food_regions()
        self.start_time = rospy.Time.now().to_time()
        self.angle_data_queue = Queue.Queue()
        self.angle_data_sub = rospy.Subscriber('/em3242_angle_sensor_data', EM3242_AngleSensorData, self.on_angle_data)

    @property
    def pre_trial_end_time(self):
        return self.param['duration']['pre_trial']

    @property
    def trial_end_time(self):
        return self.pre_trial_end_time + self.param['duration']['trial']

    @property
    def post_trial_end_time(self):
        return self.trial_end_time + self.param['duration']['post_trial']

    def get_param(self):
        self.param = rospy.get_param('/flywheel', None)
        if self.param is None:
            param_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),self.Default_Param_File)
            with open(param_file_path,'r') as f:
                self.param = yaml.load(f)
        print(self.param)
        print()

    def create_food_regions(self):
        self.food_region_list = []
        for index, position in enumerate(self.param['food_sources']['position_list']):
            food_region_param = {
                    'index': index,
                    'position': position,
                    'width': self.param['food_sources']['width'],
                    'led': self.param['food_sources']['led']
                    }
            self.food_region_list.append(FoodRegion(food_region_param))

    def on_angle_data(self,data):
        self.angle_data_queue.put(data)

    def run(self):

        while not rospy.is_shutdown():

            try:
                angle_data = self.angle_data_queue.get(block=False,timeout=1.0)
            except Queue.Empty:
                continue

            if self.start_position is None:
                self.start_position = np.deg2rad(angle_data.cumulative_angle)*self.param['wheel_radius']

            fly_position = np.deg2rad(angle_data.cumulative_angle)*self.param['wheel_radius']
            fly_position -= self.start_position

            # Get ROS time and elapsed time and update food regions
            ros_time_now = rospy.Time.now()
            current_time = ros_time_now.to_time()
            elapsed_time = current_time - self.start_time
            led_enabled = True
            for food_region in self.food_region_list:
                if elapsed_time < self.pre_trial_end_time:
                    led_enabled = False
                elif elapsed_time > self.trial_end_time:
                    led_enabled = False
                    if food_region.led_scheduler.led_on:
                        food_region.led_scheduler.turn_off_led()
                food_region.update(elapsed_time,fly_position,led_enabled=led_enabled)

            # Publish data for flywheel
            if elapsed_time < self.post_trial_end_time:
                flywheel_data = FlyWheelData()
                flywheel_data.header.stamp = ros_time_now
                flywheel_data.position = fly_position
                flywheel_data.elapsed_time = elapsed_time
                for food_region in self.food_region_list:
                    region_data = FoodRegionData()
                    region_data.location = food_region.position
                    region_data.width = food_region.width
                    region_data.contains = food_region.contains(fly_position)
                    region_data.led_data.on = food_region.led_scheduler.led_on
                    region_data.led_data.value = food_region.led_scheduler.value
                    region_data.led_data.pin = food_region.led_scheduler.pin
                    flywheel_data.region_data_array.append(region_data)
                self.flywheel_data_pub.publish(flywheel_data)
                print('t = {:1.2f}, pos = {:1.2f}'.format(elapsed_time, fly_position))

# ---------------------------------------------------------------------------------------
if __name__ == '__main__':

    node = FixedFoodSourcesNode()
    node.run()


text:
# -*- coding: utf-8 -*-

"""
В этом файле собран микс из наработок Гречко и моих + гибхабовских
Вычисляются характеристические функции, вводятся функции оценок опционов в моделях Мертона и Хестона,
калибруется модель Хестона.

По некоторому размышлению, я решил оставить этот файл без изменений, но разнести по разным файлам его содержимое
"""
from numpy import *
from scipy.optimize import fmin as sfmin
# фишка подхода в том, что параметры модели передаются в виде списка p (от "parameters")

def bs_characteristic_exp(u, p, r=0.0):
    """
     Характеристическая экспонента гауссовского процесса
     u - аргумент, который привычно видеть как ksi,
     p - параметры модели (здесь только один p[0] - волатильность),
     r - безрисковая ставка
    """
    return p[0]*p[0]/2.0*u*u-1j*u*(r-p[0]*p[0]/2.0)

def characteristic_func(char_exp, u, p, r, t):
    return exp(-t*char_exp(u, p, r))

def psi(char_exp, p, v, a, r=0.0, t=0.0):
    return exp(-r*t)*characteristic_func(char_exp, complex(v, -(a+1)), p, r, t)/complex(a*a+a-v*v, (2*a+1)*v)
    # альтернативно - return exp(-r*t)*char_func(v-1j*(a+1),p,r,t)/(a*a+a-v*v+1j*(2*a+1)*v)

def call_pricing(s, kmin, kmax, char_exp, p, r=0.0, t=0.0):
    # по формуле Карра-Мадана?
    b1 = abs(log(kmin/s))
    b2 = abs(log(kmax/s))
    b = b1
    if b2 > b1:
        b = b2
    n = 4096
    a = 1.5
    l = 2*b/n
    nu = pi/b
    d = zeros(n, dtype=complex)
    v = 0.0
    d1 = -1
    d2 = 1
    for i in range(n):
        d[i] = exp(1j*b*v)*psi(char_exp, p, v, a, r, t)*nu/3.0*(3.0+d1-d2)
        v += nu
        d1 *= -1
        d2 = 0
    c = fft.fft(d)
    ki = -b
    for i in range(n):
        k = s*exp(ki)
        c[i] = exp(-a*ki)/pi*c[i]*s
        print(k, ' ', c[i].real)
        ki += l

def call_pricing2(s, k, char_exp, p, r, t):
    # не очень ясно, по какой формуле
    n = 10000
    kl = log(k/s)
    sum = 0.0
    a = 1.5
    up = 30.0
    nu = up/n
    v = 0.0
    for i in range(n):
        sum += exp(-1j*v*kl)*psi(char_exp, p, v, a, r, t)*nu
        v += nu
    c = exp(-a*kl)/pi*sum*s
    return c.real

def put_pricing2(s, k, char_exp, p, r, t):
    # через паритет
    c = call_pricing2(s, k, char_exp, p, r, t)
    return c-s+k*exp(-r*t)

def merton_char_exp(u, p, r=0.0):
    # характеристическая экспонента для модели Мертона
    # p[0] - sigma>0, p[1] - lambda>0, p[2] - delta>0, p[3] - mu
    # Похоже, u = ksi, lj = complex(j)
    g0 = r-p[0]*p[0]/2.0+p[1]*(1-exp(p[2]*p[2]/2.0+p[3]))
    return p[0]*p[0]/2.0*u*u-1j*g0*u+p[1]*(1-exp(-p[2]*p[2]/2.0*u*u+1j*p[3]*u))

def kou_char_exp(u, p, r=0.0):
    # характеристическая экспонента для модели Коу
    # p[0] - sigma>0, p[1] - c+>0, p[2] - c->0, p[3] - l+>1, p[4] - l<0
    g0=r-p[0]*p[0]/2.0+p[1]/(1-p[3])+p[2]/(1-p[4])
    # print g0
    return p[0]*p[0]/2.0*u*u-1j*g0*u+1j*p[1]*u/(1j*u-p[3])+1j*p[2]*u/(1j*u-p[4])

# def kou_char_exp2(u,p,r=0.0):
#     a=p[1]*1j*u*p[3]/(1-1j*u*p[3])+p[2]*1j*u*p[4]/(1+1j*u*p[4])
#     g0=r-p[1]/(1-p[3])-p[2]/(1+p[4])+p[1]+p[2]
#     return (g0-p[0]*p[0]/2.0)*1j*u-p[0]*p[0]/2.0*u*u+a

# (k, option_price, put/call, t), список кортежей
# здесь  true для колла, false для пута
opts = [(160000.0, 27565.0, False, 0.22666152778261167), (135000.0, 5195.0, True, 0.26456776846204105),
      (135000.0, 7565.0, False, 0.2641434669240461), (110000.0, 23960.0, True, 0.37440672424472254),
      (115000.0, 19085.0, True, 0.3187342451566085), (115000.0, 1395.0, False, 0.3134406313934762),
      (120000.0, 15470.0, True, 0.335039269474388), (120000.0, 2290.0, False, 0.303147991328373),
      (100000.0, 33315.0, True, 0.42984421394326455), (100000.0, 305.0, False, 0.3618295814823113),
      (125000.0, 11145.0, True, 0.28968298757103794), (125000.0, 3530.0, False, 0.289925002660526),
      (130000.0, 7840.0, True, 0.27524345272031414), (130000.0, 5185.0, False, 0.27372703435372564),
      (155000.0, 22770.0, False, 0.2266502659261106), (140000.0, 3170.0, True, 0.25387291512296956),
      (140000.0, 10540.0, False, 0.2534113052757669), (145000.0, 1750.0, True, 0.24333086699998477),
      (145000.0, 13855.0, False, 0.227386877016284), (150000.0, 17915.0, False, 0.20483357470742397),
      (105000.0, 28330.0, True, 0.3716282062150906)]

def rmse(char_exp, p):
    """
    Это средняя квадратическая ошибка, в некотором смысле. Она применяется дальше при калибровке на сете опционов opts.
    Принимает любую характеристическую экспоненту, которую в состоянии обработать формула call_pricing2
    Внутри в качестве процентной ставки выбирается 0, в силу природы производных ЦБ
    """
    s = 132620.0
    t = 0.200093068239
    err = 0
    for opt in opts:
        op = 0
        if opt[2]:
            op = call_pricing2(s, opt[0], char_exp, p, 0, t)
        else:
            op = put_pricing2(s, opt[0], char_exp, p, 0, t)
        err += (op-opt[1])**2
    return sqrt(err/len(opts))

def rmse_merton(p):
    return rmse(merton_char_exp, p)

def rmse_kou(p):
    return rmse(kou_char_exp, p)

def sample_data():
    # sample market data
    x = [x.split() for x in open('marketdata.txt')]
    header = x[0]
    market_datas = []
    for market_data in x[1:]:
        market_datas.append([float(elem) for elem in market_data])
    return header, market_datas

def calibrate(init_val, market_datas):
    """
    parameter set p calibration

    Параметры:
    p[0] - sigma>0,
    p[1] - lambda>0,
    p[2] - delta>0,
    p[3] - mu

    market_datas - информация с рынка с полями
    s - цена базового актива (например, фьюч)
    k - страйк
    price - цена опциона по рынку (в некотором смысле)
    r - безрисковая процентная ставка
    t - срок до истечения

    Идея калибровки состоит в том, чтобы определить некоторую функцию ошибок и начальное приближение,
    затем применить некоторую оптимизирующую функцию (здесь используется алгоритм Нелдера-Мида)
    Возможная начальная догадка: [0.2, 1.0, 0.2, 0.2]
    """
    def error(x, market_datas):
        """
        Идея калибровки в минимизации функции ошибок,
        здесь используется наивный квадрат ошибки
        """
        sigma, lambda_m, delta, mu = x  # the name lambda_m is to avoid occasional use of the keyword
        result = 0.0
        for market_data in market_datas:
            s0, k, market_price, r, T = market_data
            merton_price = call_pricing2(s0, k, merton_char_exp, (sigma, lambda_m, delta, mu), r, t)
            result += (merton_price - market_price)**2
        print(sigma, lambda_m, delta, mu, 'sqr_error = ', result)
        return result
    opt = sfmin(error, init_val, args=(market_datas,), maxiter=20)
    return opt

if __name__ == '__main__':
    import time
    s = 132620.0
    # t=0.200093068239
    t = 0.2
    # vol=0.3618
    vol = 0.3
    # strike
    k = 100000.0

    l = 7.0
    lp = 5.0
    lm = -10.0
    p = 0.6  # похоже, в контексте она используется просто как волатильность модели Б-Ш

    time_start = time.clock()
    print(call_pricing2(s, k, merton_char_exp, (vol, 1.0, 0.2, 0.2), 0.0, t))
    time_elapsed = (time.clock() - time_start)
    print('time elapsed: %2f seconds' % time_elapsed)

    header, market_datas = sample_data()
    # Initialize vol=sigma, lambda, delta, mu
    init_val = [0.2, 1.0, 0.2, 0.2]
    print(market_datas[1])
    # calibration of parameters
    sigma, lambda_m, delta, mu = calibrate(init_val, market_datas)

    # print optimize.minimize(rmse_merton,[0.2,1.0,0.2,0.2],
    #   bounds=((0,None),(0,None,),(0,None),(None,None),),method='slsqp')
    # p=[0.2,0.5,1.4,1.0/2.0,1.0/3.0]
    # print optimize.minimize(rmse_kou,[0.4,0.3,0.7,1.0/2.0,1.0/3.0],
    #   bounds=((0,None),(0,None),(0,None),(None,None),(None,None)),
    #                        method='l-bfgs-b')

    # p1=option.euro_option_put_pricing(s,0,t,vol,0.0,k)
    # print p1
    # call_pricing(s,100.0,2500.0,bs_characteristic_exp,(vol,),0.0,t)
    # print call_pricing2(s,k,bs_characteristic_exp,(vol,),0.0,t)

    # print call_pricing2(s,k,kou_char_exp,(vol,l*p,l*(1-p),lp,lm),0.0,t)
    #
    # time_start = time.clock()
    # print optimize.minimize(rmse_kou, [vol, l * p, l * (1 - p), lp, lm],
    #                        bounds=((0, 1), (0.0001, 1), (0.0001, 1), (1.0, 1), (-1, 0)),
    #                        method='slsqp')
    # time_elapsed = (time.clock() - time_start)
    # print time_elapsed
    #
    # print kou_char_exp2(-1,p)
    # print kou_char_exp2(1,p)

    # print call_pricing2(s,k,kou_char_exp2,p,0.0,t)
    # print put_pricing2(s,k,kou_char_exp2,p,0.0,t)

    # call_pricing(s,50000.0,150000.0,kou_char_exp,(vol,l*p,l*(1-p),lp,lm),0.0,t)
    # print call_pricing2(s,k,kou_char_exp,(vol,l*p,l*(1-p),lp,lm),0.0,t)

max_stars_repo_name: inwise/Pyrgos-RFBR-

max_issues_repo_name: inwise/Pyrgos-RFBR-

max_forks_repo_name: inwise/Pyrgos-RFBR-

reason: from numpy,from scipy


text:
# Lyon Foster
# ***Not the property of Masten Space Systems***
# 5/22/20

from math import pi, sin, cos, atan, sqrt, atan2
import numpy as np
import csv
import random

##############################################################################################################
def skew(v):
    # argument is a 3x1 column vector
    # returns the skew symetric 'first half of the cross product matrix' of that vector
    # so if you have 'V x W'
    # you would input 'V'
    # and this function returns 'V x'
    # reference: Savage 3.2.1-1

    vCross = np.array([[0, -v[2], v[1]],
                       [v[2], 0, -v[0]],
                       [-v[1], v[0], 0]])
    # vCross = [[0, -v[2], v[1]], [v[2], 0, -v[0]], [-v[1], v[0], 0]]

    return vCross

##############################################################################################################
def euler2dcm(phi,theta,psi):
    # Inputs are pitch(theta), yaw(psi), and roll(phi) in radians
    # can be used as a body to Earth coordinate transform

    C11 = cos(theta) * cos(psi)
    C12 = -cos(phi) * sin(psi) + sin(phi) * sin(theta) * cos(psi)
    C13 = sin(phi) * sin(psi) + cos(phi) * sin(theta) * cos(psi)

    C21 = cos(theta) * sin(psi)
    C22 = cos(phi) * cos(psi) + sin(phi) * sin(theta) * sin(psi)
    C23 = -sin(phi) * cos(psi) + cos(phi) * sin(theta) * sin(psi)

    C31 = -sin(theta)
    C32 = sin(phi) * cos(theta)
    C33 = cos(phi) * cos(theta)

    dcm = np.array([
        [C11, C12, C13],
        [C21, C22, C23],
        [C31, C32, C33]
    ])

    return dcm

##############################################################################################################
def dcm2euler(dcm):
    # dcm[i-1,j-1]

    # theta = atan( (-dcm[2,0]) / (sqrt(dcm[2,1]^2 + dcm[2,2]^2)) )
    # input is a direction cosine matrix
    # output in radians

    psi = atan2(dcm[1, 0], dcm[0, 0])
    phi = atan2(dcm[2, 1], dcm[2, 2])

    if dcm[2, 0] ** 2 < 1:
        theta = - atan2(dcm[2, 0], sqrt(1 - dcm[2, 0] ** 2))
    else:
        theta = - atan2(dcm[2, 0], 0)

    euler = np.array([theta, psi, phi])
    return euler

##############################################################################################################
def retRand(precision):
    # I didn't feel like looking for a function that returned non integer random numbers so I made this.
    # precision is the number of decimal places you'd like your random number to be
    # these numbers are between 0 and 1
    return (random.randrange(0, precision, 1)) / precision

##############################################################################################################

# creating an output csv with a bunch of random numbers as the inputs to roll pitch and yaq
def euler2dcmCSV(desiredRowsOfCSV):
    with open('euler2dcm.csv', 'w', newline='') as e2d:
        e2dWriter = csv.writer(e2d, dialect='excel')
        headers = zip(['phi'], ['psi'], ['theta'],
                      ['dcm(1,1)'], ['dcm(1,2)'], ['dcm(1,3)'],
                      ['dcm(2,1)'], ['dcm(2,2)'], ['dcm(2,3)'],
                      ['dcm(3,1)'], ['dcm(3,2)'], ['dcm(3,3)'])
        for header in headers:  # putting the headers in teh csv
            e2dWriter.writerow(header)
        for i in range(0, desiredRowsOfCSV):
            # want to control the random numbers I put in
            a = retRand(1000)
            b = retRand(1000)
            c = retRand(1000)
            print(str(euler2dcm(a, b, c)[0, 0]))  # check first column of csv against this printout
            dcm = euler2dcm(a, b, c)
            # dcm_list = dcm.ravel()
            rows = zip([str(a)], [str(b)], [str(c)], [str(dcm[0, 0])], [str(dcm[0, 1])], [str(dcm[0, 2])],
                       [str(dcm[1, 0])], [str(dcm[1, 1])], [str(dcm[1, 2])],
                       [str(dcm[2, 0])], [str(dcm[2, 1])], [str(dcm[2, 2])])
            for row in rows:
                e2dWriter.writerow(row)

##############################################################################################################

def dcm2eulerCSV(desiredRowsOfCSV):
    with open('dcm2euler.csv', 'w', newline='') as d2e:
        d2eWriter = csv.writer(d2e, dialect='excel')
        headers = zip(['dcm(1,1)'], ['dcm(1,2)'], ['dcm(1,3)'],
                      ['dcm(2,1)'], ['dcm(2,2)'], ['dcm(2,3)'],
                      ['dcm(3,1)'], ['dcm(3,2)'], ['dcm(3,3)'],
                      ['phi'], ['psi'], ['theta']
                      )

        for header in headers:
            d2eWriter.writerow(header)

        for k in range(0, desiredRowsOfCSV):
            a = retRand(1000)
            b = retRand(1000)
            c = retRand(1000)

            d = retRand(1000)
            e = retRand(1000)
            f = retRand(1000)

            g = retRand(1000)
            h = retRand(1000)
            i = retRand(1000)

            dcm = np.array([[a, b, c],
                            [d, e, f],
                            [g, h, i]
                            ])

            rows = zip([str(dcm[0, 0])], [str(dcm[0, 1])], [str(dcm[0, 2])],
                       [str(dcm[1, 0])], [str(dcm[1, 1])], [str(dcm[1, 2])],
                       [str(dcm[2, 0])], [str(dcm[2, 1])], [str(dcm[2, 2])],
                       [str(dcm2euler(dcm)[0])], [str(dcm2euler(dcm)[1])], [str(dcm2euler(dcm)[2])]
                       )
            for row in rows:
                d2eWriter.writerow(row)

##############################################################################################################

def csvREAD(filename, rowsData, colsData):
    # This function reads in the data and spits out a numpy array of the data
    # be sure to pass it the number of rows and columns of data not including the header

    with open(filename, newline='') as e2d:
        e2dReader = csv.reader(e2d, delimiter=',', dialect='excel')  # ,quoting = csv.QUOTE_NONNUMERIC)

        csvASarray = np.zeros((rowsData, colsData))

        next(e2dReader, None)

        i = 0
        for row in e2dReader:
            csvASarray[i, :] = np.array(row)
            i = i + 1

    csvASarray = csvASarray.astype(np.float)

    return csvASarray

########################################################################################################

def compareFuctions(filename,options,rowsData,colsData,precision):
    # Input the file name of the csv to be compared
    # options should be either 'euler2dcm' or 'dcm2euler' depending on which way you go
    # rowsData is the rows of data from the csv not including the header
    # colsData in the columns of data from the csv not including the header
    # Precision is how many decimal places you'd like the csv to be compared to
        # for example if you enter 3, then .2341 == .2342
    # the return type is a boolean--either true, the csv matches the functions to the specified precision
    #                               or false, it does not

    #this is for a throw at the bottom
    class bcolors:
        FAIL = '\033[91m'

    array = csvREAD(filename,rowsData,colsData)

    if(options == 'euler2dcm'):
        # the first 3 columns are the input

        inputs = array[:,0:3]

        outputs = np.zeros((rowsData,9))
        for i in range(0,rowsData,1):
            outputs[i,:] = np.reshape( euler2dcm(inputs[i,0],inputs[i,1],inputs[i,2]), 9)

        checkAgainst = array[:,3:12]

        checkSame = (np.around(outputs, precision) == np.around(checkAgainst, precision)).all()
        if not checkSame:
            print('From CSV')
            print(checkAgainst[0,:])
            print('From Function')
            print(outputs[0,:])

    elif(options == 'dcm2euler'):

        inputs = array[:, 0:9]

        outputs = np.zeros((rowsData, 3))
        for i in range(0, rowsData, 1):
            outputs[i, :] = np.reshape(dcm2euler(np.array([[inputs[i, 0], inputs[i, 1], inputs[i, 2]],
                                                          [inputs[i, 3], inputs[i, 4], inputs[i, 5]],
                                                          [inputs[i, 6], inputs[i, 7], inputs[i, 8]]])), 3)

        checkAgainst = array[:, 9:12]

        checkSame = (np.around(outputs, precision) == np.around(checkAgainst, precision)).all()
        if not checkSame:
            print('From CSV')
            print(checkAgainst[1:12,:])
            print('From Function')
            print(outputs[1:12,:])

    else:
        print(f"{bcolors.FAIL}compareFunctions accepts 'euler2dcm' or 'dcm2euler' as options{bcolors.ENDC}")

    return checkSame

########################################################################################################

print(compareFuctions('euler2dcm_sn.csv','euler2dcm',100,12,2))

max_stars_repo_name: LyonFoster/GNC-Homework

max_issues_repo_name: LyonFoster/GNC-Homework

max_forks_repo_name: LyonFoster/GNC-Homework


text:
import numpy as np

def bandit_evn(p1, p2, init_state=0, init_action=0, init_reward=0):
    def env(s, a, trial):

        # initial state action
        if trial == -1:
            return init_state, init_reward, init_action

        rnd = np.random.uniform(0, 1, 1)
        if a == 0:
            if rnd < p1:
                return s, 1, None
            return s, 0, None

        if a == 1:
            if rnd < p2:
                return s, 1, None
            return s, 0, None

        if a == -1:
            return s, 0, None

    return env

max_stars_repo_name: kylmcgr/RL-RNN-SURF

max_issues_repo_name: kylmcgr/RL-RNN-SURF

max_forks_repo_name: kylmcgr/RL-RNN-SURF


text:
import cPickle
import os
import sys
import time
import numpy as np
import pandas as pd
import cStringIO as StringIO
import urllib
import caffe

REPO_DIRNAME = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + '/../../caffe')

class ImageClassifier(object):
  default_args = {
    'model_def_file': (
             '{}/models/bvlc_reference_caffenet/deploy.prototxt'.format(REPO_DIRNAME)),
          'pretrained_model_file': (
             '{}/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel'.format(REPO_DIRNAME)),
          'mean_file': (
           '{}/python/caffe/imagenet/ilsvrc_2012_mean.npy'.format(REPO_DIRNAME)),
          'class_labels_file': (
           '{}/data/ilsvrc12/synset_words.txt'.format(REPO_DIRNAME)),
          'bet_file': (
           '{}/data/ilsvrc12/imagenet.bet.pickle'.format(REPO_DIRNAME)),
  }
  for key, val in default_args.iteritems():
    if not os.path.exists(val):
          raise Exception(
            "File for {} is missing. Should be at: {}".format(key, val))
  default_args['image_dim'] = 256
  default_args['raw_scale'] = 255.

  def __init__(self, model_def_file, pretrained_model_file, mean_file,
                 raw_scale, class_labels_file, bet_file, image_dim):
      caffe.set_mode_cpu()
      self.net = caffe.Classifier(
      model_def_file, pretrained_model_file,
      image_dims=(image_dim, image_dim), raw_scale=raw_scale,
      mean=np.load(mean_file).mean(1).mean(1), channel_swap=(2, 1, 0)
      )

      with open(class_labels_file) as f:
              labels_df = pd.DataFrame([
                  {
                    'synset_id': l.strip().split(' ')[0],
                    'name': ' '.join(l.strip().split(' ')[1:]).split(',')[0]
                  }
                  for l in f.readlines()
              ])
      self.labels = labels_df.sort('synset_id')['name'].values

      self.bet = cPickle.load(open(bet_file))
        # A bias to prefer children nodes in single-chain paths
        # I am setting the value to 0.1 as a quick, simple model.
        # We could use better psychological models here...
      self.bet['infogain'] -= np.array(self.bet['preferences']) * 0.1

  def classify_image(self, image):
        try:
            starttime = time.time()
            scores = self.net.predict([image], oversample=True).flatten()
            endtime = time.time()

            indices = (-scores).argsort()[:5]
            predictions = self.labels[indices]

            # In addition to the prediction text, we will also produce
            # the length for the progress bar visualization.
            meta = [
                (p, '%.5f' % scores[i])
                for i, p in zip(indices, predictions)
            ]
            # Compute expected information gain
            expected_infogain = np.dot(
                self.bet['probmat'], scores[self.bet['idmapping']])
            expected_infogain *= self.bet['infogain']

            # sort the scores
            infogain_sort = expected_infogain.argsort()[::-1]
            bet_result = [(self.bet['words'][v], '%.5f' % expected_infogain[v])
                          for v in infogain_sort[:5]]
            return (True, meta, bet_result, '%.3f' % (endtime - starttime))

        except Exception as err:
            return (False, 'Something went wrong when classifying the '
                           'image. Maybe try another one?')
  def classify_url(self, imageUrl):
      try:
            string_buffer = StringIO.StringIO(
              urllib.urlopen(imageUrl).read())
            image = caffe.io.load_image(string_buffer)
      except Exception as err:
            return False

      result = self.classify_image(image)
      return  result

max_stars_repo_name: Pirolf/Cabot

max_issues_repo_name: Pirolf/Cabot

max_forks_repo_name: Pirolf/Cabot


text:
import geoplot as gplt
import geopandas as gpd
import geoplot.crs as gcrs
import imageio
import pandas as pd
import pathlib
import matplotlib.pyplot as plt
import mapclassify as mc
import numpy as np
import mapclassify as mc
import time
import colorama
import os
colorama.init()
start_time = time.time()
last_time = time.time()

def timer():
    global start_time
    old_time = start_time
    start_time=time.time()
    return start_time-old_time
def timerstring():
    s=str(timer())
    if("." in s):
        s=s[:s.index(".")+2]
    return s.ljust(5)+"s "

def laptime():
    global last_time
    old_time = last_time
    last_time=time.time()
    return last_time-old_time
def lapstring():
    s=str(laptime())
    if("." in s):
        s=s[:s.index(".")+2]
    return s.ljust(5)+"s "

laptime()
timer()

has_data = False
alldata = None

palette=['DD', 'LC', 'LR/lc', 'NT', 'LR/cd', 'VU', 'EN', 'CR', 'EW', "EX"]
names = ["Data defficient","Least concern", "Lower risk", "Near threatened", "Conservation dependent-", "Vulnerable", "Endangered", "Cricitally Endangered", "Extinct in the wild", "Extinct"]

def generate_color(database,scheme=False):
    palette=['DD', 'LC', 'LR/lc', 'NT', 'LR/cd', 'VU', 'EN', 'CR', 'EW', "EX"]
    names = ["Data defficient","Least concern", "Lower risk", "Near threatened", "Conservation dependent-", "Vulnerable", "Endangered", "Cricitally Endangered", "Extinct in the wild", "Extinct"]
    categories = database["category"].unique()
    to_remove = list()
    if(len(categories)!=len(palette)):
        for index,element in enumerate(palette):
            if element not in categories:
                to_remove.append(index)
    shift=0
    for index in to_remove:
        palette.pop(index+shift)
        names.pop(index+shift)
        shift-=1
    def get_color(cat):
        return palette.index(cat)
    #print(categories)
    #print(palette)
    #print(len(categories),len(palette))
    database["color"]=database["category"].apply(get_color)
    #database.sort_values(by="color",axis=0,ascending=True,inplace=True)
    if(scheme):
        scheme=mc.EqualInterval(database["color"],k=len(palette))
        return scheme

def get_area(entry):
    return entry.area

#path="SIMPLIFIED_DATA"
path="REDLIST"
total_count=0
for dirpath, dirnames, filenames in os.walk(path):
    for name in filenames:
        if name.endswith((".shp")):
            total_count+=1
counter = 0
for dirpath, dirnames, filenames in os.walk(path):
    for name in filenames:
        if name.endswith((".shp")): #BIRCHES BONEFISH_TARPONS
            counter+=1
            if(counter>=37):
                pathname =os.path.join(dirpath, name)
                print("[%i/%i]Open"%(counter,total_count),pathname)
                db = gpd.read_file(pathname)
                print(lapstring(),"Finished reading database")
                generate_color(db) #add "color" entry

                foldername = "ORDER_AGGREGATED_DATA/"
                try:os.mkdir(foldername)
                except:pass
                # print(list(db.columns))
                orders = db["order_"].unique()
                for index,order in enumerate(orders):
                    output = db[db["order_"]==order]
                    k = output["kingdom"].iat[0]
                    p = output["phylum"].iat[0]
                    c = output["class"].iat[0]
                    outputname = k+"_"+p+"_"+c+"_"+order+"_"+name

                    output=output.explode()
                    output["area"]=output["geometry"].apply(get_area)
                    output=output[output["area"]>0.05]
                    output["geometry"]=output.simplify(0.1)

                    for type in "marine","terrestial","freshwater":
                        outputname = k+"_"+p+"_"+c+"_"+order+"_"+type+"_"+name
                        selection = output[output[type]=="true"]
                        selection = selection.filter(["category",'color', 'geometry'])
                        if(len(selection)==0):
                            continue
                        print(len(selection),outputname,"%i/%i"%(index+1,len(orders)) if len(orders) else "")
                        selection=selection.dissolve(by="color")
                        selection.to_file(foldername+outputname)
                        print(lapstring(),"Output to ",foldername+outputname)
                        del selection

                    # output = output.filter(["category",'color', 'geometry'])
                # print(len(color_output),outputname,"%i/%i"%(index+1,len(orders)) if len(orders) else "")
                # output=output.dissolve(by="color")
                # output.to_file(foldername+outputname)
                # print(lapstring(),"Output to ",foldername+outputname)
                # del output

print(timerstring(),"Aggregating all files done")
print("Done")


text:
#/usr/bin/python3

"""
Module for analysing token counts and generating graphs
on the EuroParl dataset
"""

from collections import Counter
import re
from matplotlib import pyplot as plt
import numpy as np
import pandas as pd
from operator import itemgetter
import os
import sys

filepath = "dataset/intermediate/de-en.tsv"

if not os.path.exists(filepath):
    print("Please run this script from root project directory. Also make you already have downloaded the full dataset using the scripts we have provided")
    exit(1)

try:
    languages = [sys.argv[1]]
except:
    languages = ["de", "en"]

def make_plots():
    """
    Display plots for token count differences
    Using pandas series for quantile measurement
    """

    de_counts, en_counts = [], []

    invalid_sentence_count = 0
    valid_sentence_count = 0

    en_number_counter = Counter()
    de_number_counter = Counter()

    num_matcher = re.compile(r"\b(\d+)\b")

    print("Reading through the file, should take 10-15 seconds")

    pati = 0
    PATIENCE = -1

    with open(filepath, "r") as f:
        for line in f:
            try:
                de, en = line.split("\t")
                de_toks = de.split(" ")
                en_toks = en.split(" ")

                if PATIENCE != -1:
                    num_de_toks = []
                    num_en_toks = []

                    for tok in de_toks:
                        x = num_matcher.match(tok)
                        if x:
                            num_de_toks.append((tok, x.group(1)))
                    num_de_toks_new = list(map(itemgetter(1), num_de_toks))

                    for tok in en_toks:
                        x = num_matcher.match(tok)
                        if x:
                            num_en_toks.append((tok, x.group(1)))
                    num_en_toks_new = list(map(itemgetter(1), num_en_toks))

                    for (tok_org, tok) in num_de_toks:
                        if tok not in num_en_toks_new:
                            # timestamp
                            if tok_org.endswith(".00") or tok_org.endswith(".30"):
                                continue
                            print(f"Missing: {tok} {tok_org}")
                            print(f"Mistmatch in: {line}")
                            print(num_en_toks_new)
                            print(num_de_toks_new)
                            # print(de_toks)
                            # print(en_toks)
                            pati += 1
                            if pati == PATIENCE:
                                return
                            break
                    for (tok_org, tok) in num_en_toks:
                        if tok not in num_de_toks_new:
                            # timestamp
                            if re.search(r"\.\d0$", tok_org):
                                continue
                            print(f"Missing: {tok} {tok_org}")
                            print(f"Mistmatch in: {line}")
                            print(num_en_toks_new)
                            print(num_de_toks_new)
                            pati += 1
                            if pati == PATIENCE:
                                return
                            break

                    for tok in num_de_toks:
                        de_number_counter[tok] += 1
                    for tok in num_en_toks:
                        en_number_counter[tok] += 1

                    de_counts.append(len(de_toks))
                    en_counts.append(len(en_toks))
                valid_sentence_count += 1
            except ValueError:
                invalid_sentence_count += 1

    TOP_K = 100
    de_counts_cnt = sorted(de_number_counter.items(), key=itemgetter(1), reverse=True)[:TOP_K]
    en_counts_cnt = sorted(en_number_counter.items(), key=itemgetter(1), reverse=True)[:TOP_K]
    print("Top numbers")
    print(de_counts_cnt)
    print(en_counts_cnt)
    print("-------")

    token_diff = [de_counts[i] - en_counts[i] for i in range(len(de_counts))]
    token_diff.sort()
    token_series = pd.Series(token_diff, dtype=np.int64)

    de_counts.sort()
    en_counts.sort()
    de_series = pd.Series(de_counts, dtype=np.int64)
    en_series = pd.Series(en_counts, dtype=np.int64)

    line_colors = ['r', 'g', 'b', 'c', 'm', 'y', 'k', 'purple']

    # token_counts[0].sort()
    # token_counts[1].sort()
    print(f"Total sentences processed: {valid_sentence_count}; Sentences skipped: {invalid_sentence_count}")

    print("Statistics on the token counts")
    print("German")
    print(de_series.describe().astype(np.int64))
    print("English")
    print(en_series.describe().astype(np.int64))

    print("Token count differences")
    print(token_series.describe().astype(np.int64))
    print("Token difference 2% quantile", token_series.quantile(q=0.02))
    print("Token difference 10% quantile", token_series.quantile(q=0.1))
    print("Token difference 90% quantile", token_series.quantile(q=0.9))
    print("Token difference 97% quantile", token_series.quantile(q=0.97))
    plt.xlabel("Index of sentence pair")
    plt.ylabel("Difference value")
    plt.plot(token_series, label="German token count - English token count", color=line_colors[-1])
    plt.legend()
    plt.show()

    for language in languages:
        language_label = "English" if language == "en" else "German"

        plot_series = de_series if language == "de" else en_series
        plt.xlabel("Index of sentence pair")
        plt.ylabel("Token count")
        plt.plot(plot_series, label=f"{language_label} token counts", color=line_colors[0])

        for index, quantile in enumerate([0.5, 0.75, 0.95, 0.99, 0.9999]):
            if quantile > 0.99:
                quantile_label = quantile*100
            else:
                quantile_label = int(quantile*100)

            med = plot_series.quantile(q=quantile)
            plt.axhline(y=med, linestyle="dashed", label=f"{language_label} token count {quantile_label}% quantile", color=line_colors[index + 1])

        plt.legend()
        plt.show()

if __name__ == "__main__":
    make_plots()

max_stars_repo_name: yoogottamk/anlp-project

max_issues_repo_name: yoogottamk/anlp-project

max_forks_repo_name: yoogottamk/anlp-project


text:
# Pyrhon program to demonstrate erosion and
# dilation of image

import cv2
import numpy as np

# reading the input image
img = cv2.imread("../images/1.jpeg")

# taking a matrix of size 5 as the kernel
kernel = np.ones((3, 3), np.uint8)

# the first parameter is the original image,
# kernel is the matrix with which image is
# convolved and third parameter is the number
# of iteration, which will determine how much
# you want to erode / dilate a given image.

img_erosion = cv2.erode(img, kernel, iterations=1)
img_dilation = cv2.dilate(img, kernel, iterations=1)

cv2.imshow("Original", img)
cv2.imshow("Erosion", img_erosion)
cv2.imshow("Dilation", img_dilation)

cv2.waitKey(0)


text:
import numpy as np
import numpy.testing as npt
import pandas as pd
import pandas.testing as pdt
import pyarrow as pa
import pytest

import fletcher as fr

@pytest.fixture
def array_inhom_chunks():
    chunk1 = pa.array(list("abc"), pa.string())
    chunk2 = pa.array(list("12345"), pa.string())
    chunk3 = pa.array(list("Z"), pa.string())
    chunked_array = pa.chunked_array([chunk1, chunk2, chunk3])
    return fr.FletcherChunkedArray(chunked_array)

@pytest.mark.parametrize(
    "indices, expected",
    [
        (np.array(range(3)), np.full(3, 0)),
        (np.array(range(3, 8)), np.full(5, 1)),
        (np.array([8]), np.array([2])),
        (np.array([0, 1, 5, 7, 8]), np.array([0, 0, 1, 1, 2])),
        (np.array([5, 8, 0, 7, 1]), np.array([1, 2, 0, 1, 0])),
    ],
)
def test_get_chunk_indexer(array_inhom_chunks, indices, expected):

    actual = array_inhom_chunks._get_chunk_indexer(indices)
    npt.assert_array_equal(actual, expected)

def test_fletcherarray_constructor():
    with pytest.raises(ValueError):
        fr.FletcherChunkedArray(None)

def test_flatten():
    list_array = pa.array([[1, 2], [3, 4]])
    npt.assert_array_equal(
        fr.FletcherContinuousArray(list_array).flatten(), [1, 2, 3, 4]
    )

    chunked_list_array = pa.chunked_array([list_array, list_array])
    npt.assert_array_equal(
        fr.FletcherChunkedArray(chunked_list_array).flatten(), [1, 2, 3, 4, 1, 2, 3, 4]
    )

def test_pandas_from_arrow():
    arr = pa.array(["a", "b", "c"], pa.string())

    expected_series_woutname = pd.Series(fr.FletcherChunkedArray(arr))
    pdt.assert_series_equal(expected_series_woutname, fr.pandas_from_arrow(arr))

    expected_series_woutname = pd.Series(fr.FletcherContinuousArray(arr))
    pdt.assert_series_equal(
        expected_series_woutname, fr.pandas_from_arrow(arr, continuous=True)
    )

    rb = pa.RecordBatch.from_arrays([arr], ["column"])
    expected_df = pd.DataFrame({"column": fr.FletcherChunkedArray(arr)})
    table = pa.Table.from_arrays([arr], ["column"])
    pdt.assert_frame_equal(expected_df, fr.pandas_from_arrow(rb))
    pdt.assert_frame_equal(expected_df, fr.pandas_from_arrow(table))

    expected_df = pd.DataFrame({"column": fr.FletcherContinuousArray(arr)})
    table = pa.Table.from_arrays([arr], ["column"])
    pdt.assert_frame_equal(expected_df, fr.pandas_from_arrow(rb, continuous=True))
    pdt.assert_frame_equal(expected_df, fr.pandas_from_arrow(table, continuous=True))

def test_take_list_arrays():
    indices = [0, 1, 4, 3, 5]
    indptr = [0, 2, 3, 5]
    list_array = pa.ListArray.from_arrays(indptr, indices)
    large_list_array = pa.LargeListArray.from_arrays(indptr, indices)

    test_with_null = fr.FletcherContinuousArray(pa.array([[1, 2], [None, 3], [4, 5]]))

    assert np.all(
        pa.array(test_with_null.take([1, 2, 1])).to_pylist()
        == [[None, 3], [4, 5], [None, 3]]
    )

    test = fr.FletcherContinuousArray(pa.chunked_array([list_array, list_array])).take(
        [0, 5, 1]
    )
    test_large = fr.FletcherContinuousArray(
        pa.chunked_array([large_list_array, large_list_array])
    ).take([0, 5, 1])
    expected = [[0, 1], [3, 5], [4]]
    assert np.all(
        list(
            map(
                lambda x: np.all(np.array(test[x]) == np.array(expected)[x]),
                range(0, len(test)),
            )
        )
    )
    assert np.all(
        list(
            map(
                lambda x: np.all(np.array(test_large[x]) == np.array(expected)[x]),
                range(0, len(test_large)),
            )
        )
    )

max_stars_repo_name: simonjayhawkins/fletcher

max_issues_repo_name: simonjayhawkins/fletcher

max_forks_repo_name: simonjayhawkins/fletcher


text:
import cv2
import sys
import os
import numpy as np
from PIL import Image
import gaia.utils as cfg
import gaia.recognition as recognition

recognizer = recognition.face_recognizer()

def prepare_training_data(data_folder_path):
    dirs = os.listdir(data_folder_path)
    dirs.sort(key=cfg.natural_sort_key)
    images = []
    faces = []
    labels = []
    names = []
    for dir_name in dirs:
        if not dir_name.startswith("s"):
            continue;

        split = dir_name.split('-')
        label = int(split[0].replace("s", ""))
        subject_dir_path = data_folder_path + "/" + dir_name
        subject_images_names = os.listdir(subject_dir_path)

        for image_name in subject_images_names:
            if image_name.startswith("."):
                continue;

            image_path = subject_dir_path + "/" + image_name
            image = cv2.imread(image_path)
            images.append(image)

            cv2.imshow("Training on image...", image)
            cv2.waitKey(100)

            face, rect = recognition.detect_face(image)

            if face is not None:
                cv2.imshow("Training on face...", face)
                cv2.waitKey(100)

                faces.append(face)
                labels.append(label)
                name = split[1]
                if (names.count(name) <= 0):
                    names.append(name)

    cv2.destroyAllWindows()
    cv2.waitKey(1)
    cv2.destroyAllWindows()
    cv2.waitKey(0)

    return images, faces, labels, names

def train():
    global recognizer

    print("Preparing data...")
    (images, faces, labels, names) = prepare_training_data("data/subjects")
    recognizer.train(faces, np.array(labels))
    recognizer.write('data/trainer.yml')

    print("Data prepared")
    print("Total images: ", len(images))
    print("Total faces: ", len(faces))
    print("Total labels: ", len(labels))
    print("Total names: ", len(names))

max_stars_repo_name: thiagozampieri/gaia-loginface

max_issues_repo_name: thiagozampieri/gaia-loginface

max_forks_repo_name: thiagozampieri/gaia-loginface


text:
'''
y = b + a * x : Linear regression
y = b + a1 *x + a2 * x^2 -> 선형 회귀로 b, a1, a2를 결정 할 수 있다.

y = b+ a1 * x1 + a2 * x2 +...: 선형 회귀
y = b+ a1 * x1 + a2 * x2 + a3 * x1^2 + a4 * x1 * x2 + a5 *x2^2
'''

import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures

np.random.seed(1216)

# Training Set - data
X = 6 * np.random.rand(100,1) - 3 # -3 <= x < 3
print('X = ', X[:5])

# target
y = 0.5 + 2 * X + X**2 +np.random.randn(100,1)

# A = np.array([[1],[2],[3]]) # 3x1 행렬
# print('A =', A)
poly_feature = PolynomialFeatures(degree=2, include_bias=False)
# A_poly = poly_feature.fit_transform(A)
# print('A_poly =', A_poly)
#
# B = np.array([[1,2],[3,4]])
# print('B =', B)
# B_poly = poly_feature.fit_transform(B)
# print('B_poly =', B_poly)

X_poly = poly_feature.fit_transform(X)
print('X_poly =', X_poly[:5])

lin_reg = LinearRegression() # LR 객체생성
lin_reg.fit(X_poly,y) # Train data fitting, 학습 -> 계수들을 찾겠다
print('intercept: ', lin_reg.intercept_)
print('coefficients : ', lin_reg.coef_)

X_test = np.linspace(-3,3,100).reshape(100,1)
X_test_poly = poly_feature.fit_transform(X_test)
print(X_test[:5])
print(X_test[-5:])
y_pred = lin_reg.predict(X_test_poly)

plt.scatter(X,y)
plt.plot(X_test,y_pred, 'r')
plt.show()

max_stars_repo_name: SOOIN-KIM/lab-python

max_issues_repo_name: SOOIN-KIM/lab-python

max_forks_repo_name: SOOIN-KIM/lab-python


text:
from tkinter.tix import Tree
from numpy.core.arrayprint import _leading_trailing
from numpy.lib.polynomial import polyval
from pkg_resources import to_filename
import rasterio
from matplotlib import pyplot as plt
import numpy as np
from scipy import special
import library as sarlab
from datetime import datetime as dt
import argparse
import glob
import os
import shutil
from covariance import CovarianceMatrix
import isceio as io
import closures
import scipy.stats as stats
from sm_forward import SMForward
from closures import write_closures

def readInputs():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'path', help='Path to the interferogram.')
    parser.add_argument('-l', '--landcover', type=str, dest='landcover',
                        required=False, help='Landcover file path')
    parser.add_argument('-o', '--output', type=str,
                        dest='output', required=True, help='Output folder to save timeseries to')
    args = parser.parse_args()

    return args

def main():
    inputs = readInputs()
    stack_path = inputs.path
    stack_path = os.path.expanduser(stack_path)
    outputs = inputs.output
    files = glob.glob(stack_path)
    files = sorted(files)
    # files = files[3:6]
    dates = []
    for file in files:
        date = file.split('/')[-2]
        dates.append(date)

    # clone = None
    if inputs.landcover:
        landcover = np.squeeze(rasterio.open(inputs.landcover).read())
        print('landcover:')
        print(landcover.shape)

    SLCs = io.load_stack(files)
    inc_map = io.load_inc_from_slc(files[0])

    cov = CovarianceMatrix(SLCs, ml_size=(41, 41))
    SLCs = None
    coherence = cov.get_coherence()

    intensities = sarlab.get_intensity(cov.cov)
    epsilons = sarlab.intensity_to_epsilon(intensities)
    intensities = None
    cov = None

    k = special.comb(coherence.shape[0] - 1, 2)
    triplets = closures.get_triplets(coherence.shape[0])
    print(epsilons.shape)
    forward = SMForward(1, 1, 1)
    write_closures(
        coherence, '/Users/rbiessel/Documents/InSAR/saltLakeClosures')
    return
    for sf in np.linspace(1/10, 2, 10):
        for tf in np.linspace(0, 15, 10):

            imaginary_part = np.random.rand(3) * sf

            e1 = (epsilons[:, :, 0] * sf + tf) + imaginary_part[0] * 1j
            e2 = (epsilons[:, :, 1] * sf + tf) + imaginary_part[1] * 1j
            e3 = (epsilons[:, :, 2] * sf + tf) + imaginary_part[2] * 1j

            phi_sm_12 = forward.get_phases_dezan(e1, e2, use_epsilon=True)
            phi_sm_23 = forward.get_phases_dezan(e2, e3, use_epsilon=True)
            phi_sm_13 = forward.get_phases_dezan(e1, e3, use_epsilon=True)

            closure = phi_sm_12 * phi_sm_23 * np.conj(phi_sm_13)

            fig, ax = plt.subplots(nrows=1, ncols=3, sharex=True, sharey=True)
            ax[0].imshow(np.angle(closure), vmin=-np.pi/4,
                         vmax=np.pi/4, cmap=plt.cm.seismic)
            observed = coherence[0, 1] * \
                coherence[1, 2] * coherence[0, 2].conj()

            ax[1].imshow(np.angle(observed), vmin=-np.pi/4,
                         vmax=np.pi/4, cmap=plt.cm.seismic)
            ax[2].imshow(np.sqrt((closure.real - observed.real) **
                                 2 + (closure.imag - observed.imag)**2))

            plt.show()

main()

max_stars_repo_name: rbiessel/CovSAR

max_issues_repo_name: rbiessel/CovSAR

max_forks_repo_name: rbiessel/CovSAR

reason: import numpy,from numpy,import scipy,from scipy


text:
from argparse import ArgumentParser
import pandas as pd
import numpy as np
import sys
import os
from pdb import set_trace
from sklearn.model_selection import train_test_split

def save(df, fname):
	dname = os.path.dirname(fname)
	if not os.path.isdir(dname):
		os.makedirs(dname)
	records = df.to_records(index=False)
	records.dtype.names = [str(i) for i in records.dtype.names]
	np.save(fname, records)

def run_scan(
	outdir, indir, addsv=False,
	lr=[0.001], weight=[50], lmb=[0.04],
	igpu=0, gpu_fraction=0.25, postfix='1'):
	#
	# TF crap preamble
	#
	import keras
	from keras import backend as K
	import sys
	import tensorflow as tf
	import os
	os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
	os.environ['CUDA_VISIBLE_DEVICES'] = str(igpu)
	print('running on GPU '+str(igpu))

	gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction)
	sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
	K.set_session(sess)
	print('using gpu memory fraction: '+str(gpu_fraction))

	#
	# Load Data
	#
	from make_samples import make_sample
	from keras.layers import Input
	Inputs = Input((21,))
	X_traintest, isB_traintest , isMC_traintest = make_sample(indir, addsv)
	X_all, X_test, isB_all, isB_test, isMC_all, isMC_test = train_test_split(
		X_traintest, isB_traintest , isMC_traintest,
		test_size=0.1, random_state=42
		)

	from keras.optimizers import Adam
	from models import modelIverseGrad
	from itertools import product
	#
	# Run scan
	#
	for ilmb, ilr, iweight in product(lmb, lr, weight):
		print 'Testing lambda: %f, learning rate: %f, weight: %f' % (ilmb, ilr, iweight)
		model = modelIverseGrad(Inputs, rev_grad=ilmb)
		model.compile(
			loss = ['binary_crossentropy']*4,
			optimizer=Adam(lr=ilr),
			loss_weights=[1., 0., iweight, iweight]
			)
		history = model.fit(
			X_all,
			[isB_all, isB_all, isMC_all, isMC_all],
			batch_size=5000, epochs=75,  verbose=0, validation_split=0.2,
			sample_weight = [
				isMC_all.ravel(),
				1-isMC_all.ravel(),
				1-isB_all.ravel()*0.75,
				1+isB_all.ravel()*0.75],
			)

		history = pd.DataFrame(history.history)
		save(history, '%s/domain_adaptation_lr%f_w%f_l%f/%s/history.npy' % (outdir, ilr, iweight, ilmb, postfix))

if __name__ == '__main__':
	parser = ArgumentParser()
	parser.add_argument('outputdir')
	parser.add_argument("inputdir")
	parser.add_argument('postfix')
	parser.add_argument("--addsv", action='store_true')
	parser.add_argument("--gpu",  help="select specific GPU",   type=int, metavar="OPT", default=-1)
	parser.add_argument("--gpufraction",  help="select memory fraction for GPU",   type=float, metavar="OPT", default=-1)
	parser.add_argument("--lr", help="learning rates")
	parser.add_argument("--weight", help="domain adaptation weights")
	parser.add_argument("--lmb", help="domain adaptation lambdas")
	args = parser.parse_args()
	str2floats = lambda x: [float(i) for i in x.split(',')]

	run_scan(
	args.outputdir, args.inputdir,
	addsv=args.addsv,
	lr=str2floats(args.lr),
	weight=str2floats(args.weight),
	lmb=str2floats(args.lmb),
	igpu=args.gpu, gpu_fraction=args.gpufraction,
	postfix=args.postfix)


text:
# --------------
# Import packages
import numpy as np
import pandas as pd
from scipy.stats import mode
path
filepath = path
bank = pd.read_csv(filepath)
print(bank.head())
categorical_var  = bank.select_dtypes(include = 'object')
print("="*50)
print(categorical_var)
print("="*50)
numerical_var  = bank.select_dtypes(include = 'number')
print("="*50)
print(numerical_var)

# code starts here

# code ends here

# --------------
# code starts here
import numpy as np
import pandas as pd
from scipy.stats import mode
path
filepath = path
bank = pd.read_csv(filepath)
banks = bank.drop('Loan_ID',axis=1)
banks = pd.DataFrame(banks)
print(banks.head())
print(banks.isnull().sum())
bank_mode = banks.mode()
print("="*50)
for column in banks.columns:
    banks[column].fillna(banks[column].mode()[0], inplace=True)
print("="*50)
print(banks.head(10))

# --------------
# Code starts here
import numpy as np
import pandas as pd
from scipy.stats import mode
path
filepath = path
bank = pd.read_csv(filepath)
banks = bank.drop('Loan_ID',axis=1)
banks = pd.DataFrame(banks)
print(banks.head())
print(banks.isnull().sum())
bank_mode = banks.mode()
print("="*50)
for column in banks.columns:
    banks[column].fillna(banks[column].mode()[0], inplace=True)
print("="*50)
print(banks.head(10))
avg_loan_amount = banks.pivot_table(index=['Gender','Married','Self_Employed'],values='LoanAmount',aggfunc='mean')
print("="*50)
print(avg_loan_amount)

# code ends here

# --------------
# code starts here

# code for loan aprroved for self employed
loan_approved_se = banks.loc[(banks["Self_Employed"]=="Yes")  & (banks["Loan_Status"]=="Y"), ["Loan_Status"]].count()
print(loan_approved_se)

# code for loan approved for non self employed
loan_approved_nse = banks.loc[(banks["Self_Employed"]=="No")  & (banks["Loan_Status"]=="Y"), ["Loan_Status"]].count()
print(loan_approved_nse)

# percentage of loan approved for self employed
percentage_se = (loan_approved_se * 100 / 614)
percentage_se=percentage_se[0]
# print percentage of loan approved for self employed
print(percentage_se)

#percentage of loan for non self employed
percentage_nse = (loan_approved_nse * 100 / 614)
percentage_nse=percentage_nse[0]
#print percentage of loan for non self employed
print (percentage_nse)

# code ends here

# --------------
# code starts here

# loan amount term

loan_term = banks['Loan_Amount_Term'].apply(lambda x: int(x)/12 )

big_loan_term=len(loan_term[loan_term>=25])

print(big_loan_term)

# code ends here

# --------------
# code starts here
import numpy as np
import pandas as pd
from scipy.stats import mode
path
filepath = path
bank = pd.read_csv(filepath)
banks = bank.drop('Loan_ID',axis=1)
loan_groupby = banks.groupby('Loan_Status')['ApplicantIncome', 'Credit_History']
print(loan_groupby)
mean_values = banks.groupby('Loan_Status')['ApplicantIncome', 'Credit_History'].agg(np.mean)
print(mean_values)

# code ends here

reason: import numpy,from scipy


text:
import numpy as np
from linker.jerrl import Jerrl

from common.preprocessing.wordhashing import WordHashing

class Preprocessor:
    @staticmethod
    def qapair_to_hash(question_answer_uris):
        jerrl = Jerrl()
        hashing = WordHashing()
        hashed_qapairs = {}
        for data_item in question_answer_uris:
            question = data_item["question"]
            query = data_item["query"]
            counter = 0
            for item in jerrl.find_mentions(question, data_item["uris"]):
                question = u"{0} {1} {2}".format(question[:item["start"]], str(counter), question[item["end"]:])
                query = query.replace(item["uri"].raw_uri, str(counter))
                counter += 1

            hashed_question = hashing.hash(question)
            hashed_sparql = hashing.hash(query)
            hashed_qapairs[data_item["id"]] = (hashed_question, hashed_sparql)

        VOCAB_SIZE = 50000  # len(hashing.ids)
        DATA_SIZE = len(question_answer_uris)
        i = 0
        questions = np.zeros([DATA_SIZE, VOCAB_SIZE])
        sparqls = np.zeros([DATA_SIZE, VOCAB_SIZE])
        ids = []
        for kv in hashed_qapairs.iteritems():
            question_unique_counts = np.unique(kv[1][0], return_counts=True)
            questions[i, question_unique_counts[0]] = question_unique_counts[1]

            sparql_unique_counts = np.unique(kv[1][1], return_counts=True)
            sparqls[i, sparql_unique_counts[0]] = sparql_unique_counts[1]
            ids.append(kv[0])
            i += 1

        return questions, sparqls, ids

max_stars_repo_name: tamlhp/kbqa

max_issues_repo_name: tamlhp/kbqa

max_forks_repo_name: tamlhp/kbqa


text:
from scipy.fftpack import dct,idct
from skimage import io
import matplotlib.pyplot as plt
import numpy as np
import cv2

def dct2(block):
    return dct(dct(block.T, norm = 'ortho').T, norm = 'ortho')

def idct2(block):
    return idct(idct(block.T, norm = 'ortho').T, norm = 'ortho')

def quanStep(block,quanTable):
    dct_block = dct2(block.T.T)
    qdct_block = np.zeros((8,8))
    for i in range(0,8):
        for j in range(0,8):
            qdct_block[i,j] = np.round(float(dct_block[i,j]/(quanTable[i,j])))*quanTable[i,j]
    rect_qdct_block = np.uint8(idct2(qdct_block.T.T))
    #Dis_Res(rect_qdct_block)
    return rect_qdct_block

def Dis_Res(block):
    rows,cols = block.shape
    for i in range(0,rows):
        for j in range(0,cols):
                print(block[i,j])

def imageDCT_to_D(image,quanTable):
        rows,cols = image.shape
        dct_image = np.zeros((rows,cols))
        rows_times = np.uint8(rows/8)
        cols_times = np.uint8(cols/8)

        for i in range(0,rows_times):
            for j in range(0,cols_times):
                block = image[8*i:8*i+8,8*j:8*j+8]
                qblock = quanStep(block,quanTable)
                dct_image[8*i:8*i+8,8*j:8*j+8] = qblock
        return dct_image

def PSNR_calc(image,qimage):
    rows,cols,dims = image.shape
    dif=0.0
    for i in range(0,rows):
        for j in range(0,cols):
            dif = dif + (image[i,j] - qimage[i,j]) * (image[i,j] - qimage[i,j])
    MSE = dif / (rows*cols)
    PSNR = 10*np.log10(255*255/MSE)
    PSNR_mean=np.mean(PSNR)
    PSNR_mean = float("{0:.2f}".format(PSNR_mean))
    return PSNR_mean

def QF(QT,qu):
    QT2 = np.zeros((8,8))

    if (qu <= 0):
        qu = 1

    if (qu > 100):
        qu = 100

    if (qu <50):
        qu = 5000/qu
    else:
        qu = 200-qu*2

    for i in range(0,8):
        for j in range(0,8):
            QT2[i][j] = QT[i][j]* qu/100

    return QT2

def img_process(img,qu):
    print('Quality = ',qu)
    quanTable2 = np.int64(QF(quanTable,qu))

    imgYcc = cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
    imgTemp= np.uint32(imageDCT_to_D(imgYcc[:,:,0],quanTable2))
    imgYcc[:,:,0] = imgTemp
    img2 = cv2.cvtColor(imgYcc, cv2.COLOR_YCrCb2BGR)

    plt.subplot(1,2,2)
    plt.title('DCT Quantized Result')
    plt.imshow(imgTemp,cmap='gray')

    img_PSNR = PSNR_calc(img,img2)
    io.imsave('QF'+str(qu)+'_PSNR='+str(img_PSNR)+'dB.jpg',img2)

    print('PSNR = ',img_PSNR,'dB')

img = io.imread('image.bmp')
plt.figure(num=2,figsize=(6,6))
quality = np.array([75,50,25,15,5])

quanTable = np.array([
  [16, 11, 10, 16, 24, 40, 51, 61],
  [12, 12, 14, 19, 26, 58, 60, 55],
  [14, 13, 16, 24, 40, 57, 69, 56],
  [14, 17, 22, 29, 51, 87, 80, 62],
  [18, 22, 37, 56, 68, 109, 103, 77],
  [24, 35, 55, 64, 81, 104, 113, 92],
  [49, 64, 78, 87, 103, 121, 120, 101],
  [72, 92, 95, 98, 112, 100, 103, 99]])

plt.subplot(1,2,1)
plt.title('Original iamge')
plt.imshow(img,cmap='gray')

#for i in range(0,quality.shape[0]):
img_process(img,quality[4])