text:
#import matplotlib;
import numpy;
import os;
import pylab;
import scipy;
import scipy.linalg;
import subprocess;
import sys;

PIXEL_SIZE=15;

name=sys.argv[1];
step=int(sys.argv[2]);
width0=int(sys.argv[3]);
length0=int(sys.argv[4]);
ul_lon=float(sys.argv[5]);
ul_lat=float(sys.argv[6]);

cmd="\nsed -e '/\*/d' "+name+" > temp\nmv "+name+" "+name+".old\nmv temp "+name+"\n";
subprocess.call(cmd,shell=True);
indat=scipy.loadtxt(name);

rwin=step;
awin=step;
wsamp=1;

da_e=PIXEL_SIZE*100;	#az pixel size at earth surface, cm
dr_g=PIXEL_SIZE*100;	#ground pixel size in range direction, cm

x1ind=scipy.matrix([indat[:,0]],scipy.int32).conj().transpose();
dx=scipy.matrix([indat[:,1]]).conj().transpose();
y1ind=scipy.matrix([indat[:,2]],scipy.int32).conj().transpose();
dy=scipy.matrix([indat[:,3]]).conj().transpose();
snr=scipy.matrix([indat[:,4]]).conj().transpose();
c11=scipy.matrix([scipy.sqrt(indat[:,5])]).conj().transpose() #1 sigma drng
c22=scipy.matrix([scipy.sqrt(indat[:,6])]).conj().transpose() #1 sigma dazo

x1=x1ind * dr_g;
dx=dx * dr_g;
y1=y1ind * da_e;
dy=dy * da_e;
c11=c11 * dr_g; #1 sigma drng
c22=c22 * da_e; #1 sigma dazo
x2=x1+dx;
y2=y1+dy;

rlooks=rwin/wsamp;
alooks=awin/wsamp;

width1=scipy.floor(width0/rlooks);
length1=scipy.floor(length0/alooks);
[xg,yg]=scipy.meshgrid(scipy.arange(1,width1+1,1),scipy.arange(1,length1+1,1));
xg=xg*dr_g*rlooks/1e5;	#convert from pix to km
yg=yg*da_e*alooks/1e5;	#convert from pix to km

sigy_thresh=scipy.inf;	#cm
sigx_thresh=scipy.inf;	#cm
snr_thresh=0;		#(not log10)
mag_threshx=scipy.inf;	#cm
mag_threshy=scipy.inf;	#cm

#initial mask
c22good=scipy.matrix(pylab.find(c22<sigy_thresh)).conj().transpose();
c11good=scipy.matrix(pylab.find(c11<sigx_thresh)).conj().transpose();
snrgood=scipy.matrix(pylab.find(snr>snr_thresh)).conj().transpose();

good=scipy.matrix(scipy.unique(scipy.array(scipy.vstack((snrgood,c11good,c22good))))).conj().transpose();

x1good=x1[good].reshape(-1,1);
x1goodind=x1ind[good].reshape(-1,1);
y1good=y1[good].reshape(-1,1);
y1goodind=y1ind[good].reshape(-1,1);
x2good=x2[good].reshape(-1,1);
y2good=y2[good].reshape(-1,1);

#get and remove affine fit
good2=scipy.matrix(pylab.find(good<300000)).conj().transpose();

x1good=x1[good2].reshape(-1,1);
y1good=y1[good2].reshape(-1,1);
x2good=x2[good2].reshape(-1,1);
y2good=y2[good2].reshape(-1,1);

c0=scipy.matrix(scipy.zeros((scipy.size(good2)))).reshape(-1,1);
c1=scipy.matrix(scipy.ones((scipy.size(good2)))).reshape(-1,1);
n=c1.shape[0];

A=scipy.vstack((scipy.hstack((x1good,y1good,c0,c0,c1,c0)),scipy.hstack((c0,c0,x1good,y1good,c0,c1))));

b=scipy.vstack((x2good,y2good));

M=scipy.linalg.lstsq(A,b)[0];

pred=A*M;
res=pred-b;

# std() in python defaults to 0 degrees of freedom
resdev=res.std(axis=0,ddof=1);
q=pylab.find(abs(res)<1.5*resdev);
A1=A[q,];
b1=b[q];
M=scipy.linalg.lstsq(A1,b1)[0];
pred=A*M;

x1good=x1[good].reshape(-1,1);
x1goodind=x1ind[good].reshape(-1,1);
y1good=y1[good].reshape(-1,1);
y1goodind=y1ind[good].reshape(-1,1);
x2good=x2[good].reshape(-1,1);
y2good=y2[good].reshape(-1,1);

c0=scipy.matrix(scipy.zeros((scipy.size(good)))).reshape(-1,1);
c1=scipy.matrix(scipy.ones((scipy.size(good)))).reshape(-1,1);
n=c1.shape[0];

A=scipy.vstack((scipy.hstack((x1good,y1good,c0,c0,c1,c0)),scipy.hstack((c0,c0,x1good,y1good,c0,c1))));

b=scipy.vstack((x2good,y2good));

pred=A*M;

n=c1.shape[0];

res=pred-b;
resdx=res[0:n];
resdy=res[(n):(2*n)];

#remap into
newx=scipy.matrix(scipy.ceil(x1goodind / rlooks),scipy.int32);
newy=scipy.matrix(scipy.floor(y1goodind / alooks),scipy.int32);

vind=scipy.asarray((newy-1)*width1+newx,scipy.int32).reshape(-1);

temp=scipy.matrix(0*(scipy.arange(1,length1*width1+1,1))).conj().transpose();
temp[vind]=resdy;
dyg=temp.reshape(length1,width1);

temp=scipy.matrix(0*(scipy.arange(1,length1*width1+1,1))).conj().transpose();
temp[vind]=resdx;
dxg=temp.reshape(length1,width1);

#setup mask indicies
newx=scipy.matrix(scipy.ceil(x1ind/rlooks),scipy.int32);
newy=scipy.matrix(scipy.floor(y1ind/alooks),scipy.int32);
vind=scipy.asarray((newy-1)*width1+newx,scipy.int32).reshape(-1);
temp=scipy.NaN*scipy.matrix(scipy.arange(0,length1*width1,1)).conj().transpose();

#sigma_y mask
temp[vind]=c22;
sigyg=temp.reshape(length1,width1);
mask_sigy=scipy.zeros(dyg.shape);
mask_sigy[(sigyg>sigy_thresh)]=scipy.NaN;

#sigma_x mask
temp=scipy.NaN*scipy.matrix(scipy.arange(0,length1*width1,1)).conj().transpose();
temp[vind]=c11;
sigxg=temp.reshape(length1,width1);
mask_sigx=scipy.zeros(dxg.shape);
mask_sigx[(sigxg>sigx_thresh)]=scipy.NaN;

#SNR mask
temp=scipy.NaN*scipy.matrix(scipy.arange(0,length1*width1,1)).conj().transpose();
temp[vind]=snr;
snrg=temp.reshape(length1,width1);
mask_snr=scipy.zeros(dyg.shape);
mask_snr[(snrg<snr_thresh)]=scipy.NaN;

#mag mask y
mask_magy=scipy.zeros(dyg.shape);
mask_magy[abs(dyg)>mag_threshy]=scipy.NaN;

#mag mask x
mask_magx=scipy.zeros(dxg.shape);
mask_magx[abs(dxg)>mag_threshx]=scipy.NaN;

#final mask
mask_total=mask_snr+mask_sigy+mask_magy;
bad=scipy.isnan(mask_total);
dyg[bad]=scipy.NaN;

mask_total=mask_snr+mask_sigx+mask_magx;
bad=scipy.isnan(mask_total);
dxg[bad]=scipy.NaN;

x_step=PIXEL_SIZE*step;
y_step=-PIXEL_SIZE*step;

columns=scipy.arange(0,width1,1);
rows=scipy.arange(0,length1,1);

vect_utm_n=ul_lat+rows*y_step;
vect_utm_e=ul_long+columns*x_step;

[map_utm_e,map_utm_n]=scipy.meshgrid(vect_utm_e,vect_utm_n);
map_utm_n=scipy.flipud(map_utm_n);

cdir=os.getcwd();
cdir=cdir[cdir.rfind("/")+1:].strip();

column_utm_n=map_utm_n.reshape(scipy.size(map_utm_n,0)*scipy.size(map_utm_n,1),1);
column_utm_e=map_utm_e.reshape(scipy.size(map_utm_e,0)*scipy.size(map_utm_e,1),1);
column_dyg=scipy.flipud(dyg).reshape(scipy.size(dyg,0)*scipy.size(dyg,1),1);
column_dxg=scipy.flipud(dxg).reshape(scipy.size(dxg,0)*scipy.size(dxg,1),1);
column_snr=scipy.flipud(snrg).reshape(scipy.size(snrg,0)*scipy.size(snrg,1),1);
ampcor_optical_azimuth=scipy.concatenate((column_utm_e,column_utm_n,column_dyg,column_snr),axis=1);
ampcor_optical_range=scipy.concatenate((column_utm_e,column_utm_n,column_dxg,column_snr),axis=1);
scipy.savetxt("landsat_northxyz.txt",ampcor_optical_azimuth,delimiter=" ",fmt="%.1f");
scipy.savetxt("landsat_eastxyz.txt",ampcor_optical_range,delimiter=" ",fmt="%.1f");

#matplotlib.pyplot.imshow(scipy.array(dxg),interpolation='nearest',origin='lower');
#matplotlib.pyplot.show();

max_stars_repo_name: whyjz/CARST

max_issues_repo_name: whyjz/CARST

max_forks_repo_name: whyjz/CARST

reason: import numpy,import scipy


text:
import re
import torch
import pandas as pd
import numpy as np
import pickle

def replace_string_newline(str_start: str, str_end: str, text: str) -> str:
    """
    re.sub function stops at newline characters, but this function moves past these
    params: str_start, the start character of the string to be delted
            str_end, the end character of the string to be deleted
            text, the string to be edited
    return: text, a string without the string between str_start and str_end
    """
    while text.find(str_start) != -1:
        begin = text.find(str_start) # also just starts w/o http, has (/
        nextp = text.find(str_end, begin, len(text))
        text = text.replace(text[begin-1:nextp+1], ' ')
    return text

def filter_news(txt_file: list) -> list:
    """
    filters non-content (e.g. HTTP references, bullets, hashtags) from text data
    params: txt_file, a list of strings, e.g. from news corpora with non-content data
    return: out_txt, a list of strings filtered as designated
    """

    mid_txt = []
    out_txt = []
    for line in txt_file:
        mid_txt.extend(line.split('\n\n'))
    for line in mid_txt:
        while line.find('(http') != -1:
            begin = line.find('(http') # also just starts w/o http, has (/
            nextp1 = line.find(')\n', begin, len(line))
            nextp2 = line.find(') ', begin, len(line))
            nextp3 = line.find(')', begin, len(line))
            nextp4 = line.find(' ', begin, len(line))
            nextp = max(nextp1, nextp2, nextp3, nextp4)
            line = line.replace(line[begin-1:nextp+1], ' ')
        line = replace_string_newline('[http', ']', line)
        line = replace_string_newline('<http', '>', line)
        line = replace_string_newline('/http', ')', line)
        line = replace_string_newline('(/', ')', line)
        line = replace_string_newline('(\\', ')', line)
        line = replace_string_newline('(data:', ')', line)
        line = replace_string_newline('(java:', ')', line)
        line = re.sub('-\\n', '-', line) #replaces hyphenated newline with just hyphen
        line = re.sub('\n',' ',line) # replaces newline characters with a space
        line = re.sub('\\xa0', ' ', line) # replaces coded space with regular space
        line = re.sub('\\xad', '', line) # replaces soft-hyphens
        line = re.sub('^ *', '', line) # removes leading space
        line = re.sub('  *', ' ', line) # removes extra space
        line = re.sub('^ *', '', line) # removes leading space
        line = re.sub('\*(.*?)\\n', '', line) # removes phrases beginning with asterisk
        line = re.sub('^\[(.*?)\] .*$', '', line) # removes lines beginning and ending with brackets
        line = re.sub('^[#*](.*?)$', '', line) # removes lines beginning with hashtag
        line = re.sub('\[','', line) # delete extra brackets does not remove lines beginning with open bracket
        line = re.sub('[\*\\#\!\>\_\\\]\|]*', '', line) # removes extra characters
        line = re.sub('(.*?)Video duration(.*?)$', '', line) # removes lines of video captions
        line = re.sub(' \.', '.', line) # removes extra space before periods
        line = re.sub(' ,', ',', line) # removes extra space before commas
        line = re.sub('^[ \>]$', '', line) # removes entries with single space or single >
        line = re.sub('  *', ' ', line) # removes extra space
        line = re.sub('^ *', '', line) # removes leading space
        line = re.sub(' *$', '', line) # removes ending space
        line = re.sub('((.*?).html)', '', line) # removes other link data
         # remove single words from a list???? how??? Image captions? Image numbers?
        out_txt.append(line)
    while('' in out_txt):
        out_txt.remove('')
    return out_txt

file1 = pd.read_csv('dict_adj_to_country.csv')
adj_to_country = dict(zip(file1.Adjective, file1.Country))

ext: py

max_stars_repo_name: tphanmt/space-nlp

max_issues_repo_name: tphanmt/space-nlp

max_forks_repo_name: tphanmt/space-nlp

reason: import numpy


text:
import numpy as np

class TreeNode(object):
    """
    The foundational data structure for DecisionTree.
    Root, branches, and leaves are all TreeNodes.
    """
    def __init__(
        self,
        features=None,
        parent=None,
        recommendation=None,
        split_feature=None,
    ):
        """
        Parameters
        ----------
        features: list
            The feature values corresponding to this node: 1 or 0.
            None if that feature hasn't been split on.
        parent: TreeNode
            If None, then this is the root TreeNode.
        recommendation: float
            The departure time recommendation for this node.
        split_feature: int
            The index of the feature on which
            this node's children are split.
        """
        self.parent = parent
        # If/when this node gets split, it will get a hi and low branch.
        self.hi_branch = None
        self.lo_branch = None
        self.is_leaf = True
        # The feature index on which this node is split to create
        # its children. If this node is a leaf, this will be None.
        self.split_feature = split_feature
        # The list of all feature values associated with this node.
        # Features that haven't been split on yet are given as None.
        self.features = features
        # The departure time recommendation associated with this node.
        self.recommendation = recommendation

    def attempt_split(self, data, err_fn, n_min):
        """
        Try to split this node into two child nodes.

        Parameters
        ----------
        data: DataFrame
            Features for each of the data points
        err_fn: function
            The error function that determines the fitness of a split.
            Choose a split that minimizes the combined error of the
            two resulting branches.
        n_min: int
            The number of data points that need to remain in a node to
            make it viable.

        Returns
        -------
        success: boolean
            If a split happened, returns True.
        """
        success = False

        n_features = len(data.columns)
        # Initialize the root node properly.
        if self.features is None:
            self.features = [None] * n_features

        node_data = self.find_members(data)

        feature_candidates = [
            i for i, j in enumerate(self.features) if j is None]
        best_feature = -1
        best_split_score = 1e10
        best_hi_recommendation = self.recommendation
        best_lo_recommendation = self.recommendation

        # Check each of the potential feature dimensions
        # for split quality.
        if len(feature_candidates) > 0:
            # Shuffe these up to prevent order artifacts in ties.
            np.random.shuffle(feature_candidates)
            for i_feature in feature_candidates:
                hi_data = node_data.loc[node_data.iloc[:, i_feature] == 1, :]
                lo_data = node_data.loc[node_data.iloc[:, i_feature] == 0, :]
                # Only consider splits for which both child nodes
                # maintain more than the minimum number of data points.
                if hi_data.shape[0] >= n_min and lo_data.shape[0] >= n_min:
                    hi_score, hi_recommendation = err_fn(list(hi_data.index))
                    lo_score, lo_recommendation = err_fn(list(lo_data.index))
                    split_score = hi_score + lo_score
                    # Track the best-so-far solution
                    if split_score < best_split_score:
                        best_split_score = split_score
                        best_feature = i_feature
                        best_hi_recommendation = hi_recommendation
                        best_lo_recommendation = lo_recommendation
                        success = True

            # Act on the best split.
            if success:
                # Make a copy of the features list.
                hi_features = list(self.features)
                hi_features[best_feature] = 1
                self.hi_branch = TreeNode(
                    parent=self,
                    features=hi_features,
                    recommendation=best_hi_recommendation,
                )

                lo_features = list(self.features)
                lo_features[best_feature] = 0
                self.lo_branch = TreeNode(
                    parent=self,
                    features=lo_features,
                    recommendation=best_lo_recommendation,
                )

                self.is_leaf = False
                self.split_feature = best_feature

        return success

    def find_members(self, data):
        """
        Find all of the dates within features that belong to this node.

        Parameters
        ----------
        data: DataFrame
            A set of dates under consideration
            and their associated features.

        Returns:
        --------
        member_data: DataFrame
            A subset of the rows of the features DataFrame.
        """
        member_data = data
        for i_feature, feature in enumerate(self.features):
            if self.features[i_feature] is not None:
                member_data = member_data.loc[
                        member_data.iloc[:, i_feature] == feature]

        return member_data


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

np.random.seed(2)
data = pd.read_csv('creditcard.csv')

## Data Preprocessing

from sklearn.preprocessing import StandardScaler
data['normalizedAmount'] = StandardScaler().fit_transform(data['Amount'].values.reshape(-1,1))
data = data.drop(['Time', 'Amount'], axis = 1)

x = data.drop(['Class'], axis = 1)
y = data['Class']

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.3, random_state = 0)

## Random Forest
from sklearn.ensemble import RandomForestClassifier
classifier = RandomForestClassifier(n_estimators = 100)
classifier.fit(x_train, y_train.values.ravel())
y_pred = classifier.predict(x_test)

## Confusion Matrix

from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sns
cm = confusion_matrix(y_test, y_pred.round())
print(cm)
sns.heatmap(cm, annot = True)
plt.show()
print(classification_report(y_test, y_pred.round()))

max_stars_repo_path: 2.Project - Random Forest.py

max_issues_repo_path: 2.Project - Random Forest.py

max_forks_repo_path: 2.Project - Random Forest.py


text:
# https://deeplearningcourses.com/c/data-science-deep-learning-in-theano-tensorflow
# https://www.udemy.com/data-science-deep-learning-in-theano-tensorflow
from __future__ import print_function, division
from builtins import range
# Note: you may need to update your version of future
# sudo pip install -U future

from keras.models import Sequential
from keras.layers import Dense, Activation
from util import get_normalized_data, y2indicator

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf

import theano
import theano.tensor as T

# RMSprop experiment
# to compare TF / Keras / Theano

N = 10
D = 2
X = np.random.randn(N, D).astype(np.float32)
w = np.array([0.5, -0.5], dtype=np.float32)
Y = X.dot(w) + 1
Y = Y.reshape(-1, 1)

# keras
# the model will be a sequence of layers
model = Sequential()
model.add(Dense(units=1, input_dim=D))

# copy the weights for later
weights = model.layers[0].get_weights()
w0 = weights[0].copy()
b0 = weights[1].copy()

model.compile(
  loss='mean_squared_error',
  optimizer='rmsprop',
)

r = model.fit(X, Y, epochs=15, batch_size=10)

# print the available keys
print(r.history.keys())

# tf
inputs = tf.compat.v1.placeholder(tf.float32, shape=(None, 2))
targets = tf.compat.v1.placeholder(tf.float32, shape=(None, 1))
tfw = tf.Variable(w0)
tfb = tf.Variable(b0)
pred = tf.matmul(inputs, tfw) + tfb

loss = tf.reduce_mean(input_tensor=tf.square(targets - pred))
train_op = tf.compat.v1.train.RMSPropOptimizer(1e-3, epsilon=1e-8).minimize(loss)

tflosses = []
init = tf.compat.v1.global_variables_initializer()
with tf.compat.v1.Session() as sess:
  sess.run(init)
  for e in range(15):
    _, l = sess.run([train_op, loss], feed_dict={inputs: X, targets: Y})
    tflosses.append(l)

# theano
def rmsprop(cost, params, lr=1e-3, decay=0.9, eps=1e-8):
  # return updates
  lr = np.float32(lr)
  decay = np.float32(decay)
  eps = np.float32(eps)

  updates = []
  grads = T.grad(cost, params)

  # tf-like
  # caches = [theano.shared(np.ones_like(p.get_value(), dtype=np.float32)) for p in params]

  # keras-like
  caches = [theano.shared(np.zeros_like(p.get_value(), dtype=np.float32)) for p in params]

  new_caches = []
  for c, g in zip(caches, grads):
    new_c = decay*c + (np.float32(1) - decay)*g*g
    updates.append((c, new_c))
    new_caches.append(new_c)

  for p, new_c, g in zip(params, new_caches, grads):
    new_p = p - lr*g / T.sqrt(new_c + eps)
    updates.append((p, new_p))

  return updates

thX = T.matrix('X')
thY = T.matrix('Y')
thw = theano.shared(w0)
thb = theano.shared(b0)
thP = thX.dot(thw) + thb
cost = T.mean((thY - thP)**2)
params = [thw, thb]
updates = rmsprop(cost, params)

train_op = theano.function(
  inputs=[thX, thY],
  outputs=cost,
  updates=updates,
)

thlosses = []
for e in range(15):
  c = train_op(X, Y)
  thlosses.append(c)

# plot results
plt.plot(r.history['loss'], label='keras loss')
plt.plot(tflosses, label='tf loss')
plt.plot(thlosses, label='theano loss')
plt.legend()
plt.show()

reason: import numpy,import theano


text:
import pandas as pd
import numpy as np
from cfg.create_config import Config
from argparse import ArgumentParser
from sklearn.model_selection import train_test_split
from model.train_model import Model

def parse_args():
    parser = ArgumentParser(description='Start training')
    parser.add_argument('--data', dest='data', default='data/train.csv', type=str)
    parser.add_argument('--weights_folder', dest='weights_folder',
                        default='weights', type=str)

    args = parser.parse_args()
    return args

def prepare_data(path):
    print("Starting Preprocessing...")

    config = Config()
    cfg = config.load_cfg()
    width, height = cfg['width'], cfg['height']

    data = pd.read_csv(path)

    datapoints = data['pixels'].tolist()

    X = []
    for xseq in datapoints:
        xx = [int(xp) for xp in xseq.split(' ')]
        xx = np.asarray(xx).reshape(width, height)
        X.append(xx.astype('float32'))

    X = np.asarray(X)
    X = np.expand_dims(X, -1)

    y = pd.get_dummies(data['target']).as_matrix()

    print("Preprocessing Done!")
    print("Number of Features: " + str(len(X[0])))
    print("Number of Labels: " + str(len(y[0])))
    print("Number of examples in dataset:" + str(len(X)))

    return X, y

if __name__ == '__main__':
    args = parse_args()

    x, y = prepare_data(args.data)

    x -= np.mean(x, axis=0)
    x /= np.std(x, axis=0)

    X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.1, random_state=42)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.1, random_state=41)

    model = Model(X_train, y_train, X_valid, y_valid, args.weights_folder)
    model.train()

max_stars_repo_name: logisticAKB/syrex

max_issues_repo_name: logisticAKB/EmotionRecognition

max_forks_repo_name: logisticAKB/EmotionRecognition


text:
import argparse

from net import Net
import pickle
from do_experiment import *
import networkx as nx
import numpy as np
import random
import os
from helpers import HiddenPrints

# 2nd experiment:
#   1) see the effects of varying the connectivity of the network
#   2) compare between scenarios: vanilla vs quarantine vs tracing
#   3) visualize via tables/heatmaps the effect of the latter two on the prevalence

parser = argparse.ArgumentParser()
parser.add_argument('--recompute', action='store_true', default=False)
args = parser.parse_args()
force_recompute = args.recompute

n = 500
p_i = 0.5
mc_iterations = 100
max_t = 200

res = 50
# interval = (0.008,0.03)
interval = (0.008,0.1)
# this means <res> equidistant points on <interval>

working_dir = os.getcwd()
path = os.path.join(working_dir,'Cache')

# the plots are created in cache folder
# vary_p(res, n, p_i, mc_iterations, max_t,interval=interval, mode=None, force_recompute=force_recompute, path=path)
# vary_p(res, n, p_i, mc_iterations, max_t,interval=interval, mode='quarantine', force_recompute=force_recompute, path=path)
# vary_p(res, n, p_i, mc_iterations, max_t,interval=interval, mode='tracing', force_recompute=force_recompute, path=path)
if __name__ == '__main__':
    with HiddenPrints():
        vary_p_plot_cache(res, n, p_i, mc_iterations, max_t, interval=interval, force_recompute=force_recompute, path=path)

reason: import numpy,import networkx


text:
from keras.models import model_from_json
import json
import cv2
import h5py
import numpy as np

import utils

img_path = r'C:\Users\lap\Desktop\XRT\img\split\ha\img45\img212.png'
img_path = r'img.png'
model_filepath = 'model-1612617459.json'
weights_filepath = 'weights-1612617459-0.921989.hdf5'

label_list = ['谈','般','盏','坤','膀','脂','型','骏','童','挟','损','恋','婴','读','账','服',
              '任','茸','张','亢','耀','涉','个','随','挂','抗','贞','瞥','瘤','作','河','欲',
              '侵','吸','眺','线','捂','倾','牌','筒','渊','拥','话','赞','知','除','巩','惫',
              '揭','扬','驼','绿','渔','榆','辊','应','儡','假','崩','抬','是','讲','刷','鸿',
              '契','寒','录','教','也','艾','囤','秦','峨','括','诲','滴','凶','须','孽','巾',
              '沉','餐','暂','蒙','攘','键','厄','的','芭','岳','惜','椰','足','伴','离','笼',
              '临','胁','泉','晚','迟','汞','级','跳','轴','偶','啸','移','贾','老','节','蜗',
              '堑','帕','肖','伟','渝','撮','臀','吉','汉','反','双','坏','翔','胖','绪','固',
              '舀','再','咏','堂','尔','沟','符','涵','水','误','岿','所','摄','广','结','学',
              '苫','臭','恬','诱','递','烷','硼','茁','标','越','吏','笑','馒','耗','氟','加',
              '砧','稻','晃','臂','其','配','城','筑','痹','揖','江','连','卡','狠','瓤','乳',
              '赵','仿','睹','相','好','屿','争','袭','王','吃','疏','粕','涟','垣','逢','锤',
              '覆','薯','贴','冷','霸','聂','糕','占']

def make_tagcode_list():
    dataset_filepath = 'HWDB1.1.hdf5'
    with h5py.File(dataset_filepath, 'r') as f1, open(r'label_list.txt', 'a') as f2:
        allowed = list(set(utils.unicode_to_tagcode(c) for c in label_list))
        for i in range(897758):
            tagcode = f1['trn/tagcode'][i][0]
            if tagcode in allowed:
                f2.write(str(tagcode) + '\n')

if __name__ == "__main__":
    with open(model_filepath) as f:
        d = json.load(f)
        model = model_from_json(json.dumps(d))

    model.load_weights(weights_filepath)
    model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])

    img = cv2.imread(img_path)
    img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # img = cv2.dilate(img, None)
    img = cv2.resize(img, (64, 64))

    cv2.imwrite(r'img.png', img)
    x = np.zeros([1, 1, 64, 64])
    x[0, 0] = img

    label_i = model.predict_classes(x)[0]
    print(label_i)

    with open(r'label_list.txt', 'r') as f:
        i = 0
        for tagcode in f:
            if i == label_i:
                label = utils.tagcode_to_unicode(int(tagcode))
                print(label)
                break
            else:
                i += 1


text:
"""carla_obj_det_dev dataset."""

import collections
import json
import os
import pandas
from copy import deepcopy
from PIL import Image
import numpy as np

import tensorflow.compat.v1 as tf
import tensorflow_datasets as tfds

_DESCRIPTION = """
Synthetic multimodality (RGB, depth) dataset generated using CARLA (https://carla.org).
"""

_CITATION = """
@inproceedings{Dosovitskiy17,
  title = { {CARLA}: {An} Open Urban Driving Simulator},
  author = {Alexey Dosovitskiy and German Ros and Felipe Codevilla and Antonio Lopez and Vladlen Koltun},
  booktitle = {Proceedings of the 1st Annual Conference on Robot Learning},
  pages = {1--16},
  year = {2017}
}
"""

# fmt: off
_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_od_dev.tar.gz"
# fmt: on

class CarlaObjDetDev(tfds.core.GeneratorBasedBuilder):
    """DatasetBuilder for carla_obj_det_dev dataset."""

    VERSION = tfds.core.Version("1.0.1")
    RELEASE_NOTES = {
        "1.0.0": "Initial release.",
        "1.0.1": "Correcting error to RGB and depth image pairing",
    }

    def _info(self) -> tfds.core.DatasetInfo:
        """Returns the dataset metadata."""
        features = {
            # sequence of [RGB, depth] images
            "image": tfds.features.Sequence(
                tfds.features.Image(shape=(600, 800, 3)), length=2,
            ),
            # sequence of image features for [RGB, depth]
            "images": tfds.features.Sequence(
                tfds.features.FeaturesDict(
                    {
                        "file_name": tfds.features.Text(),
                        "height": tf.int64,
                        "width": tf.int64,
                        "id": tf.int64,
                    },
                ),
                length=2,
            ),
            # both modalities share the same categories
            "categories": tfds.features.Sequence(
                tfds.features.FeaturesDict(
                    {
                        "id": tf.int64,  # {'pedstrian':1, 'vehicles':2, 'trafficlight':3, 'patch':4}
                        "name": tfds.features.Text(),
                    }
                )
            ),
            # both modalities share the same objects
            "objects": tfds.features.Sequence(
                {
                    "id": tf.int64,
                    "image_id": tf.int64,
                    "area": tf.int64,  # un-normalized area
                    "boxes": tfds.features.BBoxFeature(),  # normalized bounding box [ymin, xmin, ymax, xmax]
                    "labels": tfds.features.ClassLabel(num_classes=5),
                    "is_crowd": tf.bool,
                }
            ),
            # these data only apply to the "green screen patch" objects, which both modalities share
            "patch_metadata": tfds.features.FeaturesDict(
                {
                    "gs_coords": tfds.features.Sequence(
                        tfds.features.Tensor(
                            shape=[2], dtype=tf.int64
                        ),  # green screen vertices in (x,y)
                        length=4,  # always rectangle shape
                    ),
                    "cc_ground_truth": tfds.features.Tensor(
                        shape=[24, 3], dtype=tf.float32
                    ),  # colorchecker color ground truth
                    "cc_scene": tfds.features.Tensor(
                        shape=[24, 3], dtype=tf.float32
                    ),  # colorchecker colors in a scene
                    # binarized segmentation mask of patch.
                    # mask[x,y] == 1 indicates patch pixel; 0 otherwise
                    "mask": tfds.features.Tensor(shape=[600, 800, 3], dtype=tf.uint8),
                    "shape": tfds.features.Text(),
                }
            ),
        }

        return tfds.core.DatasetInfo(
            builder=self,
            description=_DESCRIPTION,
            features=tfds.features.FeaturesDict(features),
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager: tfds.download.DownloadManager):
        """Returns SplitGenerators."""
        path = dl_manager.download_and_extract(_URLS)
        return [
            tfds.core.SplitGenerator(
                name="dev", gen_kwargs={"path": os.path.join(path, "dev")},
            )
        ]

    def _generate_examples(self, path):
        """yield examples"""
        # For each image, gets its annotations and yield relevant data

        yield_id = 0

        patch_rgb = (
            180,
            130,
            70,
        )  # rgb values of patch object in semantic segmentation image

        annotation_path = os.path.join(path, "annotations/coco_annotations.json")

        cocoanno = COCOAnnotation(annotation_path)

        images_rgb = (
            cocoanno.images()
        )  # list of dictionaries of RGB image id, height, width, file_name

        # sort images alphabetically
        images_rgb = sorted(images_rgb, key=lambda x: x["file_name"].lower())

        for image_rgb in images_rgb:

            # Pairing RGB and depth
            fname_rgb = image_rgb["file_name"]  # rgb image file
            fname, fext = fname_rgb.split(".")
            fname_depth = fname + "_Depth." + fext  # depth image file
            image_depth = deepcopy(image_rgb)
            image_depth["file_name"] = fname_depth

            # get binarized patch mask
            fname, fext = image_rgb["file_name"].split(".")
            fname_ss = fname + "SS." + fext
            img_ss = Image.open(os.path.join(path, fname_ss)).convert("RGB")
            img_ss = np.array(img_ss)
            mask = np.zeros_like(img_ss)
            mask[np.all(img_ss == patch_rgb, axis=-1)] = 1

            # get object annotations for each image
            annotations = cocoanno.get_annotations(image_rgb["id"])

            # convert bbox to Pytorch format
            def build_bbox(x, y, width, height):
                return tfds.features.BBox(
                    ymin=y / image_rgb["height"],
                    xmin=x / image_rgb["width"],
                    ymax=(y + height) / image_rgb["height"],
                    xmax=(x + width) / image_rgb["width"],
                )

            # convert segmentation format of (x0,y0,x1,y1,...) to ( (x0, y0), (x1, y1), ... )
            def build_coords(segmentation):
                xs = segmentation[
                    :-2:2
                ]  # last two values are repeats of the first two values
                ys = segmentation[1:-2:2]
                coords = [[int(round(x)), int(round(y))] for (x, y) in zip(xs, ys)]
                assert len(coords) == 4

                return coords

            # get colorchecker color box values. There are 24 color boxes, so output shape is (24, 3)
            def get_cc(ground_truth=True):
                if ground_truth:
                    return (
                        pandas.read_csv(
                            os.path.join(
                                path,
                                "annotations",
                                "xrite_passport_colors_sRGB-GMB-2005.csv",
                            ),
                            header=None,
                        )
                        .to_numpy()
                        .astype("float32")
                    )
                else:
                    return (
                        pandas.read_csv(
                            os.path.join(
                                path, "annotations", fname_rgb.split(".")[-2] + ".csv",
                            ),
                            header=None,
                        )
                        .to_numpy()
                        .astype("float32")
                    )

            example = {
                "image": [
                    os.path.join(path, modality,)
                    for modality in [fname_rgb, fname_depth]
                ],
                "images": [image_rgb, image_depth],
                "categories": cocoanno.categories(),
                "objects": [
                    {
                        "id": anno["id"],
                        "image_id": anno["image_id"],
                        "area": anno["areas"],
                        "boxes": build_bbox(*anno["bbox"]),
                        "labels": anno["category_id"],
                        "is_crowd": bool(anno["iscrowd"]),
                    }
                    for anno in annotations
                ],
                "patch_metadata": [
                    {
                        "gs_coords": build_coords(*anno["segmentation"]),
                        "cc_ground_truth": get_cc(),
                        "cc_scene": get_cc(ground_truth=False),
                        "mask": mask,
                        "shape": "rect",
                    }
                    for anno in annotations
                    if anno["category_id"] == 4  # is patch
                ][0],
            }

            yield_id = yield_id + 1

            yield yield_id, example

class COCOAnnotation(object):
    """COCO annotation helper class."""

    def __init__(self, annotation_path):
        with tf.io.gfile.GFile(annotation_path) as f:
            data = json.load(f)
        self._data = data

        # for each images["id"], find all annotations such that annotations["image_id"] == images["id"]
        img_id2annotations = collections.defaultdict(list)
        for a in self._data["annotations"]:
            img_id2annotations[a["image_id"]].append(a)
        self._img_id2annotations = {
            k: list(sorted(v, key=lambda a: a["id"]))
            for k, v in img_id2annotations.items()
        }

    def categories(self):
        """Return the category dicts, as sorted in the file."""
        return self._data["categories"]

    def images(self):
        """Return the image dicts, as sorted in the file."""
        return self._data["images"]

    def get_annotations(self, img_id):
        """Return all annotations associated with the image id string."""
        return self._img_id2annotations.get(img_id, [])

max_stars_repo_name: audreyruckman/armory

max_issues_repo_name: audreyruckman/armory

max_forks_repo_name: audreyruckman/armory


text:
#!/usr/bin/env python

# Copyright (c) 2016 PAL Robotics SL. All Rights Reserved
#
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all
# copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# Author:
#   * Sam Pfeiffer
#   * Job van Dieten
#   * Jordi Pages
#
# This document is a modified version of the original pick_client.py
# containing only the SphericalService class and the main.

import rospy

import pick_client
import place_client

import numpy as np
from std_srvs.srv import Empty

from moveit_msgs.msg import MoveItErrorCodes
moveit_error_dict = {}
for name in MoveItErrorCodes.__dict__.keys():
    if not name[:1] == '_':
        code = MoveItErrorCodes.__dict__[name]
        moveit_error_dict[code] = name

class SphericalService(object):
    def __init__(self):
        rospy.loginfo("Starting Spherical Grab Service")
        self.pick_type = pick_client.PickAruco()
        self.place_type = place_client.PlaceAruco()
        rospy.loginfo("Finished SphericalService constructor")
        self.place_gui = rospy.Service("/place_gui", Empty, self.start_aruco_place)
        self.pick_gui = rospy.Service("/pick_gui", Empty, self.start_aruco_pick)

    def start_aruco_pick(self, req):
        self.pick_type.pick_aruco("pick")
        return {}

    def start_aruco_place(self, req):
        self.place_type.place_aruco("place")
        return {}

if __name__ == '__main__':
    rospy.init_node('pick_aruco_demo')
    sphere = SphericalService()
    rospy.spin()


text:
# Copyright (c) 2021 Bill Silver. Licence granted to public under terms of MIT licence
# at https://github.com/GizmoBill/QuantumComputing/blob/main/LICENSE

# ************************
# *                      *
# *  Grover's Algorithm  *
# *                      *
# ************************

from qiskit import QuantumCircuit, QuantumRegister, AncillaRegister
from numpy import pi, sqrt, arcsin

# ***********************************************
# *                                             *
# *  Create Multi-Control X from Toffoli Gates  *
# *                                             *
# ***********************************************
#
# Add a controlled X gate to the specified QuantumCircuit, using the specified
# list of control qubits, the specified target qubit, and the specified list
# of ancilla qubits. Any number of controls >= 0 can be specified. The number
# of ancillas must be at least two fewer than the number of controls; these
# qubits will be left in their original state. A minimum depth circuit is
# created.
def multiX(qc, controls, target, ancilla) :
    n = len(controls)
    uncompute = []
    if n > 2 :
        ai = 0  # ancilla index

        while n > 2 :
            newControls = [];
            for i in range(0, n, 2):
                if i == n - 1 :
                    newControls.append(controls[i])
                else :
                    qc.toffoli(controls[i], controls[i + 1], ancilla[ai])
                    uncompute.insert(0, [controls[i], controls[i + 1], ancilla[ai]])
                    newControls.append(ancilla[ai])
                    ai = ai + 1
            controls = newControls
            n = len(controls)

    if n == 0 :
        qc.x(target)
    elif n == 1 :
        qc.cx(controls[0], target)
    else :
        qc.toffoli(controls[0], controls[1], target)
    for t in uncompute :
        qc.toffoli(t[0], t[1], t[2])

# **********************************
# *                                *
# *  Select Oracle's Winner State  *
# *                                *
# **********************************
#
# An oracle is created with a multi-controlled X gate (MCX), one control qubit for
# each oracle qubit. With just the MCX, the winning state would br all 1's. To select
# any winner, this function adds X gates to every qubit whose corresponding bit in
# winner is 0. The X gates must be added both before and after the MCX. so that the
# desired winner is shifted to the all 1's state before the MCX, and back after the
# MCX.

def oracle(qc, qubits, winner) :
    for q in qubits :
        if (winner & 1) == 0 :
            qc.x(q)
        winner = winner >> 1

# ************************************
# *                                  *
# *  Make and Return Grover Circuit  *
# *                                  *
# ************************************

def grover(numQbits, winner) :
    # Number of iterations
    n = int(pi / (4 * arcsin(2 ** (-numQbits / 2))))

    # Create circuit. There are quantum and classical registers whose sizes are the
    # specified numQbits. There are numQbits-2 ancillas for the MCX gates (oracle
    # and diffuser), and one more for the oracle target qubit for phase kickback.
    qc = QuantumCircuit(numQbits, numQbits, name = "Grover:q={0},i={1},w={2}".format(numQbits, n, winner))
    qList = list(qc.qregs[0])
    areg = AncillaRegister(numQbits - 1)
    qc.add_register(areg)
    target = areg[0]
    ancilla = [areg[i] for i in range(1, areg.size)]

    # Initialize the primary qubits to |+> and the target ancilla to |->
    for q in range(numQbits) :
        qc.h(qList[q])
    qc.x(target)
    qc.h(target)

    for s in range(n) :
        # Oracle
        oracle(qc, qList, winner)
        multiX(qc, qList, target, ancilla)
        oracle(qc, qList, winner)

        # Diffuse (amplitude amplification)
        for q in range(1, numQbits) :
            qc.h(qList[q])
            qc.x(qList[q])
        qc.z(qList[0])
        multiX(qc, [qList[i] for i in range(1, numQbits)], qList[0], ancilla)
        qc.z(qList[0])
        for q in range(1, numQbits) :
            qc.x(qList[q])
            qc.h(qList[q])

    # Measure the result
    for q in range(numQbits) :
        qc.measure(qList[q], qc.cregs[0][q])

    print("{0} qubits, {1} iterations, winner = {2}".format(numQbits, n, winner))

    return qc

max_stars_repo_name: GizmoBill/QuantumComputing

max_issues_repo_name: GizmoBill/QuantumComputing

max_forks_repo_name: GizmoBill/QuantumComputing

reason: from numpy


text:
import os
from os.path import dirname, join
from typing import List

import numpy as np
import scipy.stats
from loguru import logger

from deidentify.base import Document
from deidentify.dataset import brat

PREDICTIONS_PATH = join(dirname(__file__), '../../output/predictions')

class LogUniform():

    def __init__(self, a=-1, b=0, base=10):
        self.loc = a
        self.scale = b - a
        self.base = base

    def rvs(self, size=None, random_state=None):
        uniform = scipy.stats.uniform(loc=self.loc, scale=self.scale)
        if not size:
            return np.power(self.base, uniform.rvs(random_state=random_state))

        return np.power(self.base, uniform.rvs(size=size, random_state=random_state))

def model_dir(corpus_name, run_id):
    return join(PREDICTIONS_PATH, corpus_name, run_id)

def _save_predictions(path, documents: List[Document]):
    os.makedirs(path, exist_ok=True)

    for doc in documents:
        brat.write_brat_annotations(doc.annotations, join(path, '{}.ann'.format(doc.name)))

def save_predictions(corpus_name, run_id,
                     train: List[Document] = None,
                     test: List[Document] = None,
                     dev: List[Document] = None):

    for part_name, part_docs in zip(['train', 'test', 'dev'], [train, test, dev]):
        if not part_docs:
            continue

        logger.info('Write {}.{}.{} predictions (N = {})'.format(corpus_name, run_id, part_name,
                                                                 len(part_docs)))
        base_path = model_dir(corpus_name, run_id)
        _save_predictions(join(base_path, part_name), part_docs)

max_stars_repo_name: bbieniek/deidentify

max_issues_repo_name: HabibMrad/deidentify

max_forks_repo_name: HabibMrad/deidentify


text:
"""
Tests class Contingency

# Author: Vladan Lucic
# $Id: test_cluster.py 1520 2019-03-11 17:00:20Z vladan $
"""

__version__ = "$Revision$"

import warnings
import unittest

import numpy as np
import numpy.testing as np_test
import scipy as sp
import pandas as pd

from pyto.clustering.contingency import Contingency

class TestCluster(np_test.TestCase):
    """
    """

    def setUp(self):
        """
        Set some variables
        """

        self.clust_data_1 = np.array([
            [3, 4, 10, 5],
            [3, 2, 6, 12],
            [4, 2, 6, 1],
            [10, 1, 2, 3],
            [3, 5, 7, 9]])
        self.clust_matched_1 = np.array([
            [10, 1, 2, 3],
            [0, 0, 0, 0],
            [7, 6, 16, 6],
            [6, 7, 13, 21]])
        self.class_ref_1 = dict({0:2, 1:3, 2:2, 3:0, 4:3})

    def test_match(self):
        """
        Tests match()
        """

        # ndarray
        con = Contingency(data=self.clust_data_1)
        con.match()
        np_test.assert_equal(con.matched, self.clust_matched_1)
        np_test.assert_equal(con.class_ref, self.class_ref_1)

        # DataFrame
        data_df = pd.DataFrame(data=self.clust_data_1)
        con = Contingency(data=data_df)
        con.match()
        np_test.assert_equal(con.matched, self.clust_matched_1)
        np_test.assert_equal(con.class_ref, self.class_ref_1)

    def test_evaluate(self):
        """
        Tests evaluate()
        """

        # test all calculated attributes
        con = Contingency(data=self.clust_data_1)
        con.evaluate()
        np_test.assert_equal(np.array(con.class_margin), [22, 23, 13, 16, 24])
        np_test.assert_equal(
            np.array(con.class_margin_matched), [16, 0, 35, 47])
        np_test.assert_equal(np.array(con.ref_margin), [23, 14, 31, 30])
        np_test.assert_equal(np.array(con.tp), [10, 12, 6, 10, 9])
        np_test.assert_equal(np.array(con.tp_matched), [10, 0, 16, 21])
        np_test.assert_equal(np.array(con.fp_matched), [6, 0, 19, 26])
        np_test.assert_equal(np.array(con.fn_matched), [13, 14, 15, 9])
        desired_precision = (10./16 + 16./35 + 21./47) / 4
        np_test.assert_equal(con.precision_matched, desired_precision)
        desired_recall = (10./23 + 0 + 16./31 + 21./30) / 4
        np_test.assert_equal(con.recall_matched, desired_recall)
        desired_recall_adj = (10./23 + 0 + 16./31 + 21./30) / 5
        np_test.assert_equal(con.recall_matched_adj, desired_recall_adj)
        np_test.assert_almost_equal(
            con.f1_matched, 2 / (1./desired_precision + 1./desired_recall))
        np_test.assert_almost_equal(
            con.f1_matched_adj,
            2 / (1./desired_precision + 1./desired_recall_adj))

        # DataFrame
        data_df = pd.DataFrame(data=self.clust_data_1)
        con = Contingency(data=data_df)
        con.evaluate()
        desired_precision = (10./16 + 16./35 + 21./47) / 4
        np_test.assert_equal(con.precision_matched, desired_precision)
        desired_recall = (10./23 + 0 + 16./31 + 21./30) / 4
        np_test.assert_equal(con.recall_matched, desired_recall)
        desired_recall_adj = (10./23 + 0 + 16./31 + 21./30) / 5
        np_test.assert_equal(con.recall_matched_adj, desired_recall_adj)

    def test_do_g_test(self):
        """
        Tests do_g_test()
        """

        con = Contingency(data=self.clust_data_1)
        con.do_g_test()
        np_test.assert_equal(con.g_test.dof, 12)
        np_test.assert_almost_equal(
            con.g_test.p,
            sp.stats.chi2_contingency(
                self.clust_data_1, lambda_="log-likelihood",
                correction=True)[1])
        np_test.assert_almost_equal(con.g_test.g, 25.9416331, decimal=5)

if __name__ == '__main__':
    suite = unittest.TestLoader().loadTestsFromTestCase(TestCluster)
    unittest.TextTestRunner(verbosity=2).run(suite)


text:
import numpy
from defines import Types
from scaler import NumericScaler
from binomial import BinomialClassifier
from sklearn import feature_selection
from sklearn import decomposition
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_predict
from sklearn.metrics import classification_report
from sklearn.metrics import f1_score, accuracy_score
from sklearn.metrics import confusion_matrix

def baseline_classifier(data, labels):
    print 'Applying baseline classification'
    clf = BinomialClassifier(0.7)
    pred = cross_val_predict(clf, data, labels, cv=10)
    clf.best_score_ = accuracy_score(labels, pred)

    print 'accuracy: %0.3f' % accuracy_score(labels, pred)
    print '\n',
    print classification_report(labels, pred)

    return clf

def knn_classifier(data, labels, columns):
    print 'Applying k-nearest neighbor classification'
    # create param grid
    n_numeric = len([c.TYPE for c in columns if c.TYPE is Types.NUMERICAL and c.CATEGORIES is None])
    n_neighbors = list(range(1, 51, 1))
    parameters = dict(knn__n_neighbors=n_neighbors)

    # create model pipeline
    ns = NumericScaler(n_numeric)
    rf = RandomForestClassifier() #random_state=8)
    knn = KNeighborsClassifier()
    rfe = feature_selection.RFE(rf)
    pipe = Pipeline(steps=[('ns', ns),
                           ('rfe', rfe),
                           ('knn', knn)])

    # run grid search with 10-fold cross validation
    clf = GridSearchCV(pipe, parameters, cv=10, verbose=1)
    clf.fit(data, labels)
    pred = clf.predict(data)

    print 'accuracy: %0.3f' % clf.best_score_
    print 'Best parameters set: '
    best_parameters = clf.best_estimator_.get_params()
    for param_name in sorted(parameters.keys()):
        print '\t%s: %r' % (param_name, best_parameters[param_name])
    print '\n',
    print classification_report(labels, pred)

    return clf

def svm_classifier(data, labels, columns):
    print 'Applying SVM classification with RBF kernel'
    # create param grid
    n_numeric = len([c.TYPE for c in columns if c.TYPE is Types.NUMERICAL and c.CATEGORIES is None])
    C = [0.1, 1, 10, 100, 1000]
    gamma = ['auto', 1, 0.1, 0.001, 0.0001]
    parameters = dict(svm__C=C,
                      svm__gamma=gamma)

    # create model pipeline
    ns = NumericScaler(n_numeric)
    rf = RandomForestClassifier() #random_state=2)
    rfe = feature_selection.RFE(rf)
    svm = SVC(kernel='rbf') #, random_state=17)
    pipe = Pipeline(steps=[('ns', ns),
                           ('rfe', rfe),
                           ('svm', svm)])

    # run grid search with 10-fold validation
    clf = GridSearchCV(pipe, parameters, cv=10, verbose=1)
    clf.fit(data, labels)
    pred = clf.predict(data)

    print 'accuracy: %0.3f' % clf.best_score_
    print 'Best parameters set: '
    best_parameters = clf.best_estimator_.get_params()
    for param_name in sorted(parameters.keys()):
        print '\t%s: %r' % (param_name, best_parameters[param_name])
    print '\n',
    print classification_report(labels, pred)

    return clf

def naive_bayes_classifier(data, labels, columns):
    print 'Applying Naive Bayes classification'
    # create param grid
    n_numeric = len([c.TYPE for c in columns if c.TYPE is Types.NUMERICAL and c.CATEGORIES is None])
    n_components = list(range(1, data.shape[1] + 1, 1))
    parameters = dict(pca__n_components=n_components)

    # create model pipeline
    ns = NumericScaler(n_numeric, with_std=False)
    rf = RandomForestClassifier() #random_state=2)
    rfe = feature_selection.RFE(rf)
    pca = decomposition.PCA()
    gnb = GaussianNB()
    pipe = Pipeline(steps=[('ns', ns),
                           ('pca', pca),
                           ('gnb', gnb)])

    # run grid search with 10-fold validation
    clf = GridSearchCV(pipe, parameters, cv=10, verbose=1)
    clf.fit(data, labels)
    pred = clf.predict(data)

    print 'accuracy: %0.3f' % clf.best_score_
    print 'Best parameters set: '
    best_parameters = clf.best_estimator_.get_params()
    for param_name in sorted(parameters.keys()):
        print '\t%s: %r' % (param_name, best_parameters[param_name])
    print '\n',
    print classification_report(labels, pred)

    return clf

max_stars_repo_name: nikhilnrng/german-credit-risk

max_issues_repo_name: nikhilnrng/german-credit-risk

max_forks_repo_name: nikhilnrng/german-credit-risk


text:
from pprint import pprint
from datetime import datetime
import time
import os
import json
import math
import itertools as it
from pytz import timezone
from dataclasses import dataclass, asdict

from tqdm import tqdm
import numpy as np

import tensorflow as tf

from ..src.channelcoding.dataclasses import EncoderDecoderSettings
from ..src.channelcoding.encoder_decoders import EncoderDecoder
from ..src.utils import  TurboCodesJSONEncoder, safe_open_dir

@dataclass
class Stats:
    """Stats kept track of by the benchmarker"""
    ber: float
    bler: float
    total_bit_errors: int
    total_block_errors: int
    num_blocks: int
    runtime: float
    ber_var: float
    bler_var: float

@dataclass
class TestRunnerSettings:
    """Settings object for TurboRunner"""
    encoder_decoder_settings: EncoderDecoderSettings
    block_len: int
    num_blocks: int
    batch_size: int
    gpu: bool

@dataclass
class TestResult:
    """Object to hold details on test run"""
    test_number: int
    model_id: str
    test_run_settings: TestRunnerSettings
    stats: Stats

class TestRunner:

    def __init__(self, encoder_decoder: EncoderDecoder, block_len: int, num_blocks: int, batch_size: int):
        self.encoder_decoder = encoder_decoder
        self.block_len = block_len
        self.num_blocks = num_blocks
        self.batch_size = batch_size

        self._batch_starts = range(0, self.num_blocks, self.batch_size)
        self._batch_ends = it.chain(self._batch_starts[1:], [self.num_blocks])

    def __len__(self):
        return len(self._batch_starts)

    @tf.function
    def _run_iter(self, cur_batch_size):
        original_msg = tf.cast(tf.random.uniform((cur_batch_size, self.block_len, 1), minval=0, maxval=2, dtype=tf.int32), dtype=tf.float32)
        decoded_msg = tf.cast(self.encoder_decoder(original_msg) > 0, tf.float32)

        num_bit_errors = tf.reduce_sum(tf.cast(tf.not_equal(original_msg, decoded_msg), tf.float32), axis=1)[:, 0]

        return num_bit_errors

    def __iter__(self):
        for batch_start, batch_end in zip(self._batch_starts, self._batch_ends):
            cur_batch_size = batch_end - batch_start

            yield self._run_iter(cur_batch_size).numpy()

    def settings(self):
        return TestRunnerSettings(
            encoder_decoder_settings=self.encoder_decoder.settings(),
            block_len=self.block_len,
            num_blocks=self.num_blocks,
            batch_size=self.batch_size,
            gpu=tf.test.is_gpu_available()
        )

class CodeBenchmarker:

    def __init__(self, model_title: str = "", save_to_file=False, save_plot=False, show_plot=False, tzname='America/Chicago'):
        self.model_title = model_title
        self.tz = timezone(tzname)
        self.timestamp = datetime.now().astimezone(self.tz).strftime('%Y-%m-%d.%H-%M-%S')

        if self.model_title:
            self.model_id = ".".join([model_title, self.timestamp])
        else:
            self.model_id = self.timestamp

        self.save_filedir = "./results"
        self.save_filename = ".".join([self.model_id, "json"])
        self.save_to_file = save_to_file

    def _run_wrapper(self, runner):
        """Use this for callbacks"""
        total_bit_errors = 0
        total_block_errors = 0
        ber = 1.
        bler = 1.
        block_ber = np.zeros((runner.num_blocks,), dtype=np.float32)
        block_bler = np.zeros((runner.num_blocks,), dtype=np.int8)
        num_blocks = 0
        num_bits = 0

        pbar = tqdm(runner)
        start_time = time.time()
        for i, bit_errors in enumerate(pbar, 1):
            # Global trackers
            new_num_blocks = num_blocks + bit_errors.shape[0]
            block_ber[num_blocks:new_num_blocks] = bit_errors / runner.block_len
            block_has_error = (bit_errors > 0)
            block_bler[num_blocks:new_num_blocks] = block_has_error

            num_bits += runner.block_len * bit_errors.shape[0]
            num_blocks = new_num_blocks
            total_bit_errors += np.sum(bit_errors)
            total_block_errors += np.count_nonzero(block_has_error)
            ber = total_bit_errors / num_bits
            bler = total_block_errors / num_blocks
            pbar.set_postfix(ber=ber, bler=bler)
        end_time = time.time()

        ber_var = np.var(block_ber, ddof=1)
        bler_var = np.var(block_bler, ddof=1)

        print(len(block_ber))
        print(len(block_bler))
        print(f"BER VAR: {np.sum((block_ber - ber) ** 2) / (runner.num_blocks - 1)}")
        print(f"BLER VAR: {np.sum((block_bler - bler) ** 2) / (runner.num_blocks - 1)}")
        # print(block_ber.tolist())
        # print(block_bler.tolist())

        return Stats(
            total_bit_errors=total_bit_errors,
            total_block_errors=total_block_errors,
            ber=ber,
            bler=bler,
            num_blocks=num_blocks,
            runtime=end_time-start_time,
            ber_var=ber_var,
            bler_var=bler_var
        )

    def benchmark(self, encoder_decoders, block_len, num_blocks, batch_size):
        results = []
        for test_num, encoder_decoder in enumerate(encoder_decoders):
            runner = TestRunner(encoder_decoder, block_len, num_blocks, batch_size)

            print(f"Running test {test_num}")
            pprint(asdict(runner.settings()))

            stats = self._run_wrapper(runner)
            print(f"Complete test {test_num} for model {self.model_id}")
            pprint(asdict(stats))

            results.append(TestResult(test_number=test_num, model_id=self.model_id, test_run_settings=runner.settings(), stats=stats))

            if self.save_to_file:
                self.save_file(results)

        return results

    def safe_get_save_filepath(self):
        return os.path.join(safe_open_dir(self.save_filedir), self.save_filename)

    def save_file(self, results):
        with open(self.safe_get_save_filepath(), 'w') as f:
                json.dump(results, f, cls=TurboCodesJSONEncoder)


text:
#!/usr/bin/python
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import sys
import math
from optparse import OptionParser

parser = OptionParser()

parser.add_option("-b",
                  action="store_true",
                  default=False,
                  dest="black",
                  help="black background")

parser.add_option("--x_size",
                  dest="x_size",
                  type="int",
                  default=10,
                  help="Figure x size (Default 10)")

parser.add_option("--y_size",
                  dest="y_size",
                  type="int",
                  default=30,
                  help="Figure x size (Default 30)")

parser.add_option("--no_labels",
                  dest="no_labels",
                  action="store_true",
                  default=False,
                  help="Do not label x and y axis");

parser.add_option("--no_ylabels",
                  dest="no_ylabels",
                  action="store_true",
                  default=False,
                  help="Do not label y axis");

parser.add_option("--stat",
                  dest="stat",
                  default="combo",
                  help="Stat to plot (odds, sig, combo) (Default: combo)")

parser.add_option("-i",
                  "--input",
                  dest="input_file",
                  help="GIGGLE result file name with '-s'")

parser.add_option("--ablines",
                  dest="ablines")

parser.add_option("-o",
                  "--output",
                  dest="output_file",
                  help="Output file name")

parser.add_option("-s",
                  "--states",
                  dest="states_file",
                  help="States file name")

parser.add_option("-n",
                  "--cell_names",
                  dest="cells_names_file",
                  help="Cells names file name")

parser.add_option("--sort",
                  dest="cells_sort_file",
                  help="Cells sort order file name")

parser.add_option("--state_names",
                  dest="state_names",
                  help="States names")

parser.add_option("--group_names",
                  dest="group_names",
                  help="Group names")

parser.add_option("-c",
                  "--cells",
                  dest="cells_file",
                  help="Cells file name")

(options, args) = parser.parse_args()

if not options.output_file:
    parser.error('Output file not given')
if not options.input_file:
    parser.error('Input file not given')
if not options.states_file:
    parser.error('States file not given')
if not options.cells_file:
    parser.error('Cells file not given')

if options.stat not in ['odds', 'sig', 'combo']:
    parser.error('Stat "' + options.stat + '" not supported')

states = []
for l in open(options.states_file, 'r'):
    states.append(l.rstrip().split('\t')[1])

cells = []
for l in open(options.cells_file, 'r'):
    cells.append(l.rstrip().split('\t')[1])

names = []
if options.cells_names_file:
    for l in open(options.cells_names_file, 'r'):
        names.append(l.rstrip().split('\t')[1])

sorts = []
if options.cells_sort_file:
    for l in open(options.cells_sort_file, 'r'):
        sorts.append(l.rstrip().split('\t')[1])

M = {}

for c in cells:
    M[c] = {}
    for s in states:
        M[c][s] = 0

for l in open(options.input_file,'r'):
    if l[0] == '#':
        continue
    A = l.rstrip().split('\t')
    odds = float(A[3])
    sig = float(A[4])
    combo = float(A[7])

    c = ''
    s = ''
    found = False
    for i in cells:
        if '/' + i in A[0]:
            assert not found, (A[0], i)
            c = i
            found = True

    found = False
    for i in states:
        if c + '_' + i in A[0]:
            assert not found, (A[0], i)
            s = i
            found = True

    assert M[c][s] == 0, (A[0], c, s, M[c][s])

    if options.stat == 'sig':
        M[c][s] = sig
    elif options.stat == 'combo':
        M[c][s] = combo
    elif options.stat == 'odds':
        M[c][s] = odds

D=np.zeros([len(cells),len(states)])

c = 0
for cell in cells:
    s = 0
    for state in states:
        D[c,s] = M[cell][state]
        s+=1
    c+=1

column_labels = cells
row_labels = states
data = D

fig, ax = plt.subplots(figsize=(options.x_size,options.y_size), \
                       dpi=300)
if options.black:
    fig =  matplotlib.pyplot.figure(figsize=(options.x_size,options.y_size), \
                                    dpi=300, \
                                    facecolor='black')

    ax = fig.add_subplot(1,1,1,axisbg='k')

#plt.pcolor(data, cmap=plt.cm.Blues)
#plt.pcolor(data, cmap=plt.cm.bwr)

if options.cells_sort_file:
    idxs = np.argsort(sorts)
    sorts=np.array(sorts)[idxs]
    data=data[idxs]
elif options.cells_names_file:
    idxs = np.argsort(names)
    names=np.array(names)[idxs]
    data=data[idxs]

from matplotlib import colors as mcolors
from matplotlib.colors import Normalize

_seismic_data = ( (0.0, 0.0, 0.3),
                  (0.0, 0.0, 1.0),

                  (1.0, 1.0, 1.0),

                  (1.0, 0.0, 0.0),
                  (0.5, 0.0, 0.0))

hm = mcolors.LinearSegmentedColormap.from_list( \
        name='red_white_blue', \
        colors=_seismic_data, N=256)

class MidpointNormalize(Normalize):
    def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False):
        self.midpoint = midpoint
        Normalize.__init__(self, vmin, vmax, clip)

    def __call__(self, value, clip=None):
        # I'm ignoring masked values and all kinds of edge cases to make a
        # simple example...
        x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]
        return np.ma.masked_array(np.interp(value, x, y))

#norm=matplotlib.colors.LogNorm(vmin=data.min(), vmax=data.max()), \
print data.min(), data.max()

plt.pcolor(data, \
           norm = MidpointNormalize(midpoint=0),
           cmap=hm)
           #cmap=plt.cm.seismic)
ax.xaxis.tick_top()

ax.xaxis.set_ticks_position('none')
ax.yaxis.set_ticks_position('none')

if options.ablines:
    for l in [int(x) for x in options.ablines.split(',')]:
        ax.axhline(l, linestyle='--', color='k')

if options.no_labels:
    ax.set_xticklabels("")
    ax.set_yticklabels("")
elif options.no_ylabels:
    ax.set_yticklabels("")

    if options.state_names:
        state_names = []
        for l in open(options.state_names, 'r'):
            state_names.append(l.rstrip())
        if options.black:
            plt.xticks(np.arange(0.5,len(state_names)+0.5,1.0),state_names,rotation=90, fontsize=10, color='white')
        else:
            plt.xticks(np.arange(0.5,len(state_names)+0.5,1.0),state_names,rotation=90, fontsize=10)
    else:
        if options.black:
            plt.xticks(np.arange(0.5,len(states)+0.5,1.0),states,rotation=90, fontsize=10, color='white')
        else:
            plt.xticks(np.arange(0.5,len(states)+0.5,1.0),states,rotation=90, fontsize=10)
else:
    if options.group_names:

        group_names = []
        for l in open(options.group_names, 'r'):
            group_names.append(l.rstrip())
        group_names.reverse()
        if options.black:
            plt.yticks(np.arange(0.5,len(group_names)+0.5,1.0),[x.decode("utf8") for x in group_names], fontsize=10, color='white')
        else:
            plt.yticks(np.arange(0.5,len(group_names)+0.5,1.0),[x.decode("utf8") for x in group_names], fontsize=10)

    elif options.cells_names_file:
        if options.black:
            plt.yticks(np.arange(0.5,len(names)+0.5,1.0),[x.decode("utf8") for x in names], fontsize=10, color='white')
        else:
            plt.yticks(np.arange(0.5,len(names)+0.5,1.0),[x.decode("utf8") for x in names], fontsize=10)

    else:
        if options.black:
            plt.yticks(np.arange(0.5,len(cells)+0.5,1.0),cells, fontsize=10, color='white')
        else:
            plt.yticks(np.arange(0.5,len(cells)+0.5,1.0),cells, fontsize=10)

    if options.state_names:
        state_names = []
        for l in open(options.state_names, 'r'):
            state_names.append(l.rstrip())
        if options.black:
            plt.xticks(np.arange(0.5,len(state_names)+0.5,1.0),state_names,rotation=90, fontsize=10, color='white')
        else:
            plt.xticks(np.arange(0.5,len(state_names)+0.5,1.0),state_names,rotation=90, fontsize=10)
    else:
        if options.black:
            plt.xticks(np.arange(0.5,len(state_names)+0.5,1.0),state_names,rotation=90, fontsize=10, color='white')
        else:
            plt.xticks(np.arange(0.5,len(states)+0.5,1.0),states,rotation=90, fontsize=10)

plt.ylim((0,len(cells)))
plt.xlim((0,len(states)))

#cbar_min = -1 * max([abs(data.min()), data.max()])
#cbar_max = max([abs(data.min()), data.max()])

cbar = plt.colorbar(fraction=0.046, pad=0.04, ticks=[data.min(), 0, data.max()])
cbar.ax.tick_params(labelsize=10)

if options.black:
    cbar.ax.tick_params(color='white', labelcolor='white')
    plt.savefig(options.output_file,bbox_inches='tight',\
                facecolor=fig.get_facecolor(),\
                transparent=True)
else:
    plt.savefig(options.output_file,bbox_inches='tight')


text:
# Load two images
import cv2
import numpy as np
from matplotlib import pyplot as plt

# augs = torchvision.transforms.Compose([
#    torchvision.transforms.RandomHorizontalFlip(), color_aug, shape_aug])
# apply(img, augs)

"""
def apply(img_tensored_img, aug, shape=(8, 16)):
    rows, columns = shape
    fig_size = 900
    fig = plt.figure(dpi=fig_size)

    for position in range(1, (rows * columns) + 1):
        augmented = aug(img_tensored_img)
        img_altered = augmented.T.numpy() * 255
        img = np.swapaxes(img_altered, 0, 1)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(dtype=np.int32)
        fig.add_subplot(rows, columns, position)
        plt.imshow(img)
        plt.axis('off')
    plt.show()
"""

def show_image(image):
    cv2.imshow('image', image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

def embed_image():
    gray = cv2.cvtColor(blur, cv2.COLOR_BGR2GRAY)

    thresh = cv2.threshold(gray, 1, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
    thresh_inv = 255 - thresh
    contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    resulting_contours = max(contours, key=cv2.contourArea)[:, 0, :]

    # polygon = Polygon(np.array(resulting_contours))

    # rectangle = tuple(map(int, polygon.bounds))

    roi = background_image[y_offset:y_offset + o_height, x_offset:x_offset + o_width].copy()
    output = cv2.seamlessClone(object_image, background_image, thresh, (o_height + 150, o_width + 150), cv2.NORMAL_CLONE)
    show_image(output)

    roi_background = cv2.bitwise_and(roi, roi, mask=thresh_inv)
    dst = cv2.add(roi_background, object_image)
    background_image[y_offset: y_offset + o_height, x_offset:x_offset + o_width, :] = dst

if __name__ == '__main__':
    background_image = cv2.imread('../data/background_imgs/anthony-tran-iRbHr8jpmGw-unsplash.jpg')
    object_image = cv2.imread('../data/augmented_objects/0a2ca7e9d62c4a96831ad9a245b405c7.jpg')

    x_offset, y_offset = (0, 0)
    o_height, o_width = object_image.shape[:2]
    blur = cv2.GaussianBlur(object_image, (5, 5), 1)
    """
    tran = torchvision.transforms.ToTensor()
    """
    gray = cv2.cvtColor(blur, cv2.COLOR_BGR2GRAY)

    thresh = cv2.threshold(gray, 1, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
    thresh_inv = 255 - thresh
    # result = poisson_edit(object_image, background_image[y_offset:y_offset + o_height, x_offset:x_offset + o_width].copy(), thresh, (0, 0))
    # background_image[y_offset:y_offset + o_height, x_offset:x_offset + o_width] = result
    embed_image()

    """
    augmentations = [RandomHorizontalFlip(),
                     RandomVerticalFlip(),
                     RandomRotation((-360, 360)),
                     RandomErasing(scale=(0.02, 0.15), p=0.3),
                     RandomPerspective(distortion_scale=0.65),
                     RandomSolarize(0.4, p=0.2),
                     ColorJitter(brightness=0.0, contrast=0.0, saturation=0.5, hue=0.5)]
    aug_transformations = transforms.Compose(augmentations)

    apply(tran(object_image), aug_transformations)
    """


text:
# coding: utf-8
#

import numpy as np

def match_string(a, b):
    a = ' ' + a
    b = ' ' + b
    array = np.zeros((len(a), len(b)), dtype=np.int)
    steps = np.zeros((len(a), len(b)), dtype=np.int)
    array[0] = np.arange(len(b))
    array[:, 0] = np.arange(len(a))
    steps[0] = 1

    for i in range(1, len(a)):
        for j in range(1, len(b)):
            sub_cost = 0 if a[i] == b[j] else 2
            minval = array[i-1, j-1]+sub_cost # substitution
            steps[i, j] = 2

            del_cost = 1 if a[i] != ' ' else 0
            ins_cost = 1 if b[j] != ' ' else 0
            # delete, insertion
            for step, val in enumerate((array[i-1, j]+del_cost, array[i, j-1]+ins_cost)):
                if minval > val:
                    steps[i, j] = step
                    minval = val
            array[i, j] = minval

    # print(array)
    # print(steps)
    return array, steps

def backward(steps, a, b):
    assert len(steps) == len(a)+1
    assert len(steps[0]) == len(b)+1

    x = len(a)
    y = len(b)

    ss = []
    while x > 0 or y > 0:
        step = steps[x, y]
        # print(x, y)
        if step == 0:
            x, y = x-1, y
            ss.append(['d', a[x], ' '])
        elif step == 1:
            x, y = x, y-1
            ss.append(['i', ' ', b[y]])
        elif step == 2:
            x, y = x-1, y-1
            if a[x] == b[y]:
                ss.append(['=', a[x], b[y]])
            else:
                ss.append(['r', a[x], b[y]])
    # for i in range(len(ss)):
        # print()
    for line in map(list, zip(*ss)):
        print(''.join(reversed(line)))

def main():
    a = "sitting"
    b = "kitten"
    # a, b = "Monday", "Hello world"
    array, steps = match_string(a, b)
    backward(steps, a, b)

def match_distance(a, b):
    array, steps = match_string(a, b)
    if False:
        backward(steps, a, b)
    return int(array[-1, -1])

if __name__ == '__main__':
    print(match_distance('sitting', 'kitten'))

max_stars_repo_name: chenxuchun/atx-server

max_issues_repo_name: chenxuchun/atx-server

max_forks_repo_name: chenxuchun/atx-server


text:
# This file is part of the Reproducible Open Benchmarks for Data Analysis
# Platform (ROB) - Top Tagger Benchmark Demo.
#
# Copyright (C) [2019-2020] NYU.
#
# ROB is free software; you can redistribute it and/or modify it under the
# terms of the MIT License; see LICENSE file for more details.

import numpy as np
import copy
import pickle

# Data loading related

def load_from_pickle(filename, n_jets):
    jets = []
    fd = open(filename, "rb")

    for i in range(n_jets):
        jet = pickle.load(fd)
        jets.append(jet)

    fd.close()

    return jets

# Jet related

# Get the 4-vector pT
def _pt(v):
#     pz = v[2]
#     p = (v[0:3] ** 2).sum() ** 0.5
#     eta = 0.5 * (np.log(p + pz) - np.log(p - pz))
#     pt2 = p / np.cosh(eta)
    pt=(v[0:2] ** 2).sum() ** 0.5
#     print('pt2=',pt2)
#     print('pt=',pt)

    return pt

def permute_by_pt(jet, root_id=None):
    # ensure that the left sub-jet has always a larger pt than the right

    if root_id is None:
        root_id = jet["root_id"]

    if jet["tree"][root_id][0] != -1: # because if it is -1, then there are no children and so it is a leaf (SM)
        left = jet["tree"][root_id][0] #root_id left child. This gives the id of the left child (SM)
        right = jet["tree"][root_id][1] #root_id right child. This gives the id of the right child (SM)
        #So object["tree"][root_id] contains the position of the left and right children of object in jet["content"] (SM)

        pt_left = _pt(jet["content"][left]) #We get the pt of the left child (SM)
        pt_right = _pt(jet["content"][right]) #We get the pt of the right child (SM)
#         print('pt_left=',pt_left)
#         print('pt_right=',pt_right)

        if pt_left < pt_right: # We switch the left and right children if necessary so that the left child has greater pt (SM)
            jet["tree"][root_id][0] = right
            jet["tree"][root_id][1] = left
            print('pt_left=',pt_left)
            print('pt_right=',pt_right)

        # We call the function recursively to ensure each subtree satisfies this property (SM)
        permute_by_pt(jet, left)
        permute_by_pt(jet, right)

    return jet

#------------------------------------------------------------------
# NEW DIC 16 2018
# Count the number of leaves
# Recursive function to go deep up to the leaves and then go back to the root adding the children values to get the parent ones
def rewrite_content_leaves(jet):
    jet = copy.deepcopy(jet)
#     print('/////'*20)
#     print('jet=',jet)

#     if jet["content"].shape[1] == 5: #jet["content"].shape[0] is the sequence of subjets in the clustering algorithm. jet["content"].shape[1] is (px,py,pz,E,..) (SM)
#         pflow = jet["content"][:, 4].copy() #All rows and 5th column ? (SM)

    content = jet["content"]
    tree = jet["tree"]
    n_constituents=np.ones(len(content))
#     print('n_constituents =',n_constituents)
#     print('----'*20)

#     tot_leaves=0
#     for i in range(len(tree)):
#       if  tree[i, 0] == -1:
#         tot_leaves+=1

    def _rec(i):
        if tree[i, 0] == -1:
            pass
        else:
            _rec(tree[i, 0])
            _rec(tree[i, 1])
#             c = content[tree[i, 0]] + content[tree[i, 1]] #We add the 4-vectors of the two children of object i (SM)
            n_leaves=n_constituents[tree[i, 0]] + n_constituents[tree[i, 1]] #We add the 4-vectors of the two children of object i (SM)
            # print('/////'*20)
#             print('n_constituents[i] before=',n_constituents[i])
#             content[i] = c # We replace the 4-vector of object i by the sum of the 4-vector of its children
            n_constituents[i] =n_leaves
#             print('n_constituents[i] after=',n_constituents[i])

    _rec(jet["root_id"])

    jet["n_constituents"]=n_constituents
#     print('jet["n_constituents"] =',jet["n_constituents"])
#     print('tot_leaves=',tot_leaves)
#     print('/////'*20)

#     print('/////'*20)
#     print('/////'*20)
#     print('jet=',jet)
    return jet

#-----------
def permute_by_n_leaves(jet, root_id=None):
    # ensure that the left sub-jet has always a larger pt than the right

    if root_id is None:
        root_id = jet["root_id"]

    if jet["tree"][root_id][0] != -1: # because if it is -1, then there are no children and so it is a leaf (SM)
        left = jet["tree"][root_id][0] #root_id left child. This gives the id of the left child (SM)
        right = jet["tree"][root_id][1] #root_id right child. This gives the id of the right child (SM)
        #So object["tree"][root_id] contains the position of the left and right children of object in jet["content"] (SM)

        nleaves_left = jet["n_constituents"][left] #We get the pt of the left child (SM)
        nleaves_right = jet["n_constituents"][right] #We get the pt of the right child (SM)
#         print('pt_left=',pt_left)
#         print('pt_right=',pt_right)

        if nleaves_left < nleaves_right: # We switch the left and right children if necessary so that the left child has greater pt (SM)
#             print('nleaves_left before =',nleaves_left)
#             print('nleaves_right before =',nleaves_right)
#             print('----'*20)

            jet["tree"][root_id][0] = right
            jet["tree"][root_id][1] = left
#             print('nleaves_left after =',nleaves_left)
#             print('nleaves_right after =',nleaves_right)
#             print('----'*20)

        # We call the function recursively to ensure each subtree satisfies this property (SM)
        permute_by_n_leaves(jet, left)
        permute_by_n_leaves(jet, right)

    return jet

#--------------------------------------------------------------------

def rewrite_content(jet):
    jet = copy.deepcopy(jet)
#     print('/////'*20)
#     print('jet=',jet)

#     if jet["content"].shape[1] == 5: #jet["content"].shape[0] is the sequence of subjets in the clustering algorithm. jet["content"].shape[1] is (px,py,pz,E,..) (SM)
#         pflow = jet["content"][:, 4].copy() #All rows and 5th column ? (SM)

    content = jet["content"]
    tree = jet["tree"]

    def _rec(i):
        if tree[i, 0] == -1:
            pass
        else:
            _rec(tree[i, 0])
            _rec(tree[i, 1])
            c = content[tree[i, 0]] + content[tree[i, 1]] #We add the 4-vectors of the two children of object i (SM)
#             print('/////'*20)
#             print('content[i] before=',content[i])
            content[i] = c # We replace the 4-vector of object i by the sum of the 4-vector of its children
#             print('content[i] after=',content[i])
    _rec(jet["root_id"])

#     if jet["content"].shape[1] == 5:
#         jet["content"][:, 4] = pflow
#     print('/////'*20)
#     print('/////'*20)
#     print('jet=',jet)
    return jet

# We redefine the jet content in terms of eta,phi,p, etc (SM)
def extract_nyu_samples(jet, pflow=False):
    # per node feature extraction

#     jet = copy.deepcopy(jet)

    s = jet["content"].shape

    if not pflow:
        content = np.zeros((s[0], 7))
    else:
        # pflow value will be one-hot encoded
        content = np.zeros((s[0], 7+4))

    for i in range(len(jet["content"])):
        px = jet["content"][i, 0]
        py = jet["content"][i, 1]
        pz = jet["content"][i, 2]

        p = (jet["content"][i, 0:3] ** 2).sum() ** 0.5
        eta = 0.5 * (np.log(p + pz) - np.log(p - pz)) #pseudorapidity.
        theta = 2 * np.arctan(np.exp(-eta)) #angle with respect to the beam axis. np.arctan is the trigonometric inverse tangent, element-wise. Its real part is in [-pi/2, pi/2] (arctan(+/-inf) returns +/-pi/2).
#         pt = p / np.cosh(eta)
        phi = np.arctan2(py, px) # np.arctan2 it the element-wise arc tangent of x1/x2 choosing the quadrant correctly (starting from the x axis).
        pt = (jet["content"][i, 0:2] ** 2).sum() ** 0.5

        content[i, 0] = p # Absolute value of the momentum
        content[i, 1] = eta if np.isfinite(eta) else 0.0  #pseudorapidity
        content[i, 2] = phi # Azimuthal angle
        content[i, 3] = jet["content"][i, 3] #Energy
        content[i, 4] = (jet["content"][i, 3] /
                         jet["content"][jet["root_id"], 3]) #Energy/ jet energy
        content[i, 5] = pt if np.isfinite(pt) else 0.0 #Transverse momentum
        content[i, 6] = theta if np.isfinite(theta) else 0.0  #Angle with respect to the beam axis

        if pflow:
            if jet["content"][i, 4] >= 0:
                content[i, 7+int(jet["content"][i, 4])] = 1.0

    jet["content"] = content
#     print('jet["content"][0:2] =',jet["content"][0:2] )

    return jet

# We redefine the jet content in terms of eta,phi,p, etc (SM)
def extract(jet, features, pflow=False,kappa=None):
    # per node feature extraction

#     jet = copy.deepcopy(jet)

    s = jet["content"].shape

    content = np.zeros((s[0], int(features)))

#     if not pflow:
#         content = np.zeros((s[0], 7))
#     else:
#         # pflow value will be one-hot encoded
#         content = np.zeros((s[0], 7+4))

#     jetp = jetpT * np.cosh(eta)

#     jetpx = jet["content"][jet["root_id"], 0]
#     jetpy = jet["content"][jet["root_id"], 1]
    jetpz = jet["content"][jet["root_id"], 2]
    jetpT=(jet["content"][jet["root_id"], 0:2] ** 2).sum() ** 0.5
    jetp = (jet["content"][jet["root_id"], 0:3] ** 2).sum() ** 0.5
#     jetEta=0.5 * (np.log(jetp + jetpz) - np.log(jetp - jetpz))
#     jetPhi=np.arctan2(jetpy, jetpx)

#     jetPhi=np.arctan2(jetpy, jetpx)

#     print('/////'*20)
#     print('Jet 4vec=',jet["content"][jet["root_id"]])
#     print('Jet pT=',jet["pt"])
#     print('Jet pT check=',jetpT)
#     print('----'*20)

    for i in range(len(jet["content"])):
        px = jet["content"][i, 0]
        py = jet["content"][i, 1]
        pz = jet["content"][i, 2]

        p = (jet["content"][i, 0:3] ** 2).sum() ** 0.5
#         print('p before=',p)
        eta = 0.5 * (np.log(p + pz) - np.log(p - pz)) #pseudorapidity.
        theta = 2 * np.arctan(np.exp(-eta)) #angle with respect to the beam axis. np.arctan is the trigonometric inverse tangent, element-wise. Its real part is in [-pi/2, pi/2] (arctan(+/-inf) returns +/-pi/2).
#         pt = p / np.cosh(eta)
        phi = np.arctan2(py, px) # np.arctan2 it the element-wise arc tangent of x1/x2 choosing the quadrant correctly (starting from the x axis).
        pt = (jet["content"][i, 0:2] ** 2).sum() ** 0.5

#         p=pt*np.cosh(eta)
#         print('p after=',p)
        charge= jet["charge"][i]
        abs_charge= jet["abs_charge"][i]
        muon = jet["muon"][i]

        abs_pT_weighted_q=(1/jetpT**kappa) * abs_charge * pt**kappa

#         content[i, 0] = p/jetpT # Absolute value of the momentum
#         content[i, 1] = eta if np.isfinite(eta) else 0.0  #pseudorapidity
#         content[i, 2] = phi # Azimuthal angle
# #         content[i, 3] = jet["content"][i, 3] #Energy
#         content[i, 3] = (jet["content"][i, 3] /jetpT) #Energy/ jet pT
#         content[i, 4] = pt/jetpT if np.isfinite(pt) else 0.0 #Transverse momentum
#         content[i, 5] = theta if np.isfinite(theta) else 0.0  #Angle with respect to the beam axis

#         content[i, 0] = p # Absolute value of the momentum
#         content[i, 1] = p/jetpT # Absolute value of the momentum
#         content[i, 2] = pt if np.isfinite(pt) else 0.0 #Transverse momentum
#         content[i, 3] = pt/jetpT if np.isfinite(pt) else 0.0 #Transverse momentum
#         content[i, 4] = jet["content"][i, 3] #Energy/ jet pT
#         content[i, 5] = (jet["content"][i, 3] /jetpT) #Energy/ jet pT
#         content[i, 6] = phi # Azimuthal angle
#         content[i, 7] = eta if np.isfinite(eta) else 0.0  #pseudorapidity
#         content[i, 8] = theta if np.isfinite(theta) else 0.0  #Angle with respect to the beam axis

#         content[i, 9] = ((phi-jetPhi)**2+(eta-jetEta)**2)  #C/A distance with respect to the jet axis. We do not divide by R**2 given that it is just a constant
#         content[i, 10] = np.minimum(pt**2,jetpT**2)*((phi-jetPhi)**2+(eta-jetEta)**2)  #kt distance with respect to the jet axis. We do not divide by R**2 given that it is just a constant
#         content[i, 11] = np.minimum(1/(pt**2),1/(jetpT**2))*((phi-jetPhi)**2+(eta-jetEta)**2)  #anti-kt distance with respect to the jet axis. We do not divide by R**2 given that it is just a constant
     #----------------------------------------
#      Last used (nyu)
        content[i, 0] = p # Absolute value of the momentum
        content[i, 1] = eta if np.isfinite(eta) else 0.0  #pseudorapidity
        content[i, 2] = phi # Azimuthal angle
        content[i, 3] = jet["content"][i, 3] #Energy
        content[i, 4] = (jet["content"][i, 3] /
                         jet["content"][jet["root_id"], 3]) #Energy/ jet energy
        content[i, 5] = pt if np.isfinite(pt) else 0.0 #Transverse momentum
        content[i, 6] = theta if np.isfinite(theta) else 0.0  #Angle with respect to the beam axis
#         content[i, 7] = charge
#         content[i, 7] = abs_charge
#         content[i, 9] = muon
#         content[i, 7] = abs_pT_weighted_q
#
# #      #-----------------------------------------
#         content[i, 0] = p # Absolute value of the momentum
#         content[i, 1] = p/jetp
#         content[i, 2] = eta if np.isfinite(eta) else 0.0  #pseudorapidity
#         content[i, 3] = phi # Azimuthal angle
# #         content[i, 3] = jet["content"][i, 3] #Energy
# #         content[i, 4] = (jet["content"][i, 3] /
# #                          jet["content"][jet["root_id"], 3]) #Energy/ jet energy
#         content[i, 4] = pt if np.isfinite(pt) else 0.0 #Transverse momentum
#         content[i, 5] = pt/jetpT if np.isfinite(pt) else 0.0 #Transverse momentum
#         content[i, 6] = theta if np.isfinite(theta) else 0.0  #Angle with respect to the beam axis
# #         content[i, 7] = charge
# #         content[i, 8] = abs_charge
# #         content[i, 9] = muon

#         if pflow:
#             if jet["content"][i, 4] >= 0:
#                 content[i, 7+int(jet["content"][i, 4])] = 1.0

    jet["content"] = content
#     print('jet["content"][0:2] =',jet["content"][0:2] )

    return jet

def randomize(jet):
    # build a random tree

    jet = copy.deepcopy(jet)

    #We get the leaves
    leaves = np.where(jet["tree"][:, 0] == -1)[0] # This gives the positions of jet["tree"] that correspond to a leaf (SM)
    nodes = [n for n in leaves]
    content = [jet["content"][n] for n in nodes]
    nodes = [i for i in range(len(nodes))]
    tree = [[-1, -1] for n in nodes]
    pool = [n for n in nodes]
    next_id = len(nodes)

    while len(pool) >= 2:
        #We randomly pick 2 elements of pool and then delete them (SM)
        i = np.random.randint(len(pool))
        left = pool[i]
        del pool[i]
        j = np.random.randint(len(pool))
        right = pool[j]
        del pool[j]

        nodes.append(next_id)
        c = (content[left] + content[right]) #We get the next object by adding the 4-vector of the children

        if len(c) == 5:
            c[-1] = -1

        content.append(c) #We append the new object to the jet contents (SM)
        tree.append([left, right]) # We append the labels of the left and right children of the new object to the tree (SM)
        pool.append(next_id) # We add the location of the new object (in contents) to the pool of particles we pick from to do the clustering (SM)
        next_id += 1

    # We create the jet (SM)
    jet["content"] = np.array(content)
    jet["tree"] = np.array(tree).astype(int)
    jet["root_id"] = len(jet["tree"]) - 1 # The last element in this case will be the root of the jet tree (SM)

    return jet

# We do the same as in the previous function but sorting by pT in this case (SM)
def sequentialize_by_pt(jet, reverse=False):
    # transform the tree into a sequence ordered by pt

    jet = copy.deepcopy(jet)

#     print('jet["tree"]=',jet["tree"])
#     print('---'*20)

    leaves = np.where(jet["tree"][:, 0] == -1)[0] #The entries give the position of the leaves in the content list
#     print('leaves=',leaves)
#     print('---'*20)

    nodes = [n for n in leaves]# Same as leaves
#     print('nodes=',nodes)
#     print('---'*20)

    content = [jet["content"][n] for n in nodes] #Here we get the leaves content
#     print('content=',content)
#     print('---'*20)

    nodes = [i for i in range(len(nodes))]
    tree = [[-1, -1] for n in nodes] #We make a tree of length=Number of leaves, where the entries are all (-1), as expected for leaves
#     print('tree=',tree)
#     print('---'*20)

    #Order the list of leaves based on pT. reverse=False gives the list in ascending pT
    pool = sorted([n for n in nodes],
                  key=lambda n: _pt(content[n]),
                  reverse=reverse)
#     print('pool=',pool)
#     print('Ordered pT content=',[ _pt(content[n]) for n in pool])
#     print('---'*20)

    next_id = len(pool)

    #Remake the tree. We start from the end of the ordered list of leaves. So in reverse=False we cluster first the greatest pT constituents. => reverse=False gives a tree ordered in descending order in pT
    while len(pool) >= 2:
        right = pool[-1]
        left = pool[-2]
#         print('right=',_pt(content[right]))
#         print('left=',_pt(content[left]))
#         print('---'*20)
        del pool[-1]
        del pool[-1]

        # We build the tree as a ladder. So we append the right subjet to the list
        nodes.append(next_id)
#         print('next_id=',next_id)
#         print('---'*20)
        c = (content[left] + content[right])

        if len(c) == 5:
            c[-1] = -1

        content.append(c) # We append the new subjet to the contents list
#         print('pT c=',_pt(c))
#         print('pT content[next_id]=c?',_pt(content[next_id]))
        tree.append([left, right]) # We append the children locations of the new subjet (locations in the contents list)
#         print('tree=',tree)
#         print('tree[next_id]=',tree[next_id])
#         print('---'*20)
#         print('---'*20)
        pool.append(next_id) #We append the location of the new subjet 'c', so content[next_id]=c
        next_id += 1

    jet["content"] = np.array(content)
    jet["tree"] = np.array(tree).astype(int)
    jet["root_id"] = len(jet["tree"]) - 1

    return jet

#----------------------------------------------------------------------------------------
def rewrite_feature(feature,tree):
    feature = copy.deepcopy(feature)
    tree = copy.deepcopy(tree)
#     print('/////'*20)
#     print('jet=',jet)

#     if jet["content"].shape[1] == 5: #jet["content"].shape[0] is the sequence of subjets in the clustering algorithm. jet["content"].shape[1] is (px,py,pz,E,..) (SM)
#         pflow = jet["content"][:, 4].copy() #All rows and 5th column ? (SM)

#     content = jet["content"]
#     tree = jet["tree"]

    def _rec(i):
#         print('tree[i, 0]=',tree[2*i])
        if tree[i, 0] == -1:
            pass
        else:
            _rec(tree[i, 0]) #We call the function recursively before adding the children values => We will start from the leaves and go up to the root. When we hit a leaf, the 'pass' statement will just pass and so we will get 'c'
            _rec(tree[i, 1])
            c = feature[tree[i, 0]] + feature[tree[i, 1]] #We add the 4-vectors of the two children of object i (SM)
            print('/////'*20)
            print('feature[i] before=',feature[i])
            feature[i] = c # We replace the 4-vector of object i by the sum of the 4-vector of its children
            print('feature[i] after=',feature[i])

    _rec(0) # We start form the root id

#     if jet["content"].shape[1] == 5:
#         jet["content"][:, 4] = pflow
#     print('/////'*20)
#     print('/////'*20)
#     print('jet=',jet)
    return feature

max_stars_repo_name: scailfin/rob-demo-top-tagger

max_issues_repo_name: scailfin/rob-demo-top-tagger

max_forks_repo_name: scailfin/rob-demo-top-tagger