text:
import numpy as np
import tensorflow as tf

import DeepSparseCoding.tf1x.utils.plot_functions as pf
import DeepSparseCoding.tf1x.utils.data_processing as dp
import DeepSparseCoding.tf1x.utils.entropy_functions as ef
import DeepSparseCoding.tf1x.ops.init_ops as init_ops
from DeepSparseCoding.tf1x.models.base_model import Model
from DeepSparseCoding.tf1x.modules.lca_module import LcaModule
from DeepSparseCoding.tf1x.modules.mlp_module import MlpModule

class ListaModel(Model):
  def __init__(self):
    super(ListaModel, self).__init__()

  def load_params(self, params):
    """
    Load parameters into object
    Inputs:
     params: [obj] model parameters
    """
    super(ListaModel, self).load_params(params)
    # Network Size
    self.input_shape = [None, self.params.num_pixels]
    self.w_shape = [self.params.num_pixels, self.params.num_neurons]
    self.s_shape = [self.params.num_neurons, self.params.num_neurons]
    # Hyper Parameters
    self.eta = self.params.dt / self.params.tau

  def build_lca_module(self, input_node):
    module = LcaModule(input_node, self.params.num_neurons, self.sparse_mult,
      self.eta, self.params.thresh_type, self.params.rectify_a,
      self.params.num_steps, self.params.eps)
    return module

  def build_graph_from_input(self, input_node):
    """Build the TensorFlow graph object"""
    with tf.device(self.params.device):
      with self.graph.as_default():
        with tf.compat.v1.variable_scope("auto_placeholders") as scope:
          self.sparse_mult = tf.compat.v1.placeholder(tf.float32, shape=(), name="sparse_mult")
          self.train_lca = tf.compat.v1.placeholder(tf.bool, shape=(), name="train_lca")

        self.train_lca = tf.cast(self.train_lca, tf.float32)

        self.lca_module = self.build_lca_module(input_node)

        with tf.compat.v1.variable_scope("weight_inits") as scope:
          self.w_init = tf.compat.v1.truncated_normal_initializer(stddev=0.01)
          self.s_init = init_ops.GDNGammaInitializer(diagonal_gain=0.0, off_diagonal_gain=0.001)

        with tf.compat.v1.variable_scope("weights") as scope:
          self.w = tf.compat.v1.get_variable(name="w_enc", shape=self.w_shape, dtype=tf.float32,
            initializer=self.w_init, trainable=True)
          self.s = tf.compat.v1.get_variable(name="lateral_connectivity", shape=self.s_shape,
            dtype=tf.float32, initializer=self.s_init, trainable=True)
        self.trainable_variables.update({self.w.name:self.w, self.s.name:self.s})

        with tf.compat.v1.variable_scope("inference") as scope:
          feedforward_drive = tf.matmul(input_node, self.w, name="feedforward_drive")
          self.a_list = [self.lca_module.threshold_units(feedforward_drive, name="a_init")]
          for layer_id in range(self.params.num_layers):
            self.a_list.append(self.lca_module.threshold_units(feedforward_drive
              + tf.matmul(self.a_list[layer_id], self.s)))
          self.a = self.a_list[-1]

        with tf.compat.v1.variable_scope("loss") as scope:
          reduc_dim = list(range(1, len(self.lca_module.a.shape)))
          labels = tf.stop_gradient(self.lca_module.a)
          self.lista_loss = tf.reduce_mean(input_tensor=tf.reduce_sum(input_tensor=tf.square(labels - self.a),
            axis=reduc_dim))
          #Loss switches based on train_lca flag
          self.total_loss = self.train_lca * self.lca_module.total_loss + \
            (1-self.train_lca) * self.lista_loss

        with tf.compat.v1.variable_scope("norm_weights") as scope:
          self.norm_lista_w = self.w.assign(tf.nn.l2_normalize(self.w, axis=0, epsilon=self.params.eps,
            name="row_l2_norm"))
          self.norm_weights = tf.group(self.norm_lista_w, name="l2_normalization")

        with tf.compat.v1.variable_scope("performance_metrics") as scope:
          #LCA metrics
          MSE = tf.reduce_mean(input_tensor=tf.square(tf.subtract(input_node, self.lca_module.reconstruction)),
            axis=[1, 0], name="mean_squared_error")
          pixel_var = tf.nn.moments(x=input_node, axes=[1])[1]
          self.pSNRdB = tf.multiply(10.0, ef.safe_log(tf.math.divide(tf.square(pixel_var),
            MSE)), name="recon_quality")

  def get_input_shape(self):
    return self.input_shape

  def get_num_latent(self):
    return self.params.num_neurons

  def get_encodings(self):
    return self.a

  def get_total_loss(self):
    return self.total_loss

  def compute_recon_from_encoding(self, a_in):
    return self.lca_module.build_decoder(a_in, name="reconstruction")

  def generate_update_dict(self, input_data, input_labels=None, batch_step=0):
    """
    Log train progress information
    Inputs:
      input_data: data object containing the current image batch
      input_labels: data object containing the current label batch
      batch_step: current batch number within the schedule
    """
    update_dict = super(ListaModel, self).generate_update_dict(input_data, input_labels, batch_step)
    feed_dict = self.get_feed_dict(input_data, input_labels)
    eval_list = [self.global_step, self.lca_module.loss_dict["recon_loss"],
      self.lca_module.loss_dict["sparse_loss"], self.get_total_loss(),
      self.lca_module.a, self.get_encodings(), self.lca_module.reconstruction, self.pSNRdB,
      self.lista_loss]
    grad_name_list = []
    learning_rate_dict = {}
    for w_idx, weight_grad_var in enumerate(self.grads_and_vars[self.sched_idx]):
      eval_list.append(weight_grad_var[0][0]) # [grad(0) or var(1)][value(0) or name(1)]
      grad_name = weight_grad_var[0][1].name.split('/')[1].split(':')[0] # 2nd is np.split
      grad_name_list.append(grad_name)
      learning_rate_dict[grad_name] = self.get_schedule("weight_lr")[w_idx]
    out_vals =  tf.compat.v1.get_default_session().run(eval_list, feed_dict)
    current_step, recon_loss, sparse_loss, total_loss, lca_a_vals, lista_a_vals, recon, pSNRdB, lista_loss \
      = out_vals[0:9]
    input_max = np.max(input_data)
    input_mean = np.mean(input_data)
    input_min = np.min(input_data)
    recon_max = np.max(recon)
    recon_mean = np.mean(recon)
    recon_min = np.min(recon)
    lca_a_vals_max = np.array(lca_a_vals.max())
    lca_a_vals_mean = np.array(lca_a_vals.mean())
    lca_a_vals_min = np.array(lca_a_vals.min())
    lca_a_frac_act = np.array(np.count_nonzero(lca_a_vals)
      / float(lca_a_vals.size))
    lista_a_vals_max = np.array(lista_a_vals.max())
    lista_a_vals_mean = np.array(lista_a_vals.mean())
    lista_a_vals_min = np.array(lista_a_vals.min())
    lista_a_frac_act = np.array(np.count_nonzero(lista_a_vals)
      / float(lista_a_vals.size))

    stat_dict = {"global_batch_index":current_step,
      "batch_step":batch_step,
      "schedule_index":self.sched_idx,
      "lca_recon_loss":recon_loss,
      "lca_sparse_loss":sparse_loss,
      "lista_loss":lista_loss,
      "total_loss":total_loss,
      "pSNRdB": np.mean(pSNRdB),
      "lca_a_fraction_active":lca_a_frac_act,
      "lca_a_max_mean_min":[lca_a_vals_max, lca_a_vals_mean, lca_a_vals_min],
      "lista_a_fraction_active":lista_a_frac_act,
      "lista_a_max_mean_min":[lista_a_vals_max, lista_a_vals_mean, lista_a_vals_min],
      "x_max_mean_min":[input_max, input_mean, input_min],
      "x_hat_max_mean_min":[recon_max, recon_mean, recon_min]}
    grads = out_vals[9:]
    for grad, name in zip(grads, grad_name_list):
      grad_max = learning_rate_dict[name]*np.array(grad.max())
      grad_min = learning_rate_dict[name]*np.array(grad.min())
      grad_mean = learning_rate_dict[name]*np.mean(np.array(grad))
      stat_dict[name+"_grad_max_mean_min"] = [grad_max, grad_mean, grad_min]
      stat_dict[name+"_learning_rate"] = learning_rate_dict[name]
    update_dict.update(stat_dict) #stat_dict overwrites for same keys
    return update_dict

  def generate_plots(self, input_data, input_labels=None):
    """
    Plot weights, reconstruction, and gradients
    Inputs:
      input_data: data object containing the current image batch
      input_labels: data object containing the current label batch
    """
    super(ListaModel, self).generate_plots(input_data, input_labels)
    feed_dict = self.get_feed_dict(input_data, input_labels)
    eval_list = [self.global_step, self.lca_module.w, self.w,
      self.lca_module.reconstruction, self.lca_module.a, self.get_encodings()]
    eval_out = tf.compat.v1.get_default_session().run(eval_list, feed_dict)
    current_step = str(eval_out[0])
    filename_suffix = "_v"+self.params.version+"_"+current_step.zfill(5)+".png"
    lca_weights, lista_weights, recon, lca_activity, lista_activity = eval_out[1:]
    lca_weights_norm = np.linalg.norm(lca_weights, axis=0, keepdims=False)
    lista_weights_norm = np.linalg.norm(lista_weights, axis=0, keepdims=False)
    recon = dp.reshape_data(recon, flatten=False)[0]
    lca_weights = dp.reshape_data(lca_weights.T, flatten=False)[0] # [num_neurons, height, width]
    lista_weights = dp.reshape_data(lista_weights.T, flatten=False)[0] # [num_neurons, height, width]

    fig = pf.plot_activity_hist(input_data, title="Image Histogram",
      save_filename=self.params.disp_dir+"img_hist"+filename_suffix)

    #Scale image by max and min of images and/or recon
    r_max = np.max([np.max(input_data), np.max(recon)])
    r_min = np.min([np.min(input_data), np.min(recon)])

    input_data = dp.reshape_data(input_data, flatten=False)[0]
    fig = pf.plot_data_tiled(input_data, normalize=False,
      title="Scaled Images at step "+current_step, vmin=r_min, vmax=r_max,
      save_filename=self.params.disp_dir+"images"+filename_suffix)
    fig = pf.plot_data_tiled(recon, normalize=False,
      title="Recons at step "+current_step, vmin=r_min, vmax=r_max,
      save_filename=self.params.disp_dir+"recons"+filename_suffix)

    fig = pf.plot_activity_hist(lca_activity, title="LCA Activity Histogram",
      save_filename=self.params.disp_dir+"lca_act_hist"+filename_suffix)

    fig = pf.plot_activity_hist(lista_activity, title="LISTA Activity Histogram",
      save_filename=self.params.disp_dir+"lista_act_hist"+filename_suffix)

    fig = pf.plot_data_tiled(lca_weights, normalize=False,
      title="LCA Dictionary at step "+current_step, vmin=None, vmax=None,
      save_filename=self.params.disp_dir+"lca_w"+filename_suffix)

    fig = pf.plot_data_tiled(lista_weights, normalize=False,
      title="LISTA Dictionary at step "+current_step, vmin=None, vmax=None,
      save_filename=self.params.disp_dir+"lista_w"+filename_suffix)

    #for weight_grad_var in self.grads_and_vars[self.sched_idx]:
    #  grad = weight_grad_var[0][0].eval(feed_dict)
    #  shape = grad.shape
    #  name = weight_grad_var[0][1].name.split('/')[1].split(':')[0]#np.split
    #  grad = dp.reshape_data(grad.T, flatten=False)[0]
    #  fig = pf.plot_data_tiled(grad, normalize=True,
    #    title="Gradient for w at step "+current_step, vmin=None, vmax=None,
    #    save_filename=self.params.disp_dir+"dphi"+filename_suffix)

max_stars_repo_name: dpaiton/DeepSparseCoding

max_issues_repo_name: dpaiton/DeepSparseCoding

max_forks_repo_name: dpaiton/DeepSparseCoding


text:
# KERNEL SVM

# Importing libraries

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import StandardScaler # for feature scaling

from sklearn.svm import SVC

from sklearn.metrics import confusion_matrix, accuracy_score

from matplotlib.colors import ListedColormap

# Importing dataset
dataset = pd.read_csv('Social_Network_Ads.csv')

# pd.set_option('display.float_format', lambda x: '%.0f' % x)

X = dataset.iloc[:, [2, 3]].values
y = dataset.iloc[:, 4].values

# Splitting the dataset into training and test set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25, random_state = 0)

# Feature scaling
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)

# fitting the classifier to training set
# Create classifier here
# rbf is like gaussian kernel
classifier = SVC(kernel = 'rbf', random_state = 0)
classifier.fit(X_train, y_train)

# preditct test set results
y_pred = classifier.predict(X_test)

# making the confusion matrix
cm = confusion_matrix(y_test, y_pred)
# print cm

# just normal accuracy
acc = accuracy_score(y_test, y_pred)

# Visualize the training set results
X_set, y_set = X_train, y_train
X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01), np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
plt.contourf(X1, X2, classifier.predict(np.array([X1.ravel(), X2.ravel()]).T).reshape(X1.shape), alpha = 0.75, cmap = ListedColormap(('red', 'green')))
plt.xlim(X1.min(), X1.max())
plt.ylim(X2.min(), X2.max())
for i, j in enumerate(np.unique(y_set)):
	plt.scatter(X_set[y_set == j, 0], X_set[y_set == j, 1], c = ListedColormap(('red', 'green'))(i), label = j)

plt.title('SVC (Training Set)')
plt.xlabel('Age')
plt.ylabel('Estimated Salary')
plt.legend()
plt.show()

# Visualize the test set results
X_set, y_set = X_test, y_test
X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01), np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
plt.contourf(X1, X2, classifier.predict(np.array([X1.ravel(), X2.ravel()]).T).reshape(X1.shape), alpha = 0.75, cmap = ListedColormap(('red', 'green')))
plt.xlim(X1.min(), X1.max())
plt.ylim(X2.min(), X2.max())
for i, j in enumerate(np.unique(y_set)):
	plt.scatter(X_set[y_set == j, 0], X_set[y_set == j, 1], c = ListedColormap(('red', 'green'))(i), label = j)

plt.title('SVC (Test Set)')
plt.xlabel('Age')
plt.ylabel('Estimated Salary')
plt.legend()
plt.show()

ext: py

max_stars_repo_name: jsjaskaran/mllearn

max_issues_repo_name: jsjaskaran/mllearn

max_forks_repo_name: jsjaskaran/mllearn

reason: import numpy


text:
# This file is part of MAGNETAR, the set of magnetic field analysis tools
#
# Copyright (C) 2013-2017 Juan Diego Soler

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.pyplot import cm
from astropy.io import fits
from astropy.convolution import convolve, convolve_fft
from astropy.convolution import Gaussian2DKernel

# ================================================================================================================================
def strfunclist(xpos, ypos, psi, nsteps=10, EqualNbins=False):

    #Fix to include uncertainties
    s_psi=5.*np.pi/180.

    npos=np.size(xpos)

    distances=np.zeros([npos,npos])
    deltapsi=np.zeros([npos,npos])
    deltapsi2=np.zeros([npos,npos])
    cospsi=np.zeros([npos,npos])
    sinpsi=np.zeros([npos,npos])

    qint=np.sin(psi)**2 - np.cos(psi)**2
    uint=-2.*np.cos(psi)*np.sin(psi)

    for i in range(0, npos):

        deltax=xpos[i]-xpos
        deltay=ypos[i]-ypos
        dist=np.sqrt(deltax**2 + deltay**2)
        distances[:,i]=dist
        distances[i:npos,i]=0.

        deltapsi[i,:]=0.5*np.arctan2(qint[i]*uint-qint*uint[i], qint[i]*qint+uint[i]*uint)
        deltapsi2[i,:]=(deltapsi[i,:])**2
        cospsi[i,:]=np.cos(deltapsi[i,:])
        sinpsi[i,:]=np.sin(deltapsi[i,:])

    plt.imshow(distances, origin='lower', interpolation='none')
    plt.show()

    plt.imshow((180./np.pi)*np.sqrt(deltapsi2), origin='lower', interpolation='none')
    plt.colorbar()
    plt.show()
    #import pdb; pdb.set_trace()

    if (EqualNbins):
        # Invert the distribution of distances
        hist, bin_edges = np.histogram(distances[(distances > 0.).nonzero()], bins=10000)
        bin_centre=0.5*(bin_edges[0:np.size(bin_edges)-1]+bin_edges[1:np.size(bin_edges)])
        chist=np.cumsum(hist)
        pitch=np.max(chist)/float(nsteps)

        hsteps=pitch*np.arange(0,nsteps+1,1)
        dsteps=np.zeros(nsteps+1)

        for i in range(0, np.size(dsteps)-1):
            good=np.logical_and(chist>hsteps[i],chist<hsteps[i+1]).nonzero()
            dsteps[i]=np.min(bin_centre[good])
        dsteps[np.size(dsteps)-1]=np.max(distances)

    else:
        dpitch=(np.max(distances[(distances > 0.).nonzero()])-np.min(distances[(distances > 0.).nonzero()]))/float(nsteps)
        dsteps=np.arange(np.min(distances[(distances > 0.).nonzero()]), nsteps, dpitch)

    print(dsteps)
    #import pdb; pdb.set_trace()

    # -------------------------------------------------------------------------------------------------------------
    npairs=np.zeros(nsteps)
    lag=np.zeros(nsteps)

    structFunc=np.zeros(nsteps)
    s_structFunc=np.zeros(nsteps)

    cosFunc=np.zeros(nsteps)
    s_cosFunc=np.zeros(nsteps)

    for i in range(0, nsteps):
        good=np.logical_and(distances>dsteps[i],distances<dsteps[i+1]).nonzero()
        print(np.size(good), ' between ', dsteps[i], 'and', dsteps[i+1])
        lag[i]=0.5*(dsteps[i+1]+dsteps[i])
        npairs[i]=float(np.size(good))

        if(npairs[i] > 0.):
            tempdeltapsi2=deltapsi2[good]
            #tempdeltapsi2=np.arctan(np.tan(np.sqrt(deltapsi2[good])))**2
            #print(np.max((180/np.pi)*tempdeltapsi2))
            structFunc[i]=np.sqrt(np.mean(tempdeltapsi2))

            A1=(np.sum(deltapsi)**2)*(s_psi**2)
            A2=np.sum(deltapsi2*s_psi**2)
            s_structFunc[i]=np.sqrt(A1+A2)/(npairs[i]*structFunc[i])

            cosFunc[i]=np.sum(cospsi[good])/np.size(good)

    return lag, structFunc, cosFunc

# =================================================================================================================================
def strfunc(Imap, Qmap, Umap, pxsz=1., nsteps=10, beamfwhm=1.):
    # Calculates the relative orientation angle between the density structures and the magnetic field.
    # INPUTS
    # Imap - Intensity or column density map
    # Qmap - Stokes Q map
    # Umap - Stokes U map
    #

# Convert polarization maps into lists
    sz=np.shape(Imap)
    psi=0.5*np.arctan2(Umap,Qmap)
    x=pxsz*np.arange(sz[0])/float(sz[0])
    y=pxsz*np.arange(sz[1])/float(sz[1])

# Computing coordinate grid
    xv, yv = np.meshgrid(x, y) #, sparse=False, indexing='ij')

# Compute structure function using the lists
    lag, structFunc, cosFunc = strfunclist(xv.ravel(), yv.ravel(), psi.ravel(), nsteps=nsteps, EqualNbins=True)

    plt.figure()
    plt.plot(lag, (180/np.pi)*structFunc, 'ro')
    plt.show()

    good4fit=(lag > 0.3).nonzero()
    import pdb; pdb.set_trace()
    #z = np.polyfit(lag[good4fit]**2, 1.-cosFunc[good4fit], 2)

    plt.figure()
    plt.plot(lag, 1.-cosFunc, 'bo')
    plt.plot(lag, z, 'k')
    plt.show()

    import pdb; pdb.set_trace()

from astropy.convolution import convolve_fft

#npix=60
#Imap=np.random.rand(npix,npix)
#Qmap=np.random.rand(npix,npix)-0.5
#Umap=np.random.rand(npix,npix)-0.5

Imap=fits.open('data/Taurusfwhm5_logNHmap.fits')[0].data
Qmap=fits.open('data/Taurusfwhm10_Qmap.fits')[0].data
Umap=fits.open('data/Taurusfwhm10_Umap.fits')[0].data

#import pdb; pdb.set_trace()
pxksz=5
strfunc(Imap, Qmap, Umap, nsteps=20, pxsz=1.)
#strfunc(Imap, convolve_fft(Qmap, Gaussian2DKernel(pxksz), boundary='wrap'), convolve_fft(Umap, Gaussian2DKernel(pxksz), boundary='wrap'), nsteps=npix, pxsz=1.)
#strfunc(Imap, convolve_fft(Qmap, Gaussian2DKernel(pxksz), boundary='fill'), convolve_fft(Umap, Gaussian2DKernel(pxksz), boundary='fill'), nsteps=npix, pxsz=1.)

max_stars_repo_name: solerjuan/magnetar

max_issues_repo_name: solerjuan/magnetar

max_forks_repo_name: solerjuan/magnetar

reason: import numpy,from astropy


text:
# Start by importing numpy as Keras, like scikit-learn, requires data to be a numpy array.
import numpy as np

# We will use the MNIST dataset containing hand-written digits.
from sklearn import datasets
from sklearn.cross_validation import train_test_split

from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.utils import np_utils

# Set seed for reproducibility.
np.random.seed(1234)

# Load data
mnist = datasets.load_digits()
X = mnist.data
Y = mnist.target

# let's examine data
print X.shape, Y.shape
print X[0]
print Y[0]

# Split the data into a training and test set.
train_X, test_X, train_y, test_y = train_test_split(X, Y, train_size=0.7, random_state=0)

# One Hot Encoding
def one_hot_encode_object_array(arr):
    '''One hot encode a numpy array of objects (e.g. strings)'''
    uniques, ids = np.unique(arr, return_inverse=True)
    return np_utils.to_categorical(ids, len(uniques))

# One hot encode labels for training and test sets.
train_y_ohe = one_hot_encode_object_array(train_y)
test_y_ohe = one_hot_encode_object_array(test_y)

# Initialize a Sequential neural network.
model = Sequential()

# Add a fully connected layer with 32 units. Each unit recieves an input from every unit in the input layer, and since number of units in the input is equal to the dimension (64) of the input vectors, we need the input shape to be 64.
# Keras uses Dense to create a fully connected layer.
model.add(Dense(32, input_shape=(64,)))

# Add an activation after the first layer. We will use sigmoid activation, which is DEFN. Other choices like relu etc are also possible.
model.add(Activation('sigmoid'))

# Add another layer
# model.add(Dense(16))
# model.add(Activation('relu'))

# Add output layer, which is always fully connected. Since the output will be OHE labels, which are 10 dimensions, we would want the output layer to have 10 units.
model.add(Dense(10))

# Add activation for the output layer. In classification tasks, we use softmax activation, which is -- DEFN--. This provides a probilistic interpretation for the output labels...more details..
model.add(Activation('softmax'))

# Next we need to configure the model. There are some more choices we need to make before we can run the model.
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
# optimizer
# loss
# metrics

# Train the network. fit method, similar to sklearn.
model.fit(train_X, train_y_ohe, nb_epoch=10, batch_size=30)
# nb_epoch, batch_size

# Compute accuracy of the model based on this network.
# print model.predict(test_X)[0]
loss, accuracy = model.evaluate(test_X, test_y_ohe)
print accuracy

max_stars_repo_name: markovianlabs/collaborative-learning

max_issues_repo_name: markovianlabs/collaborative-learning

max_forks_repo_name: markovianlabs/collaborative-learning


text:
import sys
sys.path.append("../..")
import glob
import os.path
import smd.utils as utils
from tqdm import tqdm
import scipy.io

DATASET_PATH = "/Users/quentin/Computer/DataSet/Music/speech_music_detection/klaus_seyerlehner/"
AUDIO_PATH = DATASET_PATH
ANNOTATION_PATH = DATASET_PATH

# !!!!! This dataset is not labelled for speech, ... only for evaluation of music detection

def load_files():
    audio_files = glob.glob(AUDIO_PATH + "/*.mp3")
    label_files = glob.glob(ANNOTATION_PATH + "/*.label")
    return audio_files, label_files

def find_associated_label(audio_file, label_files):
    name = os.path.basename(audio_file).replace(".mp3", "")
    for label in label_files:
        if name in label:
            return label
    raise ValueError("Incorrect input: the associated label in not present.")

def get_event_list(label_file, type):
    events = []
    mat = scipy.io.loadmat(label_files[0])
    data = mat["data"]

    # Need to check the time betweek each data
    exit()
    return events

def concatenate_events(events_1, events_2):
    return events_1 + events_2

if __name__ == "__main__":
    audio_files, label_files = load_files()

    print("Number of audio files: " + str(len(audio_files)))
    print("Number of label files: " + str(len(label_files)))

    for audio in tqdm(audio_files):
        music = find_associated_label(audio, label_files)

        music_events = get_event_list(music, "music")

        utils.save_annotation(music_events, os.path.basename(audio).replace(".mp3", "") + ".txt", AUDIO_PATH)

max_issues_repo_name: yagyapandeya/speech-music-detection

max_forks_repo_name: yagyapandeya/speech-music-detection

reason: import scipy


text:
# prefixLSH.py

import numpy as np

MAX_NUMBER_BITS_IN_FLOC = 50
K = 50

def binary_hash(h):
    return np.binary_repr(h, width=MAX_NUMBER_BITS_IN_FLOC)

def get_cohorts_dict(hash_list, min_k=K):
    """
    Applies the prefixLSH algorithm described in paper
    "Clustering for Private Interest-based Advertising" by Epasto and Medina, et al.

    Args:
    hash_list - a list of hashes to sort, where the hashes are *not* bit arrays.
    min_k - the minimum number of hashes that can be in a cohort.

    Returns a dictionary mapping hash to (int) cohort ID.
    """
    # map hashes to their bit array representations
    # pass list with duplicates to sorting algorithm instead of values of map
    bit_hash_list = [binary_hash(h) for h in hash_list]
    # binary representation of hash -> cohortId
    bit_hash_cohorts_dict = get_bit_hash_cohorts_dict(bit_hash_list, min_k=min_k)
    # map back to non bit-array hashes
    return {h: bit_hash_cohorts_dict[binary_hash(h)] for h in hash_list}

def get_bit_hash_cohorts_dict(bit_hash_list, min_k=K):
    """
    Applies the prefixLSH algorithm described in paper
    "Clustering for Private Interest-based Advertising" by Epasto and Medina, et al.

    Args:
    bit_hash_list - a list of hashes to sort where the hashes are bit arrays.
    min_k - the minimum number of hashes that can be in a cohort.

    Returns a dictionary mapping hash to (int) cohort ID.
    """
    sorted_bit_hash_list = sorted(bit_hash_list)
    return apply_prefixLSH({}, 0, sorted_bit_hash_list, min_k=min_k)

def apply_prefixLSH(cohorts_dict, bit, sorted_bit_hash_list, min_k):
    """

    pre-order tree traversal. Each leaf corresponds to a cohort ID.
    Args:

    cohorts_dict - dict of form {bit_hash: cohort_id}. Empty on first call/root node
    bit - Bit index to split cohort on. Starts at 0.
    sorted_bit_hash_list - Sorted (asc) list of hashes, in binary rep.
    min_k - Minimum size of cohort to allow.

    Returns:

    An updated cohorts_dict ({bit_hash: cohort_id}) with a key for each bit_hash in
    `sorted_bit_hash_list`.

    """
    if len(sorted_bit_hash_list) < min_k:
        raise Exception("fewer than %s hashes to sort" % min_k)
    if bit == MAX_NUMBER_BITS_IN_FLOC:
        raise Exception("maximum number of bits exceeded: %s" % bit)
    # all items in hash list must be same length/number of bits
    if not all(len(sorted_bit_hash_list[0]) == len(s) for s in sorted_bit_hash_list):
        raise Exception("items in hash list of different lengths")
    # search through sorted hash list at bit=bit for the first hash where bit=1
    # start at 1 less than minimum k. If all have bit=1, then left list will be too small
    i = min_k - 1
    if bit < len(sorted_bit_hash_list[0]):
        while i < len(sorted_bit_hash_list):
            if sorted_bit_hash_list[i][bit] == "1":
                break
            i += 1
    left_sorted_bit_hash_list = sorted_bit_hash_list[:i]
    right_sorted_bit_hash_list = sorted_bit_hash_list[i:]

    if (len(left_sorted_bit_hash_list) < min_k) or (
        len(right_sorted_bit_hash_list) < min_k
    ):
        # Do not recurse. This is a cohort.
        # cohort ID is the next largest cohort ID that hasn't been assigned.
        cohort_id = max(cohorts_dict.values()) + 1 if cohorts_dict else 1
        cohorts_dict.update({bit_hash: cohort_id for bit_hash in sorted_bit_hash_list})
        return cohorts_dict

    cohorts_dict = apply_prefixLSH(
        cohorts_dict, bit + 1, left_sorted_bit_hash_list, min_k
    )
    cohorts_dict = apply_prefixLSH(
        cohorts_dict, bit + 1, right_sorted_bit_hash_list, min_k
    )
    return cohorts_dict

max_stars_repo_name: aberke/floc-analysis

max_issues_repo_name: aberke/floc-analysis

max_forks_repo_name: aberke/floc-analysis


text:
from elegy import utils

import jax, jax.numpy as jnp
import numpy as np
from unittest import TestCase
import tempfile, os, pickle
import PIL, urllib

import elegy

class ResNetTest(TestCase):
    def test_basic_predict(self):
        # FIXME: test succeeds if run alone or if run on the cpu-only version of jax
        # test fails with "DNN library is not found" if run on gpu with all other tests together

        model = elegy.Model(elegy.nets.resnet.ResNet18(), run_eagerly=True)
        assert isinstance(model.module, elegy.Module)

        x = np.random.random((2, 224, 224, 3)).astype(np.float32)
        y = model.predict(x)

        # update_modules results in a call to `set_default_parameters` for elegy Modules
        # it might be better to have the user call this explicitly to avoid potential OOM
        model.update_modules()

        assert jnp.all(y.shape == (2, 1000))

        # test loading weights from file
        with tempfile.TemporaryDirectory() as tempdir:
            pklpath = os.path.join(tempdir, "delete_me.pkl")
            open(pklpath, "wb").write(
                pickle.dumps(model.module.get_default_parameters())
            )

            new_r18 = elegy.nets.resnet.ResNet18(weights=pklpath)
            y2 = elegy.Model(new_r18, run_eagerly=True).predict(x)

        assert np.allclose(y, y2, rtol=0.001)

    def test_autodownload_pretrained_r18(self):
        fname, _ = urllib.request.urlretrieve(
            "https://upload.wikimedia.org/wikipedia/commons/e/e4/A_French_Bulldog.jpg"
        )
        im = np.array(PIL.Image.open(fname).resize([224, 224])) / np.float32(255)

        r18 = elegy.nets.resnet.ResNet18(weights="imagenet")
        with jax.disable_jit():
            assert elegy.Model(r18).predict(im[np.newaxis]).argmax() == 245

    def test_autodownload_pretrained_r50(self):
        fname, _ = urllib.request.urlretrieve(
            "https://upload.wikimedia.org/wikipedia/commons/e/e4/A_French_Bulldog.jpg"
        )
        im = np.array(PIL.Image.open(fname).resize([224, 224])) / np.float32(255)

        r50 = elegy.nets.resnet.ResNet50(weights="imagenet")
        with jax.disable_jit():
            assert elegy.Model(r50).predict(im[np.newaxis]).argmax() == 245

max_stars_repo_name: sooheon/elegy

max_issues_repo_name: sooheon/elegy

max_forks_repo_name: sooheon/elegy

reason: import numpy,import jax


text:
# Import Libraries
import tensorflow
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from rarfile import RarFile
from urllib.request import urlretrieve
from kerastuner import HyperModel
from kerastuner.tuners import RandomSearch, Hyperband, BayesianOptimization
from tensorflow import keras
from sklearn.metrics import confusion_matrix

sns.set(rc={'figure.figsize': (20, 10)})

# download the compressed divorce file
urlretrieve('https://archive.ics.uci.edu/ml/machine-learning-databases/00497/divorce.rar', 'divorce_file.rar')

# extract rar file
with RarFile('divorce_file.rar', mode='r') as rf:
  rf.extractall()

# read divorce data
df = pd.read_excel('divorce.xlsx')

# clean columns
clean_cols = [x.lower() for x in df.columns.to_list()]
df.columns = clean_cols

# Separate the target and features as separate dataframes
X = df.drop('class', axis=1)
y = df[['class']].astype('int')

# Stratified split based on the distribution of the target vector, y
X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                    stratify=y,
                                                    test_size=0.20,
                                                    random_state=30)

class MyHyperModel(HyperModel):

    def __init__(self, num_classes):
        self.num_classes = num_classes

    def build(self, hp):

        # specify model
        model = keras.Sequential()

        # range of models to build
        for i in range(hp.Int('num_layers', 2, 20)):

            model.add(keras.layers.Dense(units=hp.Int('units_' + str(i),
                                                min_value=32,
                                                max_value=512,
                                                step=32),
                                   activation='relu'))

        model.add(keras.layers.Dense(self.num_classes, activation='sigmoid'))

        model.compile(
            optimizer=keras.optimizers.Adam(
                hp.Choice('learning_rate',
                          values=[1e-2, 1e-3, 1e-4])),
            loss='binary_crossentropy',
            metrics=['accuracy'])

        return model

hypermodel = MyHyperModel(num_classes=1)

tuner = Hyperband(
    hypermodel,
    objective='accuracy',
    max_epochs=10,
    seed=10,
    project_name='divorce test')

tuner.search(X_train.values, y_train.values.flatten(),
             epochs=10,
             validation_data=(X_test.values, y_test.values.flatten()))

params = tuner.get_best_hyperparameters()[0]

model = tuner.hypermodel.build(params)

model.fit(X.values, y.values.flatten(), epochs=20)

hyperband_accuracy_df = pd.DataFrame(model.history.history)

hyperband_accuracy_df[['loss', 'accuracy']].plot()
plt.title('Loss & Accuracy Per EPOCH')
plt.xlabel('EPOCH')
plt.ylabel('Accruacy')
plt.show()

random_tuner = RandomSearch(
    hypermodel,
    objective='accuracy',
    max_trials=10,
    seed=10,
    project_name='divorce test')

random_tuner.search(X_train.values, y_train.values.flatten(),
             epochs=10,
             validation_data=(X_test.values, y_test.values.flatten()))

random_params = random_tuner.get_best_hyperparameters()[0]

random_model = random_tuner.hypermodel.build(params)

random_model.fit(X.values, y.values.flatten(), epochs=15)

random_accuracy_df = pd.DataFrame(random_model.history.history)

random_accuracy_df[['loss', 'accuracy']].plot()
plt.title('Loss & Accuracy Per EPOCH For Random Model')
plt.xlabel('EPOCH')
plt.ylabel('Accruacy')
plt.show()

bayesian_tuner = BayesianOptimization(
    hypermodel,
    objective='accuracy',
    max_trials=10,
    seed=10,
    project_name='divorce test')

bayesian_tuner.search(X_train.values, y_train.values.flatten(),
             epochs=10,
             validation_data=(X_test.values, y_test.values.flatten()))

bayesian_params = bayesian_tuner.get_best_hyperparameters()[0]

bayesian_model = bayesian_tuner.hypermodel.build(bayesian_params)

bayesian_model.fit(X.values, y.values.flatten(), epochs=15)

bayesian_accuracy_df = pd.DataFrame(bayesian_model.history.history)

bayesian_accuracy_df[['loss', 'accuracy']].plot()
plt.title('Loss & Accuracy Per EPOCH For Bayesian Optimisation Model')
plt.xlabel('EPOCH')
plt.ylabel('Accruacy')
plt.show()


text:
from sympy import nsolve, exp, Symbol
from sympy.core import symbol
import sympy as sym

def transformation_LP():
    return

def transformation_HP():
    return

def transformation_BP():
    return

def transformation_BS():
    return

def g2MQ_BPF(g, FBW):
    pass

def stepped_impedance(LC, Z0_high, Z0_low, lg_high, lg_low, fc, ladder_type="LC"):
    """
    Function to convert LC values into TX line physical lengths ... see Sec. 5.1.1, page 114
    @param LC: LC or CL values as an array
    @param Z0_high: Z0 of inductor line
    @param Z0_low: Z0 of capacitor line
    @param lg_high: guided wavelength of inductor line @ cut-off frequency
    @param lg_low: guided wavelength of capacitor line @ cut-off frequency
    @param fc: cut-off frequency
    @param ladder_type: "LC" or "CL"
    """
    LC_len_guess = zeros_like(LC)
    if ladder_type == "LC":
        LC_len_guess[1:-1:2] = (lg_high / 2 / pi) * arcsin(
            2 * pi * fc * LC[1:-1:2] / Z0_high
        )
        LC_len_guess[2:-1:2] = (lg_low / 2 / pi) * arcsin(
            2 * pi * fc * LC[2:-1:2] * Z0_low
        )
    if ladder_type == "CL":
        LC_len_guess[1:-1:2] = (lg_low / 2 / pi) * arcsin(
            2 * pi * fc * LC[1:-1:2] * Z0_low
        )
        print(LC_len_guess[1:-1:2])
        LC_len_guess[2:-1:2] = (lg_high / 2 / pi) * arcsin(
            2 * pi * fc * LC[2:-1:2] / Z0_high
        )

    #     lL = Symbol("lL")
    #     lC = Symbol("lC")
    #     f1 = (
    #         Z0_high * sym.sin(2 * pi * lL / lg_high)
    #         + Z0_low * sym.tan(pi * lC / lg_low)
    #         - 2 * pi * fc * LC[1]
    #     )
    #     f2 = (
    #         sym.sin(2 * pi * lC / lg_low) / Z0_low
    #         + 2 * sym.tan(pi * lL / lg_high) / Z0_high
    #         - 2 * pi * fc * LC[2]
    #     )
    #     print([LC_len_guess[1], LC_len_guess[2]])
    #     print(nsolve([f1, f2], [lL, lC], [LC_len_guess[1], LC_len_guess[2]]))

    return LC_len_guess

    # ==========================================================================
    # J. S. Hong, Ch.5, LPF Excercise
    # ==========================================================================

    #     g = Chebyshev_g(LAr=0.1, LAs=40, Ws=6)[1]
    #     LC = g2LC_LPF(g, Z0=50, fc=1e9, ladder_type="LC")
    #     LC_len = stepped_impedance(
    #         LC, Z0_high=93, Z0_low=24, lg_high=118e-3, lg_low=105e-3, fc=1e9
    #     )
    #     print(LC_len)

    g = Butterworth_g(LAs=20, Ws=1.5)
    print("order: ", g[0])
    print("parameters: ", g[1])
    g2S(g=g[1], Wstart=0.0001, Wstop=3, npoints=1001, dB_limit=-70)

    # Pozar's example ... ex 8.6, page 424
    g = Butterworth_g(LAs=20, Ws=1.5)[1]
    LC = g2LC_LPF(g, Z0=50, fc=2.5e9, ladder_type="CL")
    print("LC values:\n", LC)
    LC_len = stepped_impedance(
        LC,
        Z0_high=120,
        Z0_low=20,
        lg_high=118e-3,
        lg_low=105e-3,
        fc=1e9,
        ladder_type="CL",
    )

# def g2KJ_HP(g, LC_values, ZY0=1, ZYL=1, plot):
#
# def g2KJ_BP(g, LC_values, ZY0=1, ZYL=1, plot):
#
# def g2KJ_BS(g, LC_values, ZY0=1, ZYL=1, plot):

# def g2LC_LPF(g, Z0=50, fc=1e9, ladder_type="LC"):  # only for LPFs
#     if ladder_type == "LC":
#         g[1:-1:2] = g[1:-1:2] * Z0 / (2 * pi * fc)
#         g[2:-1:2] = g[2:-1:2] / Z0 / (2 * pi * fc)
#         g[0] = Z0
#         g[-1] = g[-1] * Z0
#         g2LC = g
#     if ladder_type == "CL":
#         g[1:-1:2] = g[1:-1:2] / Z0 / (2 * pi * fc)
#         g[2:-1:2] = g[2:-1:2] * Z0 / (2 * pi * fc)
#         g[0] = Z0
#         g[-1] = g[-1] * Z0
#         g2LC = g
#     return g2LC

def g2LC(g, Z0=50, fc=1e9, ladder_type="LC"):  # only for LPFs
    if ladder_type == "LC":
        g[1:-1:2] = g[1:-1:2] * Z0 / (2 * np.pi * fc)
        g[2:-1:2] = g[2:-1:2] / Z0 / (2 * np.pi * fc)
        g[0] = Z0
        g[-1] = g[-1] * Z0
        g2LC = g
    if ladder_type == "CL":
        g[1:-1:2] = g[1:-1:2] / Z0 / (2 * np.pi * fc)
        g[2:-1:2] = g[2:-1:2] * Z0 / (2 * np.pi * fc)
        g[0] = Z0
        g[-1] = g[-1] * Z0
        g2LC = g
    return g2LC

def stub_LPF():
    # Implement Fig. 5.3 from J. S. Hong
    pass

# ==========================================================================
# J. S. Hong, stub LPF
# ==========================================================================

# ==========================================================================
# J. S. Hong, elliptic LPF
# ==========================================================================

# ==========================================================================
# J. S. Hong, interdigital
# ==========================================================================

# ==========================================================================
# J. S. Hong, combline
# ==========================================================================

# ==========================================================================
# J. S. Hong, pseudo-combline
# ==========================================================================

# ==========================================================================
# J. S. Hong, stub bandpass
# ==========================================================================

max_stars_repo_name: zinka/Filtertool

max_issues_repo_name: zinka/Filtertool

max_forks_repo_name: zinka/Filtertool

reason: import sympy,from sympy


text:
import matplotlib.colors
import matplotlib.pyplot as plt
import numpy as np
from pcl_helper import *

def rgb_to_hsv(rgb_list):
    rgb_normalized = [1.0*rgb_list[0]/255, 1.0*rgb_list[1]/255, 1.0*rgb_list[2]/255]
    hsv_normalized = matplotlib.colors.rgb_to_hsv([[rgb_normalized]])[0][0]
    return hsv_normalized

def compute_color_histograms(cloud, using_hsv=True):

    # Compute histograms for the clusters
    point_colors_list = []

    # Step through each point in the point cloud
    for point in pc2.read_points(cloud, skip_nans=True):
        rgb_list = float_to_rgb(point[3])
        if using_hsv:
            point_colors_list.append(rgb_to_hsv(rgb_list) * 255)
        else:
            point_colors_list.append(rgb_list)

    # Populate lists with color values
    channel_1_vals = []
    channel_2_vals = []
    channel_3_vals = []

    for color in point_colors_list:
        channel_1_vals.append(color[0])
        channel_2_vals.append(color[1])
        channel_3_vals.append(color[2])

    # TODO: Compute histograms
    nbin = 22
    nrange = (0, 256)
    r_hist = np.histogram(channel_1_vals, bins=nbin, range=nrange)
    g_hist = np.histogram(channel_2_vals, bins=nbin, range=nrange)
    b_hist = np.histogram(channel_3_vals, bins=nbin, range=nrange)
    # TODO: Concatenate and normalize the histograms
    hist_features = np.concatenate((r_hist[0], g_hist[0], b_hist[0])).astype(np.float64)
    # Generate random features for demo mode.
    # Replace normed_features with your feature vector
    normed_features = hist_features/ np.sum(hist_features)
    #print("color**********", normed_features)
    return normed_features

def compute_normal_histograms(normal_cloud):
    norm_x_vals = []
    norm_y_vals = []
    norm_z_vals = []

    for norm_component in pc2.read_points(normal_cloud,
                                          field_names = ('normal_x', 'normal_y', 'normal_z'),
                                          skip_nans=True):
        norm_x_vals.append(norm_component[0])
        norm_y_vals.append(norm_component[1])
        norm_z_vals.append(norm_component[2])

    # TODO: Compute histograms of normal values (just like with color)
    nbin = 30
    nrange = (0, 1)
    norm_x = np.histogram(norm_x_vals, bins=nbin, range=nrange)
    norm_y = np.histogram(norm_y_vals, bins=nbin, range=nrange)
    norm_z = np.histogram(norm_z_vals, bins=nbin, range=nrange)

    # TODO: Concatenate and normalize the histograms
    hist_features = np.concatenate((norm_x[0], norm_y[0], norm_z[0])).astype(np.float64)
    # Generate random features for demo mode.
    # Replace normed_features with your feature vector
    normed_features = hist_features/ np.sum(hist_features)
    print("normal ",normed_features)
    #fig = plt.figure(figsize=(12,6))
    #plt.plot(normed_features)
    #plt.title('Normal Features Plot')
    #plt.tick_params(axis='both', which='major', labelsize=20)
    #fig.tight_layout()
    return normed_features


text:
# code adapted from https://github.com/huggingface/notebooks/blob/master/examples/translation.ipynb

# # need special branch of sacrebleu for thai bleu
# !pip uninstall -q sacrebleu
# !pip install -q git+https://github.com/cstorm125/sacrebleu.git@add_thai_tokenizer
# # written with transformers 4.6.0

# export WANDB_PROJECT=mariantmt-zh_cn-th
# python train_model.py --input_fname ../data/v1/Train.csv \
#     --output_dir ../models/marianmt-zh_cn-th --source_lang zh --target_lang th \
#     --metric_tokenize th_syllable --fp16

# export WANDB_PROJECT=mariantmt-th-zh_cn
# python train_model.py --input_fname ../data/v1/Train.csv \
#     --output_dir ../models/marianmt-zh_cn-th --source_lang th --target_lang zh \
#     --metric_tokenize zh --fp16

import argparse
from transformers import (
    MarianTokenizer,
    MarianMTModel,
    MarianConfig,
    AutoModelForSeq2SeqLM,
    DataCollatorForSeq2Seq,
    Seq2SeqTrainingArguments,
    Seq2SeqTrainer,
)
from datasets import load_metric, Dataset
import numpy as np
import pandas as pd
import datasets
import random
import json
from functools import partial

import logging
logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.INFO)

metric = load_metric("sacrebleu")

def preprocess_function(examples,
                        tokenizer,
                        max_input_length,
                        max_target_length,
                        source_lang,
                        target_lang,):
    inputs = [ex[source_lang] for ex in examples["translation"]]
    targets = [ex[target_lang] for ex in examples["translation"]]
    model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=True)

    with tokenizer.as_target_tokenizer():
        labels = tokenizer(targets, max_length=max_target_length, truncation=True)

    model_inputs["labels"] = labels["input_ids"]
    return model_inputs

def postprocess_text(preds, labels):
    preds = [pred.strip() for pred in preds]
    labels = [[label.strip()] for label in labels]
    return preds, labels

def compute_metrics(eval_preds,
                    tokenizer,
                    metric,
                    metric_tokenize):
    preds, labels = eval_preds
    if isinstance(preds, tuple):
        preds = preds[0]
    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)

    # Replace -100 in the labels as we can't decode them.
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

    # Some simple post-processing
    decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)

    result = metric.compute(predictions=decoded_preds,
                            references=decoded_labels,
                            tokenize= metric_tokenize)
    result = {"bleu": result["score"]}

    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]
    result["gen_len"] = np.mean(prediction_lens)
    result = {k: round(v, 4) for k, v in result.items()}
    return result

def main(args):
    df = pd.read_csv(args.input_fname, encoding='utf-8')[[args.source_lang,args.target_lang]]
    logging.info(f'Loaded {df.shape}')

    #convert to dictionary
    j = {'translation':[]}
    for i in df.itertuples():
        j['translation'] += [{args.source_lang: i[1], args.target_lang:i[2]}]

    train_dataset = Dataset.from_dict(j)
    raw_datasets = train_dataset.train_test_split(test_size=args.valid_pct,
                                                  seed=args.seed)
    logging.info(f'Datasets created {raw_datasets}')

    tokenizer = MarianTokenizer.from_pretrained(args.output_dir)
    logging.info(f'Tokenizer loaded from {args.output_dir}')

    #tokenize datasets
    tokenized_datasets = raw_datasets.map(partial(preprocess_function,
                                                  tokenizer=tokenizer,
                                                  max_input_length = args.max_input_length,
                                                  max_target_length = args.max_target_length,
                                                  source_lang = args.source_lang,
                                                  target_lang = args.target_lang),
                                          batched=True,)
    logging.info(f'Tokenized datasets: {tokenized_datasets}')

    #filter those with too few tokens
    tokenized_datasets = tokenized_datasets.filter(lambda example: len(example['translation']['zh'])>2)
    tokenized_datasets = tokenized_datasets.filter(lambda example: len(example['translation']['th'])>2)
    logging.info(f'Tokenized datasets when filtered out less than 2 tokens per sequence: {tokenized_datasets}')

    config = MarianConfig.from_pretrained(args.output_dir)
    model = MarianMTModel(config)
    logging.info(f'Loaded model from {args.output_dir}')

    training_args = Seq2SeqTrainingArguments(
        args.output_dir,
        evaluation_strategy = "epoch",
        load_best_model_at_end=True,
        learning_rate= args.learning_rate,
        warmup_ratio=args.warmup_ratio,
        per_device_train_batch_size=args.batch_size,
        per_device_eval_batch_size=args.batch_size,
        weight_decay=args.weight_decay,
        save_total_limit=args.save_total_limit,
        num_train_epochs=args.num_train_epochs,
        predict_with_generate=True,
        fp16=args.fp16,
        seed=args.seed,
    )
    logging.info(f'Training congig {training_args}')

    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
    trainer = Seq2SeqTrainer(
        model,
        training_args,
        train_dataset=tokenized_datasets["train"],
        eval_dataset=tokenized_datasets["test"],
        data_collator=data_collator,
        tokenizer=tokenizer,
        compute_metrics=partial(compute_metrics,
                                tokenizer=tokenizer,
                                metric=metric,
                                metric_tokenize=args.metric_tokenize),
    )
    logging.info(f'Trainer created')

    trainer.train()

    model.save_pretrained(f"{args.output_dir}_best")
    tokenizer.save_pretrained(f"{args.output_dir}_best")
    logging.info(f'Best model saved')

    model.cpu()
    src_text = [
        '我爱你',
        '国王有很多心事。我明白'
    ]
    translated = model.generate(**tokenizer(src_text, return_tensors="pt", padding=True))
    print([tokenizer.decode(t, skip_special_tokens=True) for t in translated])

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--input_fname', type=str)
    parser.add_argument('--output_dir', type=str)
    parser.add_argument('--source_lang', type=str)
    parser.add_argument('--target_lang', type=str)
    parser.add_argument('--metric_tokenize', type=str)

    parser.add_argument('--max_input_length', type=int, default=160)
    parser.add_argument('--max_target_length', type=int, default=160)
    parser.add_argument('--batch_size', type=int, default=64)
    parser.add_argument('--num_train_epochs', type=int, default=10)
    parser.add_argument('--learning_rate', type=float, default=2e-5)
    parser.add_argument('--warmup_ratio', type=float, default=0.1)
    parser.add_argument('--weight_decay', type=float, default=0.01)
    parser.add_argument('--save_total_limit', type=int, default=3)
    parser.add_argument('--fp16', action='store_true')

    parser.add_argument('--valid_pct', type=float, default=0.01)
    parser.add_argument('--seed', type=int, default=42)

    args = parser.parse_args()
    main(args)

max_stars_repo_name: LalitaDeelert/lalita-mt-zhth

max_forks_repo_name: LalitaDeelert/lalita-mt-zhth


text:
import functools
import os
import pickle

import numpy as np
import torch
from PIL import Image
from torch.utils.data import DataLoader, DistributedSampler
from torchvision import transforms
from torchvision.datasets import VisionDataset

class CIFAR10(VisionDataset):
    base_folder = "cifar-10-batches-py"
    train_val_test_list = {
        "train": [
            ["data_batch_1", "c99cafc152244af753f735de768cd75f"],
            ["data_batch_2", "d4bba439e000b95fd0a9bffe97cbabec"],
            ["data_batch_3", "54ebc095f3ab1f0389bbae665268c751"],
            ["data_batch_4", "634d18415352ddfa80567beed471001a"]
        ],
        "val": [
            ["data_batch_5", "482c414d41f54cd18b22e5b47cb7c3cb"],
        ],
        "augment": [
            ["data_batch_1", "c99cafc152244af753f735de768cd75f"],
            ["data_batch_2", "d4bba439e000b95fd0a9bffe97cbabec"],
            ["data_batch_3", "54ebc095f3ab1f0389bbae665268c751"],
            ["data_batch_4", "634d18415352ddfa80567beed471001a"],
            ["data_batch_5", "482c414d41f54cd18b22e5b47cb7c3cb"],
        ],
        "test": [
            ["test_batch", "40351d587109b95175f43aff81a1287e"],
        ]
    }

    def __init__(self, root, split="train", transform=None, target_transform=None):
        super(CIFAR10, self).__init__(root, transform=transform, target_transform=target_transform)

        self.split = split
        downloaded_list = self.train_val_test_list[split]
        self.data = []
        self.targets = []

        # now load the picked numpy arrays
        for file_name, _ in downloaded_list:
            file_path = os.path.join(self.root, self.base_folder, file_name)
            with open(file_path, "rb") as f:
                entry = pickle.load(f, encoding="latin1")
                self.data.append(entry["data"])
                if "labels" in entry:
                    self.targets.extend(entry["labels"])
                else:
                    self.targets.extend(entry["fine_labels"])

        self.data = np.vstack(self.data).reshape(-1, 3, 32, 32)
        self.data = self.data.transpose((0, 2, 3, 1))  # convert to HWC
        self.data = np.ascontiguousarray(self.data)  # increase locality

    def __getitem__(self, index):
        img, target = self.data[index], self.targets[index]

        # doing this so that it is consistent with all other datasets to return a PIL Image
        img = Image.fromarray(img)
        if self.transform is not None:
            img = self.transform(img)
        if self.target_transform is not None:
            target = self.target_transform(target)
        return img, target

    def __len__(self):
        return len(self.data)

    def extra_repr(self):
        return "Split: {}".format(self.split.capitalize())

def cutout_fn(img, length):
    h, w = img.size(1), img.size(2)
    mask = np.ones((h, w), np.float32)
    y = np.random.randint(h)
    x = np.random.randint(w)

    y1 = np.clip(y - length // 2, 0, h)
    y2 = np.clip(y + length // 2, 0, h)
    x1 = np.clip(x - length // 2, 0, w)
    x2 = np.clip(x + length // 2, 0, w)

    mask[y1: y2, x1: x2] = 0.
    mask = torch.from_numpy(mask)
    mask = mask.expand_as(img)
    img *= mask

    return img

class DistributableDataLoader(DataLoader):
    def __init__(self, dataset, distributed=False, **kwargs):
        self.distributed_sampler = None
        if distributed:
            self.distributed_sampler = kwargs["sampler"] = DistributedSampler(dataset)
            kwargs.pop("shuffle", None)
        super().__init__(dataset, **kwargs)
        self._epoch = 0

    def __iter__(self):
        self._epoch += 1
        if self.distributed_sampler is not None:
            self.distributed_sampler.set_epoch(self._epoch)
        return super().__iter__()

def dataloader_cifar(image_dir, split, args, batch_size=None, shuffle=None, distributed=False):  # pylint: disable=inconsistent-return-statements
    if batch_size is None:
        batch_size = args.batch_size
    if shuffle is None:
        shuffle = split in ("train", "augment")
    assert torch.cuda.is_available()
    MEAN = [0.49139968, 0.48215827, 0.44653124]
    STD = [0.24703233, 0.24348505, 0.26158768]
    transf = [
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip()
    ]
    normalize = [
        transforms.ToTensor(),
        transforms.Normalize(MEAN, STD)
    ]
    if split in ("train", "augment"):
        cutout = []
        if args.cutout > 0:
            cutout.append(functools.partial(cutout_fn, length=args.cutout))
        dataset = CIFAR10(image_dir, split, transform=transforms.Compose(transf + normalize + cutout))
        return DistributableDataLoader(dataset, distributed=distributed, batch_size=batch_size, drop_last=True, shuffle=shuffle, num_workers=args.num_threads)
    if split in ("val", "test"):
        dataset = CIFAR10(image_dir, split, transform=transforms.Compose(normalize))
        return DistributableDataLoader(dataset, distributed=distributed, batch_size=batch_size, drop_last=False, shuffle=shuffle, num_workers=args.num_threads)


text:
import logging; log = logging.getLogger(__name__)
import OpenGL.GL as gl
import ctypes
import numpy as np
import re
from importlib import resources
from .Attribute import Attribute
from .Uniform import makeUniform
from .UniformBlock import UniformBlock

class Program:
    """Base class for shader programs.

    You should use `Context.Program()` instead of
    instantiating this class.
    """

    # vtx -> tesselate -> geom -> fragment
    shader_types = {
        'fragment_shader':        gl.GL_FRAGMENT_SHADER,
        'vertex_shader':          gl.GL_VERTEX_SHADER,
        'geometry_shader':        gl.GL_GEOMETRY_SHADER,
        'tess_evaluation_shader': gl.GL_TESS_EVALUATION_SHADER,
        'tess_control_shader':    gl.GL_TESS_CONTROL_SHADER,
    }

    def __init__(self, ctx, separable=False, **files):
        self.ctx = ctx
        self.id  = ctx.glCreateProgram()

        if separable or getattr(self, 'separable', False):
            self.ctx.glProgramParameteri(self.id,
                self.ctx.GL_PROGRAM_SEPARABLE, True)

        # every program should have its own vertex array object,
        # since that stores important state info.
        self.vao = ctx.VertexArray()
        self._shaders      = {}
        self.attribs       = {}
        self.uniforms      = {}
        self.uniformBlocks = {}

        if len(files) > 0:
            self._loadShaders(**files)
            self._link()
        else: # we can also specify these as attributes of subclasses.
            shaders = {}
            for typ in self.shader_types.keys():
                path = getattr(self, typ, None)
                if path is not None:
                    shaders[typ] = path
            if len(shaders) > 0:
                self._loadShaders(**shaders)
                self._link()

    def __del__(self):
        for obj in self._shaders.values():
            self.ctx.glDeleteShader(obj)
        self.ctx.glDeleteProgram(self.id)

    def __enter__(self):
        self.bind()
        return self

    def __exit__(self, type, value, traceback):
        self.unbind()

    def bind(self):
        self.vao.bind()
        self.ctx.glUseProgram(self.id)

    def unbind(self):
        self.ctx.glUseProgram(0)
        self.vao.unbind()

    @property
    def memUsedCpu(self):
        """Estimated amount of CPU-side memory used."""
        return None # amount not known

    @property
    def memUsedGpu(self):
        """Estimated amount of GPU-side memory used."""
        return None # amount not known

    def run(self, *args, **kwargs):
        """Execute the program."""
        raise NotImplementedError

    def setUniforms(self, **values):
        """Set the values of multiple uniform variables."""
        with self:
            for name, val in values.items():
                self.uniforms[name].value = val

    def _loadShaders(self, **files):
        """Load shader sources and attach to current program.

        files: paths to shader code files.
        """
        for name, path in files.items():
            shd = self._compileShader(name, path)
            #log.debug("Bind shader %d to program %d: %s => %s", shd, self.id, name, path)
            self.ctx.glAttachShader(self.id, shd)

    def _getShaderCodeFromFile(self, path):
        """Load shader code from file path.

        Exists so that programs can override it to provide a base path.
        """
        with open(path, 'rt') as file:
            return file.read()

    def _getShaderCode(self, path, forcePath=False):
        """Load shader code from a path.

        path: A file path beginning with '@',
            or a tuple of (package, name) to use importlib.resources,
            or the code itself.
        forcePath: Whether to interpret `path` as a file path even if it
            doesn't begin with '@'. Used for '#include' function since
            including a string isn't very useful.
        """
        if type(path) is tuple:
            code = resources.read_text(*path)
        elif path.startswith('@'):
            code = self._getShaderCodeFromFile(path[1:])
        elif forcePath: # for #include
            code = self._getShaderCodeFromFile(path)
        else:
            code = path

        return self._preprocessShader(code)

    def _compileShader(self, name, path):
        if name not in self.shader_types:
            raise KeyError("Unknown shader type '%s'" % str(name))

        code = self._getShaderCode(path)
        shd = self.ctx.glCreateShader(self.shader_types[name])
        self.ctx.glShaderSource(shd, code)
        self.ctx.glCompileShader(shd)

        r = self.ctx.glGetShaderiv(shd, gl.GL_COMPILE_STATUS)
        if not r:
            raise RuntimeError("Error compiling %s:\n%s" % (
                name, self.ctx.glGetShaderInfoLog(shd).decode('utf-8')))

        self._shaders[name] = shd
        return shd

    def _preprocessShader(self, code):
        """Preprocess shader code, as a hack since we don't have
        glNamedStringARB."""
        rx = r'^\#include\s+"([^"]+)"'
        result = []
        while True:
            match = re.search(rx, code, re.MULTILINE)
            if match is None: break
            st, en, path = match.start(), match.end(), match.group(1)
            result.append(code[0:st])
            result.append("/*** include(%s) ***/\n" % path)
            result.append(self._getShaderCode(path, forcePath=True))
            result.append("\n/*** end include(%s) ***/\n" % path)
            code = code[en:]
        result.append(code)
        result = ''.join(result)
        #if result != code:
        #    log.debug("*** preprocessed code:\n%s", result)
        return result

    def _link(self):
        self.ctx.glLinkProgram(self.id)

        # check if linked OK
        stat = self.ctx.glGetProgramiv(self.id, gl.GL_LINK_STATUS)
        #log.debug("link status = %r", stat)
        if not stat:
            r = self.ctx.glGetProgramInfoLog(self.id).decode('utf-8')
            raise RuntimeError("Program linking failed:\n%s" % r)

        self.ctx.glUseProgram(self.id)
        self._getVariables()

    def _getVariables(self):
        """Get the attribues and uniforms."""
        nAttrs = self.ctx.glGetProgramiv(self.id, gl.GL_ACTIVE_ATTRIBUTES)
        #log.debug("Program has %d attrs", nAttrs)
        for i in range(nAttrs):
            a = Attribute(self, i)
            self.attribs[a.name] = a

        nUniforms = self.ctx.glGetProgramiv(self.id, gl.GL_ACTIVE_UNIFORMS)
        #log.debug("Program has %d uniforms", nUniforms)
        for i in range(nUniforms):
            u = makeUniform(self, i)
            self.uniforms[u.name] = u

        nBlocks = self.ctx.glGetProgramiv(self.id, gl.GL_ACTIVE_UNIFORM_BLOCKS)
        #log.debug("Program has %d uniform blocks", nBlocks)
        for i in range(nBlocks):
            u = UniformBlock(self, i)
            self.uniformBlocks[u.name] = u


text:
import os
import socket
import subprocess
import time
from datetime import datetime

import numpy as np
from celery import chord
from celery.exceptions import Reject

from .worker import solve
from ..app import app

## Monitoring tasks

@app.task
def monitor_queues(ignore_result=True):
    server_name = app.conf.MONITORING_SERVER_NAME
    server_port = app.conf.MONITORING_SERVER_PORT
    metric_prefix = app.conf.MONITORING_METRIC_PREFIX

    queues_to_monitor = ('server', 'worker')

    output = subprocess.check_output('rabbitmqctl -q list_queues name messages consumers', shell=True)
    lines = (line.split() for line in output.splitlines())
    data = ((queue, int(tasks), int(consumers)) for queue, tasks, consumers in lines if queue in queues_to_monitor)

    timestamp = int(time.time())
    metrics = []
    for queue, tasks, consumers in data:
        metric_base_name = "%s.queue.%s." % (metric_prefix, queue)

        metrics.append("%s %d %d\n" % (metric_base_name + 'tasks', tasks, timestamp))
        metrics.append("%s %d %d\n" % (metric_base_name + 'consumers', consumers, timestamp))

    sock = socket.create_connection((server_name, server_port), timeout=10)
    sock.sendall(''.join(metrics))
    sock.close()

## Recording the experiment status

def get_experiment_status_filename(status):
    return os.path.join(app.conf.STATUS_DIR, status)

def get_experiment_status_time():
    """Get the current local date and time, in ISO 8601 format (microseconds and TZ removed)"""
    return datetime.now().replace(microsecond=0).isoformat()

@app.task
def record_experiment_status(status):
    # ako su rezultati vec sracunati, onda nece pokrenuti, jer je iskesirano
    with open(get_experiment_status_filename(status), 'w') as fp:
        fp.write(get_experiment_status_time() + '\n')

## Seeding the computations

def y0_gen(theta_resolution):
    for theta1_init in np.linspace(0, 2 * np.pi, theta_resolution):
        for theta2_init in np.linspace(0, 2 * np.pi, theta_resolution):
            # Initial conditions: theta1, dtheta1/dt, theta2, dtheta2/dt.
            y0 = np.array([
                theta1_init,
                0.0,
                theta2_init,
                0.0
            ])
            yield y0

@app.task
def seed_computations(ignore_result=True):
    # da li su eksperimenti zapoceti, bacamo izuzetak
    # seed-ovani smo ranije

    # omogucava sleep-resume mehanizam (samo ako ne ubijemo RabitMq i REdis)
    # if os.path.exists(get_experiment_status_filename('started')):
    #     raise Reject('Computations have already been seeded!')
    # snimanje statusa
    record_experiment_status.si('started').delay()

    theta_resolution = app.conf.THETA_RESOLUTION
    L1 = app.conf.L1
    L2 = app.conf.L2
    m1 = app.conf.M1
    m2 = app.conf.M2
    tmax = app.conf.TMAX
    dt = app.conf.DT

    import logging
    logging.warn('\nL1 %s\nL2 %s\nm1 %s\nm2 %s\ntmax %s\ndt %s\ntheta_resolution %s',
                 L1, L2, m1, m2, tmax, dt, theta_resolution)
    # distribuirani map reduce
    chord(
        (
            solve.s(L1, L2, m1, m2, tmax, dt, y0)
            for y0 in y0_gen(theta_resolution)
        ),
        store_results.s()
    ).delay()

@app.task
def store_results(results):
    import logging
    import csv

    results_path = os.path.join(app.conf.RESULTS_DIR, "results.csv")
    logging.info('RESULTS PATH <- %s', results_path)

    with open(results_path, 'w') as csvfile:
        fieldnames = ['theta1_init', 'theta2_init', 'theta1', 'theta2', 'x1', 'y1', 'x2', 'y2']
        csvwriter = csv.DictWriter(csvfile, fieldnames=fieldnames)
        csvwriter.writeheader()

        for theta1_init, theta2_init, theta1, theta2, x1, y1, x2, y2 in results:
            csvwriter.writerow({'theta1_init': theta1_init,
                                'theta2_init': theta2_init,
                                'theta1': theta1[-1],
                                'theta2': theta2[-1],
                                'x1': x1[-1],
                                'y1': y1[-1],
                                'x2': x2[-1],
                                'y2': y2[-1]
                                })


text:
import numpy as np
import pandas as pd
from collections import deque

# generate some data
# define features and target values
data = {
    'wind_direction': ['N', 'S', 'E', 'W'],
    'tide': ['Low', 'High'],
    'swell_forecasting': ['small', 'medium', 'large'],
    'good_waves': ['Yes', 'No']
}

# create an empty dataframe
data_df = pd.DataFrame(columns=data.keys())

np.random.seed(42)
# randomnly create 1000 instances
for i in range(1000):
    data_df.loc[i, 'wind_direction'] = str(np.random.choice(data['wind_direction'], 1)[0])
    data_df.loc[i, 'tide'] = str(np.random.choice(data['tide'], 1)[0])
    data_df.loc[i, 'swell_forecasting'] = str(np.random.choice(data['swell_forecasting'], 1)[0])
    data_df.loc[i, 'good_waves'] = str(np.random.choice(data['good_waves'], 1)[0])

data_df.head()

# separate target from predictors
X = np.array(data_df.drop('good_waves', axis=1).copy())
y = np.array(data_df['good_waves'].copy())
feature_names = list(data_df.keys())[:3]

# import and instantiate our DecisionTreeClassifier class
from ID3 import DecisionTreeClassifier

# instantiate DecisionTreeClassifier
tree_clf = DecisionTreeClassifier(X=X, feature_names=feature_names, labels=y)
print("System entropy {:.4f}".format(tree_clf.entropy))
# run algorithm id3 to build a tree
tree_clf.id3()
tree_clf.printTree()

max_stars_repo_licenses: Unlicense MIT

max_issues_repo_licenses: Unlicense MIT

max_forks_repo_licenses: Unlicense MIT


text:
#!/usr/bin/env python
# coding=utf-8
import numpy as np
import pytest
from loadsim.read_ids import ReadIds

class TestsubGroup():
    def setup_method(self, method):
        self.rs = ReadIds('/home/mtx/work/tree/groups_066/subhalo_ids_066.')

    def teardown_method(self):
        del self.rs

    def test_fnr(self, submbid=12638380):
        self.rs.gen_sub_table()
        self.rs.get_fnr_ids()
        fnr = self.rs.get_par_files(submbid)
        sub = self.rs.subtable[submbid]
        sublen = sub['SubLen']
        suboff = sub['SubOffset']
        subend = suboff + sublen
        assert self.rs.NidOff[fnr] < suboff
        assert self.rs.NidOff[fnr] + self.rs.Nids[fnr] > subend

    def test_first_id_is_mostboundid(self, submbid=12638380):
        ids = self.rs.get_par_ids(submbid=submbid)
        assert ids[0] == submbid

    def test_ids_sublen(self, submbid=12638380):
        ids = self.rs.get_par_ids(submbid=submbid)
        assert ids.size == self.rs.subtable[submbid]['SubLen']

    def test_ids_shape(self, submbid=12638380):
        ids = self.rs.get_par_ids(submbid=submbid)
        assert ids.shape == (self.rs.subtable[submbid]['SubLen'])

    def test_ids_unique(self, submbid=12638380):
        ids = self.rs.get_par_ids(submbid=submbid)
        assert np.unique(ids).size == ids.size

    def test_sub_in_multiple_files_filenum(self):
        # submbid=8495089
        submbid = self.rs.Data_sub['SubMostBoundID'][0]
        fnrs = self.rs.get_par_files(submbid)
        assert fnrs.size > 1

    def test_sub_in_multiple_files_sublen(self):
        # submbid=8495089
        submbid = self.rs.Data_sub['SubMostBoundID'][0]
        sub = self.rs.subtable[submbid]
        print(sub.dtype)
        print(sub)
        ids = self.rs.get_par_ids(submbid=submbid)
        assert ids.size == sub['SubLen']

    def test_sub_in_multiple_files_mbid(self):
        # submbid=8495089
        submbid = self.rs.Data_sub['SubMostBoundID'][0]
        sub = self.rs.subtable[submbid]
        ids = self.rs.get_par_ids(submbid=submbid)
        assert ids[0] == submbid

    def test_sub_in_multiple_files_unique(self):
        # submbid=8495089
        submbid = self.rs.Data_sub['SubMostBoundID'][0]
        sub = self.rs.subtable[submbid]
        ids = self.rs.get_par_ids(submbid=submbid)
        assert np.unique(ids).size == ids.size

max_stars_repo_name: POFK/loadsim

max_issues_repo_name: POFK/loadsim

max_forks_repo_name: POFK/loadsim


text:
"""
Usage ::

    # Import
    import psana.pscalib.calib.MDBWebUtils as wu
    from psana.pscalib.calib.MDBWebUtils import calib_constants

    _ = wu.request(url, query=None)
    _ = wu.database_names(url=cc.URL)
    _ = wu.collection_names(dbname, url=cc.URL)
    _ = wu.find_docs(dbname, colname, query={'ctype':'pedestals'}, url=cc.URL)
    _ = wu.find_doc(dbname, colname, query={'ctype':'pedestals'}, url=cc.URL)
    _ = wu.get_doc_for_docid(dbname, colname, docid, url=cc.URL)
    _ = wu.get_data_for_id(dbname, dataid, url=cc.URL)
    _ = wu.get_data_for_docid(dbname, colname, docid, url=cc.URL)
    _ = wu.get_data_for_doc(dbname, colname, doc, url=cc.URL)
    data,doc = wu.calib_constants(det, exp=None, ctype='pedestals', run=None, time_sec=None, vers=None, url=cc.URL)

    test_*()
"""
#------------------------------

import logging
logger = logging.getLogger(__name__)

import psana.pscalib.calib.CalibConstants as cc
from requests import get
#import json
from time import time
from numpy import fromstring
from psana.pscalib.calib.MDBUtils import dbnames_collection_query, object_from_data_string

#------------------------------
#------------------------------

def request(url, query=None) :
    #logger.debug('==== query: %s' % str(query))
    t0_sec = time()
    r = get(url, query)
    dt = time()-t0_sec
    logger.debug('CONSUMED TIME by request %.6f sec\n  for url=%s  query=%s' % (dt, url, str(query)))
    return r

#------------------------------

# curl -s "https://pswww.slac.stanford.edu/calib_ws/test_db"
def database_names(url=cc.URL) :
    """Returns list of database names for url.
    """
    r = request(url)
    return r.json()

#------------------------------

# curl -s "https://pswww.slac.stanford.edu/calib_ws/test_db/test_coll"
def collection_names(dbname, url=cc.URL) :
    """Returns list of collection names for dbname and url.
    """
    r = request('%s/%s'%(url,dbname))
    return r.json()

#------------------------------
# curl -s "https://pswww.slac.stanford.edu/calib_ws/test_db/test_coll?query_string=%7B%20%22item%22..."
def find_docs(dbname, colname, query={'ctype':'pedestals'}, url=cc.URL) :
    """Returns list of documents for query, e.g. query={'ctype':'pedestals', "run":{ "$gte":80}}
    """
    query_string=str(query).replace("'",'"')
    logger.debug('find_docs query: %s' % query_string)
    r = request('%s/%s/%s'%(url,dbname,colname),{"query_string": query_string})
    return r.json()

#------------------------------

def find_doc(dbname, colname, query={'ctype':'pedestals'}, url=cc.URL) :
    """Returns document for query.
       1. finds all documents for query
       2. select the latest for run or time_sec
    """
    docs = find_docs(dbname, colname, query, url)
    if docs is None :
        logger.warning('find_docs returns None for query: %s' % query)
        return None

    if len(docs)==0 :
        logger.warning('find_docs returns list of length 0 for query: %s' % query)
        return None

    qkeys = query.keys()
    key_sort = 'time_sec' if 'time_sec' in qkeys else 'run'

    logger.debug('find_doc query: %s\nfind_doc key_sort: %s' % (str(query), key_sort))
    vals = [int(d[key_sort]) for d in docs]
    vals.sort(reverse=True)
    logger.debug('find_doc values: %s' % str(vals))
    val_sel = int(vals[0])
    logger.debug('find_doc select document for %s: %s' % (key_sort,val_sel))
    for d in docs :
        if d[key_sort]==val_sel :
            return d
    return None

#------------------------------

# curl -s "https://pswww.slac.stanford.edu/calib_ws/cdb_cxic0415/cspad_0001/5b6893e81ead141643fe4344"
def get_doc_for_docid(dbname, colname, docid, url=cc.URL) :
    """Returns document for docid.
    """
    r = request('%s/%s/%s/%s'%(url,dbname,colname,docid))
    return r.json()

#------------------------------

# curl -s "https://pswww.slac.stanford.edu/calib_ws/cdb_cxic0415/gridfs/5b6893d91ead141643fe3f6a"
def get_data_for_id(dbname, dataid, url=cc.URL) :
    """Returns raw data from GridFS, at this level there is no info for parsing.
    """
    r = request('%s/%s/gridfs/%s'%(url,dbname,dataid))
    logger.debug('get_data_for_docid:'\
                +'\n  r.status_code: %s\n  r.headers: %s\n  r.encoding: %s\n  r.content: %s...\n' %
                 (str(r.status_code),  str(r.headers),  str(r.encoding),  str(r.content[:50])))
    return r.content

#------------------------------

def get_data_for_docid(dbname, colname, docid, url=cc.URL) :
    """Returns data from GridFS using docid.
    """
    doc = get_doc_for_docid(dbname, colname, docid, url)
    logger.debug('get_data_for_docid: %s' % str(doc))
    return get_data_for_doc(dbname, colname, doc, url)

#------------------------------

# curl -s "https://pswww.slac.stanford.edu/calib_ws/cdb_cxic0415/cspad_0001/gridfs/5b6893e81ead141643fe4344"
def get_data_for_doc(dbname, colname, doc, url=cc.URL) :
    """Returns data from GridFS using doc.
    """
    logger.debug('get_data_for_doc: %s', str(doc))
    idd = doc.get('id_data', None)
    if idd is None :
        logger.debug("get_data_for_doc: key 'id_data' is missing in selected document...")
        return None

    r2 = request('%s/%s/gridfs/%s'%(url,dbname,idd))
    s = r2.content

    return object_from_data_string(s, doc)

#------------------------------

def calib_constants(det, exp=None, ctype='pedestals', run=None, time_sec=None, vers=None, url=cc.URL) :
    """Returns calibration constants and document with metadata for specified parameters.
       To get meaningful constants, at least a few parameters must be specified, e.g.:
       - det, ctype, time_sec
       - det, ctype, version
       - det, exp, ctype, run
       - det, exp, ctype, time_sec
       - det, exp, ctype, run, version
       etc...
    """
    db_det, db_exp, colname, query = dbnames_collection_query(det, exp, ctype, run, time_sec, vers)
    logger.debug('get_constants: %s %s %s %s' % (db_det, db_exp, colname, str(query)))
    dbname = db_det if exp is None else db_exp
    doc = find_doc(dbname, colname, query, url)
    if doc is None :
        logger.warning('document is not available for query: %s' % str(query))
        return (None, None)
    return (get_data_for_doc(dbname, colname, doc, url), doc)

#------------------------------
#---------  TESTS  ------------
#------------------------------

if __name__ == "__main__" :

  def test_database_names() :
    print('test_database_names:', database_names())

#------------------------------

  def test_collection_names() :
    print('test_collection_names:', collection_names('cdb_cspad_0001'))

#------------------------------

  def test_find_docs() :
    docs = find_docs('cdb_cspad_0001', 'cspad_0001')
    print('find_docs: number of docs found: %d' % len(docs))
    print('test_find_docs returns:', type(docs))
    for i,d in enumerate(docs) :
        print('%04d %12s %10s run:%04d time_sec:%10s %s' % (i, d['ctype'], d['experiment'], d['run'], str(d['time_sec']), d['detector']))

    if len(docs)==0 : return
    doc0 = docs[0]
    print('doc0 type:', type(doc0))
    print('doc0:', doc0)
    print('doc0.keys():', doc0.keys())

#------------------------------

  def test_get_random_doc_and_data_ids(det='cspad_0001') :
    from psana.pscalib.calib.MDBUtils import db_prefixed_name
    dbname = db_prefixed_name(det)
    colname = det
    doc = find_doc(dbname, colname, query={'ctype':'pedestals'})
    print('Pick up any doc for dbname:%s colname:%s pedestals: ' % (dbname,colname))
    print('Document: %s' % str(doc))
    id_doc  = doc.get('_id', None)
    id_data = doc.get('id_data', None)
    print('_id : %s   id_data : %s' % (id_doc, id_data))
    return id_doc, id_data, dbname, colname

#------------------------------

  def test_find_doc() :
    #doc = find_doc('cdb_cxic0415', 'cspad_0001', query={'ctype':'pedestals', 'run':{'$lte':40}})
    #print('====> test_find_doc for run: %s' % str(doc))

    #doc = find_doc('cdb_cxid9114', 'cspad_0001', query={'ctype':'pedestals', 'time_sec':{'$lte':1402851400}})
    #print('====> test_find_doc for time_sec: %s' % str(doc))

    _,_,_,_ = test_get_random_doc_and_data_ids(det='cspad_0001')
    _,_,_,_ = test_get_random_doc_and_data_ids(det='cspad_0002')

#------------------------------

  def test_get_data_for_id() :
    id_doc, id_data, dbname, colname = test_get_random_doc_and_data_ids(det='cspad_0001')
    o = get_data_for_id(dbname, id_data)
    print('test_get_data_for_id: r.content raw data: %s ...' % str(o[:500]))

#------------------------------

  def test_get_data_for_docid() :
    id_doc, id_data, dbname, colname = test_get_random_doc_and_data_ids(det='cspad_0001')
    o = get_data_for_docid(dbname, colname, id_doc)
    #o = get_data_for_docid('cdb_cxid9114', 'cspad_0001', '5b6cdde71ead144f115319be')
    print_ndarr(o, 'test_get_data_for_docid o:', first=0, last=10)

#------------------------------

  def test_dbnames_collection_query() :
    det='cspad_0001'
    db_det, db_exp, colname, query = dbnames_collection_query(det, exp=None, ctype='pedestals', run=50, time_sec=None, vers=None)
    print('test_dbnames_collection_query:', db_det, db_exp, colname, query)

#------------------------------

  def test_calib_constants() :
    det = 'cspad_0001'
    data, doc = calib_constants('cspad_0001', exp='cxic0415', ctype='pedestals', run=50, time_sec=None, vers=None) #, url=cc.URL)
    print_ndarr(data, '==== test_calib_constants', first=0, last=5)
    print('==== doc: %s' % str(doc))

#------------------------------

  def test_calib_constants_text() :
    det = 'cspad_0001'
    data, doc = calib_constants(det, exp='cxic0415', ctype='geometry', run=50, time_sec=None, vers=None) #, url=cc.URL)
    print('==== test_calib_constants_text data:', data)
    print('==== doc: %s' % str(doc))

#------------------------------

  def test_calib_constants_dict() :
    det = 'opal1000_0059'
    #data, doc = calib_constants(det, exp='amox23616', ctype='lasingoffreference', run=60, time_sec=None, vers=None)
    data, doc = calib_constants(det, exp=None, ctype='lasingoffreference', run=60, time_sec=None, vers=None)
    print('==== test_calib_constants_dict data:', data)
    print('XXXX ==== type(data)', type(data))
    print('XXXX ==== type(doc) ', type(doc))
    print('==== doc: %s' % doc)

#------------------------------

if __name__ == "__main__" :
  def usage() :
      return 'Use command: python %s <test-number>, where <test-number> = 0,1,2,...,9' % sys.argv[0]\
           + '\n  0: test_database_names'\
           + '\n  1: test_collection_names'\
           + '\n  2: test_find_docs'\
           + '\n  3: test_find_doc'\
           + '\n  4: test_get_data_for_id'\
           + '\n  5: test_get_data_for_docid'\
           + '\n  6: test_dbnames_collection_query'\
           + '\n  7: test_calib_constants'\
           + '\n  8: test_calib_constants_text'\
           + '\n  9: test_calib_constants_dict'

#------------------------------

if __name__ == "__main__" :
    import os
    import sys
    from psana.pyalgos.generic.NDArrUtils import print_ndarr # info_ndarr, print_ndarr
    global print_ndarr
    logging.basicConfig(format='%(message)s', level=logging.DEBUG) # logging.INFO

    logger.info('\n%s\n' % usage())
    tname = sys.argv[1] if len(sys.argv) > 1 else '0'
    logger.info('%s\nTest %s:' % (50*'_',tname))
    if   tname == '0' : test_database_names();
    elif tname == '1' : test_collection_names();
    elif tname == '2' : test_find_docs();
    elif tname == '3' : test_find_doc();
    elif tname == '4' : test_get_data_for_id();
    elif tname == '5' : test_get_data_for_docid();
    elif tname == '6' : test_dbnames_collection_query();
    elif tname == '7' : test_calib_constants();
    elif tname == '8' : test_calib_constants_text();
    elif tname == '9' : test_calib_constants_dict();
    else : logger.info('Not-recognized test name: %s' % tname)
    sys.exit('End of test %s' % tname)

#------------------------------

reason: from numpy


text:
from sklearn.linear_model import Ridge
import numpy as np
from sklearn.linear_model import SGDRegressor
from sklearn.linear_model import Lasso
from sklearn.linear_model import ElasticNet
from sklearn import datasets
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt

# Create Data with a randomized function
m = 100
x = 6 * np.random.rand(m, 1) - 3
y = 0.5 * x**2 + x + 2 + np.random.randn(m, 1)
plt.scatter(x, y)

# Ridge Regression (L2 Regularization)
ridge_reg = Ridge(alpha=2, solver='auto', max_iter=1000, tol=0.001)
ridge_reg.fit(x, y)
x_ridge = [[-2]]
y_ridge = ridge_reg.predict(x_ridge)
print(f"Ridge Regression Y value: {y_ridge}")
plt.scatter(x_ridge, y_ridge, c='orange', s=100, label="Ridge Regression")

# Stochastic Gradient Descent with l2 penalty (alternative to above)
sgd_reg = SGDRegressor(penalty='l2', max_iter=1000, tol=0.001)
sgd_reg.fit(x, y.ravel())
x_sgd = [[-1]]
y_sgd = sgd_reg.predict(x_sgd)
print(f"SGD Ridge Y value: {y_sgd}")
plt.scatter(x_sgd, y_sgd, c="red", s=100, label="SGD_l2")

# Lasso Regression (L1 Regularization)
lasso_reg = Lasso(alpha=0.1)
lasso_reg.fit(x, y)
x_lasso = [[0]]
y_lasso = lasso_reg.predict(x_lasso)
print(f"Lasso Y value: {y_lasso}")
plt.scatter(x_lasso, y_lasso, c="green", s=100, label="Lasso Regression")

# Stochastic Gradient Descent with l1 penalty (alternative to above)
sgd_lasso = SGDRegressor(penalty='l1', max_iter=100, tol=0.001)
sgd_lasso.fit(x, y.ravel())
x_sgd_lasso = [[1.0]]
y_sgd_lasso = sgd_lasso.predict(x_sgd_lasso)
print(f"SGD Lasso Y value: {y_sgd_lasso}")
plt.scatter(x_sgd_lasso, y_sgd_lasso, c="magenta", s=100, label="SGD_l1")

# Elastic Net - uses a mix of l1 and l2 regularization
elastic_reg = ElasticNet(alpha=0.1, l1_ratio=0.5)
elastic_reg.fit(x, y)
x_elastic = [[2.0]]
y_elastic = elastic_reg.predict(x_elastic)
print(f"Elastic Net Y value: {y_elastic}")
plt.scatter(x_elastic, y_elastic, c='black', s=100, label="Elastic Net")
plt.legend()
plt.show()

# Iris Dataset with Regularized Logistic Regression
# Load and plot the data
iris = datasets.load_iris()
x = iris['data'][:, (2, 3)]  # Petal length and width
y = iris['target']
plt.scatter(x[:, 0], x[:, 1], cmap='jet', c=y)
plt.xlabel("Petal Length")
plt.ylabel("Petal Width")

# Create Logistic Regression model with L2 regularization
log_reg = LogisticRegression(multi_class='multinomial', solver='lbfgs', C=10)
# The lbfgs solver is an optimization algorithm that can handle L2 regularization
log_reg.fit(x, y)
x_log_test = [[2, 0.6], [4.5, 1], [5, 1.7]]

# Make Predictions and Plot
pred_num = 1
for i in x_log_test:
    y_log = log_reg.predict([i])
    y_log_prob = log_reg.predict_proba([i])
    print(f"Class Probabilities for Prediction {pred_num}: {y_log_prob[0]}")
    plt.scatter(i[0], i[1], s=100, label=f"Predicted Class: {y_log[0]}")
    pred_num += 1
plt.legend()
plt.show()

max_stars_repo_name: KevinKronk/regularization

max_issues_repo_name: KevinKronk/regularization

max_forks_repo_name: KevinKronk/regularization


text:
import numpy as np

class SigmoidNeuron():
    def __init__(self, n):
        np.random.seed(123)
        self.synaptic_weights = 2 * np.random.random((n, 1)) - 1

    def __sigmoid(self, x):
        return 1 / (1 + np.exp(-x))

    def __sigmoid_derivative(self, x):
        return x * (1 - x)

    def train(self, training_inputs, training_output, iterations):
        for iteration in range(iterations):
            output = self.predict(training_inputs)
            error = training_output.reshape((len(training_inputs), 1)) - output
            adjustment = np.dot(training_inputs.T, error *
                                self.__sigmoid_derivative(output))
            self.synaptic_weights += adjustment

    def predict(self, inputs):
        return self.__sigmoid(np.dot(inputs, self.synaptic_weights))

if __name__ == '__main__':
    # Initialize Sigmoid Neuron:
    sigmoid = SigmoidNeuron(2)
    print("Inicialización de pesos aleatorios:")
    print(sigmoid.synaptic_weights)

    # Datos de entrenamiento:
    training_inputs = np.array([[1, 0], [0, 0], [0, 1]])
    training_output = np.array([1, 0, 1]).T.reshape((3, 1))

    # Entrenamos la neurona (100,000 iteraciones):
    sigmoid.train(training_inputs, training_output, 100000)
    print("Nuevos pesos sinápticos luego del entrenamiento: ")
    print(sigmoid.synaptic_weights)

    # Predecimos para probar la red:
    print("Predicción para [1, 1]: ")
    print(sigmoid.predict(np.array([1, 1])))

max_stars_repo_name: PythonDayMX/MLaaS


text:
import astropy.units as u
import numpy as np

class SimulatedEventsInfo:
    """
    Information about all simulated events,
    needed for calculating event weights.

    Attributes
    ----------

    n_showers: int
        Total number of simulated showers. If reuse was used, this
        should already include the reuse.
    energy_min: u.Quantity[energy]
        Lower limit of the simulated energy range
    energy_max: u.Quantity[energy]
        Upper limit of the simulated energy range
    max_impact: u.Quantity[length]
        Maximum simulated impact parameter
    spectral_index: float
        Spectral Index of the simulated power law with sign included.
    """

    __slots__ = (
        "n_showers",
        "energy_min",
        "energy_max",
        "max_impact",
        "spectral_index",
        "viewcone",
    )

    @u.quantity_input(
        energy_min=u.TeV, energy_max=u.TeV, max_impact=u.m, viewcone=u.deg
    )
    def __init__(
        self, n_showers, energy_min, energy_max, max_impact, spectral_index, viewcone
    ):
        self.n_showers = n_showers
        self.energy_min = energy_min
        self.energy_max = energy_max
        self.max_impact = max_impact
        self.spectral_index = spectral_index
        self.viewcone = viewcone

        if spectral_index > -1:
            raise ValueError("spectral index must be <= -1")

    @u.quantity_input(energy_bins=u.TeV)
    def calculate_n_showers_per_energy(self, energy_bins):
        """
        Calculate number of showers that were simulated in the given energy intervals

        This assumes the events were generated and from a powerlaw
        like CORSIKA simulates events.

        Parameters
        ----------
        energy_bins: ``~astropy.units.Quantity``[energy]
            The interval edges for which to calculate the number of simulated showers

        Returns
        -------
        n_showers: ``~numpy.ndarray``
            The expected number of events inside each of the ``energy_bins``.
            This is a floating point number.
            The actual numbers will follow a poissionian distribution around this
            expected value.
        """
        bins = energy_bins
        e_low = bins[:-1]
        e_high = bins[1:]
        e_min = self.energy_min
        e_max = self.energy_max

        integral = _powerlaw_pdf_integral(
            self.spectral_index, e_low, e_high, e_min, e_max
        )

        integral[e_high <= e_min] = 0
        integral[e_low >= e_max] = 0

        mask = (e_high > e_max) & (e_low < e_max)
        integral[mask] = _powerlaw_pdf_integral(
            self.spectral_index, e_low[mask], e_max, e_min, e_max
        )

        mask = (e_high > e_min) & (e_low < e_min)
        integral[mask] = _powerlaw_pdf_integral(
            self.spectral_index, e_min, e_high[mask], e_min, e_max
        )

        return self.n_showers * integral

    def calculate_n_showers_per_fov(self, fov_bins):
        """
        Calculate number of showers that were simulated in the given fov bins.

        This assumes the events were generated uniformly distributed per solid angle,
        like CORSIKA simulates events with the VIEWCONE option.

        Parameters
        ----------
        fov_bins: astropy.units.Quantity[angle]
            The FOV bin edges for which to calculate the number of simulated showers

        Returns
        -------
        n_showers: numpy.ndarray(ndim=2)
            The expected number of events inside each of the ``fov_bins``.
            This is a floating point number.
            The actual numbers will follow a poissionian distribution around this
            expected value.
        """
        fov_bins = fov_bins
        fov_low = fov_bins[:-1]
        fov_high = fov_bins[1:]

        fov_integral = _viewcone_pdf_integral(self.viewcone, fov_low, fov_high)
        viewcone = self.viewcone
        # check if any of the bins are outside the max viewcone
        fov_integral = np.where(fov_high <= viewcone, fov_integral, 0)

        # identify the bin with the maximum viewcone inside
        mask = (viewcone > fov_low) & (viewcone < fov_high)
        fov_integral[mask] = _viewcone_pdf_integral(viewcone, fov_low[mask], viewcone)

        return self.n_showers * fov_integral

    @u.quantity_input(energy_bins=u.TeV, fov_bins=u.deg)
    def calculate_n_showers_per_energy_and_fov(self, energy_bins, fov_bins):
        """
        Calculate number of showers that were simulated in the given
        energy and fov bins.

        This assumes the events were generated uniformly distributed per solid angle,
        and from a powerlaw in energy like CORSIKA simulates events.

        Parameters
        ----------
        energy_bins: astropy.units.Quantity[energy]
            The energy bin edges for which to calculate the number of simulated showers
        fov_bins: astropy.units.Quantity[angle]
            The FOV bin edges for which to calculate the number of simulated showers

        Returns
        -------
        n_showers: numpy.ndarray(ndim=2)
            The expected number of events inside each of the
            ``energy_bins`` and ``fov_bins``.
            Dimension (n_energy_bins, n_fov_bins)
            This is a floating point number.
            The actual numbers will follow a poissionian distribution around this
            expected value.
        """
        # energy distribution and fov distribution are independent in CORSIKA,
        # so just multiply both distributions.
        e_integral = self.calculate_n_showers_per_energy(energy_bins)
        fov_integral = self.calculate_n_showers_per_fov(fov_bins)
        return e_integral[:, np.newaxis] * fov_integral / self.n_showers

    def __repr__(self):
        return (
            f"{self.__class__.__name__}("
            f"n_showers={self.n_showers}, "
            f"energy_min={self.energy_min:.3f}, "
            f"energy_max={self.energy_max:.2f}, "
            f"spectral_index={self.spectral_index:.1f}, "
            f"max_impact={self.max_impact:.2f}, "
            f"viewcone={self.viewcone}"
            ")"
        )

def _powerlaw_pdf_integral(index, e_low, e_high, e_min, e_max):
    # strip units, make sure all in the same unit
    e_low = e_low.to_value(u.TeV)
    e_high = e_high.to_value(u.TeV)
    e_min = e_min.to_value(u.TeV)
    e_max = e_max.to_value(u.TeV)

    int_index = index + 1
    normalization = 1 / (e_max ** int_index - e_min ** int_index)
    e_term = e_high ** int_index - e_low ** int_index
    return e_term * normalization

def _viewcone_pdf_integral(viewcone, fov_low, fov_high):
    if viewcone.value == 0:
        raise ValueError("Only supported for diffuse simulations")
    else:
        norm = 1 / (1 - np.cos(viewcone))

    integral = np.cos(fov_low) - np.cos(fov_high)

    return norm * integral

max_stars_repo_name: jsitarek/pyirf

max_issues_repo_name: jsitarek/pyirf

max_forks_repo_name: jsitarek/pyirf

reason: import numpy,import astropy