text:
"""
Simple feed-forward network for MNIST digit recognition
From https://github.com/Vict0rSch/deep_learning/blob/master/keras/feedforward/feedforward_keras_mnist.py
Also https://github.com/wxs/keras-mnist-tutorial/blob/master/MNIST%20in%20Keras.ipynb

To run:
import ex_mnist
data = ex_mnist.load_data() # Do this explicitly so we can use other data
model = ex_mnist.init_model()
(model, loss) = ex_mnist.run_network(data, model)
ex_mnist.plot_losses('loss.png', loss)
"""

import time
import numpy as np
import matplotlib
matplotlib.use('Agg')

from matplotlib import pyplot as plt
from keras.utils import np_utils
import keras.callbacks as cb
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import RMSprop
from keras.datasets import mnist

class LossHistory(cb.Callback):
    def on_train_begin(self, logs={}):
        self.losses = []

    def on_batch_end(self, batch, logs={}):
        batch_loss = logs.get('loss')
        self.losses.append(batch_loss)

def load_data():
    print 'Loading data...'
    (X_train, y_train), (X_test, y_test) = mnist.load_data()

    X_train = X_train.astype('float32')
    X_test = X_test.astype('float32')

    X_train /= 255
    X_test /= 255

    y_train = np_utils.to_categorical(y_train, 10)
    y_test = np_utils.to_categorical(y_test, 10)

    X_train = np.reshape(X_train, (60000, 784))
    X_test = np.reshape(X_test, (10000, 784))

    print 'Data loaded.'
    return [X_train, X_test, y_train, y_test]

def init_model():
    start_time = time.time()
    print 'Compiling Model ... '
    model = Sequential()
    model.add(Dense(500, input_dim=784))
    model.add(Activation('relu'))
    model.add(Dropout(0.4))
    model.add(Dense(300))
    model.add(Activation('relu'))
    model.add(Dropout(0.4))
    model.add(Dense(10))
    model.add(Activation('softmax'))

    rms = RMSprop()
    model.compile(loss='categorical_crossentropy', optimizer=rms,
      metrics=['accuracy'])
    print 'Model compiled in {0} seconds'.format(time.time() - start_time)
    return model

def run_network(data=None, model=None, epochs=20, batch=256):
    try:
        start_time = time.time()
        if data is None:
            X_train, X_test, y_train, y_test = load_data()
        else:
            X_train, X_test, y_train, y_test = data

        if model is None:
            model = init_model()

        history = LossHistory()

        print 'Training model...'
        model.fit(X_train, y_train, nb_epoch=epochs, batch_size=batch,
                  callbacks=[history],
                  validation_data=(X_test, y_test), verbose=2)

        print "Training duration : {0}".format(time.time() - start_time)
        score = model.evaluate(X_test, y_test, batch_size=16)

        print "Network's test score [loss, accuracy]: {0}".format(score)
        return model, history.losses
    except KeyboardInterrupt:
        print ' KeyboardInterrupt'
        return model, history.losses

def predict(model, images):
  """
  Takes an array of images. Obviously dimensions must match training set.
  """
  return model.predict_classes(images)

def display_classes(png, images, classes, ncol=4):
  """
  Draw a number of images and their predictions

  Example:
  images = data[1][:12]
  classes = model.predict_classes('classes.png', images)
  """
  fig = plt.figure()
  nrow = len(images) / ncol
  if len(images) % ncol > 0: nrow = nrow + 1

  def draw(i):
    plt.subplot(nrow,ncol,i)
    plt.imshow(images[i].reshape(28,28), cmap='gray', interpolation='none')
    plt.title('Predicted: %s' % classes[i])
  [ draw(i) for i in range(0,len(images)) ]
  plt.tight_layout()
  plt.savefig(png)

def plot_losses(png, losses):
    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.plot(losses)
    ax.set_title('Loss per batch')
    plt.savefig(png)

if __name__ == '__main__':
  data = load_data() # Do this explicitly so we can use other data
  model = init_model()
  (model, loss) = run_network(data, model)
  plot_losses('ff_mnist_loss.png', loss)


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Apr 06 18:16:36 2015

@author: Ziang
"""
import numpy as np
from time import time
class StochasticVariationalInference:

    def __init__(self, alpha=0.5, max_iter=10000, tolerance=1.0e-02, debug=False, mute=True, plot=False):
        self.alpha = alpha
        self.max_iter = max_iter
        self.tolerance = tolerance
        self.debug = debug
        self.mute = mute
        self.plot = plot

    def sample_z(self):
        raise NotImplementedError

    def update(self, z, x, y, val, gradient, t):
        raise NotImplementedError

    def objective(self, z, X, Y):
        raise NotImplementedError

    def gradient(self, z, x, y):
        raise NotImplementedError

    def predict(self, X):
        raise NotImplementedError

    def predict_prob(self, X):
        raise NotImplementedError

    def fit(self, X, Y):
        raise NotImplementedError

    def infer_parameters(self, x_tr, y_tr, x_te=None, y_te=None):
        # initialize parameters
        isScore = False if x_te is None and y_te is None else True
        (N,D) = x_tr.shape # dimensions
        t = 0
        z = self.sample_z()
        index = np.random.choice(range(N), size=max(1, self.alpha*N), replace=True)
        grad = self.gradient(z, x_tr[index, :], y_tr[index])
        val = self.objective(z, x_tr[index, :], y_tr[index])
        # for debugging
        if self.debug:
            self.parameters_best = self.parameters
            if self.plot:
                from matplotlib import pyplot as plt
                plt.figure('Stochastic Variational Inference')
                plt.rc('text', usetex=True)
                xdata = [t] # steps
                vals = [val]
                x_min = t
                error_tr = np.sum(len(np.where(self.predict(x_tr) != y_tr)[0])) / float(x_tr.shape[0])
                ydata1 = [error_tr]
                error_tr_min = error_tr
                if isScore:
                    error_te = np.sum(len(np.where(self.predict(x_te) != y_te)[0])) / float(x_tr.shape[0])
                    ydata2 = [error_te]
                    error_te_min = error_te
                    plt.subplot(211)
                    plt.title(r'Training performance')
                    line_tr, = plt.plot(xdata, ydata1,'g', label=r'Training Error')
                    line_te, = plt.plot(xdata, ydata2,'b', label=r'Testing Error')
#                    lines1 = plt.plot(xdata, ydata1,'g', xdata, ydata2, 'b')
                else:
                    plt.subplot(211)
                    plt.title(r'Training performance')
#                    lines1 = plt.plot(xdata, ydata1,'g')
                    line_tr, = plt.plot(xdata, ydata1,'g', label=r'Training Error')
                plt.legend()
                plt.ylabel(r'Error')
                plt.ylim(0, 1)
                plt.grid(True)
                plt.subplot(212)
                plt.xlabel(r'Number of iterations')
                plt.ylabel(r'log($\textbf{y}$)')
#                lines2 = plt.plot(xdata, vals, 'r')
                line_val, = plt.plot(xdata, vals, 'r', label=r'Log marginal likelihood')
                plt.legend()
                plt.ylim(np.min(vals)-2, np.max(vals)+2)
                plt.grid(True)

        # perform stochastic gradient descent (max(func))
        t0 = time()
        while t < self.max_iter and np.linalg.norm(grad) > self.tolerance:
            t = t + 1
            self.update(z, x_tr[index, :], y_tr[index], val, grad, t)
            # sample z
            z = self.sample_z()
            # sample X
            index = np.random.choice(range(N), size=max(1, self.alpha*N), replace=True)
            grad = self.gradient(z, x_tr[index, :], y_tr[index])
            val = self.objective(z, x_tr[index, :], y_tr[index])
            # for debugging
            if self.debug:
                if self.plot:
                    xdata.append(t)
                    error_tr = np.sum(len(np.where(self.predict(x_tr) != y_tr)[0])) / float(x_tr.shape[0])
                    ydata1.append(error_tr)
                    if error_tr < error_tr_min and not isScore:
                        error_tr_min = error_tr
                        x_min = t
                        self.parameters_best = self.parameters
                    line_tr.set_data(xdata, ydata1)
                    if isScore:
                        error_te = np.sum(len(np.where(self.predict(x_te) != y_te)[0])) / float(x_te.shape[0])
                        ydata2.append(error_te)
                        if error_te < error_te_min:
                            error_te_min = error_te
                            x_min = t
                            self.parameters_best = self.parameters
                        line_te.set_data(xdata, ydata2)
                    plt.subplot(211)
                    plt.xlim(xmax=np.max(xdata))
                    plt.draw()
                    vals.append(val)
                    line_val.set_data(xdata, vals)
                    plt.subplot(212)
                    plt.xlim(xmax=np.max(xdata))
                    plt.ylim(np.min(vals)-2, np.max(vals)+2)
                    plt.draw()
                if not self.mute and self.plot:
                    if isScore:
                        print('Iter {:5}: \ntr_error = {} \nte_error = {}'.format(t, ydata1[-1], ydata2[-1]))
                    else:
                        print('Iter {:5}: tr_error = {}'.format(t, ydata1[-1]))
                if not self.mute and not self.plot:
                    print('Iter {:5}: val = {}'.format(t, val))
            if not self.mute:
                print('{:6d} iterations, val={:.16f}, time = {:.5f}s'.format(t, val, time()-t0))
        if self.debug:
            if self.plot:
                plt.subplot(211)

                plt.annotate('min tr error', \
                             xy=(x_min, \
                             error_tr_min), \
                             xytext=(x_min-0.3,error_tr_min-0.2),\
                             arrowprops=dict(facecolor='black', shrink=0.05))
                if x_te is not None and y_te is not None:
                    plt.annotate('min te error', \
                                 xy=(xdata[np.argmin(ydata2)], \
                                 np.min(ydata2)), \
                                 xytext=(x_min+0.3,error_tr_min-0.2),\
                                 arrowprops=dict(facecolor='black', shrink=0.05))
            self.parameters = self.parameters_best

ext: py

max_stars_repo_name: AlchemicalChest/gaussian-process-svi

max_issues_repo_name: AlchemicalChest/gaussian-process-svi

max_forks_repo_name: AlchemicalChest/gaussian-process-svi

reason: import numpy


text:
#!/usr/bin/env python

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf
import blocksparse.ewops as ew
from blocksparse.optimize import adafactor1d_op, adafactor2d_op

ones        = 0
out         = 1
beta2       = 0.5
learn_rate  = 0.5
grad_scale  = 1.0
clip_thresh = 1.0
epsilon     = 1e-30

config = tf.ConfigProto(
    intra_op_parallelism_threads=1,
    inter_op_parallelism_threads=1)

class AdafactorTest(tf.test.TestCase):

    def testAdafactor(self):

        with self.test_session(config=config) as sess, tf.device("/gpu:0"):
            for dtype in (tf.float32, tf.float16): # tf.float16
                for shape_g in (
                     (1024,1024*2),
                     (   1,1024*2),
                     (1024,1023*1),
                     (   1,1023*1),
                ):

                    shape_c = ( 1,shape_g[1])
                    shape_r = (shape_g[0], 1)

                    if ones:
                        G = np.ones( shape_g, dtype=np.float32)
                        P = np.ones( shape_g, dtype=np.float32)
                        C = np.zeros(shape_c, dtype=np.float32)
                        R = np.zeros(shape_r, dtype=np.float32)
                    else:
                        G = np.random.uniform(-1.0, 1.0, shape_g).astype(np.float16).astype(np.float32)
                        P = np.random.uniform(-1.0, 1.0, shape_g).astype(np.float16).astype(np.float32)
                        C = np.random.uniform( 0.0, 1.0, shape_c).astype(np.float16).astype(np.float32)
                        R = np.random.uniform( 0.0, 1.0, shape_r).astype(np.float16).astype(np.float32)

                    g = tf.placeholder(tf.float32, G.shape)
                    p = tf.Variable(initial_value=P, name="p")
                    c = tf.Variable(initial_value=C, name="c")
                    r = tf.Variable(initial_value=R, name="r")
                    sess.run( tf.global_variables_initializer() )

                    g = ew.float_cast(g, dtype=dtype)

                    # adafactor has it's own fused infinity filtering but quick test of this standalone op here.
                    g = ew.filter_infinity(g)

                    if shape_g[0] > 1:

                        p, c, r, x, _ = sess.run(
                            adafactor2d_op(p, c, r, g, beta2, learn_rate, grad_scale, clip_thresh, epsilon=epsilon, zero_nans=True),
                            feed_dict={ g: G } )

                        C = beta2 * C + (1.0 - beta2) * np.mean(np.square(G) + epsilon, axis=0, keepdims=True)
                        R = beta2 * R + (1.0 - beta2) * np.mean(np.square(G) + epsilon, axis=1, keepdims=True)
                        LTM = np.mean(R, keepdims=True)
                        X = G / (np.sqrt(R / LTM) * np.sqrt(C))
                        RMS_X = np.sqrt(np.mean(np.square(X), keepdims=True))

                    else:

                        r = R
                        p, c, x, _ = sess.run(
                            adafactor1d_op(p, c, g, beta2, learn_rate, grad_scale, clip_thresh, epsilon=epsilon, zero_nans=True),
                            feed_dict={ g: G } )

                        C = beta2 * C + (1.0 - beta2) * (np.square(G) + epsilon)
                        X = G / np.sqrt(C)
                        RMS_X = np.sqrt(np.mean(np.square(X), keepdims=True))

                    P -= learn_rate * X / np.maximum(1.0, RMS_X / clip_thresh)

                    print("testAdafactor", dtype)
                    for op, dev, cpu in [
                        [ "C", c, C ],
                        [ "R", r, R ],
                        [ "X", x, X ],
                        [ "P", p, P ],
                    ]:
                        self.compare_results(op, dev, cpu)

    def compare_results(self, op, dev, cpu):

        dif     = np.abs(cpu - dev)
        avgval  = np.average(abs(cpu))
        maxdif  = dif.max()
        max_err = maxdif if avgval == 0 else maxdif / avgval
        l2_err  = np.sqrt(np.square(dif).sum()) / np.sqrt(np.square(cpu).sum())

        print("op:%3s, err:%17.12f, l2_err:%17.12f shape:%14s" % (op, maxdif, l2_err, str(cpu.shape)))

        if out:
            np.savetxt("%s_dif.txt"%op, dif, fmt='%6.3f')
            np.savetxt("%s_cpu.txt"%op, cpu, fmt='%6.3f')
            np.savetxt("%s_gpu.txt"%op, dev, fmt='%6.3f')

if __name__ == "__main__":
  tf.test.main()

# a = np.zeros((32,32), dtype=np.bool)
# for y, x in np.ndindex(a.shape):
#     if x <= y: a[y,x] = True
# b = np.packbits(a.reshape(-1,8)[:,::-1]).view(np.uint32)

# np.unpackbits(b.view(np.uint8))

# b = np.packbits(a.reshape(-1,8)[:,::-1])

max_stars_repo_name: ThomasHagebols/blocksparse

max_issues_repo_name: ThomasHagebols/blocksparse

max_forks_repo_name: ThomasHagebols/blocksparse


text:
import numpy as np
import joblib

from tqdm import tqdm
from sklearn.tree import DecisionTreeClassifier

from sklearn.ensemble import AdaBoostClassifier

import ethologger.utils.auxiliary as aux

from ethologger.experiments.profile import ActiveFrames
from ethologger.experiments.profile import DormantIntervals
from ethologger.utils.visualization import VisualizationWrapper as VizWrap
from ethologger.project.helper import ProjectIO, Logger

class FrameExtraction(ProjectIO):
    def __init__(
        self,
        fps,
        project_path,
        expt_path_dict,
        annotation_path_dict,
        moving_threshold_args,
        dormant_intervals_args,
        activity_datums_list,
        activity_threshold_args,
        decision_tree_args=None,
    ):
        self.fps = fps
        self.project_path = project_path
        self.expt_path_dict = expt_path_dict
        self.annotation_path_dict = annotation_path_dict
        self.moving_threshold_args = moving_threshold_args
        self.dormant_intervals_args = dormant_intervals_args
        self.activity_datums_list = activity_datums_list
        self.activity_threshold_args = activity_threshold_args
        self.decision_tree_args = decision_tree_args
        if self.decision_tree_args is None:
            self.decision_tree_args = {}
        self.logger = Logger()
        ProjectIO.__init__(self)

    @aux.timeit
    def extract_dormant_intervals(self):
        ExtDormant = DormantIntervals(fps=self.fps)

        pbar = tqdm(self.expt_path_dict.items())
        for name, expt_path in pbar:
            pbar.set_description(f"Extracting dormant intervals of {name}...")

            expt_item = self._load_expt_item(expt_path)
            dfDelta = self._load_delta(expt_path)
            delta_names_dict = self._load_delta_names_dict(expt_path)

            ExtDormant.delta_names_dict = delta_names_dict
            frame_val = ExtDormant.get_movement_values(
                dfDelta, datums=list(delta_names_dict.values()), value_type="delta"
            )
            threshold = ExtDormant.get_threshold(frame_val, self.moving_threshold_args)

            _ = ExtDormant.set_bouts(frame_val, threshold)
            dormant_moving_labels = ExtDormant.get_labels(**self.dormant_intervals_args)

            expt_item.mask_dormant = dormant_moving_labels == 0
            self.logger.info_saved("Mask of dormant frames")

            viz = VizWrap(tick_stepsize=1.5)
            fig, _ = viz.ethogram(
                dormant_moving_labels,
                ylabel="state",
                title=f"Dormant-Moving Ethogram of {name}",
                label_name_dict=ExtDormant.id_to_state,
            )
            dormant_moving_figpath = (
                expt_path / "figures" / "dormant_moving_ethogram.png"
            )
            fig.savefig(dormant_moving_figpath, bbox_inches="tight")
            self.logger.file_saved(
                "Dormant-Moving ethogram", dormant_moving_figpath, sd=3
            )

            dormant_intervals = []
            intervals = aux.cont_intvls(dormant_moving_labels)
            for i in range(1, intervals.shape[0]):
                if dormant_moving_labels[intervals[i - 1]] == 0:
                    dormant_intervals.append((intervals[i - 1], intervals[i]))

            expt_item.dormant_intervals = dormant_intervals
            self.logger.info_saved("Dormant intervals")

            self._save_expt_item(expt_item, expt_path)

    @staticmethod
    def _get_threshold_labels(mask_list, shape):
        labels = np.zeros(shape, dtype=int)
        for i, mask in enumerate(mask_list):
            labels[mask] = i + 1
        return labels

    def _get_thresholds(self, ActiveFrames_instance, dfCoef, mask_dormant, datums_list):
        thresholds = []
        for datums in datums_list:
            frame_val = ActiveFrames_instance.get_activity_values(
                dfCoef, datums=datums, value_type="wavelet-coef"
            )
            th = ActiveFrames_instance.get_threshold(
                frame_val[mask_dormant], self.activity_threshold_args
            )
            thresholds.append(th)
        return thresholds

    @aux.timeit
    def extract_active_frames(self):
        ExtActive = ActiveFrames(fps=self.fps)

        pbar = tqdm(self.expt_path_dict.items())
        for name, expt_path in pbar:
            pbar.set_description(f"Extracting active frames of {name}...")

            expt_item = self._load_expt_item(expt_path)
            mask_dormant = expt_item.mask_dormant

            ftws = self._load_ws(expt_path)
            snap_names_dict = self._load_snap_names_dict(expt_path)

            ExtActive.snap_names_dict = snap_names_dict
            dfCoef = ExtActive.get_dfCoef(ftws, log=True)
            datums_list = list(self.activity_datums_list)

            thresholds = self._get_thresholds(
                ExtActive, dfCoef, mask_dormant, datums_list
            )
            _, mask_active_p = ExtActive.get_mask_active(
                dfCoef, datums_list, thresholds, value_type="wavelet-coef"
            )

            threshold_labels = self.__class__._get_threshold_labels(
                mask_active_p, ftws.shape[0]
            )
            mask_active = threshold_labels != 0

            activity_labels = np.where(mask_active, 1, 0).astype(int)
            self.logger.number_of_dormant_active(
                name, sum(activity_labels[mask_dormant])
            )

            expt_item.threshold_labels = threshold_labels
            self.logger.info_saved("Threshold labels")

            expt_item.mask_active = mask_active
            self.logger.info_saved("Mask of active frames")

            viz = VizWrap()
            fig, _ = viz.ethogram(
                threshold_labels,
                title=f"Threshold Labels of {name}",
            )
            threshold_lbl_figpath = expt_path / "figures" / "threshold_labels.png"
            fig.savefig(threshold_lbl_figpath, bbox_inches="tight")
            self.logger.file_saved(
                "Broken-hbar plot of threshold labels", threshold_lbl_figpath, sd=3
            )

            fig, _ = viz.ethogram(
                activity_labels,
                title=f"Activity Labels of {name}",
            )
            activity_lbl_figpath = expt_path / "figures" / "activity_labels.png"
            fig.savefig(activity_lbl_figpath, bbox_inches="tight")
            self.logger.file_saved(
                "Broken-hbar plot of activity labels",
                activity_lbl_figpath,
                sd=3,
            )

            self._save_expt_item(expt_item, expt_path)

    @aux.timeit
    def construct_activity_decision_tree(self):
        ExtActive = ActiveFrames(fps=self.fps)

        X_activity_list = []
        y_activity_list = []

        annotation_path_items = self.annotation_path_dict.items()
        if not annotation_path_items:
            raise RuntimeError(
                "At least one annotated experiment must be provided.\n"
                "Activity decision tree can not be constructed.\n"
                "Please use 'extract_active_frames' directly to extract active frames."
            )

        pbar = tqdm(annotation_path_items)
        for name, _ in pbar:
            pbar.set_description(f"Processing {name}...")

            expt_path = self.expt_path_dict[name]

            annotations = self._load_annotations(expt_path)
            expt_item = self._load_expt_item(expt_path)
            ftws = self._load_ws(expt_path)
            snap_names_dict = self._load_snap_names_dict(expt_path)

            mask_dormant = expt_item.mask_dormant

            ExtActive.snap_names_dict = snap_names_dict
            dfCoef = ExtActive.get_dfCoef(ftws, log=True)
            datums_list = list(self.activity_datums_list)

            thresholds = self._get_thresholds(
                ExtActive, dfCoef, mask_dormant, datums_list
            )

            _, mask_active_p = ExtActive.get_mask_active(
                dfCoef, datums_list, thresholds, value_type="wavelet-coef"
            )

            X_activity_list.append(dfCoef.to_numpy()[mask_dormant, :])

            threshold_labels = self.__class__._get_threshold_labels(
                mask_active_p, ftws.shape[0]
            )

            expt_item.threshold_labels = threshold_labels
            self.logger.info_saved("Threshold labels")

            # Alternative way.
            mask_active = threshold_labels != 0
            mask_active[annotations == 7] = False
            # Sensible way of constructing ground-truth.
            # Backup-way to go.
            ## mask_active = annotations != 7
            ## mask_active[threshold_labels == 0] = False
            ## mask_active[annotations == 2] = True
            # Alternative, "more" unsupervised
            ## mask_active = threshold_labels != 0

            y_temp = np.where(mask_active, 1, 0).astype(int)
            y_activity_list.append(y_temp[mask_dormant])

            # Might redundant, can be removed.
            expt_item.gt_activity_labels = y_temp
            self.logger.info_saved("Ground-truth activity labels")

            viz = VizWrap()
            fig, _ = viz.ethogram(
                threshold_labels,
                title=f"Threshold Labels of {name}",
            )
            threshold_lbl_figpath = expt_path / "figures" / "threshold_labels.png"
            fig.savefig(threshold_lbl_figpath, bbox_inches="tight")
            self.logger.file_saved(
                "Broken-hbar plot of threshold labels", threshold_lbl_figpath, sd=3
            )

            fig, _ = viz.ethogram(
                y_temp,
                title=f"Ground-truth Activity Labels of {name}",
            )
            gt_activity_lbl_figpath = expt_path / "figures" / "gt_activity_labels.png"
            fig.savefig(gt_activity_lbl_figpath, bbox_inches="tight")
            self.logger.file_saved(
                "Broken-hbar plot of ground-truth activity labels",
                gt_activity_lbl_figpath,
                sd=3,
            )

            self._save_expt_item(expt_item, expt_path)

        X = np.concatenate(X_activity_list)
        y = np.concatenate(y_activity_list)
        decision_tree = DecisionTreeClassifier(**self.decision_tree_args)
        clf = AdaBoostClassifier(base_estimator=decision_tree, n_estimators=20)
        ## clf = decision_tree
        clf.fit(X=X, y=y)
        decision_tree_path = self.project_path / "activity_decision_tree.z"
        joblib.dump(clf, decision_tree_path)
        self.logger.file_saved("Activity decision tree", decision_tree_path, sd=3)

    @aux.timeit
    def classify_active_frames(self):
        ExtActive = ActiveFrames(fps=self.fps)

        clf = self._load_activity_decision_tree(self.project_path)

        pbar = tqdm(self.expt_path_dict.items())
        for name, expt_path in pbar:
            pbar.set_description(f"Classifying active frames of {name}...")

            expt_item = self._load_expt_item(expt_path)
            mask_dormant = expt_item.mask_dormant

            ftws = self._load_ws(expt_path)
            snap_names_dict = self._load_snap_names_dict(expt_path)

            ExtActive.snap_names_dict = snap_names_dict

            dfCoef = ExtActive.get_dfCoef(ftws, log=True)
            X_expt = dfCoef.to_numpy()

            y_pred = clf.predict(X_expt)
            self.logger.number_of_dormant_active(name, sum(y_pred[mask_dormant]))

            expt_item.mask_active = y_pred != 0
            self.logger.info_saved("Mask of active frames")
            self._save_expt_item(expt_item, expt_path)

            viz = VizWrap()
            fig, _ = viz.ethogram(
                y_pred,
                title=f"Activiy Labels of {name}",
            )
            activity_lbl_figpath = expt_path / "figures" / "activity_labels.png"
            fig.savefig(activity_lbl_figpath, bbox_inches="tight")
            self.logger.file_saved(
                "Broken-hbar plot of activity labels", activity_lbl_figpath, sd=3
            )

            if expt_item.hasAnn:
                y_true = expt_item.gt_activity_labels
                self.logger.dormant_active_training_fscore(
                    name, y_true[mask_dormant], y_pred[mask_dormant]
                )


text:
# Python modules

# 3rd party modules
import wx
import numpy as np

# Our modules
import vespa.common.wx_gravy.plot_panel_spectrum as plot_panel_spectrum

class PlotPanelUserPrior(plot_panel_spectrum.PlotPanelSpectrum):

    def __init__(self, parent, **kwargs):
        '''
        This is a customization of the PlotPanel object for use in a specific
        location in our application. The basic matplotlib functionality is
        contained in the base class, all we are doing here is overwriting the
        various canvas Event functions to mesh with our application.

        parent - the widget to which the PlotPanel is directly attached

        dataset - the currently active dataset

        '''
        #tab.SizerSplitterWindow.Fit(tab)  # bugfix wxGlade 0.9.6 to 1.0.0

        super().__init__(parent, **kwargs)

        self._dataset = kwargs["dataset"]
        self._statusbar = wx.GetApp().GetTopWindow().statusbar

        # these are in points to facilitate area calculations
        self.ref_locations = 0,self._dataset.spectral_dims[0]-1

    # EVENT FUNCTIONS -----------------------------------------------

    def on_motion(self, xdata, ydata, val, bounds, iaxis):
        if self.prefs.xaxis_hertz:
            hz  = xdata
            ppm = xdata / self._dataset.frequency
        else:
            ppm = xdata
            hz  = xdata * self._dataset.frequency

        self._statusbar.SetStatusText(" PPM = %.3f" % (ppm, ), 0)
        self._statusbar.SetStatusText(" Hz = %.3f"  % (hz,  ), 1)

        # this may give wrong values in an axes that has multiple lines
        # displayed since there is no sure way to determine which line
        # the cursor is closest to
        self._statusbar.SetStatusText((" Value = "+str(val[0])), 2)

    def on_scroll(self, button, step, iaxis):
        self.set_vertical_scale(step)

    def on_zoom_motion(self, xmin, xmax, val, ymin, ymax, iplot=None):
        if self.prefs.xaxis_hertz:
            hz_str  = xmin
            hz_end  = xmax
            if hz_str > hz_end: hz_str, hz_end = hz_end, hz_str
            ppm_str = hz_str / self._dataset.frequency
            ppm_end = hz_end / self._dataset.frequency
        else:
            ppm_str  = xmin
            ppm_end  = xmax
            if ppm_str > ppm_end: ppm_str, ppm_end = ppm_end, ppm_str
            hz_str = ppm_str * self._dataset.frequency
            hz_end = ppm_end * self._dataset.frequency

        delta_ppm = -1*(ppm_str - ppm_end)  # keeps delta positive
        delta_hz  = delta_ppm * self._dataset.frequency
        self._statusbar.SetStatusText((" PPM Range = %.2f to %.2f" % (ppm_str, ppm_end)), 0)
        self._statusbar.SetStatusText((" Hz Range = %.1f to %.1f" % (hz_str, hz_end)), 1)
        self._statusbar.SetStatusText((" dPPM = %.2f  dHz = %.1f" % (delta_ppm, delta_hz)), 2)

    def on_refs_select(self, xmin, xmax, val, reset=False, iplot=None):
        # Calculate area of span
        area, rms = self.calculate_area()
        txt = ' Area = %1.5g  RMS = %1.5g' % (area[1], rms[1])  # summed values
        self._statusbar.SetStatusText(txt, 3)

    def on_refs_motion(self, xmin, xmax, val, iplot=None):
        if self.prefs.xaxis_hertz:
            hz_str  = xmin
            hz_end  = xmax
            if hz_str > hz_end: hz_str, hz_end = hz_end, hz_str
            ppm_str = hz_str / self._dataset.frequency
            ppm_end = hz_end / self._dataset.frequency
        else:
            ppm_str  = xmin
            ppm_end  = xmax
            if ppm_str > ppm_end: ppm_str, ppm_end = ppm_end, ppm_str
            hz_str = ppm_str * self._dataset.frequency
            hz_end = ppm_end * self._dataset.frequency

        delta_ppm = -1*(ppm_str - ppm_end) # keeps delta positive
        delta_hz  = delta_ppm * self._dataset.frequency
        self._statusbar.SetStatusText((" PPM Range = %.2f to %.2f" % (ppm_str, ppm_end)), 0)
        self._statusbar.SetStatusText((" Hz Range = %.1f to %.1f" % (hz_str, hz_end)), 1)
        self._statusbar.SetStatusText((" dPPM = %.2f  dHz = %.1f" % (delta_ppm, delta_hz)), 2)

        # convert ppm to points
        dataset = self._dataset
        pts_end = (dataset.spectral_dims[0] / 2) - (dataset.frequency * (ppm_str - dataset.resppm) / dataset.spectral_hpp)
        pts_end = np.where(pts_end > 0, pts_end, 0)
        pts_str = (dataset.spectral_dims[0] / 2) - (dataset.frequency * (ppm_end - dataset.resppm) / dataset.spectral_hpp)
        pts_str = np.where(pts_str > 0, pts_str, 0)

        # store reference span start/end values as PPM
        self.ref_locations = pts_str, pts_end

        area, rms = self.calculate_area()
        txt = ' Area = %1.5g  RMS = %1.5g' % (area[1], rms[1])      # summed values

        self._statusbar.SetStatusText(txt, 3)

max_stars_repo_name: vespa-mrs/vespa

max_issues_repo_name: vespa-mrs/vespa

max_forks_repo_name: vespa-mrs/vespa


text:
"""
test_Lx(z)

Author: Jacob Jost
Affiliation: University of Colorado at Boulder (Undergraduate)
Created on: Thu June 4 09:00:00 MDT 2015

NOTE: plots may differ than Ueda et al. (2014) because their plots show the De-absorbed
XLF and the redshift ranges arebinned, also my be cutting the plots off at a certain value***
"""

import ares
import numpy as np
from scipy import integrate
import matplotlib.pyplot as pl

u03 = ares.util.read_lit('ueda2003')
u14 = ares.util.read_lit('ueda2014')
z = np.linspace(2.4, 3.0, 100)
L = np.logspace(42.5, 47, 100)

"""
Compare to Ueda et al. (2003) LDDE model.
"""

fig1 = pl.figure(1); ax1 = fig1.add_subplot(111)

"""
Plot the QSO-LF at a few redshifts, show PLE, PDE, LDDE models.
"""
fig1 = pl.figure(1); ax1 = fig1.add_subplot(111)

colors = 'k', 'b', 'g'
for i, redshift in enumerate(z):
    lf_ple = u03.LuminosityFunction(L, redshift, evolution='ldde')
    lf_pde = u14.LuminosityFunction(L, redshift, evolution='ldde')

    ax1.loglog(L, lf_ple, color=colors[i], ls='--')
    ax1.loglog(L, lf_pde, color=colors[i], ls='-', label=r'$z={}$'.format(redshift))
    ax1.loglog(L, lf_ldde, color=colors[i], ls=':')

ax1.set_xlabel(r'$L_X$')
ax1.set_ylabel(r'$\phi(L_X)$')
ax1.legend(loc='best')

#---------------------------------------------------

#models = []
#r = []
#for i, redshift in enumerate(z):
#    model1 = []
#    models.append(model1)
#    r.append(redshift)
#    for p, Lx in enumerate(L):
#        model = u14.LuminosityFunction_LDDE(Lx, redshift, **u14.qsolf_LDDE2_hardpars)
#        model1.append(model)
#models = np.array(models)
#r = np.array(r)
#
#for i, j in enumerate(models):
#    pl.loglog(L, models[i], alpha = 0.25)
#    pl.title(r'The 2-10 KeV Band at $z$ ~ ${0:.1f} - {1:.1f}$'.format(r[0], r[-1]))
#    pl.ylim(10**-12, 10**-2)
#    pl.xlim(L[0], L[-23])
#
#ax1.set_xlabel(r'$L_X$')
#ax1.set_ylabel(r'$\phi(L_X)$')
#ax1.legend(loc='best')
#pl.show()
#
##---------------------------------------------------
#
#"""
#test_Lx(z)
#
#Author: Jacob Jost
#Affiliation: University of Colorado at Boulder (Undergraduate)
#Created on: Thu June 4 09:00:00 MDT 2015
#
#NOTE:
#    plots may differ than Ueda et al. (2014) because their plots show the De-absorbed
#    XLF and the redshift ranges arebinned, also my be cutting the plots off at a certain value***
#
#    HEADS UP: this takes a while to run, use caution.
#"""
#
##---------------------------------------------------
#
#u14 = ares.util.read_lit('ueda2014')
#z = np.linspace(0, 5, 100)
#models = 100
#
#
#hardpars = u14.qsolf_LDDE2_hardpars #parameters dictionary
#harderr = u14.qsolf_LDDE2_harderr #parameters error dictionary
#hardall = hardpars.copy() #copying parameters dictionary
#hardall.update(harderr) #combining parameters dictionary & parameters error dictionary
#hardsamples = u14.randomsamples(100, **hardall)
#
#fig2 = pl.figure(2); ax2 = fig2.add_subplot(111)
#
#integrand = []
#for i in range(models):
#    integrand1 = []
#    r = []
#    integrand.append(integrand1)
#    for j in range(len(z)):
#        x = lambda Lx: u14.LuminosityFunction_LDDE(Lx, z[j],\
#        **hardsamples[i])
#        p, err = integrate.quad(x, 10**41, 10**46)
#        r.append(z[j])
#        integrand1.append(p)
#
#for i, j in enumerate(integrand):
#    pl.semilogy(z, integrand[i], color = 'k', alpha = 0.25)
#
#pl.title(r'The 2-10 KeV Band at $z$ ~ ${0:.1f} - {1:.1f}$'.format(r[0], r[-1]))
#
#ax2.set_xlabel(r'$z$')
#ax2.set_ylabel(r'$\phi(L_X)$')
#ax2.legend(loc='best')
#pl.show()
#
##---------------------------------------------------
#
#"""
#test_Lx(z)
#
#Author: Jacob Jost
#Affiliation: University of Colorado at Boulder (Undergraduate)
#Created on: Thu June 4 09:00:00 MDT 2015
#
#NOTE: plots may differ than Ueda et al. (2014) because their plots show the De-absorbed
#XLF and the redshift ranges arebinned, also my be cutting the plots off at a certain value***
#"""
#
#z = np.linspace(2.4, 3.0, 100)
#L = np.logspace(42.5, 47, 100)
#fig3 = pl.figure(3); ax3 = fig3.add_subplot(111)
#
#hardpars = u14.qsolf_LDDE2_hardpars #parameters dictionary
#harderr = u14.qsolf_LDDE2_harderr #parameters error dictionary
#hardall = hardpars.copy() #copying parameters dictionary
#hardall.update(harderr) #combining parameters dictionary & parameters error dictionary
#hardsamples = u14.randomsamples(100, **hardall)
#
#results = []
#r = []
#for i, redshift in enumerate(z):
#    model1 = []
#    results.append(model1)
#    r.append(redshift)
#    for p, Lx in enumerate(L):
#        model = u14.LuminosityFunction_LDDE(Lx, redshift,\
#        **hardsamples[i])
#        model1.append(model)
#
#
#for i, j in enumerate(results):
#    pl.loglog(L, results[i], color = 'r', alpha = 0.25)
#
#pl.title(r'The 2-10 KeV Band at $z$ ~ ${0:.1f} - {1:.1f}$'.format(r[0], r[-1]))
#pl.ylim(10**-12, 10**-4)
#pl.xlim(L[0], L[-23])
#ax1.set_xlabel(r'$L_X$')
#ax1.set_ylabel(r'$\phi(L_X)$')
#ax1.legend(loc='best')
#pl.show()

max_stars_repo_name: jlashner/ares

max_issues_repo_name: jlashner/ares

max_forks_repo_name: jlashner/ares

reason: import numpy,from scipy


text:
'''Filter colors between lower and upper bounds in BGR space'''

import cv2
import numpy as np

PARAM_DEFAULT = {
        'lower': [17, 15, 100],
        'upper': [50, 56, 200]
    }

def init(cap, param):
    lower_color_bound = np.array(param['lower'], dtype='uint8')
    upper_color_bound = np.array(param['upper'], dtype='uint8')
    background_color = (255, 255, 255)
    _, frame0 = cap.read()
    background = np.zeros(frame0.shape, np.uint8)
    background[:, :] = background_color
    init_input = {
        'background': background,
        'lower_color_bound': lower_color_bound,
        'upper_color_bound': upper_color_bound
    }
    step_input = None
    return init_input, step_input

def step(i, frame, param, init_input, step_input):
    lower_color_bound = init_input['lower_color_bound']
    upper_color_bound = init_input['upper_color_bound']
    background = init_input['background']
    mask = cv2.inRange(frame, lower_color_bound, upper_color_bound)
    f_frame = cv2.bitwise_and(frame, frame, mask=mask)
    f_frame = f_frame + cv2.add(background, f_frame, mask=cv2.bitwise_not(mask))
    filtered_frame = f_frame
    step_input = None
    return filtered_frame, step_input

max_stars_repo_name: vwrobel/cvtools

max_issues_repo_name: vwrobel/cvtools

max_forks_repo_name: vwrobel/cvtools


text:
import numpy as np
import cv2
import pandas as pd
import os

from time import time
from pytesseract import image_to_string
from re import search
from glob import glob as listdir
from collections import Counter
from argparse import ArgumentParser
from bs4 import BeautifulSoup as bs
from requests import get as geturl

class ShardCounter():
    def __init__(self):
        # measured values from own data
        self.N_SCALE = 98
        self.N_BOUNDS = np.array([70, 93, 70, 95]) / self.N_SCALE

        # url for table of champions from wiki
        self.CHAMP_URL = "https://leagueoflegends.fandom.com/wiki/List_of_champions"

        # blue essence exchange rates
        self.EXCHANGE = {
            450 : 90,
            1350 : 270,
            3150 : 630,
            4800 : 960,
            6300 : 1260,
            7800 : 1560
        }

        # instead of 256 colours have only 64
        self.IM_QUANT_FACTOR = 4

        # bounds on boundary thresholding
        self.LOWER_B_THRESH = np.array([30, 22, 18])
        self.UPPER_B_THRESH = np.array([80, 72, 48])

        # limit on the area of a champion square
        self.SQ_AREA_LIMIT = 1500

        # confidence level of new champ or recognized
        self.CHAMP_CONFIDENCE = 0.79

        # one day for csv refresh time limit
        self.TIME_LIMIT = 86400

        # config for tesseract
        self.P_CONFIG = ("-l eng --oem 1 --psm 7")

        # pre-load the champion images
        self.champ_images = {champ : cv2.imread(champ) for champ in listdir("champs/*")}

        # keeps track of champ table update
        self.last_update = 0

    ## locate
    def assign_champ_to_square(self, imagelist, display=False):
        all_champs = dict()

        # iterate through images
        for base in imagelist:
            # quantize the image slightly
            smoothed = base // self.IM_QUANT_FACTOR
            smoothed = smoothed * self.IM_QUANT_FACTOR

            # threshold based on RGB values, more accurate than just BW
            rgbthresh = cv2.inRange(smoothed, self.LOWER_B_THRESH,
                self.UPPER_B_THRESH)
            rgbthresh = cv2.filter2D(rgbthresh, -1, np.ones((2, 2)))

            # find contours within the images, essentially shape finding
            contours, h = cv2.findContours(rgbthresh, cv2.RETR_EXTERNAL,
                cv2.CHAIN_APPROX_SIMPLE)

            # holds bounding boxes and frequency of areas
            boxes = dict()
            areas = list()

            #  iterate through contours
            for cnt in contours:
                # calculate the approximate shape
                arclength = 0.1 * cv2.arcLength(cnt, True)
                shape = cv2.approxPolyDP(cnt,arclength,True)

                # if the shape has four points, i.e. a square
                if(len(shape) == 4):
                    # obtain the bounding rectangle
                    (x, y, w, h) = cv2.boundingRect(shape)

                    # calculate the area
                    ar = w * h

                    # presume the area is above a limit
                    if ar > self.SQ_AREA_LIMIT:
                        # if not present in dictionary, add an entry
                        if ar not in boxes:
                            boxes[ar] = list()

                        # keep track of areas and bounding boxes
                        areas.append(ar)
                        boxes[ar].append([y, x, w, h])

            # find the most common area
            base_ar = Counter(areas).most_common()[0][0]

            # also look at similar areas
            freqs = {x for x in areas if abs(base_ar - x) < base_ar / 20}

            # collect the actual image slices
            imboxes = list()
            for ar in freqs:
                for idx, bbox in enumerate(boxes[ar]):
                    imboxes.append(base[bbox[0]:bbox[0] + bbox[2],
                        bbox[1]:bbox[1] + bbox[3]])

            # calculate the correlations between each of the champions
            corrs = self.list_corr(imboxes)

            # look through rates and see if any new champions have been found
            for idx, (rate, champ, box) in enumerate(corrs):
                if display:
                    print("Found {} at {}".format(champ, rate))
                if rate < self.CHAMP_CONFIDENCE:
                    print("New Champion Found")
                    cv2.imwrite("champs/champ_" + str(idx).zfill(3) + ".png", box)

                # if the champion is not in the last, add em
                if champ not in all_champs:
                    all_champs[champ] =  (rate, champ, box)

            # info
            print("Found {} champions".format(len(corrs)))

        return list(all_champs.values())

    ## produce a list of correlations of each bbox against known champions
    def list_corr(self, boxes):
        champ_corrs = list()

        # for each bbox, compute relative correlation
        for box in boxes:
            max_match = 0
            c_champ = None

            # compare against every known champion
            for champ in self.champ_images:
                match = self.correlate(box, self.champ_images[champ])

                # overwrite if higher confidence
                if match > max_match:
                    max_match = match
                    c_champ = champ

            # append highest match
            champ_corrs.append((max_match, c_champ, box))

        return champ_corrs

    ## find correlation of base image against a comparison
    def correlate(self, base, comp):
        # resize the larger image
        if base.shape[0] > comp.shape[0]:
            base = cv2.resize(base, (comp.shape[1], comp.shape[0]))
        else:
            comp = cv2.resize(comp, (base.shape[1], base.shape[0]))

        # compute normalized correlation
        results = cv2.matchTemplate(base, comp, cv2.TM_CCORR_NORMED)

        return results.max()

    ##  get datasheet of current champions and their essence costs
    def getChampTable(self):
        if (time() - self.last_update) > self.TIME_LIMIT:
            # obtain  the html
            res = geturl(self.CHAMP_URL)
            soup = bs(res.content, 'lxml')

            # find the table and convert to dataframe
            table = soup.find_all("table", {"class" : "sortable"})[0]
            self.champ_table = pd.read_html(str(table))[0]

            # sift through and extract just the name
            for idx, row in self.champ_table.iterrows():
                name = row["Champion"].replace(",", "\xa0").split("\xa0")[0]

                self.champ_table.at[idx, "Champion"] = name

        return self.champ_table

    ## convert a champion from pathname into proper name and blue essence cost
    def convChamp(self, champ, cdata):
        # find the basename of the champion
        champ = os.path.splitext(os.path.basename(champ))[0]

        # find corresponding champion in champ table
        row = cdata[cdata["Champion"].str.lower() == champ]

        # convert to cost
        cost = int(self.EXCHANGE[row["Blue Essence"].values[0]])
        name = row["Champion"].values[0]

        return cost, name

    ## get the bounding box of the number
    def getNumberBox(self, img):
        boundsy = np.int32(self.N_BOUNDS[:2] * img.shape[1])
        boundsx = np.int32(self.N_BOUNDS[2:] * img.shape[0])
        return img[boundsy[0]:boundsy[1], boundsx[0]:boundsx[1]]

    ## retrieve the digit as text from the image
    def getDigit(self, image):
        # convert to gray
        image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        # thresh on anything over 127, inverted for OCR as similar to MNIST - black on white bg
        _, image = cv2.threshold(image, 127, 255, cv2.THRESH_BINARY_INV)

        # get the string from pytesseract
        result = image_to_string(image, config=self.P_CONFIG)
        # search for digit
        groups = search("\d+", result)

        # if digit found use it, else assume one
        if groups:
            result = int(groups.group())
        else:
            result = 1

        return result

    ## count the shards in the given set of images
    def countShardCosts(self, images):
        corrs = sc.assign_champ_to_square(images, display=False)

        total_cost = 0
        total_shards = 0

        df = pd.DataFrame()

        # update known champ data
        self.getChampTable()

        # sort the list based on the champion name
        corrs.sort(key=lambda x : x[1])
        for rate, champ, box in corrs:
            # find the champion name and cost
            cost, name = self.convChamp(champ, self.champ_table)
            # retrieve number of shards via OCR
            num_shards = self.getDigit(self.getNumberBox(box))

            # calculate cost
            n_cost = cost * num_shards
            total_shards += num_shards
            total_cost += n_cost

            # print result
            print("{} champion shards of {} is worth {} blue essence".format(
                num_shards, name, n_cost))

            df = df.append({"Champion" : name, "Cost": cost, "Shards" : num_shards,
                "Total Cost" : n_cost}, ignore_index=True)

        print("Total cost of all champion shards are {} blue essence".format(
            total_cost))
        print("{} shards found in total from {} unique champions".format(total_shards,
            len(corrs)))

        return df

    ## count shards using a list of paths rather than images
    def countShardsPath(self, imagepaths):
        images = [cv2.imread(image) for image in imagepaths]
        return self.countShardCosts(images)

if __name__ == "__main__":
    parser = ArgumentParser()

    parser.add_argument("base", help="Base Image", nargs="+")

    args = parser.parse_args()

    sc = ShardCounter()

    sc.countShardsPath(args.base)

max_stars_repo_name: CheeseAnt/league-shard-counter

max_issues_repo_name: CheeseAnt/league-shard-counter

max_forks_repo_name: CheeseAnt/league-shard-counter


text:
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

from paddle.fluid.layers import data
from paddle.fluid.layers.control_flow import lod_rank_table
from paddle.fluid.executor import Executor
import paddle.fluid.core as core
import numpy
import unittest

class TestLoDRankTable(unittest.TestCase):
    def test_lod_rank_table(self):
        x = data(name='x', shape=[100])
        cpu = core.CPUPlace()
        rank_table = lod_rank_table(x=x, level=1)
        rank_table.persistable = True
        exe = Executor(cpu)
        scope = core.Scope()

        tensor = core.LoDTensor()
        tensor.set(numpy.random.random(size=(17, 100)), cpu)
        tensor.set_recursive_sequence_lengths(
            [[1, 2], [5, 1, 1], [3, 1, 5, 1, 3, 3, 1]])
        exe.run(scope=scope, feed={'x': tensor})
        var = scope.find_var(rank_table.name)
        table = var.get_lod_rank_table()
        self.assertEqual([(0, 5), (1, 1), (2, 1)], list(table.items()))

if __name__ == '__main__':
    unittest.main()


text:
import numpy as np
import math as m
from matplotlib import pyplot as plt
from matplotlib.patches import Rectangle
import time as t

fig,ax=plt.subplots()

st=t.time()
lmb=[700*(10**-9),550*(10**-9),475*(10**-9)]
i=0.2 #intenisity in watt/m2
s=0.00002 #slit sepr in metres
d=50 #screen distance in metres
y_lim=100
spc=0.075#y intervals
bh=0.2#height of spectral band width
y=np.arange(-y_lim,y_lim,spc)
ints_r,ints_g,ints_b=[],[],[]
print(t.time()-st)
st=t.time()

for x in y:
    d1=m.sqrt(d**2 + (x-s/2)**2)
    d2=m.sqrt(d**2 + (x+s/2)**2)
    pathdif=abs(d1-d2)

    phi_r=2*(np.pi)*pathdif/lmb[0]
    ints_r.append(4*i*(np.cos(phi_r/2)**2))

    phi_g=2*(np.pi)*pathdif/lmb[1]
    ints_g.append(4*i*(np.cos(phi_g/2)**2))

    phi_b=2*(np.pi)*pathdif/lmb[2]
    ints_b.append(4*i*(np.cos(phi_b/2)**2))

    ax.add_patch(Rectangle((x,-bh),spc*0.9,bh*0.9,
    color=(ints_r[-1]/0.8,ints_g[-1]/0.8,ints_b[-1]/0.8)))
print(t.time()-st)
st=t.time()
ax.plot(y,ints_r,c=(0.8,0,0))
ax.plot(y,ints_g,c=(0,0.8,0))
ax.plot(y,ints_b,c=(0,0,0.8))
plt.show()


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 28 22:59:03 2021

@author: kylei
"""

import numpy as np
from sklearn import mixture
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import GridSearchCV
import pandas as pd

class GMMDensity():
    def __init__(self):
        self.gmms = [] #Array for holding class dependent gaussian mixture models
        self.med_densities = [] #Array for holding class dependent densities
        self.params = [] #holds cv_1 and b for each component so they don't need to be recalculated
        self.dim = []

    def return_gmms(self):
        return self.gmms

    def fit_class_gmm(self, X, Y):
        n_classes = len(np.unique(Y))

        self.dim = X.shape[0]

        '''For each class need a seperate GMM model'''
        for i in range(n_classes):
            component_space = X[Y==i]

            cv = GridSearchCV(estimator=mixture.GaussianMixture(covariance_type = 'full')
                              ,param_grid = {'n_components':range(1,10)}, cv = 5)

            cv.fit(component_space)
            n_components_ = cv.best_params_['n_components']

            gmm = mixture.GaussianMixture(n_components = n_components_,
                                          covariance_type = 'full')
            gmm.fit(component_space)

            self.gmms.append(gmm)

    def estimate_densities_of_training_samples(self, X, Y):
        densities = []
        densities_ex = []

        # Compute densities of all samples
        for i in range(X.shape[0]):
            gmm = self.gmms[Y[i]]    # Select the class dependent GMM

            x = X[i,:]
            z = []
            dim = x.shape[0]
            for j in range(gmm.weights_.shape[0]):
                x_i = gmm.means_[j]
                w_i = gmm.weights_[j]
                cov = gmm.covariances_[j]
                cov = np.linalg.inv(cov)

                b = -2.*np.log(w_i) + dim*np.log(2.*np.pi) - np.log(np.linalg.det(cov))
                z.append(np.dot(x - x_i, np.dot(cov, x - x_i)) + b)

            densities.append(np.min(z))
            densities_ex.append(z)

        return np.array(densities), np.array(densities_ex, dtype = object)

    def prepare_computation_of_plausible_counterfactuals(self, X, Y):
        densities, densities_ex = self.estimate_densities_of_training_samples(X, Y)

        Y_targets = np.unique(Y)

        density_threshold = [np.median(densities[Y==y_]) for y_ in Y_targets]

        return density_threshold

    def estimate_class_density(self, X, Y):
        '''Estimate the density for each class,
            using the GMM fitted to each of those classes'''
        dense = []
        for j in range(len(self.gmms)):
            print(j)
            densities = []

            component_space = X[Y == j]
            '''
            component_space = component_space+1e-5*np.random.rand(component_space.shape[0],
                                                                  component_space.shape[1])
            '''
            cov_1 = np.linalg.inv(self.gmms[j].covariances_)
            x_i = self.gmms[j].means_.reshape(-1,1)
            w_i = self.gmms[j].weights_
            dim = X.shape[1]

            b = -2.*np.log(w_i) + dim*np.log(2.*np.pi) - np.log(np.linalg.det(cov_1))

            self.params.append([cov_1, b])
            for i in range(component_space.shape[0]):
                density = np.dot(component_space[i] - x_i,
                                 np.dot(cov_1, component_space[i] - x_i)) + b

                densities.append(density)

            med_density = -2.*np.log(np.median(densities))
            dense.append(med_density)
            self.med_densities.append(med_density)

        return dense

    def find_new_density(self, x_prime):
        densities = []

        for j in range(len(self.gmms)):
            gmm = self.gmms[j]

            x_i = gmm.means_
            w_i = gmm.weights_[j]
            cov = gmm.covariances_[j]
            cov = np.linalg.inv(cov)

            b = -2.*np.log(w_i) + self.dim*np.log(2.*np.pi) - np.log(np.linalg.det(cov))

            density = np.dot(x_prime - x_i,
                                 np.dot(cov, x_prime - x_i)) + b

        return density

def preprocessing():
    dataset = pd.read_csv('prototype cf/german_credit_data.csv').drop('Unnamed: 0', axis = 1)

    dataset.dropna(axis=0, how='any', thresh=None, subset=None, inplace=True)

    lb = LabelEncoder()

    for column in dataset.columns:
        if dataset[column].dtype != np.dtype('int64'):
            dataset[column] = lb.fit_transform(dataset[column])

    return dataset

if __name__ == '__main__':
    from sklearn.datasets import make_blobs

    n_classes = 2
    data, labels = make_blobs(n_samples=1000,
                          centers=n_classes,
                          random_state=10,
                          center_box=(-3.,8.),
                          cluster_std=2.5)

    gmm_obj = GMMDensity()
    gmm_obj.fit_class_gmm(data,labels)

    gmm_obj.estimate_densities_of_training_samples(data, labels)

    test = gmm_obj.prepare_computation_of_plausible_counterfactuals(data, labels)
    #%%

    np.random.seed()
    i = np.random.randint(0, len(data))
    sample = data[i,:]
    label = labels[i]

    gmm_obj.find_new_density(sample)
    #%%

    #%%
    dataset = preprocessing()
    X = dataset.iloc[:,:-1].values
    Y = dataset.iloc[:,[-1]].values

    gmm = GMMDensity()
    gmm.fit_class_gmm(X, Y)
    gmm.estimate_class_density(X, Y)

max_stars_repo_path: Preliminary Folder/DensityEstimator - Copy.py

max_stars_repo_name: kspurlock/ChWeiMo

max_issues_repo_name: kspurlock/ChWeiMo

max_forks_repo_name: kspurlock/ChWeiMo


text:
import numpy as np
import math
import random
import re
import sys

maxSize = 500
powerParam = 0.6

left   = 0
bottom = 0
right  = 150
top    = 150

clusterSizes = []
for i in range(1, 1000):
    nextSize = max(maxSize / math.pow(i, powerParam), 1)
    clusterSizes.append(int(nextSize))

points = []
pointClusters = []

clusterCenters = []
for clusterIdx in range(len(clusterSizes)):

    center = [random.randint(left, right), random.randint(bottom, top)]

    for idx in range(clusterSizes[clusterIdx]):
        alpha = random.random() * math.pi * 2
        radius = np.random.normal()
        point = [math.sin(alpha) * radius + center[0], math.cos(alpha) * radius + center[1]]

        points.append(point)
        pointClusters.append(clusterIdx)

similarities = []
for i in range(len(points)):
    for j in range(i + 1, len(points)):
        diffX = points[i][0] - points[j][0]
        diffY = points[i][1] - points[j][1]

        if diffX > 5 or diffY > 5:
            continue

        distance = math.sqrt(diffX * diffX + diffY * diffY)
        if distance > 5:
            continue

        similarity = 2. / (1. + math.exp(distance * distance))

        if similarity < 1e-3:
            continue

        similarities.append([i, j, similarity])

    print(i, len(points), file=sys.stderr)

random.shuffle(similarities)
similarities = similarities[: len(similarities) / 3]

markupOut = open("2d_markup.tsv", "w")
pointsOut = open("2d_points.tsv", "w")
simOut = open("2d_similarities.tsv", "w")

for similarityInfo in similarities:
    print("\t".join(map(str, similarityInfo)), file=simOut)
for i in range(len(points)):
    print("\t".join(map(str, [pointClusters[i], i])), file=markupOut)
print("\n".join(["\t".join(map(str, x)) for x in points]), file=pointsOut)

markupOut.close()
pointsOut.close()
simOut.close()


text:
import chainer
import numpy as np

def my_convertor(data, device=None):
    # data is tuple of numpy array
    return tuple([chainer.cuda.to_gpu(d.astype('float32'), device) for d in data])

class MyIterator(chainer.dataset.Iterator):
    # iterator for training

    def __init__(self, dataset, batch_size, repeat=True, shuffle=True):
        self.dataset = dataset
        self.batch_size = batch_size
        self._repeat = repeat
        self._shuffle = shuffle
        self.epoch = 0
        self.epoch_detail = 0

    def __next__(self):
        return self.dataset.get_random_batch(self.batch_size)

    next = __next__

def validation(trainer=None, model=None, dataset=None):
    # evaluate voxel IoUs on all classes
    with chainer.configuration.using_config('train', False):
        ious = {}
        for class_id in dataset.class_ids:
            iou = 0
            for batch in dataset.get_all_batches_for_evaluation(100, class_id):
                batch = my_convertor(batch)
                iou += model.evaluate_iou(*batch).sum()
            iou /= dataset.num_data[class_id] * 24
            ious['%s/iou_%s' % (dataset.set_name, class_id)] = iou
        ious['%s/iou' % dataset.set_name] = np.mean([float(v) for v in ious.values()])
        chainer.report(ious)

def lr_shift(trainer=None, optimizer=None, iterations=None, factor=0.1):
    if trainer.updater.iteration in iterations:
        optimizer.alpha *= factor


text:
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from torchvision import models

class VGG19(nn.Module):
  """Pretrained VGG19 network, without fully connected layers"""

  LAST_FEATURE_MAP = 4  # Index of default block of VGG19 to return

  def __init__(self, output_blocks=[LAST_FEATURE_MAP], requires_grad=False):
    """Build pretrained VGG19

    Parameters
    ----------
    output_blocks : list of int
      Indices of feature blocks to return. Each feature block ends
      before a max-pooling layer, i.e. the output of a feature block is the
      output of the last convolutional layer and activation right before the
      feature map is downscaled. The last possible block index is 5, a block
      which only consists of the last max-pooling in the network
    requires_grad : bool
      If true, parameters of the model require gradient. Possibly useful for
      finetuning the network
    """
    super(VGG19, self).__init__()
    assert len(output_blocks) >= 1, 'Need at least one output block'
    self.output_blocks = sorted(output_blocks)
    last_needed_block = self.output_blocks[-1]

    assert last_needed_block <= 5, 'VGG19 has at most 6 blocks'

    layers = models.vgg19(pretrained=True).features
    self.blocks = nn.ModuleList([nn.Sequential()])
    for idx, layer in enumerate(layers):
      if isinstance(layer, nn.MaxPool2d):
        if len(self.blocks) - 1 == last_needed_block:
          break
        self.blocks.append(nn.Sequential())

      self.blocks[-1].add_module(str(idx), layer)

    for param in self.parameters():
      param.requires_grad = requires_grad

    mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
    mean_tensor = torch.from_numpy(mean.reshape(1, 3, 1, 1))
    self.register_buffer('mean', mean_tensor)

    std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
    std_tensor = torch.from_numpy(std.reshape(1, 3, 1, 1))
    self.register_buffer('std', std_tensor)

  def forward(self, inp):
    """Get VGG19 feature maps

    Parameters
    ----------
    inp : torch.Tensor
      Input tensor of shape Bx3xHxW. Values are expected to be in range (0, 1)
    """

    # We need to recreate the Variable on every call, as Variables should not
    # be registered as buffers, but we also can not mix operations on Variables
    # with Tensors.
    mean = Variable(self.mean, requires_grad=False)
    std = Variable(self.std, requires_grad=False)

    # Normalize input to the statistics VGG19 expects
    x = inp.sub(mean).div(std)

    out = []
    for block_idx, block in enumerate(self.blocks):
      x = block(x)
      if block_idx in self.output_blocks:
        out.append(x)

    return out

max_stars_repo_name: mseitzer/csmri-refinement

max_issues_repo_name: mseitzer/csmri-refinement

max_forks_repo_name: mseitzer/csmri-refinement


text:
from scipy.stats import randint as sp_randint
from scipy.stats import uniform as sp_uniform
import numpy as np
import random
from sklearn.ensemble import RandomForestRegressor
from sklearn.neural_network import MLPRegressor
from lightgbm import LGBMRegressor

class LayerSizeGenerator:

    def __init__(self):
        self.num_layers = [1, 2]
        self.num_neurons = np.arange(1, 15+1, 1)

    def rvs(self, random_state=42):
        random.seed(random_state)
        # first randomly define num of layers, then pick the neuron size for each of them
        num_layers = random.choice(self.num_layers)
        layer_sizes = random.choices(self.num_neurons, k=num_layers)
        return layer_sizes

class NNCombWrapper():
    def __init__(self, model_params=None):
        self.model_name = "nncomb"
        self.search_type = 'random'
        self.param_grid = {"early_stopping": [True],
                           "learning_rate": ["invscaling"],
                           "learning_rate_init": np.linspace(0.001, 0.999, 100),
                           'alpha': np.linspace(0.001, 0.999, 100),
                           'solver': ["adam"],
                           'activation': ["relu"],
                           "hidden_layer_sizes": LayerSizeGenerator()}
        if model_params is None:
            self.ModelClass = MLPRegressor()
        else:
            self.ModelClass = MLPRegressor(**model_params)

class RandomForestWrapper():
    def __init__(self, model_params=None):
        self.model_name = "random_forest"
        self.search_type = 'random'
        self.param_grid = {"max_features": ['auto', 'sqrt', 'log2'],
                           "min_samples_split": sp_randint(2, 31),
                           "n_estimators": sp_randint(2, 301),
                           "max_depth": sp_randint(2, 20)}
        if model_params is None:
            self.ModelClass = RandomForestRegressor()
        else:
            self.ModelClass = RandomForestRegressor(**model_params)

class LGBWrapper():
    def __init__(self, model_params=None):
        self.model_name = "lgb_regression"
        self.search_type = 'random'
        self.param_grid = {'num_leaves': sp_randint(6, 50),
                           'min_child_samples': sp_randint(100, 500),
                           'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
                           'subsample': sp_uniform(loc=0.2, scale=0.8),
                           "n_estimators": sp_randint(500, 1000),
                           "max_depth": sp_randint(3, 100),
                           "learning_rate": np.linspace(0.001, 0.99, 100),
                           'colsample_bytree': sp_uniform(loc=0.4, scale=0.6),
                           'reg_alpha': [0, 1e-1, 1, 2, 5, 7, 10, 50, 100],
                           'reg_lambda': [0, 1e-1, 1, 5, 10, 20, 50, 100],
                           "objective": ["huber"]}
        if model_params is None:
            self.ModelClass = LGBMRegressor()
        else:
            self.ModelClass = LGBMRegressor(**model_params)

max_stars_repo_name: dcuoliveira/NN-VAR-Cov-Pred

max_issues_repo_name: dcuoliveira/NN-VAR-Cov-Pred

max_forks_repo_name: dcuoliveira/NN-VAR-Cov-Pred


text:
import streamlit as st
import pandas as pd
import json
import os
from datetime import datetime
import dicescoring
import numpy as np
import io

# from sklearn.metrics import (accuracy_score, auc, f1_score, precision_score, recall_score,
#                             mean_absolute_error, mean_squared_error, r2_score)
st.set_option('deprecation.showfileUploaderEncoding', False)

# funtions
def relative_time(t_diff):
    days, seconds = t_diff.days, t_diff.seconds
    if days > 0:
        return f"{days}d"
    else:
        hours = t_diff.seconds // 3600
        minutes = t_diff.seconds // 60
        if hours >0 : #hour
            return f"{hours}h"
        elif minutes >0:
            return f"{minutes}m"
        else:
            return f"{seconds}s"

def get_leaderboard_dataframe(csv_file = 'leaderboardTB.csv', greater_is_better = True):
    df_leaderboard = pd.read_csv('leaderboardTB.csv', header = None)
    df_leaderboard.columns = ['Username', 'Score', 'Submission Time']
    df_leaderboard['counter'] = 1
    df_leaderboard = df_leaderboard.groupby('Username').agg({"Score": "max",
                                                            "counter": "count",
                                                            "Submission Time": "max"})
    df_leaderboard = df_leaderboard.sort_values("Score", ascending = not greater_is_better)
    df_leaderboard = df_leaderboard.reset_index()
    df_leaderboard.columns = ['Username','Score', 'Entries', 'Last']
    df_leaderboard['Last'] = df_leaderboard['Last'].map(lambda x: relative_time(datetime.now() - datetime.strptime(x, "%Y%m%d_%H%M%S")))
    return df_leaderboard

def app():
    # Title
    st.title("TB Competition Leaderboard")

    # Username Input
    username = st.text_input("Username", value = "Joao", max_chars= 20,)
    username = username.replace(",","") # for storing csv purpose
    st.header(f"Hi {username} !!!")

    # Check if master data has been registered:
    master_files = os.listdir('master')
    if ("via_project_9Dec2020_15h40m_Les_ground_truth.json" not in master_files):
        st.text("Admin please insert ground truth data")
    else:
        greater_is_better = True# if metric_type in ["MAE", "MSE"] else True # CHANGE HERE AS YOU WANT

        uploaded_file = st.file_uploader("Upload Submission json File", type='json')
        groud_truth_file = 'master/tb_ground_truth.json'
        if st.button("SUBMIT"):
            if uploaded_file is None:
                st.text("UPLOAD FIRST")
            else:
                # save submission

                stringio = io.StringIO(uploaded_file.getvalue().decode("utf-8"))
                json_uploaded_submission = json.load(stringio)
                datetime_now = datetime.now().strftime("%Y%m%d_%H%M%S")
                datetime_now = datetime.now().strftime("%Y%m%d_%H%M%S")
                filename_submission = f"submission/sub_{username}__{datetime_now}.json"
                with open(filename_submission, 'w') as outfile:
                    json.dump(json_uploaded_submission, outfile)

                # calculate score
                dice_scores_user = dicescoring.dice_list(groud_truth_file, uploaded_file)

                score = round(np.sum(np.asarray(dice_scores_user)) * 10)
                score = round(score,5)
                st.text(f"YOUR Dice socer: {score}")
                # save score
                with open("leaderboardTB.csv", "a+") as leaderboard_csv:
                    leaderboard_csv.write(f"{username},{score},{datetime_now}\n")

        # Showing Leaderboard
        st.header("Leaderboard")
        if os.stat("leaderboardTB.csv").st_size == 0:
            st.text("NO SUBMISSION YET")
        else:
            df_leaderboard = get_leaderboard_dataframe(csv_file = 'leaderboardTB.csv', greater_is_better = greater_is_better)
            st.write(df_leaderboard)

    # To register master data
    if username == 'pokerad_admin_siim21': # CHANGE HERE AS YOU WANT
        change_master_key = st.checkbox('Change Ground Truth File')

        if change_master_key:

            # Master Data Frame
            uploaded_file_master = st.file_uploader("Upload Ground Truth File", type='json')
            if uploaded_file_master is not None:
                stringio = io.StringIO(uploaded_file_master.getvalue().decode("utf-8"))
    #            f_uploaded_submission_master = open(stringio)
    #            json_uploaded_submission_master = json.load(f_uploaded_submission_master)
                json_uploaded_submission_master = json.load(stringio)
                datetime_now = datetime.now().strftime("%Y%m%d_%H%M%S")
                with open('master/tb_ground_truth.json', 'w') as outfile:
                    json.dump(json_uploaded_submission_master, outfile)

max_stars_repo_name: JoaoSantinha/streamlit-leaderboard

max_issues_repo_name: JoaoSantinha/streamlit-leaderboard

max_forks_repo_name: JoaoSantinha/streamlit-leaderboard


text:
import matplotlib.pyplot as plt
import matplotlib.patches as patches
from PIL import Image
import numpy as np

import pycocotools.mask as mask_util
import cv2

classes = [line.rstrip('\n') for line in open('coco_classes.txt')]

# You might need to adjust score threshold
def display_objdetect_image(image, boxes, labels, scores, masks, score_threshold=0.7):
    # Resize boxes
    ratio = 800.0 / min(image.size[0], image.size[1])
    boxes /= ratio

    _, ax = plt.subplots(1, figsize=(12,9))

    image = np.array(image)

    for mask, box, label, score in zip(masks, boxes, labels, scores):
        # Showing boxes with score > 0.7
        if score <= score_threshold:
            continue

        # Finding contour based on mask
        mask = mask[0, :, :, None]
        int_box = [int(i) for i in box]
        mask = cv2.resize(mask, (int_box[2]-int_box[0]+1, int_box[3]-int_box[1]+1))
        mask = mask > 0.5
        im_mask = np.zeros((image.shape[0], image.shape[1]), dtype=np.uint8)
        x_0 = max(int_box[0], 0)
        x_1 = min(int_box[2] + 1, image.shape[1])
        y_0 = max(int_box[1], 0)
        y_1 = min(int_box[3] + 1, image.shape[0])
        mask_y_0 = max(y_0 - box[1], 0)
        mask_y_1 = mask_y_0 + y_1 - y_0
        mask_x_0 = max(x_0 - box[0], 0)
        mask_x_1 = mask_x_0 + x_1 - x_0
        im_mask[y_0:y_1, x_0:x_1] = mask[
            mask_y_0 : mask_y_1, mask_x_0 : mask_x_1
        ]
        im_mask = im_mask[:, :, None]

        # OpenCV version 4.x
        contours, hierarchy = cv2.findContours(
            im_mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE
        )

        image = cv2.drawContours(image, contours, -1, 25, 3)

        rect = patches.Rectangle((box[0], box[1]), box[2] - box[0], box[3] - box[1], linewidth=1, edgecolor='b', facecolor='none')
        ax.annotate(classes[label] + ':' + str(np.round(score, 2)), (box[0], box[1]), color='w', fontsize=12)
        ax.add_patch(rect)

    ax.imshow(image)
    plt.savefig("annotated.png")

img = Image.open('../images/preprocessed.jpg')

labels = np.fromfile("../labels.data", dtype=np.int64)
NUM_CLASSES = labels.size
boxes = np.fromfile("../boxes.data", dtype=np.float32).reshape((NUM_CLASSES, 4))
scores = np.fromfile("../scores.data", dtype=np.float32).reshape((NUM_CLASSES))
# For model exported from pytorch use 800x800
masks = np.fromfile("../masks.data", dtype=np.float32).reshape((NUM_CLASSES, 1, 28, 28))
display_objdetect_image(img, boxes, labels, scores, masks)

max_stars_repo_name: kimjungwow/onnxruntime-riscv

max_issues_repo_name: kimjungwow/onnxruntime-riscv

max_forks_repo_name: kimjungwow/onnxruntime-riscv


text:
import numpy as np
import pyqtgraph as pg
import time

from PyQt5.Qsci import QsciScintilla, QsciLexerPython

from spyre import Spyrelet, Task, Element
from spyre.widgets.task import TaskWidget
from spyre.plotting import LinePlotWidget
from spyre.widgets.rangespace import Rangespace
from spyre.widgets.param_widget import ParamWidget
from spyre.widgets.repository_widget import RepositoryWidget

from lantz import Q_
import time
import os

from lantz.drivers.spectrum import MS2721B
from lantz.log import log_to_screen, DEBUG

class SpectrumAnalyzer(Spyrelet):

    requires = {
        'spa': MS2721B
    }

    @Task()
    def set(self):
        self.dataset.clear()
        log_to_screen(DEBUG)
        freq_params = self.Frequency_Settings.widget.get()
        # amp_params = self.Frequency_Settings.widget.get()
        span = freq_params['frequency span']
        center = freq_params['center freq']
        self.spa.freq_span = span
        self.spa.freq_cent = center
        # self.spa.ref_level = ref
        # self.spa.freq_star = start

    @set.initializer
    def initialize(self):
        return

    @set.finalizer
    def finalize(self):
        return

    @Task()
    def save(self):
        self.dataset.clear()
        log_to_screen(DEBUG)
        save_params = self.Save_Settings.widget.get()

        t = save_params['sleep time']
        count = save_params['file count']

        for x in range(count):
            self.spa.savefile(x)
            time.sleep(t)

    @save.initializer
    def initialize(self):
        return

    @save.finalizer
    def finalize(self):
        return

    @Element()
    def Frequency_Settings(self):
        freq_params = [
        ('frequency span', {'type': float, 'default': 3000, 'units': 'Hz'}),
        ('center freq', {'type': float, 'default': 30000000, 'units': 'Hz'}),
        ]
        w = ParamWidget(freq_params)
        return w

    @Element()
    def Save_Settings(self):
        save_params = [
        ('sleep time', {'type': float, 'default': 30}),
        ('file count', {'type': int, 'default': 10}),
        ]
        w = ParamWidget(save_params)
        return w

max_stars_repo_name: zhong-lab/optics

max_issues_repo_name: zhong-lab/optics

max_forks_repo_name: zhong-lab/optics


text:
import gym
import zmq
import random
import numpy as np
from collections import deque
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Sequential

class DQNAgent:
    def __init__(self, state_size, action_size):
        self.state_size = state_size
        self.action_size = action_size
        self.replay_buffer = deque(maxlen=2000)
        self.gamma = 0.95  # Discount Rate
        self.epsilon = 1.0  # Exploration Rate
        self.epsilon_min = 0.01
        self.epsilon_decay = 0.995
        self.learning_rate = 0.001
        self.model = self._build_model()

    def _build_model(self):
        """Build Neural Net for Deep Q-learning Model"""

        model = Sequential()
        model.add(Dense(24, input_dim=self.state_size, activation='relu'))
        model.add(Dense(24, activation='relu'))
        model.add(Dense(self.action_size, activation='linear'))
        model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))
        return model

    def memorize(self, state, action, reward, next_state, done):
        #open('agent_data','a').write(str((state, action, reward, next_state, done)) + '\n')
        #self.replay_buffer.append((state, action, reward, next_state, done))
        message = np.array((state, action, reward, next_state, done), dtype = object)
        print(message)
        socket.send(message)

    def act(self, state):
        if np.random.rand() <= self.epsilon:
            return random.randrange(self.action_size)
        act_values = self.model.predict(state)
        return np.argmax(act_values[0])  # returns action

    def replay(self, batch_size):
        minibatch = random.sample(self.replay_buffer, batch_size)
        for state, action, reward, next_state, done in minibatch:
            target = reward
            if not done:
                target += self.gamma * np.amax(self.model.predict(next_state)[0])
            target_f = self.model.predict(state)
            target_f[0][action] = target
            self.model.fit(state, target_f, epochs=1, verbose=0)
        if self.epsilon > self.epsilon_min:
            self.epsilon *= self.epsilon_decay

    def load(self, name):
        self.model.load_weights(name)

    def save(self, name):
        self.model.save_weights(name)

if __name__ == '__main__':

    context = zmq.Context()
    socket = context.socket(zmq.REQ)
    socket.connect("tcp://172.17.0.2:6566")

    env = gym.make('CartPole-v1')
    state_size = env.observation_space.shape[0]
    action_size = env.action_space.n

    agent = DQNAgent(state_size, action_size)
    # agent.load('./save/cartpole-dqn.h5')

    done = False
    batch_size = 32
    num_episodes = 1000
    cnt = 0
    for e in range(num_episodes):
        state = env.reset()
        state = np.reshape(state, [1, state_size])
        for time in range(500):
            # env.render()
            action = agent.act(state)
            next_state, reward, done, _ = env.step(action)
            reward = reward if not done else -10
            next_state = np.reshape(next_state, [1, state_size])
            message = str((state.tolist(), action, reward, next_state.tolist(), done))
            socket.send(bytes(message, encoding = "utf8"))
            message = socket.recv()
            if message == b'Cover':
                cnt += 1
            else:
                with open('save/cartpole-dqn{}.h5'.format(e), 'wb') as f:
                    f.write(message)
            state = next_state
            if done:
                print('episode: {}/{}, score: {}, e: {:.2}'.format(e, num_episodes, time, agent.epsilon))
                break


text:
import numpy as np
import matplotlib.pyplot as plt
import bound_methods
from numpy import nan, inf
from matplotlib.pyplot import cm

#plot results for experiment4_1
#rewards are feasible in that all start states end up at goal within 25 steps
bar_width = 0.15
opacity = 0.8

sample_flag = 4
chain_length = 10000
step = 0.01
alphas = [0,1,5,10,50]
size = 9
num_reps = 200
rolloutLength = 100
stochastic = 1
randActionProb = 0.2
numDemos = [1,5,9]
tol = 0.0001
gamma = 0.9
burn = 100
skip = 20
delta_conf = 0.95
bounds = ["VaR 95"]
fmts = ['o-','s--','^-.', '*:','>-','d--']

filePath = "./data/gridworld_noisydemo_exp/"

color=iter(cm.rainbow(np.linspace(0,1,6)))

alpha = 5
index = np.arange(len(numDemos))
cnt = 0;

for alpha in alphas:
    print("******", alpha)
    bound_type = bounds[0]
    accuracies = []
    average_bound_error = []
    for numDemo in numDemos:
        true_perf_ratio = []
        predicted = []
        bound_error = []

        print("=========", numDemo, "=========")
        for rep in range(num_reps):
            #print("rep",rep)
            filename = "NoisyDemo_numdemos" +  str(numDemo) + "_alpha" + str(alpha) + "_chain" + str(chain_length) +  "_step" + str(step) + "0000_L1sampleflag" + str(sample_flag) +  "_rolloutLength" + str(rolloutLength) +   "_stochastic" + str(stochastic) + "_randActionProb" + str(randActionProb) + "00000_rep" + str(rep)+ ".txt"
            #print filename
            f = open(filePath + filename,'r')
            f.readline()                                #clear out comment from buffer
            actual = (float(f.readline())) #get the true ratio
            #print("actual", actual)
            f.readline()                                #clear out ---
            wfcb = (float(f.readline())) #get the worst-case feature count bound
            f.readline()  #clear out ---
            samples = []
            for line in f:                              #read in the mcmc chain
                val = float(line)
                samples.append(float(line))
            f.close()
            #print samples
            #burn
            burned_samples = samples[burn::skip]
            #print "max sample", np.max(burned_samples)
            #compute confidence bound
            if bound_type == "VAR":
                upper_bnd = bound_methods.value_at_risk(burned_samples, delta_conf)
            elif bound_type == "VaR 99":
                upper_bnd = bound_methods.percentile_confidence_upper_bnd(burned_samples, 0.99, delta_conf)
            elif bound_type == "VaR 95":
                upper_bnd = bound_methods.percentile_confidence_upper_bnd(burned_samples, 0.95, delta_conf)
            elif bound_type == "VaR 90":
                upper_bnd = bound_methods.percentile_confidence_upper_bnd(burned_samples, 0.9, delta_conf)
            elif bound_type == "WFCB":
                upper_bnd = wfcb

            #print ("upper bound", upper_bnd)
            predicted.append(upper_bnd)
            true_perf_ratio.append(actual)
            bound_error.append(upper_bnd - actual)
        accuracy = 0.0
        for i in range(len(predicted)):
            if (predicted[i] >= true_perf_ratio[i]) or np.abs(predicted[i] - true_perf_ratio[i]) < tol:
                accuracy += 1.0
        accuracy = accuracy / len(predicted)
        #print(bound_type)
        #print(predicted)
        #print("accuracy", accuracy)
        accuracies.append(accuracy)
        average_bound_error.append(np.mean(bound_error))
        #print()
    print(average_bound_error)
    c = next(color)
    plt.figure(1)
    rects1 = plt.bar(index + cnt * bar_width,average_bound_error, bar_width,
                 alpha=opacity, label="c = "+str(alpha), color=c)

    plt.figure(2)
    rects2 = plt.bar(index + cnt * bar_width,accuracies, bar_width,
                 alpha=opacity, label="c = "+str(alpha), color=c)

    cnt += 1
plt.figure(1)
plt.axis([-.2,3,-0.2, 2.75])
plt.yticks(fontsize=18)
plt.xticks(index + 2*bar_width, ('1', '5', '9'), fontsize=18)
plt.legend(fontsize=18)
plt.xlabel('number of demonstrations',fontsize=19)
plt.ylabel('average bound error',fontsize=19)
plt.tight_layout()
plt.savefig("./figs/noisydemo_bound_error_overAlpha.png")

plt.figure(2)
plt.yticks(fontsize=18)
plt.xticks(index + 2*bar_width, ('1', '5', '9'), fontsize=18)
plt.legend(fontsize=18, loc='center left')
plt.xlabel('number of demonstrations',fontsize=19)
plt.ylabel('accuracy',fontsize=19)
plt.tight_layout()
plt.savefig("./figs/noisydemo_accuracy_overAlpha.png")
plt.show()

reason: import numpy,from numpy


text:
# Distributed under the MIT License.
# See LICENSE.txt for details.

import numpy as np
from Evolution.Systems.CurvedScalarWave.Characteristics import (
    char_speed_vpsi, char_speed_vzero, char_speed_vplus, char_speed_vminus)

def error(face_mesh_velocity, normal_covector, normal_vector, psi, phi,
          inertial_coords, gamma1, gamma2, lapse, shift, dt_psi, dt_pi, dt_phi,
          d_psi, d_phi):
    return None

def dt_psi_constraint_preserving_spherical_radiation(
    face_mesh_velocity, normal_covector, normal_vector, psi, phi,
    inertial_coords, gamma1, gamma2, lapse, shift, dt_psi, dt_pi, dt_phi,
    d_psi, d_phi):
    char_speed_psi = char_speed_vpsi(gamma1, lapse, shift, normal_covector)

    if face_mesh_velocity is not None:
        char_speed_psi -= np.dot(normal_covector, face_mesh_velocity)

    return np.dot(normal_vector, d_psi - phi) * min(0., char_speed_psi)

def dt_phi_constraint_preserving_spherical_radiation(
    face_mesh_velocity, normal_covector, normal_vector, psi, phi,
    inertial_coords, gamma1, gamma2, lapse, shift, dt_psi, dt_pi, dt_phi,
    d_psi, d_phi):
    char_speed_zero = char_speed_vzero(gamma1, lapse, shift, normal_covector)
    if face_mesh_velocity is not None:
        char_speed_zero -= np.dot(normal_covector, face_mesh_velocity)
    return 0.5 * np.einsum("ij,j", d_phi.T - d_phi, normal_vector) * min(
        0, char_speed_zero)

def dt_pi_constraint_preserving_spherical_radiation(
    face_mesh_velocity, normal_covector, normal_vector, psi, phi,
    inertial_coords, gamma1, gamma2, lapse, shift, dt_psi, dt_pi, dt_phi,
    d_psi, d_phi):
    dt_psi_correction = dt_psi_constraint_preserving_spherical_radiation(
        face_mesh_velocity, normal_covector, normal_vector, psi, phi,
        inertial_coords, gamma1, gamma2, lapse, shift, dt_psi, dt_pi, dt_phi,
        d_psi, d_phi)
    inv_radius = 1. / np.linalg.norm(inertial_coords)
    bc_dt_pi = (2. * inv_radius**2 * psi + 4. * inv_radius * dt_psi +
                4. * inv_radius * np.dot(normal_vector, phi) +
                2. * np.dot(normal_vector, dt_phi) + np.dot(shift, dt_phi) +
                np.einsum("i,j,ij", normal_vector, normal_vector, d_phi))
    bc_dt_pi /= lapse
    return bc_dt_pi - dt_pi + gamma2 * dt_psi_correction

max_stars_repo_name: nilsvu/spectre

max_issues_repo_name: nilsvu/spectre

max_forks_repo_name: nilsvu/spectre


text:
import json
import os
import sys
import traceback
import warnings
from contextlib import contextmanager
from datetime import datetime
from typing import Optional, Tuple, Union, List

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.model_selection import BaseCrossValidator

_ALWAYS_CATCH = False

def in_colab() -> bool:
    return 'google.colab' in sys.modules

def in_kaggle() -> bool:
    return 'kaggle_web_client' in sys.modules

def set_always_catch(catch: bool):
    global _ALWAYS_CATCH
    _ALWAYS_CATCH = catch

@contextmanager
def catch_everything_in_kaggle(name: Optional[str] = None):
    try:
        yield
    except Exception:
        msg = f"WARNINGS: exception occur in {name or '(unknown)'}: {traceback.format_exc()}"
        warnings.warn(msg)
        print(msg)

        if in_kaggle() or _ALWAYS_CATCH:
            # ...catch and suppress if this is executed in kaggle
            pass
        else:
            # re-raise if this is executed outside of kaggle
            raise

def save_directory_as_kaggle_dataset(output_dir: str, title: str, id: str) -> None:
    cmd = f'kaggle datasets version -p {output_dir} -m "automatic upload" --dir-mode tar'

    metadata = {
        "title": title,
        "id": f"nyanpn/{id}",
        "licenses": [{"name": "CC0-1.0"}]
    }

    with open(os.path.join(output_dir, "dataset-metadata.json"), "w") as fp:
        json.dump(metadata, fp)

    os.system(cmd)

########################################################################################################################
# from nyaggle

def plot_importance(importance: pd.DataFrame, path: Optional[str] = None, top_n: int = 100,
                    figsize: Optional[Tuple[int, int]] = None,
                    title: Optional[str] = None):
    """
    Plot feature importance and write to image

    Args:
        importance:
            The dataframe which has "feature" and "importance" column
        path:
            The file path to be saved
        top_n:
            The number of features to be visualized
        figsize:
            The size of the figure
        title:
            The title of the plot
    Example:
        >>> import pandas as pd
        >>> import lightgbm as lgb
        >>> from nyaggle.util import plot_importance
        >>> from sklearn.datasets import make_classification

        >>> X, y = make_classification()
        >>> X = pd.DataFrame(X, columns=['col{}'.format(i) for i in range(X.shape[1])])
        >>> booster = lgb.train({'objective': 'binary'}, lgb.Dataset(X, y))
        >>> importance = pd.DataFrame({
        >>>     'feature': X.columns,
        >>>     'importance': booster.feature_importance('gain')
        >>> })
        >>> plot_importance(importance, 'importance.png')
    """
    importance = importance.groupby('feature')['importance'] \
        .mean() \
        .reset_index() \
        .sort_values(by='importance', ascending=False)

    if len(importance) > top_n:
        importance = importance.iloc[:top_n, :]

    if figsize is None:
        figsize = (10, 16)

    if title is None:
        title = 'Feature Importance'

    plt.figure(figsize=figsize)
    sns.barplot(x="importance", y="feature", data=importance)
    plt.title(title)
    plt.tight_layout()
    if path is not None:
        plt.savefig(path)

class TimeSeriesSplit(BaseCrossValidator):
    """ Time Series cross-validator

    Time Series cross-validator which provides train/test indices to split variable interval time series data.
    This class provides low-level API for time series validation strategy.
    This class is compatible with sklearn's ``BaseCrossValidator`` (base class of ``KFold``, ``GroupKFold`` etc).

    Args:
        source:
            The column name or series of timestamp.
        times:
            Splitting window, where times[i][0] and times[i][1] denotes train and test time interval in (i-1)th fold
            respectively. Each time interval should be pair of datetime or str, and the validator generates indices
            of rows where timestamp is in the half-open interval [start, end).
            For example, if ``times[i][0] = ('2018-01-01', '2018-01-03')``, indices for (i-1)th training data
            will be rows where timestamp value meets ``2018-01-01 <= t < 2018-01-03``.

    Example:
        >>> import numpy as np
        >>> import pandas as pd
        >>> from nyaggle.validation import TimeSeriesSplit
        >>> df = pd.DataFrame()
        >>> df['time'] = pd.date_range(start='2018/1/1', periods=5)

        >>> folds = TimeSeriesSplit('time',
        >>>                         [(('2018-01-01', '2018-01-02'), ('2018-01-02', '2018-01-04')),
        >>>                          (('2018-01-02', '2018-01-03'), ('2018-01-04', '2018-01-06'))])

        >>> folds.get_n_splits()
        2

        >>> splits = folds.split(df)

        >>> train_index, test_index = next(splits)
        >>> train_index
        [0]
        >>> test_index
        [1, 2]

        >>> train_index, test_index = next(splits)
        >>> train_index
        [1]
        >>> test_index
        [3, 4]
    """
    datepair = Tuple[Union[datetime, str], Union[datetime, str]]

    def __init__(self, source: Union[pd.Series, str],
                 times: List[Tuple[datepair, datepair]] = None):
        self.source = source
        self.times = []
        if times:
            for t in times:
                self.add_fold(t[0], t[1])

    def _to_datetime(self, time: Union[str, datetime]):
        return time if isinstance(time, datetime) else pd.to_datetime(time)

    def _to_datetime_tuple(self, time: datepair):
        return self._to_datetime(time[0]), self._to_datetime(time[1])

    def add_fold(self, train_interval: datepair, test_interval: datepair):
        """
        Append 1 split to the validator.

        Args:
            train_interval:
                start and end time of training data.
            test_interval:
                start and end time of test data.
        """
        train_interval = self._to_datetime_tuple(train_interval)
        test_interval = self._to_datetime_tuple(test_interval)
        assert train_interval[1], "train_interval[1] should not be None"
        assert test_interval[0], "test_interval[0] should not be None"

        assert (not train_interval[0]) or (
                train_interval[0] <= train_interval[1]), "train_interval[0] < train_interval[1]"
        assert (not test_interval[1]) or (test_interval[0] <= test_interval[1]), "test_interval[0] < test_interval[1]"

        self.times.append((train_interval, test_interval))

    def get_n_splits(self, X=None, y=None, groups=None):
        return len(self.times)

    def split(self, X, y=None, groups=None):
        """
        Generate indices to split data into training and test set.

        Args:
            X:
                Training data.
            y:
                Ignored.
            groups:
                Ignored.

        Yields:
            The training set and the testing set indices for that split.
        """
        ts = X[self.source] if isinstance(self.source, str) else self.source

        for train_interval, test_interval in self.times:
            train_mask = ts < train_interval[1]
            if train_interval[0]:
                train_mask = (train_interval[0] <= ts) & train_mask

            test_mask = test_interval[0] <= ts
            if test_interval[1]:
                test_mask = test_mask & (ts < test_interval[1])

            yield np.where(train_mask)[0], np.where(test_mask)[0]

max_stars_repo_name: copyrosicky/MlbEngagementCompetition

max_issues_repo_name: copyrosicky/MlbEngagementCompetition

max_forks_repo_name: copyrosicky/MlbEngagementCompetition