text:
# importint the necessary lirbraries

# import tensorflow.keras as kerasfrom __future__ import print_function
from __future__ import print_function

import keras
from keras.layers import Convolution2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import *
from keras.layers.normalization import BatchNormalization
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ModelCheckpoint
from keras import initializers
from keras.preprocessing import image
from keras.preprocessing.image import img_to_array
from keras import layers
from keras.models import load_model
import matplotlib.image as mpimg

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import tensorflow as tf
from keras import backend as K
from keras.layers import Layer
from keras import activations
from keras import utils
from keras import models
from keras.models import Model
from keras.callbacks import ModelCheckpoint
from keras.optimizers import Adam, SGD

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer
from os import listdir
import os
from imutils import paths
import numpy as np
import pickle
import matplotlib.pyplot as plt
import cv2
import random
import os

class Length(layers.Layer):
    """
    Compute the length of vectors. This is used to compute a Tensor that has the same shape with y_true in margin_loss.
    Using this layer as model's output can directly predict labels by using `y_pred = np.argmax(model.predict(x), 1)`
    inputs: shape=[None, num_vectors, dim_vector]
    output: shape=[None, num_vectors]
    """

    def call(self, inputs, **kwargs):
        return K.sqrt(K.sum(K.square(inputs), -1) + K.epsilon())

    def compute_output_shape(self, input_shape):
        return input_shape[:-1]

    def get_config(self):
        config = super(Length, self).get_config()
        return config

class Mask(layers.Layer):
    """
    Mask a Tensor with shape=[None, num_capsule, dim_vector] either by the capsule with max length or by an additional
    input mask. Except the max-length capsule (or specified capsule), all vectors are masked to zeros. Then flatten the
    masked Tensor.
    For example:
        ```
        x = keras.layers.Input(shape=[8, 3, 2])  # batch_size=8, each sample contains 3 capsules with dim_vector=2
        y = keras.layers.Input(shape=[8, 3])  # True labels. 8 samples, 3 classes, one-hot coding.
        out = Mask()(x)  # out.shape=[8, 6]
        # or
        out2 = Mask()([x, y])  # out2.shape=[8,6]. Masked with true labels y. Of course y can also be manipulated.
        ```
    """

    def call(self, inputs, **kwargs):
        if type(inputs) is list:  # true label is provided with shape = [None, n_classes], i.e. one-hot code.
            assert len(inputs) == 2
            inputs, mask = inputs
        else:  # if no true label, mask by the max length of capsules. Mainly used for prediction
            # compute lengths of capsules
            x = K.sqrt(K.sum(K.square(inputs), -1))
            # generate the mask which is a one-hot code.
            # mask.shape=[None, n_classes]=[None, num_capsule]
            mask = K.one_hot(indices=K.argmax(x, 1), num_classes=x.get_shape().as_list()[1])

        # inputs.shape=[None, num_capsule, dim_capsule]
        # mask.shape=[None, num_capsule]
        # masked.shape=[None, num_capsule * dim_capsule]
        masked = K.batch_flatten(inputs * K.expand_dims(mask, -1))
        return masked

    def compute_output_shape(self, input_shape):
        if type(input_shape[0]) is tuple:  # true label provided
            return tuple([None, input_shape[0][1] * input_shape[0][2]])
        else:  # no true label provided
            return tuple([None, input_shape[1] * input_shape[2]])

    def get_config(self):
        config = super(Mask, self).get_config()
        return config

def squash(vectors, axis=-1):
    """
    The non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0
    :param vectors: some vectors to be squashed, N-dim tensor
    :param axis: the axis to squash
    :return: a Tensor with same shape as input vectors
    """
    s_squared_norm = K.sum(K.square(vectors), axis, keepdims=True)
    scale = s_squared_norm / (1 + s_squared_norm) / K.sqrt(s_squared_norm + K.epsilon())
    return scale * vectors

class CapsuleLayer(layers.Layer):
    """
    The capsule layer. It is similar to Dense layer. Dense layer has `in_num` inputs, each is a scalar, the output of the
    neuron from the former layer, and it has `out_num` output neurons. CapsuleLayer just expand the output of the neuron
    from scalar to vector. So its input shape = [None, input_num_capsule, input_dim_capsule] and output shape = \
    [None, num_capsule, dim_capsule]. For Dense Layer, input_dim_capsule = dim_capsule = 1.

    :param num_capsule: number of capsules in this layer
    :param dim_capsule: dimension of the output vectors of the capsules in this layer
    :param routings: number of iterations for the routing algorithm
    """

    def __init__(self, num_capsule, dim_capsule, routings=3,
                 kernel_initializer='glorot_uniform',
                 **kwargs):
        super(CapsuleLayer, self).__init__(**kwargs)
        self.num_capsule = num_capsule
        self.dim_capsule = dim_capsule
        self.routings = routings
        self.kernel_initializer = initializers.get(kernel_initializer)

    def build(self, input_shape):
        assert len(input_shape) >= 3, "The input Tensor should have shape=[None, input_num_capsule, input_dim_capsule]"
        self.input_num_capsule = input_shape[1]
        self.input_dim_capsule = input_shape[2]

        # Transform matrix
        self.W = self.add_weight(shape=[self.num_capsule, self.input_num_capsule,
                                        self.dim_capsule, self.input_dim_capsule],
                                 initializer=self.kernel_initializer,
                                 name='W')

        self.built = True

    def call(self, inputs, training=None):
        # inputs.shape=[None, input_num_capsule, input_dim_capsule]
        # inputs_expand.shape=[None, 1, input_num_capsule, input_dim_capsule]
        inputs_expand = K.expand_dims(inputs, 1)

        # Replicate num_capsule dimension to prepare being multiplied by W
        # inputs_tiled.shape=[None, num_capsule, input_num_capsule, input_dim_capsule]
        inputs_tiled = K.tile(inputs_expand, [1, self.num_capsule, 1, 1])

        # Compute `inputs * W` by scanning inputs_tiled on dimension 0.
        # x.shape=[num_capsule, input_num_capsule, input_dim_capsule]
        # W.shape=[num_capsule, input_num_capsule, dim_capsule, input_dim_capsule]
        # Regard the first two dimensions as `batch` dimension,
        # then matmul: [input_dim_capsule] x [dim_capsule, input_dim_capsule]^T -> [dim_capsule].
        # inputs_hat.shape = [None, num_capsule, input_num_capsule, dim_capsule]
        inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]), elems=inputs_tiled)

        # Begin: Routing algorithm ---------------------------------------------------------------------#
        # The prior for coupling coefficient, initialized as zeros.
        # b.shape = [None, self.num_capsule, self.input_num_capsule].
        b = tf.zeros(shape=[K.shape(inputs_hat)[0], self.num_capsule, self.input_num_capsule])

        assert self.routings > 0, 'The routings should be > 0.'
        for i in range(self.routings):
            # c.shape=[batch_size, num_capsule, input_num_capsule]
            c = tf.nn.softmax(b, dim=1)

            # c.shape =  [batch_size, num_capsule, input_num_capsule]
            # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule]
            # The first two dimensions as `batch` dimension,
            # then matmal: [input_num_capsule] x [input_num_capsule, dim_capsule] -> [dim_capsule].
            # outputs.shape=[None, num_capsule, dim_capsule]
            outputs = squash(K.batch_dot(c, inputs_hat, [2, 2]))  # [None, 10, 16]

            if i < self.routings - 1:
                # outputs.shape =  [None, num_capsule, dim_capsule]
                # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule]
                # The first two dimensions as `batch` dimension,
                # then matmal: [dim_capsule] x [input_num_capsule, dim_capsule]^T -> [input_num_capsule].
                # b.shape=[batch_size, num_capsule, input_num_capsule]
                b += K.batch_dot(outputs, inputs_hat, [2, 3])
        # End: Routing algorithm -----------------------------------------------------------------------#

        return outputs

    def compute_output_shape(self, input_shape):
        return tuple([None, self.num_capsule, self.dim_capsule])

    def get_config(self):
        config = {
            'num_capsule': self.num_capsule,
            'dim_capsule': self.dim_capsule,
            'routings': self.routings
        }
        base_config = super(CapsuleLayer, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

def PrimaryCap(inputs, dim_capsule, n_channels, kernel_size, strides, padding):
    """
    Apply Conv2D `n_channels` times and concatenate all capsules
    :param inputs: 4D tensor, shape=[None, width, height, channels]
    :param dim_capsule: the dim of the output vector of capsule
    :param n_channels: the number of types of capsules
    :return: output tensor, shape=[None, num_capsule, dim_capsule]
    """
    output = layers.Conv2D(filters=dim_capsule * n_channels, kernel_size=kernel_size, strides=strides, padding=padding,
                           name='primarycap_conv2d')(inputs)
    outputs = layers.Reshape(target_shape=[-1, dim_capsule], name='primarycap_reshape')(output)
    return layers.Lambda(squash, name='primarycap_squash')(outputs)

"""
# The following is another way to implement primary capsule layer. This is much slower.
# Apply Conv2D `n_channels` times and concatenate all capsules
def PrimaryCap(inputs, dim_capsule, n_channels, kernel_size, strides, padding):
    outputs = []
    for _ in range(n_channels):
        output = layers.Conv2D(filters=dim_capsule, kernel_size=kernel_size, strides=strides, padding=padding)(inputs)
        outputs.append(layers.Reshape([output.get_shape().as_list()[1] ** 2, dim_capsule])(output))
    outputs = layers.Concatenate(axis=1)(outputs)
    return layers.Lambda(squash)(outputs)
"""

def CapsNet(input_shape, n_class, routings):
    """
    A Capsule Network on MNIST.
    :param input_shape: data shape, 3d, [width, height, channels]
    :param n_class: number of classes
    :param routings: number of routing iterations
    :return: Two Keras Models, the first one used for training, and the second one for evaluation.
            `eval_model` can also be used for training.
    """
    x = layers.Input(shape=input_shape)

    # Layer 1: Just a conventional Conv2D layer
    conv1 = layers.Conv2D(filters=128, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x)

    # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
    primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid')

    # Layer 3: Capsule layer. Routing algorithm works here.
    digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
                             name='digitcaps')(primarycaps)

    # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
    # If using tensorflow, this will not be necessary. :)
    out_caps = Length(name='capsnet')(digitcaps)

    # Decoder network.
    y = layers.Input(shape=(n_class,))
    masked_by_y = Mask()([digitcaps, y])  # The true label is used to mask the output of capsule layer. For training
    masked = Mask()(digitcaps)  # Mask using the capsule with maximal length. For prediction

    # Shared Decoder model in training and prediction
    decoder = models.Sequential(name='decoder')
    decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
    decoder.add(layers.Dense(1024, activation='relu'))
    decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
    decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))

    # Models for training and evaluation (prediction)
    train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
    eval_model = models.Model(x, [out_caps, decoder(masked)])

    # manipulate model
    noise = layers.Input(shape=(n_class, 16))
    noised_digitcaps = layers.Add()([digitcaps, noise])
    masked_noised_y = Mask()([noised_digitcaps, y])
    manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
    return train_model, eval_model, manipulate_model

def margin_loss(y_true, y_pred):
    """
    Margin loss for Eq.(4). When y_true[i, :] contains not just one `1`, this loss should work too. Not test it.
    :param y_true: [None, n_classes]
    :param y_pred: [None, num_capsule]
    :return: a scalar loss value.
    """
    L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \
        0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1))

    return K.mean(K.sum(L, 1))

def capsule_prediction(image_path):
    # define the model
    model, eval_model, manipulate_model = CapsNet(input_shape=(28, 28, 3),
                                                  n_class=10,
                                                  routings=3)
    eval_model.summary()
    dir_name = os.path.dirname(os.path.abspath(__file__))
    eval_model.load_weights(dir_name + "/best_weights_capsule_train.h5")

    # setting up predictions
    new_img = None
    try:
        new_img = image.load_img(image_path, target_size=(28, 28))
    except OSError:
        return {'error': 'could not identify image'}

    the_image = image.load_img(image_path)
    img = image.img_to_array(new_img)
    img = np.expand_dims(img, axis=0)
    img = img / 255

    prediction, y_recon = eval_model.predict(img)
    li = ["Bacterial_spot", "Early_blight", "healthy", "Late_blight", "Leaf_Mold", "mosaic_virus", "Septoria_leaf_spot",
          "spider_mite", "Target_Spot", "Yellow_Leaf_Curl"]
    # decode the results into a list of tuples (class, description, probability)
    # (one such list for each sample in the batch)
    class_name = li[prediction.argmax()]

    '''
    d = prediction.flatten()
    j = d.max()
    for index,item in enumerate(d):
        if item == j:
            class_name = li[index]
    '''

    # ploting image with predicted class name
    '''plt.figure(figsize=(4, 4))
    plt.imshow(the_image)
    plt.axis('off')
    plt.title(class_name)'''
    K.clear_session()
    return {'disease': class_name}

max_stars_repo_name: Tobenna-KA/mimeai-api

max_issues_repo_name: Tobenna-KA/mimeai-api

max_forks_repo_name: Tobenna-KA/mimeai-api


text:
# code refactored from Magnus Erik Hvass Pedersen tutorials

import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
from sklearn.metrics import confusion_matrix
import math

class Util(object):

    def plot_image(self, image, img_shape=(28,28)):
        plt.imshow(image.reshape(img_shape),
                   interpolation='nearest',
                   cmap='binary')

        plt.show()

    def plot_images(self, images, cls_true, cls_pred=None, img_size=28, img_shape=(28,28)):
        assert len(images) == len(cls_true) == 9

        # Create figure with 3x3 sub-plots.
        fig, axes = plt.subplots(3, 3)
        fig.subplots_adjust(hspace=0.3, wspace=0.3)

        for i, ax in enumerate(axes.flat):
            # Plot image.
            ax.imshow(images[i].reshape(img_shape), cmap='binary')

            # Show true and predicted classes.
            if cls_pred is None:
                xlabel = "True: {0}".format(cls_true[i])
            else:
                xlabel = "True: {0}, Pred: {1}".format(cls_true[i], cls_pred[i])

            # Show the classes as the label on the x-axis.
            ax.set_xlabel(xlabel)

            # Remove ticks from the plot.
            ax.set_xticks([])
            ax.set_yticks([])

        # Ensure the plot is shown correctly with multiple plots
        # in a single Notebook cell.
        plt.show()

    def plot_history(self, history, metric='acc', loc='lower right'):
        # list all data in history
        # print(history.history.keys())
        # summarize history for accuracy
        plt.plot(history.history[metric])
        plt.plot(history.history['val_'+metric])
        if metric == 'acc':
            metric = 'accuracy'
        plt.title('model ' + metric)
        plt.ylabel(metric)
        plt.xlabel('epoch')
        plt.legend(['train', 'test'], loc=loc)
        plt.show()

    def plot_images_2(self, images, cls_true, class_names, cls_pred=None, smooth=True):
        assert len(images) == len(cls_true) == 9

        # Create figure with sub-plots.
        fig, axes = plt.subplots(3, 3)

        # Adjust vertical spacing if we need to print ensemble and best-net.
        if cls_pred is None:
            hspace = 0.3
        else:
            hspace = 0.6
        fig.subplots_adjust(hspace=hspace, wspace=0.3)

        for i, ax in enumerate(axes.flat):
            # Interpolation type.
            if smooth:
                interpolation = 'spline16'
            else:
                interpolation = 'nearest'

            # Plot image.
            ax.imshow(images[i, :, :, :],
                      interpolation=interpolation)

            # Name of the true class.
            cls_true_name = class_names[cls_true[i]]

            # Show true and predicted classes.
            if cls_pred is None:
                xlabel = "True: {0}".format(cls_true_name)
            else:
                # Name of the predicted class.
                cls_pred_name = class_names[cls_pred[i]]

                xlabel = "True: {0}\nPred: {1}".format(cls_true_name, cls_pred_name)

            # Show the classes as the label on the x-axis.
            ax.set_xlabel(xlabel)

            # Remove ticks from the plot.
            ax.set_xticks([])
            ax.set_yticks([])

        # Ensure the plot is shown correctly with multiple plots
        # in a single Notebook cell.
        plt.show()

    def print_test_accuracy(self, session, data, x, y_true, y_pred_cls, num_classes,
                            show_example_errors=False,
                            show_confusion_matrix=False):

        # Split the test-set into smaller batches of this size.
        test_batch_size = 256

        # Number of images in the test-set.
        num_test = len(data.test.images)

        # Allocate an array for the predicted classes which
        # will be calculated in batches and filled into this array.
        cls_pred = np.zeros(shape=num_test, dtype=np.int)

        # Now calculate the predicted classes for the batches.
        # We will just iterate through all the batches.
        # There might be a more clever and Pythonic way of doing this.

        # The starting index for the next batch is denoted i.
        i = 0

        while i < num_test:
            # The ending index for the next batch is denoted j.
            j = min(i + test_batch_size, num_test)

            # Get the images from the test-set between index i and j.
            images = data.test.images[i:j, :]

            # Get the associated labels.
            labels = data.test.labels[i:j, :]

            # Create a feed-dict with these images and labels.
            feed_dict = {x: images,
                         y_true: labels}

            # Calculate the predicted class using TensorFlow.
            cls_pred[i:j] = session.run(y_pred_cls, feed_dict=feed_dict)

            # Set the start-index for the next batch to the
            # end-index of the current batch.
            i = j

        # Convenience variable for the true class-numbers of the test-set.
        cls_true = data.test.cls

        # Create a boolean array whether each image is correctly classified.
        correct = (cls_true == cls_pred)

        # Calculate the number of correctly classified images.
        # When summing a boolean array, False means 0 and True means 1.
        correct_sum = correct.sum()

        # Classification accuracy is the number of correctly classified
        # images divided by the total number of images in the test-set.
        acc = float(correct_sum) / num_test

        # Print the accuracy.
        msg = "Accuracy on Test-Set: {0:.1%} ({1} / {2})"
        print(msg.format(acc, correct_sum, num_test))

        # Plot some examples of mis-classifications, if desired.
        if show_example_errors:
            print("Example errors:")
            self.plot_example_errors(data=data, cls_pred=cls_pred, correct=correct)

        # Plot the confusion matrix, if desired.
        if show_confusion_matrix:
            print("Confusion Matrix:")
            self.plot_confusion_matrix(data=data, num_classes=num_classes, cls_pred=cls_pred)

    def plot_confusion_matrix(self, data, num_classes, cls_pred):
        # This is called from print_test_accuracy() below.

        # cls_pred is an array of the predicted class-number for
        # all images in the test-set.

        # Get the true classifications for the test-set.
        cls_true = data.test.cls

        # Get the confusion matrix using sklearn.
        cm = confusion_matrix(y_true=cls_true,
                              y_pred=cls_pred)

        # Print the confusion matrix as text.
        print(cm)

        # Plot the confusion matrix as an image.
        plt.matshow(cm)

        # Make various adjustments to the plot.
        plt.colorbar()
        tick_marks = np.arange(num_classes)
        plt.xticks(tick_marks, range(num_classes))
        plt.yticks(tick_marks, range(num_classes))
        plt.xlabel('Predicted')
        plt.ylabel('True')

        # Ensure the plot is shown correctly with multiple plots
        # in a single Notebook cell.
        plt.show()

    def plot_example_errors(self, data, cls_pred, correct):
        # This function is called from print_test_accuracy() below.

        # cls_pred is an array of the predicted class-number for
        # all images in the test-set.

        # correct is a boolean array whether the predicted class
        # is equal to the true class for each image in the test-set.

        # Negate the boolean array.
        incorrect = (correct == False)

        # Get the images from the test-set that have been
        # incorrectly classified.
        images = data.test.images[incorrect]

        # Get the predicted classes for those images.
        cls_pred = cls_pred[incorrect]

        # Get the true classes for those images.
        cls_true = data.test.cls[incorrect]

        # Plot the first 9 images.
        self.plot_images(images=images[0:9],
                    cls_true=cls_true[0:9],
                    cls_pred=cls_pred[0:9])

    def plot_weights(self, session, weights, img_shape=(28,28)):
        # Get the values for the weights from the TensorFlow variable.
        w = session.run(weights)

        # Get the lowest and highest values for the weights.
        # This is used to correct the colour intensity across
        # the images so they can be compared with each other.
        w_min = np.min(w)
        w_max = np.max(w)

        # Create figure with 3x4 sub-plots,
        # where the last 2 sub-plots are unused.
        fig, axes = plt.subplots(3, 4)
        fig.subplots_adjust(hspace=0.3, wspace=0.3)

        for i, ax in enumerate(axes.flat):
            # Only use the weights for the first 10 sub-plots.
            if i<10:
                # Get the weights for the i'th digit and reshape it.
                # Note that w.shape == (img_size_flat, 10)
                image = w[:, i].reshape(img_shape)

                # Set the label for the sub-plot.
                ax.set_xlabel("Weights: {0}".format(i))

                # Plot the image.
                ax.imshow(image, vmin=w_min, vmax=w_max, cmap='seismic')

            # Remove ticks from each sub-plot.
            ax.set_xticks([])
            ax.set_yticks([])

    def plot_conv_weights(self, session, weights, input_channel=0):
        # Assume weights are TensorFlow ops for 4-dim variables
        # e.g. weights_conv1 or weights_conv2.

        # Retrieve the values of the weight-variables from TensorFlow.
        # A feed-dict is not necessary because nothing is calculated.
        w = session.run(weights)

        # Get the lowest and highest values for the weights.
        # This is used to correct the colour intensity across
        # the images so they can be compared with each other.
        w_min = np.min(w)
        w_max = np.max(w)

        # Number of filters used in the conv. layer.
        num_filters = w.shape[3]

        # Number of grids to plot.
        # Rounded-up, square-root of the number of filters.
        num_grids = math.ceil(math.sqrt(num_filters))

        # Create figure with a grid of sub-plots.
        fig, axes = plt.subplots(num_grids, num_grids)

        # Plot all the filter-weights.
        for i, ax in enumerate(axes.flat):
            # Only plot the valid filter-weights.
            if i<num_filters:
                # Get the weights for the i'th filter of the input channel.
                # See new_conv_layer() for details on the format
                # of this 4-dim tensor.
                img = w[:, :, input_channel, i]

                # Plot image.
                ax.imshow(img, vmin=w_min, vmax=w_max,
                          interpolation='nearest', cmap='seismic')

            # Remove ticks from the plot.
            ax.set_xticks([])
            ax.set_yticks([])

        # Ensure the plot is shown correctly with multiple plots
        # in a single Notebook cell.
        plt.show()

    def plot_conv_layer(self, session, x, layer, image):
        # Assume layer is a TensorFlow op that outputs a 4-dim tensor
        # which is the output of a convolutional layer,
        # e.g. layer_conv1 or layer_conv2.

        # Create a feed-dict containing just one image.
        # Note that we don't need to feed y_true because it is
        # not used in this calculation.
        feed_dict = {x: [image]}

        # Calculate and retrieve the output values of the layer
        # when inputting that image.
        values = session.run(layer, feed_dict=feed_dict)

        # Number of filters used in the conv. layer.
        num_filters = values.shape[3]

        # Number of grids to plot.
        # Rounded-up, square-root of the number of filters.
        num_grids = math.ceil(math.sqrt(num_filters))

        # Create figure with a grid of sub-plots.
        fig, axes = plt.subplots(num_grids, num_grids)

        # Plot the output images of all the filters.
        for i, ax in enumerate(axes.flat):
            # Only plot the images for valid filters.
            if i<num_filters:
                # Get the output image of using the i'th filter.
                # See new_conv_layer() for details on the format
                # of this 4-dim tensor.
                img = values[0, :, :, i]

                # Plot image.
                ax.imshow(img, interpolation='nearest', cmap='binary')

            # Remove ticks from the plot.
            ax.set_xticks([])
            ax.set_yticks([])

        # Ensure the plot is shown correctly with multiple plots
        # in a single Notebook cell.
        plt.show()

    def plot_transfer_values(self, i, images, transfer_values):
        print("Input image:")

        # Plot the i'th image from the test-set.
        plt.imshow(images[i], interpolation='nearest')
        plt.show()

        print("Transfer-values for the image using Inception model:")

        # Transform the transfer-values into an image.
        img = transfer_values[i]
        img = img.reshape((32, 64))

        # Plot the image for the transfer-values.
        plt.imshow(img, interpolation='nearest', cmap='Reds')
        plt.show()

    def plot_scatter(self, values, cls, num_classes):
        # Create a color-map with a different color for each class.
        import matplotlib.cm as cm
        cmap = cm.rainbow(np.linspace(0.0, 1.0, num_classes))

        # Get the color for each sample.
        colors = cmap[cls]

        # Extract the x- and y-values.
        x = values[:, 0]
        y = values[:, 1]

        # Plot it.
        plt.scatter(x, y, color=colors)
        plt.show()

ext: py

reason: import numpy


text:
from heuslertools.tools.measurement import Measurement
import numpy as np

def conv(file):
    i=0
    for x in open(file):
        if i == 2:
            yield x.replace('#', 'Number').encode()
        else:
            yield x.replace(',', '.').encode()
        i+=1

def load_sims_data(file):
    data = np.genfromtxt((conv(file)), delimiter='\t', comments='#', skip_header=2, names=True)
    return data

class SIMSMeasurement(Measurement):
    """
    Object representing a SIMS measurement

    You can import it with

    ```from heuslertools.sims import SIMSMeasurement```
    """

    def __init__(self, file, **kwargs):
        super().__init__(file, "", **kwargs)

    def _load_data(self):
        return load_sims_data(self.file)

    def append_measurement(self, file):
        """Append data from another file.

        Parameters
        ----------
        file : str
            path of file to append
        """
        self.data = np.append(self.data, load_sims_data(file))


text:
#! -*- coding: utf-8 -*-
import os
import torch
import numpy as np
import re
import unicodedata
from msnlp.data.datasets.token_classification import convert_texts_to_features
from msnlp.data.processors.token_classification import get_entities
from msnlp.data.datasets.classification import ClassificationPredictSet

class PredictProcessing:

    @classmethod
    def pre_processing(cls, texts, tokenizer, model_type, **kwargs):
        """
        :param texts: list of str
        :param labels: list of str
        :return: vectorizer, labelencoder, x, y
        """
        raise NotImplementedError('not implement pre_processing method!')

    @classmethod
    def post_processing(cls, logits, id2label, **kwargs):
        """
        :param text: str
        :param vectorizer: such as countvectorize  word2vec
        :return: numpy of shape
        """
        raise NotImplementedError('not implement pre_processing method!')

class TokenPredictprocess(PredictProcessing):

    @classmethod
    def pre_processing(cls, texts, tokenizer, model_type, **kwargs):
        """
        :param texts: list of sentences.
        :param labels: list of label.
        :param kwargs: model_params.
        :return:
        """
        features_raw = convert_texts_to_features(
            texts,
            tokenizer
        )
        features = []
        input_ids_list = []
        attention_mask_list = []
        token_ids_list = []
        for feature in features_raw:
            input_ids = feature.input_ids
            input_ids_list.append(input_ids)
            attention_mask_list.append(feature.attention_mask)
            token_ids_list.append(feature.token_type_ids)
        input_ids_list = torch.LongTensor(input_ids_list)
        attention_mask_list = torch.LongTensor(attention_mask_list)
        token_ids_list = torch.LongTensor(token_ids_list)
        features = [input_ids_list, attention_mask_list, token_ids_list]
        return features

    @staticmethod
    def clean_space(text):
        """"
        处理多余的空格
        """
        match_regex = re.compile(
            u'[\u4e00-\u9fa5。\.,，:：《》、\(\)（）]{1} +(?<![a-zA-Z])|\d+ +| +\d+|[a-z A-Z]+')
        should_replace_list = match_regex.findall(text)
        order_replace_list = sorted(
            should_replace_list, key=lambda i: len(i), reverse=True)
        for i in order_replace_list:
            if i == u' ':
                continue
            new_i = i.strip()
            text = text.replace(i, new_i)
        return text

    @staticmethod
    def stem(token):
        """获取token的“词干”（如果是##开头，则自动去掉##）
        """
        if token[:2] == '##':
            return token[2:]
        else:
            return token

    @staticmethod
    def _is_control(ch):
        """控制类字符判断
        """
        return unicodedata.category(ch) in ('Cc', 'Cf')

    @staticmethod
    def _is_special(ch):
        """判断是不是有特殊含义的符号
        """
        return bool(ch) and (ch[0] == '[') and (ch[-1] == ']')

    @classmethod
    def rematch(cls, text, tokens):
        """给出原始的text和tokenize后的tokens的映射关系
        args:
            text:模型预测结果
            tokens: 标签id到标签的映射字典
        return:
            tokens里每个token包含的字符在原始文本中的位置。
        example:
            假设使用bert-base的tokenizer
            text: 'how do you do,中华人民共和国成立于1949年，中国总人口超过1400000000人'
            tokens: ['how', 'do', 'you', 'do', ',', '中', '华',
                     '人', '民', '共', '和', '国', '成',
                     '立', '于', '1949', '年', '，', '中',
                     '国', '总', '人', '口', '超', '过',
                     '1400', '##000', '##000', '人']
            return: [[0, 1, 2], [4, 5], [7, 8, 9],
                     [11, 12], [13], [14],
                     [15], [16], [17], [18], [19],
                     [20], [21], [22], [23], [24, 25, 26, 27],
                     [28], [29], [30], [31], [32],
                     [33], [34], [35], [36], [37, 38, 39, 40],
                     [41, 42, 43], [44, 45, 46], [47]]
        """
        normalized_text, char_mapping = '', []
        for i, ch in enumerate(text):
            # lower case
            ch = unicodedata.normalize('NFD', ch)
            ch = ''.join([c for c in ch if unicodedata.category(c) != 'Mn'])
            ch = ch.lower()
            # lower case
            ch = ''.join([
                c for c in ch
                if not (ord(c) == 0 or ord(c) == 0xfffd or cls._is_control(c))
            ])
            normalized_text += ch
            char_mapping.extend([i] * len(ch))

        text, token_mapping, offset = normalized_text, [], 0
        for token in tokens:
            if cls._is_special(token):
                token_mapping.append([])
            else:
                token = cls.stem(token)
                start = text[offset:].index(token) + offset
                end = start + len(token)
                token_mapping.append(char_mapping[start:end])
                offset = end

        return token_mapping

    @classmethod
    def post_processing(cls, logit_list, id2label, **kwargs):
        """
        由于ner返回需要提供实体在文本中的位置，因此输入需要包含原始文本
        args:
            logit_list:模型预测结果
            id2label: 标签id到标签的映射字典
            input_list:输入token转换后的ids
            input_texts:原始文本
            tokenizer:分词工具，辅助定位实体在原始文本中的位置
        """
        input_list = kwargs.get("input_list")
        input_texts = kwargs.get("input_texts", None)
        tokenizer = kwargs.get("tokenizer")
        result_list = []
        for i in range(len(input_list)):
            inputs = input_list[i]
            logits = logit_list[i]
            text = ""
            rematch_list = []
            if input_texts:
                text = input_texts[i]
                input_tokens = tokenizer.convert_ids_to_tokens(inputs)
                rematch_list = cls.rematch(text, input_tokens)
            pred_ids = np.argmax(logits, -1)
            input_ids = inputs
            labels = [id2label.get(i, "O") for i in pred_ids]
            chunk_list = get_entities(labels)
            # [('PER', 0, 1), ('LOC', 3, 3)]
            entity_list = []
            for chunk in chunk_list:
                entity_type = chunk[0]
                tokens = []
                begin = -1
                end = -1
                select_ids = []
                for i in range(chunk[1], chunk[2] + 1):
                    if input_ids[i] > 0:
                        tokens.append(
                            tokenizer._convert_id_to_token(input_ids[i]))
                        select_ids.append(i)
                if len(tokens) > 0:
                    entity = tokenizer.convert_tokens_to_string(tokens)
                    entity = cls.clean_space(entity)
                    if input_texts:
                        begin = rematch_list[select_ids[0]][0]
                        end = rematch_list[select_ids[-1]][-1]
                        entity = text[begin:(end + 1)].strip()
                    if entity:
                        entity_list.append([entity, entity_type, begin, end])

            result_list.append(entity_list)
        return result_list

class ClassificationPredictprocess(PredictProcessing):

    @classmethod
    def pre_processing(cls, texts, tokenizer, model_type, **kwargs):
        """
        :param texts: list of sentences.
        :param labels: list of label.
        :param kwargs: model_params.
        :return:
        """
        task_name = kwargs.get("task_name")
        id2label = kwargs.get("id2label")
        features_raw = ClassificationPredictSet(
            texts,
            id2label,
            tokenizer=tokenizer,
            task_name=task_name)
        features = []
        input_ids_list = []
        attention_mask_list = []
        token_ids_list = []
        for feature in features_raw:
            input_ids = feature.input_ids
            input_ids_list.append(input_ids)
            attention_mask_list.append(feature.attention_mask)
            token_ids_list.append(feature.token_type_ids)
        input_ids_list = torch.LongTensor(input_ids_list)
        attention_mask_list = torch.LongTensor(attention_mask_list)
        token_ids_list = torch.LongTensor(token_ids_list)
        features = [input_ids_list, attention_mask_list, token_ids_list]
        return features

    @classmethod
    def post_processing(cls, logits, id2label, **kwargs):
        predictions = np.argmax(logits, axis=1)
        predictions = [id2label[p] for p in predictions]
        return predictions

    @classmethod
    def post_processing_topn(cls, logits, id2label, topn=1, **kwargs):
        if topn < 1:
            topn = 1
        pred_ids = np.argsort(logits, axis=1)[:, ::-1][:, :topn]
        return [[id2label[i_sort] for i_sort in p_sort] for p_sort in pred_ids]

max_stars_repo_name: open-maip/ms-nlp-sdk

max_issues_repo_name: open-maip/ms-nlp-sdk

max_forks_repo_name: open-maip/ms-nlp-sdk


text:
import numpy as np
from NamePreferences import name_parse, alt_name_parse
def process_names(names):
    shape = np.shape(names)
    ndim = len(shape)
    if ndim==1:
        new_names = process_single_names(names)
    else:
        new_names = process_multiple_names(names)
    return new_names

def process_single_names(names):
    known_names = list(name_parse.keys())
    new_names = np.empty_like(names)
    for i, n in enumerate(names):
        n = alt_name_parse.get(n, n)
        if n not in known_names:
            print(f"New GC: {n}")
            name_parse[n] = n
        new_names[i] = name_parse[n]
    return new_names

def process_multiple_names(names):
    known_names = list(name_parse.keys())
    nGCs = np.shape(names)[0]
    new_names = np.empty((nGCs), dtype='<U50')
    for i in range(nGCs):
        gc_names = names[i, :]
        in_filt = np.isin(gc_names, known_names) * (gc_names != '')
        if in_filt.sum() > 0:
            n = gc_names[in_filt][0]
            new_names[i] = name_parse[alt_name_parse.get(n, n)]
            for n in gc_names:
                name_parse[n] = new_names[i]
        else:
            gc_nam = gc_names[gc_names != ''][0]
            gc_nam = alt_name_parse.get(gc_nam, gc_nam)
            for n in gc_names:
                name_parse[n] = gc_nam
            new_names[i] = gc_nam
    return new_names


text:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import sys
import os
import math
from statsmodels.graphics.tsaplots import plot_acf
import statsmodels.api as sm
from statsmodels.tsa.stattools import acf
import scipy.integrate as integrate
import random

beta_low=float(sys.argv[1])
beta_high=float(sys.argv[2])
nbeta=int(sys.argv[3])
e=float(sys.argv[4])
h=float(sys.argv[5])
nu=float(sys.argv[6])

beta=np.zeros((nbeta))
if( (h).is_integer()): h=int(h)
if( (nu).is_integer()): nu=int(nu)

transient_time=float(sys.argv[7])
tau_max=float(sys.argv[8])

transient_time=int(transient_time)
tau_max=int(tau_max)

L=[]
for ind in range(9, len(sys.argv)):
    L.append(int(sys.argv[ind]))

block_size=20*tau_max

plt.rc('text', usetex=True)
plt.rc('font', family='serif', size=18)
plt.rc('text.latex', preamble=r'\usepackage{bm}')
fig, ax1 = plt.subplots(1, 1, figsize=(9,6))
ax1.set_title(r"$h=%s$; $e=%s$; $\nu=%s$" %(h, e, nu))
ax1.set_xlabel(r"$\beta$")
ax1.set_ylabel(r"$L\rho$")

for l in range(len(L)):
    BASEDIR=("/home/ilaria/Desktop/MultiComponents_SC/Output_2C/L%d_e%s_h%s_nu%s_bmin%s_bmax%s" %(L[l], e,  h, nu, beta_low, beta_high))

    Ds_mean=np.zeros((nbeta))
    Ds_err=np.zeros((nbeta))

    for b in range(nbeta):
        beta[b]=beta_low +b*(beta_high -beta_low)/(nbeta-1)

#        fileDs=("%s/beta_%d/Dual_Stiffness.npy" %(BASEDIR, b))
#        Ds=np.load(fileDs)

        file=h5py.File('%s/beta_%d/Output.h5' %(BASEDIR, b), 'r')
        Ds=np.asarray(file['Measurements']['ds'])

        #cut of the transient regime:
        Ds=Ds[transient_time:]

        Ds_mean[b]=np.mean(Ds)
        Ds_err[b]=np.sqrt(np.var(Ds)/(len(Ds)-1))

    ax1.plot(beta, Ds_mean, '-')
    ax1.errorbar(beta, Ds_mean, yerr=Ds_err, capsize=2,label="L=%s" %L[l])

ax1.legend(loc="best")
plt.tight_layout()
plt.show()

reason: import numpy,import scipy,import statsmodels,from statsmodels


text:
import numpy as np
import time
import keyboard
import math
import threading

def attack_mob(boxes,classes):
    """
    recevies in the player box and the mob box and then will move the player towards the mob and then attack it
    """
    #midpoints X1 and X2
    player, closestmob = calculate_distance(boxes,classes)

    #vertical movement y axis
    if player[0]<closestmob[0]:
        keyboard.teledown()
    else:
        keyboard.teleup()

    # horizontal movement, i messed up the coordinates while creating the tuple index 1 is x, index 0 is y
    if player[1]<closestmob[1]:
        #moveleft and attack
        print("player coord:"+str(player[0])+" "+str(player[1]))
        print("\n mob coord:"+str(closestmob[0])+" "+str(closestmob[1]))
        keyboard.moveRight()
        keyboard.moveRight()
        # keyboard.moveRight()
        keyboard.attackFiveTimes()
        keyboard.loot()
    else:
        # mob is to the right and attack
        print("player coord:"+str(player[0])+" "+str(player[1]))
        print("\n mob coord:"+str(closestmob[0])+" "+str(closestmob[1]))
        keyboard.moveLeft()
        keyboard.moveLeft()
        # keyboard.moveLeft()
        keyboard.attackFiveTimes()
        keyboard.loot()

def filter(detections):
    """
   takes first five detections returns boxes,scores and classes as numpy arrays
    """
    #get first five predictions
    boxes = detections['detection_boxes'][0].numpy()[:5]
    scores = detections['detection_scores'][0].numpy()[:5]
    classes = (detections['detection_classes'][0].numpy() + 1).astype(int)[:5]

    isTherePlayer = False

    if 2 in classes[:]:
        isTherePlayer = True

    return boxes, scores, classes, isTherePlayer

def calculate_distance(boxes,classes):
    """
    calculates the distance between the player and the three mobs, and returns the mob with the shortest distance
    """
    #get the index of the player, returns a numpy array containing the index
    itemindex = np.where(classes==2)
    #get the midpoints, list of tuples
    midpoints =[]
    for i in range(np.shape(boxes)[0]):
        midpoints.append(getBoxesMidpoint(boxes[i]))

    #calculate the distance between the player and the mobs
    distance=np.zeros(5,dtype=np.float32)
    for i in range(np.shape(boxes)[0]):
        if i == itemindex[0][0]:
            distance[i]= 99999.0
        else:
            distance[i]=distance_2points(midpoints[i],midpoints[itemindex[0][0]])

    #get the min index, and return the player coord and mob coord.
    minindex = np.argmin(distance)
    return midpoints[itemindex[0][0]],midpoints[minindex]

def getBoxesMidpoint(box):
    """
    takes in normalized coordinates of the 800x600 screen. coordinates are xmin,ymin,xmax,ymax
    returns a tuple of the midpoint
    """
    #denormalize them
    normalized_coord = np.array([box[0]*806,box[1]*629,box[2]*806,box[3]*629],dtype=np.float32)
    #offset from the origin
    return (((normalized_coord[2]-normalized_coord[0])/2)+normalized_coord[0],((((normalized_coord[3]-normalized_coord[1])/2))+normalized_coord[1]))

def distance_2points(pt1,pt2):
    """
    returns distance between two points pt1(x1,y1),pt2(x2,y2). points as tuples.
    """
    return math.hypot(pt2[0]-pt1[0], pt2[1]-pt1[1])

def autobuff(stop_event):
    starttime = time.time()
    while not stop_event.wait(1):
        print("Buffing!")
        keyboard.buff()
        keyboard.buff()
        keyboard.buff()
        time.sleep(65.0 - ((time.time() - starttime) % 65.0))

def autocc(stop_event):
    starttime = time.time()
    while not stop_event.wait(1):
        print("CC'ing!")
        keyboard.cc()
        time.sleep(90.0 - ((time.time() - starttime) % 90.0))

if __name__ == "__main__":
    pass

max_stars_repo_name: Aearsears/mapleai

max_issues_repo_name: Aearsears/mapleai

max_forks_repo_name: Aearsears/mapleai


text:
from nltk.corpus import stopwords
from pulp import LpMaximize, LpProblem, LpVariable, GLPK
import numpy as np

STOPWORDS = stopwords.words('english') + ["'", '"', '!', ',', '.', '?', '-s', '-ly', '</s> ', 's']

def get_summary_parameters(concept_idf: dict, lemmas: list):
    """
    This function is used to get the parameters used in ILP. Sentences shorter than 3 or with no concept is removed.
    So n_concept is the number of the concepts and n_sentences is the number of valid sentences.

    :param concept_idf: a dictionary of the IDF of each possible concept
    :param lemmas: list of list of lemmas
    :return: occ: int8 array of shape [n_concepts, n_sentences]; w: mat of shape [n_concepts], sentence_id_map: map the
    sentences used in the computation to the original idx.
    """
    concept_vocab = dict({})
    concepts = list([])
    sentence_id_map = list([])
    num_concepts = 0
    num_sentences = 0

    for j, sentence in enumerate(lemmas):
        if len(sentence) < 5:
            continue

        concept_in_sentence = list([])
        for i in range(1, len(sentence)):
            concept = (sentence[i-1], sentence[i])
            if not (concept[0] in STOPWORDS and concept[1] in STOPWORDS):
                if concept not in concept_vocab:
                    concept_vocab[concept] = num_concepts
                    concept_in_sentence.append(num_concepts)
                    num_concepts += 1
                else:
                    concept_in_sentence.append(concept_vocab[concept])

        if len(concept_in_sentence) > 0:
            sentence_id_map.append(j)
            concepts.append(concept_in_sentence)
            num_sentences += 1

    occ = np.zeros((num_concepts, num_sentences), np.int8)
    for i, sentence in enumerate(concepts):
        for concept in sentence:
            occ[concept][i] = 1
    w = np.zeros([num_concepts])
    for concept, idx in concept_vocab.items():
        w[idx] = concept_idf[concept]

    return occ, w, sentence_id_map

def get_summary(concept_idf, lemmas, max_length):
    occ, w, sentence_id_map = get_summary_parameters(concept_idf, lemmas)
    num_concepts, num_sentences = occ.shape
    concept_vars = [LpVariable("concept_" + str(idx), cat='Binary') for idx in range(num_concepts)]
    sentence_vars = [LpVariable("sentence_" + str(idx), cat='Binary') for idx in range(num_sentences)]

    l_constrained = sum([len(lemmas[idx]) * s_var for idx, s_var in zip(sentence_id_map, sentence_vars)]) <= max_length
    con_1 = [[s_var * occ[i][j] <= c_var for j, s_var in enumerate(sentence_vars)]
             for i, c_var in enumerate(concept_vars)]
    con_2 = [sum([s_var * occ[i][j] for j, s_var in enumerate(sentence_vars)]) >= c_var
             for i, c_var in enumerate(concept_vars)]

    target = sum([c_w * c_var for c_w, c_var in zip(w, concept_vars)])

    model = LpProblem(name="Summary", sense=LpMaximize)
    model += (l_constrained, "l_constrained")
    for i, con in enumerate(con_2):
        model += (con, "con_2_" + str(i))
    cnt = 0
    for ll in con_1:
        for con in ll:
            model += (con, "con_1_" + str(cnt))
            cnt += 1

    model += target
    model.solve(solver=GLPK(msg=False))

    selected_sentence = list([])
    for var in model.variables():
        if var.name.startswith("sentence") and var.value() == 1:
            selected_sentence.append(sentence_id_map[int(var.name[9:])])
    selected_sentence_id = sorted(selected_sentence)

    return selected_sentence_id


text:
import plotly.express as px
import numpy as np
import pandas as pd
import pickle
import datetime
import zipfile
import requests
import io
import sys

#Helper function
def get_divide_cols_fn(c1, c2, res):
    def f(df):
        df[res] = df[c1]/df[c2]
        return df
    return f

def get_JHU_data(name, verbose = False):
    if verbose: print("Retrieving JHU {}".format(name))
    jhu_url = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{}_global.csv'.format(name)
    jhu_df = pd.read_csv(jhu_url, error_bad_lines = False)

    jhu_df = jhu_df.groupby('Country/Region').sum().drop(['Lat', 'Long'], axis=1)
    jhu_df = jhu_df.stack().rename(name.capitalize()).to_frame()

    jhu_df.index = jhu_df.index.rename(['Country', 'Date'])
    jhu_df.index = jhu_df.index.set_levels([jhu_df.index.levels[0], pd.to_datetime(jhu_df.index.levels[1])])

    return jhu_df

def get_us_data(verbose = False):
    if verbose: print("Retrieving US data")
    url = 'https://covidtracking.com/api/us/daily.csv'
    us_df = pd.read_csv(url, index_col = 0)
    us_df.index = pd.to_datetime(us_df.index, format = '%Y%m%d').rename('Date')
    us_df = us_df.sort_index()

    col_map = {
        'positive': 'Confirmed',
        'totalTestResults': 'TotalTests',
        'hospitalized': 'Hospitalized',
        'death': 'Deaths'
    }

    us_df = us_df[list(col_map.keys())].rename(mapper=  col_map, axis=1)
    us_df['Country'] = 'US'
    us_df = us_df.set_index([us_df.Country, us_df.index]).drop(['Country'], axis=1)

    return us_df

def get_italy_data(verbose = False):
    if verbose: print("Retrieving Italy data")
    url = 'https://raw.githubusercontent.com/pcm-dpc/COVID-19/master/dati-andamento-nazionale/dpc-covid19-ita-andamento-nazionale.csv'
    italy_df = pd.read_csv(url, index_col = 0)
    italy_df.index = pd.to_datetime(italy_df.index).normalize().rename('Date')
    col_map = {
        'dimessi_guariti': 'Recovered',
        'totale_ospedalizzati': 'Hospitalized',
        'isolamento_domiciliare': 'Home',
        'tamponi': 'TotalTests',
        'stato': 'Country',
        'totale_casi': 'Confirmed',
        'deceduti': 'Deaths'
    }
    italy_df = italy_df[list(col_map.keys())].rename(mapper =  col_map, axis=1)
    italy_df.Country = 'Italy'
    italy_df = italy_df.set_index([italy_df.Country, italy_df.index]).drop(['Country'], axis=1)

    return italy_df

################################
# Below function too slow
################################
# def clean_mx_data(df, date):
#     """
#     Should return a row of a df, indexed by Date, Country (Mexico) with columns Confirmed,
#     Totaltests (confirmed + negative), Deaths. Hospitalization would also be nice.
#     """
#     result_df = {}

#     result_df['Date'] = date
#     result_df['Country'] = 'Mexico'
#     result_df['Confirmed'] = (df['RESULTADO'] == 1).sum()
#     result_df['TotalTests'] = (df['RESULTADO'] == 2).sum() + result_df['Confirmed']
#     result_df['Deaths'] = df.query('RESULTADO == 1 & FECHA_DEF != "9999-99-99"').FECHA_DEF.count()

#     result_df = pd.DataFrame(result_df, index = [0])

#     return result_df

# def get_mexico_data(days_backwards = 3, verbose = False):
    # end = datetime.date.today()
    # start = datetime.date.today() - datetime.timedelta(days = days_backwards)
    # dates_to_see = pd.date_range(start, end)

    # res = pd.DataFrame()

    # for d in dates_to_see:
    #     if verbose: print('Mexico pulling for {}'.format(d.strftime('%d-%m-%Y')))
    #     url = "http://187.191.75.115/gobmx/salud/datos_abiertos/historicos/datos_abiertos_covid19_{}.zip".format(
    #     d.strftime('%d.%m.%Y'))
    #     r = requests.get(url, stream = True)

    #     #fof represents 404 error on first try. Means we've reached last day of data.
    #     fof = r.status_code == 404
    #     if fof:
    #         #If it's not under historical, grab most recent
    #         r = requests.get("http://187.191.75.115/gobmx/salud/datos_abiertos/datos_abiertos_covid19.zip")

    #     f = zipfile.ZipFile(io.BytesIO(r.content))
    #     with f.open(f.namelist()[0]) as thefile:
    #         df = pd.read_csv(thefile)

    #     #Takes care of case when we had to go to recent df and it's yesterday's data.
    #     if fof and df.FECHA_ACTUALIZACION.iloc[0] != d:
    #         continue

    #     c = clean_mx_data(df, d)
    #     res = res.append(c)

    # return res.set_index(['Country', 'Date']).astype('float64')

def get_mexico_data(verbose = False):
    base_url = "https://raw.githubusercontent.com/mariorz/covid19-mx-time-series/master/data/covid19_{}_mx.csv"
    types = [('confirmed', 'Confirmed'),
             ('deaths', 'Deaths'),
             ('negatives', 'Negatives')]
    base_df = pd.DataFrame()

    for url_name, my_name in types:
        if verbose: print('Retrieving Mexico {}'.format(url_name))
        df = pd.read_csv(base_url.format(url_name))
        df = df.drop(columns = ['Estado']).sum(axis=0).rename_axis('Date').rename(my_name).to_frame()
        base_df = base_df.join(df, how = 'outer')

    base_df['TotalTests'] = base_df['Confirmed'] + base_df['Negatives']
    base_df['Country'] = ['Mexico'] * len(base_df.index)

    base_df = base_df.drop(columns = 'Negatives')
    base_df.index = pd.to_datetime(base_df.index, dayfirst = True)

    base_df = base_df.replace(0, np.nan).reset_index().set_index(['Country', 'Date'])

    return base_df

def clean_chile_data(df, df_name):
    if df_name == 'TotalesNacionales':
        rename_map = {
            'Casos totales': 'Confirmed',
            'Casos recuperados': 'Recovered',
            'Fallecidos': 'Deaths',
        }
        df = df.T
        df.columns = df.iloc[0].str.rstrip()
        # df.index = pd.to_datetime(df.index)
        df = df[1:].rename(columns = rename_map)[list(rename_map.values())]
        df.index = pd.to_datetime(df.index)
    elif df_name == 'PCREstablecimiento':
        df = df.query('Establecimiento == "Total realizados"').T[2:]
        df.columns = ['TotalTests']
    elif df_name == 'UCI':
        df = df.T.drop(['Region', 'Codigo region', 'Poblacion']).sum(axis=1).rename('ICU').to_frame()
    elif df_name == 'HospitalizadosEtario':
        df.T.drop(['Grupo de edad', 'Sexo']).astype('int64').sum(axis=1).rename('Hospitalized').to_frame()

    return df

def get_chile_data(verbose = False):
    prod_filename_map = {
        5: 'TotalesNacionales',
        8: 'UCI',
        17: 'PCREstablecimiento',
    }
    general_URL_form = 'https://raw.githubusercontent.com/MinCiencia/Datos-COVID19/master/output/producto{}/{}.csv'

    master_df = pd.DataFrame()

    for num, name in prod_filename_map.items():
        url = general_URL_form.format(num, name)
        if verbose: print("Retrieving Chile {}".format(name))
        df = pd.read_csv(url)

        df = clean_chile_data(df, name)

        master_df = master_df.join(df, how = 'outer')

    master_df['Country'] = ['Chile'] * len(master_df.index)
    master_df.index = master_df.index.rename('Date')
    master_df.index = pd.to_datetime(master_df.index)

    return master_df.reset_index().set_index(['Country', 'Date']).astype('float64').replace(0, np.NaN)

def get_panama_data(verbose = False):
    if verbose: print("Retrieving Panama data")
    df = pd.read_csv('https://raw.githubusercontent.com/c0t088/DAP-Panama/master/data_covid_pma_dia.csv')

    fields_relevant = {
        'Fecha': 'Date',
        'Casos_confirmados': 'Confirmed',
        'Fallecidos_tot': 'Deaths',
        'Domicilio': 'Home',
        'Hosp_sala': 'Hospitalized',
        'Hosp_uci': 'ICU',
        'Recuperados': 'Recovered',
        'Pruebas negativas': 'NegTests',
        'Pruebas positivas': 'PosTests'
        }

    total_tests = df[['Fecha', 'Pruebas negativas', 'Pruebas positivas']].set_index('Fecha').cumsum().sum(axis=1).rename('TotalTests')

    df = df[list(fields_relevant.keys())]

    df.columns = [fields_relevant.get(c, c) for c in df.columns]
    df['Date'] = pd.to_datetime(df['Date'])
    df = df.set_index('Date')
    df.index = pd.to_datetime(df.index)

    df = df.join(total_tests).drop(columns = ['NegTests', 'PosTests'])
    df['Hospitalized'] = df['Hospitalized'] + df['ICU']
    df['Country'] = ['Panama'] * len(df.index)

    return df.reset_index().set_index(['Country', 'Date'])

def join_dfs(default, other):
    #For joining dataframes, using one as default and filling in missing data
    overlap = set(default.columns).intersection(set(other.columns))
    default = pd.merge(default, other, left_index=True, right_index = True, how = 'outer', suffixes = ('', '_other'))
    for col in overlap:
        default[col] = default[col].combine_first(default[col + '_other'])

    default = default.drop([col + '_other' for col in overlap], axis=1)

    return default

def get_pop_dict():
    pop = pd.read_csv('data/WPP2019_TotalPopulationBySex.csv')
    pop = pop[pop.Time.eq(2020) & pop.Variant.eq('Medium')][['Location', 'PopTotal']].set_index('Location')
    pop.columns = ['Population']
    pop.loc['Diamond Princess'] = 3700

    aliases = {
        'US': 'United States of America',
        'Iran': 'Iran (Islamic Republic of)',
        'Bolivia': 'Bolivia (Plurinational State of)',
        'Brunei': 'Brunei Darussalam',
        'Korea, South': 'Republic of Korea',
        'Moldova': 'Republic of Moldova',
        'Russia': 'Russian Federation',
        "Cote d'Ivoire": "Côte d'Ivoire",
        'Syria': 'Syrian Arab Republic',
        'Tanzania': 'United Republic of Tanzania',
        'West Bank and Gaza': 'State of Palestine',
        'Venezuela': 'Venezuela (Bolivarian Republic of)',
        'Vietnam': 'Viet Nam',
        'Taiwan*': 'China, Taiwan Province of China',
        'Congo (Brazzaville)': 'Congo',
        'Congo (Kinshasa)': 'Democratic Republic of the Congo'
    }

    aliases = {v:k for k, v in aliases.items()}
    pop = pop.rename(mapper = aliases, axis=0) * 1000
    return pop

def merge_pop(df, verbose = False):
    if verbose: print("Retrieving population data.")
    pop = get_pop_dict()
    df = df.merge(pop, how = 'left', left_on = 'Country', right_index = True)
    return df

def data_clean(out_file, hard_refresh = False, verbose = False):

    dat = pd.read_csv('data/Covid19_Data.csv', index_col = 0)
    dat.index = pd.to_datetime(dat.index)

    col = dat.columns.str.split('_', expand=True)
    dat.columns = col
    dat.index = dat.index.rename("Date")
    dat.columns = dat.columns.rename(['Country', 'Col'])
    dat = dat.stack(level=0)
    dat.index = dat.index.reorder_levels(['Country', 'Date'])

    #Join the dataframes, add US, Italy, Mexico, Chile, Panama data from source.
    dat = join_dfs(dat, get_us_data(verbose = verbose))
    dat = join_dfs(dat, get_italy_data(verbose = verbose))
    dat = join_dfs(dat, get_mexico_data(verbose = verbose))
    dat = join_dfs(dat, get_chile_data(verbose = verbose))
    dat = join_dfs(dat, get_panama_data(verbose = verbose))

    #Get Hopkins data
    jhu_df = get_JHU_data('confirmed', verbose = verbose)
    jhu_death_df = get_JHU_data('deaths', verbose = verbose)
    jhu_df = jhu_death_df.join(jhu_df)
    jhu_death_df = None
    jhu_recovered_df = get_JHU_data('recovered', verbose = verbose)
    jhu_df = jhu_df.join(jhu_recovered_df)
    jhu_recovered_df = None
    dat = join_dfs(dat, jhu_df)

    jhu_df = None
    dat = merge_pop(dat, verbose = verbose)
    if verbose:
        print("Merged dataframes")

    # if not hard_refresh:
    #     if verbose: print("Joining previously compiled data")
    #     old = pickle.load(open('data/compiled_data.p', 'rb')).reset_index()
    #     old.Date = pd.to_datetime(old.Date)
    #     old = old.set_index(['Country', 'Date'])
    #     dat = join_dfs(old, dat)

    #Tens of thousands of population
    dat['Pop10k'] = dat['Population'] / 10000

    #Add New Cases, New Tests
    dat['NewCases'] = dat.groupby('Country')['Confirmed'].diff()
    dat['NewTests'] = dat.groupby('Country')['TotalTests'].diff()
    dat['NewDeaths'] = dat.groupby('Country')['Deaths'].diff()
    dat['DaysSinceFirst'] = (dat['Confirmed'] >= 1).groupby('Country').cumsum().replace(0, np.nan) - 1
    dat['ConfirmedGrowth'] = dat.groupby('Country')['Confirmed'].pct_change() * 100.
    dat['DaysSinceTenthDeath'] = (dat['Deaths'] >= 10).groupby('Country').cumsum().replace(0, np.nan)
    dat['ActiveCases'] = dat['Confirmed'] - dat['Recovered']

    dat = dat.groupby('Country').apply(get_divide_cols_fn("NewCases", "NewTests", "DailyPosTestRate"))
    dat = dat.groupby('Country').apply(get_divide_cols_fn("Confirmed", "TotalTests", "CumulativePosTestRate"))
    dat = dat.groupby('Country').apply(get_divide_cols_fn("Deaths", "Confirmed", "CumulativeDeathRate"))
    dat = dat.groupby('Country').apply(get_divide_cols_fn('Hospitalized', 'Confirmed', 'HospitalizationRate'))
    dat = dat.groupby('Country').apply(get_divide_cols_fn('TotalTests', 'Pop10k', 'TotalTestsPer10k'))
    dat = dat.groupby('Country').apply(get_divide_cols_fn('Confirmed', 'Pop10k', 'ConfirmedPer10k'))
    dat = dat.groupby('Country').apply(get_divide_cols_fn('NewTests', 'Pop10k', 'NewTestsPer10k'))

    #Days Since Shutdown
    dat['DaysSinceShutdown'] = np.nan
    x = (dat[dat['Shutdown'] == 1]).groupby('Country').shift(0)
    s_countries = list(x.index.get_level_values(0))

    for s in s_countries:
        old = x.xs(s, level=0).index[0]
        first = dat.xs(s, level=0).index[0]
        d = len(dat.xs(s, level=0).loc[first:old].index)
        new = dat.xs(s, level=0).Shutdown.shift(-d + 1).fillna(1).cumsum() - (d)
        dat.loc[s, 'DaysSinceShutdown'] = new

    #Keep the ones with more than 1 confirmed day
    dat = dat[dat['Confirmed'] >= 1]

    #Set the index
    dat = dat.set_index([dat.index, "DaysSinceFirst", "DaysSinceTenthDeath", "DaysSinceShutdown"
                        ]).reorder_levels(["Country", "DaysSinceFirst", "DaysSinceTenthDeath", "DaysSinceShutdown",
                                        "Date"]).sort_index()
    #Pickle object, return success
    pickle.dump(dat, open(out_file, 'wb'))

    return dat

# sched = BackgroundScheduler()

# @sched.scheduled_job('cron', day_of_week='mon-sun',  hour = 23)
# def scheduled_job():
#     data_clean('compiled_data.p')

if __name__ == "__main__":
    # sched.start()
    data_clean('data/compiled_data.p', hard_refresh = True, verbose=True)


text:
import argparse
import pickle
import torch
from torch.utils.data import DataLoader, Dataset
import torch.nn.functional as F
from torch import nn
import numpy as np

import gym
from gym.wrappers.time_limit import TimeLimit

import pickle

from pomp.example_problems.gym_pendulum_baseenv import PendulumGoalEnv
from pomp.example_problems.robotics.fetch.reach import FetchReachEnv
from pomp.example_problems.robotics.fetch.push import FetchPushEnv
from pomp.example_problems.robotics.fetch.slide import FetchSlideEnv
from pomp.example_problems.robotics.fetch.pick_and_place import FetchPickAndPlaceEnv
from HER_mod.rl_modules.velocity_env import MultiGoalEnvironment, CarEnvironment

from collect_distance_data import my_dataset

num_episodes = 5000
epochs = 500

num_episodes = None
epochs = None

def get_args():
	parser = argparse.ArgumentParser()
	parser.add_argument('--env-name', type=str, default='FetchReach-v1', help='the environment name')
	parser.add_argument('--max-steps', type=int, default=None, help='number of steps allowed in rollout')
	parser.add_argument('--agent-location', type=str, default="saved_models/her_FetchReach.pkl", help='location the agent is stored')
	parser.add_argument('--cuda', action='store_true', help='if use gpu do the acceleration')
	parser.add_argument('--batch-size', type=int, default=512, help='the sample batch size')
	parser.add_argument('--hidden-size', type=int, default=256, help='the sample batch size')
	parser.add_argument('--episodes', type=int, default=5000, help='number of episodes to sample for data')
	parser.add_argument('--epochs', type=int, default=500, help='number of epochs to train for')
	parser.add_argument('--dropout-rate', type=float, default=.2, help='dropout rate for dropout layers')
	parser.add_argument('--p2p', action='store_true', help='if we\'re using the p2p agent')

	args = parser.parse_args()
	return args

def get_agent(filename):
    with open(filename, 'rb') as f:
        agent = pickle.load(f)
    return agent

def make_env(args):
    if args.env_name == "MultiGoalEnvironment":
        env = MultiGoalEnvironment("MultiGoalEnvironment", time=True, vel_goal=False)
    elif "Car" in args.env_name:
        env = CarEnvironment("CarEnvironment", time=True, vel_goal=False)
        # env = TimeLimit(CarEnvironment("CarEnvironment", time=True, vel_goal=False), max_episode_steps=50)
    elif args.env_name == "PendulumGoal":
        env = TimeLimit(PendulumGoalEnv(g=9.8), max_episode_steps=200)
    elif "FetchReach" in args.env_name:
        env = TimeLimit(FetchReachEnv(), max_episode_steps=50)
    elif "FetchPush" in args.env_name:
        env = TimeLimit(FetchPushEnv(), max_episode_steps=50)
    elif "FetchSlide" in args.env_name:
        env = TimeLimit(FetchSlideEnv(), max_episode_steps=50)
    elif "FetchPickAndPlace" in args.env_name:
        env = TimeLimit(FetchPickAndPlaceEnv(), max_episode_steps=50)
    else:
        env = gym.make(args.env_name)

    return env

class net(nn.Module):
	def __init__(self, input_shape, args):
		super(net, self).__init__()
		# self.norm1 = nn.LayerNorm(256)
		# self.norm2 = nn.LayerNorm(256)
		# self.norm3 = nn.LayerNorm(256)
		self.norm1 = nn.BatchNorm1d(input_shape)
		# self.norm1 = nn.BatchNorm1d(args.hidden_size)
		self.norm2 = nn.BatchNorm1d(args.hidden_size)
		self.norm3 = nn.BatchNorm1d(args.hidden_size)
		self.norm4 = nn.BatchNorm1d(args.hidden_size)
		self.fc1 = nn.Linear(input_shape, args.hidden_size)
		self.fc2 = nn.Linear(args.hidden_size, args.hidden_size)
		self.fc3 = nn.Linear(args.hidden_size, args.hidden_size)
		self.q_out = nn.Linear(args.hidden_size, 1)
		self.do1 = nn.Dropout(args.dropout_rate)
		self.do2 = nn.Dropout(args.dropout_rate)
		self.do3 = nn.Dropout(args.dropout_rate)
		# self.do1 = nn.AlphaDropout(args.dropout_rate)
		# self.do2 = nn.AlphaDropout(args.dropout_rate)
		# self.do3 = nn.AlphaDropout(args.dropout_rate)
		self.act = F.relu
		# self.act = F.selu

	# def forward(self, x):
	#     x = self.norm1(x)
	#     x = F.relu(self.fc1(x))
	#     x = self.norm2(x)
	#     x = self.do1(x)
	#     x = F.relu(self.fc2(x))
	#     x = self.norm3(x)
	#     x = self.do2(x)
	#     x = F.relu(self.fc3(x))
	#     x = self.do3(x)
	#     q_value = self.q_out(x)

	#     return q_value

	def forward(self, x):
		self.act = F.relu
		x = self.norm1(x)
		x = self.act(self.fc1(x))
		x = self.norm2(x)
		x = self.do1(x)
		x = self.act(self.fc2(x))
		x = self.norm3(x)
		x = self.do2(x)
		x = self.act(self.fc3(x))
		x = self.norm4(x)
		x = self.do3(x)
		q_value = self.q_out(x)

		return q_value

def train(args, model, training_dataloader, testing_dataloader):
	opt = torch.optim.Adam(model.parameters(), lr=.001)
	for i in range(args.epochs):
		if i % max(args.epochs//10, 1) == 0:
			print("Epoch: " + str(i))
		train_l = []
		model.train()
		for train_features, train_labels in training_dataloader:
			if args.cuda:
				train_features=train_features.cuda()
				train_labels=train_labels.cuda()
			train_out = model(train_features)
			l = ((train_labels - train_out)**2).mean()
			opt.zero_grad()
			l.backward()
			train_l.append(l.detach())
			opt.step()
		train_loss = sum(train_l)/len(train_l)

		if i % max(args.epochs//10, 1) == 0:
			print("Training loss: " + str(train_loss))

		model.eval()
		test_l = []
		for test_features, test_labels in testing_dataloader:
			if args.cuda:
				test_features=test_features.cuda()
				test_labels=test_labels.cuda()
			out = model(test_features)
			l = ((test_labels - out)**2).mean()
			test_l.append(l.detach())
		test_loss = sum(test_l)/len(test_l)
		if i % max(args.epochs//10, 1) == 0:
			print("Testing loss: " + str(test_loss))

# args = get_args()
# agent = get_agent(args.agent_location)

# data, labels, input_shape = collect_data_goal_env(agent, args)
# training_dataset = my_dataset(data[:-2*len(data)//10], labels[:-2*len(labels)//10])
# testing_dataset  = my_dataset(data[-2*len(data)//10:], labels[-2*len(labels)//10:])

if __name__ == "__main__":
	args = get_args()
	if args.p2p:
		suffix = '_p2p.pkl'
	else:
		suffix = '.pkl'

	with open('datasets/train_' + args.env_name + suffix, 'rb') as f:
		training_dataset = pickle.load(f)

	with open('datasets/test_' + args.env_name + suffix, 'rb') as f:
		testing_dataset = pickle.load(f)

	train_dataloader = DataLoader(training_dataset, batch_size=args.batch_size, shuffle=True, pin_memory=True)
	test_dataloader = DataLoader(testing_dataset, batch_size=args.batch_size, shuffle=True, pin_memory=True)

	env = make_env(args)
	obs = env.reset()

	if args.p2p:
		input_shape = 2*obs['observation'].shape[0]
	else:
		input_shape = obs['observation'].shape[0] + obs['desired_goal'].shape[0]

	model = net(input_shape, args)

	if args.cuda:
		model.cuda()

	train(args, model, train_dataloader, test_dataloader)
	model.cpu()
	with open(args.agent_location[:-4] + '_distance' + suffix, 'wb') as f:
	    pickle.dump(model, f)
	    print("Saved models")


text:
import pytest
import numpy as np
import pandas as pd

from .decorator import Decorator, frame_exclude

np.random.seed(1)

def func():
    "This is a doc for method"
    pass

def test_decorator_doc():
    f = frame_exclude(func)

    assert f.__doc__ == 'This is a doc for method'

def test_decorator_init_func():
    class a(Decorator):
        def setup_func(self, func):
            return sum

    f = a(func)

    assert f([10, 20]) == 30


text:
import numpy as np
import pandas as pd
import cv2
from PIL import Image
import json

CSV_FILE = 'driving_log.csv'
TOP_CROP = 56
BOT_CROP = 16
STEERING_ADJ = 0.045
IMSIZE = 80

def load_img(image_name):
    '''
    Load and process image. image_name is a str of path and image name.
    Return cropped, resized, and normalized image of an np.array.
    '''
    im = Image.open(image_name)
    im = np.asarray(im)
    return im

def crop_resize_img(image):
    '''
    Crop and resize image.
    '''
    image = image.crop(box=(0, TOP_CROP, 320, 160-BOT_CROP))
    image = image.resize((IMSIZE, IMSIZE))
    return image

def get_batch_data(batch_size=64):
    '''
    Get batch data from data file.
    Return image file names and steering number.
    '''
    data = pd.read_csv(CSV_FILE, sep=',')
    data_size = len(data)
    random_idx = np.random.randint(0, data_size, batch_size)
    batch_data = []
    for index in random_idx:
        img_selector = np.random.randint(0, 3)
        if img_selector == 0:
            img_file = data.iloc[index, 0].strip()
            steer = data.iloc[index, 3]
            batch_data.append((img_file, steer))
        elif img_selector == 1:
            img_file = data.iloc[index, 1].strip()
            steer = data.iloc[index, 3] + STEERING_ADJ
            batch_data.append((img_file, steer))
        else:
            img_file = data.iloc[index, 2].strip()
            steer = data.iloc[index, 3] - STEERING_ADJ
            batch_data.append((img_file, steer))
    return batch_data

def batch_generator(batch_size=64):
    """
    Generate the training batch.
    """
    while True:
        X_batch = []
        y_batch = []
        data = get_batch_data(batch_size)
        for img_file, steer in data:
            im = load_img(img_file)
            im, steer = random_transform_img(im, steer)
            im = Image.fromarray(im)
            im = crop_resize_img(im)
            im = np.asarray(im)
            im = cv2.cvtColor(im, cv2.COLOR_RGB2YUV)
            X_batch.append(im)
            y_batch.append(steer)
        yield (np.array(X_batch), np.array(y_batch))

def random_transform_img(image, steer):
    '''
    Randomly transform image in format of np.array.
    Return randomly flipped, sheared, rotated, gamma-adjusted image.
    '''
    image, steer = random_flip(image, steer)
    image, steer = random_shear(image, steer)
    image, steer = random_rotate(image, steer)
    image = random_gamma(image)
    return image, steer

def random_flip(image, steer):
    '''
    Randomly flip image.
    '''
    if np.random.binomial(1, .5, 1)[0]:
        image = np.fliplr(image)
        steer = - steer
    return image, steer

def random_shear(image, steer, shear_dist=50):
    '''
    Randomly shear image.
    '''
    rows, cols, _ = image.shape
    d = np.random.randint(-shear_dist, shear_dist+1)
    pt_1 = np.float32([[0, rows], [cols, rows], [cols/2, rows/2]])
    pt_2 = np.float32([[0, rows], [cols, rows], [cols/2+d, rows/2]])
    dsteer = d / (rows/2) * .4875
    M = cv2.getAffineTransform(pt_1, pt_2)
    image = cv2.warpAffine(image, M, (cols,rows), borderMode=1)
    steer += dsteer
    return image, steer

def random_rotate(image, steer, angle_range=5):
    '''
    Randomly rotate image.
    '''
    angle = np.random.uniform(-angle_range, angle_range)
    rows, cols, _ = image.shape
    M = cv2.getRotationMatrix2D((cols/2,rows/2),angle,1)
    image = cv2.warpAffine(image, M, (cols,rows), borderMode=1)
    dsteer = - angle * np.pi / 180 * .4875
    steer += dsteer
    return image, steer

def random_gamma(image, gamma_range=0.7):
    '''
    Random adjust image gamma.
    '''
    gamma = np.random.uniform(1.-gamma_range, 1.+gamma_range)
    invGamma = 1.0 / gamma
    table = np.array([((i / 255.0) ** invGamma) * 255
            for i in np.arange(0, 256)]).astype("uint8")
    return cv2.LUT(image, table)

def save_model(model):
    '''
    Save model files.
    '''
    json_string = model.to_json()
    with open('model.json', 'w') as outfile:
        json.dump(json_string, outfile)
    model.save_weights('model.h5')

max_stars_repo_name: hangyao/behavioral-cloning

max_issues_repo_name: hangyao/behavioral-cloning

max_forks_repo_name: hangyao/behavioral-cloning


text:
import numpy as np
import matplotlib.pyplot as plt
from . import head_draw_functions as hdf
import IL

# This guy creates a dataset matrix of dimension resolution x resolution x figNumber using the
# using the "drawFunction" method
def generator(
    datasetType,
    figure, # which figure are we drawing the dataset with
    resolution, # pixels in one dimension
    figNumber,  # how many figures do you want to draw
    gridParams, # ((gauge_min, gauge_max),(width_min, width_max))
    beauty_factor
    #foreground_white = np.random.randint(0,2,2) # fg vs bg / wb vs bw. For now we keep it random
):
    # first retrieve the name of the function to be used to generate
    drawFunction = IL.diz[figure]

    # Beauty is the minimum number of pixels that has to be different between the generated image
    # and an empty grid (see the code to understand). So with a factor of 0.1 we can accept
    # generated images that are at least 10% different from an empty grid. Basically, the more
    # beautiful the easier it is for the CNN to spot the figure.
    beauty = int(resolution*resolution*beauty_factor)

    # Print chosen parameters
    print(f' # Generating {datasetType} data tensor with {drawFunction.__name__} function.')
    print(f"    Number of figures to be generated: {figNumber}")
    print(f"    Number of channels: 1")
    print(f"    Figure resolution: {resolution} x {resolution}")
    print(f"    Grid gauge: {gridParams[0][0]}-{gridParams[0][1]}")
    print(f"    Grid width: {gridParams[1][0]}-{gridParams[1][1]}")
    print(f"    Beauty: {beauty} px")

    # Generation loop: each iteration inserts a resolution x resolution x 1 dish in the starting empty
    # dataset "cube". The explicit additional dimension 1 serves later purpose in the CNN

    payload = np.zeros((figNumber, resolution,resolution))
    for i in range(figNumber):
        foreground_white = np.random.randint(0,2,2) # we put it here because we want it random

        # draw image, reshape and add
        figure = hdf.GenerateFigure(drawFunction,resolution,gridParams,foreground_white,beauty)

        # add to the data matrix
        payload[i,:,:] = figure

    print('  Completed. #\n')
    return payload

max_stars_repo_name: lorenzosquadrani/Kaleido

max_issues_repo_name: lorenzosquadrani/Kaleido

max_forks_repo_name: lorenzosquadrani/Kaleido


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Feb 07 23:42:41 2015

@author: Seb
"""

import numpy as np
from scipy.optimize       import minimize
from reflectanceError     import ReflectanceError
from sklearn.metrics      import r2_score

def optimization(trainingParameters, trainingReflectances, testParameters, testReflectances):

    BVFs = np.unique(trainingParameters[:,0])
    Vss  = np.unique(trainingParameters[:,1])

    #%% build optimization function from rectangular reflectance grid

    reflectanceGrid3D  = np.reshape(trainingReflectances, (len(BVFs), len(Vss), trainingReflectances.shape[1]))
    functionToMinimize = ReflectanceError(BVFs, Vss, reflectanceGrid3D)

    #%% do optimization

    estimatedParameters = np.zeros_like(testParameters)
    absErrors           = np.zeros_like(testParameters)

    for idx, (testParameter, testReflectance) in enumerate(zip(testParameters, testReflectances)):
        functionToMinimize.setReflectanceToMatch(testReflectance)
        minimization = minimize(functionToMinimize.f, [np.median(BVFs), np.median(Vss)], method="Nelder-Mead")
        # interpolation extrapolates with constant values. We just crop it to the bounds
        clippedX= np.clip(minimization.x, [min(BVFs), min(Vss)], [max(BVFs), max(Vss)])

        estimatedParameters[idx, :] = clippedX
        absErrors[idx,:]            = np.abs(clippedX - testParameter)

    r2Score = r2_score(testParameters.T, estimatedParameters.T)

    return absErrors, r2Score

max_stars_repo_name: maleike/MITK

reason: import numpy,from scipy


text:
import json
import numpy as np
from tfbp.helpers.definitions import vocab_path

# Load JSON array of vocab words from file
with open(vocab_path) as f:
  vocab = json.load(f)

vocab_size = len(vocab)

# Pad_char should be last character in vocab list
pad_char = vocab[-1]

word_vectors = np.zeros([vocab_size, vocab_size])
word2index = {}

i = 0
for word in vocab:
  word2index[word] = i
  word_vectors[i][i] = 1
  i += 1

index2word = {v: k for k, v in word2index.iteritems()}

def word2vec(word):
  return word_vectors[word2index.get(word)]

def vec2word(vec):
  index = list(vec).index(1.0)
  return index2word[index]

def label2vec(label):
  return [list(word2vec(c)) for c in label]

def vec2label(vec):
  return ''.join([vec2word(v) for v in vec]).rstrip(pad_char)

max_stars_repo_name: whittlbc/tfbp

max_issues_repo_name: whittlbc/tfbp

max_forks_repo_name: whittlbc/tfbp


text:
from scipy.stats import norm
import math

def bsm_find_call_price(underlying_asset_price, strike_price, annual_volatility, annual_cc_risk_free, time_in_years = 1, annual_cc_dividend_yield = 0):

    d1 = (math.log(underlying_asset_price/strike_price) + (annual_cc_risk_free - annual_cc_dividend_yield + (annual_volatility**2)/2)*time_in_years)/(annual_volatility * math.sqrt(time_in_years))
    d2 = d1 - (annual_volatility * math.sqrt(time_in_years))

    # use d1 and d2 in a a cumulative standard normal distribution function
    n1 = norm.cdf(d1)
    n2 = norm.cdf(d2)

    # black-scholes-merton calculation:
    price = underlying_asset_price * math.exp(-annual_cc_dividend_yield*time_in_years) * n1 - strike_price * math.exp(-annual_cc_risk_free*time_in_years)*n2

    return price

def bsm_find_put_price(underlying_asset_price, strike_price, annual_volatility, annual_cc_risk_free, time_in_years = 1, annual_cc_dividend_yield = 0):

    d1 = (math.log(underlying_asset_price/strike_price) + (annual_cc_risk_free - annual_cc_dividend_yield + (annual_volatility**2)/2)*time_in_years)/(annual_volatility * math.sqrt(time_in_years))
    d2 = d1 - (annual_volatility * math.sqrt(time_in_years))

    # use d1 and d2 in a a cumulative standard normal distribution function
    n1 = norm.cdf(-d1)
    n2 = norm.cdf(-d2)

    # black-scholes-merton calculation:
    price = strike_price * math.exp(-annual_cc_risk_free*time_in_years) * n2 - underlying_asset_price * math.exp(-annual_cc_dividend_yield*time_in_years) * n1

    return price

#playing around.
# check to see if the following produces the correct results.
call_example = bsm_find_call_price(100, 110, 0.14247, 0.05, 1)
print('Call price : ${}'.format(call_example))

put_example = bsm_find_put_price(100, 110, 0.14247, 0.05, 1)
print('Put price : ${}'.format(put_example))

#double check by put-call parity:
putcall_check_1 = round(float(put_example) + 100.00 - 110.00/math.exp(0.05) - float(call_example), 10)
print('Put-call parity difference should be zero. calculations say: {}'.format(putcall_check_1))

# taken from a text book.
# check the dividend feature:
ex_call_with_dividends = bsm_find_call_price(1000, 1100, 0.14247, 0.06, 1, annual_cc_dividend_yield = 0.02)
print('Call price with dividends : ${}'.format(ex_call_with_dividends))

ex_put_with_dividends = bsm_find_put_price(1000, 1100, 0.14247, 0.06, 1, annual_cc_dividend_yield = 0.02)
print('Put price with dividends: ${}'.format(ex_put_with_dividends))

putcall_check_2 = float(ex_put_with_dividends) + 1000.00 - 1100.00/math.exp(0.06) - float(ex_call_with_dividends) -(0.02 * 1000)/math.exp(0.02)
print('Put-call parity difference should be zero. calculations say: {}'.format(putcall_check_2))

print('dangit...')
#another approach to put price using call price:
ex_put_with_dividends_repeat = ex_call_with_dividends - 1000*math.exp(-0.02*1) + 1100*math.exp(-0.06*1)
print(ex_put_with_dividends_repeat)
# everything works...

max_forks_repo_name: tmnewt/atop

reason: from scipy