text:
"""
MSM Network Plot
================
"""
from msmbuilder.example_datasets import FsPeptide
from msmbuilder.featurizer import DihedralFeaturizer
from msmbuilder.decomposition import tICA
from msmbuilder.cluster import MiniBatchKMeans
from msmbuilder.msm import MarkovStateModel

import numpy as np

import msmexplorer as msme

rs = np.random.RandomState(42)

# Load Fs Peptide Data
trajs = FsPeptide().get().trajectories

# Extract Backbone Dihedrals
featurizer = DihedralFeaturizer(types=['phi', 'psi'])
diheds = featurizer.fit_transform(trajs)

# Perform Dimensionality Reduction
tica_model = tICA(lag_time=2, n_components=2)
tica_trajs = tica_model.fit_transform(diheds)

# Perform Clustering
clusterer = MiniBatchKMeans(n_clusters=12, random_state=rs)
clustered_trajs = clusterer.fit_transform(tica_trajs)

# Construct MSM
msm = MarkovStateModel(lag_time=2)
msm.fit(clustered_trajs)

# Plot MSM Network
pos = dict(zip(range(clusterer.n_clusters), clusterer.cluster_centers_))
msme.plot_msm_network(msm, pos=pos, node_color='pomegranate',
                      edge_color='carbon')

max_stars_repo_name: smsaladi/msmexplorer

max_issues_repo_name: smsaladi/msmexplorer

max_forks_repo_name: smsaladi/msmexplorer


text:
# -*- coding: utf-8 -*-
"""
Example using the functions in TopomapArray to plot a set of 10:20 EEG sensors
onto a single subplot.

@author: KensingtonSka (Rhys Hobbs)
"""
import numpy as np
import matplotlib.pyplot as plt
from TopomapArray import project_onto_zplane, gen_grid_size, project_onto_grid

# %% Hypothetical sensor positions:
ch_names = ['Fp1', 'Fp2', 'Fz', 'F3', 'F4', 'F7', 'F8',
            'Cz', 'C3', 'C4', 'T3', 'T4',
            'Pz', 'P3', 'P4', 'T5', 'T6', 'O1', 'O2']
xyz = [np.array([-21.5,	70.2,   -0.1]),
       np.array([28.4,	69.1,	-0.4]),
       np.array([0.6,    40.9,	53.9]),
       np.array([-35.5,	49.4,	32.4]),
       np.array([40.2,   47.6,	32.1]),
       np.array([-54.8,	33.9,	-3.5]),
       np.array([56.6,	30.8,	-4.1]),
       np.array([0.8,	-14.7,	73.9]),
       np.array([-52.2,	-16.4,	57.8]),
       np.array([54.1,	-18.0,	57.5]),
       np.array([-70.2,	-21.3,	-10.7]),
       np.array([71.9,	-25.2,	-8.2]),
       np.array([0.2,	-62.1,	64.5]),
       np.array([-39.5,	-76.3,	47.4]),
       np.array([36.8,	-74.9,	49.2]),
       np.array([-61.5,	-65.3,	1.1]),
       np.array([59.3,	-67.6,	3.8]),
       np.array([-26.8,	-100.2,	12.8]),
       np.array([24.1,	-100.5,	14.1])]
xyz = np.array(xyz)

##Triangle example (uncomment to use):
#ch_names = ['U', 'L', 'R']
#length = 1
#xyz = [np.array([0, length, 0.5]),
#       np.array([-length*np.cos(np.pi/6), -length*np.sin(np.pi/6), 0.5]),
#       np.array([length*np.cos(np.pi/6), -length*np.sin(np.pi/6), 0.5])]
#xyz = np.array(xyz)

##Circle example (uncomment to use):
#radius = 1
#n_points = 10
#angle = np.linspace(0, 2*np.pi*(1-(1/n_points)), n_points)
#xyz = radius*np.array([np.cos(angle), np.sin(angle), angle*0]).T

# Generate data to plot:
X   = np.arange(0, 100, 1)
Y = (np.random.rand(len(xyz), 100)*8) - (4 + np.random.rand()*0.2)

# %% Generate subplot positions:
xy = project_onto_zplane(xyz, projection='z',
                             scale_seperation_distance=2, ch_names=ch_names)
grid_size = gen_grid_size(xy, sbp=2)
grid_pos = project_onto_grid(xy, grid_size, rotation_matrix=-90)

# %% PLOT:
# %% Using grid_size & grid_pos to plot the data onto the subplots:
# I explicitly don't use gridspec becuase I have observed poor performance
# when working with very large grids (~400 squares).

plt.style.use('dark_background')
fig, ax = plt.subplots(grid_size[0], grid_size[1], figsize=(12, 8))
[axi.set_axis_off() for axi in ax.ravel()] #Turn off axes
[axi.tick_params(axis='both', labelsize=8) for axi in ax.ravel()] #Set axis text size

# Generating equal y-limits:
ylim = round(max([abs(Y.max()), abs(Y.min())]))
plt.setp(ax, xlim=[X[0], X[-1]],
             ylim=[-ylim, ylim])

for i, pos in enumerate(grid_pos):
    ax[pos[0], pos[1]].plot(X, Y[i,:], '-', linewidth=0.5)

    #General settings:
    ax[pos[0], pos[1]].set_title(ch_names[i], size=8)

    # Ticks and border settings:
    ax[pos[0], pos[1]].axis('on')
    ax[pos[0], pos[1]].spines['right'].set_visible(False)
    ax[pos[0], pos[1]].spines['top'].set_visible(False)
    ax[pos[0], pos[1]].set_xticklabels([])
    ax[pos[0], pos[1]].set_yticklabels([])

#Make the top-left a reference:
ax[0, 0].axis('on')
ax[0, 0].spines['right'].set_visible(False)
ax[0, 0].spines['top'].set_visible(False)
ax[0, 0].set_xlabel('time (s)')
ax[0, 0].set_ylabel('voltage (V)')

ext: py

max_stars_repo_name: KensingtonSka/topomap-array

max_issues_repo_name: KensingtonSka/topomap-array

max_forks_repo_name: KensingtonSka/topomap-array

reason: import numpy


text:
# coding=utf-8
# TODO: Not yet implemented!
"""
The model for the character level speech recognizer.

Based on the paper:

http://arxiv.org/pdf/1601.06581v2.pdf

This model is:

character level RNN-LM
"""

import tensorflow as tf
from tensorflow.python.client import timeline
import numpy as np
import time
import os
from datetime import datetime
import logging
from random import randint
import util.dataprocessor as dataprocessor

class LanguageModel(object):
    def __init__(self, num_layers, hidden_size, batch_size, max_input_seq_length,
                 max_target_seq_length, input_dim):
        """
        Character level language model to help with language model predictions
        Uses lstm cells in a deep rnn

        Parameters
        ----------
        :param num_layers: number of lstm layers
        :param hidden_size: size of hidden layers
        :param batch_size: number of training examples fed at once
        :param max_input_seq_length: maximum length of input vector sequence
        :param max_target_seq_length: maximum length of ouput vector sequence
        :param input_dim: dimension of input vector
        """
        # Store model's parameters
        self.num_layers = num_layers
        self.hidden_size = hidden_size
        self.batch_size = batch_size
        self.max_input_seq_length = max_input_seq_length
        self.max_target_seq_length = max_target_seq_length
        self.input_dim = input_dim
        # Output vector as the same dimension as input
        self.num_labels = input_dim

        # Create object's variables for tensorflow ops
        self.rnn_state_zero_op = None
        self.rnn_keep_state_op = None
        self.saver_op = None

        # Create object's variable for result output
        self.prediction = None

        # Create object's variables for placeholders
        self.input_keep_prob_ph = self.output_keep_prob_ph = None
        self.inputs_ph = self.input_seq_lengths_ph = self.labels_ph = None

        # Create object's variables for dataset's iterator input
        self.iterator_get_next_op = None
        self.is_training_var = tf.Variable(initial_value=False, trainable=False, name="is_training_var", dtype=tf.bool)

        # Create object's variable for hidden state
        self.rnn_tuple_state = None

        # Create object's variables for training
        self.input_keep_prob = self.output_keep_prob = None
        self.global_step = None
        self.learning_rate_var = None
        # Create object variables for tensorflow training's ops
        self.learning_rate_decay_op = None
        self.accumulated_mean_loss = self.acc_mean_loss_op = self.acc_mean_loss_zero_op = None
        self.accumulated_error_rate = self.acc_error_rate_op = self.acc_error_rate_zero_op = None
        self.mini_batch = self.increase_mini_batch_op = self.mini_batch_zero_op = None
        self.acc_gradients_zero_op = self.accumulate_gradients_op = None
        self.train_step_op = None

        # Create object's variables for tensorboard
        self.tensorboard_dir = None
        self.timeline_enabled = False
        self.train_summaries_op = None
        self.test_summaries_op = None
        self.summary_writer_op = None

        # Create object's variables for status checking
        self.rnn_created = False

    def create_forward_rnn(self):
        """
        Create the forward-only RNN

        Parameters
        -------
        :return: the logits
        """
        if self.rnn_created:
            logging.fatal("Trying to create the language RNN but it is already.")

        # Set placeholders for input
        self.inputs_ph = tf.placeholder(tf.float32, shape=[self.max_input_seq_length, None, self.input_dim],
                                        name="inputs_ph")

        self.input_seq_lengths_ph = tf.placeholder(tf.int32, shape=[None], name="input_seq_lengths_ph")

        # Build the RNN
        self.global_step, logits, self.prediction, self.rnn_keep_state_op, self.rnn_state_zero_op, \
            _, _, self.rnn_tuple_state = self._build_base_rnn(self.inputs_ph, self.input_seq_lengths_ph, True)

        # Add the saving and restore operation
        self.saver_op = self._add_saving_op()

        return logits

    def create_training_rnn(self, input_keep_prob, output_keep_prob, grad_clip, learning_rate, lr_decay_factor,
                            use_iterator=False):
        """
        Create the training RNN

        Parameters
        ----------
        :param input_keep_prob: probability of keeping input signal for a cell during training
        :param output_keep_prob: probability of keeping output signal from a cell during training
        :param grad_clip: max gradient size (prevent exploding gradients)
        :param learning_rate: learning rate parameter fed to optimizer
        :param lr_decay_factor: decay factor of the learning rate
        :param use_iterator: if True then plug an iterator.get_next() operation for the input of the model, if None
                            placeholders are created instead
        """
        if self.rnn_created:
            logging.fatal("Trying to create the language RNN but it is already.")

        # Store model parameters
        self.input_keep_prob = input_keep_prob
        self.output_keep_prob = output_keep_prob

        if use_iterator is True:
            text_batch, input_lengths, label_batch = self.iterator_get_next_op
            # Pad if the batch is not complete
            padded_text_batch = tf.pad(text_batch, [[0, self.batch_size - tf.size(input_lengths)], [0, 0], [0, 0]])
            # Transpose padded_text_batch in order to get time serie as first dimension
            # [batch_size, time_serie, input_dim] ====> [time_serie, batch_size, input_dim]
            inputs = tf.transpose(padded_text_batch, perm=[1, 0, 2])
            # Pad input_seq_lengths if the batch is not complete
            input_seq_lengths = tf.pad(input_lengths, [[0, self.batch_size - tf.size(input_lengths)]])

            # Label tensor must be provided as a sparse tensor.
            sparse_labels = tf.SparseTensor(label_batch[0], label_batch[1], [self.batch_size, label_batch[2][1]])
            # Pad sparse_labels if the batch is not complete
            sparse_labels, _ = tf.sparse_fill_empty_rows(sparse_labels, self.num_labels - 1)
        else:
            # Set placeholders for input
            self.inputs_ph = tf.placeholder(tf.int32, shape=[self.max_input_seq_length, None, self.input_dim],
                                            name="inputs_ph")

            self.input_seq_lengths_ph = tf.placeholder(tf.int32, shape=[None], name="input_seq_lengths_ph")
            self.labels_ph = tf.placeholder(tf.int32, shape=[None, self.max_target_seq_length],
                                            name="labels_ph")
            inputs = self.inputs_ph
            input_seq_lengths = self.input_seq_lengths_ph
            label_batch = self.labels_ph

            # Label tensor must be provided as a sparse tensor.
            # First get indexes from non-zero positions
            idx = tf.where(tf.not_equal(label_batch, 0))
            # Then build a sparse tensor from indexes
            sparse_labels = tf.SparseTensor(idx, tf.gather_nd(label_batch, idx),
                                            [self.batch_size, self.max_target_seq_length])

        self.global_step, logits, prediction, self.rnn_keep_state_op, self.rnn_state_zero_op, self.input_keep_prob_ph, \
            self.output_keep_prob_ph, self.rnn_tuple_state = self._build_base_rnn(inputs, input_seq_lengths, False)

        # Add the train part to the network
        self.learning_rate_var = self._add_training_on_rnn(logits, grad_clip, learning_rate, lr_decay_factor,
                                                           sparse_labels, input_seq_lengths, prediction)

        # Add the saving and restore operation
        self.saver_op = self._add_saving_op()

    def _build_base_rnn(self, inputs, input_seq_lengths, forward_only=True):
        """
        Build the Language RNN

        Parameters
        ----------
        :param inputs: inputs to the RNN
        :param input_seq_lengths: vector containing the length of each input from 'inputs'
        :param forward_only: whether the RNN will be used for training or not (if true then add a dropout layer)

        Returns
        ----------
        :returns logits: each char probability for each timestep of the input, for each item of the batch
        :returns prediction: the best prediction for the input
        :returns rnn_keep_state_op: a tensorflow op to save the RNN internal state for the next batch
        :returns rnn_state_zero_op: a tensorflow op to reset the RNN internal state to zeros
        :returns input_keep_prob_ph: a placeholder for input_keep_prob of the dropout layer
                                     (None if forward_only is True)
        :returns output_keep_prob_ph: a placeholder for output_keep_prob of the dropout layer
                                      (None if forward_only is True)
        :returns rnn_tuple_state: the RNN internal state
        """
        # Define a variable to keep track of the learning process step
        global_step = tf.Variable(0, trainable=False, name='global_step')

        # If building the RNN for training then create dropout rate placeholders
        input_keep_prob_ph = output_keep_prob_ph = None
        if not forward_only:
            with tf.name_scope('dropout'):
                # Create placeholders, used to override values when running on the test set
                input_keep_prob_ph = tf.placeholder(tf.float32)
                output_keep_prob_ph = tf.placeholder(tf.float32)

        # Define cells of language model
        with tf.variable_scope('LSTM'):
            # Create each layer
            layers_list = []
            for _ in range(self.num_layers):
                cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_size, state_is_tuple=True)

                # If building the RNN for training then add a dropoutWrapper to the cells
                if not forward_only:
                    with tf.name_scope('dropout'):
                        cell = tf.contrib.rnn.DropoutWrapper(cell, input_keep_prob=input_keep_prob_ph,
                                                             output_keep_prob=output_keep_prob_ph)
                layers_list.append(cell)

            # Store the layers in a multi-layer RNN
            cell = tf.contrib.rnn.MultiRNNCell(layers_list, state_is_tuple=True)

        # Build the input layer between input and the RNN
        with tf.variable_scope('Input_Layer'):
            w_i = tf.get_variable("input_w", [self.input_dim, self.hidden_size], tf.float32,
                                  initializer=tf.contrib.layers.xavier_initializer())
            b_i = tf.get_variable("input_b", [self.hidden_size], tf.float32,
                                  initializer=tf.constant_initializer(0.0))

        # Apply the input layer to the network input to produce the input for the rnn part of the network
        rnn_inputs = [tf.matmul(tf.squeeze(tf.cast(i, tf.float32), axis=[0]), w_i) + b_i
                      for i in tf.split(axis=0, num_or_size_splits=self.max_input_seq_length, value=inputs)]
        # Switch from a list to a tensor
        rnn_inputs = tf.stack(rnn_inputs)

        # Define some variables to store the RNN state
        # Note : tensorflow keep the state inside a batch but it's necessary to do this in order to keep the state
        #        between batches, especially when doing live transcript
        #        Another way would have been to get the state as an output of the session and feed it every time but
        #        this way is much more efficient
        with tf.variable_scope('Hidden_state'):
            state_variables = []
            for state_c, state_h in cell.zero_state(self.batch_size, tf.float32):
                state_variables.append(tf.nn.rnn_cell.LSTMStateTuple(
                    tf.Variable(state_c, trainable=False),
                    tf.Variable(state_h, trainable=False)))
            # Return as a tuple, so that it can be fed to dynamic_rnn as an initial state
            rnn_tuple_state = tuple(state_variables)

        # Build the RNN
        with tf.name_scope('LSTM'):
            rnn_output, new_states = tf.nn.dynamic_rnn(cell, rnn_inputs, sequence_length=input_seq_lengths,
                                                       initial_state=rnn_tuple_state, time_major=True)

        # Define an op to keep the hidden state between batches
        update_ops = []
        for state_variable, new_state in zip(rnn_tuple_state, new_states):
            # Assign the new state to the state variables on this layer
            update_ops.extend([state_variable[0].assign(new_state[0]),
                               state_variable[1].assign(new_state[1])])
        # Return a tuple in order to combine all update_ops into a single operation.
        # The tuple's actual value should not be used.
        rnn_keep_state_op = tf.tuple(update_ops)

        # Define an op to reset the hidden state to zeros
        update_ops = []
        for state_variable in rnn_tuple_state:
            # Assign the new state to the state variables on this layer
            update_ops.extend([state_variable[0].assign(tf.zeros_like(state_variable[0])),
                               state_variable[1].assign(tf.zeros_like(state_variable[1]))])
        # Return a tuple in order to combine all update_ops into a single operation.
        # The tuple's actual value should not be used.
        rnn_state_zero_op = tf.tuple(update_ops)

        # Build the output layer between the RNN and the char_map
        with tf.variable_scope('Output_layer'):
            w_o = tf.get_variable("output_w", [self.hidden_size, self.num_labels], tf.float32,
                                  initializer=tf.contrib.layers.xavier_initializer())
            b_o = tf.get_variable("output_b", [self.num_labels], tf.float32,
                                  initializer=tf.constant_initializer(0.0))

        # Compute the logits (each char probability for each timestep of the input, for each item of the batch)
        logits = tf.stack([tf.matmul(tf.squeeze(i, axis=[0]), w_o) + b_o
                           for i in tf.split(axis=0, num_or_size_splits=self.max_input_seq_length, value=rnn_output)])

        # Compute the prediction which is the best "path" of probabilities for each item of the batch
        decoded, _log_prob = tf.nn.ctc_beam_search_decoder(logits, input_seq_lengths)
        # Set the RNN result to the best path found
        prediction = tf.to_int32(decoded[0])

        return global_step, logits, prediction, rnn_keep_state_op, rnn_state_zero_op, \
            input_keep_prob_ph, output_keep_prob_ph, rnn_tuple_state

    def _add_training_on_rnn(self, logits, grad_clip, learning_rate, lr_decay_factor,
                             sparse_labels, input_seq_lengths, prediction):
        """
        Build the training add-on of the Language RNN

        This add-on offer ops that can be used to train the network :
          * self.learning_rate_decay_op : will decay the learning rate
          * self.acc_mean_loss_op : will compute the loss and accumulate it over multiple mini-batchs
          * self.acc_mean_loss_zero_op : will reset the loss accumulator to 0
          * self.acc_error_rate_op : will compute the error rate and accumulate it over multiple mini-batchs
          * self.acc_error_rate_zero_op : will reset the error_rate accumulator to 0
          * self.increase_mini_batch_op : will increase the mini-batchs counter
          * self.mini_batch_zero_op : will reset the mini-batchs counter
          * self.acc_gradients_zero_op : will reset the gradients
          * self.accumulate_gradients_op : will compute the gradients and accumulate them over multiple mini-batchs
          * self.train_step_op : will clip the accumulated gradients and apply them on the RNN

        Parameters
        ----------
        :param logits: the output of the RNN before the beam search
        :param grad_clip: max gradient size (prevent exploding gradients)
        :param learning_rate: learning rate parameter fed to optimizer
        :param lr_decay_factor: decay factor of the learning rate
        :param sparse_labels: the labels in a sparse tensor
        :param input_seq_lengths: vector containing the length of each input from 'inputs'
        :param prediction: the predicted label given by the RNN

        Returns
        -------
        :returns: tensorflow variable keeping the current learning rate
        """
        # Define the variable for the learning rate
        learning_rate_var = tf.Variable(float(learning_rate), trainable=False, name='learning_rate')
        # Define an op to decrease the learning rate
        self.learning_rate_decay_op = learning_rate_var.assign(tf.multiply(learning_rate_var, lr_decay_factor))

        # Compute the CTC loss between the logits and the truth for each item of the batch
        with tf.name_scope('CTC'):
            ctc_loss = tf.nn.ctc_loss(sparse_labels, logits, input_seq_lengths, ignore_longer_outputs_than_inputs=True)

            # Compute the mean loss of the batch (only used to check on progression in learning)
            # The loss is averaged accross the batch but before we take into account the real size of the label
            mean_loss = tf.reduce_mean(tf.truediv(ctc_loss, tf.to_float(input_seq_lengths)))

            # Set an accumulator to sum the loss between mini-batchs
            self.accumulated_mean_loss = tf.Variable(0.0, trainable=False)
            self.acc_mean_loss_op = self.accumulated_mean_loss.assign_add(mean_loss)
            self.acc_mean_loss_zero_op = self.accumulated_mean_loss.assign(tf.zeros_like(self.accumulated_mean_loss))

        # Compute the error between the logits and the truth
        with tf.name_scope('Error_Rate'):
            error_rate = tf.reduce_mean(tf.edit_distance(prediction, sparse_labels, normalize=True))

            # Set an accumulator to sum the error rate between mini-batchs
            self.accumulated_error_rate = tf.Variable(0.0, trainable=False)
            self.acc_error_rate_op = self.accumulated_error_rate.assign_add(error_rate)
            self.acc_error_rate_zero_op = self.accumulated_error_rate.assign(tf.zeros_like(self.accumulated_error_rate))

        # Count mini-batchs
        with tf.name_scope('Mini_batch'):
            # Set an accumulator to count the number of mini-batchs in a batch
            # Note : variable is defined as float to avoid type conversion error using tf.divide
            self.mini_batch = tf.Variable(0.0, trainable=False)
            self.increase_mini_batch_op = self.mini_batch.assign_add(1)
            self.mini_batch_zero_op = self.mini_batch.assign(tf.zeros_like(self.mini_batch))

        # Compute the gradients
        trainable_variables = tf.trainable_variables()
        with tf.name_scope('Gradients'):
            opt = tf.train.AdamOptimizer(learning_rate_var)
            gradients = opt.compute_gradients(ctc_loss, trainable_variables)

            # Define a list of variables to store the accumulated gradients between batchs
            accumulated_gradients = [tf.Variable(tf.zeros_like(tv.initialized_value()), trainable=False)
                                     for tv in trainable_variables]

            # Define an op to reset the accumulated gradient
            self.acc_gradients_zero_op = [tv.assign(tf.zeros_like(tv)) for tv in accumulated_gradients]

            # Define an op to accumulate the gradients calculated by the current batch with
            # the accumulated gradients variable
            self.accumulate_gradients_op = [accumulated_gradients[i].assign_add(gv[0])
                                            for i, gv in enumerate(gradients)]

            # Define an op to apply the result of the accumulated gradients
            clipped_gradients, _norm = tf.clip_by_global_norm(accumulated_gradients, grad_clip)
            self.train_step_op = opt.apply_gradients([(clipped_gradients[i], gv[1]) for i, gv in enumerate(gradients)],
                                                     global_step=self.global_step)
        return learning_rate_var

    def add_tensorboard(self, session, tensorboard_dir, tb_run_name=None, timeline_enabled=False):
        """
        Add the tensorboard operations to the RNN
        This method will add ops to feed tensorboard
          self.train_summaries_op : will produce the summary for a training step
          self.test_summaries_op : will produce the summary for a test step
          self.summary_writer_op : will write the summary to disk

        Parameters
        ----------
        :param session: the tensorflow session
        :param tensorboard_dir: path to tensorboard directory
        :param tb_run_name: directory name for the tensorboard files inside tensorboard_dir, if None a default dir
                            will be created
        :param timeline_enabled: enable the output of a trace file for timeline visualization
        """
        self.tensorboard_dir = tensorboard_dir
        self.timeline_enabled = timeline_enabled

        # Define GraphKeys for TensorBoard
        graphkey_training = tf.GraphKeys()
        graphkey_test = tf.GraphKeys()

        # Learning rate
        tf.summary.scalar('Learning_rate', self.learning_rate_var, collections=[graphkey_training, graphkey_test])

        # Loss
        with tf.name_scope('Mean_loss'):
            mean_loss = tf.divide(self.accumulated_mean_loss, self.mini_batch)
            tf.summary.scalar('Training', mean_loss, collections=[graphkey_training])
            tf.summary.scalar('Test', mean_loss, collections=[graphkey_test])

        # Accuracy
        with tf.name_scope('Accuracy_-_Error_Rate'):
            mean_error_rate = tf.divide(self.accumulated_error_rate, self.mini_batch)
            tf.summary.scalar('Training', mean_error_rate, collections=[graphkey_training])
            tf.summary.scalar('Test', mean_error_rate, collections=[graphkey_test])

        # Hidden state
        with tf.name_scope('RNN_internal_state'):
            for idx, state_variable in enumerate(self.rnn_tuple_state):
                tf.summary.histogram('Training_layer-{0}_cell_state'.format(idx), state_variable[0],
                                     collections=[graphkey_training])
                tf.summary.histogram('Test_layer-{0}_cell_state'.format(idx), state_variable[0],
                                     collections=[graphkey_test])
                tf.summary.histogram('Training_layer-{0}_hidden_state'.format(idx), state_variable[1],
                                     collections=[graphkey_training])
                tf.summary.histogram('Test_layer-{0}_hidden_state'.format(idx), state_variable[1],
                                     collections=[graphkey_test])

        self.train_summaries_op = tf.summary.merge_all(key=graphkey_training)
        self.test_summaries_op = tf.summary.merge_all(key=graphkey_test)
        if tb_run_name is None:
            run_name = datetime.now().strftime('%Y-%m-%d--%H-%M-%S')
        else:
            run_name = tb_run_name
        self.summary_writer_op = tf.summary.FileWriter(tensorboard_dir + '/' + run_name + '/', graph=session.graph)

    def get_learning_rate(self):
        return self.learning_rate_var.eval()

    def set_learning_rate(self, sess, learning_rate):
        assign_op = self.learning_rate_var.assign(learning_rate)
        sess.run(assign_op)

    def set_is_training(self, sess, is_training):
        assign_op = self.is_training_var.assign(is_training)
        sess.run(assign_op)

    @staticmethod
    def initialize(sess):
        # Initialize variables
        sess.run(tf.global_variables_initializer())

    def save(self, session, checkpoint_dir):
        # Save the model
        checkpoint_path = os.path.join(checkpoint_dir, "languagemodel.ckpt")
        self.saver_op.save(session, checkpoint_path, global_step=self.global_step)
        logging.info("Checkpoint saved")

    def restore(self, session, checkpoint_dir):
        ckpt = tf.train.get_checkpoint_state(checkpoint_dir)

        # Restore from checkpoint (will overwrite variables)
        if ckpt:
            self.saver_op.restore(session, ckpt.model_checkpoint_path)
            logging.info("Restored model parameters from %s (global_step id %d)", ckpt.model_checkpoint_path,
                         self.global_step.eval())
        else:
            logging.info("Created model with fresh parameters.")
        return

    @staticmethod
    def _add_saving_op():
        """
        Define a tensorflow operation to save or restore the network

        :return: a tensorflow tf.train.Saver operation
        """
        # Define an op to save or restore the network

        # Only save needed tensors :
        #   - weight and biais from the input layer, the output layer
        #   - weight and biais from the LSTM (which are named kernel and bias respectively)
        #   - currents global_step and learning_rate

        for var in tf.global_variables():
            logging.debug("TF variable : %s - %s", var.name, var)

        save_list = [var for var in tf.global_variables()
                     if (var.name.find('/input_w:0') != -1) or (var.name.find('/input_b:0') != -1) or
                     (var.name.find('/output_w:0') != -1) or (var.name.find('/output_b:0') != -1) or
                     (var.name.find('global_step:0') != -1) or (var.name.find('learning_rate:0') != -1) or
                     (var.name.find('/kernel:0') != -1) or (var.name.find('/bias:0') != -1)]
        if len(save_list) == 0:
            raise ValueError("Trying to define the saving operation before the RNN is built")

        saver_op = tf.train.Saver(save_list)
        return saver_op

    def run_step(self, session, compute_gradients=True, run_options=None, run_metadata=None):
        """
        Returns:
        mean of ctc_loss
        """
        # Base output is to accumulate loss, error_rate, increase the mini-batchs counter and keep the hidden state for
        # next batch
        output_feed = [self.acc_mean_loss_op, self.acc_error_rate_op,
                       self.increase_mini_batch_op, self.rnn_keep_state_op]

        if compute_gradients:
            # Add the update operation
            output_feed.append(self.accumulate_gradients_op)
            # and feed the dropout layer the keep probability values
            input_feed = {self.input_keep_prob_ph: self.input_keep_prob,
                          self.output_keep_prob_ph: self.output_keep_prob}
        else:
            # No need to apply a dropout, set the keep probability to 1.0
            input_feed = {self.input_keep_prob_ph: 1.0, self.output_keep_prob_ph: 1.0}

        # Actually run the tensorflow session
        start_time = time.time()
        logging.debug("Starting a step")
        session.run(output_feed, input_feed, options=run_options, run_metadata=run_metadata)
        mini_batch_num = self.mini_batch.eval()
        logging.debug("Step duration : %.2f", time.time() - start_time)
        return mini_batch_num

    def start_batch(self, session, is_training, run_options=None, run_metadata=None):
        output = [self.acc_error_rate_zero_op, self.acc_mean_loss_zero_op, self.mini_batch_zero_op]

        self.set_is_training(session, is_training)
        if is_training:
            output.append(self.acc_gradients_zero_op)

        session.run(output, options=run_options, run_metadata=run_metadata)
        return

    def end_batch(self, session, is_training, run_options=None, run_metadata=None, rnn_state_reset_ratio=1.0):
        # Get each accumulator's value and compute the mean for the batch
        output_feed = [self.accumulated_mean_loss, self.accumulated_error_rate, self.mini_batch, self.global_step]

        # If in training...
        if is_training:
            # Append the train_step_op (this will apply the gradients)
            output_feed.append(self.train_step_op)
            # Reset the hidden state at the given random ratio (default to always)
            if randint(1, 1 // rnn_state_reset_ratio) == 1:
                output_feed.append(self.rnn_state_zero_op)

        # If a tensorboard dir is configured then run the merged_summaries operation
        if self.tensorboard_dir is not None:
            if is_training:
                output_feed.append(self.train_summaries_op)
            else:
                output_feed.append(self.test_summaries_op)

        outputs = session.run(output_feed, options=run_options, run_metadata=run_metadata)
        accumulated_loss = outputs[0]
        accumulated_error_rate = outputs[1]
        batchs_count = outputs[2]
        global_step = outputs[3]

        if self.tensorboard_dir is not None:
            summary = outputs[-1]
            self.summary_writer_op.add_summary(summary, global_step)

        mean_loss = accumulated_loss / batchs_count
        mean_error_rate = accumulated_error_rate / batchs_count
        return mean_loss, mean_error_rate, global_step

    def process_input(self, session, inputs, input_seq_lengths, run_options=None, run_metadata=None):
        """
        Returns:
          Next char
        """
        input_feed = {self.inputs_ph: np.array(inputs), self.input_seq_lengths_ph: np.array(input_seq_lengths)}

        if (self.input_keep_prob_ph is not None) and (self.output_keep_prob_ph is not None):
            input_feed[self.input_keep_prob_ph] = 1.0
            input_feed[self.output_keep_prob_ph] = 1.0

        output_feed = [self.prediction]
        outputs = session.run(output_feed, input_feed, options=run_options, run_metadata=run_metadata)
        predictions = session.run(tf.sparse_tensor_to_dense(outputs[0], default_value=self.num_labels,
                                                            validate_indices=True),
                                  options=run_options, run_metadata=run_metadata)
        return predictions

    @staticmethod
    def build_dataset(input_set, batch_size, max_input_seq_length, char_map):
        # TODO : fix size calculation
        length_set = [len(label) for label in input_set]
        input_dataset = tf.data.Dataset.from_tensor_slices(input_set)
        input_length_dataset = tf.data.Dataset.from_tensor_slices(length_set)
        label_dataset = tf.data.Dataset.from_tensor_slices(input_set)

        def _transcode_label(label):
            # Need to convert back to string because tf.py_func changed it to a numpy array
            label = str(label, encoding='UTF-8')
            label_transcoded = dataprocessor.DataProcessor.get_str_to_one_hot_encoded(char_map, label)
            return np.array(label_transcoded, dtype=np.int32)

        def _transcode_and_offset_label(label):
            # Need to convert back to string because tf.py_func changed it to a numpy array
            label = str(label, encoding='UTF-8')
            offseted_label = dataprocessor.DataProcessor.get_str_labels(char_map, label)
            offseted_label = offseted_label[1:]
            offseted_label.append(len(char_map) - 1)
            logging.debug("Returning offseted label as : %s", offseted_label)
            return np.array(offseted_label, dtype=np.int32)

        input_dataset = input_dataset.map(lambda label: tf.py_func(_transcode_label, [label], tf.int32),
                                          num_parallel_calls=2).prefetch(30)
        label_dataset = label_dataset.map(lambda label: tf.py_func(_transcode_and_offset_label, [label], tf.int32),
                                          num_parallel_calls=2).prefetch(30)

        # Batch the datasets
        input_dataset = input_dataset.padded_batch(batch_size, padded_shapes=[None, None])
        label_dataset = label_dataset.apply(tf.contrib.data.dense_to_sparse_batch(batch_size=batch_size,
                                                                                row_shape=[max_input_seq_length]))
        input_length_dataset = input_length_dataset.batch(batch_size)

        # And zip them together
        dataset = tf.data.Dataset.zip((input_dataset, input_length_dataset, label_dataset))

        # TODO : add a filter for files which are too long (currently de-structuring with Dataset.filter is not
        #        supported in python3)

        return dataset

    def add_dataset_input(self, dataset):
        """
        Add one dataset as an input to the model

        Parameters
        ----------
        :param dataset: a tensorflow Dataset
        :return iterator: tensorflow Iterator for the dataset
        """
        iterator = dataset.make_initializable_iterator()
        self.iterator_get_next_op = iterator.get_next()
        return iterator

    def add_datasets_input(self, train_dataset, valid_dataset):
        """
        Add training and evaluation datasets for input to the model
        Warning : returned iterators must be initialized before use : "tf.Session.run(iterator.initializer)" on each

        Parameters
        ----------
        :param train_dataset: a tensorflow Dataset
        :param valid_dataset: a tensorflow Dataset
        :return t_iterator: tensorflow Iterator for the train dataset
        :return v_iterator: tensorflow Iterator for the valid dataset
        """
        t_iterator = train_dataset.make_initializable_iterator()
        v_iterator = valid_dataset.make_initializable_iterator()
        self.iterator_get_next_op = tf.cond(self.is_training_var, lambda: t_iterator.get_next(),
                                            lambda: v_iterator.get_next())
        return t_iterator, v_iterator

    def _write_timeline(self, run_metadata, inter_time, action=""):
        logging.debug("--- Action %s duration : %.4f", action, time.time() - inter_time)

        if self.tensorboard_dir is None:
            logging.warning("Could not write timeline, a tensorboard_dir is required in config file")
            return

        # Create the Timeline object, and write it to a json
        trace = timeline.Timeline(step_stats=run_metadata.step_stats)
        logging.info('Writing to timeline-' + action + '.ctf.json')
        with open(self.tensorboard_dir + '/' + 'timeline-' + action + '.ctf.json', 'w') as trace_file:
            trace_file.write(trace.generate_chrome_trace_format())
        return time.time()

    def run_train_step(self, sess, mini_batch_size, rnn_state_reset_ratio, run_options=None, run_metadata=None):
        """
        Run a single train step

        Parameters
        ----------
        :param sess: a tensorflow session
        :param mini_batch_size: the number of batchs to run before applying the gradients
        :param rnn_state_reset_ratio: the ratio to which the RNN internal state will be reset to 0
             example: 1.0 mean the RNN internal state will be reset at the end of each batch
             example: 0.25 mean there is 25% chances that the RNN internal state will be reset at the end of each batch
        :param run_options: options parameter for the sess.run calls
        :param run_metadata: run_metadata parameter for the sess.run calls
        :returns float mean_loss: mean loss for the train batch run
        :returns float mean_error_rate: mean error rate for the train batch run
        :returns int current_step: new value of the step counter at the end of this batch
        :returns bool dataset_empty: `True` if the dataset was emptied during the batch
        """
        start_time = inter_time = time.time()
        dataset_empty = False

        # Start a new batch
        self.start_batch(sess, True, run_options=run_options, run_metadata=run_metadata)
        if self.timeline_enabled:
            inter_time = self._write_timeline(run_metadata, inter_time, "start_batch")

        # Run multiple mini-batchs inside the train step
        mini_batch_num = 0
        try:
            for i in range(mini_batch_size):
                # Run a step on a batch and keep the loss
                mini_batch_num = self.run_step(sess, True, run_options=run_options, run_metadata=run_metadata)
                if self.timeline_enabled:
                    inter_time = self._write_timeline(run_metadata, inter_time, "step-" + str(i))
        except tf.errors.OutOfRangeError:
            logging.debug("Dataset empty, exiting train step")
            dataset_empty = True

        # Close the batch if at least a mini-batch was completed
        if mini_batch_num > 0:
            mean_loss, mean_error_rate, current_step = self.end_batch(sess, True, run_options=run_options,
                                                                      run_metadata=run_metadata,
                                                                      rnn_state_reset_ratio=rnn_state_reset_ratio)
            if self.timeline_enabled:
                _ = self._write_timeline(run_metadata, inter_time, "end_batch")

            # Step result
            logging.info("Batch %d : loss %.5f - error_rate %.5f - duration %.2f",
                         current_step, mean_loss, mean_error_rate, time.time() - start_time)

            return mean_loss, mean_error_rate, current_step, dataset_empty
        else:
            return 0.0, 0.0, self.global_step.eval(), dataset_empty


text:
from pycocotools.coco import COCO
from PIL import Image
from sortedcontainers import SortedList
import random
import json
import math
from tqdm import tqdm
import logging
import copy
import numpy as np
import argparse

def logger_config(log_path, logging_name):
    logger = logging.getLogger(logging_name)
    logger.setLevel(level=logging.DEBUG)
    handler = logging.FileHandler(log_path, encoding='UTF-8')
    handler.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)
    console = logging.StreamHandler()
    console.setLevel(logging.DEBUG)
    logger.addHandler(handler)
    logger.addHandler(console)
    return logger

def parse_args():
    parser = argparse.ArgumentParser(description='image generator')
    parser.add_argument('gen_amount',type=int, help='number of images being generated')
    parser.add_argument(
        '--base-prob',
        type=float,
        default=0.1,
        help='probability that will be divided uniformly to all middle segments, '
             'while the rest part will be arranged corresponding to segments\' height')
    args = parser.parse_args()
    return args

args=parse_args()

path = "/home/zhouyuxuan/ICDAR2021/"
# path = "/Volumes/ROSELIA/ICDAR2021/"

logger = logger_config(log_path=path + 'gen/log.txt', logging_name='sherco')

CLASSES = ('embedded', 'isolated')

coco = COCO(path + "TrM.json")
cat_ids = coco.get_cat_ids(cat_names=CLASSES)
cat2label = {cat_id: i for i, cat_id in enumerate(cat_ids)}
img_ids = coco.get_img_ids()
img_info = coco.load_imgs(img_ids)

top_segs = []
bot_segs = []
mid_segs = []
mid_heights = []

for imgn in range(len(img_info)):
    img_id = img_info[imgn]['id']
    ann_ids = coco.get_ann_ids(img_ids=[img_id])
    ann_info = coco.load_anns(ann_ids)
    img = Image.open(path + "TrM/" + img_info[img_id]['file_name'])

    lines = [0, 2048]
    for annn in range(len(ann_info)):
        if ann_info[annn]['category_id'] == 1:
            lines.append(ann_info[annn]['bbox'][1])
            lines.append(ann_info[annn]['bbox'][1] + ann_info[annn]['bbox'][3])

    lines = sorted(lines)
    assert lines[-1] == 2048

    i=0
    while i<len(lines)-1:
        j=i+1
        while lines[j] - lines[i] <= 20:
            j=j+1
        if j>=len(lines):
            break
        annids = []
        for annn in range(len(ann_info)):
            bbox = ann_info[annn]['bbox']
            if bbox[1] >= lines[i] and bbox[1] + bbox[3] <= lines[j]:
                annids.append(ann_info[annn]['id'])

        if (lines[j]-lines[i])//500>=len(ann_ids) or len(ann_ids)==0:
            i=j+1
            continue

        segment = {'img_id': img_id,
                   'top': lines[i],
                   'bottom': lines[j],
                   'height': lines[j] - lines[i],
                   'ann_ids': annids}

        if lines[i] == 0:
            top_segs.append(segment)
        elif lines[j] == 2048:
            bot_segs.append(segment)
        else:
            mid_segs.append(segment)

        i=j+1

top_segs = SortedList(top_segs, key=lambda seg: seg['height'])
bot_segs = SortedList(bot_segs, key=lambda seg: seg['height'])
mid_segs = SortedList(mid_segs, key=lambda seg: seg['height'])
mid_tot=len(mid_segs)

mid_heights = np.array([seg['height'] for seg in mid_segs])
mid_weights = np.sqrt(mid_heights)
prefix_sum = np.array(mid_weights)
for i in range(1, mid_tot):
    prefix_sum[i] = prefix_sum[i - 1] + prefix_sum[i]

def cut_image(gen_image, segment, img_id, ann_id, current_height):
    image_info = img_info[segment['img_id']]
    ori_anns = coco.load_anns(segment['ann_ids'])
    anns=copy.deepcopy(ori_anns)
    # anns = coco.load_anns(segment['ann_ids'])
    for ann in anns:
        ann['image_id'] = img_id
        ann['id'] = ann_id
        delta = current_height - segment['top']
        ann['bbox'][1] += delta
        for segmentation in ann['segmentation']:
            for i in range(1, len(segmentation), 2):
                segmentation[i] += delta
        ann_id += 1

    image = Image.open(path + "TrM/" + image_info['file_name'])
    image = image.crop((0, segment['top'], 1447, segment['bottom']))
    # image = image.resize(image.size, Image.BILINEAR)
    gen_image.paste(image, box=(0, current_height, 1447, current_height + image.size[1]))

    return anns, segment['height']

img = Image.open(path + "TrM/" + img_info[0]['file_name'])
mode = img.mode

top_frq = np.zeros(len(top_segs))
bot_frq = np.zeros(len(bot_segs))
mid_frq = np.zeros(len(mid_segs))

TOT_GEN=args.gen_amount
BASE_PROB=args.base_prob

images = []
annotations = []
ann_id = 0
for gen_id in tqdm(range(TOT_GEN)):
    remain = 2048
    current_height = 0
    noceil = 0

    seg_id = random.randint(0,len(top_segs)-1)
    top_segment = top_segs[seg_id]
    # print(top_segment)
    remain -= top_segment['height']
    file_name = "top" + str(seg_id)
    top_frq[seg_id] += 1

    bot_segment = None
    if remain >= bot_segs[0]['height']:
        idx = min( bot_segs.bisect_left({'height': remain}) ,len(bot_segs)-1)
        seg_id = random.randint(0, idx)
        bot_segment = bot_segs[seg_id]
        remain -= bot_segment['height']
        file_name += "-bot" + str(seg_id)
        bot_frq[seg_id] += 1
    # print(bot_segment)

    gen_image = Image.new("L", (1447, 2048), color=255)

    annotation, height = cut_image(gen_image, top_segment, gen_id, ann_id, current_height)
    annotations.extend(annotation)
    current_height += math.ceil(height)
    # noceil+=height
    ann_id += len(annotation)

    if remain >= mid_segs[0]['height']:
        file_name += "-mid"
    while remain >= mid_segs[0]['height']:
        idx = min(mid_tot-1,mid_segs.bisect_left({'height': remain}))
        seg_id=np.random.choice(a=np.arange(idx+1),
                                p=BASE_PROB/(idx+1)+(1-BASE_PROB)*mid_weights[:idx+1]/prefix_sum[idx])
        mid_segment = mid_segs[seg_id]
        remain -= mid_segment['height']
        annotation, height = cut_image(gen_image, mid_segment, gen_id, ann_id, current_height)
        annotations.extend(annotation)
        current_height += math.ceil(height)
        ann_id += len(annotation)
        file_name += str(seg_id) + "-"
        mid_frq[seg_id] += 1
        # noceil+=height
        # print(ann_id,remain,current_height,noceil)

    if bot_segment != None:
        annotation, height = cut_image(gen_image, bot_segment, gen_id, ann_id, current_height)
        annotations.extend(annotation)
        current_height += math.ceil(height)
        # noceil+=height
        ann_id += len(annotation)

    file_name = file_name[:-1] + ".jpg"
    gen_image.save(path + "gen/" + file_name)

    images.append({"id": gen_id,
                   "file_name": file_name,
                   "height": 2048,
                   "width": 1447})

    # logger.info("image "+str(gen_id)+" generated")

json_file = open(path + "all.json", "w")
json_file.write(json.dumps({'images': images,
                            'annotations': annotations,
                            'categories': [{'id': 0, 'name': 'embedded'}, {'id': 1, 'name': 'isolated'}]}))
json_file.close()

top_frq.sort()
bot_frq.sort()

np.savetxt(path + "top_frq.txt", top_frq[::-1])
np.savetxt(path + "bot_frq.txt", bot_frq[::-1])

mid_rank=np.argsort(mid_frq)
f=open(path+"mid_frq.txt","w")
for i in range(mid_tot-1,-1,-1):
    f.write(str(mid_frq[mid_rank[i]])+" "+str(mid_rank[i])+" "+str(mid_segs[mid_rank[i]]['height'])+"\n")
f.close()

# coco = COCO(path + "gen.json")
# cat_ids = coco.get_cat_ids(cat_names=CLASSES)
# cat2label = {cat_id: i for i, cat_id in enumerate(cat_ids)}
# img_ids = coco.get_img_ids()
# img_info = coco.load_imgs(img_ids)
#
# for imgn in range(len(img_info)):
#     img_id = img_info[imgn]['id']
#     ann_ids = coco.get_ann_ids(img_ids=[img_id])
#     ann_info = coco.load_anns(ann_ids)
#     img = Image.open(path + "gen/" + img_info[img_id]['file_name'])
#
#     for annn in range(len(ann_info)):
#         # print(ann_info[annn])
#         bbox = ann_info[annn]['bbox']
#         # print(bbox,annn)
#         img2 = img.crop((bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]))
#         img2.save(path + "gen/crop" + str(annn) + ".jpg")

max_stars_repo_name: TriplePool/mmdetection

max_issues_repo_name: TriplePool/mmdetection

max_forks_repo_name: TriplePool/mmdetection


text:
"""
Module for DataRepositories which can get datasets
"""
from abc import ABC, abstractmethod
from pathlib import Path
import numpy as np
import pandas as pd
from .dataset import CombinedDataset

class DatasetRepositoryBase(ABC):
    """
    Abstract class for getting datasets
    """

    @abstractmethod
    def load_dataset(self) -> CombinedDataset:
        """
        Loads the dataset
        """

class DatasetRepository:
    def __init__(self, datastore_root: Path, width: int, height: int):
        self.datastore_root = datastore_root
        self.width = width
        self.height = height

    def _open_and_read(self, path: Path) -> str:
        with open(path, "r") as file:
            return file.read()

    def load_dataset(self) -> CombinedDataset:
        """
        Loads the dataset
        """
        datapoint_paths = Path(self.datastore_root).glob('*.dp')
        datastore_root_string = str(self.datastore_root) + "/"
        images = []
        renders = []
        names = []
        for path in datapoint_paths:
            file_names = self._open_and_read(path).split(", ")
            target_file_name = Path(datastore_root_string + file_names[0])
            render_file_names = [Path(datastore_root_string + x) for x in file_names[1:]]

            target = pd.read_csv(target_file_name, dtype={'x': np.float32, 'y': np.float32, 'z': np.float32}).values
            target = np.asarray(target, dtype=np.float32)
            target = target.reshape(self.height, self.width, 3)
            for render_file in render_file_names:
                render = pd.read_csv(render_file,
                                     dtype={
                                         'r': np.float32,
                                         'g': np.float32,
                                         'b': np.float32,
                                         'c': np.float32,
                                         'd': np.float32,
                                         'nx': np.float32,
                                         'ny': np.float32,
                                         'nz': np.float32,
                                     }).values
                render = np.asarray(render, dtype=np.float32)
                render = render.reshape(self.height, self.width, 8)
                renders.append(render)
                images.append(target)
                names.append(str(render_file))

        dataset = CombinedDataset(dataset_path=Path(), names=names, images=images, renders=renders)
        return dataset

class DummyDatasetRepository(DatasetRepositoryBase):
    """
    Returns dummy data
    renders should be 0.2 darker than the images, with a random factor applied
    """

    def __init__(self, samples: int):
        self.samples = samples
        self.width = 20
        self.height = 20

    def load_dataset(self) -> CombinedDataset:
        names = [f"{i}" for i in range(self.samples)]
        renders = [np.random.rand(self.width, self.height, 5) for _ in range(self.samples)]
        images = [(renders[i][:, :, :3] * 0.8 + (np.random.rand(self.width, self.height, 3) * 0.2)) * 0.8 + 0.2
                  for i in range(self.samples)]

        dataset = CombinedDataset(dataset_path=Path(), names=names, images=images, renders=renders)
        return dataset

# repo = DatasetRepository(Path("/home/anders/Documents/datasets/ppt/640x360_run02"), 640, 380)


text:
import numpy as np
import tensorflow as tf
import keras.backend as K
from keras.layers import  Dense, Activation, Input
from keras.models import Model, load_model
from keras.optimizers import Adam
from keras.regularizers import l2
import time

#Optimizers for Critic Network
class RMSprop:
    def __init__(self, weights, alpha=0.01, decay_rate=0.99, beta1=0.9, beta2=0.999, epsilon=1e-5):
        self.alpha = alpha
        self.rmsprop_cache = np.zeros_like(weights)
        self.decay_rate = decay_rate
        self.epsilon = epsilon
        self.theta = weights
    def backward_pass(self, gradient):
        self.rmsprop_cache = self.decay_rate * self.rmsprop_cache + (1 - self.decay_rate) * gradient**2
        self.theta = self.theta + self.alpha * gradient / (np.sqrt(self.rmsprop_cache) + self.epsilon)
        return self.theta

class AdamOptimizer:
    def __init__(self, weights, alpha=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):
        self.alpha = alpha
        self.beta1 = beta1
        self.beta2 = beta2
        self.epsilon = epsilon
        self.m = 0
        self.v = 0
        self.t = 0
        self.theta = weights

    def backward_pass(self, gradient):
        self.t = self.t + 1
        self.m = self.beta1*self.m + (1 - self.beta1)*gradient
        self.v = self.beta2*self.v + (1 - self.beta2)*(gradient**2)
        m_hat = self.m/(1 - self.beta1**self.t)
        v_hat = self.v/(1 - self.beta2**self.t)
        self.theta = self.theta + self.alpha*(m_hat/(np.sqrt(v_hat) - self.epsilon))
        return self.theta

class Actor():
    def __init__(self, ALPHA, n_actions =4,
        layer1_size=16,layer2_size=16, input_dims = 8):

        self.lr = ALPHA
        self.input_dims = input_dims
        self.h1_dims = layer1_size
        self.h2_dims = layer2_size
        self.n_actions = n_actions
        self.state_memory = []
        self.action_memory = []
        self.Q_memory = []

        self.actor, self.phi, self.policy = self.build_polic_network()

        self.actions_space = [i for i in range(n_actions)]

    def build_polic_network(self):
        #Bulding a network
        input = Input(shape=(self.input_dims,))
        advantages = Input(shape=[1])
         # no hidden layer
        if(self.h1_dims == 0 and self.h2_dims==0):
            probs = Dense(self.n_actions, activation='softmax')(input)
            #Output of this model will be input for the Critic Network
            phi = Model(inputs=[input], outputs=[input])
        #One hidden layer
        elif(self.h1_dims != 0 and self.h2_dims == 0):
            dense1 = Dense(self.h1_dims,activation='relu')(input)
            probs = Dense(self.n_actions, activation='softmax')(dense1)
            #Output of this model will be input for the Critic Network
            phi = Model(inputs=[input], outputs=[dense1])
        #Two hidden layers
        else:
            dense1 = Dense(self.h1_dims,activation='relu')(input)
            dense2 = Dense(self.h2_dims, activation='relu')(dense1)
            probs = Dense(self.n_actions, activation='softmax')(dense2)

            #Output of this model will be input for the Critic Network
            phi = Model(inputs=[input], outputs=[dense2])

        #Loss funciton implimenting Cross Entropy
        def custum_loss(y_true,y_pred):
            #Clipping to ignore getting 0 and 1 has input from softmax layer
            out = K.clip(y_pred, 1e-8,1-1e-8)
            log_lik = y_true*K.log(out)

            return K.sum(-log_lik*advantages)

        #Generate the model with proper inputs and outputs
        actor = Model(inputs=[input,advantages], outputs=[probs])
        actor.compile(optimizer=Adam(lr=self.lr), loss=custum_loss)
        actor.summary()

        predict = Model(inputs=[input], outputs=[probs])
        # phi.compile(optimizer=Adam(lr=self.lr), loss="custum_loss(Q,y_true,y_pred)")

        return actor, phi, predict

    def store_transition(self, observation, action, Q):
        self.action_memory.append(action)
        self.state_memory.append(observation)
        self.Q_memory.append(Q)

    def learn(self):

        state_memory = np.array(self.state_memory)
        action_memory = np.array(self.action_memory)
        Q_memory = np.array(self.Q_memory)

        Q_memory = Q_memory - np.mean(Q_memory)
        Q_memory/=np.std(Q_memory)

        actions = np.zeros([len(action_memory), self.n_actions])
        actions[np.arange(len(action_memory)),action_memory] = 1

        cost = self.actor.train_on_batch([state_memory, Q_memory], actions)

        self.state_memory = []
        self.action_memory = []
        self.Q_memory = []

        # return cost

    def choose_action(self, observation):
        state = observation[np.newaxis, :]
        #Get the probability for each action
        probabilities = self.policy.predict(state)[0]
        #Get the action by sampling from the given probability
        action = np.random.choice(self.actions_space, p=probabilities)

        return action

    def save_model(self,name):
        self.policy.save(name)

    def load_weights(self,name):
        self.policy.load_weights(name)

class Critic():
    def __init__(self, ALPHA, lambda_=0, Gamma = 0.99, n_actions =4,
        input_dims = 8,decay=0.01):

        self.gamma = Gamma
        self.decay = decay
        self.lambda_ = lambda_
        self.input_dims = input_dims
        self.n_actions = n_actions
        self.actions_space = [i for i in range(n_actions)]

        self.linear_weights = np.random.randn(input_dims, 1).ravel() / np.sqrt(input_dims)

        self.optimizer = RMSprop(self.linear_weights,alpha=ALPHA)

    def predict(self, actor_hidden_out, probs, action):

        Q_input = np.zeros(self.input_dims)

        probs = [-prob for prob in probs]
        probs[action] = 1 + probs[action]
        output_size = len(actor_hidden_out)
        #Geting input for critic linear approximation as given
        # in Policy Gradient Methods for
        #Reinforcement Learning with FunctionApproximation
        #by Richard S. Sutton, David McAllester, Satinder Singh, Yishay Mansour
        for i in range(self.n_actions):
            Q_input = Q_input + np.pad(actor_hidden_out,\
                (i*output_size,(self.n_actions-1-i)*output_size)\
                    , 'constant') * probs[i]

        return np.dot(self.linear_weights, Q_input)

    def initialize_eligibility(self, actor_output, action):

        output_size = len(actor_output)
        phi = np.pad(actor_output,(action*output_size,\
            (self.n_actions-1-action)*output_size),'constant')

        self.eligibilty = phi

    def learn(self, reward, next_state,next_action, Q, done, actor):

        #Output of last hidden layer of actor network
        actor_output = actor.phi.predict(next_state)[0]
        output_size = len(actor_output)
        #probability of actions
        probs = actor.policy.predict(next_state)[0]

        #Get Q for next state-action pair
        Q_ = self.predict(actor_output, probs, next_action)

        #When done is true no need to take value of next state and change only the target value of present action
        TD_error = reward + self.gamma * Q_ * (1 - int(done)) - Q

        td_el = TD_error * self.eligibilty
        #Normalize
        # norm = np.linalg.norm(td_el)
        # if norm != 0.0:
        #     td_el = td_el/norm

        #Update weights
        self.linear_weights = self.optimizer.backward_pass(td_el)

        phi_ = np.pad(actor_output,(next_action*output_size,\
            (self.n_actions-1-next_action)*output_size),'constant')

        #Update Eligibility Traces
        self.eligibilty = self.gamma * self.lambda_ * self.eligibilty + phi_

    def save_model(self,name):
        np.save(name,self.linear_weights)

    def load_weights(self,name):
        self.critic.load_weights(name)


text:
import glob
import os

import cv2
import matplotlib.pyplot as plt
import numpy as np
import speed_estimation
from PIL import Image

speed_list = {}
label_start = 0
images = []
no_label = []

for fname in sorted(glob.glob("*.png")):
    im = cv2.imread(fname)
    im = im[:, :, 0]

    boxes = []

    contours, _ = cv2.findContours(im, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    for cnt in contours:
        rect = cv2.minAreaRect(cnt)
        box = cv2.boxPoints(rect)
        boxes.append(box)

    speed_list, label_start = speed_estimation.parse_frame(boxes, speed_list, label_start, 10)

    t = np.zeros_like(im)
    for box in boxes:
        cv2.drawContours(t, [np.int0(box)], 0, (255), 2)

    img = cv2.cvtColor(t, cv2.COLOR_BGR2RGB)
    im_pil = Image.fromarray(img)
    no_label.append(im_pil)

    for objs in speed_list:
        if speed_list[objs].average_speed != 0:
            cv2.putText(
                t,
                str(round(speed_list[objs].average_speed, 1)*5),
                tuple(np.mean(speed_list[objs].current_position, axis=0)),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.5,
                (100, 255, 255),
                2,
                cv2.LINE_AA
            )

    img = cv2.cvtColor(t, cv2.COLOR_BGR2RGB)
    im_pil = Image.fromarray(img)
    images.append(im_pil)

    # show bounding boxes
    # plt.imshow(t)
    # show original image
    # plt.imshow(im)
    # plt.show()

images[0].save('outputs/tracking.gif', save_all=True, append_images=images[1:], optimize=False, duration=400, loop=0)
no_label[0].save('outputs/no_label.gif', save_all=True, append_images=no_label[1:], optimize=False, duration=400, loop=0)


text:
# from itertools import product
import fire
import numpy as np

from run import _run, get_stats
from compare_shards import run_exp

class Cli:
    def run(self, verbose=True, **kwargs):
        run_exp(verbose=verbose, **kwargs)

def format_stats(li):
    li = list(zip(*sorted([v.items() for v in li])))
    dt = {row[0][0]: np.array([v[1] for v in row]) for row in li}
    return dt

def get_means(dt):
    dt = {k: v.mean() for k, v in dt.items()}
    return dt

if __name__ == '__main__':
    fire.Fire(Cli)


text:
import h5py
import numpy as np
import pytest
import json

# We generate mock data files for different versions of the Bluelake HDF5 file
# format:

class MockDataFile_v1:

    def __init__(self, file):
        self.file = h5py.File(file, 'w')

    def get_file_format_version(self):
        return 1

    def write_metadata(self):
        self.file.attrs["Bluelake version"] = "unknown"
        self.file.attrs["File format version"] = self.get_file_format_version()
        self.file.attrs["Experiment"] = ""
        self.file.attrs["Description"] = ""
        self.file.attrs["GUID"] = ""
        self.file.attrs["Export time (ns)"] = -1

    def make_continuous_channel(self, group, name, start, dt, data):
        if group not in self.file:
            self.file.create_group(group)

        self.file[group][name] = data
        dset = self.file[group][name]
        dset.attrs["Start time (ns)"] = start
        dset.attrs["Stop time (ns)"] = start + len(data) * dt
        dset.attrs["Sample rate (Hz)"] = 1 / dt * 1e9
        return dset

    def make_timeseries_channel(self, group, name, data):
        if group not in self.file:
            self.file.create_group(group)

        compound_type = np.dtype([("Timestamp", np.int64), ("Value", float)])
        self.file[group][name] = np.array(data, compound_type)
        dset = self.file[group][name]
        return dset

    def make_timetags_channel(self, group, name, data):
        raise NotImplementedError

class MockDataFile_v2(MockDataFile_v1):

    def get_file_format_version(self):
        return 2

    def make_calibration_data(self, calibration_idx, group, attributes):
        if "Calibration" not in self.file:
            self.file.create_group("Calibration")

        # Numeric value converted to string
        if calibration_idx not in self.file["Calibration"]:
            self.file["Calibration"].create_group(calibration_idx)

        # e.g. Force 1x, Force 1y ... etc
        if group not in self.file["Calibration"][calibration_idx]:
            self.file["Calibration"][calibration_idx].create_group(group)

        # Attributes
        field = self.file["Calibration"][calibration_idx][group]
        for i, v in attributes.items():
            field.attrs[i] = v

    def make_continuous_channel(self, group, name, start, dt, data):
        dset = super().make_continuous_channel(group, name, start, dt, data)
        dset.attrs["Kind"] = "Continuous"

    def make_timeseries_channel(self, group, name, data):
        dset = super().make_timeseries_channel(group, name, data)
        dset.attrs["Kind"] = b"TimeSeries"

    def make_json_data(self, group, name, data):
        if group not in self.file:
            self.file.create_group(group)

        data_location = group + "_" + name
        self.file.create_group(data_location)

        self.file[group][name] = self.file[data_location].create_dataset("value0", data=data)
        return self.file[group][name]

    def make_timetags_channel(self, group, name, data):
        if group not in self.file:
            self.file.create_group(group)

        self.file[group][name] = data
        dset = self.file[group][name]
        dset.attrs["Kind"] = "TimeTags"
        return dset

@pytest.fixture(scope="session", params=[MockDataFile_v1, MockDataFile_v2])
def h5_file(tmpdir_factory, request):
    mock_class = request.param

    tmpdir = tmpdir_factory.mktemp("pylake")
    mock_file = mock_class(tmpdir.join("%s.h5" % mock_class.__class__.__name__))
    mock_file.write_metadata()

    mock_file.make_continuous_channel("Force HF", "Force 1x", 1, 10, np.arange(5.0))
    mock_file.make_continuous_channel("Force HF", "Force 1y", 1, 10, np.arange(5.0, 10.0))

    mock_file.make_timeseries_channel("Force LF", "Force 1x", [(1, 1.1), (2, 2.1)])
    mock_file.make_timeseries_channel("Force LF", "Force 1y", [(1, 1.2), (2, 2.2)])

    if mock_class == MockDataFile_v2:
        mock_file.make_timetags_channel(
            "Photon Time Tags", "Red",
            np.arange(10, 100, step=10, dtype=np.int64))

        calibration_time_field = "Stop time (ns)"
        mock_file.make_calibration_data("1", "Force 1x", {calibration_time_field: 0})
        mock_file.make_calibration_data("2", "Force 1x", {calibration_time_field: 1})
        mock_file.make_calibration_data("3", "Force 1x", {calibration_time_field: 10})
        mock_file.make_calibration_data("4", "Force 1x", {calibration_time_field: 100})
        mock_file.make_calibration_data("1", "Force 1y", {calibration_time_field: 0})
        mock_file.make_calibration_data("2", "Force 1y", {calibration_time_field: 1})
        mock_file.make_calibration_data("3", "Force 1y", {calibration_time_field: 10})
        mock_file.make_calibration_data("4", "Force 1y", {calibration_time_field: 100})

        counts = [2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 8, 0,
                  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0,
                  1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0,
                  0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8, 0]

        infowave = [1, 0, 2, 1, 0, 2, 1, 0, 2, 1, 0, 2, 1, 0, 0, 2,
                    0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 0, 2,
                    1, 0, 2, 1, 0, 2, 1, 0, 2, 1, 0, 2, 1, 0, 0, 2,
                    0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 0, 2]

        enc = json.JSONEncoder()

        json_string = enc.encode({
            "value0": {
                "cereal_class_version": 1,
                "fluorescence": True,
                "force": False,
                "scan count": 0,
                "scan volume": {
                    "center point (um)": {
                        "x": 58.075877109272604,
                        "y": 31.978375270573267,
                        "z": 0
                    },
                    "cereal_class_version": 1,
                    "pixel time (ms)": 0.2,
                    "scan axes": [
                        {
                            "axis": 0,
                            "cereal_class_version": 1,
                            "num of pixels": 5,
                            "pixel size (nm)": 10,
                            "scan time (ms)": 0,
                            "scan width (um)": 36.07468112612217
                        }
                    ]
                }
            }
        })

        # Generate lines at 1 Hz
        freq = int(1e9 / 16)
        mock_file.make_continuous_channel("Photon count", "Red", int(20e9), freq, counts)
        mock_file.make_continuous_channel("Photon count", "Green", int(20e9), freq, counts)
        mock_file.make_continuous_channel("Photon count", "Blue", int(20e9), freq, counts)
        mock_file.make_continuous_channel("Info wave", "Info wave", int(20e9), freq, infowave)
        ds = mock_file.make_json_data("Kymograph", "Kymo1", json_string)
        ds.attrs["Start time (ns)"] = int(20e9)
        ds.attrs["Stop time (ns)"] = int(20e9 + len(infowave) * freq)

        json_string = enc.encode({
            "value0": {
                "cereal_class_version": 1,
                "fluorescence": True,
                "force": False,
                "scan count": 0,
                "scan volume": {
                    "center point (um)": {
                        "x": 58.075877109272604,
                        "y": 31.978375270573267,
                        "z": 0
                    },
                    "cereal_class_version": 1,
                    "pixel time (ms)": 0.2,
                    "scan axes": [
                        {
                            "axis": 1,
                            "cereal_class_version": 1,
                            "num of pixels": 4,
                            "pixel size (nm)": 10,
                            "scan time (ms)": 0,
                            "scan width (um)": 36.07468112612217
                        },
                        {
                            "axis": 0,
                            "cereal_class_version": 1,
                            "num of pixels": 5,
                            "pixel size (nm)": 10,
                            "scan time (ms)": 0,
                            "scan width (um)": 36.07468112612217
                        }
                    ]
                }
            }
        })

        ds = mock_file.make_json_data("Scan", "Scan1", json_string)
        ds.attrs["Start time (ns)"] = int(20e9)
        ds.attrs["Stop time (ns)"] = int(20e9 + len(infowave) * freq)

    return mock_file.file

@pytest.fixture(scope="session")
def h5_file_invalid_version(tmpdir_factory):
    tmpdir = tmpdir_factory.mktemp("pylake")

    mock_file = h5py.File(tmpdir.join("invalid.h5"), 'w')
    mock_file.attrs["Bluelake version"] = "unknown"
    mock_file.attrs["File format version"] = 254

    return mock_file


text:
import numpy as np
import pandas as pd
import streamlit as st
import matplotlib.pyplot as plt
import math
import arabic_reshaper
from bidi.algorithm import get_display
import random as rd

knumber_bins_per_line=2
first_last_name=['المعدل','اسم التلميذ','ر.ت',"رقم التلميذ",]
all_subjects=['النشاط العلمي ',
       'التربية الإسلامية'
          , 'الرياضيات',
          'الاجتماعيات',
          'اللغة الفرنسية',
    'اللغة العربية']

selected_feature="Moyenne Gen"

useful_col=first_last_name+all_subjects
st.title("احصائيات المستوى السادس بمدرسة طه حسين")

def getComparisonFeature(df):
    if st.checkbox("Choisisez un seul critere pour le choix des groupes."):
    # get the list of columns
        criteria = list(df.select_dtypes(include=['float64']).columns)
        criterion = st.selectbox("", criteria)
        return criterion
    return "Moyenne Gen"

def getTaskType():
    # Select columns to display
    if st.checkbox("#1- Choisisez ce que vous voulez faire."):
    # get the list of columns
        tasks = ["تكوين مجموعات","إحصائيات"]
        tasks = st.selectbox("", tasks)
        return tasks
    return ""
#Get requested task
option = getTaskType()
if option == "إحصائيات":
    st.write("Télèchargez le fichier excel qui contient les moyennes par matières.")
elif option =="تكوين محموعات":
    st.write("Télèchargez le fichier excel qui contient les moyennes générales des élèves.")

def plotDensity(df, selected_feature, title="",generate_report=True,name_fig="PNG"):
    plt.rcParams.update({'font.size': 10})
    fig, axs = plt.subplots()
    axs=df[selected_feature].plot.density(color='red',label='Fille')
    axs.set_xlabel('Les moyennes')
    axs.set_ylabel('La densité')
    axs.set_title(title)
    st.pyplot(fig)
    if generate_report:
        plt.savefig(name_fig)

def plotDensity2(df1,df2,label1="",label2="", title="",generate_report=True,name_fig="PNG"):
    plt.rcParams.update({'font.size': 10})
    fig, axs = plt.subplots()
    axs=df1.plot.density(color='red',label=label1)
    axs=df2.plot.density(color='blue',label=label2)
    axs.legend()
    axs.set_xlabel('La moyenne')
    axs.set_ylabel('La densité')
    axs.set_title(title)
    st.pyplot(fig)
    if generate_report:
        plt.savefig(name_fig)

def group_split(df, grp_pct=.5, seed=5145):
    np.random.seed(seed)
    perm = np.random.permutation(df.index)
    m = len(df.index)
    train_end = int(grp_pct * m)
    validate_end = int(grp_pct * m) + train_end
    grp1 = df.iloc[perm[:train_end]]
    grp2 = df.iloc[perm[train_end:validate_end]]
    return grp1,grp2

def isParityBroken(grp):
    return grp['Genre'].value_counts(ascending=True)[0] - grp['Genre'].value_counts(ascending=True)[0] <2

def getBestSplit(df,selected_feature,KeepParity=True):
    best_std=10
    best_mean=100
    best_seed = 0
    for seed in range(12,100,1):
        grp1, grp2 = group_split(df,grp_pct=0.5,seed=seed)
        if KeepParity and (isParityBroken(grp1) or isParityBroken(grp2)):
            pass
        std = abs(grp1[selected_feature].std()-grp2[selected_feature].std())
        mean = abs(grp1[selected_feature].mean()-grp2[selected_feature].mean())

        if std<best_std and mean<best_mean :
            best_std = std
            best_mean = mean
            best_seed = seed
    st.write("L'erreur sur l'écart type des deux groupes est: ","**{:.4f}**".format(abs(best_std-best_std)))
    st.write("L'erreur sur la moyenne des deux groupes est: ","**{:.4f}**".format(mean))

    return group_split(df,grp_pct=0.5,seed=best_seed)

# Upload CSV or XLS files
with st.sidebar.header('#2- تحميل ملف csv/excel.'):
    uploaded_file = st.sidebar.file_uploader("", type=["csv","xls"])

# Some basic stats analysis on classes
if uploaded_file is not None:
    @st.cache
    def load_file():
        if "csv" in uploaded_file.name:
            csv = pd.read_csv(uploaded_file)
            return csv
        elif "xls" in uploaded_file.name:
            df = pd.read_excel (uploaded_file,skiprows=9)
            df.columns=[i.replace("\n","") for i in df.columns]
            df = df.loc[:, ~df.columns.str.contains('^Unnamed')]
            return df

    def getDescription(df,cols):
        return df[cols].describe(include='all')

    def getUsefulColumns():
        # Select columns to display
        if st.checkbox("ضع علامة لاختيار مواد معينة"):
        # get the list of columns
            columns = df.columns.tolist()
            delta_cols= set(columns) - set(first_last_name)
            st.write("#### اختر المواد الدي تريد")
            selected_cols = st.multiselect("", delta_cols)
            return selected_cols
        else:
            return all_subjects

    if option == "إحصائيات":
        df = load_file()
        # create excel writer object
        writer = pd.ExcelWriter('StatReport.xlsx', engine='xlsxwriter')
        # write dataframe to excel
        df.to_excel(writer, sheet_name="Fichier des données",index=False,startrow=2, startcol=2)

        st.header('** Les données **')
        st.write(df)
        cols = getUsefulColumns()
        st.write('---')
        st.header('** Statistiques globales **')
        st.write(getDescription(df,cols))
        getDescription(df,cols).to_excel(writer, sheet_name="Fichier des données",startrow=2, startcol=4+len(df.columns))
        st.header('** Liste des élèves ayant besoin de soutien par matière **')
        st.write('* Dans le cadre de ce script, on considère qu\'un élève a besoin de soutien dans une matière si sa note dans celle-ci est < 5 / 10.')
        #Get mean of each subject and print list of student below the absolute avergae (for now we enjoy 5/10)
        mean_mat=df[cols].mean()
        for i in cols:
            st.write( "{:.2f}".format(mean_mat[i])+"<>"+i)
            st.write(df[df[i]< 5][[i]+first_last_name])
            selection= df[df[i]< 5][[i]+first_last_name]
            if len(selection.index) >0:
                selection.to_excel(writer, sheet_name=str(i),index=False,startrow=2, startcol=2)

        st.header('** Histogrammes des notes par matière**')
        plt.rcParams.update({'font.size': 5})
        arabic_char = {get_display(arabic_reshaper.reshape(k)): k for k in cols}
        _ = math.ceil(len(cols)/knumber_bins_per_line)
        fig, axs = plt.subplots(_, knumber_bins_per_line, sharey=True, sharex=True)

        #Dirty way to hide useless bins
        count = 0
        for i, _c in enumerate(arabic_char.values()):
            ax = axs.flat[i]
            ax.hist(df[[_c]], bins=10,rwidth=0.5,weights=np.ones(len(df[[_c]])) / len(df[[_c]]),color='k', alpha=0.5)
            ax.set_title(get_display(arabic_reshaper.reshape(_c)))
            count +=1
        # Cleanup empty bins
        for item in range(count,knumber_bins_per_line*math.ceil(len(cols)/knumber_bins_per_line)):
            fig.delaxes(axs.flat[item])
        fig.supxlabel('Les moyennes des matières')
        fig.supylabel('Le pourcentage des élèves')
        st.pyplot(fig)

        download=st.button('Télécharger')
        if download:
            workbook  = writer.book
            worksheet = writer.sheets['Fichier des données']
            # Insert an image.
            plt.savefig("Histogrammes.png")

            worksheet.insert_image('H3', 'Histogrammes.png')
            writer.save()

    else:
        df = load_file()
        # create excel writer object
        writer = pd.ExcelWriter('GroupSplit.xlsx', engine='xlsxwriter')
        # write dataframe to excel
        df.to_excel(writer, sheet_name="données",index=False, startrow=2, startcol=2)
        st.header('** Les données **')
        st.write(df)
        st.header('** Récap **')
        selected_feature = getComparisonFeature(df)
        st.write(df[selected_feature].describe())
        df[selected_feature].describe().to_excel(writer, sheet_name="données", startrow=4+len(df.index), startcol=2)

        plotDensity(df,selected_feature,"Densité des notes ("+selected_feature+") du groupe entier",name_fig="GlobalDensity.png")

        if 'Genre' in df.columns:
            colGender = "Genre"
            g= "Garçon"
            f= "Fille"
        elif ' arabic stuf' in df.columns:
            colGender = " arabic "
            g=" garcon in arabic"
            f=" fille in arabic"
        else:
            colGender=""
        if colGender !="":
            plotDensity2(df[df[colGender]==f][selected_feature],
                df[df[colGender]!=f][selected_feature],g,f,
                title="Densité des notes ("+selected_feature+") du groupe par genre",name_fig="GenreDensity.png")

        st.header('** Création de 2 groupes homogènes **')
        KeepParity =True
        if st.checkbox("Cochez ici si vous voulez ignorer la parité fille/garcon"):
            KeepParity = False
        grp1,grp2 = getBestSplit(df,selected_feature,KeepParity)
        st.write("*Groupe 1:*")
        st.write(grp1)
        st.write(getDescription(grp1,selected_feature))
        st.write("*Groupe 2:*")
        st.write(grp2)
        st.write(getDescription(grp2,selected_feature))
        # Plot both kde
        plotDensity2(grp2[selected_feature],grp1[selected_feature],"Groupe 1","Groupe 2",
            title="Densité des notes ("+selected_feature+") des groupes constitués",name_fig="GroupsDensity.png")
        download=st.button('Télécharger')
        if download:
            grp1.to_excel(writer, sheet_name="Groupes",index=False,startrow=2, startcol=2)
            grp2.to_excel(writer, sheet_name="Groupes",index=False,startrow=6+len(grp1.index), startcol=2)
            # Get the xlsxwriter workbook and worksheet objects.
            workbook  = writer.book
            worksheet = writer.sheets['Groupes']
            # Insert an image.
            worksheet.insert_image('K3', 'GroupsDensity.png')
            worksheet2 = writer.sheets['données']
            worksheet2.insert_image('C3', 'GenreDensity.png')
            worksheet2.insert_image('C25', 'GlobalDensity.png')
            writer.save()

else:
    st.write("...")


text:
import torch
import torch.nn as nn
import time
import copy
from .utils import compute_layer_norm
import numpy as np

class Trainer:
    def __init__(self, dataloaders, model, criterion, optimizer, scheduler=None, mlp_width=None, mode='oo', learning_rate=1e-3,
                 weight_decay=1e-2, pretrained_file=None,
                 optimizer_type=None, version=None, use_cuda=True,
                 record_rate=1000, print_rate=10000,
                 **kwargs):
        """
        :param record_rate: How often the trainer logs results (in epochs)
        """

        # Dataset
        self.dataloaders = dataloaders

        # Hyperparameters
        self.mlp_width = mlp_width
        self.mode = mode
        self.learning_rate = learning_rate
        self.weight_decay = weight_decay
        self.pretrained_file = pretrained_file
        self.optimizer_type = optimizer_type

        self.model, self.criterion, self.optimizer =  model, criterion, optimizer
        self.scheduler = scheduler
        # Stored training metrics
        self.val_loss = []
        self.train_loss = []
        self.weights_norms = []
        self.best_model_wts = copy.deepcopy(self.model.state_dict())
        self.num_epochs_trained = 0
        self.best_loss = 0.0
        self.grad_norms = []
        self.use_cuda = use_cuda
        self.record_rate = record_rate
        self.print_rate = print_rate

        # Version
        self.version = version

    def train(self, num_epochs=25, eval_only=False, train_only=False, verbose=True):
        """
        Function taken from Pytorch documentation, has lots of nice functionality in it.
        But I didn't write this so it has a lot.
        """
        since = time.time()

        for epoch in range(num_epochs):
            if verbose:
                print('Epoch {}/{}'.format(epoch, num_epochs - 1))
                print('-' * 10)
            self.num_epochs_trained = self.num_epochs_trained + 1

            # Each epoch has a training and validation phase
            if eval_only:
                phases = ['val']
            elif train_only:
                phases = ['train']
            else:
                phases = ['train', 'val']

            for phase in phases:

                if phase == 'train':
                    self.model.train()  # Set model to training mode
                else:
                    self.model.eval()   # Set model to evaluate mode

                running_loss = 0.0
                running_corrects = 0
                epoch_start_time = time.time()

                # Iterate over data.
                for i, (inputs, labels) in enumerate(self.dataloaders[phase]):
                    if self.use_cuda and torch.cuda.is_available():
                        inputs = inputs.cuda().double()
                        labels = labels.cuda().double()
                    if len(inputs.shape) == 1:
                        inputs = inputs.reshape(inputs.shape[0],1)
                        labels = labels.reshape(labels.shape[0],1)

                    # zero the parameter gradients
                    self.optimizer.zero_grad()

                    # forward
                    # track history if only in train
                    with torch.set_grad_enabled(phase == 'train'):
                        # Get model outputs and calculate loss
                        # Special case for inception because in training it has an auxiliary output. In train
                        #   mode we calculate the loss by summing the final output and the auxiliary output
                        #   but in testing we only consider the final output.
                        outputs = self.model(inputs)
                        loss = self.criterion(outputs, labels)

                        preds = outputs

                        # loss.register_hook(lambda grad: print(grad))

                        # backward + optimize only if in training phase
                        if phase == 'train':
                            loss.backward()
                            self.optimizer.step()

                            if verbose:
                                if (i+1) % self.print_rate == 0:
                                    print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f'
                                        %(epoch+1, num_epochs, i+1, len(train_dataset)//batch_size, loss.data))

                    # statistics
                    running_loss += loss.item() * inputs.size(0)
                    running_corrects += torch.sum(preds == labels.data)

                if self.scheduler is not None:
                    self.scheduler.step()

                epoch_loss = running_loss / len(self.dataloaders[phase].dataset)
                epoch_acc = running_corrects.double() / len(self.dataloaders[phase].dataset)

                epoch_time_elapsed = time.time() - epoch_start_time

                if not verbose:
                    if (epoch+1) % self.print_rate == 0:
                        print('Epoch {} complete in {:.0f}m {:.0f}s'.format(epoch+1, epoch_time_elapsed // 60, epoch_time_elapsed % 60))
                        print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))
                else:
                    print('Epoch complete in {:.0f}m {:.0f}s'.format(epoch_time_elapsed // 60, epoch_time_elapsed % 60))
                    print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))

                # norms = [compute_gradient_norm(m) for m in self.model.classifier.children()]
                # self.grad_norms.append(norms)
                # print('Gradient norms of: classifier layers - {}, features - {}'.format(norms, compute_gradient_norm(self.model.features)))

                # deep copy the model
                if phase == 'val' and epoch_loss > self.best_loss:
                    self.best_loss = epoch_loss
                    self.best_model_wts = copy.deepcopy(self.model.state_dict())

                if (epoch+1) % self.record_rate == 0:
                    # try: # why do I do these things to myself
                    base = compute_layer_norm(self.model, layer_name='linear0')
                    if len(base) > 0:
                        norms = base[0].detach().numpy()[np.newaxis][np.newaxis, :]
                        norms = np.concatenate((norms,
                        compute_layer_norm(self.model, layer_name='weights')[0].detach().numpy()[np.newaxis][np.newaxis, :]),
                        axis=0)
                    else:
                        norms = np.array(compute_layer_norm(self.model, layer_name='weights')[0].detach().numpy())

                    self.weights_norms.append(norms)
                    if phase == 'val':
                        self.val_loss.append((epoch+1, epoch_loss))
                    else:
                        self.train_loss.append((epoch+1, epoch_loss))

        time_elapsed = time.time() - since
        print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
        # print('Best val Loss: {:4f}'.format(self.best_loss))

        # load best model weights
        # self.model.load_state_dict(self.best_model_wts)
        return self.model

max_stars_repo_name: meissnereric/dill-torch

max_issues_repo_name: meissnereric/dill-torch

max_forks_repo_name: meissnereric/dill-torch