text:
"""
This file exists to provide one common place for all serialisation and simplify_ and _detail
for all native python objects.
"""
from collections import OrderedDict
from typing import Collection
from typing import Dict
from typing import Union
from typing import Tuple

import numpy

from syft.workers.abstract import AbstractWorker
from syft import serde

# Simplify/Detail Collections (list, set, tuple, etc.)

def _simplify_collection(
    worker: AbstractWorker, my_collection: Collection, shallow: bool = False
) -> Tuple:
    """
    This function is designed to search a collection for any objects
    which may need to be simplified (i.e., torch tensors). It iterates
    through each object in the collection and calls _simplify on it. Finally,
    it returns the output as the tuple of simplified items of the input collection.
    This function is used to simplify list, set, and tuple. The reverse function,
    which undoes the functionality of this function is different for each of these types:
    _detail_collection_list, _detail_collection_set, _detail_collection_tuple.

    Args:
        my_collection (Collection): a collection of python objects

    Returns:
        Tuple: a tuple with simplified objects.

    """

    # Don't simplify contents
    if shallow:
        return tuple(my_collection)

    # Step 0: initialize empty list
    pieces = list()

    # Step 1: serialize each part of the collection
    for part in my_collection:
        pieces.append(serde._simplify(worker, part))

    # Step 2: return serialization as tuple of simplified items
    return tuple(pieces)

def _detail_collection_list(
    worker: AbstractWorker, my_collection: Tuple, shallow: bool = False
) -> Collection:
    """
    This function is designed to operate in the opposite direction of
    _simplify_collection. It takes a tuple of simple python objects
    and iterates through it to determine whether objects in the collection
    need to be converted into more advanced types. In particular, it
    converts binary objects into torch Tensors where appropriate.

    Args:
        worker: the worker doing the deserialization
        my_collection (Tuple): a tuple of simple python objects (including binary).

    Returns:
        Collection: a collection of the same type as the input where the objects
            in the collection have been detailed.
    """

    # Don't detail contents
    if shallow:
        return list(my_collection)

    pieces = list()

    # Step 1: deserialize each part of the collection
    for part in my_collection:
        detailed = serde._detail(worker, part)
        pieces.append(detailed)

    return pieces

def _detail_collection_set(
    worker: AbstractWorker, my_collection: Tuple, shallow: bool = False
) -> Collection:
    """
    This function is designed to operate in the opposite direction of
    _simplify_collection. It takes a tuple of simple python objects
    and iterates through it to determine whether objects in the collection
    need to be converted into more advanced types. In particular, it
    converts binary objects into torch Tensors where appropriate.

    Args:
        worker: the worker doing the deserialization
        my_collection (Tuple): a tuple of simple python objects (including binary).

    Returns:
        Collection: a collection of the same type as the input where the objects
            in the collection have been detailed.
    """

    # Don't detail contents
    if shallow:
        return set(my_collection)

    pieces = list()

    # Step 1: deserialize each part of the collection
    for part in my_collection:
        detailed = serde._detail(worker, part)
        pieces.append(detailed)
    return set(pieces)

def _detail_collection_tuple(
    worker: AbstractWorker, my_tuple: Tuple, shallow: bool = False
) -> Tuple:
    """
    This function is designed to operate in the opposite direction of
    _simplify_collection. It takes a tuple of simple python objects
    and iterates through it to determine whether objects in the collection
    need to be converted into more advanced types. In particular, it
    converts binary objects into torch Tensors where appropriate.
    This is only applicable to tuples. They need special handling because
    `msgpack` is encoding a tuple as a list.

    Args:
        worker: the worker doing the deserialization
        my_tuple (Tuple): a collection of simple python objects (including binary).

    Returns:
        tuple: a collection of the same type as the input where the objects
            in the collection have been detailed.
    """

    # Don't detail contents
    if shallow:
        return my_tuple

    pieces = list()

    # Step 1: deserialize each part of the collection
    for part in my_tuple:
        pieces.append(serde._detail(worker, part))

    return tuple(pieces)

def _simplify_dictionary(worker: AbstractWorker, my_dict: Dict, shallow: bool = False) -> Tuple:
    """
    This function is designed to search a dict for any objects
    which may need to be simplified (i.e., torch tensors). It iterates
    through each key, value in the dict and calls _simplify on it. Finally,
    it returns the output tuple of tuples containing key/value pairs. The
    reverse function to this function is _detail_dictionary, which undoes
    the functionality of this function.

    Args:
        my_dict: A dictionary of python objects.

    Returns:
        Tuple: Tuple containing tuples of simplified key/value pairs from the
            input dictionary.

    """
    pieces = list()
    # for dictionaries we want to simplify both the key and the value
    for key, value in my_dict.items():
        pieces.append(
            (serde._simplify(worker, key), serde._simplify(worker, value) if not shallow else value)
        )

    return tuple(pieces)

def _detail_dictionary(worker: AbstractWorker, my_dict: Tuple, shallow: bool = False) -> Dict:
    """
    This function is designed to operate in the opposite direction of
    _simplify_dictionary. It takes a dictionary of simple python objects
    and iterates through it to determine whether objects in the collection
    need to be converted into more advanced types. In particular, it
    converts binary objects into torch Tensors where appropriate.

    Args:
        worker: the worker doing the deserialization
        my_dict (Tuple): a simplified dictionary of simple python objects (including binary).

    Returns:
        Dict: a collection of the same type as the input where the objects
            in the collection have been detailed.
    """
    pieces = {}
    # for dictionaries we want to detail both the key and the value
    for key, value in my_dict:
        detailed_key = serde._detail(worker, key)

        if shallow:
            pieces[detailed_key] = value
        else:
            detailed_value = serde._detail(worker, value)
            pieces[detailed_key] = detailed_value

    return pieces

# Simplify/Detail native types

def _simplify_str(worker: AbstractWorker, obj: str) -> tuple:
    return (obj.encode("utf-8"),)

def _detail_str(worker: AbstractWorker, str_tuple: tuple) -> str:
    return str_tuple[0].decode("utf-8")

def _simplify_range(worker: AbstractWorker, my_range: range) -> Tuple[int, int, int]:
    """
    This function extracts the start, stop and step from the range.

    Args:
        my_range (range): a range object

    Returns:
        list: a list defining the range parameters [start, stop, step]

    Examples:

        range_parameters = _simplify_range(range(1, 3, 4))

        assert range_parameters == [1, 3, 4]

    """

    return (my_range.start, my_range.stop, my_range.step)

def _detail_range(worker: AbstractWorker, my_range_params: Tuple[int, int, int]) -> range:
    """
    This function extracts the start, stop and step from a tuple.

    Args:
        worker: the worker doing the deserialization (only here to standardise signature
            with other _detail functions)
        my_range_params (tuple): a tuple defining the range parameters [start, stop, step]

    Returns:
        range: a range object

    Examples:
        new_range = _detail_range([1, 3, 4])

        assert new_range == range(1, 3, 4)

    """

    return range(my_range_params[0], my_range_params[1], my_range_params[2])

def _simplify_ellipsis(worker: AbstractWorker, e: Ellipsis) -> Tuple[bytes]:
    return (b"",)

def _detail_ellipsis(worker: AbstractWorker, ellipsis: bytes) -> Ellipsis:
    return ...

def _simplify_slice(worker: AbstractWorker, my_slice: slice) -> Tuple[int, int, int]:
    """
    This function creates a list that represents a slice.

    Args:
        my_slice (slice): a python slice

    Returns:
        tuple : a list holding the start, stop and step values

    Examples:

        slice_representation = _simplify_slice(slice(1,2,3))

    """
    return (my_slice.start, my_slice.stop, my_slice.step)

def _detail_slice(worker: AbstractWorker, my_slice: Tuple[int, int, int]) -> slice:
    """
    This function extracts the start, stop and step from a list.

    Args:
        my_slice (tuple): a list defining the slice parameters [start, stop, step]

    Returns:
        range: a range object

    Examples:
        new_range = _detail_range([1, 3, 4])

        assert new_range == range(1, 3, 4)

    """

    return slice(my_slice[0], my_slice[1], my_slice[2])

#   Numpy array

def _simplify_ndarray(worker: AbstractWorker, my_array: numpy.ndarray) -> Tuple[bin, Tuple, Tuple]:
    """
    This function gets the byte representation of the array
        and stores the dtype and shape for reconstruction

    Args:
        my_array (numpy.ndarray): a numpy array

    Returns:
        list: a list holding the byte representation, shape and dtype of the array

    Examples:

        arr_representation = _simplify_ndarray(numpy.random.random([1000, 1000])))

    """
    arr_bytes = my_array.tobytes()
    arr_shape = serde._simplify(worker, my_array.shape)
    arr_dtype = serde._simplify(worker, my_array.dtype.name)

    return (arr_bytes, arr_shape, arr_dtype)

def _detail_ndarray(
    worker: AbstractWorker, arr_representation: Tuple[bin, Tuple, str]
) -> numpy.ndarray:
    """
    This function reconstruct a numpy array from it's byte data, the shape and the dtype
        by first loading the byte data with the appropiate dtype and then reshaping it into the
        original shape

    Args:
        worker: the worker doing the deserialization
        arr_representation (tuple): a tuple holding the byte representation, shape
        and dtype of the array

    Returns:
        numpy.ndarray: a numpy array

    Examples:
        arr = _detail_ndarray(arr_representation)

    """
    arr_shape = serde._detail(worker, arr_representation[1])
    arr_dtype = serde._detail(worker, arr_representation[2])
    res = numpy.frombuffer(arr_representation[0], dtype=arr_dtype).reshape(arr_shape)

    assert type(res) == numpy.ndarray

    return res

def _simplify_numpy_number(
    worker: AbstractWorker, numpy_nb: Union[numpy.int32, numpy.int64, numpy.float32, numpy.float64]
) -> Tuple[bin, Tuple]:
    """
    This function gets the byte representation of the numpy number
        and stores the dtype for reconstruction

    Args:
        numpy_nb (e.g numpy.float64): a numpy number

    Returns:
        list: a list holding the byte representation, dtype of the numpy number

    Examples:

        np_representation = _simplify_numpy_number(worker, numpy.float64(2.3)))

    """
    nb_bytes = numpy_nb.tobytes()
    nb_dtype = serde._simplify(worker, numpy_nb.dtype.name)

    return (nb_bytes, nb_dtype)

def _detail_numpy_number(
    worker: AbstractWorker, nb_representation: Tuple[bin, Tuple, str]
) -> Union[numpy.int32, numpy.int64, numpy.float32, numpy.float64]:
    """
    This function reconstruct a numpy number from it's byte data, dtype
        by first loading the byte data with the appropiate dtype

    Args:
        worker: the worker doing the deserialization
        np_representation (tuple): a tuple holding the byte representation
        and dtype of the numpy number

    Returns:
        numpy.float or numpy.int: a numpy number

    Examples:
        nb = _detail_numpy_number(nb_representation)

    """
    nb_dtype = serde._detail(worker, nb_representation[1])
    nb = numpy.frombuffer(nb_representation[0], dtype=nb_dtype)[0]

    assert type(nb) in [numpy.float32, numpy.float64, numpy.int32, numpy.int64]

    return nb

# Maps a type to a tuple containing its simplifier and detailer function
# IMPORTANT: serialization constants for these objects need to be defined in `proto.json` file
# in https://github.com/OpenMined/proto
MAP_NATIVE_SIMPLIFIERS_AND_DETAILERS = OrderedDict(
    {
        dict: (_simplify_dictionary, _detail_dictionary),
        list: (_simplify_collection, _detail_collection_list),
        range: (_simplify_range, _detail_range),
        set: (_simplify_collection, _detail_collection_set),
        slice: (_simplify_slice, _detail_slice),
        str: (_simplify_str, _detail_str),
        tuple: (_simplify_collection, _detail_collection_tuple),
        type(Ellipsis): (_simplify_ellipsis, _detail_ellipsis),
        numpy.ndarray: (_simplify_ndarray, _detail_ndarray),
        numpy.float32: (_simplify_numpy_number, _detail_numpy_number),
        numpy.float64: (_simplify_numpy_number, _detail_numpy_number),
        numpy.int32: (_simplify_numpy_number, _detail_numpy_number),
        numpy.int64: (_simplify_numpy_number, _detail_numpy_number),
    }
)

max_stars_repo_name: robot-ai-machinelearning/PySyft

max_issues_repo_name: robot-ai-machinelearning/PySyft

max_forks_repo_name: robot-ai-machinelearning/PySyft


text:
import csv

import pandas as pd
import numpy as np

def get_data(file_path):
    df=pd.read_csv(file_path)
    columns=list(df.columns)
    dtypes={x: np.float16 for x in columns if x.startswith(('feature','target'))}
    df=df.astype(dtypes)
    return df

def get_feature_names(df):
    return [c for c in df.columns if 'feature' in c]

ext: py

max_stars_repo_name: gregoryverghese/numerai

max_issues_repo_name: gregoryverghese/numerai

max_forks_repo_name: gregoryverghese/numerai

reason: import numpy


text:
#!/usr/sbin/python
#-*- encoding:utf-8 -*-
#
# http://www.quuxlabs.com/blog/2010/09/matrix-factorization-a-simple-tutorial-and-implementation-in-python/
#

import operator
import numpy as np

with open('user_prefs.txt') as f:
    prefs_str = ''.join(f.readlines())

# {'andy': {'霍乱时期的爱情': 1},...}
def read_prefs(prefs_str):
    itemset = set()
    userset = set()
    prefs = {}
    for line in prefs_str.split('\n'):
        parts = line.rstrip().split()
        if len(parts) == 2:
            userId, itemId = parts
            itemset.add(itemId)
            userset.add(userId)
            prefs.setdefault(userId, {})
            prefs[userId].update({itemId:1})

    mat = np.empty((len(userset), len(itemset)))
    for user in prefs:
        for item in prefs[user]:
            i = list(userset).index(user)
            j = list(itemset).index(item)
            mat[i][j] = prefs[user][item]
    return prefs, mat, list(itemset), list(userset)

def matrix_factorization(R, P, Q, K, steps=5000, alpha=0.0002, beta=0.02):
    Q = Q.T
    for step in range(steps):
        for i in range(len(R)):
            for j in range(len(R[i])):
                if R[i][j] > 0:
                    eij = R[i][j] - np.dot(P[i,:],Q[:,j])
                    for k in range(K):
                        P[i][k] = P[i][k] + alpha * (2 * eij * Q[k][j] - beta * P[i][k])
                        Q[k][j] = Q[k][j] + alpha * (2 * eij * P[i][k] - beta * Q[k][j])
        eR = np.dot(P,Q)
        e = 0
        for i in range(len(R)):
            for j in range(len(R[i])):
                if R[i][j] > 0:
                    e = e + pow(R[i][j] - np.dot(P[i,:],Q[:,j]), 2)
                    for k in range(K):
                        e = e + (beta/2) * (pow(P[i][k],2) + pow(Q[k][j],2))
        if e < 0.001:
            break
    return P, Q.T

prefs, mat, items, users = read_prefs(prefs_str)

print(mat)
print(items)
print(users)

R = mat
    # [
    #  [5,3,0,1],
    #  [4,0,0,1],
    #  [1,1,0,5],
    #  [1,0,0,4],
    #  [0,1,5,4],
    # ]

R = np.array(R)

N = len(R)
M = len(R[0])
K = 2

P = np.random.rand(N,K)
Q = np.random.rand(M,K)

nP, nQ = matrix_factorization(R, P, Q, K, steps=5000, alpha=0.0002, beta=0.02)
nR = np.dot(nP, nQ.T)
# print(repr(R))
# print(repr(nP))
# print(repr(nQ))
# print(repr(nR))

print('\n======= 用户矩阵 ========')
for i in range(0, len(nP)):
    print(users[i], nP[i])

print('\n======= 物品矩阵 ========')
for i in range(0, len(nQ)):
    print(items[i], nQ[i])

print('\n======= Matrix Factorization 推荐结果 ========')
for i in range(0, len(nP)):
    for j in range(0, len(nQ)):
        score = np.sum(nP[i] * nQ[j])
        if score > 1.1 and R[i][j] == 0:
            print(users[i], items[j], score)

max_stars_repo_name: liuslevis/CFDemo

max_issues_repo_name: liuslevis/CFDemo

max_forks_repo_name: liuslevis/CFDemo


text:
from args_parser import ArgsParser
import cv2
import copy
from detector import DetectorFactory
from tracker import Tracker
import numpy as np
from graphic import GraphicBuilder
from events import FinishVideoProcessingEvent
from event_bus import EventBus
import pika
import pickle
import json

class Analyzer(object):
    def __init__(self, object_detector, object_tracker):
        self.object_detector = object_detector
        self.object_tracker = object_tracker

    def analyze(self, video_path, output_video_path, user_id):

        # Create opencv video capture object
        cap = cv2.VideoCapture(video_path)
        writer = None

        # Variables initialization
        skip_frame_count = 0
        track_colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0),
                        (0, 255, 255), (255, 0, 255), (255, 127, 255),
                        (127, 0, 255), (127, 0, 127)]

        tracks_dictionary = {}

        frame_count = 0
        # Infinite loop to process video frames
        while (True):
            # Capture frame-by-frame
            ret, frame = cap.read()

            if frame is None:
                break

            # necessary to create dictionary
            frame_count += 1

            # Make copy of original frame
            orig_frame = copy.copy(frame)

            # ???????????
            # Skip initial frames that display logo
            if (skip_frame_count < 15):
                skip_frame_count += 1
                continue

            # Detect and return centeroids of the objects in the frame
            centers = self.object_detector.detect(frame)

            # If centroids are detected then track them
            if (len(centers) > 0):

                # Track object using Kalman Filter
                self.object_tracker.update(centers)

                # For identified object tracks draw tracking line
                # Use various colors to indicate different track_id
                for i in range(len(self.object_tracker.tracks)):

                    # check if track is in my dictionary or need to add it
                    if self.object_tracker.tracks[i].track_id not in tracks_dictionary:
                        tracks_dictionary[self.object_tracker.tracks[i].track_id] = {}

                    if (len(self.object_tracker.tracks[i].trace) > 1):
                        # Add frame and coordinates this track on this frame
                        # For Unscented Kalman Filter
                        tracks_dictionary[self.object_tracker.tracks[i].track_id][frame_count] = [
                            self.object_tracker.tracks[i].trace[len(self.object_tracker.tracks[i].trace) - 1][0],
                            self.object_tracker.tracks[i].trace[len(self.object_tracker.tracks[i].trace) - 1][1]]

                        for j in range(len(self.object_tracker.tracks[i].trace) - 1, 1, -1):
                            # For Unscented Kalman Filter
                            x1 = self.object_tracker.tracks[i].trace[j][0]
                            y1 = self.object_tracker.tracks[i].trace[j][1]
                            x2 = self.object_tracker.tracks[i].trace[j - 1][0]
                            y2 = self.object_tracker.tracks[i].trace[j - 1][1]
                            clr = self.object_tracker.tracks[i].track_id % 9
                            cv2.line(frame, (int(x1), int(y1)), (int(x2), int(y2)), track_colors[clr], 2)

                            text = "{}".format(self.object_tracker.tracks[i].track_id)
                            cv2.putText(frame, text, (int(self.object_tracker.tracks[i].trace[len(self.object_tracker.tracks[i].trace) - 1][0]),
                                                      int(self.object_tracker.tracks[i].trace[len(self.object_tracker.tracks[i].trace) - 1][
                                                              1] + 5)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)

                frame = cv2.resize(frame, (1250, 750))
                cv2.imshow('Tracking', frame)
                # check if the video writer is None
                if writer is None:
                    # initialize our video writer
                    fourcc = cv2.VideoWriter_fourcc(*"MJPG")
                    writer = cv2.VideoWriter(output_video_path, fourcc, 30, (frame.shape[1], frame.shape[0]),
                                             True)

                # write the output frame to disk
                writer.write(frame)

            #cv2.waitKey(50)

            # Check for key strokes
            #k = cv2.waitKey(50) & 0xff
            #if k == 27:  # 'esc' key has been pressed, exit program.
            #    break
            #if k == 112:  # 'p' has been pressed. this will pause/resume the code.
            #    pause = not pause
            #    if (pause is True):
            #        print("Code is paused. Press 'p' to resume..")
            #        while (pause is True):
            #            # stay in this loop until
            #            key = cv2.waitKey(30) & 0xff
            #            if key == 112:
            #                pause = False
            #                print("Resume code..!!")
            #                break

        graphic_builder = GraphicBuilder(tracks_dictionary)
        graphic_builder.build()

        finish_video_processing_event = FinishVideoProcessingEvent(output_video_path, graphic_builder.tracks_dictionary, graphic_builder.imagePaths, user_id)

        event_bus = EventBus(finish_video_processing_event)
        event_bus.publish()

        cap.release()
        writer.release()
        cv2.destroyAllWindows()

if __name__ == '__main__':
    args_parser = ArgsParser()
    args_parser.parse()

    # Create Object Detector
    detectorFactory = DetectorFactory()
    object_detector = detectorFactory.create_detector('yolo', args_parser)

    # Create Object Tracker
    object_tracker = Tracker(100, 5, 5)

    analyzer = Analyzer(object_detector, object_tracker)
    analyzer.analyze('videos/test_occlusion_short_1second.mp4', 'output/readyForRabbit.avi')


text:
import pandas as pd
import numpy as np
from sklearn.impute import KNNImputer
from sklearn.preprocessing import StandardScaler

class Preprocessor:
    """This class shall  be used to clean and transform the data before training."""

    def __init__(self, file_object, logger_object):
        self.file_object = file_object
        self.logger_object = logger_object

    def remove_columns(self,data,columns):
        """
                Method Name: remove_columns
                Description: This method removes the given columns from a pandas dataframe.
                Output: A pandas DataFrame after removing the specified columns.
                On Failure: Raise Exception"""
        self.logger_object.log(self.file_object, 'Entered the remove_columns method of the Preprocessor class')
        self.data=data
        self.columns=columns
        try:
            self.useful_data=self.data.drop(labels=self.columns, axis=1) # drop the labels specified in the columns
            self.logger_object.log(self.file_object,
                                   'Column removal Successful.Exited the remove_columns method of the Preprocessor class')
            return self.useful_data
        except Exception as e:
            self.logger_object.log(self.file_object,'Exception occured in remove_columns method of the Preprocessor class. Exception message:  '+str(e))
            self.logger_object.log(self.file_object,
                                   'Column removal Unsuccessful. Exited the remove_columns method of the Preprocessor class')
            raise Exception()

    def separate_label_feature(self, data, label_column_name):
        """
                        Method Name: separate_label_feature
                        Description: This method separates the features and a Label Coulmns.
                        Output: Returns two separate Dataframes, one containing features and the other containing Labels .
                        On Failure: Raise Exception"""
        self.logger_object.log(self.file_object, 'Entered the separate_label_feature method of the Preprocessor class')
        try:
            self.X=data.drop(labels=label_column_name,axis=1) # drop the columns specified and separate the feature columns
            self.Y=data[label_column_name] # Filter the Label columns
            self.logger_object.log(self.file_object,
                                   'Label Separation Successful. Exited the separate_label_feature method of the Preprocessor class')
            return self.X,self.Y
        except Exception as e:
            self.logger_object.log(self.file_object,'Exception occured in separate_label_feature method of the Preprocessor class. Exception message:  ' + str(e))
            self.logger_object.log(self.file_object, 'Label Separation Unsuccessful. Exited the separate_label_feature method of the Preprocessor class')
            raise Exception()

    def dropUnnecessaryColumns(self,data,columnNameList):
        """Method Name: is_null_present
           Description: This method drops the unwanted columns as discussed in EDA section."""
        data = data.drop(columnNameList,axis=1)
        return data

    def replaceInvalidValuesWithNull(self,data):

        """
                Method Name: is_null_present
                Description: This method replaces invalid values i.e. '?' with null, as discussed in EDA."""

        for column in data.columns:
            count = data[column][data[column] == '?'].count()
            if count != 0:
                data[column] = data[column].replace('?', np.nan)
        return data

    def is_null_present(self,data):
        """
                                Method Name: is_null_present
                                Description: This method checks whether there are null values present in the pandas Dataframe or not.
                                Output: Returns True if null values are present in the DataFrame, False if they are not present and
                                        returns the list of columns for which null values are present.
                                On Failure: Raise Exception"""
        self.logger_object.log(self.file_object, 'Entered the is_null_present method of the Preprocessor class')
        self.null_present = False
        self.cols_with_missing_values=[]
        self.cols = data.columns
        try:
            self.null_counts=data.isna().sum() # check for the count of null values per column
            for i in range(len(self.null_counts)):
                if self.null_counts[i]>0:
                    self.null_present=True
                    self.cols_with_missing_values.append(self.cols[i])
            if(self.null_present): # write the logs to see which columns have null values
                self.dataframe_with_null = pd.DataFrame()
                self.dataframe_with_null['columns'] = data.columns
                self.dataframe_with_null['missing values count'] = np.asarray(data.isna().sum())
                self.dataframe_with_null.to_csv('preprocessing_data/null_values.csv') # storing the null column information to file
            self.logger_object.log(self.file_object,'Finding missing values is a success.Data written to the null values file. Exited the is_null_present method of the Preprocessor class')
            return self.null_present, self.cols_with_missing_values
        except Exception as e:
            self.logger_object.log(self.file_object,'Exception occured in is_null_present method of the Preprocessor class. Exception message:  ' + str(e))
            self.logger_object.log(self.file_object,'Finding missing values failed. Exited the is_null_present method of the Preprocessor class')
            raise Exception()

    def encodeCategoricalValues(self,data):
     """
            Method Name: encodeCategoricalValues
            Description: This method encodes all the categorical values in the training set.
            Output: A Dataframe which has all the categorical values encoded.
            On Failure: Raise Exception"""
     data["class"] = data["class"].map({'p': 1, 'e': 2})

     for column in data.drop(['class'],axis=1).columns:
            data = pd.get_dummies(data, columns=[column])

     return data

    def encodeCategoricalValuesPrediction(self,data):
        """
                Method Name: encodeCategoricalValuesPrediction
                Description: This method encodes all the categorical values in the prediction set.
                Output: A Dataframe which has all the categorical values encoded.
                On Failure: Raise Exception"""

        for column in data.columns:
            data = pd.get_dummies(data, columns=[column])

        return data

    # def handleImbalanceDataset(self,X,Y):
    #     """
    #                                                   Method Name: handleImbalanceDataset
    #                                                   Description: This method handles the imbalance in the dataset by oversampling.
    #                                                   Output: A Dataframe which is balanced now.
    #                                                   On Failure: Raise Exception
    #
    #                                                   Written By: iNeuron Intelligence
    #                                                   Version: 1.0
    #                                                   Revisions: None
    #                                """
    #
    #
    #
    #     rdsmple = RandomOverSampler()
    #     x_sampled, y_sampled = rdsmple.fit_sample(X, Y)
    #
    #     return x_sampled,y_sampled

    def standardScalingData(self,X):

        scalar = StandardScaler()
        X_scaled = scalar.fit_transform(X)

        return X_scaled

    def logTransformation(self,X):

        for column in X.columns:
            X[column] += 1
            X[column] = np.log(X[column])

        return X

    def impute_missing_values(self, data):
        """
            Method Name: impute_missing_values
            Description: This method replaces all the missing values in the Dataframe using KNN Imputer.
            Output: A Dataframe which has all the missing values imputed.
            On Failure: Raise Exception."""
        self.logger_object.log(self.file_object, 'Entered the impute_missing_values method of the Preprocessor class')
        self.data= data
        try:
            imputer=KNNImputer(n_neighbors=3, weights='uniform',missing_values=np.nan)
            self.new_array=imputer.fit_transform(self.data) # impute the missing values
            # convert the nd-array returned in the step above to a Dataframe
            self.new_data=pd.DataFrame(data=(self.new_array), columns=self.data.columns)
            self.logger_object.log(self.file_object, 'Imputing missing values Successful. Exited the impute_missing_values method of the Preprocessor class')
            return self.new_data
        except Exception as e:
            self.logger_object.log(self.file_object,'Exception occured in impute_missing_values method of the Preprocessor class. Exception message:  ' + str(e))
            self.logger_object.log(self.file_object,'Imputing missing values failed. Exited the impute_missing_values method of the Preprocessor class')
            raise Exception()

    def get_columns_with_zero_std_deviation(self,data):
        """
                Method Name: get_columns_with_zero_std_deviation
                Description: This method finds out the columns which have a standard deviation of zero.
                Output: List of the columns with standard deviation of zero
                On Failure: Raise Exception."""
        self.logger_object.log(self.file_object, 'Entered the get_columns_with_zero_std_deviation method of the Preprocessor class')
        self.columns=data.columns
        self.data_n = data.describe()
        self.col_to_drop=[]
        try:
            for x in self.columns:
                if (self.data_n[x]['std'] == 0): # check if standard deviation is zero
                    self.col_to_drop.append(x)  # prepare the list of columns with standard deviation zero
            self.logger_object.log(self.file_object, 'Column search for Standard Deviation of Zero Successful. Exited the get_columns_with_zero_std_deviation method of the Preprocessor class')
            return self.col_to_drop

        except Exception as e:
            self.logger_object.log(self.file_object,'Exception occured in get_columns_with_zero_std_deviation method of the Preprocessor class. Exception message:  ' + str(e))
            self.logger_object.log(self.file_object, 'Column search for Standard Deviation of Zero Failed. Exited the get_columns_with_zero_std_deviation method of the Preprocessor class')
            raise Exception()


text:
from pathlib import Path
import pandas as pd
import gensim
import numpy as np
import sklearn
from sklearn import decomposition
from sklearn.preprocessing import normalize
from sklearn.cluster import KMeans, AffinityPropagation
from sklearn.metrics import pairwise_distances_argmin_min
import matplotlib.pyplot as plt
import csv
from operator import itemgetter
from tqdm import tqdm
import tensorflow as tf
from tensorboard.plugins import projector
from tensorflow.data import Dataset
from tqdm import tqdm
from eval import Performance
from heapq import nlargest
from transformers import TFLongformerModel, LongformerTokenizer
import os
import csv

#Script used to visualize results of clustering on Tensorboard Projector.

tokenizer = LongformerTokenizer.from_pretrained('../storage/tokenizer', max_length = 2048)
def clean_read(filename):
    return open(filename, 'r').read()\
            .replace('\n\n','. ')\
            .replace('..', '.')\
            .replace('\n', '')
def create_data_set(files, labels):
    inputs = tokenizer([clean_read(x) for x in files], return_tensors="tf", padding='max_length', max_length=2048, truncation=True)
    inputs = {k: inputs[k] for k in ['input_ids', 'attention_mask']}
    labels = tf.reshape(tf.constant(labels, dtype=tf.int32), (len(labels), 1))
    d_inputs = Dataset.from_tensor_slices(inputs)
    d_labels = Dataset.from_tensor_slices(labels)
    return inputs, labels

pca = decomposition.PCA()
PATH = Path("../storage/dataset_2/dataset")
iterd = PATH.iterdir()
filenames = []
labels = []
iterd = PATH.iterdir()
for cat in iterd :
    filenames.append([str(x) for x in cat.iterdir()])
    labels.append([('IRRELEVANT_TREATED' not in str(x)) for x in cat.iterdir()])

files_train, files_test, y_train, y_test = filenames[0], filenames[1], labels[0], labels[1]
files = files_train+files_test
y = y_train+y_test
f = open('/home/pfrod/results/clustering_num_shuffled_articles.txt', 'a')
for i, (file, x) in enumerate(zip(files, y)):
    print(f'{i}'+', '+f'{file}'+', '+f'{x}', file = f)
f.close()

inputs, labels = create_data_set(files, y)
labels = labels.numpy()[:1624]
labels = labels.flatten()
#Allows to visualize embedding for the finetuned architecture, just change the input file if you want to visualize embeddings without finetune.
trained = TFLongformerModel.from_pretrained('longformer_128', from_pt = True, attention_window = 128, trainable = False)
sh = tf.shape(inputs['input_ids'])
result = not_trained({'input_ids' : inputs['input_ids'][:8,:], 'attention_mask' :inputs['attention_mask'][:8,:]})
liste_embeddings_not_trained = []
liste_embeddings_trained = []
for i in tqdm(range(tf.shape(inputs['input_ids'])[0]//8)):
    liste_embeddings_trained.append(trained({'input_ids' : inputs['input_ids'][i*8:(i+1)*8,:], 'attention_mask' :inputs['attention_mask'][:8,:]})['pooler_output'])
embeddings_mat_trained = tf.concat(liste_embeddings_trained, axis = 0)
embeddings_mat_trained = embeddings_mat_trained.numpy()

# Set up a logs directory, so Tensorboard knows where to look for files.
log_dir='/home/pfrod/architectures/logs/viz_embeddings_train_nopretrain/'
if not os.path.exists(log_dir):
    os.makedirs(log_dir)

# Save Labels separately on a line-by-line manner.
with open(os.path.join(log_dir, 'metadata.tsv'), "w") as f:
    tsv_writer = csv.writer(f, delimiter='\t')
    for label in labels:
        tsv_writer.writerow([label])
with open(os.path.join(log_dir, 'vectors.tsv'), 'w') as f :
    tsv_writer = csv.writer(f, delimiter='\t')
    for vector in embeddings_mat_trained :
        tsv_writer.writerow(vector)

max_issues_repo_name: readerbench/IRVIN


text:
import numpy as np
import pandas as pd

from classification_model.processing.data_management import load_pipeline
from classification_model.config import config
from classification_model.processing.validation import validate_inputs
from classification_model import __version__ as _version

import logging

_logger = logging.getLogger(__name__)

pipeline_file_name = f"{config.PIPELINE_SAVE_FILE}{_version}.pkl"
_current_pipe = load_pipeline(file_name=pipeline_file_name)

def make_prediction(*, input_data) -> dict:

	data = pd.read_json(input_data)

	validated_data = validate_inputs(input_data=data)

	prediction = _current_pipe.predict_test(validated_data)

	results = {"predictions": prediction, "version": _version}

	_logger.info(
		f"Making predictions with model version: {_version} "
		f"Inputs: {validated_data} "
		f"Predictions: {results}"
	)

	return results


text:
import Teams
import csv
import datetime as dt
import numpy as np
from constants import _Const

def LoadTeamData(stop_date, input_team):
    CONST = _Const()
    data = Teams.csvopen(CONST.INPUT_FILE)
    next(data)

    fixtures = {}

    for row in data:
        date=dt.datetime.strptime(row[1], '%d/%m/%y')
        if date>stop_date:
            break
        fixture_date = dt.datetime.strptime(row[1], '%d/%m/%y').date()
        home_team = row[2]
        away_team = row[3]
        home_goals = int(row[4])
        away_goals = int(row[5])
        home_shots = int(row[11])
        away_shots = int(row[12])
        home_sht = int(row[13])
        away_sht = int(row[14])
        home_corners = int(row[17])
        away_corners = int(row[18])
        if row[6] == 'H' : winner = 2
        elif row[6] == 'A' : winner = -1
        elif row[6] == 'D' : winner = 1

        if (home_team == input_team or away_team == input_team):
            id_value = home_team + '_' + away_team

            if id_value not in fixtures:
                fixtures[id_value] = Teams.Fixture(name = id_value, fixture_date = fixture_date, winner=winner)

            fixtures[id_value].hometeam = Teams.Team(name = home_team)
            fixtures[id_value].awayteam = Teams.Team(name = away_team)

            fixtures[id_value].hometeam.update(home_goals, away_goals, home_shots, home_sht, home_corners)
            fixtures[id_value].awayteam.update(away_goals, home_goals, away_shots, away_sht, away_corners)

    return fixtures

def LoadAllData(stop_date):
    CONST = _Const()
    data = Teams.csvopen(CONST.INPUT_FILE)
    next(data)

    fixtures = {}

    for row in data:
        date=dt.datetime.strptime(row[1], '%d/%m/%y')
        if date>stop_date:
            break
        fixture_date = dt.datetime.strptime(row[1], '%d/%m/%y').date()
        home_team = row[2]
        away_team = row[3]
        home_goals = int(row[4])
        away_goals = int(row[5])
        home_shots = int(row[11])
        away_shots = int(row[12])
        home_sht = int(row[13])
        away_sht = int(row[14])
        home_corners = int(row[17])
        away_corners = int(row[18])
        if row[6] == 'H' : winner = 2
        elif row[6] == 'A' : winner = -1
        elif row[6] == 'D' : winner = 1

        id_value = home_team + '_' + away_team

        if id_value not in fixtures:
            fixtures[id_value] = Teams.Fixture(name = id_value, fixture_date = fixture_date, winner=winner)

        fixtures[id_value].hometeam = Teams.Team(name = home_team)
        fixtures[id_value].awayteam = Teams.Team(name = away_team)

        fixtures[id_value].hometeam.update(home_goals, away_goals, home_shots, home_sht, home_corners)
        fixtures[id_value].awayteam.update(away_goals, home_goals, away_shots, away_sht, away_corners)

    return fixtures

def GetAllTeamsWithTotals(stop_date):
    fixtures = LoadAllData(stop_date)

    teams = {}

    for fixture in fixtures:

        fix = fixtures.get(fixture)

        home_team = fix.hometeam.name
        away_team = fix.awayteam.name

        if home_team not in teams:
            teams[home_team] = Teams.Team(name = home_team)
        if away_team not in teams:
            teams[away_team] = Teams.Team(name = away_team)
        teams[home_team].update(fix.hometeam.goals_scored, fix.hometeam.goals_allowed, fix.hometeam.shots[0], fix.hometeam.shots_on_target[0], fix.hometeam.corners[0])
        teams[away_team].update(fix.awayteam.goals_scored, fix.awayteam.goals_allowed, fix.awayteam.shots[0], fix.awayteam.shots_on_target[0], fix.awayteam.corners[0])

    return teams

max_stars_repo_name: githubLewis/Football-data-analysis

max_issues_repo_name: githubLewis/Football-data-analysis

max_forks_repo_name: githubLewis/Football-data-analysis


text:
################################################################################################################################################
######################################################## Import required modules ###############################################################
################################################################################################################################################

import argparse
import pprint
import json
import logging
import os
import sys
import pandas as pd
import random
import time
import glob
import numpy as np
from collections import defaultdict

import torch
import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data
import torch.utils.data.distributed
from torch.utils.data import Dataset, DataLoader

from transformers import RobertaModel, RobertaConfig
from transformers import RobertaForSequenceClassification
from transformers import AdamW, get_linear_schedule_with_warmup

################################################################################################################################################
###################################################### Parse input arguments ###################################################################
################################################################################################################################################

def parse_args():

    parser = argparse.ArgumentParser()

    # CLI args

    parser.add_argument('--train_batch_size',
                        type=int,
                        default=64)

    parser.add_argument('--train_steps_per_epoch',
                        type=int,
                        default=64)

    parser.add_argument('--validation_batch_size',
                        type=int,
                        default=64)

    parser.add_argument('--validation_steps_per_epoch',
                        type=int,
                        default=64)

    parser.add_argument('--epochs',
                        type=int,
                        default=1)

    parser.add_argument('--freeze_bert_layer',
                        type=eval,
                        default=False)

    parser.add_argument('--learning_rate',
                        type=float,
                        default=0.01)

    parser.add_argument('--momentum',
                        type=float,
                        default=0.5)

    parser.add_argument('--seed',
                        type=int,
                        default=42)

    parser.add_argument('--log_interval',
                        type=int,
                        default=100)

    parser.add_argument('--backend',
                        type=str,
                        default=None)

    parser.add_argument('--max_seq_length',
                        type=int,
                        default=128)

    parser.add_argument('--run_validation',
                        type=eval,
                        default=False)

    # Container environment

    parser.add_argument('--hosts',
                        type=list,
                        default=json.loads(os.environ['SM_HOSTS']))

    parser.add_argument('--current_host',
                        type=str,
                        default=os.environ['SM_CURRENT_HOST'])

    parser.add_argument('--model_dir',
                        type=str,
                        default=os.environ['SM_MODEL_DIR'])

    parser.add_argument('--train_data',
                        type=str,
                        default=os.environ['SM_CHANNEL_TRAIN'])

    parser.add_argument('--validation_data',
                        type=str,
                        default=os.environ['SM_CHANNEL_VALIDATION'])

    parser.add_argument('--output_dir',
                        type=str,
                        default=os.environ['SM_OUTPUT_DIR'])

    parser.add_argument('--num_gpus',
                        type=int,
                        default=os.environ['SM_NUM_GPUS'])

    # Debugger args

    parser.add_argument("--save-frequency",
                        type=int,
                        default=10,
                        help="frequency with which to save steps")

    parser.add_argument("--smdebug_path",
                        type=str,
                        help="output directory to save data in",
                        default="/opt/ml/output/tensors",)

    parser.add_argument("--hook-type",
                        type=str,
                        choices=["saveall", "module-input-output", "weights-bias-gradients"],
                        default="saveall",)

    return parser.parse_args()

################################################################################################################################################
########################################################### Tools and variables ################################################################
################################################################################################################################################

# Model name according to the PyTorch documentation:
# https://github.com/aws/sagemaker-pytorch-inference-toolkit/blob/6936c08581e26ff3bac26824b1e4946ec68ffc85/src/sagemaker_pytorch_serving_container/torchserve.py#L45
MODEL_NAME = 'model.pth'
# Hugging face list of models: https://huggingface.co/models
PRE_TRAINED_MODEL_NAME = 'roberta-base'

def create_list_input_files(path):
    input_files = glob.glob('{}/*.tsv'.format(path))
    print(input_files)
    return input_files

def save_transformer_model(model, model_dir):
    path = '{}/transformer'.format(model_dir)
    os.makedirs(path, exist_ok=True)
    print('Saving Transformer model to {}'.format(path))
    model.save_pretrained(path)

def save_pytorch_model(model, model_dir):
    os.makedirs(model_dir, exist_ok=True)
    print('Saving PyTorch model to {}'.format(model_dir))
    save_path = os.path.join(model_dir, MODEL_NAME)
    torch.save(model.state_dict(), save_path)

################################################################################################################################################
########################################################### Configure the model ################################################################
################################################################################################################################################

def configure_model():
    classes = [-1, 0, 1]

    config = RobertaConfig.from_pretrained(
        PRE_TRAINED_MODEL_NAME,
        num_labels=len(classes),
        id2label={
            ### BEGIN SOLUTION - DO NOT delete this comment for grading purposes
            0: -1, # Replace all None
            1: 0, # Replace all None
            2: 1, # Replace all None
            ### END SOLUTION - DO NOT delete this comment for grading purposes
        },
        label2id={
            -1: 0,
            0: 1,
            1: 2,
        }
    )

    config.output_attentions=True

    return config

################################################################################################################################################
####################################################### PyTorch Dataset and DataLoader #########################################################
################################################################################################################################################

# PyTorch dataset retrieves the dataset’s features and labels one sample at a time
# Create a custom Dataset class for the reviews
class ReviewDataset(Dataset):

    def __init__(self, input_ids_list, label_id_list):
        self.input_ids_list = input_ids_list
        self.label_id_list = label_id_list

    def __len__(self):
        return len(self.input_ids_list)

    def __getitem__(self, item):
        # convert list of token_ids into an array of PyTorch LongTensors
        input_ids = json.loads(self.input_ids_list[item])
        label_id = self.label_id_list[item]

        input_ids_tensor = torch.LongTensor(input_ids)
        label_id_tensor = torch.tensor(label_id, dtype=torch.long)

        return input_ids_tensor, label_id_tensor

# PyTorch DataLoader helps to to organise the input training data in “minibatches” and reshuffle the data at every epoch
# It takes Dataset as an input
def create_data_loader(path, batch_size):
    print("Get data loader")

    df = pd.DataFrame(columns=['input_ids', 'label_id'])

    input_files = create_list_input_files(path)

    for file in input_files:
        df_temp = pd.read_csv(file,
                              sep='\t',
                              usecols=['input_ids', 'label_id'])
        df = df.append(df_temp)

    ds = ReviewDataset(
        input_ids_list=df.input_ids.to_numpy(),
        label_id_list=df.label_id.to_numpy(),
    )

    return DataLoader(
        ds,
        batch_size=batch_size,
        shuffle=True,
        drop_last=True,
    ), df

################################################################################################################################################
################################################################ Train model ###################################################################
################################################################################################################################################

def train_model(model,
                train_data_loader,
                df_train,
                val_data_loader,
                df_val,
                args):

    loss_function = nn.CrossEntropyLoss()
    optimizer = optim.Adam(params=model.parameters(), lr=args.learning_rate)

    if args.freeze_bert_layer:
        print('Freezing BERT base layers...')
        for name, param in model.named_parameters():
            if 'classifier' not in name:  # classifier layer
                param.requires_grad = False
        print('Set classifier layers to `param.requires_grad=False`.')

    train_correct = 0
    train_total = 0

    for epoch in range(args.epochs):
        print('EPOCH -- {}'.format(epoch))

        for i, (sent, label) in enumerate(train_data_loader):
            if i < args.train_steps_per_epoch:
                model.train()
                optimizer.zero_grad()
                sent = sent.squeeze(0)
                if torch.cuda.is_available():
                    sent = sent.cuda()
                    label = label.cuda()
                output = model(sent)[0]
                _, predicted = torch.max(output, 1)

                loss = loss_function(output, label)
                loss.backward()
                optimizer.step()

                if args.run_validation and i % args.validation_steps_per_epoch == 0:
                    print('RUNNING VALIDATION:')
                    correct = 0
                    total = 0
                    model.eval()

                    for sent, label in val_data_loader:
                        sent = sent.squeeze(0)
                        if torch.cuda.is_available():
                            sent = sent.cuda()
                            label = label.cuda()
                        output = model(sent)[0]
                        _, predicted = torch.max(output.data, 1)

                        total += label.size(0)
                        correct += (predicted.cpu() ==label.cpu()).sum()

                    accuracy = 100.00 * correct.numpy() / total
                    print('[epoch/step: {0}/{1}] val_loss: {2:.2f} - val_acc: {3:.2f}%'.format(epoch, i, loss.item(), accuracy))
            else:
                break

    print('TRAINING COMPLETED.')
    return model

################################################################################################################################################
#################################################################### Main ######################################################################
################################################################################################################################################

if __name__ == '__main__':

    # Parse args

    args = parse_args()
    print('Loaded arguments:')
    print(args)

    # Get environment variables

    env_var = os.environ
    print('Environment variables:')
    pprint.pprint(dict(env_var), width = 1)

    # Check if distributed training

    is_distributed = len(args.hosts) > 1 and args.backend is not None

    print("Distributed training - {}".format(is_distributed))
    use_cuda = args.num_gpus > 0
    print("Number of gpus available - {}".format(args.num_gpus))
    kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}

    device = torch.device('cuda' if use_cuda else 'cpu')

    # Initialize the distributed environment.

    if is_distributed:
        world_size = len(args.hosts)
        os.environ['WORLD_SIZE'] = str(world_size)
        host_rank = args.hosts.index(args.current_host)
        os.environ['RANK'] = str(host_rank)
        dist.init_process_group(backend=args.backend, rank=host_rank, world_size=world_size)
        print('Initialized the distributed environment: \'{}\' backend on {} nodes. '.format(
            args.backend, dist.get_world_size()) + 'Current host rank is {}. Number of gpus: {}'.format(
            dist.get_rank(), args.num_gpus))

    # Set the seed for generating random numbers

    torch.manual_seed(args.seed)
    if use_cuda:
        torch.cuda.manual_seed(args.seed)

    # Instantiate model

    config = None
    model = None

    successful_download = False
    retries = 0

    while (retries < 5 and not successful_download):
        try:
            # Configure model
            config = configure_model()
            model = RobertaForSequenceClassification.from_pretrained(
                'roberta-base',
                config=config
            )

            model.to(device)
            successful_download = True
            print('Sucessfully downloaded after {} retries.'.format(retries))

        except:
            retries = retries + 1
            random_sleep = random.randint(1, 30)
            print('Retry #{}.  Sleeping for {} seconds'.format(retries, random_sleep))
            time.sleep(random_sleep)

    if not model:
         print('Not properly initialized...')

    # Create data loaders

    train_data_loader, df_train = create_data_loader(args.train_data, args.train_batch_size)
    val_data_loader, df_val = create_data_loader(args.validation_data, args.validation_batch_size)

    print("Processes {}/{} ({:.0f}%) of train data".format(
        len(train_data_loader.sampler), len(train_data_loader.dataset),
        100. * len(train_data_loader.sampler) / len(train_data_loader.dataset)
    ))

    print("Processes {}/{} ({:.0f}%) of validation data".format(
        len(val_data_loader.sampler), len(val_data_loader.dataset),
        100. * len(val_data_loader.sampler) / len(val_data_loader.dataset)
    ))

    print('model_dir: {}'.format(args.model_dir))
    print('model summary: {}'.format(model))

    callbacks = []
    initial_epoch_number = 0

    # Start training

    model = train_model(
        model,
        train_data_loader,
        df_train,
        val_data_loader,
        df_val,
        args
    )

    save_transformer_model(model, args.model_dir)
    save_pytorch_model(model, args.model_dir)

    # Prepare for inference which will be used in deployment
    # You will need three files for it: inference.py, requirements.txt, config.json

    inference_path = os.path.join(args.model_dir, "code/")
    os.makedirs(inference_path, exist_ok=True)
    os.system("cp inference.py {}".format(inference_path))
    os.system("cp requirements.txt {}".format(inference_path))
    os.system("cp config.json {}".format(inference_path))

max_stars_repo_path: Course-2-Build, Train, and Deploy ML Pipelines using BERT/Week-2/src/train.py


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Aug 13 17:56:03 2019

@author: domenico
"""

# %% import packages
import sys
sys.path.append("./src/")
from utils import splitVec, tens, putZeroDiag, optim_torch, gen_test_net, soft_lu_bound, soft_l_bound,\
    degIO_from_mat, strIO_from_mat, tic, toc, rand_steps
import numpy as np
import torch
import matplotlib.pyplot as plt
from utils import tens, strIO_from_mat
from dirBin1_dynNets import  dirBin1_staNet, dirBin1_dynNet_SD

# %% Load WTN Data
#load yearly world Trade networks
ld_data = np.load("./data/world_trade_network/world_trade_net_T.npz",
                  allow_pickle=True)
wtn_T, all_y, nodes= ld_data["wtn_T"], ld_data["all_y"], ld_data["nodes"]
dist_T, scaling_infl = ld_data["dist_T"], ld_data["scaling_infl"]

# Set unit measure and  rescale for inflation
unit_measure = 1e6
Y_T = tens(wtn_T >0)
X_T = tens(dist_T).log().unsqueeze(2) # torch.tensor((dist_T - dist_T.mean())/dist_T.std()).unsqueeze(2)
N = Y_T.shape[0]
T = Y_T.shape[2]
SAVE_FOLD = './data/estimates_real_data/WTN'
t =T -2
Y_t = Y_T[:, :, t]
X_t=X_T[:, :, :, t]
A_t = Y_t>0
Y_tp1 = Y_T[:, :, t+1]
X_T_multi = X_T.repeat_interleave(2, dim=2)
X_T_multi[:, :, 1, :] += 50
# %% Test single snapshot estimates of  phi
model = dirBin1_dynNet_SD(avoid_ovflw_fun_flag=False)
model.distr = 'bernoulli'
beta_t = torch.zeros(N)
diag = []
wI, wO = 1 + torch.randn(N), torch.randn(N)
Y=Y_t
phi_0 = mdoel.start_phi_from_obs(Y_t, n_iter = 1500)
phi_0 = model.identify(phi_0)
model.exp_A(phi_0).sum(dim=1)
Y_t.sum(dim=1)
model.sta

model.zero_deg_par_fun(Y_t, phi_0)
model.identify(model.set_zero_deg_par(Y_t, phi_0))

phi = phi_0.clone()
N = phi.shape[0] // 2
zero_deg_par_i, zero_deg_par_o = model.zero_deg_par_fun(Y_t, phi)
phi[:N][phi[:N] == 0] = zero_deg_par_i
phi[N:][phi[N:] == 0] = zero_deg_par_o

phi_ss_t, diag_iter =  model.estimate_ss_t(Y_t, phi_0=phi_0, plot_flag=False, print_flag=True, max_opt_iter=2500,
                                             dist_par_un_t=None, print_every=500)
diag.append(diag_iter)
print(model.check_tot_exp(Y_t, phi_ss_t) )

# %% test quick ss estimate
model = dirBin1_dynNet_SD(avoid_ovflw_fun_flag=False, rescale_SD = False )
degIO = strIO_from_mat(Y_t)

phi_t, diag = model.estimate_ss_t_bin_from_degIO(degIO, min_opt_iter=500,
                                    max_opt_iter=2500, opt_n=1, lr=0.01,
                                    print_flag=True, plot_flag=False, print_every=500)

Y_t.sum()
model.exp_A(phi_t).sum()
print(model.check_tot_exp(Y_t, phi_t))

# %% Test joint ss estimates of phi and beta
model = dirBin1_dynNet_SD()
par_ss_t, diag = model.estimate_ss_t( Y_t, X_t=X_t, beta_t=None, phi_0=None, dist_par_un_t=None, like_type=2,
                                     est_beta=True, dim_beta=1,
                                        max_opt_iter=50, print_flag=True)

# %% test estimate of sequence of phi given static beta
model = dirBin1_staNet(avoid_ovflw_fun_flag=True)
dim_beta = N
n_reg = X_T.shape[2]
beta = torch.ones(dim_beta, n_reg) * 1
all_par_est_T, diag_T = model.ss_filt(Y_T, X_T=X_T_multi, beta=beta, phi_T_0=None,
                                         est_beta=False,
                                        max_opt_iter=15, opt_n=1, lr=0.01,
                                        print_flag=True, plot_flag=False, print_every=10)

plt.plot(diag_T)
# %% test estimate of beta constant given phi_T
model = dirBin1_staNet(avoid_ovflw_fun_flag=True)
all_par_est_T, diag_T = model.ss_filt(Y_T, X_T=None, beta=None, phi_T_0=None,
                                         est_beta=False,
                                        max_opt_iter=4, opt_n=1, lr=0.01,
                                        print_flag=True, plot_flag=False, print_every=10)

phi_ss_est_T = all_par_est_T[:2*N, :]
dim_beta = 1
model = dirBin1_dynNet_SD(avoid_ovflw_fun_flag=True  )
beta_t_est, diag_beta_t = model.estimate_beta_const_given_phi_T(Y_T, X_T_multi, phi_ss_est_T.clone().detach(),
                                                                  dim_beta=dim_beta, dist_par_un=0,
                                                                  max_opt_iter=10, print_flag=True, plot_flag=True)

# %% test estimate of beta and phi_T
model = dirBin1_dynNet_SD(avoid_ovflw_fun_flag=True  )
phi_T, dist_par_un, beta, diag = \
model.ss_filt_est_beta_const(Y_T, X_T=X_T_multi, beta=None, phi_T=None,
                                      est_const_beta=True, dim_beta=1,
                                      opt_large_steps=2, opt_n=1, max_opt_iter_phi=5, lr_phi=0.01,
                                      max_opt_iter_beta=4, lr_beta=0.01,
                                      print_flag_phi=False, print_flag_beta=True,
                                      print_every=1)

plt.close("all")
plt.plot(-np.array(diag))

# %% Define Parameters for Score Driven Dynamics
N_max_opt_iter_max = 10000
N_max_opt_iter_each_iter = 200
N_iter = N_max_opt_iter_max//N_max_opt_iter_each_iter
N_BA = N

B = torch.cat([torch.ones(N_BA) * 0.85, torch.ones(N_BA) * 0.85])
A = torch.cat([torch.ones(N_BA) * 0.1, torch.ones(N_BA) * 0.1])
wI, wO = 1 + torch.randn(N), torch.randn(N)
W = torch.cat((torch.ones(N) * wI, torch.ones(N) * wO)) * 0.001

# %% Test score
model = dirBin1_dynNet_SD(avoid_ovflw_fun_flag=True, rescale_SD=True)
s_1 = model.score_t(Y_t, W, backprop_score=True, like_type=2)
s_2 = model.score_t(Y_t, W, backprop_score=False, like_type=2)

print((s_1[0] - s_2[0]).abs().sum())
print((s_1[1] - s_2[1]).abs().sum())

# %% test starting values
strIO_from_mat(model.exp_A(model.start_phi_form_obs(Y_t))) - strIO_from_mat(Y_t)
model.check_tot_exp(Y_t, model.start_phi_form_obs(Y_t), one_dim_out=False)

# %% Tet score driven estimates
model = dirBin1_dynNet_SD(avoid_ovflw_fun_flag=True, rescale_SD=False)
W_est, B_est, A_est, diag = model.estimate_SD(Y_T, B0=B, A0=A, W0=W, max_opt_iter=20, lr=0.01,
                                                dim_dist_par_un=1, print_flag=True, plot_flag=False,
                                                                 est_dis_par_un=False)

# %% Test Score Driven Estimates with static regressors
dim_beta=N
n_reg=X_T_multi.shape[2]
n_beta_tv = 0
dim_dist_par_un = N

model = dirBin1_dynNet_SD(avoid_ovflw_fun_flag=True, rescale_SD = False )
W_est, B_est, A_est, dist_par_un_est, beta_const_est,  diag = model.estimate_SD_X0(Y_T, X_T=X_T_multi,
                                                                dim_beta=dim_beta,
                                                                n_beta_tv = n_beta_tv,
                                                                B0=torch.cat((B, torch.zeros(n_beta_tv))),
                                                                A0=torch.cat((A, torch.zeros(n_beta_tv))),
                                                                W0=torch.cat((W, torch.zeros(n_beta_tv))),
                                                                beta_const_0= torch.zeros(dim_beta, n_reg-n_beta_tv),
                                                                max_opt_iter=50,
                                                                lr=0.01,
                                                                print_flag=True, plot_flag=False)

# %% Test sd dgp with time varying regressors
model = dirBin1_dynNet_SD(avoid_ovflw_fun_flag=True, rescale_SD=False)
N = 30
T = 100
N_max_opt_iter_max = 10000
N_max_opt_iter_each_iter = 200
N_iter = N_max_opt_iter_max//N_max_opt_iter_each_iter
N_BA = N
n_reg = 2
n_beta_tv = 1
dim_beta = 1

X_T = torch.randn(N, N, n_reg, T)
A = torch.cat([torch.ones(N_BA) * 0.01, torch.ones(N_BA) * 0.01])
B = torch.cat([torch.ones(N_BA) * 0.9, torch.ones(N_BA) * 0.9])
um_sd_par, um_degs = model.dgp_phi_var_size(N, degMin=5, degMax=25)
beta_const = torch.ones(dim_beta, n_reg - n_beta_tv )

if n_beta_tv * dim_beta > 0:
    A = torch.cat((A, torch.ones(n_beta_tv * dim_beta) * 0.01))
    B = torch.cat((B, torch.ones(n_beta_tv * dim_beta) * 0.01))
    um_sd_par = torch.cat((um_sd_par, torch.ones(n_beta_tv * dim_beta) * 1/(B[- n_beta_tv * dim_beta])))

W = um_sd_par * (1-B)

model.n_reg_beta_tv = n_beta_tv
model.dim_beta = dim_beta
phi_T, beta_sd_T, Y_T_dgp = model.sd_dgp(W, B, A, p_T=None, beta_const=beta_const, X_T=X_T, N=N, T=T)

Y_T_dgp

W_est, B_est, A_est, dist_par_un_est, beta_const_est,  diag = model.estimate_SD_X0(Y_T_dgp, X_T=X_T,
                                                        dim_beta=dim_beta,
                                                        n_beta_tv=n_beta_tv,
                                                        B0=B,
                                                        A0=A,
                                                        W0=W,
                                                        beta_const_0=beta_const,
                                                        max_opt_iter=20,
                                                        lr=0.01,
                                                        print_flag=True, plot_flag=False)

# %% Test missp dgps
N = 50
T = 100
N_sample = 10

model = dirBin1_dynNet_SD(avoid_ovflw_fun_flag=True, rescale_SD=False)
degb = tens([10, N-10])
um_phi, um_degs = model.dgp_phi_var_size(N, degMin=degb[0], degMax=degb[1])

Y_T_S_dgp, phi_T_dgp, X_T_dgp, beta_T_dgp = model.sample_from_dgps(N, T, N_sample, um_phi=um_phi, dgp_type='step',
                                                                    X_T=None, n_reg=2, n_reg_beta_tv=1,
                                                                    dim_beta=1, degb=degb)
plt.close()
plt.plot(phi_T_dgp[3, :])
plt.plot(phi_T_dgp[N+2, :])

plt.close()
plt.plot(Y_T_S_dgp[:, :, :, 0].sum(dim=0).transpose(0, 1)[:, 4])
Y_T_S_dgp[:, :, :, 0].sum(dim=0)

# %%

# %%

#

max_stars_repo_name: domenicodigangi/dynwgraphs

max_issues_repo_name: domenicodigangi/dynwgraphs

max_forks_repo_name: domenicodigangi/dynwgraphs


text:
import pandas as pd
from scipy.stats import iqr
from sklearn.ensemble import IsolationForest
from sklearn.neighbors import LocalOutlierFactor
import sys

sys.path.append('../util')
from ..util.constants import logger

def outlier_detection_dataframe(df: pd.DataFrame, strategy: str='density') -> pd.Series:
    """Outlier detection of a given dataframe.

    Keyword arguments:
    df -- dataframe
    Output:
    outlier of given dataframe
    """
    logger.info(f'Start to detect outlier for the whole data set with strategy: {strategy}...')
    if strategy == 'high_dimension':
        out = outlier_detection_high_dimension(df)
    elif strategy == 'density':
        out = outlier_detection_density(df)
    else:
        logger.info('Unrecognized strategy. Use density-based strategy instead')
        out = outlier_detection_density(df)
    logger.info(f'...End with outlier detection, {out.sum()} outliers found')
    return out

def outlier_detection_feature(col: pd.Series) -> pd.Series:
    """Outlier detection of a given column.

    Keyword arguments:
    col -- column
    Output:
    outlier of given feature
    """
    logger.info(f'Start to detect outlier for given feature {col.name} with 3 IQR strategy...')
    v_iqr = iqr(col)
    v_mean = col.mean()
    ceiling = v_mean + 3*v_iqr
    floor = v_mean - 3*v_iqr
    out = col.map(lambda x: x>ceiling or x<floor)
    logger.info(f'...End with outlier detection, {out.sum()} outliers found')
    return out

def outlier_detection_high_dimension(df: pd.DataFrame) -> pd.Series:
    """High dimension outlier detection of a given dataframe with a random forest.

    Keyword arguments:
    df -- dataframe
    Output:
    outlier of given dataframe
        """
    clf = IsolationForest(n_estimators=20, warm_start=True)
    out = pd.Series(clf.fit_predict(df), index = df.index)
    out = out.map(lambda x: x == -1)
    return out

def outlier_detection_density(df: pd.DataFrame) -> pd.Series:
    """Density-based outlier detection of a given dataframe.

    Keyword arguments:
    df -- dataframe
    Output:
    outlier of given dataframe
    """
    clf = LocalOutlierFactor(n_neighbors=20, contamination=0.1)
    out = pd.Series(clf.fit_predict(df), index = df.index)
    out = out.map(lambda x: x == -1)
    return out

reason: from scipy


text:
'''
This script loads the EgoGesture dataset through different classes
'''
import os
import sys
import pickle
import numpy as np
import pandas as pd
import random
import torch
import pdb
from torch.utils.data import Dataset, DataLoader,RandomSampler
import torchvision.transforms as transforms
from torchvision.utils import save_image
from PIL import Image
import matplotlib.pyplot as plt
from tqdm import tqdm, trange
import random
import skimage.util as ski_util
from sklearn.utils import shuffle
from temporal_transforms import *
from spatial_transforms import *

# self-defined modules
import opts

class make_dataset_instance(Dataset):
    def __init__(self, data):
        self.data = data

    def __getitem__(self, idx):
        img = self.data[idx][0]
        label = self.data[idx][1]
        return img, torch.Tensor([label])[0].long()

    def __len__(self):
        return len(self.data)

def load_video_class(csv_path, mode, class_id, reverse=False):
    # mode: train, val, test
    csv_file = os.path.join(csv_path, '{}.pkl'.format(mode))
    annot_df = pd.read_pickle(csv_file)
    # annot_df = DownSample(raw_annot_df, 40)
    rgb_samples = []
    depth_samples = []
    labels = []
    # get task index in dataframe
    task_ind = []
    for i in range(annot_df.shape[0]):
        if annot_df['label'][i] in class_id:
            task_ind.append(i)
    annot_df_task = annot_df.iloc[task_ind]
    for frame_i in range(annot_df_task.shape[0]):
        rgb_list = annot_df_task['rgb'].iloc[frame_i] # convert string in dataframe to list
        depth_list = annot_df_task['depth'].iloc[frame_i] # convert string in dataframe to list
        rgb_samples.append(rgb_list)
        depth_samples.append(depth_list)
        labels.append(annot_df_task['label'].iloc[frame_i])
        # data augmentation by reversing the sequence of the video
        if reverse:
            rgb_samples.append(rgb_list[::-1])
            depth_samples.append(depth_list[::-1])
            labels.append(annot_df_task['label'].iloc[frame_i])
    return rgb_samples, depth_samples, labels

class dataset_video_class(Dataset):
    def __init__(self, root_path, mode, class_id, n_frames_per_clip, img_size,
                reverse=False, transform=None, temporal_transform=None, modality=None):
        self.root_path = root_path
        self.rgb_samples, self.depth_samples, self.labels = load_video_class(root_path, mode,
                                                                            class_id, reverse)
        self.w = img_size[0]
        self.h = img_size[1]
        self.sample_num = len(self.rgb_samples)
        self.transform = transform
        self.n_frames_per_clip = n_frames_per_clip
        self.temporal_transform = temporal_transform
        self.modality = modality
        # print('{} {} samples have been loaded'.format(class_id, self.sample_num))

    def __getitem__(self, idx):
        rgb_name = self.rgb_samples[idx]
        depth_name = self.depth_samples[idx]
        label = self.labels[idx]
        indices = [i for i in range(len(rgb_name))]
        if self.temporal_transform is None:
        # repeat frame sequence less than the n_frames_per_clip
            if len(rgb_name) < self.n_frames_per_clip:
                RepeatTimes = int(self.n_frames_per_clip/len(rgb_name))+1
                rgb_name = rgb_name*RepeatTimes
                depth_name = depth_name*RepeatTimes
            rgb = torch.zeros([3, len(rgb_name), self.h, self.w]).float()
            depth = torch.zeros([1, len(rgb_name), self.h, self.w]).float()
            for frame_name_i in range(len(rgb_name)):
                rgb_cache = Image.open(rgb_name[frame_name_i]).convert("RGB").resize((171, 128), Image.BILINEAR)  # C * W * H
                # rgb_cache = Image.open(rgb_name[frame_name_i]).convert("RGB")
                rgb_cache = self.transform(rgb_cache)
                rgb[:, frame_name_i, :, :] = rgb_cache
                depth_cache = Image.open(depth_name[frame_name_i]).convert("L").resize((171, 128), Image.BILINEAR)  # C * W * H
                # depth_cache = Image.open(depth_name[frame_name_i]).convert("L")
                depth_cache = self.transform(depth_cache)
                depth[:, frame_name_i, :, :] = depth_cache
        else:
            rgb = torch.zeros([3, self.n_frames_per_clip, self.h, self.w]).float()
            depth = torch.zeros([1, self.n_frames_per_clip, self.h, self.w]).float()
            selected_indice = self.temporal_transform(indices)
            for i, frame_name_i in enumerate(selected_indice):
                rgb_cache = Image.open(rgb_name[frame_name_i]).convert("RGB").resize((171, 128), Image.BILINEAR)  # C * W * H
                # rgb_cache = Image.open(rgb_name[frame_name_i]).convert("RGB")
                rgb_cache = self.transform(rgb_cache)
                rgb[:, i, :, :] = rgb_cache
                depth_cache = Image.open(depth_name[frame_name_i]).convert("L").resize((171, 128), Image.BILINEAR)  # C * W * H
                # depth_cache = Image.open(depth_name[frame_name_i]).convert("L")
                depth_cache = self.transform(depth_cache)
                depth[:, i, :, :] = depth_cache
        if self.modality == 'Depth':
            return depth, (torch.tensor(label)-1).long()
        elif self.modality == 'RGB':
            return rgb, (torch.tensor(label)-1).long()
        elif self.modality == 'RGB-D':
            return torch.cat([rgb, depth], 0), (torch.tensor(label)-1).long()
        else:
            return rgb, depth, (torch.tensor(label)-1).long()
        # return rgb, mask, (torch.tensor(label)-1).long()

    def __len__(self):
        return int(self.sample_num)

# root_path = './'
# class_id = [i for i in range(0,41)]
# args = opts.parse_opts()
# trans_train = Compose([
#             Scale([112,112]),
#             SpatialElasticDisplacement(),
#             ToTensor(1)
#             ])
# temporal_transform_ = Compose([
#         TemporalCenterCrop(100)
#         ])

# dataset_train = dataset_video_class(root_path, 'train',
#                                     n_frames_per_clip=100,
#                                     class_id = class_id,
#                                     img_size=(args.w, args.h),
#                                     reverse=False, transform=trans_train,
#                                     temporal_transform = temporal_transform_)
# rgb, depth, label = dataset_train.__getitem__(0)

# # dataloader_train = DataLoader(dataset_train, batch_size=32,
# #                                 shuffle=True,
# #                                 num_workers=args.num_workers, pin_memory=True)
# # trainiter = iter(dataloader_train)
# # rgbs, masks, labels = trainiter.next()

# pdb.set_trace()

max_stars_repo_name: villawang/CatNet

max_issues_repo_name: villawang/CatNet

max_forks_repo_name: villawang/CatNet


text:
import re
import os
import decimal
import argparse
import numpy as np
from sympy import Symbol
from scipy import interpolate
from sympy.stats import sample, Uniform, Exponential

def main():

    """ Add interactivity for LHE analysis """

    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser.add_argument('-m', action='store', dest='mAp', type=float)
    parser.add_argument('-e', action='store', dest='eps', type=float)
    parser.add_argument('-i', action='store', dest='infile',)
    args = parser.parse_args()
    mAp, eps = args.mAp, args.eps
    rsample_factor = 10

    # Will store information here
    nEvents = 10_000
    resamples_allowed = 500
    resamples = np.zeros( nEvents )
    decay_zs = [] # Only those within bounds on first sample
    decay_zs_resampled = [] # Within bounds after < resamples_allowed
    positions = np.zeros( (nEvents, 3) )
    report = {
            'Resampled events (normalized)': 0,
            'Average resamples': 0,
            'Max resamples': 0
            }

    ##################################################
    # Analysis (similar to writeBremDecay, but dev-y as opposed to pro-y)
    ##################################################

    # Creation XYZ
    Sym = Symbol('q')
    x_rv = Uniform(Sym, -10 , 10 )
    y_rv = Uniform(Sym, -40 , 40 )
    #z_rv = Uniform(Sym, -0.175, 0.175)
    Xs = sample( x_rv, numsamples=nEvents, seed=np.random.seed( 2 ) )
    Ys = sample( y_rv, numsamples=nEvents, seed=np.random.seed( 2 ) )
    #Zs = sample( z_rv, numsamples=nEvents, seed=np.random.seed( 2 ) )

    # Detector limits
    zmin = 300
    zmax = 4000 - 300

    # Decay time
    c_speed = 299_792_458_000 # mm/s
    t = Symbol('t')
    decay_width = gamma_ap_tot(mAp, eps)
    tau = 6.582e-25 / decay_width # hbar = 6.582e-25 GeV*s
    decay_rv = Exponential(t, 1/tau)
    decay_t = sample(
                        decay_rv,
                        numsamples=nEvents*rsample_factor,
                        seed=np.random.seed( 2 )
                        )

    # Open file for analysis
    with open(args.infile, 'r') as ogfile:

        # Skip from header/init
        for line in ogfile:
            if line == '</init>\n':
                break

        ##################################################
        #  events
        ##################################################
        event_num = 0
        event_line = 0
        current_line = 0
        for line in ogfile: # Picks up where last loop leaves off
            current_line += 1

            # Scale relevant lines
            if line == '<event>\n':
                event_num += 1
                event_line = 0
                if event_num % 1000 == 0:
                    print( 'Reformatting event: {}'.format(event_num) )

            else: event_line += 1

            if event_line == 6: # Take note of Ap info for projection
                line = rescaleLine(line, tokens=range(6,11))
                px,py,pz,en = [
                                float(v) for v in numsInLine(line)[6:10]
                                ]
                Ap_3mom = np.array((px,py,pz))

            """ Might care about these later
            elif event_line < 9: # decay electrons

                # Null parents
                line = replaceNums(line, [2,3], [-1,-1])
            """

            # Skip mgrwt. add appropriate vertex, and end event
            if event_line == 16 :

                # Add verticies
                #x,y,z,t = next(Xs), next(Ys), next(Zs), next(decay_t)*(en/mAp)
                x,y,z = next(Xs), next(Ys), 0
                c_vertex = np.array( (x,y,z) )

                t = -1 # While prep
                d_vertex = c_vertex + Ap_3mom*c_speed / mAp * t
                resamps = -1
                while not ( zmin <= d_vertex[2] <= zmax ):
                    resamps += 1
                    """
                    if resamps == 50:
                        print(f'50 samples on event {event_num}: {Ap_3mom}')
                    """
                    if resamps == resamples_allowed:
                        #print(f'500 samples on event {event_num}; p = {Ap_3mom}')
                        break
                    try:
                        t = next(decay_t)*(en/mAp)
                        d_vertex = c_vertex + Ap_3mom*c_speed / mAp * t
                    except: # ran out of decay_t
                        # Fill report
                        report['Resampled events (normalized)'] \
                                    = np.count_nonzero(resamples) / event_num
                        report['Average resamples'] \
                                    = np.average(resamples)*nEvents / event_num
                        report['Max resamples'] = max(resamples)
                        report['Events before quitting'] = event_num

                        # Print report
                        print(resamples[:event_num])
                        for k,v in report.items():
                            print( '{}: {}'.format(k,v) )

                        quit()

                if resamps == 0:
                    decay_zs.append(d_vertex[2])
                elif resamps < resamples_allowed:
                    decay_zs_resampled.append(d_vertex[2])

                resamples[event_num-1] = resamps

    # Fill report
    report['Resampled events (normalized)'] \
                                        = np.count_nonzero(resamples) / nEvents
    report['Average resamples'] = np.average(resamples)
    report['Max resamples'] = max(resamples)

    # Print report
    for k,v in report.items():
        print( '{}: {}'.format(k,v) )

    # Plot resamples
    """
    import matplotlib.pyplot as plt
    plt.hist(resamples)
    plt.title(r"$m_{A'} =$" + str(mAp) + r" GeV, $\epsilon$ = " + str(eps))

    # Plot decay_zs, decay_zs, and combo
    #plt.figure()
    fig, (ax1, ax2) = plt.subplots(nrows=2)
    ns, bins, p  = ax1.hist(
                [decay_zs,decay_zs_resampled],
                histtype='step',
                log=True,
                bins=50,
                range = (zmin,zmax),
                density=True,
                label=['decay_zs','decay_zs_resampled']
                )

    plt.bar( bins[:-1 ] , ns[1] / ns[0], label='nolog', width=68)
    plt.bar( bins[:-1 ] , np.log(ns[1]/ns[0]) / np.log(10),
            label='log', width=68 )

    plt.legend(loc='upper center')
    plt.ylabel(r'A.U.')
    plt.xlabel(r'Decay Z [mm]')

    plt.hist(
                decay_zs_resampled,
                histtype='step',
                log=True,
                bins=50,
                range = (zmin,zmax),
                density=True,
                label='decay_zs_resampled'
                )

    plt.legend(loc='upper center')
    plt.ylabel(r'A.U.')
    plt.xlabel(r'Decay Z [mm]')

    plt.hist(
                decay_zs + decay_zs_resampled,
                histtype='step',
                log=True,
                bins=50,
                range = (zmin,zmax),
                density=True,
                label='Combination'
                )

    plt.legend(loc='upper right')
    plt.ylabel(r'A.U.')
    plt.xlabel(r'Decay Z [mm]')

    plt.title(r"$m_{A'} =$" + str(mAp) + r" GeV, $\epsilon$ = " + str(eps))

    plt.show()
    """

def writeBremDecay(lhe, mAp, eps, seed, outdir=None, nevents=10_000):

    """ Break A'->ee LHEs into brem and decay files and reformat/rescale """

    # Write to input directory if outdir not provided
    if outdir is None: outdir = '/'.join( lhe.split('/')[:-1] )

    # Create outdir if needed
    if not os.path.exists(outdir): os.makedirs(outdir)

    # Outfile names
    genname = lhe.split('/')[-1].split('.lhe')[0] \
            + '_run{}'.format(seed) \
            + '_eps{}'.format(eps)
    bremfile = '{}/{}_brem.lhe'.format(outdir,genname)
    decayfile = '{}/{}_decay.lhe'.format(outdir,genname)
    print( 'Reformatting:\n{}\nInto:\n{}\n{}'.format(lhe,bremfile,decayfile) )

    # Creation XYZ
    Sym = Symbol('q')
    x_rv = Uniform(Sym, -10 , 10 )
    y_rv = Uniform(Sym, -40 , 40 )
    #z_rv = Uniform(Sym, -0.175, 0.175)
    Xs = sample( x_rv, numsamples=nevents, seed=np.random.seed( seed ) )
    Ys = sample( y_rv, numsamples=nevents, seed=np.random.seed( seed ) )
    #Zs = sample( z_rv, numsamples=nevents, seed=np.random.seed( seed ) )

    # Detector limits
    zmin = 300
    zmax = 4000 - 300

    # Decay time
    c_speed = 299_792_458_000 # mm/s
    t = Symbol('t')
    decay_width = gamma_ap_tot(mAp, eps)
    tau = 6.582e-25 / decay_width # hbar = 6.582e-25 GeV*s
    decay_rv = Exponential(t, 1/tau)
    decay_t = sample(
                        decay_rv,
                        numsamples=nevents,
                        seed=np.random.seed( seed )
                        )

    # Open original and output files
    with open(lhe, 'r') as ogfile, \
            open(bremfile, 'w') as bremf,  \
            open(decayfile, 'w') as decayf:

        ##################################################
        # Edit header (techincaly until </init>
        # Many conditions shouldn't check in events sec.
        ##################################################
        scaling_mass = False
        for line in ogfile:

            # ebeams
            if re.search(r'ebeam',line):
                line = rescaleLine(line)

            # Masses
            if line[:10] == 'BLOCK MASS':
                scaling_mass = True # Indicate following lines should be scaled
                continue
            if line[0] == '#':
                scaling_mass = False
            if scaling_mass:
                line = rescaleLine(line, tokens=[1])

            # Decay Width
            if re.match(r'DECAY +622', line):
                line = replaceNums(line, [1], [decay_width])

            # Break from header/init
            if line == '</init>\n':
                bremf.write(line)
                decayf.write(line)
                break

            bremf.write(line)
            decayf.write(line)

        ##################################################
        # Edit events
        ##################################################
        event_num = 0
        event_line = 0
        current_line = 0
        for line in ogfile: # Picks up where last loop leaves off
            current_line += 1

            # Scale relevant lines
            if line == '<event>\n':
                event_num += 1
                event_line = 0
                decayf.write(line)
                if event_num % 1000 == 0:
                    print( 'Reformatting event: {}'.format(event_num) )

            else: event_line += 1
            if 1 < event_line < 9:

                line = rescaleLine(line, tokens=range(6,11))

            # Event info line
            if event_line ==1:

                # Correct particle number
                line = replaceNums(line, [0], [5])
                bremf.write(line)

                line = replaceNums(line, [0], [2])
                decayf.write(line)

            elif event_line < 7: # If first 5 write to bremfile
                bremf.write(line)

                if event_line == 6: # Take note of Ap info for projection
                    px,py,pz,en = [
                                    float(v) for v in numsInLine(line)[6:10]
                                    ]
                    Ap_3mom = np.array((px,py,pz))

            elif event_line < 9: # decay electrons

                # Null parents
                line = replaceNums(line, [2,3], [-1,-1])
                decayf.write(line)

            # Skip mgrwt. add appropriate vertex, and end event
            elif event_line == 16 :

                # Add verticies
                #x,y,z,t = next(Xs), next(Ys), next(Zs), next(decay_t)*(en/mAp)
                x,y,z,t = next(Xs), next(Ys), 0, next(decay_t)*(en/mAp)
                bremf.write( '#vertex {} {} {}\n'.format(x,y,z) )

                c_vertex = np.array( (x,y,z) )
                d_vertex = c_vertex + Ap_3mom*c_speed / mAp * t
                decayf.write( '#vertex {} {} {} {}\n'.format(*d_vertex,t) )

                # End event
                bremf.write(line)
                decayf.write(line)

            # End both
            elif line == '</LesHouchesEvents>\n':
                bremf.write(line)
                decayf.write(line)

    return bremfile, decayfile

# <From Nikita Blinov> ##################################################
# Hadronic R ratio used to compute hadronic width of the A'
Rvals=np.loadtxt("reformat/r_fixed.dat")
#Rvals=np.loadtxt("r_fixed.dat")
Rvals_interp = interpolate.interp1d(Rvals[:,0],Rvals[:,1],kind='linear');

def Rfunc(s):
    if np.sqrt(s) >= 0.36:
        return Rvals_interp(np.sqrt(s))
    else:
        return 0.

def gamma_ap_to_ll(mAp,ml,eps):
    if mAp < 2.*ml:
        return 0.

    aEM=1/137.035999679

    beta=1. - (2*ml/mAp)**2

    return (1./3.)*(aEM*eps**2)*mAp*np.sqrt(beta)*(1 + (2*ml**2)/mAp**2)

# Total decay rate of Ap into electrons and muons and hadrons
# Valid for mAp > 2*me
# Width is returned in GeV
def gamma_ap_tot(mAp, epsilon):
    me = 0.51099895/1000.
    mmu = 105.6583745/1000.
    return gamma_ap_to_ll(mAp,me,epsilon) \
            + gamma_ap_to_ll(mAp,mmu,epsilon)*(1. + Rfunc(mAp**2))

# </From Nikita Blinov> ##################################################

##################################################
# Line editing
##################################################

def numsInLine(line_w_nums):

    """ Find numbers in line """

    nums =  re.findall(r' [\d,\.,e,\-,\+]+', line_w_nums) # Seems close enough

    return [ num[1:] for num in nums ]

def rescaleLine(line_w_nums, scale=decimal.Decimal('0.1'), tokens=[0]):

    """ Replace numbers at given tokens (indicies) with scaled versions """

    numbers = numsInLine(line_w_nums)
    numbers = [ numbers[i] for i in tokens ] # Get numbers at desired indicies
    scaled_line = line_w_nums

    for number in numbers:
        scaled_line = re.sub(re.sub(r'\+', r'\\+', number), # looks like - equiv not needed
                            str(decimal.Decimal(number)*scale), scaled_line,
                            count=1)

    return scaled_line

def replaceNums(line_w_nums, tokens, new_vals):

    """ Replace numbers at given tokens (indicies) with specific new_vals """

    numbers = numsInLine(line_w_nums)
    numbers = [ numbers[i] for i in tokens ]# Numbers we care about
    new_line = line_w_nums

    for number, new_val in zip(numbers,new_vals):
        new_line = re.sub(number, str(new_val), new_line, count=1)

    return new_line

if __name__=='__main__': main()

reason: import numpy,from scipy,from sympy


text:
#!/usr/bin/env python3
# -*- coding = utf-8 -*-
import os

import cv2
import numpy as np

def resize(img, target_shape = (48, 48)):
   '''
   Resizes an inputted 2-dimensional image, target shape is default for fer2013.
   '''
   return cv2.resize(img, dsize = target_shape, interpolation = cv2.INTER_CUBIC)

def grayscale(img):
   '''
   Converts an image to binary (grayscale).
   '''
   return np.expand_dims(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), axis = 0)

max_stars_repo_name: BTTHuyen/engagement-detection


text:
import numpy as np
import matplotlib.pyplot as plt
import os
import cv2
import random
import pickle
import tensorflow as tf
import time
from tensorflow.keras.models import Sequential
from tensorflow.keras.callbacks import TensorBoard
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D

#Load saved python serialized objects...
def loadData(trainX, trainY):
    trainX = pickle.load(open("X.pickle", "rb"))
    trainY = pickle.load(open("Y.pickle", "rb"))
    return (trainX, trainY)

imgSize = 128
denseLayers = [0, 1, 2]
layerSizes = [32, 64, 128]
conv2DLayers = [1, 2, 3]

trainX = []
trainY = []
(trainX, trainY) = loadData(trainX, trainY)

#Normalize the data (images again...)
trainX = trainX / 255.0

for denseLayer in denseLayers:
    for layerSize in layerSizes:
        for conv2DLayer in conv2DLayers:
            modelName = "{}-conv-{}-nodes-{}-dense-{}".format(conv2DLayer, layerSize, denseLayer, int(time.time()))
            tb = TensorBoard(log_dir="logs/{}".format(modelName))
            print("\nNow fitting: ", modelName, "\n")

            model = Sequential()

            model.add(Conv2D(layerSize, kernel_size=(3,3), input_shape = trainX.shape[1:]))
            model.add(Activation("relu"))
            model.add(MaxPooling2D(pool_size=(2,2)))

            for i in range(conv2DLayer-1):
                model.add(Conv2D(layerSize, kernel_size=(3,3)))
                model.add(Activation("relu"))
                model.add(MaxPooling2D(pool_size=(2,2)))

            model.add(Flatten(input_shape=(imgSize, imgSize)))

            for i in range(denseLayer):
                model.add(Dense(layerSize))
                model.add(Activation("relu"))

            model.add(Dense(1))
            #Sigmoid function is because it exists between (0 to 1)
            model.add(Activation("sigmoid"))

            model.compile(loss="binary_crossentropy",
                optimizer="adam",
                metrics=["accuracy"])

            model.fit(trainX, trainY, batch_size=32, epochs=20, validation_split=0.3, callbacks=[tb])


text:
from colorama import Fore, Back, Style
import numpy
import random
import os
from config import *
from object import *
from grid import Grid
temp = [list("(^^^^^)")]
paddle_short = [list("(^^^)")]
paddle_long = [list("(^^^^^^^)")]
paddle = paddle_long
class item:
    def __init__(self):
        self._x = 0
        self._y = 0
        self._height = 0
        self._width = 0
        self._velx =  0
        self._vely = 0
    def getX(self):
        return self._x

    def getY(self):
        return self._y

    def getHeight(self):
        return self._height

    def getWidth(self):
        return self._width
####################################################################

class Paddle(item):
    def __init__(self):
        super().__init__()
        self._height = len(paddle)
        self._width = len(paddle[0])
        self._short = False
        self._long = False
        self._time = 0
        self._score = 0
        self._lives = 3

    def placePaddle(self,grid,x,y):
        self.clear(grid)
        self._x = x
        self._y = y - len(paddle[0])//2 +1
        for i in range(self._y,self._y+len(paddle[0])):
            grid[self._x][i] = Back.BLACK+Fore.GREEN + paddle[0][i - self._y] + Fore.RESET
    def printPaddle(self,grid):
        for i in range(self._y,self._y+len(paddle[0])):
            grid[self._x][i] = Back.BLACK+Fore.GREEN + paddle[0][i - self._y] + Fore.RESET
    def clear(self,grid):
        for i in range(self._y,self._y+len(paddle[0])):
                grid[self._x][i] = Back.BLACK+ Fore.BLACK + ' '
    def dec_pos(self,grid):
        if(self._y> 2 ):
            self.clear(grid)
            self._y -= 2
    def inc_pos(self,grid):
        if(self._y < columns-len(paddle[0])-1) :
            self.clear(grid)
            self._y += 2
    def getLives(self):
        return self._lives
    def getScore(self):
        return self._score
    def changTime(self,val):
        self._time += val