text:
import numpy as np
from pyquil.gates import RZ, RX, I, CZ, ISWAP, CPHASE
from pyquil.noise_gates import _get_qvm_noise_supported_gates, THETA

def test_get_qvm_noise_supported_gates_from_compiler_isa(compiler_isa):
    gates = _get_qvm_noise_supported_gates(compiler_isa)
    for q in [0, 1, 2]:
        for g in [
            I(q),
            RX(np.pi / 2, q),
            RX(-np.pi / 2, q),
            RX(np.pi, q),
            RX(-np.pi, q),
            RZ(THETA, q),
        ]:
            assert g in gates

    assert CZ(0, 1) in gates
    assert CZ(1, 0) in gates
    assert ISWAP(1, 2) in gates
    assert ISWAP(2, 1) in gates
    assert CPHASE(THETA, 2, 0) in gates
    assert CPHASE(THETA, 0, 2) in gates

ASPEN_8_QUBITS_NO_RX = {8, 9, 10, 18, 19, 28, 29, 31}
ASPEN_8_QUBITS_NO_RZ = {8, 9, 10, 18, 19, 28, 29, 31}
ASPEN_8_EDGES_NO_CZ = {(0, 1), (10, 11), (1, 2), (21, 22), (17, 10), (12, 25)}

def test_get_qvm_noise_supported_gates_from_aspen8_isa(qcs_aspen8_quantum_processor, noise_model_dict):
    gates = _get_qvm_noise_supported_gates(qcs_aspen8_quantum_processor.to_compiler_isa())

    for q in range(len(qcs_aspen8_quantum_processor._isa.architecture.nodes)):
        if q not in ASPEN_8_QUBITS_NO_RX:
            for g in [
                RX(np.pi / 2, q),
                RX(-np.pi / 2, q),
                RX(np.pi, q),
                RX(-np.pi, q),
            ]:
                assert g in gates
        if q not in ASPEN_8_QUBITS_NO_RZ:
            assert RZ(THETA, q) in gates

    for edge in qcs_aspen8_quantum_processor._isa.architecture.edges:
        if (
            edge.node_ids[0],
            edge.node_ids[1],
        ) in ASPEN_8_EDGES_NO_CZ:
            continue
        assert CZ(edge.node_ids[0], edge.node_ids[1]) in gates
        assert CZ(edge.node_ids[1], edge.node_ids[0]) in gates

max_stars_repo_name: stjordanis/pyquil

max_issues_repo_name: stjordanis/pyquil

max_forks_repo_name: stjordanis/pyquil


text:
import math
import os

import numpy as np
from scipy import spatial

import torch
from base_controller import BaseController
from evaluation_set import EvaluationSet
from lc_candidate import LcCandidate
from model import Model
from lc_handler import LcHandler
from submap_handler import SubmapHandler

class MapBuildingController(BaseController):
    def __init__(self, export_map_folder="", bw=100, state_dict='./net_params_new_1.pkl', desc_size=128):
        super().__init__(bw, state_dict, desc_size)
        self.descriptors = None
        self.submaps = {}
        self.export_map_folder = export_map_folder

        self.alignment_engine = SubmapHandler()
        self.lc_engine = LcHandler()

    def add_submap(self, submap):
        id = submap.id
        self.submaps[id] = submap

    def clear_clouds(self):
        self.submaps = {}

    def get_submaps(self):
        return list(self.submaps.values())

    # --- SUBMAP CONSTRAINTS -------------------------------------------------

    def compute_submap_constraints(self, submaps):
        n_submaps = len(submaps)
        if n_submaps == 0:
            return None
        print(f"Computing constraints for {n_submaps} submaps.")
        return self.alignment_engine.compute_constraints(submaps)

    def publish_all_submaps(self, submaps):
        self.alignment_engine.publish_submaps(submaps)

    # --- SUBMAP DESCRIPTORS --------------------------------------------------

    def build_descriptor_map(self):
        print("Building feature...")
        eval_set = EvaluationSet(submaps, self.bw)

        '''
        self.descriptors = self.describe_all_point_clouds(self.submaps)
        if self.export_map_folder is not "":
            self.export_descriptors_to_folder(
                self.export_map_folder, descriptors)
        '''

    def find_loop_closures(self):
        if self.descriptors is None:
            print("ERROR: descriptor map is empty.")
            return False
        return self.lc_engine.find_loop_closures(self.descriptors)

    def describe_all_point_clouds(self, submaps):
        eval_set = EvaluationSet(submaps, self.bw)
        loader = torch.utils.data.DataLoader(
            eval_set, batch_size=1, shuffle=False, num_workers=1, pin_memory=True, drop_last=False)

        n_clouds = len(clouds)
        embeddings = np.empty(1)
        for batch_idx, data in enumerate(loader):
            data = data.cuda().float()
            embedded, _, _ = self.net(data, data, data)
            embeddings = np.append(
                embeddings, embedded.cpu().data.numpy().reshape([1, -1]))

        descriptors = embeddings[1:].reshape([n_clouds, self.desc_size])
        return descriptors

    def export_descriptors_to_folder(self, map_folder, descriptors):
        files = os.listdir(map_folder)
        next_index = len(files) + 1
        descriptor_filename = f'{map_folder}/descriptors{next_index}.csv'
        timestamp_filename = f'{map_folder}/timestamps{next_index}.csv'
        ts_arr = np.array(self.timestamps)

        np.savetxt(descriptor_filename, descriptors, delimiter=',')
        np.savetxt(timestamp_filename, ts_arr, delimiter=',')

if __name__ == "__main__":
    ctrl = MapBuildingController()
    ctrl.handle_point_cloud(None)

ext: py

reason: import numpy,from scipy


text:
import math

import numpy as np
import sklearn
from preprocesspack import utils
#from utils import discretizeEF, discretizeEW, entropy

class Attribute:
    def __init__(self, vector, name=""):
        """
            Basic constructor of the Attribute class
        :param vector: Numeric vector containing the information of the attribute
        :param name: Character indicating the name given to the attribute
        """
        if (isinstance(vector, list) or isinstance(vector, np.ndarray)):
            if (len(vector) >= 1):

                self.vector = vector
                self.name = name
                self.size = len(vector)
                self.type = type(vector[0])
                if (type(vector[0])== str):
                    self.vector=self.getCatAsInt()

            else:
                raise ValueError("The vector parameter must have at least one element")
        else:
            raise TypeError("The vector parameter must be a list or a numpy array")

    def isCategorical(self):
        if(type(self.vector[0]) is tuple):
            return True
        else:
            return False

    def discretize(self, num_bins, typeDisc):
        """
        This function applies equal width discretization to an Attribute
        :param num_bins: Numeric value indicating the number of intervals
        :param typeDisc: a Character indicating the type of discretization:  "EW"(Equal Width) or "EF"(Equal Frequency)
        :return:  An Attribute with the factor containing the equal width discretization
        """

        if (type(self.vector[0]) is tuple):
            return Attribute(self.vector, self.name)
        else:
            if (typeDisc == "EF"):
                vector = utils.discretizeEF(self.vector, num_bins)[0]
            else:
                vector = utils.discretizeEW(self.vector, num_bins)[0]
            return Attribute(vector, self.name)

    def entropy(self):
        if (type(self.vector[0]) is tuple or (type(self.vector[0]) is int)):
            return utils.entropy(self.vector)
        else:
            return None

    def normalize(self):
        """
        Function to normalize a given Attribute.
        :return: A normalized attribute. If the attribute is categorical it will return a copy of the original attribute
        """
        vector = self.vector.copy()
        if (type(self.vector[0]) is tuple):
            return Attribute(vector, self.name)
        else:
            # valor = min(self.vector)
            # vector = (np.array(vector))
            # vector = vector - valor
            # vector = vector / valor
            mod = math.sqrt(np.sum(np.power(self.vector, 2)))
            vector=np.divide(vector,mod)
            return Attribute(vector, self.name)

    def standardize(self):
        """
        Function to standardize a given Attribute
        :return: A standardized attribute. If the attribute is categorical it will return a copy of the original attribute
        """
        vector = self.vector.copy()
        if (type(self.vector[0]) is tuple):
            return Attribute(vector, self.name)
        else:
            media = np.mean(vector)
            desviacion = np.std(vector)
            estandarizado = (vector - media) / desviacion
            return Attribute(estandarizado, self.name)

    def variance(self):
        """
        Function to compute the variance of a given Attribute
        :return: A real number containing the variance of the attribute
        """
        if (self.isCategorical()):
            return None
        else:
            return np.var(self.vector)

    def getName(self):
        """
        Function to get the name of the Attribute
        :return: A string containing the name of the attribute.
        """
        return self.name

    def getVector(self):
        """
        Function to get the list that contain the Attribute
        :return:A vector containing the information of the attribute.
        """
        return self.vector

    def printAttribute(self):
        print(self.name)
        for i in self.vector:
            print(i)

    def getCatAsInt(self):
        if(self.isCategorical()):
            elements = set(self.vector)
            diccionario = dict(zip(elements, range(0, len(elements))))
            return [diccionario[x] for x in self.vector]
        else:
            return self.vector

def computeCorrelation(x,y,discretizationType="EW",num_bins=3):
    """
    This function computes the correlation between two vectors

    :param x: a vector
    :param y: a vector
    :param discretizationType: if x and y are a mix of continuous and discrete variables a discretization is computed
    in the continuous one. This parameter indicates the discretization type to compute: equal width "EW"
    (default) or equal frequency "EF"
    :param num_bins: if x and y are a mix of continuous and discrete variables a discretization is computed
    in the continuous one. This parameter indicates the number of intervals to use in the discretization.
    It's the  of y/3 by default
    :return: A real number with the correlation between both vectors
    """
    xVector = x.getVector()
    yVector = y.getVector()
    if(x.isCategorical()==False and y.isCategorical()==True):
        if(discretizationType=="EW"):
            xVector=utils.discretizeEW(x.getVector(),num_bins)[0]
            xDisc=Attribute(xVector)

        else:
            xVector=utils.discretizeEF(x.getVector(),num_bins)[0]
            xDisc=Attribute(xVector)

        return sklearn.metrics.mutual_info_score(xDisc.getCatAsInt(), y.getCatAsInt())

    if (x.isCategorical() == True and y.isCategorical() == False):

        if (discretizationType == "EW"):
            yVector = utils.discretizeEW(y.getVector(), num_bins)[0]
            yDisc=Attribute(yVector)

        else:
            yVector = utils.discretizeEF(y.getVector(), num_bins)[0]
            yDisc=Attribute(yVector)

        return sklearn.metrics.mutual_info_score(x.getCatAsInt(), yDisc.getCatAsInt())

    if (x.isCategorical() == True and y.isCategorical() == True):

        return sklearn.metrics.mutual_info_score(x.getCatAsInt(), y.getCatAsInt())
    else:
        return np.corrcoef(xVector,yVector)[0][1]

reason: import numpy


text:
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import PolynomialFeatures

def plot_classifier(X_train, y_train, classifier, scaler, order=1, grid=100, xlabel=r"$x_1$", ylabel=r"$x_2$"):
    """
    Affichage des résultats de classification

    REMARQUE IMPORTANTE: le code ci-dessous a été écrit à l'arrache et n'est absolument pas un modèle de beau code...
    """

    # Create polynomial features
    if classifier.__class__.__name__ == "LogisticRegression":
        poly = PolynomialFeatures(order)
        X = poly.fit_transform(X_train)
    else:
        X = X_train

    if classifier.__class__.__name__ == "KNeighborsClassifier":
        a = 0
        b = 1
    else:
        a = 1
        b = 2

    f, axes = plt.subplots(1,2)
    xx = np.linspace(X[:,a].min(), X[:,a].max(), grid)
    yy = np.linspace(X[:,b].min(), X[:,b].max(), grid)
    XX, YY = np.meshgrid(xx,yy)

    axes[0].set_xlim(X[:,a].min(),X[:,a].max())
    axes[0].set_ylim(X[:,b].min(),X[:,b].max())

    if classifier.__class__.__name__ == "KNeighborsClassifier":
        vec = np.c_[XX.ravel(), YY.ravel()]
    elif classifier.__class__.__name__ == "LogisticRegression":
        vec = poly.fit_transform(np.c_[XX.ravel(), YY.ravel()])
        vec = scaler.transform(vec)
    else:
        raise Exception()

    z = classifier.predict_proba(vec)[:, 1]
    z = z.reshape(XX.shape)

    sns.scatterplot(x=X[:,a], y=X[:,b], hue=y_train.ravel(), ax=axes[0], s=50)
    CS = axes[0].contour(XX,YY,z,[0.5], colors=["g"])
    axes[0].set_xlabel(xlabel)
    axes[0].set_ylabel(ylabel)

    labels = ['Frontière de décision']
    for i in range(len(labels)):
        CS.collections[i].set_label(labels[i])
    axes[0].legend(loc='lower left')

    CS2 = axes[1].contourf(XX, YY, z, 100, cmap="inferno")
    CS = axes[1].contour(XX,YY,z,[0.5], colors=["w"])

max_forks_repo_name: tiombo/Projet-Final


text:
import numpy as np
import pandas as pd
from  mssa.src.prediction_models.ts_svd_model import SVDModel
from math import ceil
from sklearn.preprocessing import StandardScaler

class TSMM(object):
    # kSingularValuesToKeep:    (int) the number of singular values to retain
    # T0:                       (int) the number of entries below which the model will not be trained.
    # T:                        (int) Number of entries in each submodel
    # gamma:                    (float) (0,1) fraction of T after which the model is updated
    # col_to_row_ratio:         (int) the ration of no. columns to the number of rows in each sub-model

    def __init__(self, kSingularValuesToKeep=None, T=int(1e5), gamma=0.2, T0=1000, col_to_row_ratio=1, SSVT=False, p=None, L=None, persist_L = False, no_ts = 1, normalize = True, fill_in_missing = True):
        self.kSingularValuesToKeep = kSingularValuesToKeep

        self.no_ts = no_ts
        self.col_to_row_ratio = col_to_row_ratio
        self.fill_in_missing = fill_in_missing
        # if self.col_to_row_ratio % (self.no_ts*2) != 0:
        #     self.col_to_row_ratio = self.col_to_row_ratio + (2*self.no_ts-self.col_to_row_ratio %(2*self.no_ts))
        # print(self.col_to_row_ratio)
        if L is None:
            self.L = int(np.sqrt(T / self.col_to_row_ratio))
            M = int(self.L * self.col_to_row_ratio)
            self.T = int(self.L * M)

        else:
            self.L = L
            M = int(T/L)
            self.T = int(self.L*M)
            self.col_to_row_ratio = M/L

        if M % (2*self.no_ts) != 0:
            M = M + (2*self.no_ts -M %(2*self.no_ts))
            # self.col_to_row_ratio = self.get_dimensions(M)
            # if not persist_L: self.L = self.T//M
            # subtract a small amount to avoid issues with machine precision
            self.col_to_row_ratio = M/self.L -1e-14

            if not persist_L:
                self.T = int(M*M/self.col_to_row_ratio)
                self.L = int(np.sqrt(self.T / self.col_to_row_ratio))
            else:
                self.T = self.L*M
            Warning ('Number of columns has to be even and divisible by the number of time series thus col_to_row_ratio is changed to %s'%( self.col_to_row_ratio))

        self.normalize = normalize
        self.persist_L = persist_L
        self.gamma = gamma
        self.models = {}
        self.T0 = T0
        self.TimeSeries = None
        self.TimeSeriesIndex = 0
        self.ReconIndex = 0
        self.MUpdateIndex = 0
        self.SSVT = SSVT
        self.p = p

    def get_dimensions(self, M):
        ratio = self.col_to_row_ratio
        while M%ratio !=0:
            ratio +=1
        return ratio

    def get_model_index(self, ts_index=None):
        if ts_index is None:
            ts_index = self.TimeSeriesIndex
        model_index = int(max((ts_index - 1) / (self.T / 2) - 1, 0))
        return model_index

    def update_model(self, NewEntries):
        """
        This function takes a new set of entries and update the model accordingly.
        if the number of new entries means new model need to be bulit, this function segment the new entries into
        several entries and then feed them to the update_ts and fit function
        :param NewEntries: Entries to be included in the new model
        """
        assert len(NewEntries.shape) == 2
        assert NewEntries.shape[1] == self.no_ts

        # Define update chunck for the update SVD function (Not really needed, should be resolved once the update function is fixed)
        if len(self.models) == 1 and NewEntries.size < self.T / 2:
            UpdateChunk = 20 * int(np.sqrt(self.T0))//self.no_ts
        else:
            UpdateChunk = int(self.T/(4*self.col_to_row_ratio))//self.no_ts

        # find if new models should be constructed
        N = NewEntries.size
        Rows = NewEntries.shape[0]

        if N == 0:
            return
        # if the number of models is zero, get the estimate of p
        if len(self.models) == 0 and self.p == None:
            self.p = 1.0 - np.sum(np.isnan(NewEntries))/NewEntries.size
            if self.fill_in_missing: self.p = 1.0
        current_no_models = len(self.models)
        updated_no_models = self.get_model_index(self.TimeSeriesIndex + N) + 1

        # if no new models are to be constructed
        if current_no_models == updated_no_models:
            # If it is a big update, do it at once
            last_model_size = self.models[updated_no_models - 1].M * self.models[updated_no_models - 1].N

            if N/float(last_model_size) > self.gamma: # or len(self.models)>1:
                self.updateTS(NewEntries[:,:])
                self.fitModels()
                return
            # Otherwise, update it chunk by chunk (Because Incremental Update requires small updates when we have one sub-model)
            i = -1
            for i in range(int(ceil(Rows/(UpdateChunk)))):
                self.updateTS(NewEntries[i * UpdateChunk: (i + 1) * UpdateChunk,:])
                self.fitModels()

        else:
            # first complete the last model so it would have exactly T entries
            if current_no_models > 0:
                fillFactor = (self.TimeSeriesIndex % int(self.T /2))
                FillElements = (int((self.T / 2 - fillFactor)) * (fillFactor > 0)) // self.no_ts
                if FillElements > 0:
                    self.updateTS(NewEntries[:FillElements,:])
                    self.fitModels()
                    NewEntries = NewEntries[FillElements:,:]
            # Then, build the other new models. one of the is the very first model, we will skip the second iteration.
            SkipNext = False

            for i in range(updated_no_models - current_no_models + (current_no_models == 0)):
                if SkipNext:
                    SkipNext = False
                    continue

                if len(self.models) == 0:
                    self.updateTS(NewEntries[: self.T//self.no_ts,:])
                    SkipNext = True
                    self.fitModels()
                    i += 1
                    if 0 in self.models:
                        self.kSingularValuesToKeep = self.models[0].kSingularValues

                else:
                    self.updateTS(NewEntries[i * int((self.T//self.no_ts)/2): (i + 1) * int((self.T//self.no_ts)/ 2),:])
                    self.fitModels()

    def updateTS(self, NewEntries):
        # Update the time series with the new entries.
        # only keep the last T entries

        N = NewEntries.size
        num_ts_obs = self.T//self.no_ts
        num_new_rows = N//self.no_ts

        if N > self.T / 2 and len(self.models) > 1:
            raise Exception('TimeSeries should be updated before T/2 values are assigned')

        self.TimeSeriesIndex += N

        if self.TimeSeriesIndex == N or self.TimeSeries is None:
            self.TimeSeries = NewEntries

        elif self.TimeSeries.size < self.T:
            TSarray = np.zeros([len(self.TimeSeries) + N//self.no_ts, self.no_ts])
            TSarray[:len(self.TimeSeries),:] = self.TimeSeries
            TSarray[len(self.TimeSeries):,:] = NewEntries
            self.TimeSeries = TSarray

        else:
            self.TimeSeries[:num_ts_obs-num_new_rows,:] = self.TimeSeries[-num_ts_obs + num_new_rows:,:]
            self.TimeSeries[-num_new_rows:,:] = NewEntries

        if self.TimeSeries.shape[0] > num_ts_obs:

            self.TimeSeries = self.TimeSeries[-num_ts_obs:,:]

    def fitModels(self):

        # Determine which model to fit
        ModelIndex = self.get_model_index(self.TimeSeriesIndex)
        # Determine the number of new Entries since the last reconstruction of a model
        lenEntriesSinceCons = self.TimeSeriesIndex - self.ReconIndex
        lenEntriesSinceLastUpdate = self.TimeSeriesIndex - self.MUpdateIndex
        # Do not fit very few observations
        if self.TimeSeriesIndex < self.T0:
            return
        if np.sqrt(self.TimeSeriesIndex/self.col_to_row_ratio) < 2:
            return
        # Do not fit a lot of observations
        if lenEntriesSinceLastUpdate > self.T and ModelIndex != 0:
            print (self.TimeSeriesIndex, self.MUpdateIndex, [(m.N, m.M, m.start) for m in self.models.values()])
            raise Exception('Model should be updated before T values are assigned')

        if lenEntriesSinceLastUpdate == 0:
            raise Exception('There are no new entries')

        # Build a new model
        if ModelIndex not in self.models:
            # start with the last T/2 entries from previous model
            initEntries = self.TimeSeries[int(int(self.T / 2) - self.TimeSeriesIndex % (self.T / 2))//self.no_ts:,:]
            start = self.TimeSeriesIndex - self.TimeSeriesIndex % int(self.T / 2) - int(self.T / 2)
            # if ModelIndex != 0: assert len(initEntries) == self.T / 2

            if lenEntriesSinceCons == self.T // 2:
                initEntries = self.TimeSeries[:,:]
                start = max(self.TimeSeriesIndex - self.T, 0)
            if ModelIndex == 0:
                initEntries = self.TimeSeries[:,:]
                start = 0

            if self.persist_L: N = self.L
            else:
                N = int(np.sqrt(initEntries.size / (self.col_to_row_ratio)))
                if N >  initEntries.shape[0]:
                    N = initEntries.shape[0]
            M = int(initEntries.size / N)
            if M < self.no_ts:
                raise Exception ('Number of columns in the matrix (%s) is less than the number of time series (%s)' % (M, self.no_ts))
            if M%self.no_ts != 0:
                M -= M%self.no_ts

            M_ts = M//self.no_ts
            if M_ts == 1:
                N = initEntries.shape[0]
            inc_obs = initEntries[:M_ts*N,:]

            if self.normalize:
                scaler = StandardScaler()
                inc_obs = scaler.fit_transform(inc_obs)
                norm_means = scaler.mean_
                norm_std = scaler.scale_
            else:
                norm_means = np.zeros(self.no_ts)
                norm_std = np.ones(self.no_ts)

            self.models[ModelIndex] = SVDModel('t1', self.kSingularValuesToKeep, N, M, start=int(start), SSVT=self.SSVT,
                                               probObservation=self.p, no_ts = self.no_ts, norm_mean = norm_means, norm_std = norm_std, fill_in_missing = self.fill_in_missing)
            flattened_obs = inc_obs.reshape([N,M], order = 'F')
            flattened_obs = flattened_obs[:,np.arange(M_ts*self.no_ts).reshape([self.no_ts,M_ts]).flatten('F')]
            self.models[ModelIndex].fit(pd.DataFrame(data={'t1': flattened_obs.flatten('F')}))
            self.models[ModelIndex].obs_ = flattened_obs.flatten('F')

            old_mupdate_index = self.MUpdateIndex
            self.ReconIndex = max(N * M + start, old_mupdate_index)
            self.MUpdateIndex = self.ReconIndex
            if lenEntriesSinceCons == self.T // 2 or ModelIndex == 0:
                return

        Model = self.models[ModelIndex]
        lenEntriesSinceCons = self.TimeSeriesIndex - self.ReconIndex
        ModelLength = Model.N * Model.M + Model.start
        TSlength = self.TimeSeriesIndex - Model.start
        if (TSlength <= self.TimeSeries.size and (float(lenEntriesSinceCons) / (self.ReconIndex - Model.start) >= self.gamma)) or (self.TimeSeriesIndex % (self.T / 2) == 0):  # condition to recompute SVD
            if self.persist_L: N = self.L
            else: N = int(np.sqrt(TSlength/self.col_to_row_ratio))
            M = int(TSlength / N)
            if M % self.no_ts != 0:
                M -= M%self.no_ts

            M_ts = M//self.no_ts
            TSeries = self.TimeSeries[-TSlength//self.no_ts:,:]
            TSeries = TSeries[:(N * M)//self.no_ts,:]
            if self.normalize:
                scaler = StandardScaler()
                TSeries = scaler.fit_transform(TSeries)
                norm_means = scaler.mean_
                norm_std = scaler.scale_
            else:
                norm_means = np.zeros(self.no_ts)
                norm_std = np.ones(self.no_ts)

            flattened_obs = TSeries.reshape([N,M], order = 'F')
            flattened_obs = flattened_obs[:,np.arange(M).reshape([self.no_ts,M_ts]).flatten('F')]

            self.models[ModelIndex] = SVDModel('t1', self.kSingularValuesToKeep, N, M, start= int(Model.start),
                                               TimesReconstructed=Model.TimesReconstructed + 1,
                                               TimesUpdated=Model.TimesUpdated, SSVT=self.SSVT, probObservation=self.p,
                                               no_ts = self.no_ts, norm_mean = norm_means, norm_std = norm_std, fill_in_missing = self.fill_in_missing)

            self.models[ModelIndex].fit(pd.DataFrame(data={'t1': flattened_obs.flatten('F')}))
            self.ReconIndex = N * M + Model.start
            self.MUpdateIndex = self.ReconIndex

        else:
            Model = self.models[ModelIndex]
            N = Model.N
            M = Model.M
            if self.TimeSeriesIndex - ModelLength < N*self.no_ts:
                return
            else:
                NewEntries = self.TimeSeries[-(self.TimeSeriesIndex - ModelLength)//self.no_ts:,:]
                if self.normalize:
                    NewEntries = NewEntries - Model.norm_mean
                    NewEntries = NewEntries / Model.norm_std
                num_new_columns = self.no_ts*((NewEntries[:,0]).size//N)
                flattened_obs = NewEntries[:(num_new_columns*N)//self.no_ts,:].reshape([N,num_new_columns], order = 'F')
                flattened_obs = flattened_obs[:,np.arange(num_new_columns).reshape([self.no_ts,num_new_columns//self.no_ts]).flatten('F')]
                D = flattened_obs.flatten('F')
                # p = int(len(D) / N)
                # D = D[:N * p]
                print(D.shape)
                Model.updateSVD(D, 'UP')
                self.MUpdateIndex = Model.N * Model.M + Model.start
                Model.updated = True

    def _denoiseTS(self, models=None, index=None, range_=True):
        # denoise the whole time series if no specific  submodels are selected
        if models is None:
            models = self.models

        if range_ or index is None:
            if index is None:
                index = [0, int(self.MUpdateIndex)]
            no_obs = (index[1] - index[0])//self.no_ts
            denoised = np.zeros([no_obs, self.no_ts])
            count =  np.zeros([no_obs, self.no_ts])
            y1, y2 = index[0], index[1]
            for Model in models.values():
                x1, x2 = Model.start, Model.M * Model.N + Model.start
                if x1 <= y2 and y1 <= x2:
                    RIndex = np.array([max(x1, y1), min(x2, y2)])//self.no_ts
                    RIndexS = RIndex - y1//self.no_ts
                    # call to make sure that model.matrix is estimated
                    denoise_flat = Model.denoisedTS(return_ = False)
                    M = Model.M
                    N = Model.N
                    denoised_matrix = Model.matrix
                    denoised_columns_swapped = denoised_matrix[:,np.arange(M).reshape([M//self.no_ts, self.no_ts]).flatten('F')]
                    denoised_ts = denoised_columns_swapped.reshape([denoised_columns_swapped.size//self.no_ts,self.no_ts],order ='F')
                    denoised_index = RIndex-x1//self.no_ts
                    denoised[RIndexS[0]:RIndexS[1],:] += (denoised_ts[denoised_index[0]:denoised_index[1],:]*Model.norm_std+Model.norm_mean)
                    count[RIndexS[0]:RIndexS[1],:] += 1
            denoised[count == 0] = np.nan
            denoised[count > 0] = denoised[count > 0] / count[count > 0]
            return denoised

        else:

            index = np.array(index)
            I = len(index)
            denoised = np.zeros(I)
            models = np.zeros(2 * I)
            models[:I] = (index) / (self.T / 2) - 1
            models[I:] = models[:I] + 1
            models[models < 0] = 0
            count = np.zeros(len(index))

            for ModelNumber in np.unique(models):
                Model = models[ModelNumber]
                x1, x2 = Model.start, Model.M * Model.N + Model.start
                updatedIndices = np.logical_and(index >= x1, index < x2)
                assert np.sum(updatedIndices) > 0
                count += updatedIndices
                denoised[updatedIndices] += Model.denoisedTS(index[updatedIndices] - x1, range_)

            denoised[count == 0] = np.nan
            denoised[count > 0] = denoised[count > 0] / count[count > 0]
            return denoised

    def _predict(self, index=None, method='average', NoModels=None, dataPoints=None, models=None):

        if models is None:
            models = self.models
        n = len(models)

        if NoModels is None or NoModels > n or NoModels < 1: NoModels = n
        # if index next predict
        # UsedModels = [a for a in models.values()[-NoModels:]]
        UsedModels = [models[i] for i in range(n - NoModels, n)]

        if dataPoints is None and (index is None or index == self.TimeSeriesIndex + 1):
            TSDF = pd.DataFrame(data={'t1': self.TimeSeries[-self.L:]})

            predicions = np.array([mod.predict(pd.DataFrame(data={}), TSDF) for mod in UsedModels])
            return np.mean(predicions)

        elif dataPoints is None and index <= self.TimeSeriesIndex:
            slack = self.TimeSeriesIndex - index + 1
            if slack > (self.T - self.L): raise Exception
            TSDF = pd.DataFrame(data={'t1': self.TimeSeries[-self.L - slack:-slack]})
            predicions = np.array([mod.predict(pd.DataFrame(data={}), TSDF) for mod in UsedModels])
            return np.mean(predicions)

        elif dataPoints is not None:
            assert len(dataPoints) == self.L - 1
            TSDF = pd.DataFrame(data={'t1': dataPoints})
            predicions = np.array([mod.predict(pd.DataFrame(data={}), TSDF) for mod in UsedModels])
            return np.mean(predicions)
        else:
            return 0

max_stars_repo_name: valeman/mSSA

max_issues_repo_name: valeman/mSSA

max_forks_repo_name: valeman/mSSA


text:
import numpy as np
from helpers.getCornersFrom2Points import getLineAngleFrom2Points

def validateCords(cords, screenWidth, screenHeight, maxAngleDiff, maxDistance):
    boxTopLeft = cords[0]
    boxTopRight = cords[1]
    boxBottomRight = cords[2]

    angle = getLineAngleFrom2Points(boxTopLeft, boxTopRight)
    if np.rad2deg(angle) > maxAngleDiff or np.rad2deg(angle) < -maxAngleDiff:
        return False

    if boxTopLeft[0] < -maxDistance or \
            boxTopLeft[0] > maxDistance or \
            boxTopLeft[1] < -maxDistance or \
            boxTopLeft[1] > maxDistance or \
            boxBottomRight[0] < screenWidth - maxDistance or \
            boxBottomRight[0] > screenWidth + maxDistance or \
            boxBottomRight[1] < screenHeight - maxDistance or \
            boxBottomRight[1] > screenHeight + maxDistance:
        return False

    return True


text:
"""
Utilities
---------

This package contains helper functions that did not fit into any other packages.
These functions include helper functions for common operations when dealing with :mod:`sympy`,
as well as functions that help with memoisation of CPU intensive function results.

These functions are designed to be package-specific.
The users of the software are generally discouraged to use any of these functions.
"""

import sympyhelpers
from memoisation import MemoisableObject, memoised_property
import memoisation
import decorators
from decorators import cache

max_stars_repo_name: nicktimko/means

max_issues_repo_name: nicktimko/means

max_forks_repo_name: nicktimko/means

reason: import sympy


text:
# MCMC sampling for MMPP with Pytorch
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn.functional as F

from .utils import hard_step, soft_step

class MJPTrajectory:
    def __init__(self, arrival_times, states, t_max):
        if len(arrival_times) + 1 != states.shape[0]:
            raise ValueError("Shapes of t and z don't match")
        if states.ndim != 2:
            raise ValueError("States must be in one-hot format")
        self.n_intervals = len(arrival_times) + 1
        self.t = arrival_times
        self.tau = torch.zeros(self.n_intervals)
        self.tau[-1] = t_max
        self.tau[:-1] = arrival_times
        self.tau[1:] -= arrival_times
        self.z = states

    def __repr__(self):
        return f"MJPTrajectory({self.n_intervals})"

def stack_paths(samples):
    """Convert a list of MJPTrajectories into two tensors

    Args:
        samples: List of MJPTrajectory of length B

    Returns:
        tau: Dwell times, shape [B, N_max]
        z: States, shape [B, N_max, K]
    """
    n_max = max(s.n_intervals for s in samples)
    taus = []
    zs = []
    for s in samples:
        taus.append(F.pad(s.tau, (0, n_max - s.n_intervals)).unsqueeze(0))
        zs.append(F.pad(s.z, (0, 0, 0, n_max - s.n_intervals)).unsqueeze(0))
    tau = torch.cat(taus, axis=0)
    z = torch.cat(zs, axis=0)
    return tau, z

def combine_duplicates(tau, z):
    """Merge dwell times of repeating subsequent states."""
    different = torch.ones(z.shape[0]).bool()
    different[:-1] = torch.any(z[1:] != z[:-1], dim=1)
    z_new = z[different]
    tau_new = torch.zeros(different.sum().item())
    prev = 0
    for idx, nxt in enumerate(different.nonzero().view(-1)):
        nxt = nxt.item() + 1
        tau_new[idx] = tau[slice(prev, nxt)].sum()
        prev = nxt
    return tau_new, z_new

def sample_poisson(t_max, rate):
    """Sample inter-event times from a Poisson process.

    Args:
        t_max: Duration of the observed interval.
        rate: Rate of the Poisson process.

    Returns:
        t: Arrival times.
    """
    tau = []
    t_last = 0.0
    while True:
        tau_next = np.random.exponential(scale=1./rate)
        t_next = t_last + tau_next
        if t_next > t_max:
            break
        tau.append(tau_next)
        t_last += tau_next
    return np.array(tau).cumsum()

def assign_to_intervals_tau(tau, x, temp=None):
    """Assign events to intervals.

    Args:
        tau: sample from q(t), shape [batch_size, num_intervals]
        x: event times, shape [num_events, 1]
        temp: temperature for soft step. Default: None - use hard step.

    Returns:
        assignment: event to interval, shape [batch_size, num_events, num_intervals]
    """
    batch_size, num_intervals = tau.shape
    num_samples = x.shape[0]
    t = tau.cumsum(-1)[..., :-1]
    t.unsqueeze_(-2)
    if temp is None:
        step = hard_step
    else:
        step = lambda x: soft_step(x, temp)
    assignment = torch.zeros(batch_size, num_samples, num_intervals)
    assignment[..., -1] = 1.0
    diff = step(t - x)
    assignment[..., :-1] += diff
    assignment[..., 1:] -= diff
    return assignment

def sample_ihp(path, rates, omega, t_max):
    """Sample from an inhomogeneous Poisson process with pw-constant intensity.

    Sampling is done using thinning.

    Args:
        path: MJP trajectory (defines the pw-constant rates)
        rates: Rate of each state, shape [K]
        omega: Upper bound on the rates.
        t_max: End of the observed interval.

    Returns:
        t: Arrival times.
    """
    t = torch.Tensor(sample_poisson(t_max, omega))

    assignment = assign_to_intervals_tau(path.tau.view([1, -1]),
                                     t.view([-1, 1]))[0]
    real_intensity = (assignment @ path.z @ rates).view(-1)

    accept = (torch.rand_like(t) * omega) < real_intensity
    return t[accept]

def t_to_tau(t, t_max):
    tau = torch.zeros(len(t) + 1)
    tau[-1] = t_max
    tau[:-1] = t
    tau[1:] -= t
    return tau

def tau_to_t(tau):
    t = tau.cumsum(-1)[:-1]
    return t

def log_vector_matrix(vec, mtx):
    return torch.logsumexp(vec[..., np.newaxis] + mtx, dim=-2)

def log_matrix_vector(mtx, vec):
    return torch.logsumexp(mtx + vec[..., np.newaxis, :], dim=-1)

def ffbs_logspace(log_obs_prob, log_transition, log_prior):
    """Sample from the posterior of a Hidden Markov Model.

    Implemented according to Rao & Teh, JMLR 2014.
    All the computations are performed in log-space for numerical stability.

    Args:
        log_obs_prob: Log-likelihood matrix, shape [batch_size, n_intervals, n_states]
        log_transition: Log of transition matrix, shape [n_states, n_states]
        log_prior: Log of prior on z_0, shape [n_states]

    Returns:
        z: Samples of states given jump times, shape [batch_size, n_intervals, n_states]
    """
    n_intervals = log_obs_prob.shape[-2]
    n_states = log_obs_prob.shape[-1]
    log_alpha = torch.zeros_like(log_obs_prob)
    log_beta = torch.zeros_like(log_obs_prob)
    z = torch.zeros_like(log_obs_prob)
    log_alpha[..., 0, :] = log_prior
    for i in range(1, n_intervals):
        log_alpha[..., i, :] = log_vector_matrix(log_alpha[..., i - 1, :] + log_obs_prob[..., i - 1, :],
                                                 log_transition.T)
    log_beta[..., -1, :] = log_obs_prob[..., -1, :] + log_alpha[..., -1, :]
    probs = log_beta[..., -1, :].softmax(-1)
    z[..., -1, :] = F.one_hot(torch.multinomial(probs, 1), n_states)
    transition = log_transition.exp()
    for i in reversed(range(0, n_intervals - 1)):
        log_beta[..., i, :] = log_alpha[..., i, :] + log_obs_prob[..., i, :] + (z[..., i + 1, :] @ transition.T).log()
        probs = log_beta[..., i, :].softmax(-1)
        z[..., i, :] = F.one_hot(torch.multinomial(probs, 1), n_states)
    return z

def sample_mmpp_posterior(path_init, t_max, x, lmbd, Q, pi, omega, n_burn_in=100, n_sample=1000, tqdm=None):
    """Draw samples from the posterior distribution of a MMPP

    Args:
        path_init: Initial MJPTrajectory
        t_max: Duration of the observed interval.
        x: Observed events (realizations of the visible PP), shape [M, 1]
        lmbd: Rates of each state, shape [K]
        Q: Generator matrix, shape [K, K]
        pi: Prior distribution over z_0, shape [K]
        omega: Uniformization constant
        n_burn_in: Number of samples to discard.
        n_sample: Number of samples to return.
    """
    if omega <= Q.abs().diag().max():
        raise ValueError("Uniformization constant must be larger than the largest value of Q.")
    K = Q.shape[0]
    B = torch.eye(K) + Q / omega
    B = F.normalize(B, p=1, dim=1, eps=0.0)

    path = path_init
    samples = []

    if tqdm is None:
        range_iter = range
    else:
        range_iter = tqdm.trange

    for epoch in range_iter(n_burn_in + n_sample):
        jump_rates = omega + torch.diag(Q)
        oversampling = 2.0 * jump_rates.max().item()
        t_new = sample_ihp(path, jump_rates, oversampling, t_max)
        t = torch.sort(torch.cat([t_new, path.t])).values
        tau = t_to_tau(t, t_max)

        # Compute the likelihood matrix
        assignment = assign_to_intervals_tau(tau.view(1, -1), x)[0]
        counts = assignment.sum(0).view(-1, 1)  # num of events in each interval

        log_obs_prob = (counts * lmbd.log() - tau.view(-1, 1) * lmbd).unsqueeze(0)
        log_transition = B.log()
        log_prior = pi.log()
        z = ffbs_logspace(log_obs_prob, log_transition, log_prior)[0]

        tau_new, z_new = combine_duplicates(tau, z)
        t_new = tau_to_t(tau_new)

        path = MJPTrajectory(t_new, z_new, t_max)
        samples.append(path)
    return samples[n_burn_in:]

def m_step(tau, z, x, psi, alpha, beta):
    """Compute optimal values of lmbd, Q, pi, omega given samples from the posterior."""
    # avg number of transitions i->j
    trans_counts = (z[:, :-1].transpose(-2, -1) @ z[:, 1:]).mean(0)
    # avg total time spent in each of K states
    dwell_times = (z * tau.unsqueeze(-1)).sum(-2).mean(0)
    # avg number of observations in each of K states
    event_counts = (assign_to_intervals_tau(tau, x) @ z).sum(-2).mean(0)

    lmbd_new = event_counts / dwell_times
    psi = psi - 1
    pi_new = (z[:, 0, :].sum(0) + psi) / (z.shape[0] + psi.sum())

    Q_new = (trans_counts + alpha - 1) / (dwell_times.reshape([-1, 1]) + beta)
    Q_new[np.diag_indices_from(Q_new)] = 0.0
    Q_new -= torch.diag(Q_new.sum(1))

    omega_new = 2.0 * Q_new.diag().abs().max().item()

    return lmbd_new, Q_new, pi_new, omega_new

max_stars_repo_name: shchur/triangular-tpp

max_issues_repo_name: shchur/triangular-tpp

max_forks_repo_name: shchur/triangular-tpp


text:
import numpy as np

def legendre(x, n):

    y = np.zeros((n + 1, len(x)))
    y[0, :] = 1.0
    if n > 0:
        y[1, :] = x
    for k in range(1, n):
        y[k + 1, :] = (
            1.0 / (k + 1.0) * ((2.0 * k + 1.0) * x * y[k, :] - k * y[k - 1, :])
        )

    return y

def legendre2(x, n):

    if n % 2:
        raise ValueError("Can only ask for even Legendre polynomials")

    y = legendre(x, n)

    y2 = np.zeros((n / 2 + 1, len(x)))
    for k in range(0, n + 1, 2):
        y2[k / 2, :] = y[k, :]

    return y2

def zonal(z, n):

    y = legendre(z, n)
    for k in range(0, n + 1):
        y[k, :] *= np.sqrt((2.0 * k + 1.0) / (4.0 * np.pi))
    return y

def zonal2(z, n):

    y = legendre2(z, n)
    for k in range(0, n + 1, 2):
        y[k / 2, :] *= np.sqrt((2.0 * k + 1.0) / (4.0 * np.pi))
    return y

max_stars_repo_name: cbannwarth/aimsprop

max_issues_repo_name: cbannwarth/aimsprop

max_forks_repo_name: cbannwarth/aimsprop


text:
"""Author: Brandon Trabucco, Copyright 2019, MIT License"""

from playground import maybe_initialize_process
from playground.envs.normalized_env import NormalizedEnv
from playground.distributions.gaussian import Gaussian
from playground.networks import dense
from playground.replay_buffers.path_replay_buffer import PathReplayBuffer
from playground.loggers.tensorboard_logger import TensorboardLogger
from playground.samplers.parallel_sampler import ParallelSampler
from playground.algorithms.policy_gradient import PolicyGradient
import numpy as np

policy_gradient_variant = dict(
    max_path_length=1000,
    max_num_paths=1000,
    logging_dir="./",
    hidden_size=400,
    num_hidden_layers=2,
    reward_scale=1.0,
    discount=0.99,
    policy_learning_rate=0.0003,
    exploration_noise_std=0.1,
    num_workers=2,
    num_steps_per_epoch=10000,
    num_steps_per_eval=10000,
    num_epochs_per_eval=10,
    num_epochs=1000)

def policy_gradient(
        variant,
        env_class,
        env_kwargs=None,
        observation_key="observation",
):
    # initialize tensorflow and the multiprocessing interface
    maybe_initialize_process()

    # run an experiment with multiple agents
    if env_kwargs is None:
        env_kwargs = {}

    # initialize the environment to track the cardinality of actions
    env = NormalizedEnv(env_class, **env_kwargs)
    action_dim = env.action_space.low.size
    observation_dim = env.observation_space.spaces[
        observation_key].low.size

    # create a replay buffer to store data
    replay_buffer = PathReplayBuffer(
        max_path_length=variant["max_path_length"],
        max_num_paths=variant["max_num_paths"])

    # create a logging instance
    logger = TensorboardLogger(
        replay_buffer, variant["logging_dir"])

    # create policies for each level in the hierarchy
    policy = Gaussian(
        dense(
            observation_dim,
            action_dim,
            hidden_size=variant["hidden_size"],
            num_hidden_layers=variant["num_hidden_layers"],
            output_activation="tanh"),
        optimizer_kwargs=dict(learning_rate=variant["policy_learning_rate"]),
        std=variant["exploration_noise_std"])

    # train the agent using soft actor critic
    algorithm = PolicyGradient(
        policy,
        replay_buffer,
        reward_scale=variant["reward_scale"],
        discount=variant["discount"],
        observation_key=observation_key,
        batch_size=-1,  # sample everything in the buffer
        logger=logger,
        logging_prefix="policy_gradient/")

    # make a sampler to collect data to warm up the hierarchy
    sampler = ParallelSampler(
        env,
        policy,
        max_path_length=variant["max_path_length"],
        num_workers=variant["num_workers"])

    #  train for a specified number of iterations
    for iteration in range(variant["num_epochs"]):

        # discard all previous samples for on policy learning
        replay_buffer.empty()

        if iteration % variant["num_epochs_per_eval"] == 0:

            # evaluate the policy at this step
            sampler.set_weights(policy.get_weights())
            paths, eval_returns, num_steps = sampler.collect(
                variant["num_steps_per_eval"],
                deterministic=True,
                keep_data=False,
                workers_to_use=variant["num_workers"])
            logger.record("eval_mean_return", np.mean(eval_returns))

        # collect more training samples
        sampler.set_weights(policy.get_weights())
        paths, train_returns, num_steps = sampler.collect(
            variant["num_steps_per_epoch"],
            deterministic=False,
            keep_data=True,
            workers_to_use=variant["num_workers"])
        logger.record("train_mean_return", np.mean(train_returns))

        # insert the samples into the replay buffer
        for o, a, r in paths:
            replay_buffer.insert_path(o, a, r)

        # train once with the on policy data
        algorithm.fit(iteration)

max_stars_repo_name: brandontrabucco/playground

max_issues_repo_name: brandontrabucco/playground

max_forks_repo_name: brandontrabucco/playground


text:
# License: Apache-2.0
# import os
import setuptools
import numpy
from Cython.Build import cythonize
from gators import __version__

with open('README.md', 'r') as f:
    long_description = f.read().replace('![Gators logo](doc_data/GATORS_LOGO.png)\n\n', '')

# rootdir = os.path.normpath(os.path.join(__file__, os.pardir))
# print(rootdir)
# import sys
# sys.exit()
extensions = [
    setuptools.Extension(
        'imputer',
        ['gators/imputers/imputer.pyx'],
        include_dirs=[numpy.get_include()],
        extra_compile_args=['-O3'],
    ),
    setuptools.Extension(
        'encoder',
        ['gators/encoders/encoder.pyx'],
        include_dirs=[numpy.get_include()],
        extra_compile_args=['-O3'],
    ),
    setuptools.Extension(
        'data_cleaning',
        ['gators/data_cleaning/data_cleaning.pyx'],
        include_dirs=[numpy.get_include()],
        extra_compile_args=['-O3'],
    ),
    setuptools.Extension(
        'binning',
        ['gators/binning/binning.pyx'],
        include_dirs=[numpy.get_include()],
        extra_compile_args=['-O3'],
    ),
    setuptools.Extension(
        'clipping',
        ['gators/clipping/clipping.pyx'],
        include_dirs=[numpy.get_include()],
        extra_compile_args=['-O3'],
    ),
    setuptools.Extension(
        'scaler',
        ['gators/scalers/scaler.pyx'],
        include_dirs=[numpy.get_include()],
        extra_compile_args=['-O3'],
    ),
    setuptools.Extension(
        'feature_gen',
        ['gators/feature_generation/feature_gen.pyx'],
        include_dirs=[numpy.get_include()],
        extra_compile_args=['-O3'],
    ),
    setuptools.Extension(
        'feature_gen_dt',
        ['gators/feature_generation_dt/feature_gen_dt.pyx'],
        include_dirs=[numpy.get_include()],
        extra_compile_args=['-O3'],
    ),
    setuptools.Extension(
        'feature_gen_str',
        ['gators/feature_generation_str/feature_gen_str.pyx'],
        include_dirs=[numpy.get_include()],
        extra_compile_args=['-O3'],
    )
]

setuptools.setup(
    name='gators',
    version=__version__,
    url='https://paypal.github.io/gators/',
    author='Simility Data Team',
    options={'bdist_wheel': {'universal': True}},
    long_description=long_description,
    long_description_content_type="text_markdown",
    description='Model building and Model deployment library',
    maintainer='Charles Poli',
    packages=setuptools.find_packages(exclude=['examples', 'doc']),
    classifiers=[
        'Programming Language :: Python :: 3',
        'License :: OSI Approved :: Apache-2 License',
        'Operating System :: OS Independent',
    ],
    python_requires='>=3.6',
    license='Apache-2.0 Software License',
    setup_requires=['numpy', 'Cython'],
    package_data={'gators': [
        '*.c',
    ]},
    include_package_data=True,
    ext_modules=cythonize(extensions, compiler_directives={
                          'language_level': "3"}),
    zip_safe=False,
)

max_stars_repo_name: Aditya-Kapadiya/gators

max_issues_repo_name: Aditya-Kapadiya/gators

max_forks_repo_name: Aditya-Kapadiya/gators


text:
import numpy as np

class EarlyStopping():
    def __init__(self, patience=3, verbose=1):
        self._step = 0
        self._loss = float('inf')
        self.patience = patience
        self.verbose = verbose

    def validate(self, loss):
        if self._loss < loss:
            self._step += 1
            if self._step > self.patience:
                if self.verbose:
                    print('early stopping')
                return True
        else:
            self._step = 0
            self._loss = loss

        return False

def check_converge(log, duration=20, threshold=0.01):
    ave = np.mean(log[-duration:])
    current = log[-1]

    if abs(ave-current) < threshold and len(log) > duration:
        print("early stopping")
        return True
    else:
        return False


text:
from __future__ import print_function, division

import copy
import torch
import numpy as np
from sklearn.metrics import roc_auc_score

def gen_target(LOSS_MODE, tg, device):
    target, target2 = None, None
    if LOSS_MODE == "BCELoss":
        target = torch.FloatTensor(tg)
    return target.to(device), target2

def cal_loss(LOSS_MODE, scores, target, target2, criterion, criterion2):
    loss = None
    if LOSS_MODE == 'BCELoss':
        loss = criterion(scores[0], target.unsqueeze(1))
    return loss

def trans_scores(LOSS_MODE, scores, cos_dis):
    if LOSS_MODE == 'BCELoss':
        scores_n = scores[0]
    return scores_n

def collect_metrics(metrics_records, loss_records, step_name, metrics, label_predict, total_loss, get_f1=False):
    avg_loss = np.mean(total_loss)
    loss_records[step_name].append(avg_loss)
    accuracy, recall, F1 = cal_f1(metrics)
    metrics_records[step_name]["acc"].append(accuracy)
    metrics_records[step_name]["rec"].append(recall)
    metrics_records[step_name]["f1"].append(F1)
    metrics_records[step_name]["auc"].append(
        roc_auc_score(y_true=label_predict["label"], y_score=label_predict["predict"]))
    print("{} loss:{:.4f} acc:{:.4f} rec:{:.4f} f1:{:.4f}".format(step_name, avg_loss, accuracy, recall, F1))
    if get_f1:
        return metrics_records, loss_records, avg_loss, F1
    return metrics_records, loss_records, avg_loss

def hit_rate(rank_list, topk):
    hit_list = []
    for rank in rank_list:
        rank = rank + 1
        if rank > topk:
            hit = 0
        else:
            hit = 1.0 / topk * (topk + 1 - rank)
        hit_list.append(hit)
    return hit_list

def cal_metrics_batch(metrics, scores, tg, loss_mode):
    val, idxx = scores.to("cpu").detach().topk(1)
    if loss_mode in ['BCELoss']:
        pre = (val.numpy() >= 0.5) + 0
    for i in range(len(tg)):
        if pre[i] == 1 and tg[i] == 1:
            metrics['TP'] += 1
        elif pre[i] == 1 and tg[i] == 0:
            metrics['FP'] += 1
        elif pre[i] == 0 and tg[i] == 1:
            metrics['FN'] += 1
        elif pre[i] == 0 and tg[i] == 0:
            metrics['TN'] += 1
    return metrics, val.numpy().tolist()

def cal_f1(metrics):
    if metrics["TP"] + metrics["FP"] == 0:
        accuracy = 0
    else:
        accuracy = metrics['TP'] / (metrics['TP'] + metrics['FP'])
    if metrics['TP'] + metrics['FN'] == 0:
        recall = 0
    else:
        recall = metrics['TP'] / (metrics['TP'] + metrics['FN'])
    if accuracy == 0 and recall == 0:
        F1 = 0
    else:
        F1 = 2 * accuracy * recall / (accuracy + recall)
    return accuracy, recall, F1

def collect_loss(loss):
    tmp = np.mean(loss.to("cpu").detach().numpy(), dtype=np.float64)
    return tmp

def gen_batch_similarity(samples, device=None, poi=False):
    top_lens, down_lens, target = [], [], []
    for sample in samples:
        top_lens.append(len(sample[0][0]))
        down_lens.append(len(sample[1][0]))
        target.append(sample[2])

    top_maxlen = max(top_lens)
    down_maxlen = max(down_lens)
    loc_top, loc_down = [], []
    tim_top, tim_down = [], []
    if poi:
        poi_top, poi_down = [], []
    for sample in samples:
        if poi:
            loc_tmp, tim_tmp, poi_tmp = copy.deepcopy(sample[0])
        else:
            loc_tmp, tim_tmp = copy.deepcopy(sample[0])
        loc_tmp.extend([0] * (top_maxlen - len(loc_tmp)))
        loc_top.append(loc_tmp)
        tim_tmp.extend([0] * (top_maxlen - len(tim_tmp)))
        tim_top.append(tim_tmp)
        if poi:
            for _ in range((top_maxlen - len(poi_tmp))):
                poi_tmp.append([0] * 21)
            poi_top.append(poi_tmp)

        if poi:
            loc_tmp, tim_tmp, poi_tmp = copy.deepcopy(sample[1])
        else:
            loc_tmp, tim_tmp = copy.deepcopy(sample[1])
        loc_tmp.extend([0] * (down_maxlen - len(loc_tmp)))
        loc_down.append(loc_tmp)
        tim_tmp.extend([0] * (down_maxlen - len(tim_tmp)))
        tim_down.append(tim_tmp)
        if poi:
            for _ in range((down_maxlen - len(poi_tmp))):
                poi_tmp.append([0] * 21)
            poi_down.append(poi_tmp)
    loc_top = torch.LongTensor(loc_top).to(device)
    loc_down = torch.LongTensor(loc_down).to(device)
    tim_top = torch.LongTensor(tim_top).to(device)
    tim_down = torch.LongTensor(tim_down).to(device)
    top_lens = torch.LongTensor(top_lens).to(device)
    down_lens = torch.LongTensor(down_lens).to(device)
    if poi:
        poi_top = torch.FloatTensor(poi_top).to(device)
        poi_down = torch.FloatTensor(poi_down).to(device)
        return loc_top, tim_top, poi_top, top_lens, loc_down, tim_down, poi_down, down_lens, target
    else:
        return loc_top, tim_top, top_lens, loc_down, tim_down, down_lens, target

max_stars_repo_name: lzzppp/DERT

max_issues_repo_name: lzzppp/DERT

max_forks_repo_name: lzzppp/DERT


text:
"""
Example file for the __equs__.Python workshop 2019

https://github.com/equs-python/__equs__.Python

This file contains unit tests for the measure_state
function from the measurement_2.py module.
"""
import numpy as np

from measurement_2 import measure_state

def test_pure_state_measurement():
    """ Testing the measurement of a pure state in Z """
    # Test 1: State |0>, one measurement
    state = np.array([[1, 0], [0, 0]])
    expected_result = np.array([1, 0])
    actual_result = measure_state(state, number_of_samples=1)

    # Numpy's version of the assert statement for arrays
    assert np.allclose(expected_result, actual_result)

    # Test 2: State |1>, one measurement
    state = np.array([[0, 0], [0, 1]])
    expected_result = np.array([0, 1])
    actual_result = measure_state(state, number_of_samples=1)

    assert np.allclose(expected_result, actual_result)

def test_pure_state_measurement_stats():
    """ Testing the statistics of a pure state measurement in Z """
    # Test 1: State |0>, 100 measurements
    state = np.array([[1, 0], [0, 0]])
    expected_result = np.array([1, 0])
    actual_results = measure_state(state, number_of_samples=100)
    # Loop over the results
    for actual_result in actual_results:
        assert np.allclose(expected_result, actual_result)

def test_mixed_state_measurement_stats():
    """ Testing the statistics of a pure state measurement in Z """
    # State (|0> + |1>)/sqrt(2), 5000 measurements
    state = np.array([[0.5, 0.5], [0.5, 0.5]])
    number_of_samples = 5000
    actual_results = measure_state(
        state, number_of_samples=number_of_samples
    )
    # Count how often the |0> and |1> state were measured
    outcomes, counts = np.unique(
        actual_results, return_counts=True, axis=0
    )
    tolerance = 0.05
    assert abs(counts[0] / number_of_samples - 0.5) < tolerance


text:
# --------------------------------------------------------
# Visual Detection: State-of-the-Art
# Copyright: Hanbo Zhang
# Licensed under The MIT License [see LICENSE for details]
# Written by Hanbo Zhang
# --------------------------------------------------------

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

from model.utils.config import cfg

from model.fcgn.classifier import _Classifier
from model.fcgn.grasp_proposal_target import _GraspTargetLayer

from model.fcgn.bbox_transform_grasp import points2labels
from model.utils.net_utils import _smooth_l1_loss, weights_normal_init

import numpy as np

from model.fcgn.generate_grasp_anchors import generate_oriented_anchors
from model.Detectors import graspDetector

import pdb

class FCGN(graspDetector):

    def __init__(self, feat_name = 'res101', feat_list = ('conv4',), pretrained = True):
        super(FCGN, self).__init__(feat_name, feat_list, pretrained)
        ##### Important to set model to eval mode before evaluation ####
        self.FeatExt.eval()
        rand_img = torch.zeros(size=(1, 3, 224, 224))
        rand_feat = self.FeatExt(rand_img)
        self.FeatExt.train()
        self.dout_base_model = rand_feat.size(1)

        self.size = cfg.SCALES[0]
        self.FCGN_as = cfg.FCGN.ANCHOR_SCALES
        self.FCGN_ar = cfg.FCGN.ANCHOR_RATIOS
        self.FCGN_aa = cfg.FCGN.ANCHOR_ANGLES
        self.FCGN_fs = cfg.FCGN.FEAT_STRIDE[0]

        self.FCGN_classifier = _Classifier(self.dout_base_model, 5, self.FCGN_as, self.FCGN_ar, self.FCGN_aa)
        self.FCGN_proposal_target = _GraspTargetLayer(self.FCGN_fs, self.FCGN_ar, self.FCGN_as, self.FCGN_aa)

        self.FCGN_anchors = torch.from_numpy(generate_oriented_anchors(base_size=self.FCGN_fs,
                                    scales=np.array(self.FCGN_as), ratios=np.array(self.FCGN_ar),
                                    angles=np.array(self.FCGN_aa))).float()

        self.FCGN_num_anchors = self.FCGN_anchors.size(0)
        # [x1, y1, x2, y2] -> [xc, yc, w, h]
        self.FCGN_anchors = self._grasp_anchor_transform()

        self.iter_counter = 0

    def _grasp_anchor_transform(self):
        return torch.cat([
            (self.FCGN_anchors[:, 0:1] + self.FCGN_anchors[:, 2:3]) / 2,
            (self.FCGN_anchors[:, 1:2] + self.FCGN_anchors[:, 3:4]) / 2,
            self.FCGN_anchors[:, 2:3] - self.FCGN_anchors[:, 0:1] + 1,
            self.FCGN_anchors[:, 3:4] - self.FCGN_anchors[:, 1:2] + 1,
            self.FCGN_anchors[:, 4:5]
        ], dim=1)

    def forward(self, data_batch):
        x = data_batch[0]
        im_info = data_batch[1]
        gt_boxes = data_batch[2]
        num_boxes = data_batch[3]

        if self.training:
            self.iter_counter += 1

        # features
        x = self.FeatExt(x)
        pred = self.FCGN_classifier(x)
        loc, conf = pred
        self.batch_size = loc.size(0)

        all_anchors = self._generate_anchors(conf.size(1), conf.size(2))
        all_anchors = all_anchors.type_as(gt_boxes)
        all_anchors = all_anchors.expand(self.batch_size, all_anchors.size(1),all_anchors.size(2))

        loc = loc.contiguous().view(loc.size(0), -1, 5)
        conf = conf.contiguous().view(conf.size(0), -1, 2)
        prob = F.softmax(conf, 2)

        bbox_loss = 0
        cls_loss = 0
        conf_label = None
        if self.training:
            # inside weights indicate which bounding box should be regressed
            # outside weidhts indicate two things:
            # 1. Which bounding box should contribute for classification loss,
            # 2. Balance cls loss and bbox loss
            gt_xywhc = points2labels(gt_boxes)
            loc_label, conf_label, iw, ow = self.FCGN_proposal_target(conf, gt_xywhc, all_anchors)

            keep = conf_label.view(-1).ne(-1).nonzero().view(-1)
            conf = torch.index_select(conf.view(-1, 2), 0, keep.data)
            conf_label = torch.index_select(conf_label.view(-1), 0, keep.data)
            cls_loss = F.cross_entropy(conf, conf_label)

            bbox_loss = _smooth_l1_loss(loc, loc_label, iw, ow, dim = [2,1])

        return loc, prob, bbox_loss , cls_loss, conf_label, all_anchors

    def _generate_anchors(self, feat_height, feat_width):
        shift_x = np.arange(0, feat_width) * self.FCGN_fs
        shift_y = np.arange(0, feat_height) * self.FCGN_fs
        shift_x, shift_y = np.meshgrid(shift_x, shift_y)
        shifts = torch.from_numpy(np.vstack((shift_x.ravel(), shift_y.ravel())).transpose())
        shifts = torch.cat([
            shifts,
            torch.zeros(shifts.size(0), 3).type_as(shifts)
        ], dim = 1)
        shifts = shifts.contiguous().float()

        A = self.FCGN_num_anchors
        K = shifts.size(0)

        # anchors = self.FCGN_anchors.view(1, A, 5) + shifts.view(1, K, 5).permute(1, 0, 2).contiguous()
        anchors = self.FCGN_anchors.view(1, A, 5).type_as(shifts) + shifts.view(K, 1, 5)
        anchors = anchors.view(1, K * A, 5)

        return anchors

    def create_architecture(self):
        self._init_weights()

    def _init_weights(self):
        weights_normal_init(self.FCGN_classifier.conf, 0.01, 0.)
        weights_normal_init(self.FCGN_classifier.loc, 0.001, 0.)


text:
# THIS SOURCE CODE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND, AND ITS
# AUTHOR AND THE JOURNAL OF MACHINE LEARNING RESEARCH (JMLR) AND JMLR’S
# PUBLISHERS AND DISTRIBUTORS, DISCLAIM ANY AND ALL WARRANTIES, INCLUDING BUT
# NOT LIMITED TO ANY IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE, AND ANY WARRANTIES OR NON INFRINGEMENT. THE USER ASSUMES
# ALL LIABILITY AND RESPONSIBILITY FOR USE OF THIS SOURCE CODE, AND NEITHER THE
# AUTHOR NOR JMLR, NOR JMLR’S PUBLISHERS AND DISTRIBUTORS, WILL BE LIABLE FOR
# DAMAGES OF ANY KIND RESULTING FROM ITS USE. Without limiting the generality
# of the foregoing, neither the author, nor JMLR, nor JMLR’s publishers and
# distributors, warrant that the Source Code will be error-free, will operate
# without interruption, or will meet the needs of the user.

import numpy as np
import pandas as pd

COLUMNS = np.array(['TIMESTAMP_START', 'MONTH', 'SW_IN_F', 'TA_F', 'NEE_VUT_USTAR50', 'VPD_F', 'H_F_MDS', 'LE_F_MDS'])
VAR_NAMES = np.array(['DATETIME', 'MONTH', 'Rg', 'T', 'NEE', 'VPD', 'H', 'LE'])

def parse_date(x):
    date_str = x['TIMESTAMP_START'].astype(str)
    x['YEAR'] = int(date_str[:4])
    x['MONTH'] = int(date_str[4:6])
    x['DATE'] = int(date_str[:8])
    return x

def process_file(df, save_file, start=201404010000, end=201503312330):
    # Only keep observations from given time period
    df = df[(df.TIMESTAMP_START >= start) & (df.TIMESTAMP_START <= end)]
    df = df.apply(lambda x: parse_date(x), axis=1)

    columns = COLUMNS
    var_names = VAR_NAMES

    # Compute the daily maximum potential downward shortwave radiation
    df = df.merge(df[['DATE', 'SW_IN_POT']].groupby('DATE').max(), left_on='DATE', right_on='DATE', suffixes=('', '_MAX'))

    # Add noise to radiation when the potential is 0
    df.loc[df.SW_IN_POT == 0, 'SW_IN_F'] = np.random.normal(0, 0.000001, sum(df.SW_IN_POT == 0))

    # Filter out observations where potential radiation less than four fifths of the daily maximum
    df = df[df.SW_IN_POT >= (4 / 5) * df.SW_IN_POT_MAX]

    # Filter out gapfilled observations
    df = df[(df.NEE_VUT_USTAR50_QC == 0) & ((df.H_F_MDS_QC == 0) | (df.H_F_MDS_QC == -9999)) & ((df.LE_F_MDS_QC == 0) | (df.LE_F_MDS_QC == -9999))]

    # Filter columns
    df = df[columns]

    # Rename columns
    df = df.rename(columns=dict(zip(columns, var_names)))

    # Save to file without row names and without date
    df.drop(columns=['DATETIME']).to_csv(save_file, index=False)

    return df

df = pd.read_csv('../../FLX_FI-Hyy_FLUXNET2015_FULLSET_HH_1996-2018_beta-3.csv') # Path to file with full FLUXNET database.

process_file(df, './FI_HYY_Preprocessed_2014.csv')
process_file(df, './FI_HYY_Preprocessed_2013-2015.csv', start=201301010000, end=201512312330)

max_stars_repo_name: edahelsinki/ICSD

max_issues_repo_name: edahelsinki/ICSD

max_forks_repo_name: edahelsinki/ICSD


text:
import pytest
import hashlib
import json
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn import datasets, linear_model
from testbook import testbook

V_HASH = 'lkfidhzwpt_'
DEBUG = True

@pytest.fixture(scope='module')
def tb():
  with testbook('hw3.ipynb', execute=True, timeout=120) as tb:
    yield tb

def verify_exists(tb, name, num_test_case):
    try:
        val = tb.ref(name)
    except Exception as e:
        if DEBUG:
            print(type(e))
            print(e.args)
            print(e)
        assert False, f'STEP {num_test_case}: {name} does not exist. Ensure that {name} is defined.'

    return val

def get_array_size(tb, name):
    tb.inject(f"""
        {V_HASH}val_size = np.size({name},0)
    """)

    exec(f'global array_size; array_size = tb.ref("{V_HASH}val_size")')
    return array_size

def get_array_dim_count(tb, name):
    tb.inject(f"""
        {V_HASH}val_size = {name}.ndim
    """)
    exec(f'global array_size; array_size = tb.ref("{V_HASH}val_size")')
    return array_size

def verify_array_size(tb, name, size, num_test_case):
    array_size = get_array_size(tb, name)
    assert array_size == size, f'STEP {num_test_case}: {name} is the wrong size. {name} should be {size} observations. Instead, it is {array_size} observations.'
    return array_size

def print_hash(tb, name):
    print(hashlib.md5(str(name).encode('utf-8')).hexdigest())

def verify_hash(tb, name, hash_string, num_test_case, message = None):
    if message:
        assert hashlib.md5(str(name).encode('utf-8')).hexdigest() == hash_string, f'STEP {num_test_case}: {message}.'
    else:
        assert hashlib.md5(str(name).encode('utf-8')).hexdigest() == hash_string, f'STEP {num_test_case}: {name} has the wrong value.'

def test_step_1(tb):
    try:
        complete = None
        complete = tb.ref('STEP_1_COMPLETE')
    except:
        # STEP_1_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 1: not complete.'

def test_step_2(tb):
    try:
        complete = None
        complete = tb.ref('STEP_2_COMPLETE')
    except:
        # STEP_2_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 2: not complete.'

def test_step_3(tb):
    try:
        complete = None
        complete = tb.ref('STEP_3_COMPLETE')
    except:
        # STEP_3_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 3: not complete.'

def test_step_4(tb):
    try:
        complete = None
        complete = tb.ref('STEP_4_COMPLETE')
    except:
        # STEP_4_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 4: not complete.'

def test_step_5(tb):
    try:
        complete = None
        complete = tb.ref('STEP_5_COMPLETE')
    except:
        # STEP_5_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 5: not complete.'

def test_step_6(tb):
    try:
        complete = None
        complete = tb.ref('STEP_6_COMPLETE')
    except:
        # STEP_6_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 6: not complete.'

def test_step_7(tb):
    try:
        complete = None
        complete = tb.ref('STEP_7_COMPLETE')
    except:
        # STEP_7_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 7: not complete.'

def test_step_8(tb):
    try:
        complete = None
        complete = tb.ref('STEP_8_COMPLETE')
    except:
        # STEP_8_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 8: not complete.'

def test_step_9(tb):
    try:
        complete = None
        complete = tb.ref('STEP_9_COMPLETE')
    except:
        # STEP_9_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 9: not complete.'

def test_step_10(tb):
    try:
        complete = None
        complete = tb.ref('STEP_10_COMPLETE')
    except:
        # STEP_10_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 10: not complete.'

def test_step_11(tb):
    try:
        complete = None
        complete = tb.ref('STEP_11_COMPLETE')
    except:
        # STEP_11_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 11: not complete.'

def test_step_12(tb):
    try:
        complete = None
        complete = tb.ref('STEP_12_COMPLETE')
    except:
        # STEP_12_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 12: not complete.'

def test_step_13(tb):
    try:
        complete = None
        complete = tb.ref('STEP_13_COMPLETE')
    except:
        # STEP_13_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 13: not complete.'

def test_step_14(tb):
    try:
        complete = None
        complete = tb.ref('STEP_14_COMPLETE')
    except:
        # STEP_14_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 14: not complete.'

def test_step_15(tb):
    try:
        complete = None
        complete = tb.ref('STEP_15_COMPLETE')
    except:
        # STEP_16_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 15: not complete.'

def test_step_16(tb):
    try:
        complete = None
        complete = tb.ref('STEP_16_COMPLETE')
    except:
        # STEP_16_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 16: not complete.'

def test_step_17(tb):
    try:
        complete = None
        complete = tb.ref('STEP_17_COMPLETE')
    except:
        # STEP_17_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 17: not complete.'

def test_step_18(tb):
    try:
        complete = None
        complete = tb.ref('STEP_18_COMPLETE')
    except:
        # STEP_18_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        assert complete, 'STEP 18: not complete.'

def test_step_19(tb):
    try:
        complete = None
        complete = tb.ref('STEP_19_COMPLETE')
    except:
        # STEP_19_COMPLETE constant has been removed, set to true
        complete = True
    finally:
        pass # STEP 19 is optional


text:
import sys
import typing

import numpy as np

def solve(a: np.ndarray) -> typing.NoReturn:
  h, w = a.shape
  sy = a.sum(axis=0)
  sx = a.sum(axis=1)
  b = sy[:, None] + sx[None, :] - a
  for x in b:
    print(' '.join(map(str, x.tolist())))

def main() -> typing.NoReturn:
  h, w = map(int, input().split())
  a = np.array(
    sys.stdin.read().split(),
    dtype=np.int64,
  ).reshape(h, w)
  solve(a)

main()

max_stars_repo_name: kagemeka/atcoder-submissions

max_issues_repo_name: kagemeka/atcoder-submissions

max_forks_repo_name: kagemeka/atcoder-submissions


text:
import os
import cv2
import numpy as np

data_path = '/home/lfx/Downloads/dtd/images'
images = []

for subdir, dirs, files in os.walk(data_path):
    for file in files:
        #print(os.path.join(subdir, file))
        image = cv2.imread(os.path.join(subdir, file))
        output_path = os.path.join('/home/lfx/Downloads', 'New Folder', file)

        print(output_path)
        cv2.imwrite(output_path, image)

max_stars_repo_name: MagazzuGaetano/Face-Detector

max_issues_repo_name: MagazzuGaetano/Face-Detector

max_forks_repo_name: MagazzuGaetano/Face-Detector


text:
import numpy as np

pulse_len = 1000

config = {
    "version": 1,
    "controllers": {
        "con1": {
            "type": "opx1",
            "analog_outputs": {
                1: {"offset": +0.0},
            },
            "analog_inputs": {
                1: {"offset": +0.0},
            },
        }
    },
    "elements": {
        "qe1": {
            "singleInput": {"port": ("con1", 1)},
            "outputs": {"output1": ("con1", 1)},
            "intermediate_frequency": 100e6,
            "operations": {
                "readoutOp": "readoutPulse",
                "readoutOp2": "readoutPulse2",
            },
            "time_of_flight": 180,
            "smearing": 0,
        },
    },
    "pulses": {
        "readoutPulse": {
            "operation": "measure",
            "length": pulse_len,
            "waveforms": {"single": "ramp_wf"},
            "digital_marker": "ON",
            "integration_weights": {"x": "xWeights", "y": "yWeights"},
        },
        "readoutPulse2": {
            "operation": "measure",
            "length": 2 * pulse_len,
            "waveforms": {"single": "ramp_wf2"},
            "digital_marker": "ON",
            "integration_weights": {"x": "xWeights2", "y": "yWeights"},
        },
    },
    "waveforms": {
        "const_wf": {"type": "constant", "sample": 0.2},
        "ramp_wf": {
            "type": "arbitrary",
            "samples": np.linspace(0, -0.5, pulse_len).tolist(),
        },
        "ramp_wf2": {
            "type": "arbitrary",
            "samples": np.linspace(0, -0.5, pulse_len).tolist()
            + np.linspace(0, -0.5, pulse_len).tolist(),
        },
    },
    "digital_waveforms": {
        "ON": {"samples": [(1, 0)]},
    },
    "integration_weights": {
        "xWeights": {
            "cosine": [1.0] * (pulse_len // 4),
            "sine": [0.0] * (pulse_len // 4),
        },
        "xWeights2": {
            "cosine": [1.0] * (2 * pulse_len // 4),
            "sine": [0.0] * (2 * pulse_len // 4),
        },
        "yWeights": {
            "cosine": [0.0] * (pulse_len // 4),
            "sine": [1.0] * (pulse_len // 4),
        },
    },
}

max_stars_repo_name: qua-platform/qua-libs

max_issues_repo_name: qua-platform/qua-libs

max_forks_repo_name: qua-platform/qua-libs


text:
# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the LICENSE file in
# the root directory of this source tree. An additional grant of patent rights
# can be found in the PATENTS file in the same directory.

import math
import numpy as np
import torch

from fairseq import search, utils
from fairseq.models import FairseqIncrementalDecoder

class SequenceGenerator(object):
    def __init__(
        self, models, task
    ):
        self.args = task.args
        self.models = models
        self.tokenizer = task.tokenizer

    def cuda(self):
        for model in self.models:
            model.cuda()
        return self

    def generate_batched_itr(
        self, data_itr, cuda=False
    ):
        """Iterate over a batched dataset and yield individual translations.
        Args:
            maxlen_a/b: generate sequences of maximum length ax + b,
                where x is the source sentence length.
            cuda: use GPU for generation
            timer: StopwatchMeter for timing generations.
        """

        for sample in data_itr:
            s = utils.move_to_cuda(sample) if cuda else sample
            if 'net_input' not in s:
                continue
            input = s['net_input']
            targets = s["target"]
            origin_target = self.convert_to_tokens(targets)

            with torch.no_grad():
                tokens, ppl_probs = self.generate(
                    input
                )
            for i, id in enumerate(s['id'].data):
                yield id, self.merge_bert_tokens(tokens[i]), self.merge_bert_tokens(origin_target[i]), ppl_probs[i]/len(tokens[i])

    def generate(self, encoder_input):

        with torch.no_grad():

            model= self.models[0]
            # model.eval()
            encoder_out =  model.greedy_generater(**encoder_input, max_lens=self.args.max_tokens_generate)
            generate_ids = encoder_out[0]
            ppl_probs = encoder_out[1]
            tokens = self.convert_to_tokens(generate_ids)
            return tokens, ppl_probs.tolist()

    def convert_to_tokens(self, ids):

        def to_tokens(t):
            return self.tokenizer.convert_ids_to_tokens(t)

        eos_ids = self.tokenizer.eos()
        if not isinstance(ids, list):
            ids = ids.tolist()
        tokens = []
        for item in ids:
            if eos_ids in item:
                eos_index = item.index(eos_ids)
                item = item[:eos_index]
            tokens.append(to_tokens(item))
        return tokens

    def merge_bert_tokens(self, tokens):
        new_tokens = []
        new_tokens.append(tokens[0])
        for tok in tokens[1:]:
            if tok.startswith("##"):
                new_tokens[-1] += tok[2:]
            else:
                new_tokens.append(tok)
        return new_tokens

max_stars_repo_name: initc/bert-fairseq

max_issues_repo_name: initc/bert-fairseq

max_forks_repo_name: initc/bert-fairseq


text:
# https://www.hackerrank.com/challenges/np-inner-and-outer/problem

import numpy as np
a = np.array(input().split(), int)
b = np.array(input().split(), int)
print(np.inner(a,b), np.outer(a,b), sep='\n')


text:
import numpy as np
import random
from model import ActorNet, CriticNet
import torch
import torch.nn.functional as F
import torch.optim as optim

"""
The "A2C_Agent" class implements the agent learning to solve the Continuous Control environment problem using an Actor-Critic method (see report.md for more info).
"""

class A2C_Agent():
    """Interacts with and learns from the environment."""

    def __init__(self, state_size, action_size, device, seed, LR=5e-4, gamma=0.95, entropy_weight=0.02, actor_network_max_grad_norm = 5, critic_network_max_grad_norm = 5, nstepqlearning_size=5, gae_lambda = 1.0):
        """Initialize an Agent object.

        Params
        ======
            state_size (int): dimension of each state
            action_size (int): dimension of each action
            seed (int): random seed
            LR (float): learning rate
            GAMMA (float): factor used to discount future values
            entropy_weight (float): weight of the entropy value using with the entropy_loss
            actor_network_max_grad_norm (float): threshold value used in gradient clipping in the actor model
            critic_network_max_grad_norm (float): threshold value used in gradient clipping in the critic model
            nstepqlearning_size (int): the number of steps used for the N-step bootstrapping algorithm
            gae_lambda (float): lambda used in GAE algorithm to use as discount factor in getting a mixture of every available estimated N-step bootstrapping results (from 1-5).

        """
        self.state_size = state_size
        self.action_size = action_size
        self.entropy_weight = entropy_weight
        random.seed(seed)
        self.gamma=gamma
        self.actor_network_max_grad_norm = actor_network_max_grad_norm
        self.critic_network_max_grad_norm = critic_network_max_grad_norm
        self.nstepqlearning_size = nstepqlearning_size
        self.gae_lambda = gae_lambda
        self.device=device

        print("----Dumping agent hyperparameters---- ")
        print("LR: ", LR)
        print("gamma: ", gamma)
        print("actor_network_max_grad_norm: ", self.actor_network_max_grad_norm)
        print("critic_network_max_grad_norm: ", self.critic_network_max_grad_norm)
        print("nstepqlearning_size: ", self.nstepqlearning_size)
        print("gae_lambda: ", self.gae_lambda)
        print("entropy_weight: ", self.entropy_weight)
        print("------------------------------------- ")

        self.actor_net = ActorNet(state_size, action_size, device, seed).to(self.device)       # Theta
        self.critic_net = CriticNet(state_size, action_size, seed).to(self.device)     # Thetav
        self.actor_optimizer = optim.RMSprop(self.actor_net.parameters(), lr=LR)
        self.critic_optimizer = optim.RMSprop(self.critic_net.parameters(), lr=LR)

    def tensor(self, x):
        if isinstance(x, torch.Tensor):
            return x
        x = np.asarray(x, dtype=np.float32)
        x = torch.from_numpy(x).to(self.device)
        return x

    def act(self, state):
        """Returns actions for given state as per current policy.
        Params
        ======
            state (array_like): current state
        """

        state = torch.from_numpy(state).float().unsqueeze(0).to(self.device)
        self.actor_net.eval()
        self.critic_net.eval()
        ( actor_values, log_prob, entropy ) = self.actor_net(state)
        critic_values = self.critic_net(state)
        self.actor_net.train()
        self.critic_net.train()
        return actor_values, log_prob, entropy, critic_values

    def train_one_episode(self, env, brain_name):
        env_info = env.reset(train_mode=True)[brain_name]     # reset the environment
        num_agents = len(env_info.agents)

        states = env_info.vector_observations                  # get the current state (for each agent)
        episode_terminated = False
        scores = np.zeros(num_agents)                          # initialize the score (for each agent)

        while episode_terminated == False:
            l_states = []
            l_actions = []
            l_rewards = []           #np.zeros(( nstepqlearning_size, num_agents ))
            l_masks = []
            l_next_states = []
            l_values = []
            l_log_probs = []
            l_entropy = []

            nstep_memory_size = self.nstepqlearning_size
            for i in range(self.nstepqlearning_size):

                # Get a(t) according to actor policy
                (actions, log_prob, entropy, values) = self.act(states)
                actions = np.clip(actions, -1, 1)                  # Put all actions between -1 and 1. ( The last activation of the Actor is tanh, which puts the out values in this range,
                                                                   # but later we are sampling it which can produce values outside of this range)

                # Perform a(t) in all environments
                env_info = env.step(actions)[brain_name]           # send all actions to tne environment

                # get s(t+1), r(t) and wasLastAction(t)
                next_states = env_info.vector_observations         # get next state (for each agent)
                rewards = env_info.rewards                         # get reward (for each agent)
                dones = env_info.local_done                        # see if episode finished

                masks = 1 - np.asarray(dones, np.int)

                l_states.append(states)
                l_actions.append(actions)
                l_rewards.append( rewards )
                l_masks.append(masks)
                l_next_states.append(next_states)
                l_values.append( values )
                l_log_probs.append( log_prob )
                l_entropy.append(entropy)

                # update score
                scores += env_info.rewards                         # update the score (for each agent)

                states = next_states                               # roll over states to next time step
                if np.any(dones):                                  # exit loop if episode terminated
                    nstep_memory_size = i + 1
                    episode_terminated = True
                    break

            # get one prediction for the last estimated Q value
            (_, _, _, values) = self.act(states)
            l_values.append( values )    # Add to the list, GAE will use it

            advantages = self.tensor( torch.zeros((num_agents)) ).to(self.device)
            returns = values.reshape(( num_agents, )).to(self.device)     # last estimated Value ( of s(t+nstep_memory_size) )

            l_advantages = [None] * nstep_memory_size
            l_rets = [None] * nstep_memory_size
            l_masks = torch.tensor(np.array(l_masks)).to(self.device)
            l_rewards = torch.tensor(np.array(l_rewards)).to(self.device)

            for i in reversed(range(nstep_memory_size)):
                returns = l_rewards[i] + self.gamma * l_masks[i] * returns

                # Normal advantage calculation.
                #advantages = returns - l_values[i].detach().reshape((num_agents, ))

                # GAE
                td_error = l_rewards[i] + self.gamma * l_masks[i] * l_values[i+1] - l_values[i]
                advantages = advantages * self.gae_lambda * self.gamma * l_masks[i] + td_error
                # GAE end

                l_advantages[i] = advantages.detach()
                l_rets[i] = returns.detach()

            # bring log_probs list to Tensor with shape [ num_agents,nstepqlearning_size ]
            logprobs = torch.cat(l_log_probs).squeeze()
            logprobs = logprobs.reshape(( nstep_memory_size*num_agents )).to(self.device)

            ents = torch.cat(l_entropy).squeeze()
            advantages_tensor = torch.cat(l_advantages, dim=0).squeeze().detach().to(self.device)

            policy_loss = -(logprobs * advantages_tensor).mean()

            # entropy: currently it's constant but I left it here, to make it possible to use different distribution parameters during the training process
            entropy_loss = ents.mean()

            # ==== train Critic ====
            self.critic_optimizer.zero_grad()
            l_rets = torch.cat(l_rets, dim=0).squeeze().detach().to(self.device)
            l_values = torch.cat(l_values[:nstep_memory_size], dim=0).squeeze().to(self.device)
            v = 0.5 * ( l_rets - l_values )
            value_loss = v.pow(2).mean()
            value_loss.backward()
            torch.nn.utils.clip_grad_norm_(self.critic_net.parameters(), self.critic_network_max_grad_norm)
            self.critic_optimizer.step()

            # ==== train Actor ====
            self.actor_optimizer.zero_grad()
            # Add entropy term to the loss function to encourage having evenly distributed actions
            (policy_loss - self.entropy_weight * entropy_loss).backward()
            torch.nn.utils.clip_grad_norm_(self.actor_net.parameters(), self.actor_network_max_grad_norm)
            self.actor_optimizer.step()

        return scores


text:
import pandas as pd
import matplotlib.pyplot as plt
from numpy import arange

def pre_processamento_dpa(nome_arquivo: str) -> pd.DataFrame:
    """Função para fazer o pre-processamento de dados da dano potencial
       associado (DPA) para modelagem de machine learning.
       :param nome_arquivo: nome do arquivo a ser carregado pelo pandas
       :type nome_arquivo: str
    """

    # Carregamento dos dados necessários para classificação (DPA)
    dados_dpa: pd.DataFrame = pd.read_csv(
        nome_arquivo.lower(),
        sep=';',
        encoding='latin-1',
        decimal=',',
        thousands='.',
        usecols=
        [
            'Volume de projeto licenciado do Reservatório (m³)',
            'Existência de população a jusante',
            'Impacto ambiental',
            'Impacto sócio-econômico',
            'Dano Potencial Associado - DPA'
        ])

    #dados_dpa['nome_coluna'].isnull().sum()
    #dados_dpa.value_counts('nome_coluna')

    # Tratamento dos dados
    dados_dpa.dropna(how='all',
                     subset=[
                         'Volume de projeto licenciado do Reservatório (m³)',
                         'Existência de população a jusante',
                         'Impacto ambiental',
                         'Impacto sócio-econômico'],
                     inplace=True)

    # Convertendo as variáveis do DPA para label encoder
    dados_dpa.loc[dados_dpa[
        'Volume de projeto licenciado do Reservatório (m³)'] <= 5000000, \
            'Volume de projeto licenciado do Reservatório (m³)'] = 1
    dados_dpa.loc[(dados_dpa[
        'Volume de projeto licenciado do Reservatório (m³)'] > 5000000) & \
        (dados_dpa['Volume de projeto licenciado do Reservatório (m³)'] \
        <= 75000000), 'Volume de projeto licenciado do Reservatório (m³)'] = 2
    dados_dpa.loc[(dados_dpa[
        'Volume de projeto licenciado do Reservatório (m³)'] > 75000000) & \
            (dados_dpa['Volume de projeto licenciado do Reservatório (m³)'] \
            <= 200000000),
            'Volume de projeto licenciado do Reservatório (m³)'] = 3
    dados_dpa.loc[
        dados_dpa['Volume de projeto licenciado do Reservatório (m³)'] > \
        200000000, 'Volume de projeto licenciado do Reservatório (m³)'] = 4

    dados_dpa.replace(
        {
         'Existência de população a jusante':
         {'Inexistente (Não existem pessoas permanentes/residentes ou ' +
          'temporárias/transitando na área afetada a jusante da barragem)': 0,
          'Pouco Frequente (Não existem pessoas ocupando permanentemente a ' +
          'área afetada a jusante da barragem, mas existe estrada vicinal ' +
          'de uso local)': 4,
          'Frequente (Não existem pessoas ocupando permanentemente a área ' +
          'afetada a jusante da barragem, mas existe rodovia municipal ou ' +
          'estadual ou federal ou outro local e/ou empreendimento de ' +
          'permanência eventual de pessoas que poderão ser atingidas)': 8,
          'Existente (Existem pessoas ocupando permanentemente a área afetada'+
          ' a jusante da barragem, portanto, vidas humanas poderão ser ' +
          'atingidas)': 12,
          'Indefinido': 12
         },
         'Impacto ambiental':
         {'Insignificante (Área afetada a jusante da barragem encontra-se ' +
          'totalmente descaracterizada de suas condições naturais e a ' +
          'estrutura armazena apenas resíduos Classe II B - Inertes, segundo '
          'a NBR 10004/2004 da ABNT)': 3,
          'Pouco Significativo (Área afetada a jusante da barragem não ' +
          'apresenta área de interesse ambiental relevante ou áreas ' +
          'protegidas em legislação específica (excluidas APPs) e armazena ' +
          'apenas resíduos Classe II B - Inertes, segundo a NBR 10004/2004 ' +
          'da ABNT)': 3,
          'Significativo (Área afetada a jusante da barragem apresenta área ' +
          'de interesse ambiental relevante ou áreas protegidas em ' +
          'legislação específica (excluidas APPs)) e armazena apenas ' +
          'resíduos Classe II B - Inertes, segundo a NBR 10004/2004 ' +
          'da ABNT)': 3,
          'Muito Significativo (Barragem armazena rejeitos ou resíduos ' +
          'sólidos classificados na Classe II A - Não Inertes, segundo a NBR '+
          '10004/2004)': 5,
          'Muito Significativo Agravado (Barragem armazena rejeitos ou ' +
          'resíduos sólidos classificados na Classe I - Perigosos segundo a' +
          ' NBR 10004/2004)': 5
         },
         'Impacto sócio-econômico':
         {'Inexistente (Não existem quaisquer instalações na área afetada a ' +
          'jusante da barragem)': 0,
          'BAIXO (Existe pequena concentração de instalações residenciais, ' +
          'agrícolas, industriais ou de infraestrutura de relevância ' +
          'sócio-econômico-cultural na área afetada a jusante da barragem)': 4,
          'MÉDIO (Existe moderada concentração de instalações residenciais, ' +
          'agrícolas, industriais ou de infraestrutura de relevância ' +
          'sócio-econômico-cultural na área afetada a jusante da barragem)': 8,
          'ALTO (Existe alta concentração de instalações residenciais, ' +
          'agrícolas, industriais ou de infraestrutura de relevância ' +
          'sócio-econômico-cultural na área afetada a jusante da barragem)': 8,
          'Indefinido': 8
         },
         'Dano Potencial Associado - DPA':
         {'Não se aplica': 0,
          'Baixa': 1,
          'Média': 2,
          'Alta': 3
         }
        }, inplace=True)

    # Criação da coluna somatório DPA
    dados_dpa['DPA'] = dados_dpa['Volume de projeto licenciado do ' +
    'Reservatório (m³)'] + dados_dpa['Existência de população a jusante'] \
    + dados_dpa['Impacto ambiental'] + dados_dpa['Impacto sócio-econômico']
    return dados_dpa

if __name__ == '__main__':
    teste: pd.DataFrame = pre_processamento_dpa('barragens.csv')

    #Visualização da contagem de classes do DPA
    plt.figure(figsize=(15, 5))
    nome_colunas: list = ['Não se aplica', 'Baixa', 'Média', 'Alta']
    plt.bar(nome_colunas,
            teste.value_counts('Dano Potencial Associado - DPA'),
            color=plt.cm.Set2(arange(len(nome_colunas))))
    plt.title('Contagem das classes do Dano Potencial Associado (DPA)',
              fontdict={'fontsize': 20, 'fontweight':'bold'})
    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)
    plt.grid(axis='y')
    plt.show();

reason: from numpy