text:
from datetime import datetime

import geopandas as gpd
import pandas as pd
import numpy as np
from copy import deepcopy

import STARTHER
import lastnedvegnett
import skrivdataframe
import nvdbapiv3
import nvdbgeotricks

t0 = datetime.now()

# Teller motorveg av klasse A og motortrafikkveg

# mittfilter = lastnedvegnett.kostraFagdataFilter( mittfilter={} )
mittfilter = { 'tidspunkt': '2021-12-15'}
# mittfilter['adskiltelop'] = 'med,nei'
# mittfilter['sideanlegg'] = 'false'

sok = nvdbapiv3.nvdbFagdata( 595  )
sok.filter( mittfilter )
myGdf = nvdbgeotricks.records2gdf( sok.to_records( ) )

myGdf = myGdf[ myGdf['trafikantgruppe'] == 'K']    # Kun kjørende
myGdf = myGdf[ myGdf['adskilte_lop'] != 'Mot' ]     # Adskilte løp = Med,Nei
myGdf = myGdf[ ~myGdf['vref'].str.contains( 'SD') ] # Fjerner sideanlegg
myGdf = myGdf[ ~myGdf['vref'].str.contains( 'KD') ] # Fjerner kryssdeler
myGdf = myGdf[ myGdf['typeVeg'] != 'Rampe' ] # Fjerner ramper

# # For debugging
# lengde = myGdf.groupby( ['fylke', 'vegkategori', 'nummer' ]).agg( {'segmentlengde' : 'sum' } ).reset_index()
# lengde['Veg'] = 'FV' + lengde['nummer'].astype(str)
# lengde['Lengde (m)'] = lengde['segmentlengde']
# lengde = lengde[[ 'fylke', 'Veg', 'Lengde (m)']]

telling = myGdf.groupby( ['fylke', 'Motorvegtype' ]).agg( { 'segmentlengde' : 'sum'} ).astype(int).reset_index()
telling.rename( columns={ 'segmentlengde' : 'Lengde (m)' }, inplace=True)
lengde = myGdf.groupby( [ 'Motorvegtype' ]).agg( { 'segmentlengde' : 'sum'} ).astype(int).reset_index()
print( "Motorveg i i fjor", lengde)

metadata = { 'tidspunkt' : '2021-12-15', 'Kryssdeler' : 'Kryssdeler filtrert vekk', 'Sideanlegg' : 'Sideanlegg filtrert vekk', 'Ramper' : 'Ramper filtrert vekk', 'trafikantgruppe' : 'K' }
skrivdataframe.skrivdf2xlsx( telling, '../kostraleveranse2021/EKSTRARAPPORT motorveger.xlsx', sheet_name='Motorveger', metadata=metadata )

tidsbruk = datetime.now() - t0

# Debugger og sammenligner med i dag
myGdf.to_file( 'debugmotorveg.gpkg', layer='motorveg-2021-12-15', driver='GPKG')

sok = nvdbapiv3.nvdbFagdata( 595  )
myGdf2 = nvdbgeotricks.records2gdf( sok.to_records( ) )
myGdf2 = myGdf2[ myGdf2['trafikantgruppe'] == 'K']    # Kun kjørende
myGdf2 = myGdf2[ myGdf2['adskilte_lop'] != 'Mot' ]     # Adskilte løp = Med,Nei
myGdf2 = myGdf2[ ~myGdf2['vref'].str.contains( 'SD') ] # Fjerner sideanlegg
myGdf2 = myGdf2[ ~myGdf2['vref'].str.contains( 'KD') ] # Fjerner kryssdeler
myGdf2 = myGdf2[ myGdf2['typeVeg'] != 'Rampe' ] # Fjerner ramper

telling2 = myGdf2.groupby( ['fylke', 'Motorvegtype' ]).agg( { 'segmentlengde' : 'sum'} ).astype(int).reset_index()
lengde2 = myGdf2.groupby( [ 'Motorvegtype' ]).agg( { 'segmentlengde' : 'sum'} ).astype(int).reset_index()
print( "Motorveg i dag", lengde2)

myGdf2.to_file( 'debugmotorveg.gpkg', layer='motorveg-2022-05-03', driver='GPKG')


text:
"""The WaveBlocks Project

This file contains code for the representation of potentials for a single component.
These potential are of course scalar ones.

@author: R. Bourquin
@copyright: Copyright (C) 2010, 2011 R. Bourquin
@license: Modified BSD License
"""

import sympy
import numpy

from MatrixPotential import MatrixPotential

class MatrixPotential1S(MatrixPotential):
    r"""
    This class represents a scalar potential :math:`V\left(x\right)`. The potential is given as an
    analytical :math:`1 \times 1` matrix expression. Some symbolic calculations with
    the potential are supported. For example calculation of eigenvalues and
    exponentials and numerical evaluation. Further, there are methods for
    splitting the potential into a Taylor expansion and for basis transformations
    between canonical and eigenbasis.
    """

    def __init__(self, expression, variables):
        r"""
        Create a new ``MatrixPotential1S`` instance for a given potential matrix :math:`V\left(x\right)`.

        :param expression: An expression representing the potential.
        """
        #: The variable :math:`x` that represents position space.
        self.x = variables[0]
        #: The matrix of the potential :math:`V\left(x\right)`.
        self.potential = expression
        # Unpack single matrix entry
        self.potential = self.potential[0,0]
        self.exponential = None

        self.number_components = 1

        # prepare the function in every potential matrix cell for numerical evaluation
        self.potential_n = sympy.vectorize(0)(sympy.lambdify(self.x, self.potential, "numpy"))

        # Symbolic and numerical eigenvalues and eigenvectors
        self.eigenvalues_s = None
        self.eigenvalues_n = None
        self.eigenvectors_s = None
        self.eigenvectors_n = None

        self.taylor_eigen_s = None
        self.taylor_eigen_n = None

        self.remainder_eigen_s = None
        self.remainder_eigen_n = None

    def __str__(self):
        r"""
        Put the number of components and the analytical expression (the matrix) into a printable string.
        """
        return """Scalar potential given by the expression: V(x) = \n""" + str(self.potential)

    def get_number_components(self):
        r"""
        :return: The number :math:`N` of components the potential supports. In the one dimensional case, it's just 1.
        """
        return 1

    def evaluate_at(self, nodes, component=0, as_matrix=False):
        r"""
        Evaluate the potential matrix elementwise at some given grid nodes :math:`\gamma`.

        :param nodes: The grid nodes :math:`\gamma` we want to evaluate the potential at.
        :param component: The component :math:`V_{i,j}` that gets evaluated or 'None' to evaluate all.
        :param as_matrix: Dummy parameter which has no effect here.
        :return: A list with the single entry evaluated at the nodes.
        """
        return tuple([ numpy.array(self.potential_n(nodes), dtype=numpy.floating) ])

    def calculate_eigenvalues(self):
        r"""
        Calculate the eigenvalue :math:`\lambda_0\left(x\right)` of the potential :math:`V\left(x\right)`.
        In the scalar case this is just the matrix entry :math:`V_{0,0}`.

        .. note:: This function is idempotent and the eigenvalues are memoized for later reuse.
        """
        if self.eigenvalues_s is None:
            self.eigenvalues_s = self.potential
            self.eigenvalues_n = sympy.vectorize(0)(sympy.lambdify(self.x, self.potential, "numpy"))

    def evaluate_eigenvalues_at(self, nodes, component=None, as_matrix=False):
        r"""
        Evaluate the eigenvalue :math:`\lambda_0\left(x\right)` at some grid nodes :math:`\gamma`.

        :param nodes: The grid nodes :math:`\gamma` we want to evaluate the eigenvalue at.
        :param diagonal_component: Dummy parameter that has no effect here.
        :param as_matrix: Dummy parameter which has no effect here.
        :return: A list with the single eigenvalue evaluated at the nodes.
        """
        self.calculate_eigenvalues()

        return tuple([ numpy.array(self.eigenvalues_n(nodes)) ])

    def calculate_eigenvectors(self):
        r"""
        Calculate the eigenvector :math:`nu_0\left(x\right)` of the potential :math:`V\left(x\right)`.
        In the scalar case this is just the value :math:`1`.

        .. note:: This function is idempotent and the eigenvectors are memoized for later reuse.
        """
        if self.eigenvectors_s is None:
            self.eigenvectors_s = sympy.Matrix([[1]])
            self.eigenvectors_n = sympy.vectorize(0)(sympy.lambdify(self.x, 1, "numpy"))

    def evaluate_eigenvectors_at(self, nodes):
        r"""
        Evaluate the eigenvector :math:`nu_0\left(x\right)` at some grid nodes :math:`\gamma`.

        :param nodes: The grid nodes :math:`\gamma` we want to evaluate the eigenvector at.
        :return: A list with the eigenvector evaluated at the given nodes.
        """
        self.calculate_eigenvectors()

        return tuple([ numpy.ones((1, len(nodes)), dtype=numpy.floating) ])

    def project_to_eigen(self, nodes, values, basis=None):
        r"""
        Project a given vector from the canonical basis to the eigenbasis of the potential.

        :param nodes: The grid nodes :math:`\gamma` for the pointwise transformation.
        :param values: The list of vectors :math:`\varphi_i` containing the values we want to transform.
        :param basis: A list of basisvectors :math:`nu_i`. Allows to use this function for external data, similar to a static function.
        :return: This method does nothing and returns the values.
        """
        return [ values[0].copy() ]

    def project_to_canonical(self, nodes, values, basis=None):
        r"""
        Project a given vector from the potential's eigenbasis to the canonical basis.

        :param nodes: The grid nodes :math:`\gamma` for the pointwise transformation.
        :param values: The list of vectors :math:`\varphi_i` containing the values we want to transform.
        :param basis: A list of basis vectors :math:`nu_i`. Allows to use this function for external data, similar to a static function.
        :return: This method does nothing and returns the values.
        """
        return [ values[0].copy() ]

    def calculate_exponential(self, factor=1):
        r"""
        Calculate the matrix exponential :math:`E = \exp\left(\alpha M\right)`. In this case
        the matrix is of size :math:`1 \times 1` thus the exponential simplifies to the scalar exponential function.

        :param factor: A prefactor :math:`\alpha` in the exponential.

        .. note:: This function is idempotent.
        """
        if self.exponential is None:
            self.exponential = sympy.exp(factor*self.potential)

    def evaluate_exponential_at(self, nodes):
        r"""
        Evaluate the exponential of the potential matrix :math:`V` at some grid nodes :math:`\gamma`.

        :param nodes: The grid nodes :math:`\gamma` we want to evaluate the exponential at.
        :return: The numerical approximation of the matrix exponential at the given grid nodes.
        """
        # Hack for older sympy versions, see recent issue:
        # http://www.mail-archive.com/sympy@googlegroups.com/msg05137.html
        lookup = {"I" : 1j}

        # prepare the function of every potential matrix exponential cell for numerical evaluation
        self.expfunctions = sympy.vectorize(0)(sympy.lambdify(self.x, self.exponential, (lookup, "numpy")))

        return tuple([ numpy.array(self.expfunctions(nodes)) ])

    def calculate_jacobian(self):
        r"""
        Calculate the jacobian matrix for the component :math:`V_{0,0}` of the potential.
        For potentials which depend only one variable :math:`x`, this equals the first derivative.
        """
        self.jacobian_s = sympy.diff(self.potential, self.x)
        self.jacobian_n = sympy.vectorize(0)(sympy.lambdify(self.x, self.jacobian_s, "numpy"))

    def evaluate_jacobian_at(self, nodes, component=None):
        r"""
        Evaluate the potential's jacobian at some grid nodes :math:`\gamma`.

        :param nodes: The grid nodes :math:`\gamma` the jacobian gets evaluated at.
        :param component: Dummy parameter that has no effect here.
        :return: The value of the potential's jacobian at the given nodes.
        """
        return tuple([ self.jacobian_n(nodes) ])

    def calculate_hessian(self):
        r"""
        Calculate the hessian matrix for component :math:`V_{0,0}` of the potential.
        For potentials which depend only one variable :math:`x`, this equals the second derivative.
        """
        self.hessian_s = sympy.diff(self.potential, self.x, 2)
        self.hessian_n = sympy.vectorize(0)(sympy.lambdify(self.x, self.hessian_s, "numpy"))

    def evaluate_hessian_at(self, nodes, component=None):
        r"""
        Evaluate the potential's hessian at some grid nodes :math:`\gamma`.

        :param nodes: The grid nodes :math:`\gamma` the hessian gets evaluated at.
        :param component: Dummy parameter that has no effect here.
        :return: The value of the potential's hessian at the given nodes.
        """
        return tuple([ self.hessian_n(nodes) ])

    def calculate_local_quadratic(self, diagonal_component=None):
        r"""
        Calculate the local quadratic approximation :math:`U` of the potential's eigenvalue :math:`\lambda`.

        :param diagonal_component: Dummy parameter that has no effect here.

        .. note:: This function is idempotent.
        """
        # Calculation already done at some earlier time?
        if self.taylor_eigen_s is not None:
            return

        self.calculate_eigenvalues()
        self.calculate_jacobian()
        self.calculate_hessian()

        self.taylor_eigen_s = [ (0, self.eigenvalues_s), (1, self.jacobian_s), (2, self.hessian_s) ]

        # Construct function to evaluate the approximation at point q at the given nodes
        assert(self.taylor_eigen_n is None)

        self.taylor_eigen_n = [
            (order, sympy.vectorize(0)(sympy.lambdify([self.x], f, "numpy")))
            for order, f in self.taylor_eigen_s
            ]

    def evaluate_local_quadratic_at(self, nodes, diagonal_component=None):
        r"""
        Numerically evaluate the local quadratic approximation :math:`U` of
        the potential's eigenvalue :math:`\lambda` at the given grid nodes :math:`\gamma`.
        This function is used for the homogeneous case.

        :param nodes: The grid nodes :math:`\gamma` we want to evaluate the quadratic approximation at.
        :return: An array containing the values of :math:`U` at the nodes :math:`\gamma`.
        """
        return tuple([ numpy.array(f(nodes), dtype=numpy.floating) for order, f in self.taylor_eigen_n ])

    def calculate_local_remainder(self, diagonal_component=None):
        r"""
        Calculate the non-quadratic remainder :math:`W` of the quadratic
        approximation :math:`U` of the potential's eigenvalue :math:`\lambda`.
        This function is used for the homogeneous case and takes into account
        the leading component :math:`\chi`.

        :param diagonal_component: Dummy parameter that has no effect here.

        .. note:: This function is idempotent.
        """
        # Calculation already done at some earlier time?
        if self.remainder_eigen_s is not None:
            return

        self.calculate_eigenvalues()

        f = self.eigenvalues_s

        # point where the taylor series is computed
        q = sympy.Symbol("q")

        p = f.subs(self.x, q)
        j = sympy.diff(f, self.x)
        j = j.subs(self.x, q)
        h = sympy.diff(f, self.x, 2)
        h = h.subs(self.x, q)

        quadratic =  p + j*(self.x-q) + sympy.Rational(1,2)*h*(self.x-q)**2

        # Symbolic expression for the taylor expansion remainder term
        self.remainder_eigen_s = self.potential - quadratic

        # Construct functions to evaluate the approximation at point q at the given nodes
        assert(self.remainder_eigen_n is None)

        self.remainder_eigen_n = sympy.vectorize(1)(sympy.lambdify([q, self.x], self.remainder_eigen_s, "numpy"))

    def evaluate_local_remainder_at(self, position, nodes, diagonal_component=None, component=None):
        r"""
        Numerically evaluate the non-quadratic remainder :math:`W` of the quadratic
        approximation :math:`U` of the potential's eigenvalue :math:`\lambda` at the given nodes :math:`\gamma`.
        This function is used for the homogeneous and the inhomogeneous case and
        just evaluates the remainder :math:`W`.

        :param position: The point :math:`q` where the Taylor series is computed.
        :param nodes: The grid nodes :math:`\gamma` we want to evaluate the potential at.
        :param component: Dummy parameter that has no effect here.
        :return: A list with a single entry consisting of an array containing the values of :math:`W` at the nodes :math:`\gamma`.
        """
        return tuple([ numpy.array(self.remainder_eigen_n(position, nodes), dtype=numpy.floating) ])

ext: py

max_stars_repo_name: WaveBlocks/WaveBlocks

max_issues_repo_name: WaveBlocks/WaveBlocks

max_forks_repo_name: WaveBlocks/WaveBlocks

reason: import numpy,import sympy


text:
from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM
import torch

import pandas as pd
from numpy import load
from numpy import savez_compressed
import numpy as np
from tqdm import tqdm

#BERT_MODEL = 'bert-base-multilingual-uncased'
BERT_MODEL = 'bert-base-cased'

def process_bert_tokenize(text):

    marked_text = "[CLS] " + text + " [SEP]"

    # Tokenize our sentence with the BERT tokenizer.
    tokenized_text = tokenizer.tokenize(marked_text)

    # Map the token strings to their vocabulary indeces.
    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
    #print("len(indexed_tokens):", len(indexed_tokens))

    # Mark each of the 22 tokens as belonging to sentence "1".
    segments_ids = [1] * len(tokenized_text)

    # Convert inputs to PyTorch tensors
    tokens_tensor = torch.tensor([indexed_tokens])
    segments_tensors = torch.tensor([segments_ids])

    # Put the model in "evaluation" mode, meaning feed-forward operation.
    #print("Evaluating the model")
    model.eval()

    # Predict hidden states features for each layer
    with torch.no_grad():
        encoded_layers, pooled_output = model(tokens_tensor, segments_tensors)

    # Concatenate the tensors for all layers. We use `stack` here to
    # create a new dimension in the tensor.
    token_embeddings = torch.stack(encoded_layers, dim=0).numpy()

    sentence_embeddings = pooled_output.numpy()

    #print("token_embeddings:", token_embeddings)
    #print("token_embeddings.shape:", token_embeddings.shape)
    #print("sentence_embeddings.shape:", sentence_embeddings.shape)

    return token_embeddings, sentence_embeddings

if __name__ == "__main__":
    # Load pre-trained model tokenizer (vocabulary)
    tokenizer = BertTokenizer.from_pretrained(BERT_MODEL)

    # Load pre-trained model (weights)
    print("Loading Model")
    model = BertModel.from_pretrained(BERT_MODEL)

    # Load dataset
    print("Loading data")
    split = "test"
    data = pd.read_csv("/Users/d22admin/USCGDrive/BeyondAssignment/small_dataset_useful/"
                       "df_tripets_with_teams_2_4k_"+split+"_useful.csv")

    print("Getting embeddings for the tweets")
    tweet_arr_vec = []
    for i in tqdm(range(0, len(list(data["text"])))):
        word_embeddings, sent_embeddings = process_bert_tokenize(list(data["text"])[i])
        word_num = word_embeddings.shape[2]
        tweet_arr_vec.append(np.squeeze(sent_embeddings))

    tweet_arr_np = np.array(tweet_arr_vec)

    print("Saving the embeddings in a numpy array")

    savez_compressed('/Users/d22admin/USCGDrive/BeyondAssignment/Deliverables/Embeddings/'
                     'text_emb_'+split+'_2k_bert_base_cased_useful.npz', tweet_arr_vec)

reason: import numpy,from numpy


text:
#!/usr/bin/env python

""" check_pseudo.py calculates energy for 7 alat points near SIESTA equilibrium to fine tune the delta-factor.
"""

import os
import sys
import uuid
import glob
import numpy as np
import shutil
import matplotlib.pyplot as plt
from generate import PGInputFile, PTInputFile
from get_energies import read_energy
from calc_delta import BM, read_ref_data, calcDelta, get_alats, get_volumes

def check_pseudo(settings, data_dir):
    """ Checks pseudopotential for delta factor calculation

    Arguments:
        settings {[type]} -- [description]
        data_dir {[type]} -- [description]
    """
    cwd = os.getcwd()
    element = settings.calc["element"]
    x, y = [], []
    os.chdir(data_dir)
    pseudo_file = glob.glob("*.psf")[0]

    for root, dirs, _ in os.walk(os.getcwd()):
        if "check" in root: continue
        for dir_i in dirs:
            try:
                alat = float(dir_i)
            except:
                continue
            energies = read_energy(element, alat)
            if energies is not None:
                x_i, y_i = energies
                x.append(x_i)
                y.append(y_i)
    x = np.array(x) / settings.calc["nat"]
    y = np.array(y) / settings.calc["nat"]
    p = np.polyfit(x, y, 2)

    # make 7 points out of existing data
    if len(x) == 7:
        x_p = x
        y_p = y
    else:
        x_p = get_volumes(7, settings.calc) / settings.calc["nat"]
        y_p = np.poly1d(p)(x_p)

    # get check directory
    if not os.path.exists("check"):
        os.makedirs("check")
    shutil.copy(pseudo_file, "check")
    os.chdir("check")

    # write original data
    np.savetxt("energies_original.txt", np.vstack((x, y)).T)

    vol, bulk_mod, bulk_deriv, _ = BM(np.vstack((x_p, y_p)).T)
    np.savetxt("energies_BM.txt", np.vstack((x_p, y_p)).T)

    our_data = np.core.records.fromrecords([(element, vol, bulk_mod, bulk_deriv), ], names=('element', 'V0', 'B0', 'BP'))
    ref_data = read_ref_data(os.path.join(cwd, "delta", "WIEN2k.txt"))
    ref_data_el = ref_data[ref_data['element'] == element]
    delta, delta_rel, _ = calcDelta(our_data, ref_data_el, useasymm=False)
    with open("BP.dat", "w") as f:
        f.write("Our data: {}\n".format(our_data))
        f.write("Reference data: {}\n".format(ref_data_el))
        f.write("Delta factor: {} {}\n".format(delta, delta_rel))

max_stars_repo_name: ansobolev/PseudoGenerator

max_issues_repo_name: ansobolev/PseudoGenerator

max_forks_repo_name: ansobolev/PseudoGenerator

reason: import numpy


text:
# TM35FIN.py
#
# convert WGS84 (or other) lat/lon to ETRS-TM35FIN
# and figure out their TM35 map tiles
#
# 2012-2014 Lauri Kangas
# License: ICCLEIYSIUYA (http://evvk.com/evvktvh.html)

import numpy

#      ['L4', 'L4L', 'L41', 'L41L', 'L411', 'L411R', 'L4113', 'L4131R', 'L4113H', 'L4113H3']

# list for using row letters as calculatable indices
rows200 = ['K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X']

# tile sizes indexed by printed map scale (e.g. 1:200k)
size = { 200 : (192000, 96000),
         100 : (96000, 48000),
          50 : (48000, 24000),
          25 : (24000, 12000),
          20 : (12000, 12000),
          10 : (6000, 6000),
           5 : (3000, 3000)}

# tile sizes with list index corresponding to tile level (0..9)
size_level = [ (192000, 96000),
               (96000, 96000),
               (96000, 48000),
               (48000, 48000),
               (48000, 24000),
               (24000, 24000),
               (24000, 12000),
               (12000, 12000),
               (6000, 6000),
               (3000, 3000)]

grid4 = (('1','2'),('3','4'))
grid8 = (('A','B'), ('C','D'),('E','F'),('G','H'))

K4pos = (500000,6570000) # position of lower right of K4 as reference
refpos = (K4pos[0]-5*size[200][0], K4pos[1]) # "K0" lower left

def latlon_to_xy(lat, lon, src='epsg:4326'):
    import pyproj
    # in:  lat,lon as numpy arrays (dd.dddd, WGS84 default)
    # out: x,y as numpy arrays (ETRS-TM35FIN)

    proj_latlon = pyproj.Proj(init=src) # default: WGS84
    proj_etrs = pyproj.Proj(init='epsg:3067') # ETRS-TM35FIN

    return pyproj.transform(proj_latlon, proj_etrs, lon, lat)

def xy_to_latlon(x, y, dst='epsg:4326'):
    # in:  x,y as numpy arrays (ETRS-TM35FIN)
    # out: lat,lon as numpy arrays (dd.ddd, WGS84 default)
    #      (might also be lon,lat...)
    import pyproj

    proj_latlon = pyproj.Proj(init=src) # default: WGS84
    proj_etrs = pyproj.Proj(init='epsg:3067') # ETRS-TM35FIN

    return pyproj.transform(proj_etrs, proj_latlon, x, y)

def tile_of_latlon(lat, lon, level=9, uniq=True, src='epsg:4326'):
    # in:  lat,lon as numpy arrays (dd.dddd, WGS84 default)
    # out: python list of TM35 map tiles

    x, y = latlon_to_xy(lat, lon, src)
    return tile_of_xy(x, y, level=level, uniq=uniq)

def tile_of_xy(x, y, level=9, uniq=True):
    # in:  x,y as numpy arrays (ETRS-TM35FIN)
    # out: python list of TM35 map tiles
    #
    # calculates full level 9 tile for every (x,y)
    # returns list of shorten()'ed names if level < 9,
    # non-unique (one for every (x,y)) if uniq=False

    Ndist = (y - refpos[1])
    Edist = (x - refpos[0])

    N200 = divmod(Ndist, size[200][1])
    E200 = divmod(Edist, size[200][0])

    N100 = divmod(N200[1], size[100][1])
    E100 = divmod(E200[1], size[100][0])

    N50 = divmod(N100[1], size[50][1])
    E50 = divmod(E100[1], size[50][0])

    N25 = divmod(N50[1], size[25][1])
    E25 = divmod(E50[1], size[25][0])

    N10 = divmod(N25[1], size[10][1])
    E10 = divmod(E25[1], size[10][0])

    N5 = divmod(N10[1], size[5][1])
    E5 = divmod(E10[1], size[5][0])

    # i've no idea anymore what this does
    tile_parts = numpy.array([[rows200[int(N)] for N in N200[0]],
        ["%d" % E for E in E200[0]],
        [grid4[int(E)][int(N)] for N,E in zip(N100[0],E100[0])],
        [grid4[int(E)][int(N)] for N,E in zip(N50[0], E50[0] )],
        [grid4[int(E)][int(N)] for N,E in zip(N25[0], E25[0] )],
        [grid8[int(E)][int(N)] for N,E in zip(N10[0], E10[0] )],
        [grid4[int(E)][int(N)] for N,E in zip(N5[0],  E5[0]  )]]).T

    if level == 9:
        tiles = [''.join(t) for t in tile_parts]
    else:
        tiles = [shorten(''.join(t))[level] for t in tile_parts]

    if uniq:
        return uniqs(tiles)
    else:
        return tiles

def xy_of_tile(tile, corner='sw'):
    # in:  tile name, e.g. 'L4131F'
    #      corner of coordinate, any of these
    #
    #      ul u ur         nw n ne
    #       l c r     or    w 0 e
    #      ll d lr         sw s se
    #
    # out: (x,y) coordinates within tile (center or upper left)
    #      None for broken tile

    level = tile_level(tile)
    if level < 0:
        return None

    # now we know len(tile) is within 2..7
    row200_letter = tile[0]
    row200_number = rows200.index(row200_letter)
    column200_number = tile[1]

    x,y = refpos # lower left of "K0"

    y += row200_number * size[200][1]
    x += int(column200_number) * size[200][0]

    if len(tile) > 2:
        if tile[2] in '34R':
            x += size[100][0]
        if tile[2] in '24':
            y += size[100][1]

    if len(tile) > 3:
        if tile[3] in '34R':
            x += size[50][0]
        if tile[3] in '24':
            y += size[50][1]

    if len(tile) > 4:
        if tile[4] in '34R':
            x += size[25][0]
        if tile[4] in '24':
            y += size[25][1]

    if len(tile) > 5:
        offset = 0
        if tile[5] in 'CD':
            offset = 1
        if tile[5] in 'EFR':
            offset = 2
        if tile[5] in 'GH':
            offset = 3
        x += size[10][0]*offset
        if tile[5] in 'BDFH':
            y += size[10][1]

    if len(tile) == 7:
        if tile[6] in '34':
            x += size[5][0]
        if tile[6] in '24':
            y += size[5][1]

    # now x,y points to lower left of tile

    if corner in ['ll','l','ul','sw','w','nw']:
        x_offset = 0

    if corner in ['u','c','d','n','0','s']:
        x_offset = 0.5

    if corner in ['lr','r','ur','se','e','ne']:
        x_offset = 1

    if corner in ['ll','d','lr','sw','s','se']:
        y_offset = 0

    if corner in ['l','c','r','w','0','e']:
        y_offset = 0.5

    if corner in ['ul','u','ur','nw','n','ne']:
        y_offset = 1

    x += size_level[level][0]*x_offset
    y += size_level[level][1]*y_offset

    return x,y

def tile_size(tile):
    # in:  tile name, e.g. 'L4131E'
    # out: tile size in meters, e.g. (6000, 6000), None if broken tile

    level = tile_level(tile)

    if level < 0:
        return None

    return size_level[level]

def tile_level(tile):
    # in:  tile name, e.g. 'L4131E'
    # out: tile level index (0..9), -1 if broken name

    if len(tile) < 2 or len(tile) > 7:
        return -1

    legal = ['KLMNPQRSTUVWX', '23456', '1234', '1234', '1234', 'ABCDEFGH', '1234']

    LR = False
    if tile[-1] in 'LR':
        if len(tile) == 7:
            return -1 # e.g. L4131AR illegal
        LR = True
        tile = tile[:-1]

    # possible level based on tile length, corrected later for LR tiles
    levels = [-1, -1, 0, 2, 4, 6, 8, 9]

    for i,c in enumerate(tile):
        if not c in legal[i]:
            return -1

    level = levels[len(tile)]

    if LR:
        level += 1

    return level

def shorten(tile):
    # in:  tile name, e.g.
    #      'L4113H3'
    # out: list of shortened tile names, with L&R, e.g.
    #      ['L4', 'L4L', 'L41', 'L41L', 'L411', 'L411R', 'L4113', 'L4131R', 'L4113H', 'L4113H3']

    shortened_tiles = [tile[0:c+2] for c in range(len(tile)-1)]

    LRtiles = [t[:-1]+('L' if t[-1] in '12' else 'R') for t in shortened_tiles[1:4]]

    for LR,k in zip(LRtiles, [1,3,5]):
        shortened_tiles.insert(k, LR)

    t = shortened_tiles[7]

    shortened_tiles.insert(7, t[:-1] + ('L' if t[-1] in 'ABCD' else 'R'))

    return shortened_tiles

# pick unique values from list, preserve order
def uniqs(seq, idfun=None):
   if idfun is None:
       def idfun(x): return x
   seen = {}
   result = []
   for item in seq:
       marker = idfun(item)
       if marker in seen: continue
       seen[marker] = 1
       result.append(item)
   return result

max_stars_repo_name: tanisc/FMIPROT

max_issues_repo_name: tanisc/FMIPROT

max_forks_repo_name: tanisc/FMIPROT


text:
import argparse
import sys
import os
import numpy as np
from dvtraitsim_shared import DVTreeData, DVParamLiang
import dvtraitsim_cpp as dvcpp
from scipy.stats import norm
import csv
import dendropy
from dendropy.model import continuous
from itertools import repeat
from itertools import starmap
from pic_compute import pic_compute
from tp_update_theta import tp_update

# parse arguments
parser = argparse.ArgumentParser(description='BaleenWhale arguments')
parser.add_argument("--treedata", required=True, type=str, help="treedata directories")
parser.add_argument("--result", required=True, type=str, help="result npy file")
parser.add_argument("--num_threads", default=-1, required=False, type=int, help="number of threads")
args = parser.parse_args()
files = args.treedata
output = args.result
num_threads = args.num_threads

#
# argsort of 2-dimensional arrays is awkward
# returns i,j so that X[i,j] == np.sort(X)
def argsort2D(X):
    return np.arange(len(X))[:, np.newaxis], np.argsort(X, axis=1)

def normalized_norm(x, y):
    diff_norm = np.linalg.norm(x - y, axis=1)
    max_err = np.nanmax(diff_norm)
    return diff_norm / max_err

K_TVP=1e6
K_TV = 1e6
K_TVM = 1e12
nu=1e-4

td = DVTreeData(path=files, scalar=20000)

with open(files + 'extantspecieslabels.csv') as csv_file:
    csv1_reader = csv.reader(csv_file, delimiter=',')
    extantlabels = list(csv1_reader)

with open(files + 'slater_length_data.csv') as csv_file:
    csv2_reader = csv.reader(csv_file, delimiter=',')
    lengthdata = list(csv2_reader)

extantlabels_array = np.array(extantlabels)[1:, 1]
lengthdata_array = np.array(lengthdata)
length_index = []
for label in extantlabels_array:
    length_index.append(np.where(lengthdata_array[:, 0] == label)[0][0])

logTL = lengthdata_array[length_index, 1].astype(np.float)
length = logTL

# reorder the traits according to the simulation order
sim_species_label = ["Balaena_mysticetus", "Balaenoptera_acutorostrata", "Caperea_marginata",
                     "Balaenoptera_borealis",
                     "Balaenoptera_physalus", "Eschrichtius_robustus", "Balaenoptera_musculus",
                     "Balaenoptera_omurai",
                     "Eubalaena_australis", "Megaptera_novaeangliae", "Balaenoptera_bonaerensis",
                     "Balaenoptera_brydei",
                     "Balaenoptera_edeni", "Eubalaena_glacialis", "Eubalaena_japonica"]

obsZ_ordered_sim = length[
    [np.where(sim_species_label[i] == extantlabels_array)[0][0] for i in range(15)]]
obsZ = obsZ_ordered_sim
meantrait = np.mean(obsZ)
# PIC calculation
taxa1 = dendropy.TaxonNamespace()
dataset_combined = dendropy.DataSet.get(path=files + "bw_char.nex", schema="nexus")
tree_emp = dataset_combined.tree_lists[0][0]
chars_emp = dataset_combined.char_matrices[0]
pic_emp = continuous.PhylogeneticIndependentConstrasts(tree=tree_emp,
                                                       char_matrix=chars_emp)
ctree_emp = pic_emp.contrasts_tree(character_index=0,
                                   annotate_pic_statistics=True, state_values_as_node_labels=False,
                                   corrected_edge_lengths=False)
emp_pic = []
label = []
for nd in ctree_emp.postorder_internal_node_iter():
    emp_pic.append(nd.pic_contrast_standardized)
    label.append(int(nd.label))

emp_pic_orded_node = abs(np.array(emp_pic)[np.argsort(label)])

tree_sim = dendropy.Tree.get(
    path=files + "bw.nex", schema="nexus",
    taxon_namespace=taxa1)
print('trying to estimate the parameters', '...')

# let's try to find a true simulation:
sampleparam_TVP = DVParamLiang(gamma=1, a=1, K=K_TVP, h=1, nu=nu, r=1, theta=0, V00=.0001,
                               V01=.0001, Vmax=100, inittrait=meantrait, initpop=1e5,
                               initpop_sigma=10.0, break_on_mu=False, num_threads=num_threads)
sampleparam_TV = DVParamLiang(gamma=1, a=1, K=K_TV, h=1, nu=nu, r=1, theta=0, V00=.0001, V01=.0001,
                              Vmax=100, inittrait=meantrait, initpop=1e5,
                              initpop_sigma=10.0, break_on_mu=False, num_threads=num_threads)
sampleparam_TVM = DVParamLiang(gamma=1, a=1, K=K_TVM, h=1, nu=nu, r=1, theta=0, V00=.0001,
                               V01=.0001, Vmax=100.0, inittrait=meantrait, initpop=1e5,
                               initpop_sigma=10.0, break_on_mu=False, num_threads=num_threads)

# pop = dvcpp.DVSim(td, obs_param)

population = 40000
generations = 30
total_population = population * 3

lefttrait = np.min(obsZ)
righttrait = np.max(obsZ)

prior = [3, 5.1, 80, 270, 0.0, nu * 100, 0, 1e-3, lefttrait, righttrait]
prior_TVM = [0.5, 5, 800, 1500, 0.0, nu * 100, 0, 0.5e-3, lefttrait, righttrait]

params_TVP = np.tile(sampleparam_TVP, (population, 1))  # duplicate
params_TVP[:, 0] = np.random.uniform(prior[0], prior[1], params_TVP.shape[0])  # randomize 'gamma'
params_TVP[:, 1] = np.random.uniform(prior[2], prior[3], params_TVP.shape[0])  # randomize 'a'
params_TVP[:, 4] = np.random.uniform(prior[4], prior[5], params_TVP.shape[0])  # randomize 'nu'
params_TVP[:, 6] = np.random.uniform(prior[8], prior[9], params_TVP.shape[0])  # randomize 'theta'
params_TVP[:, 9] = np.random.uniform(prior[6], prior[7], params_TVP.shape[0])  # randomize 'Vm'

params_TV = np.tile(sampleparam_TV, (population, 1))  # duplicate
params_TV[:, 0] = np.random.uniform(prior[0], prior[1], params_TV.shape[0])  # randomize 'gamma'
params_TV[:, 1] = np.random.uniform(prior[2], prior[3], params_TV.shape[0])  # randomize 'a'
params_TV[:, 4] = np.random.uniform(prior[4], prior[5], params_TV.shape[0])  # randomize 'nu'
params_TV[:, 6] = np.random.uniform(prior[8], prior[9], params_TV.shape[0])  # randomize 'theta'
params_TV[:, 9] = np.random.uniform(prior[6], prior[7], params_TV.shape[0])  # randomize 'Vm'

params_TVM = np.tile(sampleparam_TVM, (population, 1))  # duplicate
params_TVM[:, 0] = np.random.uniform(prior_TVM[0], prior_TVM[1],
                                     params_TVM.shape[0])  # randomize 'gamma'
params_TVM[:, 1] = np.random.uniform(prior_TVM[2], prior_TVM[3],
                                     params_TVM.shape[0])  # randomize 'a'
params_TVM[:, 4] = np.random.uniform(prior_TVM[4], prior_TVM[5],
                                     params_TVM.shape[0])  # randomize 'nu'
params_TVM[:, 6] = np.random.uniform(prior_TVM[8], prior_TVM[9],
                                     params_TVM.shape[0])  # randomize 'theta'
params_TVM[:, 9] = np.random.uniform(prior_TVM[6], prior_TVM[7],
                                     params_TVM.shape[0])  # randomize 'Vm'

# model choice
model_index = np.array([0, 1, 2])
model_params = np.repeat(model_index, repeats=population)
model_data = np.zeros(shape=(generations + 1, total_population))
model_data[0, :] = model_params
propose_model = model_params

# store parameters used
# TVP
gamma_data_TVP = np.zeros(shape=(generations + 1, population))
a_data_TVP = np.zeros(shape=(generations + 1, population))
nu_data_TVP = np.zeros(shape=(generations + 1, population))
vm_data_TVP = np.zeros(shape=(generations + 1, population))
theta_data_TVP = np.zeros(shape=(generations + 1, population))

# TV
gamma_data_TV = np.zeros(shape=(generations + 1, population))
a_data_TV = np.zeros(shape=(generations + 1, population))
nu_data_TV = np.zeros(shape=(generations + 1, population))
vm_data_TV = np.zeros(shape=(generations + 1, population))
theta_data_TV = np.zeros(shape=(generations + 1, population))

# TVM
gamma_data_TVM = np.zeros(shape=(generations + 1, population))
a_data_TVM = np.zeros(shape=(generations + 1, population))
nu_data_TVM = np.zeros(shape=(generations + 1, population))
vm_data_TVM = np.zeros(shape=(generations + 1, population))
theta_data_TVM = np.zeros(shape=(generations + 1, population))

gamma_data_TVP[0, :] = params_TVP[:, 0]
a_data_TVP[0, :] = params_TVP[:, 1]
nu_data_TVP[0, :] = params_TVP[:, 4]
vm_data_TVP[0, :] = params_TVP[:, 9]
theta_data_TVP[0, :] = params_TVP[:, 6]

gamma_data_TV[0, :] = params_TV[:, 0]
a_data_TV[0, :] = params_TV[:, 1]
nu_data_TV[0, :] = params_TV[:, 4]
vm_data_TV[0, :] = params_TV[:, 9]
theta_data_TV[0, :] = params_TV[:, 6]

gamma_data_TVM[0, :] = params_TVM[:, 0]
a_data_TVM[0, :] = params_TVM[:, 1]
nu_data_TVM[0, :] = params_TVM[:, 4]
vm_data_TVM[0, :] = params_TVM[:, 9]
theta_data_TVM[0, :] = params_TVM[:, 6]

fitness = np.zeros(shape=(generations, total_population))
# Initialize the weights.
weight_model = np.zeros(total_population)
weight_model.fill(1 / total_population)

# weights for paras of TVP
weight_gamma_TVP = np.zeros(population)
weight_gamma_TVP.fill(1 / population)
weight_a_TVP = np.zeros(population)
weight_a_TVP.fill(1 / population)
weight_nu_TVP = np.zeros(population)
weight_nu_TVP.fill(1 / population)
weight_vm_TVP = np.zeros(population)
weight_vm_TVP.fill(1 / population)
weight_theta_TVP = np.zeros(population)
weight_theta_TVP.fill(1 / population)

# weights for paras of TV
weight_gamma_TV = np.zeros(population)
weight_gamma_TV.fill(1 / population)
weight_a_TV = np.zeros(population)
weight_a_TV.fill(1 / population)
weight_nu_TV = np.zeros(population)
weight_nu_TV.fill(1 / population)
weight_vm_TV = np.zeros(population)
weight_vm_TV.fill(1 / population)
weight_theta_TV = np.zeros(population)
weight_theta_TV.fill(1 / population)

# weights for paras of TVM
weight_gamma_TVM = np.zeros(population)
weight_gamma_TVM.fill(1 / population)
weight_a_TVM = np.zeros(population)
weight_a_TVM.fill(1 / population)
weight_nu_TVM = np.zeros(population)
weight_nu_TVM.fill(1 / population)
weight_vm_TVM = np.zeros(population)
weight_vm_TVM.fill(1 / population)
weight_theta_TVM = np.zeros(population)
weight_theta_TVM.fill(1 / population)

for g in range(generations):
    # model 0
    TVP_sample_length = len(np.where(model_data[g, :] == 0)[0])
    TV_sample_length = len(np.where(model_data[g, :] == 1)[0])
    TVM_sample_length = len(np.where(model_data[g, :] == 2)[0])
    Z = np.zeros((1, td.total_species))
    pics = np.zeros((1, td.total_species - 1))

    if TVP_sample_length > 0:
        print('TVP simulations start...')

        simmodelTVP = dvcpp.DVSimTVP(td, params_TVP)
        valid_TVP = np.where(simmodelTVP['sim_time'] == td.sim_evo_time)[0]
        num_valid_sims_TVP = len(valid_TVP)
        if num_valid_sims_TVP < 20:
            print("WARNING:Valid simulations are too scarce!")
        if num_valid_sims_TVP > 0:
            Z_modelTVP = simmodelTVP['Z'][valid_TVP]
            pic_ordered_list = list(starmap(pic_compute,
                                                 zip(repeat(tree_sim), Z_modelTVP, repeat(taxa1),
                                                     range(num_valid_sims_TVP))))
            # in case the parallel computation returns disordered output
            order_list = []
            contrast_list = []
            for i in range(num_valid_sims_TVP):
                order_list.append(pic_ordered_list[i][1])
                contrast_list.append(pic_ordered_list[i][0])
            ordered_contrast_list = [contrast_list[item] for item in np.argsort(order_list)]
            contrast_array = abs(np.vstack(ordered_contrast_list))
            # i, j = argsort2D(Z_modelTVP)
            # Z_modelTVP = Z_modelTVP[i, j]
            # V = pop['V'][valid][i, j]
            Z_modelTVP = np.nan_to_num(Z_modelTVP)
            Z = np.vstack([Z, Z_modelTVP])
            pics = np.vstack([pics, contrast_array])
            # V = np.nan_to_num(V)
            # GOF: Goodness of fit
        if len(valid_TVP) == 0:
            print('No complete results from TVP model ')

    if TV_sample_length > 0:
        print('TV simulations start...')
        # model 1
        # for param_drury in params_DR:
        simmodelTV = dvcpp.DVSimTV(td, params_TV)
        valid_TV = np.where(simmodelTV['sim_time'] == td.sim_evo_time)[0]
        num_valid_sims_TV = len(valid_TV)
        Z_modelTV = simmodelTV['Z'][valid_TV]
        pic_ordered_list = list(starmap(pic_compute,
                                             zip(repeat(tree_sim), Z_modelTV, repeat(taxa1),
                                                 range(num_valid_sims_TV))))
        # in case the parallel computation returns disordered output
        order_list = []
        contrast_list = []
        for i in range(num_valid_sims_TV):
            order_list.append(pic_ordered_list[i][1])
            contrast_list.append(pic_ordered_list[i][0])
        ordered_contrast_list = [contrast_list[item] for item in np.argsort(order_list)]
        contrast_array = abs(np.vstack(ordered_contrast_list))
        # i, j = argsort2D(Z_modelTV)
        # Z_modelTV = Z_modelTV[i, j]
        # V = pop['V'][valid][i, j]
        Z_modelTV = np.nan_to_num(Z_modelTV)
        Z = np.vstack([Z, Z_modelTV])
        pics = np.vstack([pics, contrast_array])

    if TVM_sample_length > 0:
        print('TVM simulations start...')
        simmodelTVM = dvcpp.DVSimTVMLog10(td, params_TVM)
        valid_TVM = np.where(simmodelTVM['sim_time'] == td.sim_evo_time)[0]
        num_valid_sims_TVM = len(valid_TVM)
        Z_modelTVM = simmodelTVM['Z'][valid_TVM]
        pic_ordered_list = list(starmap(pic_compute,
                                             zip(repeat(tree_sim), Z_modelTVM, repeat(taxa1),
                                                 range(num_valid_sims_TVM))))
        # in case the parallel computation returns disordered output
        order_list = []
        contrast_list = []
        for i in range(num_valid_sims_TVM):
            order_list.append(pic_ordered_list[i][1])
            contrast_list.append(pic_ordered_list[i][0])
        ordered_contrast_list = [contrast_list[item] for item in np.argsort(order_list)]
        contrast_array = abs(np.vstack(ordered_contrast_list))
        # i, j = argsort2D(Z_modelTVM)
        # Z_modelTVM = Z_modelTVM[i, j]
        # V = pop['V'][valid][i, j]
        Z_modelTVM = np.nan_to_num(Z_modelTVM)
        Z = np.vstack([Z, Z_modelTVM])
        pics = np.vstack([pics, contrast_array])

    Z = Z[1:, ]
    pics = pics[1:, ]

    valid = np.concatenate([valid_TVP, np.array(valid_TV) + TVP_sample_length,
                            np.array(valid_TVM) + TVP_sample_length + TV_sample_length
                            ]).astype(int)

    eudis = normalized_norm(Z, obsZ)
    eudis_pic = normalized_norm(pics, emp_pic_orded_node)
    # eudis = eudistance(Z, obsZ)

    fitness[g, valid] += 1 - eudis
    fitness[g, valid] += 1 - eudis_pic

    # fitness[g,valid] += 1.0 - normalized_norm(np.sqrt(V), np.sqrt(obsV))

    # print something...
    q5 = np.argsort(fitness[g, :])[-int(total_population // 4)]  # best 25%
    fit_index = np.where(fitness[g, :] > fitness[g, q5])[0]

    modelTVPperc = len(np.where(propose_model[fit_index] == 0)[0]) / len(fit_index)
    modelTVperc = len(np.where(propose_model[fit_index] == 1)[0]) / len(fit_index)
    modelTVMperc = len(np.where(propose_model[fit_index] == 2)[0]) / len(fit_index)

    print('Iteration = %d 25th Model TVP: %.1f%% ;  Model TV: %.1f%% ; Model TVM: %.1f%%...'
          % (g, modelTVPperc * 100, modelTVperc * 100, modelTVMperc * 100))
    print('Average fitness: %f' % np.mean(fitness[g, fit_index]))
    # reevaluate the weight of the best fitted  models
    weight_model_bestfitted = weight_model[fit_index] * fitness[g, fit_index] / sum(
        weight_model[fit_index] * fitness[g, fit_index])
    # sample new models from the fitness of previous best fitted models
    sample_model_index = sorted(
        np.random.choice(fit_index, total_population, p=weight_model_bestfitted))

    propose_model = model_params

    q5_TVP = np.argsort(fitness[g, :population])[-int(population // 200)]  # best 5%
    q5_TV = np.argsort(fitness[g, population:2 * population])[
                -int(population // 200)] + population  # best 5%
    q5_TVM = np.argsort(fitness[g, 2 * population:3 * population])[
                 -int(population // 200)] + 2 * population  # best 5%

    fit_index_TVP = np.where(fitness[g, :population] > fitness[g, q5_TVP])[0]
    fit_index_TV = np.where(fitness[g, population:2 * population] > fitness[g, q5_TV])[
                       0] + population
    fit_index_TVM = np.where(fitness[g, 2 * population:] > fitness[g, q5_TVM])[0] + 2 * population

    previous_bestfitted_index_TVP = fit_index_TVP
    previous_bestfitted_index_TV = fit_index_TV - population
    previous_bestfitted_index_TVM = fit_index_TVM - 2 * population

    chosengamma_TVP, chosena_TVP, chosennu_TVP, chosenvm_TVP, chosentheta_TVP = np.mean(
        params_TVP[previous_bestfitted_index_TVP, 0]), \
                                                                                np.mean(params_TVP[
                                                                                previous_bestfitted_index_TVP, 1]), \
                                                                                np.mean(params_TVP[
                                                                                previous_bestfitted_index_TVP, 4]), \
                                                                                np.mean(params_TVP[
                                                                                previous_bestfitted_index_TVP, 9]), \
                                                                                np.mean(params_TVP[
                                                                                previous_bestfitted_index_TVP, 6])

    chosengamma_TV, chosena_TV, chosennu_TV, chosenvm_TV, chosentheta_TV = np.mean(
        params_TV[previous_bestfitted_index_TV, 0]), \
                                                                           np.mean(params_TV[
                                                                                previous_bestfitted_index_TV, 1]), \
                                                                           np.mean(params_TV[
                                                                                previous_bestfitted_index_TV, 4]), \
                                                                           np.mean(params_TV[
                                                                                previous_bestfitted_index_TV, 9]), \
                                                                           np.mean(params_TV[
                                                                                previous_bestfitted_index_TV, 6])

    chosengamma_TVM, chosena_TVM, chosennu_TVM, chosenvm_TVM, chosentheta_TVM = np.mean(
        params_TVM[previous_bestfitted_index_TVM, 0]), \
                                                                                np.mean(params_TVM[
                                                                                previous_bestfitted_index_TVM, 1]), \
                                                                                np.mean(params_TVM[
                                                                                previous_bestfitted_index_TVM, 4]), \
                                                                                np.mean(params_TVM[
                                                                                previous_bestfitted_index_TVM, 9]), \
                                                                                np.mean(params_TVM[
                                                                                previous_bestfitted_index_TVM, 6])

    print('Mean estimates: TVP gamma: %.3e ; a: %.3e ; nu: %.3e ; Vm : %f; theta : %f' % (
    chosengamma_TVP, chosena_TVP, chosennu_TVP, chosenvm_TVP, chosentheta_TVP))
    print('Mean estimates: TV gamma: %.3e ; a: %.3e ; nu: %.3e ; Vm : %f; theta : %f' % (
    chosengamma_TV, chosena_TV, chosennu_TV, chosenvm_TV, chosentheta_TV))
    print('Mean estimates: TVM gamma: %.3e ; a: %.3e ; nu: %.3e ; Vm : %f; theta : %f' % (
    chosengamma_TVM, chosena_TVM, chosennu_TVM, chosenvm_TVM, chosentheta_TVM))

    model_data[g + 1, :] = propose_model
    gamma_data_TVP[g + 1, :] = params_TVP[:, 0]
    a_data_TVP[g + 1, :] = params_TVP[:, 1]
    nu_data_TVP[g + 1, :] = params_TVP[:, 4]
    vm_data_TVP[g + 1, :] = params_TVP[:, 9]
    theta_data_TVP[g + 1, :] = params_TVP[:, 6]

    gamma_data_TV[g + 1, :] = params_TV[:, 0]
    a_data_TV[g + 1, :] = params_TV[:, 1]
    nu_data_TV[g + 1, :] = params_TV[:, 4]
    vm_data_TV[g + 1, :] = params_TV[:, 9]
    theta_data_TV[g + 1, :] = params_TV[:, 6]

    gamma_data_TVM[g + 1, :] = params_TVM[:, 0]
    a_data_TVM[g + 1, :] = params_TVM[:, 1]
    nu_data_TVM[g + 1, :] = params_TVM[:, 4]
    vm_data_TVM[g + 1, :] = params_TVM[:, 9]
    theta_data_TVM[g + 1, :] = params_TVM[:, 6]

    if len(np.where(propose_model == 0)[0]) > 0:
        params_TVP_update = params_TVP[:, [0, 1, 4, 9, 6]]
        modelinex = 0
        # update TP paras and weights
        weight_gamma_TVP, weight_a_TVP, weight_nu_TVP, weight_vm_TVP, weight_theta_TVP, \
        propose_gamma_TVP, propose_a_TVP, propose_nu_TVP, propose_vm_TVP, propose_theta_TVP = \
            tp_update(previous_bestfitted_index_TVP, propose_model, params_TVP_update,
                      weight_gamma_TVP,
                      weight_a_TVP, weight_nu_TVP, weight_vm_TVP, weight_theta_TVP, modelinex)
        modelTVP = np.where(propose_model == modelinex)
        params_TVP = np.tile(sampleparam_TVP, (len(modelTVP[0]), 1))
        params_TVP[:, 0] = propose_gamma_TVP
        params_TVP[:, 1] = propose_a_TVP
        params_TVP[:, 4] = propose_nu_TVP
        params_TVP[:, 9] = propose_vm_TVP
        params_TVP[:, 6] = propose_theta_TVP

    if len(np.where(propose_model == 1)[0]) > 0:
        params_TV_update = params_TV[:, [0, 1, 4, 9, 6]]
        modelinex = 1
        if len(valid_TV) > 0:
            # update TP paras and weights
            weight_gamma_TV, weight_a_TV, weight_nu_TV, weight_vm_TV, weight_theta_TV, \
            propose_gamma_TV, propose_a_TV, propose_nu_TV, propose_vm_TV, propose_theta_TV = \
                tp_update(previous_bestfitted_index_TV, propose_model, params_TV_update,
                          weight_gamma_TV,
                          weight_a_TV, weight_nu_TV, weight_vm_TV, weight_theta_TV, modelinex)
            modelTV = np.where(propose_model == modelinex)
            params_TV = np.tile(sampleparam_TV, (len(modelTV[0]), 1))
            params_TV[:, 0] = propose_gamma_TV
            params_TV[:, 1] = propose_a_TV
            params_TV[:, 4] = propose_nu_TV
            params_TV[:, 9] = propose_vm_TV
            params_TV[:, 6] = propose_theta_TV

    if len(np.where(propose_model == 2)[0]) > 0:
        params_TVM_update = params_TVM[:, [0, 1, 4, 9, 6]]
        modelinex = 2
        # update TP paras and weights
        weight_gamma_TVM, weight_a_TVM, weight_nu_TVM, weight_vm_TVM, weight_theta_TVM, \
        propose_gamma_TVM, propose_a_TVM, propose_nu_TVM, propose_vm_TVM, propose_theta_TVM = \
            tp_update(previous_bestfitted_index_TVM, propose_model, params_TVM_update,
                      weight_gamma_TVM,
                      weight_a_TVM, weight_nu_TVM, weight_vm_TVM, weight_theta_TVM, modelinex)
        modelTVM = np.where(propose_model == modelinex)
        params_TVM = np.tile(sampleparam_TVM, (len(modelTVM[0]), 1))
        params_TVM[:, 0] = propose_gamma_TVM
        params_TVM[:, 1] = propose_a_TVM
        params_TVM[:, 4] = propose_nu_TVM
        params_TVM[:, 9] = propose_vm_TVM
        params_TVM[:, 6] = propose_theta_TVM

    #

    para_data = {'model_data': model_data, 'fitness': fitness, 'Z': Z,
                 'gamma_data_TVP': gamma_data_TVP, 'a_data_TVP': a_data_TVP,
                 'nu_data_TVP': nu_data_TVP,
                 'vm_data_TVP': vm_data_TVP, 'theta_data_TVP': theta_data_TVP,
                 'gamma_data_TV': gamma_data_TV, 'a_data_TV': a_data_TV, 'nu_data_TV': nu_data_TV,
                 'vm_data_TV': vm_data_TV, 'theta_data_TV': theta_data_TV,
                 'gamma_data_TVM': gamma_data_TVM, 'a_data_TVM': a_data_TVM,
                 'nu_data_TVM': nu_data_TVM,
                 'vm_data_TVM': vm_data_TVM, 'theta_data_TVM': theta_data_TVM
                 }

    np.save(output, para_data)

reason: import numpy,from scipy


text:
import tensorflow as tf
import numpy as np
import os
import gym

def build_z(Vmin, Vmax, nb_atoms, numpy=False):
    dz = (Vmax - Vmin) / (nb_atoms - 1)
    if numpy:
        z = np.arange(Vmin, Vmax + dz / 2, dz)
    else:
        z = tf.range(Vmin, Vmax + dz / 2, dz, dtype=tf.float32, name='z')  # TODO: reuse?

    return z, dz

def parent_path(path):
    if path.endswith('/'):
        path = path[:-1]
    return os.path.join(*os.path.split(path)[:-1])

atari_actions = ['noop', 'fire', 'up', 'right', 'left',
                 'down', 'up-right', 'up-left', 'down-right', 'down-left',
                 'up-fire', 'right-fire', 'left-fire', 'down-fire', 'up-right-fire',
                 'up-left-fire', 'down-right-fire', 'down-left-fire']

def actions_from_env(env):
    """ Propagate through all wrappers to get action indices. """
    while True:
        if isinstance(env, gym.Wrapper):
            env = env.env
        else:
            break
    if isinstance(env, gym.Env):
        if hasattr(env, 'ale'):
            actions = env.ale.getMinimalActionSet()
            return [atari_actions[i] for i in actions]

def make_env(game_name):
    from baselines.common.atari_wrappers import wrap_deepmind, make_atari
    env = make_atari(game_name + "NoFrameskip-v4")
    monitored_env = SimpleMonitor(env)
    env = wrap_deepmind(monitored_env, frame_stack=True, scale=True)
    return env, monitored_env

# hard copy from old baselines.common.misc_util
# TODO: remove?
import time

class SimpleMonitor(gym.Wrapper):
    def __init__(self, env):
        """Adds two qunatities to info returned by every step:
            num_steps: int
                Number of steps takes so far
            rewards: [float]
                All the cumulative rewards for the episodes completed so far.
        """
        super().__init__(env)
        # current episode state
        self._current_reward = None
        self._num_steps = None
        # temporary monitor state that we do not save
        self._time_offset = None
        self._total_steps = None
        # monitor state
        self._episode_rewards = []
        self._episode_lengths = []
        self._episode_end_times = []

    def _reset(self):
        obs = self.env.reset()
        # recompute temporary state if needed
        if self._time_offset is None:
            self._time_offset = time.time()
            if len(self._episode_end_times) > 0:
                self._time_offset -= self._episode_end_times[-1]
        if self._total_steps is None:
            self._total_steps = sum(self._episode_lengths)
        # update monitor state
        if self._current_reward is not None:
            self._episode_rewards.append(self._current_reward)
            self._episode_lengths.append(self._num_steps)
            self._episode_end_times.append(time.time() - self._time_offset)
        # reset episode state
        self._current_reward = 0
        self._num_steps = 0

        return obs

    def _step(self, action):
        obs, rew, done, info = self.env.step(action)
        self._current_reward += rew
        self._num_steps += 1
        self._total_steps += 1
        info['steps'] = self._total_steps
        info['rewards'] = self._episode_rewards
        return (obs, rew, done, info)

    def get_state(self):
        return {
            'env_id': self.env.unwrapped.spec.id,
            'episode_data': {
                'episode_rewards': self._episode_rewards,
                'episode_lengths': self._episode_lengths,
                'episode_end_times': self._episode_end_times,
                'initial_reset_time': 0,
            }
        }

    def set_state(self, state):
        assert state['env_id'] == self.env.unwrapped.spec.id
        ed = state['episode_data']
        self._episode_rewards = ed['episode_rewards']
        self._episode_lengths = ed['episode_lengths']
        self._episode_end_times = ed['episode_end_times']

max_stars_repo_name: Silvicek/distributional-dqn

max_issues_repo_name: Silvicek/distributional-dqn

max_forks_repo_name: Silvicek/distributional-dqn


text:
r"""
Polynomial morphisms for products of projective spaces

This class builds on the projective space class and its point and morphism classes.

EXAMPLES::

    sage: P1xP1.<x,y,u,v> = ProductProjectiveSpaces(QQ, [1, 1])
    sage: H = End(P1xP1)
    sage: H([x^2*u, y^2*v, x*v^2, y*u^2])
    Scheme endomorphism of Product of projective spaces P^1 x P^1 over Rational Field
      Defn: Defined by sending (x : y , u : v) to
            (x^2*u : y^2*v , x*v^2 : y*u^2).
"""
#*****************************************************************************
# Copyright (C) 2014 Ben Hutz <bn4941@gmail.com>
#
# Distributed under the terms of the GNU General Public License (GPL)
# as published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
# http://www.gnu.org/licenses/
#*****************************************************************************
from sage.categories.homset import Hom, End
from sage.rings.quotient_ring import QuotientRing_generic
from sage.schemes.generic.morphism import SchemeMorphism_polynomial

class ProductProjectiveSpaces_morphism_ring(SchemeMorphism_polynomial):
    r"""
    The class of morphisms on products of projective spaces.

    The components are projective space morphisms.

    EXAMPLES::

        sage: T.<x,y,z,w,u> = ProductProjectiveSpaces([2, 1], QQ)
        sage: H = T.Hom(T)
        sage: H([x^2, y^2, z^2, w^2, u^2])
        Scheme endomorphism of Product of projective spaces P^2 x P^1 over Rational Field
          Defn: Defined by sending (x : y : z , w : u) to
                (x^2 : y^2 : z^2 , w^2 : u^2).
    """

    def __init__(self, parent, polys, check=True):
        r"""
        The Python constructor.

        INPUT:

        - ``parent`` -- Hom-set.

        - ``polys`` -- anything that defines a point in the class.

        - ``check`` -- Boolean. Whether or not to perform input checks.
          (Default:`` True``)

        EXAMPLES::

            sage: T.<x,y,z,w,u> = ProductProjectiveSpaces([2, 1], QQ)
            sage: H = T.Hom(T)
            sage: H([x^2*u, y^2*w, z^2*u, w^2, u^2])
            Scheme endomorphism of Product of projective spaces P^2 x P^1 over Rational Field
              Defn: Defined by sending (x : y : z , w : u) to
                    (x^2*u : y^2*w : z^2*u , w^2 : u^2).

        ::

            sage: T.<x,y,z,w,u> = ProductProjectiveSpaces([2, 1], QQ)
            sage: H = T.Hom(T)
            sage: H([x^2*u, y^2*w, z^2*u, w^2, u*z])
            Traceback (most recent call last):
            ...
            TypeError: polys (=[x^2*u, y^2*w, z^2*u, w^2, z*u]) must be
            multi-homogeneous of the same degrees (by component)

        ::

            sage: R.<s,t> = PolynomialRing(QQ)
            sage: Z.<a,b,x,y,z> = ProductProjectiveSpaces([1, 2], QQ)
            sage: P.<u,v,w,s,t,r> = ProductProjectiveSpaces([3, 1], QQ)
            sage: H = Hom(Z,P)
            sage: f = H([a^2,b^2,a^2,a*b,a*x,b*z]); f
            Scheme morphism:
              From: Product of projective spaces P^1 x P^2 over Rational Field
              To:   Product of projective spaces P^3 x P^1 over Rational Field
              Defn: Defined by sending (a : b , x : y : z) to
                    (a^2 : b^2 : a^2 : a*b , a*x : b*z).

        ::

            sage: Z.<a,b,c,x,y,z> = ProductProjectiveSpaces([1, 3], QQ)
            sage: P.<u,v,w,s,t,r> = ProductProjectiveSpaces([2, 2], QQ)
            sage: H = Hom(Z,P)
            sage: f = H([a^2,b^2,c^2,x^2,y^2,z^2])
            Traceback (most recent call last):
            ...
            TypeError: polys (=[a^2, b^2, c^2, x^2, y^2, z^2]) must be
            multi-homogeneous of the same degrees (by component)
        """
        if check:
            #check multi-homogeneous
            #if self is a subscheme, we may need the lift of the polynomials
            try:
                polys[0].exponents()
            except AttributeError:
                polys = [f.lift() for f in polys]

            target = parent.codomain().ambient_space()
            dom = parent.domain().ambient_space()
            from sage.schemes.product_projective.space import is_ProductProjectiveSpaces
            if is_ProductProjectiveSpaces(target):
                splitpolys = target._factors(polys)
                for m in range(len(splitpolys)):
                    d = dom._degree(splitpolys[m][0])
                    if not all(d == dom._degree(f) for f in splitpolys[m]):
                        raise  TypeError("polys (=%s) must be multi-homogeneous of the same degrees (by component)"%polys)
            else:
                #we are mapping into some other kind of space
                target._validate(polys)

        SchemeMorphism_polynomial.__init__(self, parent, polys, check)

    def __getitem__(self, i):
        r"""
        Return the ``i``-th coordinate polynomial.

        INPUT:

        - ``i`` -- integer.

        OUTPUT:

        The (multi)-homomgeneous polynomial that is the ``i``-th coordinate.

        EXAMPLES::

            sage: T.<x,y,z,w,u> = ProductProjectiveSpaces([2, 1], QQ)
            sage: H = T.Hom(T)
            sage: F = H([x^2*u, y^2*w, z^2*u, w^2, u^2])
            sage: F[2]
            z^2*u
        """
        return(self._polys[i])

    def _repr_defn(self):
        r"""
        Return a string representation of this morphism.

        OUTPUT: String.

        EXAMPLES::

            sage: P.<x,y,z,w> = ProductProjectiveSpaces([1, 1], QQ)
            sage: H = Hom(P,P)
            sage: f = H([x^2, y^2, z, w])
            sage: f._repr_defn()
            'Defined by sending (x : y , z : w) to \n(x^2 : y^2 , z : w).'
        """
        s  = 'Defined by sending '
        s += self.domain().ambient_space()._repr_generic_point()
        s += ' to \n'
        s += self.codomain().ambient_space()._repr_generic_point(self._polys)
        s += '.'
        return s

    def __call__(self, P, check=True):
        r"""
        Make morphisms of products of projective spaces callable.

        INPUT:

        - ``P`` -- a point in the domain.

        - ``check`` -- Boolean - whether or not to perform the input checks
          on the image point (Default: ``True``).

        OUTPUT: The image point in the codomain.

        EXAMPLES::

            sage: T.<x,y,z,w,u> = ProductProjectiveSpaces([2, 1], QQ)
            sage: H = T.Hom(T)
            sage: F = H([x^2*u, y^2*w, z^2*u, w^2, u^2])
            sage: F(T([2, 1, 3, 0, 1]))
            (4/9 : 0 : 1 , 0 : 1)

        ::

            sage: PP.<x,y,z,u,v,w> = ProductProjectiveSpaces(QQ, [1, 1, 1])
            sage: HP = End(PP)
            sage: f = HP([v*x^2,w*y^2,z^2,u^2,v^2,w^2])
            sage: Q = PP([0,1,1,1,1,1])
            sage: f(Q)
            (0 : 1 , 1 : 1 , 1 : 1)

        ::

            sage: PP.<t0,t1,t2,t3,t4> = ProductProjectiveSpaces([2,1], ZZ)
            sage: Q = PP([1,1,1,2,1])
            sage: Z.<a,b,x,y,z> = ProductProjectiveSpaces([1,2], ZZ)
            sage: H = End(Z)
            sage: f = H([a^3, b^3+a*b^2, x^2, y^2-z^2, z*y])
            sage: f(Q)
            Traceback (most recent call last):
            ...
            TypeError: (1 : 1 : 1 , 2 : 1) fails to convert into the map's domain
            Product of projective spaces P^1 x P^2 over Integer Ring, but a
            `pushforward` method is not properly implemented
            sage: f([1,1,1,2,1])
            (1 : 2 , 1 : 3 : 2)

        ::

            sage: PP.<x,y,u,v> = ProductProjectiveSpaces(ZZ, [1, 1])
            sage: HP = End(PP)
            sage: g = HP([x^2, y^2, u^2, v^2])
            sage: g([0, 0, 0, 0],check=False)
            (0 : 0 , 0 : 0)
        """
        from sage.schemes.product_projective.point import ProductProjectiveSpaces_point_ring
        if check:
            if not isinstance(P, ProductProjectiveSpaces_point_ring):
                try:
                    P = self.domain()(P)
                except (TypeError, NotImplementedError):
                    raise TypeError("%s fails to convert into the map's domain %s, but a `pushforward` method is not properly implemented"%(P, self.domain()))
            elif self.domain()!= P.codomain():
                raise TypeError("%s fails to convert into the map's domain %s, but a `pushforward` method is not properly implemented"%(P, self.domain()))

        A = self.codomain()
        Q = list(P)
        newP = [f(Q) for f in self.defining_polynomials()]
        return(A.point(newP, check))

    def is_morphism(self):
        r"""
        Returns ``True`` if this mapping is a morphism of products of projective spaces.

        For each component space of the codomain of this mapping we consider the subscheme of
        the domain of this map generated by the corresponding coordinates of the map.
        This map is a morphism if and only if each of these subschemes has no points.

        OUTPUT: Boolean.

        EXAMPLES::

            sage: Z.<a,b,x,y,z> = ProductProjectiveSpaces([1, 2], ZZ)
            sage: H = End(Z)
            sage: f = H([a^2, b^2, x*z-y*z, x^2-y^2, z^2])
            sage: f.is_morphism()
            False

        ::

            sage: P.<x,y,z,u,v,w>=ProductProjectiveSpaces([2, 2], QQ)
            sage: H = End(P)
            sage: f = H([u, v, w, u^2, v^2, w^2])
            sage: f.is_morphism()
            True

        ::

            sage: P.<x,y,z,w,u> = ProductProjectiveSpaces([2, 1], QQ)
            sage: Q.<a,b,c,d,e> = ProductProjectiveSpaces([1, 2], QQ)
            sage: H = Hom(P, Q)
            sage: f = H([x^2, y^2, u^3, w^3, u^3])
            sage: f.is_morphism()
            False
        """
        m = 0
        T = self.domain().ambient_space()
        S = self.codomain().ambient_space()

        if T.base_ring().is_field():
            f = self
        else:
            f = self.change_ring(T.base_ring().fraction_field())
            T = T.change_ring(T.base_ring().fraction_field())

        for i in range(S.num_components()):
            t = S[i].dimension_relative() + 1
            X = T.subscheme(list(f)[m : m + t])
            if X.dimension() > -1:
                return False
            m = m + t

        return True

    def nth_iterate(self, P, n, normalize=False):
        r"""
        For a map of this morphism and a point `P` in ``self.domain()``
        this function returns the nth iterate of `P` by this morphism.

        If ``normalize`` is ``True``, then the coordinates are
        automatically normalized.

        .. TODO:: Is there a more efficient way to do this?

        INPUT:

        - ``P`` -- a point in ``self.domain()``.

        - ``n`` -- a positive integer.

        - ``normalize`` - Boolean (optional Default: ``False``).

        OUTPUT:

        - A point in ``self.codomain()``.

        EXAMPLES::

            sage: Z.<a,b,x,y,z> = ProductProjectiveSpaces([1, 2], QQ)
            sage: H = End(Z)
            sage: f = H([a^3, b^3 + a*b^2, x^2, y^2 - z^2, z*y])
            sage: P = Z([1, 1, 1, 1, 1])
            sage: f.nth_iterate(P, 3)
            (1/1872 : 1 , 1 : 1 : 0)
        """
        return(P.nth_iterate(self, n, normalize))

    def orbit(self, P, N, **kwds):
        r"""
        Returns the orbit of `P` by this morphism.

        If `N` is an integer it returns `[P,self(P),\ldots,self^N(P)]`.

        If `n` is a list or tuple `n = [m, k]` it returns `[self^m(P),\ldots,self^k(P)]`.
        Automatically normalize the points if ``normalize == True``. Perform the checks on point initialize if
        ``check==True``.

        INPUT:

        - ``P`` -- a point in ``self.domain()``.

        - ``n`` -- a non-negative integer or list or tuple of two non-negative integers.

        kwds:

        - ``check`` -- boolean (optional - default: ``True``).

        - ``normalize`` -- boolean (optional - default: ``False``).

        OUTPUT:

        - a list of points in ``self.codomain()``.

        EXAMPLES::

            sage: Z.<a,b,x,y,z> = ProductProjectiveSpaces([1, 2], QQ)
            sage: H = End(Z)
            sage: f = H([a^3, b^3 + a*b^2, x^2, y^2 - z^2, z*y])
            sage: P = Z([1, 1, 1, 1, 1])
            sage: f.orbit(P, 3)
            [(1 : 1 , 1 : 1 : 1), (1/2 : 1 , 1 : 0 : 1), (1/12 : 1 , -1 : 1 : 0), (1/1872 : 1 , 1 : 1 : 0)]
        """
        return(P.orbit(self, N, **kwds))

    def nth_iterate_map(self, n):
        r"""
        This function returns the nth iterate of this morphsim as a
        function on ``self.domain()``.

        ALGORITHM:

        Uses a form of successive squaring to reduce computations.

        .. TODO:: This could be improved.

        INPUT:

        - ``n`` -- a positive integer.

        OUTPUT:

        - A map between products of projective spaces.

        EXAMPLES::

            sage: Z.<a,b,x,y,z> = ProductProjectiveSpaces([1 , 2], QQ)
            sage: H = End(Z)
            sage: f = H([a^3, b^3, x^2, y^2, z^2])
            sage: f.nth_iterate_map(3)
            Scheme endomorphism of Product of projective spaces P^1 x P^2 over
            Rational Field
              Defn: Defined by sending (a : b , x : y : z) to
                    (a^27 : b^27 , x^8 : y^8 : z^8).
        """
        if not self.is_endomorphism():
            raise TypeError("domain and codomain of function not equal")

        E = self.domain()
        D = int(n)
        if D < 0:
            raise TypeError("iterate number must be a nonnegative integer")
        N = sum([E.ambient_space()[i].dimension_relative() + 1 for i in range(E.ambient_space().num_components())])
        F = list(self._polys)
        Coord_ring = E.coordinate_ring()
        if isinstance(Coord_ring, QuotientRing_generic):
            PHI = [Coord_ring.gen(i).lift() for i in range(N)]
        else:
            PHI = [Coord_ring.gen(i) for i in range(N)]

        while D:
            if D&1:
                PHI = [PHI[j](*F) for j in range(N)]
            if D > 1: #avoid extra iterate
                F = [F[j](*F) for j in range(N)] #'square'
            D >>= 1
        return End(E)(PHI)

max_stars_repo_name: fredstro/sage

max_issues_repo_name: boothby/sage

max_forks_repo_name: boothby/sage

reason: from sage


text:
# coding: utf-8
###
 # @file   ps.py
 # @author  Anton Ragot <anton.ragot@epfl.ch>, Jérémy Plassmann <jeremy.plassmann@epfl.ch>
 #
 # @section LICENSE
 #
 # MIT License
 #
 # Copyright (c) 2020 Distributed Computing Laboratory, EPFL
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in all
 # copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
###

#!/usr/bin/env python

import time

import numpy as np
from tensorflow.keras.optimizers import Adam

from . import garfield_pb2
from . import tools
from .server import Server

class PS(Server):
    """ Parameter Server node, handles the updates of the parameter of the model. """

    def __init__(self, network=None, log=False, dataset="mnist", model="Small", batch_size=128, nb_byz_worker= 0):
        """ Create a Parameter Server node.

            args:
                - network:   State of the cluster
                - log:      Boolean indicating whether to log or not
                - asyncr:   Boolean

        """
        super().__init__(network, log, dataset, model, batch_size, nb_byz_worker)

        self.optimizer = Adam(lr=1e-3)

    def get_gradients(self, iter):
        """ Get gradients from the workers at a specific iteration.

            args:
                - iter: integer
            Returns:
                Gradients from the different PS.
        """

        gradients = []

        for i, connection in enumerate(self.worker_connections):
            counter = 0
            read = False
            while not read:
                try:
                    response = connection.GetGradient(garfield_pb2.Request(iter=iter,
                                                                        job="ps",
                                                                        req_id=self.task_id))
                    serialized_gradient = response.gradients
                    gradient = np.frombuffer(serialized_gradient, dtype=np.float32)
                    gradients.append(gradient)
                    read = True
                except Exception as e:
                    print("Trying to connect to Worker node ", i)
                    time.sleep(5)
                    counter+=1
                    if counter > 10:			#any reasonable large enough number
                        exit(0)
        return gradients

    def upate_model(self, gradient):
        """ Update the model with the aggregated gradients.

            Args:
                - gradient: gradient to update the model.
            Returns:
                Model after update.
        """
        reshape_gradient = tools.reshape_weights(self.model, gradient)
        self.optimizer.apply_gradients(zip(reshape_gradient, self.model.trainable_variables))
        return tools.flatten_weights(self.model.trainable_variables)

    def commit_model(self, model):
        """ Make the model available on the network.

            Args:
                - model: model to commit
        """
        self.service.model_wieghts_history.append(model)

max_stars_repo_name: sahareslami/Garfield

max_issues_repo_name: sahareslami/Garfield

max_forks_repo_name: sahareslami/Garfield


text:
# coding: utf-8

import os
import itertools
import numpy as np
import pandas as pd
import networkx as nx

measures = [
    (nx.degree_centrality, 1),
    (nx.closeness_centrality, -1),
    (nx.betweenness_centrality, 1),
]

def parse_graph(graph_file, distance_threshold=200):
    dtypes = {
        'POINT_X': np.float64,
        'POINT_X': np.float64,
        'POINT_Z': np.float64}
    stops = pd.read_excel(graph_file, dtype=dtypes)

    graph = nx.Graph()
    for i in range(stops.shape[0]):
        node_attrs = {
            'name': stops['Name_GJCZ'][i],
            'routes': stops['XL_GJCZ'][i],
            'x': float(stops['POINT_X'][i]),
            'y': float(stops['POINT_Y'][i]),
            'z': float(stops['POINT_Z'][i])
        }
        graph.add_node(i, **node_attrs)

    for pair in itertools.combinations(graph.nodes, 2):
        n0 = graph.nodes[pair[0]]
        n1 = graph.nodes[pair[1]]

        dist = np.sqrt((n0['x'] - n1['x']) ** 2 + (n0['y'] - n1['y']) ** 2)

        routes0 = n0['routes'].split(';')
        routes1 = n1['routes'].split(';')
        routes = list(set(routes0) & set(routes1))

        edge_attrs = {
            'distance': float(dist),
            'routes': ';'.join(routes)
        }

        if dist <= distance_threshold or len(routes) > 0:
            print(u'Edge: {} and {}, dist: {}, routes: {}'.format(
                n0['name'], n1['name'], dist, ';'.join(routes)))
            graph.add_edge(pair[0], pair[1], **edge_attrs)

    save_file = '{}.gexf'.format(os.path.splitext(graph_file)[0])
    nx.write_gexf(graph, save_file)

    return graph

def score_nodes(graph, watch_nodes):
    scores = np.zeros((len(watch_nodes), len(measures)))

    for meas_idx, (meas_func, meas_weight) in enumerate(measures):
        meas_res = meas_func(graph)
        for node_idx, node in enumerate(watch_nodes):
            scores[node_idx, meas_idx] = meas_res[node] * meas_weight

    return scores

def find_comb(graph, watch_nodes=None, n_combs=2):
    if watch_nodes == None:
        watch_nodes = range(graph.number_of_nodes())

    new_node = 'new_node'
    best_combs = None
    max_diff = -np.inf

    scores = score_nodes(graph, watch_nodes)
    for comb in itertools.combinations(graph.nodes, n_combs):
        comb_nodes = list(comb)
        graph.add_node(new_node)
        for node in comb_nodes:
            graph.add_edge(new_node, node)

        comb_scores = score_nodes(graph, watch_nodes)
        diff = np.sum(comb_scores - scores)
        if diff > max_diff:
            max_diff = diff
            best_combs = comb_nodes

        graph.remove_node(new_node)
        print('Node combination: {}, diff: {}'.format(comb_nodes, diff))

    return best_combs, max_diff

if __name__ == '__main__':
    # graph = nx.sedgewick_maze_graph()
    # graph = nx.tutte_graph()
    graph = parse_graph('stops_56.xlsx')

    best_combs, max_diff = find_comb(graph)
    print(best_combs)

max_stars_repo_name: zhougt/landscape

max_issues_repo_name: zhougt/landscape

max_forks_repo_name: zhougt/landscape

reason: import numpy,import networkx


text:
#coding=utf-8
import tensorflow as tf
import wml_tfutils as wmlt
import wnn
import functools
import tfop
import object_detection2.bboxes as odbox
from object_detection2.standard_names import *
import wmodule
from object_detection2.modeling.onestage_heads.onestage_tools import *
from object_detection2.datadef import *
from object_detection2.config.config import global_cfg
import object_detection2.wlayers as odl
from object_detection2.modeling.build import HEAD_OUTPUTS
from object_detection2.data.dataloader import DataLoader
import wsummary
from itertools import count
import numpy as np

'''
Use GIOU loss instated the official huber loss for box regression.
'''
@HEAD_OUTPUTS.register()
class FCOSGIOUOutputsV2(wmodule.WChildModule):
    def __init__(
        self,
        cfg,
        parent,
        box2box_transform,
        pred_logits,
        pred_regression,
        pred_center_ness,
        gt_boxes=None,
        gt_labels=None,
        gt_length=None,
        max_detections_per_image=100,
        batched_inputs=None,
        **kwargs,
    ):
        """
        Args:
            cfg: Only the child part
            box2box_transform (Box2BoxTransform): :class:`Box2BoxTransform` instance for
                anchor-proposal transformations.
            anchor_matcher (Matcher): :class:`Matcher` instance for matching anchors to
                ground-truth boxes; used to determine training labels.
            images (ImageList): :class:`ImageList` instance representing N input images
            pred_objectness_logits (list[Tensor]): A list of L elements.
                Element i is a tensor of shape (N, A, Hi, Wi) representing
                the predicted objectness logits for anchors.
            pred_anchor_deltas (list[Tensor]): A list of L elements. Element i is a tensor of shape
                (N, A*4, Hi, Wi) representing the predicted "deltas" used to transform anchors
                to proposals.
            anchors (list[list[Boxes]]): A list of N elements. Each element is a list of L
                Boxes. The Boxes at (n, l) stores the entire anchor array for feature map l in image
                n (i.e. the cell anchors repeated over all locations in feature map (n, l)).
            boundary_threshold (int): if >= 0, then anchors that extend beyond the image
                boundary by more than boundary_thresh are not used in training. Set to a very large
                number or < 0 to disable this behavior. Only needed in training.
            gt_boxes (list[Boxes], optional): A list of N elements. Element i a Boxes storing
                the ground-truth ("gt") boxes for image i.
        """
        super().__init__(cfg,parent=parent,**kwargs)
        assert len(cfg.SIZE_THRESHOLD) == len(pred_logits)-1,"Error size threshold num."
        self.num_classes      = cfg.NUM_CLASSES
        self.focal_loss_alpha         = cfg.FOCAL_LOSS_ALPHA
        self.focal_loss_gamma         = cfg.FOCAL_LOSS_GAMMA
        self.topk_candidates          = cfg.TOPK_CANDIDATES_TEST
        self.score_threshold          = cfg.SCORE_THRESH_TEST
        self.nms_threshold            = cfg.NMS_THRESH_TEST
        self.max_detections_per_image = max_detections_per_image
        self.size_threshold = [0]+cfg.SIZE_THRESHOLD+[-1]
        self.box2box_transform = box2box_transform
        self.pred_logits = pred_logits
        self.pred_regression = pred_regression
        self.pred_center_ness = pred_center_ness

        self.gt_boxes = gt_boxes
        self.gt_labels = gt_labels
        self.gt_length = gt_length
        self.num_feature_maps = len(pred_logits)
        self.mid_results = {}
        self.batched_inputs = batched_inputs

    def _get_ground_truth(self):
        """
        Returns:
            gt_objectness_logits: list of N tensors. Tensor i is a vector whose length is the
                total number of anchors in image i (i.e., len(anchors[i])). Label values are
                in {-1, 0, 1}, with meanings: -1 = ignore; 0 = negative class; 1 = positive class.
            gt_anchor_deltas: list of N tensors. Tensor i has shape (len(anchors[i]), 4).
        """

        img_size = tf.shape(self.batched_inputs[IMAGE])[1:3]

        res_list = []

        for i,logits,regression,center_ness in zip(count(),self.pred_logits,self.pred_regression,self.pred_center_ness):
            res = self.box2box_transform.get_deltas(gboxes=self.gt_boxes,
                                                    glabels=self.gt_labels,
                                                    glength=self.gt_length,
                                                    min_size=self.size_threshold[i],
                                                    max_size=self.size_threshold[i+1],
                                                    fm_shape=tf.shape(logits)[1:3],
                                                    img_size=img_size)
            if global_cfg.GLOBAL.SUMMARY_LEVEL <= SummaryLevel.DEBUG:
                for k,v in res.items():
                    if len(v.get_shape()) == 3:
                        v = tf.expand_dims(v,axis=-1)
                    wsummary.feature_map_summary(v,k)
            res_list.append(res)

        return res_list

    @wmlt.add_name_scope
    def losses(self):
        """
        Args:
            For `gt_classes` and `gt_anchors_deltas` parameters, see
                :meth:`FCOSGIou.get_ground_truth`.
            Their shapes are (N, R) and (N, R, 4), respectively, where R is
            the total number of anchors across levels, i.e. sum(Hi x Wi x A)
            For `pred_class_logits` and `pred_anchor_deltas`, see
                :meth:`FCOSGIouHead.forward`.

        Returns:
            dict[str: Tensor]:
                mapping from a named loss to a scalar tensor
                storing the loss. Used during training only. The dict keys are:
                "loss_cls" and "loss_box_reg"
        """

        assert len(self.pred_logits[0].get_shape()) == 4,"error logits dim"

        gt_results = self._get_ground_truth()
        loss_cls_list = []
        loss_regression_list = []
        loss_center_ness_list = []
        total_num_foreground = []

        img_size = tf.shape(self.batched_inputs[IMAGE])[1:3]

        for i,gt_results_item in enumerate(gt_results):
            gt_classes = gt_results_item['g_classes']
            gt_boxes = gt_results_item['g_boxes']
            g_center_ness = gt_results_item['g_center_ness']
            pred_class_logits = self.pred_logits[i]
            pred_regression = self.pred_regression[i]
            pred_center_ness = self.pred_center_ness[i]

            foreground_idxs = (gt_classes > 0)
            num_foreground = tf.reduce_sum(tf.cast(foreground_idxs,tf.int32))
            total_num_foreground.append(num_foreground)

            gt_classes_target = tf.one_hot(gt_classes,depth=self.num_classes+1)
            gt_classes_target = gt_classes_target[...,1:]

            #
            pred_center_ness = tf.expand_dims(pred_center_ness,axis=-1)
            wsummary.histogram_or_scalar(pred_center_ness,"center_ness")
            # logits loss
            loss_cls = tf.reduce_sum(wnn.sigmoid_cross_entropy_with_logits_FL(
                labels = gt_classes_target,
                logits = pred_class_logits,
                alpha=self.focal_loss_alpha,
                gamma=self.focal_loss_gamma))

            # regression loss
            pred_boxes = self.box2box_transform.apply_deltas(regression=pred_regression,img_size=img_size)
            if global_cfg.GLOBAL.SUMMARY_LEVEL <= SummaryLevel.DEBUG and gt_classes.get_shape().as_list()[0]>1:
                log_boxes = self.box2box_transform.apply_deltas(regression=gt_results_item['g_regression'],
                                                                img_size=img_size)
                log_boxes = odbox.tfabsolutely_boxes_to_relative_boxes(log_boxes,width=img_size[1],height=img_size[0])
                boxes1 = tf.reshape(log_boxes[1:2],[1,-1,4])
                wsummary.detection_image_summary(images=self.batched_inputs[IMAGE][1:2],
                                                 boxes=boxes1,
                                                 name="FCOSGIou_decode_test")
            pred_center_ness = tf.boolean_mask(pred_center_ness,foreground_idxs)
            g_center_ness = tf.boolean_mask(g_center_ness,foreground_idxs)
            pred_boxes = tf.boolean_mask(pred_boxes,foreground_idxs)
            gt_boxes = tf.boolean_mask(gt_boxes,foreground_idxs)
            wsummary.histogram_or_scalar(pred_center_ness,"center_ness_pos")
            reg_loss_sum = (1.0-odl.giou(pred_boxes,gt_boxes))
            wmlt.variable_summaries_v2(reg_loss_sum,f"giou_loss{i}")
            pred_center_ness = tf.squeeze(pred_center_ness,axis=-1)
            reg_norm = tf.reduce_sum(g_center_ness)+1e-5
            reg_loss_sum = reg_loss_sum*g_center_ness
            wmlt.variable_summaries_v2(reg_loss_sum,f"loss_sum{i}")
            loss_box_reg = tf.reduce_sum(reg_loss_sum)*300/reg_norm
            wmlt.variable_summaries_v2(loss_box_reg,f"box_reg_loss_{i}")

            loss_center_ness = 0.5*tf.nn.sigmoid_cross_entropy_with_logits(labels=g_center_ness,
                                                                       logits=pred_center_ness)
            loss_center_ness = tf.reduce_sum(loss_center_ness)*0.1
            wmlt.variable_summaries_v2(loss_center_ness,f"center_ness_loss{i}")

            loss_cls_list.append(loss_cls)
            loss_regression_list.append(loss_box_reg)
            loss_center_ness_list.append(loss_center_ness)

        total_num_foreground = tf.to_float(tf.maximum(tf.add_n(total_num_foreground),1))
        return {"fcos_loss_cls": tf.add_n(loss_cls_list)/total_num_foreground,
                "fcos_loss_center_ness": tf.add_n(loss_center_ness_list)/total_num_foreground,
                "fcos_loss_box_reg": tf.add_n(loss_regression_list)/total_num_foreground}

    @wmlt.add_name_scope
    def inference(self, inputs,box_cls, box_regression,center_ness,nms=None,pad=True):
        """
        Arguments:
            inputs: same as FCOS.forward's batched_inputs
            box_cls: list of Tensor, Tensor's shape is [B,H,W,A*num_classes]
            box_delta: list of Tensor, Tensor's shape is [B,H,W,A*4]
        Returns:
            results:
            RD_BOXES: [B,N,4]
            RD_LABELS: [B,N]
            RD_PROBABILITY:[ B,N]
            RD_LENGTH:[B]
        """
        assert len(box_cls[0].get_shape()) == 4, "error box cls dims"
        assert len(box_regression[0].get_shape()) == 4, "error box delta dims"

        B,_,_,_ = wmlt.combined_static_and_dynamic_shape(box_regression[0])
        fm_sizes = [tf.shape(x)[1:3] for x in box_regression]
        box_cls = [reshape_to_N_HWA_K(x,self.num_classes) for x in box_cls]
        box_regression = [reshape_to_N_HWA_K(x,4) for x in box_regression]
        center_ness = [tf.reshape(x,[B,-1]) for x in center_ness]
        box_cls = tf.concat(box_cls,axis=1)
        box_regression = tf.concat(box_regression,axis=1)
        center_ness = tf.concat(center_ness,axis=1)

        results = wmlt.static_or_dynamic_map_fn(lambda x:self.inference_single_image(x[0],x[1],x[2],fm_sizes,nms=nms,pad=pad),
                                                elems=[box_cls,box_regression,center_ness],
                                                dtype=[tf.float32,tf.int32,tf.float32,tf.int32],
                                                back_prop=False)
        outdata = {RD_BOXES:results[0],RD_LABELS:results[1],RD_PROBABILITY:results[2],RD_LENGTH:results[3]}
        if global_cfg.GLOBAL.SUMMARY_LEVEL<=SummaryLevel.DEBUG:
            wsummary.detection_image_summary(images=inputs[IMAGE],
                                             boxes=outdata[RD_BOXES], classes=outdata[RD_LABELS],
                                             lengths=outdata[RD_LENGTH],
                                             scores=outdata[RD_PROBABILITY],
                                              name="FCOSGIou_result",
                                             category_index=DataLoader.category_index)
        return outdata

    @wmlt.add_name_scope
    def inference_single_image(self, box_cls, box_regression, center_ness,fm_sizes,nms = None,pad=True):
        """
        Single-image inference. Return bounding-box detection results by thresholding
        on scores and applying non-maximum suppression (NMS).

        Arguments:
            box_cls;[WxHxA(concat),K]
            box_delta [WxHxA(concat),box_dim]
            anchors [WxHxA(concat),box_dim]
            anchors_size: anchors'size per level
        Returns:
            Same as `inference`, but for only one image.
        """

        assert len(box_cls.get_shape())==2,"Error box cls tensor shape"
        assert len(box_regression.get_shape())==2,"Error box delta tensor shape"

        boxes_all = []
        scores_all = []
        class_idxs_all = []

        # Iterate over every feature level
        sizes = [tf.reduce_prod(x) for x in fm_sizes]
        box_cls = tf.split(box_cls,num_or_size_splits=sizes)
        box_regression = tf.split(box_regression,num_or_size_splits=sizes)
        center_ness = tf.split(center_ness,num_or_size_splits=sizes)
        img_size = tf.shape(self.batched_inputs[IMAGE])[1:3]
        for fm_size,box_cls_i, box_reg_i, centern_ness_i in zip(fm_sizes,box_cls, box_regression, center_ness):

            boxes_i = self.box2box_transform.apply_deltas(regression=box_reg_i,img_size=img_size,
                                                             fm_size=fm_size)
            boxes_i = odbox.tfabsolutely_boxes_to_relative_boxes(boxes_i,
                                                                 width=img_size[1], height=img_size[0])

            # (HxWxAxK,)
            center_ness_i = tf.reshape(tf.nn.sigmoid(centern_ness_i),[-1,1])
            box_cls_i = tf.nn.sigmoid(tf.reshape(box_cls_i,[-1,self.num_classes]))*center_ness_i
            box_cls_i = tf.reshape(box_cls_i,[-1])

            # Keep top k top scoring indices only.
            num_topk = tf.minimum(self.topk_candidates, tf.shape(box_reg_i)[0])
            # torch.sort is actually faster than .topk (at least on GPUs)
            predicted_prob, topk_idxs = tf.nn.top_k(box_cls_i,num_topk)
            predicted_prob = predicted_prob[:num_topk]
            topk_idxs = topk_idxs[:num_topk]

            if self.score_threshold>1e-5:
                keep_idxs = tf.greater(predicted_prob,self.score_threshold)
                predicted_prob = tf.boolean_mask(predicted_prob,keep_idxs)
                topk_idxs = tf.boolean_mask(topk_idxs,keep_idxs)

            boxes_idxs = topk_idxs // self.num_classes
            classes_idxs = topk_idxs % self.num_classes

            # filter out the proposals with low confidence score
            boxes_i = tf.gather(boxes_i,boxes_idxs)

            boxes_all.append(boxes_i)
            scores_all.append(predicted_prob)
            class_idxs_all.append(classes_idxs)

        boxes_all, scores_all, class_idxs_all= [
            tf.concat(x,axis=0) for x in [boxes_all, scores_all, class_idxs_all]
        ]
        x,y = wmlt.sort_data(key=scores_all,datas=[boxes_all,class_idxs_all])
        boxes_all,class_idxs_all = y
        scores_all,_ = x
        if nms is None:
            nms = functools.partial(tfop.boxes_nms, threshold=self.nms_threshold, classes_wise=True,k=self.max_detections_per_image)
        boxes,labels,nms_idxs = nms(bboxes=boxes_all,classes=class_idxs_all)
        scores = tf.gather(scores_all,nms_idxs)

        candiate_nr = self.max_detections_per_image
        #labels = tf.Print(labels,[tf.shape(labels)],name="XXXXXXXXXXXXXXXXXXXX",summarize=100)
        labels = labels+1 #加上背景
        lens = tf.shape(labels)[0]
        if pad:
            boxes = tf.pad(boxes, paddings=[[0, candiate_nr - lens], [0, 0]])
            labels = tf.pad(labels, paddings=[[0, candiate_nr - lens]])
            scores = tf.pad(scores, paddings=[[0, candiate_nr - lens]])

        return [boxes,labels,scores,lens]


text:
from typing import List, Dict

import numpy as np
from torch import Tensor
from allennlp.common import Params
from allennlp.common.util import START_SYMBOL, END_SYMBOL
from allennlp.data import Vocabulary
from allennlp.data.vocabulary import DEFAULT_OOV_TOKEN, DEFAULT_PADDING_TOKEN

from rulm.language_model import LanguageModel
from rulm.transform import Transform

@LanguageModel.register("vocabulary_chain")
class VocabularyChainLanguageModel(LanguageModel):
    def __init__(self, vocab: Vocabulary, transforms: List[Transform]=None):
        LanguageModel.__init__(self, vocab, transforms)

    def train(self,
              file_name: str,
              train_params: Params,
              serialization_dir: str=None,
              **kwargs):
        pass

    def _load(self,
              params: Params,
              vocab: Vocabulary,
              serialization_dir: str,
              weights_file: str,
              cuda_device: int=-1.,
              **kwargs):
        pass

    def predict(self, batch: Dict[str, Dict[str, Tensor]], **kwargs) -> np.ndarray:
        inputs = batch["source_tokens"]["tokens"]
        batch_size = inputs.size(0)
        vocab_size = self.vocab.get_vocab_size("tokens")
        result = np.zeros((batch_size, vocab_size), dtype="float")
        for sample_number in range(batch_size):
            probabilities = np.zeros(vocab_size)
            aux = (START_SYMBOL, END_SYMBOL, DEFAULT_OOV_TOKEN, DEFAULT_PADDING_TOKEN)
            aux_indices = [self.vocab.get_token_index(s) for s in aux]
            bos_index = aux_indices[0]
            eos_index = aux_indices[1]
            last_index = inputs[sample_number][-1]
            first_not_aux_index = 0
            for i in range(self.vocab.get_vocab_size()):
                if i in aux_indices:
                    continue
                first_not_aux_index = i
                break
            if last_index == bos_index:
                probabilities[first_not_aux_index] = 1.
            elif last_index != self.vocab.get_vocab_size() - 1:
                probabilities[last_index + 1] = 1.
            elif last_index == self.vocab.get_vocab_size() - 1:
                probabilities[eos_index] = 1.
            result[sample_number] = probabilities
        return result

max_stars_repo_name: IlyaGusev/rulm

max_issues_repo_name: IlyaGusev/rulm

max_forks_repo_name: IlyaGusev/rulm