text:
from collections.abc import Iterable
from copy import deepcopy
import gc
import networkx as nx
import numba
from numba import jit
import numpy as np
import os
import pandas as pd
from scipy import sparse
from sklearn.decomposition import TruncatedSVD
import time
import warnings

from csrgraph.methods import (
    _row_norm, _node_degrees, _src_multiply, _dst_multiply
)
from csrgraph.random_walks import (
    _random_walk, _node2vec_walks,_node2vec_walks_with_rejective_sampling
)
from csrgraph import methods, random_walks
from csrgraph import ggvec, glove, grarep

UINT32_MAX = (2**32) - 1
UINT16_MAX = (2**16) - 1

class csrgraph():
    """
    This top level python class either calls external JIT'ed methods
        or methods from the JIT'ed internal graph
    """
    def __init__(self, data, nodenames=None, copy=True, threads=0):
        """
        A class for larger graphs.

        NOTE: this class tends to "steal data" by default.
            If you pass a numpy array/scipy matrix/csr graph
            Chances are this object will point to the same instance of data

        Parameters:
        -------------------

        data : The graph data. Can be one of:
            **NetworkX Graph**
            **Numpy dense matrix**
            **CSR Matrix**
            **(data, indices, indptr)**
            **CSRGraph object**

        nodenames (array of str or int) : Node names
            The position in this array should correspond with the node ID
            So if passing a CSR Matrix or raw data, it should be co-indexed
            with the matrix/raw data arrays

        copy : bool
            Wether to copy passed data to create new object
            Default behavior is to point to underlying ctor passed data
            For networkX graphs and numpy dense matrices we create a new object anyway
        threads : int
            number of threads to leverage for methods in this graph.
            WARNING: changes the numba environment variable to do it.
            Recompiles methods and changes it when changed.
            0 is numba default (usually all threads)

        TODO: add numpy mmap support for very large on-disk graphs
            Should be in a different class
            This also requires routines to read/write edgelists, etc. from disk

        TODO: subclass scipy.csr_matrix???
        """
        # If input is a CSRGraph, same object
        if isinstance(data, csrgraph):
            if copy:
                self.mat = data.mat.copy()
                self.names = deepcopy(data.names)
            else:
                self.mat = data.mat
                self.names = data.names
            if not nodenames:
                nodenames = self.names
            else:
                self.names = nodenames
        # NetworkX Graph input
        elif isinstance(data, (nx.Graph, nx.DiGraph)):
            mat = nx.adjacency_matrix(data)
            mat.data = mat.data.astype(np.float32)
            self.mat = mat
            nodenames = list(data)
        # CSR Matrix Input
        elif isinstance(data, sparse.csr_matrix):
            if copy: self.mat = data.copy()
            else: self.mat = data
        # Numpy Array Input
        elif isinstance(data, np.ndarray):
            self.mat = sparse.csr_matrix(data)
        else:
            raise ValueError(f"Incorrect input data type: {type(data).__name__}")
        # Now that we have the core csr_matrix, alias underlying arrays
        assert hasattr(self, 'mat')
        self.weights = self.mat.data
        self.src = self.mat.indptr
        self.dst = self.mat.indices
        # indptr has one more element than nnodes
        self.nnodes = self.src.size - 1
        # node name -> node ID
        if nodenames is not None:
            self.names = pd.Series(nodenames)
        else:
            self.names = pd.Series(np.arange(self.nnodes))
        # Bounds check once here otherwise there be dragons later
        max_idx = np.max(self.dst)
        if self.nnodes < max_idx:
            raise ValueError(f"""
                Out of bounds node: {max_idx}, nnodes: {self.nnodes}
            """)
        self.set_threads(threads)

    def set_threads(self, threads):
        self.threads = threads
        # Manage threading through Numba hack
        if type(threads) is not int:
            raise ValueError("Threads argument must be an int!")
        if threads == 0:
            threads = numba.config.NUMBA_DEFAULT_NUM_THREADS
        threads = str(threads)
        # If we change the number of threads, recompile
        try:
            prev_numba_value = os.environ['NUMBA_NUM_THREADS']
        except KeyError:
            prev_numba_value = threads
        if threads != prev_numba_value:
            os.environ['NUMBA_NUM_THREADS'] = threads
            _random_walk.recompile()
            _row_norm.recompile()
            _node2vec_walks.recompile()
            _node_degrees.recompile()
            _src_multiply.recompile()
            _dst_multiply.recompile()

    def __getitem__(self, node):
        """
        [] operator
        like networkX, gets names of neighbor nodes
        """
        # Get node ID from names array
        # This is O(n) by design -- we more often get names from IDs
        #    than we get IDs from names and we don't want to hold 2 maps
        # TODO : replace names with a pd.Index and use get_loc
        node_id = self.names[self.names == node].index[0]
        edges = self.dst[
            self.src[node_id] : self.src[node_id+1]
        ]
        return self.names.iloc[edges].values

    def nodes(self):
        """
        Returns the graph's nodes, in order
        """
        if self.names is not None:
            return self.names
        else:
            return np.arange(self.nnodes)

    def normalize(self, return_self=True):
        """
        Normalizes edge weights per node

        For any node in the Graph, the new edges' weights will sum to 1

        return_self : bool
            whether to change the graph's values and return itself
            this lets us call `G.normalize()` directly
        """
        new_weights = _row_norm(self.weights, self.src)
        if return_self:
            self.mat = sparse.csr_matrix((new_weights, self.dst, self.src))
            # Point objects to the correct places
            self.weights = self.mat.data
            self.src = self.mat.indptr
            self.dst = self.mat.indices
            gc.collect()
            return self
        else:
            return csrgraph(sparse.csr_matrix(
                (new_weights, self.dst, self.src)),
                nodenames=self.names)

    def random_walks(self,
                walklen=10,
                epochs=1,
                start_nodes=None,
                normalize_self=False,
                return_weight=1.,
                neighbor_weight=1.,
                rejective_sampling=False):
        """
        Create random walks from the transition matrix of a graph
            in CSR sparse format

        Parameters
        ----------
        T : scipy.sparse.csr matrix
            Graph transition matrix in CSR sparse format
        walklen : int
            length of the random walks
        epochs : int
            number of times to start a walk from each nodes
        return_weight : float in (0, inf]
            Weight on the probability of returning to node coming from
            Having this higher tends the walks to be
            more like a Breadth-First Search.
            Having this very high  (> 2) makes search very local.
            Equal to the inverse of p in the Node2Vec paper.
        neighbor_weight : float in (0, inf]
            Weight on the probability of visitng a neighbor node
            to the one we're coming from in the random walk
            Having this higher tends the walks to be
            more like a Depth-First Search.
            Having this very high makes search more outward.
            Having this very low makes search very local.
            Equal to the inverse of q in the Node2Vec paper.
        threads : int
            number of threads to use.  0 is full use
        rejective_sampling: bool
            for deepwalk (p=1, q=1), this parameters is of no use
            for node2vec walks, it determines if we use rejective sampling or not
            Rejective sampling is faster, but less stable
            Credit to https://github.com/louisabraham/fastnode2vec for original idea

        Returns
        -------
        out : 2d np.array (n_walks, walklen)
            A matrix where each row is a random walk,
            and each entry is the ID of the node
        """
        # Make csr graph
        if normalize_self:
            self.normalize(return_self=True)
            T = self
        else:
            T = self.normalize(return_self=False)
        n_rows = T.nnodes
        if start_nodes is None:
            start_nodes = np.arange(n_rows)
        sampling_nodes = np.tile(start_nodes, epochs)
        # Node2Vec Biased walks if parameters specified
        if (return_weight > 1. or return_weight < 1.
                or neighbor_weight < 1. or neighbor_weight > 1.):
            if rejective_sampling:
                walks = _node2vec_walks_with_rejective_sampling(T.weights, T.src, T.dst,
                                        sampling_nodes=sampling_nodes,
                                        walklen=walklen,
                                        return_weight=return_weight,
                                        neighbor_weight=neighbor_weight)
            else:
                walks = _node2vec_walks(T.weights, T.src, T.dst,
                                        sampling_nodes=sampling_nodes,
                                        walklen=walklen,
                                        return_weight=return_weight,
                                        neighbor_weight=neighbor_weight)
        # much faster implementation for regular walks
        else:
            walks = _random_walk(T.weights, T.src, T.dst,
                                 sampling_nodes, walklen)
        return walks

    def ggvec(self, n_components=2,
              learning_rate=0.05,
              tol="auto",
              max_epoch=500,
              negative_ratio=1.0,
              order=1,
              negative_decay=0.,
              exponent=0.5,
              max_loss=30.,
              tol_samples=100,
              verbose=True):
        """
        GGVec: Fast global first (and higher) order local embeddings.

        This algorithm directly minimizes related nodes' distances.
        It uses a relaxation pass (negative sample) + contraction pass (loss minimization)
        To find stable embeddings based on the minimal dot product of edge weights.

        Parameters:
        -------------
        n_components (int):
            Number of individual embedding dimensions.
        order : int >= 1
            Meta-level of the embeddings. Improves link prediction performance.
            Setting this higher than 1 ~quadratically slows down algorithm
                Order = 1 directly optimizes the graph.
                Order = 2 optimizes graph plus 2nd order edges
                    (eg. neighbours of neighbours)
                Order = 3 optimizes up to 3rd order edges
            Higher order edges are automatically weighed using GraRep-style graph formation
            Eg. the higher-order graph is from stable high-order random walk distribution.
        negative_ratio : float in [0, 1]
            Negative sampling ratio.
            Setting this higher will do more negative sampling.
            This is slower, but can lead to higher quality embeddings.
        exponent : float
            Weighing exponent in loss function.
            Having this lower reduces effect of large edge weights.
        tol : float in [0, 1] or "auto"
            Optimization early stopping criterion.
            Stops average loss < tol for tol_samples epochs.
            "auto" sets tol as a function of learning_rate
        tol_samples : int
            Optimization early stopping criterion.
            This is the number of epochs to sample for loss stability.
            Once loss is stable over this number of epochs we stop early.
        negative_decay : float in [0, 1]
            Decay on negative ratio.
            If >0 then negative ratio will decay by (1-negative_decay) ** epoch
            You should usually leave this to 0.
        max_epoch : int
            Stopping criterion.
        max_count : int
            Ceiling value on edge weights for numerical stability
        learning_rate : float in [0, 1]
            Optimization learning rate.
        max_loss : float
            Loss value ceiling for numerical stability.
        """
        if tol == 'auto':
            tol = max(learning_rate / 2, 0.05)
        # Higher order graph embeddings
        # Method inspired by GraRep (powers of transition matrix)
        if order > 1:
            norm_weights = _row_norm(self.weights, self.src)
            tranmat = sparse.csr_matrix((norm_weights, self.dst, self.src))
            target_matrix = tranmat.copy()
            res = np.zeros((self.nnodes, n_components))
            for _ in range(order - 1):
                target_matrix = target_matrix.dot(tranmat)
                w = ggvec.ggvec_main(
                    data=target_matrix.data,
                    src=target_matrix.indptr,
                    dst=target_matrix.indices,
                    n_components=n_components, tol=tol,
                    tol_samples=tol_samples,
                    max_epoch=max_epoch, learning_rate=learning_rate,
                    negative_ratio=negative_ratio,
                    negative_decay=negative_decay,
                    exponent=exponent,
                    max_loss=max_loss, verbose=verbose)
                res = np.sum([res, w], axis=0)
            return res
        else:
            return ggvec.ggvec_main(
                data=self.weights, src=self.src, dst=self.dst,
                n_components=n_components, tol=tol,
                tol_samples=tol_samples,
                max_epoch=max_epoch, learning_rate=learning_rate,
                negative_ratio=negative_ratio,
                negative_decay=negative_decay,
                exponent=exponent,
                max_loss=max_loss, verbose=verbose)

    def glove(self, n_components=2,
              tol=0.0001, max_epoch=10_000,
              max_count=50, exponent=0.5,
              learning_rate=0.1, max_loss=10.,
              verbose=True):
        """
        Global first order embedding for positive, count-valued sparse matrices.

        This algorithm is normally used in NLP on word co-occurence matrices.
        The algorithm fails if any value in the sparse matrix < 1.
        It is also a poor choice for matrices with homogeneous edge weights.

        Parameters:
        -------------
        n_components (int):
            Number of individual embedding dimensions.
        tol : float in [0, 1]
            Optimization early stopping criterion.
            Stops when largest gradient is < tol
        max_epoch : int
            Stopping criterion.
        max_count : int
            Ceiling value on edge weights for numerical stability
        exponent : float
            Weighing exponent in loss function.
            Having this lower reduces effect of large edge weights.
        learning_rate : float in [0, 1]
            Optimization learning rate.
        max_loss : float
            Loss value ceiling for numerical stability.

        References:
        -------------
        Paper: https://nlp.stanford.edu/pubs/glove.pdf
        Original implementation: https://github.com/stanfordnlp/GloVe/blob/master/src/glove.c
        """
        w = glove.glove_main(weights=self.weights, dst=self.dst, src=self.src,
                           n_components=n_components, tol=tol,
                           max_epoch=max_epoch, learning_rate=learning_rate,
                           max_count=max_count, exponent=exponent,
                           max_loss=max_loss, verbose=verbose)
        return w

    def grarep(self, n_components=2,
               order=5,
               embedder=TruncatedSVD(
                    n_iter=10,
                    random_state=42),
                verbose=True):
        """Implementation of GraRep: Embeddings of powers of the PMI matrix
            of the graph adj matrix.

        NOTE: Unlike GGVec and GLoVe, this method returns a LIST OF EMBEDDINGS
              (one per order). You can sum them, or take the last one for embedding.
            Default pooling is to sum :
                `lambda x : np.sum(x, axis=0)`
            You can also take only the highest order embedding:
                `lambda x : x[-1]`
            Etc.

        Original paper: https://dl.acm.org/citation.cfm?id=2806512

        Parameters :
        ----------------
        n_components (int):
            Number of individual embedding dimensions.
        order (int):
            Number of PMI matrix powers.
            The performance degrades close to quadratically as a factor of this parameter.
            Generally should be kept under 5.
        embedder : (instance of sklearn API compatible model)
            Should implement the `fit_transform` method:
                https://scikit-learn.org/stable/glossary.html#term-fit-transform
            The model should also have `n_components` as a parameter
            for number of resulting embedding dimensions. See:
                https://scikit-learn.org/stable/modules/manifold.html#manifold
            If not compatible, set resulting dimensions in the model instance directly
        merger : function[list[array]] -> array
            GraRep returns one embedding matrix per order.
            This function reduces it to a single matrix.
        """
        w_array = grarep.grarep_main(weights=self.weights, dst=self.dst, src=self.src,
                           n_components=n_components, order=order,
                           embedder=embedder,verbose=verbose)
        return w_array

    def random_walk_resample(self, walklen=4, epochs=30):
        """
        Create a new graph from random walk co-occurences.

        First, we generate random walks on the graph
        Then, any nodes appearing together in a walk get an edge
        Edge weights are co-occurence counts.

        Recommendation: many short walks > fewer long walks

        TODO: add node2vec walk parameters
        """
        walks = self.random_walks(walklen=walklen, epochs=epochs)
        elist = random_walks.walks_to_edgelist(walks)
        if 'weight' in elist.columns:
            weights = elist.weight.to_numpy()
        else:
            weights = np.ones(dst.shape[0])
        return methods._edgelist_to_graph(
            elist.src.to_numpy(), elist.dst.to_numpy(),
            weights, nnodes=self.nnodes, nodenames=self.names)

    #
    #
    # TODO: Organize Graph method here
    # Layout nodes by their 1d embedding's position
    # Also layout by hilbert space filling curve
    #
    #

def read_edgelist(f, directed=True, sep=r"\s+", header=None, keep_default_na=False, **readcsvkwargs):
    """
    Creates a csrgraph from an edgelist.

    The edgelist should be in the form
       [source  destination]
        or
       [source  destination  edge_weight]

    The first column needs to be the source, the second the destination.
    If there is a third column it's assumed to be edge weights.

    Otherwise, all arguments from pandas.read_csv can be used to read the file.

    f : str
        Filename to read
    directed : bool
        Whether the graph is directed or undirected.
        All csrgraphs are directed, undirected graphs simply add "return edges"
    sep : str
        CSV-style separator. Eg. Use "," if comma separated
    header : int or None
        pandas read_csv parameter. Use if column names are present
    keep_default_na: bool
        pandas read_csv argument to prevent casting any value to NaN
    read_csv_kwargs : keyword arguments for pd.read_csv
        Pass these kwargs as you would normally to pd.read_csv.
    Returns : csrgraph
    """
    # Read in csv correctly to each column
    elist = pd.read_csv(
        f, sep=sep, header=header,
        keep_default_na=keep_default_na, **readcsvkwargs
    )
    if len(elist.columns) == 2:
        elist.columns = ['src', 'dst']
        elist['weight'] = np.ones(elist.shape[0])
    elif len(elist.columns) == 3:
        elist.columns = ['src', 'dst', 'weight']
    else:
        raise ValueError(f"""
            Invalid columns: {elist.columns}
            Expected 2 (source, destination)
            or 3 (source, destination, weight)
            Read File: \n{elist.head(5)}
        """)
    # Create name mapping to normalize node IDs
    # Somehow this is 1.5x faster than np.union1d. Shame on numpy.
    allnodes = list(
        set(elist.src.unique())
        .union(set(elist.dst.unique())))
    # Factor all nodes to unique IDs
    names = (
        pd.Series(allnodes).astype('category')
        .cat.categories
    )
    nnodes = names.shape[0]
    # Get the input data type
    if nnodes > UINT16_MAX:
        dtype = np.uint32
    if nnodes > UINT32_MAX:
        dtype = np.uint64
    else:
        dtype = np.uint16
    name_dict = dict(zip(names,
                         np.arange(names.shape[0], dtype=dtype)))
    elist.src = elist.src.map(name_dict).astype(dtype)
    elist.dst = elist.dst.map(name_dict).astype(dtype)
    # clean up temp data
    allnodes = None
    name_dict = None
    gc.collect()
    # If undirected graph, append edgelist to reversed self
    if not directed:
        other_df = elist.copy()
        other_df.columns = ['dst', 'src', 'weight']
        elist = pd.concat([elist, other_df])
        other_df = None
        gc.collect()
    # Need to sort by src for _edgelist_to_graph
    elist = elist.sort_values(by='src')
    # extract numpy arrays and clear memory
    src = elist.src.to_numpy()
    dst = elist.dst.to_numpy()
    weight = elist.weight.to_numpy()
    elist = None
    gc.collect()
    G = methods._edgelist_to_graph(
        src, dst, weight,
        nnodes, nodenames=names
    )
    return G

reason: import numpy,from scipy,import numba,from numba,import networkx


text:
# coding=utf-8

from typing import List, Optional, Tuple, Union

import numpy as np
import tensorflow as tf
import tensorflow.keras.layers as tfkl

from deepreg.model import layer_util
from deepreg.model.backbone.u_net import UNet
from deepreg.registry import REGISTRY

class AffineHead(tfkl.Layer):
    def __init__(
        self,
        image_size: tuple,
        name: str = "AffineHead",
    ):
        """
        Init.

        :param image_size: such as (dim1, dim2, dim3)
        :param name: name of the layer
        """
        super().__init__(name=name)
        self.reference_grid = layer_util.get_reference_grid(image_size)
        self.transform_initial = tf.constant_initializer(
            value=list(np.eye(4, 3).reshape((-1)))
        )
        self._flatten = tfkl.Flatten()
        self._dense = tfkl.Dense(units=12, bias_initializer=self.transform_initial)

    def call(
        self, inputs: Union[tf.Tensor, List], **kwargs
    ) -> Tuple[tf.Tensor, tf.Tensor]:
        """

        :param inputs: a tensor or a list of tensor with length 1
        :param kwargs: additional args
        :return: ddf and theta

            - ddf has shape (batch, dim1, dim2, dim3, 3)
            - theta has shape (batch, 4, 3)
        """
        if isinstance(inputs, list):
            inputs = inputs[0]
        theta = self._dense(self._flatten(inputs))
        theta = tf.reshape(theta, shape=(-1, 4, 3))
        # warp the reference grid with affine parameters to output a ddf
        grid_warped = layer_util.warp_grid(self.reference_grid, theta)
        ddf = grid_warped - self.reference_grid
        return ddf, theta

    def get_config(self):
        """Return the config dictionary for recreating this class."""
        config = super().get_config()
        config.update(image_size=self.reference_grid.shape[:3])
        return config

@REGISTRY.register_backbone(name="global")
class GlobalNet(UNet):
    """
    Build GlobalNet for image registration.

    GlobalNet is a special UNet where the decoder for up-sampling is skipped.
    The network's outputs come from the bottom layer from the encoder directly.

    Reference:

    - Hu, Yipeng, et al.
      "Label-driven weakly-supervised learning
      for multimodal deformable image registration,"
      https://arxiv.org/abs/1711.01666
    """

    def __init__(
        self,
        image_size: tuple,
        num_channel_initial: int,
        extract_levels: Optional[Tuple[int, ...]] = None,
        depth: Optional[int] = None,
        name: str = "GlobalNet",
        **kwargs,
    ):
        """
        Image is encoded gradually, i from level 0 to E.
        Then, a densely-connected layer outputs an affine
        transformation.

        :param image_size: tuple, such as (dim1, dim2, dim3)
        :param num_channel_initial: int, number of initial channels
        :param extract_levels: list, which levels from net to extract, deprecated.
            If depth is not given, depth = max(extract_levels) will be used.
        :param depth: depth of the encoder. If given, extract_levels is not used.
        :param name: name of the backbone.
        :param kwargs: additional arguments.
        """
        if depth is None:
            if extract_levels is None:
                raise ValueError(
                    "GlobalNet requires `depth` or `extract_levels` "
                    "to define the depth of encoder. "
                    "If `depth` is not given, "
                    "the maximum value of `extract_levels` will be used."
                    "However the argument `extract_levels` is deprecated "
                    "and will be removed in future release."
                )
            depth = max(extract_levels)
        super().__init__(
            image_size=image_size,
            num_channel_initial=num_channel_initial,
            depth=depth,
            extract_levels=(depth,),
            name=name,
            **kwargs,
        )

    def build_output_block(
        self,
        image_size: Tuple[int, ...],
        extract_levels: Tuple[int, ...],
        out_channels: int,
        out_kernel_initializer: str,
        out_activation: str,
    ) -> Union[tf.keras.Model, tfkl.Layer]:
        """
        Build a block for output.

        The input to this block is a list of length 1.
        The output has two tensors.

        :param image_size: such as (dim1, dim2, dim3)
        :param extract_levels: not used
        :param out_channels: not used
        :param out_kernel_initializer: not used
        :param out_activation: not used
        :return: a block consists of one or multiple layers
        """
        return AffineHead(image_size=image_size)

ext: py

max_stars_repo_name: Zhiyuan-w/DeepReg

max_issues_repo_name: Zhiyuan-w/DeepReg

max_forks_repo_name: Zhiyuan-w/DeepReg

reason: import numpy


text:
import pandas
import matplotlib
import scipy
import pykov

import sys
import pickle
import re

readName = input("Select which user to read from: ")
#readName = "Suzanne Reed"
readPath = readName + ".dat"
rawDataFile = open(readPath, 'rb')
rawData = pickle.load(rawDataFile)

# Create vector for individual symbol probabilities
dataVector = pykov.Vector()

for message in rawData:
    #Preprocess
    message = message.lower()

    # Split message into words and punctuation
    messageList = re.findall('\w+|[.!?@;()\\-]', message);

    # Add a start for each message
    if '[' in dataVector:
        dataVector['['] = dataVector['['] + 1
    else:
        dataVector['['] = 1;

    # Add all the symbols
    for symbol in messageList:
        if symbol in dataVector:
            dataVector[symbol] = dataVector[symbol] + 1
        else:
            dataVector[symbol] = 1;

    # Add an end for each message
    if ']' in dataVector:
        dataVector[']'] = dataVector[']'] + 1
    else:
        dataVector[']'] = 1;

dataVector.normalize()

# Create matrix of all transitions in data
dataMatrix = pykov.Matrix()

for message in rawData:
    #Preprocess
    message = message.lower()

    # Split message into words and punctuation
    messageList = re.findall('\w+|[.!?@;()\\-]', message);

    # Start with beginner symbol
    lastSymbol = '['

    # step through message and add symbol pairs
    for i in range(len(messageList)):
        currentSymbol = messageList[i]
        if (lastSymbol, currentSymbol) in dataMatrix:
            dataMatrix[(lastSymbol, currentSymbol)] = dataMatrix[(lastSymbol, currentSymbol)] + 1
        else:
            dataMatrix[(lastSymbol, currentSymbol)] = 1

        lastSymbol = currentSymbol

    # End with end symbol
    currentSymbol = ']'
    if (lastSymbol, currentSymbol) in dataMatrix:
        dataMatrix[(lastSymbol, currentSymbol)] = dataMatrix[(lastSymbol, currentSymbol)] + 1
    else:
        dataMatrix[(lastSymbol, currentSymbol)] = 1

# Build square, normalized transition matrix
transitionMatrix = pykov.Matrix()

for symbol in dataVector:
    transitionVector = pykov.Vector()

    for transitionSymbol in dataVector:
        if (symbol, transitionSymbol) in dataMatrix:
            transitionVector[transitionSymbol] = dataMatrix[(symbol, transitionSymbol)]
        else:
            transitionVector[transitionSymbol] = 0

    transitionVector.normalize()

    for transitionSymbol in transitionVector:
        transitionMatrix[(symbol, transitionSymbol)] = transitionVector[transitionSymbol]

# Build chain from transition matrix
dataChain = pykov.Chain(transitionMatrix)

# Save chain due to processing time
writePath = readName + " Chain.dat"
writeDataFile = open(writePath, 'wb')
pickle.dump(dataChain, writeDataFile)
writeDataFile.close()

reason: import scipy


text:
import artm
import numpy as np
import pandas as pd

from topicnet.cooking_machine import Dataset
from topicnet.cooking_machine.models import TopicModel
from typing import List

from topnum.search_methods.topic_bank.phi_initialization import utils as init_phi_utils

def initialize_randomly(
        dataset: Dataset,
        model_number: int,
        num_topics: int) -> pd.DataFrame:

    phi_template = _get_phi_template(dataset, num_topics)

    random = np.random.RandomState(seed=model_number)
    phi_values = random.random(phi_template.shape)

    return pd.DataFrame(
        index=phi_template.index,
        columns=phi_template.columns,
        data=phi_values
    )

def initialize_with_copying_topics(
        dataset: Dataset,
        model_number: int,
        num_topics: int,
        phi: pd.DataFrame,
        num_topics_to_copy: int = None,
        topic_indices_to_copy: List[int] = None) -> pd.DataFrame:

    random = np.random.RandomState(seed=model_number)

    if num_topics_to_copy is None and topic_indices_to_copy is None:
        raise ValueError(
            'Either `num_topics_to_copy` or `topic_indices_to_copy` should be specified!'
        )
    elif topic_indices_to_copy is None:
        topic_indices_to_copy = list(range(len(phi.columns)))
    elif num_topics_to_copy is None:
        num_topics_to_copy = len(topic_indices_to_copy)
    elif num_topics_to_copy != len(topic_indices_to_copy):
        raise ValueError(
            'If both `num_topics_to_copy` and `topic_indices_to_copy` are specified,'
            ' they shouldn\'t contradict each other!'
        )
    else:
        assert False

    topics_to_copy = random.choice(
        topic_indices_to_copy,
        size=num_topics_to_copy,
        replace=False
    )
    artm_model_template = _get_artm_model_template(
        dataset,
        num_topics
    )

    init_phi_utils._copy_phi(
        artm_model_template,
        phi.iloc[:, topics_to_copy]
    )
    model_template = TopicModel(
        artm_model=artm_model_template
    )

    return model_template.get_phi()

def _get_phi_template(dataset: Dataset, num_topics: int) -> pd.DataFrame:
    artm_model = _get_artm_model_template(dataset, num_topics)
    model = TopicModel(artm_model=artm_model)
    phi_template = model.get_phi()

    del model
    del artm_model

    return phi_template

def _get_artm_model_template(dataset: Dataset, num_topics: int) -> artm.ARTM:
    artm_model = artm.ARTM(num_topics=num_topics, num_processors=1)
    artm_model.initialize(dictionary=dataset.get_dictionary())

    return artm_model


text:
import YOLNP
from plate_manager import consult_plate,treat_plate,make_draw
import cv2
import time
import numpy as np
from sys import exit as quitt
from os import system as sy

all_infos = []

video_path = "VIDEO_PATH"

cap = cv2.VideoCapture(video_path)
ret, img = cap.read()
mean_time = []
ss = time.time()

try:
    if ret:
        frameSize = (img.shape[1], img.shape[0])
        out = cv2.VideoWriter('output_video.avi',cv2.VideoWriter_fourcc(*'DIVX'), 30, frameSize)
        detector = YOLNP.Detector()

        while (cv2.waitKey(1) < 0):
            start = time.time()
            ret, image = cap.read()
            if not ret: break

            image_copy = image.copy()
            image_h, image_w, image_c = image.shape
            scale_percent = 10
            width = int(image_w * scale_percent / 100)
            height = int(image_h * scale_percent / 100)
            dim = (width,height)
            image_copy = cv2.resize(image_copy,dim)
            plate_crops, positions = detector.plate(image,False,True)

            plates_content = []
            original_text = []
            img_plate_index = 0

            if plate_crops:
                for crop in plate_crops:
                    plate_content, score, coordinates = detector.paddle_ocr(crop)
                    plates_content.append((treat_plate(plate_content),img_plate_index))
                    original_text.append(plate_content)
                    img_plate_index += 1

            for plate_content,index in plates_content:
                info = consult_plate(plate_content)
                roi = ((positions[index][0],positions[index][1]),( positions[index][0] + positions[index][-2],positions[index][1] + positions[index][-1]))
                cv2.rectangle(image,roi[0],roi[1],(0,255,0),5)
                if info:
                    all_infos.append(info)
                    plate_resource = info[-1]
                    venal = info[0].replace(".",",")
                    if venal == "Aliquota:": venal = "Não Estimado"
                    else: venal = "R$ " + info[0].replace(".",",")
                    marca = info[1]
                    cor = info[5]
                    image = make_draw(image,positions[index],plate_resource, venal, marca, cor)

            out.write(image)
            end = time.time()
            mean_time.append(end-start)
            sy("cls")
            print('frame processed in: {:.2f} seconds\naverage processing time: {:.2f} seconds'.format(end - start,np.mean(mean_time)))

        out.release()

    else:
        raise ValueError("Could not open the file, check the path!")
except:
    out.release()
    quitt()
finally:
    print("Time processed: {:.2f}".format(time.time()-ss))


text:
#
# sawb.py
# Francesco Conti <fconti@iis.ee.ethz.ch>
#
# Copyright (C) 2018-2021 ETH Zurich and University of Bologna
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch
from nemo.precision import Precision
from nemo.quant.pact import *
from nemo.graph import DeployGraph
from torch.nn.modules.utils import _single,_pair
from collections import OrderedDict
import types
import logging
import numpy as np
import copy
import math
import torchvision.models
import re
from nemo.transf.common import *

# first entry is c1, second is c2 with alpha_w^* = c1 * sqrt(Ew2) - c2 * Ew1
__sawb_asymm_lut = {
    2: [8.356, 7.841],
    3: [4.643, 3.729],
    4: [8.356, 7.841],
    5: [12.522, 12.592],
    6: [15.344, 15.914],
    7: [19.767, 21.306],
    8: [26.294, 29.421]
}

# Disable gradients for alpha,beta params
def _disable_grad_sawb(self, layer_bits={}):

    # Colab with SAWB LUT: https://colab.research.google.com/drive/1UEQnvVcSP3N-QTZLEumbbGCv_oLv-JtL
    module_dict = {}
    use_default = False
    if not layer_bits:
        layer_bits = {}
        use_default = True
    for n,m in self.named_modules():
        if (m.__class__.__name__ == "PACT_Conv2d" or \
            m.__class__.__name__ == "PACT_Conv1d" or \
            m.__class__.__name__ == "PACT_Linear"):
            m.W_alpha.requires_grad = False
            m.W_beta.requires_grad = False

# Set weight clipping parameters according to Statistics-Aware Weight Binning
def _weight_clip_sawb(self, asymmetric=True, layer_bits={}, check_minmax=True, verbose=False):

    # Colab with SAWB LUT: https://colab.research.google.com/drive/1UEQnvVcSP3N-QTZLEumbbGCv_oLv-JtL
    module_dict = {}
    use_default = False
    if not layer_bits:
        layer_bits = {}
        use_default = True
    for n,m in self.named_modules():
        if (m.__class__.__name__ == "PACT_Conv2d" or \
            m.__class__.__name__ == "PACT_Conv1d" or \
            m.__class__.__name__ == "PACT_Linear"):
            if use_default:
                module_dict[n] = m
                layer_bits[n] = m.W_precision.get_bits()
            elif n in layer_bits.keys():
                module_dict[n] = m

    for n in module_dict.keys():
        m = module_dict[n]
        # compute E[|w|]
        Ew1 = m.weight.abs().mean()
        # compute E[w^2]
        Ew2 = (m.weight.abs() ** 2).mean()
        # compute alpha
        alpha = __sawb_asymm_lut[layer_bits[n]][0] * torch.sqrt(Ew2) - __sawb_asymm_lut[layer_bits[n]][1] * Ew1
        # compute beta
        eps = 2*alpha / (2**layer_bits[n])
        if asymmetric:
            beta = alpha + eps * (2**layer_bits[n]-1)
        else:
            beta = alpha + eps * 2**layer_bits[n]
        if check_minmax:
            m.W_alpha.data[:] = min(alpha, m.weight.min().abs())
            m.W_beta.data[:]  = min(beta, m.weight.max().abs())
        else:
            m.W_alpha.data[:] = alpha
            m.W_beta.data[:]  = beta
        if verbose:
            print("[weight clip SAWB] %s: Ew1=%.3e Ew2=%.3e alpha=%.3e beta=%.3e" % (n, Ew1, Ew2, m.W_alpha.data.item(), m.W_beta.data.item()))

max_stars_repo_name: SimlaBurcu/nemo

max_issues_repo_name: SimlaBurcu/nemo

max_forks_repo_name: SimlaBurcu/nemo


text:
import numpy as np
import nibabel as nib

def set_new_data(image, new_data, new_dtype=None, remove_nan=True):
    """
    From a nibabel image and a numpy array it creates a new image with
    the same header of the image and the new_data as its data.
    :param image: nibabel image
    :param new_data:
    :param new_dtype: numpy array
    :param remove_nan:
    :return: nibabel image
    """
    hd = image.header
    if remove_nan:
        new_data = np.nan_to_num(new_data)
    # if nifty1
    if hd['sizeof_hdr'] == 348:
        new_image = nib.Nifti1Image(new_data, image.affine, header=hd)
    # if nifty2
    elif hd['sizeof_hdr'] == 540:
        new_image = nib.Nifti2Image(new_data, image.affine, header=hd)
    else:
        raise IOError('Input image header problems in sizeof_hdr.')
    if new_dtype is None:
        new_image.set_data_dtype(new_data.dtype)
        return new_image
    else:
        new_image.set_data_dtype(new_dtype)
        return new_image

def compare_two_nib(im1, im2):
    """
    Comparison between two nibabel imagse.
    :param im1: one nibabel image (or str with path to a nifti image)
    :param im2: another nibabel image (or another str with path to a nifti image)
    :return: true false and plot to console if the images are the same or not (up to a tollerance in the data)
    """
    im1_name = 'First image'
    im2_name = 'Second image'
    msg = ''

    hd1 = im1.header
    hd2 = im2.header

    images_are_equal = True
    # compare nifty version:
    if not hd1['sizeof_hdr'] == hd2['sizeof_hdr']:

        if hd1['sizeof_hdr'] == 348:
            msg += '{0} is nifti1\n{1} is nifti2.'.format(im1_name, im2_name)
        else:
            msg += '{0} is nifti2\n{1} is nifti1.'.format(im1_name, im2_name)

        images_are_equal = False
    # Compare headers:
    else:
        for k in hd1.keys():
            if k not in ['scl_slope', 'scl_inter']:
                val1, val2 = hd1[k], hd2[k]
                are_different = val1 != val2
                if isinstance(val1, np.ndarray):
                    are_different = are_different.any()

                if are_different:
                    images_are_equal = False
                    msg += 'Header Key {0} for {1} is {2} - for {3} is {4}  \n'.format(
                        k, im1_name , hd1[k], im2_name, hd2[k])

            elif not np.isnan(hd1[k]) and np.isnan(hd2[k]):
                images_are_equal = False
                msg += 'Header Key {0} for {1} is {2} - for {3} is {4}  \n'.format(
                    k, im1_name, hd1[k], im2_name, hd2[k])
    # Compare values and type:
    if not im1.get_data_dtype() == im2.get_data_dtype():
        msg += 'Dtype are different consistent {0} {1} - {2} {3} \n'.format(
            im1_name, im1.get_data_dtype(), im2_name, im2.get_data_dtype())
        images_are_equal = False
    if not np.array_equal(im1.get_data(), im2.get_data()):
        msg += 'Data are different. \n'
        images_are_equal = False
        if np.array_equal(np.nan_to_num(im1.get_data()), np.nan_to_num(im2.get_data())):
            msg += '(data are different only up to nans)'
    if not np.array_equal(im1.affine, im2.affine):
        msg += 'Affine transformations are different. \n'
        images_are_equal = False

    print(msg)
    print('\n -- Is it {} that images are equal!'.format(images_are_equal))
    return images_are_equal

def one_voxel_volume(im, decimals=6):
    """
    Volume of a single voxel of the given nifti image
    :param im: nibabel image
    :param decimals: number of decimals rouding the volume
    :return: volume of a single voxel of the input image.
    """
    return np.round(np.abs(np.prod(np.diag(im.affine)[:3])), decimals=decimals)

# ---------- Header modifications ---------------

def modify_image_data_type(im_input, new_dtype, update_descrip_field_header=None, verbose=1, remove_nan=True):
    """
    Change the data dtype of an input image with the specified new dtype.
    The new data dtype must be from numpy data dtype.
    :param im_input: input image
    :param new_dtype: new data dtype
    :param update_descrip_field_header: string to update the 'descrip' parameter of the image header.
    :param verbose: if True, it prompts data dtype before and after.
    :param remove_nan: to remove the nan to the image data if any.
    :return: input image with new data dtype.
    """
    if update_descrip_field_header is not None:
        if not isinstance(update_descrip_field_header, str):
            raise IOError('update_description must be a string')
        hd = im_input.header
        hd['descrip'] = update_descrip_field_header
        im_input.update_header()
    new_im = set_new_data(im_input, im_input.get_data(), new_dtype=new_dtype, remove_nan=remove_nan)
    if verbose > 0:
        print('Data type before {}'.format(im_input.get_data_dtype()))
        print('Data type after {}'.format(new_im.get_data_dtype()))
    return new_im

def modify_affine_transformation(im_input, new_aff, q_form=True, s_form=True, verbose=1, multiplication_side='left'):
    """
    Change q_form or s_form or both translational part and rotational part.
    :param im_input: nibabel input image
    :param new_aff: new affine transformation to be multiplied or replaced
    :param q_form: [True] affect q_form
    :param s_form: [True] affect s_form
    :param multiplication_side: can be 'lef', 'right', or 'replace'.
    :param verbose:

    :return: None. It creates a new image in pfi_nifti_output with defined translational part.

    NOTE: please see the documentation http://nipy.org/nibabel/nifti_images.html#choosing-image-affine for more on the
    relationships between s_form affine, q_form affine and fall-back header affine.
    """
    if np.linalg.det(new_aff) < 0 :
        print('WARNING: affine matrix proposed has negative determinant.')
    if multiplication_side is 'left':
        new_transf = new_aff.dot(im_input.affine)
    elif multiplication_side is 'right':
        new_transf = im_input.affine.dot(new_aff)
    elif multiplication_side is 'replace':
        new_transf = new_aff
    else:
        raise IOError('multiplication_side parameter can be lef, right, or replace.')

    # create output image on the input
    if im_input.header['sizeof_hdr'] == 348:
        new_image = nib.Nifti1Image(im_input.get_data(), new_transf, header=im_input.header)
    # if nifty2
    elif im_input.header['sizeof_hdr'] == 540:
        new_image = nib.Nifti2Image(im_input.get_data(), new_transf, header=im_input.header)
    else:
        raise IOError('Input image header problems in sizeof_hdr.')

    if s_form:
        new_image.set_sform(new_transf)
    else:
        new_image.set_sform(im_input.affine)
    if q_form:
        new_image.set_qform(new_transf)
    else:
        new_image.set_qform(im_input.affine)

    # new_image.update_header()

    if verbose > 0:
        # print intermediate results
        print('Affine input image:')
        print(im_input.affine)
        print('Affine after update:')
        print(new_image.affine)
        print('Q-form after update:')
        print(new_image.get_qform(coded=True))
        print('S-form after update:')
        print(new_image.get_sform(coded=True))

    return new_image

def replace_translational_part(im_input, new_translation, q_form=True, s_form=True):
    """
    Create new image with translational part replaced. Not destructive.
    :param im_input: nibabel input image
    :param new_translation: new translational part
    :param q_form: new translation will be applied to the q_form
    :param s_form: new translation will be applied to the s_form
    :return: copy of the input image with the provided new_translation as translational part.
    Everything else of the image is the same.
    """
    new_affine = np.copy(im_input.affine)
    new_affine[:3, 3] = new_translation[:3]
    im_out = modify_affine_transformation(im_input, new_aff=new_affine, q_form=q_form, s_form=s_form,
                                          multiplication_side='replace')
    return im_out

def remove_nan_from_im(im_input):
    """
    Returns the same input images without nan
    :param im_input: nibabel input image
    :return: set_new_data(im_input, np.nan_to_num(im_input.get_data()))
    """
    return set_new_data(im_input, np.nan_to_num(im_input.get_data()))

def images_are_overlapping(im1, im2):
    """
    Check if two images have overlapping (congruent domain) in the real space (same data and same shape)
    :param im1: nibabel image.
    :param im2: nibabel image.
    :return: np.array_equal(im1.shape[:3], im2.shape[:3]) * np.array_equal(im1.affine, im2.affine)
    """
    return np.array_equal(im1.shape[:3], im2.shape[:3]) and np.array_equal(im1.affine, im2.affine)

max_stars_repo_name: nipy/nilabels

max_issues_repo_name: SebastianoF/LabelsManager

max_forks_repo_name: nipy/nilabels

reason: import numpy,from numpy


text:
import pandas as pd
import numpy as np
import pandas_profiling as pdf

class DataCleaner(object):

    def __init__(self, file_path, test_data_path="NULL"):
        self.data = pd.read_csv(file_path)
        self.original_data = self.data
        if test_data_path == "NULL":
            self.test_data = []
            self.original_test_data = []
        else:
            self.test_data = pd.read_csv(test_data_path)
            self.original_test_data = self.test_data

    def get_data(self):
        return self.data

    def get_original_data(self):
        return self.original_data

    def get_test_data(self):
        return self.test_data

    def get_original_test_data(self):
        return self.original_test_data

    def get_profile(self, profile_location, input_data="NULL"):
        if input_data == "NULL":
            profile = pdf.ProfileReport(self.data)
        else:
            profile = pdf.ProfileReport(input_data)
        profile.to_file(outputfile=profile_location)

    # use "NA" to replace all the missing data
    def replace_missing_data(self, variable_name, current_value, data_indicator=0, target_value="NA"):
        if data_indicator == 0:
            self.data[variable_name] = np.where(
                self.data[variable_name] == current_value, target_value, self.data[variable_name])
        else:
            self.test_data[variable_name] = np.where(
                self.data[variable_name] == current_value, target_value, self.data[variable_name])

    # Missing data imputation
    def impute_missing_data(self, variable, method, data_indicator=0, new_variable="NULL", value=0):
        # numerical: zero
        if new_variable == "NULL":
            new_variable = variable

        # numerical: median (Mean/median imputation has the assumption that the
        # data are missing completely at random (MCAR))
        if method == "median":
            temp_variable = self[variable].fillna(
                self.data[variable].median())

        # numerical: mean
        if method == "mean":
            temp_variable = self[
                variable].fillna(self.data[variable].mean())

        if method == "mode":
            temp_variable = self[
                variable].fillna(self.data[variable].mode())

        # numerical: end of distribution value (data are not missing completely at
        # random)
        if method == "eod":
            extreme = self.data[variable].mean() + 3 * \
                self.data[variable].std()
            temp_variable = self[variable].fillna(extreme)

        # numerical: arbitrary value(data are not missing completely at random)
        if method == "arbitrary":
            temp_variable = self[variable].fillna(value)

        # numerical/categorical: create variable indicating missingness (data are
        # not missing completely at random)
        if method == "binary":
            temp_variable = np.where(
                self.data[variable].isnull(), 1, 0)

        # numerical/categorical: frequency category (the data are missing
        # completely at random (MCAR))
        if method == "frequency":
            most_frequent_category = self.data.groupby(
                [variable])[variable].count().sort_values(ascending=False).index[0]
            temp_variable = self.data[variable].fillna(
                most_frequent_category, inplace=False)

        # categorical: create a category indicating missingness (no assumption)
        if method == "na category":
            temp_variable = np.where(
                self.data[variable].isnull(), "Missing", self.data[variable])

        if data_indicator == 0:
            self.data[new_variable] = temp_variable
        else:
            self.test_data[new_variable] = temp_variable

            # numerical/categorical: random sampling (Random sample imputation assumes
        # that the data are missing completely at random (MCAR))
        if method == "random":
            # extract the random sample to fill the na
            # pandas needs to have the same index in order to merge datasets
            if data_indicator == 0:
                data_sample = self.data[variable].dropna().sample(self.data[variable].isnull().sum(), random_state=0)
                data_sample.index = self.data[self.data[variable].isnull()].index
                self.data.loc[self.data[variable].isnull(), new_variable] = data_sample
            else:
                data_sample = self.data[variable].dropna().sample(self.test_data[variable].isnull().sum(), random_state=0)
                data_sample.index = self.test_data[self.test_data[variable].isnull()].index
                self.test_data.loc[self.test_data[variable].isnull(), new_variable] = data_sample

        # delete whole row
        if method == "drop":
            if data_indicator == 0:
                self.data[pd.notnull(self.data[variable])]
            else:
                self.test_data[pd.notnull(self.data[variable])]


text:
import pandas as pd
import numpy as np
import os
from tqdm import tqdm; tqdm.monitor_interval = 0  # noqa
from data import DicomDataset, load_mask_counts, load_gt

sub_fn = 'x_ensemble/CLF_ADJUST_Both_ENS_0024.csv'
out_fn = os.path.join(os.path.dirname(sub_fn), 'ADJUST_V2_' + os.path.basename(
    sub_fn))
sub = load_gt(sub_fn, rle_key='EncodedPixels')
val = '_VAL' in sub_fn
if val:
  ids = load_mask_counts('train-rle.csv')
else:
  ids = load_mask_counts('sample_submission.csv')
adjusted_sub = {'ImageId': [], 'EncodedPixels': []}

num_removed = 0
num_added = 0
num_missed = 0
for image_id in tqdm(sub):
  rles = sub[image_id]
  num_masks = ids.get(image_id, 1)
  masks = DicomDataset.rles_to_mask(rles, merge_masks=False)
  num_pred = masks.max()
  if num_pred > num_masks:
    sizes = np.float32([(masks == i).sum() for i in range(1, num_pred + 1)])
    inds = np.argsort(-sizes)[:num_masks]
    inds = [range(1, num_pred + 1)[ind] for ind in inds]
    num_removed += len(sizes) - len(inds)
    rles = []
    for ind in inds:
      rles.append(DicomDataset.mask_to_rle((masks == ind).astype(
          'uint8') * 255))

  elif num_masks > num_pred and num_masks >= 2:
    if num_pred >= 1:
      sizes = np.float32([(masks == i).sum() for i in range(1, num_pred + 1)])
      inds = np.argsort(-sizes)[:num_masks]
      inds = [range(1, num_pred + 1)[ind] for ind in inds]
      num_removed += len(sizes) - len(inds)
      for kk in range(num_masks - num_pred):
        ind = inds[kk % len(inds)]
        rles.append(DicomDataset.mask_to_rle((masks == ind).astype(
            'uint8') * 255))
        num_added += 1
    else:
      num_missed += num_masks

  for rle in rles:
    adjusted_sub['EncodedPixels'].append(rle)
    adjusted_sub['ImageId'].append(image_id)

adjusted_sub = pd.DataFrame(adjusted_sub, columns=['ImageId', 'EncodedPixels'])
adjusted_sub.to_csv(out_fn, index=False)
print("Wrote to: %s" % out_fn)
print("Removed: %d | Added: %d | Missed: %d" % (num_removed, num_added,
    num_missed))
# assert len(sub) == adjusted_sub.ImageId.nunique()
print(len(sub))


text:
import unittest
import numpy as np

from openmdao.api import Group, IndepVarComp
from openaerostruct.structures.structural_cg import StructuralCG
from openaerostruct.utils.testing import run_test, get_default_surfaces

np.random.seed(1)

class Test(unittest.TestCase):

    def test(self):
        surface = get_default_surfaces()[0]

        group = Group()

        comp = StructuralCG(surface=surface)

        indep_var_comp = IndepVarComp()

        ny = surface['mesh'].shape[1]

        indep_var_comp.add_output('nodes', val=np.random.random((ny, 3)), units='m')
        indep_var_comp.add_output('structural_mass', val=1., units='kg')
        indep_var_comp.add_output('element_mass', val=np.ones((ny-1)), units='kg')

        group.add_subsystem('indep_var_comp', indep_var_comp, promotes=['*'])
        group.add_subsystem('structural_cg', comp, promotes=['*'])

        run_test(self, group, complex_flag=True, compact_print=False)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: EricUrbineer/OpenAeroStruct

max_issues_repo_name: EricUrbineer/OpenAeroStruct

max_forks_repo_name: EricUrbineer/OpenAeroStruct


text:
import sys
sys.path.append('../src/meta_rule/')
sys.path.append('../dd_lnn/')

import time
import copy
import argparse
from meta_interpretive import BaseMetaPredicate, MetaRule, Project, DisjunctionRule
from train_test import score, align_labels
from read import load_data, load_metadata, load_labels

import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as func
import torch.optim as optim
from utils import evaluate, MyBatchSampler
from torch.utils.data import DataLoader, TensorDataset

#driver for experiments with wn18 link prediction data
if __name__ == "__main__":
    begtime = time.time()
    alpha = 0.95

    parser = argparse.ArgumentParser()
    parser.add_argument("-bk", "--background", required=True, help="file containing schema information")
    parser.add_argument("-ftr", "--facts_train", required=True, help="file containing facts for training")
    parser.add_argument("-fte", "--facts_test", required=True, help="file containing facts for testing")
    parser.add_argument("-t", "--target", required=True, help="the target predicate")

    cl_args = parser.parse_args()
    background_fname = cl_args.background
    facts_fname_train = cl_args.facts_train
    facts_fname_test = cl_args.facts_test
    target = cl_args.target

    dfs_train = load_metadata(background_fname)
    load_data(facts_fname_train, dfs_train)
    print("done reading (" + str(time.time()-begtime) + ")")

    attr_name = None
    labels_df_train = None
    relations_train = []
    rel_names_train = []
    for name, df in dfs_train.items():
        colnames = df.columns.values.tolist()
        attr_name = colnames[0]
        df.columns = [attr_name + "0", attr_name + "1"]

        if target == name:
            labels_df_train = df
        else:
            rel_names_train.append(name)
            relations_train.append(df)

            #creating inv
            name = "inv" + name
            cols = df.columns.tolist()
            inv_df = df[[cols[1], cols[0]]].copy()
            inv_df.columns = [cols[0], cols[1]]
            rel_names_train.append(name)
            relations_train.append(inv_df)

    labels_df_train.columns = [attr_name + "0", attr_name + "3"]
    labels_df_train['Label'] = 1.0

    body0 = BaseMetaPredicate(relations_train)
    print("done body0 (" + str(time.time()-begtime) + "s)")
    body0.df.columns = [attr_name + "0", attr_name + "1"]
    body1 = copy.deepcopy(body0)
    body1.df.columns = [attr_name + "2", attr_name + "3"]
    join = MetaRule([body0, body1], [[[attr_name + "1"], [attr_name + "2"]]], alpha, False)
    print("done join (" + str(time.time()-begtime) + "s)")
    proj = Project(join, [attr_name + "0", attr_name + "3"])
    print("done project (" + str(time.time()-begtime) + "s)")

    meta = proj
    df = labels_df_train
    label = 'Label'
    step = 1e-3
    batch_size = 32
    epochs = 15
    y = align_labels(meta, df, label)

    print("done label alignment (" + str(time.time()-begtime) + "s)")
    data = TensorDataset(torch.arange(y.size()[0]), y)
    all_loader = DataLoader(data, batch_size=batch_size, shuffle=True)

    pos_idx = np.nonzero(y.numpy())[0].tolist()
    print("+ve: " + str(len(pos_idx)))

    if len(pos_idx) > 0:
        pos_loader = DataLoader(TensorDataset(torch.LongTensor(pos_idx)), batch_size=batch_size, shuffle=True)
        optimizer = optim.Adam(meta.parameters(), lr=step)
        loss_fn = nn.BCEWithLogitsLoss(reduction="sum")

        iter = 0
        for epoch in range(epochs):
            pos_loss = 0.0
            for idx in pos_loader:
                yb = torch.ones(idx[0].size()[0], 1)
                meta.train()
                optimizer.zero_grad()
                yhat, slacks = meta(idx[0])
                #yhat = torch.min(yhat, torch.ones(yhat.size()[0], 1)) #making sure yhat does not exceed 1
                #loss = - torch.sum(yhat)
                loss = loss_fn(yhat, yb)
                pos_loss = pos_loss + loss.item()
                print("Epoch " + str(epoch) + " (iteration=" + str(iter) + "): " + str(loss.item()))

                loss.backward()
                optimizer.step()
                iter = iter + 1

            print("Epoch=" + str(epoch) + " +ve.loss=" + str(pos_loss) + " (" + str(time.time()-begtime) + "s)")

    print("done training (" +str(time.time()-begtime) + "s)")

    beta, params = body0.alpha['default'].get_params()
    params = params.detach().numpy()[0].tolist()
    print("LNN-pred (0): ", end='')
    for rel, wt in sorted(zip(rel_names_train, params), key=lambda t: t[1], reverse=True):
        np.set_printoptions(precision=3, suppress=True)
        print(rel + " (" + str(wt) + ") ", end='')
    beta = beta.detach().numpy()
    np.set_printoptions(precision=3, suppress=True)
    print("beta (" + str(beta[0,0]) + ")")
    beta, params = body1.alpha['default'].get_params()
    params = params.detach().numpy()[0].tolist()
    print("LNN-pred (1): ", end='')
    for rel, wt in sorted(zip(rel_names_train, params), key=lambda t: t[1], reverse=True):
        np.set_printoptions(precision=3, suppress=True)
        print(rel + " (" + str(wt) + ") ", end='')
    beta = beta.detach().numpy()
    np.set_printoptions(precision=3, suppress=True)
    print("beta (" + str(beta[0,0]) + ")")
    lnn_beta, lnn_wts, slacks = join.AND['default'].cdd()
    np.set_printoptions(precision=3, suppress=True)
    print("LNN AND beta, weights: " + \
          str(np.around(lnn_beta.item(), decimals=3)) + " " + str(lnn_wts.detach().numpy()))

    #get the target relation's type
    target_relname = -1
    with open('./../data/dbpedia/insnet/relmap') as f:
        for line in f:
            line = line.strip('\n')
            relname, arrow, relid = line.split(" ")
            if target == relid:
                target_relname = relname

    domain_name = "null"
    range_name = "null"
    with open('./../data/dbpedia/insnet/reltype') as f:
        for line in f:
            line = line.strip('\n')
            relname, _domain, _range = line.split(' ')
            if relname == target_relname:
                dummy, domain_name = _domain.split('=')
                dummy, range_name = _range.split('=')

    print(target_relname + " " + domain_name + " " + range_name)
    dest2type = pd.read_csv('./../data/dbpedia/insnet/instancetype', sep=' ', header=0, names=['id', 'type'])

    dfs_test = load_metadata(background_fname)
    load_data(facts_fname_test, dfs_test)

    labels_df_test = dfs_test[target]
    labels_df_test.columns = [attr_name + "0", attr_name + "3"]

    true_links = pd.concat([labels_df_train, labels_df_test])
    true_links.drop(['Label'], axis=1, inplace=True)

    cols = true_links.columns.tolist()
    inv_true_links = true_links[[cols[1], cols[0]]].copy()
    inv_true_links.columns = [cols[0], cols[1]]

    yhat = score(proj, 1024)
    yhat[['new_' + attr_name + "3"]] = yhat[[attr_name + "3"]].apply(pd.to_numeric)
    yhat = pd.merge(yhat, dest2type, \
                    left_on=['new_' + attr_name + "3"], \
                    right_on=['id'], \
                    how='left')
    yhat.drop(['new_' + attr_name + "3", "id"], axis=1, inplace=True)
    yhat.fillna("null", inplace=True)
    print("done evaluation (" + str(time.time()-begtime) + "s)")

    mrr = 0.0
    hits10 = 0.0
    hits1 = 0.0
    hits3 = 0.0
    ranks = []

    scores = pd.merge(labels_df_test, yhat,
                      left_on=[attr_name + "0", attr_name + "3"],
                      right_on=[attr_name + "0", attr_name + "3"],
                      how='left')
    scores.fillna(0, inplace=True)
    scores = scores \
        .groupby([attr_name + "0", attr_name + "3"]) \
        .agg({'prediction': 'max'}) \
        .reset_index()
    print("done retrieving scores (" + str(time.time()-begtime) + "s)")

    false_links = None
    if range_name == 'null':
        false_links = pd.merge(yhat, true_links,
                               left_on=[attr_name + "0", attr_name + "3"],
                               right_on=[attr_name + "0", attr_name + "3"],
                               how='left', indicator=True) \
                        .query("_merge != 'both'") \
                        .drop('_merge', axis=1)
    else:
        false_links = pd.merge(yhat, true_links,
                               left_on=[attr_name + "0", attr_name + "3"],
                               right_on=[attr_name + "0", attr_name + "3"],
                               how='left', indicator=True) \
                        .query("_merge != 'both' and (type == '" + range_name + "' or type == 'null')") \
                        .drop('_merge', axis=1)
    false_links = false_links \
        .groupby([attr_name + "0"])['prediction'] \
        .agg(list)
    print("done join for false links (" + str(time.time()-begtime) + "s)")
    src2false = {}
    for index, value in false_links.iteritems():
        src2false[index] = value
    print("done groupby for false links (" + str(time.time()-begtime) + "s)")

    inv_false_links = None
    if range_name == 'null':
        inv_false_links = pd.merge(yhat, inv_true_links,
                                   left_on=[attr_name + "0", attr_name + "3"],
                                   right_on=[attr_name + "3", attr_name + "0"],
                                   suffixes = ['_left', '_right'],
                                   how='left', indicator=True) \
                            .query("_merge != 'both'") \
                            .drop('_merge', axis=1) \
                            .groupby([attr_name + "0_left"])['prediction'] \
                            .agg(list)
    else:
        inv_false_links = pd.merge(yhat, inv_true_links,
                                   left_on=[attr_name + "0", attr_name + "3"],
                                   right_on=[attr_name + "3", attr_name + "0"],
                                   suffixes = ['_left', '_right'],
                                   how='left', indicator=True) \
                            .query("_merge != 'both' and (type == '" + range_name + "' or type == 'null')") \
                            .drop('_merge', axis=1) \
                            .groupby([attr_name + "0_left"])['prediction'] \
                            .agg(list)
    print("done join for inv_false links (" + str(time.time()-begtime) + "s)")
    src2inv_false = {}
    for index, value in inv_false_links.iteritems():
        src2inv_false[index] = value
    print("done groubby for inv_false links (" + str(time.time()-begtime) + "s)")

    for index, row in scores.iterrows():
        src = row[attr_name + "0"]
        dest = row[attr_name + "3"]
        s = row['prediction']

        rank4dest = 1
        if src in src2false:
            rank4dest = 1 + len([i for i in src2false[src] if i > s])

        rank4src_inv = 1
        if dest in src2inv_false:
            rank4src_inv = 1 + len([i for i in src2inv_false[dest] if i > s])

        rank = min(rank4dest, rank4src_inv)

        mrr += 1.0 / rank
        hits10 = hits10 + 1 if rank <= 10 else hits10
        hits1 = hits1 + 1 if rank <= 1 else hits1
        hits3 = hits3 + 1 if rank <= 3 else hits3
        ranks += [rank]

    numqueries = labels_df_test.shape[0]
    print("queries=" + str(numqueries) + " (" + str(scores.shape[0]) + \
          ") mrr=" + str(mrr) + \
          " hits10=" + str(hits10) + \
          " hits3=" + str(hits3) + \
          " hits1=" + str(hits1) +  \
          " (" + str(time.time()-begtime) + "s)")

max_stars_repo_name: IBM/LOA

max_issues_repo_name: IBM/LOA

max_forks_repo_name: IBM/LOA


text:
from __future__ import division
import sys
import numpy as np
import lxmls.sequences.discriminative_sequence_classifier as dsc
import pdb

class StructuredPerceptron(dsc.DiscriminativeSequenceClassifier):
    """ Implements a first order CRF"""

    def __init__(self, observation_labels, state_labels, feature_mapper,
                 num_epochs=10, learning_rate=1.0, averaged=True):
        dsc.DiscriminativeSequenceClassifier.__init__(self, observation_labels, state_labels, feature_mapper)
        self.num_epochs = num_epochs
        self.learning_rate = learning_rate
        self.averaged = averaged
        self.params_per_epoch = []

    def train_supervised(self, dataset):
        self.parameters = np.zeros(self.feature_mapper.get_num_features())
        num_examples = dataset.size()
        for epoch in xrange(self.num_epochs):
            num_labels_total = 0
            num_mistakes_total = 0
            for i in xrange(num_examples):
                sequence = dataset.seq_list[i]
                num_labels, num_mistakes = self.perceptron_update(sequence)
                num_labels_total += num_labels
                num_mistakes_total += num_mistakes
            self.params_per_epoch.append(self.parameters.copy())
            acc = 1.0 - num_mistakes_total / num_labels_total
            print "Epoch: %i Accuracy: %f" % (epoch, acc)
        self.trained = True

        if self.averaged:
            new_w = 0
            for old_w in self.params_per_epoch:
                new_w += old_w
            new_w /= len(self.params_per_epoch)
            self.parameters = new_w

    def perceptron_update(self, sequence):

        # ----------
        # Solution to Exercise 3.3

        num_labels = 0
        num_mistakes = 0

        predicted_sequence, _ = self.viterbi_decode(sequence)

        y_hat = predicted_sequence.y

        # Update initial features.
        y_t_true = sequence.y[0]
        y_t_hat = y_hat[0]

        if y_t_true != y_t_hat:
            true_initial_features = self.feature_mapper.get_initial_features(sequence, y_t_true)
            self.parameters[true_initial_features] += self.learning_rate
            hat_initial_features = self.feature_mapper.get_initial_features(sequence, y_t_hat)
            self.parameters[hat_initial_features] -= self.learning_rate

        for pos in xrange(len(sequence.x)):
            y_t_true = sequence.y[pos]
            y_t_hat = y_hat[pos]

            # Update emission features.
            num_labels += 1
            if y_t_true != y_t_hat:
                num_mistakes += 1
                true_emission_features = self.feature_mapper.get_emission_features(sequence, pos, y_t_true)
                self.parameters[true_emission_features] += self.learning_rate
                hat_emission_features = self.feature_mapper.get_emission_features(sequence, pos, y_t_hat)
                self.parameters[hat_emission_features] -= self.learning_rate

            if pos > 0:
                # update bigram features
                # If true bigram != predicted bigram update bigram features
                prev_y_t_true = sequence.y[pos-1]
                prev_y_t_hat = y_hat[pos-1]
                if y_t_true != y_t_hat or prev_y_t_true != prev_y_t_hat:
                    true_transition_features = self.feature_mapper.get_transition_features(
                        sequence, pos-1, y_t_true, prev_y_t_true)
                    self.parameters[true_transition_features] += self.learning_rate
                    hat_transition_features = self.feature_mapper.get_transition_features(
                        sequence, pos-1, y_t_hat, prev_y_t_hat)
                    self.parameters[hat_transition_features] -= self.learning_rate

        pos = len(sequence.x)
        y_t_true = sequence.y[pos-1]
        y_t_hat = y_hat[pos-1]

        if y_t_true != y_t_hat:
            true_final_features = self.feature_mapper.get_final_features(sequence, y_t_true)
            self.parameters[true_final_features] += self.learning_rate
            hat_final_features = self.feature_mapper.get_final_features(sequence, y_t_hat)
            self.parameters[hat_final_features] -= self.learning_rate

        return num_labels, num_mistakes

        # End of solution to Exercise 3.3
        # ----------

    def save_model(self, dir):
        fn = open(dir + "parameters.txt", 'w')
        for p_id, p in enumerate(self.parameters):
            fn.write("%i\t%f\n" % (p_id, p))
        fn.close()

    def load_model(self, dir):
        fn = open(dir + "parameters.txt", 'r')
        for line in fn:
            toks = line.strip().split("\t")
            p_id = int(toks[0])
            p = float(toks[1])
            self.parameters[p_id] = p
        fn.close()

max_stars_repo_name: khui/lxmls-toolkit

max_issues_repo_name: khui/lxmls-toolkit

max_forks_repo_name: khui/lxmls-toolkit


text:
import argparse
from py_utils import utils
import sys
import numpy as np
from tqdm import tqdm

IN_EXTENSIONS = [".mp4", ".avi"]

def main():

    if (
        g_args.input == g_args.output
        or g_args.output == "."
        or g_args.output == g_args.script_dir
    ):
        exit("IN cannot be the same as OUT, '.' or script path.")

    # if not (utils.confirm_overwrite(g_args.output, "n")):
    #     exit("Aborted folder creation.")

    utils.make_dir(g_args.output)

    in_files = utils.get_file_paths(g_args.input, *IN_EXTENSIONS)

    for f in tqdm(in_files, desc="processing"):
        fn = utils.get_file_name(f)
        out_file = utils.join_paths(g_args.output, f"{fn}.webm")
        if not utils.exists_file(out_file):
            cmd = f"ffmpeg -i {f} -b:v 1M  -vf chromakey=0x00FF00:0.27:0.1 -c:v libvpx -pix_fmt yuva420p -metadata:s:v:0 alpha_mode='1' -auto-alt-ref 0 {out_file}"
            utils.exec_shell_command(cmd, True)

def exit(msg=None):
    if msg:
        print(f"{msg}")
    print("Exit script.")
    sys.exit()

def parse_args():
    # construct the argument parser and parse the arguments
    ap = argparse.ArgumentParser()
    ap.add_argument(
        "-i",
        "--input",
        dest="input",
        type=utils.to_path,
        help="path to input folder",
        required=True,
        # multiple (at least one) arguments gathered into list
        # nargs='+',
    )
    ap.add_argument(
        "-o",
        "--output",
        dest="output",
        type=utils.to_path,
        help="path to output folder",
        # either argument is given or current './out' is used by default
        nargs="?",
        default=utils.to_path("./out"),
    )
    args = ap.parse_args()
    args.script_dir = utils.get_script_dir()
    args.current_dir = utils.get_current_dir()
    return args

if __name__ == "__main__":
    # parse args
    g_args = parse_args()  # can be accessed globally
    # call main function
    main()

max_stars_repo_name: Kruemelkatze/whistling-weirdos

max_issues_repo_name: Kruemelkatze/whistling-weirdos

max_forks_repo_name: Kruemelkatze/whistling-weirdos


text:
import numpy as np

def getTotalCurrentBaseInversion(purchase_operations):
    if len(purchase_operations) > 0:
        return np.sum([x[3] for x in purchase_operations])
    else:
        return 0.0

def getTotalCurrentQuoteInversion(purchase_operations):
    if len(purchase_operations) > 0:
        return np.sum([x[4] for x in purchase_operations])
    else:
        return 0.0

def getCurrentPercentBenefit(purchase_operations, current_sell_price):
    if len(purchase_operations) == 0:
        return 0
    purchase_prices = [x[0] for x in purchase_operations]
    purchase_fees = [x[2] for x in purchase_operations]
    purchase_amounts = [x[4] for x in purchase_operations]

    purchases = []
    total_amounts = 0.0

    for price, fee, amount in zip(purchase_prices, purchase_fees, purchase_amounts):
        real_price = price + (price*fee)
        total_amounts += amount
        purchases.append(real_price*amount)
    if total_amounts == 0:
        return 0
    average_purchase_price = np.sum(purchases) / total_amounts

    current_benefit = (current_sell_price/average_purchase_price) - 1.0
    return 100*current_benefit

def improveMeanPrice(purchase_operations, current_price):
    if len(purchase_operations) == 0:
        return True
    purchase_prices = [x[0] for x in purchase_operations]
    average_purchase_price = np.mean(purchase_prices)

    if current_price < (average_purchase_price - (average_purchase_price*0.002)):
        return True
    else:
        return False

def percentImproveMeanPrice(purchase_operations, current_price):
    if len(purchase_operations) == 0:
        return 0
    purchase_prices = [x[0] for x in purchase_operations]
    average_purchase_price = np.mean(purchase_prices)

    r = (current_price/average_purchase_price) - 1.0

    if r < 0:
        return abs(r)*100
    else:
        return 0

def getCurrentMeanPrice(purchase_operations):
    if len(purchase_operations) == 0:
        return 0
    purchase_prices = [x[0] for x in purchase_operations]
    average_purchase_price = np.mean(purchase_prices)
    return average_purchase_price

def getCurrentRealMeanPrice(purchase_operations):
    if len(purchase_operations) == 0:
        return 0
    purchase_prices = [x[0] for x in purchase_operations]
    purchase_fees = [x[2] for x in purchase_operations]
    purchase_amounts = [x[4] for x in purchase_operations]
    real_purchase_operations = []
    for price, fee, amount in zip(purchase_prices, purchase_fees, purchase_amounts):
        real_price = price + (price*fee)
        real_purchase_operations.append((real_price, amount))
    num = 0.0
    den = 0.0
    for p, a in real_purchase_operations:
        num += p*a
        den += a
    if den == 0:
        return 0
    average_mean_price = num/den
    return average_mean_price

def get_amount_to_buy_in_quote(logger,purchase_operations,purchase_price,initial_amount_to_buy_in_base,min_amount_to_buy_in_base,max_amount_to_buy_in_base,r):
    if len(purchase_operations) < 1:
        return initial_amount_to_buy_in_base/purchase_price
    purchase_prices = [x[0] for x in purchase_operations ]
    purchase_fees = [x[2] for x in purchase_operations ]
    purchase_amounts = [x[4] for x in purchase_operations ]
    real_purchase_operations = []
    for price,fee,amount in  zip(purchase_prices,purchase_fees,purchase_amounts):
        real_price = price + (price*fee)
        real_purchase_operations.append((real_price,amount))
    C = 0.0
    I = 0.0
    for p,a in real_purchase_operations:
        C += p*a
        I += a
    if C == 0:
        return 0
    amount_to_buy_in_quote = (C - (purchase_price*(1+r)*I))/(purchase_price*r)
    logger.debug("calculating amount_to_buy_in_quote:{}".format(amount_to_buy_in_quote))
    amount_to_buy_in_base = amount_to_buy_in_quote * purchase_price
    logger.debug("calculating amount_to_buy_in_base:{}".format(amount_to_buy_in_base))
    if amount_to_buy_in_base < min_amount_to_buy_in_base:
        amount_to_buy_in_base = min_amount_to_buy_in_base
        amount_to_buy_in_quote = amount_to_buy_in_base/purchase_price
    if amount_to_buy_in_base > max_amount_to_buy_in_base:
        amount_to_buy_in_base = max_amount_to_buy_in_base
        amount_to_buy_in_quote = amount_to_buy_in_base/purchase_price

    logger.debug("final amount_to_buy_in_base:{}".format(amount_to_buy_in_base))
    logger.debug("final amount_to_buy_in_quote:{}".format(amount_to_buy_in_quote))
    return amount_to_buy_in_quote


text:
import numpy as np
from sklearn.model_selection import StratifiedKFold, train_test_split, GroupShuffleSplit, ShuffleSplit
from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, roc_curve
from sklearn.ensemble import RandomForestClassifier

def block_hour(dataset, labels, patient_id_array, hours=4):
    new_dataset = []
    for column in dataset.T:
        new_feature = []
        new_labels = []
        for patient_id in np.unique(patient_id_array):
            print("Patient ID: ", patient_id)
            patient_index = np.where(patient_id_array == patient_id)[0]
            patient = dataset[patient_index]
            patient_labels = labels[patient_index]
            for i in range(len(patient_labels) - hours):
                if 1 in patient_labels[i:i + hours]:
                    num = 1
                else:
                    num = 0
                new_feature.append(column[i:i + hours])
                new_labels.append(num)
        new_dataset.append(np.array(new_feature))
    return np.array(new_dataset), np.array(new_labels)

features = np.nan_to_num(np.load('Datasets/training_1_nanfill.npy'))
labels = np.load('Datasets/training_1_Y.npy')
patient_id = np.load('Datasets/training_1_patient.npy')

X, y = block_hour(features, labels, patient_id, 4)
# X, y = features, labels

subs = ['MEAN', 'NORMAL', 'ZERO']
i = 0
acc = {}
f1 = {}
auc = {}
skf = StratifiedKFold(n_splits=10)

acc[subs[i]] = []
f1[subs[i]] = []
auc[subs[i]] = []
res = []
y_test_all = []

for train_index, test_index in skf.split(X[0], y):
    print("TRAIN:", train_index, "TEST:", test_index)
    aux_res = []
    for feature in X:
        X_train, X_test = feature[train_index], feature[test_index]
        y_train, y_test = y[train_index], y[test_index]

        elf = RandomForestClassifier(n_estimators=20)
        print("Start training...")

        elf = elf.fit(X_train, y_train)
        print("Start testing...")
        pred = elf.predict_proba(X_test)[:, 1]
        results = elf.predict(X_test)
        aux_res.append(pred)
        print(subs[i], " Accuracy: ", accuracy_score(results, y_test))
        print(subs[i], "F1-Score: ", f1_score(results, y_test))

    res.append(pred)
    y_test_all.append(y_test)
    acc[subs[i]].append(accuracy_score(results, y_test))
    f1[subs[i]].append(f1_score(results, y_test))
    auc[subs[i]].append(roc_auc_score(y_test, pred))
    print(subs[i], " Accuracy: ", accuracy_score(results, y_test))
    print(subs[i], "F1-Score: ", f1_score(results, y_test))
    print(subs[i], "AUC: ", auc)

res = np.concatenate(res)
y_test_all = np.concatenate(y_test_all)

fpr, tpr, thresholds = roc_curve(y_test_all, res, pos_label=1)

threshold = 0
accuracy = []
f1_score_list = []
step = 0.001

for threshold in np.arange(0, 1, step):
    print(threshold)
    new_results = np.zeros(len(res))
    new_results[np.where(res > threshold)[0]] = 1
    new_results[np.where(res <= threshold)[0]] = 0
    accuracy.append(accuracy_score(y_test_all, new_results))
    f1_score_list.append(f1_score(y_test_all, new_results))
print(accuracy_score(y_test_all, new_results))
print(f1_score(y_test_all, new_results))

new_threshold = np.array(f1_score_list).argmax() * step
print(subs[i], "Threshold:", new_threshold)

new_results = np.zeros(len(res))
new_results[np.where(res > new_threshold)[0]] = 1
new_results[np.where(res <= new_threshold)[0]] = 0

acc[subs[i]].append(accuracy_score(y_test_all, new_results))
f1[subs[i]].append(f1_score(y_test_all, new_results))
print(subs[i], "Accuracy th: ", accuracy_score(y_test_all, new_results))
print(subs[i], "F1-Score th: ",f1_score(y_test_all, new_results))


text:
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import functools
import itertools

from absl.testing import absltest
from absl.testing import parameterized

import numpy as onp
import scipy.misc as osp_misc
import scipy.special as osp_special
import scipy.stats as osp_stats

from jax import api
from jax import test_util as jtu
from jax.scipy import misc as lsp_misc
from jax.scipy import special as lsp_special
from jax.scipy import stats as lsp_stats

from jax.config import config
config.parse_flags_with_absl()
FLAGS = config.FLAGS

all_shapes = [(), (4,), (3, 4), (3, 1), (1, 4), (2, 1, 4)]

float_dtypes = [onp.float32, onp.float64]
complex_dtypes = [onp.complex64]
int_dtypes = [onp.int32, onp.int64]
bool_dtypes = [onp.bool_]
default_dtypes = float_dtypes + int_dtypes
numeric_dtypes = float_dtypes + complex_dtypes + int_dtypes

OpRecord = collections.namedtuple("OpRecord", ["name", "nargs", "dtypes", "rng",
                                               "test_autodiff", "test_name"])

def op_record(name, nargs, dtypes, rng, test_grad, test_name=None):
  test_name = test_name or name
  return OpRecord(name, nargs, dtypes, rng, test_grad, test_name)

JAX_SPECIAL_FUNCTION_RECORDS = [
    # TODO: digamma has no JVP implemented.
    op_record("digamma", 1, float_dtypes, jtu.rand_positive(), False),
    op_record("erf", 1, float_dtypes, jtu.rand_small_positive(), True),
    op_record("erfc", 1, float_dtypes, jtu.rand_small_positive(), True),
    op_record("erfinv", 1, float_dtypes, jtu.rand_small_positive(), True),
    op_record("expit", 1, float_dtypes, jtu.rand_small_positive(), True),
    # TODO: gammaln has slightly high error.
    op_record("gammaln", 1, float_dtypes, jtu.rand_positive(), False),
    # TODO: NaNs in gradient for logit.
    op_record("logit", 1, float_dtypes, jtu.rand_small_positive(), False),
    op_record("log_ndtr", 1, float_dtypes, jtu.rand_default(), True),
    op_record("ndtri", 1, float_dtypes, jtu.rand_uniform(0., 1.), True),
    op_record("ndtr", 1, float_dtypes, jtu.rand_default(), True),
]

CombosWithReplacement = itertools.combinations_with_replacement

class LaxBackedScipyTests(jtu.JaxTestCase):
  """Tests for LAX-backed Scipy implementation."""

  def _GetArgsMaker(self, rng, shapes, dtypes):
    return lambda: [rng(shape, dtype) for shape, dtype in zip(shapes, dtypes)]

  @parameterized.named_parameters(jtu.cases_from_list(
      {"testcase_name": "_inshape={}_axis={}_keepdims={}".format(
          jtu.format_shape_dtype_string(shape, dtype), axis, keepdims),
       "rng": jtu.rand_default(), "shape": shape, "dtype": dtype,
       "axis": axis, "keepdims": keepdims}
      for shape in all_shapes for dtype in float_dtypes
      for axis in range(-len(shape), len(shape))
      for keepdims in [False, True]))
  @jtu.skip_on_flag("jax_xla_backend", "xrt")
  def testLogSumExp(self, rng, shape, dtype, axis, keepdims):
    # TODO(mattjj): test autodiff
    def scipy_fun(array_to_reduce):
      return osp_special.logsumexp(array_to_reduce, axis, keepdims=keepdims)

    def lax_fun(array_to_reduce):
      return lsp_special.logsumexp(array_to_reduce, axis, keepdims=keepdims)

    args_maker = lambda: [rng(shape, dtype)]
    self._CheckAgainstNumpy(scipy_fun, lax_fun, args_maker, check_dtypes=True)
    self._CompileAndCheck(lax_fun, args_maker, check_dtypes=True)

  @parameterized.named_parameters(jtu.cases_from_list(
      {"testcase_name": jtu.format_test_name_suffix(
          rec.test_name, shapes, dtypes),
       "rng": rec.rng, "shapes": shapes, "dtypes": dtypes,
       "test_autodiff": rec.test_autodiff,
       "scipy_op": getattr(osp_special, rec.name),
       "lax_op": getattr(lsp_special, rec.name)}
      for rec in JAX_SPECIAL_FUNCTION_RECORDS
      for shapes in CombosWithReplacement(all_shapes, rec.nargs)
      for dtypes in CombosWithReplacement(rec.dtypes, rec.nargs)))
  def testScipySpecialFun(self, scipy_op, lax_op, rng, shapes, dtypes,
                          test_autodiff):
    # TODO(mattjj): unskip this test combination when real() on tpu is improved
    if (FLAGS.jax_test_dut and FLAGS.jax_test_dut.startswith("tpu")
        and not shapes[0]):
      return absltest.unittest.skip("real() on scalar not supported on tpu")

    args_maker = self._GetArgsMaker(rng, shapes, dtypes)
    args = args_maker()
    self.assertAllClose(scipy_op(*args), lax_op(*args), atol=1e-3, rtol=1e-3,
                        check_dtypes=False)
    self._CompileAndCheck(lax_op, args_maker, check_dtypes=True)

    if test_autodiff:
      jtu.check_grads(lax_op, args, order=1, atol=1e-3, rtol=3e-3)

if __name__ == "__main__":
  absltest.main()

max_stars_repo_name: jekbradbury/jax

reason: import numpy,import scipy,from jax