text:
"""
# Filename: nsgaiiengine.py
# Description:
# Created by ngocjr7 on [09-06-2020 16:05:40]
"""
from __future__ import absolute_import

from functools import wraps
from typing import List, Union, Callable, Dict
from random import Random
from tqdm.auto import tqdm

from geneticpython.core import Individual, Population, Pareto
from geneticpython.core.operators import Selection, Crossover, Mutation
from geneticpython.core.operators import Replacement, RankReplacement, TournamentSelection
from geneticpython.engines.geneticengine import GeneticEngine
from geneticpython.callbacks import CallbackList, Callback, History
from geneticpython.engines.multi_objective.multi_objective_engine import MultiObjectiveEngine, is_dominated
from geneticpython.utils.validation import check_random_state

import random
import math
import itertools
import numpy as np

def default_crowded_comparator(p1: Individual, p2: Individual) -> int:
    if p1.nondominated_rank < p2.nondominated_rank:
        return -1
    elif p1.nondominated_rank > p2.nondominated_rank:
        return 1
    else:
        if p1.crowding_distance > p2.crowding_distance:
            return -1
        elif p1.crowding_distance < p2.crowding_distance:
            return 1
        else:
            return 0

class NSGAIIEngine(MultiObjectiveEngine):

    def __init__(self, population: Population,
                 objectives: List[Callable[[Individual], Union[float, int]]] = None,
                 tournament_size: int = 2,
                 selection_size: int = None,
                 crossover: Crossover = None,
                 mutation: Mutation = None,
                 callbacks: List[Callback] = None,
                 generations: int = 100,
                 random_state: int = None,
                 crowded_comparator: Callable[[Individual, Individual], int] = None):

        replacement = RankReplacement()
        selection = TournamentSelection(tournament_size)
        self.crowded_comparator = crowded_comparator or default_crowded_comparator

        super(NSGAIIEngine, self).__init__(population=population,
                                           objectives=objectives,
                                           selection=selection,
                                           selection_size=selection_size,
                                           crossover=crossover,
                                           mutation=mutation,
                                           replacement=replacement,
                                           callbacks=callbacks,
                                           generations=generations,
                                           random_state=random_state)

    @staticmethod
    def nondominated_sort(population: List[Individual]) -> List[List[Individual]]:
        """
            This function uses Individuals.objectives for compararing

            :param population: list of individual
            :type population: List[Individual]

            :return: List of front (List[Individual]) after nondominated sorting
            :rtype: List[List[Individual]]
        """
        fronts = list()
        num_dominators = dict()
        slaves = dict()

        # find fisrt pareto front
        fronts.append(list())
        for p1 in population:
            slaves[p1] = list()
            num_dominators[p1] = 0

            for p2 in population:
                if is_dominated(p1, p2):
                    slaves[p1].append(p2)
                elif is_dominated(p2, p1):
                    num_dominators[p1] += 1

            if num_dominators[p1] == 0:
                fronts[0].append(p1)

        # find other pareto front
        i = 0
        while len(fronts[i]) > 0:
            current_front = list()
            for p1 in fronts[i]:
                for p2 in slaves[p1]:
                    num_dominators[p2] -= 1
                    if num_dominators[p2] == 0:
                        current_front.append(p2)

            i += 1
            fronts.append(current_front)

        if len(fronts[-1]) == 0:
            fronts.pop()

        return fronts

    @staticmethod
    def calc_crowding_distance(front: List[Individual]) -> Dict[Individual, float]:
        """
            compute crowding distance for each individual in pareto front

            :param front: a list of individual in pareto front
            :type front: List[Individual]

            :return: crowding distance of each individual, stored in a dict
            :rtype: Dict[Individual, float]
        """
        crowding_distance = dict()
        n = len(front)
        for i in range(n):
            crowding_distance[front[i]] = 0.0

        for m in range(len(front[0]._objectives)):
            front.sort(key=lambda indv: indv._objectives[m])
            crowding_distance[front[0]
                              ] = crowding_distance[front[n - 1]] = float('inf')
            delta_f = front[n - 1]._objectives[m] - front[0]._objectives[m]
            if delta_f != 0.0:
                for i in range(1, n - 1):
                    crowding_distance[front[i]] += (
                        front[i + 1]._objectives[m] - front[i - 1]._objectives[m]) / delta_f

        return crowding_distance

    @staticmethod
    def sort(population: List[Individual], random_state=None) -> List[Individual]:
        random_state = check_random_state(random_state)
        # non-dominated sorting
        pareto_fronts = NSGAIIEngine.nondominated_sort(population)

        # assign nondominated rank
        for rank in range(len(pareto_fronts)):
            for i in range(len(pareto_fronts[rank])):
                pareto_fronts[rank][i].nondominated_rank = rank

        # compute and assign  crowding distance
        for rank in range(len(pareto_fronts)):
            crowding_distance_dict = NSGAIIEngine.calc_crowding_distance(
                pareto_fronts[rank])
            for i in range(len(pareto_fronts[rank])):
                pareto_fronts[rank][i].crowding_distance = crowding_distance_dict[pareto_fronts[rank][i]]

            random_state.shuffle(pareto_fronts[rank])

            pareto_fronts[rank].sort(
                key=lambda indv: indv.crowding_distance, reverse=True)

        flatten_population = list(itertools.chain.from_iterable(pareto_fronts))

        return flatten_population

    def do_initialization(self) -> List[Individual]:
        # Injecting nondominated_rank and crowding_distance for template individuals
        # This just makes it easier to understand
        self.population.individual_temp.nondominated_rank = None
        self.population.individual_temp.crowding_distance = None
        # init population
        population = self.population.init_population(self.random_state)

        return population

    def do_evaluation(self, population: List[Individual]) -> List[Individual]:
        population = NSGAIIEngine.sort(population, self.random_state)
        return population

    def do_reproduction(self, mating_population: List[Individual]) -> List[Individual]:
        return super(NSGAIIEngine, self).do_reproduction(mating_population)

    def do_selection(self) -> List[Individual]:
        try:
            mating_population = self.selection.select(self.selection_size,
                                                      population=self.population.individuals,
                                                      comparator=self.crowded_comparator,
                                                      random_state=self.random_state)
        except TypeError as err:
            raise TypeError(" {}\nSelection does not support nondominated comparator,\n \
                    TournamentSelection is recommended".format(err))

        return mating_population

    def do_replacement(self, new_population) -> List[Individual]:
        new_population = self.replacement.replace(
            self.population.size,
            new_population,
            comparator=self.crowded_comparator,
            sorted=True,
            random_state=self.random_state)
        return new_population

    def run(self, generations: int = None) -> History:
        return super(NSGAIIEngine, self).run(generations)

    def get_pareto_front(self) -> Pareto:
        pareto_front = list()
        for indv in self.population.individuals:
            try:
                if indv.nondominated_rank == 0:
                    pareto_front.append(indv)
            except:
                raise ValueError(f"Cannot get nondominated_rank from individuals\
                                 The engine has not run yet")

        return Pareto(pareto_front)

if __name__ == '__main__':
    pass


text:
import time
from enum import Enum

import numpy as np

class EpisodeStateCode(Enum):
    IN_PROGRESS = 1
    DRAW = 2
    WIN = 3

class TickTacToeEnvironment:
    player_sign_dict = {
        0: '_',
        1: 'X',
        2: '0'
    }

    def __init__(self):
        self.game_cells = np.zeros((3, 3), dtype=np.int32)

    def _check_win(self):
        for i in range(3):
            if self.game_cells[i, 0] == self.game_cells[i, 1] == self.game_cells[i, 2] and self.game_cells[i, 0] != 0:
                return True

        for j in range(3):
            if self.game_cells[0, j] == self.game_cells[1, j] == self.game_cells[2, j] and self.game_cells[0, j] != 0:
                return True

        if self.game_cells[0, 0] == self.game_cells[1, 1] == self.game_cells[2, 2] and self.game_cells[0, 0] != 0:
            return True
        if self.game_cells[0, 2] == self.game_cells[1, 1] == self.game_cells[2, 0] and self.game_cells[0, 2] != 0:
            return True
        return False

    def _set_cell(self, action, player_number):
        row = action // 3
        column = action % 3

        if self.game_cells[row, column] != 0:
            raise ValueError("Action can only be made for for a free cell, cell {0} is occupied".format(action + 1))
        self.game_cells[row, column] = player_number

    def _obtain_observation(self, player_number):
        game_cells_copy = self.game_cells.copy()
        if player_number == 2:
            for i in range(3):
                for j in range(3):
                    if game_cells_copy[i, j] == 1:
                        game_cells_copy[i, j] = 2
                    elif game_cells_copy[i, j] == 2:
                        game_cells_copy[i, j] = 1
        return game_cells_copy.ravel()

    def reset(self):
        self.game_cells = np.zeros((3, 3), dtype=np.int32)
        return self.game_cells.ravel()

    def step(self, action, action_player_id, observation_player_id):
        episode_state_code = EpisodeStateCode.IN_PROGRESS
        self._set_cell(action, action_player_id)
        if self._check_win():
            episode_state_code = EpisodeStateCode.WIN
        else:
            free_cells, _ = self.get_free_and_occupied_cells()
            if len(free_cells) == 0:
                episode_state_code = EpisodeStateCode.DRAW

        return self._obtain_observation(observation_player_id), episode_state_code

    def render(self):
        "Can be improved"
        print('\n' * 25)
        for i in range(3):
            for j in range(3):
                print(self.player_sign_dict[self.game_cells[i, j]] + ' ', end="")
                if j == 2:
                    print()
        time.sleep(0.8)
        print()

    def get_free_and_occupied_cells(self):
        free_cells = []
        occupied_cells = []
        for i in range(3):
            for j in range(3):
                cell_index_raveled = i * 3 + j
                if self.game_cells[i, j] == 0:
                    free_cells.append(cell_index_raveled)
                else:
                    occupied_cells.append(cell_index_raveled)
        return free_cells, occupied_cells

ext: py

max_stars_repo_name: ViacheslavBobrov/ReinforcementLearning

max_issues_repo_name: ViacheslavBobrov/ReinforcementLearning

max_forks_repo_name: ViacheslavBobrov/ReinforcementLearning

reason: import numpy


text:
from pygrn.grns.base import GRN
from pygrn import config
from copy import deepcopy
import numpy as np
import tensorflow as tf

class DiffGRN(GRN):

    def __init__(self):
        pass

    def reset(self):
        self.tf_input_conc = tf.multiply(1.0/self.size(),
                                         tf.ones([self.num_input]))
        self.tf_output_conc = tf.multiply(1.0/self.size(),
                                          tf.ones([self.num_output]))
        self.tf_regulatory_conc = tf.multiply(1.0/self.size(),
                                              tf.ones([self.num_regulatory]))
        return self

    def warmup(self, nsteps):
        self.set_input(np.zeros(self.num_input))
        for i in range(nsteps):
            self.step()

    def convert_to_tensor(self):
        self.tf_identifiers = tf.convert_to_tensor(self.identifiers,
                                                   dtype=tf.float32)
        self.tf_enhancers = tf.convert_to_tensor(self.enhancers,
                                                 dtype=tf.float32)
        self.tf_inhibitors = tf.convert_to_tensor(self.inhibitors,
                                                  dtype=tf.float32)
        self.tf_beta = tf.convert_to_tensor(self.beta, dtype=tf.float32)
        self.tf_delta = tf.convert_to_tensor(self.delta, dtype=tf.float32)

    def setup(self):
        if not hasattr(self, "tf_identifiers"):
            self.convert_to_tensor()
        self.reset()
        ids = tf.maximum(0.0, tf.minimum(1.0, self.tf_identifiers))
        enh = tf.maximum(0.0, tf.minimum(1.0, self.tf_enhancers))
        inh = tf.maximum(0.0, tf.minimum(1.0, self.tf_inhibitors))
        beta = tf.maximum(config.BETA_MIN, tf.minimum(config.BETA_MAX,
                                                      self.tf_beta))
        delta = tf.maximum(config.DELTA_MIN, tf.minimum(config.DELTA_MAX,
                                                        self.tf_delta))

        ids = tf.reshape(
            tf.tile(ids, [self.size()]), [self.size(), self.size()])
        enh = tf.transpose(tf.reshape(
            tf.tile(enh, [self.size()]), [self.size(), self.size()]))
        inh = tf.transpose(tf.reshape(
            tf.tile(inh, [self.size()]), [self.size(), self.size()]))
        self.tf_enhance_match = tf.exp(-beta * tf.abs(enh - ids))
        self.tf_inhibit_match = tf.exp(-beta * tf.abs(inh - ids))
        self.tf_sigs = self.tf_enhance_match - self.tf_inhibit_match
        self.tf_delta_n = tf.divide(delta, tf.to_float(self.size()))
        self.tf_output_mask = tf.convert_to_tensor(
            np.concatenate((np.ones(self.num_input),
                            np.zeros(self.num_output),
                            np.ones(self.num_regulatory))),
            dtype=tf.float32)

    def get_signatures(self):
        with tf.Session() as s:
            return s.run(self.tf_sigs)

    def get_concentrations(self):
        with tf.Session() as s:
            return s.run(tf.concat([self.tf_input_conc,
                                    self.tf_output_conc,
                                    self.tf_regulatory_conc], 0))

    def set_input(self, input_t):
        inp_concs = tf.convert_to_tensor(input_t, dtype=tf.float32)
        self.tf_input_conc = inp_concs

    def step(self):
        concs = tf.concat([self.tf_input_conc, self.tf_output_conc,
                           self.tf_regulatory_conc], 0)
        conc_diff = tf.multiply(concs, self.tf_output_mask)
        conc_diff = tf.reshape(conc_diff, [1, self.size()])
        conc_diff = tf.matmul(conc_diff, self.tf_sigs)
        conc_diff = tf.multiply(self.tf_delta_n, conc_diff)
        concs = tf.add(concs, conc_diff)
        concs = tf.maximum(0.0, concs)
        concs = tf.reshape(concs, [self.size()])
        _, regs = tf.split(concs, [self.num_input,
                                   self.num_regulatory+self.num_output])
        sumconcs = tf.reduce_sum(regs)
        concs = tf.cond(tf.greater(sumconcs, 0),
                        lambda: tf.div(concs, sumconcs), lambda: concs)
        _, self.tf_output_conc, self.tf_regulatory_conc = tf.split(
            concs, [self.num_input, self.num_output, self.num_regulatory])

    def get_output_tensor(self):
        return self.tf_output_conc

    def get_output(self):
        with tf.Session() as s:
            return s.run(self.get_output_tensor())

    def input_step(self, inputs):
        self.set_input(inputs)
        self.step()
        return self.get_output_tensor()

    def clone(self):
        g = DiffGRN()
        g.identifiers = deepcopy(self.identifiers)
        g.enhancers = deepcopy(self.enhancers)
        g.inhibitors = deepcopy(self.inhibitors)
        g.beta = deepcopy(self.beta)
        g.delta = deepcopy(self.delta)

        g.num_input = deepcopy(self.num_input)
        g.num_output = deepcopy(self.num_output)
        g.num_regulatory = deepcopy(self.num_regulatory)
        return g


text:
import cv2
import numpy as np
import torch
from torchvision.transforms import Compose, Normalize, ToTensor

def preprocess_image(img: np.ndarray, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) -> torch.Tensor:
    preprocessing = Compose([
        ToTensor(),
        Normalize(mean=mean, std=std)
    ])
    return preprocessing(img.copy()).unsqueeze(0)

def deprocess_image(img):
    """ see https://github.com/jacobgil/keras-grad-cam/blob/master/grad-cam.py#L65 """
    img = img - np.mean(img)
    img = img / (np.std(img) + 1e-5)
    img = img * 0.1
    img = img + 0.5
    img = np.clip(img, 0, 1)
    return np.uint8(img * 255)

def show_cam_on_image(img: np.ndarray,
                      mask: np.ndarray,
                      use_rgb: bool = False,
                      colormap: int = cv2.COLORMAP_JET) -> np.ndarray:
    """ This function overlays the cam mask on the image as an heatmap.
    By default the heatmap is in BGR format.

    :param img: The base image in RGB or BGR format.
    :param mask: The cam mask.
    :param use_rgb: Whether to use an RGB or BGR heatmap, this should be set to True if 'img' is in RGB format.
    :param colormap: The OpenCV colormap to be used.
    :returns: The default image with the cam overlay.
    """
    heatmap = cv2.applyColorMap(np.uint8(255 * mask), colormap)
    if use_rgb:
        heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)
    heatmap = np.float32(heatmap) / 255

    if np.max(img) > 1:
        raise Exception(
            "The input image should np.float32 in the range [0, 1]")

    cam = heatmap + img
    cam = cam / np.max(cam)
    return np.uint8(255 * cam)

max_stars_repo_name: hddlovefxx/-pytorch-grad-cam

max_issues_repo_name: hddlovefxx/-pytorch-grad-cam

max_forks_repo_name: hddlovefxx/-pytorch-grad-cam


text:
##########################################################################
# NSAp - Copyright (C) CEA, 2013 - 2016
# Distributed under the terms of the CeCILL-B license, as published by
# the CEA-CNRS-INRIA. Refer to the LICENSE file or to
# http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
# for details.
##########################################################################

"""
Registration utilities.
"""

# System import
import os
import subprocess
import nibabel
import numpy
import shutil
import multiprocessing

# Hopla import
from hopla.converter import hopla

# Package import
from pypreclin.utils.reorient import check_orientation
from pypreclin.utils.export import ungzip_file
from pypreclin.utils.export import gzip_file
from pypreclin.utils.reorient import swap_affine
from pypreclin import preproc

# Third party import
import pyconnectome
from pyconnectome.utils.filetools import apply_mask
from pyconnectome import DEFAULT_FSL_PATH
from pyconnectome.utils.regtools import flirt
from nipype.interfaces.ants import AffineInitializer
from nipype.interfaces.ants import ApplyTransforms
from transforms3d.affines import decompose44
from transforms3d.euler import mat2euler
from scipy.io import loadmat
import nibabel

def timeserie_to_reference(tfile, outdir, rindex=None,
                           restrict_deformation=(1, 1, 1), rfile=None, njobs=1,
                           clean_tmp=True):
    """ Register all the fMRI volumes to a reference volume identified by his
    index in the timeserie.

    The registration used is a non-linear regisration.

    Parameters
    ----------
    tfile: str
        the file that containes the timeserie.
    outdir: str
        the destination folder.
    rindex: int, default None
        the reference volume index in the timeserie.
    restrict_deformation: 3-uplet
        restrict the deformation in the given axis.
    rfile: str, default None
        the reference volume.
    njobs: int, default 1
        the desired number of parallel job during the registration.
    clean_tmp: bool, default True
        if set, clean the temporary results.

    Returns
    -------
    resfile: str
        the registration result.
    """
    # Check input index
    im = nibabel.load(tfile)
    array = im.get_data()
    if array.ndim != 4:
        raise ValueError("A timeserie (4d volume) is expected.")
    reference_image = None
    if rindex is not None:
        if rindex < 0 or rindex > array.shape[3]:
            raise ValueError(
                "Index '{0}' is out of bound considering the last dimension "
                "as the time dimension '{1}'.".format(rindex, array.shape))
    elif rfile is not None:
        reference_image = rfile
    else:
        raise ValueError("You need to specify a reference file or index.")

    # Split the timeserie
    tmpdir = os.path.join(outdir, "tmp")
    if not os.path.isdir(tmpdir):
        os.mkdir(tmpdir)
    moving_images = []
    outdirs = []
    for i in range(array.shape[3]):
        _im = nibabel.Nifti1Image(array[..., i], im.affine)
        _outfile = os.path.join(tmpdir, str(i).zfill(4) + ".nii.gz")
        outdirs.append(os.path.join(tmpdir, str(i).zfill(4)))
        if not os.path.isdir(outdirs[-1]):
            os.mkdir(outdirs[-1])
        nibabel.save(_im, _outfile)
        moving_images.append(_outfile)
        if reference_image is None and i == rindex:
            reference_image = _outfile

    # Start volume to volume non rigid registration
    scriptdir = os.path.join(os.path.dirname(pyconnectome.__file__), "scripts")
    script = os.path.join(scriptdir, "pyconnectome_ants_register")
    python_cmd = "python3"
    if not os.path.isfile(script):
        script = "pyconnectome_ants_register"
        python_cmd = None
    logfile = os.path.join(tmpdir, "log")
    if os.path.isfile(logfile):
        os.remove(logfile)
    status, exitcodes = hopla(
        script,
        b="/usr/lib/ants",
        o=outdirs,
        i=moving_images,
        r=reference_image,
        w=1,
        D=3,
        G=0.2,
        J=1,
        N=True,
        B=True,
        R=list(restrict_deformation),
        V=2,
        hopla_iterative_kwargs=["o", "i"],
        hopla_cpus=njobs,
        hopla_logfile=logfile,
        hopla_python_cmd=python_cmd,
        hopla_use_subprocess=True,
        hopla_verbose=1)
    if not (numpy.asarray(list(exitcodes.values())) == 0).all():
        raise ValueError("The registration failed, check the log "
                         "'{0}'.".format(logfile))

    # Start timeserie concatenation
    timeserie = []
    affine = None
    for path in outdirs:
        sid = os.path.basename(path)
        _im = nibabel.load(os.path.join(
            path,  "ants_2WarpToTemplate_{0}.nii.gz".format(sid)))
        if affine is None:
            affine = _im.affine
        elif not numpy.allclose(affine, im.affine, atol=1e-3):
            raise ValueError("Affine matrices must be the same: {0} - "
                             "{1}.".format(outdirs[0], path))
        data = _im.get_data()
        data.shape += (1, )
        timeserie.append(data)
    registered_array = numpy.concatenate(timeserie, axis=3)
    _im = nibabel.Nifti1Image(registered_array, affine)
    resfile = os.path.join(outdir, "ants_WarpToTemplate.nii.gz")
    nibabel.save(_im, resfile)

    # Clean temporary files if requested
    if clean_tmp:
        shutil.rmtree(tmpdir, ignore_errors=True)

    return resfile

def jip_align(source_file, target_file, outdir, jipdir, prefix="w",
              auto=False, non_linear=False, fslconfig=DEFAULT_FSL_PATH):
    """ Register a source image to a taget image using the 'jip_align'
    command.

    Parameters
    ----------
    source_file: str (mandatory)
        the source Nifti image.
    target_file: str (mandatory)
        the target Nifti masked image.
    outdir: str (mandatory)
        the destination folder.
    jipdir: str (mandatory)
        the jip binary path.
    prefix: str (optional, default 'w')
        prefix the generated file with this character.
    auto: bool (optional, default False)
        if set control the JIP window with the script.
    non_linear: bool (optional, default False)
        in the automatic mode, decide or not to compute the non-linear
        deformation field.
    fslconfig: str (optional)
        the FSL .sh configuration file.

    Returns
    -------
    register_file: str
        the registered image.
    register_mask_file: str
        the registered and masked image.
    native_masked_file: str
        the masked image in the native space.
    """
    # Check input image orientation: must be the same
    same_orient, orients = check_orientation([source_file, target_file])
    if not same_orient:
        print(
            "[WARNING] Source file '{0}' ({2}) and taget file '{1}' ({3}) "
            "must have the same orientation for JIP to work properly.".format(
                source_file, target_file, orients[0], orients[1]))

    # Try to init the affine deformation
    # ToDo: find a way to init the JIP deformation
    if False:
        nb_cpus = multiprocessing.cpu_count()
        init_file = os.path.join(outdir, "init_affine.mat")
        init = AffineInitializer()
        init.inputs.fixed_image = target_file
        init.inputs.moving_image = source_file
        init.inputs.num_threads = nb_cpus
        init.inputs.out_file = init_file
        init.run()
        init_source_file = os.path.join(outdir, "init_source.nii.gz")
        at = ApplyTransforms()
        at.inputs.dimension = 3
        at.inputs.input_image = source_file
        at.inputs.reference_image = target_file
        at.inputs.output_image = init_source_file
        at.inputs.transforms = init_file
        at.inputs.interpolation = "BSpline"
        at.inputs.num_threads = nb_cpus
        at.run()
        # From https://github.com/ANTsX/ANTs/wiki/ITK-affine-transform-conversion
        # From https://github.com/netstim/leaddbs/blob/master/helpers/ea_antsmat2mat.m
        init_mat = loadmat(init_file)
        afftransform = init_mat["AffineTransform_double_3_3"]
        m_center = init_mat["fixed"]
        mat = numpy.eye(4)
        mat[:3, :3] = afftransform[:9].reshape(3, 3).T
        m_translation = afftransform[9:]
        m_offset = m_translation + m_center - numpy.dot(mat[:3, :3], m_center)
        mat[:3, 3] = m_offset.squeeze()
        mat = numpy.linalg.inv(mat)
        #rotation = swap_affine(axes="LPS")
        #mat = numpy.dot(rotation, mat)
        ras_to_lps = numpy.ones((4, 4))
        ras_to_lps[2, :2] = -1
        ras_to_lps[:2, 2:] = -1
        mat = mat * ras_to_lps
        print(mat)
        trans, rot, zoom, shear = decompose44(mat)
        euler = numpy.asarray(mat2euler(rot, axes='sxyz')) * 180. / numpy.pi
        print(trans, euler, zoom, shear)
        align_file = os.path.join(outdir, "align.com")
        im = nibabel.load(source_file)
        orig = im.affine[:3, 3]
        print(orig)
        shift = orig - trans
        with open(align_file, "wt") as of:
            of.write("set registration-translations {0} \n".format(
                " ".join([str(elem) for elem in shift])))

    # Change current working directory
    cwd = os.getcwd()
    os.chdir(outdir)

    # Only support uncompressed Nifti
    source_file = ungzip_file(source_file, prefix="", outdir=outdir)
    target_file = ungzip_file(target_file, prefix="", outdir=outdir)

    # Create jip environment
    jip_envriron = os.environ
    jip_envriron["JIP_HOME"] = os.path.dirname(jipdir)
    if "PATH" in jip_envriron:
        jip_envriron["PATH"] = jip_envriron["PATH"] + ":" + jipdir
    else:
        jip_envriron["PATH"] = jipdir

    # Copy source file
    align_file = os.path.join(outdir, "align.com")
    cmd = ["align", source_file, "-t", target_file]
    if auto:
        if non_linear:
            auto_cmd = cmd + ["-L", "111111111111", "-W", "111", "-a"]
        else:
            auto_cmd = cmd + ["-L", "111111000000", "-W", "000", "-A"]
        if os.path.isfile(align_file):
            auto_cmd += ["-I"]
        subprocess.call(auto_cmd, env=jip_envriron)
    else:
        print(" ".join(cmd))
        subprocess.call(cmd, env=jip_envriron)
    if not os.path.isfile(align_file):
        raise ValueError(
            "No 'align.com' file in '{0}' folder. JIP has probably failed: "
            "'{1}'".format(outdir, " ".join(cmd)))

    # Get the apply nonlinear deformation jip batches
    aplly_nonlin_batch = os.path.join(
        os.path.dirname(preproc.__file__), "resources", "apply_nonlin.com")
    aplly_inv_nonlin_batch = os.path.join(
        os.path.dirname(preproc.__file__), "resources", "apply_inv_nonlin.com")

    # Resample the source image
    register_file = os.path.join(outdir, prefix + os.path.basename(source_file))
    cmd = ["jip", aplly_nonlin_batch, source_file, register_file]
    subprocess.call(cmd, env=jip_envriron)

    # Apply mask
    if os.path.isfile(register_file + ".gz"):
        os.remove(register_file + ".gz")
    register_mask_fileroot = os.path.join(
        outdir, "m" + prefix + os.path.basename(source_file).split(".")[0])
    register_mask_file = apply_mask(
        input_file=register_file,
        output_fileroot=register_mask_fileroot,
        mask_file=target_file,
        fslconfig=fslconfig)

    # Send back masked image to original space
    register_mask_file = ungzip_file(register_mask_file, prefix="",
                                     outdir=outdir)
    native_masked_file = os.path.join(
        outdir, "n" + prefix + os.path.basename(register_mask_file))
    cmd = ["jip", aplly_inv_nonlin_batch, register_mask_file,
           native_masked_file]
    subprocess.call(cmd, env=jip_envriron)

    # Restore current working directory and gzip output
    os.chdir(cwd)
    register_file = gzip_file(
        register_file, prefix="", outdir=outdir,
        remove_original_file=True)
    register_mask_file = gzip_file(
        register_mask_file, prefix="", outdir=outdir,
        remove_original_file=True)
    native_masked_file = gzip_file(
        native_masked_file, prefix="", outdir=outdir,
        remove_original_file=True)

    return register_file, register_mask_file, native_masked_file, align_file

def apply_jip_align(apply_to_files, align_with, outdir, jipdir, prefix="w",
                    apply_inv=False):
    """ Apply a jip deformation.

    Parameters
    ----------
    apply_to_files: list of str (mandatory)
        a list of image path to apply the computed deformation.
    align_with: str ot list of str (mandatory)
        the alignement file containind the deformation parameters. Expect
        an 'align.com' file.
    outdir: str (mandatory)
        the destination folder.
    jipdir: str (mandatory)
        the jip binary path.
    prefix: str (optional, default 'w')
        prefix the generated file with this character.
    apply_inv: bool (optional, default False)
        if set apply the inverse deformation.

    Returns
    -------
    deformed_files: list of str
        the deformed input files.
    """
    # Check input parameters
    if not isinstance(align_with, list):
        align_with = [align_with]
    if not isinstance(apply_to_files, list):
        raise ValueError("The 'apply_to_files' function parameter must "
                         "contains a list of files.")
    for path in apply_to_files + align_with:
        if not os.path.isfile(path):
            raise ValueError("'{0}' is not a valid file.".format(path))

    # Get the apply nonlinear deformation jip batches
    if apply_inv:
        batch = os.path.join(os.path.dirname(preproc.__file__), "resources",
                             "apply_inv_nonlin.com")
    else:
        batch = os.path.join(os.path.dirname(preproc.__file__), "resources",
                             "apply_nonlin.com")
    batch = os.path.abspath(batch)

    # Create jip environment
    jip_envriron = os.environ
    jip_envriron["JIP_HOME"] = os.path.dirname(jipdir)
    if "PATH" in jip_envriron:
        jip_envriron["PATH"] = jip_envriron["PATH"] + ":" + jipdir
    else:
        jip_envriron["PATH"] = jipdir

    # Apply the jip deformation
    deformed_files = []
    cwd = os.getcwd()
    for path in apply_to_files:
        extra_file = ungzip_file(path, prefix="", outdir=outdir)
        deformed_file = os.path.join(
            outdir, prefix + os.path.basename(extra_file))
        for align_file in align_with:
            os.chdir(os.path.dirname(align_file))
            cmd = ["jip", batch, extra_file, deformed_file]
            subprocess.call(cmd, env=jip_envriron)
            extra_file = deformed_file
        deformed_file = gzip_file(
            deformed_file, prefix="", outdir=outdir, remove_original_file=True)
        deformed_files.append(deformed_file)
    os.chdir(cwd)

    return deformed_files

def check_jip_install(jipdir):
    """ Simple function to test if the JIP software bibaries are present.

    Parameters
    ----------
    jip_dir: str
        the JIP binaries directory.

    Returns
    -------
    status: bool
        the JIP installation status.
    """
    status = True
    for name in ("jip", "align"):
        binary_path = os.path.join(jipdir, name)
        if not os.path.isfile(binary_path):
            status = False
            break
    return status

def resample_image(source_file, target_file, out_file,
                   fslconfig=DEFAULT_FSL_PATH):
    """ Resample a source image to fit witha target image.

    Parameters
    ----------
    source_file: str
        the source Nifti image.
    target_file: str
        the target Nifti image.
    out_file: str
        the destination file.
    fslconfig: str (optional)
        the FSL .sh configuration file.
    """
    flirt(
        in_file=source_file,
        ref_file=target_file,
        init=None, cost="corratio",
        applyxfm=True,
        out=out_file,
        interp="nearestneighbour",
        shfile=fslconfig)
    return out_file

max_stars_repo_name: VisionandCognition/pypreclin

max_issues_repo_name: VisionandCognition/pypreclin

max_forks_repo_name: VisionandCognition/pypreclin

reason: import numpy,from scipy


text:
from kinematics import *
from generator import *
import blender_utils
import sympy
import sys

def main(argv):
    if len(argv) < 2:
        print("usage : " + argv[0] + " <blend_file> [<endpoint>]")
        exit()

    x = sympy.Symbol("x")
    y = sympy.Symbol("y")
    z = sympy.Symbol("z")

    blender_utils.call_blender_export(argv[1])

    if len(argv) == 2:
        l = blender_utils.read_json("blender.out.json")
        for e in l:
            print(e["name"])
    elif len(argv) == 3:
        endpoint = argv[2].replace("/", "_")
        chain = blender_utils.extract_chain(
            blender_utils.read_json("blender.out.json"),
            argv[2]
        )
        chain.name = chain.name.replace("/", "_")
        gen = CppProjectGenerator()
        add_ik_module(gen, chain)
        for fname in gen:
            with open(fname, "w+") as f:
                f.write(str(gen[fname]))

max_stars_repo_name: aversiveplusplus/aversiveplusplus

max_issues_repo_name: aversiveplusplus/aversiveplusplus

max_forks_repo_name: aversiveplusplus/aversiveplusplus

reason: import sympy


text:
import os
import sys
if sys.path[0] != os.getcwd():
    sys.path.insert(0, os.getcwd())
lib_local_path = os.path.normpath(r'C:\WORKSPACE\DEV\pyc3dserver')
if os.path.exists(lib_local_path):
    if sys.path[1] != lib_local_path:
        sys.path.insert(1, lib_local_path)
import pyc3dserver as c3d
import numpy as np
#%%
current_f_dir_path = os.path.dirname(__file__)
c3d_sample_dir_path = os.path.join(current_f_dir_path, r'..\Samples_C3D\Sample04')
src_c3d_path = os.path.join(c3d_sample_dir_path, 'sub_labels.c3d')

itf = c3d.c3dserver()
ret = c3d.open_c3d(itf, src_c3d_path)
dict_header = c3d.get_dict_header(itf)
dict_groups = c3d.get_dict_groups(itf)
dict_markers = c3d.get_dict_markers(itf, blocked_nan=True, resid=True)
dict_forces = c3d.get_dict_forces(itf, desc=True)
dict_analogs = c3d.get_dict_analogs(itf, desc=True)
ret = c3d.close_c3d(itf)


text:
"""Plot DM/SNR distributions of repeater populations."""
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import ks_2samp

from frbpoppy import CosmicPopulation, Survey, SurveyPopulation, plot
from frbpoppy import split_pop, pprint, hist

from tests.convenience import plot_aa_style, rel_path

DAYS = 4
INTERACTIVE_PLOT = False
PLOTTING_LIMIT_N_SRCS = 0
SNR = False

r = CosmicPopulation.simple(n_srcs=int(1e5), n_days=DAYS, repeaters=True)
r.set_dist(z_max=0.01)
r.set_lum(model='powerlaw', low=1e35, high=1e45, power=-1.5,
          per_source='different')
r.set_time(model='poisson', rate=3)
r.set_dm_igm(model='ioka', slope=1000, std=0)
r.set_dm(mw=False, igm=True, host=False)
r.set_w('constant', value=1)

r.generate()

# Set up survey
survey = Survey('perfect', n_days=DAYS)
survey.set_beam(model='perfect')
survey.snr_limit = 1e6

surv_pop = SurveyPopulation(r, survey)

pprint(f'{r.n_bursts()}:{surv_pop.n_bursts()}')
pprint(f'{surv_pop.n_sources()} sources detected')

if r.n_bursts() < PLOTTING_LIMIT_N_SRCS:
    pprint('Not sufficient FRB sources for plotting')
    exit()

# Split population into seamingly one-off and repeater populations
mask = ((~np.isnan(surv_pop.frbs.time)).sum(1) > 1)
pop_rep, pop_one = split_pop(surv_pop, mask)
pop_rep.name += ' (> 1 burst)'
pop_one.name += ' (1 burst)'

if INTERACTIVE_PLOT:
    plot(r, pop_rep, pop_one, tns=False, mute=False)

# Plot dm distribution
if SNR:
    plot_aa_style(cols=2)
    f, (ax1, ax2) = plt.subplots(1, 2)
else:
    plot_aa_style(cols=1)
    f, ax1 = plt.subplots(1, 1)

prop_cycle = plt.rcParams['axes.prop_cycle']
colors = prop_cycle.by_key()['color']

pops = (r, pop_rep, pop_one)

for i, pop in enumerate(pops):

    # Distinguish populations
    if pop.name.endswith('(1 burst)'):
        label = '1 burst'
        linestyle = 'solid'
    elif pop.name.endswith('(> 1 burst)'):
        label = '$>$1 burst'
        linestyle = 'dashed'
    else:
        label = 'cosmic'
        linestyle = 'dashdot'

    pprint(f'Number of bursts in {label}: {pop.n_bursts()}')

    # Do stuff with data
    dm = pop.frbs.dm
    x, y = hist(dm)
    x *= 200  # Normalise x-axis z=0.01, z=2

    # Plot DM distributions
    ax1.step(x, y, where='mid', linestyle=linestyle, label=label,
             color=colors[i])

    # Plot fluence distributions
    snr = pop.frbs.snr

    if snr is None:
        continue

    if not SNR:
        continue

    try:
        ax2.step(*hist(snr, bin_type='log'), where='mid', linestyle=linestyle,
                 color=colors[i])
    except ValueError:
        pprint('Zero sources available to plot')
        continue

ax1.set_xlabel(r'DM$_{\textrm{ex}}$ ($\textrm{pc}\ \textrm{cm}^{-3}$)')
ax1.set_ylabel('Fraction')

if SNR:
    ax2.set_xlabel(r'SNR')
    ax2.set_xscale('log')
    ax2.set_yscale('log')
    ax2.yaxis.tick_right()
    plt.figlegend(loc='upper center', ncol=len(pops), framealpha=1)
else:
    plt.figlegend(loc='upper center', ncol=3, framealpha=1, prop={'size': 8},
                  bbox_to_anchor=(0.5, 1.07), bbox_transform=ax1.transAxes)

# Test the difference between the distributions
rep = pops[1].frbs.dm[:20]  # First 20 repeaters
one = pops[2].frbs.dm[:min([200, len(pops[2].frbs.dm)])]  # First x one-offs
print(f'KS test: {ks_2samp(rep, one)}')

plt.tight_layout()
plt.savefig(rel_path('plots/rep_dm_dist.pdf'))
plt.clf()

max_stars_repo_name: macrocosme/frbpoppy

max_issues_repo_name: macrocosme/frbpoppy

max_forks_repo_name: macrocosme/frbpoppy


text:
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import csv
import numpy as np
import os
import sys

from observations.util import maybe_download_and_extract

def rock_art(path):
  """Pacific Rock Art features

  Data characterise rock art at 103 sites in the Pacific.

  A data frame with 103 observations on the following 641 variables.

  `Site.No.`
      a numeric vector

  `Site.Name`
      a character vector

  `Site.Code`
      a character vector

  `District`
      a character vector

  `Island`
      a character vector

  `Country`
      a character vector

  `Technique`
      a character vector

  `Engtech`
      a character vector

  `red`
      a numeric vector

  `black`
      a numeric vector

  `yellow`
      a numeric vector

  `white`
      a numeric vector

  `green`
      a numeric vector

  `red.blk`
      a numeric vector

  `red.wh`
      a numeric vector

  `red.yell`
      a numeric vector

  `r.w.y`
      a numeric vector

  `black.white`
      a numeric vector

  `blue`
      a numeric vector

  `Geology`
      a character vector

  `Topography`
      a character vector

  `Location`
      a character vector

  `Proxhab.km.`
      a character vector

  `Proxcoast.km.`
      a numeric vector

  `Maxheight.m.`
      a numeric vector

  `Language`
      a character vector

  `No.motif`
      a character vector

  `Ca1`
      a numeric vector

  `Ca2`
      a numeric vector

  `Ca3`
      a numeric vector

  `Ca4`
      a numeric vector

  `Cb5`
      a numeric vector

  `Cb6`
      a numeric vector

  `Cc7`
      a numeric vector

  `Cc8`
      a numeric vector

  `Cc9`
      a numeric vector

  `Cc10`
      a numeric vector

  `Cc11`
      a numeric vector

  `Cc12`
      a numeric vector

  `Cc13`
      a numeric vector

  `Cc14`
      a numeric vector

  `Cc15`
      a numeric vector

  `Cc16`
      a numeric vector

  `Cc17`
      a numeric vector

  `Cc18`
      a numeric vector

  `Cc19`
      a numeric vector

  `Cc20`
      a numeric vector

  `Cd21`
      a numeric vector

  `Cd22`
      a numeric vector

  `Cd23`
      a numeric vector

  `Cd24`
      a numeric vector

  `Cd25`
      a numeric vector

  `Cd26`
      a numeric vector

  `Cd27`
      a numeric vector

  `Ce28`
      a numeric vector

  `Ce29`
      a numeric vector

  `Cf30`
      a numeric vector

  `Cf31`
      a numeric vector

  `Cf32`
      a numeric vector

  `Cf33`
      a numeric vector

  `Cf34`
      a numeric vector

  `Cf35`
      a numeric vector

  `Cf36`
      a numeric vector

  `Cf37`
      a numeric vector

  `Cf38`
      a numeric vector

  `Cg39`
      a numeric vector

  `Cg40`
      a numeric vector

  `Ch41`
      a numeric vector

  `Ch42`
      a numeric vector

  `Ci43`
      a numeric vector

  `Ci44`
      a numeric vector

  `Cj45`
      a numeric vector

  `Ck46`
      a numeric vector

  `Ck47`
      a numeric vector

  `Cl48`
      a numeric vector

  `Cm49`
      a numeric vector

  `Cm50`
      a numeric vector

  `Cm51`
      a numeric vector

  `Cm52`
      a numeric vector

  `Cm53`
      a numeric vector

  `Cm54`
      a numeric vector

  `Cm55`
      a numeric vector

  `Cm56`
      a numeric vector

  `Cm57`
      a numeric vector

  `Cm58`
      a numeric vector

  `Cn59`
      a numeric vector

  `Cn60`
      a numeric vector

  `Cn61`
      a numeric vector

  `Cn62`
      a numeric vector

  `Cn63`
      a numeric vector

  `Cn64`
      a numeric vector

  `Cn65`
      a numeric vector

  `Cn66`
      a numeric vector

  `Cn67`
      a numeric vector

  `Cn68`
      a numeric vector

  `Cn69`
      a numeric vector

  `Cn70`
      a numeric vector

  `Cn71`
      a numeric vector

  `Co72`
      a numeric vector

  `Co73`
      a numeric vector

  `Co74`
      a numeric vector

  `Co75`
      a numeric vector

  `Co76`
      a numeric vector

  `Co77`
      a numeric vector

  `Co78`
      a numeric vector

  `Co79`
      a numeric vector

  `Cp80`
      a numeric vector

  `Cq81`
      a numeric vector

  `Cq82`
      a numeric vector

  `Cq83`
      a numeric vector

  `Cq84`
      a numeric vector

  `Cq85`
      a numeric vector

  `Cq86`
      a numeric vector

  `Cq87`
      a numeric vector

  `Cq88`
      a numeric vector

  `Cq89`
      a numeric vector

  `Cq90`
      a numeric vector

  `Cq91`
      a numeric vector

  `Cq92`
      a numeric vector

  `Cq93`
      a numeric vector

  `Cq94`
      a numeric vector

  `Cq95`
      a numeric vector

  `Cq96`
      a numeric vector

  `Cq97`
      a numeric vector

  `Cr98`
      a numeric vector

  `Cr99`
      a numeric vector

  `Cr100`
      a numeric vector

  `Cr101`
      a numeric vector

  `Cs102`
      a numeric vector

  `Cs103`
      a numeric vector

  `Cs104`
      a numeric vector

  `Cs105`
      a numeric vector

  `Cs106`
      a numeric vector

  `Ct107`
      a numeric vector

  `C108`
      a numeric vector

  `C109`
      a numeric vector

  `C110`
      a numeric vector

  `C111`
      a numeric vector

  `SSa1`
      a numeric vector

  `SSd2`
      a numeric vector

  `SSd3`
      a numeric vector

  `SSd4`
      a numeric vector

  `SSd5`
      a numeric vector

  `SSd6`
      a numeric vector

  `SSd7`
      a numeric vector

  `SSd8`
      a numeric vector

  `SSf9`
      a numeric vector

  `SSg10`
      a numeric vector

  `SSj11`
      a numeric vector

  `SSj12`
      a numeric vector

  `SSj13`
      a numeric vector

  `SSl14`
      a numeric vector

  `SSm15`
      a numeric vector

  `SSm16`
      a numeric vector

  `SSn17`
      a numeric vector

  `SSn18`
      a numeric vector

  `SSn19`
      a numeric vector

  `SSn20`
      a numeric vector

  `SSn21`
      a numeric vector

  `SSn22`
      a numeric vector

  `SSn23`
      a numeric vector

  `SSn24`
      a numeric vector

  `SSn25`
      a numeric vector

  `SSn26`
      a numeric vector

  `SSn27`
      a numeric vector

  `SSn28`
      a numeric vector

  `SSn29`
      a numeric vector

  `SSn30`
      a numeric vector

  `SSn31`
      a numeric vector

  `SSn32`
      a numeric vector

  `SSn33`
      a numeric vector

  `SSn34`
      a numeric vector

  `SSn35`
      a numeric vector

  `SSo36`
      a numeric vector

  `SSo37`
      a numeric vector

  `SSp38`
      a numeric vector

  `SSq39`
      a numeric vector

  `SSq40`
      a numeric vector

  `SSt41`
      a numeric vector

  `SSu42`
      a numeric vector

  `Oa1`
      a numeric vector

  `Oc2`
      a numeric vector

  `Od3`
      a numeric vector

  `Od4`
      a numeric vector

  `Oe5`
      a numeric vector

  `Of6`
      a numeric vector

  `Of7`
      a numeric vector

  `Of8`
      a numeric vector

  `Of9`
      a numeric vector

  `Og10`
      a numeric vector

  `Og11`
      a numeric vector

  `Og12`
      a numeric vector

  `Og13`
      a numeric vector

  `Og14`
      a numeric vector

  `Og15`
      a numeric vector

  `Oi16`
      a numeric vector

  `Om17`
      a numeric vector

  `Om18`
      a numeric vector

  `Om19`
      a numeric vector

  `Om20`
      a numeric vector

  `Om21`
      a numeric vector

  `On22`
      a numeric vector

  `On23`
      a numeric vector

  `On24`
      a numeric vector

  `Oq25`
      a numeric vector

  `Oq26`
      a numeric vector

  `Oq27`
      a numeric vector

  `.u28`
      a numeric vector

  `Ov29`
      a numeric vector

  `Ov30`
      a numeric vector

  `O31`
      a numeric vector

  `O32`
      a numeric vector

  `O33`
      a numeric vector

  `Sa1`
      a numeric vector

  `Sb2`
      a numeric vector

  `Sb3`
      a numeric vector

  `Sd4`
      a numeric vector

  `Sd5`
      a numeric vector

  `Sd6`
      a numeric vector

  `Sd7`
      a numeric vector

  `Se8`
      a numeric vector

  `Si9`
      a numeric vector

  `Sm10`
      a numeric vector

  `Sm11`
      a numeric vector

  `S12`
      a numeric vector

  `S13`
      a numeric vector

  `Sx14`
      a numeric vector

  `Sx15`
      a numeric vector

  `Sx16`
      a numeric vector

  `Sx17`
      a numeric vector

  `Sy18`
      a numeric vector

  `Sz19`
      a numeric vector

  `S20`
      a numeric vector

  `S21`
      a numeric vector

  `S22`
      a numeric vector

  `S23`
      a numeric vector

  `S24`
      a numeric vector

  `S25`
      a numeric vector

  `SCd1`
      a numeric vector

  `SCd2`
      a numeric vector

  `SCd3`
      a numeric vector

  `SCd4`
      a numeric vector

  `SCd5`
      a numeric vector

  `SCd6`
      a numeric vector

  `SCd7`
      a numeric vector

  `SCm8`
      a numeric vector

  `SCn9`
      a numeric vector

  `SCn10`
      a numeric vector

  `SCw11`
      a numeric vector

  `SCx12`
      a numeric vector

  `SCx13`
      a numeric vector

  `SCx14`
      a numeric vector

  `SCx15`
      a numeric vector

  `SCx16`
      a numeric vector

  `SCy17`
      a numeric vector

  `SCy18`
      a numeric vector

  `SC19`
      a numeric vector

  `SC20`
      a numeric vector

  `SC21`
      a numeric vector

  `SC22`
      a numeric vector

  `SC23`
      a numeric vector

  `SC24`
      a numeric vector

  `SC25`
      a numeric vector

  `SC26`
      a numeric vector

  `SRd1`
      a numeric vector

  `SRd2`
      a numeric vector

  `SRd3`
      a numeric vector

  `SRd4`
      a numeric vector

  `SRf5`
      a numeric vector

  `SRf6`
      a numeric vector

  `SRf7`
      a numeric vector

  `SRj8`
      a numeric vector

  `SR9`
      a numeric vector

  `SR10`
      a numeric vector

  `Bd1`
      a numeric vector

  `Bn2`
      a numeric vector

  `Bn3`
      a numeric vector

  `Bn4`
      a numeric vector

  `Bt5`
      a numeric vector

  `Bx6`
      a numeric vector

  `Ha1`
      a numeric vector

  `Hg2`
      a numeric vector

  `Hn3`
      a numeric vector

  `Hq4`
      a numeric vector

  `Hq5`
      a numeric vector

  `TDd1`
      a numeric vector

  `TDf2`
      a numeric vector

  `TDj3`
      a numeric vector

  `TDn4`
      a numeric vector

  `TDq5`
      a numeric vector

  `TD6`
      a numeric vector

  `TD7`
      a numeric vector

  `TD8`
      a numeric vector

  `TD9`
      a numeric vector

  `Dc1`
      a numeric vector

  `Dg2`
      a numeric vector

  `Dh3`
      a numeric vector

  `Dk4`
      a numeric vector

  `Dm5`
      a numeric vector

  `Dm6`
      a numeric vector

  `D7`
      a numeric vector

  `D8`
      a numeric vector

  `D9`
      a numeric vector

  `D10`
      a numeric vector

  `D11`
      a numeric vector

  `D12`
      a numeric vector

  `D13`
      a numeric vector

  `Ta1`
      a numeric vector

  `Tc2`
      a numeric vector

  `Tc3`
      a numeric vector

  `Tc4`
      a numeric vector

  `Td5`
      a numeric vector

  `Tf6`
      a numeric vector

  `Tf7`
      a numeric vector

  `Tg8`
      a numeric vector

  `Th9`
      a numeric vector

  `To10`
      a numeric vector

  `T11`
      a numeric vector

  `T12`
      a numeric vector

  `T13`
      a numeric vector

  `T14`
      a numeric vector

  `T15`
      a numeric vector

  `T16`
      a numeric vector

  `CNg1`
      a numeric vector

  `CN2`
      a numeric vector

  `CN3`
      a numeric vector

  `CN4`
      a numeric vector

  `CN5`
      a numeric vector

  `CN6`
      a numeric vector

  `CN7`
      a numeric vector

  `CN8`
      a numeric vector

  `Ld1`
      a numeric vector

  `Lf2`
      a numeric vector

  `Lg3`
      a numeric vector

  `Lp4`
      a numeric vector

  `L5`
      a numeric vector

  `L6`
      a numeric vector

  `L7`
      a numeric vector

  `L8`
      a numeric vector

  `L9`
      a numeric vector

  `L10`
      a numeric vector

  `L11`
      a numeric vector

  `LS1`
      a numeric vector

  `LS2`
      a numeric vector

  `LL1`
      a numeric vector

  `LL2`
      a numeric vector

  `LL3`
      a numeric vector

  `LL4`
      a numeric vector

  `LL5`
      a numeric vector

  `EGd1`
      a numeric vector

  `EGf2`
      a numeric vector

  `CCd1`
      a numeric vector

  `CCn2`
      a numeric vector

  `CCn3`
      a numeric vector

  `EMc1`
      a numeric vector

  `EMd2`
      a numeric vector

  `EMd3`
      a numeric vector

  `EMf4`
      a numeric vector

  `EMf5`
      a numeric vector

  `EMn6`
      a numeric vector

  `EMx7`
      a numeric vector

  `EM8`
      a numeric vector

  `EM9`
      a numeric vector

  `EM10`
      a numeric vector

  `EM11`
      a numeric vector

  `EM12`
      a numeric vector

  `TE1`
      a numeric vector

  `TE2`
      a numeric vector

  `TE3`
      a numeric vector

  `TE4`
      a numeric vector

  `TE5`
      a numeric vector

  `BWe1`
      a numeric vector

  `BWn2`
      a numeric vector

  `BWn3`
      a numeric vector

  `TS1`
      a numeric vector

  `TS2`
      a numeric vector

  `TS3`
      a numeric vector

  `TS4`
      a numeric vector

  `TS5`
      a numeric vector

  `TS6`
      a numeric vector

  `TS7`
      a numeric vector

  `TS8`
      a numeric vector

  `TS9`
      a numeric vector

  `Pg1`
      a numeric vector

  `Pg2`
      a numeric vector

  `Pg3`
      a numeric vector

  `DUaa1`
      a numeric vector

  `DUw2`
      a numeric vector

  `DU3`
      a numeric vector

  `CP1`
      a numeric vector

  `CP2`
      a numeric vector

  `CP3`
      a numeric vector

  `CP4`
      a numeric vector

  `CP5`
      a numeric vector

  `CP6`
      a numeric vector

  `CP7`
      a numeric vector

  `CP8`
      a numeric vector

  `CP9`
      a numeric vector

  `CP10`
      a numeric vector

  `CP11`
      a numeric vector

  `CP12`
      a numeric vector

  `STd1`
      a numeric vector

  `STd2`
      a numeric vector

  `STd3`
      a numeric vector

  `STg4`
      a numeric vector

  `STaa5`
      a numeric vector

  `STaa6`
      a numeric vector

  `STaa7`
      a numeric vector

  `STaa8`
      a numeric vector

  `ST9`
      a numeric vector

  `ST10`
      a numeric vector

  `ST11`
      a numeric vector

  `ST12`
      a numeric vector

  `Wd1`
      a numeric vector

  `Wd2`
      a numeric vector

  `Wd3`
      a numeric vector

  `Wd4`
      a numeric vector

  `Wn5`
      a numeric vector

  `Waa6`
      a numeric vector

  `Waa7`
      a numeric vector

  `W8`
      a numeric vector

  `W9`
      a numeric vector

  `W10`
      a numeric vector

  `W11`
      a numeric vector

  `W12`
      a numeric vector

  `W13`
      a numeric vector

  `Zd1`
      a numeric vector

  `Zd2`
      a numeric vector

  `Zn3`
      a numeric vector

  `Zw4`
      a numeric vector

  `Zw5`
      a numeric vector

  `Zaa6`
      a numeric vector

  `Z7`
      a numeric vector

  `Z8`
      a numeric vector

  `Z9`
      a numeric vector

  `Z10`
      a numeric vector

  `Z11`
      a numeric vector

  `Z12`
      a numeric vector

  `CLd1`
      a numeric vector

  `CLd2`
      a numeric vector

  `CLd3`
      a numeric vector

  `CLd4`
      a numeric vector

  `CLd5`
      a numeric vector

  `CLd6`
      a numeric vector

  `CLd7`
      a numeric vector

  `CLd8`
      a numeric vector

  `CLd9`
      a numeric vector

  `CLd10`
      a numeric vector

  `CLd11`
      a numeric vector

  `CLd12`
      a numeric vector

  `CLd13`
      a numeric vector

  `CLd14`
      a numeric vector

  `CLd15`
      a numeric vector

  `CLd16`
      a numeric vector

  `CLd17`
      a numeric vector

  `CLd18`
      a numeric vector

  `CLd19`
      a numeric vector

  `CLd20`
      a numeric vector

  `CLd21`
      a numeric vector

  `CLd22`
      a numeric vector

  `CLd23`
      a numeric vector

  `CLd24`
      a numeric vector

  `CLd25`
      a numeric vector

  `CLd26`
      a numeric vector

  `CLd27`
      a numeric vector

  `CLd28`
      a numeric vector

  `CLd29`
      a numeric vector

  `CLd30`
      a numeric vector

  `CLd31`
      a numeric vector

  `CLd32`
      a numeric vector

  `CLd33`
      a numeric vector

  `CLd34`
      a numeric vector

  `CLd35`
      a numeric vector

  `CLd36`
      a numeric vector

  `CLd37`
      a numeric vector

  `CLd38`
      a numeric vector

  `CLn39`
      a numeric vector

  `CLn40`
      a numeric vector

  `CLn41`
      a numeric vector

  `CLn42`
      a numeric vector

  `CLn43`
      a numeric vector

  `CLn44`
      a numeric vector

  `CLn45`
      a numeric vector

  `CLn46`
      a numeric vector

  `CLn47`
      a numeric vector

  `CLn48`
      a numeric vector

  `CLw49`
      a numeric vector

  `CL50`
      a numeric vector

  `CL51`
      a numeric vector

  `CL52`
      a numeric vector

  `CL53`
      a numeric vector

  `CL54`
      a numeric vector

  `CL55`
      a numeric vector

  `CL56`
      a numeric vector

  `CL57`
      a numeric vector

  `CL58`
      a numeric vector

  `CL59`
      a numeric vector

  `Xd1`
      a numeric vector

  `Xd2`
      a numeric vector

  `Xd3`
      a numeric vector

  `Xd4`
      a numeric vector

  `Xd5`
      a numeric vector

  `Xd6`
      a numeric vector

  `Xd7`
      a numeric vector

  `Xd8`
      a numeric vector

  `Xd9`
      a numeric vector

  `Xd10`
      a numeric vector

  `Xd11`
      a numeric vector

  `Xd12`
      a numeric vector

  `Xd13`
      a numeric vector

  `Xf14`
      a numeric vector

  `Xk15`
      a numeric vector

  `Xn16`
      a numeric vector

  `Xn17`
      a numeric vector

  `Xn18`
      a numeric vector

  `Xn19`
      a numeric vector

  `Xn20`
      a numeric vector

  `Xn21`
      a numeric vector

  `Xn22`
      a numeric vector

  `Xn23`
      a numeric vector

  `Xn24`
      a numeric vector

  `Xn25`
      a numeric vector

  `Xn26`
      a numeric vector

  `Xn27`
      a numeric vector

  `Xn28`
      a numeric vector

  `Xn29`
      a numeric vector

  `Xn30`
      a numeric vector

  `Xn31`
      a numeric vector

  `Xn32`
      a numeric vector

  `Xp33`
      a numeric vector

  `Xp34`
      a numeric vector

  `Xp35`
      a numeric vector

  `Xq36`
      a numeric vector

  `Xq37`
      a numeric vector

  `Xq38`
      a numeric vector

  `X39`
      a numeric vector

  `X40`
      a numeric vector

  `X41`
      a numeric vector

  `X42`
      a numeric vector

  `X43`
      a numeric vector

  `X44`
      a numeric vector

  `X45`
      a numeric vector

  `X46`
      a numeric vector

  `X47`
      a numeric vector

  `X48`
      a numeric vector

  `X49`
      a numeric vector

  `X50`
      a numeric vector

  `Qd1`
      a numeric vector

  `Qe2`
      a numeric vector

  `Qe3`
      a numeric vector

  `Qh4`
      a numeric vector

  `Qh5`
      a numeric vector

  `Qh6`
      a numeric vector

  `Qh7`
      a numeric vector

  `Qh8`
      a numeric vector

  `Qh9`
      a numeric vector

  `Qn10`
      a numeric vector

  `Qn11`
      a numeric vector

  `Qt12`
      a numeric vector

  `Q13`
      a numeric vector

  `Q14`
      a numeric vector

  `Q15`
      a numeric vector

  `Q16`
      a numeric vector

  `Q17`
      a numeric vector

  `Q18`
      a numeric vector

  `Q19`
      a numeric vector

  `Q20`
      a numeric vector

  `Q21`
      a numeric vector

  `Q22`
      a numeric vector

  `TZd1`
      a numeric vector

  `TZf2`
      a numeric vector

  `TZh3`
      a numeric vector

  `TZ4`
      a numeric vector

  `CRd1`
      a numeric vector

  `CR2`
      a numeric vector

  `CR3`
      a numeric vector

  `EUd1`
      a numeric vector

  `EUd2`
      a numeric vector

  `EUg3`
      a numeric vector

  `EUm4`
      a numeric vector

  `EUw5`
      a numeric vector

  `EU6`
      a numeric vector

  `Ud1`
      a numeric vector

  `Ud2`
      a numeric vector

  `Ud3`
      a numeric vector

  `Uaa4`
      a numeric vector

  `U5`
      a numeric vector

  `Vd1`
      a numeric vector

  `V2`
      a numeric vector

  `V3`
      a numeric vector

  `V4`
      a numeric vector

  `V5`
      a numeric vector

  `LWE1`
      a numeric vector

  `LWE2`
      a numeric vector

  `Ad1`
      a numeric vector

  `Al2`
      a numeric vector

  `Am3`
      a numeric vector

  `An4`
      a numeric vector

  `Aw5`
      a numeric vector

  `Aaa6`
      a numeric vector

  `A7`
      a numeric vector

  `A8`
      a numeric vector

  `A9`
      a numeric vector

  `EVd1`
      a numeric vector

  `EVg2`
      a numeric vector

  `TK1`
      a numeric vector

  `ECL1`
      a numeric vector

  `EFe1`
      a numeric vector

  `EFm2`
      a numeric vector

  `EFm3`
      a numeric vector

  `EF4`
      a numeric vector

  `LPo1`
      a numeric vector

  `LPq2`
      a numeric vector

  `LP3`
      a numeric vector

  `LP4`
      a numeric vector

  `LP5`
      a numeric vector

  `PT1`
      a numeric vector

  `CSC`
      a numeric vector

  `CSR`
      a numeric vector

  `CCRC`
      a numeric vector

  `SA`
      a numeric vector

  `Anthrop`
      a numeric vector

  `Turtle`
      a numeric vector

  `Boat`
      a numeric vector

  `Canoe`
      a numeric vector

  `Hand`
      a numeric vector

  `Foot`
      a numeric vector

  `Lizard`
      a numeric vector

  `Crocodile`
      a numeric vector

  `Jellyfish`
      a numeric vector

  `Bird`
      a numeric vector

  `Anthrobird`
      a numeric vector

  `Axe`
      a numeric vector

  `Marine`
      a numeric vector

  `Face`
      a numeric vector

  `Zoo1`
      a numeric vector

  `Zoo2`
      a numeric vector

  `Zoo3`
      a numeric vector

  `Zoo4`
      a numeric vector

  `Zoo5`
      a numeric vector

  `Zoo6`
      a numeric vector

  Meredith Wilson: *Picturing Pacific Pre-History* (PhD thesis), 2002,
  Australian National University.

  Args:

    path: str.
      Path to directory which either stores file or otherwise file will
      be downloaded and extracted there.
      Filename is `rock_art.csv`.

  Returns:

    Tuple of np.ndarray `x_train` with 103 rows and 641 columns and
    dictionary `metadata` of column headers (feature names).
  """
  import pandas as pd
  path = os.path.expanduser(path)
  filename = 'rock_art.csv'
  if not os.path.exists(os.path.join(path, filename)):
    url = 'http://dustintran.com/data/r/DAAG/rockArt.csv'
    maybe_download_and_extract(path, url,
                               save_file_name='rock_art.csv',
                               resume=False)

  data = pd.read_csv(os.path.join(path, filename), index_col=0,
                     parse_dates=True)
  x_train = data.values
  metadata = {'columns': data.columns}
  return x_train, metadata


text:
# -*- coding: utf-8 -*-

"""A collection of basic fitting and statistic functions for EpiPy."""

import numpy as np
from inspect import getargspec
from random import uniform
from scipy import integrate, optimize
from epipy.lib.core.utils import corr_coef

def fit(model, xdata, N0, ydata=None, params=None, iter=10):
    """Fit given model base initial coordinates with paramters. Are no paramter
    given, this function will use initial guesses for paramters.

    :param callable model: Function which defines an epidemic mo
    :param array xdata: Observation steps
    :param tuple N0: Model values at initial observation step
    :param array ydata: Infected individuals
    :param tuple params: Epidemic model parameters
    :param int iter: Number of iterations

    :returns: The fitted epidemic model, used parameters and correlation
    coefficient.
    :rtype: tuple(array, array, float/None)
    """
    # Population
    N = np.sum(N0)

    def fit_odeint(xdata, *params):
        return integrate.odeint(model, N0, xdata, args=((N,) + params))[:, 1]

    # static fit - Fit a given epidemic model based on parameters.
    if params:
        return fit_odeint(xdata, *params), np.array(params), None
    # Iterative - Fit a given epidemic model and initial guesses for parameters
    # and return the best fitted model based on the correlation coefficient.
    else:
        args, _, _, values = getargspec(model)
        param_size = len(args) - 3
        result = []
        p0 = [uniform(0., 1.) for x in range(param_size)]
        for n in range(iter):
            params, _ = optimize.curve_fit(fit_odeint,
                                           xdata, ydata,
                                           p0=p0,
                                           maxfev=1000)
            fitted = fit_odeint(xdata, *params)
            _corr_coef = corr_coef(ydata, fitted)
            result.append((fitted, params, _corr_coef))
        return sorted(result, key=lambda x: x[2], reverse=True)[0]

max_stars_repo_name: ckaus/EpiPy

max_issues_repo_name: ckaus/EpiPy

max_forks_repo_name: ckaus/EpiPy


text:
import numpy as np
import plotly.graph_objects as go
import plotly.offline as pyo

x_values = np.linspace(0, 1, 100)  # 100 evenly spaced values
y_values = np.random.randn(100)   # 100 random values

# create traces
trace0 = go.Scatter(
    x=x_values,
    y=y_values+5,
    mode='markers',
    name='markers'
)
trace1 = go.Scatter(
    x=x_values,
    y=y_values,
    mode='lines+markers',
    name='lines+markers'
)
trace2 = go.Scatter(
    x=x_values,
    y=y_values-5,
    mode='lines',
    name='lines'
)
data = [trace0, trace1, trace2]  # assign traces to data
layout = go.Layout(
    title='Line chart showing three different modes'
)
fig = go.Figure(data=data, layout=layout)
pyo.plot(fig)


text:
import base64
import enum
import hashlib
import imghdr
import json
from distutils.util import strtobool as strtobool
from enum import Enum
from typing import Any

import numpy as np

import conf
import cv2
import processors
from boxes.box_processor import PSMode
from flask import Blueprint, jsonify
from flask_restful import Resource, reqparse, request
from logger import create_info_logger
from numpyencoder import NumpyEncoder
from skimage import io
from utils.network import find_open_port, get_ip_address
from utils.utils import current_milli_time, ensure_exists

logger = create_info_logger(__name__, "marie.log")

ALLOWED_TYPES = {'png', 'jpeg', 'tiff'}
TYPES_TO_EXT = {'png': 'png', 'jpeg': 'jpg', 'tiff': 'tif'}

def encodeToBase64(img: np.ndarray) -> str:
    """encode image to base64"""
    retval, buffer = cv2.imencode('.png', img)
    png_as_text = base64.b64encode(buffer).decode()
    return png_as_text

def base64StringToBytes(data: str):
    """conver base 64 string to byte"""
    if data is None:
        return ""
    base64_message = data
    base64_bytes = base64_message.encode('utf-8')
    message_bytes = base64.b64decode(base64_bytes)
    return message_bytes

def load_image(fname, image_type):
    """"
        Load image, if the image is a TIFF, we will load the image as a multipage tiff, otherwise we return an
        array with image as first element
    """
    if fname is None:
        return False, None

    if image_type == 'tiff':
        loaded, frames = cv2.imreadmulti(fname, [], cv2.IMREAD_ANYCOLOR)
        if not loaded:
            return False, []
        # each frame needs to be converted to RGB format
        converted = []
        for frame in frames:
            if len(frame.shape) == 2:
                frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
            converted.append(frame)

        return loaded, converted

    img = io.imread(fname)  # RGB order
    if img.shape[0] == 2:
        img = img[0]
    if len(img.shape) == 2:
        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
    if img.shape[2] == 4:
        img = img[:, :, :3]
    img = np.array(img)

    return True, [img]

# Blueprint Configuration
blueprint = Blueprint(
    name='icr_bp',
    import_name=__name__,
    url_prefix=conf.API_PREFIX
)

logger.info('IcrAPIRoutes inited')
box_processor = processors.box_processor
icr_processor = processors.icr_processor
show_error = True  # show predition errors

@blueprint.route('/', methods=['GET'])
def status():
    """Get application status"""
    import os
    build = {}
    if os.path.exists('.build'):
        with open('.build', 'r') as fp:
            build = json.load(fp)
    host = get_ip_address()

    return jsonify(
        {
            "name": "marie-icr",
            "host": host,
            "component": [
                {
                    "name": "craft",
                    "version": "1.0.0"
                },
                {
                    "name": "craft-benchmark",
                    "version": "1.0.0"
                }
            ],
            "build": build
        }
    ), 200

def store_temp_file(message_bytes, queue_id, file_type, store_raw):
    """Store temp file from decoded payload"""
    m = hashlib.sha256()
    m.update(message_bytes)
    checksum = m.hexdigest()

    upload_dir = ensure_exists(f'/tmp/marie/{queue_id}')
    ext = TYPES_TO_EXT[file_type]
    tmp_file = f"{upload_dir}/{checksum}.{ext}"

    if store_raw:
        # message read directly from a file
        with open(tmp_file, "wb") as tmp:
            tmp.write(message_bytes)
    else:
        # TODO : This does not handle multipage tiffs
        # convert to numpy array as the message has been passed from base64
        npimg = np.frombuffer(message_bytes, np.uint8)
        # convert numpy array to image
        img = cv2.imdecode(npimg, cv2.IMREAD_UNCHANGED)
        cv2.imwrite(tmp_file, img)

    return tmp_file, checksum

def extract_payload(payload, queue_id):  # -> tuple[bytes, str]:
    """Extract data from payload"""
    import io
    import os

    from utils.utils import FileSystem, ensure_exists

    print(f"Payload info ")
    # determine how to extract payload based on the type of the key supplied
    # Possible keys
    # data, srcData, srcFile, srcUrl

    store_raw = False
    if "data" in payload:
        raw_data = payload["data"]
        data = base64StringToBytes(raw_data)
    elif "srcData" in payload:
        raw_data = payload["srcData"]
        data = base64StringToBytes(raw_data)
    elif "srcFile" in payload:
        img_path = payload["srcFile"]
        # FIXME: relative path resolution is now working as expected
        base_dir = FileSystem.get_share_directory()
        path = os.path.abspath(os.path.join(base_dir, img_path))
        print(f'base_dir = {base_dir}')
        print(f'raw_data = {img_path}')
        print(f'resolved path = {path}')
        if not os.path.exists(path):
            raise Exception(f"File not found : {img_path}")
        with open(path, 'rb') as file:
            data = file.read()
        store_raw = True
    else:
        raise Exception("Unable to determine datasource in payload")

    with io.BytesIO(data) as memfile:
        file_type = imghdr.what(memfile)

    if file_type not in ALLOWED_TYPES:
        raise Exception(F"Unsupported file type, expected one of : {ALLOWED_TYPES}")

    tmp_file, checksum = store_temp_file(data, queue_id, file_type, store_raw)
    print(f'Filetype : {file_type}')
    print(f'tmp_file : {tmp_file}')

    return tmp_file, checksum, file_type

def process_extract_fullpage(frames, queue_id, checksum, pms_mode, args):
    """Process full page extraction """
    # TODO : Implement multipage tiff processing

    page_index = 0
    img = frames[page_index]
    h = img.shape[0]
    w = img.shape[1]
    # allow for small padding around the component
    padding = 4
    overlay = np.ones((h + padding * 2, w + padding * 2, 3), dtype=np.uint8) * 255
    overlay[padding:h + padding, padding:w + padding] = img

    boxes, img_fragments, lines, _ = box_processor.extract_bounding_boxes(
        queue_id, checksum, overlay, pms_mode)
    result, overlay_image = icr_processor.recognize(
        queue_id, checksum, overlay, boxes, img_fragments, lines)

    cv2.imwrite(f'/tmp/marie/overlay_image_{page_index}.png', overlay_image)
    result['overlay_b64'] = encodeToBase64(overlay_image)

    return result

def process_extract_regions(frames, queue_id, checksum, pms_mode, regions, args):
    """Process region based extract"""
    filter_snippets = bool(strtobool(args['filter_snippets'])) if 'filter_snippets' in args else False
    output = []
    extended = []

    for region in regions:
        # validate required fields
        if not all(key in region for key in ("id", "pageIndex", "x", "y", "w", "h")):
            raise Exception(f"Required key missing in region : {region}")

    for region in regions:
        try:
            rid = region['id']
            page_index = region['pageIndex']
            x = region['x']
            y = region['y']
            w = region['w']
            h = region['h']

            img = frames[page_index]
            img = img[y:y + h, x:x + w].copy()
            # allow for small padding around the component
            padding = 4
            overlay = np.ones((h + padding * 2, w + padding * 2, 3), dtype=np.uint8) * 255
            overlay[padding:h + padding, padding:w + padding] = img

            boxes, img_fragments, lines, _ = box_processor.extract_bounding_boxes(
                queue_id, checksum, overlay, pms_mode)
            result, overlay_image = icr_processor.recognize(
                queue_id, checksum, overlay, boxes, img_fragments, lines)

            cv2.imwrite(f'/tmp/marie/overlay_image_{page_index}_{rid}.png', overlay_image)
            result["overlay_b64"] = encodeToBase64(overlay_image)
            result["id"] = rid

            extended.append(result)

            # TODO : Implement rendering modes
            # 1 - Simple
            # 2 - Full
            # 3 - HOCR

            rendering_mode = 'simple'
            region_result = {}
            if rendering_mode == "simple":
                if "lines" in result:
                    lines = result["lines"]
                    line = lines[0]
                    region_result["id"] = rid
                    region_result["text"] = line["text"]
                    region_result["confidence"] = line["confidence"]
                    output.append(region_result)
        except Exception as ex:
            print(ex)

    # Filter out base 64 encoded fragments(fragment_b64, overlay_b64)
    # This is useful when we like to display or process image in the output but has significant payload overhead

    def filter_base64(node, filters):
        if isinstance(node, (list, tuple, np.ndarray)):
            for v in node:
                filter_base64(v, filters)
        elif isinstance(node, dict):
            for flt in filters:
                try:
                    del node[flt]
                except KeyError:
                    pass
            for key, value in node.items():
                filter_base64(value, filters)
        else:
            pass
        return node

    if filter_snippets:
        extended = filter_base64(extended, filters=["fragment_b64", "overlay_b64"])

    return {"regions": output, "extended": extended}

@blueprint.route("/extract/<queue_id>", methods=["POST"])
def extract(queue_id: str):
    """
    ICR payload to process
        Process image via ICR, this is low level API, to get more usable results call extract_icr.

    Args:
        queue_id: Unique queue to tie the extraction to
    """

    logger.info("Starting ICR processing request", extra={"session": queue_id})
    try:
        print(request)
        payload = request.json
        if payload is None:
            return {"error": "empty payload"}, 200

        pms_mode = PSMode.from_value(payload["mode"] if 'mode' in payload else '')
        regions = payload["regions"] if "regions" in payload else []

        # due to compatibility issues with other frameworks we allow passing same arguments in the 'args' object
        args = {}
        if 'args' in payload:
            args = payload['args']
            pms_mode = PSMode.from_value(payload['args']['mode'] if 'mode' in payload['args'] else '')

        tmp_file, checksum, file_type = extract_payload(payload, queue_id)
        loaded, frames = load_image(tmp_file, file_type)

        print(f'Frame size : {len(frames)}')
        if not loaded:
            print(f'Unable to read image : {tmp_file}')
            raise Exception(f'Unable to read image : {tmp_file}')

        frame_len = len(frames)
        print(f'frame_len : {frame_len}')
        print(f'regions_len : {len(regions)}')

        if len(regions) == 0:
            result = process_extract_fullpage(frames, queue_id, checksum, pms_mode, args)
        else:
            result = process_extract_regions(frames, queue_id, checksum, pms_mode, regions, args)

        serialized = json.dumps(result, sort_keys=True, separators=(',', ': '), ensure_ascii=False, indent=2,
                                cls=NumpyEncoder)

        return serialized, 200
    except BaseException as error:
        # raise error
        # print(str(error))
        if show_error:
            return {"error": str(error)}, 500
        else:
            return {"error": 'inference exception'}, 500

max_stars_repo_name: gregbugaj/marie-ai

max_issues_repo_name: gregbugaj/marie-icr

max_forks_repo_name: gregbugaj/marie-ai

reason: import numpy,from numpy


text:
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os

def read_file(filename):
    """Read a csv data file located in the ./data/ directory
    Return design matrix X and y-data"""
    DATA_DIR = "./data"
    if not os.path.exists(DATA_DIR):
        os.mkdir(DATA_DIR)
    DATA_FILE = os.path.join(DATA_DIR,filename)
    df = pd.read_csv(DATA_FILE).values
    y = np.array(df[:,1])
    X = np.array(df[:,2:])
    return X, y

def save_fig(y_test, y_pred, score, fig_id):
    """Save a matplotlib figure to the ./figure/ directory.
    If/else statements ask the user if they wish to overwrite an
    already existing figure."""
    # Plot
    l = y_test.shape[0]
    x = np.linspace(1,l,l)
    plt.plot(y_test, y_pred, 'ro', label=fr"prediction, $R^2 = ${score:2.2f}")
    min = np.min(y_test)
    max = np.max(y_test)
    plt.plot([min,max],[min,max])
    plt.xlabel("CADi true values")
    plt.ylabel("CADi prediction")
    plt.legend()
    plt.grid()
    # Saving of figure
    FIGURE_DIR = "./figures"
    if not os.path.exists(FIGURE_DIR):
        os.mkdir(FIGURE_DIR)
    FIGURE_FILE = os.path.join(FIGURE_DIR, fig_id)
    if os.path.exists(FIGURE_FILE):
        overwrite = str(input("The file \"" + FIGURE_FILE + \
            "\" already exists, do you wish to overwrite it [y/n]?\n"))
        if overwrite == "y":
            plt.savefig(FIGURE_FILE, format="png")
            plt.close()
            print("Figure was overwritten.")
        else:
            print("Figure was not saved.")
    else:
        plt.savefig(FIGURE_FILE, format="png")
        plt.close()
    return None

max_stars_repo_name: simehaa/MachineLearning

max_issues_repo_name: simehaa/MachineLearning

max_forks_repo_name: simehaa/MachineLearning


text:
import os
import numpy as np
from astropy.io import fits
from collections import OrderedDict

__all__ = ["CCD", ]

class CCD(np.ndarray):
    """ CCD class, to cope with ccd images

    Note
    ----
    This class is a subclass of numpy.ndarray.

    References
    ----------
    https://docs.scipy.org/doc/numpy/user/basics.subclassing.html
    https://docs.scipy.org/doc/numpy/reference/arrays.classes.html

    Updates
    -------
    remove mean, median and std methods.

    """

    # ccd info
    gain = 1
    ron = 0
    unit = "adu"

    # file info, only when read from fits file
    fp = ""
    hdu = 0
    header = None
    meta = OrderedDict()

    # post-processing info
    _trim = None
    _rot90 = 0

    extr_attr_list = ["gain", "ron", "unit",            # ccd info
                      "fp", "hdu", "header", "meta",    # file info
                      "_trim", "_rot90"]                # post-proc info

    ###########################
    # print info
    ###########################
    def pprint(self):
        s = ("< twodspec.ccd.CCD instance {} x {} >\n"
             "gain  = {}\n"
             "ron   = {}\n"
             "unit  = {}\n"
             "trim  = {}\n"
             "rot90 = {}\n"
             "fp    = {}\n"
             "{}".format(*self.shape,
                         self.gain, self.ron, self.unit, self._trim,
                         self._rot90, self.fp,
                         self.__repr__()))
        print(s)
        return

    @property
    def info(self):
        """ print basic info

        Usage:
        ------
        CCD.info

        """
        s = ("< twodspec.ccd.CCD instance {} x {} >\n"
             "gain  = {}\n"
             "ron   = {}\n"
             "unit  = {}\n"
             "trim  = {}\n"
             "rot90 = {}\n"
             "fp    = {}\n"
             "hdu   = {}".format(*self.shape,
                                 self.gain, self.ron, self.unit, self._trim,
                                 self._rot90, self.fp, self.hdu))
        print(s)
        return

    ###########################
    # to inherit numpy.ndarray
    ###########################
    def __new__(cls, data, gain=1., ron=0., unit="adu",
                trim=None, rot90=0, header=None, meta=None,
                offset=0, strides=None, order="C", info=None):
        # Create the ndarray instance of our type, given the usual
        # ndarray input arguments.  This will call the standard
        # ndarray constructor, but return an object of our type.
        # It also triggers a call to InfoArray.__array_finalize__

        data = np.array(data, dtype=np.float, order=order)

        # substantiate
        ccd = super(CCD, cls).__new__(cls, shape=data.shape,
                                      dtype=data.dtype, buffer=data,
                                      offset=offset, strides=strides,
                                      order=order)
        # set info
        ccd.gain = gain
        ccd.ron = ron
        ccd.unit = unit
        ccd._trim = trim
        ccd._rot90 = rot90
        ccd.header = header
        if header is None:
            ccd.meta = OrderedDict()
        else:
            ccd.meta = OrderedDict(header)

        return ccd

    # def __array_finalize__(self, obj):
    #     if obj is None:
    #         return

    # def __array_wrap__(self, out_arr, context=None):
    #     print('In __array_wrap__:')
    #     print('   self is %s' % repr(self))
    #     print('   arr is %s' % repr(out_arr))
    #     # then just call the parent
    #     return super(CCD, self).__array_wrap__(self, out_arr, context)

    ###########################
    # read from fits
    ###########################
    @staticmethod
    def reads(fps=["",], method="median", std=False, hdu=0, gain=1., ron=0., unit="adu", trim=None, rot90=0):
        """ read and combine multiple ccd frames """
        # read
        ccds = CCD([CCD.read(fp, hdu=hdu, gain=gain, ron=ron, unit=unit, trim=trim, rot90=rot90) for fp in fps])

        # combine
        if method == "median":
            ccd_comb = np.median(ccds, axis=0)
        elif method == "mean":
            ccd_comb = np.mean(ccds, axis=0)
        else:
            raise ValueError("@CCD.combine: bad method [{}]".format(method))
        # set info
        ccd_comb.gain = gain
        ccd_comb.ron = ron
        ccd_comb.unit = unit
        ccd_comb._trim = trim
        ccd_comb._rot90 = rot90

        if not std:
            return ccd_comb
        else:
            # evaluate std
            ccd_std = np.std(ccds, axis=0)
            # set info
            ccd_std.gain = gain
            ccd_std.ron = ron
            ccd_std.unit = unit
            ccd_std._trim = trim
            ccd_std._rot90 = rot90
            return ccd_comb, ccd_std

    @staticmethod
    def read(fp="", hdu=0, gain=1., ron=0., unit="adu", trim=None, rot90=0):
        if os.path.exists(fp):
            # read fits
            hdu_list = fits.open(fp)

            # get data
            data = hdu_list[hdu].data
            header = hdu_list[hdu].header
            meta = OrderedDict(header)

            # trim
            if trim is not None:
                l, r, t, b = trim
                data = data[l:r + 1, t:b + 1]

            # rotate
            data = np.rot90(data, rot90)

            # initialize CCD
            ccd = CCD(data, gain=gain, ron=ron, unit=unit,
                      trim=trim, rot90=rot90, header=header, meta=meta)
            ccd.fp = fp

            return ccd

        else:
            raise ValueError("@CCD.read(): file not found! [{}]".format(fp))

    def trim(self, trim=(0, 2048, 0, 2048)):
        """ trim CCD data

        Parameters
        ----------
        trim : [left, right, top, bottom]
            the trimed section
        """
        l, r, t, b = trim
        trimed_data = self[l:r, t:b]
        trimed_data.copy_info(self)
        return trimed_data

    def rot90(self, k):
        """ Rotate self by 90 degrees in the counter-clockwise direction. <-\

        Parameters
        ----------
        k : integer
            Number of times the array is rotated by 90 degrees.
        """
        rotated_data = np.rot90(self, k)
        rotated_data.copy_info(self)
        return rotated_data

    ###########################
    # copy option
    ###########################
    def copy_info(self, ccd1):
        """ copy info from an other CCD instance """
        for k in self.extr_attr_list:
            self.__setattr__(k, ccd1.__getattribute__(k))

    # def copy(self):
    #     return np.copy(self)

    ###########################
    # get config
    ###########################
    @property
    def config(self):
        return dict(hdu=self.hdu,
                    gain=self.gain,
                    ron=self.ron,
                    unit=self.unit,
                    trim=self._trim,
                    rot90=self._rot90)

    def write(self, fp, overwrite=True):
        phdu = fits.PrimaryHDU(data=self, header=fits.Header(self.meta))
        phdu.writeto(fp, overwrite=overwrite)
        return

    ###########################
    # arithmetic options
    ###########################
    # def subtract(self, ccd1):
    #     """ ccd2 = self - ccd1 """
    #     ccd2 = self/ccd1
    #     ccd2.copy_info(self)
    #     return ccd2
    #
    # def devide(self, ccd1):
    #     """ ccd2 = self / ccd1 """
    #     ccd2 = self / ccd1
    #     ccd2.copy_info(self)
    #     return ccd2

    # @staticmethod
    # def combine(ccds, method="median"):
    #     """ combine ccd frames """
    #     ccds = CCD(ccds)
    #     if method == "median":
    #         return np.median(ccds, axis=0)
    #     elif method == "mean":
    #         return np.mean(ccds, axis=0)
    #     else:
    #         raise ValueError("@CCD.combine: bad method [{}]".format(method))

    # def mean(self, axis=None):
    #     return CCD(np.mean(self, axis=axis))
    #
    # def median(self, axis=None):
    #     return CCD(np.median(self, axis=axis))
    #
    # def std(self, axis=None):
    #     return CCD(np.std(self, axis=axis))

def test2():
    fp = os.getenv("HOME") + "/PycharmProjects/songcn/twodspec/data/s2_2017-01-13T16-43-05.fits"
    # check image --> consistent with fits.getdata & ccdproc.CCDData.read
    print("---")
    data = fits.getdata(fp)
    print(data)

    print("---")
    from ccdproc import CCDData
    ccddata = CCDData.read(fp, unit="adu")
    print(ccddata)

    print("---")
    for i in range(4):
        ccd = CCD.read(fp, rot90=i, gain=5)
        print(ccd)
    print("---")

def test1():
    fp = os.getenv("HOME") + "/PycharmProjects/songcn/twodspec/data/s2_2017-01-13T16-43-05.fits"
    ccd = CCD.read(fp, rot90=0, gain=5)
    print(ccd)

    print(ccd.info)
    print(ccd.config)

    from matplotlib import pyplot as plt
    plt.figure()
    plt.imshow(ccd, origin="upper")

if __name__ == "__main__":
    test1()
    test2()

max_stars_repo_name: hypergravity/songcn

max_issues_repo_name: hypergravity/songcn

max_forks_repo_name: hypergravity/songcn

reason: import numpy,from astropy


text:
import io
from pathlib import Path

from .model import MusicHighlighter
from .lib import *
import tensorflow as tf
import numpy as np
import os

os.environ["CUDA_VISIBLE_DEVICES"] = ''

def extract(fs, length=30, save_score=True, save_thumbnail=True, save_wav=True):
    for f in fs:
        with tf.Session() as sess:

            model = MusicHighlighter()
            sess.run(tf.global_variables_initializer())
            # model.saver.restore(sess, "pop_music_highlighter" + os.path.sep + "model" + os.path.sep + "model")
            model.saver.restore(sess, "model/model")

            name = f.name
            audio, spectrogram, duration = audio_read(f)
            n_chunk, remainder = np.divmod(duration, 3)
            chunk_spec = chunk(spectrogram, n_chunk)
            pos = positional_encoding(batch_size=1, n_pos=n_chunk, d_pos=model.dim_feature * 4)

            n_chunk = n_chunk.astype('int')
            chunk_spec = chunk_spec.astype('float')
            pos = pos.astype('float')

            attn_score = model.calculate(sess=sess, x=chunk_spec, pos_enc=pos, num_chunk=n_chunk)
            attn_score = np.repeat(attn_score, 3)
            attn_score = np.append(attn_score, np.zeros(remainder))

            # score
            attn_score = attn_score / attn_score.max()
            if save_score:
                if (not os.path.exists("output" + os.path.sep + "attention")):
                    os.mkdir("output" + os.path.sep + "attention")
                np.save('output' + os.path.sep + 'attention\\{}_score.npy'.format(name), attn_score)

            # thumbnail
            attn_score = attn_score.cumsum()
            attn_score = np.append(attn_score[length], attn_score[length:] - attn_score[:-length])
            index = np.argmax(attn_score)
            highlight = [index, index + length]

            if save_thumbnail:
                np.save('output' + os.path.sep + 'attention' + os.path.sep + '{}_highlight.npy'.format(name), highlight)

            if save_wav:
                librosa.output.write_wav('output' + os.path.sep + 'attention' + os.path.sep + '{}_audio.wav'.format(name),
                                         audio[highlight[0] * 22050:highlight[1] * 22050], 22050)

if __name__ == '__main__':
    fs = ["data/Pink Floyd - The Great Gig in The Sky.wav", "data/FMP_C4_Audio_Beatles_YouCantDoThat.wav"]
    # fs = ["data/Pink Floyd - The Great Gig in The Sky.wav"]
    extract(fs, length=10, save_score=True, save_thumbnail=True, save_wav=True)


text:
from collections import OrderedDict
from datetime import datetime, timedelta

import netCDF4 as nc
import numpy as np
from PyQt5.QtCore import pyqtSignal, pyqtSlot, QMutex
from PyQt5.QtWidgets import QWidget, QDateTimeEdit, QFormLayout

try:
    # for netCDF4 versions before 1.4.0
    from netCDF4._netCDF4 import _dateparse
except ImportError:
    # netcdf4 version 1.4.0 removes netcdftime to a separate package "cftime"
    from cftime._cftime import _dateparse

from pyntpg.dataset_var_picker.dataset_var_picker import CONSOLE_TEXT
from pyntpg.dataset_var_picker.dataset_var_picker import DatasetVarPicker

try:
    from cftime import DatetimeGregorian
    datetime_types = (datetime, DatetimeGregorian)
except ImportError:
    # fallback compat if cftime < 1.2.0 installed.
    datetime_types = (datetime,)

def datetime_units(units):
    """ Detect if the str units is a parsable datetime units format. """
    try:
        try:
            _dateparse(units)
            return True
        except TypeError:
            # CASE: cftime > 1.2.0, signature chagned,
            # needs to be called with calendard "standard" arg.
            # unfortunately inspect.getfullargspec doesn't work
            # on builtin functions (which _dateparse is b/c it's
            # in C. So, can't do this more elegantly by inspection.
            _dateparse(units, "standard")
            return True
    except (AttributeError, ValueError):
        return False

class DatetimePicker(DatasetVarPicker):

    signal_status_message = pyqtSignal(str)

    def __init__(self, *args, **kwargs):
        self.slices = OrderedDict()
        self.target_len = None

        super(DatetimePicker, self).__init__(*args, **kwargs)

        self.date_range_container = QWidget()
        self.date_range_layout = QFormLayout()
        self.date_range_container.setLayout(self.date_range_layout)
        self.layout.addWidget(self.date_range_container)

        # create the date slice widgets
        self.start_time = QDateTimeEdit()
        self.start_time.setDisplayFormat("yyyy-MM-dd hh:mm:ss")
        self.date_range_layout.addRow("start", self.start_time)
        self.end_time = QDateTimeEdit()
        self.end_time.setDisplayFormat("yyyy-MM-dd hh:mm:ss")
        self.date_range_layout.addRow("end", self.end_time)

        # a mutex to prevent programatic changes from being registered as user inputs
        self.datetime_change_mutex = QMutex()
        self.datetime_user_modified = False

        self.start_time.dateTimeChanged.connect(self.accept_datetime_change)
        self.end_time.dateTimeChanged.connect(self.accept_datetime_change)

        self.variable_widget.currentIndexChanged[str].connect(self.variable_selected)

    @pyqtSlot()
    def accept_datetime_change(self):
        if self.datetime_change_mutex.tryLock():  # MAKE SURE TO UNLOCK
            self.datetime_user_modified = True
            self.datetime_change_mutex.unlock()   # MADE SURE TO UNLCOK

    @pyqtSlot(int)
    def accept_target_len(self, val):
        self.target_len = val

    @pyqtSlot(OrderedDict)
    def accept_slices(self, slices):
        self.slices = slices
        self.dataset_selected(self.dataset_widget.currentText())

    @pyqtSlot(str)
    def variable_selected(self, variable):
        """ Once variable selected, must set the min and max datetimes. """
        dataset = self.dataset_widget.currentText()
        if not dataset or not variable:
            return  # don't follow through for changed to nothing

        num_dims = len(self.get_original_shape(dataset, variable))

        # This is going to get funky....
        # 1. Assume that the start and end are the min and max values.... in other words, assume
        # that the time array is in order and sorted.
        bounds = super(DatetimePicker, self).get_data(oslice=[[0, -1] for _ in range(num_dims)])
        # 2. If either of the bound are masked, take the hit and read the whole thing and calculate min max.
        if np.ma.count_masked(bounds) > 0:
            full_data = super(DatetimePicker, self).get_data()
            bounds = np.array([np.nanmin(full_data), np.nanmax(full_data)])

        if not isinstance(bounds.item(0), datetime_types):
            value = self.get_value()
            # must have units if not already datetime because of show_var condition
            bounds = nc.num2date(bounds, value.units)

        # super annoying... cftime 1.2.0 returns a custom type that does not
        # inherit from datetime, so it bascially can't be passed to ANYTHING
        # that expects plain old datetimes anymore.... so, roundabout conversion
        # to ensure we have a datetime object. This will work for pre cftime 1.2.0
        # returning native datetime since dattetime has isoformat method as well.
        start = datetime.fromisoformat(bounds.item(0).isoformat())
        end = datetime.fromisoformat(bounds.item(-1).isoformat())
        if start is None or end is None:
            self.signal_status_message.emit(
                "Error: fill in time array bound for dataset {}, var {}. Cannot use.".format(dataset, variable)
            )
            return  # also abort here... don't follow through

        # must grab the original values before setting the range because setting the
        # range will set the value to start or range if it's outside of range when changed.
        original_start_dt = self.start_time.dateTime().toPyDateTime()
        original_end_dt = self.end_time.dateTime().toPyDateTime()

        self.datetime_change_mutex.lock()  # A LOCK!!!! ENSURE UNLOCK.
        self.start_time.setDateTimeRange(start, end)
        self.end_time.setDateTimeRange(start, end)

        # smc@20181217 keep original user modified dates if they are valid.
        # emphasis on user modified! That's why new listeners required.
        # only set the times to the bounds if the original datetimes were not in the range.
        if original_start_dt < start or original_end_dt > end or not self.datetime_user_modified :
            self.start_time.setDateTime(start)
        if original_end_dt > end or original_end_dt < start or not self.datetime_user_modified:
            self.end_time.setDateTime(end)
        self.datetime_change_mutex.unlock()

    def show_var_condition(self, dataset, variable):
        if not super(DatetimePicker, self).show_var_condition(dataset, variable):
            return False

        dimensions = self.get_dimensions(dataset, variable)
        if not set(list(dimensions.keys())).issubset(list(self.slices.keys())):
            return False

        if not np.prod(list(dimensions.values())) == self.target_len:
            return False

        value = self.get_value(dataset, variable)
        if dataset == CONSOLE_TEXT:
            return ((hasattr(value, "units") and datetime_units(value.units))
                    or isinstance(np.array(value).item(0), datetime_types))
        else:
            # separate these out so don't try to read from the netcdf here.
            return hasattr(value, "units") and datetime_units(value.units)

    def get_data(self, _=None):
        num_dims = len(self.get_original_shape())
        oslices = [v[0] for v in self.slices.values()]

        data = super(DatetimePicker, self).get_data(oslice=oslices[:num_dims])
        mask = np.ma.getmaskarray(data)  # hopefully none!

        if not isinstance(data.item(0), datetime_types):
            # not datetime already, convert through num2date
            # by assumption value has a units attribute since
            # show_var_condition, would not allow the variable to be displayed
            # unless it was already a datetime or had num2date parseable units field
            value = self.get_value()
            data = nc.num2date(data, value.units)

        if len(self.slices) > 1:
            data = data.flatten()
            mask = mask.flatten()

        start_bound_dt = self.start_time.dateTime().toPyDateTime()
        end_bound_dt = self.end_time.dateTime().toPyDateTime()

        if np.any(mask):
            # if any data values are masked, must go through and remove the Nones from the data array...
            # the None values are introduced by the nc.num2date call on masked elements
            mask_date_detector = np.vectorize(lambda x: x is None or x < start_bound_dt or x > end_bound_dt)
            return np.ma.masked_where(mask_date_detector(data), data)
        else:
            # otherwise, this approach seems to be much more efficient.
            return np.ma.masked_where((data < start_bound_dt) | (data > end_bound_dt), data)

    def get_config(self):
        default = super(DatetimePicker, self).get_config()
        default.update({"type": "datetime"})
        return default