text:
# pylint: disable=not-callable, no-member, invalid-name, line-too-long
"""
Real Spherical Harmonics equivariant with respect to o3.rot and o3.irr_repr
"""
import math
from functools import lru_cache

import torch
from sympy import Integer, Poly, diff, factorial, pi, sqrt, symbols

from e3nn import rs
from e3nn.util.eval_code import eval_code

def spherical_harmonics_expand_matrix(ls, like=None):
    """
    convertion matrix between a flatten vector (L, m) like that
    (0, 0) (1, -1) (1, 0) (1, 1) (2, -2) (2, -1) (2, 0) (2, 1) (2, 2)

    and a bidimensional matrix representation like that
                    (0, 0)
            (1, -1) (1, 0) (1, 1)
    (2, -2) (2, -1) (2, 0) (2, 1) (2, 2)

    :return: tensor [l, m, l * m]
    """
    lmax = max(ls)
    zeros = torch.zeros if like is None else like.new_zeros
    m = zeros(len(ls), 2 * lmax + 1, sum(2 * l + 1 for l in ls))
    i = 0
    for j, l in enumerate(ls):
        m[j, lmax - l: lmax + l + 1, i:i + 2 * l + 1] = torch.eye(2 * l + 1)
        i += 2 * l + 1
    return m

@torch.jit.script
def mul_m_lm(Rs: rs.TY_RS_STRICT, x_m: torch.Tensor, x_lm: torch.Tensor) -> torch.Tensor:  # pragma: no cover
    """
    multiply tensor [..., l * m] by [..., m]
    """
    lmax = x_m.shape[-1] // 2
    out = []
    i = 0
    for mul, l, _ in Rs:
        d = mul * (2 * l + 1)
        x1 = x_lm[..., i: i + d]  # [..., mul * m]
        x1 = x1.reshape(x1.shape[:-1] + (mul, 2 * l + 1))  # [..., mul, m]
        x2 = x_m[..., lmax - l: lmax + l + 1]  # [..., m]
        x2 = x2.reshape(x2.shape[:-1] + (1, 2 * l + 1))  # [..., mul=1, m]
        x = x1 * x2
        x = x.reshape(x.shape[:-2] + (d,))
        out.append(x)
        i += d
    return torch.cat(out, dim=-1)

@torch.jit.script
def mul_radial_angular(Rs: rs.TY_RS_STRICT, radial, angular):  # pragma: no cover
    """
    :param Rs: output representation
    :param angular: [..., l * m]
    :param radial: [..., l * mul]
    """
    n = 0
    for mul, l, _ in Rs:
        n += mul * (2 * l + 1)

    y = radial[..., 0] * angular[..., 0]
    out = radial.new_empty(y.shape + (n,))

    a = 0
    r = 0
    i = 0
    for mul, l, _ in Rs:
        dim = mul * (2 * l + 1)
        x = radial[..., r: r + mul, None] * angular[..., None, a: a + 2 * l + 1]
        x = x.reshape(y.shape + (dim,))
        out[..., i: i + dim] = x
        i += dim
        r += mul
        a += 2 * l + 1

    assert r == radial.shape[-1]
    assert a == angular.shape[-1]

    return out

def sympy_legendre(l, m):
    """
    en.wikipedia.org/wiki/Associated_Legendre_polynomials
    - remove two times (-1)^m
    - use another normalization such that P(l, -m) = P(l, m)

    y = sqrt(1 - z^2)
    """
    l = Integer(l)
    m = Integer(abs(m))
    z, y = symbols('z y', real=True)
    ex = 1 / (2**l * factorial(l)) * y**m * diff((z**2 - 1)**l, z, l + m)
    ex *= (-1)**l * sqrt((2 * l + 1) / (4 * pi) * factorial(l - m) / factorial(l + m))
    return ex

def poly_legendre(l, m):
    """
    polynomial coefficients of legendre

    y = sqrt(1 - z^2)
    """
    z, y = symbols('z y', real=True)
    return Poly(sympy_legendre(l, m), domain='R', gens=(z, y)).as_dict()

_legendre_code = """
import torch
from e3nn import rsh

@torch.jit.script
def main(z: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
    out = z.new_zeros(z.shape + (lsize,))

# fill out

    return out
"""

@lru_cache()
def _legendre_genjit(ls):
    ls = list(ls)
    fill = ""
    i = 0
    for l in ls:
        for m in range(l + 1):
            p = poly_legendre(l, m)
            formula = " + ".join("{:.25f} * z**{} * y**{}".format(c, zn, yn) for (zn, yn), c in p.items())
            fill += "    l{} = {}\n".format(m, formula)

        for m in range(-l, l + 1):
            fill += "    out[..., {}] = l{}\n".format(i, abs(m))
            i += 1

    code = _legendre_code
    code = code.replace("lsize", str(sum(2 * l + 1 for l in ls)))
    code = code.replace("# fill out", fill)
    return eval_code(code).main

def legendre(ls, z, y=None):
    """
    associated Legendre polynomials

    :param ls: list
    :param z: tensor of shape [...]
    :return: tensor of shape [..., l * m]
    """
    if y is None:
        y = (1 - z**2).relu().sqrt()

    return _legendre_genjit(tuple(ls))(z, y)

def spherical_harmonics_z(Rs, z, y=None):
    """
    the z component of the spherical harmonics
    (useful to perform fourier transform)

    :param z: tensor of shape [...]
    :return: tensor of shape [..., l * m]
    """
    Rs = rs.simplify(Rs)
    for _, l, p in Rs:
        assert p in [0, (-1)**l]
    ls = [l for mul, l, _ in Rs]
    return legendre(ls, z, y)  # [..., l * m]

@torch.jit.script
def spherical_harmonics_alpha(l: int, alpha: torch.Tensor) -> torch.Tensor:  # pragma: no cover
    """
    the alpha (x, y) component of the spherical harmonics
    (useful to perform fourier transform)

    :param alpha: tensor of shape [...]
    :return: tensor of shape [..., m]
    """
    alpha = alpha.unsqueeze(-1)  # [..., 1]

    m = torch.arange(1, l + 1, dtype=alpha.dtype, device=alpha.device)  # [1, 2, 3, ..., l]
    cos = torch.cos(m * alpha)  # [..., m]

    m = torch.arange(l, 0, -1, dtype=alpha.dtype, device=alpha.device)  # [l, l-1, l-2, ..., 1]
    sin = torch.sin(m * alpha)  # [..., m]

    out = torch.cat([
        math.sqrt(2) * sin,
        torch.ones_like(alpha),
        math.sqrt(2) * cos,
    ], dim=alpha.ndim-1)

    return out  # [..., m]

def spherical_harmonics_alpha_beta(Rs, alpha, beta):
    """
    spherical harmonics

    :param Rs: list of L's
    :param alpha: float or tensor of shape [...]
    :param beta: float or tensor of shape [...]
    :return: tensor of shape [..., m]
    """
    if alpha.device.type == 'cuda' and beta.device.type == 'cuda' and not alpha.requires_grad and not beta.requires_grad and rs.lmax(Rs) <= 10:  # pragma: no cover
        xyz = torch.stack([beta.sin() * alpha.cos(), beta.sin() * alpha.sin(), beta.cos()], dim=-1)
        try:
            return spherical_harmonics_xyz_cuda(Rs, xyz)
        except ImportError:
            pass

    return spherical_harmonics_alpha_z_y(Rs, alpha, beta.cos(), beta.sin().abs())

def spherical_harmonics_alpha_z_y(Rs, alpha, z, y):
    """
    cpu version of spherical_harmonics_alpha_beta
    """
    Rs = rs.simplify(Rs)
    sha = spherical_harmonics_alpha(rs.lmax(Rs), alpha.flatten())  # [z, m]
    shz = spherical_harmonics_z(Rs, z.flatten(), y.flatten())  # [z, l * m]
    out = mul_m_lm(Rs, sha, shz)
    return out.reshape(alpha.shape + (shz.shape[1],))

def spherical_harmonics_xyz(Rs, xyz, eps=0.):
    """
    spherical harmonics

    :param Rs: list of L's
    :param xyz: tensor of shape [..., 3]
    :param eps: epsilon for denominator of atan2
    :return: tensor of shape [..., m]

    The eps parameter is only to be used when backpropogating to coordinates xyz.
    To determine a stable eps value, we recommend benchmarking against numerical
    gradients before setting this parameter. Use the smallest epsilon that prevents NaNs.
    For some cases, we have used 1e-10. Your case may require a different value.
    Use this option with care.
    """

    if xyz.device.type == 'cuda' and not xyz.requires_grad and rs.lmax(Rs) <= 10:  # pragma: no cover
        try:
            return spherical_harmonics_xyz_cuda(Rs, xyz)
        except ImportError:
            pass

    norm = torch.norm(xyz, 2, dim=-1, keepdim=True)
    xyz = xyz / (norm + eps)

    alpha = torch.atan2(xyz[..., 1], xyz[..., 0] + eps)  # [...]
    z = xyz[..., 2]  # [...]
    y = (xyz[..., 0].pow(2) + xyz[..., 1].pow(2) + eps).sqrt()  # [...]

    return spherical_harmonics_alpha_z_y(Rs, alpha, z, y)

def spherical_harmonics_xyz_cuda(Rs, xyz):  # pragma: no cover
    """
    cuda version of spherical_harmonics_xyz
    """
    from e3nn import cuda_rsh  # pylint: disable=no-name-in-module, import-outside-toplevel

    Rs = rs.simplify(Rs)

    *size, _ = xyz.size()
    xyz = xyz.reshape(-1, 3)
    xyz = xyz / torch.norm(xyz, 2, -1, keepdim=True)

    lmax = rs.lmax(Rs)
    out = xyz.new_empty(((lmax + 1)**2, xyz.size(0)))  # [ filters, batch_size]
    cuda_rsh.real_spherical_harmonics(out, xyz)

    # (-1)^L same as (pi-theta) -> (-1)^(L+m) and 'quantum' norm (-1)^m combined  # h - halved
    norm_coef = [elem for lh in range((lmax + 1) // 2) for elem in [1.] * (4 * lh + 1) + [-1.] * (4 * lh + 3)]
    if lmax % 2 == 0:
        norm_coef.extend([1.] * (2 * lmax + 1))
    norm_coef = out.new_tensor(norm_coef).unsqueeze(1)
    out.mul_(norm_coef)

    if not rs.are_equal(Rs, list(range(lmax + 1))):
        out = torch.cat([out[l**2: (l + 1)**2] for mul, l, _ in Rs for _ in range(mul)])
    return out.T.reshape(*size, -1)

reason: from sympy


text:
import matplotlib.pyplot as plt
import seaborn as sns

from matplotlib.offsetbox import OffsetImage, AnnotationBbox
from matplotlib.patches import Rectangle
from svgpathtools import svg2paths, wsvg
import os
import re
import numpy as np
import glob
import cv2

def plot_heatmap(data,
                xlabel,
                ylabel,
                minimization=False,
                savefig_path=None,
                 ):
    plt.clf()
    plt.cla()

    # figure
    fig, ax = plt.subplots(figsize=(8, 8))

    cmap = sns.cubehelix_palette(dark=0.1, light=0.9, as_cmap=True)
    # Cells have a value between 0.0 and 1.0 since they represent probabilities

    # Set the color for the under the limit to be white (0.0) so empty cells are not visualized
    # cmap.set_under('0.0')
    # Plot NaN in white
    cmap.set_bad(color='white')

    ax = sns.heatmap(data, cmap=cmap)
    ax.invert_yaxis()
    plt.xlabel(xlabel.name)
    plt.ylabel(ylabel.name)

    # get figure to save to file
    if savefig_path:
        ht_figure = ax.get_figure()
        fig_name = savefig_path+"/heatmap_"+xlabel.name+"_"+ylabel.name
        print(os.path.abspath(fig_name))
        ht_figure.savefig(fig_name)

    plt.clf()
    plt.cla()
    plt.close()

def plot_heatmap_rescaled(data,
                xlabel,
                ylabel,
                min_value_x=0,
                max_value_x=25,
                min_value_y=0,
                max_value_y=25,
                minimization=False,
                savefig_path=None,
                 ):
    plt.clf()
    plt.cla()
    ax = sns.heatmap(data)
    ax.invert_yaxis()
    plt.xlabel(xlabel.name)
    plt.ylabel(ylabel.name)

    num_cells_x = 25
    num_cells_y = 25

    if max_value_x > 25:
        num_cells_x = 25
    else:
        num_cells_x = max_value_x + 1

    if max_value_y > 25:
        num_cells_y = 25
    else:
        num_cells_y = max_value_y + 1

    xtickslabel = [round(the_bin, 1) for the_bin in np.linspace(min_value_x, max_value_x, num_cells_x)]
    ytickslabel = [round(the_bin, 1) for the_bin in np.linspace(min_value_y, max_value_y, num_cells_y)]

    ax.set_xticklabels(xtickslabel)
    plt.xticks(rotation=30)
    ax.set_yticklabels(ytickslabel)
    plt.yticks(rotation=0)
    #ax.set_aspect("equal")
    ax.tick_params(axis='both', which='major', labelsize=7)
    # get figure to save to file
    if savefig_path:
        ht_figure = ax.get_figure()
        fig_name = savefig_path+"/heatmap_"+xlabel.name+"_"+ylabel.name
        print(os.path.abspath(fig_name))
        ht_figure.savefig(fig_name, dpi=400)

    plt.clf()
    plt.cla()
    plt.close()

def plot_archive_rescaled(data,
                labels,
                xlabel,
                ylabel,
                min_value_x=0,
                max_value_x=25,
                min_value_y=0,
                max_value_y=25,
                minimization=False,
                savefig_path=None,
                 ):
    plt.clf()
    plt.cla()

    str_labels = np.full(labels.shape, "", dtype=(str))

    for (i, j), value in np.ndenumerate(labels):
        if value > 0:
            str_labels[i][j] = value

    annotations = str_labels

    ax = sns.heatmap(data,  annot=annotations, fmt="s")
    ax.invert_yaxis()
    plt.xlabel(xlabel.name)
    plt.ylabel(ylabel.name)

    num_cells_x = 25
    num_cells_y = 25

    if max_value_x > 25:
        num_cells_x = 25
    else:
        num_cells_x = max_value_x + 1

    if max_value_y > 25:
        num_cells_y = 25
    else:
        num_cells_y = max_value_y + 1

    xtickslabel = [round(the_bin, 1) for the_bin in np.linspace(min_value_x, max_value_x, num_cells_x)]
    ytickslabel = [round(the_bin, 1) for the_bin in np.linspace(min_value_y, max_value_y, num_cells_y)]

    ax.set_xticklabels(xtickslabel)
    plt.xticks(rotation=30)
    ax.set_yticklabels(ytickslabel)
    plt.yticks(rotation=0)

    #ax.set_aspect("equal")
    # get figure to save to file
    if savefig_path:
        ht_figure = ax.get_figure()
        fig_name = savefig_path+"/archive_"+xlabel.name+"_"+ylabel.name
        print(os.path.abspath(fig_name))
        ht_figure.savefig(fig_name, dpi=400)

    plt.clf()
    plt.cla()
    plt.close()

def plot_heatmap_rescaled_expansion(data, expansion,
                xlabel,
                ylabel,
                min_value_x=0,
                max_value_x=25,
                min_value_y=0,
                max_value_y=25,
                minimization=False,
                savefig_path=None,
                 ):
    plt.clf()
    plt.cla()
    ax = sns.heatmap(data)
    ax.invert_yaxis()
    plt.xlabel(xlabel.name)
    plt.ylabel(ylabel.name)

    num_cells_x = 25
    num_cells_y = 25

    if max_value_x > 25:
        num_cells_x = 25
    else:
        num_cells_x = max_value_x + 1

    if max_value_y > 25:
        num_cells_y = 25
    else:
        num_cells_y = max_value_y + 1

    xtickslabel = [round(the_bin, 1) for the_bin in np.linspace(min_value_x, max_value_x, num_cells_x)]
    ytickslabel = [round(the_bin, 1) for the_bin in np.linspace(min_value_y, max_value_y, num_cells_y)]

    ax.set_xticklabels(xtickslabel)
    plt.xticks(rotation=30)
    ax.set_yticklabels(ytickslabel)
    plt.yticks(rotation=0)

    for index in expansion:
        ax.add_patch(Rectangle((index[0], index[1]), 1, 1, fill=False, edgecolor='red', lw=2))
    ax.set_aspect("equal")
    # get figure to save to file
    if savefig_path:
        ht_figure = ax.get_figure()
        fig_name = savefig_path+"/expansion_"+xlabel.name+"_"+ylabel.name
        print(os.path.abspath(fig_name))
        ht_figure.savefig(fig_name, dpi=400)

    plt.clf()
    plt.cla()
    plt.close()

def plot_svg(xml, filename):
    root = ET.fromstring(xml)
    svg_path = root.find(NAMESPACE + 'path').get('d')
    wsvg(svg_path, filename=filename+'.svg')

def getImage(path):
    img = plt.imread(path)
    res = cv2.resize(img, dsize=(150, 150), interpolation=cv2.INTER_AREA)
    return OffsetImage(res)

def plot_roads(dir_path, xlabel, ylabel):
    paths = glob.glob(dir_path + "/"+xlabel+"_"+ylabel+"/*.jpg")
    x=[]
    y=[]
    for a in paths:
        pattern = re.compile('[\d\.]+, [\d\.]+')
        segments = pattern.findall(a)
        for se in segments:
            se = se.split(', ')
            x.append(int(se[0]))
            y.append(int(se[1]))

    plt.cla()

    fig, ax = plt.subplots(figsize=(35,35))
    #ax.scatter(x, y)

    for x0, y0, path in zip(x, y, paths):

        ab = AnnotationBbox(getImage(path), (y0, x0), frameon=False)
        ax.add_artist(ab)

    ax.set_xlim(6,24)
    ax.set_ylim(6,24)

    plt.xlabel(xlabel)
    plt.ylabel(ylabel)

    plt.xticks(np.arange(6, 25, 1))
    plt.yticks(np.arange(6, 25, 1))

    plt.grid(color='blue', linestyle='-', linewidth=0.09)
    ht_figure = ax.get_figure()
    ht_figure.savefig(dir_path+"/roads_"+xlabel+"_"+ylabel, dpi=400)

    plt.clf()
    plt.close()

ext: py

max_stars_repo_name: IharBakhanovich/DeepHyperion

max_issues_repo_name: IharBakhanovich/DeepHyperion

max_forks_repo_name: IharBakhanovich/DeepHyperion

reason: import numpy


text:
'''
The Fisheries Habitat Scenarios Tool IO module contains functions for handling
inputs and outputs
'''

import logging
import os
import csv
import copy

import numpy as np

from .. import utils

LOGGER = logging.getLogger('natcap.invest.fisheries.hst_io')

# Fetch and Verify Arguments
def fetch_args(args):
    '''
    Fetches input arguments from the user, verifies for correctness and
    completeness, and returns a list of variables dictionaries

    Args:
        args (dictionary): arguments from the user (same as Fisheries
            Preprocessor entry point)

    Returns:
        vars_dict (dictionary): dictionary containing necessary variables

    Raises:
        ValueError: parameter mismatch between Population and Habitat CSV files

    Example Returns::

        vars_dict = {
            'workspace_dir': 'path/to/workspace_dir/',
            'output_dir': 'path/to/output_dir/',
            'sexsp': 2,
            'gamma': 0.5,

            # Pop Vars
            'population_csv_path': 'path/to/csv_path',
            'Surv_nat_xsa': np.array(
                [[[...], [...]], [[...], [...]], ...]),
            'Classes': np.array([...]),
            'Class_vectors': {
                'Vulnfishing': np.array([...], [...]),
                'Maturity': np.array([...], [...]),
                'Duration': np.array([...], [...]),
                'Weight': np.array([...], [...]),
                'Fecundity': np.array([...], [...]),
            },
            'Regions': np.array([...]),
            'Region_vectors': {
                'Exploitationfraction': np.array([...]),
                'Larvaldispersal': np.array([...]),
            },

            # Habitat Vars
            'habitat_chg_csv_path': 'path/to/csv',
            'habitat_dep_csv_path': 'path/to/csv',
            'Habitats': ['habitat1', 'habitat2', ...],
            'Hab_classes': ['class1', 'class2', ...],
            'Hab_regions': ['region1', 'region2', ...],
            'Hab_chg_hx': np.array(
                [[[...], [...]], [[...], [...]], ...]),
            'Hab_dep_ha': np.array(
                [[[...], [...]], [[...], [...]], ...]),
            'Hab_class_mvmt_a': np.array([...]),
            'Hab_dep_num_a': np.array([...]),
        }

    '''
    args = args.copy()
    sexsp_dict = {
        'no': 1,
        'yes': 2,
    }
    args['sexsp'] = sexsp_dict[args['sexsp'].lower()]
    args['gamma'] = float(args['gamma'])

    # Fetch Data
    args['output_dir'] = os.path.join(args['workspace_dir'], 'output')
    utils.make_directories([args['workspace_dir'], args['output_dir']])
    pop_dict = read_population_csv(args)
    habitat_chg_dict = read_habitat_chg_csv(args)
    habitat_dep_dict = read_habitat_dep_csv(args)

    # Check that habitat names match between habitat parameter files
    assert habitat_dep_dict['Habitats'] == habitat_chg_dict['Habitats'], (
        "Mismatch between Habitat names in Habitat Paramater CSV files.")

    del habitat_dep_dict['Habitats']
    habitat_dict = dict(habitat_chg_dict.items() + habitat_dep_dict.items())

    # Check that classes and regions match
    P_Classes = [x.lower() for x in pop_dict['Classes']]
    H_Classes = [x.lower() for x in habitat_dict['Hab_classes']]
    assert P_Classes == H_Classes, (
        "Mismatch between class names in Population and Habitat CSV files "
        "(%s vs %s)") % (P_Classes, H_Classes)

    P_Regions = [x.lower() for x in pop_dict['Regions']]
    H_Regions = [x.lower() for x in habitat_dict['Hab_regions']]
    assert P_Regions == H_Regions, (
        "Mismatch between region names in Population and Habitat CSV files")

    # Combine Data
    vars_dict = dict(args.items() + pop_dict.items() + habitat_dict.items())

    return vars_dict

def read_population_csv(args):
    '''
    Parses and verifies a single Population Parameters CSV file

    Parses and verifies inputs from the Population Parameters CSV file.
    If not all necessary vectors are included, the function will raise a
    MissingParameter exception. Survival matrix will be arranged by
    class-elements, 2nd dim: sex, and 3rd dim: region. Class vectors will
    be arranged by class-elements, 2nd dim: sex (depending on whether model
    is sex-specific) Region vectors will be arraged by region-elements,
    sex-agnostic.

    Args:
        args (dictionary): arguments provided by user

    Returns:
        pop_dict (dictionary): dictionary containing verified population
            arguments

    Raises:
        MissingParameter: required parameter not included
        ValueError: values are out of bounds or of wrong type

    Example Returns::

        pop_dict = {
            'population_csv_path': 'path/to/csv',
            'Surv_nat_xsa': np.array(
                [[...], [...]], [[...], [...]], ...),

            # Class Vectors
            'Classes': np.array([...]),
            'Class_vector_names': [...],
            'Class_vectors': {
                'Vulnfishing': np.array([...], [...]),
                'Maturity': np.array([...], [...]),
                'Duration': np.array([...], [...]),
                'Weight': np.array([...], [...]),
                'Fecundity': np.array([...], [...]),
            },

            # Region Vectors
            'Regions': np.array([...]),
            'Region_vector_names': [...],
            'Region_vectors': {
                'Exploitationfraction': np.array([...]),
                'Larvaldispersal': np.array([...]),
            },
        }
    '''
    path = args['population_csv_path']
    pop_dict = _parse_population_csv(path, args['sexsp'])

    # Check that required information exists
    assert 'Surv_nat_xsa' in pop_dict, (
        "Population Parameters File does not contain a Survival Matrix")

    Necessary_Params = ['Classes', 'Regions', 'Surv_nat_xsa']
    Matching_Params = [i for i in pop_dict.keys() if i in Necessary_Params]

    assert len(Matching_Params) == len(Necessary_Params), (
        "Population Parameters File does not contain all necessary "
        "parameters. %s") % path

    # Checks that all Survival Matrix elements exist between [0, 1]
    A = pop_dict['Surv_nat_xsa']
    assert all((A.min() >= 0.0, A.max() <= 1.0)), (
        "Surivial Matrix contains values outside [0, 1]")

    return pop_dict

def _parse_population_csv(path, sexsp):
    '''
    Parses the given Population Parameters CSV file and returns a dictionary
    of lists, arrays, and matrices

    Dictionary items containing lists, arrays or matrices are capitalized,
    while single variables are lowercase.

    Keys: Surv_nat_xsa, Vulnfishing, Maturity, Duration, Weight, Fecundity,
            Exploitationfraction, Larvaldispersal, Classes, Regions

    Returns:
        pop_dict (dictionary): verified population arguments

    Example Returns::

        pop_dict = {
            'Surv_nat_xsa': np.array(
                [[...], [...]], [[...], [...]], ...),

            # Class Vectors
            'Classes': np.array([...]),
            'Class_vector_names': [...],
            'Class_vectors': {
                'Vulnfishing': np.array([...], [...]),
                'Maturity': np.array([...], [...]),
                'Duration': np.array([...], [...]),
                'Weight': np.array([...], [...]),
                'Fecundity': np.array([...], [...]),
            }

            # Region Vectors
            'Regions': np.array([...]),
            'Region_vector_names': [...],
            'Region_vectors': {
                'Exploitationfraction': np.array([...]),
                'Larvaldispersal': np.array([...]),
            }
        }
    '''
    assert sexsp in (1, 2), 'Sexsp value %s unknown' % sexsp
    csv_data = []
    pop_dict = {}

    with open(path, 'rb') as csvfile:
        reader = csv.reader(csvfile)
        for line in reader:
            csv_data.append(line)

    start_rows = _get_table_row_start_indexes(csv_data)
    start_cols = _get_table_col_start_indexes(csv_data, start_rows[0])
    end_rows = _get_table_row_end_indexes(csv_data)
    end_cols = _get_table_col_end_indexes(csv_data, start_rows[0])

    classes = _get_col(
        csv_data, start_rows[0])[0:end_rows[0]+1]

    pop_dict["Classes"] = classes[1:]
    if sexsp == 2:
        pop_dict["Classes"] = pop_dict["Classes"][0:len(pop_dict["Classes"])/2]

    regions = _get_row(csv_data, start_cols[0])[0: end_cols[0]+1]

    pop_dict["Regions"] = regions[1:]

    surv_table = _get_table(csv_data, start_rows[0]+1, start_cols[0]+1)
    class_attributes_table = _get_table(
        csv_data, start_rows[0], start_cols[1])
    region_attributes_table = _get_table(
        csv_data, start_rows[1], start_cols[0])

    if sexsp == 1:
        # Sex Neutral
        pop_dict['Surv_nat_xsa'] = np.array(
            [surv_table], dtype=np.float_).swapaxes(1, 2).swapaxes(0, 1)

    elif sexsp == 2:
        # Sex Specific
        female = np.array(surv_table[0:len(surv_table)/sexsp], dtype=np.float_)
        male = np.array(surv_table[len(surv_table)/sexsp:], dtype=np.float_)
        pop_dict['Surv_nat_xsa'] = np.array(
            [female, male]).swapaxes(1, 2).swapaxes(0, 1)

    Class_vector_names = class_attributes_table[0]
    for i in range(0, len(Class_vector_names)):
        Class_vector_names[i] = Class_vector_names[i].capitalize()
    pop_dict['Class_vector_names'] = Class_vector_names
    pop_dict['Class_vectors'] = {}
    for col in range(0, len(class_attributes_table[0])):
        pop_dict['Class_vectors'].update(_vectorize_attribute(
            _get_col(class_attributes_table, col), sexsp))

    Region_vector_names = []
    for attribute in region_attributes_table:
        Region_vector_names.append(attribute[0].capitalize())
    pop_dict['Region_vector_names'] = Region_vector_names
    pop_dict['Region_vectors'] = {}
    for row in range(0, len(region_attributes_table)):
        pop_dict['Region_vectors'].update(_vectorize_reg_attribute(
            _get_row(region_attributes_table, row)))

    return pop_dict

def read_habitat_dep_csv(args):
    '''
    Parses and verifies a Habitat Dependency Parameters CSV file and returns a
    dictionary of information related to the interaction between a species and
    the given habitats.

    Parses the Habitat Parameters CSV file for the following vectors:

        + Names of Habitats and Classes
        + Habitat-Class Dependency

    The following vectors are derived from the information given in the file:

        + Classes where movement between habitats occurs
        + Number of habitats that a particular class depends upon

    Args:
        args (dictionary): arguments from the user (same as Fisheries
            HST entry point)

    Returns:
        habitat_dep_dict (dictionary): dictionary containing necessary
            variables

    Raises:
        MissingParameter - required parameter not included
        ValueError - values are out of bounds or of wrong type
        IndexError - likely a file formatting issue

    Example Returns::

        habitat_dep_dict = {
            'Habitats': ['habitat1', 'habitat2', ...],
            'Hab_classes': ['class1', 'class2', ...],
            'Hab_dep_ha': np.array(
                [[[...], [...]], [[...], [...]], ...]),
            'Hab_class_mvmt_a': np.array([...]),
            'Hab_dep_num_a': np.array([...]),
        }
    '''
    habitat_dep_dict = _parse_habitat_dep_csv(args)

    # Verify provided information
    A = habitat_dep_dict['Hab_dep_ha']
    assert (A.min() >= 0.0 and A.max() <= 1.0), (
        "At least one element of the Habitat Dependency by Class vectors is "
        "out of bounds. Values must be between [0, 1] inclusive.")

    # Derive additional information
    # TRANSITION BITMAP NEEDS CLEARER DEFINITION
    A_ah = copy.copy(habitat_dep_dict['Hab_dep_ha'].swapaxes(0, 1))
    A_ah[A_ah != 0] = 1
    Hab_class_mvmt_a = [0]
    for i in range(1, len(A_ah)):
        if np.all(A_ah[i] == A_ah[i-1]):
            Hab_class_mvmt_a.append(0)
        else:
            Hab_class_mvmt_a.append(1)
    habitat_dep_dict['Hab_class_mvmt_a'] = np.array(Hab_class_mvmt_a)

    Hab_dep_num_a = []
    for A_h in A_ah:
        Hab_dep_num_a.append(int(len(np.nonzero(A_h)[0])))
    habitat_dep_dict['Hab_dep_num_a'] = np.array(Hab_dep_num_a)

    return habitat_dep_dict

def _parse_habitat_dep_csv(args):
    '''
    Parses the Habitat Dependency Parameters CSV file for the following vectors
        + Names of Habitats and Classes
        + Habitat-Class Dependency

    Args:
        args (dictionary): arguments from the user (same as Fisheries
            HST entry point)

    Returns:
        habitat_dep_dict (dictionary): dictionary containing necessary
            variables

    Raises:
        MissingParameter: required parameter not included
        IndexError: likely a file formatting issue

    Example Returns::

        habitat_dep_dict = {
            'Habitats': ['habitat1', 'habitat2', ...],
            'Hab_classes': ['class1', 'class2', ...],
            'Hab_dep_ha': np.array(
                [[[...], [...]], [[...], [...]], ...]),
            'Hab_class_mvmt_a': np.array([...]),
            'Hab_dep_num_a': np.array([...]),
        }
    '''
    path = args['habitat_dep_csv_path']
    csv_data = []
    habitat_dep_dict = {}

    with open(path, 'rb') as csvfile:
        reader = csv.reader(csvfile)
        for line in reader:
            csv_data.append(line)

    # Find Boundary Information
    start_rows = _get_table_row_start_indexes(csv_data)
    start_cols = _get_table_col_start_indexes(csv_data, start_rows[0])
    end_rows = _get_table_row_end_indexes(csv_data)
    end_cols = _get_table_col_end_indexes(csv_data, start_rows[0])

    # Start Parsing Data
    Habitats = _get_col(
        csv_data, start_rows[0])[start_rows[0]+1:end_rows[0]+1]
    Hab_classes = _get_row(
        csv_data, start_cols[0])[start_cols[0]+1:end_cols[0]+1]
    Hab_dep_ha = _get_table(
        csv_data, start_rows[0]+1, start_cols[0]+1)

    # Standardize capitalization
    Habitats = map(lambda x: x.capitalize(), Habitats)
    Hab_classes = map(lambda x: x.capitalize(), Hab_classes)

    # Reformat and add data to dictionary
    habitat_dep_dict['Habitats'] = Habitats
    habitat_dep_dict['Hab_classes'] = Hab_classes
    habitat_dep_dict['Hab_dep_ha'] = np.array(Hab_dep_ha, dtype=float)

    return habitat_dep_dict

def read_habitat_chg_csv(args):
    '''
    Parses and verifies a Habitat Change Parameters CSV file and returns a
    dictionary of information related to the interaction between a species
    and the given habitats.

    Parses the Habitat Change Parameters CSV file for the following vectors:

        + Names of Habitats and Regions
        + Habitat Area Change

    Args:
        args (dictionary): arguments from the user (same as Fisheries
            HST entry point)

    Returns:
        habitat_chg_dict (dictionary): dictionary containing necessary
            variables

    Raises:
        MissingParameter: required parameter not included
        ValueError: values are out of bounds or of wrong type
        IndexError: likely a file formatting issue

    Example Returns::

        habitat_chg_dict = {
            'Habitats': ['habitat1', 'habitat2', ...],
            'Hab_regions': ['region1', 'region2', ...],
            'Hab_chg_hx': np.array(
                [[[...], [...]], [[...], [...]], ...]),
        }
    '''
    habitat_chg_dict = _parse_habitat_chg_csv(args)

    # Verify provided information
    A = habitat_chg_dict['Hab_chg_hx']
    assert A.min() >= -1.0, (
        "At least one element of the Habitat Area Change vectors is out of "
        "bounds. Values must be between [-1.0, +inf).")

    return habitat_chg_dict

def _parse_habitat_chg_csv(args):
    '''
    Parses the Habitat Change Parameters CSV file for the following vectors
        + Names of Habitats and Regions
        + Habitat Area Change

    Args:
        args (dictionary): arguments from the user (same as Fisheries
            HST entry point)

    Returns:
        habitat_chg_dict (dictionary): dictionary containing necessary
            variables

    Raises:
        MissingParameter: required parameter not included
        IndexError: likely a file formatting issue

    Example Returns::

        habitat_chg_dict = {
            'Habitats': ['habitat1', 'habitat2', ...],
            'Hab_regions': ['region1', 'region2', ...],
            'Hab_chg_hx': np.array(
                [[[...], [...]], [[...], [...]], ...]),
        }
    '''
    path = args['habitat_chg_csv_path']
    csv_data = []
    habitat_chg_dict = {}

    with open(path, 'rb') as csvfile:
        reader = csv.reader(csvfile)
        for line in reader:
            csv_data.append(line)

    # Find Boundary Information
    start_rows = _get_table_row_start_indexes(csv_data)
    start_cols = _get_table_col_start_indexes(csv_data, start_rows[0])
    end_rows = _get_table_row_end_indexes(csv_data)
    end_cols = _get_table_col_end_indexes(csv_data, start_rows[0])

    # Start Parsing Data
    Habitats = _get_col(
        csv_data, start_rows[0])[start_rows[0]+1:end_rows[0]+1]
    Hab_regions = _get_row(
        csv_data, start_cols[0])[start_cols[0]+1:end_cols[0]+1]
    Hab_chg_hx = _get_table(
        csv_data, start_rows[0]+1, start_cols[0]+1)

    # Standardize capitalization
    Habitats = map(lambda x: x.capitalize(), Habitats)
    Hab_regions = map(lambda x: x.capitalize(), Hab_regions)

    # Reformat and add data to dictionary
    habitat_chg_dict['Habitats'] = Habitats
    habitat_chg_dict['Hab_regions'] = Hab_regions
    habitat_chg_dict['Hab_chg_hx'] = np.array(Hab_chg_hx, dtype=float)

    return habitat_chg_dict

# Helper functions for navigating CSV files
def _get_col(lsts, col):
    l = []
    for row in range(0, len(lsts)):
        l.append(lsts[row][col])
    return l

def _get_row(lsts, row):
    l = []
    for entry in range(0, len(lsts[row])):
        l.append(lsts[row][entry])
    return l

def _get_table(lsts, row, col):
    table = []

    end_col = col
    while end_col + 1 < len(lsts[0]) and lsts[0][end_col + 1] != '':
        end_col += 1

    end_row = row
    while end_row + 1 < len(lsts) and lsts[end_row + 1][0] != '':
        end_row += 1

    for line in range(row, end_row + 1):
        table.append(lsts[line][col:end_col + 1])
    return table

def _get_table_row_start_indexes(lsts):
    indexes = []
    if lsts[0][0] != '':
        indexes.append(0)
    for line in range(1, len(lsts)):
        if lsts[line - 1][0] == '' and lsts[line][0] != '':
            indexes.append(line)
    return indexes

def _get_table_col_start_indexes(lsts, top):
    indexes = []
    if lsts[top][0] != '':
        indexes.append(0)
    for col in range(1, len(lsts[top])):
        if lsts[top][col - 1] == '' and lsts[top][col] != '':
            indexes.append(col)
    return indexes

def _get_table_row_end_indexes(lsts):
    indexes = []
    for i in range(1, len(lsts)):
        if (lsts[i][0] == '' and lsts[i-1][0] != ''):
            indexes.append(i-1)
    if lsts[-1] != '':
        indexes.append(len(lsts)-1)

    return indexes

def _get_table_col_end_indexes(lsts, top):
    indexes = []
    for i in range(1, len(lsts[top])):
        if (lsts[top][i] == '' and lsts[top][i-1] != ''):
            indexes.append(i-1)
    if lsts[top][-1] != '':
        indexes.append(len(lsts[top])-1)

    return indexes

def _vectorize_attribute(lst, rows):
    d = {}
    a = np.array(lst[1:], dtype=np.float_)
    a = np.reshape(a, (rows, a.shape[0] / rows))
    d[lst[0].strip().capitalize()] = a
    return d

def _vectorize_reg_attribute(lst):
    d = {}
    a = np.array(lst[1:], dtype=np.float_)
    d[lst[0].strip().capitalize()] = a
    return d

# Generate Outputs
def save_population_csv(vars_dict):
    '''
    Creates a new Population Parameters CSV file based the provided inputs.

    Args:
        vars_dict (dictionary): variables generated by preprocessor arguments
            and run.

    Example Args::

        args = {
            'workspace_dir': 'path/to/workspace_dir/',
            'output_dir': 'path/to/output_dir/',
            'sexsp': 2,
            'population_csv_path': 'path/to/csv',  # original csv file
            'Surv_nat_xsa': np.ndarray([...]),
            'Surv_nat_xsa_mod': np.ndarray([...]),

            # Class Vectors
            'Classes': np.array([...]),
            'Class_vector_names': [...],
            'Class_vectors': {
                'Vulnfishing': np.array([...], [...]),
                'Maturity': np.array([...], [...]),
                'Duration': np.array([...], [...]),
                'Weight': np.array([...], [...]),
                'Fecundity': np.array([...], [...]),
            },

            # Region Vectors
            'Regions': np.array([...]),
            'Region_vector_names': [...],
            'Region_vectors': {
                'Exploitationfraction': np.array([...]),
                'Larvaldispersal': np.array([...]),
            },

            # other arguments are ignored ...
        }

    Note:
        + Creates a modified Population Parameters CSV file located in the 'workspace/output/' folder
        + Currently appends '_modified' to original filename for new filename
    '''
    Surv_nat_asx_mod = vars_dict['Surv_nat_xsa_mod'].swapaxes(0, 2)
    num_classes = len(vars_dict['Classes'])
    l = []
    # Create header
    l.append('Class')
    for region in vars_dict['Regions']:
        l.append(region)
    l.append('')
    l = [l]

    # Add survival matrix
    for c in range(1, num_classes + 1):
        l.append([])
        l[c].append(vars_dict['Classes'][c-1])
        for i in Surv_nat_asx_mod[c-1][0].tolist():
            l[c].append(i)
        l[c].append('')
    # l[c].append('')
    if vars_dict['sexsp'] == 2:
        for c in range(1, num_classes + 1):
            l.append([])
            l[c + num_classes].append(vars_dict['Classes'][c-1])
            for i in Surv_nat_asx_mod[c-1][1].tolist():
                l[c + num_classes].append(i)
            l[c + num_classes].append('')

    # Add class vectors
    for key in vars_dict['Class_vector_names']:
        l[0].append(key)
        vector = vars_dict['Class_vectors'][key].tolist()
        if vars_dict['sexsp'] == 2:
            vector = vector[0] + vector[1]
        else:
            vector = vector[0]
        map(lambda l, v: l.append(v), l[1:], vector)

    # Add row of spaces
    l.append([])
    while (len(l[-1]) < len(l[0])):
        l[-1].append('')

    # Add region vectors
    for key in vars_dict['Region_vector_names']:
        l.append([])
        l[-1].append(key)
        vector = vars_dict['Region_vectors'][key].tolist()
        for i in vector:
            l[-1].append(i)
        while (len(l[-1]) < len(l[0])):
            l[-1].append('')

    # Write List to File
    basename, ext = os.path.splitext(os.path.basename(
        vars_dict['population_csv_path']))
    filename = basename + '_modified' + ext
    output_path = os.path.join(vars_dict['output_dir'], filename)
    f = open(output_path, 'wb')
    wr = csv.writer(f)
    for row in l:
        wr.writerow(row)

max_stars_repo_name: danielgis/invest

max_issues_repo_name: danielgis/invest

max_forks_repo_name: danielgis/invest


text:
import argparse as arp
import os, pathlib
import os.path as osp
import numpy as np
import pandas

from reinforcement_learning.gym.envs.donkey_car.donkey_env import DonkeyEnv
from reinforcement_learning.ddpg.ddpg import DDPG as ddpg
from reinforcement_learning.common.vec_env.subproc_vec_env import SubprocVecEnv
from reinforcement_learning.ddpg.policies import MlpPolicy
from reinforcement_learning import logger
from reinforcement_learning.common.callbacks import CheckpointCallback

def find_checkpoint_with_latest_date(checkpoint_dir, prefix='rl_model_'):
    checkpoint_files = [item for item in os.listdir(checkpoint_dir) if osp.isfile(osp.join(checkpoint_dir, item)) and item.startswith(prefix) and item.endswith('.zip')]
    checkpoint_fpaths = [osp.join(checkpoint_dir, item) for item in checkpoint_files]
    checkpoint_dates = [pathlib.Path(item).stat().st_mtime for item in checkpoint_fpaths]
    idx = sorted(range(len(checkpoint_dates)), key=lambda k: checkpoint_dates[k])
    return checkpoint_fpaths[idx[-1]]

def make_env(env_class, *args):
    fn = lambda: env_class(*args)
    return fn

def generate_traj(env, model, nsteps):
    n = len(env.remotes)
    states = [[] for _ in range(n)]
    actions = [[] for _ in range(n)]
    obs = env.reset()
    for i in range(nsteps):
        action, state = model.predict(obs)
        next_obs, reward, done, info = env.step(action)
        for e in range(n):
            states[e].append(obs[e][0])
            actions[e].append(action[e])
        obs = np.array(next_obs)
    return states, actions

if __name__ == '__main__':

    parser = arp.ArgumentParser(description='Train RL agent.')
    parser.add_argument('-e', '--exe', default='/home/mizolotu/DonkeyCar/donkey_sim.x86_64', help='Simulator exe')
    parser.add_argument('-p', '--port', help='Port', default=9091)
    parser.add_argument('-n', '--nenvs', help='Number of environments', type=int, default=1)
    parser.add_argument('-s', '--nsteps', help='Number of steps', type=int, default=256)
    parser.add_argument('-u', '--nupdates', help='Number of episodes', type=int, default=1000000)
    parser.add_argument('-l', '--level', help='Level', default='generated_track')
    parser.add_argument('-c', '--checkpoint', help='Checkpoint')  # e.g. 'rl_model_384_steps.zip
    args = parser.parse_args()

    if args.nenvs is not None:
        nenvs = args.nenvs

    env_class = DonkeyEnv
    conf = {'exe_path': args.exe, 'port' : args.port}
    algorithm = ddpg
    policy = MlpPolicy
    total_steps = args.nsteps * args.nupdates

    modeldir = '{0}/{1}/{2}'.format('models', env_class.__name__, algorithm.__name__)
    logdir = '{0}/{1}/{2}'.format('results', env_class.__name__, algorithm.__name__)
    format_strs = os.getenv('', 'stdout,log,csv').split(',')
    logger.configure(os.path.abspath(logdir), format_strs)

    # create environments

    env_fns = [make_env(env_class, args.level, conf) for env_idx in range(nenvs)]
    env = SubprocVecEnv(env_fns)

    if args.checkpoint is None:
        chkpt = find_checkpoint_with_latest_date(modeldir)
    else:
        chkpt = args.checkpoint
    print(chkpt)

    model = algorithm.load(chkpt)
    model.set_env(env)
    print('Model has been loaded from {0}!'.format(chkpt))

    try:
        p = pandas.read_csv('data/trajs.csv')
        trajs = p.values
    except Exception as e:
        print(e)
        trajs = []
        for i in range(100):
            states, actions = generate_traj(env, model, args.nsteps)
            for se, ae in zip(states, actions):
                trajs.append([])
                for s, a in zip(se, ae):
                    trajs[-1].append(np.hstack([s, a]))
                trajs[-1] = np.vstack(trajs[-1])
        trajs = np.vstack(trajs)
        pandas.DataFrame(trajs).to_csv('data/trajs.csv', index=False, header=False)
    print(trajs.shape)

    del model

    print('Continue training with RL!')
    model = algorithm(MlpPolicy, env, verbose=1)
    model.pretrain(trajs, n_epochs=1000)
    model.learn(total_timesteps=total_steps)

max_stars_repo_name: mizolotu/DonkeyCarExperiments

max_issues_repo_name: mizolotu/DonkeyCarExperiments

max_forks_repo_name: mizolotu/DonkeyCarExperiments


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Feb 15 14:07:04 2016
A script to analyze some of the genes identified by Engelmann et al 2011,
A Comprehensive Analysis of Gene Expression Changes Provoked by Bacterial
and Fungal Infection in C elegans.

@author: dangeles
"""
import tissue_enrichment_analysis as tea  # the library to be used
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import os

# tissue dictionary to use
tissue_df = pd.read_csv('../input/final_cutoff33_threshold0.95_methodany.csv')

# rna-seq datasets to reanalyze
#Bacterial
dfLum = pd.read_csv('../input/Engelmann/luminescens_Engelmann_2011.csv')
dfMarc = pd.read_csv('../input/Engelmann/marcescens_Engelmann_2011.csv')
dfFaec = pd.read_csv('../input/Engelmann/faecalis_Engelmann_2011.csv')

# Fungal
dfHarp = pd.read_csv('../input/Engelmann/harposporium_Engelmann_2011.csv')
dfCon = pd.read_csv('../input/Engelmann/coniospora_Engelmann_2011.csv')
# dfSin= pd.read_csv('../input/Engelmann/sinha_2012.csv')

# dictionary of gene names
names = pd.read_csv('../input/Engelmann/c_elegans.PRJNA13758.WS241.livegeneIDs.unmaprm.txt',
                    sep='\t', comment='#')

# make the directories to place the analyses
dirEngelmann = '../output/Engelmann/'
dirAnalysis = '../output/Engelmann/Analysis'
dirGraphs = '../output/Engelmann/Graphs'
DIRS = [dirEngelmann, dirAnalysis, dirGraphs]

# Make all the necessary dirs if they don't already exist
for d in DIRS:
    if not os.path.exists(d):
        os.makedirs(d)

# Selector functions to draw WBIDs from
f = lambda x, y: (names.HumanReadable.isin(x[y]))
g = lambda x, y: (names.GeneName.isin(x[y]))

# ==============================================================================
# ==============================================================================
# # Engelmann Analysis
# ==============================================================================
# ==============================================================================

class receptacle(object):

    def __init__(self, name):
        self.name = name
        self.result_dict = {}
        self.n = 0

    def add_result(self, name, df):
        self.result_dict[name] = df
        self.n += 1

Lnames = ['Otorhabdus luminescens', 'Serratia marcescens',
          'Enterococcus faecalis', 'Harposporium sp', 'Drechmeria coniospora']
Ldf = [dfLum, dfMarc, dfFaec, dfHarp, dfCon]
Ldirection = ['Infection_upregulated', 'Infection_downregulated']

# direction specific
Lreceptacles = {}
n_genes = []
for i, df in enumerate(Ldf):
    fname = Lnames[i]
    obj = receptacle(fname)

    for direction in Ldirection:
        ind = g(df[df[direction] == 1.0], 'SequenceNameGene')
        x = names[ind].WBID

        print('---------')
        print(fname + ' ' + direction)
        print('Number of genes submitted for analysis ', len(x))
        y = tissue_df[tissue_df.wbid.isin(x)].wbid.unique().shape[0]
        print('Number of genes used for analysis ', y)
        print('\n')

        df_res, unused = tea.enrichment_analysis(x, tissue_df,
                                                 show=False, alpha=0.1)

        print(df_res.empty)
        if len(df_res) > 0:
            obj.add_result(direction, df_res)

        n_genes.append(['{0}, {1}'.format(fname, direction), y, len(unused)])

    Lreceptacles[fname] = obj

# make
for species in Lreceptacles:
    current = Lreceptacles[species]

    n = current.n  # number of dfs
    keys = current.result_dict.keys()

    if n == 0:
        next

    i = 0
    if n > 1:
        fig, ax = plt.subplots(nrows=n, figsize=(8, 8))
        fig.subplots_adjust(top=2)
        fig.suptitle(species, fontsize=15, y=1.02)

        tea.plot_enrichment_results(current.result_dict['Infection_downregulated'],
                                    title='Name', save=False, fig=fig, ax=ax[0])

        # suppress xlabel
        ax[0].set_xlabel('')
        ax[0].set_ylabel('Down-Regulated Tissues')
        ax[0].yaxis.set_label_position('right')
        tea.plot_enrichment_results(current.result_dict['Infection_upregulated'],
                                    title='Name', save=False, fig=fig, ax=ax[1])
        ax[1].set_ylabel('Up-Regulated Tissues')
        xlabel = ax[1].set_xlabel('Enrichment Fold Change - {0}'.format(species))
        ax[1].yaxis.set_label_position('right')
        fig.tight_layout()
        plt.savefig('../output/Engelmann/Graphs/'+species+'Enrichment.pdf',
                    rect=[0, 0.03, 1, 0.95], bbox_extra_artists=[xlabel],
                    bbox_inches='tight')
#        plt.close()

    else:
        fig, ax = plt.subplots(nrows=n, figsize=(8, 4))
        fig.subplots_adjust(top=0.85)
        fig.suptitle(species, fontsize=15, y=1.05)

        if 'Infection_downregulated' in current.result_dict:
            ax = tea.plot_enrichment_results(current.result_dict['Infection_downregulated'],
                                             title='Name', save=False, fig=fig,
                                             ax=ax)
            ax.set_ylabel('Down-Regulated Tissues')
        else:
            ax = tea.plot_enrichment_results(current.result_dict['Infection_upregulated'],
                                             title='Name', save=False, fig=fig,
                                             ax=ax)
            ax.set_ylabel('Up-Regulated Tissues')

        xlabel = ax.set_xlabel('Enrichment Fold Change - {0}'.format(species))
        ax.yaxis.set_label_position('right')
        plt.tight_layout()
        plt.savefig('../output/Engelmann/Graphs/'+species+'Enrichment.pdf',
                    rect=[0, 0.03, 1, 0.95], bbox_extra_artists=[xlabel],
                    bbox_inches='tight')
    i += 1

max_stars_repo_name: Lioscro/TissueEnrichmentAnalysis

max_issues_repo_name: Lioscro/TissueEnrichmentAnalysis

max_forks_repo_name: Lioscro/TissueEnrichmentAnalysis


text:
import numpy as np

from ._layout import Layout
from ._multivector import MultiVector

class ConformalLayout(Layout):
    r"""
    A layout for a conformal algebra, which adds extra constants and helpers.

    Typically these should be constructed via :func:`clifford.conformalize`.

    .. versionadded:: 1.2.0

    Attributes
    ----------
    ep : MultiVector
        The first added basis element, :math:`e_{+}`, usually with :math:`e_{+}^2 = +1`
    en : MultiVector
        The first added basis element, :math:`e_{-}`, usually with :math:`e_{-}^2 = -1`
    eo : MultiVector
        The null basis vector at the origin, :math:`e_o = 0.5(e_{-} - e_{+})`
    einf : MultiVector
        The null vector at infinity, :math:`e_\infty = e_{-} + e_{+}`
    E0 : MultiVector
        The minkowski subspace bivector, :math:`e_\infty \wedge e_o`
    I_base : MultiVector
        The pseudoscalar of the base ga, in cga layout
    """
    def __init__(self, *args, layout=None, **kwargs):
        super().__init__(*args, **kwargs)
        self._base_layout = layout

        ep, en = self.basis_vectors_lst[-2:]

        # setup  null basis, and minkowski subspace bivector
        eo = .5 ^ (en - ep)
        einf = en + ep
        E0 = einf ^ eo
        I_base = self.pseudoScalar*E0

        # helper properties
        self.ep = ep
        self.en = en
        self.eo = eo
        self.einf = einf
        self.E0 = E0
        self.I_base = I_base

    @classmethod
    def _from_base_layout(cls, layout, added_sig=[1, -1], **kwargs) -> 'ConformalLayout':
        """ helper to implement :func:`clifford.conformalize` """
        sig_c = list(layout.sig) + added_sig
        return cls(
            sig_c,
            ids=layout._basis_vector_ids.augmented_with(len(added_sig)),
            layout=layout, **kwargs)

    # some convenience functions
    def up(self, x: MultiVector) -> MultiVector:
        """ up-project a vector from GA to CGA """
        try:
            if x.layout == self._base_layout:
                # vector is in original space, map it into conformal space
                old_val = x.value
                new_val = np.zeros(self.gaDims)
                new_val[:len(old_val)] = old_val
                x = self.MultiVector(value=new_val)
        except(AttributeError):
            # if x is a scalar it doesnt have layout but following
            # will still work
            pass

        # then up-project into a null vector
        return x + (.5 ^ ((x**2)*self.einf)) + self.eo

    def homo(self, x: MultiVector) -> MultiVector:
        """ homogenize a CGA vector """
        return x/(-x | self.einf)[()]

    def down(self, x: MultiVector) -> MultiVector:
        """ down-project a vector from CGA to GA """
        x_down = (self.homo(x) ^ self.E0)*self.E0
        # new_val = x_down.value[:self.base_layout.gaDims]
        # create vector in self.base_layout (not cga)
        # x_down = self.base_layout.MultiVector(value=new_val)
        return x_down

max_stars_repo_name: hugohadfield/clifford

max_issues_repo_name: hugohadfield/clifford

max_forks_repo_name: hugohadfield/clifford


text:
import numpy as np
from hypothesis import strategies, given, example, settings
import wfsim

@settings(max_examples=100, deadline=None)
@given(strategies.integers(min_value=0, max_value=1_000),
       strategies.integers(min_value=0, max_value=4),
       strategies.integers(min_value=0, max_value=1_000))
@example(data_length=101, n_channels=4, noise_data_length=1000)
def test_noise(data_length, n_channels, noise_data_length):
    """Testing wfsim.RawData.add_noise"""
    if data_length <= 0 or noise_data_length <= 0:
        # Double check the input, cannot make np.arrays with negative
        # dimensions
        return
    if n_channels > 4 or n_channels < 0:
        # Double check input
        return

    # Data are random integers. NB: we are at sim rr so pulse is negative
    max_pulse_size = 100  # ADC counts
    data = np.random.randint(-max_pulse_size, 0, size=(n_channels, data_length))
    channel_mask = np.array([(False, 9223372036399775857, -454999850),
                             (False, 9223372036399775857, -454999850),
                             (False, 9223372036399775857, -454999850),
                             (True, n_channels - 1, noise_data_length - n_channels),
                             ],
                            dtype=[('mask', '?'), ('left', '<i8'), ('right', '<i8')])
    # Take a copy of the channel mask
    channel_mask = channel_mask[:n_channels]
    # Noise is a string with random floats
    noise_data = np.random.randint(-10, 10, size=noise_data_length).astype(np.float)

    RawData = wfsim.RawData
    noise_function = RawData.add_noise

    # Actually test that we can run the function
    noise_function(data, channel_mask, noise_data, noise_data_length)

max_stars_repo_name: WenzDaniel/WFSim

max_issues_repo_name: WenzDaniel/WFSim

max_forks_repo_name: WenzDaniel/WFSim


text:
import copy
import os
import pickle
import time
from collections import Counter

import matplotlib.pyplot as plt
import numpy as np
from direct.stdpy import threading

import config
import utils.pcd_utils as pcdu
import utils.run_utils as ru
from localenv import envloader as el

class ForceRecorder(object):
    def __init__(self, motion_planner_x, f_id):
        self.motion_planner_x = motion_planner_x
        self.f_name = "./log/force_armjnts_log_" + f_id + ".pkl"
        self.__rbtx = self.motion_planner_x.rbtx
        self.__thread_recorder = None
        self.__thread_plot = None
        self.__flag = True
        self.__plotflag = False
        self.__ploton = True
        self.__armname = self.motion_planner_x.armname
        self.__force = []
        # record
        self.__info = []
        self.__force = []
        self.__armjnts = []

        # zero force
        self.__rbtx.zerotcpforce(armname="lft")
        self.__rbtx.zerotcpforce(armname="rgt")

    def start_record(self):
        self.__rbtx.zerotcpforce(armname=self.__armname)

        def recorder():
            while self.__flag:
                self.__force.append(self.__rbtx.getinhandtcpforce(armname=self.__armname))
                self.__armjnts = self.motion_planner_x.get_armjnts()
                self.__info.append([self.__force, self.__armjnts, time.time()])
                self.__plotflag = True
                time.sleep(.1)

        def plot():
            fig = plt.figure(1)
            plt.ion()
            plt.show()
            plt.ylim((-10, 10))
            while self.__ploton:
                if self.__plotflag:
                    plt.clf()
                    # print(len(self.__force))
                    force = [l[:3] for l in self.__force]
                    torque = [l[3:] for l in self.__force]
                    # distance = [np.linalg.norm(self.fixgoal - np.array(dis[1])) for dis in self.__tcp]

                    x = [0.02 * i for i in range(len(force))]
                    plt.xlim((0, max(x)))
                    plt.subplot(211)
                    plt.plot(x, force, label=["x", "y", "z"])
                    plt.legend("xyz", loc='upper left')
                    plt.subplot(212)
                    plt.plot(x, torque, label=["Rx", "Ry", "Rz"])
                    plt.pause(0.005)
                time.sleep(.1)
            plt.savefig(f"{time.time()}.png")
            pickle.dump(self.__info, open(self.f_name, "wb"))
            plt.close(fig)

        self.__thread_recorder = threading.Thread(target=recorder, name="recorder")
        self.__thread_recorder.start()
        self.__thread_plot = threading.Thread(target=plot, name="plot")
        self.__thread_plot.start()
        print("start finish")

    def finish_record(self):
        if self.__thread_recorder is None:
            return
        self.__flag = False
        self.__thread_recorder.join()
        self.__thread_recorder = None

        if self.__thread_plot is None:
            return
        self.__ploton = False
        time.sleep(.05)
        self.__plotflag = False
        self.__thread_plot.join()
        self.__thread_plot = None

def load_motion_f(folder_name, f_name, root=config.MOTIONSCRIPT_REL_PATH):
    return pickle.load(open(os.path.join(root, folder_name, f_name), "rb"))

def load_motion_sgl(motion, folder_name, id_list, root=config.MOTIONSCRIPT_REL_PATH):
    motion_dict = pickle.load(open(root + folder_name + motion + ".pkl", "rb"))
    value = []
    for id in id_list:
        try:
            value = motion_dict[id]
            motion_dict = value
        except:
            continue
    objrelpos, objrelrot, path = value
    return objrelpos, objrelrot, path

def get_script_id_dict_multiplepen(folder_name, root=config.MOTIONSCRIPT_REL_PATH):
    id_list = []
    f_dict = load_motion_f(folder_name, "cam2place_pen.pkl", root=root)
    for k, v in f_dict.items():
        print(k, list(v.keys()))

    id = int(input("Select a grasp id from above:"))
    id_list.append(id)

    id = int(input("Select a objmat4 id from above:"))
    id_list.append(id)

    return id_list

def load_motion_seq(motion_seq, folder_name, id_list, armname=None, root=config.MOTIONSCRIPT_REL_PATH):
    path = []
    objrelpos = None
    objrelrot = None
    for motion in motion_seq:
        try:
            if armname is None:
                path_dict_path = root + folder_name + motion + ".pkl"
            else:
                path_dict_path = root + folder_name + "_".join([armname, motion]) + ".pkl"
            path_dict = pickle.load(open(path_dict_path, "rb"))
            for id in id_list:
                try:
                    path_dict = path_dict[id]
                except:
                    continue
            objrelpos, objrelrot, path_temp = path_dict
            # print(motion, path_temp)
            path.extend(path_temp)
        except:
            print(motion, "failed!")
            continue
    return objrelpos, objrelrot, path

def input_script_id(folder_name, armname=None, root=config.MOTIONSCRIPT_REL_PATH):
    def __have_dict(dict_list):
        bool = False
        for item in dict_list:
            if type(item) == dict:
                bool = True
                break
        return bool

    def __get_common_key(dict_list):
        key_list = []
        dict_cnt = 0
        for item in dict_list:
            try:
                key_list.extend(list(item.keys()))
                dict_cnt += 1
            except:
                continue
        return [key for key, cnt in Counter(key_list).items() if cnt == dict_cnt]

    def __dict_list_update(dict_list, id):
        new_dict_list = []
        for path_dict in dict_list:
            try:
                v = path_dict[id]
            except:
                v = path_dict
            new_dict_list.append(v)
        return new_dict_list

    id_list = []
    path_dict_list = []

    for f_name in os.listdir(os.path.join(root, folder_name)):
        if f_name[:4] == "draw" or f_name[:4] == "time":
            continue
        if armname is None:
            path_dict_list.append(load_motion_f(folder_name, f_name, root=root))
        else:
            if f_name[:3] == armname:
                path_dict_list.append(load_motion_f(folder_name, f_name, root=root))

    while __have_dict(dict_list=path_dict_list):
        print(__get_common_key(path_dict_list))
        id = int(input("Select a id from the list above:"))
        id_list.append(id)
        path_dict_list = __dict_list_update(path_dict_list, id)
    return id_list

def setting_sim(stl_f_name, pos=(600, 0, 780)):
    if stl_f_name[-3:] != 'stl':
        stl_f_name += '.stl'
    objcm = el.loadObj(stl_f_name, pos=pos, rot=(0, 0, 0))
    objcm.reparentTo(base.render)

def setting_hand(phxilocator, pcd_f_name="/dataset/pcd/a_lft_0.pkl"):
    amat = phxilocator.amat
    hand_pcd = pickle.load(open(config.ROOT + pcd_f_name, "rb"))
    hand_pcd = pcdu.trans_pcd(pcdu.remove_pcd_zeros(hand_pcd), amat)
    hand = pcdu.reconstruct_surface(hand_pcd)

    pen_pos = (800, 400, 880)
    pen_pcd = el.loadObjpcd('pen.stl', pos=pen_pos, rot=(30, -90, 0), toggledebug=False)
    # base.pg.genpointcloudnp(pen_pcd).reparentTo(base.render)
    pen, pen_w, pen_h, pen_origin_pos, pen_rot_angle = pcdu.get_std_convexhull(pen_pcd, origin="tip", color=(1, 1, 0),
                                                                               toggledebug=False, toggleransac=False)
    hand.reparentTo(base.render)
    pen.reparentTo(base.render)
    base.pggen.plotSphere(base.render, pen_origin_pos, radius=5, rgba=(0, 1, 0, 1))

    return pen

def setting_real_simple(phoxi_f_path, amat):
    grayimg, depthnparray_float32, pcd = ru.load_phxiinfo(phoxi_f_name=phoxi_f_path, load=True)
    pcd = np.array([p for p in pcdu.trans_pcd(pcd, amat) if 790 < p[2] < 1100])
    pcdu.show_pcd(pcd)

def setting_real(phxilocator, phoxi_f_path, pen_stl_f_name, paintingobj_stl_f_name, resolution=1):
    pen_item = ru.get_obj_from_phoxiinfo_withmodel_nobgf(phxilocator, pen_stl_f_name, phoxi_f_name=phoxi_f_path,
                                                         x_range=(600, 1000), y_range=(200, 600), z_range=(810, 1000))
    if paintingobj_stl_f_name is not None:
        paintingobj_item = \
            ru.get_obj_from_phoxiinfo_withmodel_nobgf(phxilocator, paintingobj_stl_f_name, x_range=(400, 1080),
                                                      phoxi_f_name=phoxi_f_path, resolution=resolution)
    else:
        paintingobj_item = ru.get_obj_from_phoxiinfo_nobgf(phxilocator, phoxi_f_name=phoxi_f_path, load=True,
                                                           resolution=resolution)
    paintingobj_item.show_objcm(rgba=(1, 1, 1, .5))
    # paintingobj_item.show_objpcd(rgba=(1, 1, 0, 1))
    pen_item.show_objcm(rgba=(0, 1, 0, 1))

def show_drawmotion_ms(motion_planner, pen_cm, motion_f_path, grasp_id_list, jawwidth=20):
    draw_dict = pickle.load(open(motion_f_path, "rb"))
    objrelpos, objrelrot = None, None

    for grasp_id in grasp_id_list:
        path = []
        if grasp_id not in draw_dict.keys():
            continue
        for stroke_key, v in draw_dict[grasp_id].items():
            print("------", stroke_key, "--------")
            objrelpos, objrelrot, path_stroke = v
            if path != []:
                path_gotodraw = motion_planner.plan_start2end(start=path[-1], end=path_stroke[0])
            else:
                path_gotodraw = []
            path_up = motion_planner.get_moveup_path(path_stroke[-1], pen_cm, objrelpos, objrelrot, length=30)
            path.extend(path_gotodraw + path_stroke + path_up)
        motion_planner.ah.show_animation_hold(path, copy.deepcopy(pen_cm), objrelpos, objrelrot, jawwidth=jawwidth)

def show_drawmotion_ss(motion_planner, pen_cm, motion_f_path, grasp_id_list, jawwidth=20):
    draw_dict = pickle.load(open(motion_f_path, "rb"))

    for grasp_id in grasp_id_list:
        objrelpos, objrelrot, path = draw_dict[grasp_id]
        motion_planner.ah.show_animation_hold(path, copy.deepcopy(pen_cm), objrelpos, objrelrot, jawwidth=jawwidth)

max_stars_repo_name: takuya-ki/wrs

max_issues_repo_name: takuya-ki/wrs

max_forks_repo_name: takuya-ki/wrs


text:
import spotipy.util as util
from progress.bar import Bar
import numpy as np
from bs4 import BeautifulSoup
import pandas as pd
from wordcloud import WordCloud, STOPWORDS
import json, requests, urllib.parse, re, time, sys, click, spotipy, os

def auth():
    username = 'default'
    scope = 'user-read-private user-read-playback-state user-modify-playback-state user-library-read'

    token = util.prompt_for_user_token(username,
                            scope,
                            client_id='', # CHANGE
                            client_secret='', # CHANGE
                            redirect_uri='http://localhost:8000/')

    if token:
        sp = spotipy.Spotify(auth=token)
        # print('Authenticated')
        return sp
    else:
        print("Failed to get token")
        exit()

def get_lyric(artist, name, idx):
    try:
        query = urllib.parse.quote_plus(artist + ' ' + name)
        # print(idx, query)
        url = "https://genius.com/api/search/multi?per_page=5&q=" + query

        response = requests.get(url)

        result = json.loads(response.text)

        try:
            for i in result['response']['sections']:
                if i['type'] == 'song':
                    lyric_url = i['hits'][0]['result']['url']
        except:
            return np.NaN

        lyric_response = requests.get(lyric_url)

        soup = BeautifulSoup(lyric_response.content, 'html.parser')

        lyric_divs = soup.find_all('div', class_='lyrics')

        lyrics = ''
        if lyric_divs:
            for i in lyric_divs:
                lyrics += i.text.replace('\n', ' ').replace('"', "' ")
        else:
            lyrics += soup.find('div', class_='Lyrics__Container-sc-1ynbvzw-2 jgQsqn').getText(separator=" ").replace('"', "' ")

        rules = [r'\[.*?\]', r'\(.*?\)', r'[?,.]']

        for rule in rules:
            lyrics = re.sub(rule, '', lyrics)

        # lyrics = re.sub(r'la.', '', lyrics)
        # lyrics = re.sub(r'[ \t]{2,}', '', lyrics)

        return lyrics
    except:
        return -1

def generate_lyrics(URI):
    sp = auth()
    df = pd.DataFrame(columns=['uri', 'name', 'artist', 'lyrics'])

    data = []

    try:
        results = sp.playlist_items(URI)
        for t in results['items']:
            data.append({'uri': t['track']['uri'], 'name': t['track']['name'], 'artist': t['track']['artists'][0]['name']})

    except spotipy.exceptions.SpotifyException:
        print('The URI does not exist')
        exit()

    while results['next']:
        results = sp.next(results)
        for t in results['items']:
            data.append({'uri': t['track']['uri'], 'name': t['track']['name'], 'artist': t['track']['artists'][0]['name']})

    df = df.append(data, ignore_index=True)

    bar = Bar('Getting Lyrics', max=len(df))

    for idx, row in df.iterrows():
        for attempt in range(10):
            lyrics = get_lyric(row['artist'], row['name'], idx)
            if lyrics != -1:
                row['lyrics'] = lyrics
                break
            else:
                print('sleeping', attempt)
                time.sleep(5)
        bar.next()

    bar.finish()

    print('Could not find lyrics for: {} songs'.format(df['lyrics'].isna().sum()))
    df = df.dropna()

    df.to_csv(URI[-22:] + '_lyrics.csv', index=False)
    print("Saved lyrics to file")

def create_database(URI):
    print("Gettng songs this may take a while...")
    generate_lyrics(URI)

def run_blender(URI):
    if click.confirm("Do you want to run the blender file now?", default=True):
        os.system("blender -b -P WordPile.py -- " + URI)
    return None

def generate_freq(URI):
    df = pd.read_csv(URI[-22:] + '_lyrics.csv')
    all_words = []

    for i in df['lyrics'].values:
        all_words += str(i).lower().split()

    word_freq = WordCloud().process_text(' '.join(all_words))

    with open(URI[-22:] + '.txt', 'w') as f:
        json.dump(word_freq, f)

    print("Saved frequencies to " + URI[-22:] + '.txt')
    run_blender(URI)

def look_for_lyrics(URI):
    try:
        open(URI[-22:] + '_lyrics.csv')
        return -1
    except IOError:
        print("New Playlist...")
        return

def validate_uri(URI_maybe):
    URI = re.findall(r"spotify:playlist:[A-Za-z0-9]{22}", URI_maybe)
    if URI:
        return URI[0]
    else:
        print("Not a valid URI")
        exit()

def main():
    argv = sys.argv
    if len(argv) > 1:
        URI_maybe = argv[1]
        print('\n' + URI_maybe)
        URI = validate_uri(URI_maybe)
        response = look_for_lyrics(URI)
        if response == -1:
            if click.confirm("You have alread created a lyrics file, create another?", default=True):
                create_database(URI)
                generate_freq(URI)
            else:
                generate_freq(URI)
        else:
            print("Creating a new database...")
            create_database(URI)
            generate_freq(URI)
    else:
        print("Usage: python generat_freq.py <URI>")

if __name__ == "__main__":
    main()

# command = 'blender -b --python WordPile.py <URI>'


text:
import numpy as np

def Gaussian_normalizing_z(sigma):
    """ Normalizing coefficient for Gaussian distribution
        sigma is convariance matrix
    """
    d = len(sigma)
    det = np.linalg.det(sigma)
    return np.sqrt((2.0 * np.pi)**d * det)

def Gaussian_log_normalizing_z(sigma):
    """ Log-normalizing coefficient for Gaussian distribution
        sigma is convariance matrix
    """
    d = len(sigma)
    sig, logdet = np.linalg.slogdet(sigma)
    return 0.5 * (np.log(2.0 * np.pi) * d + logdet)

def remove_zero_diagonal(m, etha = 1.0e-8):
    """ Remove every i-th row and column if m[i, i] == 0
    """
    v = np.abs(np.diag(m)) > etha
    return m[v, :][:, v]

def marginal_p_data_Laplace(p_model_MAP, hessian_MAP):
    """ Compute p(D) via laplace approximation
    """
    hessian_MAP = remove_zero_diagonal(hessian_MAP)
    sigma = - p_model_MAP * np.linalg.inv(hessian_MAP)
    return p_model_MAP * Gaussian_normalizing_z(sigma)

def marginal_log_p_data_Laplace(log_p_model_MAP, hessian_log_MAP):
    """ Compute log(p(D)) via laplace approximation
    """
    hessian_log_MAP = remove_zero_diagonal(hessian_log_MAP)
    sigma = - np.linalg.inv(hessian_log_MAP)
    return log_p_model_MAP + Gaussian_log_normalizing_z(sigma)

if __name__ == "__main__":
    m = np.array([[1, 0, 2],
                    [0, 0, 0],
                    [2, 0, 3]])
    m = remove_zero_diagonal(m)
    assert np.allclose(m, [[1, 2], [2, 3]])

    import torch
    torch.set_default_tensor_type(torch.DoubleTensor)

    loc = torch.tensor([0.1, 1.0])
    covar = torch.tensor([[2.1, 1], [1, 5.1]])
    mn = torch.distributions.MultivariateNormal(loc=loc, covariance_matrix=covar)

    scale = 3.4
    p_model_MAP = scale * torch.exp(mn.log_prob(loc))
    hessian_MAP = - p_model_MAP * torch.inverse(covar)
    p_D = marginal_p_data_Laplace(p_model_MAP.data.numpy(), hessian_MAP.data.numpy())
    assert np.allclose(p_D, scale)

    log_p_model_MAP = torch.log(p_model_MAP)
    hessian_log_MAP = - torch.inverse(covar)
    log_p_D = marginal_log_p_data_Laplace(log_p_model_MAP.data.numpy(), hessian_log_MAP.data.numpy())
    p_D = np.exp(log_p_D)
    assert np.allclose(p_D, scale)

max_stars_repo_name: dmytrov/stochasticcontrol

max_issues_repo_name: dmytrov/stochasticcontrol

max_forks_repo_name: dmytrov/stochasticcontrol


text:
from abc import ABC, abstractmethod

from gym.spaces import Box
import numpy as np

from abmarl.sim.gridworld.base import GridWorldBaseComponent
from abmarl.sim.gridworld.agent import GridObservingAgent
import abmarl.sim.gridworld.utils as gu

class ObserverBaseComponent(GridWorldBaseComponent, ABC):
    """
    Abstract Observer Component base from which all observer components will inherit.
    """
    @property
    @abstractmethod
    def key(self):
        """
        The key in the observation dictionary.

        The observation space of all observing agents in the gridworld framework is a dict.
        We can build up complex observation spaces with multiple components by
        assigning each component an entry in the observation dictionary. Observations
        will be a dictionary even if your simulation only has one Observer.
        """
        pass

    @property
    @abstractmethod
    def supported_agent_type(self):
        """
        The type of Agent that this Observer works with.

        If an agent is this type, the Observer will add its entry to the
        agent's observation space and will produce observations for this agent.
        """
        pass

    @abstractmethod
    def get_obs(self, agent, **kwargs):
        """
        Observe the state of the simulation.

        Args:
            agent: The agent for which we return an observation.

        Returns:
            This agent's observation.
        """
        pass

class SingleGridObserver(ObserverBaseComponent):
    """
    Observe a subset of the grid centered on the agent's position.

    The observation is centered around the observing agent's position. Each agent
    in the "observation window" is recorded in the relative cell using its encoding.
    If there are multiple agents on a single cell with different encodings, the
    agent will observe only one of them chosen at random.
    """
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        for agent in self.agents.values():
            if isinstance(agent, self.supported_agent_type):
                agent.observation_space[self.key] = Box(
                    -np.inf, np.inf, (agent.view_range * 2 + 1, agent.view_range * 2 + 1), np.int
                )

    @property
    def key(self):
        """
        This Observer's key is "grid".
        """
        return 'grid'

    @property
    def supported_agent_type(self):
        """
        This Observer works with GridObservingAgents.
        """
        return GridObservingAgent

    def get_obs(self, agent, **kwargs):
        """
        The agent observes a sub-grid centered on its position.

        The observation may include other agents, empty spaces, out of bounds, and
        masked cells, which can be blocked from view by other blocking agents.

        Returns:
            The observation as a dictionary.
        """
        if not isinstance(agent, self.supported_agent_type):
            return {}

        # Generate a local grid and an observation mask
        local_grid, mask = gu.create_grid_and_mask(
            agent, self.grid, agent.view_range, self.agents
        )

        # Convolve the grid observation with the mask.
        obs = np.zeros((2 * agent.view_range + 1, 2 * agent.view_range + 1), dtype=np.int)
        for r in range(2 * agent.view_range + 1):
            for c in range(2 * agent.view_range + 1):
                if mask[r, c]: # We can see this cell
                    candidate_agents = local_grid[r, c]
                    if candidate_agents is None: # This cell is out of bounds
                        obs[r, c] = -1
                    elif not candidate_agents: # In bounds empty cell
                        obs[r, c] = 0
                    else: # Observe one of the agents at this cell
                        obs[r, c] = np.random.choice(
                            [other.encoding for other in candidate_agents.values()]
                        )
                else: # Cell blocked by agent. Indicate invisible with -2
                    obs[r, c] = -2

        return {self.key: obs}

class MultiGridObserver(ObserverBaseComponent):
    """
    Observe a subset of the grid centered on the agent's position.

    The observation is centered around the observing agent's position. The observing
    agent sees a stack of observations, one for each positive encoding, where the
    number of agents of each encoding is given rather than the encoding
    itself. Out of bounds and masked indicators appear in every grid.
    """
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.number_of_encodings = 1
        for agent in self.agents.values():
            self.number_of_encodings = max(self.number_of_encodings, agent.encoding)
        for agent in self.agents.values():
            if isinstance(agent, self.supported_agent_type):
                agent.observation_space[self.key] = Box(
                    -2,
                    len(self.agents),
                    (agent.view_range * 2 + 1, agent.view_range * 2 + 1, self.number_of_encodings),
                    np.int
                )

    @property
    def key(self):
        """
        This Observer's key is "grid".
        """
        return 'grid'

    @property
    def supported_agent_type(self):
        """
        This Observer works with GridObservingAgents.
        """
        return GridObservingAgent

    def get_obs(self, agent, **kwargs):
        """
        The agent observes one or more sub-grids centered on its position.

        The observation may include other agents, empty spaces, out of bounds, and
        masked cells, which can be blocked from view by other blocking agents.
        Each grid records the number of agents on a particular cell correlated
        to a specific encoding.

        Returns:
            The observation as a dictionary.
        """
        if not isinstance(agent, self.supported_agent_type):
            return {}

        # Generate a local grid and an observation mask.
        local_grid, mask = gu.create_grid_and_mask(
            agent, self.grid, agent.view_range, self.agents
        )

        # Convolve the grid observation with the mask.
        obs = np.zeros(
            (2 * agent.view_range + 1, 2 * agent.view_range + 1, self.number_of_encodings),
            dtype=np.int
        )
        for encoding in range(self.number_of_encodings):
            for r in range(2 * agent.view_range + 1):
                for c in range(2 * agent.view_range + 1):
                    if mask[r, c]: # We can see this cell
                        candidate_agents = local_grid[r, c]
                        if candidate_agents is None: # This cell is out of bounds
                            obs[r, c, encoding] = -1
                        elif not candidate_agents: # In bounds empty cell
                            obs[r, c, encoding] = 0
                        else: # Observe the number of agents at this cell with this encoding
                            obs[r, c, encoding] = sum([
                                True if other.encoding == encoding + 1 else False
                                for other in candidate_agents.values()
                            ])
                    else: # Cell blocked by agent. Indicate invisible with -2
                        obs[r, c, encoding] = -2

        return {self.key: obs}

max_stars_repo_name: LLNL/Admiral

max_issues_repo_name: LLNL/Admiral

max_forks_repo_name: LLNL/Abmarl


text:
import cv2
import numpy as np
import os
from PIL import Image
from keras.preprocessing import image
from keras.preprocessing.image import array_to_img

img_path = 'pictures/'
NO = 1

def resize(x):
    x_out = []

    for i in range(len(x)):
        img = cv2.resize(x[i],dsize=(96,96))
        x_out.append(img)

    return np.array(x_out)

x = []

while True:
    if not os.path.exists(img_path + str(NO) + ".jpg"):
        break
    img = Image.open(img_path + str(NO) + ".jpg")
    img = image.img_to_array(img)
    x.append(img)
    NO += 1

x_train = resize(x)


text:
# /mpcutilities/tests/test_kepcart.py

"""

# --------------------------------------------------------------
#
# Nov 2018
#
# Payne
#
# Testing...
#
# --------------------------------------------------------------

"""

# Import third-party packages
# --------------------------------------------------------------
import numpy as np

# Import the specific package/module/function we are testing
# --------------------------------------------------------------
import mpcutilities.helloworld  as HW

# Basic test(s) of the "phys_const" package
# ----------------------------------------------------------
def test_helloworld():
    '''
    Testing ...
    '''
    assert HW.HelloWorld().HW == "Hello World"

max_stars_repo_name: matthewjohnpayne/MPCUtilities

max_issues_repo_name: matthewjohnpayne/MPCUtilities

max_forks_repo_name: matthewjohnpayne/MPCUtilities


text:
from unittest.mock import Mock
import numpy as np

import pytest
import traitlets
import traittypes

from crowddynamics.traits import shape_validator, length_validator, \
    trait_to_dtype

def test_shape_validator():
    trait = None
    value = Mock()

    correct = (1, 1)
    validator = shape_validator(*correct)
    value.shape = correct
    assert validator(trait, value).shape == value.shape

    incorrect = (1, 2)
    value.shape = incorrect
    pytest.raises(traitlets.TraitError, validator, trait, value)

def test_length_validator():
    trait = None
    value = np.array((1.0, 1.0))

    lengths = np.sqrt(2), 1.0
    validator = length_validator(*lengths)
    assert np.allclose(validator(trait, value), value)

    value = np.array((0.0, 0.0))
    pytest.raises(traitlets.TraitError, validator, trait, value)

class Traits(traitlets.HasTraits):
    int = traitlets.Int()
    float = traitlets.Float()
    complex = traitlets.Complex()
    bool = traitlets.Bool()
    array = traittypes.Array(default_value=(0, 0), dtype=np.float64)

def test_trait_to_primitive_dtype():
    for name, trait in Traits.class_traits().items():
        dtypespec = trait_to_dtype(name, trait)
        assert True


text:
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.

# This program is free software; you can redistribute it and/or modify
# it under the terms of the Apache License Version 2.0.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License Version 2.0 for more details.
# ============================================================================

import os.path as osp
import numpy as np
import scipy as sp
import scipy.io
import networkx as nx

def standardize():
    filepath = osp.dirname(osp.abspath(__file__))

    data = sp.io.loadmat(f"{filepath}/Homo_sapiens.mat")
    network, group = data["network"], data["group"]

    network.setdiag(0.0)
    network.eliminate_zeros()
    degree = np.asarray(network.sum(1)).flatten()

    network = network[degree > 0][:, degree > 0]
    group = group[degree > 0]

    G = nx.Graph()
    # add nodes
    for i in range(network.shape[0]):
        G.add_node(i)
    # add edges
    for i, j in zip(*network.nonzero()):
        if i != j:
            G.add_edge(i, j)
            G.add_edge(j, i)
    # add groups
    for i, row in enumerate(group):
        label = row.toarray().flatten().astype(np.int32).tolist()
        G.nodes[i]["label"] = label

    label_array = np.asarray(
        [G.nodes[i]["label"] for i in range(G.number_of_nodes())],
        dtype=np.int64)
    G.graph["label_array"] = label_array

    nx.write_gpickle(G, f"{filepath}/PPI_nx.pkl")
    nx.write_gpickle(G, f"{filepath}/PPI_nx.pkl2", protocol=2)

    return G

max_stars_repo_name: llan-ml/MetaTNE

max_issues_repo_name: llan-ml/MetaTNE

max_forks_repo_name: llan-ml/MetaTNE

reason: import numpy,import scipy,import networkx


text:
import copy
import time
import pickle
import logging
from collections import deque, defaultdict
from heapq import heapify, heappush, heappop
from tqdm import tqdm

from tqdm import tqdm
import numpy as np
import scipy.sparse as sp
from scipy.sparse import csr_matrix

from sparse_dot_mkl import dot_product_mkl
from ortools.sat.python import cp_model

from IPython import embed

logger = logging.getLogger(__name__)

def greedy_assign(pred_tree_nodes, constraints, viable_placements):
    logger.debug('Reformating viable placements')
    viable_placements = [[(s, n.uid) for s, n in placements]
                                for placements in viable_placements]

    # compute the tree incompatibility matrix
    logger.debug('Computing tree incompatibility matrix')
    num_tree_nodes = len(pred_tree_nodes)
    tree_incompat_mx = np.zeros((num_tree_nodes, num_tree_nodes), dtype=float)
    for node in pred_tree_nodes:
        if node.parent is None:
            tree_incompat_mx[node.uid][node.uid] = 1.0
            assert node == pred_tree_nodes[-1]
            break
        a, b = node.uid, node.parent.uid
        tree_incompat_mx[a][a] = 1.0
        tree_incompat_mx[a][b] = 1.0

    # compute the transitive closure of the initial matrix
    prev_tree_incompat_mx = np.zeros_like(tree_incompat_mx)
    curr_tree_incompat_mx = tree_incompat_mx
    while not np.array_equal(prev_tree_incompat_mx, curr_tree_incompat_mx):
        prev_tree_incompat_mx = copy.deepcopy(curr_tree_incompat_mx)
        curr_tree_incompat_mx = (
                (curr_tree_incompat_mx @ curr_tree_incompat_mx)
                + prev_tree_incompat_mx
        ).astype(bool).astype(float)

    curr_tree_incompat_mx = curr_tree_incompat_mx + curr_tree_incompat_mx.T
    tree_incompat_mx = curr_tree_incompat_mx.astype(bool)
    tree_incompat_mx = csr_matrix(tree_incompat_mx)

    # compute constraint incompatibility matrix
    logger.debug('Computing constraint incompatibility matrix')
    Xi = sp.vstack(constraints)
    extreme_constraints = copy.deepcopy(Xi)
    extreme_constraints.data *= np.inf
    constraint_incompat_mx = dot_product_mkl(
        extreme_constraints, extreme_constraints.T, dense=True
    )
    constraint_incompat_mx = ((constraint_incompat_mx == -np.inf)
                                | np.isnan(constraint_incompat_mx))
    constraint_incompat_mx = csr_matrix(constraint_incompat_mx)

    logger.debug('Begin greedy assignment of constraints to tree nodes')

    # Setup:
    # - queue for every constraint (based on sorted list of viable placements)
    # - heap of proposed assignments for unassigned constraints
    # - list(?) of chosen constraints and assignments

    max_assign_nuid = num_tree_nodes
    num_constraints = len(viable_placements)

    while True: # this is the "push-down" loop

        # running set of viable placements
        running_vp = [deque(l) for l in viable_placements]

        # note this is a min-heap so we negate the score
        to_pick_heap = [(i, *d.popleft()) for i, d in enumerate(running_vp)]
        to_pick_heap = [(-s, (c, t)) for c, s, t in to_pick_heap]
        heapify(to_pick_heap)

        picked_cidxs = np.array([], dtype=int)
        picked_tidxs = np.array([], dtype=int)

        while len(picked_cidxs) < num_constraints:
            _, (c, t) = heappop(to_pick_heap)

            if t > max_assign_nuid: # "push-down" condition
                # if incompatible do the following
                try:
                    _s, _t = running_vp[c].popleft()
                except:
                    max_assign_nuid = np.max(picked_tidxs) - 1
                    break
                c_next_best = (-_s, (c, _t))
                heappush(to_pick_heap, c_next_best)
                continue

            if len(picked_cidxs) == 0:
                picked_cidxs = np.append(picked_cidxs, int(c))
                picked_tidxs = np.append(picked_tidxs, int(t))
                continue

            c_incompat_mask = constraint_incompat_mx[
                c, picked_cidxs
            ].toarray().reshape(-1,)

            t_incompat_mask = tree_incompat_mx[
                t, picked_tidxs[c_incompat_mask]
            ].toarray().reshape(-1,)

            if np.sum(t_incompat_mask) > 0:
                # if incompatible do the following
                try:
                    _s, _t = running_vp[c].popleft()
                except:
                    max_assign_nuid = np.max(picked_tidxs) - 1
                    break
                c_next_best = (-_s, (c, _t))
                heappush(to_pick_heap, c_next_best)
            else:
                picked_cidxs = np.append(picked_cidxs, int(c))
                picked_tidxs = np.append(picked_tidxs, int(t))

        assert len(picked_cidxs) <= num_constraints
        if len(picked_cidxs) == num_constraints:
            break

    placements_out = defaultdict(set)
    for cuid, tuid in zip(picked_cidxs, picked_tidxs):
        placements_out[tuid].add(cuid)

    return placements_out

max_stars_repo_name: rangell/icff

max_issues_repo_name: rangell/icff

max_forks_repo_name: rangell/icff

reason: import numpy,import scipy,from scipy