text:
import random
from dataclasses import dataclass
import pandas as pd
import numpy as np
import tensorflow as tf

@dataclass
class RamanDataGenerator(tf.keras.utils.Sequence):
    df: pd.DataFrame
    batch_size: int
    max_classes: int

    weighted_sum: bool = True
    roll: bool = True
    roll_factor: int = 12

    slope: bool = True
    slope_factor: float = 0.2

    noise: bool = True
    noise_range: tuple = (80, 100)

    sparse_labels: bool = False

    def __post_init__(self):
        # transform to numpy for performance reasons
        self.samples = self.df.iloc[:, 10:-1].to_numpy().astype("float32")
        self.labels = self.df.loc[:, "labels"].to_numpy().astype("uint32")

    def __len__(self):
        return int(len(self.df) // self.batch_size)

    def __getitem__(self, index):
        # selection of mini-batch
        BOTTOM = index * self.batch_size
        TOP = (index + 1) * self.batch_size
        batch_samples = self.samples[BOTTOM:TOP]
        batch_labels = self.labels[BOTTOM:TOP].reshape((self.batch_size, 1))

        batch_samples = self._augmentation(batch_samples, batch_labels)
        batch_labels = batch_labels.reshape((self.batch_size,))

        # in case of categorical crossentropy loss, labels are translated
        # form sparse to categorical
        if not self.sparse_labels:
            batch_labels = tf.keras.utils.to_categorical(
                batch_labels, num_classes=self.max_classes
            )

        return (
            batch_samples,
            batch_labels,
        )

    def _augmentation(self, batch_samples, batch_labels):
        """Compute data augmentation on 'batch_samples', applying
        weighted sum + roll(horizontal shift) + baseline noise + adittive white gaussian noise

        Args:
            batch_samples (np.array): Batch of spectra (1d array)
            batch_labels (np.array): Batch of label, number that are class identifier

        Returns:
            np.array: Batch of augmented data
        """

        if self.weighted_sum:
            alpha = np.random.rand(self.batch_size)

            other_samples = np.apply_along_axis(
                self._get_random_sample_from_class, 1, batch_labels
            ).reshape(self.batch_size, batch_samples.shape[1])

            if self.roll:
                other_samples = np.apply_along_axis(self._random_roll, 1, other_samples)

            batch_samples = (
                np.multiply(
                    batch_samples - other_samples, alpha.reshape(self.batch_size, 1)
                )
                + other_samples
            )

        if self.slope:
            batch_samples = np.apply_along_axis(
                lambda x: self._produce_background_baseline(x, batch_samples.shape[1]),
                1,
                batch_samples,
            )

        if self.noise:
            batch_samples = np.apply_along_axis(self._random_noise, 1, batch_samples)

        return batch_samples

    def _get_random_sample_from_class(self, label):
        """Extract a random sample from the datas marked as 'label'

        Args:
            label (int): Number that describe the class identifier of data to select

        Returns:
            np.array: Random sample of the 'label' class
        """
        class_indexes = np.where(self.labels == label)[0]
        CLASS_INDEX = np.random.choice(class_indexes, 1)[0]

        return self.samples[CLASS_INDEX : CLASS_INDEX + 1]

    def _random_noise(self, arr):
        """Apply adittive white gaussian noise to 'arr' of magnitued 'noise_range'

        Args:
            arr (np.array): Sample to wich apply the noise

        Returns:
            np.array: Noise 'arr'
        """
        rnd_snr = random.randint(self.noise_range[0], self.noise_range[1])
        NOISE_FACTOR = 1 / (10 ** (rnd_snr / 10))

        return arr + np.random.normal(0, NOISE_FACTOR, len(arr))

    def _random_roll(self, arr):
        """Apply random roll (numpy way to say horizontal shift) to 'arr' of magnitude 'roll_factor'

        Args:
            arr (np.array): Sample to wich apply the roll

        Returns:
            np.array: Random rolled sample
        """

        SHIFT_FACTOR = self.roll_factor
        random_shift = random.randint(-1 * SHIFT_FACTOR, SHIFT_FACTOR)

        rolled = np.roll(arr, random_shift)
        padded = (
            np.pad(rolled[random_shift:], (random_shift, 0), "edge")
            if random_shift >= 0
            else np.pad(rolled[:random_shift], (0, abs(random_shift)), "edge")
        )

        return padded

    def _produce_background_baseline(self, arr, steps):
        """Apply a random baseline noise to 'arr' of magnitude 'slope_factor'

        Args:
            arr (np.array): Spectrum to wich apply the baseline noise
            steps (int): Length of the 'arr' argument

        Returns:
            np.array: Noised spectrum
        """

        SLOPE = random.triangular(-1 * self.slope_factor, self.slope_factor)
        line = (
            np.linspace(abs(SLOPE), 0, steps)
            if SLOPE < 0
            else np.linspace(0, SLOPE, steps)
        )

        alpha = random.random()

        return arr * alpha + line * (1 - alpha)

max_stars_repo_name: fmolivato/RamanDataGenerator

max_issues_repo_name: fmolivato/RamanDataGenerator

max_forks_repo_name: fmolivato/RamanDataGenerator


text:
import matplotlib.pyplot as plt
import numpy as np

k = np.linspace(-3.14, 3.14, num=200)
m=0.1

fig, ax = plt.subplots()

sink = np.sin(k/2)
k0_lattice = 2*np.arcsinh(np.sqrt(sink**2+m**2/4))
k0_continuum = np.sqrt(k**2+m**2)
plt.plot(k, k0_lattice, label='lattice')
plt.plot(k, k0_continuum, label='lattice')

plt.xlabel(r'$|\mathbf{k}|$')
plt.ylabel(r'$k_0$')
ax.legend()
plt.show()

ext: py

max_stars_repo_name: rantahar/Lattice-Field-Theory

max_issues_repo_name: rantahar/Lattice-Field-Theory

max_forks_repo_name: rantahar/Lattice-Field-Theory

reason: import numpy


text:
from GrapHiC.Dataset import HiC_Dataset
import numpy as np

CHROMS = [f'chr{idx+1}' for idx in np.arange(19)]+['chrX']

def mytransform(vec,perc=95):
    absvec = abs(vec)
    vperc = np.percentile(absvec[absvec>0],
                          perc)
    out = np.divide(vec,vperc)
    out[out>1] = 1

    return out

if __name__=="__main__":

    import argparse
    parser = argparse.ArgumentParser(description='Construct a datset of per-gene graph objects based off contact info, some target (e.g. rnaseq) and protein binding data', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("-c","--cooler_files",
                        nargs = "+",
                        help="Cooler file(s) from a Hi-C experiment(s)",
                        default = 'KR_downsampled_WT_merged_10000.cool',
                        type=str)
    parser.add_argument("-t","--target",
                        help="path tsv with columns ['chrom','promoter_start','promoter_end','target','name']",
                        default = 'rnaseq.tsv',
                        type=str)
    parser.add_argument("-tr","--tracks",
                        help="track paths",
                        nargs = "+",
                        default = ['Mbd3_ESC.bw'],
                        type=str)
    parser.add_argument("-n","--names",
                        help="names of each datatrack",
                        nargs = "+",
                        default = None,
                        type=str)
    parser.add_argument("-s","--statistics_types",
                        help="Statistics of each track to collect over each specified region and cooler bin",
                        nargs = "+",
                        default = 'max',
                        type=str)
    parser.add_argument("-tf","--transform",
                        help="Transform to apply to the data. Can either provide a custom transform as in this script or specify 'robust', 'standard' or 'power' to use a scikit-learn defined data transform. Not specifying will result in no transform.",
                        default = None)
    parser.add_argument("-r","--root",
                        help="root folder containing both raw_dir and processed_dir",
                        type = str,
                        default = 'Data/')
    parser.add_argument("-bf", "--buffer",
                        help="Buffer to add each side of a gene to create a graph object",
                        type = int,
                        default = 1e6)
    parser.add_argument("-ch","--chunk_size",
                        help="How many genes to try and process at once",
                        default = 500,
                        type = int
                       )
    parser.add_argument("-chr","--chromosomes",
                        nargs = "+",
                        help="Chromsomes to retrieve",
                        default = CHROMS,
                        type = str)
    args = parser.parse_args()

    if isinstance(args.cooler_files,str):
        args.cooler_files = [args.cooler_files]

    if isinstance(args.tracks,str):
        args.tracks = [args.tracks]

    dset = HiC_Dataset(args.root,
                       contacts=args.cooler_files,
                       tracks=args.tracks,
                       names=args.names,
                       target=args.target,
                       track_transform=mytransform,
                       pre_transform=None,
                       buffer = args.buffer,
                       chunk_size = args.chunk_size,
                       track_statistic_types=args.statistics_types,
                       chromosomes = args.chromosomes
                      )


text:
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===========================================================================
"""Get dataset"""
import os

import mindspore.dataset as ds
import numpy as np

def get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=1000,
                           rank_size=None, rank_id=None, line_per_sample=1000):
    """Get Mindrecord dataset"""
    file_prefix_name = 'train_input_part.mindrecord' if train_mode else 'test_input_part.mindrecord'
    file_suffix_name = '00' if train_mode else '0'
    shuffle = train_mode
    if rank_size is not None and rank_id is not None:
        data_set = ds.MindDataset(os.path.join(directory, file_prefix_name + file_suffix_name),
                                  columns_list=['cats_vals', 'num_vals', 'label'],
                                  num_shards=rank_size, shard_id=rank_id, shuffle=shuffle,
                                  num_parallel_workers=8)
    else:
        data_set = ds.MindDataset(os.path.join(directory, file_prefix_name + file_suffix_name),
                                  columns_list=['cats_vals', 'num_vals', 'label'],
                                  shuffle=shuffle, num_parallel_workers=8)
    data_set = data_set.batch(int(batch_size / line_per_sample), drop_remainder=True)
    data_set = data_set.map(operations=(lambda x, y, z: (np.array(x).flatten().reshape(batch_size, 26),
                                                         np.array(y).flatten().reshape(batch_size, 13),
                                                         np.array(z).flatten().reshape(batch_size, 1))),
                            input_columns=['cats_vals', 'num_vals', 'label'],
                            column_order=['cats_vals', 'num_vals', 'label'],
                            num_parallel_workers=8)
    data_set = data_set.repeat(epochs)
    return data_set

max_stars_repo_name: leelige/mindspore

max_issues_repo_name: leelige/mindspore

max_forks_repo_name: leelige/mindspore


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 IBM Corporation and others
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__author__ = "Deepta Rajan"

import os
import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader
from sklearn.preprocessing import OneHotEncoder

class Heartbeat_Dataset(Dataset):
    def __init__(self, data_dir, mode='train'):
        self.data_dir = data_dir
        self.mode = mode
        self.X = np.loadtxt(os.path.join(self.data_dir, 'mitbih_'+self.mode+'.csv'), delimiter=',')
        self.labels = np.asarray([int(self.X[i][-1]) for i in range(self.X.shape[0])])
        self.X = np.expand_dims(self.X[:,0:-1], 1) # last column is labels
        enc = OneHotEncoder(sparse=False, categories='auto')
        self.labels_onehot = enc.fit_transform(np.expand_dims(self.labels,1))
        print(self.mode + " dataset: " + "X.shape: ", self.X.shape, " Y.shape", self.labels_onehot.shape)

    def __getitem__(self, index):
        ecg = self.X[index]
        label = self.labels_onehot[index]
        label = torch.from_numpy(label)
        ecg = torch.from_numpy(ecg)

        return ecg, label

    def __len__(self):
        return len(self.labels)

def get_loader(data_dir, mode, batch_size, shuffle):
    heartbeat = Heartbeat_Dataset(data_dir, mode)
    data_loader = DataLoader(dataset=heartbeat, batch_size=batch_size, shuffle=shuffle)

    return data_loader

max_stars_repo_name: Drajan/DDxNet

max_issues_repo_name: Drajan/DDxNet

max_forks_repo_name: Drajan/DDxNet


text:
# This file is part of h5py, a Python interface to the HDF5 library.
#
# http://www.h5py.org
#
# Copyright 2008-2013 Andrew Collette and contributors
#
# License:  Standard 3-clause BSD; see "license.txt" for full license terms
#           and contributor agreement.

"""
    Tests for the (internal) selections module
"""

import numpy as np
import h5py
import h5py._hl.selections as sel
import h5py._hl.selections2 as sel2

from .common import TestCase, ut

class BaseSelection(TestCase):
    def setUp(self):
        self.f = h5py.File(self.mktemp(), 'w')
        self.dsid = self.f.create_dataset('x', ()).id

    def tearDown(self):
        if self.f:
            self.f.close()

class TestTypeGeneration(BaseSelection):

    """
        Internal feature: Determine output types from dataset dtype and fields.
    """

    def test_simple(self):
        """ Non-compound types are handled appropriately """
        dt = np.dtype('i')
        out, format = sel2.read_dtypes(dt, ())
        self.assertEqual(out, format)
        self.assertEqual(out, np.dtype('i'))

    def test_simple_fieldexc(self):
        """ Field names for non-field types raises ValueError """
        dt = np.dtype('i')
        with self.assertRaises(ValueError):
            out, format = sel2.read_dtypes(dt, ('a',))

    def test_compound_simple(self):
        """ Compound types with elemental subtypes """
        dt = np.dtype( [('a','i'), ('b','f'), ('c','|S10')] )

        # Implicit selection of all fields -> all fields
        out, format = sel2.read_dtypes(dt, ())
        self.assertEqual(out, format)
        self.assertEqual(out, dt)

        # Explicit selection of fields -> requested fields
        out, format = sel2.read_dtypes(dt, ('a','b'))
        self.assertEqual(out, format)
        self.assertEqual(out, np.dtype( [('a','i'), ('b','f')] ))

        # Explicit selection of exactly one field -> no fields
        out, format = sel2.read_dtypes(dt, ('a',))
        self.assertEqual(out, np.dtype('i'))
        self.assertEqual(format, np.dtype( [('a','i')] ))

        # Field does not apear in named typed
        with self.assertRaises(ValueError):
            out, format = sel2.read_dtypes(dt, ('j', 'k'))

class TestScalarSliceRules(BaseSelection):

    """
        Internal feature: selections rules for scalar datasets
    """

    def test_args(self):
        """ Permissible arguments for scalar slicing """
        shape, selection = sel2.read_selections_scalar(self.dsid, ())
        self.assertEqual(shape, None)
        self.assertEqual(selection.get_select_npoints(), 1)

        shape, selection = sel2.read_selections_scalar(self.dsid, (Ellipsis,))
        self.assertEqual(shape, ())
        self.assertEqual(selection.get_select_npoints(), 1)

        with self.assertRaises(ValueError):
            shape, selection = sel2.read_selections_scalar(self.dsid, (1,))

        dsid = self.f.create_dataset('y', (1,)).id
        with self.assertRaises(RuntimeError):
            shape, selection = sel2.read_selections_scalar(dsid, (1,))

class TestSelection(BaseSelection):

    """ High-level routes to generate a selection
    """

    def test_selection(self):
        dset = self.f.create_dataset('dset', (100,100))
        regref = dset.regionref[0:100, 0:100]

        # args is list, return a FancySelection
        st = sel.select((10,), list([1,2,3]), dset)
        self.assertIsInstance(st, sel.FancySelection)

        # args is a Boolean mask, return a PointSelection
        st1 = sel.select((5,), np.array([True,False,False,False,True]), dset)
        self.assertIsInstance(st1, sel.PointSelection)

        # args is int, return a SimpleSelection
        st2 = sel.select((10,), 1, dset)
        self.assertIsInstance(st2, sel.SimpleSelection)

        # args is RegionReference, return a Selection instance
        st3 = sel.select((100,100), regref, dset)
        self.assertIsInstance(st3, sel.Selection)

        # args is RegionReference, but dataset is None
        with self.assertRaises(TypeError):
            sel.select((100,), regref, None)

        # args is RegionReference, but its shape doesn't match dataset shape
        with self.assertRaises(TypeError):
            sel.select((100,), regref, dset)

        # args is a single Selection instance, return the arg
        st4 = sel.select((100,100), st3, dset)
        self.assertEqual(st4,st3)

        # args is a single Selection instance, but args shape doesn't match Shape
        with self.assertRaises(TypeError):
            sel.select((100,), st3, dset)


text:
import tensorflow as tf
import tensorflow.contrib.slim as slim

import neural_toolbox.ft_utils as ft_utils

def film_layer(ft, context, reuse=False):
    """
    A very basic FiLM layer with a linear transformation from context to FiLM parameters
    :param ft: features map to modulate. Must be a 3-D input vector (+batch size)
    :param context: conditioned FiLM parameters. Must be a 1-D input vector (+batch size)
    :param reuse: reuse variable, e.g, multi-gpu
    :return: modulated features
    """

    height = int(ft.get_shape()[1])
    width = int(ft.get_shape()[2])
    feature_size = int(ft.get_shape()[3])

    film_params = slim.fully_connected(context,
                                       num_outputs=2 * feature_size,
                                       activation_fn=None,
                                       reuse=reuse,
                                       scope="film_projection")

    film_params = tf.expand_dims(film_params, axis=[1])
    film_params = tf.expand_dims(film_params, axis=[1])
    film_params = tf.tile(film_params, [1, height, width, 1])

    gammas = film_params[:, :, :, :feature_size]
    betas = film_params[:, :, :, feature_size:]

    output = (1 + gammas) * ft + betas

    return output

class FiLMResblock(object):
    def __init__(self, features, context, is_training,
                 feature_size,
                 film_layer_fct=film_layer,
                 kernel1=list([1, 1]),
                 kernel2=list([3, 3]),
                 spatial_location=True, reuse=None):

        # Append a mask with spatial location to the feature map
        if spatial_location:
            features = ft_utils.append_spatial_location(features)

        # First convolution
        self.conv1_out = slim.conv2d(features,
                                     num_outputs=feature_size,
                                     kernel_size=kernel1,
                                     activation_fn=tf.nn.relu,
                                     scope='conv1',
                                     reuse=reuse)

        # Second convolution
        self.conv2 = slim.conv2d(self.conv1_out,
                                 num_outputs=feature_size,
                                 kernel_size=kernel2,
                                 activation_fn=None,
                                 scope='conv2',
                                 reuse=reuse)

        # Center/reduce output (Batch Normalization with no training parameters)
        self.conv2_bn = slim.batch_norm(self.conv2,
                                        center=False,
                                        scale=False,
                                        decay=0.9,
                                        is_training=is_training,
                                        reuse=reuse)

        # Apply FILM layer Residual connection
        with tf.variable_scope("FiLM_layer", reuse=reuse):
            self.conv2_film = film_layer_fct(self.conv2_bn, context, reuse=reuse)

        # Apply ReLU
        self.conv2_out = tf.nn.relu(self.conv2_film)

        # Residual connection
        self.output = self.conv2_out + self.conv1_out

    def get(self):
        return self.output

if __name__ == '__main__':
    import numpy as np

    feature_maps = tf.placeholder(tf.float32, shape=[None, 3, 3, 2])
    lstm_state = tf.placeholder(tf.float32, shape=[None, 6])

    modulated_feat1 = film_layer(ft=feature_maps, context=lstm_state)
    modulated_feat2 = FiLMResblock(features=feature_maps, context=lstm_state, is_training=True).get()

    sess = tf.InteractiveSession()
    sess.run(tf.initialize_variables(tf.trainable_variables()))

    feature_maps_np = np.array(
    [
        [
          [ [1,2,3  ], [1, 2, 3] , [0, 0, 0]],
          [ [1, 2,3 ], [1, 2, 3] , [1, 1, 1] ]
        ],
        [
            [[-1, 1, 0], [0, 0, 0], [-1, 1, 0]],
            [[-1, 1, 0], [-1, 1, 1], [-4, 1, 4]]
        ]
    ]    )
    feature_maps_np = np.transpose(feature_maps_np, axes=[0,2,3,1])

    feature_maps_cst = tf.constant(feature_maps_np, dtype=tf.float32)
    lstm_state_cst = tf.constant( np.array([[1,0,1,0,1, 0], [0, 1, 1, 0, 1, 0 ]]), dtype=tf.float32)

    modulated_feat_cst = film_layer(feature_maps_cst, lstm_state_cst, reuse=True)
    print(modulated_feat_cst)


text:
from __future__ import print_function, division
import numpy as np
from ahoy import directions, c_measurers, dc_dx_measurers, noise_measurers
import test

class TestChemoNoiseMeasurer1D(test.TestBase):
    dim = 1
    L = np.array([0.7])
    noise_measurer_cls = noise_measurers.ChemoNoiseMeasurer

    def setUp(self):
        super(TestChemoNoiseMeasurer1D, self).setUp()
        self.noise_0 = 1.5
        self.v_0 = 2.2

    def run_valchemo(self, ds, chi, noise_expected):
        grad_c_measurer = c_measurers.ConstantGradCMeasurer(self.n, self.dim)
        dc_dx_measurer = dc_dx_measurers.SpatialDcDxMeasurer(ds,
                                                             grad_c_measurer)
        noise_measurer = self.noise_measurer_cls(self.noise_0, chi,
                                                 dc_dx_measurer)
        noise = noise_measurer.get_noise()
        self.assertTrue(np.allclose(noise, noise_expected))

    def get_ds_parallel(self):
        u_0 = np.zeros([self.n, self.dim])
        u_0[:, 0] = 1.0
        return directions.directions_nd(u_0)

    def get_ds_antiparallel(self):
        u_0 = np.zeros([self.n, self.dim])
        u_0[:, 0] = -1.0
        return directions.directions_nd(u_0)

    def test_parallel_nochemo(self):
        ds = self.get_ds_parallel()
        self.run_valchemo(ds, 0.0, self.noise_0)

    def test_antiparallel_nochemo(self):
        ds = self.get_ds_antiparallel()
        self.run_valchemo(ds, 0.0, self.noise_0)

    def test_parallel_maxchemo(self):
        ds = self.get_ds_parallel()
        self.run_valchemo(ds, 1.0, 0.0)

    def test_antiparallel_maxchemo(self):
        ds = self.get_ds_antiparallel()
        self.run_valchemo(ds, 1.0, 2.0 * self.noise_0)

    def test_parallel_halfchemo(self):
        ds = self.get_ds_parallel()
        self.run_valchemo(ds, 0.5, self.noise_0 / 2.0)

    def test_antiparallel_halfchemo(self):
        ds = self.get_ds_antiparallel()
        self.run_valchemo(ds, 0.5, 1.5 * self.noise_0)

class TestChemoNoiseMeasurer2D(TestChemoNoiseMeasurer1D):
    dim = 2
    L = np.array([0.7, 1.1])

    def get_ds_perp(self):
        u_0 = np.zeros([self.n, self.dim])
        u_0[:, 1] = 1.0
        return directions.directions_nd(u_0)

    def test_perp_nochemo(self):
        ds = self.get_ds_perp()
        self.run_valchemo(ds, 0.0, self.noise_0)

    def test_perp_maxchemo(self):
        ds = self.get_ds_perp()
        self.run_valchemo(ds, 1.0, self.noise_0)

    def test_perp_halfchemo(self):
        ds = self.get_ds_perp()
        self.run_valchemo(ds, 0.5, self.noise_0)

class TestOneSidedChemoNoiseMeasurer1D(TestChemoNoiseMeasurer1D):
    noise_measurer_cls = noise_measurers.OneSidedChemoNoiseMeasurer

    def test_antiparallel_maxchemo(self):
        ds = self.get_ds_antiparallel()
        self.run_valchemo(ds, 1.0, self.noise_0)

    def test_antiparallel_halfchemo(self):
        ds = self.get_ds_antiparallel()
        self.run_valchemo(ds, 0.5, self.noise_0)

class TestOneSidedChemoRudderControllers2D(TestChemoNoiseMeasurer2D):
    noise_measurer_cls = noise_measurers.OneSidedChemoNoiseMeasurer

    def test_antiparallel_maxchemo(self):
        ds = self.get_ds_antiparallel()
        self.run_valchemo(ds, 1.0, self.noise_0)

    def test_antiparallel_halfchemo(self):
        ds = self.get_ds_antiparallel()
        self.run_valchemo(ds, 0.5, self.noise_0)

max_stars_repo_name: eddiejessup/ships

max_issues_repo_name: eddiejessup/ships

max_forks_repo_name: eddiejessup/ships


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

# ----------------------------------------------
# Predict age
# ----------------------------------------------

import cv2
import sys
import numpy as np
import os
import caffe
import glob
import codecs

# ----------------------------------------------
# MODE
# ----------------------------------------------

DATASET_ROOT_PATH="/Volumes/TB4/Keras/"
DATASET_SUB_PATH="dataset/vggface2/"

# ----------------------------------------------
# Argument
# ----------------------------------------------

#DATASET_ROOT_PATH=""

if len(sys.argv) >= 1:
  if len(sys.argv) >= 2:
    DATASET_ROOT_PATH=sys.argv[1]
  if len(sys.argv) >= 3:
    if sys.argv[2]=="gpu":
      caffe.set_mode_gpu()
    else:
      if(sys.argv[2]=="cpu"):
        None
      else:
        print("argv[2] must be gpu or cpu")
else:
  print("usage: python estimate_age.py [datasetroot(optional)] [gpu/cpu(optional)]")
  sys.exit(1)

# ----------------------------------------------
# Load Model
# ----------------------------------------------

net  = caffe.Net('pretrain/age.prototxt', 'pretrain/dex_imdb_wiki.caffemodel', caffe.TEST)

def calc_age(image):
	img = cv2.imread(image)

	image_size=224

	img = cv2.resize(img, (image_size, image_size))
	img = img[...,::-1]  #BGR 2 RGB

	data = np.array(img, dtype=np.float32)
	data.shape = (1,) + data.shape

	data -= 128

	data = data.transpose((0, 3, 1, 2))
	out = net.forward_all(data = data)
	pred = out[net.outputs[0]]

	age=0
	for i in range(0,101):
		age=age+pred[0][i]*i

	return age

# ----------------------------------------------
# FileList
# ----------------------------------------------

lines=codecs.open(DATASET_ROOT_PATH+DATASET_SUB_PATH+"identity_meta.csv", 'r', 'utf-8').readlines()

OUTPUT_PATH=DATASET_ROOT_PATH+DATASET_SUB_PATH+"identity_meta_with_estimated_age.csv"
if(os.path.exists(OUTPUT_PATH)):
	output_data=open(OUTPUT_PATH).readlines()	#for continue
else:
	output_data=[]

cache={}
for line in output_data:
	obj=line.split(", ")
	path=obj[0]
	age=obj[5].strip()
	cache[path]=age

DEBUG_MODE=0	#If 1, Age estimate from one image

with codecs.open(OUTPUT_PATH, 'w', 'utf-8') as f:
	cnt=0
	for line in lines:
		obj=line.split(", ")
		path=obj[0]
		trainset=obj[3]

		print("target person : "+path+" "+str(cnt)+"/"+str(len(lines)))
		cnt=cnt+1

		if path in cache.keys():
			#already estimated
			f.write(line.strip()+", "+str(cache[path])+"\n")
			print("  use cached age "+str(cache[path]))
		else:
			#yet estimated
			if trainset=="0" or trainset=="1":
				if trainset=="0":
					path2=DATASET_ROOT_PATH+DATASET_SUB_PATH+"test/"+path
				else:
					path2=DATASET_ROOT_PATH+DATASET_SUB_PATH+"train/"+path
				print("target folder : "+path2)
				age_sum=0
				age_cnt=0
				for image_path in glob.glob(path2+"/*.jpg"):
					age=calc_age(image_path)
					age_sum+=age
					age_cnt=age_cnt+1
					print(" path : "+image_path+" age : "+str(age))
					if DEBUG_MODE==1:
						break
				age=int(round(age_sum/age_cnt))
				print("  avarage age : "+str(age))
				f.write(line.strip()+", "+str(age)+"\n")


text:
# AUTOGENERATED! DO NOT EDIT! File to edit: notebooks/metrics.ipynb (unless otherwise specified).

__all__ = ['peace', 'ece', 'ece_v2', 'class_wise_error']

# Cell

from riverreliability import utils

import numpy as np
import scipy.stats
import scipy.integrate

import riverreliability.beta
import sklearn.metrics
import sklearn.datasets
import sklearn.model_selection
import sklearn.svm
import sklearn.preprocessing
import sklearn.utils

# Cell

def peace(y_probs, y_preds, y_true, samples=1000, bins="fd", **bin_args):
    """Compute the posterior expected balanced accuracy-based calibration error (PEACE).

    Parameters:
    y_probs (np.array): predicted class probabilities
    y_preds (np.array): predicted class labels
    y_true (np.array): true class labels
    samples (int): number of samples for numerical integration

    Returns:
    exp_ce (float): expected calibration error

    """

    sklearn.utils.check_consistent_length(y_preds, y_true)
    classes = sklearn.utils.multiclass.unique_labels(y_preds, y_true)

    # define the bin function
    def bin_func(y_probs_bin, y_preds_bin, y_true_bin):

        xs = np.linspace(0, 1, samples)
        conf = y_probs_bin.mean()

        if len(np.unique(y_preds_bin)) > 1:
            # estimate beta parameters
            confusion = sklearn.metrics.confusion_matrix(y_true_bin, y_preds_bin, labels=classes)
            params = riverreliability.beta.get_beta_parameters(confusion)
            ys = abs(xs - conf) * riverreliability.beta.beta_avg_pdf(xs, params, fft=True)
        else:
            params = sum(y_preds_bin == y_true_bin)+1, sum(y_preds_bin != y_true_bin)+1
            ys = abs(xs - conf) * scipy.stats.beta.pdf(xs, params[0], params[1])

        # approximate the integral using Simpson's rule
        return scipy.integrate.simps(ys, xs)

    # compute the full result
    bin_indices = utils.get_bin_indices(y_probs, bins=bins, lower=0, upper=1, **bin_args)
    return utils.binning(y_probs, y_preds, y_true, bin_indices, bin_func)

# Cell

def ece(y_probs, y_preds, y_true, balanced=False, bins="fd", **bin_args):
    """Compute the expected calibration error (ECE).

    Parameters:
    y_probs (np.array): predicted class probabilities
    y_preds (np.array): predicted class labels
    y_true (np.array): true class labels

    Returns:
    exp_ce (float): expected calibration error

    """

    sklearn.utils.check_consistent_length(y_preds, y_true)

    # define the bin function
    def bin_func(y_probs_bin, y_preds_bin, y_true_bin):
        acc = (y_preds_bin == y_true_bin).mean()
        conf = y_probs_bin.mean()
        return abs(acc - conf)

    # define the balanced bin function
    def balanced_bin_func(y_probs_bin, y_preds_bin, y_true_bin):
        balacc = sklearn.metrics.balanced_accuracy_score(y_true_bin, y_preds_bin)
        conf = y_probs_bin.mean()
        return abs(balacc - conf)

    # compute the full result
    bin_indices = utils.get_bin_indices(y_probs, bins=bins, lower=0, upper=1, **bin_args)
    func = balanced_bin_func if balanced else bin_func
    return utils.binning(y_probs, y_preds, y_true, bin_indices, func)

# Cell

def ece_v2(y_probs, y_preds, y_true, bins="fd", **bin_args):
    """Compute the expected calibration error based on the expected posterior balanced accuracy (ECEv2).

    Parameters:
    y_probs (np.array): predicted class probabilities
    y_preds (np.array): predicted class labels
    y_true (np.array): true class labels

    Returns:
    exp_ce (float): expected calibration error

    """

    sklearn.utils.check_consistent_length(y_preds, y_true)
    classes = sklearn.utils.multiclass.unique_labels(y_preds, y_true)

    # define the bin function
    def bin_func(y_probs_bin, y_preds_bin, y_true_bin):
        confusion = sklearn.metrics.confusion_matrix(y_true_bin, y_preds_bin, labels=classes)
        acc = riverreliability.beta.balanced_accuracy_expected(confusion, fft=True)
        conf = y_probs_bin.mean()
        return abs(acc - conf)

    # compute the full result
    bin_indices = utils.get_bin_indices(y_probs, bins=bins, lower=0, upper=1, **bin_args)
    return utils.binning(y_probs, y_preds, y_true, bin_indices, bin_func)

# Cell

def class_wise_error(y_probs, y_preds, y_true, base_error, *base_error_args, **base_error_kwargs):
    """Compute classwise-error as proposed in "Beyond temperature scaling: Obtaining well-calibrated
    multiclass probabilities with Dirichlet calibration" (Kull, 2019).

    Parameters:
    y_probs (np.array): predicted class probabilities
    y_preds (np.array): predicted class labels
    y_true (np.array): true class labels
    base_error (callable): function that returns ECE for given probabilities, label predictions and true labels
    base_error_[kw]args ([kw]args): [Keyword ]arguments that should be passed to the base_ece callable.

    Returns:
    exp_ce (float): class-wise expected calibration error

    """

    if hasattr(y_preds, "to_numpy"):
        y_preds = y_preds.to_numpy()
    if hasattr(y_preds, "to_numpy"):
        y_true = y_true.to_numpy()

    sklearn.utils.check_consistent_length(y_preds, y_true)
    classes = sklearn.utils.multiclass.unique_labels(y_preds, y_true)

    result = 0.
    for i, c in enumerate(classes):
        selector = y_preds == c
        if sum(selector) == 0:
            continue

        result += base_error(y_probs[selector, i], y_preds[selector], y_true[selector], *base_error_args, **base_error_kwargs)

    return result/len(classes)

max_stars_repo_name: MaximLippeveld/riverreliability

max_issues_repo_name: MaximLippeveld/riverreliability

max_forks_repo_name: MaximLippeveld/riverreliability

reason: import numpy,import scipy


text:
import matplotlib.pyplot as plt
import numpy as np

skip_num = 1000

with open('data.txt', 'r') as f:
    lines = f.readlines()

    line_size = int(len(lines) / skip_num)

    data = np.zeros((line_size, 2))

    for i in range(line_size):
        data[i][0] = float(lines[skip_num * i].split(',')[0].split('{')[1])
        data[i][1] = float(lines[skip_num * i].split(',')[1].split('}')[0])

    plt.plot(data[:, 0], data[:, 1])
    plt.show()


text:
# -------------------------------------------------------------
# Authors:  Tim van Zalingen (10784012)
#           Maico Timmerman (10542590)
# Date:     12 April 2016
# File: 21.py
#
# The file for assignment 2.1. Plots N number pairs using
# uniform.
# -------------------------------------------------------------
import numpy as np
import matplotlib.pyplot as plt

def ibm_rnd(N, seed):
    a = 65539
    c = 0
    m = 2**31
    numbers = [seed]
    for i in range(1, N):
        numbers.append((a * numbers[i - 1] + c) % float(m))
    return np.array(numbers) / m

def main():
    N = 100
    x = ibm_rnd(N, 983)
    y = ibm_rnd(N, 759)
    plt.title('{0} number pairs created with IBM RND'.format(N))
    plt.plot(x, y, 'o')
    plt.show()

if __name__ == '__main__':
    main()


text:
# Copyright (c) 2020, Huawei Technologies.All rights reserved.
#
# Licensed under the BSD 3-Clause License  (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import torch
import numpy as np
from common_utils import TestCase, run_tests
from common_device_type import dtypes, instantiate_device_type_tests
from util_test import create_common_tensor

class TestAny(TestCase):
    def create_bool_tensor(self, shape, minValue, maxValue):
        input1 = np.random.uniform(minValue, maxValue, shape)
        cpu_input = torch.from_numpy(input1) > 0.5
        npu_input = (torch.from_numpy(input1) > 0.5).to("npu")
        return cpu_input, npu_input

    def cpu_op_exec(self, input):
        output = input.any()
        output = output.numpy()
        return output

    def npu_op_exec(self, input):
        output = input.any()
        output = output.to("cpu")
        output = output.numpy()
        return output

    def test_any_shape_format(self, device):
        shape_list = [[],
                      [1024],
                      [32, 1024],
                      [32, 8, 1024],
                      [128, 32, 8, 1024]]

        for item in shape_list:
            cpu_input, npu_input = self.create_bool_tensor(item, 0, 1)
            cpu_output = self.cpu_op_exec(cpu_input)
            npu_output = self.npu_op_exec(npu_input)
            self.assertRtolEqual(
                cpu_output.astype(np.int32),
                npu_output.astype(np.int32))

    def cpu_op_exec1(self, input, dim, keepdim):
        output = input.any(dim=dim, keepdim=keepdim)
        output = output.numpy()
        return output

    def npu_op_exec1(self, input, dim, keepdim):
        output = input.any(dim=dim, keepdim=keepdim)
        output = output.to("cpu")
        output = output.numpy()
        return output

    def npu_op_out_exec1(self, input, dim, keepdim):
        shape = list(input.shape)
        output0 = torch.randn(shape)>0
        output1 = torch.randn(shape.pop())>0
        output0 = output0.npu()
        output1 = output1.npu()
        torch.any(input, dim=dim, keepdim=keepdim, out=output0)
        torch.any(input, dim=dim, keepdim=keepdim, out=output1)
        output0 = output0.to("cpu").numpy()
        output1 = output1.to("cpu").numpy()
        return output0, output1

    def test_anyd_shape_format(self, device):
        shape_list = [[ [1024],             0, False],
                      [ [32, 1024],         1, False],
                      [ [32, 8, 1024],      2, True ],
                      [ [128, 32, 8, 1024], 3, True ]]

        for item in shape_list:
            cpu_input, npu_input = self.create_bool_tensor(item[0], 0, 1)
            cpu_output = self.cpu_op_exec1(cpu_input, item[1], item[2])
            npu_output = self.npu_op_exec1(npu_input, item[1], item[2])
            npu_out0, npu_out1 = self.npu_op_out_exec1(npu_input, item[1], item[2])
            self.assertRtolEqual(cpu_output.astype(np.int32),npu_output.astype(np.int32))
            self.assertRtolEqual(cpu_output.astype(np.int32),npu_out0.astype(np.int32))
            self.assertRtolEqual(cpu_output.astype(np.int32),npu_out1.astype(np.int32))

instantiate_device_type_tests(TestAny, globals(), except_for="cpu")
if __name__ == "__main__":
    run_tests()

max_stars_repo_name: Ascend/pytorch

max_issues_repo_name: Ascend/pytorch

max_forks_repo_name: Ascend/pytorch


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import numpy as np
from gmprocess.core.streamcollection import StreamCollection
from gmprocess.io.read import read_data
from gmprocess.utils.test_utils import read_data_dir
from gmprocess.waveform_processing.integrate import get_disp, get_vel
from gmprocess.utils.config import get_config

def test_get_disp():

    data_files, origin = read_data_dir("geonet", "us1000778i", "*.V1A")
    data_files.sort()
    streams = []
    for f in data_files:
        streams += read_data(f)

    sc = StreamCollection(streams)

    config = get_config()
    config["integration"]["frequency"] = True

    final_disp = []
    for st in sc:
        for tr in st:
            tmp_tr = get_disp(tr, config=config)
            final_disp.append(tmp_tr.data[-1])

    target_final_disp = np.array(
        [
            -0.07689,
            0.082552,
            -0.024509,
            -0.00047,
            -0.000257,
            -0.000152,
            -0.003425,
            0.000671,
            0.000178,
        ]
    )

    np.testing.assert_allclose(final_disp, target_final_disp, atol=1e-6)

    config["integration"]["frequency"] = False
    config["integration"]["initial"] = 0.0
    config["integration"]["demean"] = True

    final_disp = []
    for st in sc:
        for tr in st:
            tmp_tr = get_disp(tr, config=config)
            final_disp.append(tmp_tr.data[-1])

    target_final_disp = np.array(
        [
            -0.076882,
            0.082549,
            -0.024512,
            -0.000469,
            -0.000259,
            -0.000152,
            -0.003425,
            0.000672,
            0.000178,
        ]
    )

    np.testing.assert_allclose(final_disp, target_final_disp, atol=1e-6)

def test_get_vel():
    data_files, origin = read_data_dir("geonet", "us1000778i", "*.V1A")
    data_files.sort()
    streams = []
    for f in data_files:
        streams += read_data(f)

    sc = StreamCollection(streams)

    config = get_config()
    config["integration"]["frequency"] = True

    final_vel = []
    for st in sc:
        for tr in st:
            tmp_tr = get_vel(tr, config=config)
            final_vel.append(tmp_tr.data[-1])

    target_final_vel = np.array(
        [
            -2.182293e-03,
            -1.417545e-03,
            2.111492e-03,
            -9.395322e-04,
            1.662219e-03,
            -2.690978e-04,
            1.376186e-04,
            -7.358185e-05,
            1.741465e-05,
        ]
    )

    np.testing.assert_allclose(final_vel, target_final_vel, atol=1e-6)

def test_integrate_taper():
    data_files, origin = read_data_dir("geonet", "us1000778i", "*.V1A")
    data_files.sort()
    streams = []
    for f in data_files:
        streams += read_data(f)

    sc = StreamCollection(streams)

    config = get_config()
    config["integration"]["taper"]["taper"] = True

    final_vel = []
    for st in sc:
        for tr in st:
            tmp_tr = tr.integrate(config=config)
            final_vel.append(tmp_tr.data[-1])

    target_final_vel = np.array(
        [
            3.896186e00,
            -4.901823e00,
            -5.722080e-01,
            1.621672e-01,
            -1.654317e-01,
            -8.242356e-04,
            -1.482590e-02,
            1.504334e-01,
            1.021050e-01,
        ]
    )

    np.testing.assert_allclose(final_vel, target_final_vel, atol=1e-6)

if __name__ == "__main__":
    os.environ["CALLED_FROM_PYTEST"] = "True"
    test_get_disp()
    test_get_vel()
    test_integrate_taper()

max_stars_repo_name: usgs/groundmotion-processing-

max_issues_repo_name: usgs/groundmotion-processing-

max_forks_repo_name: usgs/groundmotion-processing-


text:
"""
main training script
"""

import os
import math
from decimal import Decimal

import utility

import numpy as np
from math import floor
import torch
import torch.nn.utils as utils
from tqdm import tqdm
import torch.nn.functional as F

class Trainer():
    def __init__(self, args, loader, my_model, my_loss, ckp):
        self.args = args
        self.scale = args.scale

        self.ckp = ckp
        self.loader_train = loader.loader_train
        self.loader_test = loader.loader_test
        self.model = my_model
        self.loss = my_loss
        self.losstype = args.loss
        self.task = args.task
        self.noise_eval = args.noise_eval
        self.optimizer = utility.make_optimizer(args, self.model)

        if self.args.load != '':
            self.optimizer.load(ckp.dir, epoch=len(ckp.log))
        self.noiseL_B = [0, 55]  # ingnored when opt.mode=='S'
        self.error_last = 1e8

        self.ckp.write_log("-------options----------")
        self.ckp.write_log(args)

    def train(self):
        self.loss.step()
        epoch = self.optimizer.get_last_epoch() + 1
        lr = self.optimizer.get_lr()

        self.ckp.write_log(
            '[Epoch {}]\tLearning rate: {:.2e}'.format(epoch, Decimal(lr))
        )
        self.loss.start_log()
        self.model.train()

        timer_data, timer_model, timer_backward = utility.timer(), utility.timer(), utility.timer()
        # TEMP
        self.loader_train.dataset.set_scale(0)
        for batch, (lr, hr, mask, _,) in enumerate(self.loader_train):
            if self.args.debug:
                print("into train loop")
                print(lr.shape,hr.shape,mask.shape)

            if self.task =='denoise' and (not self.args.real_isp):
            #if self.args.compute_grads or self.args.predict_groups:

                noise = torch.randn(lr.size())*(self.noise_eval)
                lr = torch.clamp((lr + noise),0,255)

            lr, hr, mask = self.prepare(lr, hr, mask)
            # lr = lr.to(self.model.device)
            # hr = hr.to(self.model.device)
            # mask = mask.to(self.model.device)
            mask.requires_grad_(False)
            if self.args.debug:
                print('lr shape:', lr.shape)
                print("hr shape: ", hr.shape)
                print("mask shape", mask.shape)

            timer_data.hold()
            timer_model.tic()

            self.optimizer.zero_grad()

            sr = self.model(lr, mask, 0)
            if self.args.debug:
                print("training forwarded: ")
                print(lr.shape,hr.shape,mask.shape)
                print(mask.dtype)
                print(mask[:,10:12,10:12])
            if self.args.compute_grads:
                loss = self.loss(sr, mask)
            elif self.args.predict_groups:
                mask = mask.long()
                loss = self.loss_expand(sr[0],mask[:,0,:,:])+self.loss_expand(sr[1],mask[:,1,:,:])+self.loss_expand(sr[2],mask[:,2,:,:])
            else:
                if 'WeightedL1' in self.losstype:
                    loss = self.loss(sr, hr, mask)
                else:
                    loss = self.loss(sr, hr)
            timer_backward.tic()
            loss.backward()
            timer_backward.hold()
            if self.args.debug:
                print("loss backwarded: ")
            if self.args.gclip > 0:
                utils.clip_grad_value_(
                    self.model.parameters(),
                    self.args.gclip
                )
            self.optimizer.step()

            timer_model.hold()

            if (batch + 1) % self.args.print_every == 0:
                if self.args.timer:
                    self.ckp.write_log('[{}/{}]\t{}\t{:.3f}+{:.3f}+{:.3f}s\t{:.3f}+{:.3f}+{:.3f}s'.format(
                        (batch + 1) * self.args.batch_size,
                        len(self.loader_train.dataset),
                        self.loss.display_loss(batch),
                        timer_model.release(),
                        timer_data.release(),
                        timer_backward.release(),
                        self.args.timer_total_forward.release(),
                        self.args.timer_embedding_forward.release(),
                        self.args.timer_kconv_forward.release(),
                    ))
                else:
                    self.ckp.write_log('[{}/{}]\t{}\t{:.1f}+{:.1f}+{:.1f}s'.format(
                        (batch + 1) * self.args.batch_size,
                        len(self.loader_train.dataset),
                        self.loss.display_loss(batch),
                        timer_model.release(),
                        timer_data.release(),
                    timer_backward.release()))

            timer_data.tic()

        self.loss.end_log(len(self.loader_train))
        self.error_last = self.loss.log[-1, -1]
        self.optimizer.schedule()

    def loss_expand(self,pred,ground):
        if self.args.debug:
            print("loss expand function:")
            print(pred.shape, ground.shape)
        pred = pred.permute(0, 2, 3, 1)
        ncs = pred.shape[3]
        pred = pred.view(-1, ncs)
        mask_flat = ground.view(-1)
        lossp = self.loss(pred, mask_flat)

        return lossp
    """
    def merge_grads(self,grads_tensor):
        stre = np.linspace(0, 0.2, self.args.Qstr + 1)[1:-1]
        cohe = np.linspace(0, 1, self.args.Qcohe + 1)[1:-1]
        grads = grads_tensor.cpu().numpy()
        H, W, C = grads.shape
        if self.args.debug:
            print('merging grads:')
            print(stre, cohe)
            print(grads.shape, grads.dtype)
        grad_map = np.zeros((H, W), dtype=np.int32)
        for i in range(H):
            for j in range(W):
                if self.args.debug:
                    print('grads:')
                    print(grads[i,j,0],grads[i,j,1],grads[i,j,2])
                tempu = floor(grads[i, j, 0] * self.args.Qangle)
                if tempu < 0:
                    tempu = 0
                if tempu > self.args.Qangle - 1:
                    tempu = self.args.Qangle - 1
                if self.args.debug:
                    print('tempu:')
                    print(tempu)
                lamda =  np.searchsorted(stre, grads[i, j, 1])
                mu = np.searchsorted(cohe, grads[i, j, 2])
                if self.args.debug:
                    print('lamda&mu:')
                    print(lamda, mu)
                grad_map[i,j] = (tempu * self.args.Qstr * self.args.Qcohe) + lamda * self.args.Qcohe + mu

        if self.args.debug:
            print('grad_map:')
            print(grad_map[10:12,10:12])

        return grad_map
    """

    def merge_grads(self,grads_tensor):
        """

        :param grads_tensor: of shape (H,W,3)
        :return:
        """
        if self.args.Qstr ==2:
            stre = np.array([0.05],dtype=np.float32)
        else:
            stre = np.linspace(0, 0.2, self.args.Qstr + 1)[1:-1]
        if self.args.Qcohe ==2:
            cohe = np.array([0.3], dtype=np.float32)
        else:
            cohe = np.linspace(0, 1, self.args.Qcohe + 1)[1:-1]
        grads = grads_tensor.cpu().numpy()
        H, W, C = grads.shape
        if self.args.debug:
            print('merging grads:')
            print(stre, cohe)
            print(grads.shape, grads.dtype)
        tempus = np.clip(np.floor(grads[:,:,0]*self.args.Qangle),0,self.args.Qangle-1)
        lamdas = np.clip(np.searchsorted(stre,grads[:,:,1]),0,self.args.Qstr)
        mus = np.clip(np.searchsorted(cohe,grads[:,:,2]),0,self.args.Qcohe)
        grad_map = (tempus * self.args.Qstr * self.args.Qcohe) + lamdas * self.args.Qcohe + mus

        if self.args.debug:
            print('grad_map:')
            print(grad_map.shape)
            print(grad_map[10:12,10:12])

        return grad_map

    def stride_cut(self,lr,hr,mask=None, split_size=400, overlap_size=100):
        if self.args.debug:
            print('stride cutting the following tensors: ')
            print('lr: ',lr.shape)
            print('hr: ',hr.shape)
            if mask is not None:
                print('mask: ',mask.shape)
            print('split_size: ',split_size,'  overlap_size: ',overlap_size)
        stride = split_size - overlap_size
            ## 构建图像块的索引
        orig_shape = (lr.shape[2], lr.shape[3])

        imhigh = lr.shape[2]
        imwidth = lr.shape[3]
        range_y = np.arange(0, imhigh - split_size, stride)
        range_x = np.arange(0, imwidth - split_size, stride)
        if self.args.debug:
            print(range_x)
            print(range_y)
        if range_y[-1] != imhigh - split_size:
            range_y = np.append(range_y, imhigh - split_size)
        if range_x[-1] != imwidth - split_size:
            range_x = np.append(range_x, imwidth - split_size)

        sz = len(range_y) * len(range_x)  ## 图像块的数量
        if self.args.debug:
            print('sz: ',sz)
        res_lr=  torch.zeros((sz,lr.shape[1], split_size,split_size),dtype=lr.dtype)
        res_hr = torch.zeros((sz, hr.shape[1], split_size, split_size), dtype = hr.dtype)
        res_mask = torch.zeros((sz, split_size, split_size), dtype = mask.dtype)
        if self.args.debug:
            print(range_x)
            print(range_y)
            print('sz: ',sz, res_lr.shape,res_lr.dtype,res_hr.shape,res_mask.shape,res_mask.dtype)
        index = 0
        for y in range_y:
            for x in range_x:
                res_lr[index,:,:,:] = lr[0,:,y:y + split_size, x:x + split_size]
                res_hr[index, :, :, :] = hr[0, :, y:y + split_size, x:x + split_size]
                res_mask[index, :, :] = mask[0, y:y + split_size, x:x + split_size]
                index = index + 1

        if self.args.debug:
            print('finished cutting: ', res_lr.shape,res_hr.shape,res_mask.shape)

        return res_lr,res_hr,res_mask

    def recon_from_cols(self,sr_cols,imsize, stride=300,split_size=400):

        sr_recon = torch.zeros((1,sr_cols.shape[1],imsize[0],imsize[1]),dtype = sr_cols.dtype)

        w = torch.zeros((1,sr_cols.shape[1],imsize[0],imsize[1]),dtype = sr_cols.dtype)
        if self.args.debug:
            print('reconstructing patches: ', sr_recon.shape, w.shape)
        range_y = np.arange(0, imsize[0] - split_size, stride)
        range_x = np.arange(0, imsize[1] - split_size, stride)
        if range_y[-1] != imsize[0] - split_size:
            range_y = np.append(range_y, imsize[0] - split_size)
        if range_x[-1] != imsize[1] - split_size:
            range_x = np.append(range_x, imsize[1] - split_size)
        if self.args.debug:
            print('range x and y: ', range_x, range_y)
        index = 0
        for y in range_y:
            for x in range_x:
                sr_recon[0,:,y:y + split_size, x:x + split_size] = sr_recon[0,:,y:y + split_size, x:x + split_size] + sr_cols[
                    index,:,:,:]
                w[0,:,y:y + split_size, x:x + split_size] = w[0,:,y:y + split_size, x:x + split_size] + 1
                index = index + 1
        if self.args.debug:
            print('reconstruction finished: ', sr_recon.shape,sr_recon.max(), w.shape)

        return sr_recon / w

    def test(self):
        torch.set_grad_enabled(False)
        #print(self.loader_test)
        epoch = self.optimizer.get_last_epoch()
        self.ckp.write_log('\nEvaluation:')
        self.ckp.add_log(
            torch.zeros(1, len(self.loader_test), len(self.scale))
        )
        #self.model.to
        self.model.eval()

        timer_test = utility.timer()
        if self.args.save_results: self.ckp.begin_background()
        if self.args.debug:
            print("Eval loop")
        for idx_data, d in enumerate(self.loader_test):
            if self.args.debug:
                print("idx data retrieved")
            for idx_scale, scale in enumerate(self.scale):
                if self.args.debug:
                    print("scale fetched")
                d.dataset.set_scale(idx_scale)
                torch.manual_seed(2)
                for lr, hr, mask, filename in tqdm(d, ncols=80):
                    if self.args.debug:
                        print("eval started")

                    #print("prepared")
                    if self.task =='denoise' and (not self.args.real_isp):
                    #if self.args.compute_grads or self.args.predict_groups:
                        noise = (torch.randn(lr.size())*(self.noise_eval))
                        lr = torch.clamp((lr + noise),0,255)

                    if self.args.predict_groups:

                        mask = mask.long()

                    if self.args.debug:
                        print('lr shape:', lr.shape)
                        print("hr shape: ", hr.shape)
                        print("mask shape", mask.shape)
                    #print("noise added", lr.shape, hr.shape, mask.shape)
                    #print(hr.shape)
                    if lr.shape[2]*lr.shape[3]>250*250 and(not (self.args.compute_grads or self.args.predict_groups)):
                        if self.args.model == 'kpn':
                            if self.args.debug:
                                print('oversized input: ', lr.shape, hr.shape, mask.shape)

                            orig_shape = (lr.shape[2], lr.shape[3])
                            lr_cols, hr_cols, mask_cols = self.stride_cut(lr, hr, mask, split_size=96,
                                                                          overlap_size=48)
                            sr_cols = torch.zeros(hr_cols.shape, dtype=hr_cols.dtype)
                            for colidx in range(lr_cols.shape[0]):
                                lr_slice = lr_cols[colidx, :, :, :].unsqueeze(0)
                                hr_slice = hr_cols[colidx, :, :, :].unsqueeze(0)
                                mask_slice = mask_cols[colidx, :, :].unsqueeze(0)
                                lr_slice, hr_slice, mask_slice = self.prepare(lr_slice, hr_slice, mask_slice)
                                if self.args.debug:
                                    print('cut forwarding: ', colidx, lr_slice.shape)
                                sr_slice = self.model(lr_slice, mask_slice, idx_scale)
                                sr_cols[colidx, :, :, :] = sr_slice[0, :, :, :]
                            if self.args.debug:
                                print('finished patch forwarding: ', sr_cols.shape)
                            sr = self.recon_from_cols(sr_cols, orig_shape, stride=48, split_size=96)
                        else:

                            if self.args.debug:
                                print('oversized input: ',lr.shape, hr.shape, mask.shape)

                            orig_shape = (lr.shape[2],lr.shape[3])
                            lr_cols, hr_cols, mask_cols = self.stride_cut(lr, hr, mask, split_size=200, overlap_size=100)
                            sr_cols = torch.zeros(hr_cols.shape, dtype = hr_cols.dtype)
                            for colidx in range(lr_cols.shape[0]):
                                lr_slice= lr_cols[colidx,:,:,:].unsqueeze(0)
                                hr_slice= hr_cols[colidx,:,:,:].unsqueeze(0)
                                mask_slice = mask_cols[colidx, :, :].unsqueeze(0)
                                lr_slice,hr_slice,mask_slice = self.prepare(lr_slice,hr_slice,mask_slice)
                                sr_slice = self.model(lr_slice, mask_slice,idx_scale)
                                sr_cols[colidx,:,:,:] = sr_slice[0,:,:,:]
                            if self.args.debug:
                                print('finished patch forwarding: ',sr_cols.shape)
                            sr = self.recon_from_cols(sr_cols,orig_shape, stride=100,split_size=200)

                    else:
                        lr, hr, mask = self.prepare(lr, hr, mask)
                        if self.args.debug:
                            print('prepared: ', lr.shape, hr.shape,mask.shape)
                        sr = self.model(lr, mask, idx_scale)
                    if self.args.debug:
                        print("forwarded")
                        print("eval sr shape & mask shape:")
                        if self.args.predict_groups:
                            print(sr[0].shape, mask.shape)
                        else:
                            print(sr.shape, mask.shape)
                    if not (self.args.compute_grads or self.args.predict_groups):
                        sr = utility.quantize(sr, self.args.rgb_range)

                    if self.args.predict_groups:

                        _, sr_predict0 = torch.max(sr[0], 1)
                        _, sr_predict1 = torch.max(sr[1], 1)
                        _, sr_predict2 = torch.max(sr[2], 1)
                        sr_predict = torch.cat((sr_predict0.unsqueeze(-1),sr_predict1.unsqueeze(-1),sr_predict2.unsqueeze(-1)),dim=-1)
                        sr_predict = sr_predict.to(dtype=torch.int32)
                        if self.args.debug:
                            print("calculate prediction")
                            print("prediction shape:")
                            print(sr_predict.shape)
                            print(sr_predict.dtype)
                        save_list = [((sr_predict.squeeze(0)[:,:,0]*self.args.Qcohe*self.args.Qstr)+ (sr_predict.squeeze(0)[:,:,1]*self.args.Qcohe)+(sr_predict.squeeze(0)[:,:,2]))]
                        if self.args.debug:
                            print("savelist saved")
                            print("savelist shape:")
                            print(save_list[0].shape)
                    elif self.args.compute_grads:
                        if self.args.debug:
                            print("saving mask: ")
                        if self.args.real_isp:

                            save_list = [sr.permute(0, 2, 3, 1).contiguous().squeeze(0)]

                        elif self.args.outgrads:
                            save_list = [sr.permute(0, 2, 3, 1).contiguous().squeeze(0)]
                        else:
                            save_list = [self.merge_grads(sr.permute(0,2,3,1).contiguous().squeeze(0))]
                        #save_list = [sr.permute(0, 2, 3, 1).contiguous().squeeze(0)]
                    else:

                        save_list = [sr]

                    if self.args.compute_grads:
                        if self.args.debug:
                            print("calculate mse")
                            print(sr.shape, sr.dtype, mask.shape, mask.dtype)
                        if self.args.no_gt:
                            self.ckp.log[-1, idx_data, idx_scale] += 0
                        else:
                            self.ckp.log[-1, idx_data, idx_scale] += utility.calc_mse(
                                sr, mask
                            )
                    elif self.args.predict_groups:
                        #print('permulting')
                        #print(mask.permute(0,2,3,1).contiguous().shape)
                        #mask_per = mask.permute(0,2,3,1).contiguous()
                        #print('permulted')
                        #print(mask_per.shape,mask_per.dtype)
                        #print(sr_predict.shape,sr_predict.dtype)

                        #diffs = (sr_predict==mask.permute(0,2,3,1).contiguous())
                        #if self.args.debug:
                        #    print("calculate difference:")
                        #    print(diffs.shape,diffs.dtype)
                        #    print("number of elements:")
                        #    print(mask.numel())
                        #    print("correct prediction:")
                        #    print(torch.sum((sr_predict==mask.permute(0,2,3,1)).float()))
                        #acc = torch.sum((sr_predict[1]==mask.permute(0,2,3,1)).float())/mask.numel()
                        if self.args.debug:
                            print(sr_predict.shape, sr_predict.dtype)
                            print(mask.shape, mask.dtype)
                        acc = torch.sum((sr_predict[:,:,:,1] == mask.to(dtype=torch.int32)[:,1,:,:]).float()) / mask[:,1,:,:].numel()
                        if self.args.debug:
                            print("calculate accuracy:")
                            print(acc)
                        self.ckp.log[-1, idx_data, idx_scale] += acc

                    else:
                        self.ckp.log[-1, idx_data, idx_scale] += utility.calc_psnr(
                            sr, hr, scale, self.args.rgb_range, dataset=d
                        )
                    if self.args.debug:
                        print("loss logged!")

                    if self.args.save_gt:
                        if self.args.compute_grads or self.args.predict_groups:
                            save_list.extend([mask.permute(0,2,3,1).contiguous().squeeze(0)])
                        else:
                            if self.args.use_real:
                                if self.args.debug:
                                    print('saving lr image when using real isp',lr[:,0:self.args.n_colors,:,:].shape)
                                save_list.extend([lr[:,0:self.args.n_colors,:,:], hr])
                            else:
                                save_list.extend([lr, hr])
                    if self.args.debug:
                        print("SAVING RESULTS!")
                    if self.args.save_results:
                        self.ckp.save_results(d, filename[0], save_list, scale)

                    if self.args.debug:
                        print("results saved!")

                self.ckp.log[-1, idx_data, idx_scale] /= len(d)
                best = self.ckp.log.max(0)
                self.ckp.write_log(
                    '[{} x{}]\tPSNR: {:.3f} (Best: {:.3f} @epoch {})'.format(
                        d.dataset.name,
                        scale,
                        self.ckp.log[-1, idx_data, idx_scale],
                        best[0][idx_data, idx_scale],
                        best[1][idx_data, idx_scale] + 1
                    )
                )

        self.ckp.write_log('Forward: {:.2f}s\n'.format(timer_test.toc()))
        self.ckp.write_log('Saving...')

        if self.args.save_results:
            self.ckp.end_background()

        if not self.args.test_only:
            self.ckp.save(self, epoch, is_best=(best[1][0, 0] == epoch))

        self.ckp.write_log(
            'Total: {:.2f}s\n'.format(timer_test.toc()), refresh=True
        )

        torch.set_grad_enabled(True)

    def prepare(self, *args, test=False):
        device = torch.device('cpu' if self.args.cpu or test else 'cuda')
        self.model = self.model.to(device)
        def _prepare(tensor):
            if self.args.precision == 'half': tensor = tensor.half()
            return tensor.to(device)

        return [_prepare(a) for a in args]

    def terminate(self):
        if self.args.test_only:
            self.test()
            return True
        else:
            epoch = self.optimizer.get_last_epoch() + 1
            return epoch >= self.args.epochs

max_stars_repo_name: XenonLamb/CSConvNet

max_issues_repo_name: XenonLamb/CSConvNet

max_forks_repo_name: XenonLamb/CSConvNet


text:
"""
===========================
Hill5 analytic infall model
===========================
.. moduleauthor:: Adam Ginsburg <adam.g.ginsburg@gmail.com>

Code translated from:
https://bitbucket.org/devries/analytic_infall/overview

Original source:
http://adsabs.harvard.edu/abs/2005ApJ...620..800D

Module API
^^^^^^^^^^
"""
import numpy as np
from . import model

def hill5_model(xarr, tau, v_lsr,  v_infall,  sigma,  tpeak, TBG=2.73):
    """
    The hill5 from de Vries and Myers 2005.  This model implicitly has zero
    optical depth in the envelope, no envelope velocity, and has a fixed
    background radiation temperature (see Table 2 in the paper).

    Parameters
    ----------
    xarr : np.ndarray
        array of x values
    tau : float
        tau_c, the core-center optical depth
    v_lsr : float
        The centroid velocity in km/s
    v_infall : float
        The infall velocity
    sigma : float
        The line width
    tpeak : float
        The peak brightness temperature
    TBG : float
        The background temperature
    """

    vf = v_lsr+v_infall
    vr = v_lsr-v_infall

    velocity_array = np.array(xarr.as_unit('km/s'))
    tauf = tau*np.exp(-((velocity_array-vf)/sigma)**2 / 2.0 )
    taur = tau*np.exp(-((velocity_array-vr)/sigma)**2 / 2.0 )

    subf = ((1-np.exp(-tauf))/tauf * (tauf>1e-4) + (tauf < 1e-4))
    subr = ((1-np.exp(-taur))/taur * (taur>1e-4) + (taur < 1e-4))
    subf[subf != subf] = 1.0
    subr[subr != subr] = 1.0

    frequency = np.array(xarr.as_unit('Hz'))
    hill_array =(jfunc(tpeak,frequency)-jfunc(TBG,frequency))*(subf-np.exp(-tauf)*subr)

    if np.isnan(hill_array).any():
        raise ValueError("Hill5 model has a NAN")

    return hill_array

def jfunc(t, nu):
    """
    t- kelvin
    nu - Hz?
    """

    H=6.6260755e-27
    K=1.380658e-16
    NSIG=2.0

    # I think this is only necessary for C
    #if(nu<1.0e-6) return t

    to = H*nu/K
    return to/(np.exp(to/t)-1.0)

hill5_fitter = model.SpectralModel(hill5_model, 5,
    parnames=['tau', 'v_lsr',  'v_infall',  'sigma', 'tpeak'],
    parlimited=[(True,False),(False,False),(True,False),(True,False), (True,False)],
    parlimits=[(0,0), (0,0), (0,0), (0,0), (0,0)],
    shortvarnames=("\\tau","v_{lsr}","v_{infall}","\\sigma","T_{peak}"), # specify the parameter names (TeX is OK)
    fitunit='km/s',
    guess_types=[1.0, 'center', 1.0, 'width', 'amplitude'],
   )

max_stars_repo_name: glangsto/pyspeckit

max_issues_repo_name: glangsto/pyspeckit

max_forks_repo_name: glangsto/pyspeckit


text:
# -*- coding: utf-8 -*-
# @Time: 2020/6/20 0020 0:46
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard, ReduceLROnPlateau
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.utils import to_categorical
from .model_structure.BiLSTM import BILSTM_Model
from .model_structure.GRU import GRU_Model
from .model_structure.TextCNN_m import TextCNN_m
from sklearn import metrics
import numpy as np
import pickle
class SA_Model_Train:
    def __init__(self, max_words, embedding_dim, maxlen, tokenize_path, sa_model_path_m, train_method=''):
        if train_method == 'gru':
            self.init_model = GRU_Model()
        elif train_method == 'bilstm':
            self.init_model = BILSTM_Model()
        else:
            # 默认textcnn
            self.init_model = TextCNN_m()
        self.max_words = max_words
        self.tokenize_path = tokenize_path
        self.embedding_dim = embedding_dim
        self.maxlen = maxlen
        self.sa_model_path_m = sa_model_path_m
        self.model = self.init_model.create_model(self.max_words, self.embedding_dim, self.maxlen)
    def train_tk(self,train_data):
        tokenizer = Tokenizer(filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', num_words=self.max_words)
        tokenizer.fit_on_texts(train_data)
        with open(self.tokenize_path, 'wb') as tokenize:
            pickle.dump(tokenizer, tokenize)
        return tokenizer
    def train(self,train_data,label,num_classes,batch_size=256,epochs=10,verbose=1,evaluate=True):
        # to_categorical
        targets_values = to_categorical(label, num_classes=num_classes)
        # data split
        x_train, y_train, x_test, y_test = train_test_split(train_data, targets_values, test_size=0.2, random_state=1)
        # pad_sequences
        tokenizer = self.train_tk(train_data)
        x_train, x_test = pad_sequences(tokenizer.texts_to_sequences(x_train), self.maxlen), np.array(x_test)
        y_train, y_test = pad_sequences(tokenizer.texts_to_sequences(y_train), self.maxlen), np.array(y_test)
        self.model.fit(x_train, x_test, batch_size, epochs, verbose,
                       validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss', min_delta=0.0001)])
        self.model.save(self.sa_model_path_m)
        try:
            pre_result = self.model.predict(y_train, batch_size=256, verbose=0)
        except:
            result_ = self.model.predict_classes(y_train, batch_size=256, verbose=0)
            pre_result = np.argmax(result_, axis=1)
        if evaluate:
            result = [np.argmax(i) for i in pre_result]
            y_test = [np.argmax(i) for i in y_test]
            report = metrics.classification_report(y_test, result)
            acc = metrics.accuracy_score(y_test, result)
            auc = metrics.roc_auc_score(y_test, result)
            print(report)
            print('acc:  {}    auc:  {}'.format(acc, auc))
# if __name__ == '__main__':
#     # C-CNN-SA(字符级卷积网络)
#     train_data = pd.read_csv('data/sa_data_train.csv')
#     # list sentence
#     train_data['text_cut'] = train_data['text'].apply(lambda x: " ".join(list(x)))
#     model = SA()
#     tokenizer = model.train_tk()
#     # 2-8的分割数据,固定测试数据
#     targets_values = to_categorical(train_data['label'], num_classes=num_classes)
#     x_train, y_train, x_test, y_test = train_test_split(train_data['text_cut'],targets_values, test_size=0.2, random_state=1)
#     # pad_sequences
#     x_train, x_test = pad_sequences(tokenizer.texts_to_sequences(x_train), maxlen), np.array(x_test)
#     y_train, y_test = pad_sequences(tokenizer.texts_to_sequences(y_train), maxlen), np.array(y_test)
#     # train
#     pre_result = model.train(x_train, x_test)
#     # evaluate
#     model.evaluate(pre_result, y_test)

max_stars_repo_name: CarryChang/litNLP

max_issues_repo_name: CarryChang/litNLP

max_forks_repo_name: CarryChang/litNLP


text:
# 12 March 2018 Miroslav Gasparek
# Python bootcamp, exercise 4: Practice with Pandas

# Import modules
import glob
import re

import numpy as np
import pandas as pd
import numba

# This is how we import the module of Matplotlib we'll be using
import matplotlib.pyplot as plt

# Some pretty Seaborn settings
import seaborn as sns
rc={'lines.linewidth': 2, 'axes.labelsize': 18, 'axes.titlesize': 18}
sns.set(rc=rc)

# Close figures
plt.close('all')
#
# # Exercise 4.1
# # a) Get list of CSV files
# csv_list = glob.glob('data/grant*.csv')
# # Initialize list of DataFrames
# df_list = []
# # Load in each sequentially.
# for csv_file in csv_list:
#     # Read in DataFrame
#     df = pd.read_csv(csv_file, comment='#')
#     # Place in list
#     df_list.append(df)
#
# # b) Initialize years
# years = []
# for csv_file in csv_list:
#     years.append(int(csv_file[-8:-4]))
#
# # Rename 'yearband' to 'year'
# df_list[0] = df_list[0].rename(columns={'yearband': 'year'})
# # No worries about Y2K
# df_list[0]['year'] += 1900
#
# for i, df in enumerate(df_list):
#     df_list[i]['year'] = np.ones(len(df), dtype=int)*years[i]
#
# # Change the names of each column for each DataFrame
# # Select a DataFrame
# df = df_list[3]
# # See which column had 'len' in it and extract it
# df.columns[df.columns.str.contains('len')]
#
# # Define a renaming function
# def rename_cols(df):
#     """ Rename columns so that all DataFrames have same column headings."""
#     # Sniff out the key names from names that are close
#     band_key = df.columns[df.columns.str.contains('and')][0]
#     species_key = df.columns[df.columns.str.contains('ecies')][0]
#     length_key = df.columns[df.columns.str.contains('len')][0]
#     depth_key = df.columns[df.columns.str.contains('dep')][0]
#     year_key = df.columns[df.columns.str.contains('year')][0]
#
#     # Rename the columns using renaming dictionary
#     return df.rename(columns={  band_key: 'band',
#                                 species_key: 'species',
#                                 depth_key: 'beak depth (mm)',
#                                 length_key: 'beak length (mm)',
#                                 year_key: 'year'})
#
# # Loop through the DataFrames and rename the columns
# for i, df in enumerate(df_list):
#     df_list[i] = rename_cols(df)
#
# # Do the concatenation
# df = pd.concat(df_list, axis=0, ignore_index=True)
#
# # c) Drop the duplicated bird numberrs
# # Stats about how many birds were measured more than once
# # print(  'There were', len(df['band'][df['band'].duplicated()].unique()),
# #         'birds that were measured more than once.')
# # print('There were', len(df['band'].unique()), 'total birds measured.')
#
# # Drop all rows with matching year and band (keep first)
# df = df.drop_duplicates(subset=['year', 'band'])
#
# # Save DataFrame as a CSV file
# df.to_csv('data/final_grant_complete.csv', index=False)
#
# # d) Plot the ECDF of the beak depths
# # Define ECDF function
# def ecdf(data):
#     """ Computes the ECDF of input data"""
#     return np.sort(data), np.arange(1,len(data) + 1) / len(data)
# # Pull out data for ECDF plotting
# # Beak depth
# bd_fortis = \
# bd_fortis = df.loc[(df['species']=='fortis') & (df['year']==1987), 'beak depth (mm)']
# bd_scandens = \
# bd_scandens = df.loc[(df['species']=='scandens') & (df['year']==1987), 'beak depth (mm)']
# # Beak length
# bl_fortis = \
# bl_fortis = df.loc[(df['species']=='fortis') & (df['year']==1987), 'beak length (mm)']
# bl_scandens = \
# bl_scandens = df.loc[(df['species']=='scandens') & (df['year']==1987), 'beak length (mm)']
#
# # Plot the results
# fig1, ax = plt.subplots(2, 1, figsize=(8, 5))
#
# # Plot beak depth ECDFs
# x, y = ecdf(bd_fortis)
# ax[0].plot(x, y, marker='.', linestyle='none')
# x, y = ecdf(bd_scandens)
# ax[0].plot(x, y, marker='.', linestyle='none')
# ax[0].margins(0.02)
#
# # Plot beak length ECDFs
# x, y = ecdf(bl_fortis)
# ax[1].plot(x, y, marker='.', linestyle='none')
# x, y = ecdf(bl_scandens)
# ax[1].plot(x, y, marker='.', linestyle='none')
# ax[1].margins(0.02)
#
# # Legends and axis labels, tidying
# ax[0].legend(('fortis', 'scandens'), loc='lower right')
# ax[0].set_xlabel('beak depth (mm)')
# ax[0].set_ylabel('ECDF')
# ax[1].set_xlabel('beak length (mm)')
# ax[1].set_ylabel('ECDF')
# fig1.tight_layout(h_pad=3)
# # fig1.show()
#
# # e) Beak depth vs beak length
# # Extract data
# bd_fortis = df.loc[(df['species']=='fortis') & (df['year']==1987), 'beak depth (mm)']
# bd_scandens = df.loc[(df['species']=='scandens') & (df['year']==1987), 'beak depth (mm)']
#
# bl_fortis = df.loc[(df['species']=='fortis') & (df['year']==1987), 'beak length (mm)']
# bl_scandens = df.loc[(df['species']=='scandens') & (df['year']==1987), 'beak length (mm)']
# # Plot beak depth vs. beak length
# fig2 = plt.figure(2)
# plt.plot(bl_fortis, bd_fortis, marker='.', color='blue', linestyle='none', alpha=0.5)
# plt.plot(bl_scandens, bd_scandens, marker='.', color='red', linestyle='none', alpha=0.5)
# plt.xlabel('beak length (mm)')
# plt.ylabel('beak depth (mm)')
# plt.margins(0.02)
# plt.legend(('Fortis','Scandens'),loc='lower right')
# # fig2.show()
#
# # f) Repetition of the measurement for all the beaks.
# def plot_beak_data(ax, df, year, legend=False):
#     """ Plot beak length and beak depth data for a given year on a given axis. """
#     # Extract data we want
#     df_fortis = df[(df['year']==year) & (df['species']=='fortis')]
#     df_scandens = df[(df['year']==year) & (df['species']=='scandens')]
#
#     # Plot the result
#     ax.plot(df_fortis['beak length (mm)'], df_fortis['beak depth (mm)'],
#             marker='.', linestyle='none', alpha=0.5)
#     ax.plot(df_scandens['beak length (mm)'], df_scandens['beak depth (mm)'],
#             marker='.', linestyle='none', alpha=0.5)
#
#     # Clean up
#     ax.margins(0.02)
#     ax.set_xlabel('beak length (mm)', fontsize=12)
#     ax.set_ylabel('beak depth (mm)', fontsize=12)
#     ax.set_title(str(year), fontsize=14)
#     if legend:
#         ax.legend(('fortis', 'scandens'), loc='upper left')
#
#     return ax
#
# # Plots layout
# fig3, ax = plt.subplots(2, 3, figsize=(8, 5), sharex=True, sharey=True)
# # Which axes to use
# ax_inds = ((0,0), (0,1), (0,2), (1,0), (1,1))
#
# # Loop through years and make plots
# for i, year in enumerate(years):
#     if i == 0:
#         legend = True
#     else:
#         legend=False
#     _ = plot_beak_data(ax[ax_inds[i]], df, year, legend=legend)
#
# # Tidy up
# ax[1, 2].axis('off')
# fig3.tight_layout()
# # fig3.show()
#
# ###############
# # Exercise 4.2
# # a) Load the drone weight data in as a Pandas DataFrame
# df_weight = pd.read_csv('data/bee_weight.csv', comment='#')
# # b) plot ecdf of the weight data for control and pesticide
# df_control = df_weight.loc[df_weight['Treatment']=='Control', ('Weight')]
# df_pesticide = df_weight.loc[df_weight['Treatment']=='Pesticide', ('Weight')]
#
# # Plot the ECDFs
# fig4 = plt.figure(4)
#
# x_c, y_c = ecdf(df_control)
# plt.plot(x_c, y_c, marker='.', color='red', linestyle='none', alpha=0.5)
# x_p, y_p = ecdf(df_pesticide)
# plt.plot(x_p, y_p, marker='.', color='blue', linestyle='none', alpha=0.5)
# plt.xlabel('Weight (mg)')
# plt.ylabel('ECDF')
# plt.legend(('Control','Pesticide'), loc='lower right')
# plt.margins(0.02)
# fig4.show()
#
# # c) Compute the mean drone weight for control and pesticide groups
# # Get data
# mean_control = np.mean(df_weight.loc[df_weight['Treatment']=='Control', 'Weight'])
# mean_pest = np.mean(df_weight.loc[df_weight['Treatment']=='Pesticide','Weight'])
# # Print means
# print('Mean control: ', mean_control, 'mg')
# print('Mean pesticide: ', mean_pest, 'mg')
# # Means are close, so we use boootstrapping
# # Define replicate generating function
# def draw_bs_reps(data, func, size=1):
#     """ Draw bootstrap replicates from a data set."""
#     n = len(data)
#
#     # Initialize array of replicates
#     reps = np.empty(size)
#
#     for i in range(size):
#         # Generate bootstrap sample
#         bs_sample = np.random.choice(data, n)
#         # Compute replicate
#         reps[i] = func(bs_sample)
#
#     return reps
#
# # Draw 100 000 bootstrap reps for both.
# bs_reps_control = draw_bs_reps(
#         df_weight.loc[df_weight['Treatment']=='Control', 'Weight'], np.mean, size=100000)
# bs_reps_pest = draw_bs_reps(
#         df_weight.loc[df_weight['Treatment']=='Pesticide', 'Weight'], np.mean, size=100000)
#
# conf_int_control = np.percentile(bs_reps_control, [2.5, 97.5])
# conf_int_pest = np.percentile(bs_reps_pest, [2.5, 97.5])
# # print('Confidence interval for weight of control group:', conf_int_control)
# # print('Confidence interval for weight of pesticide group:', conf_int_pest)
#
# # d) Repeat the previous measurements with drone sperm
# # Load the drone sperm data
# df_sperm = pd.read_csv('data/bee_sperm.csv', comment='#')
# df_sperm_control = df_sperm.loc[df_sperm['Treatment']=='Control', 'Quality']
# df_sperm_pesticide = df_sperm.loc[df_sperm['Treatment']=='Pesticide', 'Quality']
# # Plot the ECDF of the data
# fig5 = plt.figure(5)
# (x_sperm_c, y_sperm_c) = ecdf(df_sperm_control)
# plt.plot(x_sperm_c, y_sperm_c, marker='.', color='red', linestyle='none', alpha=0.5)
# (x_sperm_p, y_sperm_p) = ecdf(df_sperm_pesticide)
# plt.plot(x_sperm_p, y_sperm_p, marker='.', color='blue', linestyle='none', alpha=0.5)
# plt.xlabel('sperm quality')
# plt.ylabel('ECDF')
# plt.legend(('Control','Pesticide'), loc='lower right')
# plt.margins(0.02)
# fig5.show()
#
# # Calculate the means
# mean_sperm_control = np.mean(df_sperm.loc[df_sperm['Treatment']=='Control', 'Quality'])
# mean_sperm_pest = np.mean(df_sperm.loc[df_sperm['Treatment']=='Pesticide','Quality'])
# # Print means
# print('Mean control: ', mean_sperm_control, 'mg')
# print('Mean pesticide: ', mean_sperm_pest, 'mg')
#
# # Get the bootstrap samples
# # Draw 100 000 bootstrap reps for both.
# bs_reps_control = draw_bs_reps(
#         df_sperm.loc[df_sperm['Treatment']=='Control', 'Quality'].dropna(), np.mean, size=100000)
# bs_reps_pest = draw_bs_reps(
#         df_sperm.loc[df_sperm['Treatment']=='Pesticide', 'Quality'].dropna(), np.mean, size=100000)
#
# conf_int_control = np.percentile(bs_reps_control, [2.5, 97.5])
# conf_int_pest = np.percentile(bs_reps_pest, [2.5, 97.5])
# print('Confidence interval for sperm of control group:', conf_int_control)
# print('Confidence interval for sperm of pesticide group:', conf_int_pest)
#
#
# # Draw 100,000 bootstrap reps for both for median
# bs_reps_control = draw_bs_reps(
#         df_sperm.loc[df_sperm['Treatment']=='Control', 'Quality'].dropna(),
#         np.median,size=100000)
# bs_reps_pest = draw_bs_reps(
#         df_sperm.loc[df_sperm['Treatment']=='Pesticide', 'Quality'].dropna(),
#         np.median, size=100000)
#
# # Compute and print confidence interval for median
# conf_int_control = np.percentile(bs_reps_control, [2.5, 97.5])
# conf_int_pest = np.percentile(bs_reps_pest, [2.5, 97.5])
#
# print('Confidence interval for median sperm of control group:', conf_int_control)
# print('Confidence interval for median sperm of pesticide group:', conf_int_pest)

############
# Exercise 4.3: Monte Carlo simulation of transcriptional pausing

def ecdf(data):
    """ Computes the ECDF of input data"""
    return np.sort(data), np.arange(1,len(data) + 1) / len(data)

# a)
# We use @Numba to speed up the calculations
@numba.jit(nopython=True)
def backtrack_steps():
    """
    Computes the number of steps it takes for a random walker to get from x = 0 to x = 1.
    """
    # Initialize the position
    x = 0
    # Initialize the number of steps
    n_steps = 0
    #
    # Walk until we get to positive 1
    while x < 1:
        x += 2*np.random.randint(0, 2) - 1
        n_steps += 1

    return n_steps

# b) Generate 10 000 backtracks

# Initialize the total time
tau = 0.5 # seconds

# Number of samples
n_samples = 10000

# Define empty array of times
t_bt = np.empty(n_samples)

# Generate the samples
for i in range(n_samples):
    t_bt[i] = backtrack_steps()

# Convert to seconds
t_bt *= tau

# c) Plot histogram of backtrack times
fig6 = plt.figure(6)
_ = plt.hist(t_bt, bins=100, normed=True)
plt.xlabel('time (s)')
plt.ylabel('PDF')
fig6.show()

# d) Plot ECDF of samples
x, y = ecdf(t_bt)

fig7 = plt.figure(7)
plt.semilogx(x, y, marker='.', linestyle='none', alpha=0.5)
plt.xlabel('time(s)')
plt.ylabel('ECDF')
plt.margins(0.02)
fig7.show()

# e)
# Plot the CCDF
fig8 = plt.figure(8)
plt.loglog(x, 1 - y, '.')
# Plot the asymptotic power law
t_smooth = np.logspace(0.5, 8, 100)
plt.loglog(t_smooth, 1/np.sqrt(t_smooth))

# Label axes
plt.xlabel('time (s)')
plt.ylabel('CCDF')
fig8.show()

reason: import numpy,import numba


text:
import os
from copy import deepcopy
from multiprocessing import Pool, cpu_count

import networkx as nx
import pandas as pd

from analysis.statics import YEARS
from analysis.utils import (get_basic_parser, get_country,
                            get_crossreference_path, get_node_and_edge_files,
                            get_preprocessed_graph_files,
                            get_preprocessed_graph_path)

def get_networkx_graph_components(basepath, nodefile, edgefile):
    cols = ["level", "type", "document_type", "heading", "law_name", "tokens_n", "key"]
    dtypes = dict(
        level=int,
        type=str,
        document_type=str,
        heading=str,
        law_name=str,
        tokens_n=int,
        key=str,
    )
    nodes = pd.read_csv(
        f"{basepath}/{nodefile}",
        low_memory=True,
        usecols=cols,
        dtype=dtypes,
        skiprows=[1],
    ).set_index("key")
    edges = pd.read_csv(f"{basepath}/{edgefile}", low_memory=True).query(
        "u != 'root' and v != 'root'"
    )
    edges = edges.join(nodes[["document_type"]], on="u").rename(
        dict(document_type="u_document_type"), axis=1
    )
    edges = edges.join(nodes[["document_type"]], on="v").rename(
        dict(document_type="v_document_type"), axis=1
    )
    G = nx.MultiDiGraph()
    G.add_nodes_from(nodes.index)
    G.add_edges_from(
        [
            (u, v, dict(edge_type=edge_type))
            for u, v, edge_type in zip(edges.u, edges.v, edges.edge_type)
        ]
    )
    assert len(edges) == G.number_of_edges()
    assert len(nodes) == G.number_of_nodes()
    return (nodes, edges, G)

def generate_individual_profiles(
    country,
    year,
    years,
    crossreference_path,
    preprocessed_graph_path,
    preprocessed_graph_files,
    nodefiles,
    edgefiles,
):
    print(f"Starting {country}, {year}...")
    qG = nx.read_gpickle(
        f"{preprocessed_graph_path}/{preprocessed_graph_files[years.index(year)]}"
    )
    qG_nodes = pd.DataFrame(
        [x for n, x in qG.nodes(data=True) if x.get("key") is not None]
    ).set_index("key")[
        ["type", "document_type", "heading", "law_name", "tokens_n", "tokens_unique"]
    ]

    nf = nodefiles[years.index(year)]
    ef = edgefiles[years.index(year)]
    nodes, edges, G = get_networkx_graph_components(crossreference_path, nf, ef)

    hG = G
    edges_to_remove = [
        (u, v, k)
        for u, v, k, d in G.edges(data="edge_type", keys=True)
        if d != "containment"
    ]
    hG.remove_edges_from(edges_to_remove)
    hG_leaves = {n for n in hG.nodes() if hG.out_degree(n) == 0}
    G = None

    max_leaf_tokens = []
    items_n = []
    seqitems_n = []
    subseqitems_n = []
    internal_references_n = []
    outgoing_references_n = []
    incoming_references_n = []
    outgoing_references_diversity_n = []
    incoming_references_diversity_n = []
    for idx, n in enumerate(qG_nodes.index):
        descendants = nx.descendants(hG, n)
        descendants_leaves = [v for v in descendants if v in hG_leaves]
        desc_nodes = nodes.loc[descendants]
        type_counts = desc_nodes.value_counts("type")
        n_items = type_counts.get("item", 0)
        n_seqitems = type_counts.get("seqitem", 0)
        n_subseqitems = type_counts.get("subseqitem", 0)
        max_tokens_in_leaf = desc_nodes.loc[descendants_leaves].tokens_n.max()
        internal_refs = len(
            edges.query(
                "u in @descendants and v in @descendants and edge_type == 'reference'"
            ).index
        )
        outgoing_refs = qG.out_degree(n)  # weighted out-degree
        incoming_refs = qG.in_degree(n)  # weighted in-degree
        outgoing_refs_diversity = len(set(qG.successors(n)))  # binary out-degree
        incoming_refs_diversity = len(set(qG.predecessors(n)))  # binary in-degree
        items_n.append(n_items)
        seqitems_n.append(n_seqitems)
        subseqitems_n.append(n_subseqitems)
        max_leaf_tokens.append(max_tokens_in_leaf)
        internal_references_n.append(internal_refs)
        outgoing_references_n.append(outgoing_refs)
        incoming_references_n.append(incoming_refs)
        outgoing_references_diversity_n.append(outgoing_refs_diversity)
        incoming_references_diversity_n.append(incoming_refs_diversity)

    qG_nodes["max_leaf_tokens"] = max_leaf_tokens
    qG_nodes["items_n"] = items_n
    qG_nodes["seqitems_n"] = seqitems_n
    qG_nodes["subseqitems_n"] = subseqitems_n
    qG_nodes["self_loops_n"] = internal_references_n
    qG_nodes["reliance_n"] = outgoing_references_n
    qG_nodes["responsibility_n"] = incoming_references_n
    qG_nodes["reliance_diversity_n"] = outgoing_references_diversity_n
    qG_nodes["responsibility_diversity_n"] = incoming_references_diversity_n
    qG_nodes["tokens_n"] = qG_nodes["tokens_n"].fillna(0).astype(int)
    qG_nodes["tokens_unique"] = qG_nodes["tokens_unique"].fillna(0).astype(int)
    qG_nodes["max_leaf_tokens"] = qG_nodes["max_leaf_tokens"].fillna(0).astype(int)

    qG_nodes.to_csv(
        f"../results/chapter-profiles/{year}-chapter-profiles-{country}.csv"
    )

if __name__ == "__main__":
    parser = get_basic_parser()
    parser.add_argument("--years", "-y", nargs="+", type=int, default=YEARS)
    args = parser.parse_args()
    country = get_country(args)
    years = args.years
    crossreference_path = get_crossreference_path(country)
    nodefiles, edgefiles = get_node_and_edge_files(crossreference_path, YEARS)
    preprocessed_graph_path = get_preprocessed_graph_path(country)
    preprocessed_graph_files = get_preprocessed_graph_files(
        preprocessed_graph_path, YEARS
    )
    result_path = "../results/chapter-profiles"
    if not os.path.exists(result_path):
        os.makedirs(result_path)
    if country == "de":
        with Pool(cpu_count() - 4) as p:
            p.starmap(
                generate_individual_profiles,
                [
                    (
                        country,
                        year,
                        YEARS,
                        crossreference_path,
                        preprocessed_graph_path,
                        preprocessed_graph_files,
                        nodefiles,
                        edgefiles,
                    )
                    for year in years
                ],
            )
    else:  # 'us'
        for year in years:
            generate_individual_profiles(
                country,
                year,
                YEARS,
                crossreference_path,
                preprocessed_graph_path,
                preprocessed_graph_files,
                nodefiles,
                edgefiles,
            )

max_stars_repo_name: QuantLaw/Measuring-Law-Over-Time

max_issues_repo_name: QuantLaw/Measuring-Law-Over-Time

max_forks_repo_name: QuantLaw/Measuring-Law-Over-Time

reason: import networkx


text:
import numpy as np
import subprocess
import glob as gb
import shutil
import time
import os

 # Function changing input mesh inside the model.inp file
def change_model_inp (input_file, change_mesh):

    # Input mesh path
    mesh_path = "*INCLUDE, INPUT="+ change_mesh + "\n"

    # Read model.np
    with open((input_file + ".inp"),"r") as model_inp:

    # Read all the lines of model.inp
        list_of_lines = model_inp.readlines()
        list_of_lines[0] = mesh_path

    with open((input_file + ".inp"),"w") as model_inp:
    	model_inp.writelines(list_of_lines)

# Function which starts ccx solver and copy results
def run_calculix(input_file, mesh_files_inp):

    name_fild_dir =  mesh_files_inp.replace('.inp','')
    new_name_fil_dir =  name_fild_dir.replace('Mesh/','')

    # Run the CalculiX solver with ccx_2.17_MT commnad
    subprocess.run(["ccx_2.17_MT", input_file])
    os.mkdir("Results/" + new_name_fil_dir)

    #Variables for copying/saving files
    frd_result_file = input_file + ".frd"
    dat_result_file = input_file + ".dat"

    frd_new_name = new_name_fil_dir + ".frd"
    dat_mew_name = new_name_fil_dir + ".dat"

    frd_dir = "Results/" + new_name_fil_dir + "/" + frd_new_name
    dat_dir = "Results/" + new_name_fil_dir + "/" + dat_mew_name

    # Copy the results files
    shutil.copyfile(frd_result_file, frd_dir)
    shutil.copyfile(dat_result_file, dat_dir)

os.mkdir("Results")
# Check the meshes in Mesh folder
mesh_file_mames = gb.glob("Mesh/*.inp")

# Name of the input inp file for CalculiX
ccx_input_file = ("model")

for mesh_file_mame in mesh_file_mames:

    change_model_inp(ccx_input_file, mesh_file_mame)
    run_calculix(ccx_input_file, mesh_file_mame)

print ("All the simualations are done!")

max_stars_repo_name: dantelimac/CoFEA

max_issues_repo_name: dantelimac/CoFEA

max_forks_repo_name: dantelimac/CoFEA


text:
import cv2
import numpy as np
from mtcnn.mtcnn import MTCNN
from imutils import build_montages
from imutils import paths

detector = MTCNN()

img = cv2.imread("input & output/crowd.jpg")
location = detector.detect_faces(img)
count = 1
location_new = []
for loc in location:
    if loc['confidence'] >= 0.99:
        location_new.append(loc)
print(type(location_new))
location_new.sort(key=lambda item: item.get("confidence"), reverse=True)
print(location_new)

if len(location_new) > 0:
    # if location['confidence']>=0.99:
    for face in location_new:
        try:
            x, y, width, height = face['box']
            # print(x,y,width,height)
            x2, y2 = x + width, y + height
            # print(x2,y2)
            cv2.rectangle(img, (x, y), (x2, y2), (0, 0, 255), 1)
            sub_face = img[y:y + height, x:x + width]
            # cv2.imshow("crop_image", sub_face)
            # cv2.waitKey(0)
            FaceFileName = "cluster_faces/face_" + str(count) + ".jpg"
            count += 1
            cv2.imwrite(FaceFileName, sub_face)

        except Exception as e:
            print(e)

# Creating montage of face images------------------------------------------

image_folder = './cluster_faces'
image_path = list(paths.list_images(image_folder))
imgs = []
for img_path in image_path:
    img = cv2.imread(img_path)
    imgs.append(img)

montages = build_montages(imgs, (75, 75), (12, 11))

for montage in montages:
    cv2.imshow("montage_output", montage)
    cv2.imwrite("output_montage_new.jpg", montage)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

#Finding blurness of that face images------------------------------

image_new_folder = './cluster_faces'
def variance_of_laplacian(image):
    # compute the Laplacian of the image and then return the focus
    # measure, which is simply the variance of the Laplacian
    return cv2.Laplacian(image, cv2.CV_64F).var()

for imagePath in paths.list_images(image_new_folder):
    image_new = cv2.imread(imagePath)
    gray = cv2.cvtColor(image_new, cv2.COLOR_BGR2GRAY)
    fm = variance_of_laplacian(gray)
    # fm = np.array([fm])
    # fm = np.sort(fm)
    print(fm)
    text = "Yes"
    if fm < 3000:
        text = "No"
    cv2.putText(image_new, "{}: {:.2f}".format(text, fm), (2, 15),
                cv2.FONT_HERSHEY_PLAIN, 0.5, (0, 255, 0), 1)
    # cv2.imshow("Image", image_new)
    key = cv2.waitKey(0)
    dir_loc = './blur_cluster_face/face_' + str(count) + '.jpg'
    count += 1
    cv2.imwrite(dir_loc, image_new)

#creating montages of blur calculated images---------------------
blur_image_folder = './blur_cluster_face'
blur_image_path = list(paths.list_images(blur_image_folder))
blur_imgs = []
for blur_img_path in blur_image_path:
    blur_img = cv2.imread(blur_img_path)
    blur_imgs.append(blur_img)

montages = build_montages(blur_imgs, (100, 100), (12, 11))

for montage in montages:
    cv2.imshow("blur_montage_output", montage)
    cv2.imwrite("output_montage_new.jpg", montage)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

cv2.waitKey(0)
cv2.destroyAllWindows()
# cv2.imwrite("input & output/crowd_output.jpg",img)
# print("The Image was successfully saved")