text:
# Licensed under a 3-clause BSD style license - see LICENSE.rst
from typing import List, Tuple
from copy import deepcopy
import warnings
import urllib.request
from io import BytesIO
from pathlib import Path
import numpy as np
from PIL import Image
from astropy.utils.exceptions import AstropyWarning
from astropy.io.fits.verify import VerifyWarning
from astropy.coordinates import SkyCoord
from astropy.io import fits
from ..utils.healpix import healpix_pixel_corners
from .io import tile_default_url, tile_default_path

__all__ = [
    'HipsTileMeta',
    'HipsTile',
]

__doctest_skip__ = [
    'HipsTile',
    'HipsTileMeta',
]

# TODO: this could be a dict. Would that be better?
def compute_image_shape(width: int, height: int, fmt: str) -> tuple:
    """Compute numpy array shape for a given image.

    The output image shape is 2-dim for grayscale, and 3-dim for color images:

    * ``shape = (height, width)`` for FITS images with one grayscale channel
    * ``shape = (height, width, 3)`` for JPG images with three RGB channels
    * ``shape = (height, width, 4)`` for PNG images with four RGBA channels

    Parameters
    ----------
    width, height : int
        Width and height of the image
    fmt : {'fits', 'jpg', 'png'}
        Image format

    Returns
    -------
    shape : tuple
        Numpy array shape
    """
    if fmt == 'fits':
        return height, width
    elif fmt == 'jpg':
        return height, width, 3
    elif fmt == 'png':
        return height, width, 4
    else:
        raise ValueError(f'Invalid format: {fmt}')

class HipsTileMeta:
    """HiPS tile metadata.

    Parameters
    ----------
    order : int
        HEALPix order
    ipix : int
        HEALPix pixel number
    file_format : {'fits', 'jpg', 'png'}
        File format
    frame : {'icrs', 'galactic', 'ecliptic'}
        Sky coordinate frame
    width : int
        Tile width (tiles always have width = height)

    Examples
    --------
    >>> from hips import HipsTileMeta
    >>> tile_meta = HipsTileMeta(order=3, ipix=450, file_format='fits')
    >>> tile_meta
    HipsTileMeta(order=3, ipix=450, file_format='fits', frame='icrs', width=512)
    >>> tile_meta.skycoord_corners
    <SkyCoord (ICRS): (ra, dec) in deg
    [( 264.375, -24.62431835), ( 258.75 , -30.        ),
    ( 264.375, -35.68533471), ( 270.   , -30.        )]>
    >>> tile_meta.tile_default_url
    'Norder3/Dir0/Npix450.fits'
    >>> tile_meta.tile_default_path.as_posix()
    'Norder3/Dir0/Npix450.fits'
    """

    def __init__(self, order: int, ipix: int, file_format: str,
                 frame: str = 'icrs', width: int = 512) -> None:
        self.order = order
        self.ipix = ipix
        self.file_format = file_format
        self.frame = frame
        self.width = width

    def __repr__(self):
        return (
            'HipsTileMeta('
            f'order={self.order}, ipix={self.ipix}, '
            f'file_format={self.file_format!r}, frame={self.frame!r}, '
            f'width={self.width}'
            ')'
        )

    def __eq__(self, other: 'HipsTileMeta') -> bool:
        return (
            self.order == other.order and
            self.ipix == other.ipix and
            self.file_format == other.file_format and
            self.frame == other.frame and
            self.width == other.width
        )

    def copy(self):
        """An independent copy."""
        return deepcopy(self)

    @property
    def skycoord_corners(self) -> SkyCoord:
        """Tile corner sky coordinates (`~astropy.coordinates.SkyCoord`)."""
        return healpix_pixel_corners(self.order, self.ipix, self.frame)

    @property
    def tile_default_url(self) -> str:
        """Tile relative URL (str)."""
        return tile_default_url(self.order, self.ipix, self.file_format)

    @property
    def tile_default_path(self) -> Path:
        """Tile relative filename path (`~pathlib.Path`)."""
        return tile_default_path(self.order, self.ipix, self.file_format)

class HipsTile:
    """HiPS tile container.

    This class provides methods for fetching, reading, and writing a HiPS tile.

    .. note::

        In HiPS, the pixel data is flipped in the y direction for
        ``jpg`` and ``png`` format with respect to FITS.
        In this package, we handle this by flipping ``jpg`` and ``png``
        data to match the ``fits`` orientation, at the I/O boundary,
        i.e. in `from_numpy` and `to_numpy`.

    Parameters
    ----------
    meta : `~hips.HipsTileMeta`
        Metadata of HiPS tile
    raw_data : `bytes`
        Raw data (copy of bytes from file)

    Examples
    --------

    Fetch a HiPS tile:

    >>> from hips import HipsTile, HipsTileMeta
    >>> meta = HipsTileMeta(order=6, ipix=30889, file_format='fits')
    >>> url = 'http://alasky.unistra.fr/2MASS/H/Norder6/Dir30000/Npix30889.fits'
    >>> tile = HipsTile.fetch(meta, url)

    The tile pixel data is available as a Numpy array:

    >>> type(tile.data)
    numpy.ndarray
    >>> tile.data.shape
    (512, 512)
    >>> tile.data.dtype.name
    int16
    """

    def __init__(self, meta: HipsTileMeta, raw_data: bytes) -> None:
        self.meta = meta
        self.raw_data = raw_data
        self._data = None

    def __eq__(self, other: 'HipsTile') -> bool:
        return (
            self.meta == other.meta and
            self.raw_data == other.raw_data
        )

    @classmethod
    def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> 'HipsTile':
        """Create a tile from given pixel data.

        Parameters
        ----------
        meta : `~hips.HipsTileMeta`
            Metadata of HiPS tile
        data : `~numpy.ndarray`
            Tile pixel data

        Returns
        -------
        tile : `~hips.HipsTile`
            HiPS tile object in the format requested in ``meta``.
        """
        fmt = meta.file_format
        bio = BytesIO()

        if fmt == 'fits':
            hdu = fits.PrimaryHDU(data)
            hdu.writeto(bio)
        elif fmt == 'jpg':
            # Flip tile to be consistent with FITS orientation
            data = np.flipud(data)
            image = Image.fromarray(data)
            image.save(bio, format='jpeg')
        elif fmt == 'png':
            # Flip tile to be consistent with FITS orientation
            data = np.flipud(data)
            image = Image.fromarray(data)
            image.save(bio, format='png')
        else:
            raise ValueError(f'Tile file format not supported: {fmt}. '
                             'Supported formats: fits, jpg, png')

        bio.seek(0)
        raw_data = bio.read()

        return cls(meta, raw_data)

    @property
    def children(self) -> List['HipsTile']:
        """Create four children tiles from parent tile."""
        w = self.data.shape[0] // 2
        data = [
            self.data[w: w * 2, 0: w],
            self.data[0: w, 0: w],
            self.data[w: w * 2, w: w * 2],
            self.data[0: w, w: w * 2]
        ]

        tiles = []
        for idx in range(4):
            meta = HipsTileMeta(
                self.meta.order + 1,
                self.meta.ipix * 4 + idx,
                self.meta.file_format,
                self.meta.frame,
                len(data[0])
            )
            tile = self.from_numpy(meta, data[idx])
            tiles.append(tile)

        return tiles

    @property
    def data(self) -> np.ndarray:
        """Tile pixel data (`~numpy.ndarray`).

        This is a cached property, it will only be computed once.

        See the `to_numpy` function.
        """
        if self._data is None:
            self._data = self.to_numpy(
                self.raw_data,
                self.meta.file_format,
            )

        return self._data

    @staticmethod
    def to_numpy(raw_data: bytes, fmt: str) -> np.ndarray:
        """Convert raw image bytes to Numpy array pixel data.

        Parameters
        ----------
        raw_data : bytes
            Raw image bytes (usually read from file or fetched from URL)
        fmt : {'fits', 'jpg', 'png'}
            File format

        Returns
        -------
        data : `~numpy.ndarray`
            Pixel data as a numpy array
        """
        bio = BytesIO(raw_data)

        if fmt == 'fits':
            # At the moment CDS is serving FITS tiles in non-standard FITS files
            # https://github.com/hipspy/hips/issues/42
            # The following warnings handling is supposed to suppress these warnings
            # (but hopefully still surface other issues in a useful way).
            with warnings.catch_warnings():
                warnings.simplefilter('ignore', AstropyWarning)
                warnings.simplefilter('ignore', VerifyWarning)
                with fits.open(bio) as hdu_list:
                    data = hdu_list[0].data
        elif fmt in {'jpg', 'png'}:
            with Image.open(bio) as image:
                data = np.array(image)
            # Flip tile to be consistent with FITS orientation
            data = np.flipud(data)
        else:
            raise ValueError(f'Tile file format not supported: {fmt}. '
                             'Supported formats: fits, jpg, png')

        return data

    @classmethod
    def read(cls, meta: HipsTileMeta, filename: str = None) -> 'HipsTile':
        """Read HiPS tile data from a directory and load into memory (`~hips.HipsTile`).

        Parameters
        ----------
        meta : `~hips.HipsTileMeta`
            Metadata of HiPS tile
        filename : str
            Filename
        """
        raw_data = Path(filename).read_bytes()
        return cls(meta, raw_data)

    @classmethod
    def fetch(cls, meta: HipsTileMeta, url: str) -> 'HipsTile':
        """Fetch HiPS tile and load into memory (`~hips.HipsTile`).

        Parameters
        ----------
        meta : `~hips.HipsTileMeta`
            Metadata of HiPS tile
        url : str
            URL containing HiPS tile
        """
        with urllib.request.urlopen(url) as response:
            raw_data = response.read()

        return cls(meta, raw_data)

    def write(self, filename: str) -> None:
        """Write to file.

        Parameters
        ----------
        filename : str
            Filename
        """
        Path(filename).write_bytes(self.raw_data)

max_stars_repo_name: adonath/hips

max_issues_repo_name: adonath/hips

max_forks_repo_name: adonath/hips

reason: import numpy,from astropy


text:
import os
import sys
from datetime import datetime

import numpy as np
import PIL.Image
import torch.optim as optim
import torch
from torch.nn import functional as F
from loss import MSE_with_regulariztion
from merge_utils import create_layer_weight, calc_loss_one_model, torch_img_preprocess, torch_vid_preprocess

sys.path.append('../icnn_torch')
from utils import img_deprocess, img_preprocess, normalise_img, \
    vid_deprocess, normalise_vid, vid_preprocess, clip_extreme_value, get_cnn_features, create_feature_masks,\
    save_video, save_gif, gaussian_blur, clip_small_norm_value

def reconstruct_stim(features_list, net_list, #[video, image] if video model was used

                     img_mean_list,
                     img_std_list,
                     norm_list,
                     bgr_list,
                     prep_list,
                     weight_list=None,
                     initial_input=None,
                     input_size=(16, 224, 224, 3),
                     feature_masks_list=None,
                     layer_weight_list=None, channel=None, mask=None,
                     opt_name='SGD',

                     lr_start=0.002, lr_end=1e-13,
                      momentum_start=0.9, momentum_end=0.9,

                      decay_start=0.01, decay_end=0.01,
                      grad_normalize = False,
                      image_jitter=False, jitter_size=4,
                      image_blur=False, sigma_start=2, sigma_end=0.5,

                      clip_extreme=False, clip_extreme_every=4, e_pct_start=1, e_pct_end=1,
                      clip_small_norm = False,

                     loss_type='l2', iter_n=200,  save_intermediate=False,
                     save_intermediate_every=1, save_intermediate_path=None,
                     disp_every=1,
                     ):

    loss_fun = torch.nn.MSELoss(reduction='sum')

    if save_intermediate:
        if save_intermediate_path is None:
            save_intermediate_path = os.path.join('./recon_stim_combine_snapshots' + datetime.now().strftime('%Y%m%dT%H%M%S'))
            os.makedirs(save_intermediate_path, exist_ok=True)

    if weight_list is None:
        weight_list = [1.0 /len(net_list) for i in range(len(net_list))]

    if layer_weight_list is None:
        layer_weight_list = [create_layer_weight(feat_dict) for feat_dict in features_list]

    if initial_input is None:
        initial_input = np.random.randint(0,256, (input_size)).astype(np.float32)

    #save if save intermediate is True
    if save_intermediate:
        if len(input_size) == 3:
            #image
            save_name = 'initial_image.jpg'
            if bgr_list[0]:
                PIL.Image.fromarray(np.uint8(initial_input[...,[2,1,0]])).save(os.path.join(save_intermediate_path, save_name))
            else:
                PIL.Image.fromarray(np.uint8(initial_input)).save(os.path.join(save_intermediate_path, save_name))
        elif len(input_size) == 4:
            # video
            # if you install cv2 and ffmpeg, you can use save_video function which save preferred video as video format
            save_name = 'initial_video.avi'
            save_video(initial_input, save_name, save_intermediate_path, bgr_list[0])

            #save_name = 'initial_video.gif'
            #save_gif(initial_input, save_name, save_intermediate_path, bgr_list[0],
            #         fr_rate=150)

        else:
            print('Input size is not appropriate for save')
            assert len(input_size) not in [3,4]

    #create feature mask
    if feature_masks_list is None:
        feature_masks_list = []
        for features in features_list:
            feat_mask = create_feature_masks(features)
            feature_masks_list.append(feat_mask)

    loss_list = np.zeros(iter_n, dtype='float32')

    input_ = initial_input.copy().astype(np.float32)

    for t in range(iter_n):
        # parameters
        lr = lr_start + t * (lr_end - lr_start) / iter_n
        momentum = momentum_start + t * (momentum_end - momentum_start) / iter_n
        decay = decay_start + t * (decay_end - decay_start) / iter_n
        sigma = sigma_start + t * (sigma_end - sigma_start) / iter_n
        # optimize based on the first element
        input = torch.tensor(input_, requires_grad=True)

        if opt_name == 'Adam':
            op = optim.Adam([input], lr = lr)
        elif opt_name == 'SGD':
            op = optim.SGD([input], lr=lr, momentum=momentum)
            #op = optim.SGD([input], lr=lr)
        elif opt_name == 'Adadelta':
            op = optim.Adadelta([input],lr = lr)
        elif opt_name == 'Adagrad':
            op = optim.Adagrad([input], lr = lr)
        elif opt_name == 'AdamW':
            op = optim.AdamW([input], lr = lr)
        elif opt_name == 'SparseAdam':
            op = optim.SparseAdam([input], lr = lr)
        elif opt_name == 'Adamax':
            op = optim.Adamax([input], lr = lr)
        elif opt_name == 'ASGD':
            op = optim.ASGD([input], lr = lr)
        elif opt_name == 'RMSprop':
            op = optim.RMSprop([input], lr = lr)
        elif opt_name == 'Rprop':
            op = optim.Rprop([input], lr = lr)

        op.zero_grad()
        net_loss_list = []
        # preprocess input for each network
        feat_inp_list = []
        for i in range(len(net_list)):
            net = net_list[i]
            features = features_list[i]
            target_layer_list = features.keys()
            img_mean = torch.tensor(img_mean_list[i])
            img_std = torch.tensor(img_std_list[i])
            norm = norm_list[i]
            bgr = bgr_list[i]
            prep_l = prep_list[i]
            feature_masks = feature_masks_list[i]
            feature_weight = layer_weight_list[i]
            net_weight = weight_list[i]
            inp = input
            for prep in prep_l:
                if prep == 'mean':
                    inp = inp.mean(0)

                elif prep == "resize(224,224)":
                    #upsample
                    inp = inp

            # preprocess
            if len(inp.shape) == 3:
                inp = torch_img_preprocess(inp, img_mean, img_std, norm)
            else:
                inp = torch_vid_preprocess(inp, img_mean, img_std, norm)

            for prep in prep_l:
                if prep == "resize(56,56)":
                    inp = F.interpolate(inp, (56,56),mode='bilinear', align_corners =True)

            inp = inp.view([1]  + list(inp.shape))
            #obtain features
            feat_inp = get_cnn_features(net, inp, target_layer_list)
            feat_inp_list.append(feat_inp)
            #calc loss
            loss = calc_loss_one_model(net, feat_inp, features, feature_masks, feature_weight, loss_fun, w_model=net_weight)
            net_loss_list.append(loss)

        op.step()
        input_ = input.detach().numpy()
        err = np.sum(net_loss_list)
        loss_list[t] = err

        # clip pixels with extreme value
        if clip_extreme and (t+1) % clip_extreme_every == 0:
            e_pct = e_pct_start + t * (e_pct_end - e_pct_start) / iter_n
            input = clip_extreme_value(input, e_pct)

        # clip pixels with small norm
        if clip_small_norm and (t+1) % clip_small_norm_every == 0:
            n_pct = n_pct_start + t * (n_pct_end - n_pct_start) / iter_n
            input = clip_small_norm_value(input, n_pct)

        # unshift
        if image_jitter:
            input = np.roll(np.roll(input, -ox, -1), -oy, -2)

        # L_2 decay
        input = (1-decay) * input

        # gaussian blur
        if image_blur:
            if len(input_size) == 3:
                input = gaussian_blur(input, sigma)
            else:
                for i in range(input.shape[1]):
                    input[:, i] = gaussian_blur(input[:, i], sigma)

        # disp info
        if (t+1) % disp_every == 0:
            print('iter=%d; err=%g;' % (t+1, err))

        # save image
        if save_intermediate and ((t+1) % save_intermediate_every == 0):
            if len(input_size) == 3:
                save_name = '%05d.jpg' % (t+1)
                PIL.Image.fromarray(normalise_img(clip_extreme_value(input_, pct=0.04))).save(
                    os.path.join(save_intermediate_path, save_name))
            else:
                save_stim = input_
                # if you install cv2 and ffmpeg, you can use save_video function which save preferred video as video format
                save_name = '%05d.avi' % (t + 1)
                save_video(normalise_vid(clip_extreme_value(save_stim, pct=0.04)), save_name,
                           save_intermediate_path, bgr_list[0], fr_rate=30)
                #save_name = '%05d.gif' % (t + 1)
                #save_gif(normalise_vid(clip_extreme_value(save_stim, pct=0.04)), save_name,
                #         save_intermediate_path,
                #         bgr_list[0], fr_rate=150)
    # return img
    if len(input_size) == 3:
        return input_, loss_list
    else:
        return input_, loss_list

ext: py

reason: import numpy


text:
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import gc
import os
import random
import shutil
import numpy as np

import torch
import tqdm
from examples.textless_nlp.gslm.speech2unit.pretrained.cpc_feature_reader import (
    CpcFeatureReader,
)
from examples.textless_nlp.gslm.speech2unit.pretrained.hubert_feature_reader import (
    HubertFeatureReader,
)
from examples.textless_nlp.gslm.speech2unit.pretrained.logmel_feature_reader import (
    LogMelFeatureReader,
)
from examples.textless_nlp.gslm.speech2unit.pretrained.w2v2_feature_reader import (
    Wav2VecFeatureReader,
)

def get_feature_reader(feature_type):
    if feature_type == "logmel":
        return LogMelFeatureReader
    elif feature_type == "hubert":
        return HubertFeatureReader
    elif feature_type == "w2v2":
        return Wav2VecFeatureReader
    elif feature_type == "cpc":
        return CpcFeatureReader
    else:
        raise NotImplementedError(f"{feature_type} is not supported.")

def get_feature_iterator(
    feature_type, checkpoint_path, layer, manifest_path, sample_pct
):
    feature_reader_cls = get_feature_reader(feature_type)
    with open(manifest_path, "r") as fp:
        lines = fp.read().split("\n")
        root = lines.pop(0).strip()
        file_path_list = [
            os.path.join(root, line.split("\t")[0])
            for line in lines
            if len(line) > 0
        ]
        if sample_pct < 1.0:
            file_path_list = random.sample(
                file_path_list, int(sample_pct * len(file_path_list))
            )
        num_files = len(file_path_list)
        reader = feature_reader_cls(
            checkpoint_path=checkpoint_path, layer=layer
        )

        def iterate():
            for file_path in file_path_list:
                feats = reader.get_feats(file_path)
                yield feats.cpu().numpy()

    return iterate, num_files

def get_features(
    feature_type, checkpoint_path, layer, manifest_path, sample_pct, flatten
):
    generator, num_files = get_feature_iterator(
        feature_type=feature_type,
        checkpoint_path=checkpoint_path,
        layer=layer,
        manifest_path=manifest_path,
        sample_pct=sample_pct,
    )
    iterator = generator()

    features_list = []
    for features in tqdm.tqdm(iterator, total=num_files):
        features_list.append(features)

    # Explicit clean up
    del iterator
    del generator
    gc.collect()
    torch.cuda.empty_cache()

    if flatten:
        return np.concatenate(features_list)

    return features_list

def get_and_dump_features(
    feature_type,
    checkpoint_path,
    layer,
    manifest_path,
    sample_pct,
    flatten,
    out_features_path,
):
    # Feature extraction
    features_batch = get_features(
        feature_type=feature_type,
        checkpoint_path=checkpoint_path,
        layer=layer,
        manifest_path=manifest_path,
        sample_pct=sample_pct,
        flatten=flatten,
    )

    # Save features
    out_dir_path = os.path.dirname(out_features_path)
    os.makedirs(out_dir_path, exist_ok=True)
    shutil.copyfile(
        manifest_path,
        os.path.join(out_dir_path, os.path.basename(manifest_path)),
    )
    np.save(out_features_path, features_batch)

    return features_batch

max_stars_repo_name: Shiguang-Guo/fairseq

max_issues_repo_name: Shiguang-Guo/fairseq

max_forks_repo_name: Shiguang-Guo/fairseq


text:
import copy
import pickle
from multiprocessing.pool import Pool
from time import time
from contextlib import suppress

import numpy as np
import pandas as pd
import os

import logging

logger = logging.getLogger(__name__)

def apply_pipeline(text, pipeline):
    for stage in pipeline:
        text = stage(text)
    return text

class Corpus:
    def __init__(self, file_path, preprocessing_pipeline, load_from_cache=True,
                 id_column="id", text_column="text", n_jobs=8):
        logger.info(f"Start preprocessing pipeline "
                    f"\"{_get_pipeline_name(preprocessing_pipeline)}\" "
                    f"for file {file_path}.")
        self.pipeline = copy.deepcopy(preprocessing_pipeline)
        cache_file_path = self._get_cache_file_path(file_path)
        if load_from_cache:
            if self._load_from_file(cache_file_path, id_column, text_column):
                logger.info(f"Loaded cached preprocessed corpus from {cache_file_path}")
                return
        df = pd.read_csv(file_path).fillna("")
        self.ids = df[id_column]
        self.data = df[text_column]
        pool = None
        for stage in self.pipeline:
            logger.info(f"Start stage \"{stage.name}\"")
            start_time = time()
            with suppress(AttributeError):
                stage.fit_corpus(self.data)
            if n_jobs > 1:
                if pool is None:
                    pool = Pool(n_jobs)
                self.data = pd.concat(
                    pool.map(ApplyStage(stage), np.array_split(self.data, n_jobs)))
            else:
                self.data = self.data.progress_apply(stage)
            logger.info(f"Finished stage \"{stage.name}\" in "
                        f"{time() - start_time:.2f} seconds")
        if pool is not None:
            pool.close()
        self._save_to_file(cache_file_path)
        self.data = self.data.str.split(" ")
        logger.info(f"Finished preprocessing pipeline. "
                    f"Saved preprocessed corpus to cache file {cache_file_path}")

    def _save_to_file(self, file_path):
        pd.concat([self.ids, self.data], axis=1).to_csv(file_path + ".csv")
        with open(file_path + ".pipeline", "wb") as file:
            pickle.dump(self.pipeline, file)

    def _load_from_file(self, file_path, id_column, text_column):
        data_exists = os.path.isfile(file_path + ".csv")
        pipeline_exists = os.path.isfile(file_path + ".pipeline")
        if not data_exists or not pipeline_exists:
            return False
        df = pd.read_csv(file_path + ".csv")
        self.ids = df[id_column]
        self.data = df[text_column].str.split(" ")
        with open(file_path + ".pipeline", "rb") as file:
            self.pipeline = pickle.load(file)
        return True

    def _get_cache_file_path(self, original_file_path):
        pipeline_name = _get_pipeline_name(self.pipeline)
        root, ext = os.path.splitext(original_file_path)
        return root + "_" + pipeline_name

def _get_pipeline_name(pipeline):
    return "_".join([stage.name for stage in pipeline])

class ApplyStage:
    def __init__(self, stage):
        self.stage = stage

    def __call__(self, data):
        return data.progress_apply(self.stage)

max_stars_repo_name: JNKielmann/Master-Thesis

max_issues_repo_name: JNKielmann/Master-Thesis

max_forks_repo_name: JNKielmann/Master-Thesis


text:
"""
StarGAN v2
Copyright (c) 2020-present NAVER Corp.

This work is licensed under the Creative Commons Attribution-NonCommercial
4.0 International License. To view a copy of this license, visit
http://creativecommons.org/licenses/by-nc/4.0/ or send a letter to
Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
"""

import copy
import math

from munch import Munch
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from core.wing import FAN
from core.utils import load_vgg16

class ResBlk(nn.Module):
    def __init__(self, dim_in, dim_out, actv=nn.LeakyReLU(0.2),
                 normalize=False, downsample=False):
        super().__init__()
        self.actv = actv
        self.normalize = normalize
        self.downsample = downsample
        self.learned_sc = dim_in != dim_out
        self._build_weights(dim_in, dim_out)

    def _build_weights(self, dim_in, dim_out):
        self.conv1 = nn.Conv2d(dim_in, dim_in, 3, 1, 1)
        self.conv2 = nn.Conv2d(dim_in, dim_out, 3, 1, 1)
        if self.normalize:
            self.norm1 = nn.InstanceNorm2d(dim_in, affine=True)
            self.norm2 = nn.InstanceNorm2d(dim_in, affine=True)
        if self.learned_sc:
            self.conv1x1 = nn.Conv2d(dim_in, dim_out, 1, 1, 0, bias=False)

    def _shortcut(self, x):
        if self.learned_sc:
            x = self.conv1x1(x)
        if self.downsample:
            x = F.avg_pool2d(x, 2)
        return x

    def _residual(self, x):
        if self.normalize:
            x = self.norm1(x)
        x = self.actv(x)
        x = self.conv1(x)
        if self.downsample:
            x = F.avg_pool2d(x, 2)
        if self.normalize:
            x = self.norm2(x)
        x = self.actv(x)
        x = self.conv2(x)
        return x

    def forward(self, x):
        x = self._shortcut(x) + self._residual(x)
        return x / math.sqrt(2)  # unit variance

class AdaIN(nn.Module):
    def __init__(self, style_dim, num_features):
        super().__init__()
        self.norm = nn.InstanceNorm2d(num_features, affine=False)
        self.fc = nn.Linear(style_dim, num_features*2)

    def forward(self, x, s):
        h = self.fc(s)
        h = h.view(h.size(0), h.size(1), 1, 1)
        gamma, beta = torch.chunk(h, chunks=2, dim=1)
        return (1 + gamma) * self.norm(x) + beta

class AdainResBlk(nn.Module):
    def __init__(self, dim_in, dim_out, style_dim=64, w_hpf=0,
                 actv=nn.LeakyReLU(0.2), upsample=False):
        super().__init__()
        self.w_hpf = w_hpf
        self.actv = actv
        self.upsample = upsample
        self.learned_sc = dim_in != dim_out
        self._build_weights(dim_in, dim_out, style_dim)

    def _build_weights(self, dim_in, dim_out, style_dim=64):
        self.conv1 = nn.Conv2d(dim_in, dim_out, 3, 1, 1)
        self.conv2 = nn.Conv2d(dim_out, dim_out, 3, 1, 1)
        self.norm1 = AdaIN(style_dim, dim_in)
        self.norm2 = AdaIN(style_dim, dim_out)
        if self.learned_sc:
            self.conv1x1 = nn.Conv2d(dim_in, dim_out, 1, 1, 0, bias=False)

    def _shortcut(self, x):
        if self.upsample:
            x = F.interpolate(x, scale_factor=2, mode='nearest')
        if self.learned_sc:
            x = self.conv1x1(x)
        return x

    def _residual(self, x, s):
        x = self.norm1(x, s)
        x = self.actv(x)
        if self.upsample:
            x = F.interpolate(x, scale_factor=2, mode='nearest')
        x = self.conv1(x)
        x = self.norm2(x, s)
        x = self.actv(x)
        x = self.conv2(x)
        return x

    def forward(self, x, s):
        out = self._residual(x, s)
        if self.w_hpf == 0:
            out = (out + self._shortcut(x)) / math.sqrt(2)
        return out

class AdainResBlkGanilla(nn.Module):
    def __init__(self, dim_in, dim_out, style_dim=64, actv=nn.LeakyReLU(0.2), upsample=True, do_residual=True):
        super().__init__()

        self.actv = actv
        self.upsample = upsample
        self.learned_sc = dim_in != dim_out
        self._build_weights(dim_in, dim_out, style_dim)
        self.do_residual = do_residual
    def _build_weights(self, dim_in, dim_out, style_dim=64):
        self.conv1 = nn.Conv2d(dim_in, dim_out, 3, 1, 1)
        self.conv2 = nn.Conv2d(dim_out, dim_out, 3, 1, 1)
        self.norm1 = AdaIN(style_dim, dim_out)
        self.norm2 = AdaIN(style_dim, dim_out)
        if self.learned_sc:
            self.conv1x1 = nn.Conv2d(dim_in, dim_out, 1, 1, 0, bias=False)

    def _shortcut(self, x):
        if self.upsample:
            x = F.interpolate(x, scale_factor=2, mode='nearest')
        if self.learned_sc:
            x = self.conv1x1(x)
        return x

    def _residual(self, x, s):
        x = self.conv1(x)
        x = self.norm1(x, s)
        x = self.actv(x)
        if self.upsample:
            x = F.interpolate(x, scale_factor=2, mode='nearest')
        x = self.conv2(x)
        x = self.norm2(x, s)
        x = self.actv(x)
        return x

    def forward(self, x, s):
        out = self._residual(x, s)
        if self.do_residual:
            out = (out + self._shortcut(x)) / math.sqrt(2)
        return out

class HighPass(nn.Module):
    def __init__(self, w_hpf, device):
        super(HighPass, self).__init__()
        self.filter = torch.tensor([[-1, -1, -1],
                                    [-1, 8., -1],
                                    [-1, -1, -1]]).to(device) / w_hpf

    def forward(self, x):
        filter = self.filter.unsqueeze(0).unsqueeze(1).repeat(x.size(1), 1, 1, 1)
        return F.conv2d(x, filter, padding=1, groups=x.size(1))

class Generator(nn.Module):
    def __init__(self, img_size=256, style_dim=64, max_conv_dim=512, w_hpf=1):
        super().__init__()
        dim_in = 2**14 // img_size
        self.img_size = img_size
        self.from_rgb = nn.Conv2d(3, dim_in, 3, 1, 1)
        self.encode = nn.ModuleList()
        self.decode = nn.ModuleList()
        self.to_rgb = nn.Sequential(
            nn.InstanceNorm2d(dim_in, affine=True),
            nn.LeakyReLU(0.2),
            nn.Conv2d(dim_in, 3, 1, 1, 0))

        # down/up-sampling blocks
        repeat_num = int(np.log2(img_size)) - 4
        if w_hpf > 0:
            repeat_num += 1
        for _ in range(repeat_num):
            dim_out = min(dim_in*2, max_conv_dim)
            self.encode.append(
                ResBlk(dim_in, dim_out, normalize=True, downsample=True))
            self.decode.insert(
                0, AdainResBlk(dim_out, dim_in, style_dim,
                               w_hpf=w_hpf, upsample=True))  # stack-like
            dim_in = dim_out

        # bottleneck blocks
        for _ in range(2):
            self.encode.append(
                ResBlk(dim_out, dim_out, normalize=True))
            self.decode.insert(
                0, AdainResBlk(dim_out, dim_out, style_dim, w_hpf=w_hpf))

        if w_hpf > 0:
            device = torch.device(
                'cuda' if torch.cuda.is_available() else 'cpu')
            self.hpf = HighPass(w_hpf, device)

    def forward(self, x, s, masks=None):
        x = self.from_rgb(x)
        cache = {}
        for block in self.encode:
            if (masks is not None) and (x.size(2) in [32, 64, 128]):
                cache[x.size(2)] = x
            x = block(x)
        for block in self.decode:
            x = block(x, s)
            if (masks is not None) and (x.size(2) in [32, 64, 128]):
                mask = masks[0] if x.size(2) in [32] else masks[1]
                mask = F.interpolate(mask, size=x.size(2), mode='bilinear')
                x = x + self.hpf(mask * cache[x.size(2)])
        return self.to_rgb(x), self.to_rgb(x)

class BasicBlock_Ganilla(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, use_dropout, stride=1):
        super(BasicBlock_Ganilla, self).__init__()
        self.rp1 = nn.ReflectionPad2d(1)
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=0, bias=False)
        self.bn1 = nn.InstanceNorm2d(planes)
        self.use_dropout = use_dropout
        if use_dropout:
            self.dropout = nn.Dropout(0.5)
        self.rp2 = nn.ReflectionPad2d(1)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=0, bias=False)
        self.bn2 = nn.InstanceNorm2d(planes)
        self.out_planes = planes

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
                nn.InstanceNorm2d(self.expansion*planes)
            )

            self.final_conv = nn.Sequential(
                nn.ReflectionPad2d(1),
                nn.Conv2d(self.expansion * planes * 2, self.expansion * planes, kernel_size=3, stride=1,
                                        padding=0, bias=False),
                nn.InstanceNorm2d(self.expansion * planes)
            )
        else:
            self.final_conv = nn.Sequential(
                nn.ReflectionPad2d(1),
                nn.Conv2d(planes*2, planes, kernel_size=3, stride=1, padding=0, bias=False),
                nn.InstanceNorm2d(planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(self.rp1(x))))
        if self.use_dropout:
            out = self.dropout(out)
        out = self.bn2(self.conv2(self.rp2(out)))
        inputt = self.shortcut(x)
        catted = torch.cat((out, inputt), 1)
        out = self.final_conv(catted)
        out = F.relu(out)
        return out

class PyramidFeatures(nn.Module):
    def __init__(self, C2_size, C3_size, C4_size, C5_size, fpn_weights =[1.0, 1.0, 1.0, 1.0], feature_size=128, style_dim=64, do_residual=True):
        super().__init__()

        self.sum_weights = fpn_weights #[1.0, 0.5, 0.5, 0.5]

        # upsample C5 to get P5 from the FPN paper
        self.P5_1 = nn.Conv2d(C5_size, feature_size, kernel_size=1, stride=1, padding=0)
        self.P5_2 = AdainResBlkGanilla(C5_size, feature_size, style_dim, do_residual=do_residual)

        # add P5 elementwise to C4
        self.P4_1 = nn.Conv2d(C4_size, feature_size, kernel_size=1, stride=1, padding=0)
        self.P4_2 = AdainResBlkGanilla(feature_size, feature_size, style_dim, do_residual=do_residual)

        # add P4 elementwise to C3
        self.P3_1 = nn.Conv2d(C3_size, feature_size, kernel_size=1, stride=1, padding=0)
        self.P3_2 = AdainResBlkGanilla(feature_size, feature_size, style_dim, do_residual=do_residual)

        self.P2_1 = nn.Conv2d(C2_size, feature_size, kernel_size=1, stride=1, padding=0)
        self.P2_upsampled = nn.Upsample(scale_factor=2, mode='nearest')
        self.rp4 = nn.ReflectionPad2d(1)
        self.P2_2 = nn.Conv2d(int(feature_size), int(feature_size/2), kernel_size=3, stride=1, padding=0)

    def forward(self, inputs, s):
        C2, C3, C4, C5 = inputs

        i = 0
        P5_adain_x = self.P5_2(C5, s)

        i += 1
        P4_x = self.P4_1(C4)
        P4_x = P5_adain_x + P4_x
        P4_adain_x = self.P4_2(P4_x, s)

        i += 1
        P3_x = self.P3_1(C3)
        P3_x = P3_x + P4_adain_x
        P3_adain_x = self.P3_2(P3_x, s)

        i += 1
        P2_x = self.P2_1(C2)
        P2_x = P2_x + P3_adain_x
        P2_upsampled_x = self.P2_upsampled(P2_x)
        P2_x = self.rp4(P2_upsampled_x)
        P2_x = self.P2_2(P2_x)

        return P2_x

class GeneratorGanilla(nn.Module):

    def __init__(self, img_size=256,  style_dim=64, input_nc=3, output_nc=3, ngf=64, ganilla_layer_nb=[2, 2, 2, 2], use_dropout=True, masks=None, do_residual=True):
        super().__init__()

        ganilla_block_nf = [64, 128, 128, img_size]
        self.inplanes = ngf

        # first conv
        self.pad1 = nn.ReflectionPad2d(3)
        self.conv1 = nn.Conv2d(input_nc, ngf, kernel_size=7, stride=1, padding=0, bias=True)
        self.in1 = nn.InstanceNorm2d(ngf)
        self.relu = nn.ReLU(inplace=True)
        self.pad2 = nn.ReflectionPad2d(1)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=0)
        self.avgpool = nn.AvgPool2d(kernel_size=3, stride=2, padding=0)

        # Output layer
        self.pad3 = nn.ReflectionPad2d(3)
        self.conv2 = nn.Conv2d(64, output_nc, 7)
        self.tanh = nn.Tanh()

        self.layer1 = self._make_layer_ganilla(BasicBlock_Ganilla, ganilla_block_nf[0], ganilla_layer_nb[0], use_dropout, stride=1)
        self.layer2 = self._make_layer_ganilla(BasicBlock_Ganilla, ganilla_block_nf[1], ganilla_layer_nb[1], use_dropout, stride=2)
        self.layer3 = self._make_layer_ganilla(BasicBlock_Ganilla, ganilla_block_nf[2], ganilla_layer_nb[2], use_dropout, stride=2)
        self.layer4 = self._make_layer_ganilla(BasicBlock_Ganilla, ganilla_block_nf[3], ganilla_layer_nb[3], use_dropout, stride=2)

        # for the skip connections
        fpn_sizes = [self.layer1[ganilla_layer_nb[0] - 1].conv2.out_channels,
                     self.layer2[ganilla_layer_nb[1] - 1].conv2.out_channels,
                     self.layer3[ganilla_layer_nb[2] - 1].conv2.out_channels,
                     self.layer4[ganilla_layer_nb[3] - 1].conv2.out_channels]

        self.fpn = PyramidFeatures(fpn_sizes[0], fpn_sizes[1], fpn_sizes[2], fpn_sizes[3], style_dim=style_dim, do_residual=do_residual)

    def _make_layer_ganilla(self, block, planes, blocks, use_dropout, stride=1):
        strides = [stride] + [1] * (blocks - 1)
        layers = []
        for stride in strides:
            layers.append(block(self.inplanes, planes, use_dropout, stride))
            self.inplanes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, inputs, s, masks=None):

        img_batch = inputs

        x = self.pad1(img_batch)
        x = self.conv1(x)
        x = self.in1(x)
        x = self.relu(x)
        x = self.pad2(x)
        x = self.maxpool(x)

        x1 = self.layer1(x)
        x2 = self.layer2(x1)
        x3 = self.layer3(x2)
        x4 = self.layer4(x3)

        out = self.fpn([x1, x2, x3, x4], s)

        out = self.pad3(out)
        out = self.conv2(out)
        out = self.tanh(out)

        return out, x4

class MappingNetwork(nn.Module):
    def __init__(self, latent_dim=16, style_dim=64, num_domains=2):
        super().__init__()
        layers = []
        layers += [nn.Linear(latent_dim, 512)]
        layers += [nn.ReLU()]
        for _ in range(3):
            layers += [nn.Linear(512, 512)]
            layers += [nn.ReLU()]
        self.shared = nn.Sequential(*layers)

        self.unshared = nn.ModuleList()
        for _ in range(num_domains):
            self.unshared += [nn.Sequential(nn.Linear(512, 512),
                                            nn.ReLU(),
                                            nn.Linear(512, 512),
                                            nn.ReLU(),
                                            nn.Linear(512, 512),
                                            nn.ReLU(),
                                            nn.Linear(512, style_dim))]

    def forward(self, z, y):
        h = self.shared(z)
        out = []
        for layer in self.unshared:
            out += [layer(h)]
        out = torch.stack(out, dim=1)  # (batch, num_domains, style_dim)
        idx = torch.LongTensor(range(y.size(0))).to(y.device)
        s = out[idx, y]  # (batch, style_dim)
        return s

class StyleEncoder(nn.Module):
    def __init__(self, img_size=256, style_dim=64, num_domains=2, max_conv_dim=512):
        super().__init__()
        dim_in = 2**14 // img_size
        blocks = []
        blocks += [nn.Conv2d(3, dim_in, 3, 1, 1)]

        repeat_num = int(np.log2(img_size)) - 2
        for _ in range(repeat_num):
            dim_out = min(dim_in*2, max_conv_dim)
            blocks += [ResBlk(dim_in, dim_out, downsample=True)]
            dim_in = dim_out

        blocks += [nn.LeakyReLU(0.2)]
        blocks += [nn.Conv2d(dim_out, dim_out, 4, 1, 0)]
        blocks += [nn.LeakyReLU(0.2)]
        self.shared = nn.Sequential(*blocks)

        self.unshared = nn.ModuleList()
        for _ in range(num_domains):
            self.unshared += [nn.Linear(dim_out, style_dim)]

    def forward(self, x, y):
        h = self.shared(x)
        h = h.view(h.size(0), -1)
        out = []
        for layer in self.unshared:
            out += [layer(h)]
        out = torch.stack(out, dim=1)  # (batch, num_domains, style_dim)
        idx = torch.LongTensor(range(y.size(0))).to(y.device)
        s = out[idx, y]  # (batch, style_dim)
        return s

class Discriminator(nn.Module):
    def __init__(self, img_size=256, num_domains=2, max_conv_dim=512):
        super().__init__()
        dim_in = 2**14 // img_size
        blocks = []
        blocks += [nn.Conv2d(3, dim_in, 3, 1, 1)]

        repeat_num = int(np.log2(img_size)) - 2
        for _ in range(repeat_num):
            dim_out = min(dim_in*2, max_conv_dim)
            blocks += [ResBlk(dim_in, dim_out, downsample=True)]
            dim_in = dim_out

        blocks += [nn.LeakyReLU(0.2)]
        blocks += [nn.Conv2d(dim_out, dim_out, 4, 1, 0)]
        blocks += [nn.LeakyReLU(0.2)]
        blocks += [nn.Conv2d(dim_out, num_domains, 1, 1, 0)]
        self.main = nn.Sequential(*blocks)

    def forward(self, x, y):
        out = self.main(x)
        out = out.view(out.size(0), -1)  # (batch, num_domains)
        idx = torch.LongTensor(range(y.size(0))).to(y.device)
        out = out[idx, y]  # (batch)
        return out

class VggExtract(torch.nn.Module):
    def __init__(self, vgg_model):
        super(VggExtract, self).__init__()
        self.vgg_layers = vgg_model.features
        self.layer_name_mapping = {
            '8': "relu2_2",
            '15': "relu3_3",
            '22': "relu4_3"
        }

    def forward(self, x):
        output = {}
        for name, module in self.vgg_layers._modules.items():
            x = module(x)
            if name in self.layer_name_mapping:
                output[self.layer_name_mapping[name]] = x
        return output

def build_model(args):
    if args.use_star_gen:
        generator = Generator(args.img_size, args.style_dim, w_hpf=args.w_hpf)
    else:
        if args.use_residual_upsample:
            generator = GeneratorGanilla(img_size=args.img_size, style_dim=args.style_dim, do_residual=True)
        else:
            generator = GeneratorGanilla(img_size=args.img_size, style_dim=args.style_dim, do_residual=False)

    mapping_network = MappingNetwork(args.latent_dim, args.style_dim, args.num_domains)
    style_encoder = StyleEncoder(args.img_size, args.style_dim, args.num_domains)
    discriminator = Discriminator(args.img_size, args.num_domains)
    generator_ema = copy.deepcopy(generator)
    mapping_network_ema = copy.deepcopy(mapping_network)
    style_encoder_ema = copy.deepcopy(style_encoder)

    nets = Munch(generator=generator,
                 mapping_network=mapping_network,
                 style_encoder=style_encoder,
                 discriminator=discriminator)
    nets_ema = Munch(generator=generator_ema,
                     mapping_network=mapping_network_ema,
                     style_encoder=style_encoder_ema)

    if args.w_hpf > 0:
        fan = FAN(fname_pretrained=args.wing_path).eval()
        nets.fan = fan
        nets_ema.fan = fan

    vgg = load_vgg16()
    vgg.eval()
    for param in vgg.parameters():
        param.requires_grad = False

    return nets, nets_ema, vgg, VggExtract(vgg)

max_stars_repo_name: NoaBrazilay/DeepLearningProject

max_issues_repo_name: NoaBrazilay/MISSGAN

max_forks_repo_name: NoaBrazilay/MISSGAN


text:
#!/usr/bin/env python
import sqlite3
import sys
import numpy as np
import pylab as pl
from collections import Counter

filesql=sys.argv[1]
outfile=sys.argv[2]

def RepresentsInt(s):
    try:
        int(s)
        return True
    except ValueError:
        return False

def readSqlall(sqlname,segment):
        sqlstatement = "SELECT {0} FROM pairs".format(segment)
        con = sqlite3.connect(sqlname)
        with con:
                cur = con.cursor()
                cur.execute(sqlstatement)
                rows = cur.fetchall()

        sqlall=(np.array(rows))
        return sqlall

ids1=readSqlall(filesql,"seg1")
ids2=readSqlall(filesql,"seg2")
ids=np.vstack((ids1,ids2))
numofids=np.max(ids)
hist=np.histogram(ids,bins=range(numofids+2))[0][1:]

segid=np.arange(np.size(hist))+1
minsegid=np.where(hist==hist.min())[0]+1

print "Segments with least number of neighbors(Number= {}):{}".format(hist.min(), minsegid)
print "Segment with least number of neighbors vmd notation(Number= {}):{}".format(hist.min(), minsegid-1)

maxsegid=np.where(hist==hist.max())[0]+1
print "Segment with most number of neighbors(Number= {}):{}".format(hist.max(), maxsegid)
print "Segment with most number of neighbors vmd notation(Number= {}):{}".format(hist.max(), maxsegid-1)
print "Mean number of neighbors", np.sum(hist)/np.size(hist)

if outfile =="plot":
        pl.plot(segid,hist,'bo')
        pl.show()
else:
        np.savetxt(outfile,np.array([segid,hist]).T,header="pairid, numberofneighbors")

max_stars_repo_name: pelagia/votca-scripts

max_issues_repo_name: pelagia/votca-scripts

max_forks_repo_name: pelagia/votca-scripts


text:
import park
from park import core, spaces, logger
from park.param import config
from park.utils import seeding
from park.spaces.box import Box

import numpy as np
from subprocess import Popen
import os
import zmq

from time import time, sleep

from park.envs.aqm.mahimahi_interface import MahimahiInterface

try:
    from park.envs.aqm.ipc_msg_pb2 import IPCMessage, IPCReply
except:
    os.system("protoc -I=" + park.__path__[0] + "/envs/aqm/ --python_out=" + park.__path__[0] + "/envs/aqm/ " + park.__path__[0] + "/envs/aqm/ipc_msg.proto")
    from park.envs.aqm.ipc_msg_pb2 import IPCMessage, IPCReply
class AQMEnv(core.Env):
    """
    TODO: write a description
    """
    def __init__(self):

        # Setup Mahimahi

        # TODO: sudo apt-get install apache2-dev
        # sudo apt-get install libssl-dev
        # sudo apt-get install libxcb-xrm-dev
        # libxcb-present-dev
        # libcairo2-dev
        # libpango1.0-dev
        # sudo apt-get install libzmq3-dev

        # TODO: check os
        mahimahi_path = park.__path__[0]+"/envs/aqm/mahimahi"

        if os.path.exists (mahimahi_path) == False :
            # create folder
            Popen("mkdir %s" % mahimahi_path, shell=True).wait()
            # get mahimahi
            # Popen("cd %s; git clone https://github.com/songtaohe/mahimahi.git" % (park.__path__[0]+"/envs/aqm/"), shell=True).wait()
            Popen("cd %s; git clone https://github.com/mehrdadkhani/mahimahi-1 mahimahi" % (park.__path__[0]+"/envs/aqm/"), shell=True).wait()
            # Make mahimahi
            # Popen("cd %s; git fetch; git checkout mahimahi_stable2; ./autogen.sh; ./configure; make; sudo make install" % mahimahi_path, shell=True).wait()
            Popen("cd %s; git fetch; git checkout mahimahi_stable2; ./autogen.sh; ./configure; make; sudo make install" % mahimahi_path, shell=True).wait()

        # compile protoc
        # protoc -I=./park/envs/aqm/ --cpp_out=./park/envs/aqm/mahimahi/src/packet ./park/envs/aqm/ipc_msg.proto

        # TODO: disable pie

        #self.mm_delay = mahimahi_path + "/src/frontend/mm-delay"
        #self.mm_link = mahimahi_path + "/src/frontend/mm-link"

        self.mm_delay = "mm-delay"
        self.mm_link = "mm-link"

        # Setup link

        self.linkDelay = config.aqm_link_delay
        self.uplinkTraceFile = config.aqm_uplink_trace
        self.downlinkTraceFile = config.aqm_downlink_trace

        # Setup workload generator

        self.workloadGeneratorSender   = "iperf -c 100.64.0.1 -P 1 -i 2 -t 10000"
        self.workloadGeneratorReceiver = "iperf -s -w 16m"
        self.workloadGeneratorKiller   = "pkill -9 iperf"

        with open("sender.sh","w") as fout:
            fout.write(self.workloadGeneratorSender+"\n")

        Popen("chmod a+x sender.sh", shell=True).wait()

        # Setup RL
        self.aqm_step_interval = config.aqm_step_interval
        self.aqm_step_num = config.aqm_step_num

        self.state_space = Box(low=np.array([0, 0, 0]), high=np.array([1e4, 1e4, 1e3]))
        self.action_space = Box(low=np.array([0]), high=np.array([1]))

        self.step_counter = 0

        # Start
        # self.reset()

        self.last_action_ts = None

    def _observe(self):
        ret = self.mahimahi.GetState()
        obs = np.array([ret['enqueued_packet'], ret['dequeued_packet'], ret['average_queueing_delay']])
        reward = -ret["average_queueing_delay"]
        info = {'message': ret['info']}
        return obs, reward, info

    def render(self):
        # TODO: depends on a switch (in __init__), visualize the mahimahi console
        pass

    def step(self, action):

        assert self.action_space.contains(action)

        self.step_counter += 1

        if self.last_action_ts is None:
            t_sleep = self.aqm_step_interval/1000.0
            sleep(t_sleep)
        else:
            t0 = time()
            t_sleep = self.last_action_ts + self.aqm_step_interval/1000.0 - t0

            if t_sleep > 0 :
                sleep(t_sleep)

        self.mahimahi.SetDropRate(action[0])
        self.last_action_ts = time()

        obs, reward, info = self._observe()
        assert self.state_space.contains(obs)
        info['wall_time_elapsed'] = t_sleep

        done = False
        if self.step_counter >= self.aqm_step_num:
            done = True

        return obs, reward, done, info

    def reset(self):
        # kill Mahimahi and workload generator
        Popen("pkill mm-delay", shell=True).wait()
        Popen(self.workloadGeneratorKiller, shell=True).wait()
        sleep(1.0)  # pkill has delay

        # start workload generator receiver
        Popen(self.workloadGeneratorReceiver, shell=True)

        # start Mahimahi
        config_dict = {}
        config_dict["mmdelay"] = self.mm_delay
        config_dict["mmlink"] = self.mm_link
        config_dict["delay"] = int(self.linkDelay)
        config_dict["uplinktrace"] = self.uplinkTraceFile
        config_dict["downlinktrace"] = self.downlinkTraceFile
        config_dict["workloadSender"] = "./sender.sh"

        start_mahimahi_cmd = \
        "%(mmdelay)s %(delay)d %(mmlink)s %(uplinktrace)s %(downlinktrace)s  \
        --meter-uplink --meter-uplink-delay --uplink-queue=pie --downlink-queue=infinite --uplink-queue-args=\"packets=2000, qdelay_ref=20, max_burst=1\" \
        %(workloadSender)s "% config_dict

        Popen(start_mahimahi_cmd, shell=True)
        sleep(1.0)  # mahimahi start delay

        # Connect to Mahimahi
        self.mahimahi = MahimahiInterface()
        self.mahimahi.ConnectToMahimahi()

        # Gain control
        #self.mahimahi.SetRLState(1)

        ## get obs
        #obs, _, _ = self._observe()
        #self.step_counter = 0
        return True
        #return obs
#    def reset(self):
#        # kill Mahimahi and workload generator
#
#        Popen("pkill mm-delay", shell=True).wait()
#        Popen(self.workloadGeneratorKiller, shell=True).wait()
#
#        sleep(1.0)  # pkill has delay
#
#        # start workload generator receiver
#        Popen(self.workloadGeneratorReceiver, shell=True)
#
#
#        # start Mahimahi
#        config_dict = {}
#
#        config_dict["mmdelay"] = self.mm_delay
#        config_dict["mmlink"] = self.mm_link
#        config_dict["delay"] = int(self.linkDelay)
#
#        config_dict["uplinktrace"] = self.uplinkTraceFile
#        config_dict["downlinktrace"] = self.downlinkTraceFile
#
#        config_dict["workloadSender"] = "./sender.sh"
#
#        start_mahimahi_cmd = \
#        "%(mmdelay)s %(delay)d %(mmlink)s %(uplinktrace)s %(downlinktrace)s  \
#        --meter-uplink --meter-uplink-delay --uplink-queue=pie --downlink-queue=infinite --uplink-queue-args=\"packets=2000, qdelay_ref=20, max_burst=1\" \
#        %(workloadSender)s "% config_dict
#
#        Popen(start_mahimahi_cmd, shell=True)
#
#        sleep(1.0)  # mahimahi start delay
#
#        # Connect to Mahimahi
#        self.mahimahi = MahimahiInterface()
#
#        self.mahimahi.ConnectToMahimahi()
#
#
#        # Gain control
#        self.mahimahi.SetRLState(1)
#
#        # get obs
#        obs, _, _ = self._observe()
#        self.step_counter = 0
#
#        return obs

    def run(self, agent):
        # set up ipc communication
        context = zmq.Context(1)
        socket = context.socket(zmq.REP)
        ipc_msg = IPCMessage()
        ipc_reply = IPCReply()

        os.system('rm /tmp/aqm_cpp_python_ipc')
        socket.bind("ipc:///tmp/aqm_cpp_python_ipc")
        self.reset()

        last_eqc = 0
        last_dqc = 0
        last_qdelay = 0

        eqc = 0
        dqc = 0
        qdelay = 0

        reg = 0

        while True:
            #print('inside')
            msg = socket.recv()
            ipc_msg.ParseFromString(msg)
            #print('got message', ipc_msg.msg)

            if ipc_msg.msg == 'get_prob':
                try:
                    eqc = ipc_msg.eqc
                    dqc = ipc_msg.dqc
                    qdelay = ipc_msg.qdelay
                except:
                    print("error reading ipc msg")

                avg_queue_delay = (qdelay - last_qdelay)/float(dqc-last_dqc + 0.01)
                eq_counter = eqc - last_eqc
                dq_counter = dqc - last_dqc

                last_qdelay = qdelay
                last_eqc = eqc
                last_dqc = dqc

                #print(avg_queue_delay, eq_counter, dq_counter)

                reward = -abs(avg_queue_delay-20) + reg

                prob = 0.2*agent.get_action(np.array([avg_queue_delay, eq_counter, dq_counter]), reward)

                prob = prob[0][0]
                #print(prob)

                if prob > 0.5 :
                    reg = -10.0 * (prob-0.5)

                if prob < 0.0:
                    reg = -10.0 * (0.0-prob)

                ipc_reply.prob = prob#agent(ipc_msg.state)
                ipc_reply.msg = 'set_prob'
                socket.send(ipc_reply.SerializeToString(),0)

                sleep(0.1)

    def seed(self, seed=None):
        # no controllable randomness
        pass

    def clean(self):
        Popen("pkill mm-delay", shell=True).wait()
        Popen(self.workloadGeneratorKiller, shell=True).wait()

        Popen("rm mahimahi_pipe1 mahimahi_pipe2 sender.sh", shell=True).wait()

        sleep(1.0)


text:
"""Mapping Vector Field of Single Cells
"""

# module to deal with reaction/diffusion/advection.
# code was loosely based on PBA, WOT and PRESCIENT.

import numpy as np
import pandas as pd
from scipy.sparse import issparse
from sklearn.neighbors import NearestNeighbors

def score_cells(adata,
                genes=None,
                layer=None,
                basis=None,
                n_neighbors=30,
                beta=0.1,
                iteration=5,
                metric="euclidean",
                metric_kwds=None,
                cores=1,
                seed=19491001,
                return_score=True,
                **kwargs,
                ):
    """Score cells based on a set of genes.

    Parameters
    ----------
        adata: :class:`~anndata.AnnData`
            AnnData object that contains the reconstructed vector field function in the `uns` attribute.
        genes: `list` or None (default: None)
            The gene names whose gene expression will be used for predicting cell fate. By default (when genes is set to
            None), the genes used for velocity embedding (var.use_for_transition) will be used for vector field
            reconstruction. Note that the genes to be used need to have velocity calculated and corresponds to those used
            in the `dyn.tl.VectorField` function.
        layer: `str` or None (default: 'X')
            Which layer of the data will be used for predicting cell fate with the reconstructed vector field function.
            The layer once provided, will override the `basis` argument and then predicting cell fate in high dimensional
            space.
        basis: `str` or None (default: `None`)
            The embedding data to use for predicting cell fate. If `basis` is either `umap` or `pca`, the reconstructed
            trajectory will be projected back to high dimensional space via the `inverse_transform` function.
        n_neighbors: `int` (default: `30`)
            Number of nearest neighbors.
        beta: `float` (default: `0.1`)
            The weight that will apply to the current query cell.
        iteration: `int` (default: `0.5`)
            Number of smooth iterations.
        metric: `str` or callable, default='euclidean'
            The distance metric to use for the tree.  The default metric is , and with p=2 is equivalent to the standard
            Euclidean metric. See the documentation of :class:`DistanceMetric` for a list of available metrics. If metric
            is "precomputed", X is assumed to be a distance matrix and must be square during fit. X may be a
            :term:`sparse graph`, in which case only "nonzero" elements may be considered neighbors.
        metric_kwds : dict, default=None
            Additional keyword arguments for the metric function.
        cores: `int` (default: 1)
            The number of parallel jobs to run for neighbors search. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
            ``-1`` means using all processors.
        seed: `int` (default `19491001`)
            Random seed to ensure the reproducibility of each run.
        return_score: `bool` (default: `False`)
            Whether to return the score. If False, save the smoothed score to `cell_scores` column in the `.obs`
            attribute and also to the dictionary corresponding to the `score_cells` key in the .uns attribute.
        kwargs:
            Additional arguments that will be passed to each nearest neighbor search algorithm.

    Returns
    -------
        Depending on return_score, it either return the cell scores or a updated adata object that contains the cell
        score information.
    """

    if basis is None and "X_pca" not in adata.obsm.keys():
        raise ValueError(f"Your adata doesn't have 'X_pca' basis in .obsm.")
    elif basis is not None and "X_" + basis not in adata.obsm.keys():
        raise ValueError(f"Your adata doesn't have the {basis} you inputted in .obsm attribute of your adata.")

    if genes is None and "use_for_pca" not in adata.obs.keys():
        raise ValueError(f"Your adata doesn't have 'use_for_pca' column in .obs.")

    if genes is None:
        genes = adata.var_names[adata.use_for_pca]
    else:
        genes = list(adata.var_names.intersection(genes)) if adata.var_names[0].isupper() else \
            list(adata.var_names.intersection([i.capitalize() for i in genes])) if \
                 adata.var_names[0][0].isupper() and adata.var_names[0][1:].islower() else \
                     list(adata.var_names.intersection([i.lower() for i in genes]))

    if len(genes) < 1:
        raise ValueError(f"Your inputted gene list doesn't overlap any gene in your adata object.")

    X_basis = adata.obsm['X_pca'] if basis is None else adata.obsm['X_' + basis]

    if X_basis.shape[0] > 5000 and X_basis.shape[1] > 2:
        from pynndescent import NNDescent

        nbrs = NNDescent(X_basis, metric=metric, metric_kwds=metric_kwds, n_neighbors=30, n_jobs=cores,
                              random_state=seed, **kwargs)
        knn, distances = nbrs.query(X_basis, k=n_neighbors)
    else:
        alg = 'ball_tree' if X_basis.shape[1] > 10 else 'kd_tree'
        nbrs = NearestNeighbors(n_neighbors=n_neighbors, algorithm=alg, n_jobs=cores).fit(X_basis)
        distances, knn = nbrs.kneighbors(X_basis)

    X_data = adata[:, genes].X if layer in [None, 'X'] else adata[:, genes].layers[layer]

    prev_score = X_data.mean(1).A1 if issparse(X_data) else X_data.mean(1)
    cur_score = np.zeros(prev_score.shape)

    for _ in range(iteration):
        for i in range(len(prev_score)):
            xn = prev_score[knn[i]]
            cur_score[i] = (beta * xn[0]) + ((1 - beta) * xn[1:].mean(axis=0))
        prev_score = cur_score

    smoothed_score = cur_score

    if return_score:
        return smoothed_score
    else:
        adata.uns['score_cells'] = {"smoothed_score": smoothed_score,
                                    "genes": genes,
                                    "layer": layer,
                                    "basis": basis}
        adata.obs['cell_score'] = smoothed_score

def cell_growth_rate(adata,
                     group,
                     source,
                     target,
                     L0=0.3,
                     L=1.2,
                     k=1e-3,
                     birth_genes=None,
                     death_genes=None,
                     clone_column=None,
                     **kwargs):
    """Estimate the growth rate via clone information or logistic equation of population dynamics.

    Growth rate is calculated as 1) number_of_cell_at_source_time_in_the_clone / number_of_cell_at_end_time_in_the_clone
    when there is clone information (`[clone_column, time_column, source_time, target_time]` are all not None); 2)
    estimate via logistic equation of population growth and death.

    Parameters
    ----------
        adata: :class:`~anndata.AnnData`
            AnnData object that contains the reconstructed vector field function in the `uns` attribute.
        group: str or None (default: `None`)
            The column key in .obs points to the collection time of each cell, required for calculating growth rate with
            clone information.
        source: str or None (default: `None`)
            The column key in .obs points to the starting point from collection time of each cell, required for
            calculating growth rate with clone information.
        target: str or None (default: `None`)
            The column key in .obs points to the end point from collection time of each cell, required for
            calculating growth rate with clone information.
        L0: float (default: `0.3`)
            The base growth/death rate.
        L: float (default: `1.2`)
            The maximum growth/death rate.
        k: float (default: `0.001)
            The steepness of the curve.
        birth_genes: list or None (default: `None`)
            The gene list associated with the cell cycle process. If None, GSEA's KEGG_CELL_CYCLE will be used.
        death_genes: list or None (default: `None`)
            The gene list associated with the cell cycle process. If None, GSEA's KEGG_APOPTOSIS will be used.
        clone_column: str or None (default: `None`)
            The column key in .obs points to the clone id if there is any. If a cell doesn't belong to any clone, the
            clone id of that cell should be assigned as `np.nan`
        kwargs
            Additional arguments that will be passed to score_cells function.

    Returns
    -------
        A updated adata object that includes `growth_rate` column or `growth_rate, birth_score, death_score` in its
        `.obs` attribute when the clone based or purely expression based growth rate was calculated.
    """

    # calculate growth rate when there is clone information.
    all_clone_info = [clone_column, group, source, target]

    obs = adata.obs
    source_mask_, target_mask_ = obs[group].values == source, obs[group].values == target

    if all(i is not None for i in all_clone_info):

        if any(i not in adata.obs.keys() for i in all_clone_info[:2]):
            raise ValueError(f"At least one of your input clone information {clone_column}, {group} "
                             f"is not in your adata .obs attribute.")
        if any(i not in adata.obs[group] for i in all_clone_info[2:]):
            raise ValueError(f"At least one of your input source/target information {source}, {target} "
                             f"is not in your adata.obs[{group}] column.")

        clone_time_count = obs.groupby([clone_column])[group].value_counts().unstack().fillna(0).astype(int)
        source_meta = obs.loc[source_mask_]
        source_mask = (source_meta[clone_column] != np.nan).values

        target_meta = obs.loc[target_mask_]
        target_mask = (target_meta[clone_column] != np.nan).values

        source_num = clone_time_count.loc[source_meta.loc[source_mask, clone_column], source].values + 1
        target_num = clone_time_count.loc[target_meta.loc[target_mask, clone_column], target].values + 1

        growth_rates = target_num / source_num
    else:
        # calculate growth rate when there is no clone information.
        if birth_genes is None:
            birth_genes = pd.read_csv('https://raw.githubusercontent.com/Xiaojieqiu/jungle/master/Cell_cycle.txt',
                                      header=None, dtype=str)
            birth_genes = birth_genes[0].values

        if death_genes is None:
            death_genes = pd.read_csv('https://raw.githubusercontent.com/Xiaojieqiu/jungle/master/Apoptosis.txt',
                                      header=None, dtype=str)
            death_genes = death_genes[0].values

        birth_score = score_cells(adata, genes=birth_genes, **kwargs)
        death_score = score_cells(adata, genes=death_genes, **kwargs)
        adata.obs['birth_score'] = birth_score
        adata.obs['death_score'] = death_score

        kb = np.log(k) / np.min(birth_score)
        kd = np.log(k) / np.min(death_score)

        b = birth_score[source_mask_]
        d = death_score[source_mask_]

        b = L0 + L / (1 + np.exp(-kb * b))
        d = L0 + L / (1 + np.exp(-kd * d))
        growth_rates = b - d

    adata.obs['growth_rate'] = np.nan
    adata.obs.loc[source_mask_, 'growth_rate'] = growth_rates

    return adata

def n_descentants(birth, death, dt):
    return np.exp(dt * (birth - death))

def growth_rate(n, dt):
    return np.log(n) / dt

max_stars_repo_name: davisidarta/dynamo-release

max_issues_repo_name: davisidarta/dynamo-release

max_forks_repo_name: davisidarta/dynamo-release

reason: import numpy,from scipy


text:
#!/usr/bin/env python3
# Copyright 2021 Christophe Bedard
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Plot cumulative time based on daily time reporting data."""

from typing import List
from typing import Optional
from typing import Union

from datetime import date
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
import pandas as pd

include_plot_title = True
save_plots = True

def load_csv(filename: str) -> np.array:
    with open(filename, 'rb') as file:
        return np.loadtxt(
            file,
            delimiter=',',
            skiprows=1,
            usecols=(0,1),
            dtype=str,
        )

def filter_data(data: np.array) -> np.array:
    return np.array([d for d in data if d[0] not in ('', 'total')])

def convert_data(data: np.array) -> np.array:
    return np.array([[date.fromisoformat(d[0]), float(d[1])] for d in data])

def add_zeroth_datapoint(data: np.array) -> np.array:
    return np.vstack([[[data[0,0], 0.0]], data])

def data_to_cumsum(data: np.array, col: int = 1) -> np.array:
    data[:,col] = np.cumsum(data[:,col])
    return data

def get_data(filename: str):
    data = load_csv(filename)
    data = filter_data(data)
    data = convert_data(data)
    data = add_zeroth_datapoint(data)
    data = data_to_cumsum(data)
    return data

def format_filename(string: str) -> str:
    string = string.replace('(', '')
    string = string.replace(')', '')
    string = string.replace(' ', '_')
    string = string.replace('\\', '')
    return string.lower()

def plot_data(
    data: np.array,
    title: str,
    major_formatter_str: str,
    major_locator: Optional[mdates.RRuleLocator] = None,
    yaxis_multiple_locator: Optional[int] = None,
    colour: str = 'blue',
) -> None:
    fig, ax = plt.subplots(1, 1)

    ax.plot(data[:,0], data[:,1], '-', color=colour)

    if include_plot_title:
        ax.set(title=title)
    ax.set(ylabel='cumulative time (h)')
    if major_locator:
        ax.xaxis.set_major_locator(major_locator)
    ax.xaxis.set_major_formatter(mdates.DateFormatter(major_formatter_str))
    if yaxis_multiple_locator:
        ax.yaxis.set_major_locator(ticker.MultipleLocator(yaxis_multiple_locator))
    ax.set_ylim(0)
    ax.grid()
    fig.autofmt_xdate()

    if save_plots:
        filename = format_filename(title)
        fig.savefig(f'{filename}.png', bbox_inches='tight')
        fig.savefig(f'{filename}.svg', bbox_inches='tight')

def plot_data_compare(
    data: List[np.array],
    title: str,
    legends: List[str],
    major_formatter_str: str,
    major_locator: Optional[mdates.RRuleLocator] = None,
    yaxis_multiple_locator: Optional[int] = None,
    colours: Union[str, List[str]] = 'blue',
) -> None:
    fig, ax = plt.subplots(1, 1)

    for i in range(len(data)):
        colour = colours if isinstance(colours, str) else colours[i]
        d = data[i]
        ax.plot(d[:,0], d[:,1], '-', color=colour)
        total_time = d[-1,1]
        legends[i] = legends[i] + f' ({total_time:g} h)'

    if include_plot_title:
        ax.set(title=title)
    ax.set(ylabel='cumulative time (h)')
    if major_locator:
        ax.xaxis.set_major_locator(major_locator)
    ax.xaxis.set_major_formatter(mdates.DateFormatter(major_formatter_str))
    if yaxis_multiple_locator:
        ax.yaxis.set_major_locator(ticker.MultipleLocator(yaxis_multiple_locator))
    ax.set_ylim(0)
    ax.legend(legends)#, loc='center', bbox_to_anchor=(0.3, 0.8))
    ax.grid()
    fig.autofmt_xdate()

    if save_plots:
        filename = format_filename(title)
        fig.savefig(f'{filename}.png', bbox_inches='tight')
        fig.savefig(f'{filename}.svg', bbox_inches='tight')

def main():
    plt.rc('text', usetex=True)
    plt.rc('font', family='serif', size=14)
    plt.rc('axes', titlesize=20)
    plt.rc('legend', fontsize=14)

    # Under File, Download -> Comma-separated values (.csv, current sheet),
    # download the 'Time' and 'Blog' sheets
    data_time = get_data('rmw_email time tracking - Code.csv')
    data_blog = get_data('rmw_email time tracking - Blog.csv')

    plot_data(
        data_time,
        'rmw\_email code time investment',
        '%Y %B',
        colour='green',
    )
    plot_data(
        data_blog,
        'rmw\_email blog post time investment',
        '%Y-%b-%d',
        mdates.DayLocator((1,5,10,15,20,25)),
        yaxis_multiple_locator=5,
        colour='blue',
    )

    plot_data_compare(
        [data_time, data_blog],
        'Overall rmw\_email time investment',
        ['code', 'blog post'],
        '%Y %B',
        colours=['green', 'blue'],
    )

    plt.show()

if __name__ == '__main__':
    main()


text:
"""

The script declare the functions used in get_data.py

"""

import pathlib
from logzero import logger, logfile
import matplotlib.pyplot as plt
import numpy as np

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LSTM

def create_log(log_name):
    tgt_path = pathlib.Path.cwd().parent.joinpath('log')
    logfile(tgt_path/log_name)     # Creating logfile

def analyze_data(df):
    logger.info('\n * Size of dataframe: {}\n'.format(df.shape))
    logger.info('* Column-wise NaNs can be identified as: ')
    logger.info('{}\n'.format(df.isnull().sum()))
    logger.info('Total NaNs:{}'.format(df.isnull().sum().sum()))

def plot_prices(df):
    plt.figure(figsize=(10,6))
    plt.grid(True)
    plt.xlabel('Date')
    plt.ylabel('Close Prices')
    plt.plot(df['Close'])
    plt.title('Historical Price of Cadila Healthcare')
    plt.savefig("../visualizations/historical_price.png")
    plt.close()

def create_dataset(dataset, time_step):      # time_step = 1 changed
    dataX, dataY = [], []
    for i in range(len(dataset)-time_step-1):    # To allow validation, i cannot not exceed len(df) - time_step
        a = dataset[i: (i+time_step), 0]         # Dataset is created, from i to time_step. This is input for prediction

        dataX.append(a)                          # append input to prediction to dataX
        dataY.append(dataset[i + time_step, 0])  # append the value (single) after time step to dataY. This is output

    return np.array(dataX), np.array(dataY)

def create_model():
    model = Sequential()      # Declaring sequential layer
    model.add(LSTM(50, return_sequences = True, input_shape = (100, 1) ))  # First LSTM layer--- Input must be equal to input step
    model.add(LSTM(50, return_sequences = True))   # Second LSTM layer
    model.add(LSTM(50))     #

    model.add(Dense(1))    # Final layer to obtain output
    model.compile(loss = 'mean_squared_error', optimizer = 'adam')   # Assignign loss metric and optimizer to the model

    logger.info("The model of following config was created: \n {}" .format(model.summary()))

    return model


text:
"""

4. SORM - Nonlinear function - Two Random variables
=====================================================

In the fourth example we have the limit state to be a nonlinear function of two (:math:`d=2`) random variables

.. math:: g(X_1, X_2) = X_1X_1 - 80

where :math:`X_1` follows a normal distribution with mean :math:`\mu_{X_1}=20` and standard deviation
:math:`\sigma_{X_1}=7` and :math:`X_2` follows a lognormal distribution with mean :math:`\mu_{X_2}=7` and
standard deviation :math:`\sigma_{X_2}=1.4`.
"""

#%% md
#
# Initially we have to import the necessary modules.

#%%

import shutil

import numpy as np
from UQpy.run_model.RunModel import RunModel
from UQpy.run_model.model_execution.PythonModel import PythonModel
from UQpy.distributions import Normal
from UQpy.reliability import FORM
from UQpy.reliability import SORM
from UQpy.distributions import Lognormal

m0 = 7
v0 = 1.4
mu = np.log(m0) - np.log(np.sqrt(1 + (v0 / m0) ** 2))
scale = np.exp(mu)
s = np.sqrt(np.log(1 + (v0 / m0) ** 2))
loc_ = 0.0

dist1 = Normal(loc=20., scale=2)
dist2 = Lognormal(s=s, loc=0.0, scale=scale)
model = PythonModel(model_script='pfn.py', model_object_name="example4",)
RunModelObject4 = RunModel(model=model)
form = FORM(distributions=[dist1, dist2], runmodel_object=RunModelObject4)
form.run()
Q0 = SORM(form_object=form)

# print results
print('SORM probability of failure: %s' % Q0.failure_probability)

max_stars_repo_name: SURGroup/UncertaintyQuantification

max_issues_repo_name: SURGroup/UncertaintyQuantification

max_forks_repo_name: SURGroup/UncertaintyQuantification


text:
import numpy
import cv2

flower_img = cv2.imread("./data/flower_pink.jpg")

pixel = flower_img[200, 250]

print(pixel)

# Change the pixel color value to blue
flower_img[300, 250] = (255, 0, 0)

# Change the pixel color value to green in a region range
flower_img[200:250, 200:350] = (0, 255, 0)

cv2.imshow("modified_pixel", flower_img)
cv2.waitKey(0)