text:
"""Data generators for the COCO dataset.

For a detailed explanation of data format see
https://cocodataset.org/#format-data

This data loader supports the following tasks in this dataset:
- Panoptic Segmentation, which combines semantic and instance segmentation
  such that all pixels are assigned a class label and all object instances are
  uniquely segmented.
"""

import functools
from typing import Optional
from absl import logging

from flax import jax_utils
import jax
import jax.numpy as jnp
from scenic.dataset_lib import dataset_utils
from scenic.dataset_lib import datasets
from scenic.dataset_lib.coco_dataset import coco_utils
from scenic.projects.baselines.detr import transforms
import tensorflow as tf
import tensorflow_datasets as tfds

# Computed from the training set by taking the per-channel mean/std-dev
# over sample, height and width axes of all training samples
MEAN_RGB = [0.48, 0.456, 0.406]
STDDEV_RGB = [0.229, 0.224, 0.225]

def make_coco_transforms(image_set, max_size=1333):
  """Returns a preprocessing function that operates on inputs and labels."""
  scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800]
  ratio = max_size / 1333.

  scales = [int(s * ratio) for s in scales]
  scales2 = [int(s * ratio) for s in [400, 500, 600]]

  normalize_boxes = transforms.NormalizeBoxes()
  init_padding_mask = transforms.InitPaddingMask()

  if image_set == 'train':
    return transforms.Compose(
        [transforms.RandomHorizontalFlip(),
         transforms.RandomSelect(
             transforms.RandomResize(scales, max_size=max_size),
             transforms.Compose([
                 transforms.RandomResize(scales2),
                 transforms.RandomSizeCrop(int(ratio * 384), int(ratio * 600)),
                 transforms.RandomResize(scales, max_size=max_size),
             ])),
         normalize_boxes,
         init_padding_mask])

  elif image_set == 'validation':
    return transforms.Compose(
        [transforms.Resize(max(scales), max_size=max_size),
         normalize_boxes,
         init_padding_mask])

  else:
    raise ValueError(f'Unknown image_set: {image_set}')

def decode_boxes(bbox, size):
  """Convert yxyx [0, 1] normalized boxes to xyxy unnormalized format."""
  y0, x0, y1, x1 = tf.split(bbox, 4, axis=-1)
  h = tf.cast(size[0], tf.float32)
  w = tf.cast(size[1], tf.float32)

  y0 = tf.clip_by_value(y0 * h, 0.0, h)
  x0 = tf.clip_by_value(x0 * w, 0.0, w)
  y1 = tf.clip_by_value(y1 * h, 0.0, h)
  x1 = tf.clip_by_value(x1 * w, 0.0, w)

  bbox = tf.concat([x0, y0, x1, y1], axis=-1)
  return bbox

def decode_coco_detection_example(example, input_range=None):
  """Given an instance and raw labels, creates <inputs, label> pair.

  Decoding includes.
  1. Converting images from uint8 [0, 255] to [0, 1.] float32.
  2. Mean subtraction and standardization using hard-coded mean and std.
  3. Convert boxes from yxyx [0-1] to xyxy un-normalized.
  4. Add 1 to all labels to account for background/padding object at label 0.
  5. Shuffling dictionary keys to be consistent with the rest of the code.

  Args:
    example: dict; Input image and raw labels.
    input_range: tuple; Range of input. By default we use Mean and StdDev
      normalization.

  Returns:
    A dictionary of {'inputs': input image, 'labels': task label}.
  """
  image = tf.image.convert_image_dtype(example['image'], dtype=tf.float32)

  ### normalize
  if input_range:
    image = image * (input_range[1] - input_range[0]) + input_range[0]
  else:
    mean_rgb = tf.constant(MEAN_RGB, shape=[1, 1, 3], dtype=tf.float32)
    std_rgb = tf.constant(STDDEV_RGB, shape=[1, 1, 3], dtype=tf.float32)
    image = (image - mean_rgb) / std_rgb

  boxes = decode_boxes(example['objects']['bbox'], tf.shape(image)[0:2])

  target = {
      'area': example['objects']['area'],
      'boxes': boxes,
      'objects/id': example['objects']['id'],
      'is_crowd': example['objects']['is_crowd'],
      'labels': example['objects']['label'] + 1,  # 0'th class is padding.
  }

  # Filters objects to exclude degenerate boxes.
  keep = tf.where(tf.logical_and(boxes[:, 2] > boxes[:, 0],
                                 boxes[:, 3] > boxes[:, 1]))[:, 0]
  target_kept = {k: tf.gather(v, keep) for k, v in target.items()}
  target_kept = target

  target_kept['orig_size'] = tf.cast(tf.shape(image)[0:2], dtype=tf.int32)
  target_kept['size'] = tf.identity(target_kept['orig_size'])
  target_kept['image/id'] = example['image/id']

  return {
      'inputs': image,
      'label': target_kept,
  }

def coco_load_split_from_tfds(batch_size,
                              *,
                              train,
                              preprocess_fn,
                              decode_fn,
                              cache=False,
                              max_size=1333,
                              max_boxes=100,
                              shuffle_buffer_size=1000,
                              shuffle_seed=0):
  """Loads a split from the COCO dataset using TensorFlow Datasets.

  Args:
    batch_size: int; The batch size returned by the data pipeline.
    train: bool; Whether to load the train or evaluation split.
    preprocess_fn: function; A function that given an example, train flag,
      and dtype returns the preprocessed the example. Note that the
      preprocessing is done BEFORE caching to re-use them.
    decode_fn: A function that given an example decodes the image, converts
      it to float32, mean-subtracts it, and pulls out the relevant parts from
      the tfds features.
    cache: bool; whether to use the ds.cache or nor.
    max_size: int; Maximum image size.
    max_boxes: int; Maximum number of boxes.
    shuffle_buffer_size: int; Size of the shuffle buffer.
    shuffle_seed: int; Seed for shuffling the training data.

  Returns:
    A `tf.data.Dataset`, and dataset info.
  """
  split = 'train' if train else 'validation'
  builder = tfds.builder('coco/2017')

  # Each host is responsible for a fixed subset of data.
  base_split_name, host_start, host_end = dataset_utils.get_data_range(
      builder, split, jax.process_index(), jax.process_count())
  data_range = tfds.core.ReadInstruction(
      base_split_name, unit='abs', from_=host_start, to=host_end)
  ds = builder.as_dataset(split=data_range, shuffle_files=False)
  options = tf.data.Options()
  options.threading.private_threadpool_size = 48
  ds = ds.with_options(options)
  ds = ds.map(decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE)
  if cache:
    ds = ds.cache()

  # TLDR: make sure max_boxes is set >=64.
  # NOTE: the number of boxes/labels always needs to be strictly larger than 63
  #   to ensure that there is at least one dummy target corresponding
  #   to an empty bounding box, and that the last target box is such a dummy
  #   empty target. This is needed for matching functions that in principle only
  #   produce matches with non-empty target boxes, and produce dummy matches
  #   with an empty target for the rest of the unmatched predicted boxes. The
  #   latter behaviour is necessary to ensure that the number of matches per
  #   datapoint is the same for all datapoints and shapes are static and jit
  #   compatible.
  padded_shapes = {
      'inputs': [max_size, max_size, 3],
      'padding_mask': [max_size, max_size],
      'label': {
          'area': [max_boxes,],
          'boxes': [max_boxes, 4],
          'objects/id': [max_boxes,],
          'is_crowd': [max_boxes,],
          'labels': [max_boxes,],
          'image/id': [],
          'orig_size': [2,],
          'size': [2,]
      },
  }

  if train:
    # First repeat then batch.
    ds = ds.shuffle(shuffle_buffer_size, seed=shuffle_seed)
    ds = ds.repeat()
    # Augmentation should be done after repeat for true randomness.
    ds = ds.map(preprocess_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE)
    ds = ds.padded_batch(batch_size, padded_shapes=padded_shapes,
                         drop_remainder=True)

  else:
    ds = ds.map(preprocess_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE)
    # First batch then repeat.
    ds = ds.padded_batch(batch_size, padded_shapes=padded_shapes,
                         drop_remainder=False)
    ds = ds.repeat()

  ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  return ds, builder.info

@datasets.add_dataset('coco_detr_detection')
def get_dataset(*,
                batch_size,
                eval_batch_size,
                num_shards,
                dtype_str='float32',
                shuffle_seed=0,
                rng=None,
                dataset_configs=None,
                dataset_service_address: Optional[str] = None):
  """Returns generators for COCO object detection 2017 train & validation set.

  Args:
    batch_size: int; Determines the train batch size.
    eval_batch_size: int; Determines the evaluation batch size.
    num_shards: int;  Number of shards --> batch shape: [num_shards, bs, ...].
    dtype_str: Data type of the image. Only 'float32' is currently supported.
    shuffle_seed: int; Seed for shuffling the training data.
    rng: JAX rng key, which can be used for augmentation, shuffling, etc.
    dataset_configs: dict; Dataset specific configurations. Must be empty.
    dataset_service_address: If set, will distribute the training dataset using
      the given tf.data service at the given address.

  Returns:
    A dataset_utils.Dataset() which includes a train_iter, a valid_iter,
    a test_iter, and a dict of meta_data.
  """
  del rng
  assert dtype_str == 'float32', (
      f'coco_detr_dataset invoked with unsupported dtype_str: {dtype_str}')
  del dtype_str

  dataset_configs = dataset_configs or {}

  max_size = dataset_configs.get('max_size', 1333)
  max_boxes = dataset_configs.get('max_boxes', 100)

  train_preprocess_fn = make_coco_transforms('train', max_size)
  eval_preprocess_fn = make_coco_transforms('validation', max_size)

  decode_fn = functools.partial(decode_coco_detection_example,
                                input_range=dataset_configs.get('input_range'))

  train_ds, train_ds_info = coco_load_split_from_tfds(
      batch_size, train=True,
      preprocess_fn=train_preprocess_fn,
      decode_fn=decode_fn,
      shuffle_buffer_size=dataset_configs.get('shuffle_buffer_size', 1000),
      max_size=max_size,
      max_boxes=max_boxes,
      shuffle_seed=shuffle_seed)

  if dataset_service_address:
    if shuffle_seed is not None:
      raise ValueError('Using dataset service with a random seed causes each '
                       'worker to produce exactly the same data. Add '
                       'config.shuffle_seed = None to your config if you '
                       'want to run with dataset service.')
    logging.info('Using the tf.data service at %s', dataset_service_address)
    train_ds = dataset_utils.distribute(train_ds, dataset_service_address)

  eval_ds, _ = coco_load_split_from_tfds(
      eval_batch_size,
      train=False,
      preprocess_fn=eval_preprocess_fn,
      max_size=max_size,
      max_boxes=max_boxes,
      decode_fn=decode_fn)

  # Labels take on values 1-80. We set 0 to be padded objects.
  num_classes = train_ds_info.features['objects']['label'].num_classes + 1

  maybe_pad_batches_train = functools.partial(
      dataset_utils.maybe_pad_batch, train=True, batch_size=batch_size)
  maybe_pad_batches_eval = functools.partial(
      dataset_utils.maybe_pad_batch, train=False, batch_size=eval_batch_size)
  shard_batches = functools.partial(dataset_utils.shard, n_devices=num_shards)

  train_iter = iter(train_ds)
  train_iter = map(dataset_utils.tf_to_numpy, train_iter)
  train_iter = map(maybe_pad_batches_train, train_iter)
  train_iter = map(shard_batches, train_iter)
  if dataset_configs.get('prefetch_to_device'):
    # Async bind batch to device which speeds up training.
    train_iter = jax_utils.prefetch_to_device(
        train_iter, dataset_configs.get('prefetch_to_device'))

  eval_iter = iter(eval_ds)
  eval_iter = map(dataset_utils.tf_to_numpy, eval_iter)
  eval_iter = map(maybe_pad_batches_eval, eval_iter)
  eval_iter = map(shard_batches, eval_iter)

  meta_data = {
      'num_classes':
          num_classes,
      'input_shape': [-1, max_size, max_size, 3],
      'num_train_examples':
          dataset_utils.get_num_examples('coco/2017', 'train'),
      'num_eval_examples':
          dataset_utils.get_num_examples('coco/2017', 'validation'),
      'input_dtype':
          jnp.float32,
      'target_is_onehot':
          False,
      'label_to_name':
          coco_utils.get_label_map('coco/2017_panoptic'),
  }
  return dataset_utils.Dataset(train_iter, eval_iter, None, meta_data)

max_stars_repo_name: NielsRogge/scenic

max_issues_repo_name: NielsRogge/scenic

max_forks_repo_name: NielsRogge/scenic

reason: import jax


text:
"""Utils
"""
import pprint
import scipy.misc
import numpy as np
import matplotlib.pyplot as plt
from time import gmtime, strftime

pp = pprint.PrettyPrinter()

def imread(path, is_grayscale=False):
    if is_grayscale:
        return scipy.misc.imread(path, flatten=True).astype(np.float)
    else:
        return scipy.misc.imread(path).astype(np.float)

def imsave(images, path):
    """

    :param images:
    :param path:
    :return:
    """
    return scipy.misc.imsave(path, images)

def inverse_transform(images):
    return (images + 1) / 2.

def generate_images(model, test_input):
    prediction = model(test_input)

    plt.figure(figsize=(12, 12))

    display_list = [test_input[0], prediction[0]]
    title = ['Input Image', 'Predicted Image']

    for i in range(2):
        plt.subplot(1, 2, i + 1)
        plt.title(title[i])
        # getting the pixel values between [0, 1] to plot it.
        plt.imshow(display_list[i] * 0.5 + 0.5)
        plt.axis('off')
    plt.savefig('fig.png')

ext: py

max_stars_repo_name: nallab/style-transfer

max_issues_repo_name: nallab/style-transfer

max_forks_repo_name: nallab/style-transfer

reason: import numpy,import scipy


text:
# Copyright (C) 2016-2019 Semtech (International) AG. All rights reserved.
#
# This file is subject to the terms and conditions defined in file 'LICENSE',
# which is part of this source code package.

from typing import Any, Awaitable, Callable, List, MutableMapping, Optional, Set, TextIO, Tuple, Union
from typing import cast

import argparse
import asyncio
import math
import numpy
import os
import struct
import sys

from intelhex import IntelHex
from intervaltree import Interval, IntervalTree
from peripherals import InterruptController, GPIO, UART
from vtimeloop import VirtualTimeLoop

import unicorn as uc
import unicorn.arm_const as uca

import loramsg as lm

class Rps:
    @staticmethod
    def makeRps(sf:int=7, bw:int=125000, cr:int=1, crc:int=1, ih:int=0) -> int:
        return ((sf-6) | ([125000,250000,500000].index(bw)<<3)
                | ((cr-1)<<5) | ((crc^1)<<7) | ((ih&0xFF)<<8)) if sf else 0

    @staticmethod
    def getSf(rps:int) -> int:
        sf = rps & 0x7
        return sf + 6 if sf else 0

    @staticmethod
    def getBw(rps:int) -> int:
        return (1 << ((rps >> 3) & 0x3)) * 125000

    @staticmethod
    def getCr(rps:int) -> int:
        return ((rps >> 5) & 0x3) + 1

    @staticmethod
    def getCrc(rps:int) -> int:
        return ((rps >> 7) & 0x1) ^ 1

    @staticmethod
    def getIh(rps:int) -> int:
        return (rps >> 8) & 0xff

    @staticmethod
    def getParams(rps:int) -> Tuple[int,int,int,int,int]:
        return (Rps.getSf(rps),
                Rps.getBw(rps),
                Rps.getCr(rps),
                Rps.getCrc(rps),
                Rps.getIh(rps))

    @staticmethod
    def validate(rps:int) -> None:
        (sf, bw, cr, crc, ih) = Rps.getParams(rps)
        if sf:
            assert bw in [125000,250000,500000], f'unsupported bw: {bw}'
            assert sf >= 7 and sf <= 12,         f'unsupported sf: {sf}'
            assert cr >= 1 and cr <= 4,          f'unsupported cr: {cr}'
            assert ih==0 or ih==1,               f'unsupported ih: {ih}'
            assert crc==0 or crc==1,             f'unsupported crc: {crc}'

    @staticmethod
    def isFSK(rps:int) -> bool:
        return (rps & 0x7) == 0

class LoraMsg:
    def __init__(self, time:float, pdu:bytes, freq:int, rps:int,
            xpow:Optional[int]=None, snr:Optional[int]=None,
            dro:Optional[int]=None, npreamble:int=8,
            rssi:Optional[float]=None) -> None:
        Rps.validate(rps)
        (sf, bw, cr, crc, ih) = Rps.getParams(rps)
        if sf:
            if dro is None:
                dro = 1 if ((sf>=11 and bw==125000)
                        or  (sf==12 and bw==250000)) else 0
            else:
                assert dro==0 or dro==1
            assert len(pdu) >= 0 and len(pdu) <= 255
        else:
            # TODO: FSK
            dro = 0

        self.pdu = pdu
        self.freq = freq
        self.rps = rps
        self.xpow = xpow
        self.snr = snr
        self.dro = dro
        self.npreamble = npreamble
        self.rssi = rssi

        self.xbeg = time
        self.xend = time + self.airtime()

    def __str__(self) -> str:
        sf = Rps.getSf(self.rps)
        bw = Rps.getBw(self.rps)
        return 'xbeg=%.6f, xend=%.6f, freq=%d, %s, pdu=%s' % (
                self.xbeg, self.xend,
                self.freq,
                'sf=%d, bw=%d' % (sf, bw) if sf else 'fsk',
                self.pdu.hex())

    def __repr__(self) -> str:
        return 'LoraMsg<%s>' % self.__str__()

    # xbeg, xend, freq, rps, xpow, snr, dlen, data....
    SIM_RXTX_SPEC = '<iiIIiiI'
    SIM_RXTX_SPEC_SZ = struct.calcsize(SIM_RXTX_SPEC)
    SIM_RXTX_SPEC_MAXSZ = SIM_RXTX_SPEC_SZ + 256

    @staticmethod
    def from_simrxtx(rxtx:bytes, c:Optional[int]=None) -> 'LoraMsg':
        (xbeg, xend, freq, rps, xpow, snr, dlen) = struct.unpack_from(
                LoraMsg.SIM_RXTX_SPEC, rxtx)
        return LoraMsg(Simulation.ticks2time(xbeg, c),
                rxtx[LoraMsg.SIM_RXTX_SPEC_SZ:LoraMsg.SIM_RXTX_SPEC_SZ+dlen],
                freq, rps, xpow, snr)

    def simrxtx(self) -> bytes:
        return struct.pack(LoraMsg.SIM_RXTX_SPEC,
                numpy.int32(Simulation.time2ticks(self.xbeg)),
                numpy.int32(Simulation.time2ticks(self.xend)),
                self.freq, self.rps, self.xpow or 0, self.snr or 0,
                len(self.pdu)) + self.pdu

    def match(self, freq:int, rps:int) -> bool:
        return (self.freq == freq) and (Rps.isFSK(rps) if Rps.isFSK(self.rps)
                else (self.rps == rps))

    @staticmethod
    def symtime(rps:int, nsym:int=1) -> float:
        (sf, bw, cr, crc, ih) = Rps.getParams(rps)
        if sf == 0:
            return 8*nsym / 50000
        # Symbol rate / time for one symbol (secs)
        Rs = bw / (1<<sf)
        Ts = 1 / Rs
        return nsym * Ts

    def airtime(self) -> float:
        Ts = LoraMsg.symtime(self.rps)
        if Rps.isFSK(self.rps):
            return (5+3+1+2+len(self.pdu)) * Ts
        # Length/time of preamble
        Tpreamble = (self.npreamble + 4.25) * Ts
        # Symbol length of payload and time
        (sf, bw, cr, crc, ih) = Rps.getParams(self.rps)
        tmp = math.ceil(
                (8*len(self.pdu) - 4*sf + 28 + 16*crc - ih*20)
                / (4*sf-self.dro*8)) * (cr+4)
        npayload = 8 + max(0,tmp)
        Tpayload = npayload * Ts
        # Time on air
        return Tpreamble + Tpayload

    def tpreamble(self) -> float:
        return LoraMsg.symtime(self.rps, self.npreamble)

class Medium:
    def __init__(self, put_up:Optional[Callable[[LoraMsg],None]]) -> None:
        self._put_up = put_up

    def reset_medium (self) -> None:
        raise NotImplementedError()

    def get_dn(self, rxon:int, rxtout:int, freq:int, rps:int, nsym:int=4, peek=False) -> Optional[LoraMsg]:
        raise NotImplementedError()

    def prune(self, ticks:int) -> List[LoraMsg]:
        raise NotImplementedError()

    def add_dn(self, msg:LoraMsg) -> None:
        raise NotImplementedError()

class SimpleMedium(Medium):
    def __init__(self, put_up:Optional[Callable[[LoraMsg],None]]) -> None:
        self._put_up = put_up
        self.msgs = IntervalTree()

    def reset_medium (self) -> None:
        self.msgs.clear()

    def add_dn(self, msg:LoraMsg) -> None:
        t0 = Simulation.time2ticks(msg.xbeg)
        t1 = t0 + Simulation.time2ticks(msg.tpreamble())
        self.msgs[t0:t1] = msg

    @staticmethod
    def overlap(i1:Interval, i2:Interval) -> int:
        return min(i1.end, i2.end) - max(i1.begin, i2.begin) # type: ignore

    def get_dn(self, rxon:int, rxtout:int, freq:int, rps:int, nsym:int=4, peek=False) -> Optional[LoraMsg]:
        rxw = Interval(rxon, rxon + rxtout)
        tpn = Simulation.time2ticks(LoraMsg.symtime(rps, nsym))
        for i in self.msgs.overlap(rxw[0], rxw[1]):
            m = i.data # type: LoraMsg
            if m.match(freq, rps) and (peek or SimpleMedium.overlap(i, rxw) >= tpn):
                break
        else:
            return None
        if not peek:
            self.msgs.remove(i)
        return m

    def prune(self, ticks:int) -> List[LoraMsg]:
        exp = cast(List[Interval], self.msgs.envelop(0, ticks))
        if exp:
            self.msgs.remove_envelop(0, ticks)
        return [iv[2] for iv in exp]

class TraceWriter:
    def __init__(self, buf:TextIO) -> None:
        self.buf = buf

    def write(self, s:str, **kwargs:Any) -> None:
        self.buf.write(s)

class TrafficTrace:
    def __init__(self, io:Union[TextIO,TraceWriter]) -> None:
        if not isinstance(io, TraceWriter):
            io = TraceWriter(io)
        self.tw = io

    def trace(self, msg:LoraMsg, d2n:bool, lost:bool=False) -> None:
        self.tw.write('%s->%s: %s%s\n' % (
            'D' if d2n else 'N', 'x' if lost else 'N' if d2n else 'D',
            msg, self.msginfo(msg)),
            strike=lost)

    def msginfo(self, msg:LoraMsg) -> str:
        return ''

class SimEvent:
    def __init__(self) -> None:
        self.e = asyncio.Event()
        self.s:Set[str] = set()

    def set(self, key:str) -> None:
        self.s.add(key)
        self.e.set()

    def clear(self, key:str) -> None:
        self.s.discard(key)
        if not self.s:
            self.e.clear()

    def is_set(self, key:str) -> bool:
        return key in self.s

class Simulation:
    RAM_BASE   = 0x10000000
    FLASH_BASE = 0x20000000
    EE_BASE    = 0x30000000

    class ResetException(BaseException):
        pass

    @staticmethod
    def time2ticks(t:float) -> int:
        return int(t * 32768)

    @staticmethod
    def ticks2time(t:int, c:Optional[int]=None) -> float:
        if c:
            t = Simulation.ticks_extend(t, c)
        return t / 32768

    @staticmethod
    def ticks_extend(t:int, c:int) -> int:
        return c + int(numpy.int32((int(numpy.int32(t)) - int(numpy.int32(c)))))

    def now2ticks(self) -> int:
        return Simulation.time2ticks(asyncio.get_event_loop().time() - self.epoch)

    @property
    def now(self) -> float:
        return asyncio.get_event_loop().time() - self.epoch

    async def sleep_until(self, deadline:float) -> None:
        delay = deadline - self.now
        await asyncio.sleep(delay)

    def __init__(self, put_up:Optional[Callable[[LoraMsg],None]],
                 hexfiles:List[str], debug:Optional[TraceWriter]=None, traffic:Optional[TrafficTrace]=None,
                 ramsz:int=16*1024, flashsz:int=128*1024, eesz:int=8*1024) -> None:

        self.medium:Optional[Medium] = SimpleMedium(put_up)

        self.emu = uc.Uc(uc.UC_ARCH_ARM, uc.UC_MODE_THUMB)
        #self.emu.hook_add(uc.UC_HOOK_CODE,
        #        lambda uc, addr, size, sim: sim.trace(addr), self)
        self.emu.hook_add(uc.UC_HOOK_INTR,
                lambda uc, intno, sim: sim.intr(intno), self)
        self.emu.hook_add(uc.UC_HOOK_BLOCK,
                lambda uc, address, size, sim:
                sim.block_special(address), self,
                begin=0xfffff000, end=0xffffffff)

        self.emu.mem_map(Simulation.RAM_BASE, ramsz)
        self.emu.mem_map(Simulation.FLASH_BASE, flashsz)
        self.emu.mem_map(Simulation.EE_BASE, eesz)
        self.emu.mem_map(0xfffff000, 0x1000) # special

        self.ex:Optional[BaseException] = None

        # Peripherals
        self.event = SimEvent()
        self.ic = InterruptController(self.event)
        self.gpio = GPIO(self.ic, 0)
        self.uart = UART(self.ic, 1)

        # Load firmware
        for hf in hexfiles:
            self.load_hexfile(hf)

        # Debug settings
        self.debug = debug
        self.traffic = traffic

        # read SP and entry point address from header in flash
        (sp, ep) = struct.unpack('<II',
                self.emu.mem_read(Simulation.FLASH_BASE, 8))
        self.isp = sp
        self.ipc = ep

        self._reset()

        self.rxparams:MutableMapping[str,int] = {}
        self.rxmsg:Optional[LoraMsg] = None

        self.evlog : asyncio.Queue[Tuple[int,int,int]] = asyncio.Queue()

    def load_hexfile(self, hexfile:str) -> None:
        ih = IntelHex()
        ih.loadhex(hexfile)
        beg = ih.minaddr()
        end = ih.maxaddr() + 1
        mem = bytes(ih.gets(beg, end - beg))
        try:
            self.emu.mem_write(beg, mem)
        except:
            print('Error loading %s at 0x%08x (%d bytes):' % (hexfile, beg, len(mem)))
            raise

    @property
    def isHW(self) -> bool:
        return False

    def filterTraffic(self, deveui:Optional[str], devaddr:Optional[int]) -> None:
        pass

    def step(self) -> int:
        exp = self.medium.prune(self.ticks if not self.rxing else self.rxparams['rxbeg'])
        if exp and self.traffic:
            for lm in exp:
                self.traffic.trace(lm, False, lost=True)
        self.emu.emu_start(self.pc, 0xffffffff)
        if self.ex is not None:
            raise self.ex
        return self.sleep

    @staticmethod
    async def _sleepto(ticks:int, ev:Optional[asyncio.Event]) -> None:
        if ticks > 0:
            tts = Simulation.ticks2time(ticks)
            if ev:
                try:
                    await asyncio.wait_for(asyncio.shield(ev.wait()), timeout=tts)
                except asyncio.TimeoutError:
                    pass
            else:
                await asyncio.sleep(tts)

    async def vsleepto(self, tticks:int, ev:Optional[asyncio.Event]) -> int:
        await Simulation._sleepto(tticks - self.now2ticks(), ev)
        return self.now2ticks()

    async def rsleepto(self, tticks:int, ev:Optional[asyncio.Event]) -> int:
        now = self.now2ticks()
        # get closer with sleep
        while ev is None or not ev.is_set():
            tts = (tticks - now) - 10
            if tts <= 0:
                break
            await Simulation._sleepto(tts, ev)
            now = self.now2ticks()
        # last few steps busy-wait
        if ev and ev.is_set():
            return now
        while now < tticks:
            now = self.now2ticks()
        return tticks # pretend we didn't overshoot

    def check_irqs(self) -> bool:
        if self.irq_enabled() and self.event.is_set('irq'):
            self.irq_invoke()
            return True
        else:
            return False

    async def shutdown(self) -> None:
        pass

    async def run(self) -> None:
        sleepto = self.vsleepto if isinstance(asyncio.get_event_loop(),
                VirtualTimeLoop) else self.rsleepto
        self.ticks = self.now2ticks()
        while True:
            nt = self.step()
            if self.check_irqs():
                continue
            self.ticks = await sleepto(nt, self.event.e)
            self.check_irqs()
            self.event.clear('reset')

    def _reset(self) -> None:
        self.vtor:Optional[int] = None

        self.gpio.reset()
        self.uart.reset()

        self.irq_invoking = False

        self.rxing:bool = False

        self.pc = self.ipc
        self.emu.reg_write(uca.UC_ARM_REG_SP, self.isp)
        self.emu.reg_write(uca.UC_ARM_REG_LR, 0xcafebabe)
        self.emu.reg_write(uca.UC_ARM_REG_CPSR, 0x33)

        self.epoch = asyncio.get_event_loop().time()
        self.ticks = 0
        self.sleep = 0

        self.event.set('reset')
        self.medium.reset_medium()

    async def reset(self) -> None:
        self._reset()

    async def get_evlog(self) -> Tuple[int,int,int]:
        return await self.evlog.get()

    def clear_evlog(self) -> List[Tuple[int,int,int]]:
        l : List[Tuple[int,int,int]] = []
        try:
            while True:
                l.append(self.evlog.get_nowait())
        except asyncio.QueueEmpty:
            pass
        return l

    def irq_enabled(self) -> bool:
        return (self.get_cpsr() & (1 << 7)) == 0

    def get_cpsr(self) -> int:
        return cast(int, self.emu.reg_read(uca.UC_ARM_REG_CPSR))

    def stack_push(self, value:int) -> None:
        sp = self.emu.reg_read(uca.UC_ARM_REG_SP) - 4
        self.emu.mem_write(sp, struct.pack('<I', value))
        self.emu.reg_write(uca.UC_ARM_REG_SP, sp)

    def stack_pop(self) -> int:
        sp = self.emu.reg_read(uca.UC_ARM_REG_SP)
        (value,) = cast(Tuple[int], struct.unpack('<I', self.emu.mem_read(sp, 4)))
        self.emu.reg_write(uca.UC_ARM_REG_SP, sp + 4)
        return value

    def irq_invoke(self) -> None:
        if self.irq_invoking:
            raise RuntimeError('Nested interrupt invocation')
        if self.vtor is None:
            raise RuntimeError('VTOR not initialized')
        self.irq_invoking = True
        while self.event.is_set('irq') and self.irq_enabled():
            irq = self.ic.next()
            (addr,) = struct.unpack('<I',
                    self.emu.mem_read(self.vtor + 4*irq, 4))
            # push LR to stack
            self.stack_push(self.emu.reg_read(uca.UC_ARM_REG_LR))
            # set LR to magic value
            self.emu.reg_write(uca.UC_ARM_REG_LR, 0xfffffff1)
            self.pc = addr
            self.step()
        self.irq_invoking = False

    def get_string(self, addr:int, length:int) -> str:
        return self.emu.mem_read(addr, length).decode('utf-8')

    def svc_panic(self, params:Tuple[int,int,int], lr:int) -> bool:
        raise RuntimeError(
                'PANIC: type=%d (%s), reason=0x%x, addr=0x%08x, lr=0x%08x'
                % (params[0], {0: 'ex', 1: 'bl', 2: 'fw'}.get(params[0], '??'),
                    params[1], params[2], lr))

    def svc_debug_str(self, params:Tuple[int,int,int], lr:int) -> bool:
        if self.debug is not None:
            self.debug.write(self.get_string(params[0], params[1]))
        return True

    def svc_ticks(self, params:Tuple[int,int,int], lr:int) -> bool:
        self.emu.reg_write(uca.UC_ARM_REG_R0, numpy.uint32(self.ticks))
        self.emu.reg_write(uca.UC_ARM_REG_R1, numpy.uint32((self.ticks >> 32)))
        return True

    def svc_sleep(self, params:Tuple[int,int,int], lr:int) -> bool:
        if params[0] == 2: # forever
            target = self.ticks + Simulation.time2ticks(10 * 3600)
        else:
            target = Simulation.ticks_extend(params[1], self.ticks)
        if target > self.ticks:
            self.sleep = target
            self.emu.reg_write(uca.UC_ARM_REG_R0, 1)
            return False
        else:
            self.emu.reg_write(uca.UC_ARM_REG_R0, 0)
            return True

    def svc_tx(self, params:Tuple[int,int,int], lr:int) -> bool:
        m = LoraMsg.from_simrxtx(bytes(self.emu.mem_read(params[0],
            LoraMsg.SIM_RXTX_SPEC_MAXSZ)), self.ticks)
        if self.traffic is not None:
            self.traffic.trace(m, True)
        self.medium._put_up(m)
        return True

    def svc_rx_start(self, params:Tuple[int,int,int], lr:int) -> bool:
        self.rxparams['freq'] = params[0]
        self.rxparams['rps'] = params[1]
        self.rxparams['rxbeg'] = self.ticks
        self.rxparams['rxtout'] = params[2]
        self.rxing = True
        return True

    def svc_rx_on(self, params:Tuple[int,int,int], lr:int) -> int:
        self.rxparams['freq'] = params[0]
        self.rxparams['rps'] = params[1]
        self.rxparams['rxbeg'] = self.ticks
        self.rxparams['rxtout'] = params[2]
        self.rxing = True
        v = 0
        m = self.medium.get_dn(
            self.rxparams['rxbeg'], self.rxparams['rxtout'],
            self.rxparams['freq'], self.rxparams['rps'], peek=True)
        if m:
            # Don't return 0 as deadline here - 0 means there's no frame
            v = numpy.int32(Simulation.time2ticks(m.xbeg + m.tpreamble())) or 1
        self.emu.reg_write(uca.UC_ARM_REG_R0, v)
        return True

    def svc_rx_cad(self, params:Tuple[int,int,int], lr:int) -> int:
        self.rxparams['freq'] = params[0]
        self.rxparams['rps'] = params[1]
        self.rxparams['rxbeg'] = self.ticks
        self.rxparams['rxtout'] = LoraMsg.symtime(params[1], params[2])
        v = 0
        m = self.medium.get_dn(
            self.rxparams['rxbeg'], self.rxparams['rxtout'],
            self.rxparams['freq'], self.rxparams['rps'], peek=True)
        if m:
            v = 1  # activity detected
        self.emu.reg_write(uca.UC_ARM_REG_R0, v)
        return True

    def get_dn(self) -> Optional[LoraMsg]:
        return None if self.medium is None else self.medium.get_dn(
                self.rxparams['rxbeg'], self.rxparams['rxtout'],
                self.rxparams['freq'], self.rxparams['rps'])

    def svc_rx_do(self, params:Tuple[int,int,int], lr:int) -> bool:
        m = self.get_dn()
        self.rxmsg = m
        if m is None:
            self.rxing = False
            r = 0
        else:
            r = numpy.int32(Simulation.time2ticks(m.xend)) or 1
        self.emu.reg_write(uca.UC_ARM_REG_R0, r)
        return True

    def svc_rx_done(self, params:Tuple[int,int,int], lr:int) -> bool:
        m = self.rxmsg
        if m:
            if self.traffic is not None:
                self.traffic.trace(m, False)
            self.emu.mem_write(params[0], m.simrxtx())
        self.emu.reg_write(uca.UC_ARM_REG_R0, 1 if m else 0)
        self.rxing = False
        return True

    def svc_rng_seed(self, params:Tuple[int,int,int], lr:int) -> bool:
        self.emu.reg_write(uca.UC_ARM_REG_R0, 0x12345678)
        return True

    def svc_vtor(self, params:Tuple[int,int,int], lr:int) -> bool:
        self.vtor = params[0]
        return True

    def svc_irq(self, params:Tuple[int,int,int], lr:int) -> bool:
        if not self.irq_invoking:
            self.sleep = self.ticks
            return False
        else:
            return True

    def svc_pio(self, params:Tuple[int,int,int], lr:int) -> bool:
        lut: List[Callable] = [
                lambda: self.gpio.config(self.emu.mem_read(params[1], 6 * 4)),
                lambda: self.emu.reg_write(uca.UC_ARM_REG_R0, self.gpio.read()),
                lambda: self.emu.reg_write(uca.UC_ARM_REG_R0,
                    self.gpio.read_irq()),
                lambda: self.gpio.clear_irq(params[1]) ]
        lut[params[0]]()
        return True

    def svc_uart(self, params:Tuple[int,int,int], lr:int) -> bool:
        cmd, p1, p2 = params
        if cmd == 0:
            if p1 == 0:
                v = self.uart.sr
            else:
                v = self.uart.read_rx()
            self.emu.reg_write(uca.UC_ARM_REG_R0, v)
        elif cmd == 1:
            self.uart.tx_char(p1)
        elif cmd == 2:
            self.uart.config(p1)
        elif cmd == 3:
            if p1 == 0:
                self.uart.setrx(not not p2)
            else:
                self.uart.settx(not not p2)
        else:
            raise KeyError('Unknown UART command %d' % cmd)

        return True

    def svc_reset(self, params:Tuple[int,int,int], lr:int) -> bool:
        raise Simulation.ResetException()

    def svc_log_ev(self, params:Tuple[int,int,int], lr:int) -> bool:
        self.evlog.put_nowait(params)

    svc_lookup = {
            0   : svc_panic,
            128 : svc_debug_str,
            129 : svc_ticks,
            130 : svc_sleep,
            131 : svc_tx,
            132 : svc_rx_start,
            133 : svc_rx_do,
            134 : svc_rx_done,
            135 : svc_rng_seed,
            136 : svc_vtor,
            137 : svc_irq,
            138 : svc_pio,
            139 : svc_uart,
            140 : svc_reset,
            141 : svc_rx_on,
            142 : svc_rx_cad,
            143 : svc_log_ev,
            }

    def trace(self, addr:int) -> None:
        print('PC=%08x' % addr)

    def intr(self, intno:int) -> None:
        try:
            lr = self.emu.reg_read(uca.UC_ARM_REG_LR)
            if intno == 2: # SVC
                svcid = self.emu.reg_read(uca.UC_ARM_REG_R0)
                handler = Simulation.svc_lookup.get(svcid)
                if handler:
                    params = (self.emu.reg_read(uca.UC_ARM_REG_R1),
                            self.emu.reg_read(uca.UC_ARM_REG_R2),
                            self.emu.reg_read(uca.UC_ARM_REG_R3))
                    try:
                        if handler(self, params, lr):
                            self.emu.reg_write(uca.UC_ARM_REG_PC, lr)
                        else:
                            self.pc = lr
                            self.emu.emu_stop()
                    except Simulation.ResetException:
                        self._reset()
                        self.emu.emu_stop()
                else:
                    raise RuntimeError('Unknown SVCID %d, lr=0x%08x'
                            % (svcid, lr))
            else:
                raise RuntimeError('Unexpected interrupt %d, lr=0x%08x'
                        % (intno, lr))
        except BaseException as ex:
            self.ex = ex
            self.emu.emu_stop()

    def block_special(self, address:int) -> None:
        if address == 0xfffffff0:
            self.pc = self.stack_pop()
            self.emu.emu_stop()
        else:
            raise RuntimeError('Unknown special PC: 0x%08x' % address)

    gpio_cmd_pin:int = 0
    gpio_rst_pin:int = 0
    gpio_rdy_pin:int = 0

    async def modemInit(self, gpio_cmd_pin:int, gpio_rst_pin:int, gpio_rdy_pin:int) -> None:
        self.gpio_cmd_pin = gpio_cmd_pin
        self.gpio_rst_pin = gpio_rst_pin
        self.gpio_rdy_pin = gpio_rdy_pin
        self.gpio.drive(gpio_cmd_pin, 1)
        self.gpio.extconfig(gpio_rst_pin, epup=True)
        await asyncio.sleep(1)

    async def modemSend(self, pdu:bytes) -> None:
        self.gpio.drive(self.gpio_cmd_pin, 0)
        await asyncio.sleep(0.01)
        await self.uart.xfr_todevice(pdu)
        await asyncio.sleep(0.025)
        self.gpio.drive(self.gpio_cmd_pin, 1)

    async def modemRecv(self) -> bytes:
        buf = bytearray()
        buf.extend(await self.uart.xfr_fromdevice(2))
        buf.extend(await self.uart.xfr_fromdevice(buf[1] + 1))
        await asyncio.sleep(0.1)
        return bytes(buf)

    async def modemWaitDataready(self, timeout:Optional[int]=None) -> None:
        await asyncio.wait_for(self.gpio.waitfor(self.gpio_rdy_pin, True), timeout=timeout)

    async def modemDataready(self) -> bool:
        return self.gpio[self.gpio_rdy_pin]

    async def modemWaitHostreset(self, timeout:Optional[int]=None) -> None:
        assert self.gpio[self.gpio_rst_pin]
        await asyncio.wait_for(self.gpio.waitfor(self.gpio_rst_pin, True), timeout=timeout)
        await asyncio.wait_for(self.gpio.waitfor(self.gpio_rst_pin, False), timeout=2)

if __name__ == '__main__':
    p = argparse.ArgumentParser()
    p.add_argument('-v', '--virtual-time', action='store_true',
            help='Use virtual time')
    p.add_argument('hexfiles', metavar='HEXFILE', nargs='+',
            help='Firmware files to load')
    args = p.parse_args()

    asyncio.get_event_loop().run_until_complete(Simulation(args.hexfiles).run())

max_stars_repo_name: povik/basicmac

max_issues_repo_name: povik/basicmac

max_forks_repo_name: povik/basicmac

reason: import numpy


text:
"""
A collection of unrefactored functions.
"""
import os
import sys
import numpy as np
import argparse
import logging
import open3d as o3d

sys.path.append("/home/akin/workspace/workspace_applications/Deep_3D_Search/FCGF_submit/")

from lib.timer import Timer, AverageMeter

from util.misc import extract_features

from model import load_model
from util.file import ensure_dir, get_folder_list, get_file_list
from util.trajectory import read_trajectory, write_trajectory
from util.pointcloud import make_open3d_point_cloud, evaluate_feature_3dmatch
from scripts.benchmark_util import do_single_pair_matching, gen_matching_pair, gather_results

import torch

from pathlib import Path

import MinkowskiEngine as ME

ch = logging.StreamHandler(sys.stdout)
logging.getLogger().setLevel(logging.INFO)
logging.basicConfig(
    format='%(asctime)s %(message)s', datefmt='%m/%d %H:%M:%S', handlers=[ch])

o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Error)

def main(args):

    target_dir = Path(args.target)
    target_dir.mkdir(exist_ok=True, parents=True)

    device = torch.device('cuda' if args.with_cuda else 'cpu')

    checkpoint = torch.load(args.model)
    config = checkpoint['config']

    num_feats = 1
    Model = load_model(config.model)
    model = Model(
        num_feats,
        config.model_n_out,
        bn_momentum=0.05,
        normalize_feature=config.normalize_feature,
        conv1_kernel_size=config.conv1_kernel_size,
        D=3)
    model.load_state_dict(checkpoint['state_dict'])
    model.eval()

    model = model.to(device)

    voxel_size = config.voxel_size

    with torch.no_grad():

        fi = args.source_file
        pcd = o3d.io.read_point_cloud(fi)

        xyz_down, feature = extract_features(
                  model,
                  xyz=np.array(pcd.points),
                  rgb=None,
                  normal=None,
                  voxel_size=voxel_size,
                  device=device,
                  skip_check=True)

        fi_base = os.path.basename(fi)
        print('fi_base = ', fi_base)
        target_path = args.target
        path_save_feature = target_path + fi_base.split('.')[0] + ".npz"
        print('path_save_feature = ', path_save_feature)

        np.savez_compressed(
            path_save_feature,
            points=np.array(pcd.points),
            xyz=xyz_down,
            feature=feature.detach().cpu().numpy())

if __name__ == '__main__':

    parser = argparse.ArgumentParser()

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--source_file', default=None, type=str, help='path to ply file')
    parser.add_argument(
        '--source_high_res',
        default=None,
        type=str,
        help='path to high_resolution point cloud')
    parser.add_argument(
        '--target', default=None, type=str, help='path to produce generated data')
    parser.add_argument(
        '-m',
        '--model',
        default=None,
        type=str,
        help='path to latest checkpoint (default: None)')
    parser.add_argument(
        '--voxel_size',
        default=0.05,
        type=float,
        help='voxel size to preprocess point cloud')
    parser.add_argument('--with_cuda', action='store_true')
    parser.add_argument(
        '--num_rand_keypoints',
        type=int,
        default=5000,
        help='Number of random keypoints for each scene')

    args = parser.parse_args()

    main(args)


text:
import sys
import argparse
from glob import glob
from datetime import datetime, timedelta

# data tools
import h5py
import pandas as pd
import numba as nb
import numpy as np

import warnings
warnings.filterwarnings("ignore")

# custom tools
sys.path.insert(0, '/glade/u/home/ksha/WORKSPACE/utils/')
sys.path.insert(0, '/glade/u/home/ksha/WORKSPACE/Analog_BC/utils/')
sys.path.insert(0, '/glade/u/home/ksha/WORKSPACE/Analog_BC/')
sys.path.insert(0, '/glade/u/home/ksha/PUBLISH/fcstpp/')

from sklearn.metrics import brier_score_loss
from fcstpp import metrics, utils
import analog_utils as ana
import data_utils as du
from namelist import *

def reliability_diagram(cate_true, prob_model, bins):
    binids = np.searchsorted(bins, prob_model)
    bin_sums = np.bincount(binids, weights=prob_model, minlength=len(bins))
    bin_true = np.bincount(binids, weights=cate_true, minlength=len(bins))
    bin_total = np.bincount(binids, minlength=len(bins))
    flag = bin_total > 0
    prob_true = bin_true/bin_total
    prob_pred = bin_sums/bin_total
    prob_true[~flag] = np.nan
    return prob_true, prob_pred

def fcst_to_prob(data, thres, mon_inds):
    dim1, dim2, dim3, dim4 = data.shape

    out = np.empty((dim1, dim2, dim3, dim4))

    for i in range(dim1):
        thres_ = thres[mon_inds[i], :]
        for j in range(dim2):
            for k in range(dim3):
                out[i, j, k, :] = data[i, j, k, :] > thres_
    return np.sum(out, axis=2)/dim3

def fcst_to_flag(data, thres, mon_inds):
    dim1, dim2, dim3 = data.shape
    out = np.empty((dim1, dim2, dim3))

    for i in range(dim1):
        thres_ = thres[mon_inds[i], :]
        for j in range(dim2):
            flag_nan = np.isnan(data[i, j, :])
            out[i, j, :] = data[i, j, :] > thres_
            out[i, j, flag_nan] = np.nan
    return out

# ---------- Parsers ---------- #
parser = argparse.ArgumentParser()
parser.add_argument('out', help='out')
args = vars(parser.parse_args())

type_ind = int(args['out'])

if type_ind == 0:
    prefix_raw = 'BASE_final_SS'
    prefix_out = 'BASE_final'
    key_raw = 'AnEn'
    EN = 25

elif type_ind == 1:
    prefix_raw = 'SL_final_SS'
    prefix_out = 'SL_final'
    key_raw = 'AnEn'
    EN = 25

elif type_ind == 2:
    prefix_raw = 'BASE_CNN'
    prefix_out = 'BASE_CNN'
    key_raw = 'cnn_pred'
    EN = 75 # 25 members dressed to 75

elif type_ind == 3:
    prefix_raw = 'SL_CNN'
    prefix_out = 'SL_CNN'
    key_raw = 'cnn_pred'
    EN = 75 # 25 members dressed to 75

# ========== Gridded input ========== #

# three watershed groups
with h5py.File(save_dir+'BCH_wshed_groups.hdf', 'r') as h5io:
    flag_sw = h5io['flag_sw'][...]
    flag_si = h5io['flag_si'][...]
    flag_n = h5io['flag_n'][...]
FLAGs = (flag_sw, flag_si, flag_n)

with h5py.File(save_dir+'BC_domain_info.hdf', 'r') as h5io:
    land_mask_bc = h5io['land_mask_bc'][...]
grid_shape = land_mask_bc.shape

# ========== BCH obs preprocessing ========== #

# import station obsevations and grid point indices
with h5py.File(save_dir+'BCH_ERA5_3H_verif.hdf', 'r') as h5io:
    BCH_obs = h5io['BCH_obs'][...]
    indx = h5io['indx'][...]
    indy = h5io['indy'][...]

# subsetting BCH obs into a given year
N_days_bch = 366 + 365*3
date_base_bch = datetime(2016, 1, 1)
date_list_bch = [date_base_bch + timedelta(days=x) for x in np.arange(N_days_bch, dtype=np.float)]

flag_pick = []
for date in date_list_bch:
    if date.year in [2017, 2018, 2019]:
        flag_pick.append(True)
    else:
        flag_pick.append(False)

flag_pick = np.array(flag_pick)

BCH_obs = BCH_obs[flag_pick, ...]

# number of stations
N_stn = BCH_obs.shape[-1]

# ========== ERA5 stn climatology preprocessing ========== #

# importing girdded ERA5 quantiles
with h5py.File(ERA_dir+'PT_3hour_q.hdf', 'r') as h5io:
    CDF_era = h5io['era_3hq_bc'][...]
    q_bins = h5io['q_bins'][...]

CDF_obs = np.empty((12, 107,)+land_mask_bc.shape)
CDF_obs[..., ~land_mask_bc] = CDF_era
CDF_obs = CDF_obs[..., indx, indy]

# station and monthly (contains neighbouring months) wise 90th
BCH_90th = CDF_obs[:, 93, :]

# ========== Merging multi-year post-processed files ========== #

# datelist for 2017-2019
N_days = 365 + 365 + 365
date_base = datetime(2017, 1, 1)
date_list = [date_base + timedelta(days=x) for x in np.arange(N_days, dtype=np.float)]

# allocation
AnEn_stn = np.empty((N_days, N_fcst, EN, N_stn))

for y in [2017, 2018, 2019]:
    if y%4 == 0:
        n_days = 366
    else:
        n_days = 365

    if type_ind == 0 or type_ind == 1:
        AnEn_full = np.empty((n_days, EN)+grid_shape)
        AnEn_full[...] = np.nan

    # ---------- #
    flag_pick = []
    for date in date_list:
        if date.year == y:
            flag_pick.append(True)
        else:
            flag_pick.append(False)
    flag_pick = np.array(flag_pick)
    # ---------- #

    for lead in range(N_fcst):

        with h5py.File(REFCST_dir + "{}_{}_lead{}.hdf".format(prefix_raw, y, lead), 'r') as h5io:
            AnEn_ = h5io[key_raw][:, :EN, ...]

        if type_ind == 0 or type_ind == 1:
            # id 0 and 1 are flattened grid points, reshape them to 2d.
            AnEn_full[..., ~land_mask_bc] = AnEn_
            AnEn_stn[flag_pick, lead, ...] = AnEn_full[..., indx, indy]
        else:
            # cnn outputs can be negative, fix it here.
            AnEn_stn[flag_pick, lead, ...] = ana.cnn_precip_fix(AnEn_[..., indx, indy])

# ========== Calibration ========== #
# params
N_bins = 15
N_boost = 100
hist_bins = np.linspace(0, 1, N_bins)
N_lead_day = 7 # number of forecasted days

# 3-hr lead times to days
fcst_leads_ini = np.arange(0, 72*3+3, 3, dtype=np.float)
# date_base = datetime(2017, 1, 1, 0) # 2017 as a reference

DAYS = []
for lead in fcst_leads_ini:
    date_temp = date_base + timedelta(hours=lead)
    DAYS.append(date_temp.day-1)
DAYS = np.array(DAYS[2:56])

# converting post-processed GEFS to flags and then probabilities
mon_inds = []
for d, date in enumerate(date_list):
    mon_inds.append(date.month-1)
mon_inds = np.array(mon_inds)

prob = fcst_to_prob(AnEn_stn, BCH_90th, mon_inds)
binary = fcst_to_flag(BCH_obs, BCH_90th, mon_inds)

o_bar = np.empty((N_lead_day,))
use = np.empty((N_lead_day, N_bins))

prob_true = np.empty((N_lead_day, N_bins, N_boost))
prob_pred = np.empty((N_lead_day, N_bins, N_boost))
brier = np.empty((N_lead_day, N_boost,))

for r in range(3):
    for d in range(N_lead_day):

        day_ind = DAYS == d

        obs = binary[:, day_ind, :][..., FLAGs[r]].flatten()
        fcst = prob[:, day_ind, :][..., FLAGs[r]].flatten()

        flag_nonan = np.logical_not(np.isnan(obs))
        obs = obs[flag_nonan]
        fcst = fcst[flag_nonan]
        L = np.sum(flag_nonan)

        o_bar_ = np.mean(obs)

        o_bar[d] = o_bar_

        for n in range(N_boost):

            ind_bagging = np.random.choice(L, size=L, replace=True)
            obs_ = obs[ind_bagging]
            fcst_ = fcst[ind_bagging]

            prob_true_, prob_pred_ = reliability_diagram(obs_, fcst_, hist_bins)
            brier_ = brier_score_loss(obs_, fcst_)

            prob_true[d, :, n] = prob_true_
            prob_pred[d, :, n] = prob_pred_
            brier[d, n] = brier_

        hist_bins_ = np.mean(prob_pred[d, ...], axis=1)
        use_, _ = np.histogram(fcst, bins=np.array(list(hist_bins_)+[1.0]))
        use[d, :] = use_

    tuple_save = (brier, prob_true, prob_pred, use, o_bar)
    label_save = ['brier', 'pos_frac', 'pred_value', 'use', 'o_bar']
    du.save_hdf5(tuple_save, label_save, save_dir, '{}_Calib_loc{}.hdf'.format(prefix_out, r))

max_stars_repo_name: yingkaisha/rainbow

max_issues_repo_name: yingkaisha/rainbow

max_forks_repo_name: yingkaisha/rainbow

reason: import numpy,import numba


text:
import numpy as np

"""
The Difference Between Copy and View
The main difference between a copy and a view of an array is that the copy is a new array, and the view is just a view of the original array.

The copy owns the data and any changes made to the copy will not affect original array, and any changes made to the original array will not affect the copy.

The view does not own the data and any changes made to the view will affect the original array, and any changes made to the original array will affect the view.
"""

# 1. Copy:
arr_1 = np.array([1, 2, 3, 4, 5])
copy = arr_1.copy()

print(f"Array: {arr_1} - Copy: {copy}")

arr_1[0] = 111
print(f"Array: {arr_1} - Copy: {copy}")

copy[0] = 222
print(f"Array: {arr_1} - Copy: {copy}")

# 2. View:

arr_2 = np.array([1, 2, 3, 4, 5])
view = arr_2.view()

print(f"Array: {arr_2} - Copy: {view}")

arr_2[0] = 111
print(f"Array: {arr_2} - Copy: {view}")

view[1] = 222
print(f"Array: {arr_2} - Copy: {view}")

# 3. Check if Array Owns its Data
"""
As mentioned above, copies owns the data, and views does not own the data, but how can we check this?
Every NumPy array has the attribute base that returns None if the array owns the data.
Otherwise, the base  attribute refers to the original object.
"""

print(f"Copy: {copy.base}")
print(f"View: {view.base}")

max_stars_repo_path: Tutorial/Code/5. Array Copy vs View.py

max_stars_repo_name: Deve-BlackHeart/Numpy

max_issues_repo_path: Tutorial/Code/5. Array Copy vs View.py

max_issues_repo_name: Deve-BlackHeart/Numpy

max_forks_repo_path: Tutorial/Code/5. Array Copy vs View.py

max_forks_repo_name: Deve-BlackHeart/Numpy


text:
from torch import nn
import torch
import torch.optim as optim
import numpy as np
import argparse
import sys
import time
import os
import copy
from torch.autograd import Variable
from torch.utils.data import DataLoader
import glob
import resource
import itertools
from collections import Counter
import pickle
from gqa_dataset import *
from mnnm_models.soft_logic_transformer import *
from workflow.mnnm_arguments import *
from workflow.metrics import *

device = torch.device('cuda')
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (40000, rlimit[1]))

def main():
    args = parse_opt()
    if args.do_submission:
        args.length += 1
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)
    torch.backends.cudnn.benchmark = True

    if args.do_finetune:
        lr_decay_step = 2
        args.lr_decrease_start = 2
        args.num_epochs = 12
        lr_decay_epochs = range(args.lr_decrease_start, args.num_epochs, lr_decay_step)
        gradual_warmup_steps = []
    else:
        lr_decay_step = 2
        lr_decay_epochs = range(args.lr_decrease_start, args.num_epochs, lr_decay_step)
        gradual_warmup_steps = [1.0 * args.lr_default, 1.0 *
                                args.lr_default, 1.5 * args.lr_default, 2.0 * args.lr_default]

    print(args)
    repo = os.path.join(args.output, args.id)
    if not os.path.exists(args.output):
        os.mkdir(args.output)
    if not os.path.exists(repo):
        os.mkdir(repo)

    with open('{}/full_vocab.json'.format(args.meta), 'r') as f:
        vocab = json.load(f)
        ivocab = {v: k for k, v in vocab.items()}

    with open('{}/answer_vocab.json'.format(args.meta), 'r') as f:
        answer = json.load(f)
        inv_answer = {v: k for k, v in answer.items()}

    MAX_LAYER = args.max_layer

    if args.ensemble:
        if args.do_testdev_pred:
            filenames = glob.glob('ensembles/testdev_ensemble_*.pkl')
            print("loading {}".format(filenames))
            ensembles = []
            for filename in filenames:
                with open(filename, 'rb') as f:
                    ensembles.append((filename, pickle.load(f)))

            keys = ensembles[0][1].keys()
            iterator = itertools.combinations(ensembles, 3)
            best_performer = 0
            best_comb = []
            for ens_comb in iterator:
                succ, total = 0, 0
                for k in keys:
                    prob = [ens[1][k]['prob'] for ens in ens_comb]
                    prob = sum(prob) / len(prob)
                    result = prob.argmax()
                    if result == ens_comb[0][1][k]['target']:
                        succ += 1
                    total += 1
                comb = [_[0] for _ in ens_comb]
                acc = succ / total
                #print('the accuracy is {} for {}'.format(acc, comb))
                if acc > best_performer:
                    best_comb = comb
                    best_performer = acc

            print("===========================")
            print(comb, best_performer)
            exit()

        elif args.do_submission:
            filenames = glob.glob('ensembles/mcan/submission_ensemble_*.npy')
            print("loading {}".format(filenames))
            ensembles = []
            for filename in filenames:
                ensembles.append(np.load(filename))

            final_prob = sum(ensembles)
            preds = np.argmax(final_prob, -1)

            with open('ensembles/quesionIds.json', 'r') as f:
                questionIds = json.load(f)

            submissions = []
            for q, pred in zip(questionIds, preds):
                submissions.append({'questionId': q, 'prediction': inv_answer[pred]})

            with open('submission_results.json', 'w') as f:
                json.dump(submissions, f)
            exit()

    print("running model with {}".format(args.model))
    if args.model == "SoftLogicTransformer":
        model = SoftLogicTransformer(vocab_size=len(vocab), stacking=args.stacking, answer_size=len(answer),
                                     visual_dim=args.visual_dim, coordinate_dim=args.additional_dim, n_head=args.n_head,
                                     n_layers=MAX_LAYER, dropout=args.dropout, intermediate_dim=args.num_regions + 1,
                                     pre_layers=args.pre_layers, intermediate_layer=args.intermediate_layer)
        print("Running Modular Transformer model with {} layers with post layer".format(args.stacking))
    else:
        raise NotImplementedError

    if not args.single:
        model = nn.DataParallel(model)
    model.to(device)

    basic_kwargs = dict(length=args.length, object_info=os.path.join(args.meta, args.object_info),
                        num_regions=args.num_regions, distribution=args.distribution,
                        vocab=vocab, answer=answer, max_layer=MAX_LAYER, num_tokens=args.num_tokens,
                        spatial_info='{}/gqa_spatial_merged_info.json'.format(args.meta),
                        forbidden=args.forbidden)

    # generate restriction dict.
    program_type_to_cand = pickle.load(open('meta_info/type2cand_dict.pkl', 'rb'))
    pt_to_cand_mask = {k: torch.zeros(len(answer)) for k in program_type_to_cand}
    for key in program_type_to_cand:
        for ele in program_type_to_cand[key]:
            pt_to_cand_mask[key][ele] = 1

    if args.do_train or args.do_train_all or args.do_finetune:
        if args.do_train_all:
            train_split = 'trainval_all_fully'
            print("using the all programs for bootstrapping")
        else:
            train_split = 'trainval_unbiased_fully'
            print("using the generated programs for training")
        testdev_split = 'testdev_pred'

        if args.do_finetune:
            if args.single:
                model = nn.DataParallel(model)

            model.load_state_dict(torch.load(args.load_from))
            print("Loading the bootstrapped model from {}".format(args.load_from))

        train_dataset = GQA(split=train_split, mode='train', contained_weight=args.contained_weight,
                            threshold=args.threshold, folder=args.data, cutoff=args.cutoff, **basic_kwargs)
        test_dataset = GQA(split=testdev_split, mode='val', contained_weight=args.contained_weight,
                           threshold=args.threshold, folder=args.data, cutoff=args.cutoff, **basic_kwargs)

        if args.num_workers == 1:
            train_dataloader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
            test_dataloader = DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False)
        else:
            train_dataloader = DataLoader(train_dataset, batch_size=args.batch_size,
                                          shuffle=True, num_workers=args.num_workers)
            test_dataloader = DataLoader(test_dataset, batch_size=args.batch_size,
                                         shuffle=False, num_workers=args.num_workers)

        cross_entropy = nn.CrossEntropyLoss(ignore_index=-1)
        KL_loss = KLDivergence()

        optimizer = optim.Adam(filter(lambda x: x.requires_grad, model.parameters()), lr=args.lr_default)

        if args.resume != -1:
            filename = glob.glob('{}/{}/model_ep{}*'.format(args.output, args.id, args.resume))[0]
            print("resuming from {}".format(filename))
            model.load_state_dict(torch.load(filename))

        for epoch in range(args.resume + 1, args.num_epochs):
            if epoch < len(gradual_warmup_steps):
                for i in range(len(optimizer.param_groups)):
                    optimizer.param_groups[i]['lr'] = gradual_warmup_steps[epoch]
                print('lr', optimizer.param_groups[-1]['lr'])
            if epoch in lr_decay_epochs:
                for i in range(len(optimizer.param_groups)):
                    optimizer.param_groups[i]['lr'] *= args.lr_decay_rate
                print('lr', optimizer.param_groups[-1]['lr'])
            else:
                print('lr', optimizer.param_groups[-1]['lr'])

            model.eval()
            success, total = 0, 0
            for i, batch in enumerate(test_dataloader):
                questionId = batch[-2]
                batch = tuple(Variable(t).to(device) for t in batch[:-2])
                results = model(*batch[:-1])
                if isinstance(results, tuple):
                    logits = results[1]
                else:
                    logits = results
                preds = torch.argmax(logits, -1)
                success_or_not = (preds == batch[-1]).float()

                success += torch.sum(success_or_not).item()
                total += success_or_not.size(0)

            acc = round(success / (total + 0.), 4)
            print("epoch {}, accuracy = {}".format(epoch, acc))

            torch.save(model.state_dict(), os.path.join(repo, 'model_ep{}_{}'.format(epoch, acc)))

            model.train()
            start_time = time.time()
            for i, batch in enumerate(train_dataloader):
                if i == 4999:  # earlier stopping
                    break
                questionId, image_id = batch[-2], batch[-1]
                batch = tuple(Variable(t).to(device) for t in batch[:-2])

                model.zero_grad()
                optimizer.zero_grad()

                results = model(*batch[:-1])
                if isinstance(results, tuple):
                    pre_logits, logits = results
                    length = pre_logits.size(-1)
                    pre_loss = KL_loss(batch[-2].view(-1, length), pre_logits.view(-1, length))
                    pred_loss = cross_entropy(logits, batch[-1])
                else:
                    logits = results
                    pre_loss = torch.FloatTensor([0]).to(device)
                    pred_loss = cross_entropy(logits, batch[-1])

                loss = args.weight * pre_loss + pred_loss

                loss.backward()
                optimizer.step()
                if i % 100 == 0 and i > 0:
                    print("epoch: {}, iteration {}/{}: module loss = {}, pred_loss = {} iter time = {}".
                          format(epoch, i, len(train_dataloader), pre_loss.item(), pred_loss.item(), time.time() - start_time))

                start_time = time.time()

    elif args.do_analyze:
        train_split = 'trainval_unbiased_fully'
        train_dataset = GQA_v2(split=train_split, mode='train', contained_weight=args.contained_weight,
                               threshold=args.threshold, folder=args.data, cutoff=args.cutoff, **basic_kwargs)
        train_dataloader = DataLoader(train_dataset, batch_size=args.batch_size,
                                      shuffle=True, num_workers=args.num_workers)
        KL_loss = KLDivergence()
        if args.load_from != "":
            repo = os.path.join(args.output, args.id, args.load_from)
        print("evaluating model from {}".format(repo))
        model.load_state_dict(torch.load(repo))
        model.eval()
        start_time = time.time()
        for i, batch in enumerate(train_dataloader):
            questionId = batch[-1]
            answerId = batch[-2]
            batch = tuple(Variable(t).to(device) for t in batch[:-1])
            results = model(*batch[:-2])
            pre_logits, logits = results

            length = pre_logits.size(-1)
            pre_loss = KL_loss(batch[-2].view(-1, length), pre_logits.view(-1, length))
            answer_prediction = logits.argmax(-1)

            groundtruth = batch[-2].argmax(-1)
            prediction = pre_logits.topk(1, -1)[1]
            bounding_boxes = batch[7]

            succ, fail = 0, 0
            succ_counter = Counter()
            fail_counter = Counter()
            for progs, idx, gt, pred, ans_pred, ans, bbox, q_id in zip(batch[2], batch[3], groundtruth, prediction, answer_prediction, answerId, bounding_boxes, questionId):
                cur_length = (torch.sum(idx) - 1).long().item()
                for j in range(cur_length):
                    prog = progs[j]
                    if gt[j] in pred[j]:
                        succ += 1
                        succ_counter.update([ivocab[prog[0].item()]])
                    else:
                        fail += 1
                        #print(q_id, bbox[gt[i]], bbox[pred[j][0]])
                        fail_counter.update([ivocab[prog[0].item()]])

                if ans_pred.item() == ans.item():
                    succ_counter.update([ivocab[progs[cur_length][0].item()]])
                else:
                    fail_counter.update([ivocab[progs[cur_length][0].item()]])

            if i == 50:
                break
            else:
                sys.stdout.write("finished {}/40 batches \r".format(i))

        print()
        print(succ_counter, fail_counter)
        print("success rate = {}".format(succ / (succ + fail + 0.)))

    elif args.do_testdev or args.do_testdev_pred or args.do_val:
        split = 'testdev_pred'
        print("Using the split of {}".format(split))
        if args.gqa_loader == 'v2':
            test_dataset = GQA(split=split, mode='val', folder=args.data, cutoff=args.cutoff,
                               threshold=args.threshold, contained_weight=args.contained_weight,
                               pt2cand=pt_to_cand_mask, **basic_kwargs)
        else:
            raise NotImplementedError

        test_dataloader = DataLoader(test_dataset, batch_size=args.batch_size,
                                     shuffle=False, num_workers=args.num_workers)

        def show(string):
            return [inv_answer[_.item()] for _ in string]

        if args.load_from != "":
            repo = os.path.join(args.output, args.id, args.load_from)
        else:
            filenames = glob.glob(os.path.join(args.output, args.id, 'model_ep*'))
            best = 0
            repo = ""
            for file in filenames:
                acc = float(file.split('_')[-1])
                if acc > best:
                    best = acc
                    repo = file
            args.load_from = repo.split('/')[-1]

        print("evaluating model from {}".format(repo))
        loaded_model = torch.load(repo)
        model.load_state_dict(loaded_model)

        model.eval()
        success, total = 0, 0
        start_idx = 0
        submissions = {}
        ensemble = {}
        failure_id = []
        for i, batch in enumerate(test_dataloader):
            questionId, image_id = batch[-2], batch[-1]
            batch = tuple(Variable(t).to(device) for t in batch[:-2])

            results = model(*batch[:-1])
            if isinstance(results, tuple):
                pre_logits, logits = results
                selected_bbox = torch.argmax(pre_logits[0], -1)
                bounding_bboxes = batch[7][0][:, :4]
            else:
                logits = results

            probs = nn.functional.softmax(logits, -1)
            preds = torch.argmax(probs, -1)

            for qid, prob, t in zip(questionId, probs, batch[-1]):
                ensemble[qid] = {'prob': prob.data.cpu().numpy(), 'target': t.item()}

            if args.debug:
                start = 40
                if i > start:
                    length = len(test_dataset.data[i][3])
                    print(test_dataset.data[i][0], "@", test_dataset.data[i][1], "@", test_dataset.data[i][3], "@",
                          inv_answer[preds[0].item()], "@", inv_answer[test_dataset[i][-2]])

                    print("SELECTION :", selected_bbox)
                    for b in selected_bbox[:length - 1]:
                        if b < args.num_regions:
                            print('[{},{},{},{}]'.format(bounding_bboxes[b][0].item(),
                                                         bounding_bboxes[b][1].item(),
                                                         bounding_bboxes[b][2].item(),
                                                         bounding_bboxes[b][3].item()))
                        elif b == args.num_regions:
                            print(None)
                        elif b == args.num_regions + 1:
                            print("yes")
                        elif b == args.num_regions + 2:
                            print("no")
                        else:
                            print("")
                    print()
                if i == start + 20:
                    break

            for pred, question_id in zip(preds, questionId):
                submissions[question_id] = inv_answer[pred.item()]
            success_or_not = []
            for idx, ele in enumerate(preds):
                cur_pred_inv = inv_answer[ele.item()]
                cur_gt_inv = inv_answer[batch[-1][idx].item()]
                stem_pred, stem_gt = stem_func(cur_pred_inv), stem_func(cur_gt_inv)
                success_or_not.append(stem_pred == stem_gt)
            # success_or_not = (preds == batch[-1]).float()
            success += sum(success_or_not)
            for idx, ele in enumerate(success_or_not):
                if ele < 0.5:   # incorrect
                    failure_id.append([questionId[idx], image_id[idx], inv_answer[preds[idx].item()]])
            failure_p = os.path.join(args.output, args.id, args.id + '_failure.pkl')
            pickle.dump(failure_id, open(failure_p, 'wb'))
            total += len(success_or_not)
            start_idx += args.batch_size

            acc = round(success / (total + 0.), 4)
            print("Validation accuracy =", acc)
        if args.do_testdev:
            with open('/tmp/gt_results.json', 'w') as f:
                json.dump(submissions, f, indent=2)
        else:
            with open('/tmp/pred_results.json', 'w') as f:
                json.dump(submissions, f, indent=2)

    elif args.do_submission:
        test_dataset = GQA(split='submission', mode='val', contained_weight=args.contained_weight,
                           threshold=args.threshold, folder=args.data, cutoff=args.cutoff, **basic_kwargs)

        test_dataloader = DataLoader(test_dataset, batch_size=args.batch_size,
                                     shuffle=False, num_workers=args.num_workers)

        if args.load_from != "":
            repo = os.path.join(args.output, args.id, args.load_from)
        else:
            filenames = glob.glob(os.path.join(args.output, args.id, 'model_ep*'))
            best = 0
            repo = ""
            for file in filenames:
                acc = float(file.split('_')[-1])
                if acc > best:
                    best = acc
                    repo = file
            args.load_from = repo.split('/')[-1]

        print("evaluating model from {}".format(repo))
        model.load_state_dict(torch.load(repo))

        model.eval()
        success, total = 0, 0
        submissions = []
        full_prob = []
        questionIds = []
        for i, batch in enumerate(test_dataloader):
            questionId = batch[-1]
            batch = tuple(Variable(t).to(device) for t in batch[:-1])

            results = model(*batch[:-2])
            if isinstance(results, tuple):
                pre_logits, logits = results
            else:
                logits = results

            if isinstance(results, tuple):
                logits = results[1]
            else:
                logits = results

            prob = torch.softmax(logits, -1)
            preds = torch.argmax(prob, -1)
            questionIds.extend(questionId)
            full_prob.append(prob.data.cpu().numpy())
            for pred, question_id in zip(preds, questionId):
                submissions.append({'questionId': question_id, 'prediction': inv_answer[pred.item()]})
            sys.stdout.write('finished {}/{}\r'.format(i, len(test_dataloader)))
        with open('submission_results.json', 'w') as f:
            json.dump(submissions, f)

    else:
        raise ValueError("Unseen Option")

if __name__ == "__main__":
    main()


text:
from typing import List, Set
from itertools import groupby
import numpy as np
import re
from app.logic import vectors as v
from app.logic.word import Word

"""
Load and cleanup data.
:param dir: File path of word embedding
:return: Word embedding model, vocabulary, dimensions
:rtype: list, dict, int
"""
def load_embedding(dir):
    print(f"Loading {dir}...")
    words, voc = load_embedding_raw(dir)
    print(f"Loaded {len(words)} words.")
    num_dimensions = most_common_dimension(words)
    words = [w for w in words if len(w.vector) == num_dimensions]
    print(f"Using {num_dimensions}-dimensional vectors, {len(words)} remain.")
    # words = remove_stop_words(words)
    # print(f"Removed stop words, {len(words)} remain.")
    # words = remove_duplicates(words)
    # print(f"Removed duplicates, {len(words)} remain.")
    return words, voc, num_dimensions

"""
Read data file and load embedding.
:param dir: File path of word embedding
:return: Word embedding model, vocabulary
:rtype: list, dict
"""
def load_embedding_raw(file_path):
    def parse_line(line, frequency):
        tokens = line.split()
        word = tokens[0]
        vector = v.normalize(np.array([float(x) for x in tokens[1:]]))  # with normalization
        # vector = np.array([float(x) for x in tokens[1:]])
        return Word(word, vector, frequency)
    words = []
    voc = {}
    i = 0
    # Words are sorted from the most common to the least common ones
    frequency = 1
    with open(file_path, encoding="utf8") as f:
        for line in f:
            w = parse_line(line, frequency)
            words.append(w)
            frequency += 1
            voc[i] = w.text
            i+=1
    return words, voc

def iter_len(iter):
    return sum(1 for _ in iter)

"""
In case of not all vectors having same dimesionality, most common one is tracked.
:param words: Word embedding model
:return: Most common dimensionality
:rtype: int
"""
def most_common_dimension(words):
    lengths = sorted([len(word.vector) for word in words])
    dimensions = [(k, iter_len(v)) for k, v in groupby(lengths)]
    for (dim, num_vectors) in dimensions:
        print(f"{num_vectors} {dim}-dimensional vectors")
    most_common = sorted(dimensions, key=lambda t: t[1], reverse=True)[0]
    return most_common[0]

'''U.S -> US'''
ignore_char_regex = re.compile("[\W_]")

'''Word has to start and end with an alphanumeric character.'''
is_valid_word = re.compile("^[^\W_].*[^\W_]$")

"""
Removes duplicate words from embedding.
:param words: Word embedding model
:return: Word embedding model without duplicate words
:rtype: list
"""
def remove_duplicates(words):
    seen_words: Set[str] = set()
    unique_words: List[Word] = []
    for w in words:
        canonical = ignore_char_regex.sub("", w.text)
        if not canonical in seen_words:
            seen_words.add(canonical)
            # Keep the original ordering
            unique_words.append(w)
    return unique_words

"""
Removes stop words from embedding.
:param words: Word embedding model
:return: Word embedding model without stop words
:rtype: list
"""
def remove_stop_words(words):
    return [w for w in words if (
        len(w.text) > 1 and is_valid_word.match(w.text))]

# # Run "smoke tests" on import
# assert [w.text for w in remove_stop_words([
#     Word('a', [], 1),
#     Word('ab', [], 1),
#     Word('-ab', [], 1),
#     Word('ab_', [], 1),
#     Word('a.', [], 1),
#     Word('.a', [], 1),
#     Word('ab', [], 1),
# ])] == ['ab', 'ab']
# assert [w.text for w in remove_duplicates([
#     Word('a.b', [], 1),
#     Word('-a-b', [], 1),
#     Word('ab_+', [], 1),
#     Word('.abc...', [], 1),
# ])] == ['a.b', '.abc...']

max_stars_repo_name: alXKl/taxonomy-editor

max_issues_repo_name: alXKl/taxonomy-editor

max_forks_repo_name: alXKl/taxonomy-editor


text:
"""
calculator.py

General functions that would be in a finacial calculator.
"""

import numpy as np

# Typical Financial Calculator Functions
# ------------------------------------------------------------------------------------------------------------------------------
def future_value(pv=100.0, r=0.07, n=1.0, f=1.0):
    """
    Calculate the future value of pv present value after compounding for n periods at r rate every f frequency.
    """
    return pv * np.exp(1.0 + r / f, n * f)

def present_value(fv=100.0, r=0.07, n=1.0, f=1.0):
    """
    Calculate the present value of fv future value before compounding for n periods at r rate every f frequency.
    """
    return fv / np.exp(1.0 + r / f, n * f)

def rate(fv=100.0, pv=90.0, n=1.0, f=1.0):
    """
    Calculate the rate needed to compound a pv present value into a fv future value compounding over n periods every f
    frequency.
    """
    return f * np.power(fv / pv, 1.0 / (n * f)) - 1.0

def periods(fv=0.0, pv=0.0, r=0.0, f=0.0):
    """
    Calculate the period needed to compound a pv present value into a fv future value compounding at r rate every f frequency.
    """
    return np.log(fv / pv) / (f * np.log(1.0 + r / f))

def effective_return(r=0.07, f=2.0):
    """
    Calculate the annual rate needed to equal an r rate at f frequency.
    """
    return np.power(1.0 + (r / f), f) - 1.0

def annual_return(r=0.07, f=1.0):
    """
    Calculate annual return from semiannual return.
    """
    return np.power(1.0 + r, f) - 1.0

def inflation_adjusted(r=0.07, i=0.03):
    """
    Calculate inflation adjusted returns.
    """
    return (1.0 + r) / (1.0 + i) - 1.0

def gain(xi=100.0, xf=110.0):
    """
    Calculate gain from intial to final value.
    """
    return (xf - xi) / xi

def amortization(p=1000.0, r=0.05, n=10.0, f=1.0):
    """
    Calculate periodic payments needed to pay off p principle at r rate over n periods every f frequency.
    """
    return p * (r / f) / (1.0 - np.power(1.0 + r / f, -f*n))

def cagr(xi=100.0, xf=110.0, n=1.0):
    """
    Calculate compund annual growth rate.
    """
    return np.power(xf / xi, 1.0 / n) - 1.0

def length_of_payment(b=1000.0, p=100.0, apr=0.18):
    """
    Calculate the length of payments of b balance with p payment at apr APR.
    """
    i = apr / 30.0
    return (-1.0 / 30.0) * np.log(1.0 + (b / p)*(1.0 - np.power(1.0 + i, 30.0))) / np.log(1.0 + i)

def annuity(p=100.0, r=0.07, n=10.0, f=1.0):
    """
    Calculate future value based on periodic p investment payment at r rate over n periods every f frequency - check this
    formula.
    """
    return p * ((np.power(1.0 + r / f, n * f) - 1.0) / r / f)

max_stars_repo_name: tmthydvnprt/compfipy

max_issues_repo_name: aarigs/compfipy

max_forks_repo_name: aarigs/compfipy


text:
#!/usr/bin/env python
from os.path import join
import sys

from nitime import  __version__, __doc__

def configuration(parent_package='',top_path=None):
    from numpy.distutils.misc_util import Configuration

    config = Configuration(None, parent_package, top_path)
    config.set_options(ignore_setup_xxx_py=True,
                       assume_default_configuration=True,
                       delegate_options_to_subpackages=True,
                       quiet=True)
    # The quiet=True option will silence all of the name setting warnings:
    # Ignoring attempt to set 'name' (from 'nitime.core' to
    #    'nitime.core.image')
    # Robert Kern recommends setting quiet=True on the numpy list, stating
    # these messages are probably only used in debugging numpy distutils.

    config.get_version('nitime/version.py') # sets config.version

    config.add_subpackage('nitime', 'nitime')

    return config

def main():
    from numpy.distutils.core import setup

    setup( name = 'nitime',
           description = 'Nitime: timeseries analysis for neuroscience data',
           author = 'Various',
           author_email = 'nipy-devel@neuroimaging.scipy.org',
           url = 'http://neuroimaging.scipy.org',
           long_description = __doc__,
           configuration = configuration)

if __name__ == "__main__":
    main()

max_stars_repo_name: dcoates/nitime

max_issues_repo_name: dcoates/nitime

max_forks_repo_name: dcoates/nitime

reason: from numpy


text:
# Copyright (c) MONAI Consortium
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
A collection of generic interfaces for MONAI transforms.
"""

import warnings
from typing import Any, Callable, Mapping, Optional, Sequence, Union

import numpy as np

from monai.transforms.inverse import InvertibleTransform

# For backwards compatibility (so this still works: from monai.transforms.compose import MapTransform)
from monai.transforms.transform import (  # noqa: F401
    MapTransform,
    Randomizable,
    RandomizableTransform,
    Transform,
    apply_transform,
)
from monai.utils import MAX_SEED, ensure_tuple, get_seed
from monai.utils.enums import TraceKeys

__all__ = ["Compose", "OneOf"]

class Compose(Randomizable, InvertibleTransform):
    """
    ``Compose`` provides the ability to chain a series of callables together in
    a sequential manner. Each transform in the sequence must take a single
    argument and return a single value.

    ``Compose`` can be used in two ways:

    #. With a series of transforms that accept and return a single
       ndarray / tensor / tensor-like parameter.
    #. With a series of transforms that accept and return a dictionary that
       contains one or more parameters. Such transforms must have pass-through
       semantics that unused values in the dictionary must be copied to the return
       dictionary. It is required that the dictionary is copied between input
       and output of each transform.

    If some transform takes a data item dictionary as input, and returns a
    sequence of data items in the transform chain, all following transforms
    will be applied to each item of this list if `map_items` is `True` (the
    default).  If `map_items` is `False`, the returned sequence is passed whole
    to the next callable in the chain.

    For example:

    A `Compose([transformA, transformB, transformC],
    map_items=True)(data_dict)` could achieve the following patch-based
    transformation on the `data_dict` input:

    #. transformA normalizes the intensity of 'img' field in the `data_dict`.
    #. transformB crops out image patches from the 'img' and 'seg' of
       `data_dict`, and return a list of three patch samples::

        {'img': 3x100x100 data, 'seg': 1x100x100 data, 'shape': (100, 100)}
                             applying transformB
                                 ---------->
        [{'img': 3x20x20 data, 'seg': 1x20x20 data, 'shape': (20, 20)},
         {'img': 3x20x20 data, 'seg': 1x20x20 data, 'shape': (20, 20)},
         {'img': 3x20x20 data, 'seg': 1x20x20 data, 'shape': (20, 20)},]

    #. transformC then randomly rotates or flips 'img' and 'seg' of
       each dictionary item in the list returned by transformB.

    The composed transforms will be set the same global random seed if user called
    `set_determinism()`.

    When using the pass-through dictionary operation, you can make use of
    :class:`monai.transforms.adaptors.adaptor` to wrap transforms that don't conform
    to the requirements. This approach allows you to use transforms from
    otherwise incompatible libraries with minimal additional work.

    Note:

        In many cases, Compose is not the best way to create pre-processing
        pipelines. Pre-processing is often not a strictly sequential series of
        operations, and much of the complexity arises when a not-sequential
        set of functions must be called as if it were a sequence.

        Example: images and labels
        Images typically require some kind of normalization that labels do not.
        Both are then typically augmented through the use of random rotations,
        flips, and deformations.
        Compose can be used with a series of transforms that take a dictionary
        that contains 'image' and 'label' entries. This might require wrapping
        `torchvision` transforms before passing them to compose.
        Alternatively, one can create a class with a `__call__` function that
        calls your pre-processing functions taking into account that not all of
        them are called on the labels.
    """

    def __init__(
        self,
        transforms: Optional[Union[Sequence[Callable], Callable]] = None,
        map_items: bool = True,
        unpack_items: bool = False,
    ) -> None:
        if transforms is None:
            transforms = []
        self.transforms = ensure_tuple(transforms)
        self.map_items = map_items
        self.unpack_items = unpack_items
        self.set_random_state(seed=get_seed())

    def set_random_state(self, seed: Optional[int] = None, state: Optional[np.random.RandomState] = None) -> "Compose":
        super().set_random_state(seed=seed, state=state)
        for _transform in self.transforms:
            if not isinstance(_transform, Randomizable):
                continue
            _transform.set_random_state(seed=self.R.randint(MAX_SEED, dtype="uint32"))
        return self

    def randomize(self, data: Optional[Any] = None) -> None:
        for _transform in self.transforms:
            if not isinstance(_transform, Randomizable):
                continue
            try:
                _transform.randomize(data)
            except TypeError as type_error:
                tfm_name: str = type(_transform).__name__
                warnings.warn(
                    f'Transform "{tfm_name}" in Compose not randomized\n{tfm_name}.{type_error}.', RuntimeWarning
                )

    def flatten(self):
        """Return a Composition with a simple list of transforms, as opposed to any nested Compositions.

        e.g., `t1 = Compose([x, x, x, x, Compose([Compose([x, x]), x, x])]).flatten()`
        will result in the equivalent of `t1 = Compose([x, x, x, x, x, x, x, x])`.

        """
        new_transforms = []
        for t in self.transforms:
            if isinstance(t, Compose) and not isinstance(t, OneOf):
                new_transforms += t.flatten().transforms
            else:
                new_transforms.append(t)

        return Compose(new_transforms)

    def __len__(self):
        """Return number of transformations."""
        return len(self.flatten().transforms)

    def __call__(self, input_):
        for _transform in self.transforms:
            input_ = apply_transform(_transform, input_, self.map_items, self.unpack_items)
        return input_

    def inverse(self, data):
        invertible_transforms = [t for t in self.flatten().transforms if isinstance(t, InvertibleTransform)]
        if not invertible_transforms:
            warnings.warn("inverse has been called but no invertible transforms have been supplied")

        # loop backwards over transforms
        for t in reversed(invertible_transforms):
            data = apply_transform(t.inverse, data, self.map_items, self.unpack_items)
        return data

class OneOf(Compose):
    """
    ``OneOf`` provides the ability to randomly choose one transform out of a
    list of callables with pre-defined probabilities for each.

    Args:
        transforms: sequence of callables.
        weights: probabilities corresponding to each callable in transforms.
            Probabilities are normalized to sum to one.

    ``OneOf`` inherits from ``Compose`` and uses args ``map_items`` and ``unpack_items`` in
    the same way.
    """

    def __init__(
        self,
        transforms: Optional[Union[Sequence[Callable], Callable]] = None,
        weights: Optional[Union[Sequence[float], float]] = None,
        map_items: bool = True,
        unpack_items: bool = False,
    ) -> None:
        super().__init__(transforms, map_items, unpack_items)
        if len(self.transforms) == 0:
            weights = []
        elif weights is None or isinstance(weights, float):
            weights = [1.0 / len(self.transforms)] * len(self.transforms)
        if len(weights) != len(self.transforms):
            raise AssertionError("transforms and weights should be same size if both specified as sequences.")
        self.weights = ensure_tuple(self._normalize_probabilities(weights))

    def _normalize_probabilities(self, weights):
        if len(weights) == 0:
            return weights
        weights = np.array(weights)
        if np.any(weights < 0):
            raise AssertionError("Probabilities must be greater than or equal to zero.")
        if np.all(weights == 0):
            raise AssertionError("At least one probability must be greater than zero.")
        weights = weights / weights.sum()
        return list(weights)

    def flatten(self):
        transforms = []
        weights = []
        for t, w in zip(self.transforms, self.weights):
            # if nested, probability is the current weight multiplied by the nested weights,
            # and so on recursively
            if isinstance(t, OneOf):
                tr = t.flatten()
                for t_, w_ in zip(tr.transforms, tr.weights):
                    transforms.append(t_)
                    weights.append(w_ * w)
            else:
                transforms.append(t)
                weights.append(w)
        return OneOf(transforms, weights, self.map_items, self.unpack_items)

    def __call__(self, data):
        if len(self.transforms) == 0:
            return data
        index = self.R.multinomial(1, self.weights).argmax()
        _transform = self.transforms[index]
        data = apply_transform(_transform, data, self.map_items, self.unpack_items)
        # if the data is a mapping (dictionary), append the OneOf transform to the end
        if isinstance(data, Mapping):
            for key in data.keys():
                if self.trace_key(key) in data:
                    self.push_transform(data, key, extra_info={"index": index})
        return data

    def inverse(self, data):
        if len(self.transforms) == 0:
            return data
        if not isinstance(data, Mapping):
            raise RuntimeError("Inverse only implemented for Mapping (dictionary) data")

        # loop until we get an index and then break (since they'll all be the same)
        index = None
        for key in data.keys():
            if self.trace_key(key) in data:
                # get the index of the applied OneOf transform
                index = self.get_most_recent_transform(data, key)[TraceKeys.EXTRA_INFO]["index"]
                # and then remove the OneOf transform
                self.pop_transform(data, key)
        if index is None:
            # no invertible transforms have been applied
            return data

        _transform = self.transforms[index]
        # apply the inverse
        return _transform.inverse(data) if isinstance(_transform, InvertibleTransform) else data

max_stars_repo_name: surajpaib/MONAI


text:
#!/usr/bin/env python
# coding: utf-8

# In[2]:

#!/usr/bin/env python
# coding: utf-8

# Import necessary modules
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_iris

# Loading data
irisData = load_iris()

# Create feature and target arrays
X = irisData.data
y = irisData.target

# Split into training and test set
X_train, X_test, y_train, y_test = train_test_split(
             X, y, test_size = 0.2, random_state=42)

knn = KNeighborsClassifier(n_neighbors=7)

knn.fit(X_train, y_train)

# Predict on dataset which model has not seen before
print(knn.predict(X_test))

# Import necessary modules
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_iris

# Loading data
irisData = load_iris()

# Create feature and target arrays
X = irisData.data
y = irisData.target

# Split into training and test set
X_train, X_test, y_train, y_test = train_test_split(
             X, y, test_size = 0.2, random_state=42)

knn = KNeighborsClassifier(n_neighbors=7)

knn.fit(X_train, y_train)

# Calculate the accuracy of the model
print(knn.score(X_test, y_test))

# Import necessary modules
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_iris
import numpy as np
import matplotlib.pyplot as plt

irisData = load_iris()

# Create feature and target arrays
X = irisData.data
y = irisData.target

# Split into training and test set
X_train, X_test, y_train, y_test = train_test_split(
             X, y, test_size = 0.2, random_state=42)

neighbors = np.arange(1, 9)
train_accuracy = np.empty(len(neighbors))
test_accuracy = np.empty(len(neighbors))

# Loop over K values
for i, k in enumerate(neighbors):
    knn = KNeighborsClassifier(n_neighbors=k)
    knn.fit(X_train, y_train)

    # Compute training and test data accuracy
    train_accuracy[i] = knn.score(X_train, y_train)
    test_accuracy[i] = knn.score(X_test, y_test)

# Generate plot
plt.plot(neighbors, test_accuracy, label = 'Testing dataset Accuracy')
plt.plot(neighbors, train_accuracy, label = 'Training dataset Accuracy')

plt.legend()
plt.xlabel('n_neighbors')
plt.ylabel('Accuracy')
plt.show()

# In[ ]:

max_stars_repo_path: Final Report and Code/k-nearest neighbors.py