text:
# Copyright 2018 Alibaba Group. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import xdl

import os
import numpy as np
import json
import time
import datetime
from xdl.python.lib.graph import current_graph
from xdl.python.proto import graph_def_pb2
from google.protobuf.text_format import Parse as TextParse
from xdl.python.utils.file_io import write_string_to_file
from xdl.python.framework.session import Hook
from xdl.python.training.export import output
from xdl.python.training.training_utils import get_global_step
from xdl.python.lib.graph import execute
from xdl.python.utils.config import get_ckpt_dir
from xdl.python.lib.tensor import Tensor
from xdl.python.utils.file_io import write_string_to_file

def _string_to_int8(src):
    return np.array([ord(ch) for ch in src], dtype=np.int8)

def _graphdef_to_pb(graph_def):
    pb = graph_def_pb2.GraphDef()
    TextParse(graph_def.to_proto_string(), pb)
    return pb

'''
Usage:
    xdl.graph_tag().set_input(data_io)
    xdl.graph_tag().set_mx_output(prop)    # in xdl.mxnet_wrapper
    xdl.graph_tag().set_tf_output(prop)    # in xdl.tf_wrapper
'''
class GraphTag(object):
    def __init__(self):
        self._inputs = list()
        self._output_op_name = 'default'
        self._sparse_list = list()
        self._fea_dict = dict()
    def append_input(self, op_name, input_name, type, size=1, table=0):
        if type == xdl.features.sparse:
            input_type = graph_def_pb2.kSparse
        else:
            input_type = graph_def_pb2.kDense
        self._inputs.append((op_name, input_name, input_type, size, table))
    def set_input(self, data_io):
        for (idx, name, type, nvec, table) in data_io.tags:
            if type == xdl.features.sparse:
                input_type = graph_def_pb2.kSparse
            else:
                input_type = graph_def_pb2.kDense
            self._inputs.append(('/GetBatch:%d' % idx, name, input_type, nvec, table))
        self._sparse_list = data_io._sparse_list
        self._fea_dict = data_io._fea_dict
    def set_mx_output(self, backend_symbol):
        import mxnet
        if isinstance(backend_symbol, mxnet.symbol.symbol.Symbol):
            self._output_op_name = backend_symbol.name
    def set_tf_output(self, backend_symbol):
        import tensorflow
        if isinstance(backend_symbol, tensorflow.Tensor):
            self._output_op_name = backend_symbol.name
    @property
    def inputs(self):
        return self._inputs
    @property
    def output_op_name(self):
        return self._output_op_name
    @property
    def sparse_list(self):
        return self._sparse_list
    @property
    def fea_dict(self):
        return self._fea_dict

_GRAPH_TAG = GraphTag()

def graph_tag():
    global _GRAPH_TAG
    return _GRAPH_TAG

class EmbedOpArg(object):
    def __init__(self, str, dig, desc):
        self.str = str
        self.dig = dig
        self.desc = desc

class EmbedCodeConf(object):
    def __init__(self, name, fea_type, emb_dim, fea_groupids, op, op_args=None):
        self.name = name
        self.fea_type = fea_type
        self.emb_dim = emb_dim
        self.fea_groupids = fea_groupids
        self.op = op
        self.op_args = op_args

class Saver(object):
    def __init__(self, ckpt_dir=None, tf_graph_name=None):
        self._ckpt_dir = ckpt_dir
        if self._ckpt_dir is None:
          self._ckpt_dir = get_ckpt_dir()
        self._graph_def = _graphdef_to_pb(current_graph()._graph_def)
        self._tf_graph_name = tf_graph_name
    def save(self, version):
        execute(self.save_op(version))

    def save_meta(self, version, **kwargs):
      kwargs['xdl_global_step'] = get_global_step().value
      values = []
      for k, v in kwargs.iteritems():
        if isinstance(v, Tensor):
          v = execute(v)
          values.append(v)
        else:
          values.append(np.array(v))
      values = [min(v.flatten().tolist()) for v in values]
      keys = kwargs.keys()
      assert len(keys) == len(values)
      buf = json.dumps(dict(zip(keys, values)))
      path = os.path.join(self._ckpt_dir, version, '.meta')
      write_string_to_file(path, buf)
    def restore(self, version):
        execute(self.restore_op(version))
    def save_op(self, version):
        return xdl.ps_save_op(_string_to_int8(version))
    def restore_op(self, version):
        return xdl.ps_restore_op(_string_to_int8(version))
    def export_graph(self, output_dir, as_text=False):
        print 'start export graph'
        for (op_name, input_name, input_type, size, table) in graph_tag().inputs:
            self.append_input(op_name, input_name, input_type, size, table)
        self._graph_def.tag.output.op_name = graph_tag().output_op_name
        if as_text:
            write_string_to_file("graph.txt", str(self._graph_def))
            output(output_dir, "graph.txt")
        else:
            write_string_to_file("graph.pb", self._graph_def.SerializeToString())
            output(output_dir, "graph.pb")
        print 'finish export graph'
    def append_input(self, op_name, input_name, input_type, size=1, table=0):
        inp = self._graph_def.tag.input.add()
        inp.op_name = op_name
        inp.input_name = input_name
        inp.input_type = input_type
        inp.size = size
        inp.table = table
    def export_sparse_conf_v4(self, sparse_v4_dir, space_list=list(list())):
        print 'start export sparse conf v4'
        from xdl.python.training.v4 import f2id_pb2
        from google.protobuf.text_format import MessageToString
        f2id_list = f2id_pb2.F2IdList()
        for gid in xrange(len(space_list)):
            for feature_groupid in space_list[gid]:
                f2_id = f2id_list.item.add()
                f2_id.feature_groupid = feature_groupid
                f2_id.fid = gid
        with open('embed.best.meta', 'wb') as f:  # f2id.pb
            f.write(MessageToString(f2id_list))
        output(sparse_v4_dir, 'embed.best.meta')
        print 'finish export sparse conf v4'
    def export_dense_conf_v4(self, dense_v4_dir, embed_code_conf_list):
        print 'start export dense conf v4'
        from xdl.python.training.v4 import dense_input_conf_pb2, embed_dimension_xdl_code_pb2
        from google.protobuf.text_format import MessageToString
        embed_dim_list = embed_dimension_xdl_code_pb2.EmbedDimList()
        embed_dim_list.model_signature = str(long(time.mktime(datetime.datetime.utcnow().timetuple()) * 1000000))
        global_offset = 1
        index = 0
        block = None
        for embed_conf in embed_code_conf_list:
            if block is None or embed_conf.name != block.name:
                block = embed_dim_list.dense_input.input_blocks.add()
                block.name = embed_conf.name
                if embed_conf.fea_type == 'common':
                    block.hint_type = dense_input_conf_pb2.COMMON
                elif embed_conf.fea_type == 'ncommon':
                    block.hint_type = dense_input_conf_pb2.UNCOMMON
                else:
                    block.hint_type = dense_input_conf_pb2.UNKNOWN
                index = 0
            for fea_groupid in embed_conf.fea_groupids:
                embed_dim = embed_dim_list.embed_dim_list.add()
                embed_dim.embed_dim = embed_conf.emb_dim
                embed_dim.fea_groupid = fea_groupid
                embed_dim.fea_type = 'common' if embed_conf.fea_type == 'unknown' else embed_conf.fea_type  # 'common' | 'ncommon'
                embed_dim.fea_group_global_offset = global_offset
                global_offset += 1
                embed_field = block.embed_fields.add()
                embed_field.index = index
                index += 1
                embed_field.dim = embed_conf.emb_dim
                embed_field.fea_group_id = fea_groupid
                embed_field.op = dense_input_conf_pb2.KSUM if embed_conf.op == 'ksum' else dense_input_conf_pb2.ASSIGN
                if embed_conf.op_args is not None:
                    for arg in embed_conf.op_args:
                        op_arg = embed_field.op_args.add()
                        if arg.str is not None:
                           op_arg.str = arg.str
                        if arg.dig is not None:
                           op_arg.dig = arg.dig
                        if arg.desc is not None:
                           op_arg.desc = arg.desc
        with open('embed-dim-xdl-code-conf', 'wb') as f:  # dense_input_conf.pb
            f.write(MessageToString(embed_dim_list))
        output(dense_v4_dir, 'embed-dim-xdl-code-conf')
        output(dense_v4_dir, 'network_desc.pb')
        print 'finish export dense conf v4'
    def export_graph_v4(self, output_v4_dir, emb_dim, space_list=list(list()), order=1, comm_hint_type='common'):
        print 'start export graph v4 @deprecated'
        self.export_sparse_graph_v4(output_v4_dir, emb_dim, space_list, order, comm_hint_type)
        print 'finish export graph v4 @deprecated'
    def export_sparse_graph_v4(self, output_v4_dir, emb_dim, space_list, order=1, comm_hint_type='common'):
        from xdl.python.training.v4 import dense_input_conf_pb2, embed_dimension_xdl_code_pb2, f2id_pb2
        from google.protobuf.text_format import MessageToString
        embed_dim_list = embed_dimension_xdl_code_pb2.EmbedDimList()
        embed_dim_list.model_signature = str(long(time.mktime(datetime.datetime.utcnow().timetuple()) * 1000000))
        if order == 1:
            comm_block = embed_dim_list.dense_input.input_blocks.add()
            ncomm_block = embed_dim_list.dense_input.input_blocks.add()
        else:
            ncomm_block = embed_dim_list.dense_input.input_blocks.add()
            comm_block = embed_dim_list.dense_input.input_blocks.add()
        comm_block.name = 'comm'
        ncomm_block.name = 'ncomm'
        comm_block.hint_type = dense_input_conf_pb2.UNKNOWN if comm_hint_type == 'unknown' else dense_input_conf_pb2.COMMON
        ncomm_block.hint_type = dense_input_conf_pb2.UNCOMMON
        comm_index = 0
        ncomm_index = 0
        for i in xrange(len(graph_tag().sparse_list)):
            name = graph_tag().sparse_list[i]
            table = graph_tag().fea_dict[name]['table']
            embed_dim = embed_dim_list.embed_dim_list.add()
            embed_dim.embed_dim = emb_dim
            embed_dim.fea_groupid = name
            embed_dim.fea_group_global_offset = i + 1
            embed_dim.fea_type = "ncommon" if table == 0 else "common"
            embed_field = ncomm_block.embed_fields.add() if table == 0 else comm_block.embed_fields.add()
            if table == 0:
                embed_field.index = ncomm_index
                ncomm_index += 1
            else:
                embed_field.index = comm_index
                comm_index += 1
            embed_field.dim = emb_dim
            embed_field.fea_group_id = name
            embed_field.op = dense_input_conf_pb2.KSUM
        with open('embed-dim-xdl-code-conf', 'wb') as f:  # dense_input_conf.pb
            f.write(MessageToString(embed_dim_list))
        output(output_v4_dir, 'embed-dim-xdl-code-conf')
        f2id_list = f2id_pb2.F2IdList()
        for gid in xrange(len(space_list)):
            for feature_groupid in space_list[gid]:
                f2_id = f2id_list.item.add()
                f2_id.feature_groupid = feature_groupid
                f2_id.fid = gid
        with open('embed.best.meta', 'wb') as f:  # f2id.pb
            f.write(MessageToString(f2id_list))
        output(output_v4_dir, 'embed.best.meta')

class CheckpointMeta:
  def __init__(self, **kwargs):
    self._meta = {}
    for k, v in kwargs.iteritems():
      if not isinstance(v, Tensor):
        continue
      self._meta[k] = v
    self._meta['xdl_global_step'] = get_global_step().value

  def save(self, ckpt_dir, version, values):
    if len(self._meta) == 0:
      return
    keys = self._meta.keys()
    values = [min(v.flatten().tolist()) for v in values]

    assert len(keys) == len(values)
    buf = json.dumps(dict(zip(keys, values)))
    path = os.path.join(ckpt_dir, version, '.meta')
    write_string_to_file(path, buf)

  def values(self):
    return self._meta.values()

class CheckpointHook(Hook):
    def __init__(self, save_interval_step=None, save_interval_secs=None,
                 is_training=True, meta=None, tf_backend=False, max_to_keep=5,
                 tf_graph_name=None):
        super(CheckpointHook, self).__init__(priority=3000)
        self._global_step = get_global_step()
        self._save_interval_step = save_interval_step
        self._save_interval_secs = save_interval_secs
        self._ckpt_dir = get_ckpt_dir()
        self._saver = Saver(self._ckpt_dir, tf_graph_name)
        self._is_training = is_training
        self._last_save_step = 0
        self._last_save_time = time.time()
        self._meta = meta
        self._max_to_keep = max_to_keep
        self._ckpt_queue = []

        if self._save_interval_step is None and self._save_interval_secs is None:
            print("Checkpoint interval_steps and interval_secs both not set, use default 10000 steps.")
            self._save_interval_step = 10000
        elif self._save_interval_step is not None and self._save_interval_secs is not None:
            raise ValueError("Checkpoint interval_steps and interval_secs can't be both set.")

        self.gstep_val = 0
        self.meta_val = None

    def before_run(self, v):
        if self._ckpt_dir is None:
            return []

        if (self._is_training):
            res = [self._global_step.value]
            if self._meta is not None:
              res.extend(self._meta.values())
            return res

    def after_run(self, v):
        if self._ckpt_dir is None:
            return
        self.gstep_val = v[0] if isinstance(v, list) else v
        if self._meta:
            self.meta_val = v[1:]
        if self._save_interval_step is not None:
            if self.gstep_val - self._last_save_step >= self._save_interval_step:
                self._save_ckpt(self.gstep_val, self.meta_val)
                self._last_save_step = self.gstep_val
                self._ckpt_queue.append(self._create_version(self.gstep_val))
                self._check_ckpt_queue()
        elif self._save_interval_secs is not None:
            if time.time() - self._last_save_time >= self._save_interval_secs:
                self._save_ckpt(self.gstep_val, self.meta_val)
                self._last_save_time = time.time()
                self._ckpt_queue.append(self._create_version(self.gstep_val))
                self._check_ckpt_queue()

    def end(self):
        self.gstep_val = xdl.execute(self._global_step.value) + 1
        if self._meta is not None:
            self.meta_val = xdl.execute(self._meta.values())
        self._save_ckpt(self.gstep_val, self.meta_val)
        if self.gstep_val != self._last_save_step:
            self._ckpt_queue.append(self._create_version(self.gstep_val))
        self._check_ckpt_queue()

    def _check_ckpt_queue(self):
        while len(self._ckpt_queue) > self._max_to_keep:
            del_version = self._ckpt_queue.pop(0)
            del_ckpt = self._ckpt_dir.rstrip("/") + "/" + del_version
            if self._ckpt_dir.startswith('hdfs://'):
                cmd = "hadoop fs -rm -r %s" % del_ckpt
            else:
                cmd = "rm -rf %s" % del_ckpt
            ret = os.system(cmd)
            if ret == 0:
                print("ckpt number is larger than max_to_keep setting, delete oldest ckpt %s" % del_ckpt)
            else:
                raise ValueError("Failed: %s" % cmd)

    def _save_ckpt(self, global_step, meta_values):
        version = self._create_version(global_step)
        print('save checkpoint at global_step[%d], ckpt version[%s]' % (global_step, version))
        self._saver.save(version)
        if meta_values:
            self._meta.save(self._ckpt_dir, version, meta_values)

    def _create_version(self, global_step):
        return "ckpt-{:.>20}".format(global_step)

class RestoreFromHook(Hook):
    def __init__(self, ckp_model):
        super(RestoreFromHook, self).__init__(priority=1000)
        self._ckp_model = ckp_model
        self._saver = Saver(get_ckpt_dir())
    def create_session(self):
        if self._ckp_model and len(self._ckp_model) > 0:
            if xdl.get_task_index() == 0:
                self._saver.restore(self._ckp_model)
                print("restore checkpoint from " + str(self._ckp_model))
            else:
                time.sleep(120)

max_stars_repo_name: Ru-Xiang/x-deeplearning


text:
import numpy as np

from .base_element import BaseElement

class Sphere(BaseElement):
    def __init__(self, object_index, medium_index, fl_brightness,
                 center, radius):
        """Sphere element

        Parameters
        ----------
        object_index: float
            Refractive index of the element
        medium_index: float
            Refractive index of surrounding medium
        fl_brightness: float
            Fluorescence brightness
        center: list-like
            Center coordinates (x, y, z) of the sphere [m]
        radius:
            Radius of the sphere [m]
        """
        #: radius of the sphere
        self.radius = radius
        points = np.atleast_2d(center)
        super(Sphere, self).__init__(object_index=object_index,
                                     medium_index=medium_index,
                                     fl_brightness=fl_brightness,
                                     points=points)

    @property
    def center(self):
        return self.points[0]

    def draw(self, grid_size, pixel_size):
        ri = np.ones(grid_size, dtype=float) * self.medium_index
        fl = np.zeros(grid_size, dtype=float)
        center = np.array(grid_size) / 2 - .5
        x = (np.arange(grid_size[0]) - center[0]) * pixel_size
        y = (np.arange(grid_size[1]) - center[1]) * pixel_size
        z = (np.arange(grid_size[2]) - center[2]) * pixel_size

        xx, yy, zz = np.meshgrid(x, y, z, indexing="ij", sparse=True)
        cy, cz, cx = self.center

        volume = (cx-xx)**2 + (cy-yy)**2 + (cz-zz)**2
        inside = volume <= self.radius**2
        ri[inside] = self.object_index
        fl[inside] = self.fl_brightness
        return ri, fl

ext: py

max_stars_repo_name: RI-imaging/cellsino

max_issues_repo_name: RI-imaging/cellsino

max_forks_repo_name: RI-imaging/cellsino

reason: import numpy


text:
"""
A noop estimator which will pause for 15 seconds after the chief exits to test that the chief is
allowed to finish its post-terminate callback.
"""

import random
import time
from typing import List

import numpy as np
import tensorflow as tf

from determined import estimator

def rand_rand_reducer(batch_metrics: List):
    return random.random()

def np_rand_reducer(batch_metrics: List):
    return np.random.random()

class ChiefPauseOnTerminateRunHook(estimator.RunHook):
    def __init__(self, ctx):
        self.ctx = ctx

    def on_trial_close(self) -> None:
        if self.ctx.distributed.get_rank() == 0:
            time.sleep(15)
            print("rank 0 has completed on_trial_close")

class NoopEstimator(estimator.EstimatorTrial):
    def __init__(self, context: estimator.EstimatorTrialContext) -> None:
        self.context = context

    def create_model_fn(self):
        def noop_model_fn(features, labels, mode):
            if mode == tf.estimator.ModeKeys.EVAL:

                eval_metric_ops = {
                    name: self.context.make_metric(metric, reducer, np.float64)
                    for name, metric, reducer in [
                        ("rand_rand", tf.constant([[]]), rand_rand_reducer),
                        ("np_rand", tf.constant([[]]), np_rand_reducer),
                    ]
                }

                return tf.estimator.EstimatorSpec(
                    mode, loss=tf.constant([0.0], dtype=tf.float32), eval_metric_ops=eval_metric_ops
                )

            if mode == tf.estimator.ModeKeys.TRAIN:
                return tf.estimator.EstimatorSpec(
                    mode,
                    loss=tf.constant([0.0], dtype=tf.float32),
                    train_op=tf.no_op(),
                )

        return noop_model_fn

    def build_estimator(self) -> tf.estimator.Estimator:
        _ = [tf.feature_column.numeric_column("x", shape=(), dtype=tf.int64)]
        optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.0)
        _ = self.context.wrap_optimizer(optimizer)

        return tf.compat.v1.estimator.Estimator(model_fn=self.create_model_fn())

    def build_train_spec(self) -> tf.estimator.TrainSpec:
        def fn():
            ds = tf.data.Dataset.range(100).batch(self.context.get_per_slot_batch_size())
            return self.context.wrap_dataset(ds)

        return tf.estimator.TrainSpec(fn, hooks=[ChiefPauseOnTerminateRunHook(self.context)])

    def build_validation_spec(self) -> tf.estimator.EvalSpec:
        def fn():
            ds = tf.data.Dataset.range(100).batch(self.context.get_per_slot_batch_size())
            return self.context.wrap_dataset(ds)

        return tf.estimator.EvalSpec(fn)

max_stars_repo_name: gh-determined-ai/determined


text:
# -*- encoding: utf-8 -*-
# ! python3
from collections import defaultdict
from io import BytesIO
from pathlib import Path

import numpy as np
import torch
from PIL import Image, ImageOps
from loguru import logger
from torchvision import datasets
from torchvision.datasets.folder import make_dataset
from torchvision.transforms import transforms
from tqdm import tqdm

from .transforms import get_crop_params, crop, FixedColorJitter

class TrainDataset(datasets.ImageFolder):
    def __init__(self,
                 img_root,
                 annotation_root,
                 cropping=256,
                 frame_num=10,
                 transform=None,
                 target_transform=None,
                 color_jitter=False):
        super(TrainDataset, self).__init__(img_root,
                                           transform=transform,
                                           target_transform=target_transform)
        # img root and annotation root should have the same class_to_idx
        self.annotations = make_dataset(annotation_root, self.class_to_idx, extensions=('png', 'jpg', 'jpeg'))
        self.cropping = cropping
        self.frame_num = frame_num
        self.color_jitter = color_jitter
        self.rgb_normalize = transforms.Compose([transforms.ToTensor(),
                                                 transforms.Normalize(
                                                     mean=[0.485, 0.456, 0.406],
                                                     std=[0.229, 0.224, 0.225])])
        # read all jpgs and annotations into mem to speed up training
        self.img_bytes = []
        self.annotation_bytes = []

        logger.info(f'Loading {len(self.imgs)} train images.')
        for path, _ in tqdm(self.imgs):
            with Path(path).open(mode='rb') as f:
                self.img_bytes.append(f.read())
        logger.info(f"JPEGImages loaded: {len(self.img_bytes)}.")

        logger.info(f'Loading {len(self.imgs)} train annotations.')
        for path, _ in tqdm(self.annotations):
            with Path(path).open(mode='rb') as f:
                self.annotation_bytes.append(f.read())
        logger.info(f"Annotations loaded: {len(self.annotation_bytes)}.")

    def __getitem__(self, index):
        img_output = []
        annotation_output = []

        # if index reaches end of dataset, get the last frames
        if index + self.frame_num > len(self.imgs):
            index = len(self.imgs) - self.frame_num
        while not self.__is_from_same_video__(index):
            index -= 1
        # get transform params
        if self.color_jitter:
            color_transform = FixedColorJitter(brightness=0.4, contrast=0.4,
                                               saturation=0.4, hue=0.4)
        else:
            color_transform = lambda t: t
        crop_i, crop_j, th, tw = 0, 0, 0, 0
        h_flip = True if torch.rand(size=(1,)).item() < 0.5 else False
        v_flip = True if torch.rand(size=(1,)).item() < 0.5 else False
        for i in range(self.frame_num):
            img = Image.open(BytesIO(self.img_bytes[index + i]))
            img = img.convert('RGB')
            annotation = Image.open(BytesIO(self.annotation_bytes[index + i]))  # (W, H), -P mode
            annotation = annotation.convert('RGB')

            if h_flip:
                img = img.transpose(Image.FLIP_LEFT_RIGHT)
                annotation = annotation.transpose(Image.FLIP_LEFT_RIGHT)
            if v_flip:
                img = img.transpose(Image.FLIP_TOP_BOTTOM)
                annotation = annotation.transpose(Image.FLIP_TOP_BOTTOM)
            if i == 0:
                W, H = img.size
                crop_i, crop_j, th, tw = get_crop_params((W, H), self.cropping)

            # all images and annotations should cropped in the same way
            img_cropped = crop(img, crop_i, crop_j, th, tw)
            annotation_cropped = crop(annotation, crop_i, crop_j, th, tw)
            img_cropped = color_transform(img_cropped)

            img_cropped = self.rgb_normalize(img_cropped).numpy()
            annotation_cropped = np.asarray(annotation_cropped).transpose((2, 0, 1))
            img_output.append(img_cropped)
            annotation_output.append(annotation_cropped)

        img_output = torch.from_numpy(np.asarray(img_output)).float()
        annotation_output = torch.from_numpy(np.asarray(annotation_output)).float()
        _, video_index = self.imgs[index + self.frame_num - 1]
        return img_output, annotation_output, video_index

    def __is_from_same_video__(self, index):
        _, indexStart = self.imgs[index]
        _, indexEnd = self.imgs[index + self.frame_num - 1]
        return indexStart == indexEnd

class InferenceDataset(datasets.ImageFolder):
    """
        Load one frame at a time.
        Used for inference.
    """

    def __init__(self,
                 root,
                 transform=None,
                 target_transform=None,
                 disable=False,
                 inference_strategy='single',
                 scale=None):
        super(InferenceDataset, self).__init__(root,
                                               transform=transform,
                                               target_transform=target_transform)
        self.img_bytes = []
        self.rgb_normalize = transforms.Compose([transforms.ToTensor(),
                                                 transforms.Normalize(
                                                     mean=[0.485, 0.456, 0.406],
                                                     std=[0.229, 0.224, 0.225])])
        logger.info(f'Loading {len(self.imgs)} inference images.')
        for path, _ in tqdm(self.imgs, disable=disable):
            with Path(path).open(mode='rb') as f:
                self.img_bytes.append(f.read())
        logger.info(f'Loaded {len(self.img_bytes)} inference images.')
        self.idx_to_class = {v: k for k, v in self.class_to_idx.items()}
        self.inference_strategy = inference_strategy
        self.scale = scale

    def __getitem__(self, index):
        path, video_index = self.imgs[index]
        img = Image.open(BytesIO(self.img_bytes[index]))
        img = img.convert('RGB')
        img_size = np.ceil(np.array(img.size)).astype(np.int)
        img = img.resize(img_size, Image.ANTIALIAS)
        normalized = self.rgb_normalize(np.asarray(img))
        if self.inference_strategy == 'hor-flip':
            img = ImageOps.mirror(img)
            normalized_flipped = self.rgb_normalize(np.asarray(img))
            return (normalized, normalized_flipped), self.idx_to_class[video_index]
        elif self.inference_strategy == 'vert-flip':
            img = ImageOps.flip(img)
            normalized_flipped = self.rgb_normalize(np.asarray(img))
            return (normalized, normalized_flipped), self.idx_to_class[video_index]
        elif self.inference_strategy == '2-scale' or self.inference_strategy == 'hor-2-scale':
            img_2_size = np.ceil(np.array(img.size) * self.scale).astype(np.int)
            if self.inference_strategy == 'hor-2-scale':
                img = ImageOps.mirror(img)
            img_2 = img.resize(img_2_size, Image.ANTIALIAS)
            normalized_2 = self.rgb_normalize(np.asarray(img_2))
            return (normalized, normalized_2), self.idx_to_class[video_index]

        return normalized, self.idx_to_class[video_index]

    def __len__(self):
        return len(self.imgs)

class TripletLossTrainDataset(datasets.ImageFolder):
    def __init__(self,
                 img_root,
                 annotation_root,
                 transform=None,
                 target_transform=None):
        super(TripletLossTrainDataset, self).__init__(img_root,
                                                      transform=transform,
                                                      target_transform=target_transform)
        annotations_data = make_dataset(annotation_root, self.class_to_idx, extensions=('png', 'jpg', 'jpeg'))
        self.rgb_normalize = transforms.Compose([transforms.ToTensor(),
                                                 transforms.Normalize(
                                                     mean=[0.485, 0.456, 0.406],
                                                     std=[0.229, 0.224, 0.225])])
        self.annotation_convert = transforms.Compose([transforms.ToTensor()])
        self.data = defaultdict(lambda: [])

        assert len(self.imgs) == len(annotations_data)

        logger.info(f'Loading {len(self.imgs)} train image, annotation pairs.')
        for (image_path, image_sequence_idx), (annotation_path, annotation_sequence_idx) in tqdm(
                zip(self.imgs, annotations_data), total=len(self.imgs)):
            assert image_sequence_idx == annotation_sequence_idx
            with Path(image_path).open(mode='rb') as f:
                img = f.read()
            with Path(annotation_path).open(mode='rb') as f:
                annotation = f.read()

            self.data[image_sequence_idx].append((img, annotation))

        logger.info(f"Pairs loaded: {len(self.data)}.")

    def __getitem__(self, index):
        sequence = []

        for img, annotation in self.data[index]:
            img = Image.open(BytesIO(img))
            img = self.rgb_normalize(img.convert('RGB'))

            annotation = Image.open(BytesIO(annotation))
            annotation = annotation.convert('RGB')
            annotation = np.asarray(annotation).transpose((2, 0, 1))
            annotation = torch.from_numpy(annotation.copy()).float()

            sequence.append((img, annotation))

        return sequence

    def __len__(self):
        return len(self.data)

max_stars_repo_name: hynekdav/semi-supervised-VOS

max_issues_repo_name: hynekdav/semi-supervised-VOS

max_forks_repo_name: hynekdav/semi-supervised-VOS


text:
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Script to cluster mapping vectors created with :py:mod:`src.mapping.mapthreading`.
"""
# Ugly import hack
import sys
import os
sys.path.insert(0, os.path.abspath('../../'))

# STANDARD
import argparse
import codecs
from collections import defaultdict

# EXTERNAL
import numpy
from scipy.optimize import minimize
from sklearn.cluster import DBSCAN
from sklearn.decomposition import PCA
from sklearn.metrics import silhouette_score

# PROJECT
from src.misc.helpers import alt

def main():
	"""
	This is the main function. It uses the parsed command line arguments to pick the right function to execute.
	"""
	argparser = init_argparser()
	args = argparser.parse_args()
	if args.train:
		train_clustering_parameters(args.input[0])
	else:
		if args.resolve:
			cluster_mappings(args.input[0], True, args.pca, indices_inpath=args.input[1], epsilon=args.epsilon, min_s=args.min)
		else:
			cluster_mappings(args.input[0], True, args.pca, epsilon=args.epsilon, min_s=args.min)

def train_clustering_parameters(vector_inpath):
	"""
	Functions that tries to figure out the optimal clustering parameters in regard to DBSCAN's epsilon,
	min_samples and p.

	Args:
		vector_inpath (str): Path to vector file. File has to have the following format (separated by spaces):
			<index of original vector #1> <index of original vector #2> <Dimension 1> ... <Dimension n>
	"""
	x0 = numpy.array([2.5, 20, 2])  # First parameters
	print alt("Load mappings...")
	indices, model = load_mappings_from_model(vector_inpath)
	X = numpy.array([model[key] for key in indices])  # Arrange data for optimization
	print alt("Start training...")

	# Cluster data and calculate loss as silhouette coefficient per cluster
	def simple_clustering(x):
		# Start clustering
		print alt("Current parameters: %s" % (str(x)))
		dbscan = DBSCAN(eps=x[0], min_samples=x[1], p=x[2])
		dbscan.fit(X)

		# Evaluate results
		cluster_sizes = get_cluster_size(dbscan.labels_)
		print alt("Current cluster sizes: %s" % (cluster_sizes))
		sscore = silhouette_score(X, dbscan.labels_)
		tscore = (sscore / (len(cluster_sizes.keys()) - 1))
		print alt("Current value of objective function: %.5f" % (tscore))
		print "-" * 50
		return tscore

	# Start minimizing
	result = minimize(simple_clustering, x0, method="Nelder-Mead")
	print result.x  # Print resulting parameter configuration

def cluster_mappings(vector_inpath, do_pca=False, target_dim=100, indices_inpath=None, epsilon=2.625, min_s=20):
	"""
	Cluster mapping vectors created with :py:mod:`src.mapping.mapthreading` or :py:mod:`rc.mapping.map_vectors.py`.
	Because just reading about the number of clusters and their sizes, there's an option to resolve the indices of
	the vectors in the cluster to their original word pairs.

	Args:
		vector_inpath (str): Path to vector file. File should have the following format (separated by spaces):
			<index of original vector #1> <index of original vector #2> <Dimension 1> ... <Dimension n>
		do_pca (bool): Flag to indicate whether PCA should be executed before clustering to reduce amount of
		computation.
		target_dim (int): Number of dimensions vectors should be shrunk to in case PCA is performed.
		indices_inpath (str): Path to file with the indices given to words. The file should have the following format:
			<index of word>	<word> (separated by tab)
		epsilon (float): Radius of circle DBSCAN uses to look for other data points.
		min_s (int): Minimum number of points in radius epsilon DBSCAN needs to declare a point a core object.
	"""
	print alt("Load mappings...")
	indices, model = load_mappings_from_model(vector_inpath)
	X = numpy.array([model[key] for key in indices])
	del model  # free up memory

	# do PCA if wanted
	if do_pca:
		print alt("Truncate vectors with PCA to %i dimensions..." % target_dim)
		pca = PCA(n_components=target_dim)
		pca.fit(X)
		X = pca.transform(X)

	# Start clustering
	print alt("Cluster points...")
	dbscan = DBSCAN(eps=epsilon, min_samples=min_s, p=2)
	dbscan.fit(X)

	# Get results
	labels = dbscan.labels_
	print alt("Cluster sizes:")
	print get_cluster_size(labels)
	print alt("Finished clustering!")
	sscore = silhouette_score(X, labels)
	print("Silhouette Coefficient: %0.3f" % (sscore))

	# Resolve indices and print all word-pairs in clusters if wanted
	if indices_inpath:
		resolve_indices(indices, labels, indices_inpath, model)

def resolve_indices(points, labels, indices_inpath):
	"""
	Resolves the indices of word pairs found in a cluster to their real names.

	Args:
		points (list): List of datapoints
		labels (list): List of unique cluster labels
		indices_inpath (str): Path to file with the indices given to words. The file should have the following format:
			<index of word>	<word> (separated by tab)
	"""
	indices = load_indices(indices_inpath)
	clusters = aggregate_cluster(points, labels)
	for cluster in clusters.keys():
		cluster_points = clusters[cluster]
		print "\nCluster %i:\n%s\n" % (cluster, 12 * "-")
		for cluster_point in cluster_points:
			words = (indices[cluster_point[0]], indices[cluster_point[1]])
			print "%s - %s" % (words[0], words[1])

def aggregate_cluster(points, labels):
	"""
	Arranges all clusters in a list, where a sublist with all points at index i corresponds with the
	custer with label i.

	Args:
		points (list): List of datapoints
		labels (list): List of unique cluster labels

	Returns:
		list: list of lists of datapoints belonging to the i-th cluster
	"""
	print alt("Aggregate clusters...")
	clusters = defaultdict(tuple)
	for i in range(len(labels)):
		label = labels[i]
		if label == -1:
			continue
		if label in clusters.keys():
			clusters[label].append(points[i])
		else:
			clusters[label] = [points[i]]
	return clusters

def load_indices(indices_inpath):
	"""
	Load word indices from a file. The file should have the following format: <index of word>	<word> (separated by
	tab)

	Args:
		indices_inpath (str): Path to index file.
	"""
	print alt("Load indices...")
	indices = defaultdict(str)
	with codecs.open(indices_inpath, "rb", "utf-8") as indices_inpath:
		line = indices_inpath.readline().strip()
		while line:
			parts = line.split("\t")
			index = int(parts[0])
			word = parts[1].replace(" ", "_")
			indices[index] = word
			line = indices_inpath.readline().strip()
	return indices

def get_cluster_size(labels):
	"""
	Calculate the size of every cluster found by DBSCAN.

	Args:
		labels (list): List of cluster IDs assigned to every data point.

	Returns:
		defaultdict: Dictionary of cluster sizes with cluster id as key and cluster size as value.
	"""
	sizes = defaultdict(int)
	for label in labels:
		if label in sizes.keys():
			sizes[label] += 1
		else:
			sizes[label] = 0
	return sizes

def load_mappings_from_model(mapping_inpath):
	"""
	Load mapping vectors from file.

	Args:
		mapping_inpath: Path mapping vector file.

	Returns:
		tuple: A tuple of a list of word index pairs and a dictionary (defaultdict) with index pair tuple as key
		and mapping vector (as numpy.array) as value.
	"""
	indices_list = []
	mappings_dict = defaultdict(numpy.array)
	with codecs.open(mapping_inpath, "rb", "utf-8") as mapping_infile:
		line = mapping_infile.readline().strip()
		while line:
			parts = line.strip().split(" ")
			indices = (int(parts[0]), int(parts[1]))
			vector = numpy.array([float(dimension.strip()) for dimension in parts[2:]])
			indices_list.append(indices)
			mappings_dict[indices] = vector
			line = mapping_infile.readline().strip()
	return indices_list, mappings_dict

def init_argparser():
	"""
	Initialize all possible arguments for the argument parser.

	Returns:
		:py:mod:`argparse.ArgumentParser`: ArgumentParser object with command line arguments for this script.
	"""
	argparser = argparse.ArgumentParser()
	argparser.add_argument('--input',
							required=True,
							nargs='+',
							help='Evaluation inputs (Path to vector file / additional data')
	argparser.add_argument('--pca',
							type=int,
							help='Shorten number of dimension of mapping vectors')
	argparser.add_argument('--train', action='store_true')
	argparser.add_argument('--resolve', action='store_true')
	argparser.add_argument('--epsilon', type=float, help='Epsilon parameter for DBSCAN.')
	argparser.add_argument('--min', type=int, help='Minimum number of neighbouring objects of a point to become a core object in DBSCAN')
	return argparser

if __name__ == "__main__":
	main()

max_stars_repo_name: Kaleidophon/BARBarA

max_issues_repo_name: Kaleidophon/BARBarA

max_forks_repo_name: Kaleidophon/BARBarA

reason: import numpy,from scipy


text:
"""Testing dataset stats generation.

Author: Yuhuang Hu
Email : duguyue100@gmail.com
"""

from __future__ import print_function
import os
import numpy as np
import cv2
import cPickle as pickle
import h5py
from spikefuel import dvsproc, helpers
from time import gmtime, strftime

option = "export-vot-bounding-boxes"
data_path = os.environ["SPIKEFUEL_DATA"]
stats_path = os.path.join(data_path, "sf_data")

if option == "vot":
    # Load VOT Challenge Dataset
    vot_fn = "INI_VOT_30fps_20160424.hdf5"
    vot_path = os.path.join(data_path, vot_fn)
    vot_db = h5py.File(vot_path, mode="r")
    vot_stats_path = os.path.join(stats_path, "vot_stats.pkl")

    # load vot stats
    f = file(vot_stats_path, mode="r")
    vot_stats = pickle.load(f)
    f.close()
    vot_list = vot_stats['vot_list']
    num_frames = vot_stats['num_frames']

    avg_num_frames = np.average(np.asarray(num_frames))
    print("Average Number of Frames: %.2f" % (avg_num_frames))

    tot_t = 0.
    tot_freq = 0.
    avg_freq = 0.
    for vidseq in vot_list:
        timestamps = vot_db[vidseq]["timestamps"][()]
        tot_t += (timestamps[-1]-timestamps[0])/1e6
        event_arr = dvsproc.cal_event_count(timestamps)
        event_freq = dvsproc.cal_event_freq(event_arr, window=1000000)
        tot_freq += np.max(event_freq[:, 1])
        t = float(timestamps[-1]-timestamps[0])/1e6
        avg_freq += float(timestamps.shape[0])/float(t)
        print("Video sequence %s is processed" % (vidseq))

    print("Average Recording Length: %.2f s" % (tot_t/len(vot_list)))
    print("Average Maximum Firing Rate: %.2f K" % (tot_freq/len(vot_list)/1e3))
    print("Average Firing Rate: %.2f K" % (avg_freq/len(vot_list)/1e3))

if option == "tracking":
    tracking_fn = "INI_TrackingDataset_30fps_20160424.hdf5"
    tracking_path = os.path.join(data_path, tracking_fn)
    tracking_db = h5py.File(tracking_path, mode="r")
    tracking_stats_path = os.path.join(stats_path, "tracking_stats.pkl")

    f = file(tracking_stats_path, mode="r")
    tracking_stats = pickle.load(f)
    f.close()

    pl = tracking_stats["primary_list"]
    sl = tracking_stats["secondary_list"]

    num_videos = 0.
    tot_frames = 0.
    tot_t = 0.
    tot_freq = 0.
    avg_freq = 0.
    for pc in pl:
        # remove sequence Kalal until I got more memory
        if pc != "Kalal":
            for sc in sl[pc]:
                num_videos += 1
                tot_frames += int(tracking_db[pc][sc].attrs["num_frames"])
                timestamps = tracking_db[pc][sc]["timestamps"][()]
                tot_t += (timestamps[-1]-timestamps[0])/1e6
                event_arr = dvsproc.cal_event_count(timestamps)
                event_freq = dvsproc.cal_event_freq(event_arr, window=1000000)
                tot_freq += np.max(event_freq[:, 1])
                t = float(timestamps[-1]-timestamps[0])/1e6
                avg_freq += float(timestamps.shape[0])/float(t)
                print("Video sequence %s is processed" % (sc))

    print("Total Number of Videos: %.2f" % (num_videos))
    print("Average Number of Frames: %.2f" % (tot_frames/num_videos))
    print("Average Recording Length: %.2f s" % (tot_t/num_videos))
    print("Average Maximum Firing Rate: %.2f K" % (tot_freq/num_videos/1e3))
    print("Average Firing Rate: %.2f K" % (avg_freq/num_videos/1e3))

if option == "ucf50":
    ucf50_fn = "INI_UCF50_30fps_20160424.hdf5"
    ucf50_path = os.path.join(data_path, ucf50_fn)
    ucf50_db = h5py.File(ucf50_path, mode="r")
    ucf50_stats_path = os.path.join(stats_path, "ucf50_stats.pkl")

    f = file(ucf50_stats_path, mode="r")
    ucf50_stats = pickle.load(f)
    f.close()

    ucf50_list = ucf50_stats["ucf50_list"]

    num_videos = 0.
    tot_frames = 0.
    tot_t = 0.
    tot_freq = []
    avg_freq = []
    for cn in ucf50_list:
        for vid_name in ucf50_stats[cn]:
            vid_n, vid_ex = os.path.splitext(vid_name)
            num_videos += 1
            tot_frames += int(ucf50_db[cn][vid_n].attrs["num_frames"])
            timestamps = ucf50_db[cn][vid_n]["timestamps"][()]

            tot_t += (timestamps[-1]-timestamps[0])/1e6
            event_arr = dvsproc.cal_event_count(timestamps)
            event_freq = dvsproc.cal_event_freq(event_arr, window=1000000)
            t = float(timestamps[-1]-timestamps[0])/1e6
            tot_freq.append(np.max(event_freq[:, 1]))
            avg_freq.append(float(timestamps.shape[0])/float(t))
            print("Video sequence %s is processed" % (vid_n))

    average_freq = np.average(np.asarray(tot_freq))
    mean_freq = np.average(np.asarray(avg_freq))
    print("Total Number of Videos: %.2f" % (num_videos))
    print("Average Number of Frames: %.2f" % (tot_frames/num_videos))
    print("Average Recording Length: %.2f s" % (tot_t/num_videos))
    print("Average Maximum Firing Rate: %.2f K" % (average_freq/1e3))
    print("Average Firing Rate: %.2f K" % (mean_freq/1e3))

if option == "caltech256":
    caltech_fn = "INI_Caltech256_10fps_20160424.hdf5"
    caltech_path = os.path.join(data_path, caltech_fn)
    caltech_db = h5py.File(caltech_path, mode="r")
    caltech_stats_path = os.path.join(stats_path, "caltech256_stats.pkl")

    f = file(caltech_stats_path, mode="r")
    caltech_stats = pickle.load(f)
    f.close()

    caltech_list = caltech_stats["caltech256_list"]

    num_videos = 0.
    tot_t = 0.
    tot_freq = []
    avg_freq = []
    wrong_recordings = []
    for cn in caltech_list:
        for img_name in caltech_stats[cn]:
            img_n, img_ex = os.path.splitext(img_name)
            num_videos += 1
            timestamps = caltech_db[cn][img_n]["timestamps"][()]
            if timestamps.size != 0:
                tot_t += (timestamps[-1]-timestamps[0])/1e6
                event_arr = dvsproc.cal_event_count(timestamps)
                event_freq = dvsproc.cal_event_freq(event_arr, window=1000000)
                t = float(timestamps[-1]-timestamps[0])/1e6
                tot_freq.append(np.max(event_freq[:, 1]))
                avg_freq.append(float(timestamps.shape[0])/float(t))
            else:
                wrong_recordings.append(img_n)
            print("Video sequence %s is processed" % (img_n))

    average_freq = np.average(np.asarray(tot_freq))
    mean_freq = np.average(np.asarray(avg_freq))
    print("Total Number of Videos: %.2f" % (num_videos))
    print("Average Recording Length: %.2f s" % (tot_t/num_videos))
    print("Average Maximum Firing Rate: %.2f K" % (average_freq/1e3))
    print("Average Firing Rate: %.2f K" % (mean_freq/1e3))
    print(wrong_recordings)

if option == "export-vot-bounding-boxes":
    vot_fn = "INI_VOT_30fps_20160610.hdf5"
    vot_path = os.path.join(data_path, vot_fn)
    vot_db = h5py.File(vot_path, mode="r")
    vot_stats_path = os.path.join(stats_path, "vot_stats.pkl")
    vot_gt_path = os.path.join(data_path, "vot-gt")
    if not os.path.isdir(vot_gt_path):
        os.mkdir(vot_gt_path)

    # load vot stats
    f = file(vot_stats_path, mode="r")
    vot_stats = pickle.load(f)
    f.close()
    vot_list = vot_stats['vot_list']
    num_frames = vot_stats['num_frames']
    for vidseq in vot_list:
        gt_filename = vidseq+"-groundtruth.txt"
        gt_savepath = os.path.join(vot_gt_path, gt_filename)
        header = "File is created at: "
        sys_time = strftime("%Y-%m-%d %H:%M:%S", gmtime())
        header += sys_time+"\n"
        header += "Each line is a bounding box and has 9 values.\n"
        header += "The structure is as follows:\n"
        header += "[Timestamps] [X1, Y1] [X2, Y2] [X3, Y3] [X4, Y4]"
        gt = vot_db[vidseq]["bounding_box"][()]
        # gt[:, 0] -= 133332

        np.savetxt(gt_savepath, gt, fmt='%.2f', delimiter=',', header=header)
        print("Ground Truth for %s is saved at %s" % (vidseq, gt_savepath))

if option == "export-td-bounding-boxes":
    tracking_fn = "INI_TrackingDataset_30fps_20160610.hdf5"
    tracking_path = os.path.join(data_path, tracking_fn)
    tracking_db = h5py.File(tracking_path, mode="r")
    tracking_stats_path = os.path.join(stats_path, "tracking_stats.pkl")
    tracking_gt_path = os.path.join(data_path, "tracking-gt")

    f = file(tracking_stats_path, mode="r")
    tracking_stats = pickle.load(f)
    f.close()

    pl = tracking_stats["primary_list"]
    sl = tracking_stats["secondary_list"]

    for pc in pl:
        # remove sequence Kalal until I got more memory
        if pc != "Kalal":
            for sc in sl[pc]:
                pc_path = os.path.join(tracking_gt_path, pc)
                if not os.path.isdir(pc_path):
                    os.mkdir(pc_path)
                sc_path = os.path.join(pc_path, sc+"-groundtruth.txt")
                header = "File is created at: "
                sys_time = strftime("%Y-%m-%d %H:%M:%S", gmtime())
                header += sys_time+"\n"
                header += "Each line is a bounding box and has 9 values. \n"
                header += "The structure is as follows:\n"
                header += "[Timestamps] [X1, Y1] [X2, Y2] [X3, Y3] [X4, Y4]"

                gt = tracking_db[pc][sc]["bounding_box"][()]

                np.savetxt(sc_path, gt, fmt='%.2f', delimiter=',',
                           header=header)

                print("Ground Truth for %s is saved at %s" % (sc, sc_path))

if option == "calculate-tracking-event-burst-timing":
    tracking_fn = "INI_TrackingDataset_30fps_20160610.hdf5"
    td_path = os.path.join(data_path, "TrackingDataset")
    tracking_path = os.path.join(data_path, tracking_fn)
    tracking_db = h5py.File(tracking_path, mode="a")
    tracking_stats_path = os.path.join(stats_path, "tracking_stats.pkl")
    tracking_gt_path = os.path.join(data_path, "tracking-gt")

    f = file(tracking_stats_path, mode="r")
    tracking_stats = pickle.load(f)
    f.close()

    pl = tracking_stats["primary_list"]
    sl = tracking_stats["secondary_list"]
    key_idx_list = []
    key_idx_ts = []
    for pc in pl:
        # remove sequence Kalal until I got more memory
        if pc != "Kalal":
            for sc in sl[pc]:
                timestamps = tracking_db[pc][sc]["timestamps"][()]

                key_idx = dvsproc.cal_first_response(timestamps)
                key_idx_list.append(key_idx)
                key_idx_ts.append(timestamps[key_idx]-timestamps[0])
                print("%s: %d" % (sc, timestamps[key_idx]-timestamps[0]))

    key_idx_ts = np.array(key_idx_ts)
    key_idx_ts = dvsproc.remove_outliers(key_idx_ts)
    key_idx_time = round(np.mean(key_idx_ts))
    print(key_idx_time)

    key_idx_list_new = []
    for pc in pl:
        # remove sequence Kalal until I got more memory
        if pc != "Kalal":
            for sc in sl[pc]:
                gt_path = os.path.join(td_path, pc, sc, "groundtruth.txt")
                gt = np.loadtxt(gt_path, dtype=np.float32, delimiter=",")
                gt = helpers.trans_groundtruth(gt, method="size")
                gt = np.reshape(gt, (gt.shape[0], 4, 2))

                # load one original frame
                frame_path = os.path.join(td_path, pc, sc,
                                          tracking_stats[sc][0])
                origin_frame = cv2.imread(frame_path)

                num_frames = int(tracking_db[pc][sc].attrs["num_frames"])

                timestamps = tracking_db[pc][sc]["timestamps"][()]
                x_pos = tracking_db[pc][sc]["x_pos"][()]
                y_pos = tracking_db[pc][sc]["y_pos"][()]
                pol = tracking_db[pc][sc]["pol"][()]

                key_idx = dvsproc.find_nearest(timestamps,
                                               key_idx_time+timestamps[0])
                key_idx_list_new.append(key_idx)
                print("%s: %d" % (sc, key_idx))

                (timestamps, x_pos,
                 y_pos, pol) = dvsproc.clean_up_events(timestamps, x_pos,
                                                       y_pos, pol,
                                                       key_idx=key_idx)
                frames, fs, ts = dvsproc.gen_dvs_frames(timestamps, x_pos,
                                                        y_pos, pol, num_frames,
                                                        fs=3)
                ts = np.array(ts)

                shift = helpers.cal_img_shift(origin_frame.shape,
                                              frames[0].shape)
                ratio = helpers.cal_bound_box_ratio(gt, origin_frame.shape[0],
                                                    origin_frame.shape[1])
                gt = helpers.cal_bound_box_position(
                        ratio,
                        frames[0].shape[0]-shift[1],
                        frames[0].shape[1]-shift[0])
                gt[:, :, 0] += shift[0]/2.
                gt[:, :, 1] += shift[1]/2.

                gt = np.reshape(gt, (gt.shape[0], 8))
                print("[MESSAGE] Size of groundtruth: "+str(gt.shape))

                gt = np.vstack((ts, gt.T)).T

                del tracking_db[pc][sc]["bounding_box"]
                tracking_db[pc][sc].create_dataset(
                    "bounding_box",
                    data=gt.astype(np.float32),
                    dtype=np.float32)
                print("[MESSAGE] Sequence %s bounding box is saved" % (sc))

    print(key_idx_list)
    print(key_idx_list_new)

if option == "calculate-vot-event-burst-timing":
    vot_fn = "INI_VOT_30fps_20160424.hdf5"
    vot_path = os.path.join(data_path, vot_fn)
    vot_data_path = os.path.join(data_path, "vot2015")
    vot_db = h5py.File(vot_path, mode="a")
    vot_stats_path = os.path.join(stats_path, "vot_stats.pkl")
    vot_gt_path = os.path.join(data_path, "vot-gt-shifted")
    if not os.path.isdir(vot_gt_path):
        os.mkdir(vot_gt_path)

    # load vot stats
    f = file(vot_stats_path, mode="r")
    vot_stats = pickle.load(f)
    f.close()
    vot_list = vot_stats['vot_list']
    num_frames = vot_stats['num_frames']
    num_seq = len(vot_list)
    key_idx_list = []
    key_idx_ts = []
    for vidseq in vot_list:
        timestamps = vot_db[vidseq]["timestamps"][()]

        key_idx = dvsproc.cal_first_response(timestamps)
        key_idx_list.append(key_idx)
        key_idx_ts.append(timestamps[key_idx] - timestamps[0])
        print("%s: %d" % (vidseq, timestamps[key_idx] - timestamps[0]))

    key_idx_ts = np.array(key_idx_ts)
    key_idx_ts = dvsproc.remove_outliers(key_idx_ts)
    key_idx_time = round(np.mean(key_idx_ts))
    print(key_idx_time)

    key_idx_list_new = []
    for i in xrange(num_seq):
        vidseq = vot_list[i]
        # load groundtruth
        gt_path = os.path.join(vot_data_path, vot_list[i]+"/groundtruth.txt")
        gt = np.loadtxt(gt_path, dtype=float, delimiter=",")
        gt = np.reshape(gt, (gt.shape[0], 4, 2))

        # load a frame as reference
        frame_path = os.path.join(vot_data_path, vot_list[i]+"/00000001.jpg")
        origin_frame = cv2.imread(frame_path)
        print("[MESSAGE] Loading sequence %s" % (vot_list[i]))

        timestamps = vot_db[vidseq]["timestamps"][()]
        x_pos = vot_db[vidseq]["x_pos"][()]
        y_pos = vot_db[vidseq]["y_pos"][()]
        pol = vot_db[vidseq]["pol"][()]

        key_idx = dvsproc.find_nearest(timestamps,
                                       key_idx_time+timestamps[0])
        key_idx_list_new.append(key_idx)
        print("%s: %d" % (vidseq, key_idx))

        (timestamps, x_pos,
         y_pos, pol) = dvsproc.clean_up_events(timestamps, x_pos, y_pos,
                                               pol, key_idx=key_idx)
        frames, fs, ts = dvsproc.gen_dvs_frames(timestamps, x_pos, y_pos,
                                                pol, num_frames[i], fs=3)
        ts = np.array(ts)

        shift = helpers.cal_img_shift(origin_frame.shape, frames[0].shape)
        ratio = helpers.cal_bound_box_ratio(gt, origin_frame.shape[0],
                                            origin_frame.shape[1])
        gt = helpers.cal_bound_box_position(ratio,
                                            frames[0].shape[0]-shift[1],
                                            frames[0].shape[1]-shift[0])
        gt[:, :, 0] += shift[0]/2.
        gt[:, :, 1] += shift[1]/2.

        gt = np.reshape(gt, (gt.shape[0], 8))
        print("[MESSAGE] Size of groundtruth: "+str(gt.shape))

        gt = np.vstack((ts, gt.T)).T

        del vot_db[vidseq]["bounding_box"]
        vot_db[vidseq].create_dataset("bounding_box",
                                      data=gt.astype(np.float32),
                                      dtype=np.float32)
        print("[MESSAGE] Sequence %s bounding box is saved" % (vidseq))

    print(key_idx_list)
    print(key_idx_list_new)

max_issues_repo_name: colinshane/spikefuel

max_forks_repo_name: colinshane/spikefuel


text:
import numpy as np
try:
    import PyQt4.QtGui as QtGui
    import PyQt4.QtCore as QtCore
    from PyQt4.QtGui import QApplication
except:
    import PyQt5.QtGui as QtGui
    import PyQt5.QtCore as QtCore
    from PyQt5.QtWidgets import QApplication

import os
import pyqtgraph as pg
import pandas as pd

import ibeatles.fitting.fitting_handler
from ibeatles.utilities.array_utilities import get_min_max_xy
from ibeatles.utilities.math_tools import get_random_value

class TableDictionaryHandler(object):

    selected_color = {'pen': (0,0,0,30),
                      'brush': (0,255,0,150)}

    lock_color = {'pen': (0,0,0,30),
                  'brush': (255,0,0,240)}

    header = ['x0','y0','x1','y1','row_index','column_index','lock','active',
                      'fitting_confidence','d_spacing_value','d_spacing_err','d_spacing_fixed',
                      'sigma_value','sigma_err','sigma_fixed',
                      'intensity_value','intensity_err','intensity_fixed',
                      'alpha_value','alpha_err','alpha_fixed',
                      'a1_value','a1_err','a1_fixed',
                      'a2_value','a2_err','a2_fixed',
                      'a5_value','a5_err','a5_fixed',
                      'a6_value','a6_err','a6_fixed']

    def __init__(self, parent=None):
        self.parent = parent

    def fill_table_with_variable(self, variable_name='d_spacing', value=np.NaN, list_keys=[], all_keys=False):
        table_dictionary = self.parent.table_dictionary
        if all_keys:
            list_keys = table_dictionary.keys()

        for _key in list_keys:
            table_dictionary[_key][variable_name]['val'] = value

        self.parent.table_dictionary = table_dictionary

    def populate_table_dictionary_entry(self, index=0, array=[]):
        table_dictionary = self.parent.table_dictionary

        table_dictionary[str(index)] = {'bin_coordinates': {'x0': array[0],
                                                            'x1': array[2],
                                                            'y0': array[1],
                                                            'y1': array[3]},
                                        'selected_item': None,
                                        'locked_item': None,
                                        'row_index': array[4],
                                        'column_index': array[5],
                                        'selected': False,
                                        'lock': array[6],
                                        'active': array[7],
                                        'fitting_confidence': array[8],
                                        'd_spacing': {'val': array[9],
                                                      'err': array[10],
                                                      'fixed': array[11]},
                                        'sigma': {'val': array[12],
                                                  'err': array[13],
                                                  'fixed': array[14]},
                                        'intensity': {'val': array[15],
                                                      'err': array[16],
                                                      'fixed': array[17]},
                                        'alpha': {'val': array[18],
                                                  'err': array[19],
                                                  'fixed': array[20]},
                                        'a1': {'val': array[21],
                                               'err': array[22],
                                               'fixed': array[23]},
                                        'a2': {'val': array[24],
                                               'err': array[25],
                                               'fixed': array[26]},
                                        'a5': {'val': array[27],
                                               'err': array[28],
                                               'fixed': array[29]},
                                        'a6': {'val': array[30],
                                               'err': array[31],
                                               'fixed': array[32]},
                                        }

        self.parent.table_dictionary = table_dictionary

    def create_table_dictionary(self):
        '''
        this will define the corner position and index of each cell
        '''
        if len(np.array(self.parent.data_metadata['normalized']['data_live_selection'])) == 0:
            return

        if not self.parent.table_dictionary == {}:
            return

        bin_size = self.parent.binning_bin_size
        pos = self.parent.binning_line_view['pos']

        # calculate outside real edges of bins
        min_max_xy = get_min_max_xy(pos)

        from_x = min_max_xy['x']['min']
        to_x = min_max_xy['x']['max']

        from_y = min_max_xy['y']['min']
        to_y = min_max_xy['y']['max']

        table_dictionary = {}
        _index = 0
        _index_col = 0
        for _x in np.arange(from_x, to_x, bin_size):
            _index_row = 0
            for _y in np.arange(from_y, to_y, bin_size):
                _str_index = str(_index)

                table_dictionary[_str_index] = {'bin_coordinates': {'x0': np.NaN,
                                                                    'x1': np.NaN,
                                                                    'y0': np.NaN,
                                                                    'y1': np.NaN},
                                                'selected_item': None,
                                                'locked_item': None,
                                                'row_index': _index_row,
                                                'column_index': _index_col,
                                                'selected': False,
                                                'lock': False,
                                                'active': False,
                                                'fitting_confidence': np.NaN,
                                                'd_spacing': {'val': np.NaN,
                                                              'err': np.NaN,
                                                              'fixed': False},
                                                'sigma': {'val': np.NaN,
                                                          'err': np.NaN,
                                                          'fixed': False},
                                                'intensity': {'val': np.NaN,
                                                              'err': np.NaN,
                                                              'fixed': False},
                                                'alpha': {'val': np.NaN,
                                                          'err': np.NaN,
                                                          'fixed': False},
                                                'a1': {'val': np.NaN,
                                                       'err': np.NaN,
                                                       'fixed': False},
                                                'a2': {'val': np.NaN,
                                                       'err': np.NaN,
                                                       'fixed': False},
                                                'a5': {'val': np.NaN,
                                                       'err': np.NaN,
                                                       'fixed': False},
                                                'a6': {'val': np.NaN,
                                                       'err': np.NaN,
                                                       'fixed': False},
                                                }
                table_dictionary[_str_index]['bin_coordinates']['x0'] = _x
                table_dictionary[_str_index]['bin_coordinates']['x1'] = _x + bin_size
                table_dictionary[_str_index]['bin_coordinates']['y0'] = _y
                table_dictionary[_str_index]['bin_coordinates']['y1'] = _y + bin_size

                # create the box to show when bin is selected
                selection_box = pg.QtGui.QGraphicsRectItem(_x, _y,
                                                           bin_size,
                                                           bin_size)
                selection_box.setPen(pg.mkPen(self.selected_color['pen']))
                selection_box.setBrush(pg.mkBrush(self.selected_color['brush']))
                table_dictionary[_str_index]['selected_item'] = selection_box

                # create the box to show when bin is locked
                lock_box = pg.QtGui.QGraphicsRectItem(_x, _y,
                                                           bin_size,
                                                           bin_size)
                lock_box.setPen(pg.mkPen(self.lock_color['pen']))
                lock_box.setBrush(pg.mkBrush(self.lock_color['brush']))
                table_dictionary[_str_index]['locked_item'] = lock_box

                _index += 1
                _index_row += 1

            _index_col += 1

        self.parent.table_dictionary = table_dictionary

        #self.parent.fitting_ui.table_dictionary = table_dictionary

        self.parent.fitting_selection['nbr_row'] = _index_row
        self.parent.fitting_selection['nbr_column'] = _index_col

    def full_table_selection_tool(self, status=True):

        table_dictionary = self.parent.table_dictionary
        for _index in table_dictionary:
            _item = table_dictionary[_index]
            _item['active'] = status
            table_dictionary[_index] = _item

        self.parent.table_dictionary = table_dictionary

    def unselect_full_table(self):
        self.full_table_selection_tool(status = False)

    def select_full_table(self):
        self.full_table_selection_tool(status = True)

    def get_average_parameters_activated(self):
        table_dictionary = self.parent.table_dictionary

        d_spacing = []
        alpha = []
        sigma = []
        a1 = []
        a2 = []
        a5 = []
        a6 = []

        for _index in table_dictionary.keys():
            _entry = table_dictionary[_index]

            if _entry['active']:
                _d_spacing = _entry['d_spacing']['val']
                _alpha = _entry['alpha']['val']
                _sigma = _entry['sigma']['val']
                _a1 = _entry['a1']['val']
                _a2 = _entry['a2']['val']
                _a5 = _entry['a5']['val']
                _a6 = _entry['a6']['val']

                d_spacing.append(_d_spacing)
                alpha.append(_alpha)
                sigma.append(_sigma)
                a1.append(_a1)
                a2.append(_a2)
                a5.append(_a5)
                a6.append(_a6)

        mean_d_spacing = self.get_mean_value(d_spacing)
        mean_alpha = self.get_mean_value(alpha)
        mean_sigma = self.get_mean_value(sigma)
        mean_a1 = self.get_mean_value(a1)
        mean_a2 = self.get_mean_value(a2)
        mean_a5 = self.get_mean_value(a5)
        mean_a6 = self.get_mean_value(a6)

        return {'d_spacing': mean_d_spacing,
                'alpha': mean_alpha,
                'sigma': mean_sigma,
                'a1': mean_a1,
                'a2': mean_a2,
                'a5': mean_a5,
                'a6': mean_a6}

    def get_mean_value(self, array=[]):
        if array == []:
            return np.NaN
        else:
            return np.nanmean(array)

    def import_table(self):
        default_file_name = str(self.parent.ui.normalized_folder.text()) + '_fitting_table.csv'
        table_file = str(QtGui.QFileDialog.getOpenFileName(self.parent,
                                                              'Define Location and File Name Where to Export the Table!',
                                                              os.path.join(self.parent.normalized_folder, default_file_name)))

        if table_file:
            pandas_data_frame = pd.read_csv(table_file)
            o_table = TableDictionaryHandler(parent=self.parent)

            numpy_table = pandas_data_frame.values
            # loop over each row in the pandas data frame
            for _index, _row_values in enumerate(numpy_table):
                o_table.populate_table_dictionary_entry(index=_index,
                                                        array=_row_values)

            o_fitting = ibeatles.fitting.fitting_handler.FittingHandler(parent=self.parent)
            o_fitting.fill_table()

    def export_table(self):
        default_file_name = str(self.parent.ui.normalized_folder.text()) + '_fitting_table.csv'
        table_file = str(QtGui.QFileDialog.getSaveFileName(self.parent,
                                                               'Select or Define Name of File!',
                                                               default_file_name,
                                                               "CSV (*.csv)"))

        if table_file:
            table_dictionary = self.parent.table_dictionary
            o_table_formated = FormatTableForExport(table=table_dictionary)
            pandas_data_frame = o_table_formated.pandas_data_frame
            header = self.header
            pandas_data_frame.to_csv(table_file, header=header)

class FormatTableForExport(object):

    pandas_data_frame = []

    def __init__(self, table={}):

        pandas_table = []

        for _key in table:

            _entry = table[_key]

            x0 = _entry['bin_coordinates']['x0']
            y0 = _entry['bin_coordinates']['y0']
            x1 = _entry['bin_coordinates']['x1']
            y1 = _entry['bin_coordinates']['y1']

            row_index = _entry['row_index']
            column_index = _entry['column_index']

            lock = _entry['lock']
            active = _entry['active']

            fitting_confidence = _entry['fitting_confidence']

            [d_spacing_val,
             d_spacing_err,
             d_spacing_fixed] = self.get_val_err_fixed(_entry['d_spacing'])

            [sigma_val,
             sigma_err,
             sigma_fixed] = self.get_val_err_fixed(_entry['sigma'])

            [intensity_val,
             intensity_err,
             intensity_fixed] = self.get_val_err_fixed(_entry['intensity'])

            [alpha_val,
             alpha_err,
             alpha_fixed] = self.get_val_err_fixed(_entry['alpha'])

            [a1_val,
             a1_err,
             a1_fixed] = self.get_val_err_fixed(_entry['a1'])

            [a2_val,
             a2_err,
             a2_fixed] = self.get_val_err_fixed(_entry['a2'])

            [a5_val,
            a5_err,
            a5_fixed] = self.get_val_err_fixed(_entry['a5'])

            [a6_val,
             a6_err,
             a6_fixed] = self.get_val_err_fixed(_entry['a6'])

            _row = [x0, x1, y0, y1,
                    row_index, column_index,
                    lock, active,
                    fitting_confidence,
                    d_spacing_val, d_spacing_err, d_spacing_fixed,
                    sigma_val, sigma_err, sigma_fixed,
                    intensity_val, intensity_err, intensity_fixed,
                    alpha_val, alpha_err, alpha_fixed,
                    a1_val, a1_err, a1_fixed,
                    a2_val, a2_err, a2_fixed,
                    a5_val, a5_err, a5_fixed,
                    a6_val, a6_err, a6_fixed,
                    ]

            pandas_table.append(_row)

        pandas_data_frame = pd.DataFrame.from_dict(pandas_table)
        self.pandas_data_frame = pandas_data_frame

    def get_val_err_fixed(self, item):
        return [item['val'], item['err'], item['fixed']]

max_stars_repo_name: indudhiman/bragg-edge

max_issues_repo_name: indudhiman/bragg-edge

max_forks_repo_name: indudhiman/bragg-edge


text:
"""Containers for storing data products and fit results."""

import numpy as np

from draco.core.containers import *

class StackSet1D(FrequencyStackByPol):
    """Container for all data required to perform a model fit to a 1D stack."""

    _axes = ("mock",)

    _dataset_spec = {
        "mock": {
            "axes": ["mock", "pol", "freq"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "transfer_function": {
            "axes": ["pol", "freq"],
            "dtype": np.float64,
            "initialise": False,
            "distributed": False,
        },
        "template": {
            "axes": ["pol", "freq"],
            "dtype": np.float64,
            "initialise": False,
            "distributed": False,
        },
    }

class StackSet3D(Stack3D):
    """Container for all data required to perform a model fit to a 3D stack."""

    _axes = ("mock",)

    _dataset_spec = {
        "mock": {
            "axes": ["mock", "pol", "delta_ra", "delta_dec", "freq"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "transfer_function": {
            "axes": ["pol", "delta_ra", "delta_dec", "freq"],
            "dtype": np.float64,
            "initialise": False,
            "distributed": False,
        },
        "template": {
            "axes": ["pol", "delta_ra", "delta_dec", "freq"],
            "dtype": np.float64,
            "initialise": False,
            "distributed": False,
        },
    }

class MCMCFit(ContainerBase):
    """Base container for the results of a model fit."""

    _axes = ("step", "walker", "param", "percentile")

    _dataset_spec = {
        "chain": {
            "axes": ["step", "walker", "param"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "chisq": {
            "axes": ["step", "walker"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "fixed": {
            "axes": ["param"],
            "dtype": np.bool,
            "initialise": True,
            "distributed": False,
        },
        "autocorr_time": {
            "axes": ["param"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "acceptance_fraction": {
            "axes": ["walker"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "percentile": {
            "axes": ["param", "percentile"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "median": {
            "axes": ["param"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "span_lower": {
            "axes": ["param"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "span_upper": {
            "axes": ["param"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
    }

    def samples(self, flat=True):
        """Return independent samples from the joint posterior distribution.

        Parameters
        ----------
        flat : bool
            If True, then the samples will be flattened over the
            step and walker axis.  Default is True.

        Returns
        -------
        samples : np.ndarray[nsample, nparam] or np.ndarray[nstep, nwalker, nparam]
            Samples from the joint posterior distribution after discarding
            burn-in and thinning based on the maximum autocorrelation length.
            This will be 2D if flat is True and 3D if flat is False.
        """

        samples = self.datasets["chain"][self.slice_chain, :, :]

        if flat:
            samples = samples.reshape(-1, samples.shape[-1])

        return samples

    @property
    def fit_index(self):
        return np.flatnonzero(~self.datasets["fixed"][:])

    @property
    def fixed_index(self):
        return np.flatnonzero(self.datasets["fixed"][:])

    @property
    def slice_chain(self):
        """Create a slice along the step axis that discards burn-in and thins.

        Discards 10 times the autocorrelation length and thins by the
        autocorrelation length divided by 2.
        """

        tau = int(np.max(self.datasets["autocorr_time"][:][self.fit_index]))
        thin = tau // 2
        discard = tau * 10

        return slice(discard, self.nstep, thin)

    @property
    def min_chisq(self):
        """Return the minimum chi-squared."""
        return np.min(self.datasets["chisq"][:])

    @property
    def nstep(self):
        """Return the total number of steps taken."""
        return self.index_map["step"].size

class MCMCFit1D(MCMCFit, StackSet1D):
    """Container for the results of a model fit to 1D stack and all associated data."""

    _axes = ("x",)

    _dataset_spec = {
        "cov": {
            "axes": ["pol", "pol", "freq", "freq"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "error": {
            "axes": ["pol", "freq"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "freq_flag": {
            "axes": ["freq"],
            "dtype": np.bool,
            "initialise": True,
            "distributed": False,
        },
        "flag": {
            "axes": ["x"],
            "dtype": np.bool,
            "initialise": True,
            "distributed": False,
        },
        "precision": {
            "axes": ["x", "x"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "model_min_chisq": {
            "axes": ["pol", "freq"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "model_percentile": {
            "axes": ["pol", "freq", "percentile"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
    }

    @property
    def ndof(self):
        """Return the number of degrees of freedom."""
        return np.sum(self.datasets["flag"][:]) - self.index_map["param"].size

class MCMCFit3D(MCMCFit, StackSet3D):
    """Container for the results of a model fit to 3D stack and all associated data."""

    _axes = ("pixel",)

    _dataset_spec = {
        "error": {
            "axes": ["pol", "delta_ra", "delta_dec", "freq"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "errori": {
            "axes": ["pol", "delta_ra", "delta_dec", "freq"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "cov": {
            "axes": ["pol", "pixel", "pixel"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
        "inv_cov": {
            "axes": ["pol", "pixel", "pixel"],
            "dtype": np.float64,
            "initialise": True,
            "distributed": False,
        },
    }

max_stars_repo_name: ssiegelx/fitstack

max_issues_repo_name: ssiegelx/fitstack

max_forks_repo_name: ssiegelx/fitstack


text:
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F

__all__ = [
    'SharpSepConv',
    'MixSeparableConv',
    'InvertedResidual',
    'InvertedResidualSE',
    'StdConv',
    'FactorizedDownsample',
    'ReductionLayer',
    'FactorizedUpsample',
    'DilConv',
    'SeparableConv',
    'Calibration',
    'ZeroLayer',
    'Hswish',
    'Hsigmoid',
    'SEModule',
    'ShuffleLayer',
    'ResidualBlock',
    'topk_gumbel_softmax',
    'TopKGumbelSoftmax'
]

class ShuffleLayer(nn.Module):

    def __init__(self, groups):
        super(ShuffleLayer, self).__init__()
        self.groups = groups

    def forward(self, x):
        batch_size, num_channels, height, width = x.size()
        channels_per_group = num_channels // self.groups
        # reshape
        x = x.view(batch_size, self.groups, channels_per_group, height, width)
        x = torch.transpose(x, 1, 2).contiguous()
        # flatten
        x = x.view(batch_size, -1, height, width)
        return x

    def __repr__(self):
        return 'ShuffleLayer(groups=%d)' % self.groups

class Hswish(nn.Module):

    def __init__(self, inplace=True):
        super(Hswish, self).__init__()
        self.inplace = inplace

    def forward(self, x):
        return x * F.relu6(x + 3., inplace=self.inplace) / 6.

    def __repr__(self):
        return 'Hswish()'

class Hsigmoid(nn.Module):

    def __init__(self, inplace=True):
        super(Hsigmoid, self).__init__()
        self.inplace = inplace

    def forward(self, x):
        return F.relu6(x + 3., inplace=self.inplace) / 6.

    def __repr__(self):
        return 'Hsigmoid()'

class SEModule(nn.Module):
    REDUCTION = 4

    def __init__(self, channel, reduction=None):
        super(SEModule, self).__init__()

        self.channel = channel
        self.reduction = SEModule.REDUCTION if reduction is None else reduction

        num_mid = make_divisible(self.channel // self.reduction, divisor=MyNetwork.CHANNEL_DIVISIBLE)

        self.fc = nn.Sequential(OrderedDict([
            ('reduce', nn.Conv2d(self.channel, num_mid, 1, 1, 0, bias=True)),
            ('relu', nn.ReLU(inplace=True)),
            ('expand', nn.Conv2d(num_mid, self.channel, 1, 1, 0, bias=True)),
            ('h_sigmoid', Hsigmoid(inplace=True)),
        ]))

    def forward(self, x):
        y = x.mean(3, keepdim=True).mean(2, keepdim=True)
        y = self.fc(y)
        return x * y

    def __repr__(self):
        return 'SE(channel=%d, reduction=%d)' % (self.channel, self.reduction)

class ZeroLayer(nn.Module):

    def __init__(self, stride=None):
        super(ZeroLayer, self).__init__()
        self.stride = stride

    def forward(self, x):
        '''n, c, h, w = x.size()
        h //= self.stride
        w //= self.stride
        device = x.get_device() if x.is_cuda else torch.device('cpu')
        # noinspection PyUnresolvedReferences
        padding = torch.zeros(n, c, h, w, device=device, requires_grad=False)
        return padding'''
        if self.stride == 1:
            return x.mul(0.)
        if len(x.shape)==4:
            return x[:,:,::self.stride,::self.stride].mul(0.)
        elif len(x.shape)==5:
            return x[:,:,::self.stride,::self.stride,::self.stride].mul(0.)

    @staticmethod
    def is_zero_layer():
        return True

# sharpSepConv
class SharpSepConv(nn.Module):
    def __init__(self, C_in, C_out, kernel_size, stride,
            padding=1, dilation=1, affine=True, C_mid_mult=1, C_mid=None):
        super(SharpSepConv, self).__init__()
        cmid = int(C_out * C_mid_mult)
        cmid = C_mid if C_mid else cmid
        self.op = nn.Sequential(
            nn.ReLU(inplace=False),
            nn.Conv2d(C_in, C_in, kernel_size, stride, padding, dilation, groups=C_in, bias=0),
            nn.Conv2d(C_in, cmid, kernel_size=1, padding=0, bias=0),
            nn.BatchNorm2d(cmid, affine=affine),

            nn.ReLU(inplace=False),
            nn.Conv2d(cmid, cmid, kernel_size, stride=1, padding=(kernel_size-1)//2, dilation=1, groups=cmid, bias=0),
            nn.Conv2d(cmid, C_out, kernel_size=1, padding=0, bias=0),
            nn.BatchNorm2d(C_out, affine=affine))
    def forward(self, x):
        return self.op(x)

# Mix Separable Conv
def split_layer(total_channels, num_groups):
    split = [int(np.ceil(total_channels / num_groups)) for _ in range(num_groups)]
    split[num_groups - 1] += total_channels - sum(split)
    return split

class MDConv(nn.Module):
    def __init__(self, C_in, n_chunks, stride=1, bias=False):
        super(MDConv, self).__init__()
        self.n_chunks = n_chunks
        self.split_out_channels = split_layer(C_in, n_chunks)

        self.layers = nn.ModuleList()
        for idx in range(self.n_chunks):
            kernel_size = 2 * idx + 3
            padding = (kernel_size - 1) // 2
            c_in_group = self.split_out_channels[idx] # the input channels of each group
            depthwise = nn.Conv2d(c_in_group, c_in_group, kernel_size=kernel_size, padding=padding,
                                  stride=stride, groups=c_in_group, bias=bias)
            self.layers.append(depthwise)

    def forward(self, x):
        split = torch.split(x, self.split_out_channels, dim=1)
        out = torch.cat([layer(s) for layer, s in zip(self.layers, split)], dim=1)
        return out

class MixSeparableConv(nn.Module):
    def __init__(self, C_in, C_out, n_chunks, stride=1, bias=False):
        super(MixSeparableConv, self).__init__()
        self.depthwise = MDConv(C_in, n_chunks, stride=1, bias=bias)
        self.pointwise = nn.Conv2d(C_in, C_out, 1, stride=stride, bias=bias)
        self.bn = nn.BatchNorm2d(C_out, affine=True)

    def forward(self, x):
        out = self.depthwise(x)
        out = self.pointwise(out)
        out = self.bn(out)
        out = nn.ReLU()(out)
        return out

# InvertedResidual
class InvertedResidual(nn.Module):
    def __init__(self, C_in, C_out, expand_ratio=3, stride=1, kernel=3):
        super(InvertedResidual, self).__init__()
        self.stride = stride
        assert stride in [1, 2]

        self.use_res_connect = self.stride == 1 and C_in == C_out

        self.conv = nn.Sequential(
            # pw
            nn.Conv2d(C_in, C_in * expand_ratio, 1, 1, 0, bias=False),
            nn.BatchNorm2d(C_in * expand_ratio),
            nn.ReLU6(inplace=True),
            # dw
            nn.Conv2d(C_in * expand_ratio, C_in * expand_ratio, kernel, stride, kernel // 2, groups=C_in * expand_ratio, bias=False),
            nn.BatchNorm2d(C_in * expand_ratio),
            nn.ReLU6(inplace=True),
            # pw-linear
            nn.Conv2d(C_in * expand_ratio, C_out, 1, 1, 0, bias=False),
            nn.BatchNorm2d(C_out),
        )

    def forward(self, x):
        if self.use_res_connect:
            return x + self.conv(x)
        else:
            return self.conv(x)

# InvertedResidualSE
class SELayer(nn.Module):
    def __init__(self, channel, reduction=4):
        super(SELayer, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.fc = nn.Sequential(
            nn.Linear(channel, channel // reduction, bias=False),
            nn.ReLU(inplace=True),
            nn.Linear(channel // reduction, channel, bias=False),
            nn.Sigmoid()
        )

    def forward(self, x):
        b, c, _, _ = x.size()
        y = self.avg_pool(x).view(b, c)
        y = self.fc(y).view(b, c, 1, 1)
        return x * y.expand_as(x)

class InvertedResidualSE(nn.Module):
    '''The InvertedResidual with SELayer
    '''
    def __init__(self, C_in, C_out, se_ratio=4, expand_ratio=3, stride=1, kernel=3):
        super(InvertedResidualSE, self).__init__()
        self.stride = stride
        assert stride in [1, 2]

        self.use_res_connect = self.stride == 1 and C_in == C_out

        self.conv = nn.Sequential(
            # pw
            nn.Conv2d(C_in, C_in * expand_ratio, 1, 1, 0, bias=False),
            nn.BatchNorm2d(C_in * expand_ratio),
            nn.ReLU6(inplace=True),
            # dw
            nn.Conv2d(C_in * expand_ratio, C_in * expand_ratio, kernel, stride, kernel // 2, groups=C_in * expand_ratio, bias=False),
            nn.BatchNorm2d(C_in * expand_ratio),
            nn.ReLU6(inplace=True),
            # se-layer
            SELayer(C_in * expand_ratio, reduction=se_ratio),
            # pw-linear
            nn.Conv2d(C_in * expand_ratio, C_out, 1, 1, 0, bias=False),
            nn.BatchNorm2d(C_out),
        )

    def forward(self, x):
        if self.use_res_connect:
            return x + self.conv(x)
        else:
            return self.conv(x)

# stdconv
class StdConv(nn.Module):
    def __init__(self, C_in, C_out):
        super(StdConv, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(C_in, C_out, 1, stride=1, padding=0, bias=False),
            nn.BatchNorm2d(C_out, affine=False),
            nn.ReLU()
        )

    def forward(self, x):
        return self.conv(x)

class FactorizedDownsample(nn.Module):
    '''used for downsample layer
    '''
    def __init__(self, C_in, C_out, affine=False):
        super().__init__()
        self.conv1 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False)
        self.conv2 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False)
        self.bn = nn.BatchNorm2d(C_out, affine=affine)

    def forward(self, x):
        out = torch.cat([self.conv1(x), self.conv2(x[:, :, 1:, 1:])], dim=1)
        out = self.bn(out)
        out = nn.ReLU(inplace=True)(out)
        return out

class ReductionLayer(nn.Module):
    def __init__(self, in_channels_pp, in_channels_p, out_channels):
        super().__init__()
        self.reduce0 = FactorizedDownsample(in_channels_pp, out_channels, affine=False)
        self.reduce1 = FactorizedDownsample(in_channels_p, out_channels, affine=False)

    def forward(self, pprev, prev):
        return self.reduce0(pprev), self.reduce1(prev)

class FactorizedUpsample(nn.Module):
    '''used for Upsample layer
    h_out = (h_in - 1)*stride - 2*padding + dilation*(k-1) + out_padding + 1
    '''
    def __init__(self, C_in, C_out, affine=False):
        super().__init__()
        self.conv1 = nn.ConvTranspose2d(C_in, C_out//2, kernel_size=3, stride=2, padding=1, output_padding=1, bias=False)
        self.conv2 = nn.ConvTranspose2d(C_in, C_out//2, kernel_size=3, stride=2, padding=1, output_padding=1, bias=False, dilation=2)
        self.bn = nn.BatchNorm2d(C_out, affine=affine)

    def forward(self, x):
        out = torch.cat([self.conv1(x), self.conv2(x[:, :, 1:, 1:])], dim=1)
        out = self.bn(out)
        out = nn.LeakyReLU(inplace=True)(out)
        return out

class DilConv(nn.Module):
    """
    (Dilated) depthwise separable conv.
    ReLU - (Dilated) depthwise separable - Pointwise - BN.
    If dilation == 2, 3x3 conv => 5x5 receptive field, 5x5 conv => 7x7 receptive field.
    o = [i + 2*p - k - (k-1)*(d-1)]/s + 1
    To keep the same size as the input:
        for 3x3 kernel, padding=dilation=2
        for 5x5 kernel, padding=2*dilation
    """
    def __init__(self, C_in, C_out, kernel_size=3, stride=1, padding=2, dilation=2, affine=True):
        super().__init__()
        assert kernel_size in [3, 5], "kernel_size must be 3 or 5"
        if kernel_size == 3:
            padding = dilation
        elif kernel_size == 5:
            padding = 2*dilation
        self.dil_conv = nn.Conv2d(C_in, C_out, kernel_size, stride, padding, dilation)
        self.bn = nn.BatchNorm2d(C_out, affine=affine)

    def forward(self, x):
        out = self.dil_conv(x)
        out = self.bn(out)
        out = nn.ReLU()(out)
        return out

class SeparableConv(nn.Module):
    def __init__(self, C_in, C_out, kernel_size, padding, stride=1):
        super(SeparableConv, self).__init__()
        self.depthwise = nn.Conv2d(C_in, C_in, kernel_size=kernel_size, padding=padding, stride=stride,
                                   groups=C_in, bias=False)
        self.pointwise = nn.Conv2d(C_in, C_out, kernel_size=1, bias=False)
        self.bn = nn.BatchNorm2d(C_out, affine=True)

    def forward(self, x):
        out = self.depthwise(x)
        out = self.pointwise(out)
        out = self.bn(out)
        out = nn.ReLU()(out)
        return out

class Calibration(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.process = None
        if in_channels != out_channels:
            self.process = StdConv(in_channels, out_channels)

    def forward(self, x):
        if self.process is None:
            return x
        return self.process(x)

class ResidualBlock(nn.Module):

    def __init__(self, conv, shortcut):
        super(ResidualBlock, self).__init__()

        self.conv = conv
        self.shortcut = shortcut

    def forward(self, x):
        if self.conv is None or isinstance(self.conv, ZeroLayer):
            res = x
        elif self.shortcut is None or isinstance(self.shortcut, ZeroLayer):
            res = self.conv(x)
        else:
            res = self.conv(x) + self.shortcut(x)
        return res

class TopKGumbelSoftmax(nn.Module):
    def __init__(
        self,
        tau: float=1,
        hard: bool=False,
        topk: int=3,
        eps: float=1e-10,
        dim: int=-1
    ):
        super(TopKGumbelSoftmax, self).__init__()
        self.tau = tau
        self.hard = hard
        self.topk = topk
        self.eps = eps
        self.dim = dim

    def forward(self, x):
        return topk_gumbel_softmax(x, self.tau, self.hard, self.topk, self.eps, self.dim)

def topk_gumbel_softmax(logits: torch.Tensor, tau: float = 1, hard: bool = False,
        topk: int = 2, eps: float = 1e-10, dim: int = -1):
    # gumbels = (
    #     -torch.empty_like(
    #         logits, memory_format=torch.legacy_contiguous_format
    #     ).exponential_().log()
    # )
    gumbels = torch.empty_like(logits, memory_format=torch.legacy_contiguous_format).uniform_()
    gumbels = -torch.log(eps - torch.log(gumbels+eps))
    gumbels = (logits + gumbels) / tau
    y_soft = gumbels.softmax(dim)
    if topk is None:
        topk = (gumbels>gumbels.mean(dim, keepdim=True)).int().sum(dim)
    if hard:
        # Straight through.
        if len(torch.tensor(topk).view(-1)) == 1:
            indices = y_soft.topk(topk)[1]
            y_hard = torch.zeros_like(logits, memory_format=torch.legacy_contiguous_format).scatter_(dim, indices, 1.0)
        else:
            y_hard = torch.zeros_like(logits, memory_format=torch.legacy_contiguous_format)
            for i, k in enumerate(topk):
                indices = y_soft[i].topk(k)[1]
                y_hard[i].scatter_(dim, indices, 1.0)
        ret = y_hard - y_soft.detach() + y_soft
    else:
        # Reparametrization trick.
        ret = y_soft
    return ret

max_stars_repo_name: marsggbo/hyperbox

max_issues_repo_name: marsggbo/hyperbox

max_forks_repo_name: marsggbo/hyperbox