text:
# -*- coding: utf-8 -*-
"""Fully Convolutional One-Stage Object Detection."""
from __future__ import absolute_import

import os
import warnings

import numpy as np
import mxnet as mx
from mxnet import autograd
from mxnet.gluon import nn, HybridBlock
from ...nn.bbox import BBoxClipToImage
# from IPython import embed

from .fcos_target import FCOSBoxConverter
from ...nn.feature import RetinaFeatureExpander
from ...nn.protomask import Protonet

__all__ = ['MaskFCOS', 'get_maskfcos',
           'maskfcos_resnet50_v1_740_coco',
           'maskfcos_resnet50_v1b_740_coco',
           'maskfcos_resnet101_v1d_740_coco',
           'maskfcos_resnet50_v1_1024_coco']

class GroupNorm(nn.HybridBlock):
    """
    If the batch size is small, it's better to use GroupNorm instead of BatchNorm.
    GroupNorm achieves good results even at small batch sizes.
    Reference:
      https://arxiv.org/pdf/1803.08494.pdf
    """
    def __init__(self, num_channels, num_groups=32, eps=1e-5,
                 multi_precision=False, prefix=None, **kwargs):
        super(GroupNorm, self).__init__(**kwargs)
        with self.name_scope():
            self.weight = self.params.get('{}_weight'.format(prefix), grad_req='write',
                                          shape=(1, num_channels, 1, 1))
            self.bias = self.params.get('{}_bias'.format(prefix), grad_req='write',
                                        shape=(1, num_channels, 1, 1))
        self.C = num_channels
        self.G = num_groups
        self.eps = eps
        self.multi_precision = multi_precision
        assert self.C % self.G == 0

    def hybrid_forward(self, F, x, weight, bias):
        # (N,C,H,W) -> (N,G,H*W*C//G)
        x_new = F.reshape(x, (0, self.G, -1))
        if self.multi_precision:
            # (N,G,H*W*C//G) -> (N,G,1)
            mean = F.mean(F.cast(x_new, "float32"), axis=-1, keepdims=True)
            mean = F.cast(mean, "float16")
        else:
            mean = F.mean(x_new, axis=-1, keepdims=True)
        # (N,G,H*W*C//G)
        centered_x_new = F.broadcast_minus(x_new, mean)
        if self.multi_precision:
            # (N,G,H*W*C//G) -> (N,G,1)
            var = F.mean(F.cast(F.square(centered_x_new), "float32"), axis=-1, keepdims=True)
            var = F.cast(var, "float16")
        else:
            var = F.mean(F.square(centered_x_new), axis=-1, keepdims=True)
        # (N,G,H*W*C//G) -> (N,C,H,W)
        x_new = F.broadcast_div(centered_x_new, F.sqrt(var + self.eps)).reshape_like(x)
        x_new = F.broadcast_add(F.broadcast_mul(x_new, weight), bias)
        return x_new

class ConvPredictor(nn.HybridBlock):
    def __init__(self, num_channels, share_params=None, bias_init=None, **kwargs):
        super(ConvPredictor, self).__init__(**kwargs)
        with self.name_scope():
            if share_params is not None:
                self.conv = nn.Conv2D(num_channels, 3, 1, 1, params=share_params,
                        bias_initializer=bias_init)
            else:
                self.conv = nn.Conv2D(num_channels, 3, 1, 1,
                        weight_initializer=mx.init.Normal(sigma=0.01),
                        bias_initializer=bias_init)

    def get_params(self):
        return self.conv.params

    def hybrid_forward(self, F, x):
        x = self.conv(x)
        return x

class RetinaHead(nn.HybridBlock):
    def __init__(self, share_params=None, prefix=None, **kwargs):
        super(RetinaHead, self).__init__(**kwargs)
        with self.name_scope():
            self.conv = nn.HybridSequential()
            for i in range(4):
                if share_params is not None:
                    self.conv.add(nn.Conv2D(256, 3, 1, 1, activation='relu',
                        params=share_params[i]))
                else:
                    self.conv.add(nn.Conv2D(256, 3, 1, 1, activation='relu',
                        weight_initializer=mx.init.Normal(sigma=0.01),
                        bias_initializer='zeros'))
                self.conv.add(GroupNorm(num_channels=256, prefix=prefix))

    def set_params(self, newconv):
        for b, nb in zip(self.conv, newconv):
            b.weight.set_data(nb.weight.data())
            b.bias.set_data(nb.bias.data())

    def get_params(self):
        param_list = []
        for opr in self.conv:
            param_list.append(opr.params)
        return param_list

    def hybrid_forward(self, F, x):
        x = self.conv(x)
        return x

@mx.init.register
class ClsBiasInit(mx.init.Initializer):
    def __init__(self, num_class, cls_method="sigmoid", pi=0.01, **kwargs):
        super(ClsBiasInit, self).__init__(**kwargs)
        self._num_class = num_class
        self._cls_method = cls_method
        self._pi = pi

    def _init_weight(self, name, data):
        if self._cls_method == "sigmoid":
            arr = -1 * np.ones((data.size, ))
            arr = arr *  np.log((1 - self._pi) / self._pi)
            data[:] = arr
        elif self._cls_method == "softmax":
            pass

def cor_target(short, scale):
    h, w, = short, short
    fh = int(np.ceil(np.ceil(np.ceil(h / 2) / 2) / 2))
    fw = int(np.ceil(np.ceil(np.ceil(w / 2) / 2) / 2))
    #
    fm_list = []
    for _ in range(len(scale)):
        fm_list.append((fh, fw))
        fh = int(np.ceil(fh / 2))
        fw = int(np.ceil(fw / 2))
    fm_list = fm_list[::-1]
    #
    cor_targets = []
    for i, stride in enumerate(scale):
        fh, fw = fm_list[i]
        cx = mx.nd.arange(0, fw).reshape((1, -1))
        cy = mx.nd.arange(0, fh).reshape((-1, 1))
        sx = mx.nd.tile(cx, reps=(fh, 1))
        sy = mx.nd.tile(cy, reps=(1, fw))
        syx = mx.nd.stack(sy.reshape(-1), sx.reshape(-1)).transpose()
        cor_targets.append(mx.nd.flip(syx, axis=1) * stride)
    cor_targets = mx.nd.concat(*cor_targets, dim=0)
    return cor_targets

class MaskFCOS(HybridBlock):
    def __init__(self, network, features, classes, steps=None, short=600,
                 valid_range=[(512, np.inf), (256, 512), (128, 256), (64, 128), (0, 64)],
                 num_prototypes=32,
                 use_p6_p7=True, pretrained_base=False,
                 nms_thresh=0.45, nms_topk=400, post_nms=100, share_params = False, **kwargs):
        super(MaskFCOS, self).__init__(**kwargs)
        num_layers = len(features) + int(use_p6_p7)*2
        self._num_layers = num_layers
        self.classes = classes
        self.nms_thresh = nms_thresh
        self.nms_topk = nms_topk
        self.post_nms = post_nms
        self.share_params = share_params
        self._scale = steps[::-1]
        self.short = short
        self.base_stride = steps[-1]
        self.valid_range = valid_range
        self.k = num_prototypes

        # input size are solid
        self.cor_targets = self.params.get_constant(name='cor_', value=cor_target(self.short, self._scale))

        with self.name_scope():
            bias_init = ClsBiasInit(len(self.classes))
            self.box_converter = FCOSBoxConverter()
            self.cliper = BBoxClipToImage()
            self.features = RetinaFeatureExpander(network=network, pretrained=pretrained_base, outputs=features)
            self.protonet = Protonet([256, 256, 256, 256, self.k])
            self.classes_head = nn.HybridSequential()
            self.box_head = nn.HybridSequential()
            self.class_predictors = nn.HybridSequential()
            self.box_predictors = nn.HybridSequential()
            # self.center_predictors = nn.HybridSequential()
            self.maskcoe_predictors = nn.HybridSequential()
            share_cls_params, share_box_params = None, None
            share_cls_pred_params, share_ctr_pred_params, \
            share_box_pred_params, share_mask_pred_params = None, None, None, None
            for i in range(self._num_layers):
                # classes
                cls_head = RetinaHead(share_params=share_cls_params, prefix='cls_{}'.format(i))
                self.classes_head.add(cls_head)
                share_cls_params = cls_head.get_params()

                # bbox
                box_head = RetinaHead(share_params=share_box_params, prefix='box_{}'.format(i))
                self.box_head.add(box_head)
                share_box_params = box_head.get_params()

                # classes preds
                cls_pred = ConvPredictor(num_channels=len(self.classes),
                                         share_params=share_cls_pred_params, bias_init=bias_init)
                self.class_predictors.add(cls_pred)
                share_cls_pred_params = cls_pred.get_params()

                # center preds
                # center_pred = ConvPredictor(num_channels=1,
                #                             share_params=share_ctr_pred_params, bias_init='zeros')
                # self.center_predictors.add(center_pred)
                # share_ctr_pred_params = center_pred.get_params()

                # mask coefficient preds
                maskcoe_pred = ConvPredictor(num_channels=self.k,
                                             share_params=share_mask_pred_params, bias_init='zeros')
                self.maskcoe_predictors.add(maskcoe_pred)
                share_mask_pred_params = maskcoe_pred.get_params()

                # bbox_pred
                bbox_pred = ConvPredictor(num_channels=4,
                                          share_params=share_box_pred_params, bias_init='zeros')
                self.box_predictors.add(bbox_pred)
                share_box_pred_params = bbox_pred.get_params()

                if not self.share_params:
                    share_cls_params, share_box_params = None, None
                    share_cls_pred_params, share_ctr_pred_params, \
                    share_box_pred_params, share_mask_pred_params = None, None, None, None
            # trainable scales
            # self.s1 = self.params.get('scale_p1', shape=(1,), differentiable=True,
            #         allow_deferred_init=True, init='ones')
            # self.s2 = self.params.get('scale_p2', shape=(1,), differentiable=True,
            #         allow_deferred_init=True, init='ones')
            # self.s3 = self.params.get('scale_p3', shape=(1,), differentiable=True,
            #         allow_deferred_init=True, init='ones')
            # self.s4 = self.params.get('scale_p4', shape=(1,), differentiable=True,
            #         allow_deferred_init=True, init='ones')
            # self.s5 = self.params.get('scale_p5', shape=(1,), differentiable=True,
            #         allow_deferred_init=True, init='ones')

    @property
    def num_classes(self):
        """Return number of foreground classes.

        Returns
        -------
        int
            Number of foreground classes

        """
        return len(self.classes)

    # pylint: disable=arguments-differ
    def hybrid_forward(self, F, x, cor_targets):
        """Hybrid forward"""
        features = self.features(x)
        masks = self.protonet(features[-1])
        masks = F.relu(masks)
        cls_head_feat = [cp(feat) for feat, cp in zip(features, self.classes_head)]
        box_head_feat = [cp(feat) for feat, cp in zip(features, self.box_head)]
        cls_preds = [F.transpose(F.reshape(cp(feat), (0, 0, -1)), (0, 2, 1))
                     for feat, cp in zip(cls_head_feat, self.class_predictors)]
        # center_preds = [F.transpose(F.reshape(bp(feat), (0, 0, -1)), (0, 2, 1))
        #              for feat, bp in zip(cls_head_feat, self.center_predictors)]

        box_preds = [F.transpose(F.exp(F.reshape(bp(feat), (0, 0, -1))), (0, 2, 1)) * sc
                     for feat, bp, sc in zip(box_head_feat, self.box_predictors, self._scale)]

        maskeoc_preds = [F.transpose(F.reshape(bp(feat), (0, 0, -1)), (0, 2, 1))
                         for feat,  bp in zip(box_head_feat, self.maskcoe_predictors)]

        cls_preds = F.concat(*cls_preds, dim=1)
        # center_preds = F.concat(*center_preds, dim=1)
        box_preds = F.concat(*box_preds, dim=1)
        maskeoc_preds = F.concat(*maskeoc_preds, dim=1)
        maskeoc_preds = F.tanh(maskeoc_preds)
        center_preds = F.sum(maskeoc_preds, axis=-1, keepdims=True)
        # with autograd.pause():
        #     h, w, = self.short, self.short
        #     fh = int(np.ceil(np.ceil(np.ceil(h / 2) / 2) / 2))
        #     fw = int(np.ceil(np.ceil(np.ceil(w / 2) / 2) / 2))
        #     #
        #     fm_list = []
        #     for _ in range(len(self._scale)):
        #         fm_list.append((fh, fw))
        #         fh = int(np.ceil(fh / 2))
        #         fw = int(np.ceil(fw / 2))
        #     fm_list = fm_list[::-1]
        #     #
        #     cor_targets = []
        #     for i, stride in enumerate(self._scale):
        #         fh, fw = fm_list[i]
        #         cx = F.arange(0, fw).reshape((1, -1))
        #         cy = F.arange(0, fh).reshape((-1, 1))
        #         sx = F.tile(cx, reps=(fh, 1))
        #         sy = F.tile(cy, reps=(1, fw))
        #         syx = F.stack(sy.reshape(-1), sx.reshape(-1)).transpose()
        #         cor_targets.append(F.flip(syx, axis=1) * stride)
        #     cor_targets = F.concat(*cor_targets, dim=0)
        box_preds = self.box_converter(box_preds, cor_targets)

        if autograd.is_training():
            return [cls_preds, center_preds, box_preds, masks, maskeoc_preds]

        box_preds = self.cliper(box_preds, x) # box_preds: [B, N, 4]
        cls_prob = F.sigmoid(cls_preds)
        center_prob = F.sigmoid(center_preds)
        cls_prob = F.broadcast_mul(cls_prob, center_prob)
        cls_id = cls_prob.argmax(axis=-1)
        probs = F.pick(cls_prob, cls_id)
        result = F.concat(cls_id.expand_dims(axis=-1), probs.expand_dims(axis=-1), box_preds, maskeoc_preds, dim=-1)
        if self.nms_thresh > 0 and self.nms_thresh < 1:
            result = F.contrib.box_nms(result, overlap_thresh=self.nms_thresh,
                                         topk=self.nms_topk, valid_thresh=0.001,
                                         id_index=0, score_index=1, coord_start=2, force_suppress=False,
                                         in_format='corner', out_format='corner')
            if self.post_nms > 0:
                result = result.slice_axis(axis=1, begin=0, end=self.post_nms)
        ids = F.slice_axis(result, axis=2, begin=0, end=1)
        scores = F.slice_axis(result, axis=2, begin=1, end=2)
        bboxes = F.slice_axis(result, axis=2, begin=2, end=6)
        maskeoc = F.slice_axis(result, axis=2, begin=6, end=6 + self.k)
        return ids, scores, bboxes, maskeoc, masks

def get_maskfcos(name, dataset, pretrained=False, ctx=mx.cpu(),
             root=os.path.join('~', '.mxnet', 'models'), **kwargs):
    "return FCOS network"
    base_name = name
    net = MaskFCOS(base_name, **kwargs)
    if pretrained:
        from ..model_store import get_model_file
        full_name = '_'.join(('fcos', name, dataset))
        net.load_parameters(get_model_file(full_name, tag=pretrained, root=root), ctx=ctx)
    return net

# def fcos_resnet50_v1_coco(pretrained=False, pretrained_base=True, **kwargs):
#     from ..resnet import resnet50_v1
#     from ...data import COCODetection
#     classes = COCODetection.CLASSES
#     pretrained_base = False if pretrained else pretrained_base
#     base_network = resnet50_v1(pretrained=pretrained_base, **kwargs)
#     features = RetinaFeatureExpander(network=base_network,
#                                      pretrained=pretrained_base,
#                                      outputs=['stage2_activation3',
#                                               'stage3_activation5',
#                                               'stage4_activation2'])
#     return get_fcos(name="resnet50_v1", dataset="coco", pretrained=pretrained,
#                     features=features, classes=classes, base_stride=128, short=800,
#                     max_size=1333, norm_layer=None, norm_kwargs=None,
#                     valid_range=[(512, np.inf), (256, 512), (128, 256), (64, 128), (0, 64)],
#                     nms_thresh=0.6, nms_topk=1000, save_topk=100)

def maskfcos_resnet50_v1_740_coco(pretrained=False, pretrained_base=True, **kwargs):
    from ...data import COCOInstance
    classes = COCOInstance.CLASSES
    return get_maskfcos('resnet50_v1',
                   features=['stage2_activation3', 'stage3_activation5', 'stage4_activation2'],
                   classes=classes,
                   steps=[8, 16, 32, 64, 128],
                   short = 740,
                   valid_range=[(512, np.inf), (256, 512), (128, 256), (64, 128), (0, 64)],
                   nms_thresh=0.45, nms_topk=1000, post_nms=100,
                   dataset='coco', pretrained=pretrained,
                   num_prototypes=32,
                   pretrained_base=pretrained_base, share_params = True, **kwargs)

def maskfcos_resnet50_v1b_740_coco(pretrained=False, pretrained_base=True, **kwargs):
    from ...data import COCOInstance
    classes = COCOInstance.CLASSES
    return get_maskfcos('resnet50_v1d',
                   features=['layers2_relu11_fwd', 'layers3_relu17_fwd', 'layers4_relu8_fwd'],
                   classes=classes,
                   steps=[8, 16, 32, 64, 128],
                   short = 740,
                   valid_range=[(512, np.inf), (256, 512), (128, 256), (64, 128), (0, 64)],
                   nms_thresh=0.45, nms_topk=1000, post_nms=100,
                   dataset='coco', pretrained=pretrained,
                   num_prototypes=32,
                   pretrained_base=pretrained_base, share_params = True, **kwargs)

def maskfcos_resnet101_v1d_740_coco(pretrained=False, pretrained_base=True, **kwargs):
    from ...data import COCOInstance
    classes = COCOInstance.CLASSES
    return get_maskfcos('resnet101_v1d',
                   features=['layers2_relu11_fwd', 'layers3_relu68_fwd', 'layers4_relu8_fwd'],
                   classes=classes,
                   steps=[8, 16, 32, 64, 128],
                   short = 740,
                   valid_range=[(512, np.inf), (256, 512), (128, 256), (64, 128), (0, 64)],
                   nms_thresh=0.45, nms_topk=1000, post_nms=100,
                   dataset='coco', pretrained=pretrained,
                   num_prototypes=32,
                   pretrained_base=pretrained_base, share_params = True, **kwargs)

def maskfcos_resnet50_v1_1024_coco(pretrained=False, pretrained_base=True, **kwargs):
    from ...data import COCOInstance
    classes = COCOInstance.CLASSES
    return get_maskfcos('resnet50_v1',
                   features=['stage2_activation3', 'stage3_activation5', 'stage4_activation2'],
                   classes=classes,
                   steps=[8, 16, 32, 64, 128],
                   short = 1024,
                   valid_range=[(512, np.inf), (256, 512), (128, 256), (64, 128), (0, 64)],
                   nms_thresh=0.45, nms_topk=1000, post_nms=100,
                   dataset='coco', pretrained=pretrained,
                   num_prototypes=32,
                   pretrained_base=pretrained_base, share_params = True, **kwargs)


text:
from keras.models import Model
from keras.layers import Input, Embedding, LSTM, Lambda
from keras import backend as K
import numpy
from nse import NSE, MultipleMemoryAccessNSE, InputMemoryMerger

input_val1 = numpy.random.randint(low=0, high=10, size=(1000, 10))
input_val2 = numpy.random.randint(low=0, high=10, size=(1000, 10))
target_val = numpy.random.rand(1000, 50)

input1 = Input(shape=(10,))
input2 = Input(shape=(10,))
embedding_for_nse = Embedding(input_dim=10, output_dim=50)
nse_encoder = NSE(50, return_mode="output_and_memory")

nse_embed_input1 = embedding_for_nse(input1)
nse_embed_input2 = embedding_for_nse(input2)  # (None, 10, 50)

nse_output = nse_encoder(nse_embed_input1)  # (None, 11, 50)

mmanse_input = InputMemoryMerger()([nse_output, nse_embed_input2])

mmanse_encoder = MultipleMemoryAccessNSE(50)
mmanse_output = mmanse_encoder(mmanse_input)

nse_model = Model(input=[input1, input2], output=mmanse_output)
nse_model.compile(optimizer='adam', loss='mse')

nse_model.summary()
nse_model.fit([input_val1, input_val2], target_val)

ext: py

max_stars_repo_name: pdasigi/neural-semantic-encoders

max_issues_repo_name: pdasigi/neural-semantic-encoders

max_forks_repo_name: pdasigi/neural-semantic-encoders

reason: import numpy


text:
from __future__ import print_function
from __future__ import absolute_import
from future.utils import iteritems
from collections import OrderedDict
import docrep
import xarray as xr
import numpy as np

from . import comodo
from .duck_array_ops import _pad_array

docstrings = docrep.DocstringProcessor(doc_key='My doc string')

class Axis:
    """
    An object that represents a group of coodinates that all lie along the same
    physical dimension but at different positions with respect to a grid cell.
    There are four possible positions::

         Center
         |------o-------|------o-------|------o-------|------o-------|
               [0]            [1]            [2]            [3]

         Left
         |------o-------|------o-------|------o-------|------o-------|
        [0]            [1]            [2]            [3]

         Right
         |------o-------|------o-------|------o-------|------o-------|
                       [0]            [1]            [2]            [3]

         Inner
         |------o-------|------o-------|------o-------|------o-------|
                       [0]            [1]            [2]

         Outer
         |------o-------|------o-------|------o-------|------o-------|
        [0]            [1]            [2]            [3]            [4]

    The `center` position is the only one without the `c_grid_axis_shift`
    attribute, which must be present for the other four. However, the actual
    value of `c_grid_axis_shift` is ignored for `inner` and `outer`, which are
    differentiated by their length.
    """

    def __init__(self, ds, axis_name, periodic=True, default_shifts={}):
        """
        Create a new Axis object from an input dataset.

        Parameters
        ----------
        ds : xarray.Dataset
            Contains the relevant grid information. Coordinate attributes
            should conform to Comodo conventions [1]_.
        axis_name : str
            The name of the axis (should match axis attribute)
        periodic : bool, optional
            Whether the domain is periodic along this axis
        default_shifts : dict, optional
            Default mapping from and to grid positions
            (e.g. `{'center': 'left'}`). Will be inferred if not specified.

        REFERENCES
        ----------
        .. [1] Comodo Conventions http://pycomodo.forge.imag.fr/norm.html
        """

        self._ds = ds
        self._name = axis_name
        self._periodic = periodic

        # figure out what the grid dimensions are
        coord_names = comodo.get_axis_coords(ds, axis_name)
        ncoords = len(coord_names)
        if ncoords == 0:
            # didn't find anything for this axis
            raise ValueError("Couldn't find any coordinates for axis %s"
                             % axis_name)

        # now figure out what type of coordinates these are:
        # center, left, right, or outer
        coords = {name: ds[name] for name in coord_names}

        # some tortured logic for dealing with malforme c_grid_axis_shift
        # attributes such as produced by old versions of xmitgcm.
        # This should be a float (either -0.5 or 0.5)
        # this function returns that, or True of the attribute is set to
        # anything at all
        def _maybe_fix_type(attr):
            if attr is not None:
                try:
                    return float(attr)
                except TypeError:
                    return True

        axis_shift = {name:
                          _maybe_fix_type(coord.attrs.get('c_grid_axis_shift'))
                      for name, coord in coords.items()}
        coord_len = {name: len(coord) for name, coord in coords.items()}

        # look for the center coord, which is required
        # this list will potential contain "center" and "face" points
        coords_without_axis_shift = {name: coord_len[name]
                                     for name, shift in axis_shift.items()
                                     if not shift}
        if len(coords_without_axis_shift) == 0:
            raise ValueError("Couldn't find a center coordinate for axis %s"
                             % axis_name)
        elif len(coords_without_axis_shift) > 1:
            raise ValueError("Found two coordinates without "
                             "`c_grid_axis_shift` attribute for axis %s"
                             % axis_name)
        center_coord_name = list(coords_without_axis_shift)[0]
        # knowing the length of the center coord is key to decoding the other
        # coords
        axis_len = coord_len[center_coord_name]

        # now we can start filling in the information about the different coords
        axis_coords = OrderedDict()
        axis_coords['center'] = coords[center_coord_name]

        # now check the other coords
        coord_names.remove(center_coord_name)
        for name in coord_names:
            shift = axis_shift[name]
            clen = coord_len[name]
            if clen == axis_len + 1:
                axis_coords['outer'] = coords[name]
            elif clen == axis_len - 1:
                axis_coords['inner'] = coords[name]
            elif shift == -0.5:
                if clen == axis_len:
                    axis_coords['left'] = coords[name]
                else:
                    raise ValueError("Left coordinate %s has incompatible "
                                     "length %g (axis_len=%g)"
                                     % (name, clen, axis_len))
            elif shift == 0.5:
                if clen == axis_len:
                    axis_coords['right'] = coords[name]
                else:
                    raise ValueError("Right coordinate %s has incompatible "
                                     "length %g (axis_len=%g)"
                                     % (name, clen, axis_len))
            else:
                raise ValueError("Coordinate %s has invalid or missing "
                                 "`c_grid_axis_shift` attribute `%s`" %
                                 (name, repr(shift)))

        self.coords = axis_coords

        # set default position shifts
        fallback_shifts = {'center': ('left', 'right', 'outer', 'inner'),
                           'left': ('center',), 'right': ('center',),
                           'outer': ('center',), 'inner': ('center',)}
        self._default_shifts = {}
        for pos in axis_coords:
            # use user-specified value if present
            if pos in default_shifts:
                self._default_shifts[pos] = default_shifts[pos]
            else:
                for possible_shift in fallback_shifts[pos]:
                    if possible_shift in axis_coords:
                        self._default_shifts[pos] = possible_shift
                        break

    def __repr__(self):
        is_periodic = 'periodic' if self._periodic else 'not periodic'
        summary = ["<xgcm.Axis '%s' %s>" % (self._name, is_periodic)]
        summary.append('Axis Coodinates:')
        summary += self._coord_desc()
        return '\n'.join(summary)

    def _coord_desc(self):
        summary = []
        for name, coord in iteritems(self.coords):
            coord_info = ('  * %-8s %s (%g)' % (name, coord.name, len(coord)))
            if name in self._default_shifts:
                coord_info += ' --> %s' % self._default_shifts[name]
            summary.append(coord_info)
        return summary

    @docstrings.get_sectionsf('neighbor_binary_func')
    @docstrings.dedent
    def _neighbor_binary_func(self, da, f, to, boundary=None, fill_value=0.0,
                              boundary_discontinuity=None):
        """
        Apply a function to neighboring points.

        Parameters
        ----------
        da : xarray.DataArray
            The data on which to operate
        f : function
            With signature f(da_left, da_right, shift)
        to : {'center', 'left', 'right', 'inner', 'outer'}
            The direction in which to shift the array. If not specified,
            default will be used.
        boundary : {None, 'fill', 'extend'}
            A flag indicating how to handle boundaries:

            * None:  Do not apply any boundary conditions. Raise an error if
              boundary conditions are required for the operation.
            * 'fill':  Set values outside the array boundary to fill_value
              (i.e. a Neumann boundary condition.)
            * 'extend': Set values outside the array to the nearest array
              value. (i.e. a limited form of Dirichlet boundary condition.)

        fill_value : float, optional
             The value to use in the boundary condition with `boundary='fill'`.

        Returns
        -------
        da_i : xarray.DataArray
            The differenced data
        """
        position_from, dim = self._get_axis_coord(da)
        if to is None:
            to = self._default_shifts[position_from]

        data_new = self._neighbor_binary_func_raw(da, f, to,
                                                  boundary=boundary,
                                                  fill_value=fill_value,
                                                  boundary_discontinuity=\
                                                  boundary_discontinuity)
        # wrap in a new xarray wrapper
        da_new = self._wrap_and_replace_coords(da, data_new, to)

        return da_new

    docstrings.delete_params('neighbor_binary_func.parameters', 'f')

    def _neighbor_binary_func_raw(self, da, f, to, boundary=None,
                                  fill_value=0.0,
                                  boundary_discontinuity=None):

        # get the two neighboring sets of raw data
        data_left, data_right = \
            self._get_neighbor_data_pairs(da,
                                          to,
                                          boundary=boundary,
                                          fill_value=fill_value,
                                          boundary_discontinuity=\
                                          boundary_discontinuity)

        # apply the function
        data_new = f(data_left, data_right)

        return data_new

    def _get_neighbor_data_pairs(self, da, position_to, boundary=None,
                                 fill_value=0.0, boundary_discontinuity=None):
        """Returns data_left, data_right.
        boundary_discontinuity option enables periodic coordinate interpolation
        (see xgcm.autogenerate)"""

        position_from, dim = self._get_axis_coord(da)

        valid_positions = ['outer', 'inner', 'left', 'right', 'center']
        if position_to not in valid_positions:
            raise ValueError("`%s` is not a valid axis position" % position_to)

        if self._periodic and boundary:
            raise ValueError("`boundary=%s` is not allowed with periodic "
                             "axis %s." % (boundary, self._name))

        if position_from == position_to:
            raise ValueError("Can't get neighbor pairs for the same position.")

        transition = (position_from, position_to)

        if ((transition == ('outer', 'center')) or
            (transition == ('center', 'inner'))):
            # doesn't matter if domain is periodic or not
            left = da.isel(**{dim: slice(None, -1)}).data
            right = da.isel(**{dim: slice(1, None)}).data
        elif ((transition == ('center', 'outer')) or
              (transition == ('inner', 'center'))):
            # pad both sides of the array
            left = _pad_array(da, dim, left=True,
                              boundary=boundary, fill_value=fill_value)
            right = _pad_array(da, dim, boundary=boundary,
                               fill_value=fill_value)
        # TODO: figure out if it matters whether we slice the original array
        # before or after padding
        elif (not self._periodic and ((transition == ('center', 'left')) or
                                       (transition == ('right', 'center')))):
            # pad only left
            left = _pad_array(da.isel(**{dim: slice(0, -1)}), dim, left=True,
                              boundary=boundary, fill_value=fill_value)
            right = da.data
        elif (not self._periodic and ((transition == ('center', 'right')) or
                                      (transition == ('left', 'center')))):
            # pad only left
            right = _pad_array(da.isel(**{dim: slice(1, None)}), dim,
                               boundary=boundary, fill_value=fill_value)
            left = da.data
        elif (self._periodic and ((transition == ('center', 'left')) or
                                  (transition == ('right', 'center')))):

            left = da.roll(**{dim: 1})
            if boundary_discontinuity is not None:
                left = add_to_slice(left, dim, 0, -boundary_discontinuity)
            left = left.data
            right = da.data
        elif (self._periodic and ((transition == ('center', 'right')) or
                                  (transition == ('left', 'center')))):
            left = da.data
            right = da.roll(**{dim: -1})
            if boundary_discontinuity is not None:
                right = add_to_slice(right, dim, -1, boundary_discontinuity)
            right = right.data
        else:
            is_periodic = 'periodic' if self._periodic else 'non-periodic'
            raise NotImplementedError(' to '.join(transition) +
                                      ' (%s) transition not yet supported.'
                                      % is_periodic)

        return left, right

    @docstrings.dedent
    def interp(self, da, to=None, boundary=None, fill_value=0.0,
               boundary_discontinuity=None):
        """
        Interpolate neighboring points to the intermediate grid point along
        this axis.

        Parameters
        ----------
        %(neighbor_binary_func.parameters.no_f)s

        Returns
        -------
        da_i : xarray.DataArray
            The interpolated data

        """

        return self._neighbor_binary_func(da, raw_interp_function, to,
                                          boundary, fill_value,
                                          boundary_discontinuity)

    @docstrings.dedent
    def diff(self, da, to=None, boundary=None, fill_value=0.0,
             boundary_discontinuity=None):
        """
        Difference neighboring points to the intermediate grid point.

        Parameters
        ----------
        %(neighbor_binary_func.parameters.no_f)s

        Returns
        -------
        da_i : xarray.DataArray
            The differenced data
        """

        return self._neighbor_binary_func(da, raw_diff_function, to,
                                          boundary, fill_value,
                                          boundary_discontinuity)

    @docstrings.dedent
    def cumsum(self, da, to=None, boundary=None, fill_value=0.0):
        """
        Cumulatively sum a DataArray, transforming to the intermediate axis
        position.

        Parameters
        ----------
        %(neighbor_binary_func.parameters.no_f)s

        Returns
        -------
        da_cum : xarray.DataArray
            The cumsummed data
        """

        pos, dim = self._get_axis_coord(da)

        if to is None:
            to = self._default_shifts[pos]

        # first use xarray's cumsum method
        da_cum = da.cumsum(dim=dim)

        boundary_kwargs = dict(boundary=boundary, fill_value=fill_value)

        # now pad / trim the data as necessary
        # here we enumerate all the valid possible shifts
        if ((pos == 'center' and to == 'right') or
            (pos == 'left' and to == 'center')):
            # do nothing, this is the default for how cumsum works
            data = da_cum.data
        elif ((pos == 'center' and to == 'left') or
              (pos == 'right' and to == 'center')):
            data = _pad_array(da_cum.isel(**{dim: slice(0, -1)}), dim,
                              left=True, **boundary_kwargs)
        elif ((pos == 'center' and to == 'inner') or
              (pos == 'outer' and to == 'center')):
            data = da_cum.isel(**{dim: slice(0, -1)}).data
        elif ((pos == 'center' and to == 'outer') or
              (pos == 'inner' and to == 'center')):
            data = _pad_array(da_cum, dim, left=True, **boundary_kwargs)
        else:
            raise ValueError("From `%s` to `%s` is not a valid position "
                             "shift for cumsum operation." % (pos, to))

        da_cum_newcoord = self._wrap_and_replace_coords(da, data, to)
        return da_cum_newcoord

    def _wrap_and_replace_coords(self, da, data_new, position_to):
        """
        Take the base coords from da, the data from data_new, and return
        a new DataArray with a coordinate on position_to.
        """
        position_from, old_dim = self._get_axis_coord(da)
        try:
            new_coord = self.coords[position_to]
        except KeyError:
            raise KeyError("Position '%s' was not found in axis.coords."
                           % position_to)
        new_dim = new_coord.name

        orig_dims = da.dims

        coords = OrderedDict()
        dims = []
        for d in orig_dims:
            if d == old_dim:
                dims.append(new_dim)
                coords[new_dim] = new_coord
            else:
                dims.append(d)
                coords[d] = da.coords[d]

        return xr.DataArray(data_new, dims=dims, coords=coords)

    def _get_axis_coord(self, da):
        """Return the position and name of the axis coordiante in a DataArray.
        """
        for position, coord in iteritems(self.coords):
            # TODO: should we have more careful checking of alignment here?
            if coord.name in da.dims:
                return position, coord.name

        raise KeyError("None of the DataArray's dims %s were found in axis "
                       "coords." % repr(da.dims))

class Grid:
    """
    An object with multiple :class:`xgcm.Axis` objects representing different
    independent axes.
    """

    def __init__(self, ds, check_dims=True, periodic=True, default_shifts={}):
        """
        Create a new Grid object from an input dataset.

        Parameters
        ----------
        ds : xarray.Dataset
            Contains the relevant grid information. Coordinate attributes
            should conform to Comodo conventions [1]_.
        check_dims : bool, optional
            Whether to check the compatibility of input data dimensions before
            performing grid operations.
        periodic : {True, False, list}
            Whether the grid is periodic (i.e. "wrap-around"). If a list is
            specified (e.g. `['X', 'Y']`), the axis names in the list will be
            be periodic and any other axes founds will be assumed non-periodic.
        default_shifts : dict
            A dictionary of dictionaries specifying default grid position
            shifts (e.g. `{'X': {'center': 'left', 'left': 'center'}}`)

        REFERENCES
        ----------
        .. [1] Comodo Conventions http://pycomodo.forge.imag.fr/norm.html
        """
        self._ds = ds
        self._check_dims = check_dims

        all_axes = comodo.get_all_axes(ds)

        self.axes = OrderedDict()
        for axis_name in all_axes:
            try:
                is_periodic = axis_name in periodic
            except TypeError:
                is_periodic = periodic
            if axis_name in default_shifts:
                axis_default_shifts = default_shifts[axis_name]
            else:
                axis_default_shifts = {}
            self.axes[axis_name] = Axis(ds, axis_name, is_periodic,
                                        default_shifts=axis_default_shifts)

    def __repr__(self):
        summary = ['<xgcm.Grid>']
        for name, axis in iteritems(self.axes):
            is_periodic = 'periodic' if axis._periodic else 'not periodic'
            summary.append('%s Axis (%s):' % (name, is_periodic))
            summary += axis._coord_desc()
        return '\n'.join(summary)

    @docstrings.dedent
    def interp(self, da, axis, **kwargs):
        """
        Interpolate neighboring points to the intermediate grid point along
        this axis.

        Parameters
        ----------
        axis : str
            Name of the axis on which ot act
        %(neighbor_binary_func.parameters.no_f)s

        Returns
        -------
        da_i : xarray.DataArray
            The interpolated data
        """

        ax = self.axes[axis]
        return ax.interp(da, **kwargs)

    @docstrings.dedent
    def diff(self, da, axis, **kwargs):
        """
        Difference neighboring points to the intermediate grid point.

        Parameters
        ----------
        axis : str
            Name of the axis on which ot act
        %(neighbor_binary_func.parameters.no_f)s

        Returns
        -------
        da_i : xarray.DataArray
            The differenced data
        """

        ax = self.axes[axis]
        return ax.diff(da, **kwargs)

    @docstrings.dedent
    def cumsum(self, da, axis, **kwargs):
        """
        Cumulatively sum a DataArray, transforming to the intermediate axis
        position.

        Parameters
        ----------
        axis : str
            Name of the axis on which ot act
        %(neighbor_binary_func.parameters.no_f)s

        Returns
        -------
        da_i : xarray.DataArray
            The cumsummed data
        """

        ax = self.axes[axis]
        return ax.cumsum(da, **kwargs)

def add_to_slice(da, dim, sl, value):
    # split array into before, middle and after (if slice is the
    # beginning or end before or after will be empty)
    before = da[{dim: slice(0, sl)}]
    middle = da[{dim: sl}]
    after = da[{dim: slice(sl+1, None)}]
    if sl < -1:
        raise RuntimeError('slice can not be smaller value than -1')
    elif sl == -1:
        da_new = xr.concat([before, middle+value], dim=dim)
    else:
        da_new = xr.concat([before, middle+value, after], dim=dim)
    # then add 'value' to middle and concatenate again
    return da_new

def raw_interp_function(data_left, data_right):
    # linear, centered interpolation
    # TODO: generalize to higher order interpolation
    return 0.5*(data_left + data_right)

def raw_diff_function(data_left, data_right):
    return data_right - data_left

_other_docstring_options="""
    * 'dirichlet'
       The value of the array at the boundary point is specified by
       `fill_value`.
    * 'neumann'
       The value of the array diff at the boundary point is
       specified[1]_ by `fill_value`.

        .. [1] https://en.wikipedia.org/wiki/Dirichlet_boundary_condition
        .. [2] https://en.wikipedia.org/wiki/Neumann_boundary_condition
"""

max_stars_repo_name: roxyboy/xgcm

max_issues_repo_name: roxyboy/xgcm

max_forks_repo_name: roxyboy/xgcm


text:
from tensorflow.python.keras import layers
from tensorflow.python.keras import models
from tensorflow.keras.optimizers import Adam
from tensorflow import keras
import numpy as np
BITS = 6
OPS_BITS = 4

max_stars_repo_name: HendrickZhou/BigBrainALU

max_issues_repo_name: HendrickZhou/BigBrainALU

max_forks_repo_name: HendrickZhou/BigBrainALU


text:
# -*- coding: utf-8 -*-
"""Setup script for refer.

mask.so is compiled as a Cython extension used to visualize the segmentation
of referred object.
All "mask" related code is copied from https://github.com/pdollar/coco.git
"""

# Standard library imports
import os
import ast

# Third party imports
import numpy as np
from Cython.Build import cythonize
from setuptools import Extension, find_packages, setup

HERE = os.path.abspath(os.path.dirname(__file__))

def get_version(module='referit'):
    """Get version from text file and avoids importing the module."""
    with open(os.path.join(HERE, module, '__init__.py'), 'r') as f:
        data = f.read()
    lines = data.split('\n')
    for line in lines:
        if line.startswith('VERSION_INFO'):
            version_tuple = ast.literal_eval(line.split('=')[-1].strip())
            version = '.'.join(map(str, version_tuple))
            break
    return version

ext_modules = [
    Extension(
        'referit.external._mask',
        sources=['referit/external/maskApi.c', 'referit/external/_mask.pyx'],
        include_dirs=[np.get_include(), 'external'],
        extra_compile_args=['-Wno-cpp', '-Wno-unused-function', '-std=c99'],
    )
]

REQUIREMENTS = ['cython', 'numpy', 'matplotlib', 'scikit-image']

setup(
    name='referit',
    version=get_version(),
    keywords=['referit', 'segmentation', 'dataset'],
    url='https://github.com/xvjiarui/refer',
    license='MIT',
    author='Jiarui Xu',
    author_email='xvjiarui0826@gmail.com',
    description='Python wrapper to load the ReferIt Game dataset',
    ext_modules=cythonize(ext_modules),
    packages=find_packages(exclude=['contrib', 'docs', 'tests*']),
    include_package_data=True,
    install_requires=REQUIREMENTS,
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: MIT License',
        'Operating System :: MacOS',
        'Operating System :: Microsoft :: Windows',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6'
    ]
)

max_stars_repo_name: xvjiarui/refer

max_issues_repo_name: xvjiarui/refer

max_forks_repo_name: xvjiarui/refer


text:
import numpy as np
from scipy.stats import norm
import matplotlib
matplotlib.rcParams['text.usetex'] = True
import matplotlib.pyplot as plt
import argparse
import os
import json

from em_pe.models import model_dict, param_dict

parser = argparse.ArgumentParser(description="Script to generate PP plot")
parser.add_argument("--m", help="Model to use")
parser.add_argument("--directory", help="PP plot run directory")
parser.add_argument("--name", help="Name")
parser.add_argument("--exclude-param", action="append", help="parameter to exclude from plot")
parser.add_argument("--exclude-dir", action="append", help="Exclude a run by directory name")
args = parser.parse_args()

exclude_params = args.exclude_param if args.exclude_param is not None else []
exclude_dir = args.exclude_dir if args.exclude_dir is not None else []

eff_samp_cutoff = 10.0

def binomial_credible_interval(phat, n, z):
    offset = np.sqrt(phat * (1.0 - phat) / n + z**2 / (4 * n**2))
    my_1 = np.ones((len(phat), 2))
    return 1.0 / (1.0 + z**2 / n) *  (np.outer(phat, np.array([1, 1])) + my_1 * z**2 / (2 * n) + np.outer(z * offset, np.array([-1, 1])))

pvalFiducial = 0.9 # fiducial credible level for outermost CI.  (Will change depending on number of parameters)

def binomial_credible_interval_default(phat, n, nParams=2):
    z_fiducial = norm.ppf((np.power(1.0 - (1.0 - pvalFiducial) / 2.0 , 1.0 / nParams)))
    #print("fiducial z", z_fiducial, " for nParams = ", nParams)
    return binomial_credible_interval(phat, n, z_fiducial)

tex_dict = {'mej':'$m_{ej}$',
            'mej_red':'$m_{ej}$ [red]',
            'mej_purple':'$m_{ej}$ [purple]',
            'mej_blue':'$m_{ej}$ [blue]',
            'vej':'$v_{ej}$',
            'vej_red':'$v_{ej}$ [red]',
            'vej_purple':'$v_{ej}$ [purple]',
            'vej_blue':'$v_{ej}$ [blue]',
            'Tc_red':'Tc [red]',
            'Tc_purple':'Tc [purple]',
            'Tc_blue':'Tc [blue]',
            'dist':'Distance',
            'kappa':'$\kappa$',
            'sigma':'$\\sigma$'
}

base_dir = args.directory
if base_dir[-1] != "/":
    base_dir += "/"

base_dir += args.name
if base_dir[-1] != "/":
    base_dir += "/"

m = model_dict[args.m]()
ordered_params = m.param_names + ["dist"]
params = {p:param_dict[p]() for p in ordered_params}
params_used = set()

cache = {}
eff_samp_list = []
dir_list = []

for d in os.listdir(base_dir):
    curr_dir = base_dir + d + "/"
    if not os.path.isdir(curr_dir) or "samples.txt" not in os.listdir(curr_dir) or curr_dir in exclude_dir or d in exclude_dir:
        continue
    print("reading samples from", curr_dir)
    s = np.loadtxt(curr_dir + "samples.txt")
    truths = np.loadtxt(curr_dir + "test_truths.txt")
    truths = dict(zip(ordered_params, truths))
    with open(curr_dir + "samples.txt") as f:
        ### the "header" contains the column names
        header = f.readline().strip().split(" ")[1:]
    samples = {header[i]:s[:,i] for i in range(len(header))}
    lnL = samples["lnL"]
    p = samples["p"]
    p_s = samples["p_s"]
    temp = np.exp(lnL) * p / p_s
    eff_samp = np.sum(temp) / np.max(temp)
    print("    eff_samp = {}".format(eff_samp))
    eff_samp_list.append(eff_samp)
    dir_list.append(d)
    L = np.exp(lnL - np.max(lnL))
    weights = L * p / p_s
    weights /= np.sum(weights)
    for i in range(3, len(header)):
        p = header[i]
        if p in exclude_params: continue
        if p not in params_used:
            cache[p] = {"CDF":[], "true":[], "ML":[]}
            params_used.add(p)
        cdf = np.sum(weights[samples[p] < truths[p]])
        cache[p]["CDF"].append(cdf)
        cache[p]["true"].append(truths[p])
        cache[p]["ML"].append(samples[p][np.argmax(lnL)])

eff_samp = np.array(eff_samp_list)
mask = eff_samp > eff_samp_cutoff
npts = eff_samp.size
npts_valid = np.sum(mask)

out_dict = {
        i:{"params":{p:{} for p in params_used}, "eff_samp":eff_samp[i], "directory":dir_list[i]
    } for i in range(npts)
}

plt.figure(figsize=(8, 8))

markers = {
        "mej":"x",
        "mej_red":"x",
        "mej_purple":"x",
        "mej_blue":"x",
        "vej":"o",
        "vej_red":"o",
        "vej_purple":"o",
        "vej_blue":"o",
        "sigma":"+"
}
colors = {
        "mej":"red",
        "mej_red":"red",
        "mej_purple":"purple",
        "mej_blue":"blue",
        "vej":"blue",
        "vej_red":"red",
        "vej_purple":"purple",
        "vej_blue":"blue",
        "sigma":"black"
}

for p in params_used:
    if p in exclude_params: continue
    cdf = np.array(cache[p]["CDF"])
    ind = np.argsort(cdf[mask])
    j = 0.0
    for i in range(npts):
        out_dict[i]["params"][p]["CDF"] = cache[p]["CDF"][i]
        out_dict[i]["params"][p]["true"] = cache[p]["true"][i]
        out_dict[i]["params"][p]["ML"] = cache[p]["ML"][i]
        if mask[i]:
            (xval,) = np.where(ind == j)[0] / npts_valid
            j += 1.0
        else:
            xval = np.nan
        out_dict[i]["params"][p]["xval"] = xval
    cdf = cdf[mask][ind]
    plt.scatter(np.arange(npts_valid) / float(npts_valid), cdf,
            label=(tex_dict[p] if p in tex_dict.keys() else p),
            marker=(markers[p] if p in markers.keys() else None),
            color=(colors[p] if p in colors.keys() else None)
    )

xvals = np.linspace(0.0, 1.0, 100)
plt.plot(xvals, xvals, "--", color="black")
pvals_lims = binomial_credible_interval_default(xvals, npts_valid, nParams=(len(params_used) + len(exclude_params)))
plt.plot(xvals, pvals_lims[:,0], color='k', ls=':')
plt.plot(xvals, pvals_lims[:,1], color='k', ls=':')

plt.xlabel("$P(x_{\\rm inj})$")
plt.ylabel("$\hat{P}$")
plt.legend()
plt.tight_layout()

plt.savefig("pp_plot.png")

with open("out.json", "w") as f:
    json.dump(out_dict, f, indent=4)

reason: import numpy,from scipy


text:
#%matplotlib notebook
import ipywidgets as ipw
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm
import matplotlib.ticker as mtick
from mpl_toolkits.axes_grid1 import make_axes_locatable
import pandas as pd
import geopandas as gpd

class livePlot():
    def __init__(self, geo_data, dates):
        def format_map():
            size = "5%"
            sm = cm.ScalarMappable(cmap=self.cmap, norm=self.norm)
            # empty array for the data range
            sm._A = []
            # make space for colorbar
            divider = make_axes_locatable(self.ax)
            cax = divider.append_axes("right", size = size, pad = 0.1)
            # add colorbar to figure
            cbar = self.fig.colorbar(sm, cax=cax, cmap = self.cmap)
            cbar.ax.tick_params(labelsize=18)
            vals = list(cbar.ax.get_yticks())
            vals.append(self.vmax)
            #if "Daily" not in key:
            vals[0] = 0
            # format display of values on colorbar
            # if log:
            #     cbar.ax.yaxis.set_major_formatter(mtick.LogFormatter())
            # else:
            cbar.ax.yaxis.set_major_formatter(mtick.Formatter())
            # format colorbar values as int
            cbar.ax.set_yticklabels([int(x) for x in vals])
            cbar.ax.set_ylabel(self.key, fontsize = 20)

        def interact():
            def update(date_index = 0):
                date = self.dates[date_index]
                # self.ax.clear()
                plt.cm.ScalarMappable(cmap=self.cmap, norm=self.norm)
                plot_df = self.geo_data[self.geo_data.index.get_level_values("date")==date]
                plot_df.plot(ax=self.ax, cax = self.ax, column=self.key, vmin=self.vmin ,vmax = self.vmax,
                             cmap = self.cmap, legend=False, linewidth=.5, edgecolor='lightgrey',
                             norm = self.norm)
                self.ax.set_title(str(date)[:10] + "\n" + "COVID-19 in the U.S.", fontsize = 30)

            self.widget = ipw.interact(update,
                          date_index = ipw.IntSlider(
                            value=len(self.dates) - 1,
                            min=0,
                            max=len(self.dates) - 1,
                            step=1,
                            description='Date Index:',
                            disabled=False,
                            continuous_update=True,
                            orientation='horizontal',
                            readout=True,
                            readout_format="d"))

        plt.rcParams["font.size"] = 20
        self.fig, self.ax = plt.subplots(figsize = (8, 6))
        self.geo_data = geo_data
        self.dates = dates
        self.date = self.dates[-1]
        self.key="Cases per Million"
        self.vmin, self.vmax = 0, geo_data[self.key].max()
        self.cmap = cm.get_cmap('Reds', 10)
        self.norm = cm.colors.Normalize(vmin = self.vmin, vmax = self.vmax)
        # format_map()
        interact()

        # self.maxx = 1000
        # self.maxy = 130
        # self.ax.set_xlim(0, self.maxx)
        # self.ax.set_ylim(0, self.maxy)
        # plt.xticks([])
        # plt.yticks([])

        # self.M = 10 ** 3
        # self.V = 8

        # self.y = np.linspace(1, self.M, self.M)
        # self.y0 = 10 ** 2 * 5

        # self.AD, = self.ax.plot(self.y, self.M * self.V / self.y)
        # self.LRAS = self.ax.axvline(self.y0)

        # x_int, y_int = self.get_intersect(self.AD, self.LRAS, line2_vert = True)
        # self.h_line_intersect = self.ax.axhline(y_int, xmin = 0, xmax = x_int,
        #                               ls = "--", color = "k")

        # self.text_vert_shift = 1
        # self.text_horiz_shift = 20
        # self.P_text = self.ax.text(-50, y_int, "$P$")
        # self.AD_text = self.ax.text(900, self.AD.get_ydata()[900] + self.text_vert_shift * -2, "$AD$")
        # self.LRAS_text = self.ax.text(self.LRAS.get_xdata(orig=False)[0] + self.text_horiz_shift ,
        #                               110, "$LRAS$")
        # self.y0_text = self.ax.text(self.LRAS.get_xdata(orig=False)[0] - self.text_horiz_shift * .75, -10, "$y$")
        # # self.MV_val_text = self.ax.text(1001, 100, "MV =" +str(self.M * self.V))
        # # self.str_vals = "MV =" +str(int(self.M * self.V)) + "   P =" + str(int(self.M * self.V / self.y0)) + "   y ="+ str(int(self.y0))
        # self.show_vals = self.ax.text(1, self.maxy * 1.03,
        #                               "MV =" +str(int(self.M * self.V)) + "    P =" + str(int(self.M * self.V / self.y0)) + "    y ="+ str(int(self.y0)),
        #                               fontsize = 14)

    # def get_intersect(self, line1, line2, line2_vert = False):
    #     if line2_vert == False:
    #         x = np.argwhere(np.diff(np.sign(line1 - line2))).flatten()
    #     else:
    #         line1_data = line1.get_data()
    #         # set orig = False or else list reads as float
    #         line2_xdata = line2.get_xdata(orig=False)[0]
    #         dist = [np.abs(i - line2_xdata) for i in line1_data[0]]
    #         min_dist = min(dist)
    #         x = dist.index(min_dist)
    #         y = line1_data[1][x]

    #     return x, y


text:
import tensorflow as tf
import numpy as np
from tensorflow.keras import layers
import keras
from keras.engine.topology import Layer
import math

class GeM(Layer):
    def __init__(self, exp, **kwargs):
        super(GeM, self).__init__(**kwargs)
        self.exp = exp

    def call(self, x, mask=None):
        return tf.reduce_mean(x ** self.exp, axis=(1,2)) ** (1/self.exp)

    def get_output_shape_for(self, input_shape):
        return (input_shape[0], input_shape[2])

def normalize_model(tensor_in):
    last_axis = len(tensor_in.shape) - 1
    return layers.Lambda(lambda x: tf.linalg.normalize(x, axis=last_axis)[0])(tensor_in)

def combine_model(tensor_list_in):
    return layers.Lambda(lambda x: tf.add_n(x))(tensor_list_in)

def reduce_model(tensor_in):
    return layers.Lambda(lambda x: tf.math.reduce_sum(x, axis=(1,2)))(tensor_in)

def self_attention_model(tensor_in):
    atten = layers.Reshape((-1, tensor_in.shape[3]))(tensor_in)
    atten = layers.Attention()([atten,atten])
    atten = layers.Reshape(tensor_in.shape[1:])(atten)
    return atten

def mac_model(tensor_in, exp):
    gem = GeM(exp)(tensor_in)
    return gem

def rmac_model(tensor_in, exp, depth):
    exp_out = layers.Lambda(lambda x: x ** exp)(tensor_in)

    short_side = min(exp_out.shape[1], exp_out.shape[2])

    maps = []

    for d in range(1,depth+1):
        size = math.floor(short_side/d)
        stride = math.floor(size * 0.5)
        print(size)
        print(stride)
        pool = layers.AveragePooling2D((size,size),(stride,stride), padding='VALID')(exp_out)
        pool = layers.Lambda(lambda x: x ** (1/exp))(pool)
        pool = normalize_model(pool)
        maps.append(pool)

    return maps

def build_model(in_shape, exp, vgg_output=False, attention=False, mac=False, rmac=False, regions=False, depth=10):
    vgg = tf.keras.applications.VGG16(include_top=False,
                                          weights='imagenet',
                                          input_shape=in_shape)

    vgg_out = vgg.get_layer('block5_conv3').output
    print(vgg_out.shape)
    if attention:
        atten = self_attention_model(vgg_out)

    if vgg_output and attention and mac:
        vgg_out = mac_model(vgg_out, exp)
        vgg_out = normalize_model(vgg_out)
        atten = mac_model(atten, exp)
        atten = normalize_model(atten)
        combined = combine_model([vgg_out, atten])
        combined = normalize_model(combined)
        return keras.models.Model(vgg.input, combined)

    if vgg_output and mac:
        vgg_out = mac_model(vgg_out, exp)
        vgg_out = normalize_model(vgg_out)
        return keras.models.Model(vgg.input, vgg_out)

    if attention and mac:
        atten = mac_model(atten, exp)
        atten = normalize_model(atten)
        return keras.models.Model(vgg.input, atten)

    if vgg_output and attention and rmac:
        vgg_out = rmac_model(vgg_out, exp, depth)
        atten = rmac_model(atten, exp, depth)
        comb = []
        for d in range(depth):
            c = combine_model([vgg_out[d], atten[d]])
            c = reduce_model(c)
            comb.append(c)
        combined = combine_model(comb)
        combined =  normalize_model(combined)
        return keras.models.Model(vgg.input, combined)

    if vgg_output and rmac:
        vgg_out = rmac_model(vgg_out, exp, depth)
        comb = []
        for d in range(depth):
            c = reduce_model(vgg_out[d])
            comb.append(c)
        vgg_out = combine_model(comb)
        vgg_out = normalize_model(vgg_out)
        return keras.models.Model(vgg.input, vgg_out)

    if attention and rmac:
        atten = rmac_model(atten, exp, depth)
        comb = []
        for d in range(depth):
            c = reduce_model(atten[d])
            comb.append(c)
        atten = combine_model(comb)
        atten = normalize_model(atten)
        return keras.models.Model(vgg.input, atten)

    if vgg_output and attention and regions:
        vgg_out = rmac_model(vgg_out, exp, depth)
        atten = rmac_model(atten, exp, depth)
        comb = []
        for d in range(depth):
            c = combine_model([vgg_out[d], atten[d]])
            c = normalize_model(c)
            comb.append(c)
        return keras.models.Model(vgg.input, comb)

    if vgg_output and regions:
        vgg_out = rmac_model(vgg_out, exp, depth)
        return keras.models.Model(vgg.input, vgg_out)

    if attention and regions:
        atten = rmac_model(atten, exp, depth)
        return keras.models.Model(vgg.input, atten)

    #sdf


text:
import torch
import torchvision
import torchvision.transforms as transforms

import matplotlib.pyplot as plt
import numpy as np

#download dataset start
transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

trainset = torchvision.datasets.CIFAR10(root='./PyTorch/data', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,
                                          shuffle=True, num_workers=0)

testset = torchvision.datasets.CIFAR10(root='./PyTorch/data', train=False,
                                       download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=4,
                                         shuffle=False, num_workers=0)

classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

#download dataset end

def imshow(img):
    img = img / 2 + 0.5     # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()

# get some random training images
dataiter = iter(trainloader)
images, labels = dataiter.next()

# show images
imshow(torchvision.utils.make_grid(images))
# print labels
print(' '.join('%5s' % classes[labels[j]] for j in range(4)))

max_stars_repo_name: sagasu/ImageRecognition

max_issues_repo_name: sagasu/ImageRecognition

max_forks_repo_name: sagasu/ImageRecognition


text:
import math
import numpy

def pivot(l, left, right):  # right should be len(l)-1
	"""
	Returns the position of the median of medians.
	Left and right values should be the start and end positions of the part of the array to use.
	"""
	if (right - left < 5):
		return partition5(l, left, right)

	for i in xrange(left, right + 1, 5):  # maybe right+1
		subRight = i + 4
		if subRight > right:
			subRight = right
		median5 = partition5(l, i, subRight)

		tmp = numpy.copy(l[median5])
		l[median5] = l[left + int(math.floor((i - left) / 5))]
		l[left + int(math.floor((i - left)/5))] = tmp
		return pivot(l, left, left + ((right - left) / 5))  # no ceil((right-left)/5.0-1) because /5 already takes floor
# 	return select(l, left, left + ((right - left) / 5), left + (right - left) / 10)  # no ceil((right-left)/5.0-1) because /5 already takes floor

def partition5(l, left, right):
	"""
	Insertion Sort of list of at most 5 elements and return the position of the median.
	"""
	j = left
	for i in xrange(left, right + 1):
		t = numpy.copy(l[i])
		for j in xrange(i, left - 1, -1):
			if l[j - 1][0] < t[0]:
				break
			l[j] = l[j - 1]
		l[j] = t
	return int(math.floor((left + right) / 2))

def select(l, left, right, n):
	while(True):
		if left == right:
			return left
		pivotIndex = pivot(l, left, right)
		pivotIndex = partition(l, left, right - 1, pivotIndex)
		if (n == pivotIndex):
			return n
		elif n < pivotIndex:
			right = pivotIndex - 1
			left = pivotIndex + 1

def partition(l, left, right, pivotIndex):  # right is included
	pivotValue = numpy.copy(l[pivotIndex])
	l[pivotIndex] = l[right]
	l[right] = pivotValue
	storeIndex = left
	tmp = 0

	for i in xrange(left, right):
		if l[i][0] < pivotValue[0]:
			tmp = l[storeIndex]
			l[storeIndex] = l[i]
			l[i] = tmp
			storeIndex += 1

	l[right] = l[storeIndex]
	l[storeIndex] = pivotValue
	return storeIndex

def for2DArray(l):
	return pivot(l, 0, l.shape[0]-1)

max_stars_repo_name: SynerClust/SynerClust

max_issues_repo_name: broadinstitute/SynerClust

max_forks_repo_name: broadinstitute/SynerClust


text:
from . import ProgressiveTest
from progressivis.table.table_base import TableSelectedView
from progressivis.table.table import Table
from progressivis.core.bitmap import bitmap
from progressivis.table.changemanager_table_selected import TableSelectedChangeManager, FakeSlot
from progressivis.table.tablechanges import TableChanges
import numpy as np

class __FakeSlot(object):
    def __init__(self, table):
        self.table = table

    def data(self):
        return self.table

class TestTableSelectedChangeManager(ProgressiveTest):
    def setUp(self):
        super(TestTableSelectedChangeManager, self).setUp()
        self.s = self.scheduler()

    def test_tablechangemanager(self):
        #pylint: disable=protected-access
        table = Table('test_changemanager_table_selected',
                      data={'a': [ 1, 2, 3], 'b': [10.1, 0.2, 0.3]})
        selection = bitmap([1,2])
        table_selected = TableSelectedView(table, selection)

        s = self.s
        s._run_number = 1
        last = s._run_number
        slot = FakeSlot(table_selected)

        mid1 = 1
        cm = TableSelectedChangeManager(slot,
                                        buffer_exposed=True,
                                        buffer_updated=True,
                                        buffer_deleted=True,
                                        buffer_masked=True)
        self.assertEqual(cm.last_update(), 0)
        self.assertEqual(cm.created.length(), 0)
        self.assertEqual(cm.updated.length(), 0)
        self.assertEqual(cm.deleted.length(), 0)

        mid2 = 2
        cm2 = TableSelectedChangeManager(slot,
                                         buffer_exposed=True,
                                         buffer_updated=True,
                                         buffer_deleted=True,
                                         buffer_masked=True)
        self.assertEqual(cm2.last_update(), 0)
        self.assertEqual(cm2.created.length(), 0)
        self.assertEqual(cm2.updated.length(), 0)
        self.assertEqual(cm2.deleted.length(), 0)

        mid3 = 3
        cm3 = TableSelectedChangeManager(slot,
                                         buffer_exposed=True,
                                         buffer_updated=True,
                                         buffer_deleted=True,
                                         buffer_masked=True)
        self.assertEqual(cm3.last_update(), 0)
        self.assertEqual(cm3.created.length(), 0)
        self.assertEqual(cm3.updated.length(), 0)
        self.assertEqual(cm3.deleted.length(), 0)
        cm.update(last, table_selected, mid=mid1)
        self.assertEqual(cm.last_update(), last)
        self.assertEqual(cm.created.next(),slice(1, 3)) # without the mask
        self.assertEqual(cm.updated.length(), 0)
        self.assertEqual(cm.deleted.length(), 0)

        s._run_number += 1
        last = s._run_number
        table.append({'a': [ 4], 'b': [0.5]}) # invisible since id=3
        cm.update(last, table_selected, mid=mid1)
        self.assertEqual(cm.last_update(), last)
        self.assertEqual(cm.created.length(), 0)
        self.assertEqual(cm.updated.length(), 0)
        self.assertEqual(cm.deleted.length(), 0)

        s._run_number += 1
        last = s._run_number
        table.append({'a': [ 5, 6, 7, 8], 'b': [0.5, 0.6, 0.7, 0.8] })
        table_selected.selection = bitmap(range(1,8))
        cm.update(last, table_selected, mid=mid1)
        self.assertEqual(cm.last_update(), last)
        self.assertEqual(cm.created.next(),slice(3, 8))
        self.assertEqual(cm.updated.length(), 0)
        self.assertEqual(cm.deleted.length(), 0)

        s._run_number += 1
        last = s._run_number
        del table.loc[[1,2,3]]
        table_selected.selection = bitmap([3,4]) # i.e 1,2,5,6,7 were deleted in selection
        cm.update(last, table_selected, mid=mid1)
        self.assertEqual(cm.last_update(), last)
        self.assertEqual(cm.created.length(), 0)
        self.assertEqual(cm.updated.length(), 0)
        self.assertEqual(cm.base.deleted.length(), 3) # 1, 2, 3
        self.assertEqual(cm.selection.deleted.length(), 6) # 1, 2, 5, 6, 7[+3 removed because it was perm.deleted]
        self.assertEqual(cm.deleted.length(), 6) # 1, 2, 3, 5, 6, 7
        cm.base.deleted.next()
        cm.selection.deleted.next()
        s._run_number += 1
        last = s._run_number
        table.append({'a': [ 15, 16, 17, 18], 'b': [0.51, 0.61, 0.71, 0.81] })
        table_selected.selection = slice(5, None)
        cm.update(last, table_selected, mid=mid1)
        self.assertEqual(cm.last_update(), last)
        self.assertEqual(cm.base.created.changes, bitmap([8, 9, 10, 11]))
        self.assertEqual(cm.selection.created.changes, bitmap([5, 6, 7, 8, 9, 10, 11]))
        self.assertEqual(cm.selection.deleted.changes, bitmap([4]))
        self.assertEqual(cm.updated.length(), 0)
        self.assertEqual(cm.base.deleted.length(), 0)
        self.assertEqual(cm.deleted.length(), 1)
        cm.deleted.next()
        self.assertEqual(cm.deleted.length(), 0)
        cm.created.next()
        self.assertEqual(cm.base.created.length(), 0)
        self.assertEqual(cm.selection.created.length(), 0)
        # s._run_number += 1
        # a[3] = 42
        # b[3] = 0.42
        # b[4] = 0.52
        # last = s._run_number
        # cm.update(last, table, mid=mid1)
        # self.assertEqual(cm.last_update(), last)
        # self.assertEqual(cm.created.length(), 0)
        # self.assertEqual(cm.updated.next(), slice(3,5))
        # self.assertEqual(cm.deleted.length(), 0)

        # s._run_number += 1
        # last = s._run_number
        # cm.update(last, table, mid=mid1)
        # self.assertEqual(cm.last_update(), last)
        # self.assertEqual(cm.created.length(), 0)
        # self.assertEqual(cm.updated.length(), 0)
        # self.assertEqual(cm.deleted.length(), 0)

        # s._run_number += 1
        # last2 = 0
        # a[2] = 22
        # b[2] = 0.22
        # b[1] = 0.12

        # last2 = s._run_number
        # cm2.update(last2, table, mid=mid2)
        # self.assertEqual(cm2.last_update(), last2)
        # self.assertEqual(cm2.created.next(), slice(0, 5))
        # self.assertEqual(cm2.updated.length(), 0)
        # self.assertEqual(cm2.deleted.length(), 0)

        # s._run_number += 1
        # a[0] = 11
        # b[0] = 0.11
        # b[2] = 0.32
        # table.append({'a': [ 6], 'b': [0.6] })

        # tv = table.loc[1:2]
        # last3 = s._run_number
        # cm3.update(last3, tv, mid=mid3)
        # self.assertEqual(cm3.created.next(), slice(1, 3)) # test ids, not indices
        # self.assertEqual(cm2.updated.length(), 0)
        # self.assertEqual(cm2.deleted.length(), 0)

        # s._run_number += 1
        # last = s._run_number
        # # with self.assertRaises(ValueError):
        # #     cm.update(last+1, table, mid=mid1)
        # cm.update(last, table, mid=mid1)
        # self.assertEqual(cm.last_update(), last)
        # self.assertEqual(cm.created.next(), slice(5,6))
        # self.assertEqual(cm.updated.next(), slice(0,3))
        # self.assertEqual(cm.deleted.length(), 0)

        # s._run_number += 1
        # last2 = s._run_number
        # cm2.update(last2, table, mid=mid2)
        # self.assertEqual(cm2.last_update(), last2)
        # self.assertEqual(cm2.created.next(), slice(5,6))
        # self.assertEqual(list(cm2.updated.next()), [0,2])
        # self.assertEqual(cm2.deleted.length(), 0)

        # s._run_number += 1
        # a[0] = 1
        # b[0] = 0.11
        # b[2] = 0.22
        # last3 = s._run_number
        # cm3.update(last3, tv, mid=mid3)
        # self.assertEqual(cm3.last_update(), last3)
        # self.assertEqual(cm3.created.length(), 0)
        # self.assertEqual(cm3.updated.next(), slice(2,3))
        # self.assertEqual(cm3.deleted.length(), 0)

        # # test deletes
        # s._run_number += 1
        # del table.loc[2]
        # last = s._run_number
        # cm.update(last, table, mid=mid1)
        # self.assertEqual(cm.last_update(), last)
        # self.assertEqual(cm.created.length(), 0)
        # self.assertEqual(cm.updated.length(), 0)
        # self.assertEqual(cm.deleted.next(), slice(2,3))
        # self.assertTrue(np.all(a[:]==np.array([1,2,a.fillvalue,42,5,6])))
        # self.assertTrue(np.all(b[:]==np.array([0.11,0.12,a.fillvalue,0.42,.52,0.6])))

        # s._run_number += 1
        # del table.loc[4]
        # table.append({'a': [ 7,8], 'b': [0.7,0.8] })
        # a[5] = 0.55
        # last2 = s._run_number
        # cm2.update(last2, table, mid=mid2)
        # self.assertEqual(cm2.last_update(), last2)
        # self.assertEqual(cm2.created.next(), slice(6,8))
        # self.assertEqual(cm2.updated.next(), slice(5,6))
        # self.assertEqual(list(cm2.deleted.next()), [2,4])

        #TODO test reset
        cm.reset()
        self.assertEqual(cm.last_update(), 0)

if __name__ == '__main__':
    ProgressiveTest.main()

max_stars_repo_name: jdfekete/progressivis

max_issues_repo_name: jdfekete/progressivis

max_forks_repo_name: jdfekete/progressivis