text:
# ==========================================================================
#
#   Copyright NumFOCUS
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#          https://www.apache.org/licenses/LICENSE-2.0.txt
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# ==========================================================================

import h5py as h5
import numpy as np

class AbstractDatasetHandler:
    """
    Abstract base class for dataset handlers.
    """

    def __init__(self):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::__init__ should not be called."
        )

    def read_all_features_from_h5py(self, filename, data_name="features"):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::read_all_features_from_h5py "
            "should not be called."
        )

    def write_all_features_to_h5py(self, filename, data_name="features"):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::write_all_features_to_h5py "
            "should not be called."
        )

    def set_all_features(self, features):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::set_all_features "
            "should not be called."
        )

    def get_all_features(self):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::get_all_features "
            "should not be called."
        )

    def clear_all_features(self):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::clear_all_features "
            "should not be called."
        )

    def set_some_features(self, feature_indices, features):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::set_some_features "
            "should not be called."
        )

    def get_some_features(self, feature_indices):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::get_some_features "
            "should not be called."
        )

    def read_all_labels_from_h5py(self, filename, data_name="labels"):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::read_all_labels_from_h5py "
            "should not be called."
        )

    def write_all_labels_to_h5py(self, filename, data_name="labels"):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::write_all_labels_to_h5py "
            "should not be called."
        )

    def set_all_labels(self, labels):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::set_all_labels "
            "should not be called."
        )

    def get_all_labels(self):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::get_all_labels "
            "should not be called."
        )

    def clear_all_labels(self):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::clear_all_labels "
            "should not be called."
        )

    def set_some_labels(self, label_indices, labels):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::set_some_labels "
            "should not be called."
        )

    def get_some_labels(self, label_indices):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::get_some_labels "
            "should not be called."
        )

    def set_all_dictionaries(self, dictionaries):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::set_all_dictionaries "
            "should not be called."
        )

    def get_all_dictionaries(self):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::get_all_dictionaries "
            "should not be called."
        )

    def clear_all_dictionaries(self):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::clear_all_dictionaries "
            "should not be called."
        )

    def set_some_dictionaries(self, dictionary_indices, dictionaries):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::set_some_dictionaries "
            "should not be called."
        )

    def get_some_dictionaries(self, dictionary_indices):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::get_some_dictionaries "
            "should not be called."
        )

    def set_all_label_definitions(self, label_definitions):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::set_all_label_definitions "
            "should not be called."
        )

    def get_all_label_definitions(self):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::get_all_label_definitions "
            "should not be called."
        )

    def check_data_consistency(self):
        raise NotImplementedError(
            "Abstract method AbstractDatasetHandler::check_data_consistency "
            "should not be called."
        )

class BasicDatasetHandler(AbstractDatasetHandler):
    def __init__(self):
        self.features = None
        self.labels = None
        self.dictionaries = None
        self.label_definitions = None

    """
    Handle the vector of features for each stored entity.
    """

    def read_all_features_from_h5py(self, filename, data_name="features"):
        """
        Read the entire database of features from a supplied h5py file.
        """
        with h5.File(filename) as ds:
            self.features = np.array(ds[data_name])

    def write_all_features_to_h5py(self, filename, data_name="features"):
        """
        Write the entire database of features to a h5py file.
        """
        with h5.File(filename, "w") as f:
            f.create_dataset(data_name, self.features.shape, data=self.features)

    def set_all_features(self, features):
        """
        Set the entire database of features from a supplied numpy array.
        """
        if isinstance(features, np.ndarray) and len(features.shape) == 2:
            self.features = features
        else:
            raise ValueError(
                "The argument to set_all_features must be a 2-dimensional numpy "
                "ndarray."
            )

    def get_all_features(self):
        """
        Get the entire database of features as a numpy array.
        """
        return self.features

    def clear_all_features(self):
        """
        Remove all features from the database
        """
        self.features = None

    def set_some_features(self, feature_indices, features):
        """
        This overwrites existing features.  It does not (yet) handle insert, delete, or
        append.

        N.B. as with numpy arrays in general, a single feature index and a list with one
        feature index will have different behaviors, in that the former drops an array
        dimension but the latter does not.  That is, when changing one feature use
        feature_indices=5, features=[3,1.2,4] OR use feature_indices=[5],
        features=[[3,1.2,4]].
        """
        self.features[feature_indices] = features

    def get_some_features(self, feature_indices):
        """
        This fetches a subset of existing features.

        N.B. as with numpy arrays in general, a single feature index and a list with one
        feature index will have different behaviors, in that the former drops an array
        dimension but the latter does not.  That is, use of feature_indices=5 returns
        [3,1.2,4] but use of feature_indices=[5] returns [[3,1.2,4]].
        """
        return self.features[feature_indices]

    """
    Handle the label(s) for each stored entity.
    """

    def read_all_labels_from_h5py(self, filename, data_name="labels"):
        """
        Read the entire database of labels from a supplied h5py file.
        """
        with h5.File(filename) as ds:
            self.labels = np.array(ds[data_name])

    def write_all_labels_to_h5py(self, filename, data_name="labels"):
        """
        Write the entire database of labels to a h5py file.
        """
        with h5.File(filename, "w") as f:
            f.create_dataset(data_name, self.labels.shape, data=self.labels)

    def set_all_labels(self, labels):
        """
        Set the entire database of labels from a supplied numpy array.
        """
        if isinstance(labels, np.ndarray) and 1 <= len(labels.shape) <= 2:
            self.labels = labels
        else:
            raise ValueError(
                "The argument to set_all_labels must be a 1-dimensional or "
                "2-dimensional numpy ndarray."
            )

    def get_all_labels(self):
        """
        Get the entire database of labels as a numpy array.
        """
        return self.labels

    def clear_all_labels(self):
        """
        Remove all labels from the database
        """
        self.labels = None

    def set_some_labels(self, label_indices, labels):
        """
        This overwrites existing labels.  It does not (yet) handle insert, delete, or
        append.

        N.B. as with numpy arrays in general, a single label index and a list with one
        label index will have different behaviors, in that the former drops an array
        dimension but the latter does not.  That is, when changing one label use
        label_indices=5, labels=[3,1,4] OR use label_indices=[5],
        labels=[[3,1,4]].
        """
        self.labels[label_indices] = labels

    def get_some_labels(self, label_indices):
        """
        This fetches a subset of existing labels.

        N.B. as with numpy arrays in general, a single label index and a list with one
        label index will have different behaviors, in that the former drops an array
        dimension but the latter does not.  That is, use of label_indices=5 returns
        [3,1,4] but use of label_indices=[5] returns [[3,1,4]].
        """
        return self.labels[
            label_indices,
        ]

    """
    Handle the dictionary of supplemental information for each stored entity.
    """

    def set_all_dictionaries(self, dictionaries):
        """
        Set the entire database of dictionaries -- one per feature vector -- from a
        supplied list of Python dict objects.

        N.B. if we were handed a (read-only) tuple then we cannot change a subset of
        them.  If this functionality is needed then supply `list(dictionaries)` to this
        function.
        """
        if (
            isinstance(dictionaries, (tuple, list))
            and len(dictionaries) > 0
            and isinstance(dictionaries[0], dict)
        ):
            self.dictionaries = dictionaries
        else:
            raise ValueError(
                "The argument to set_all_dictionaries must be a non-empty list or "
                "tuple of dictionaries."
            )

    def get_all_dictionaries(self):
        """
        Get the entire database of dictionaries as a list (or tuple) of Python dict
        objects.
        """
        return self.dictionaries

    def clear_all_dictionaries(self):
        """
        Remove all dictionaries from the database
        """
        self.dictionaries = None

    def set_some_dictionaries(self, dictionary_indices, dictionaries):
        """
        This overwrites existing dictionaries.  It does not (yet) handle insert, delete,
        or append.

        N.B. as with Python lists in general, a single dictionary index and a list with
        one dictionary index will have different behaviors, in that the former drops an
        array dimension but the latter does not.  That is, when changing one dictionary
        use dictionary_indices=5, dictionaries={'a': 1, 'b': 2} OR use
        dictionary_indices=[5], dictionaries=[{'a': 1, 'b': 2}].

        N.B. This will fail if the intially supplied value for dictionaries was a
        *tuple* of dictionaries rather than a *list* of dictionaries, because tuples are
        read-only.
        """
        if isinstance(self.dictionaries, tuple):
            raise ValueError(
                "set_some_dictionaries cannot be used unless the initially supplied "
                "dictionaries are supplied in a list instead of a tuple"
            )

        self.dictionaries[dictionary_indices] = dictionaries

    def get_some_dictionaries(self, dictionary_indices):
        """
        This fetches a subset of existing dictionaries.

        N.B. as with numpy arrays in general, a single dictionary index and a list with
        one dictionary index will have different behaviors, in that the former drops an
        array dimension but the latter does not.  That is, use of dictionary_indices=5
        returns {'a': 1, 'b': 2} but use of dictionary_indices=[5] returns [{'a': 1,
        'b': 2}].
        """
        return self.dictionaries[dictionary_indices]

    """
    Handle operations not specific to the feature vectors, labels, or dictionaries of
    supplemental information of each stored entity.
    """

    def set_all_label_definitions(self, label_definitions):
        """
        Parameters
        ----------
        label_definitions: dict
            The argument is, for example, label_definitions = {
                np.nan: {"description": "unlabeled", "color": "#FFFF00"},
                1:      {"description": "necrotic",  "color": "#FF0000"},
            }
        """

        if isinstance(label_definitions, list) and all(
            [isinstance(e, dict) for e in label_definitions]
        ):
            self.label_definitions = label_definitions
        else:
            raise ValueError(
                "The argument to set_all_label_definitions must be a list of Python "
                "dict objects"
            )

    def get_all_label_definitions(self):
        """
        Returns
        -------
        label_definitions: dict
            See set_all_label_definitions for the dict format.
        """
        return self.label_definitions

    def check_data_consistency(self):
        # Check whether among features, labels, and dictionaries that were supplied, are
        # they for the same number of entities?
        features_length = 0 if self.features is None else self.features.shape[0]
        labels_length = 0 if self.labels is None else self.labels.shape[0]
        dictionaries_length = 0 if self.dictionaries is None else len(self.dictionaries)
        # Eliminate duplicates
        all_lengths = set([features_length, labels_length, dictionaries_length])
        lengths_test = len(all_lengths) == 1 or (
            len(all_lengths) == 2 and 0 in all_lengths
        )

        # Check whether among labels and label_definitions that were supplied, are
        # they for the same number of kinds of labels?
        labels_width = (
            0
            if self.labels is None
            else 1
            if len(self.labels.shape) == 1
            else self.labels.shape[1]
        )
        label_definitions_width = (
            0 if self.label_definitions is None else len(self.label_definitions)
        )
        all_widths = set([labels_width, label_definitions_width])
        widths_test = len(all_widths) == 1 or (len(all_widths) == 2 and 0 in all_widths)

        ## Check whether every supplied label category has a definition.
        definitions_test = (
            not widths_test  # already failed previous test
            or labels_width == 0  # nothing to compare
            or label_definitions_width == 0  # nothing to compare
            or (
                len(self.labels.shape) == 1  # 1-dimensional numpy array
                and len(
                    set(self.labels).difference(set(self.label_definitions[0].keys()))
                )
                == 0
            )
            or (
                len(self.labels.shape) == 2  # 2-dimensional numpy array
                and all(
                    [
                        len(
                            set(self.labels[:, col]).difference(
                                set(self.label_definitions[col].keys())
                            )
                        )
                        == 0
                        for col in range(self.labels.shape[1])
                    ]
                )
            )
        )

        mesgs = list()
        if not lengths_test:
            mesgs += [
                f"height(features) = {features_length}, "
                f"height(labels) = {labels_length}, and "
                f"height(dictionaries) = {dictionaries_length} do not match."
            ]
        if not widths_test:
            mesgs += [
                f"width(labels) = {labels_width} and "
                f"width(label_definitions) = {label_definitions_width} do not match."
            ]
        if not definitions_test:
            mesgs += [f"Some labels have categories without definitions."]
        if len(mesgs) > 0:
            raise ValueError("\n".join(mesgs))

        return len(mesgs) == 0

max_stars_repo_name: DigitalSlideArchive/ALBench

max_issues_repo_name: DigitalSlideArchive/ALBench

max_forks_repo_name: DigitalSlideArchive/ALBench


text:
import torch
import torch.nn as nn
import numpy as np
from marl.rlcore.distributions import Categorical
import torch.nn.functional as F
import math

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1 or classname.find('Linear') != -1:
        nn.init.orthogonal_(m.weight.data)
        if m.bias is not None:
            m.bias.data.fill_(0)

class MPNN(nn.Module):
    def __init__(self, action_space, num_agents, num_entities, input_size=16, hidden_dim=128, embed_dim=None,
                 pos_index=2, norm_in=False, nonlin=nn.ReLU, n_heads=1, mask_dist=None, entity_mp=False, num_actions=4):
        super().__init__()

        self.h_dim = hidden_dim
        self.nonlin = nonlin
        self.num_agents = num_agents # number of agents
        self.num_entities = num_entities # number of entities
        self.K = 3 # message passing rounds
        self.embed_dim = self.h_dim if embed_dim is None else embed_dim
        self.n_heads = n_heads
        self.mask_dist = mask_dist
        self.input_size = input_size
        self.entity_mp = entity_mp
        # this index must be from the beginning of observation vector
        self.pos_index = pos_index

        self.encoder = nn.Sequential(nn.Linear(self.input_size,self.h_dim),
                                     self.nonlin(inplace=True))

        self.messages = MultiHeadAttention(n_heads=self.n_heads,input_dim=self.h_dim,embed_dim=self.embed_dim)

        self.update = nn.Sequential(nn.Linear(self.h_dim+self.embed_dim,self.h_dim),
                                    self.nonlin(inplace=True))

        self.value_head = nn.Sequential(nn.Linear(self.h_dim, self.h_dim),
                                        self.nonlin(inplace=True),
                                        nn.Linear(self.h_dim,1))

        self.policy_head = nn.Sequential(nn.Linear(self.h_dim, self.h_dim),
                                         self.nonlin(inplace=True))

        if self.entity_mp:
            self.entity_encoder = nn.Sequential(nn.Linear(2,self.h_dim),
                                                self.nonlin(inplace=True))

            self.entity_messages = MultiHeadAttention(n_heads=1,input_dim=self.h_dim,embed_dim=self.embed_dim)

            self.entity_update = nn.Sequential(nn.Linear(self.h_dim+self.embed_dim,self.h_dim),
                                               self.nonlin(inplace=True))

        self.dist = Categorical(self.h_dim, num_actions)

        self.is_recurrent = False

        if norm_in:
            self.in_fn = nn.BatchNorm1d(self.input_size)
            self.in_fn.weight.data.fill_(1)
            self.in_fn.bias.data.fill_(0)
        else:
            self.in_fn = lambda x: x
        self.apply(weights_init)

        self.attn_mat = np.ones((num_agents, num_agents))

        self.dropout_mask = None

    def calculate_mask(self, inp):
        # inp is batch_size x self.input_size where batch_size is num_processes*num_agents

        pos = inp[:, self.pos_index:self.pos_index+2]
        bsz = inp.size(0)//self.num_agents
        mask = torch.full(size=(bsz,self.num_agents,self.num_agents),fill_value=0,dtype=torch.uint8)
        #print(self.mask_dist)
        if self.mask_dist is not None and self.mask_dist > 0:
            for i in range(1,self.num_agents):
                shifted = torch.roll(pos,-bsz*i,0)
                dists = torch.norm(pos-shifted,dim=1)
                restrict = dists > self.mask_dist
                for x in range(self.num_agents):
                    if abs(x-i)<=4 or abs(x-i)!=0:
                        mask[:,x,(x+i)%self.num_agents].copy_(torch.tensor([1]))
                    #mask[:,x,(x+i)%self.num_agents].copy_(restrict[bsz*x:bsz*(x+1)])

        elif self.mask_dist is not None and self.mask_dist <0:# -10:
           if self.dropout_mask is None or bsz!=self.dropout_mask.shape[0] or np.random.random_sample() < 0.1: # sample new dropout mask
               temp = torch.rand(mask.size()) > 0.85
               temp.diagonal(dim1=1,dim2=2).fill_(0)
               self.dropout_mask = (temp+temp.transpose(1,2))!=0
           mask.copy_(self.dropout_mask)

        return mask

    def _fwd(self, inp):
        # inp should be (batch_size,input_size)
        # inp - {iden, vel(2), pos(2), entities(...)}
        agent_inp = inp[:,:self.input_size]
        mask = self.calculate_mask(agent_inp) # shape <batch_size/N,N,N> with 0 for comm allowed, 1 for restricted

        h = self.encoder(agent_inp) # should be (batch_size,self.h_dim)
        if self.entity_mp:
            landmark_inp = inp[:,self.input_size:] # x,y pos of landmarks wrt agents
            # should be (batch_size,self.num_entities,self.h_dim)
            he = self.entity_encoder(landmark_inp.contiguous().view(-1,2)).view(-1,self.num_entities,self.h_dim)
            entity_message = self.entity_messages(h.unsqueeze(1),he).squeeze(1) # should be (batch_size,self.h_dim)
            h = self.entity_update(torch.cat((h,entity_message),1)) # should be (batch_size,self.h_dim)

        h = h.view(self.num_agents,-1,self.h_dim).transpose(0,1) # should be (batch_size/N,N,self.h_dim)

        for k in range(self.K):
            m, attn = self.messages(h, mask=mask, return_attn=True) # should be <batch_size/N,N,self.embed_dim>
            h = self.update(torch.cat((h,m),2)) # should be <batch_size/N,N,self.h_dim>
        h = h.transpose(0,1).contiguous().view(-1,self.h_dim)

        self.attn_mat = attn.squeeze().detach().cpu().numpy()
        return h # should be <batch_size, self.h_dim> again

    def forward(self, inp, state, mask=None):
        raise NotImplementedError

    def _value(self, x):
        return self.value_head(x)

    def _policy(self, x):
        return self.policy_head(x)

    def act(self, inp, state, mask=None, deterministic=False):
        x = self._fwd(inp)
        value = self._value(x)
        dist = self.dist(self._policy(x))
        if deterministic:
            action = dist.mode()
        else:
            action = dist.sample()
        action_log_probs = dist.log_probs(action).view(-1,1)
        return value,action,action_log_probs,state

    def evaluate_actions(self, inp, state, mask, action):
        x = self._fwd(inp)

        value = self._value(x)
        dist = self.dist(self._policy(x))
        action_log_probs = dist.log_probs(action)
        dist_entropy = dist.entropy().mean()
        return value,action_log_probs,dist_entropy,state

    def get_value(self, inp, state, mask):
        x = self._fwd(inp)
        value = self._value(x)
        return value

class MultiHeadAttention(nn.Module):
    # taken from https://github.com/wouterkool/attention-tsp/blob/master/graph_encoder.py
    def __init__(
            self,
            n_heads,
            input_dim,
            embed_dim=None,
            val_dim=None,
            key_dim=None
    ):
        super(MultiHeadAttention, self).__init__()

        if val_dim is None:
            assert embed_dim is not None, "Provide either embed_dim or val_dim"
            val_dim = embed_dim // n_heads
        if key_dim is None:
            key_dim = val_dim

        self.n_heads = n_heads
        self.input_dim = input_dim
        self.embed_dim = embed_dim
        self.val_dim = val_dim
        self.key_dim = key_dim

        self.norm_factor = 1 / math.sqrt(key_dim)  # See Attention is all you need

        self.W_query = nn.Parameter(torch.Tensor(n_heads, input_dim, key_dim))
        self.W_key = nn.Parameter(torch.Tensor(n_heads, input_dim, key_dim))
        self.W_val = nn.Parameter(torch.Tensor(n_heads, input_dim, val_dim))

        if embed_dim is not None:
            self.W_out = nn.Parameter(torch.Tensor(n_heads, key_dim, embed_dim))

        self.init_parameters()

    def init_parameters(self):

        for param in self.parameters():
            stdv = 1. / math.sqrt(param.size(-1))
            param.data.uniform_(-stdv, stdv)

    def forward(self, q, h=None, mask=None, return_attn=False):
        """
        :param q: queries (batch_size, n_query, input_dim)
        :param h: data (batch_size, graph_size, input_dim)
        :param mask: mask (batch_size, n_query, graph_size) or viewable as that (i.e. can be 2 dim if n_query == 1)
        Mask should contain 1 if attention is not possible (i.e. mask is negative adjacency)
        :return:
        """
        if h is None:
            h = q  # compute self-attention

        # h should be (batch_size, graph_size, input_dim)
        batch_size, graph_size, input_dim = h.size()
        n_query = q.size(1)
        assert q.size(0) == batch_size
        assert q.size(2) == input_dim
        assert input_dim == self.input_dim, "Wrong embedding dimension of input"

        hflat = h.contiguous().view(-1, input_dim)
        qflat = q.contiguous().view(-1, input_dim)

        # last dimension can be different for keys and values
        shp = (self.n_heads, batch_size, graph_size, -1)
        shp_q = (self.n_heads, batch_size, n_query, -1)

        # Calculate queries, (n_heads, n_query, graph_size, key/val_size)
        Q = torch.matmul(qflat, self.W_query).view(shp_q)
        # Calculate keys and values (n_heads, batch_size, graph_size, key/val_size)
        K = torch.matmul(hflat, self.W_key).view(shp)
        V = torch.matmul(hflat, self.W_val).view(shp)

        # Calculate compatibility (n_heads, batch_size, n_query, graph_size)
        compatibility = self.norm_factor * torch.matmul(Q, K.transpose(2, 3))
        # Optionally apply mask to prevent attention
        if mask is not None:
            mask = mask.view(1, batch_size, n_query, graph_size).expand_as(compatibility)
            compatibility[mask] = -math.inf

        attn = F.softmax(compatibility, dim=-1)
        #print(attn)
        # If there are nodes with no neighbours then softmax returns nan so we fix them to 0
        if mask is not None:
            attnc = attn.clone()
            attnc[mask] = 0
            attn = attnc

        heads = torch.matmul(attn, V)

        out = torch.mm(
            heads.permute(1, 2, 0, 3).contiguous().view(-1, self.n_heads * self.val_dim),
            self.W_out.view(-1, self.embed_dim)
        ).view(batch_size, n_query, self.embed_dim)

        if return_attn:
            return out, attn
        return out

ext: py

reason: import numpy


text:
from pyqc.gates import *
from .import numpy as np

class Var:
    def __init__(self, value=0, grad=0, coefficient=1, id=-1):
        self.value = value
        self.grad = grad
        self.coef = coefficient
        self.id = id

    @property
    def angle(self):
        return self.value*self.coef

    @property
    def name(self):
        return '%s:var(%s)'%(self.id,round(self.value,3))

def var(value, coefficient):
    res = None
    if not isinstance(value, np.ndarray):
        raise ValueError('type of var must be numpy.ndarray are not ',type(value))
    else:
        res = []
        for i in range(len(value)):
            res.append(Var(value=value[i],coefficient=coefficient[i],id=i))
    return res

class Optimizer:
    def __init__(self, v, lr):
        self.var = v
        self.lr = lr

    def update(self):
        for v in self.var:
            v.value += self.lr*v.grad

class  VariationalQuantumFramework:
    """
    """
    def __init__(self, backend, vqc, cost_Hamiltonian):
        """
        """
        self.backend = backend
        self.vqc = vqc
        self.cost_h = cost_Hamiltonian

    def loss(self):
        """
        """
        loss = 0
        for _,value in self.cost_h.items():
            target = list(value.keys())
            loss += self.backend.getExpectation(target)
        return loss

    def _update_helper(self, gate, target, control):
        if isinstance(gate, OneGate):
            to_array = gate.matrix.H.A.reshape(4)
            self.backend.fullASim.applyOneGate(to_array,target[0],0)
            self.backend.fullASim.applyOneGate(to_array,target[0],1)
        elif gate.name=="CNOT" or gate.name=="CNOT.dag" or gate.name=="CZ" or gate.name=="CZ.dag":
            to_array = gate.cmatrix.A.reshape(4)
            self.backend.fullASim.applyControlOneGate(to_array, target[0], control[0], 0)
            self.backend.fullASim.applyControlOneGate(to_array, target[0], control[0], 1)
        elif gate.name == "Toffili":
            h_array = H.matrix.A.reshape(4)
            s_array = S.matrix.A.reshape(4)
            s_d_array = SDag.matrix.A.reshape(4)
            x_array = X.matrix.A.reshape(4)
            self.backend.fullASim.applyOneGate(h_array,target[0], 0)
            self.backend.fullASim.applyControlOneGate(s_d_array, target[0], control[0], 0)
            self.backend.fullASim.applyControlOneGate(x_array, control[1], control[0], 0)
            self.backend.fullASim.applyControlOneGate(s_array, target[0], control[1], 0)
            self.backend.fullASim.applyControlOneGate(x_array, control[1], control[0], 0)
            self.backend.fullASim.applyControlOneGate(s_d_array, target[0], control[1], 0)
            self.backend.fullASim.applyOneGate(h_array,target[0], 0)
            self.backend.fullASim.applyOneGate(h_array,target[0], 1)
            self.backend.fullASim.applyControlOneGate(s_d_array, target[0], control[0], 1)
            self.backend.fullASim.applyControlOneGate(x_array, control[1], control[0], 1)
            self.backend.fullASim.applyControlOneGate(s_array, target[0], control[1], 1)
            self.backend.fullASim.applyControlOneGate(x_array, control[1], control[0], 1)
            self.backend.fullASim.applyControlOneGate(s_d_array, target[0], control[1], 1)
            self.backend.fullASim.applyOneGate(h_array,target[0], 1)
        elif gate.name == "Toffili.dag":
            h_array = H.matrix.A.reshape(4)
            s_array = S.matrix.A.reshape(4)
            s_d_array = SDag.matrix.A.reshape(4)
            x_array = X.matrix.A.reshape(4)
            self.backend.fullASim.applyOneGate(h_array,target[0], 0)
            self.backend.fullASim.applyControlOneGate(s_array, target[0], control[1], 0)
            self.backend.fullASim.applyControlOneGate(x_array, control[1], control[0], 0)
            self.backend.fullASim.applyControlOneGate(s_d_array, target[0], control[1], 0)
            self.backend.fullASim.applyControlOneGate(x_array, control[1], control[0], 0)
            self.backend.fullASim.applyControlOneGate(s_array, target[0], control[0], 0)
            self.backend.fullASim.applyOneGate(h_array,target[0], 0)
            self.backend.fullASim.applyOneGate(h_array,target[0], 1)
            self.backend.fullASim.applyControlOneGate(s_array, target[0], control[1], 1)
            self.backend.fullASim.applyControlOneGate(x_array, control[1], control[0], 1)
            self.backend.fullASim.applyControlOneGate(s_d_array, target[0], control[1], 1)
            self.backend.fullASim.applyControlOneGate(x_array, control[1], control[0], 1)
            self.backend.fullASim.applyControlOneGate(s_array, target[0], control[0], 1)
            self.backend.fullASim.applyOneGate(h_array,target[0], 1)
        elif gate.name=="Swap" or gate.name=="Swap.dag":
            to_array = X.matrix.A.reshape(4)
            self.backend.fullASim.applyControlOneGate(to_array, target[1], target[0], 0)
            self.backend.fullASim.applyControlOneGate(to_array, target[0], target[1], 0)
            self.backend.fullASim.applyControlOneGate(to_array, target[1], target[0], 0)
            self.backend.fullASim.applyControlOneGate(to_array, target[1], target[0], 1)
            self.backend.fullASim.applyControlOneGate(to_array, target[0], target[1], 1)
            self.backend.fullASim.applyControlOneGate(to_array, target[1], target[0], 1)
        else:
            print(gate.name)
            raise RuntimeError('error')

    def backward(self):
        """
        """
        size = []
        target = []
        to_array = np.array([])
        for _,value in self.cost_h.items():
            for id,gate in value.items():
                target.append(id)
                to_array = np.append(gate.matrix.A.reshape(4), to_array)
            size.append(len(value))
        self.backend.fullASim.grad_helper_init(to_array,target,size,len(size))            #初始化|psi_b>
        layer_id = list(self.vqc.layer_info.keys())
        layer_id.reverse()
        for layer in layer_id:
            item = self.vqc.layer_info[layer]
            theta, op_dict, gate_ops = item['theta'], item['op_dict'], item['gate_ops']
            size = []
            target = []
            to_array = np.array([])
            for _,value in op_dict.items():
                for id,gate in value.items():
                    target.append(id)
                    to_array = np.append(gate.matrix.A.reshape(4), to_array)
                size.append(len(value))
            grad = self.backend.fullASim.grad_helper(to_array,target,size,len(size))     #得到梯度cost_Hamiltonian关于theta_j的梯度
            gate_ops.reverse()
            for gid in gate_ops:
                gate, target, control = self.vqc.qgate_list[gid]
                self._update_helper(gate, target, control)                               #更新|psi_a>、|psi_b>
            theta.grad = grad

max_stars_repo_name: shunzgim/PyQC

max_issues_repo_name: shunzgim/PyQC

max_forks_repo_name: shunzgim/PyQC


text:
import matplotlib.pyplot as plt
from matplotlib import dates
import numpy as np
import os
import sys
from pprint import pprint
from datetime import datetime
from datetime import timedelta
import pickle
import copy
import collections
import calendar
import mysql.connector
import math

#database connection
cnx = mysql.connector.connect(user='root', password='Suresh15', host='localhost', database='black_carbon')
cursor = cnx.cursor()

def get_data(cluster,cluster_interval):
	cursor.execute('''(SELECT
		bin_LL,
		bin_UL,
		mass,
		number,
		volume
		FROM whi_sp2_distr_data
		WHERE
		cluster_interval = %s
		AND	cluster = %s
		)''',
		(6,cluster))

	return cursor.fetchall()

def parse_data(distr_data):
	new_plot_data = []
	total_volume = 0
	for row in distr_data:
		bin_LL = row[0]
		bin_UL = row[1]
		mass  = row[2]
		number  = row[3]
		volume  = row[4]
		total_volume +=  volume
		numb_conc_norm = number/(math.log(bin_UL,10)-math.log(bin_LL,10)) #normalize number
		mass_conc_norm = mass/(math.log(bin_UL,10)-math.log(bin_LL,10)) #normalize number

		new_plot_data.append([(bin_LL+(bin_UL-bin_LL)/2),mass_conc_norm,numb_conc_norm])

	return [new_plot_data,total_volume]

clus_list = ['NPac','SPac','NCan','WPac','BB']
plot_data = {}
for cluster in clus_list:
	distr_data = get_data(cluster, 6)
	distr_results = parse_data(distr_data)
	plot_data[cluster] = distr_results[0]
	tot_volume = distr_results[1]

fig = plt.figure()
ax1 = fig.add_subplot(111)
for cluster in clus_list:
	print cluster
	mass_max_val = max([row[1] for row in plot_data[cluster]])
	numb_max_val = max([row[2] for row in plot_data[cluster]])
	bin =          [row[0] for row in plot_data[cluster]]
	mass_distr =   [row[1] for row in plot_data[cluster]]
	number_distr = [row[2]/numb_max_val for row in plot_data[cluster]]

	ax1.plot(bin,mass_distr,marker = 'o',label = cluster)
	#ax1.plot(bin,number_distr,marker = 's',label = cluster)
	#ax1.set_xlim(0,300)
	#ax1.set_ylim(0,60)
	ax1.set_ylabel('mass conc')

plt.legend(loc=4)
plt.show()

cnx.close()


text:
import csv
import numpy as np
from typing import Dict, List
from PyQt5.QtGui import QImage, QColor

import src.core.config as config

def parse(path: str, num_classes: int) -> Dict[int, List[np.ndarray]]:
    with open(path, newline='\n') as csv_file:
        data_set = prepare_data_set_dict(num_classes)
        data_reader = csv.reader(csv_file, delimiter=',')
        i = 0
        for row in data_reader:
            set_label = int(row[0])

            np_set = np.asarray(row[1:], dtype=np.float32)
            np_set = np.reshape(np_set, (-1, 28))

            data_set[set_label].append(np_set)

            i += 1
            if i % 1000 == 0:
                print('csv string parsed: ', i)

    return data_set

def prepare_data_set_dict(num_classes: int) -> Dict[int, list]:
    data_set_dict = {}
    for i in range(0, num_classes):
        data_set_dict[i] = []
    return data_set_dict

def dump_image(matrix: np.ndarray, number: int) -> None:
    image = QImage(28, 28, QImage.Format_RGB32)
    for i in range(0, matrix.shape[0]):
        for j in range(0, matrix.shape[1]):
            grayscale = matrix[i][j]
            image.setPixelColor(i, j, QColor(grayscale, grayscale, grayscale))
    image.save('dump/img' + str(number) + '.png')


text:
"""
=============================
Computing ICS data by tttrlib
=============================

Demonstrate the use of the tttrlib ICS features and compare to
numpy ICS implementation use CLSMImage as an input
"""
import tttrlib
import numpy as np
import pylab as p
import matplotlib.patches

data = tttrlib.TTTR('../../tttr-data/imaging/leica/sp5/LSM_1.ptu', 'PTU')
reading_parameter = {
    "tttr_data": data,
    "reading_routine": 'SP5',
    "channels": [0],
    "fill": True
}
clsm = tttrlib.CLSMImage(**reading_parameter)

# # specifies a selection on the image
x_range = [120, 152]
y_range = [100, 132]

# if nothing or the following ranges are specified
# an ICS of the entire image is computed
# x_range = [0, -1]
# y_range = [0, -1]

# the ICS routines computes the correlation between pairs or
# images. This way the CCF between different images can be computed, e.g.,
# to compute the correlation between different frames.
# If no image pairs are specified the ACF of the images is computed.
frames = np.arange(0, clsm.n_frames)
frame_shift = 1

ics_parameter = {
    'tttr_data': data,
    'x_range': x_range,
    'y_range': y_range,
    'clsm': clsm,
    'frames_index_pairs': list(
        zip(
            frames.tolist(),
            np.roll(frames, frame_shift).tolist()
        )
    ),
    'subtract_average': "stack"
}
ics = tttrlib.CLSMImage.compute_ics(**ics_parameter)

img = clsm.intensity
ics_mean = ics.mean(axis=0)
ics_std = ics.std(axis=0)

fig, ax = p.subplots(ncols=2)
ax[0].imshow(img.sum(axis=0), cmap='inferno')
rect = matplotlib.patches.Rectangle(
    xy=(x_range[0], y_range[0]),
    width=x_range[1]-x_range[0],
    height=y_range[1]-y_range[0],
    edgecolor='r',
    facecolor="none"
)
ax[0].add_patch(rect)
ax[1].imshow(
    np.fft.fftshift(ics_mean),
    cmap='inferno',
    vmax=0.05
)
p.show()

max_stars_repo_name: Fluorescence-Tools/tttrlib

max_issues_repo_name: Fluorescence-Tools/tttrlib

max_forks_repo_name: Fluorescence-Tools/tttrlib


text:
import numpy as np

def makedata(n, miu, sigma, theta):
    a = 1
    b = 0.2
    th=np.linspace(0, 10, int(n / 10))
    x = np.expand_dims(np.tile(a * np.exp(b * th) * np.cos(th), 10) + np.random.normal(miu, sigma, (n)), axis=0)
    y = np.expand_dims(np.tile(a * np.exp(b * th) * np.sin(th), 10) + np.random.normal(miu, sigma, (n)), axis=0)
    z = np.expand_dims(np.array(np.linspace(-1.5, 1.5, n) + np.random.normal(miu, sigma, (n))), axis=0)
    mo = np.concatenate([np.concatenate([np.concatenate([x, y], axis=0), z], axis=0), np.ones((1, n))], axis=0)
    ro_x = np.zeros((4, 4))
    ro_x[0][0] = 1.0
    ro_x[1][1] = np.cos(theta)
    ro_x[1][2] = -np.sin(theta)
    ro_x[2][1] = np.sin(theta)
    ro_x[2][2] = np.cos(theta)
    ro_x[3][3] = 1.0
    ro_y = np.zeros((4, 4))
    ro_y[0][0] = np.cos(theta)
    ro_y[1][1] = 1
    ro_y[0][2] = -np.sin(theta)
    ro_y[2][0] = np.sin(theta)
    ro_y[2][2] = np.cos(theta)
    ro_y[3][3] = 1.0
    mo = np.dot(ro_x, mo)
    mo = np.dot(ro_y, mo)
    return np.delete(mo, -1, axis=0).T

max_stars_repo_name: MartrixG/machine-learning

max_issues_repo_name: MartrixG/machine-learning

max_forks_repo_name: MartrixG/machine-learning


text:
# -*- coding:utf-8 -*-
# Copyright (c) 2022 https://github.com/WangXuan95
#
# 这是一个灰度图像的 JPEG 压缩算法。
# 它不调用除了 numpy 以外的任何库，完整而简洁地展示了 JPEG 算法的原理。
#
# 另外为了进行测试，它还调用了 PIL.Image 库用来读取待压缩的原始文件，但在JPEG 压缩算法中没用 PIL.Image 库。
#
# 你可以用它来进行图像图像压缩，比如，运行以下命令可以把 image.pgm (原始像素文件) 压缩成 image.jpg (JPEG压缩文件) 。
#     python JpegEncoder.py image.pgm image.jpg
#
# Note: 可以用Windows的图片查看器打开 .jpg 文件，来验证压缩算法的正确性
#

import sys
import numpy as np
from PIL.Image import open as imgopen

class BitstreamWriter():
    def __init__(self):
        self.bitpos = 7
        self.byte = 0x00
        self.stream = bytearray()
    def writebyte(self, _byte):
        self.stream.append(_byte)
    def writebytes(self, _bytes):
        self.stream += _bytes
    def writeword(self, word):
        self.writebyte((word>>8) & 0xFF)
        self.writebyte((word>>0) & 0xFF)
    def writebits(self, _value, _bitlen):
        for ii in range(_bitlen-1, -1, -1):
            if _value & (1<<ii):
                self.byte |= (1<<self.bitpos)
            self.bitpos -= 1
            if self.bitpos<0:
                self.writebyte(self.byte)
                self.bitpos = 7
                self.byte = 0x00
    def flush(self):
        self.writebyte(self.byte)
        self.bitpos = 7
        self.byte = 0x00
    def get(self):
        return self.stream

dct_mat = np.matrix( [
            [ 32, 32, 32, 32, 32, 32, 32, 32],
            [ 44, 38, 25,  9, -9,-25,-38,-44],
            [ 42, 17,-17,-42,-42,-17, 17, 42],
            [ 38, -9,-44,-25, 25, 44,  9,-38],
            [ 32,-32,-32, 32, 32,-32,-32, 32],
            [ 25,-44,  9, 38,-38, -9, 44,-25],
            [ 17,-42, 42,-17,-17, 42,-42, 17],
            [  9,-25, 38,-44, 44,-38, 25, -9] ], dtype = np.int32 )

zig_idxs = np.array( [
            [ 0, 1, 5, 6,14,15,27,28],
            [ 2, 4, 7,13,16,26,29,42],
            [ 3, 8,12,17,25,30,41,43],
            [ 9,11,18,24,31,40,44,53],
            [10,19,23,32,39,45,52,54],
            [20,22,33,38,46,51,55,60],
            [21,34,37,47,50,56,59,61],
            [35,36,48,49,57,58,62,63]  ], dtype = np.int32 )

def shift_round_clip(x):
    y = np.int8(x>>16)
    if x>>15 & 0x1:
        y = y + 1
    if y>63:
        y = 63
    elif y<-63:
        y = -63
    return y

def dct_quant_zig(tile):  # input tile must be (8*8)
    tile = np.matrix(tile, dtype=np.int32)
    zig_vect = np.zeros((64,), dtype=np.int8)
    dct_tile = ( dct_mat * tile * dct_mat.T )
    for i in range(8):
        for j in range(8):
            pos = zig_idxs[i][j]
            quant_level = (1 if pos==0 else pos//16)
            zig_vect[pos] = shift_round_clip( dct_tile[i,j] >> quant_level )
    return zig_vect

def get_code(val):
    absval = val if val>=0 else -val
    length = 0
    while absval:
        absval >>= 1
        length += 1
    code = val if val>=0 else (val-1)
    return length, code

def bit_encoding(stream_writer, zig_vect):
    zero_cnt = 0
    for ii, val in enumerate(zig_vect):
        length, code = get_code(val)
        zero_cnt += 0 if (ii==0 or val!=0) else 1
        if ii==0 or val!=0 or zero_cnt>=16 :
            stream_writer.writebits( zero_cnt&0x0f, 5 )
            stream_writer.writebits( length-1, 3 )
            stream_writer.writebits( code, length )
            zero_cnt = 0
        elif ii==63:
            stream_writer.writebits( 0x0f, 8 )

def jpeg_encoding(img_map):   # img_map must be a 2-dim numpy array, and has a height and width which can divide 8
    h, w = img_map.shape
    JpegStreamWriter = BitstreamWriter()
    JpegStreamWriter.writebytes(b'\xff\xd8\xff\xe0\x00\x10\x4a\x46\x49\x46\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00\xff\xdb\x00\x43\x00\x10\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x40\x40\x40\x40\x40\x40\x40\x40\x40\x40\x40\x40\x40\x40\x40\x40\xff\xc0\x00\x0b\x08')
    JpegStreamWriter.writeword(h)
    JpegStreamWriter.writeword(w)
    JpegStreamWriter.writebytes(b'\x01\x01\x11\x00\xff\xc4\x00\xab\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x00\x10\x00\x00\x00\x00\x00\x00\x00\x7f\x00\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\xf0\x11\x12\x13\x14\x15\x16\x17\x00\x21\x22\x23\x24\x25\x26\x27\x28\x31\x32\x33\x34\x35\x36\x37\x38\x41\x42\x43\x44\x45\x46\x47\x48\x51\x52\x53\x54\x55\x56\x57\x58\x61\x62\x63\x64\x65\x66\x67\x68\x71\x72\x73\x74\x75\x76\x77\x78\x81\x82\x83\x84\x85\x86\x87\x88\x91\x92\x93\x94\x95\x96\x97\x98\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xff\xda\x00\x08\x01\x01\x00\x00\x3f\x00')
    dc_prev = 0
    for yblock in range(0, h, 8):
        for xblock in range(0, w, 8):
            tile = np.array(img_map[yblock:yblock+8,xblock:xblock+8], dtype=np.int32) - 128
            zig_vect = dct_quant_zig(tile)
            zig_vect[0], dc_prev = zig_vect[0] - dc_prev, zig_vect[0]
            bit_encoding(JpegStreamWriter, zig_vect)
    JpegStreamWriter.flush()
    JpegStreamWriter.writebytes(b'\xFF\xD9')
    return JpegStreamWriter.get()

# this main program reads a image file, get its raw pixel map,
# using jpeg_encoding() to compress it to jpeg stream (bytearray type),
# and write it to .jpg file
if __name__ == '__main__':

    try:
        jpg_name   = sys.argv[2]                # get output .jpg file name from command line argument 2
        img_object = imgopen(sys.argv[1])       # get input image file name from command line argument 1, and open it as img_object
    except:
        print('  Usage:\n       python %s <input-image-file> <output-jpg-file>' % (sys.argv[0],) )
        exit(-1)

    # convert img_object to monochrome (grayscale), if it is not.
    if img_object.mode != 'L':
        print("  warning: input image is not monochrome (grayscale), converting to monochrome...")
        img_object = img_object.convert('L')

    # convert img_object to numpy 2-D array
    img_map = np.asarray(img_object)

    # 检查输入图片是否满足要求：是2维数组（即灰度图像）
    if img_map.ndim != 2:
        print("  error: image map's dimision must be 2")

    # 检查输入图片是否满足要求：每个像素占1字节（即像素深度=256）
    if img_map.dtype != np.uint8:
        print("  error: image depth must be 256")

    # 对图像 img_map 进行裁剪，让 width 和 height 都是8的倍数，这是本 JpegEncoder 的要求
    # 虽然 JPEG 算法支持压缩长宽不是8的倍数的图片，但我没有实现
    hraw, wraw = img_map.shape
    hcut, wcut = 8*(hraw//8), 8*(wraw//8)
    if hcut != hraw:
        print("  warning: height is %d, cut to %d" % (hraw, hcut))
    if hcut == 0:
        print("  error: height is cut to 0")
        exit(-1)
    if wcut != wraw:
        print("  warning: width  is %d, cut to %d" % (wraw, wcut))
    if wcut == 0:
        print("  error: width  is cut to 0")
        exit(-1)
    img_map = img_map[:hcut, :wcut]

    # 调用 jpeg_encoding() 进行图像压缩，得到 jpeg_stream （是 bytearray 类型）
    jpeg_stream = jpeg_encoding( img_map )

    # 打印压缩率等信息
    print("  origin=%dPixels   jpg-size=%dB   compress-ratio=%.2f" % (hcut*wcut, len(jpeg_stream), hcut*wcut/len(jpeg_stream)) )

    # 写入 .jpg 文件
    open(jpg_name, 'wb').write(jpeg_stream)


text:
from keras.models import Sequential
from keras.layers import Conv2D, Conv2DTranspose, MaxPooling2D, Flatten, Dense, Reshape, ReLU
from keras.utils import plot_model
import numpy as np
from sklearn.feature_extraction import image
from modalities.mimshow import compare

def __build_network(in_dims, out_dims, params):
    model = Sequential()

    model.add(Conv2D(2 * in_dims, kernel_size=5, activation='relu',
                     input_shape=params["patch_size"] + (in_dims,)))
    model.add(MaxPooling2D())
    model.add(Conv2D(in_dims, kernel_size=3, activation='relu'))
    model.add(MaxPooling2D())
    model.add(Conv2D(in_dims, kernel_size=3, activation='relu'))
    model.add(MaxPooling2D())
    model.add(Flatten())
    model.add(Dense(25))
    inverse_shape = tuple(int(t/np.power(2, 3)) for t in params["patch_size"])
    model.add(Dense(np.prod(inverse_shape) * out_dims))
    model.add(ReLU(1))
    model.add(Reshape(inverse_shape + (out_dims,)))
    model.add(Conv2DTranspose(4 * out_dims, kernel_size=3, strides=(2, 2), padding="same"))
    model.add(Conv2DTranspose(2 * out_dims, kernel_size=3, strides=(2, 2), padding="same"))
    model.add(Conv2DTranspose(out_dims, kernel_size=5, strides=(2, 2), padding="same"))

    if "print" in params and params["print"]:
        plot_model(model, to_file='model.png', show_shapes=True)
    return model

def information_gain(input, target, params=None):
    """
    Method build a CNN according to params. CNN will be composed of several convolutional layers which should convert
    input hyperspectral cube into target hyperspectral cube.
    :param input: hyperspectral cube of input intensities
    :param target: hyperspectral cube of target intensities
    :param params: parameters for building CNN
    :return: tuple
        diff = difference between target and approximation,
        approx = approximated target from input,
        model = builded (and trained) CNN
    """

    if params is None:
        params = {
            "batch": 64,
            "patch_size": (64, 64)
        }

    patches = image.extract_patches_2d(input, params["patch_size"], max_patches=500, random_state=1234)
    targets = image.extract_patches_2d(target, params["patch_size"], max_patches=500, random_state=1234)

    model = __build_network(input.shape[2], target.shape[2], params)
    model.compile(optimizer="adam", metrics=['accuracy'], loss="mean_squared_error")
    model.fit(patches, targets, batch_size=params["batch"], epochs=10)

    approx_patches = model.predict(patches, batch_size=params["batch"])
    approx = image.reconstruct_from_patches_2d(approx_patches, target.shape)
    diff = target - approx
    return diff, approx, model

if __name__ == "__main__":
    # Import all necessary libraries
    import modalities.dir_dataset as dataset
    import cv2
    import matplotlib.pyplot as plt
    import ig
    import rescale_range as rr
    import modalities.mimshow as mplot
    import modalities.stats as stats

    np.set_printoptions(suppress=True)

    vis = np.asarray(cv2.cvtColor(cv2.imread("/Users/gimli/Qsync/datasets/Suicide_Of_Saul/outputs/visw.png"),
                                  cv2.COLOR_BGR2HSV)).astype(np.float32) / 255.0
    saul, saul_metadata = dataset.load_all_images("/Users/gimli/Qsync/datasets/Suicide_Of_Saul/total_per_tif")

    params = {
        "batch": 64,
        "patch_size": (20, 20)
    }

    diff, approx, model = information_gain(saul[0], vis, params)
    plt.imshow(vis)
    plt.show()

    plt.imshow(rr.rescale_range(approx) * 255)
    plt.show()

max_stars_repo_name: gimlidc/ig-cnn

max_issues_repo_name: gimlidc/ig-cnn

max_forks_repo_name: gimlidc/ig-cnn


text:
import sys
import os
# import pandas as pd
sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
import numpy as np
from collections import defaultdict
import sys
sys.path.append("..")
# import utils.FastFinder
from utils.RepeatFinder import call_repeatFinder

class seq_assess_repeat(object):
    def __init__(self, sequence=None, begin=0):
        # defaults
        self.rc = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}
        self.sequence = sequence.upper()
        self.begin = begin
        self.size = len(sequence)
        if sequence:
            self.load(sequence)

    def load(self, sequence):
        # input validation
        try:
            if 'U' in sequence.seq or 'u' in sequence.seq:
                # print(sequence.seq
                sequence = sequence.back_transcribe()
            sequence1 = str(sequence.seq).upper()
            sequence2 = str(sequence.seq.reverse_complement()).upper()

        except:
            try:
                if 'U' in sequence or 'u' in sequence:
                    # print(sequence
                    sequence = sequence.back_transcribe()
                sequence1 = str(sequence).upper()
                sequence2 = str(sequence.reverse_complement()).upper()
            except:
                try:
                    sequence1 = str(sequence)
                    sequence1 = sequence1.upper()
                    # print(sequence1
                    bases = list(sequence1)
                    bases = reversed([self.rc.get(base, base) for base in bases])
                    sequence2 = ''.join(bases)
                    # print(sequence2
                except:
                    # print("The sequence provided was neither a Biopython record, Biopython Seq class, nor a valid string with your DNA sequence. Check what you are passing to seq_assess."
                    self.bad_seqs.append(
                        "The sequence provided was neither a Biopython record, Biopython Seq class, nor a valid string with your DNA sequence. Check what you are passing to seq_assess.")

        self.sequence = sequence1.upper()
        self.sequence1 = sequence1.upper()
        self.sequence2 = sequence2.upper()

    def run(self):
        self.find_repeats(RL=8, Rmax=11)

    def find_repeats(self, RL=8,
                     Rmax=11,
                     Rthreshold=[0.69, 0.40, 0.90, 0.60],
                     Rwindow=[70, 500, 60],
                     Rtandem=5):
        ### code for testing the following six rules:
        # A) Any sequence that contains more that 69 % of  bases that are part of repeats 8 bases or longer *   "X= 69 Y=8" 10
        # B) Any single repeated sequence contains more that 40 % of  bases that are part of repeats 8 bases or longer *    "X=40 Y=8"  6
        # C) Any sequence that contains more that 90 % of  bases that are part of repeats 8 bases or longer within a window of 70 length *  "X=90 Y=8 Z=70" 10
        # D) Any sequence that contains more that 60 % of  bases that are part of repeats 8 bases or longer within a window of 500 length for sequences longer than 1000 *  "X=60 Y=8 Z=500 L=1000" 10
        # E) a tandem repeat of length 5 or greater beginning or ending at either termini of the sequence   X=5 10
        # F) repeats more than 12 bp long risk impede synthesis

        ## outputs for Operon Calculator

        # metrics for tracking repeats for ML analysis
        self.tandem = 0  # number of tandem repeats

        self.high_local_density_1 = 0  # number of windows that exceed local density screen (0.90 of sequence is in some way repetitive for window of 70bp)
        self.high_local_density_2 = 0  # number of windows that exceed local density screen (0.60 of sequence is in some way repetitive for window of 500bp)
        self.high_specific_density = 0  # number of windows that exceed specific density screen (0.40 of sequence is repetitive to a given repeat)
        self.high_total_density = 0  # whether sequence exceeds repeat screen (0.69 of sequence is repetitive in some way)

        self.terminal_repeat = 0
        self.repeat9 = 0

        # ============================================
        # calc_repeats_10_15_20_25_40_;arge
        # ============================================
        # RP = FastFinder()
        # self.RepeatDict = RP.get_repeat_dict([self.sequence], RL, verbose=False)

        self.sequence = [self.sequence]
        self.RepeatDict = call_repeatFinder(self.sequence, k_low=9, k_high=100, verb=False)

        total = np.zeros(self.size, dtype=bool)
        total2 = np.zeros(self.size)

        repeat9_numerator = 0

        # ==================================================
        # RepeatDitc --> location_list --> 1 repeat_9 2 terminal_repeat 3 val=[]
        # ==================================================
        for i in self.RepeatDict.keys():
            val = np.zeros(self.size, dtype=bool)
            # ==================================================
            # calculate location_list
            # ==================================================
            for j in self.RepeatDict[i].keys():
                location_list = list(self.RepeatDict[i][j][0])
                # ==================================================
                # calculate repeat 9 mer
                # ==================================================
                for k in location_list:
                    if len(i) > 8:
                        for l in range(8, len(i)):
                            repeat9_numerator += 9
                    # ==================================================
                    # calculate terminal repeat
                    # ==================================================
                    if (self.begin + k) < 60 or (self.begin + k) >= self.size - 60:
                        self.terminal_repeat += 1

                    # ==================================================
                    # calculate val = []
                    # ==================================================
                    for v in range(k, k + len(i)):
                        val[v] = True
            # ============================================
            # calculate ind_fract using val
            # ============================================
            ind_fract = float(sum(val.astype(int))) / self.size
            # ==================================================
            # calculate high_specific_density using ind_fract
            # ==================================================
            if ind_fract > Rthreshold[1]:  # B)
                self.high_specific_density += 1
            # ==================================================
            # calculate total, total2 using val
            # ==================================================
            total += val
            total2 += val.astype(int)

        # ============================================
        # calculate total_fract using total, size
        # calculate high_total_density using total_fract
        # ============================================
        total_fract = float(sum(total.astype(int))) / self.size
        if total_fract > Rthreshold[0]:  # A)
            self.high_total_density += 1

        # ==================================================
        # calculate repeat9 using  repeat9_numerator, size
        # ==================================================
        self.repeat9 = repeat9_numerator / float(self.size)    # gen_9_metric???
        # ============================================
        # calc_local_density
        # ============================================
        for i in range(len(Rwindow) - 1):  # C) & D)
            for j in range(self.size - Rwindow[i]):
                peek = float(sum(total[j:j + Rwindow[i]].astype(int)))
                if peek / Rwindow[i] > Rthreshold[2 + i]:
                    # self.bad_seqs.append("%i bp window starting at %i is too repeat rich. Redesign to remove "%(Rwindow[i],j))
                    if i == 0:
                        self.high_local_density_1 += 1
                    if i == 1:
                        self.high_local_density_2 += 1

        # E)
        # ============================================
        # calculate tandem
        # ============================================
        # split the terminal part and call tandem_subroutine to calculate tandem
        # using tandem_subroutine()
        # using get_tandem_dict()
        # ============================================
        term1 = self.sequence1[:Rwindow[2]]
        self.tandem_subroutine(term1, Rtandem, "five")
        term2 = self.sequence1[len(self.sequence1) - Rwindow[2]:]
        self.tandem_subroutine(term2, Rtandem, "three")

    def tandem_subroutine(self, term, Rtandem, end):
        # subroutine that finds tandem repeats using regex

        run = self.get_tandem_dict(term, Rtandem, 5, 0)
        if len(run) > 0:
            for i in run.keys():
                for j in range(len(run[i])):
                    self.tandem += 1

    def is_tandem(self, seq, start, gap, rulen, max_x):
        x = 0
        for i in range(rulen):
            if seq[start + i] != seq[start + i + gap + rulen]:
                x += 1
            if x > max_x:
                return False
        return True

    def hdist(self, seq, start, gap, rulen, max_x):
        x = 0
        for i in range(rulen):
            # print(len(seq)
            # print(start+i
            # print(start+i+gap+rulen
            if seq[start + i] != seq[start + i + gap + rulen]:
                x += 1
            if x > max_x:
                return False
        self.tandem += 1
        return True

    def get_tandem_dict(self, seq, rulen, max_g, max_x):
        tr_chains = defaultdict(list)
        for i in range(len(seq) - rulen - rulen - 1):
            for g in range(min(len(seq) - i - rulen - max_g, max_g + 1)):
                if self.hdist(seq, i, g, rulen, max_x):
                    tr_chains[i].append(i + rulen + g)
        return tr_chains

if __name__ == "__main__":
    seq = 'AACGCAGTCAGGCACCGTGTATGAAATCTAACAATGCGCTCATCGTCATCCTCGGCACCGTCACCCTGGATGCTGTAGGCATAGGCTTGGTTATGCCGGTGCTCTTCAACGCGGGAATACTGTTAAGATTCATATATTACCTCTCAAATAATTAGCGGAAGGTTGCGATGAAACCGGCAAGGCTCTCTCAAACTGTCGTTGCGCCCGGATGTTGGGGTGAGTTGCCCTGGGGCAATTACTACCGTGAGGCGCTGGAACAGCAGCTAAATCCGTGGTTTGCAAAAATGTATGGTTTCCATTTGCTTAAAATCGGTAATTTAAGCGCAGAAATCAATTCCGAAGCGTGCGCGGTCTCCCATCAGGTGAATGTTTCATCGCAGGGGTCGCCGATGCAGGTTCTGGCCGATCCGCTACATCTTCCTTTTGCAGATAAATCCGTCGATGTTTGTCTGCTGGCGCATACTTTGCCGTGGTGTACCGACCCGCACCGTTTATTGCGGGAAGCCGACCGCGTATTGATTGATGACGGTTGGCTGGTCATTAGTGGATTTAACCCGCTGAGTTTGATGGGGTTACGTAAACTGGTACCCGTTTTACGTAAAACACCGCCCTATAATAGTCGGATGTTTACCCTTATGCGGCAACTGGACTGGCTGTCTTTACTCAATTTCGAAGTGCTACATTATAGCCGTTTTCATGTCTTACCCTGGAAAAAGCAGGGGGGGCGGCTTTTAAATACGCATATCCCGGCGCTGGGCTGTTTACAGCTTATTGTGGCCCGTAAGCGGACCATCCCGCTTACGCTTAATCCGCTGCGACATAATAAAAGTAAAACCCCTATCCGCCAGACCGTTGGCGCCACCCGGCAATATCGCAAACCGGATGGCTAAGCTTCCGCCTGGTAGCCACTATCTTCCTGCGTGGGATTCATTGCCGCGGCGCGCGCCAGTTCATCACAACGCTCGTTTTCGGGATGGCCTGCATGGCCTTTGACCCATACCCATTTGATCTGATGCTGACCTAACGCAGCATCGAGACGTTTCCAGAGATCGACATTTTTTACGGGTTTCTTTTCCGCTGTTTTCCAGCCGCGTTTCTTCCAGTTATGAATCCATTGGGTAATTCCTTGCCGCACATATTGGCTGTCGGTGCTCAATGTTACTTCGCAATGTTCTTTTAACGCTTCAAGCGCGACGATCGCCGCCATCAGTTCCATACGGTTATTGGTGGTCAGCGTGTAACCTTCACTAAACGTTTTTTCATGACCGCGATAGCGTAGGATAGCGCCATAACCACCAGGCCCTGGATTCCCCAGGCAAGAGCCATCGGTGAAAATTTCTACCTGTTTAAGCATCTCTGGTAGACTTCCTGTAATTGAAATCGATAACAAAACGCA'
    assess = seq_assess_repeat(seq, begin=0)
    assess.run()
    print("Processing....")
    print(assess.repeat9)                 # 2.25716104392
    print(assess.high_local_density_1)       # 474
    print(assess.high_local_density_2)       # 711
    print(assess.high_specific_density)      # 0
    print(assess.high_total_density)         # 0
    print(assess.tandem)                     # 2
    print(assess.terminal_repeat)            # 3

    # df = pd.read_csv('TIB_repeat2.csv').iloc[:, 0:8]
    # # print(df)
    # for index, row in df.iterrows():
    #     print(index)
    #     #     print(row)
    #     # print(row[0]) # 'AT..CG'
    #     assess = seq_assess(row[0], begin=0)
    #     assess.run()
    #     # result = calc_repeat(row[0])  # data = {'repeat_25': 0, 'freq_repeat': 4, 'repeat_40': 30, 'repeat_large': 29, 'longest_repeat': 85, 'repeat_20': 11, 'repeat_10': 17, 'repeat_15': 0}
    #     #     print(df.iloc[index,1])
    #     df.iloc[index, 1] = assess.repeat9
    #     df.iloc[index, 2] = assess.high_local_density_1
    #     df.iloc[index, 3] = assess.high_local_density_2
    #     df.iloc[index, 4] = assess.high_specific_density
    #     df.iloc[index, 5] = assess.high_total_density
    #     df.iloc[index, 6] = assess.tandem
    #     df.iloc[index, 7] = assess.terminal_repeat
    # #
    # #
    # df.to_csv("TIB_repeat_feature22.csv")

max_stars_repo_name: JustinDoIt/september

max_issues_repo_name: JustinDoIt/september

max_forks_repo_name: JustinDoIt/september


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jun  4 11:46:32 2020

@author: petrapoklukar
"""

from matplotlib import pyplot as plt
import numpy as np
import h5py

# methods for creating splits for the sample efficiency test
def create_top_splits(images):
    """ Randomly splits the entire 3dshapes dataset into model split,
        task split and holdout split.

    Returns:
        model_indices: list of length model_split
        task_indices: list of length task_split
        holdout_indices: list of length holdout_split
    """
    np.random.seed(2610)
    all_indices = np.arange(len(images))
    np.random.shuffle(all_indices)

    model_split = 250000
    task_split = 150000
    holdout_split = 80000
    model_indices = all_indices[:model_split]
    task_indices = all_indices[model_split:model_split + task_split]
    holdout_indices = all_indices[model_split + task_split:]
    assert(model_indices.shape[0] == model_split)
    assert(task_indices.shape[0] == task_split)
    assert(holdout_indices.shape[0] == holdout_split)
    assert(not np.intersect1d(model_indices, task_indices).size > 0)
    assert(not np.intersect1d(task_indices, holdout_indices).size > 0)
    return model_indices, task_indices, holdout_indices

# load dataset
dataset = h5py.File('datasets/3dshapes.h5', 'r')
print(dataset.keys())
images = dataset['images']  # array shape [480000,64,64,3], uint8 in range(256)
labels = dataset['labels']  # array shape [480000,6], float64
image_shape = images.shape[1:]  # [64,64,3]
label_shape = labels.shape[1:]  # [6]
n_samples = labels.shape[0]  # 10*10*10*8*4*15=480000

_FACTORS_IN_ORDER = ['floor_hue', 'wall_hue', 'object_hue', 'scale', 'shape',
                     'orientation']
_NUM_VALUES_PER_FACTOR = {'floor_hue': 10, 'wall_hue': 10, 'object_hue': 10,
                          'scale': 8, 'shape': 4, 'orientation': 15}

print(images.shape)

images_np=images[:480000]
print(images_np.shape)
model_indices, task_indices, holdout_indices = create_top_splits(images)
print(holdout_indices.shape)
images_np_sel=images_np[holdout_indices]
lables_np_=images_np[holdout_indices]
print(images_np_sel.shape)

# methods for sampling unconditionally/conditionally on a given factor
def get_index(factors):
  """ Converts factors to indices in range(num_data)
  Args:
    factors: np array shape [6,batch_size].
             factors[i]=factors[i,:] takes integer values in
             range(_NUM_VALUES_PER_FACTOR[_FACTORS_IN_ORDER[i]]).

  Returns:
    indices: np array shape [batch_size].
  """
  indices = 0
  base = 1
  for factor, name in reversed(list(enumerate(_FACTORS_IN_ORDER))):
      print(factor, name)
      print(indices, base)
      indices += factors[factor] * base
      base *= _NUM_VALUES_PER_FACTOR[name]
      print(indices, base)
  return indices

def sample_random_batch(batch_size):
  """ Samples a random batch of images.
  Args:
    batch_size: number of images to sample.

  Returns:
    batch: images shape [batch_size,64,64,3].
  """
  indices = np.random.choice(n_samples, batch_size)
  ims = []
  labs = []
  for ind in indices:
    im = images[ind]
    im = np.asarray(im)
    ims.append(im)
    lab = labels[ind]
    lab = np.asarray(lab)
    labs.append(lab)
  ims = np.stack(ims, axis=0)
  ims = ims / 255. # normalise values to range [0,1]
  ims = ims.astype(np.float32)
  return ims.reshape([batch_size, 64, 64, 3]), labs

def sample_batch(batch_size, fixed_factor, fixed_factor_value):
  """ Samples a batch of images with fixed_factor=fixed_factor_value, but with
      the other factors varying randomly.
  Args:
    batch_size: number of images to sample.
    fixed_factor: index of factor that is fixed in range(6).
    fixed_factor_value: integer value of factor that is fixed
      in range(_NUM_VALUES_PER_FACTOR[_FACTORS_IN_ORDER[fixed_factor]]).

  Returns:
    batch: images shape [batch_size,64,64,3]
  """
  factors = np.zeros([len(_FACTORS_IN_ORDER), batch_size],
                     dtype=np.int32)
  for factor, name in enumerate(_FACTORS_IN_ORDER):
    num_choices = _NUM_VALUES_PER_FACTOR[name]
    factors[factor] = np.random.choice(num_choices, batch_size)
  factors[fixed_factor] = fixed_factor_value
  indices = get_index(factors)
  ims = []
  for ind in indices:
    im = images[ind]
    im = np.asarray(im)
    ims.append(im)
  ims = np.stack(ims, axis=0)
  ims = ims / 255. # normalise values to range [0,1]
  ims = ims.astype(np.float32)
  return ims.reshape([batch_size, 64, 64, 3])

def show_images_grid(imgs_, num_images=25):
  ncols = int(np.ceil(num_images**0.5))
  nrows = int(np.ceil(num_images / ncols))
  _, axes = plt.subplots(ncols, nrows, figsize=(nrows * 3, ncols * 3))
  axes = axes.flatten()

  for ax_i, ax in enumerate(axes):
    if ax_i < num_images:
      ax.imshow(imgs_[ax_i], cmap='Greys_r', interpolation='nearest')
      ax.set_xticks([])
      ax.set_yticks([])
    else:
      ax.axis('off')

def create_dataset(indices, savename, images, labels):
    """ Creates a partial 3dshapes datasets from the given set of indices.
    Args:
        indices: list of indices in the original 3dshapes datasets
        name: name of the file to save the samples
    """
    ims, labs = [], []
    counter = 0
    dataset_len = len(indices)
    print("lets go")
    for ind in indices:
        image = np.asarray(images[ind])
        ims.append(image)
        label = np.asarray(labels[ind])
        labs.append(label)
        counter += 1
        if counter % 1 == 0:
            print('Processed {0}/{1}'.format(counter, dataset_len))

    hf = h5py.File('datasets/{0}.h5'.format(savename), 'w')
    hf.create_dataset('images', data=ims)
    hf.create_dataset('labels', data=labs)
    hf.create_dataset('indices', data=indices)
    hf.close()

def plot_label_distribution(indices, max_ind=10000):
    """ Plots the distribution over generative factors obtained from the labels
        list given the list of indices.
    Args:
        indices: list of indices corresponding to samples in the original
            3dshapes datasets
        max_ind: limit the set of indices to consider in the plot
    """
    labels_plot = np.stack([labels[j] for j in indices[:max_ind]])
    for i in range(len(_FACTORS_IN_ORDER)):
        plt.figure(1 + i)
        plt.clf()
        plt.hist(labels_plot[:, i], label=_FACTORS_IN_ORDER[i],
                 bins=_NUM_VALUES_PER_FACTOR[_FACTORS_IN_ORDER[i]])
        plt.show()

def create_top_datasets():
    """ Creates random model split, task split and holdout splits. The splits
        are disjoint.
    """
    model_indices, task_indices, holdout_indices = create_top_splits()
    create_dataset(model_indices, '3dshapes_model_all', images, labels)
    create_dataset(task_indices, '3dshapes_task', images, labels)
    create_dataset(holdout_indices, '3dshapes_holdout', images, labels)

def create_model_splits(filename, savename):
    """ Randomly splits the model split into smaller datasets of different
        sizes.

    Returns:

    """
    dataset_split = h5py.File('datasets/{0}.h5'.format(filename), 'r')
    print(dataset_split.keys())
    images_split = dataset_split['images']
    labels_split = dataset_split['labels']
    np.random.seed(2610)
    all_indices = np.arange(len(images))
    split_sizes = [10000, 50000, 100000, 150000, 250000]

    split_indices = []
    for split_size in split_sizes:
        model_split_indices = all_indices[:split_size]
        assert(model_split_indices.shape[0] == split_size)
        split_indices.append(model_split_indices)

    assert(np.intersect1d(split_indices[0],
                          split_indices[num_splits]).size == split_sizes[0] \
            for num_splits in range(len(split_size)))

    for split_size in range(len(split_sizes)):
        savename = '3dshapes_model_s{0}'.format(split_sizes[split_size])
        create_dataset(split_indices[split_size], savename,
                       images_split, labels_split)
    dataset_split.close()

#img_batch = sample_batch(batch_size, 5, 15)
#show_images_grid(img_batch, num_images=16)

#hf = h5py.File('datasets/3dshapes_test.h5', 'w')
#hf.create_dataset('images', data=random_batch[0])
#hf.create_dataset('labels', data=random_batch[1])
#hf.close()
#
#dataset_test = h5py.File('datasets/3dshapes_test.h5', 'r')
#dataset_test.close()


text:
# this is here as a hack to deal with aliasing between utils.py and utils dir

import numpy as np
import pandas as pd

# XXX TODO support complea numbers (in particular, handle nans)
def allclose(a, b, rtol=1e-5, atol=1e-5, equal_nan=True,
             require_same_dtype=True, require_same_shape=True,
             # case_insensitive=False,
             return_failing_idxs=False):
    """Like numpy allclose, but handles pandas nullable scalar types"""
    if isinstance(a, (tuple, list)):
        a = np.array(a)
    if isinstance(b, (tuple, list)):
        b = np.array(b)

    immediate_fail_ret = ((False, np.array([], dtype=np.int32))
                          if return_failing_idxs else False)

    # shape and dtype checks
    if require_same_shape and (len(a) != len(b)):
        # print("len mismatch!")
        return immediate_fail_ret
    if require_same_shape and (a.shape != b.shape):
        # print("shape mismatch!")
        return immediate_fail_ret
    if require_same_dtype and (a.dtype != b.dtype):
        # print("dtype mismatch!")
        return immediate_fail_ret

    a_numeric = pd.api.types.is_numeric_dtype(a.dtype)
    b_numeric = pd.api.types.is_numeric_dtype(b.dtype)
    if a_numeric != b_numeric:
        # print("failing because only one dtype is numeric!")
        return immediate_fail_ret

    # compare locations / presence of nans
    # print("checking for location/presence of nans...")
    a_mask = pd.notna(a)
    b_mask = pd.notna(b)
    mismatches = a_mask != b_mask
    if np.any(mismatches):
        if return_failing_idxs:
            return False, np.where(mismatches)[0]
        else:
            return False

    # fail immediately if there are nans and this isn't allowed
    if (not equal_nan) and np.any(a_mask):
        if return_failing_idxs:
            return False, np.where(mismatches)[0]
        return False

    # print("comparing non-nan values...")
    # extract and compare values at non-nan indices
    notnan_idxs = np.where(a_mask)[0]
    try:
        a_nonnan = a.iloc[notnan_idxs]
    except (AttributeError, NotImplementedError):
        a_nonnan = a[notnan_idxs]
    try:
        b_nonnan = b.iloc[notnan_idxs]
    except (AttributeError, NotImplementedError):
        b_nonnan = b[notnan_idxs]

    if not a_numeric:  # exact comparison for non-numeric data
        mismatches = a_nonnan != b_nonnan
        # print("mismatches", mismatches[:10])
        # print("checking for equality since not numeric!")
        # print("where mismatches: ", np.where(mismatches)[0])
        # print("any mismatches: ", np.any(mismatches))
        # print("mismatches dtype: ", mismatches.dtype, type(mismatches))
        # print("about to return val: ", np.any(mismatches), notnan_idxs[np.where(mismatches)[0]])
        if return_failing_idxs:
            return not np.any(mismatches), notnan_idxs[np.where(mismatches)[0]]
        return not np.any(mismatches)

    # print("max absdiff: ", np.max(absdiffs))
    # print("median absdiff: ", np.median(absdiffs))
    # print("mean absdiff: ", np.mean(absdiffs))
    # print("worst absdiffs:\n", np.sort(absdiffs)[-10:])

    absdiffs = np.abs(a_nonnan - b_nonnan)
    fails = absdiffs > (atol + rtol * np.abs(b_nonnan))
    if return_failing_idxs:
        fail_idxs = np.where(fails)[0]
        return not np.any(fails), notnan_idxs[fail_idxs]
    return not np.any(fails)

# # XXX TODO support complex numbers (in particular, handle nans)
# def allclose(a, b, rtol=1e-5, atol=1e-5, equal_nan=True):
#     """Like numpb allclose, but handles pandas nullable scalar types"""
#     if isinstance(a, (tuple, list)):
#         a = np.array(a)
#     if isinstance(b, (tuple, list)):
#         b = np.array(b)

#     # shape checks
#     assert len(a) == len(b)
#     assert a.shape == b.shape
#     assert a.dtype == b.dtype

#     # compare locations / presence of nans
#     a_mask = pd.notna(a)
#     b_mask = pd.notna(b)
#     assert np.array_equal(a_mask, b_mask)
#     if not equal_nan:
#         assert np.all(a_mask)

#     # extract and compare values at non-nan indices
#     try:
#         a_nonnan = a.iloc[a_mask]
#     except (AttributeError, NotImplementedError):
#         a_nonnan = a[a_mask]
#     try:
#         b_nonnan = b.iloc[b_mask]
#     except (AttributeError, NotImplementedError):
#         b_nonnan = b[b_mask]
#     absdiffs = np.abs(a_nonnan - b_nonnan)
#     return np.all(absdiffs <= (atol + rtol * np.abs(b_nonnan)))

max_stars_repo_name: dblalock/sprintz

max_issues_repo_name: memetb/sprintz

max_forks_repo_name: memetb/sprintz


text:
# Copyright 2021 Jungwoo Park
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import random
from abc import ABCMeta, abstractmethod
from typing import List, Union

import cv2
import numpy as np
import torch
import torch.nn.functional as F

from differentiable_randaugment.base import BaseOperation

TransformMatrix = List[List[Union[float, torch.Tensor]]]

class GeometricOperation(BaseOperation, metaclass=ABCMeta):
    """Template class for geometric transform operations.

    Usually the geometric operations use affine transformation. This class performs the
    image augmentation by applying certain affine transform matrix. All geometric
    operations which inherit this class must implement `transform` method which returns
    the affine matrix. The input `value` must be differentiable if it is torch tensor.

    Note:
        The background color of transformed images is `#808080` (or `0.5` for tensors
        which are in range `[0, 1]`).
    """

    @abstractmethod
    def transform(self, value: Union[float, torch.Tensor]) -> TransformMatrix:
        ...

    def apply_numpy_affine(
        self, x: np.ndarray, value: float, background: int = 0x80
    ) -> np.ndarray:
        height, width = x.shape[:2]

        # Since `cv2` apply affine transformation by fixing center point to the left-top
        # corner, the coordinate normalization is required.
        normalize = np.array([[2 / width, 0, -1], [0, 2 / height, -1], [0, 0, 1]])
        matrix = self.transform(value) + [[0, 0, 1]]
        matrix = np.linalg.inv(matrix @ normalize) @ normalize

        return cv2.warpAffine(
            x,
            matrix[:2, :],
            (width, height),
            flags=cv2.INTER_LINEAR,
            borderValue=(background, background, background),
        )

    def apply_tensor_affine(
        self, x: torch.Tensor, value: torch.Tensor, background: float = 0.5
    ) -> torch.Tensor:
        matrix = torch.zeros((2, 3), dtype=torch.float32, device=x.device)

        # To make the matrix differentiable, we copy all values to the empty matrix
        # tensor directly.
        transform = self.transform(value)
        for i, j in itertools.product(range(2), range(3)):
            matrix[i, j] = transform[i][j]

        matrix = matrix.expand(x.size(0), -1, -1)
        grid = F.affine_grid(matrix, x.shape, align_corners=True).float()

        # Unfortunately, torch does not support background color in affine
        # transformation. Instead, it fills the background with zero. So we will
        # normalize the input tensor to make the background color be `0.5`.
        x = F.grid_sample(x - background, grid, "bilinear", "zeros", align_corners=True)
        return x + background

    def apply_numpy(self, x: np.ndarray, value: float) -> np.ndarray:
        return self.apply_numpy_affine(x, value, background=0x80)

    def apply_tensor(self, x: torch.Tensor, value: torch.Tensor) -> torch.Tensor:
        return self.apply_tensor_affine(x, value, background=0.5)

class ShearX(GeometricOperation):
    """Apply shear operation along the horizontal axis.

    +---------------+-------------+-----------+
    |               | Input Image | Magnitude |
    +===============+=============+===========+
    | ShearX        | ✔           | ✔         |
    +---------------+-------------+-----------+

    The magnitude of this operation must be positive and relative. It performs shear
    operation with entire width pixels for `magnitude==1` and half pixels for
    `magnitude==0.5`. Note that the shear direction is randomly selected.
    """

    def transform(self, sx: Union[float, torch.Tensor]) -> TransformMatrix:
        sx = random.choice([-1, 1]) * sx * 2
        return [[1, sx, 0], [0, 1, 0]]

class ShearY(GeometricOperation):
    """Apply shear operation along the vertical axis.

    +---------------+-------------+-----------+
    |               | Input Image | Magnitude |
    +===============+=============+===========+
    | ShearY        | ✔           | ✔         |
    +---------------+-------------+-----------+

    The magnitude of this operation must be positive and relative. It performs shear
    operation with entire height pixels for `magnitude==1` and half pixels for
    `magnitude==0.5`. Note that the shear direction is randomly selected.
    """

    def transform(self, sy: Union[float, torch.Tensor]) -> TransformMatrix:
        sy = random.choice([-1, 1]) * sy * 2
        return [[1, 0, 0], [sy, 1, 0]]

class TranslateX(GeometricOperation):
    """Translate the image in the horizontal direction.

    +---------------+-------------+-----------+
    |               | Input Image | Magnitude |
    +===============+=============+===========+
    | TranslateX    | ✔           | ✔         |
    +---------------+-------------+-----------+

    A magnitude of this operation must be positive and relative. It performs translate
    operation with entire width pixels for `magnitude==1` and half pixels for
    `magnitude==0.5`. Note that the translate direction is randomly selected.
    """

    def transform(self, tx: Union[float, torch.Tensor]) -> TransformMatrix:
        tx = random.choice([-1, 1]) * tx * 2
        return [[1, 0, tx], [0, 1, 0]]

class TranslateY(GeometricOperation):
    """Translate an image in the vertical direction.

    +---------------+-------------+-----------+
    |               | Input Image | Magnitude |
    +===============+=============+===========+
    | TranslateX    | ✔           | ✔         |
    +---------------+-------------+-----------+

    A magnitude of this operation must be positive and relative. It performs translate
    operation with entire height pixels for `magnitude==1` and half pixels for
    `magnitude==0.5`. Note that the translate direction is randomly selected.
    """

    def transform(self, ty: Union[float, torch.Tensor]) -> TransformMatrix:
        ty = random.choice([-1, 1]) * ty * 2
        return [[1, 0, 0], [0, 1, ty]]

class Rotate(GeometricOperation):
    """Rotate an image clockwise and counter-clockwise.

    +---------------+-------------+-----------+
    |               | Input Image | Magnitude |
    +===============+=============+===========+
    | TranslateX    | ✔           | ✔         |
    +---------------+-------------+-----------+

    A magnitude of this operation must be positive and in degrees. The rotation
    direction is randomly selected (clockwise or counter-clockwise).
    """

    def transform(self, deg: Union[float, torch.Tensor]) -> TransformMatrix:
        if not isinstance(deg, torch.Tensor):
            deg = torch.tensor(deg, dtype=torch.float)

        rad = random.choice([-1, 1]) * torch.deg2rad(deg)
        alpha, beta = torch.cos(rad), torch.sin(rad)

        return [[alpha, -beta, 0], [beta, alpha, 0]]

class Cutout(GeometricOperation):
    """Set a random square patch to gray.

    +---------------+-------------+-----------+
    |               | Input Image | Magnitude |
    +===============+=============+===========+
    | Cutout        | ✔           | ✔         |
    +---------------+-------------+-----------+

    A magnitude of this operation must be positive and relative. This class erase a
    random square patch from the image and fill the region with gray color. The
    magnitude decides the patch size. The patch position is randomly selected.
    """

    def transform(self, size: Union[float, torch.Tensor]) -> TransformMatrix:
        if isinstance(size, float):
            raise RuntimeError(
                "this operation does not support affine transform for numpy array"
            )

        tx = (1 - 2 * random.random() * (1 - size.item())) / size - 1
        ty = (1 - 2 * random.random() * (1 - size.item())) / size - 1

        return [[1 / size, 0, tx], [0, 1 / size, ty]]

    def apply_numpy(self, x: np.ndarray, value: float) -> np.ndarray:
        height, width = x.shape[:2]

        size = int(value * min(width, height))
        startx = int(random.random() * (width - size))
        starty = int(random.random() * (height - size))

        return cv2.rectangle(
            np.copy(x),
            (startx, starty),
            (startx + size, starty + size),
            color=(0x80, 0x80, 0x80),
            thickness=-1,
        )

    def apply_tensor(self, x: torch.Tensor, value: torch.Tensor) -> torch.Tensor:
        ratio = self.apply_tensor_affine(torch.ones_like(x), value, background=0)
        return ratio * 0.5 + (1 - ratio) * x


text:
'''
A data feeding class. It generates a list of data samples, each of which is
a tuple of a string (image path) and an integer (beam index), and it defines
a data-fetching method.
Author: Muhammad Alrabeiah
Aug. 2019
'''

import os
import random
from natsort import natsorted
import torch
import numpy as np
import random
from skimage import io
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils

################## Creating data samples ####################
def create_samples(root,shuffle,nat_sort = False):
    init_names = os.listdir(root) # List all sub-directories in root
    if nat_sort:
        sub_dir_names = natsorted(init_names) # sort directory names in natural order
                                              # (Only for directories with numbers for names)
    else:
        sub_dir_names = init_names

    class_to_ind = {name:int(name) for name in sub_dir_names}
    data_samples = []
    for sub_dir in sub_dir_names: # Loop over all sub-directories
        img_per_dir = os.listdir(root+'/'+sub_dir) # Get a list of image names from sub-dir # i
        if img_per_dir: # If img_per_dir is NOT empty
            for img_name in img_per_dir:
                sample = (root + '/' + sub_dir + '/' + img_name, class_to_ind[sub_dir])
                data_samples.append(sample)
    if shuffle:
        random.shuffle(data_samples)

    return data_samples
#############################################################

class DataFeed(Dataset):
    '''
    A class retrieving a tuple of (image,label). It can handle the case
    of empty classes (empty folders).
    '''
    def __init__(self,root_dir, nat_sort = False, transform=None, init_shuflle = True):
        self.root = root_dir
        self.samples = create_samples(self.root,shuffle=init_shuflle,nat_sort=nat_sort)
        self.transform = transform

    def __len__(self):
        return len( self.samples )

    def __getitem__(self, idx):
        sample = self.samples[idx]
        img = io.imread(sample[0])
        if self.transform:
            img = self.transform(img)
        label = sample[1]
        return (img,label)

max_stars_repo_name: malrabeiah/CameraPredictsBeams


text:
from collections import OrderedDict
import math
from time import time

import numpy as np

class BasicLogger(object):
    """A minimal class for logging training progress."""

    def __init__(self, fts, baseline_loss=0.0):
        """Pass a list of fts as argument."""
        self.fts = fts
        self.train_fts = OrderedDict()
        self.val_fts = OrderedDict()
        self.id_val_fts = OrderedDict()
        for ft in self.fts:
            self.train_fts[ft] = [[], []]
            self.val_fts[ft] = [[], []]
            self.id_val_fts[ft] = [[], []]
        self.n_epochs = 0
        self.baseline_loss = baseline_loss

    def training_step(self, losses):
        for i, ft in enumerate(self.fts):
            self.train_fts[ft][0].append(losses[i])

    def val_step(self, losses):
        for i, ft in enumerate(self.fts):
            self.val_fts[ft][0].append(losses[i])

    def id_val_step(self, losses):
        for i, ft in enumerate(self.fts):
            self.id_val_fts[ft][0].append(losses[i])

    def end_epoch(self):
        self.n_epochs += 1
        for i, ft in enumerate(self.fts):
            mean = np.array(self.train_fts[ft][0]).mean()
            self.train_fts[ft][1].append(mean)
            #reset train_fts log
            self.train_fts[ft][0] = []
            if len(self.val_fts[ft][0]) > 0:
                mean = np.array(self.val_fts[ft][0]).mean()
                self.val_fts[ft][1].append(mean)
                #reset val_fts log
                self.val_fts[ft][0] = []
            if len(self.id_val_fts[ft][0]) > 0:
                mean = np.array(self.id_val_fts[ft][0]).mean()
                self.id_val_fts[ft][1].append(mean)
                #reset id_val_fts log
                self.id_val_fts[ft][0] = []

class IpynbLogger(BasicLogger):
    """Plots Logging Data in jupyter notebook"""

    def __init__(self, *args, **kwargs):
        super(IpynbLogger, self).__init__(*args, **kwargs)
        import matplotlib.pyplot as plt
        from IPython.display import clear_output
        self.plt = plt
        self.clear_output = clear_output

    def end_epoch(self, val_losses=None):
        super(IpynbLogger, self).end_epoch()
        if self.n_epochs > 1:
            self.plot_progress()

    def plot_progress(self):
        self.clear_output()
        x = list(range(1, self.n_epochs+1))
        train_loss = [self.train_fts[ft][1] for ft in self.fts]
        train_loss = np.array(train_loss).mean(axis=0)
        self.plt.plot(x, train_loss, label='train loss', color='orange')

        if len(self.val_fts[self.fts[0]]) > 0:
            self.plt.axhline(
                y=self.baseline_loss,
                linestyle='dotted',
                label='baseline val loss',
                color='blue'
            )
            val_loss = [self.val_fts[ft][1] for ft in self.fts]
            val_loss = np.array(val_loss).mean(axis=0)
            self.plt.plot(x, val_loss, label='val loss', color='blue')

        if len(self.id_val_fts[self.fts[0]]) > 0:
            id_val_loss = [self.id_val_fts[ft][1] for ft in self.fts]
            id_val_loss = np.array(id_val_loss).mean(axis=0)
            self.plt.plot(x, id_val_loss, label='identity val loss', color='pink')

        self.plt.ylim(0, max(1, math.floor(2*self.baseline_loss)))
        self.plt.legend()
        self.plt.xlabel('epochs')
        self.plt.ylabel('loss')
        self.plt.show();

class TensorboardXLogger(BasicLogger):

    def __init__(self, logdir='logdir/', run=None, *args, **kwargs):
        super(TensorboardXLogger, self).__init__(*args, **kwargs)
        from tensorboardX import SummaryWriter
        import os

        if run is None:
            try:
                n_runs = len(os.listdir(logdir))
            except FileNotFoundError:
                n_runs = 0
            logdir = logdir+f'{n_runs:04d}'
        else:
            logdir = logdir + str(run)
        self.writer = SummaryWriter(logdir)
        self.n_train_step = 0
        self.n_val_step = 0
        self.n_id_val_step = 0

    def training_step(self, losses):
        self.n_train_step += 1
        losses = np.array(losses)
        for i, ft in enumerate(self.fts):
            self.writer.add_scalar('online' + f'_{ft}_' + 'train_loss', losses[i], self.n_train_step)
            self.train_fts[ft][0].append(losses[i])
        self.writer.add_scalar('online' + '_mean_' + 'train_loss', losses.mean(), self.n_train_step)

    def val_step(self, losses):
        #self.n_val_step += 1
        for i, ft in enumerate(self.fts):
            #self.writer.add_scalar(f'_{ft}_' + 'val_loss', losses[i], self.n_val_step)
            self.val_fts[ft][0].append(losses[i])

    def id_val_step(self, losses):
        #self.n_id_val_step += 1
        for i, ft in enumerate(self.fts):
            #self.writer.add_scalar(f'_{ft}_' + 'id_loss', losses[i], self.n_id_val_step)
            self.id_val_fts[ft][0].append(losses[i])

    def end_epoch(self, val_losses=None):
        super(TensorboardXLogger, self).end_epoch()

        train_loss = [self.train_fts[ft][1][-1] for ft in self.fts]
        for i, ft in enumerate(self.fts):
            self.writer.add_scalar(f'{ft}_' + 'train_loss', train_loss[i], self.n_epochs)
        train_loss = np.array(train_loss).mean()
        self.writer.add_scalar('mean_train_loss', train_loss, self.n_epochs)

        val_loss = [self.val_fts[ft][1][-1] for ft in self.fts]
        for i, ft in enumerate(self.fts):
            self.writer.add_scalar(f'{ft}_' + 'val_loss', val_loss[i], self.n_epochs)
        val_loss = np.array(val_loss).mean()
        self.writer.add_scalar('mean_val_loss', val_loss, self.n_epochs)

        id_val_loss = [self.id_val_fts[ft][1][-1] for ft in self.fts]
        for i, ft in enumerate(self.fts):
            self.writer.add_scalar(f'{ft}_' + 'train_loss', id_val_loss[i], self.n_epochs)
        id_val_loss = np.array(id_val_loss).mean()
        self.writer.add_scalar('mean_id_val_loss', id_val_loss, self.n_epochs)

    def show_embeddings(self, categories):
        for ft in categories:
            feature = categories[ft]
            cats = feature['cats'] + ['_other']
            emb = feature['embedding']
            mat = emb.weight.data.cpu().numpy()
            self.writer.add_embedding(mat, metadata=cats, tag=ft, global_step=self.n_epochs)

max_stars_repo_name: arjdesign/dfencoder

max_issues_repo_name: arjdesign/dfencoder

max_forks_repo_name: arjdesign/dfencoder


text:
import logging
import os
import time

import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.optim as optim

import foolbox
from lib.adv_model import *
from lib.cwl2_attack import CWL2Attack
from lib.dataset_utils import *
from lib.dknn import DKNNL2
from lib.dknn_attack import DKNNAttack
from lib.dknn_attack_exp import DKNNExpAttack
from lib.dknn_attack_l2 import DKNNL2Attack
from lib.dknn_attack_linf import DKNNLinfAttack
from lib.lip_model import *
from lib.mnist_model import *
from lib.utils import *

os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"

# Set all random seeds
seed = 2019
np.random.seed(seed)
torch.manual_seed(seed)

device = 'cuda' if torch.cuda.is_available() else 'cpu'

(x_train, y_train), (x_valid, y_valid), (x_test, y_test) = load_mnist_all(
    '/data', val_size=0.1, seed=seed)

for epoch in range(100):

    print('epoch %d' % epoch)
    model_name = 'adv_mnist_nca_exp21_epoch%d.h5' % epoch
    net = NCAModel(output_dim=100, init_it=1)

    # Set up model directory
    save_dir = os.path.join(os.getcwd(), 'saved_models/')
    if not os.path.isdir(save_dir):
        os.makedirs(save_dir)
    model_path = os.path.join(save_dir, model_name)

    net = net.to(device)
    net.load_state_dict(torch.load(model_path))
    net.eval()

    dknn = DKNNL2(net, x_train, y_train, x_valid, y_valid, ['fc'],
                  k=5, num_classes=10)

    with torch.no_grad():
        y_pred = dknn.classify(x_test)
        ind = np.where(y_pred.argmax(1) == y_test.numpy())[0]
        print('  accuracy: %.4f' %
              ((y_pred.argmax(1) == y_test.numpy()).sum() / y_test.size(0)))

    attack = DKNNExpAttack(dknn)

    def attack_batch(x, y, batch_size):
        x_adv = torch.zeros_like(x)
        total_num = x.size(0)
        num_batches = total_num // batch_size
        for i in range(num_batches):
            begin = i * batch_size
            end = (i + 1) * batch_size
            x_adv[begin:end] = attack(
                x[begin:end], y[begin:end],
                guide_layer='fc', m=40, binary_search_steps=10,
                max_iterations=1000, learning_rate=1e-1,
                initial_const=1e1, random_start=False,
                thres_steps=200, check_adv_steps=200, verbose=False,
                max_linf=None)
        return x_adv

    num = 100
    x_adv = attack_batch(x_test[ind][:num].cuda(), y_test[ind][:num], 100)

    with torch.no_grad():
        y_pred = dknn.classify(x_adv)
        ind_adv = np.where(y_pred.argmax(1) != y_test[ind][:num].numpy())[0]
        print('  adv accuracy: %.4f' % (
            (y_pred.argmax(1) == y_test[ind][:num].numpy()).sum() / y_pred.shape[0]))
    dist = (x_adv.cpu() - x_test[ind][:num]).view(num, -1).norm(2, 1)[ind_adv]
    print('  mean dist: %.4f' % dist.mean().item())

max_stars_repo_name: chawins/entangle-rep

max_issues_repo_name: chawins/entangle-rep

max_forks_repo_name: chawins/entangle-rep


text:
from numpy import *

x,y,z = mgrid[-100:101:25., -100:101:25., -100:101:25.]

V = 2*x**2 + 3*y**2 - 4*z # just a random function for the potential

Ex,Ey,Ez = gradient(V)

max_stars_repo_name: gustavodiel/PIM

max_issues_repo_name: gustavodiel/PIM

max_forks_repo_name: gustavodiel/PIM

reason: from numpy