text:
import argparse
import inspect

import json

from sklearn.utils import all_estimators
from numpydoc.docscrape import ClassDoc
import sklearn

from .. import get_metadata
from ._d3m import convert_hyperparam_to_d3m

__all__ = ["get_output_for_module"]

def _get_output_for_estimator(name, estimator):
    metadata = get_metadata(estimator.__name__)
    init_annotations = metadata["parameters"]
    # attribute_annotations = metadata["attributes"]
    init_params = inspect.signature(estimator).parameters

    class_doc = ClassDoc(estimator)
    param_descriptions = {
        param.name: " ".join(param.desc) for param in class_doc["Parameters"]
    }
    # attribute_descriptions = {
    #     attr.name.split(":")[0].strip(): " ".join(attr.desc)
    #     for attr in class_doc["Attributes"]
    # }

    hyperparmas = []

    for param, annotation in init_annotations.items():

        # Remove parameters that is not JSON serializable
        default = init_params[param].default

        if callable(default):
            continue

        try:
            description = param_descriptions[param]
        except KeyError:
            description = ""

        try:
            hyperparam = convert_hyperparam_to_d3m(
                param,
                annotation,
                description=description,
                default=init_params[param].default,
            )
        except ValueError as e:
            raise ValueError(f"Failed parsing {name}: {e}")

        hyperparmas.append(hyperparam)

    # params = []
    # for attribute, annotation in attribute_annotations.items():

    #     try:
    #         description = attribute_descriptions[attribute]
    #     except KeyError:
    #         description = ""

    #     try:
    #         param = convert_attribute_to_d3m(
    #             attribute,
    #             annotation,
    #             description=description,
    #         )
    #     except ValueError as e:
    #         raise ValueError(f"Failed parsing {name}: {e}")

    #     params.append(param)

    estimator_output = {}
    estimator_output["name"] = f"{estimator.__module__}.{name}"
    estimator_output["common_name"] = name
    estimator_output["description"] = " ".join(
        class_doc["Summary"] + class_doc["Extended Summary"]
    )
    estimator_output["sklearn_version"] = sklearn.__version__
    estimator_output["Hyperparams"] = hyperparmas
    # estimator_output["Params"] = params

    return estimator_output

def get_output_for_module(module):
    estimators = [
        (name, est)
        for name, est in all_estimators()
        if est.__module__.split(".")[1] == module
    ]
    output = {}
    for name, Est in estimators:
        output[name] = _get_output_for_estimator(name, Est)

    return output

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Get d3m overlay for module")
    parser.add_argument("module")

    args = parser.parse_args()
    output = get_output_for_module(args.module)
    print(json.dumps(output, indent=2))

reason: from numpy


text:
from sklearn.metrics import check_scoring
import numpy as np

def check_partial_fit(estimator):
    return callable(getattr(estimator, "partial_fit", None))

def check_warm_start(estimator):
    from sklearn.tree import BaseDecisionTree
    from sklearn.ensemble import BaseEnsemble
    is_not_tree_subclass = not issubclass(type(estimator), BaseDecisionTree)
    is_not_ensemble_subclass = not issubclass(type(estimator), BaseEnsemble)

    return (hasattr(estimator, "warm_start")
            and hasattr(estimator, "max_iter") and is_not_ensemble_subclass
            and is_not_tree_subclass)

def _aggregate_score_dicts(scores):
    """Aggregate the list of dict to dict of np ndarray
    The aggregated output of _fit_and_score will be a list of dict
    of form [{'prec': 0.1, 'acc':1.0}, {'prec': 0.1, 'acc':1.0}, ...]
    Convert it to a dict of array {'prec': np.array([0.1 ...]), ...}
    Parameters
    ----------
    scores : list of dict
        List of dicts of the scores for all scorers. This is a flat list,
        assumed originally to be of row major order.
    Example
    -------
    >>> scores = [{'a': 1, 'b':10}, {'a': 2, 'b':2}, {'a': 3, 'b':3},
    ...           {'a': 10, 'b': 10}]                         # doctest: +SKIP
    >>> _aggregate_score_dicts(scores)                        # doctest: +SKIP
    {'a': array([1, 2, 3, 10]),
     'b': array([10, 2, 3, 10])}
    """
    return {
        key: np.asarray([score[key] for score in scores])
        for key in scores[0]
    }

def _check_multimetric_scoring(estimator, scoring=None):
    """Check the scoring parameter in cases when multiple metrics are allowed
    Parameters
    ----------
    estimator : sklearn estimator instance
        The estimator for which the scoring will be applied.
    scoring : string, callable, list/tuple, dict or None, default: None
        A single string (see :ref:`scoring_parameter`) or a callable
        (see :ref:`scoring`) to evaluate the predictions on the test set.
        For evaluating multiple metrics, either give a list of (unique) strings
        or a dict with names as keys and callables as values.
        NOTE that when using custom scorers, each scorer should return a single
        value. Metric functions returning a list/array of values can be wrapped
        into multiple scorers that return one value each.
        See :ref:`multimetric_grid_search` for an example.
        If None the estimator's score method is used.
        The return value in that case will be ``{'score': <default_scorer>}``.
        If the estimator's score method is not available, a ``TypeError``
        is raised.
    Returns
    -------
    scorers_dict : dict
        A dict mapping each scorer name to its validated scorer.
    is_multimetric : bool
        True if scorer is a list/tuple or dict of callables
        False if scorer is None/str/callable
    """
    if callable(scoring) or scoring is None or isinstance(scoring, str):
        scorers = {"score": check_scoring(estimator, scoring=scoring)}
        return scorers, False
    else:
        err_msg_generic = (
            "scoring should either be a single string or "
            "callable for single metric evaluation or a "
            "list/tuple of strings or a dict of scorer name "
            "mapped to the callable for multiple metric "
            "evaluation. Got %s of type %s" % (repr(scoring), type(scoring)))

        if isinstance(scoring, (list, tuple, set)):
            err_msg = ("The list/tuple elements must be unique "
                       "strings of predefined scorers. ")
            invalid = False
            try:
                keys = set(scoring)
            except TypeError:
                invalid = True
            if invalid:
                raise ValueError(err_msg)

            if len(keys) != len(scoring):
                raise ValueError(err_msg + "Duplicate elements were found in"
                                 " the given list. %r" % repr(scoring))
            elif len(keys) > 0:
                if not all(isinstance(k, str) for k in keys):
                    if any(callable(k) for k in keys):
                        raise ValueError(err_msg +
                                         "One or more of the elements were "
                                         "callables. Use a dict of score name "
                                         "mapped to the scorer callable. "
                                         "Got %r" % repr(scoring))
                    else:
                        raise ValueError(
                            err_msg + "Non-string types were found in "
                            "the given list. Got %r" % repr(scoring))
                scorers = {
                    scorer: check_scoring(estimator, scoring=scorer)
                    for scorer in scoring
                }
            else:
                raise ValueError(err_msg +
                                 "Empty list was given. %r" % repr(scoring))

        elif isinstance(scoring, dict):
            keys = set(scoring)
            if not all(isinstance(k, str) for k in keys):
                raise ValueError("Non-string types were found in the keys of "
                                 "the given dict. scoring=%r" % repr(scoring))
            if len(keys) == 0:
                raise ValueError(
                    "An empty dict was passed. %r" % repr(scoring))
            scorers = {
                key: check_scoring(estimator, scoring=scorer)
                for key, scorer in scoring.items()
            }
        else:
            raise ValueError(err_msg_generic)
        return scorers, True

ext: py

max_stars_repo_name: holgern/tune-sklearn

max_issues_repo_name: holgern/tune-sklearn

max_forks_repo_name: holgern/tune-sklearn

reason: import numpy


text:
import keras
from keras import models
import os
import pickle
import numpy

class ModelMemorizer:
    """
    for keras instance
    mm = ModelMemorizer('test')
    mm.dump(model, history.history)
    (model, history_dict) = mm.load()
    """
    def __init__(self, name, path='./model-memorizer', standalone_path=True, mkdir=True):
        self.name = name
        if standalone_path:
            path='%s/%s'%(path, name)
        self.path = path
        if mkdir:
            os.makedirs(self.path, exist_ok=True)
        self.model_path = '%s/%s.model.h5'%(path, name)
        self.history_dict_path = '%s/%s.history_dict.pickle'%(path, name)

    def mkdir(self):
        os.makedirs(self.path, exist_ok=True) # python above 3.4

    def dump_history_dict(self, history_dict):
        assert isinstance(history_dict, dict)
        f = open(self.history_dict_path, 'wb')
        pickle.dump(history_dict, f)
        f.close()

    def load_history_dict(self):
        f = open(self.history_dict_path, 'rb')
        history_dict = pickle.load( f)
        f.close()
        return history_dict

    def dump_model(self, model):
        assert isinstance(model, keras.engine.sequential.Sequential)
        model.save(self.model_path)

    def load_model(self):
        return models.load_model(self.model_path)

    def dump(self, model, history_dict):
        self.dump_history_dict(history_dict)
        self.dump_model(model)

    def load(self):
        return self.load_model(), self.load_history_dict()

max_stars_repo_name: xsthunder/python-lib

max_issues_repo_name: xsthunder/python-lib

max_forks_repo_name: xsthunder/python-lib


text:
from irec.recommendation.agents.action import Action
from .action import UserItemAction
from .base import Agent
from typing import Any, List
import numpy as np

class SimpleEnsembleAgent(Agent):
    """SimpleEnsembleAgent.

    iRec also allows agents based on ensemble strategies by using the tag
    EnsembleAgent and, in this case, more than one Action Selection Policy
    and/or more than one ValueFunction can be set.
    """

    def __init__(
        self,
        agents: List[Agent],
        use_name_meta_actions: bool = True,
        save_meta_actions: bool = True,
        *args,
        **kwargs
    ):
        """__init__.

        Args:
            agents (List[Agent]): agents
            use_name_meta_actions (bool): use_name_meta_actions
            save_meta_actions (bool): save_meta_actions
            args:
            kwargs:
        """

        super().__init__(*args, **kwargs)
        self.agents = agents
        self.use_name_meta_actions = use_name_meta_actions
        if self.use_name_meta_actions:
            self.ensemble_candidate_actions = np.array([i.name for i in self.agents])
            self.actions_name_object_map = {
                act_name: self.agents[i]
                for i, act_name in enumerate(self.ensemble_candidate_actions)
            }
        else:
            self.ensemble_candidate_actions = np.array(list(range(len(self.agents))))
        self.default_actions_num = 1
        self.save_meta_actions = save_meta_actions

    def act(self, candidate_actions: Action, actions_num: int):
        """act.

        Args:
            candidate_actions (Action): candidate_actions
            actions_num (int): actions_num
        """
        info = {}
        meta_actions_estimate, meta_vf_info = self.value_function.actions_estimate(
            self.ensemble_candidate_actions
        )
        meta_actions, meta_asp_info = self.action_selection_policy.select_actions(
            self.ensemble_candidate_actions,
            meta_actions_estimate,
            self.default_actions_num,
        )
        meta_action = meta_actions[0]
        if self.use_name_meta_actions:
            selected_agent = self.actions_name_object_map[meta_action]
        else:
            selected_agent = self.agents[meta_action]
        selected_agent_actions, selected_agent_info = selected_agent.act(
            candidate_actions, actions_num
        )
        if meta_vf_info is not None:
            info["vf_info"] = meta_vf_info
        if meta_asp_info is not None:
            info["asp_info"] = meta_asp_info
        if selected_agent_info is not None:
            info["selected_agent_info"] = selected_agent_info
        if self.save_meta_actions:
            if self.use_name_meta_actions:
                info["meta_action_name"] = meta_action
                info["mai"] = meta_action
            else:
                info["meta_action_name"] = self.agents[meta_action].name
                info["mai"] = meta_action
        if info == dict():
            info = None
        return selected_agent_actions, info

    def observe(self, observation: Any, action: UserItemAction, reward: float, info: dict):
        """observe.

        Args:
            observation (Any): observation
            action (UserItemAction): action
            reward (float): reward
            info (dict): info
        """
        vf_info = info.get("vf_info", None)
        asp_info = info.get("asp_info", None)
        self.value_function.update(observation, info["mai"], reward, vf_info)
        self.action_selection_policy.update(observation, info["mai"], reward, asp_info)
        for agent in self.agents:
            agent.observe(observation, action, reward, info)
        return None

    def reset(self, observation: Any):
        """reset.

        Args:
            observation (Any): observation
        """
        if observation is not None:
            self.value_function.reset(observation)
            self.action_selection_policy.reset(observation)
            for agent in self.agents:
                agent.reset(observation)
            return None
        else:
            raise NotImplementedError

max_stars_repo_name: irec-org/irec

max_issues_repo_name: irec-org/irec

max_forks_repo_name: irec-org/irec


text:
import torch
import torch.nn as nn
from torch.optim import Adam
import torch.nn.functional as F
from data_loader_UAST import get_fusion_dataloader
import config
from tqdm import tqdm
import math
import numpy as np
import os
import pickle
import matplotlib.pyplot as plt
import seaborn as sns
import palettable

from torch.nn.parameter import Parameter
from torch.nn.modules.module import Module

dict_ast_path = pickle.load(open('./vocabulary/unified_vocab.pkl', 'rb'))

#SAST
class ASTModal(nn.Module):
    def __init__(self):
        super(ASTModal, self).__init__()
        # 1.input embedding
        self.ASTEmbedding = nn.Embedding(len(dict_ast_path), config.embedding_dim)

        # 2.self-attention
        self.attention = SelfAttention(hidden_size=config.embedding_dim, num_attention_heads=4, dropout_prob=0.2)

        # 3.bi-LSTM
        self.lstm = nn.LSTM(input_size=config.embedding_dim,
                            hidden_size=config.hidden_size,
                            num_layers=config.num_layers,
                            batch_first=True,
                            dropout=config.dropout,
                            bidirectional=config.bidirectional)

        # 4.fully connect
        self.fc = nn.Linear(config.hidden_size * 2, 50)

    def forward(self, input_AST, x_mask):
        # 1. embedding: [batchsize,max_len,embedding_dim]
        AST_path = self.ASTEmbedding(input_AST)

        AST_att = self.attention(AST_path, x_mask)

        #3. bi-lstm
        token_att, (h_n_t, c_n_t) = self.lstm(AST_att)
        output_fw_t = h_n_t[-2, :, :]
        output_bw_t = h_n_t[-1, :, :]
        output_t = torch.cat([output_fw_t, output_bw_t], dim=-1)  # [batchsize,hidden_size*2]

        #4. fully connect + softmax
        sast_out = self.fc(output_t)  # [batchsize,2]
        # return F.log_softmax(out, dim=-1)  # [batchsize,2]
        return sast_out

class SelfAttention(nn.Module):

    def __init__(self, hidden_size, num_attention_heads, dropout_prob):
        super(SelfAttention, self).__init__()
        if hidden_size % num_attention_heads != 0:
            raise ValueError(
                "The hidden size (%d) is not a multiple of the number of attention "
                "heads (%d)" % (hidden_size, num_attention_heads))
        self.num_attention_heads = num_attention_heads
        self.attention_head_size = int(hidden_size / num_attention_heads)
        self.all_head_size = int(self.num_attention_heads * self.attention_head_size)
        self.query = nn.Linear(hidden_size, self.all_head_size)
        self.key = nn.Linear(hidden_size, self.all_head_size)
        self.value = nn.Linear(hidden_size, self.all_head_size)

        self.dropout = nn.Dropout(dropout_prob)

    def transpose_for_scores(self, x):
        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
        x = x.view(*new_x_shape)
        return x.permute(0, 2, 1, 3)

    def forward(self, hidden_states, attention_mask):
        attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
        attention_mask = (1.0 - attention_mask) * -10000.0

        mixed_query_layer = self.query(hidden_states)
        mixed_key_layer = self.key(hidden_states)
        mixed_value_layer = self.value(hidden_states)

        query_layer = self.transpose_for_scores(mixed_query_layer)
        key_layer = self.transpose_for_scores(mixed_key_layer)
        value_layer = self.transpose_for_scores(mixed_value_layer)

        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
        attention_scores = attention_scores + attention_mask

        attention_probs = nn.Softmax(dim=-1)(attention_scores)
        attention_probs = self.dropout(attention_probs)

        context_layer = torch.matmul(attention_probs, value_layer)
        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
        context_layer = context_layer.view(*new_context_layer_shape)
        return context_layer

#GAST
class GCN(nn.Module):
    def __init__(self, nfeat, nhid, nclass, dropout):
        super(GCN, self).__init__()

        self.gc1 = GraphConvolution(nfeat, nhid)
        self.gc2 = GraphConvolution(nhid, nclass)
        self.fc = nn.Linear(nclass*config.max_len, 50)
        self.dropout = dropout

    def forward(self, x, adj):
        x = F.relu(self.gc1(x, adj))
        x = F.dropout(x, self.dropout, training=self.training)
        x = self.gc2(x, adj) # [batchsize, node_num , class_num]
        x = x.view(x.size(0), x.size(1) * x.size(2))
        gast_out = self.fc(x)
        # print(x.shape)
        # return F.log_softmax(x, dim=1)
        return gast_out

class GraphConvolution(nn.Module):

    def __init__(self, in_features, out_features, bias=True):
        super(GraphConvolution, self).__init__()
        self.in_features = in_features
        self.out_features = out_features
        # parameter
        self.weight = Parameter(torch.FloatTensor(in_features, out_features))
        if bias:
            self.bias = Parameter(torch.FloatTensor(out_features))
        else:
            self.register_parameter('bias', None)
        self.reset_parameters()

    def reset_parameters(self):

        stdv = 1. / math.sqrt(self.weight.size(1))
        self.weight.data.uniform_(-stdv, stdv)
        if self.bias is not None:
            self.bias.data.uniform_(-stdv, stdv)

    def forward(self, input_X, adj):

        support = torch.matmul(input_X, self.weight)
        # output = torch.spmm(adj, support)
        output = torch.matmul(adj, support)

        if self.bias is not None:
            return output + self.bias
        else:
            return output

    def __repr__(self):
        return self.__class__.__name__ + ' (' \
               + str(self.in_features) + ' -> ' \
               + str(self.out_features) + ')'

class UAST(nn.Module):
    def __init__(self):
        super(UAST, self).__init__()

        self.sast = ASTModal()
        self.gast = GCN(nfeat=len(dict_ast_path),
                        nhid=64,  #
                        nclass=16,
                        dropout=0.5)
        # self.fc = nn.Linear(100, 50)

    def forward(self, ast_path, adj_matrix,fea_matrix,x_mask):
        sast_feature = self.sast(ast_path, x_mask) # [batchsize,50]
        gast_feature = self.gast(fea_matrix,adj_matrix) # [batchsize,50]
        fusion = sast_feature + gast_feature
        # print(fusion.shape)
        # out = self.fc(fusion)  # [batchsize,50]
        return F.log_softmax(fusion, dim=-1) #[bs,50]

def sequence_mask(input_):
    batch_size, seq_len = input_.size()
    mask = 1 - torch.triu(torch.ones((batch_size, seq_len), dtype=torch.uint8),
                          diagonal=1)
    # [B, L, L]
    return mask

def drawGraph(input_matrix):
    sns.set()
    plt.figure(dpi=120)
    sns.heatmap(data=input_matrix.cpu().numpy(),
                cmap=plt.get_cmap('Greens'),  # matplotlib
                )
    plt.title("Token original")
    plt.show()

def train(epoch, model, data_loader):
    print("start training...")

    # model = MyModel().to(config.device)
    # optimizer = Adam(model.parameters(),lr=0.001).to(config.device)
    # train_dataloader = get_fusion_dataloader(train=True) #loading data

    model.train()
    for i in range(epoch):
        for batch, (ast_path, adj_matrix, fea_matrix, label) in tqdm(enumerate(data_loader), total=len(data_loader)):

            ast_path = ast_path.to(config.device)
            adj_matrix = adj_matrix.to(config.device)
            fea_matrix = fea_matrix.to(config.device)
            label = label.to(config.device)
            x_mask = sequence_mask(ast_path).to(config.device)  # self attention

            optimizer.zero_grad()
            output = model(ast_path, adj_matrix,fea_matrix,x_mask)
            loss = F.nll_loss(output, label)
            loss.backward()
            optimizer.step()

        loss_epoch = loss.item()
        print("epoch: ", i, "  loss: ", loss_epoch)

        # if epoch % 2 == 0:
        #     for params in optimizer.param_groups:
        #         params['lr'] *= 0.8

        # torch.save(model.state_dict(), "./checkpoints/test_mymodel_%d.pth" % i)
        torch.save(model.state_dict(), "./checkpoints/UAST_model_3_len700.pth")
        # torch.save(optimizer.state_dict(), "./checkpoints/test_optimizer_%d.pth" % i)

def evaluate(model, data_loader):

    model.eval()

    loss_list = []
    acc_list = []
    label_list = []
    pred_list = []

    print("start evaluating...")
    for batch, (ast_path, adj_matrix, fea_matrix, label) in tqdm(enumerate(data_loader), total=len(data_loader)):

        ast_path = ast_path.to(config.device)
        adj_matrix = adj_matrix.to(config.device)
        fea_matrix = fea_matrix.to(config.device)
        label = label.to(config.device)
        x_mask = sequence_mask(ast_path).to(config.device)  # self attention

        with torch.no_grad():
            output = model(ast_path, adj_matrix,fea_matrix,x_mask)  # [batchsize,2]
            # print("\n output:  ",output)
            cur_loss = F.nll_loss(output, label)
            loss_list.append(cur_loss.cpu().item())

            pred = output.max(dim=-1)[-1]
            # print("\n prediction:  ",pred)
            for i in label:
                label_list.append(i.cpu().numpy())
            for j in pred:
                pred_list.append(j.cpu().numpy())
            cur_acc = pred.eq(label).float().mean()
            acc_list.append(cur_acc.cpu().item())

    print("total loss, ", np.mean(loss_list), "acc:", np.mean(acc_list))

    from sklearn.metrics import recall_score
    recall_val = recall_score(label_list, pred_list, average='macro')
    from sklearn.metrics import precision_score
    precision_val = precision_score(label_list, pred_list, average='macro')
    from sklearn.metrics import f1_score
    f1_val = f1_score(label_list, pred_list, average='macro')
    result = {
        "eval_recall": float(recall_val),
        "eval_precision": float(precision_val),
        "eval_f1": float(f1_val),
    }
    print(result)

    # model.train()

def test():
    print("starting testing!")

    loss_list = []
    acc_list = []
    label_list = []
    pred_list = []

    test_dataloder = get_fusion_dataloader(train="test")

    model = UAST().eval()
    model.load_state_dict(torch.load("./checkpoints/UAST_model_3_len700.pth"))
    # optimizer.load_state_dict(torch.load("./checkpoints/optimizer_4.pth"))

    model.to(config.device)
    # optimizer.cuda()

    for batch, (ast_path, adj_matrix, fea_matrix, label) in tqdm(enumerate(test_dataloder), total=len(test_dataloder)):

        ast_path = ast_path.to(config.device)
        adj_matrix = adj_matrix.to(config.device)
        fea_matrix = fea_matrix.to(config.device)
        label = label.to(config.device)
        x_mask = sequence_mask(ast_path).to(config.device)  # self attention

        with torch.no_grad():
            output = model(ast_path, adj_matrix,fea_matrix,x_mask)
            cur_loss = F.nll_loss(output, label)
            loss_list.append(cur_loss.cpu().item())

            pred = output.max(dim=-1)[-1]
            for i in label:
                label_list.append(i.cpu().numpy())
            for j in pred:
                pred_list.append(j.cpu().numpy())
            cur_acc = pred.eq(label).float().mean()
            acc_list.append(cur_acc.cpu().item())

    print("total loss, ", np.mean(loss_list), "acc:", np.mean(acc_list))

    from sklearn.metrics import recall_score
    recall_val = recall_score(label_list, pred_list, average='macro')
    from sklearn.metrics import precision_score
    precision_val = precision_score(label_list, pred_list, average='macro')
    from sklearn.metrics import f1_score
    f1_val = f1_score(label_list, pred_list, average='macro')
    result = {
        "eval_recall": float(recall_val),
        "eval_precision": float(precision_val),
        "eval_f1": float(f1_val),
    }
    print(result)

if __name__ == '__main__':
    print(len(dict_ast_path))
    model = UAST().to(config.device)
    optimizer = Adam(model.parameters(),lr=0.001)
    train_dataloader = get_fusion_dataloader(train="train") #loading data
    train(3,model,train_dataloader)
    # valid_dataloader = get_fusion_dataloader(train="valid") #loading data
    # evaluate(model, valid_dataloader)
    # test()
    # for i in range(10):
    #     test()


text:
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2004-2018 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
"""A QDialog widget to set-up the colormap.

It uses a description of colormaps as dict compatible with :class:`Plot`.

To run the following sample code, a QApplication must be initialized.

Create the colormap dialog and set the colormap description and data range:

>>> from silx.gui.dialog.ColormapDialog import ColormapDialog
>>> from silx.gui.colors import Colormap

>>> dialog = ColormapDialog()
>>> colormap = Colormap(name='red', normalization='log',
...                     vmin=1., vmax=2.)

>>> dialog.setColormap(colormap)
>>> colormap.setVRange(1., 100.)  # This scale the width of the plot area
>>> dialog.show()

Get the colormap description (compatible with :class:`Plot`) from the dialog:

>>> cmap = dialog.getColormap()
>>> cmap.getName()
'red'

It is also possible to display an histogram of the image in the dialog.
This updates the data range with the range of the bins.

>>> import numpy
>>> image = numpy.random.normal(size=512 * 512).reshape(512, -1)
>>> hist, bin_edges = numpy.histogram(image, bins=10)
>>> dialog.setHistogram(hist, bin_edges)

The updates of the colormap description are also available through the signal:
:attr:`ColormapDialog.sigColormapChanged`.
"""  # noqa

from __future__ import division

__authors__ = ["V.A. Sole", "T. Vincent", "H. Payno"]
__license__ = "MIT"
__date__ = "23/05/2018"

import logging

import numpy

from .. import qt
from ..colors import Colormap, preferredColormaps
from ..plot import PlotWidget
from silx.gui.widgets.FloatEdit import FloatEdit
import weakref
from silx.math.combo import min_max
from silx.third_party import enum
from silx.gui import icons
from silx.math.histogram import Histogramnd

_logger = logging.getLogger(__name__)

_colormapIconPreview = {}

class _BoundaryWidget(qt.QWidget):
    """Widget to edit a boundary of the colormap (vmin, vmax)"""
    sigValueChanged = qt.Signal(object)
    """Signal emitted when value is changed"""

    def __init__(self, parent=None, value=0.0):
        qt.QWidget.__init__(self, parent=None)
        self.setLayout(qt.QHBoxLayout())
        self.layout().setContentsMargins(0, 0, 0, 0)
        self._numVal = FloatEdit(parent=self, value=value)
        self.layout().addWidget(self._numVal)
        self._autoCB = qt.QCheckBox('auto', parent=self)
        self.layout().addWidget(self._autoCB)
        self._autoCB.setChecked(False)

        self._autoCB.toggled.connect(self._autoToggled)
        self.sigValueChanged = self._autoCB.toggled
        self.textEdited = self._numVal.textEdited
        self.editingFinished = self._numVal.editingFinished
        self._dataValue = None

    def isAutoChecked(self):
        return self._autoCB.isChecked()

    def getValue(self):
        return None if self._autoCB.isChecked() else self._numVal.value()

    def getFiniteValue(self):
        if not self._autoCB.isChecked():
            return self._numVal.value()
        elif self._dataValue is None:
            return self._numVal.value()
        else:
            return self._dataValue

    def _autoToggled(self, enabled):
        self._numVal.setEnabled(not enabled)
        self._updateDisplayedText()

    def _updateDisplayedText(self):
        # if dataValue is finite
        if self._autoCB.isChecked() and self._dataValue is not None:
            old = self._numVal.blockSignals(True)
            self._numVal.setValue(self._dataValue)
            self._numVal.blockSignals(old)

    def setDataValue(self, dataValue):
        self._dataValue = dataValue
        self._updateDisplayedText()

    def setFiniteValue(self, value):
        assert(value is not None)
        old = self._numVal.blockSignals(True)
        self._numVal.setValue(value)
        self._numVal.blockSignals(old)

    def setValue(self, value, isAuto=False):
        self._autoCB.setChecked(isAuto or value is None)
        if value is not None:
            self._numVal.setValue(value)
        self._updateDisplayedText()

class _ColormapNameCombox(qt.QComboBox):
    def __init__(self, parent=None):
        qt.QComboBox.__init__(self, parent)
        self.__initItems()

    ORIGINAL_NAME = qt.Qt.UserRole + 1

    def __initItems(self):
        for colormapName in preferredColormaps():
            index = self.count()
            self.addItem(str.title(colormapName))
            self.setItemIcon(index, self.getIconPreview(colormapName))
            self.setItemData(index, colormapName, role=self.ORIGINAL_NAME)

    def getIconPreview(self, colormapName):
        """Return an icon preview from a LUT name.

        This icons are cached into a global structure.

        :param str colormapName: str
        :rtype: qt.QIcon
        """
        if colormapName not in _colormapIconPreview:
            icon = self.createIconPreview(colormapName)
            _colormapIconPreview[colormapName] = icon
        return _colormapIconPreview[colormapName]

    def createIconPreview(self, colormapName):
        """Create and return an icon preview from a LUT name.

        This icons are cached into a global structure.

        :param str colormapName: Name of the LUT
        :rtype: qt.QIcon
        """
        colormap = Colormap(colormapName)
        size = 32
        lut = colormap.getNColors(size)
        if lut is None or len(lut) == 0:
            return qt.QIcon()

        pixmap = qt.QPixmap(size, size)
        painter = qt.QPainter(pixmap)
        for i in range(size):
            rgb = lut[i]
            r, g, b = rgb[0], rgb[1], rgb[2]
            painter.setPen(qt.QColor(r, g, b))
            painter.drawPoint(qt.QPoint(i, 0))

        painter.drawPixmap(0, 1, size, size - 1, pixmap, 0, 0, size, 1)
        painter.end()

        return qt.QIcon(pixmap)

    def getCurrentName(self):
        return self.itemData(self.currentIndex(), self.ORIGINAL_NAME)

    def findColormap(self, name):
        return self.findData(name, role=self.ORIGINAL_NAME)

    def setCurrentName(self, name):
        index = self.findColormap(name)
        if index < 0:
            index = self.count()
            self.addItem(str.title(name))
            self.setItemIcon(index, self.getIconPreview(name))
            self.setItemData(index, name, role=self.ORIGINAL_NAME)
        self.setCurrentIndex(index)

@enum.unique
class _DataInPlotMode(enum.Enum):
    """Enum for each mode of display of the data in the plot."""
    NONE = 'none'
    RANGE = 'range'
    HISTOGRAM = 'histogram'

class ColormapDialog(qt.QDialog):
    """A QDialog widget to set the colormap.

    :param parent: See :class:`QDialog`
    :param str title: The QDialog title
    """

    visibleChanged = qt.Signal(bool)
    """This event is sent when the dialog visibility change"""

    def __init__(self, parent=None, title="Colormap Dialog"):
        qt.QDialog.__init__(self, parent)
        self.setWindowTitle(title)

        self._colormap = None
        self._data = None
        self._dataInPlotMode = _DataInPlotMode.RANGE

        self._ignoreColormapChange = False
        """Used as a semaphore to avoid editing the colormap object when we are
        only attempt to display it.
        Used instead of n connect and disconnect of the sigChanged. The
        disconnection to sigChanged was also limiting when this colormapdialog
        is used in the colormapaction and associated to the activeImageChanged.
        (because the activeImageChanged is send when the colormap changed and
        the self.setcolormap is a callback)
        """

        self._histogramData = None
        self._minMaxWasEdited = False
        self._initialRange = None

        self._dataRange = None
        """If defined 3-tuple containing information from a data:
        minimum, positive minimum, maximum"""

        self._colormapStoredState = None

        # Make the GUI
        vLayout = qt.QVBoxLayout(self)

        formWidget = qt.QWidget(parent=self)
        vLayout.addWidget(formWidget)
        formLayout = qt.QFormLayout(formWidget)
        formLayout.setContentsMargins(10, 10, 10, 10)
        formLayout.setSpacing(0)

        # Colormap row
        self._comboBoxColormap = _ColormapNameCombox(parent=formWidget)
        self._comboBoxColormap.currentIndexChanged[int].connect(self._updateName)
        formLayout.addRow('Colormap:', self._comboBoxColormap)

        # Normalization row
        self._normButtonLinear = qt.QRadioButton('Linear')
        self._normButtonLinear.setChecked(True)
        self._normButtonLog = qt.QRadioButton('Log')
        self._normButtonLog.toggled.connect(self._activeLogNorm)

        normButtonGroup = qt.QButtonGroup(self)
        normButtonGroup.setExclusive(True)
        normButtonGroup.addButton(self._normButtonLinear)
        normButtonGroup.addButton(self._normButtonLog)
        self._normButtonLinear.toggled[bool].connect(self._updateLinearNorm)

        normLayout = qt.QHBoxLayout()
        normLayout.setContentsMargins(0, 0, 0, 0)
        normLayout.setSpacing(10)
        normLayout.addWidget(self._normButtonLinear)
        normLayout.addWidget(self._normButtonLog)

        formLayout.addRow('Normalization:', normLayout)

        # Min row
        self._minValue = _BoundaryWidget(parent=self, value=1.0)
        self._minValue.textEdited.connect(self._minMaxTextEdited)
        self._minValue.editingFinished.connect(self._minEditingFinished)
        self._minValue.sigValueChanged.connect(self._updateMinMax)
        formLayout.addRow('\tMin:', self._minValue)

        # Max row
        self._maxValue = _BoundaryWidget(parent=self, value=10.0)
        self._maxValue.textEdited.connect(self._minMaxTextEdited)
        self._maxValue.sigValueChanged.connect(self._updateMinMax)
        self._maxValue.editingFinished.connect(self._maxEditingFinished)
        formLayout.addRow('\tMax:', self._maxValue)

        # Add plot for histogram
        self._plotToolbar = qt.QToolBar(self)
        self._plotToolbar.setFloatable(False)
        self._plotToolbar.setMovable(False)
        self._plotToolbar.setIconSize(qt.QSize(8, 8))
        self._plotToolbar.setStyleSheet("QToolBar { border: 0px }")
        self._plotToolbar.setOrientation(qt.Qt.Vertical)

        group = qt.QActionGroup(self._plotToolbar)
        group.setExclusive(True)

        action = qt.QAction("Nothing", self)
        action.setToolTip("No range nor histogram are displayed. No extra computation have to be done.")
        action.setIcon(icons.getQIcon('colormap-none'))
        action.setCheckable(True)
        action.setData(_DataInPlotMode.NONE)
        action.setChecked(action.data() == self._dataInPlotMode)
        self._plotToolbar.addAction(action)
        group.addAction(action)
        action = qt.QAction("Data range", self)
        action.setToolTip("Display the data range within the colormap range. A fast data processing have to be done.")
        action.setIcon(icons.getQIcon('colormap-range'))
        action.setCheckable(True)
        action.setData(_DataInPlotMode.RANGE)
        action.setChecked(action.data() == self._dataInPlotMode)
        self._plotToolbar.addAction(action)
        group.addAction(action)
        action = qt.QAction("Histogram", self)
        action.setToolTip("Display the data histogram within the colormap range. A slow data processing have to be done. ")
        action.setIcon(icons.getQIcon('colormap-histogram'))
        action.setCheckable(True)
        action.setData(_DataInPlotMode.HISTOGRAM)
        action.setChecked(action.data() == self._dataInPlotMode)
        self._plotToolbar.addAction(action)
        group.addAction(action)
        group.triggered.connect(self._displayDataInPlotModeChanged)

        self._plotBox = qt.QWidget(self)
        self._plotInit()

        plotBoxLayout = qt.QHBoxLayout()
        plotBoxLayout.setContentsMargins(0, 0, 0, 0)
        plotBoxLayout.setSpacing(2)
        plotBoxLayout.addWidget(self._plotToolbar)
        plotBoxLayout.addWidget(self._plot)
        plotBoxLayout.setSizeConstraint(qt.QLayout.SetMinimumSize)
        self._plotBox.setLayout(plotBoxLayout)
        vLayout.addWidget(self._plotBox)

        # define modal buttons
        types = qt.QDialogButtonBox.Ok | qt.QDialogButtonBox.Cancel
        self._buttonsModal = qt.QDialogButtonBox(parent=self)
        self._buttonsModal.setStandardButtons(types)
        self.layout().addWidget(self._buttonsModal)
        self._buttonsModal.accepted.connect(self.accept)
        self._buttonsModal.rejected.connect(self.reject)

        # define non modal buttons
        types = qt.QDialogButtonBox.Close | qt.QDialogButtonBox.Reset
        self._buttonsNonModal = qt.QDialogButtonBox(parent=self)
        self._buttonsNonModal.setStandardButtons(types)
        self.layout().addWidget(self._buttonsNonModal)
        self._buttonsNonModal.button(qt.QDialogButtonBox.Close).clicked.connect(self.accept)
        self._buttonsNonModal.button(qt.QDialogButtonBox.Reset).clicked.connect(self.resetColormap)

        # Set the colormap to default values
        self.setColormap(Colormap(name='gray', normalization='linear',
                         vmin=None, vmax=None))

        self.setModal(self.isModal())

        vLayout.setSizeConstraint(qt.QLayout.SetMinimumSize)
        self.setFixedSize(self.sizeHint())
        self._applyColormap()

    def showEvent(self, event):
        self.visibleChanged.emit(True)
        super(ColormapDialog, self).showEvent(event)

    def closeEvent(self, event):
        if not self.isModal():
            self.accept()
        super(ColormapDialog, self).closeEvent(event)

    def hideEvent(self, event):
        self.visibleChanged.emit(False)
        super(ColormapDialog, self).hideEvent(event)

    def close(self):
        self.accept()
        qt.QDialog.close(self)

    def setModal(self, modal):
        assert type(modal) is bool
        self._buttonsNonModal.setVisible(not modal)
        self._buttonsModal.setVisible(modal)
        qt.QDialog.setModal(self, modal)

    def exec_(self):
        wasModal = self.isModal()
        self.setModal(True)
        result = super(ColormapDialog, self).exec_()
        self.setModal(wasModal)
        return result

    def _plotInit(self):
        """Init the plot to display the range and the values"""
        self._plot = PlotWidget()
        self._plot.setDataMargins(yMinMargin=0.125, yMaxMargin=0.125)
        self._plot.getXAxis().setLabel("Data Values")
        self._plot.getYAxis().setLabel("")
        self._plot.setInteractiveMode('select', zoomOnWheel=False)
        self._plot.setActiveCurveHandling(False)
        self._plot.setMinimumSize(qt.QSize(250, 200))
        self._plot.sigPlotSignal.connect(self._plotSlot)

        self._plotUpdate()

    def sizeHint(self):
        return self.layout().minimumSize()

    def _plotUpdate(self, updateMarkers=True):
        """Update the plot content

        :param bool updateMarkers: True to update markers, False otherwith
        """
        colormap = self.getColormap()
        if colormap is None:
            if self._plotBox.isVisibleTo(self):
                self._plotBox.setVisible(False)
                self.setFixedSize(self.sizeHint())
            return

        if not self._plotBox.isVisibleTo(self):
            self._plotBox.setVisible(True)
            self.setFixedSize(self.sizeHint())

        minData, maxData = self._minValue.getFiniteValue(), self._maxValue.getFiniteValue()
        if minData > maxData:
            # avoid a full collapse
            minData, maxData = maxData, minData
        minimum = minData
        maximum = maxData

        if self._dataRange is not None:
            minRange = self._dataRange[0]
            maxRange = self._dataRange[2]
            minimum = min(minimum, minRange)
            maximum = max(maximum, maxRange)

        if self._histogramData is not None:
            minHisto = self._histogramData[1][0]
            maxHisto = self._histogramData[1][-1]
            minimum = min(minimum, minHisto)
            maximum = max(maximum, maxHisto)

        marge = abs(maximum - minimum) / 6.0
        if marge < 0.0001:
            # Smaller that the QLineEdit precision
            marge = 0.0001

        minView, maxView = minimum - marge, maximum + marge

        if updateMarkers:
            # Save the state in we are not moving the markers
            self._initialRange = minView, maxView
        elif self._initialRange is not None:
            minView = min(minView, self._initialRange[0])
            maxView = max(maxView, self._initialRange[1])

        x = [minView, minData, maxData, maxView]
        y = [0, 0, 1, 1]

        self._plot.addCurve(x, y,
                            legend="ConstrainedCurve",
                            color='black',
                            symbol='o',
                            linestyle='-',
                            resetzoom=False)

        if updateMarkers:
            minDraggable = (self._colormap().isEditable() and
                            not self._minValue.isAutoChecked())
            self._plot.addXMarker(
                self._minValue.getFiniteValue(),
                legend='Min',
                text='Min',
                draggable=minDraggable,
                color='blue',
                constraint=self._plotMinMarkerConstraint)

            maxDraggable = (self._colormap().isEditable() and
                            not self._maxValue.isAutoChecked())
            self._plot.addXMarker(
                self._maxValue.getFiniteValue(),
                legend='Max',
                text='Max',
                draggable=maxDraggable,
                color='blue',
                constraint=self._plotMaxMarkerConstraint)

        self._plot.resetZoom()

    def _plotMinMarkerConstraint(self, x, y):
        """Constraint of the min marker"""
        return min(x, self._maxValue.getFiniteValue()), y

    def _plotMaxMarkerConstraint(self, x, y):
        """Constraint of the max marker"""
        return max(x, self._minValue.getFiniteValue()), y

    def _plotSlot(self, event):
        """Handle events from the plot"""
        if event['event'] in ('markerMoving', 'markerMoved'):
            value = float(str(event['xdata']))
            if event['label'] == 'Min':
                self._minValue.setValue(value)
            elif event['label'] == 'Max':
                self._maxValue.setValue(value)

            # This will recreate the markers while interacting...
            # It might break if marker interaction is changed
            if event['event'] == 'markerMoved':
                self._initialRange = None
                self._updateMinMax()
            else:
                self._plotUpdate(updateMarkers=False)

    @staticmethod
    def computeDataRange(data):
        """Compute the data range as used by :meth:`setDataRange`.

        :param data: The data to process
        :rtype: Tuple(float, float, float)
        """
        if data is None or len(data) == 0:
            return None, None, None

        dataRange = min_max(data, min_positive=True, finite=True)
        if dataRange.minimum is None:
            # Only non-finite data
            dataRange = None

        if dataRange is not None:
            min_positive = dataRange.min_positive
            if min_positive is None:
                min_positive = float('nan')
            dataRange = dataRange.minimum, min_positive, dataRange.maximum

        if dataRange is None or len(dataRange) != 3:
            qt.QMessageBox.warning(
                None, "No Data",
                "Image data does not contain any real value")
            dataRange = 1., 1., 10.

        return dataRange

    @staticmethod
    def computeHistogram(data):
        """Compute the data histogram as used by :meth:`setHistogram`.

        :param data: The data to process
        :rtype: Tuple(List(float),List(float)
        """
        _data = data
        if _data.ndim == 3:  # RGB(A) images
            _logger.info('Converting current image from RGB(A) to grayscale\
                in order to compute the intensity distribution')
            _data = (_data[:, :, 0] * 0.299 +
                     _data[:, :, 1] * 0.587 +
                     _data[:, :, 2] * 0.114)

        if len(_data) == 0:
            return None, None

        xmin, xmax = min_max(_data, min_positive=False, finite=True)
        nbins = min(256, int(numpy.sqrt(_data.size)))
        data_range = xmin, xmax

        # bad hack: get 256 bins in the case we have a B&W
        if numpy.issubdtype(_data.dtype, numpy.integer):
            if nbins > xmax - xmin:
                nbins = xmax - xmin

        nbins = max(2, nbins)
        _data = _data.ravel().astype(numpy.float32)

        histogram = Histogramnd(_data, n_bins=nbins, histo_range=data_range)
        return histogram.histo, histogram.edges[0]

    def _getData(self):
        if self._data is None:
            return None
        return self._data()

    def setData(self, data):
        """Store the data as a weakref.

        According to the state of the dialog, the data will be used to display
        the data range or the histogram of the data using :meth:`setDataRange`
        and :meth:`setHistogram`
        """
        oldData = self._getData()
        if oldData is data:
            return

        if data is None:
            self._data = None
        else:
            self._data = weakref.ref(data, self._dataAboutToFinalize)

        self._updateDataInPlot()

    def _setDataInPlotMode(self, mode):
        if self._dataInPlotMode == mode:
            return
        self._dataInPlotMode = mode
        self._updateDataInPlot()

    def _displayDataInPlotModeChanged(self, action):
        mode = action.data()
        self._setDataInPlotMode(mode)

    def _updateDataInPlot(self):
        data = self._getData()
        if data is None:
            self.setDataRange()
            self.setHistogram()
            return

        if data.size == 0:
            # One or more dimensions are equal to 0
            self.setHistogram()
            self.setDataRange()
            return

        mode = self._dataInPlotMode

        if mode == _DataInPlotMode.NONE:
            self.setHistogram()
            self.setDataRange()
        elif mode == _DataInPlotMode.RANGE:
            result = self.computeDataRange(data)
            self.setHistogram()
            self.setDataRange(*result)
        elif mode == _DataInPlotMode.HISTOGRAM:
            # The histogram should be done in a worker thread
            result = self.computeHistogram(data)
            self.setHistogram(*result)
            self.setDataRange()

    def _colormapAboutToFinalize(self, weakrefColormap):
        """Callback when the data weakref is about to be finalized."""
        if self._colormap is weakrefColormap:
            self.setColormap(None)

    def _dataAboutToFinalize(self, weakrefData):
        """Callback when the data weakref is about to be finalized."""
        if self._data is weakrefData:
            self.setData(None)

    def getHistogram(self):
        """Returns the counts and bin edges of the displayed histogram.

        :return: (hist, bin_edges)
        :rtype: 2-tuple of numpy arrays"""
        if self._histogramData is None:
            return None
        else:
            bins, counts = self._histogramData
            return numpy.array(bins, copy=True), numpy.array(counts, copy=True)

    def setHistogram(self, hist=None, bin_edges=None):
        """Set the histogram to display.

        This update the data range with the bounds of the bins.

        :param hist: array-like of counts or None to hide histogram
        :param bin_edges: array-like of bins edges or None to hide histogram
        """
        if hist is None or bin_edges is None:
            self._histogramData = None
            self._plot.remove(legend='Histogram', kind='histogram')
        else:
            hist = numpy.array(hist, copy=True)
            bin_edges = numpy.array(bin_edges, copy=True)
            self._histogramData = hist, bin_edges
            norm_hist = hist / max(hist)
            self._plot.addHistogram(norm_hist,
                                    bin_edges,
                                    legend="Histogram",
                                    color='gray',
                                    align='center',
                                    fill=True)
        self._updateMinMaxData()

    def getColormap(self):
        """Return the colormap description as a :class:`.Colormap`.

        """
        if self._colormap is None:
            return None
        return self._colormap()

    def resetColormap(self):
        """
        Reset the colormap state before modification.

        ..note :: the colormap reference state is the state when set or the
                  state when validated
        """
        colormap = self.getColormap()
        if colormap is not None and self._colormapStoredState is not None:
            if self._colormap()._toDict() != self._colormapStoredState:
                self._ignoreColormapChange = True
                colormap._setFromDict(self._colormapStoredState)
                self._ignoreColormapChange = False
                self._applyColormap()

    def setDataRange(self, minimum=None, positiveMin=None, maximum=None):
        """Set the range of data to use for the range of the histogram area.

        :param float minimum: The minimum of the data
        :param float positiveMin: The positive minimum of the data
        :param float maximum: The maximum of the data
        """
        if minimum is None or positiveMin is None or maximum is None:
            self._dataRange = None
            self._plot.remove(legend='Range', kind='histogram')
        else:
            hist = numpy.array([1])
            bin_edges = numpy.array([minimum, maximum])
            self._plot.addHistogram(hist,
                                    bin_edges,
                                    legend="Range",
                                    color='gray',
                                    align='center',
                                    fill=True)
            self._dataRange = minimum, positiveMin, maximum
        self._updateMinMaxData()

    def _updateMinMaxData(self):
        """Update the min and max of the data according to the data range and
        the histogram preset."""
        colormap = self.getColormap()

        minimum = float("+inf")
        maximum = float("-inf")

        if colormap is not None and colormap.getNormalization() == colormap.LOGARITHM:
            # find a range in the positive part of the data
            if self._dataRange is not None:
                minimum = min(minimum, self._dataRange[1])
                maximum = max(maximum, self._dataRange[2])
            if self._histogramData is not None:
                positives = list(filter(lambda x: x > 0, self._histogramData[1]))
                if len(positives) > 0:
                    minimum = min(minimum, positives[0])
                    maximum = max(maximum, positives[-1])
        else:
            if self._dataRange is not None:
                minimum = min(minimum, self._dataRange[0])
                maximum = max(maximum, self._dataRange[2])
            if self._histogramData is not None:
                minimum = min(minimum, self._histogramData[1][0])
                maximum = max(maximum, self._histogramData[1][-1])

        if not numpy.isfinite(minimum):
            minimum = None
        if not numpy.isfinite(maximum):
            maximum = None

        self._minValue.setDataValue(minimum)
        self._maxValue.setDataValue(maximum)
        self._plotUpdate()

    def accept(self):
        self.storeCurrentState()
        qt.QDialog.accept(self)

    def storeCurrentState(self):
        """
        save the current value sof the colormap if the user want to undo is
        modifications
        """
        colormap = self.getColormap()
        if colormap is not None:
            self._colormapStoredState = colormap._toDict()
        else:
            self._colormapStoredState = None

    def reject(self):
        self.resetColormap()
        qt.QDialog.reject(self)

    def setColormap(self, colormap):
        """Set the colormap description

        :param :class:`Colormap` colormap: the colormap to edit
        """
        assert colormap is None or isinstance(colormap, Colormap)
        if self._ignoreColormapChange is True:
            return

        oldColormap = self.getColormap()
        if oldColormap is colormap:
            return
        if oldColormap is not None:
            oldColormap.sigChanged.disconnect(self._applyColormap)

        if colormap is not None:
            colormap.sigChanged.connect(self._applyColormap)
            colormap = weakref.ref(colormap, self._colormapAboutToFinalize)

        self._colormap = colormap
        self.storeCurrentState()
        self._updateResetButton()
        self._applyColormap()

    def _updateResetButton(self):
        resetButton = self._buttonsNonModal.button(qt.QDialogButtonBox.Reset)
        rStateEnabled = False
        colormap = self.getColormap()
        if colormap is not None and colormap.isEditable():
            # can reset only in the case the colormap changed
            rStateEnabled = colormap._toDict() != self._colormapStoredState
        resetButton.setEnabled(rStateEnabled)

    def _applyColormap(self):
        self._updateResetButton()
        if self._ignoreColormapChange is True:
            return

        colormap = self.getColormap()
        if colormap is None:
            self._comboBoxColormap.setEnabled(False)
            self._normButtonLinear.setEnabled(False)
            self._normButtonLog.setEnabled(False)
            self._minValue.setEnabled(False)
            self._maxValue.setEnabled(False)
        else:
            self._ignoreColormapChange = True

            if colormap.getName() is not None:
                name = colormap.getName()
                self._comboBoxColormap.setCurrentName(name)
                self._comboBoxColormap.setEnabled(self._colormap().isEditable())

            assert colormap.getNormalization() in Colormap.NORMALIZATIONS
            self._normButtonLinear.setChecked(
                colormap.getNormalization() == Colormap.LINEAR)
            self._normButtonLog.setChecked(
                colormap.getNormalization() == Colormap.LOGARITHM)
            vmin = colormap.getVMin()
            vmax = colormap.getVMax()
            dataRange = colormap.getColormapRange()
            self._normButtonLinear.setEnabled(self._colormap().isEditable())
            self._normButtonLog.setEnabled(self._colormap().isEditable())
            self._minValue.setValue(vmin or dataRange[0], isAuto=vmin is None)
            self._maxValue.setValue(vmax or dataRange[1], isAuto=vmax is None)
            self._minValue.setEnabled(self._colormap().isEditable())
            self._maxValue.setEnabled(self._colormap().isEditable())
            self._ignoreColormapChange = False

        self._plotUpdate()

    def _updateMinMax(self):
        if self._ignoreColormapChange is True:
            return

        vmin = self._minValue.getFiniteValue()
        vmax = self._maxValue.getFiniteValue()
        if vmax is not None and vmin is not None and vmax < vmin:
            # If only one autoscale is checked constraints are too strong
            # We have to edit a user value anyway it is not requested
            # TODO: It would be better IMO to disable the auto checkbox before
            # this case occur (valls)
            cmin = self._minValue.isAutoChecked()
            cmax = self._maxValue.isAutoChecked()
            if cmin is False:
                self._minValue.setFiniteValue(vmax)
            if cmax is False:
                self._maxValue.setFiniteValue(vmin)

        vmin = self._minValue.getValue()
        vmax = self._maxValue.getValue()
        self._ignoreColormapChange = True
        colormap = self._colormap()
        if colormap is not None:
            colormap.setVRange(vmin, vmax)
        self._ignoreColormapChange = False
        self._plotUpdate()
        self._updateResetButton()

    def _updateName(self):
        if self._ignoreColormapChange is True:
            return

        if self._colormap():
            self._ignoreColormapChange = True
            self._colormap().setName(
                self._comboBoxColormap.getCurrentName())
            self._ignoreColormapChange = False

    def _updateLinearNorm(self, isNormLinear):
        if self._ignoreColormapChange is True:
            return

        if self._colormap():
            self._ignoreColormapChange = True
            norm = Colormap.LINEAR if isNormLinear else Colormap.LOGARITHM
            self._colormap().setNormalization(norm)
            self._ignoreColormapChange = False

    def _minMaxTextEdited(self, text):
        """Handle _minValue and _maxValue textEdited signal"""
        self._minMaxWasEdited = True

    def _minEditingFinished(self):
        """Handle _minValue editingFinished signal

        Together with :meth:`_minMaxTextEdited`, this avoids to notify
        colormap change when the min and max value where not edited.
        """
        if self._minMaxWasEdited:
            self._minMaxWasEdited = False

            # Fix start value
            if (self._maxValue.getValue() is not None and
                    self._minValue.getValue() > self._maxValue.getValue()):
                self._minValue.setValue(self._maxValue.getValue())
            self._updateMinMax()

    def _maxEditingFinished(self):
        """Handle _maxValue editingFinished signal

        Together with :meth:`_minMaxTextEdited`, this avoids to notify
        colormap change when the min and max value where not edited.
        """
        if self._minMaxWasEdited:
            self._minMaxWasEdited = False

            # Fix end value
            if (self._minValue.getValue() is not None and
                    self._minValue.getValue() > self._maxValue.getValue()):
                self._maxValue.setValue(self._minValue.getValue())
            self._updateMinMax()

    def keyPressEvent(self, event):
        """Override key handling.

        It disables leaving the dialog when editing a text field.
        """
        if event.key() == qt.Qt.Key_Enter and (self._minValue.hasFocus() or
                                               self._maxValue.hasFocus()):
            # Bypass QDialog keyPressEvent
            # To avoid leaving the dialog when pressing enter on a text field
            super(qt.QDialog, self).keyPressEvent(event)
        else:
            # Use QDialog keyPressEvent
            super(ColormapDialog, self).keyPressEvent(event)

    def _activeLogNorm(self, isLog):
        if self._ignoreColormapChange is True:
            return
        if self._colormap():
            self._ignoreColormapChange = True
            norm = Colormap.LOGARITHM if isLog is True else Colormap.LINEAR
            self._colormap().setNormalization(norm)
            self._ignoreColormapChange = False
        self._updateMinMaxData()

max_stars_repo_name: vallsv/silx

max_issues_repo_name: vallsv/silx

max_forks_repo_name: vallsv/silx


text:
"""
test.py - Test network using low-shot pairs
"""

# TODO: Put all the copied files in a temp folder and operate out of there. Or no copy, just shift to the fcn dir
# current system is bad
import caffe
import os
import sys
import numpy as np
import ss_datalayer
import csv
from skimage.io import imsave
from matplotlib.pyplot import imshow,show, figure
import time

# Get image pairs
class LoaderOfPairs(object):
    def __init__(self, profile):
        profile_copy = profile.copy()
        profile_copy['first_label_params'].append(('original_first_label', 1.0, 0.0))
	profile_copy['deploy_mode'] = True
        dbi = ss_datalayer.DBInterface(profile)
        self.PLP = ss_datalayer.PairLoaderProcess(None, None, dbi, profile_copy)
    def get_items(self):
        self.out = self.PLP.load_next_frame(try_mode=False)
	#print  self.out['deploy_info']
        return (np.asarray(self.out['first_img']), #[np.newaxis,:,:,:],
                np.asarray(self.out['first_label']), #[np.newaxis,:,:,:],
                np.asarray(self.out['second_img']), #[np.newaxis,:,:,:],
                np.asarray(self.out['second_label']), #[np.newaxis,:,:,:],
                self.out['deploy_info'])
    def get_items_no_return(self):
        self.out = self.PLP.load_next_frame(try_mode=False)
    # corrects the image to be displayable
    def correct_im(self, im):
        im = (np.transpose(im, (0,2,3,1)))/255.
        im += np.array([0.40787055,  0.45752459,  0.4810938])
        return im[:,:,:,::-1]
    # returns the outputs as images and also the first label in original img size
    def get_items_im(self):
        self.out = self.PLP.load_next_frame(try_mode=False)
        return (self.correct_im(self.out['first_img']),
                self.out['original_first_label'],
                self.correct_im(self.out['second_img']),
                self.out['second_label'][0],
                self.out['deploy_info'])

def __stack(obj_list):
    isinstance(obj_list, list)
    if isinstance(obj_list[0], np.ndarray):
        #arrays = np.stack(obj_list, axis = 0)
        arrays = np.concatenate(obj_list, axis = 0).reshape((len(obj_list),) + obj_list[0].shape)
    else:
        #It should be a list of numbers
        assert(not hasattr(obj_list[0], '__len__'))
        arrays = np.array(obj_list)
    return arrays

def compute_net_inputs(pair_item, input_names):
    inputs = dict()
    for input_name in input_names:
        inputs[input_name] = __stack(pair_item[input_name])
    return inputs
# the z(h)en function for measuring IOU
def measure(y_in, pred_in):
    thresh = .65
    y = y_in>thresh
    pred = pred_in>thresh
    tp = np.logical_and(y,pred).sum()
    tn = np.logical_and(np.logical_not(y), np.logical_not(pred)).sum()
    fp = np.logical_and(np.logical_not(y), pred).sum()
    fn = np.logical_and(y, np.logical_not(pred)).sum()
    return tp, tn, fp, fn

def test_net(loader, model_path, weights_path, output_dir, test_iters, gpu, test_cats_indicator=range(20), save_images=True):
    '''
        Test our one-shot net on test categories

        Inputs:

            loader: data loader
            model_path: specifies the model path, i.e. deploy.prototxt
            weights_path: path to weights file, i.e. caffemodel
            output_dir: where to store your test results
            gpu: specifies which gpu to run on
            test_cats_indicator: a list of indices (from 0 to 19, since we have 20 image classes) that specifies which categories are used for testing
            save_images: whether to visualize the segmented images and save them into output_dir

    '''

    caffe.set_mode_gpu()
    caffe.set_device(gpu)

    # load the parent net
    print model_path
    net = caffe.Net(model_path, weights_path, caffe.TEST)
    pred_dir = os.path.join(output_dir, 'pred')
    soft_mask_dir = os.path.join(pred_dir, 'soft')
    final_mask_dir = os.path.join(pred_dir, 'hard')
    first_label_dir = os.path.join(output_dir, 'first_label')
    os.makedirs(pred_dir)
    os.makedirs(first_label_dir)
    os.makedirs(soft_mask_dir)
    os.makedirs(final_mask_dir)

    # record all the stats
    recorder=[]

    # list of True Pos, True neg, False Pos, False Neg
    # Note: IOUs for all training categories will be zero; they are simply placeholders
    num_classes = 20
    tp_list = [0]*num_classes
    fp_list = [0]*num_classes
    fn_list = [0]*num_classes
    iou_list = [0]*num_classes

    total_time = 0

    total_time_iter = 0

    for ti in range(test_iters):
        iter_start_time = time.time()
        first_img, first_label, second_img, second_label, deploy_info = loader.get_items()
        print '>'*10, 'Test Iteration: ', ti
        inputs = compute_net_inputs(loader.out, net.inputs)
        #TODO: REMOVE THIS OR THE NET DOESNT DO GRAD DESCENT
        # make learning rate zero
        #net.params['w1s'][2].data[...] = 0.0

        #old_w1 = net.params['w1s'][0].data.copy()

        start_time = time.time()
        net.forward(**inputs)
        total_time += time.time()-start_time
        #assert((np.absolute(old_w1 -net.blobs['w1s'].data)).sum()==0.0)

        # get the input, output mask
        pred = net.blobs['score'].data[0,0]
	print "I'm not as u expected"
	print np.unique(second_label[0,0])
        tp, tn, fp, fn = measure(second_label[0,0], net.blobs['score'].data[0,0])
        iou_img = tp/float(max(tp+fp+fn,1))
        class_ind = int(deploy_info['first_semantic_labels'][0][0])-1 # because class indices from 1 in data layer
        tp_list[class_ind] += tp
        fp_list[class_ind] += fp
        fn_list[class_ind] += fn
        # max in case both pred and label are zero
        iou_list = [tp_list[ic] /
                       float(max(tp_list[ic] + fp_list[ic] + fn_list[ic],1))
                       for ic in range(num_classes)]
	print iou_list
        # record
        recorder.append([ti, 0, class_ind, tp, tn, fp, fn, iou_img])
        #debug
        #print 'Min, Max of mask = ', pred.min(), pred.max()

        if save_images:
            # save image
            imsave(os.path.join(soft_mask_dir, '%05d.png' % (ti)), pred)

        Threshold = 0.5
        pred[pred < Threshold] = 0
        pred[pred > 0] = 1

        if save_images:
            imsave(os.path.join(final_mask_dir, '%05d.png' % (ti)), pred)
        # # Debug: Show Stuff
        # imshow(net.blobs['score'].data[0,0])
        # figure()
        # imshow(net.blobs['second_img'].data[0,0])
        # figure()
        # # run net for first img
        # imshow(net.blobs['first_img'].data[0,0])
        # figure()
        # imshow(net.blobs['first_label'].data[0,0])
        # #print iou
        # show()

        # record everything
        if ti%10==0:
            with open(os.path.join(output_dir,'iou_results.txt'), 'w') as iou_file:
                iou_file.write('tp: ' +  ' '.join([str(x) for x in tp_list]) + '\n')
                iou_file.write('fp: ' +  ' '.join([str(x) for x in fp_list]) + '\n')
                iou_file.write('fn: ' +  ' '.join([str(x) for x in fn_list]) + '\n')
                iou_file.write('iou (all categories): ' +  ' '.join([str(x) for x in iou_list]) + '\n')
                iou_file.write('mean_iou (only test categories): ' + str(np.mean(np.take(iou_list, test_cats_indicator))) + '\n')
            with open(os.path.join(output_dir, 'all_results.csv'), 'wb') as allresults:
                    writer = csv.writer(allresults)
                    writer.writerows(recorder)
        total_time_iter += time.time() - iter_start_time

    # record the time per iter
    print '\n--Time per the iteration forward is {}'.format(total_time/float(test_iters))
    print '\n--Time per the whole iteration is {}'.format(total_time_iter/float(test_iters))

    # Print meanIOU
    print '\n--The mean IOU for this fold is {}'.format(np.mean(np.take(iou_list, test_cats_indicator)))

"""
    Test FCN on 15 training categories
"""
def test_net_fcn(loader, model_path, weights_path, output_dir, test_iters, gpu):
    caffe.set_mode_gpu()
    caffe.set_device(gpu)
    # load the parent net
    print model_path
    net = caffe.Net(model_path, weights_path, caffe.TEST)
    # record all the stats
    recorder=[]
    # list of True Pos, True neg, False Pos, False Neg
    num_classes=15
    tp_list = [0]*num_classes
    fp_list = [0]*num_classes
    fn_list = [0]*num_classes
    iou_list = [0]*num_classes

    for ti in range(test_iters):
        loader.get_items_no_return()
        print '>'*10, 'Test Iteration: ', ti
        inputs = compute_net_inputs(loader.out, net.inputs)
        first_img = inputs['first_img']
        first_label = np.asarray(loader.out['first_label'])
        print 'First Img = ', first_img.shape
        print 'First Label = ', first_label.shape
        print 'Label Max = ', first_label.max()
        net.forward(**inputs)
        # get the input, output mask
        pred = net.blobs['score'].data[0,0]

        for c in range(num_classes):
            cind =c+1 # ignore background class
            tp, tn, fp, fn = measure(first_label==cind, net.blobs['score'].data[0,cind])
            tp_list[c] += tp
            fp_list[c] += fp
            fn_list[c] += fn
            # if (first_label==cind).sum() > 5:
            #     # Debug: Show Stuff
            #     imshow(net.blobs['score'].data[0,cind])
            #     figure()
            #     imshow(net.blobs['first_img'].data[0,0])
            #     figure()
            #     imshow(first_label[0,0]==cind)
            #     print (first_label==cind).sum()
            #     print tp, fp, fn
            #     show()

        # record
        recorder.append([ti, 0, 0, tp, tn, fp, fn, 0.0])
        Threshold = 0.5
        pred[pred < Threshold] = 0
        pred[pred > 0] = 1
        #imsave(os.path.join(final_mask_dir, '%05d.png' % (ti)), pred)

    # record everything
    # max in case both pred and label are zero
    iou_list = [tp_list[ic] /
                float(max(tp_list[ic] + fp_list[ic] + fn_list[ic],1))
                for ic in range(num_classes)]
    with open(os.path.join(output_dir,'iou_results.txt'), 'w') as iou_file:
        iou_file.write('tp: ' +  ' '.join([str(x) for x in tp_list]) + '\n')
        iou_file.write('fp: ' +  ' '.join([str(x) for x in fp_list]) + '\n')
        iou_file.write('fn: ' +  ' '.join([str(x) for x in fn_list]) + '\n')
        iou_file.write('iou: ' +  ' '.join([str(x) for x in iou_list]) + '\n')
        iou_file.write('mean_iou: ' + str(np.mean(iou_list)) + '\n')
    with open(os.path.join(output_dir, 'all_results.csv'), 'wb') as allresults:
        writer = csv.writer(allresults)
        writer.writerows(recorder)

if __name__=='__main__':
    # check commandline args
    assert (len(sys.argv) >= 6), 'Usage: test.py [model_path] [weights_path] [output_dir] [test_iters] [ss_profile] [gpu_number=0] [save_images=0/1]'

    model_path = os.path.abspath(sys.argv[1])
    weights_path = os.path.abspath(sys.argv[2])
    output_dir = os.path.abspath(sys.argv[3])
    test_iters = int(sys.argv[4])
    ss_profile = sys.argv[5]
    gpu = 0
    if (len(sys.argv)>6):
        gpu = int(sys.argv[6])
    save_images = True
    if (len(sys.argv)>7):
        save_images = bool(int(sys.argv[7]))

    settings = __import__('ss_settings')
    profile = getattr(settings, ss_profile)

    # Specify what the test categories are (for calculating final meanIOU)
    test_cats_indicator = [ind for ind in range(20) if (profile.default_pascal_cats[ind] in profile.pascal_cats)]

    os.makedirs(output_dir)

    # Pair Loader Process
    loader = LoaderOfPairs(profile)

    # test
    test_net(loader, model_path, weights_path, output_dir, test_iters, gpu, test_cats_indicator, save_images)

max_stars_repo_name: vamsirk/FewShotLearning

max_issues_repo_name: vamsirk/FewShotLearning

max_forks_repo_name: vamsirk/FewShotLearning


text:
from enum import Enum
import ruamel.yaml as yaml
import os
import numpy as np
import exdir

from . import exdir_object as exob

def _quote_strings(value):
    if isinstance(value, str):
        return yaml.scalarstring.DoubleQuotedScalarString(value)
    else:
        try:
            new_result = {}
            for key, val in value.items():
                new_result[key] = _quote_strings(val)
            return new_result
        except AttributeError:
            pass
    return value

class Attribute(object):
    """Attribute class."""

    class Mode(Enum):
        ATTRIBUTES = 1
        METADATA = 2

    def __init__(self, parent, mode, io_mode, path=None, plugin_manager=None):
        self.parent = parent
        self.mode = mode
        self.io_mode = io_mode
        self.path = path or []
        self.plugin_manager = plugin_manager

    def __getitem__(self, name=None):
        meta_data = self._open_or_create()

        for plugin in self.plugin_manager.attribute_plugins.read_order:
            meta_data = plugin.prepare_read(meta_data)

        for i in self.path:
            meta_data = meta_data[i]
        if name is not None:
            meta_data = meta_data[name]
        if isinstance(meta_data, dict):
            return Attribute(
                self.parent, self.mode, self.io_mode, self.path + [name],
                plugin_manager=self.plugin_manager
            )
        else:
            return meta_data

    def __setitem__(self, name, value):
        meta_data = self._open_or_create()

        # if isinstance(name, np.integer):
        #     key = int(name)
        # else:
        #     key = name
        key = name

        sub_meta_data = meta_data
        for i in self.path:
            sub_meta_data = sub_meta_data[i]
        sub_meta_data[key] = value

        self._set_data(meta_data)

    def __contains__(self, name):
        meta_data = self._open_or_create()
        for i in self.path:
            meta_data = meta_data[i]
        return name in meta_data

    def keys(self):
        meta_data = self._open_or_create()
        for i in self.path:
            meta_data = meta_data[i]
        return meta_data.keys()

    def to_dict(self):
        meta_data = self._open_or_create()
        for i in self.path:  # TODO check if this is necesary
            meta_data = meta_data[i]

        for plugin in self.plugin_manager.attribute_plugins.read_order:
            meta_data = plugin.prepare_read(meta_data)

        return meta_data

    def items(self):
        meta_data = self._open_or_create()
        for i in self.path:
            meta_data = meta_data[i]
        return meta_data.items()

    def values(self):
        meta_data = self._open_or_create()
        for i in self.path:
            meta_data = meta_data[i]
        return meta_data.values()

    def _set_data(self, meta_data):
        if self.io_mode == exob.Object.OpenMode.READ_ONLY:
            raise IOError("Cannot write in read only ("r") mode")

        for plugin in self.plugin_manager.attribute_plugins.write_order:
            meta_data = plugin.prepare_write(meta_data)

        meta_data_quoted = _quote_strings(meta_data)

        with self.filename.open("w", encoding="utf-8") as meta_file:
            yaml.dump(
                meta_data_quoted,
                meta_file,
                default_flow_style=False,
                allow_unicode=True,
                Dumper=yaml.RoundTripDumper
            )

    # TODO only needs filename, make into free function
    def _open_or_create(self):
        meta_data = {}
        if self.filename.exists():  # NOTE str for Python 3.5 support
            with self.filename.open("r", encoding="utf-8") as meta_file:
                meta_data = yaml.safe_load(meta_file)
        return meta_data

    def __iter__(self):
        for key in self.keys():
            yield key

    @property
    def filename(self):
        if self.mode == self.Mode.METADATA:
            return self.parent.meta_filename
        else:
            return self.parent.attributes_filename

    def __len__(self):
        return len(self.keys())

    def update(self, value):
        for key in value:
            self[key] = value[key]

    def __str__(self):
        string = ""
        for key in self:
            string += "{}: {},".format(key, self[key])
        return "Attribute({}, {{{}}})".format(self.parent.name, string)

max_stars_repo_name: quentinsf/exdir

max_issues_repo_name: quentinsf/exdir

max_forks_repo_name: quentinsf/exdir


text:
"""An example of training PPO against OpenAI Gym Atari Envs.

This script is an example of training a PPO agent on Atari envs.

To train PPO for 10M timesteps on Breakout, run:
    python train_ppo_ale.py

To train PPO using a recurrent model on a flickering Atari env, run:
    python train_ppo_ale.py --recurrent --flicker --no-frame-stack
"""
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import
from builtins import *  # NOQA
from future import standard_library
standard_library.install_aliases()  # NOQA
import argparse
import os

import chainer
from chainer import functions as F
from chainer import links as L
import numpy as np

import chainerrl
from chainerrl.agents import PPO
from chokozainerrl import experiments
from chokozainerrl import tools
from chainerrl import misc
from chainerrl.wrappers import atari_wrappers

def make_args(argstr):
    parser = argparse.ArgumentParser()
    parser.add_argument('--mode', type=str, default='check')
    parser.add_argument('--num-envs', type=int, default=1)
    parser.add_argument('--env', type=str, default='BreakoutNoFrameskip-v4')
    parser.add_argument('--outdir', type=str, default='results')
    parser.add_argument('--gpu', type=int, default=0)
    parser.add_argument('--load-agent', type=str, default=None)
    parser.add_argument('--log-type',type=str,default="full_stream")
    parser.add_argument('--save-mp4',type=str,default="test.mp4")

    parser.add_argument('--steps', type=int, default=5 * 10 ** 7)
    parser.add_argument('--step-offset', type=int, default=0)
    parser.add_argument('--checkpoint-frequency', type=int,default=None)
    parser.add_argument('--max-frames', type=int,default=30 * 60 * 60)  # 30 minutes with 60 fps
    parser.add_argument('--eval-interval', type=int, default=10 ** 5)
    parser.add_argument('--eval-n-runs', type=int, default=10)

    parser.add_argument('--seed', type=int, default=0)
    parser.add_argument('--lr', type=float, default=2.5e-4)
    parser.add_argument('--update-interval', type=int, default=128 * 8)
    parser.add_argument('--batchsize', type=int, default=32 * 8)
    parser.add_argument('--epochs', type=int, default=4)
    parser.add_argument('--log-interval', type=int, default=10000)
    parser.add_argument('--recurrent', action='store_true', default=False)
    parser.add_argument('--flicker', action='store_true', default=False)
    parser.add_argument('--no-frame-stack', action='store_true', default=False)
    parser.add_argument('--render', action='store_true', default=False)
    parser.add_argument('--monitor', action='store_true', default=False)

    myargs = parser.parse_args(argstr)
    return myargs

def main(args):
    import logging
    logging.basicConfig(level=logging.INFO, filename='log')

    if(type(args) is list):
        args=make_args(args)
    if not os.path.exists(args.outdir):
        os.makedirs(args.outdir)

    # Set a random seed used in ChainerRL.
    misc.set_random_seed(args.seed, gpus=(args.gpu,))

    # Set different random seeds for different subprocesses.
    # If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
    # If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
    process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
    assert process_seeds.max() < 2 ** 32

    def make_env(idx, test):
        # Use different random seeds for train and test envs
        process_seed = int(process_seeds[idx])
        env_seed = 2 ** 32 - 1 - process_seed if test else process_seed
        env = atari_wrappers.wrap_deepmind(
            atari_wrappers.make_atari(args.env, max_frames=args.max_frames),
            episode_life=not test,
            clip_rewards=not test,
            flicker=args.flicker,
            frame_stack=not args.no_frame_stack,
        )
        env.seed(env_seed)
        if args.monitor:
            env = chainerrl.wrappers.Monitor(
                env, args.outdir,
                mode='evaluation' if test else 'training')
        if args.render:
            env = chainerrl.wrappers.Render(env)
        return env

    def make_env_check():
        # Use different random seeds for train and test envs
        env_seed = args.seed
        env = atari_wrappers.wrap_deepmind(
            atari_wrappers.make_atari(args.env, max_frames=args.max_frames),
            episode_life=True,
            clip_rewards=True)
        env.seed(int(env_seed))
        return env

    def make_batch_env(test):
        return chainerrl.envs.MultiprocessVectorEnv(
            [(lambda: make_env(idx, test))
             for idx, env in enumerate(range(args.num_envs))])

    sample_env = make_env(0, test=False)
    print('Observation space', sample_env.observation_space)
    print('Action space', sample_env.action_space)
    n_actions = sample_env.action_space.n

    winit_last = chainer.initializers.LeCunNormal(1e-2)
    if args.recurrent:
        model = chainerrl.links.StatelessRecurrentSequential(
            L.Convolution2D(None, 32, 8, stride=4),
            F.relu,
            L.Convolution2D(None, 64, 4, stride=2),
            F.relu,
            L.Convolution2D(None, 64, 3, stride=1),
            F.relu,
            L.Linear(None, 512),
            F.relu,
            L.NStepGRU(1, 512, 512, 0),
            chainerrl.links.Branched(
                chainer.Sequential(
                    L.Linear(None, n_actions, initialW=winit_last),
                    chainerrl.distribution.SoftmaxDistribution,
                ),
                L.Linear(None, 1),
            )
        )
    else:
        model = chainer.Sequential(
            L.Convolution2D(None, 32, 8, stride=4),
            F.relu,
            L.Convolution2D(None, 64, 4, stride=2),
            F.relu,
            L.Convolution2D(None, 64, 3, stride=1),
            F.relu,
            L.Linear(None, 512),
            F.relu,
            chainerrl.links.Branched(
                chainer.Sequential(
                    L.Linear(None, n_actions, initialW=winit_last),
                    chainerrl.distribution.SoftmaxDistribution,
                ),
                L.Linear(None, 1),
            )
        )

    # Draw the computational graph and save it in the output directory.
    fake_obss = np.zeros(
        sample_env.observation_space.shape, dtype=np.float32)[None]
    if args.recurrent:
        fake_out, _ = model(fake_obss, None)
    else:
        fake_out = model(fake_obss)
    chainerrl.misc.draw_computational_graph(
        [fake_out], os.path.join(args.outdir, 'model'))

    opt = chainer.optimizers.Adam(alpha=args.lr, eps=1e-5)
    opt.setup(model)
    opt.add_hook(chainer.optimizer.GradientClipping(0.5))

    def phi(x):
        # Feature extractor
        return np.asarray(x, dtype=np.float32) / 255

    agent = PPO(
        model,
        opt,
        gpu=args.gpu,
        phi=phi,
        update_interval=args.update_interval,
        minibatch_size=args.batchsize,
        epochs=args.epochs,
        clip_eps=0.1,
        clip_eps_vf=None,
        standardize_advantages=True,
        entropy_coef=1e-2,
        recurrent=args.recurrent,
    )

    if args.load_agent:
        agent.load(args.load_agent)

    if (args.mode=='train'):
        step_hooks = []
        # Linearly decay the learning rate to zero
        def lr_setter(env, agent, value):
            agent.optimizer.alpha = value

        step_hooks.append(
            experiments.LinearInterpolationHook(
                args.steps, args.lr, 0, lr_setter))

        experiments.train_agent_batch_with_evaluation(
            agent=agent,
            env=make_batch_env(False),
            eval_env=make_batch_env(True),
            outdir=args.outdir,
            steps=args.steps,
            eval_n_steps=None,
            eval_n_episodes=args.eval_n_runs,
            step_offset=args.step_offset,
            checkpoint_freq=args.checkpoint_frequency,
            eval_interval=args.eval_interval,
            log_interval=args.log_interval,
            save_best_so_far_agent=False,
            step_hooks=step_hooks,
            log_type=args.log_type
        )
    elif (args.mode=='check'):
        return tools.make_video.check(env=make_env_check(),agent=agent,save_mp4=args.save_mp4)

    elif (args.mode=='growth'):
        return tools.make_video.growth(env=make_env_check(),agent=agent,outdir=args.outdir,max_num=args.max_frames,save_mp4=args.save_mp4)

max_stars_repo_name: chokozainer/chokozainerrl

max_issues_repo_name: chokozainer/chokozainerrl

max_forks_repo_name: chokozainer/chokozainerrl


text:
import pandas as pd
import numpy as np

def recall(label_col, prediction_col, k=None, use_random_shuffle=True):
    """Both map should be pandas dataframe column called 'score' with index 'id'!"""
    if use_random_shuffle:
        # do random shuffle due to ties in the predicted scores (prediction_order can change between ties)
        prediction_col = prediction_col.sample(frac=1)
    if k == None:
        raise RuntimeError("You must define 'k' for recall!")
    predicted_items = set(prediction_col.sort_values("score",ascending=False).index[:k])
    relevant_items = set(label_col[label_col["score"] > 0.999].index)
    if len(relevant_items) > 0.0:
        intersection = predicted_items.intersection(relevant_items)
        return 1.0 * len(intersection) / len(relevant_items)
    else:
        return 0.0

def precision(label_col, prediction_col, k=None, use_random_shuffle=True):
    """Both map should be pandas dataframe column called 'score' with index 'id'!"""
    if use_random_shuffle:
        # do random shuffle due to ties in the predicted scores (prediction_order can change between ties)
        prediction_col = prediction_col.sample(frac=1)
    if k == None:
        raise RuntimeError("You must define 'k' for precision!")
    predicted_items = set(prediction_col.sort_values("score",ascending=False).index[:k])
    relevant_items = set(label_col[label_col["score"] > 0.999].index)
    if len(relevant_items) > 0.0:
        intersection = predicted_items.intersection(relevant_items)
        return 1.0 * len(intersection) / k
    else:
        return 0.0

max_stars_repo_name: vishalbelsare/online-centrality

max_issues_repo_name: vishalbelsare/online-centrality

max_forks_repo_name: vishalbelsare/online-centrality


text:
import unittest

import networkx as nx

import dimod

import dwave_networkx as dnx
import dwave_networkx.algorithms.TSP as tsp

class TestIsHamiltonCycle(unittest.TestCase):
    def test_empty(self):
        G = nx.Graph()

        self.assertTrue(tsp.is_hamiltonian_path(G, []))

    def test_K1(self):
        G = nx.complete_graph(1)

        self.assertTrue(tsp.is_hamiltonian_path(G, [0]))
        self.assertFalse(tsp.is_hamiltonian_path(G, []))

    def test_K2(self):
        G = nx.complete_graph(2)

        self.assertTrue(tsp.is_hamiltonian_path(G, [0, 1]))
        self.assertTrue(tsp.is_hamiltonian_path(G, [1, 0]))
        self.assertFalse(tsp.is_hamiltonian_path(G, [0]))
        self.assertFalse(tsp.is_hamiltonian_path(G, [1]))
        self.assertFalse(tsp.is_hamiltonian_path(G, []))

    def test_K3(self):
        G = nx.complete_graph(3)

        self.assertTrue(tsp.is_hamiltonian_path(G, [0, 1, 2]))
        self.assertTrue(tsp.is_hamiltonian_path(G, [1, 0, 2]))
        self.assertFalse(tsp.is_hamiltonian_path(G, [0, 1]))
        self.assertFalse(tsp.is_hamiltonian_path(G, [0]))
        self.assertFalse(tsp.is_hamiltonian_path(G, [1]))
        self.assertFalse(tsp.is_hamiltonian_path(G, []))

class TestTSP(unittest.TestCase):

    def test_TSP_basic(self):
        """Runs the function on some small and simple graphs, just to make
        sure it works in basic functionality.
        """
        G = nx.complete_graph(4)
        for u,v in G.edges(): G[u][v]['weight']=1
        route = tsp.traveling_salesman(G, dimod.ExactSolver())
        self.assertTrue(tsp.is_hamiltonian_path(G, route))

        G = nx.complete_graph(4)
        for u,v in G.edges(): G[u][v]['weight']=u+v
        route = tsp.traveling_salesman(G, dimod.ExactSolver(), lagrange=10.0)
        self.assertTrue(tsp.is_hamiltonian_path(G, route))

    def test_dimod_vs_list(self):
        G = nx.complete_graph(4)
        for u,v in G.edges(): G[u][v]['weight']=1

        route = tsp.traveling_salesman(G, dimod.ExactSolver())
        route = tsp.traveling_salesman(G, dimod.SimulatedAnnealingSampler())

reason: import networkx


text:
from styx_msgs.msg import TrafficLight
import tensorflow as tf
import numpy as np
import rospy
from cv_bridge import CvBridge, CvBridgeError
from sensor_msgs.msg import Image
import os.path

TRAFFIC_LIGHT_COLORS = ["GREEN", "RED", "YELLOW"]
CONFIDENT_THRESHOLD = 0.5

class TLClassifier(object):
    def __init__(self):
        #TODO load classifier
        MODEL_PATH = os.path.abspath(rospy.get_param('model_name'))
        # rospy.loginfo("MODEL_PATH: {}".format(MODEL_PATH))
        self.COLOR_ARRAY = [(0, 255, 0), (255, 0, 0), (255, 255, 0)]

        self.bridge = CvBridge()
        self.image_pub = rospy.Publisher("processed_image", Image, queue_size=1)

        # Load tensorflow graph
        self.detection_graph = tf.Graph()
        with self.detection_graph.as_default():
            loaded_graph_def = tf.GraphDef()
            with tf.gfile.GFile(MODEL_PATH, 'rb') as fid:
                serialized_graph = fid.read()
                loaded_graph_def.ParseFromString(serialized_graph)
                tf.import_graph_def(loaded_graph_def, name='')

            # Get tensors from loaded graph
            self.image_tensor = self.detection_graph.get_tensor_by_name('image_tensor:0')
            self.detection_boxes = self.detection_graph.get_tensor_by_name('detection_boxes:0')
            self.detection_scores = self.detection_graph.get_tensor_by_name('detection_scores:0')
            self.detection_classes = self.detection_graph.get_tensor_by_name('detection_classes:0')
            self.num_detections = self.detection_graph.get_tensor_by_name('num_detections:0')

        # Get session
        config = tf.ConfigProto(log_device_placement=False)
        config.gpu_options.allow_growth = True
        config.gpu_options.per_process_gpu_memory_fraction = 0.5
        self.sess = tf.Session(graph=self.detection_graph, config=config)

    def get_classification(self, image):
        """Determines the color of the traffic light in the image

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """
        #TODO implement light color prediction
        with self.detection_graph.as_default():
            # expand simention to reshape input image to [1, None, None, 3].
            img_expanded = np.expand_dims(image, axis=0)
            feed_dict = {
                self.image_tensor: img_expanded
            }
            boxes, scores, classes, num_detections = self.sess.run(
                [self.detection_boxes, self.detection_scores,
                 self.detection_classes, self.num_detections], feed_dict=feed_dict)
            outimage = image
            try:
                self.image_pub.publish(self.bridge.cv2_to_imgmsg(outimage, "rgb8"))
            except CvBridgeError as e:
                rospy.logerror(e)

            # Get the most likely color with the highest score
            color = None
            highest_score = scores[0][0]
            # rospy.logdebug("num_detections: {}\n".format(num_detections))
            # rospy.logdebug("scores: {}\n".format(scores))
            # rospy.logdebug("classes: {}\n".format(classes))
            most_likely_class = classes[0][0]
            if num_detections > 0:
                if highest_score >= CONFIDENT_THRESHOLD:
                    color = most_likely_class
                else:
                    log_msg = '# detections is {}, but confident {} is not enough'.format(
                        num_detections, highest_score)
                    rospy.loginfo(log_msg)

            if color == 1:
                rospy.loginfo('Infered traffic light is GREEN')
                return TrafficLight.GREEN
            elif color == 2:
                rospy.loginfo('Infered traffic light is RED')
                return TrafficLight.RED
            elif color == 3:
                rospy.loginfo('Infered traffic light is YELLOW')
                return TrafficLight.YELLOW
            else:
                rospy.loginfo('UNKNOWN traffic light')
                return TrafficLight.UNKNOWN


text:
"""Computing energy in PPMI-weighted matrices and in singular values."""

import os
import numpy as np
from scipy import sparse

import entropix.utils.metrix as metrix

if __name__ == '__main__':
    SVD_DIRPATH = '/Users/akb/Github/entropix/models/frontiers/ppmi/'
    RESULTS_FILEPATH = '/Users/akb/Github/entropix/models/frontiers/results/xp600.results'
    START = 0
    END = 300
    print('Running entropix XP#600')
    # MODEL_NAMES = ['enwiki07', 'oanc', 'enwiki2', 'acl', 'enwiki4', 'bnc',
    #                'enwiki']
    # MODEL_NAMES = ['enwiki07', 'oanc', 'enwiki2', 'enwiki4', 'bnc', 'enwiki']
    MODEL_NAMES = ['acl']
    for name in MODEL_NAMES:
        ppmi_filepath = os.path.join(SVD_DIRPATH, '{}-ppmi.npz'.format(name))
        singvalues_filepath = os.path.join(SVD_DIRPATH, '{}-singvalues.npy'.format(name))
        ppmi = sparse.load_npz(ppmi_filepath)
        singvalues = np.load(singvalues_filepath)
        total_energy = metrix.energy(ppmi)
        total_svd_energy = metrix.energy(singvalues)
        reduced_svd_energy = metrix.energy(singvalues, start=START, end=END)
        per_full = round((total_svd_energy / total_energy) * 100, 1)
        per_reduced = round((reduced_svd_energy / total_energy) * 100, 1)
        print('{}\t{}\t{}'.format(name, per_full, per_reduced))

reason: import numpy,from scipy