text:
#!/usr/bin/env python
# -*- mode: python; coding: utf-8 -*-

##################################################################
# Documentation
"""Module providing abstract base for all sentiment analyzers.

Attributes:
  BaseAnalyzer (class): abstract base for all sentiment analyzers

"""

##################################################################
# Imports
from __future__ import absolute_import, print_function, unicode_literals
from six import iteritems
try:
    from cPickle import dump, load
except ImportError:
    from _pickle import dump, load

import gc
import numpy as np
import os

from cgsa.base import BaseAnalyzer
from cgsa.utils.common import LOGGER
from cgsa.constants import (DFLT_MODEL_PATH, DFLT_W2V_PATH, BAZIOTIS,
                            BILSTM, GAMON, GUENTHER, HU_LIU, JUREK,
                            KOLCHYNA, LBA, MOHAMMAD, MUSTO, MVRNN, RNN, RNTN,
                            SEVERYN, TABOADA, YESSENALINA, CLS2IDX, IDX2CLS)
from cgsa.dl.base import DLBaseAnalyzer
from cgsa.judge import DefaultJudge
from cgsa.utils import balance_data

from cgsa.utils.word2vec import Word2Vec

##################################################################
# Variables and Constants
DEBUG_NFEATS = 10

##################################################################
# Classes
class SentimentAnalyzer(object):
    """Main class for coarse-grained sentiment analyzers.

    Attributes:
      models (list(BaseSenser)):
        sense disambiguation models
      judge (cgsa.Judge):
        meta-classifier

    """
    @staticmethod
    def load(a_path, w2v_path, on_demand=False):
        """Load serialized model(s) from disc.

        Args:
          a_path (str): path to file from which to load the model
          on_demand (bool): load models later (when explicitly asked for)

        Returns:
          (void)

        """
        # load paths to serialized models
        LOGGER.debug("Loading analyzer from file: %s", a_path)
        with open(a_path, "rb") as ifile:
            analyzer = load(ifile)
        # normalize paths to serialized models
        analyzer._dirname = os.path.dirname(a_path)
        analyzer._logger = LOGGER
        if analyzer._w2v or analyzer._lstsq:
            if w2v_path != analyzer._w2v_path:
                LOGGER.warn(
                    "Classifier %r was trained with a different embedding"
                    " file: trained with %s vs. testing with %s", analyzer,
                    analyzer._w2v_path, w2v_path
                )
            analyzer._embeddings = Word2Vec(w2v_path)
        if not on_demand:
            analyzer._models = [
                model_i
                for model_i in SentimentAnalyzer._load_models(
                        analyzer
                )
            ]
        return analyzer

    @staticmethod
    def _load_models(a_analyzer):
        """Load serialized sub-model(s) from disc.

        Args:
          a_analyzer (SentimentAnalyzer): instance of this class

        Yields:
          (cgsa.base.BaseAnalyzer): loaded submodel

        """
        for mpath_i in a_analyzer._model_paths:
            a_analyzer._logger.debug(
                "Loading model from file: %s (dirname: %s)",
                mpath_i, a_analyzer._dirname)
            with open(os.path.join(a_analyzer._dirname,
                                   mpath_i), "rb") as ifile:
                model_i = BaseAnalyzer.load(a_analyzer._dirname, ifile)
                model_i.restore(a_analyzer._embeddings)
                yield model_i

    def __init__(self, a_models, *args, **kwargs):
        """Class constructor.

        Args:
          a_models (list[str]): type of the models to train
          a_args (list[str]): arguments to use for initializing models
          a_kwargs (dict): keyword arguments to use for initializing models

        """
        self._models = []
        self.judge = DefaultJudge()

        self._dirname = None
        self._n_cls = 0
        self._wbench = None
        self._logger = LOGGER
        self._model_paths = []
        self._trained_models = []
        self._w2v = kwargs.get("w2v", False)
        self._lstsq = kwargs.get("lstsq", False)
        self._w2v_path = os.path.abspath(
            kwargs.pop("w2v_path", DFLT_W2V_PATH)
        )
        self._embeddings = None
        if self._w2v or self._lstsq:
            self._embeddings = Word2Vec(self._w2v_path)
        kwargs["embeddings"] = self._embeddings
        self._init_models(a_models, *args, **kwargs)

    def train(self, a_train_data, a_dev_data=None,
              a_path=DFLT_MODEL_PATH, a_grid_search=True,
              a_multi_gpu=False, a_balance=False):
        """Train specified model(s) on the provided data.

        Args:
          a_train_data (list or None):
            training set
          a_dev_data (list or None):
            development set
          a_path (str):
            path for storing the model
          a_grid_search (bool):
            use grid search in order to determine hyper-paramaters of
            the model
          a_multi_gpu (bool): train model on multiple GPUs
          a_balance (bool): balance dataset to get equal number of instances
            for all classes (via downsampling)

        Returns:
          void:

        """
        # convert polarity classes to integer indices
        self._n_cls = len(
            set(t.label for t in a_train_data) |
            set(t.label for t in (a_dev_data if a_dev_data else []))
        )
        train_x, train_y = self._generate_ts(a_train_data)
        dev_x, dev_y = self._generate_ts(a_dev_data)
        dirname = os.path.dirname(a_path) if a_path else None
        # separately train and dump each model
        for i, model_i in enumerate(self._models):
            try:
                if a_balance:
                    train_x, train_y = balance_data(train_x, train_y)
                model_i.train(train_x, train_y, dev_x, dev_y,
                              a_grid_search=a_grid_search,
                              a_multi_gpu=a_multi_gpu)
            except:
                # in the end, we are going to switch to the fault-tolerant mode
                raise
            else:
                if dirname:
                    self._save_model(model_i, dirname)
                    self._models[i] = None
        if a_path:
            LOGGER.debug("Saving analyzer in %s...", a_path)
            self._reset()
            with open(a_path, "wb") as ofile:
                dump(self, ofile)
            LOGGER.debug("Analyzer saved...")

    def batch_predict(self, a_instances):
        """Predict multiple input instances at once.

        Models will be loaded

        Args:
          a_models (list[str]): type of the models to train

        """
        # create a workspace for doing the predictions
        probs = np.zeros((len(a_instances),
                          len(self._model_paths),
                          len(IDX2CLS)
                          ))
        # load each trained model and let it predict the classes
        for i, model_i in enumerate(SentimentAnalyzer._load_models(self)):
            for inst_j, y_j in zip(a_instances, probs):
                model_i.predict_proba(inst_j, y_j[i])
            # unload the moddel to save some disc space
            model_i = None
            gc.collect()
        # let judge merge the decisions
        for inst_j, y_j in zip(a_instances, probs):
            lbl_idx, _ = self.judge.predict(y_j)
            inst_j.label = IDX2CLS[lbl_idx]

    def predict(self, instance):
        """Predict label of a single input instance.

        Args:
          instance (cgsa.utils.data.Tweet): input instance to classify

        Returns:
          str: predicted label

        Note:
          modifies input tweet in place

        """
        if self._wbench is None:
            self._wbench = np.zeros((len(self._models), len(CLS2IDX)))
        else:
            self._wbench *= 0
        # let each trained model predict the probabilities of classes
        for i, model_i in enumerate(self._models):
            model_i.predict_proba(instance, self._wbench[i])
        # let the judge unite the decisions
        lbl_idx, _ = self.judge.predict(self._wbench)
        return IDX2CLS[lbl_idx]

    def debug(self, instance):
        """Explain predictions of each classifier.

        Args:
          instance (cgsa.utils.data.Tweet): input instance to classify

        Returns:
          str: predicted label

        Note:
          modifies input tweet in place

        """
        n_classes = len(CLS2IDX)
        if self._wbench is None:
            self._wbench = np.zeros((len(self._models), n_classes))
        else:
            self._wbench *= 0

        from .explainer import Explainer
        explainer = Explainer(class_names=IDX2CLS)
        # let each trained model predict the probabilities of classes
        for i, model_i in enumerate(self._models):
            self._logger.info("Considering model (%d): %r", i, model_i)
            model_i.predict_proba(instance, self._wbench[i])
            self._logger.info("Predicted scores: %r", self._wbench[i])
            # explanations
            predict_proba_raw = model_i.predict_proba
            if hasattr(model_i, "predict_proba_raw"):
                predict_proba_raw = model_i.predict_proba_raw

            explanations = explainer.explain_instance(
                instance,
                predict_proba_raw,
                num_features=DEBUG_NFEATS,
                labels=[y for y in range(len(IDX2CLS))]
            )
            for i, cls in iteritems(IDX2CLS):
                self._logger.info("Class: %s", cls)
                explanation = explanations.as_list(label=i)
                self._logger.info("Explanation: %r", explanation)
        # let the judge unite the decisions
        self._logger.info("All predicted scores: %r", self._wbench)
        lbl_idx, _ = self.judge.predict(self._wbench)
        self._logger.info("Judge prediction: %r", self._wbench)
        return IDX2CLS[lbl_idx]

    def save(self, a_path):
        """Dump model to disc.

        Args:
          a_models (list[str]): type of the models to train

        """
        dirname = self._check_path(a_path)
        # store each trained model
        for i, model_i in enumerate(self._models):
            self._logger.debug("Saving model in %s", dirname)
            self._save_model(model_i, dirname)
            self._models[i] = model_i = None
            gc.collect()
        self._reset()
        dump(self, a_path)

    def _check_path(self, a_path):
        """Check whether path can be created and is writable.

        Args:
          a_path (str): file path to be checked

        Returns:
          str: directory name of the path

        """
        dirname = os.path.dirname(a_path)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        elif not os.path.exists(a_path):
            if not os.access(dirname, os.W_OK) or \
               not os.path.isdir(dirname):
                raise RuntimeError("Cannot write to directory '{:s}'.".format(
                    dirname))
        else:
            if not os.access(a_path, os.W_OK):
                raise RuntimeError("Cannot write to file '{:s}'.".format(
                    a_path))
        return dirname

    def _init_models(self, a_models, *a_args, **a_kwargs):
        """Initialize specified models.

        Args:
          a_models (list[str]): type of the models to train
          a_args (list[str]): arguments to use for initializing models
          a_kwargs (dict): keyword arguments to use for initializing models

        Returns:
          void:

        """
        for model_i in a_models:
            if model_i == BAZIOTIS:
                from cgsa.dl.baziotis import BaziotisAnalyzer
                analyzer_cls = BaziotisAnalyzer
            elif model_i == BILSTM:
                from cgsa.dl.bilstm import BiLSTMAnalyzer
                analyzer_cls = BiLSTMAnalyzer
            elif model_i == GAMON:
                from cgsa.ml.gamon import GamonAnalyzer
                analyzer_cls = GamonAnalyzer
            elif model_i == GUENTHER:
                from cgsa.ml.guenther import GuentherAnalyzer
                analyzer_cls = GuentherAnalyzer
            elif model_i == HU_LIU:
                from cgsa.lexicon.hu_liu import HuLiuAnalyzer
                analyzer_cls = HuLiuAnalyzer
            elif model_i == JUREK:
                from cgsa.lexicon.jurek import JurekAnalyzer
                analyzer_cls = JurekAnalyzer
            elif model_i == KOLCHYNA:
                from cgsa.lexicon.kolchyna import KolchynaAnalyzer
                analyzer_cls = KolchynaAnalyzer
            elif model_i == LBA:
                from cgsa.dl.lba import LBAAnalyzer
                analyzer_cls = LBAAnalyzer
            elif model_i == MUSTO:
                from cgsa.lexicon.musto import MustoAnalyzer
                analyzer_cls = MustoAnalyzer
            elif model_i == MOHAMMAD:
                from cgsa.ml.mohammad import MohammadAnalyzer
                analyzer_cls = MohammadAnalyzer
            elif model_i == MVRNN:
                from cgsa.dl.mvrnn import MVRNNAnalyzer
                analyzer_cls = MVRNNAnalyzer
            elif model_i == RNN:
                from cgsa.dl.rnn import RNNAnalyzer
                analyzer_cls = RNNAnalyzer
            elif model_i == RNTN:
                from cgsa.dl.rntn import RNTNAnalyzer
                analyzer_cls = RNTNAnalyzer
            elif model_i == SEVERYN:
                from cgsa.dl.severyn import SeverynAnalyzer
                analyzer_cls = SeverynAnalyzer
            elif model_i == TABOADA:
                from cgsa.lexicon.taboada import TaboadaAnalyzer
                analyzer_cls = TaboadaAnalyzer
            elif model_i == YESSENALINA:
                from cgsa.dl.yessenalina import YessenalinaAnalyzer
                analyzer_cls = YessenalinaAnalyzer
            else:
                raise NotImplementedError
            self._models.append(analyzer_cls(*a_args, **a_kwargs))

    def _generate_ts(self, a_data):
        """Generate training set.

        Args:
          a_data (list): input instances

        Returns:
          2-tuple(list, list):
            lists of input features and expected classes

        """
        x, y = [], []
        if not a_data:
            return (x, y)
        for msg_i in a_data:
            if not msg_i:
                continue
            x.append(msg_i)
            y_i = msg_i.label
            # we use a pre-defined mapping of symbolic labels to integers, as
            # we need these labels to be sorted proportionally to the
            # subjective scores they get assigned when optimizing threshold of
            # lexicon-based methods
            assert y_i in CLS2IDX, "Unknown label {:s}".format(y_i)
            y_i = CLS2IDX[y_i]
            y.append(y_i)
        return (x, y)

    def _prejudge(self, a_rel, a_data):
        """Collect judgments of single classifiers.

        Args:
          a_rel (dict):
            discourse relation whose sense should be predicted
          a_data (2-tuple(dict, dict)):
            list of input JSON data

        Returns:
          np.array: modified ``a_ret``

        """
        if self._wbench is None:
            self._wbench = np.zeros((len(self._models), len(CLS2IDX)))
        else:
            self._wbench *= 0
        for i, model_i in enumerate(self._models):
            model_i.predict(a_rel, a_data, self._wbench, i)
        return self._wbench

    def _reset(self):
        """Remove members which cannot be serialized.

        """
        self._wbench = None
        self._logger = None
        self._models = []
        self._embeddings = None

    def _save_model(self, a_model, a_path):
        """Save single model to specifid path.

        Args:
          a_model (cgsa.BaseAnalyzer): prediction model
          a_path (str): file path to be checked

        Returns:
          str: directory name of the path

        """
        relpath = a_model.name + ".model"
        abspath = os.path.join(a_path, relpath)
        self._logger.info("Saving model %s to %s...",
                          a_model.name, abspath
                          )
        a_model.reset()
        if isinstance(a_model, DLBaseAnalyzer):
            a_model.save(abspath)
        else:
            with open(abspath, "wb") as ofile:
                dump(a_model, ofile)
        self._logger.info(
            "Model %s saved to %s...",
            a_model.name, abspath
        )
        self._model_paths.append(
            os.path.relpath(abspath, a_path)
        )
        return self._model_paths[-1]

max_stars_repo_name: WladimirSidorenko/CGSA

max_issues_repo_name: WladimirSidorenko/CGSA

max_forks_repo_name: WladimirSidorenko/CGSA


text:
import numpy as np
import pickle as pkl
from flask import Flask, request, render_template

app = Flask(__name__)

model = pkl.load(open('../ufo-model.pkl', 'rb'))

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/predict', methods=['POST'])
def predict():
    features = [int(x) for x in request.form.values()]
    final_features = [np.array(features)]
    prediction = model.predict(final_features)
    value = prediction[0]
    countries = ["Australia", "Canada", "Germany", "UK", "US"]

    return render_template(
        'index.html',
        prediction_text = 'The predicted country of Appearance is {}'.format(countries[value])
    )

if __name__ == "__main__":
    app.run()

ext: py

max_stars_repo_name: RupeshRavindraDharme/ML-For-Beginners

max_issues_repo_name: RupeshRavindraDharme/ML-For-Beginners

max_forks_repo_name: RupeshRavindraDharme/ML-For-Beginners

reason: import numpy


text:
import torch
import os
import torch.optim as optim
from torch.autograd import Variable
import numpy as np
from model import Net
import torch.nn as nn
import sys

from preprocessing import Processing

from data import Data

dtype = torch.FloatTensor

NUM_EPOCH = 100
LEARNING_RATE = 1e-3
NUM_CLASSES = 5
EMBEDDING_SIZE = 300
BATCH_SIZE = 1
NUM_HIDDEN = 128
NUM_LAYERS = 3

data = Data("imdb_data", "dictionary.txt", "GoogleNews-vectors-negative300.bin", "sentiment_labels.txt")
numSentences = data.numSentences

model = Net(data.maxLength, EMBEDDING_SIZE)

criterion = nn.NLLLoss() # dont need cross entropy since we do softmax in the model

optimizer = optim.Adam(model.parameters(), lr = LEARNING_RATE)

for epoch in range(NUM_EPOCH):
	print("On epoch " + str(epoch))
	for sentenceCount in range(numSentences):
		print(sentenceCount)
		if sentenceCount % 200 == 0:
			print("sentenceCount " + str(sentenceCount))
		model.zero_grad()

		sentence, label, sentenceLen = data.getNextSentence()

		if sentenceLen < 3:
			print('invalid sentence')
			continue

		label = Variable(torch.LongTensor([label]))

		sentence = np.resize(sentence, (data.maxLength, BATCH_SIZE, EMBEDDING_SIZE))
		sentence = torch.from_numpy(sentence)
		sentence = sentence.type(new_type=dtype)
		sentence = Variable(sentence)

		logProb = model.forward(sentence)

		loss = criterion(logProb, label)

		loss.backward()
		optimizer.step()

max_stars_repo_name: sinhasam/Sentalyze

max_issues_repo_name: sinhasam/Sentalyze

max_forks_repo_name: sinhasam/Sentalyze


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 21 12:49:57 2020

@author: sdoggett
"""

##Use Open Street Maps to find grocery stores and transit stops

from OSMPythonTools.overpass import Overpass, overpassQueryBuilder
from OSMPythonTools.nominatim import Nominatim

nominatim = Nominatim()
overpass = Overpass()
areaId = nominatim.query('Oakland, California').areaId()

def get_supermarkets(overpass,areaId):
    query = overpassQueryBuilder(area=areaId,
                                 elementType='node',
                                 selector='"shop"="supermarket"',
                                 includeGeometry=True,
                                 out='body')
    result = overpass.query(query)
    output = {}
    for x in result.elements():
        nodeID = x.id()
        shopName = x.tag('name')
        coord = [x.lat(),x.lon()]
        output[nodeID] = [shopName, coord]

    return output

def get_convenience_store(overpass,areaId):
    query = overpassQueryBuilder(area=areaId,
                                 elementType='node',
                                 selector='"shop"="convenience"',
                                 includeGeometry=True,
                                 out='body')
    result = overpass.query(query)
    output = {}
    for x in result.elements():
        nodeID = x.id()
        shopName = x.tag('name')
        coord = [x.lat(),x.lon()]
        output[nodeID] = [shopName, coord]

    return output

def get_pharmacy(overpass,areaId):
    query = overpassQueryBuilder(area=areaId,
                                 elementType='node',
                                 selector='"amenity"="pharmacy"',
                                 includeGeometry=True,
                                 out='body')
    result = overpass.query(query)
    output = {}
    for x in result.elements():
        nodeID = x.id()
        shopName = x.tag('name')
        coord = [x.lat(),x.lon()]
        output[nodeID] = [shopName, coord]

    return output

def get_bus_stops(overpass,areaId):
    query = overpassQueryBuilder(area=areaId,
                                 elementType='node',
                                 selector='"highway"="bus stop"',
                                 includeGeometry=True,
                                 out='body')
    result = overpass.query(query)
    output = {}
    for x in result.elements():
        nodeID = x.id()
        stopName = x.tag('name')
        routes = x.tag('route_ref')
        coord = [x.lat(),x.lon()]
        output[nodeID] = [stopName,routes, coord]

    return output

def get_BART(overpass,areaId):
    query = overpassQueryBuilder(area=areaId,
                                 elementType='node',
                                 selector=['"network" = "BART"','"railway"="station"'],
                                 includeGeometry=True,
                                 out='body')
    result = overpass.query(query)
    output = {}
    for x in result.elements():
        nodeID = x.id()
        stopName = x.tag('name')
        coord = [x.lat(),x.lon()]
        output[nodeID] = [stopName, coord]

    return output

x = get_pharmacy(overpass, areaId)
print(x)

import geopandas as gpd
import osmnx as ox
import networkx as nx
ox.config(log_console=True, use_cache=True)
ox.__version__

location_point = (37.829617, -122.278865)

dest = (37.829405, -122.280013)

# same point again, but create network only of nodes within 500m along the network from point
G3 = ox.graph_from_point(location_point, distance=500, distance_type='network',network_type='walk')
fig, ax = ox.plot_graph(G3)

origin_node = ox.get_nearest_node(G3, location_point)
dest_node = ox.get_nearest_node(G3, dest)

G_projected = ox.project_graph(G3)
nc = ['r' if (node==origin_node or node==dest_node) else '#336699' for node in G_projected.nodes()]
ns = [50 if (node==origin_node or node==dest_node) else 8 for node in G_projected.nodes()]
fig, ax = ox.plot_graph(G_projected, node_size=ns, node_color=nc, node_zorder=2)

route = nx.shortest_path(G_projected, origin_node, dest_node, weight='length')
fig, ax = ox.plot_graph_route(G_projected, route, node_size=0)
l =nx.shortest_path_length(G_projected, origin_node, dest_node, weight='length')

import pandas as pd
import numpy as np
import pandana as pdna
from pandana.loaders import osm

import warnings
warnings.filterwarnings('ignore')

tmp = [-122.8662, 37.1373, -121.4798, 38.2158]

# Reordered for Pandana functions
bbox = [tmp[1], tmp[0], tmp[3], tmp[2]]

poi_df = osm.node_query(*bbox, tags='shop=supermarket')
x, y = poi_df['lon'], poi_df['lat']

store = pd.HDFStore('osm_bayarea.h5')

nodes = store.nodes
edges = store.edges
print(nodes.head(3))
print(edges.head(3))

net=pdna.Network(nodes.x,
                 nodes.y,
                 edges["from"],
                 edges["to"],
                 edges[["weight"]])
net.precompute(3000)

net.set_pois("supermarkets", 2000, 10, x, y)

a = net.nearest_pois(2000, "supermarkets", num_pois=10)
print(a.head(1))

max_stars_repo_name: sdoggett/apartment-finder

max_issues_repo_name: sdoggett/apartment-finder

max_forks_repo_name: sdoggett/apartment-finder

reason: import numpy,import networkx


text:
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

from os.path import join
import abc
import time
import numpy as np
from tqdm import tqdm
import tensorflow as tf
from tensorflow import keras

from reco_utils.recommender.deeprec.deeprec_utils import cal_metric

__all__ = ["BaseModel"]

class BaseModel:
    """Basic class of models

    Attributes:
        hparams (obj): A tf.contrib.training.HParams object, hold the entire set of hyperparameters.
        iterator_creator_train (obj): An iterator to load the data in training steps.
        iterator_creator_train (obj): An iterator to load the data in testing steps.
        graph (obj): An optional graph.
        seed (int): Random seed.
    """

    def __init__(
        self,
        hparams,
        iterator_creator,
        seed=None,
    ):
        """Initializing the model. Create common logics which are needed by all deeprec models, such as loss function,
        parameter set.

        Args:
            hparams (obj): A tf.contrib.training.HParams object, hold the entire set of hyperparameters.
            iterator_creator_train (obj): An iterator to load the data in training steps.
            iterator_creator_train (obj): An iterator to load the data in testing steps.
            graph (obj): An optional graph.
            seed (int): Random seed.
        """
        self.seed = seed
        tf.compat.v1.set_random_seed(seed)
        np.random.seed(seed)

        self.train_iterator = iterator_creator(
            hparams,
            hparams.npratio,
            col_spliter="\t",
        )
        self.test_iterator = iterator_creator(
            hparams,
            col_spliter="\t",
        )

        self.hparams = hparams
        self.support_quick_scoring = hparams.support_quick_scoring

        # set GPU use with on demand growth
        gpu_options = tf.compat.v1.GPUOptions(allow_growth=True)
        sess = tf.compat.v1.Session(
            config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)
        )

        # set this TensorFlow session as the default session for Keras
        tf.compat.v1.keras.backend.set_session(sess)

        # IMPORTANT: models have to be loaded AFTER SETTING THE SESSION for keras!
        # Otherwise, their weights will be unavailable in the threads after the session there has been set
        self.model, self.scorer = self._build_graph()

        self.loss = self._get_loss()
        self.train_optimizer = self._get_opt()

        self.model.compile(loss=self.loss, optimizer=self.train_optimizer)

    def _init_embedding(self, file_path):
        """Load pre-trained embeddings as a constant tensor.

        Args:
            file_path (str): the pre-trained glove embeddings file path.

        Returns:
            np.array: A constant numpy array.
        """

        return np.load(file_path)

    @abc.abstractmethod
    def _build_graph(self):
        """Subclass will implement this."""
        pass

    @abc.abstractmethod
    def _get_input_label_from_iter(self, batch_data):
        """Subclass will implement this"""
        pass

    def _get_loss(self):
        """Make loss function, consists of data loss and regularization loss

        Returns:
            obj: Loss function or loss function name
        """
        if self.hparams.loss == "cross_entropy_loss":
            data_loss = "categorical_crossentropy"
        elif self.hparams.loss == "log_loss":
            data_loss = "binary_crossentropy"
        else:
            raise ValueError("this loss not defined {0}".format(self.hparams.loss))
        return data_loss

    def _get_opt(self):
        """Get the optimizer according to configuration. Usually we will use Adam.
        Returns:
            obj: An optimizer.
        """
        lr = self.hparams.learning_rate
        optimizer = self.hparams.optimizer

        if optimizer == "adam":
            train_opt = keras.optimizers.Adam(lr=lr)

        return train_opt

    def _get_pred(self, logit, task):
        """Make final output as prediction score, according to different tasks.

        Args:
            logit (obj): Base prediction value.
            task (str): A task (values: regression/classification)

        Returns:
            obj: Transformed score
        """
        if task == "regression":
            pred = tf.identity(logit)
        elif task == "classification":
            pred = tf.sigmoid(logit)
        else:
            raise ValueError(
                "method must be regression or classification, but now is {0}".format(
                    task
                )
            )
        return pred

    def train(self, train_batch_data):
        """Go through the optimization step once with training data in feed_dict.

        Args:
            sess (obj): The model session object.
            feed_dict (dict): Feed values to train the model. This is a dictionary that maps graph elements to values.

        Returns:
            list: A list of values, including update operation, total loss, data loss, and merged summary.
        """
        train_input, train_label = self._get_input_label_from_iter(train_batch_data)
        rslt = self.model.train_on_batch(train_input, train_label)
        return rslt

    def eval(self, eval_batch_data):
        """Evaluate the data in feed_dict with current model.

        Args:
            sess (obj): The model session object.
            feed_dict (dict): Feed values for evaluation. This is a dictionary that maps graph elements to values.

        Returns:
            list: A list of evaluated results, including total loss value, data loss value,
                predicted scores, and ground-truth labels.
        """
        eval_input, eval_label = self._get_input_label_from_iter(eval_batch_data)
        imp_index = eval_batch_data["impression_index_batch"]

        pred_rslt = self.scorer.predict_on_batch(eval_input)

        return pred_rslt, eval_label, imp_index

    def fit(
        self,
        train_news_file,
        train_behaviors_file,
        valid_news_file,
        valid_behaviors_file,
        test_news_file=None,
        test_behaviors_file=None,
    ):
        """Fit the model with train_file. Evaluate the model on valid_file per epoch to observe the training status.
        If test_news_file is not None, evaluate it too.

        Args:
            train_file (str): training data set.
            valid_file (str): validation set.
            test_news_file (str): test set.

        Returns:
            obj: An instance of self.
        """

        for epoch in range(1, self.hparams.epochs + 1):
            step = 0
            self.hparams.current_epoch = epoch
            epoch_loss = 0
            train_start = time.time()

            tqdm_util = tqdm(
                self.train_iterator.load_data_from_file(
                    train_news_file, train_behaviors_file
                )
            )

            for batch_data_input in tqdm_util:

                step_result = self.train(batch_data_input)
                step_data_loss = step_result

                epoch_loss += step_data_loss
                step += 1
                if step % self.hparams.show_step == 0:
                    tqdm_util.set_description(
                        "step {0:d} , total_loss: {1:.4f}, data_loss: {2:.4f}".format(
                            step, epoch_loss / step, step_data_loss
                        )
                    )

            train_end = time.time()
            train_time = train_end - train_start

            self.model.save_weights('./para/dkn_' + str(epoch) + '.h5')

            eval_start = time.time()

            train_info = ",".join(
                [
                    str(item[0]) + ":" + str(item[1])
                    for item in [("logloss loss", epoch_loss / step)]
                ]
            )

            _, group_labels, group_preds = self.run_slow_eval(valid_news_file, valid_behaviors_file)
            eval_res = cal_metric(group_labels, group_preds, self.hparams.metrics)

            eval_info = ", ".join(
                [
                    str(item[0]) + ":" + str(item[1])
                    for item in sorted(eval_res.items(), key=lambda x: x[0])
                ]
            )
            if test_news_file is not None:
                test_res = self.run_eval(test_news_file, test_behaviors_file)
                test_info = ", ".join(
                    [
                        str(item[0]) + ":" + str(item[1])
                        for item in sorted(test_res.items(), key=lambda x: x[0])
                    ]
                )
            eval_end = time.time()
            eval_time = eval_end - eval_start

            if test_news_file is not None:
                print(
                    "at epoch {0:d}".format(epoch)
                    + "\ntrain info: "
                    + train_info
                    + "\neval info: "
                    + eval_info
                    + "\ntest info: "
                    + test_info
                )
            else:
                print(
                    "at epoch {0:d}".format(epoch)
                    + "\ntrain info: "
                    + train_info
                    + "\neval info: "
                    + eval_info
                )
            print(
                "at epoch {0:d} , train time: {1:.1f} eval time: {2:.1f}".format(
                    epoch, train_time, eval_time
                )
            )

        return self

    def group_labels(self, labels, preds, group_keys):
        """Devide labels and preds into several group according to values in group keys.

        Args:
            labels (list): ground truth label list.
            preds (list): prediction score list.
            group_keys (list): group key list.

        Returns:
            all_labels: labels after group.
            all_preds: preds after group.

        """

        all_keys = list(set(group_keys))
        all_keys.sort()
        group_labels = {k: [] for k in all_keys}
        group_preds = {k: [] for k in all_keys}

        for l, p, k in zip(labels, preds, group_keys):
            group_labels[k].append(l)
            group_preds[k].append(p)

        all_labels = []
        all_preds = []
        for k in all_keys:
            all_labels.append(group_labels[k])
            all_preds.append(group_preds[k])

        return all_keys, all_labels, all_preds

    def run_eval(self, news_filename, behaviors_file):
        """Evaluate the given file and returns some evaluation metrics.

        Args:
            filename (str): A file name that will be evaluated.

        Returns:
            dict: A dictionary contains evaluation metrics.
        """

        if self.support_quick_scoring:
            _, group_labels, group_preds = self.run_fast_eval(
                news_filename, behaviors_file
            )
        else:
            _, group_labels, group_preds = self.run_slow_eval(
                news_filename, behaviors_file
            )
        res = cal_metric(group_labels, group_preds, self.hparams.metrics)
        return res

    def user(self, batch_user_input):
        user_input = self._get_user_feature_from_iter(batch_user_input)
        user_vec = self.userencoder.predict_on_batch(user_input)
        user_index = batch_user_input["impr_index_batch"]

        return user_index, user_vec

    def news(self, batch_news_input):
        news_input = self._get_news_feature_from_iter(batch_news_input)
        news_vec = self.newsencoder.predict_on_batch(news_input)
        news_index = batch_news_input["news_index_batch"]

        return news_index, news_vec

    def run_user(self, news_filename, behaviors_file):
        if not hasattr(self, "userencoder"):
            raise ValueError("model must have attribute userencoder")

        user_indexes = []
        user_vecs = []
        for batch_data_input in tqdm(
            self.test_iterator.load_user_from_file(news_filename, behaviors_file)
        ):
            user_index, user_vec = self.user(batch_data_input)
            user_indexes.extend(np.reshape(user_index, -1))
            user_vecs.extend(user_vec)

        return dict(zip(user_indexes, user_vecs))

    def run_news(self, news_filename):
        if not hasattr(self, "newsencoder"):
            raise ValueError("model must have attribute newsencoder")

        news_indexes = []
        news_vecs = []
        for batch_data_input in tqdm(
            self.test_iterator.load_news_from_file(news_filename)
        ):
            news_index, news_vec = self.news(batch_data_input)
            news_indexes.extend(np.reshape(news_index, -1))
            news_vecs.extend(news_vec)

        return dict(zip(news_indexes, news_vecs))

    def run_slow_eval(self, news_filename, behaviors_file):
        preds = []
        labels = []
        imp_indexes = []

        for batch_data_input in tqdm(
            self.test_iterator.load_data_from_file(news_filename, behaviors_file)
        ):
            step_pred, step_labels, step_imp_index = self.eval(batch_data_input)
            preds.extend(np.reshape(step_pred, -1))
            labels.extend(np.reshape(step_labels, -1))
            imp_indexes.extend(np.reshape(step_imp_index, -1))

        group_impr_indexes, group_labels, group_preds = self.group_labels(
            labels, preds, imp_indexes
        )
        return group_impr_indexes, group_labels, group_preds

    def run_fast_eval(self, news_filename, behaviors_file):
        news_vecs = self.run_news(news_filename)
        user_vecs = self.run_user(news_filename, behaviors_file)

        self.news_vecs = news_vecs
        self.user_vecs = user_vecs

        group_impr_indexes = []
        group_labels = []
        group_preds = []

        for (
            impr_index,
            news_index,
            user_index,
            label,
        ) in tqdm(self.test_iterator.load_impression_from_file(behaviors_file)):
            pred = np.dot(
                np.stack([news_vecs[i] for i in news_index], axis=0),
                user_vecs[impr_index],
            )
            group_impr_indexes.append(impr_index)
            group_labels.append(label)
            group_preds.append(pred)

        return group_impr_indexes, group_labels, group_preds

max_stars_repo_name: chungdz/recommenders

max_issues_repo_name: chungdz/recommenders

max_forks_repo_name: chungdz/recommenders


text:
from gym.envs.mujoco import HalfCheetahEnv as GymHalfCheetahEnv
import numpy as np

from .mujoco_wrapper import MujocoWrapper

class HalfCheetahEnv(GymHalfCheetahEnv, MujocoWrapper):
    def __init__(self):
        super().__init__()
        self._max_episode_steps = 1000

    @staticmethod
    def check_done(states):
        return np.zeros(len(states), dtype=bool)

    def qposvel_from_obs(self, obs):
        qpos = self.sim.data.qpos.copy()
        qpos[1:] = obs[:8]
        qvel = obs[8:]
        return qpos, qvel

max_stars_repo_name: gwthomas/Safe-MBPO

max_issues_repo_name: gwthomas/Safe-MBPO

max_forks_repo_name: gwthomas/Safe-MBPO


text:
# coding=utf-8
# Copyright 2019 project LXRT.

from collections import defaultdict
import json
import random

import numpy as np
from torch.utils.data import Dataset

from param import args
from pretrain.qa_answer_table import AnswerTable
from utils import load_obj_tsv

TINY_IMG_NUM = 500
FAST_IMG_NUM = 5000

Split2ImgFeatPath = {
    "mscoco_train": "data/mscoco_imgfeat/train2014_obj36.tsv",
    "mscoco_minival": "data/mscoco_imgfeat/val2014_obj36.tsv",
    "mscoco_nominival": "data/mscoco_imgfeat/val2014_obj36.tsv",
    "vgnococo": "data/vg_gqa_imgfeat/vg_gqa_obj36.tsv",
}

class InputExample(object):
    """A single training/test example for the language model."""

    def __init__(
        self,
        uid,
        sent,
        visual_feats=None,
        obj_labels=None,
        attr_labels=None,
        is_matched=None,
        label=None,
    ):
        self.uid = uid
        self.sent = sent
        self.visual_feats = visual_feats
        self.obj_labels = obj_labels
        self.attr_labels = attr_labels
        self.is_matched = is_matched  # whether the visual and obj matched
        self.label = label

class LXMERTDataset:
    def __init__(self, splits: str, qa_sets=None):
        """
        :param splits: The data sources to be loaded
        :param qa_sets: if None, no action
                        o.w., only takes the answers appearing in these dsets
                              and remove all unlabeled data (MSCOCO captions)
        """
        self.name = splits
        self.sources = splits.split(",")

        # Loading datasets to data
        self.data = []
        for source in self.sources:
            self.data.extend(json.load(open("data/lxmert/%s.json" % source)))
        print("Load %d data from %s" % (len(self.data), self.name))

        # Create answer table according to the qa_sets
        self.answer_table = AnswerTable(qa_sets)
        print(
            "Load an answer table of size %d." % (len(self.answer_table.ans2id_map()))
        )

        # Modify the answers
        for datum in self.data:
            labelf = datum["labelf"]
            for cat, labels in labelf.items():
                for label in labels:
                    for ans in list(label.keys()):
                        new_ans = self.answer_table.convert_ans(ans)
                        if self.answer_table.used(new_ans):
                            if ans != new_ans:
                                label[new_ans] = label.pop(ans)
                        else:
                            label.pop(ans)

    def __len__(self):
        return len(self.data)

def make_uid(img_id, dset, sent_idx):
    return ("%s_%s_%03d" % (img_id, dset, sent_idx),)

"""
Example in obj tsv:
FIELDNAMES = ["img_id", "img_h", "img_w", "objects_id", "objects_conf",
              "attrs_id", "attrs_conf", "num_boxes", "boxes", "features"]
"""

class LXMERTTorchDataset(Dataset):
    def __init__(self, dataset: LXMERTDataset, topk=-1):
        super().__init__()
        self.raw_dataset = dataset
        self.task_matched = args.task_matched

        if args.tiny:
            topk = TINY_IMG_NUM
        elif args.fast:
            topk = FAST_IMG_NUM

        # Load the dataset
        img_data = []
        for source in self.raw_dataset.sources:
            img_data.extend(load_obj_tsv(Split2ImgFeatPath[source], topk))

        self.imgid2img = {}
        for img_datum in img_data:
            self.imgid2img[img_datum["img_id"]] = img_datum

        # Filter out the dataset
        used_data = []
        for datum in self.raw_dataset.data:
            if datum["img_id"] in self.imgid2img:
                used_data.append(datum)

        # Flatten the dataset (into one sent + one image entries)
        self.data = []
        for datum in used_data:
            sentf = datum["sentf"]
            for sents_cat, sents in sentf.items():
                if sents_cat in datum["labelf"]:
                    labels = datum["labelf"][sents_cat]
                else:
                    labels = None
                for sent_idx, sent in enumerate(sents):
                    new_datum = {
                        "uid": make_uid(datum["img_id"], sents_cat, sent_idx),
                        "img_id": datum["img_id"],
                        "sent": sent,
                    }
                    if labels is not None:
                        new_datum["label"] = labels[sent_idx]
                    self.data.append(new_datum)
        print("Use %d data in torch dataset" % (len(self.data)))

    def __len__(self):
        return len(self.data)

    def random_feat(self):
        """Get a random obj feat from the dataset."""
        datum = self.data[random.randint(0, len(self.data) - 1)]
        img_id = datum["img_id"]
        img_info = self.imgid2img[img_id]
        feat = img_info["features"][random.randint(0, 35)]
        return feat

    def __getitem__(self, item: int):
        datum = self.data[item]

        uid = datum["uid"]
        img_id = datum["img_id"]

        # Get image info
        img_info = self.imgid2img[img_id]
        obj_num = img_info["num_boxes"]
        feats = img_info["features"].copy()
        boxes = img_info["boxes"].copy()
        obj_labels = img_info["objects_id"].copy()
        obj_confs = img_info["objects_conf"].copy()
        attr_labels = img_info["attrs_id"].copy()
        attr_confs = img_info["attrs_conf"].copy()
        assert obj_num == len(boxes) == len(feats)

        # Normalize the boxes (to 0 ~ 1)
        img_h, img_w = img_info["img_h"], img_info["img_w"]
        boxes = boxes.copy()
        boxes[:, (0, 2)] /= img_w
        boxes[:, (1, 3)] /= img_h
        np.testing.assert_array_less(boxes, 1 + 1e-5)
        np.testing.assert_array_less(-boxes, 0 + 1e-5)

        # If calculating the matched loss, replace the sentence with an sentence
        # corresponding to other image.
        is_matched = 1
        sent = datum["sent"]
        if self.task_matched:
            if random.random() < 0.5:
                is_matched = 0
                other_datum = self.data[random.randint(0, len(self.data) - 1)]
                while other_datum["img_id"] == img_id:
                    other_datum = self.data[random.randint(0, len(self.data) - 1)]
                sent = other_datum["sent"]

        # Label, convert answer to id
        if "label" in datum:
            label = datum["label"].copy()
            for ans in list(label.keys()):
                label[self.raw_dataset.answer_table.ans2id(ans)] = label.pop(ans)
        else:
            label = None

        # Create target
        example = InputExample(
            uid,
            sent,
            (feats, boxes),
            (obj_labels, obj_confs),
            (attr_labels, attr_confs),
            is_matched,
            label,
        )
        return example

class LXMERTEvaluator:
    def __init__(self, dataset: LXMERTDataset):
        self.raw_dataset = dataset

        # Create QA Eval Data
        self.data = []
        for datum in self.raw_dataset.data:
            sentf = datum["sentf"]
            for sents_cat, sents in sentf.items():
                if sents_cat in datum["labelf"]:  # A labeled dataset
                    labels = datum["labelf"][sents_cat]
                    for sent_idx, sent in enumerate(sents):
                        new_datum = {
                            "uid": make_uid(datum["img_id"], sents_cat, sent_idx),
                            "img_id": datum["img_id"],
                            "sent": sent,
                            "dset": sents_cat,
                            "label": labels[sent_idx],
                        }
                        self.data.append(new_datum)

        # uid2datum
        self.uid2datum = {}
        for datum in self.data:
            self.uid2datum[datum["uid"]] = datum

    def evaluate(self, uid2ans: dict, pprint=False):
        score = 0.0
        cnt = 0
        dset2score = defaultdict(lambda: 0.0)
        dset2cnt = defaultdict(lambda: 0)
        for uid, ans in uid2ans.items():
            if uid not in self.uid2datum:  # Not a labeled data
                continue
            datum = self.uid2datum[uid]
            label = datum["label"]
            dset = datum["dset"]
            if ans in label:
                score += label[ans]
                dset2score[dset] += label[ans]
            cnt += 1
            dset2cnt[dset] += 1
        accu = score / cnt
        dset2accu = {}
        for dset in dset2cnt:
            dset2accu[dset] = dset2score[dset] / dset2cnt[dset]

        if pprint:
            accu_str = "Overall Accu %0.4f, " % (accu)
            sorted_keys = sorted(dset2accu.keys())
            for key in sorted_keys:
                accu_str += "%s Accu %0.4f, " % (key, dset2accu[key])
            print(accu_str)

        return accu, dset2accu

    def dump_result(self, uid2ans: dict, path):
        raise NotImplemented

max_stars_repo_name: allenai/faithful-nmn

max_issues_repo_name: arjunakula/faithful-nmn

max_forks_repo_name: arjunakula/faithful-nmn


text:
import six
import numbers
import logging

import numpy as np

from sklearn.metrics import log_loss
from sklearn.utils import check_consistent_length, check_array
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import LabelBinarizer

import mxnet as mx
from mxnet.metric import EvalMetric

__all__ = ['Activation', 'Dense', 'SoftmaxOutput', 'Variable',
           'BatchNormalization', 'Dropout', 'Sequential', 'Adam']

def _weighted_sum(sample_score, sample_weight, normalize=False):
    if normalize:
        return np.average(sample_score, weights=sample_weight)
    elif sample_weight is not None:
        return np.dot(sample_score, sample_weight)
    else:
        return sample_score.sum()

class LogLoss(object):
    def __init__(self):
        self.lb_ = None

    @property
    def __name__(self):
        return 'log_loss'

    def __call__(self, y_true, y_pred, eps=1e-15, normalize=True, sample_weight=None):
        if self.lb_ is None:
            self.lb_ = LabelBinarizer()
            T = self.lb_.fit_transform(y_true)
        else:
            T = self.lb_.transform(y_true)

        if T.shape[1] == 1:
            T = np.append(1 - T, T, axis=1)

        Y = np.clip(y_pred, eps, 1 - eps)

        if not isinstance(Y, np.ndarray):
            raise ValueError('y_pred should be an array of floats.')

        if Y.ndim == 1:
            Y = Y[:, np.newaxis]
        if Y.shape[1] == 1:
            Y = np.append(1 - Y, Y, axis=1)

        check_consistent_length(T, Y)
        T = check_array(T)
        Y = check_array(Y)
        if T.shape[1] != Y.shape[1]:
            raise ValueError('y_true and y_pred have different number of classes '
                             '%d, %d' % (T.shape[1], Y.shape[1]))

        Y /= Y.sum(axis=1)[:, np.newaxis]
        loss = -(T * np.log(Y)).sum(axis=1)

        return _weighted_sum(loss, sample_weight, normalize)

LOSS_MAP = {'categorical_crossentropy': mx.metric.np(LogLoss())}

class MXNetSymbol(object):
    def __init__(self, *args, **kwargs):
        self.logger = logging.getLogger(__name__)
        self.args = args
        self.kwargs = kwargs

    @property
    def symbol(self):
        pass

    def __call__(self, prev_symbol=None):
        if prev_symbol:
            return self.symbol(prev_symbol, *self.args, **self.kwargs)
        return self.symbol(*self.args, **self.kwargs)

class Activation(MXNetSymbol):
    @property
    def symbol(self):
        return mx.symbol.Activation

    def __call__(self, prev_symbol=None):
        """ Overwrite to allow for passing of the name of the activation
            directly. In addition, alllow for detection of output layers,
            i.e. SoftmaxOutput
        """
        assert len(self.args) == 1  # this should be the name of the activaiton

        # pop off the activation
        activation = self.args[0]
        self.args = self.args[1:]

        if not isinstance(activation, six.string_types):
            raise ValueError('activation type must be a string')

        if activation == 'softmax':
            self.logger.debug('Detected SoftmaxOutput in activation.')
            return mx.symbol.SoftmaxOutput(
                prev_symbol, name='softmax', *self.args, **self.kwargs)
        elif prev_symbol:
            return self.symbol(
                prev_symbol, *self.args, act_type=activation, **self.kwargs)

        return self.symbol(*self.args, act_type=activation, **self.kwargs)

class LeakyReLU(MXNetSymbol):
    @property
    def symbol(self):
        return mx.symbol.LeakyReLU

    @property
    def act_type(self):
        pass

    def __call__(self, prev_symbol=None):
        if prev_symbol:
            return self.symbol(
                prev_symbol, *self.args, act_type=self.act_type, **self.kwargs)
        return self.symbol(*self.args, **self.kwargs)

class PReLU(LeakyReLU):
    @property
    def act_type(self):
        return 'prelu'

class Dense(MXNetSymbol):
    """ We are going to use the Keras naming convention. We need a base
        layer class eventually.
    """
    @property
    def symbol(self):
        return mx.symbol.FullyConnected

    def __call__(self, prev_symbol=None):
        """ Overwrite to allow for passing num_hidden directly. """
        assert len(self.args) == 1  # this should be the number of hidden units

        # pop off the activation
        num_hidden = self.args[0]
        self.args = self.args[1:]
        if not isinstance(num_hidden, numbers.Integral) or num_hidden < 0:
            raise ValueError('number of hidden units must be a '
                             'positive integer.')

        if prev_symbol:
            # HACK: input_shape is used in keras and not mxnet. Lets pop
            #       it off for now and figure out a better inference later.
            if 'input_shape' in self.kwargs:
                del self.kwargs['input_shape']
            return self.symbol(
                prev_symbol, *self.args, num_hidden=num_hidden, **self.kwargs)
        return self.symbol(*self.args, num_hidden=num_hidden, **self.kwargs)

class SoftmaxOutput(MXNetSymbol):
    @property
    def symbol(self):
        return mx.symbol.SoftmaxOutput

class Variable(MXNetSymbol):
    @property
    def symbol(self):
        return mx.symbol.Variable

class BatchNormalization(MXNetSymbol):
    @property
    def symbol(self):
        return mx.symbol.BatchNorm

class Dropout(MXNetSymbol):
    @property
    def symbol(self):
        return mx.symbol.Dropout

class Sequential(object):
    def __init__(self):
        self.logger = logging.getLogger(__name__)
        self.prev_symbol = Variable('data')()

    @property
    def model(self):
        return self._model

    @model.setter
    def model(self, value):
        self._model = value

    def compile(self, optimizer, loss, class_mode='categorical'):
        """ for mxnet this is not necessary, but we have it here for
            convenience.
        """
        try:
            self.loss = LOSS_MAP[loss]
        except KeyError:
            self.logger.debug('Loss function not found.')
            self.loss = 'acc'

        self.optimizer = optimizer

    def visualize(self):
        return mx.viz.plot_network(self.prev_symbol)

    def add(self, symbol):
            self.prev_symbol = symbol(self.prev_symbol)

    def fit(self, X, y, nb_epoch=10, learning_rate=0.01, batch_size=128, validation_split=0.15):
        self.model = mx.model.FeedForward(self.prev_symbol,
                                          num_epoch=nb_epoch,
                                          optimizer=self.optimizer,
                                          numpy_batch_size=batch_size,
                                          learning_rate=learning_rate)

        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=validation_split)

        self.model.fit(X_train,
                       y_train,
                       eval_metric=self.loss,
                       eval_data=[X_test, y_test])

    def predict(self, X):
        return self.model.predict(X)

# direct imports
Adam = mx.optimizer.Adam

max_stars_repo_name: joshloyal/Nettie

max_issues_repo_name: joshloyal/Nettie

max_forks_repo_name: joshloyal/Nettie


text:
import numpy as np
from ..activation import activation_detection, drivative_of_activation
from ..metrics import get_loss_func
from ..scheduling import learning_scheduling
from ..utility_functions import progress_bar

class Network:
    VALIDATION_LOSS = "val_loss"
    TRAIN_LOSS = "train_loss"
    WEIGHT_HOLDER = "WEIGHT_HOLDER"

    def __init__(self, layers: list) -> None:
        self.history = {
            Network.VALIDATION_LOSS: [],
            Network.TRAIN_LOSS: [],
            Network.WEIGHT_HOLDER: []
        }
        self.layers = layers
        self.epochs = 1000

    def feed_forward(self, feature_inputs):
        self.layers[0].net = feature_inputs
        self.layers[0].O = feature_inputs
        for layer_index in range(1, len(self.layers)):
            self.layers[layer_index].net = np.dot(self.layers[layer_index].W, self.layers[layer_index - 1].O) + self.layers[layer_index].B
            self.layers[layer_index].O = self.layers[layer_index].activation(self.layers[layer_index].net)
        return self.layers[-1].O

# assert (feed_forward(X_train).shape == Y_train.shape)
    def back_propagation(self, y_pred, y_true, is_batch = False):
        error = y_pred - y_true # for get rid of -1
        delta = error # last layer delta and error and drivative is one(f'(net) = 1)
        for i in range(len(self.layers) - 1, 0, -1):
            if i != len(self.layers) - 1:
                delta = np.dot(self.layers[i+1].W.T, delta) * drivative_of_activation(self.layers[i].net, self.layers[i].activation_function)

            if not is_batch:
                assert delta.shape[0] == self.layers[i].neuron

                self.layers[i].delta = delta
                self.layers[i].dw = np.dot(delta, self.layers[i - 1].O.T)
                self.layers[i].db = delta
            else:
                self.layers[i].delta = delta
                self.layers[i].dw = np.dot(delta, self.layers[i - 1].O.T) / y_pred.shape[-1]
                self.layers[i].db = np.sum(delta, axis=1, keepdims=True) / y_pred.shape[-1]
                # print(delta.shape, self.layers[i].db.shape)
                # import sys
                # sys.exit(1)

            assert self.layers[i].dw.shape == self.layers[i].W.shape
            assert self.layers[i].db.shape == self.layers[i].B.shape

    def update_weights(self, lrate, optimizer, ß = 0.9):
        weight_layer = []
        for layer in self.layers[1:]:
            weight_layer.append((layer.W, layer.B))
            if optimizer == "momentum":
                if not layer.is_freezed():
                    layer.VDW = ß * layer.VDW + (1-ß) * layer.dw
                    layer.VDB = ß * layer.VDB + (1-ß) * layer.db
                    layer.W = layer.W - lrate * layer.VDW
                    layer.B = layer.B - lrate * layer.VDB
            elif optimizer == "sgd" or optimizer == "batch":
                if not layer.is_freezed():
                    # if not layer.is_parameter_freez(layer.W):
                    layer.W = layer.W - lrate * layer.dw
                    # if not layer.is_parameter_freez(layer.B):
                    layer.B = layer.B - lrate * layer.db
            else:
                raise Exception(f"optimizer: {optimizer} is not supported")

            self.history[Network.WEIGHT_HOLDER].append(weight_layer)

    def fit(self, lrate: float, epochs: int,
        X_train: np.ndarray, y_train: np.ndarray,
        X_valid: np.ndarray, y_valid: np.ndarray,
        cost_func: str = "mse",
        learning_decay: bool = False, tolorance: float=0.05) -> dict:

        self.history = {
            Network.VALIDATION_LOSS: [],
            Network.TRAIN_LOSS: [],
            Network.WEIGHT_HOLDER: []
        }

        self.train(lrate, epochs, X_train, y_train,
                    X_valid, y_valid, cost_func,
                   learning_decay, tolorance)

        return self.history

    def train(lrate: float, epochs: int,
            X_train: np.ndarray, y_train: np.ndarray,
            X_valid: np.ndarray, y_valid: np.ndarray,
            cost_func: str = "mse",
            learning_decay: bool = False, tolorance: float=0.05):
        raise NotImplementedError

    def evalute(self, X_test: np.ndarray, Y_label: np.ndarray, mean=False):
        Y_pred = self.feed_forward(X_test)
        loss = get_loss_func(Y_pred, Y_label, mean=mean)
        self.history["loss"] = loss
        return loss

    def ploting_learning_curves(self, ax, ax1):
        ax.set_title("learning_curves")

        validation_losses = np.array(self.history[Network.VALIDATION_LOSS])
        best_epoch = np.argmin(validation_losses.ravel()[50:]) + 50

        test_point = self.history[Network.VALIDATION_LOSS][best_epoch]

        arrow_prop = dict(
            arrowstyle = "->",
            connectionstyle = "angle, angleA = 0, angleB = 90, rad = 10"
        )
        ax.annotate(
            text= "early stopping" if best_epoch in range(self.epochs-10, self.epochs+2) else "maybe overfiting occured",
            xy=(best_epoch, test_point),
            xytext=(best_epoch - 40, np.max(validation_losses[100:].ravel()) - np.max(validation_losses[100:].ravel()) * 0.1),
            arrowprops = arrow_prop,
            bbox = dict(boxstyle ="round", fc ="0.8")
        )
        ax.plot(self.history[Network.TRAIN_LOSS][100:], label="traning_loss")
        ax.plot(self.history[Network.VALIDATION_LOSS][100:], label="validation_loss")
        ax1.plot(self.history[Network.VALIDATION_LOSS][10:], color='g',
                 marker=".", label="validation_loss", linewidth=0.5)
        # top_val = np.max(self.history[Network.VALIDATION_LOSS])
        # ax.set_ylim([0, top_val if top_val > 1 else top_val * 2 ])
        return best_epoch

    def ploting_errors(self, ax, prediction, y_true):
        errors = np.abs(prediction.ravel() - y_true.ravel())
        ax.plot(errors, color="r", marker=".", label="error of data test_set", linewidth=0.5)
        # ax.set_ylim(-1, 1)

class SGDNetwork(Network):
    def __init__(self, layers: list) -> None:
        super().__init__(layers)
        self.optimizer = "sgd"

    def train(self, lrate: float, epochs: int,
            X_train: np.ndarray, y_train: np.ndarray,
            X_valid: np.ndarray, y_valid: np.ndarray,
            cost_func: str = "mse",
            learning_decay: bool = False, tolorance: float = 0.05):

        self.epochs = epochs
        eta = lrate
        for epoch in range(1, epochs+1):

            if learning_decay:
                eta = learning_scheduling(lrate, epoch)
                # _, _, X_valid, y_valid =  apply_k_fold(X_train, y_train)

            for i in range(X_train.shape[-1]):
                sample = X_train[:, i].reshape(-1, 1)
                y_pred = self.feed_forward(sample)
                assert y_pred.shape == y_train[:, i].reshape(-1, 1).shape
                self.back_propagation(y_pred, y_train[:, i].reshape(-1, 1))
                self.update_weights(eta, optimizer=self.optimizer)

            print(progress_bar(epoch, epochs))

            train_pred = self.feed_forward(X_train)
            train_loss = get_loss_func(train_pred, y_train, mean=False)
            self.history[Network.TRAIN_LOSS].append(train_loss)

            valid_pred = self.feed_forward(X_valid)
            validation_loss = get_loss_func(valid_pred, y_valid, mean=False)
            self.history[Network.VALIDATION_LOSS].append(validation_loss)
            print("train_loss: %.6f \t validation_loss: %.6f" % (train_loss, validation_loss))

            # if validation_loss < tolorance:
            #     break

class BatchNetwork(Network):
    def __init__(self, layers: list) -> None:
        super().__init__(layers)
        self.optimizer = "batch"

    def train(self, lrate: float, epochs: int,
                X_train: np.ndarray, y_train: np.ndarray,
                X_valid: np.ndarray, y_valid: np.ndarray,
                cost_func: str = "mse",
                learning_decay: bool = False, tolorance: float = 0.05):

        eta = lrate
        for epoch in range(epochs):
            if learning_decay:
                eta = learning_scheduling(lrate, epoch)

            y_hats = self.feed_forward(X_train)
            assert y_hats.shape == y_train.shape
            self.back_propagation(y_hats, y_train, is_batch=True)
            self.update_weights(eta, optimizer=self.optimizer)

            train_pred = self.feed_forward(X_train)
            train_loss = get_loss_func(train_pred, y_train, mean=False)
            self.history[Network.TRAIN_LOSS].append(train_loss)

            valid_pred = self.feed_forward(X_valid)
            test_loss = get_loss_func(valid_pred, y_valid, mean=False)
            self.history[Network.VALIDATION_LOSS].append(test_loss)
            if epoch %100 == 0:
                print(progress_bar(epoch, epochs))
                print("train_loss: %.6f \t validation_loss: %.6f" % (train_loss, test_loss))


text:
##获取landmark 坐标 并且将其存入CSV中
# 具体文件为landmark_Train.csv Landmark_Test.csv...

import os
import numpy as np
import pandas as pd
from skimage import io, transform
# import numpy as np
import matplotlib.pyplot as plt

def getTargetCoord(annotationFile):
    voxelCoords = []
    with open(annotationFile) as f:
        tag = False
        for lines in f:
            if tag:
                curLine = lines.strip().split('\t')
                voxelCoords.append([int(curLine[-2]), int(curLine[-1])])
                #加入坐标
                #例如annotator301  734   945
                #加入[734,945]

                tag = False
            else:
                tag = True

    return np.array(voxelCoords)

def show_landmarks(image, landmarks):
    """Show image with landmarks"""
    plt.imshow(image,cmap=plt.cm.gray)
    plt.scatter(landmarks[:,0], landmarks[:, 1], s=50, marker='.', c='r')
    plt.pause(0.001)  # pause a bit so that plots are updated

    plt.savefig('1-landmark.png')

if __name__ == "__main__":

    state = "Produce"  #Produce or any
    data = "Test"      #Train or Test
    crop = True        #True or False
    h ,w = 1678 , 1678
    outputshape = 512
    new_h,new_w = int(outputshape) , int(outputshape)
    ############ 生成部分############################
    if state == "Produce":
        if data == "Train":
    # landmark_path = "E:\BME\DCGAN-pytorch\TestDataCrop\landmark_001.txt"
            landmarkOri_path = "E:\Hughie\GAN\Data\TrainLabel\landmark_001.txt"
            len = 301
        else:
            landmarkOri_path = "E:\Hughie\GAN\Data\TestLabel\landmark_001.txt"
            len = 101
    #
        landmark = getTargetCoord(landmarkOri_path)

        # print(landmark)
        # print(landmark[0])

        # landmark = np.reshape(landmark,(2,-1))
        # print(landmark[:,0])
        # print(landmark)

        Keypoint1_x =  (landmark[:,0] - 142)  * (new_w/w)
        Keypoint1_y =  (landmark[:,1] - 682) * (new_h/h)

        Key = pd.DataFrame({"Keypoint1_x": Keypoint1_x,
                            "Keypoint1_y": Keypoint1_y},index = [i for i in range(1,len)])

        for i in range(2,20):
            # landmark_path = "E:\BME\DCGAN-pytorch\TestDataCrop\landmark_%03d.txt"%i
            landmarkOri_path = "E:\Hughie\GAN\Data\%sLabel\landmark_%03d.txt"%(data,i)
            k_x = "Keypoint%d_x"%i
            k_y = "Keypoint%d_y"%i

            landmark = getTargetCoord(landmarkOri_path)
            Keypoint1_x = (landmark[:, 0]  - 142)  * (new_w / w)
            Keypoint1_y = (landmark[:, 1]  - 682)  * (new_h / h)

            Key[k_x] = Keypoint1_x
            Key[k_y] = Keypoint1_y

        print(Key)
        print(Key.shape)
        print(Key.head())
        # max_Key = Key.max()
        # min_Key = Key.min()
        # max_x = 0
        # max_y = 0
        # min_x = 10000
        # min_y = 10000
        # for i in range(1,20):
        #     k_x = "Keypoint%d_x" % i
        #     k_y = "Keypoint%d_y" % i
        #
        #
        #     if max_Key[k_x] > max_x:
        #         max_x = max_Key[k_x]
        #     if max_Key[k_y] > max_y:
        #         max_y = max_Key[k_y]
        #     if min_Key[k_x] < min_x:
        #         min_x = min_Key[k_x]
        #     if min_Key[k_y] < min_y:
        #         min_y = min_Key[k_y]
        #
        #
        # print("左上:(%d,%d)"%(min_x,min_y))
        # print("左下:(%d,%d)"%(min_x,max_y))
        # print("右上:(%d,%d)"%(max_x,min_y))
        # print("右下:(%d,%d)"%(max_x,max_y))
        # print("中间点:(%d,%d)"%((max_x + min_x)/2,(min_y + max_y)/2))
        # print("图像大小:%d x %d"%(max_x - min_x,max_y - min_y))

        Key.to_csv('E:\BME\HRNet-Facial-Landmark-Detection-master\CEP\Landmark_%sCrop512.csv'%data, index=True)

##########################################################################

    # print(Key.loc[[1]]["Keypoint1_x"])
    else:
        if crop:
            Key = pd.read_csv("E:\BME\DCGAN-pytorch\Landmark_%sCrop.csv"%data)
        else:
            Key = pd.read_csv("E:\BME\DCGAN-pytorch\Landmark_%s.csv"%data)

        list_x = []
        list_y = []
        img_num = 69

        for i in range(1,20):
            k_x = "Keypoint%d_x" % i
            k_y = "Keypoint%d_y" % i

            list_x.append(Key[k_x][img_num-1])
            list_y.append(Key[k_y][img_num-1])

        print(list_x)
        print(list_y)
        array_xy = np.array(list(zip(list_x,list_y)))
        if crop:
            img = io.imread(os.path.join('E:\BME\DCGAN-pytorch\%sDataCrop'%data, "%03d.bmp"%(img_num+300)))
        else:
            img = io.imread(os.path.join('E:\Hughie\GAN\Data\%sData' % data, "%03d.bmp" % (img_num+300)))
        plt.figure()
        plt.title("%03d.bmp"%(img_num+300))
        show_landmarks(img,array_xy)

        plt.show()

        # print(zip(list_x,list_y))

max_stars_repo_name: Waterbearbear/cep-HRnet

max_issues_repo_name: Waterbearbear/cep-HRnet

max_forks_repo_name: Waterbearbear/cep-HRnet


text:
import gc
import logging

import numpy as np
import pandas as pd

from arekit.common.data.input.providers.columns.base import BaseColumnsProvider
from arekit.common.utils import progress_bar_defined, progress_bar_iter

logger = logging.getLogger(__name__)

class BaseRowsStorage(object):

    def __init__(self):
        self._df = None

    # region properties

    # TODO. Temporary hack, however this should be removed in future.
    @property
    def DataFrame(self):
        return self._df

    # endregion

    @classmethod
    def from_tsv(cls, filepath, sep='\t', compression='gzip', encoding='utf-8', header="infer"):
        instance = cls()
        instance._df = pd.read_csv(filepath,
                                   sep=sep,
                                   encoding=encoding,
                                   compression=compression,
                                   header=header)
        return instance

    # region private methods

    @staticmethod
    def __create_empty(cols_with_types):
        """ cols_with_types: list of pairs ("name", dtype)
        """
        assert(isinstance(cols_with_types, list))
        data = np.empty(0, dtype=np.dtype(cols_with_types))
        return pd.DataFrame(data)

    def __fill_with_blank_rows(self, row_id_column_name, rows_count):
        assert(isinstance(row_id_column_name, str))
        assert(isinstance(rows_count, int))
        self._df[row_id_column_name] = list(range(rows_count))
        self._df.set_index(row_id_column_name, inplace=True)

    def __set_value(self, row_ind, column, value):
        self._df.at[row_ind, column] = value

    def __log_info(self):
        logger.info(self._df.info())

    def __filter(self, column_name, value):
        return self._df[self._df[column_name] == value]

    # endregion

    # region protected methods

    def _balance(self, column_name):
        """ Performs oversampled balancing.
        """
        assert (isinstance(self._df, pd.DataFrame))

        max_size = self._df[column_name].value_counts().max()

        dframes = [self._df]
        for class_index, group in self._df.groupby(column_name):
            dframes.append(group.sample(max_size - len(group), replace=True))

        # Clear resources
        self._df = pd.concat(dframes)
        for df in dframes:
            del df

        gc.collect()

    # endregion

    # region public methods

    def find_by_value(self, column_name, value):
        # TODO. Return new storage. (Encapsulation)
        return self.__filter(column_name=column_name, value=value)

    def find_first_by_value(self, column_name, value):
        # TODO. Return new storage. (Encapsulation)
        row = self.__filter(column_name=column_name, value=value)
        return row.iloc[0]

    def iter_column_values(self, column_name, dtype=None):
        values = self._df[column_name]
        if dtype is None:
            return values
        return values.astype(dtype)

    def get_row(self, row_index):
        return self._df.iloc[row_index]

    def get_cell(self, row_index, column_name):
        return self._df.iloc[row_index][column_name]

    def init_empty(self, columns_provider):
        cols_with_types = columns_provider.get_columns_list_with_types()
        self._df = self.__create_empty(cols_with_types)

    def fill(self, iter_rows_func, columns_provider, desc=""):
        assert(callable(iter_rows_func))
        assert(isinstance(columns_provider, BaseColumnsProvider))

        logged_rows_it = progress_bar_iter(iterable=iter_rows_func(True),
                                           desc="Calculating rows count",
                                           unit="rows")
        rows_count = sum(1 for _ in logged_rows_it)

        logger.info("Filling with blank rows: {}".format(rows_count))
        self.__fill_with_blank_rows(row_id_column_name=columns_provider.ROW_ID,
                                    rows_count=rows_count)
        logger.info("Completed!")

        it = progress_bar_defined(iterable=iter_rows_func(False),
                                  desc="{fmt}".format(fmt=desc),
                                  total=rows_count)

        for row_index, row in enumerate(it):
            for column, value in row.items():
                self.__set_value(row_ind=row_index,
                                 column=column,
                                 value=value)

        self.__log_info()

    def free(self):
        del self._df
        gc.collect()

    # endregion

    # region base methods

    def __iter__(self):
        for row_index, row in self._df.iterrows():
            yield row_index, row

    # endregion

max_stars_repo_name: nicolay-r/AREk

max_issues_repo_name: nicolay-r/AREk

max_forks_repo_name: nicolay-r/AREk


text:
#############################################################################################################
# NCM evaluation : mix summets from simplex with mean of crops
#############################################################################################################

import os
from tqdm import tqdm
import torch
import numpy as np
import random
import warnings
import pickle
from fstools.args import process_arguments
from fstools.utils import fix_seed, load_features, stats
from fstools.few_shot_utils import define_runs, generate_runs
warnings.filterwarnings("ignore")

def ncmInterpol(features, run_classes, run_indices, n_shots, n_runs, args, AS_feats=None, lam_mix=1):
    """
    Compute NCM based on simplex summets
    Perform a linear combination between mean of summets and each  summets
    - AS_feats : Tensor of Average of feature crops.
    - lam_mix : coefficient interpolatation between average crops of each image and the summets from the simplex
    """
    with torch.no_grad():
        batch_few_shot_runs = 1
        dim = features[0].shape[-1]
        K = [feat.shape[0] for feat in features]
        dim = features[0].shape[-1]
        minK, maxK = min(K), max(K)
        D = torch.ones(len(K), maxK, dim).to(args.device) # convert simplex data to Tensor format
        if minK!=maxK:
            for k in range(len(D)):
                D[k, :K[k]] = features[k]
        D = D.reshape(20,600, -1, dim)
        K_t = torch.Tensor(K).reshape(20,600, 1) # number of summets per image
        scores = []
        targets = torch.arange(args.n_ways).unsqueeze(1).unsqueeze(0).to(args.device)

        for batch_idx in tqdm(range(n_runs // batch_few_shot_runs)):
            runs = torch.stack([generate_runs(D[:, :,k, :].to(args.device), run_classes, run_indices, batch_idx, batch_few_shot_runs=batch_few_shot_runs) for k in range(maxK)], dim=3).squeeze(0) # Generate runs
            runs_K = generate_runs(K_t.to(args.device), run_classes, run_indices, batch_idx, batch_few_shot_runs=batch_few_shot_runs).squeeze(0).squeeze(-1) # Number of summets per image
            AS_runs = generate_runs(AS_feats.to(args.device), run_classes, run_indices, batch_idx, batch_few_shot_runs=batch_few_shot_runs).squeeze(0)

            support = runs[:,:n_shots]
            support_K = runs_K[:,:n_shots]
            query = runs[:,n_shots:n_shots+args.n_queries]
            query_K = runs_K[:,n_shots:n_shots+args.n_queries]

            # Retrieve true summets of support
            support_cat_K = [[support[c, n, :support_K[c, n].int().item()] for n in range(n_shots)] for c in range(args.n_ways)]

            # Mix support data
            mixed_supports = [lam_mix*torch.cat(support_cat_K[c])+(1-lam_mix)*AS_runs[c, 0] for c in range(args.n_ways)]
            distances_all_queries = []
            for cq in range(args.n_ways):
                for q in range(args.n_queries):
                    onequery = lam_mix*query[cq][q][:query_K[cq,q].int()]+(1-lam_mix)*AS_runs[cq, n_shots+q] # mix queries with mean
                    distances_c = torch.stack([torch.norm(onequery.unsqueeze(1)-mixed_supports[c].unsqueeze(0), dim=2).min() for c in range(args.n_ways)]) # compute distances of all summets between the shot and the query and get their min value
                    distances_all_queries.append(distances_c)

            distances_all_queries = torch.stack(distances_all_queries).reshape(args.n_ways, args.n_queries, args.n_ways) # out of all the min distances get the argmin distance between the classes and the query
            winners = torch.min(distances_all_queries, dim = 2)[1]
            scores += list((winners == targets).float().mean(dim = 1).mean(dim = 1).to("cpu").numpy())
        return stats(scores, ""), torch.Tensor(scores)

if __name__=='__main__':

    args = process_arguments()
    fix_seed(args.seed, deterministic=args.deterministic)

    novel_features, AS_feats, base_features = load_features(args.features_path, args.features_base_path)
    print('Features Loaded')
    mean_base_features = torch.mean(base_features.reshape(-1, base_features.shape[-1]), dim=0).to('cpu')

    # For each image, get closest crop to each simplex summet

    with open(args.centroids_file, 'rb') as pickle_file:
        features = pickle.load(pickle_file)
    print('Summets loaded')

    num_elements = [600]*20
    runs = list(zip(*[define_runs(args.n_ways, s, args.n_queries, 20, num_elements, args.n_runs) for s in args.n_shots]))
    run_classes, run_indices = runs[0], runs[1]

    if args.preprocessing == 'ME':
        features = [x - mean_base_features.unsqueeze(0) for x in features]
        features = [x / torch.norm(x, dim = 1, keepdim = True) for x in features]
        AS_feats = AS_feats - mean_base_features.unsqueeze(0)
        AS_feats = AS_feats / torch.norm(AS_feats, dim = 2, keepdim = True)

    (acc, conf), scores = ncmInterpol(features, run_classes[0], run_indices[0], args.n_shots[0], args.n_runs, args, AS_feats=AS_feats, lam_mix=args.lamda_mix)
    print(f'{args.lamda_mix}: {np.round(100*acc,2)}% ±{np.round(conf*100, 2)}%')

    if args.log_perf:
        with open(args.log_perf, 'wb') as handle:
            pickle.dump(scores, handle, protocol=pickle.HIGHEST_PROTOCOL)

max_stars_repo_name: ybendou/banana-penguin

max_issues_repo_name: ybendou/banana-penguin

max_forks_repo_name: ybendou/banana-penguin


text:
import matplotlib.pyplot as plt
from scipy.sparse import coo_matrix

def plot_sparse_matrix(m):
    if not isinstance(m, coo_matrix):
        m = coo_matrix(m)
    fig = plt.figure()
    ax = fig.add_subplot(111, facecolor='black')
    ax.plot(m.col, m.row, 's', color='white', ms=0.1)
    ax.set_xlim(0, m.shape[1])
    ax.set_ylim(0, m.shape[0])
    ax.set_aspect('equal')
    for spine in ax.spines.values():
        spine.set_visible(False)
    ax.invert_yaxis()
    ax.set_aspect('equal')
    ax.set_xticks([])
    ax.set_yticks([])
    return ax

max_stars_repo_name: saullocastro/tudaesasII

max_issues_repo_name: saullocastro/tudaesasII

max_forks_repo_name: saullocastro/tudaesasII

reason: from scipy


text:
import numpy as np
#from glob import glob

dtype_halo_ahf = np.dtype([('id','<i4'),
            ('hostHalo','<i4'),
                ('nSub','<i4'),
                ('mvir','<f4'),
                  ('np','<i4'),
                   ('x','<f4'),
                   ('y','<f4'),
                   ('z','<f4'),
                  ('vx','<f4'),
                  ('vy','<f4'),
                  ('vz','<f4'),
                ('rvir','<f4'),
                ('Rmax','<f4'),
                  ('R2','<f4'),
             ('mbp_off','<f4'),
             ('com_off','<f4'),
                ('Vmax','<f4'),
               ('V_esc','<f4'),
                ('sigV','<f4'),
              ('lambda','<f4'),
             ('lambdaE','<f4'),
                ('Lx','<f4'),
                ('Ly','<f4'),
                ('Lz','<f4'),
                ('b','<f4'),
                ('c','<f4'),
                ('Eax','<f4'),
                ('Eay','<f4'),
                ('Eaz','<f4'),
                ('Ebx','<f4'),
                ('Eby','<f4'),
                ('Ebz','<f4'),
                ('Ecx','<f4'),
                ('Ecy','<f4'),
                ('Ecz','<f4'),
           ('overdens','<f4'),
              ('nbins','<i4'),
            ('fMhires','<f4'),
               ('Ekin','<f4'),
                ('Epot','<f4'),
                ('SurfP','<f4'),
                ('Phi0','<f4'),
                ('cNFW','<f4'),
                ('n_gas','<i4'),
                ('M_gas','<f4'),
                ('lambda_gas','<f4'),
                ('lambdaE_gas','<f4'),
                ('Lx_gas','<f4'),
                ('Ly_gas','<f4'),
                ('Lz_gas','<f4'),
                ('b_gas','<f4'),
                ('c_gas','<f4'),
                ('Eax_gas','<f4'),
                ('Eay_gas','<f4'),
                ('Eaz_gas','<f4'),
                ('Ebx_gas','<f4'),
                ('Eby_gas','<f4'),
                ('Ebz_gas','<f4'),
                ('Ecx_gas','<f4'),
                ('Ecy_gas','<f4'),
                ('Ecz_gas','<f4'),
                ('Ekin_gas','<f4'),
                ('Epot_gas','<f4'),
                ('n_star','<i4'),
                ('M_star','<f4'),
                ('lambda_star','<f4'),
                ('lambdaE_star','<f4'),
                ('Lx_star','<f4'),
                ('Ly_star','<f4'),
                ('Lz_star','<f4'),
                ('b_star','<f4'),
                ('c_star','<f4'),
                ('Eax_star','<f4'),
                ('Eay_star','<f4'),
                ('Eaz_star','<f4'),
                ('Ebx_star','<f4'),
                ('Eby_star','<f4'),
                ('Ebz_star','<f4'),
                ('Ecx_star','<f4'),
                ('Ecy_star','<f4'),
                ('Ecz_star','<f4'),
                ('Ekin_star','<f4'),
                ('Epot_star','<f4')])

class AHF_halo():
    def __init__(self, fn=None, load=True, return_id=False):
        """
        Assumes the files follow the default AHF naming scheme.

        To be merged with Tipsy class because a snapshot and a halo catalogue
        are always closely related!
        """

        self.data = None
        self._has_data = False

        self.set_fn(fn)# = fn

        if load: self.load_ahf_data()

        if return_id is False:
            self.return_id = False
            self._return_id_list = None
        else:
            if hasattr(return_id, "__len__"):
                # String also has __len__, but let's just ignore such cases.
                self._return_id_list = return_id
            else:
                self._return_id_list = None # None = load all halo's ids.
            self.return_id = True

        if self.return_id:
            self.load_halo_member()

    def set_fn(self, fn):
        """
        TODO
        It's very annoying to give a full file name everytime.
        Let' me just use nout!

        # NOTE: The file name is split at "_halos". So, "_halos" should not appear
        more than once in the file name.

        A conventional file name consists of three parts.
        snapshot_base + redshift + AHF_output_type

        """
        #file_candidates = glob(fn+"*_halos")

        self._fn = fn # Whatever that is...
        last = fn.split(".")[-1]
        if "_halos" in last:
            self._fn_base = fn.split(last)[0]
        else:
            try:
                int(last)
                self._fn_base = fn
            except IOError as e:
                print("file name not understood:" + str(e))

    def load_ahf_data(self, fn=None):
        if fn is None:
            self._fn = self._fn_base + "AHF_halos"

        self.data = np.genfromtxt(self._fn, dtype=dtype_halo_ahf, skip_header=1)
        self._has_data = True

    def physical_unit(self):
        """

        """

    def load_halo_member(self, fn=None):
        """
        Load paticle membership from ".AHF_particles" file.

        Needs the halo catalogue, self.data.

        To match is with the simulation data,
        I need a complete particle list or the particle ID.
        """
        if not self._has_data:
            print("[AHF.load_halo_member] Load catalog data first.")
            return

        if fn is None:
            # always assum hcat._fn is available.
            # Otherwise asserting npart will fail anyway.
            fn = self._fn.replace("_halos", "_particles")

        self.ind_list=[]

        inds = []
        types = []

        with open(fn, "r")  as f:
            f.readline()
            for hid, hnp in zip(self.data["id"], self.data["np"]):
                l = f.readline()
                nparts = int(l.split()[0])
                assert nparts == hnp

                if self._return_id_list is not None:
                    if hid in self._return_id_list:
                        chunk = np.fromfile(f, dtype=int, sep=" ", count=nparts * 2).reshape(nparts, 2)
                        self.ind_list.append(dict(hid=hid,
                        idgas = chunk[chunk[:,1]==0,0],
                        iddm = chunk[chunk[:,1]==1,0],
                        idstar = chunk[chunk[:,1]==4,0]))
                    else:
                        # Skip n lines. This is a fast way.
                        for nn in npart: next(f)
                else:
                    chunk = np.fromfile(f, dtype=int, sep=" ", count=nparts * 2).reshape(nparts, 2)
                    self.ind_list.append(dict(hid=hid,
                                              idgas = chunk[chunk[:,1]==0,0],
                                              iddm = chunk[chunk[:,1]==1,0],
                                              idstar = chunk[chunk[:,1]==4,0]))

    def load_subs(self, fn=None):
        if not self._has_data:
            print("[AHF.load_sub] Load catalog data first.")
            return

        if fn is None:
            # always assum hcat._fn is available.
            # Otherwise asserting npart will fail anyway.
            fn = self._fn.replace("_halos", "_substructure")

        sub_ids = []
        hosts = []
        with open(fn, "r") as f:
            while True:
                sss = f.readline().split()
                try:
                    hid, nsub = int(sss[0]), int(sss[1])
                except:
                    break
                subs = np.array(f.readline().split()).astype(np.int32)
                assert len(subs) == nsub
                sub_ids.append(subs)
                hosts.append(hid)

        self.subs = dict(zip(hosts, sub_ids))

    def to_physical():
        """
        convert into physical units and save the state
        """

max_stars_repo_name: Hoseung/pyRamAn

max_issues_repo_name: Hoseung/pyRamAn

max_forks_repo_name: Hoseung/pyRamAn


text:
import ast, json
import pandas as pd
import numpy as np
from PIL import Image
import tables
from torch.utils import data
from torchvision import transforms
from collections import Counter

def split_dataset(file_path, label_list=None):
    df = pd.read_csv(file_path, sep='\t')
    df = df.drop_duplicates(subset='current_image_id', keep="last")
    # Keep specific labels
    if label_list: df = df[df['comparison'].isin(label_list)]
    train_split = pd.read_csv('/home/ilourentzou/ChestXGenome/splits/train.csv')
    valid_split = pd.read_csv('/home/ilourentzou/ChestXGenome/splits/valid.csv')
    test_split = pd.read_csv('/home/ilourentzou/ChestXGenome/splits/test.csv')
    pid = list(train_split['dicom_id'].unique())
    train = df[df['current_image_id'].isin(pid)]
    #train2 = df[df['previous_image_id'].isin(pid)] #either works
    pid = list(valid_split['dicom_id'].unique())
    dev = df[df['current_image_id'].isin(pid)]
    pid = list(test_split['dicom_id'].unique())
    test = df[df['current_image_id'].isin(pid)]
    print(Counter(train['comparison']))
    print(Counter(dev['comparison']))
    print(Counter(test['comparison']))
    return train, dev, test

class H5Reader(object):

    def __init__(self, h5_path, filename_idx=None):
        """For fast reading from a h5 file. A file name to index dict is created for fast indexing.
        :param h5_path: the h5 file path to read from.
        """
        self.h5_path = h5_path
        self.h5_file = tables.open_file(self.h5_path, 'r')
        self.data = self.h5_file.root.data
        self.header = self.h5_file.root.header if '/header' in self.h5_file else None
        if filename_idx is None:
            self.filename_idx = dict(enumerate(self.h5_file.root.filename.iterrows()))
            self.filename_idx = dict(zip(self.filename_idx.values(), self.filename_idx.keys()))
            #{name: i for i, name in enumerate(self.h5_file.root.filename.iterrows())}
        else:
            self.filename_idx = filename_idx
        self.h5_file.close()

    def read_image(self, filename, gray_scale=True):
        """Read an image given the file name.
        :param filename: the file name.
        :param gray_scale: if True, convert to gray-scale image.
        :return: the image. None if the file name does not exist.
        """
        self.h5_file = tables.open_file(self.h5_path, 'r')
        self.data = self.h5_file.root.data
        if filename not in self.filename_idx:
            return None
        image = self.data[self.filename_idx[filename]]
        image = image.squeeze()
        if gray_scale and image.ndim == 3:
            image = image.mean(axis=-1).astype(image.dtype)
        self.h5_file.close()
        return image

    def read_header(self, filename):
        """Read the header given the file name.
        :param filename: the file name.
        :return: the with original image shape and resizing info. None if the file name does not exist.
        """
        if filename not in self.filename_idx:
            return None
        header = self.header[self.filename_idx[filename]]
        header = json.loads(header)
        return header.copy()

class ComparisonsDataset(data.Dataset):
    def __init__(self, csv_file, h5file, labelset, transform=None):
        """
        Args:
            csv_file (pd.dataframe): dataframe table containing image names, disease severity category label, and other metadata
            image_dir (string): directory containing all of the image files
            transform (callable, optional): optional transform to be applied on a sample
        """
        self.h5file = H5Reader(h5file)
        self.csv_file = csv_file
        self.transform = transform
        self.labelset = labelset
        if self.transform is None:
            self.transform = transforms.Compose([transforms.ToTensor()])

    def __len__(self):
        return len(self.csv_file)

    def __getitem__(self, idx):
        item = self.csv_file.iloc[idx]
        current_image_id = item['current_image_id']
        previous_image_id = item['previous_image_id']
        current_image = self.h5file.read_image(current_image_id+'.dcm', gray_scale=True)
        previous_image = self.h5file.read_image(previous_image_id+'.dcm', gray_scale=True)
        bbox_current = ast.literal_eval(item['bbox_coord_224_subject'])
        bbox_previous = ast.literal_eval(item['bbox_coord_224_object'])
        assert len(bbox_current) == len(bbox_previous) == 4
        current_image = Image.fromarray(current_image/np.max(current_image))
        previous_image = Image.fromarray(previous_image/np.max(previous_image))
        cropped_current = current_image.crop(bbox_current)
        cropped_previous = previous_image.crop(bbox_previous)
        if self.transform is not None:
            cropped_current = self.transform(cropped_current)
            cropped_previous = self.transform(cropped_previous)
        label = item['comparison']
        labelidx = self.labelset.index(label)
        meta = dict(item)
        return cropped_current, cropped_previous, labelidx, meta
        #[batch_size x channels x height x width]