text:
"""Base class for an agent that defines the possible actions. """

import numpy as np

import utility_funcs as util

# basic moves every agent should do
BASE_ACTIONS = {
    0: "MOVE_LEFT",  # Move left
    1: "MOVE_RIGHT",  # Move right
    2: "MOVE_UP",  # Move up
    3: "MOVE_DOWN",  # Move down
    4: "STAY",  # don't move
    5: "TURN_CLOCKWISE",  # Rotate counter clockwise
    6: "TURN_COUNTERCLOCKWISE",
}  # Rotate clockwise

class Agent(object):
    def __init__(self, agent_id, team_num, credo, rogue_flag, rogue_deg, start_pos, start_orientation, full_map, row_size, col_size):
        """Superclass for all agents.

        Parameters
        ----------
        agent_id: (str)
            a unique id allowing the map to identify the agents
        start_pos: (np.ndarray)
            a 2d array indicating the x-y position of the agents
        start_orientation: (np.ndarray)
            a 2d array containing a unit vector indicating the agent direction
        full_map: (2d array)
            a reference to this agent's view of the environment
        row_size: (int)
            how many rows up and down the agent can look
        col_size: (int)
            how many columns left and right the agent can look
        """
        self.agent_id = agent_id
        self.team_num = team_num
        self.credo = credo
        self.is_rogue = rogue_flag
        self.rogue_deg = rogue_deg

        self.pos = np.array(start_pos)
        self.orientation = start_orientation
        self.full_map = full_map
        self.row_size = row_size
        self.col_size = col_size
        self.reward_this_turn = 0
        self.prev_visible_agents = None

    @property
    def action_space(self):
        """Identify the dimensions and bounds of the action space.

        MUST BE implemented in new environments.

        Returns
        -------
        gym Box, Discrete, or Tuple type
            a bounded box depicting the shape and bounds of the action space
        """
        raise NotImplementedError

    @property
    def observation_space(self):
        """Identify the dimensions and bounds of the observation space.

        MUST BE implemented in new environments.

        Returns
        -------
        gym Box, Discrete or Tuple type
            a bounded box depicting the shape and bounds of the observation
            space
        """
        raise NotImplementedError

    def action_map(self, action_number):
        """Maps action_number to a desired action in the map"""
        raise NotImplementedError

    def get_char_id(self):
        # return bytes(str(int(self.agent_id[-1]) + 1), encoding="ascii")
        return bytes(str(self.team_num + 1), encoding="ascii")              #changed for agents on same team = same color

    def get_state(self):
        return util.return_view(self.full_map, self.pos, self.row_size, self.col_size)

    def compute_reward(self):
        reward = self.reward_this_turn
        self.reward_this_turn = 0
        return reward

    def set_pos(self, new_pos):
        self.pos = np.array(new_pos)

    def get_pos(self):
        return self.pos

    def translate_pos_to_egocentric_coord(self, pos):
        offset_pos = pos - self.pos
        ego_centre = [self.row_size, self.col_size]
        return ego_centre + offset_pos

    def set_orientation(self, new_orientation):
        self.orientation = new_orientation

    def get_orientation(self):
        return self.orientation

    def return_valid_pos(self, new_pos):
        """Checks that the next pos is legal, if not return current pos"""
        ego_new_pos = new_pos  # self.translate_pos_to_egocentric_coord(new_pos)
        new_row, new_col = ego_new_pos
        # You can't walk through walls, closed doors or switches
        if self.is_tile_walkable(new_row, new_col):
            return new_pos
        else:
            return self.pos

    def update_agent_pos(self, new_pos):
        """Updates the agents internal positions

        Returns
        -------
        old_pos: (np.ndarray)
            2 element array describing where the agent used to be
        new_pos: (np.ndarray)
            2 element array describing the agent positions
        """
        old_pos = self.pos
        ego_new_pos = new_pos  # self.translate_pos_to_egocentric_coord(new_pos)
        new_row, new_col = ego_new_pos
        if self.is_tile_walkable(new_row, new_col):
            validated_new_pos = new_pos
        else:
            validated_new_pos = self.pos
            # TODO(dr): set action to stay
        self.set_pos(validated_new_pos)
        # TODO(ev) list array consistency
        return self.pos, np.array(old_pos)

    def is_tile_walkable(self, row, column):
        return (
            0 <= row < self.full_map.shape[0]
            and 0 <= column < self.full_map.shape[1]
            # You can't walk through walls, closed doors or switches
            and self.full_map[row, column] not in [b"@", b"D", b"w", b"W"]
        )

    def update_agent_rot(self, new_rot):
        self.set_orientation(new_rot)

    def hit(self, char):
        """Defines how an agent responds to being hit by a beam of type char"""
        raise NotImplementedError

    def consume(self, char):
        """Defines how an agent interacts with the char it is standing on"""
        raise NotImplementedError

HARVEST_ACTIONS = BASE_ACTIONS.copy()
HARVEST_ACTIONS.update({7: "FIRE"})  # Fire a penalty beam

class HarvestAgent(Agent):
    def __init__(self, agent_id, team_num, credo, rogue_flag, rogue_deg, start_pos, start_orientation, full_map, view_len):
        self.view_len = view_len
        super().__init__(agent_id, team_num, credo, rogue_flag, rogue_deg, start_pos, start_orientation, full_map, view_len, view_len)
        self.update_agent_pos(start_pos)
        self.update_agent_rot(start_orientation)

    # Ugh, this is gross, this leads to the actions basically being
    # defined in two places
    def action_map(self, action_number):
        """Maps action_number to a desired action in the map"""
        return HARVEST_ACTIONS[action_number]

    def hit(self, char):
        if char == b"F":
            self.reward_this_turn -= 50

    def fire_beam(self, char):
        if char == b"F":
            self.reward_this_turn -= 1

    def get_done(self):
        return False

    def consume(self, char):
        """Defines how an agent interacts with the char it is standing on"""
        if char == b"A":
            self.reward_this_turn += 1
            return b" "
        else:
            return char

CLEANUP_ACTIONS = BASE_ACTIONS.copy()
CLEANUP_ACTIONS.update({7: "FIRE", 8: "CLEAN"})  # Fire a penalty beam  # Fire a cleaning beam

class CleanupAgent(Agent):
    def __init__(self, agent_id, team_num, credo, rogue_flag, rogue_deg, start_pos, start_orientation, full_map, view_len):
        self.view_len = view_len
        super().__init__(agent_id, team_num, credo, rogue_flag, rogue_deg, start_pos, start_orientation, full_map, view_len, view_len)
        # remember what you've stepped on
        self.update_agent_pos(start_pos)
        self.update_agent_rot(start_orientation)

    # Ugh, this is gross, this leads to the actions basically being
    # defined in two places
    def action_map(self, action_number):
        """Maps action_number to a desired action in the map"""
        return CLEANUP_ACTIONS[action_number]

    def fire_beam(self, char):
        if char == b"F":
            self.reward_this_turn -= 1

    def get_done(self):
        return False

    def hit(self, char):
        if char == b"F":
            self.reward_this_turn -= 50

    def consume(self, char):
        """Defines how an agent interacts with the char it is standing on"""
        if char == b"A":
            self.reward_this_turn += 1
            return b" "
        else:
            return char

SWITCH_ACTIONS = BASE_ACTIONS.copy()
SWITCH_ACTIONS.update({7: "TOGGLE_SWITCH"})  # Fire a switch beam

class SwitchAgent(Agent):
    def __init__(self, agent_id, start_pos, start_orientation, full_map, view_len):
        self.view_len = view_len
        super().__init__(agent_id, start_pos, start_orientation, full_map, view_len, view_len)
        # remember what you've stepped on
        self.update_agent_pos(start_pos)
        self.update_agent_rot(start_orientation)
        self.is_done = False

    # Ugh, this is gross, this leads to the actions basically being
    # defined in two places
    def action_map(self, action_number):
        """Maps action_number to a desired action in the map"""
        return SWITCH_ACTIONS[action_number]

    def fire_beam(self, char):
        # Cost of firing a switch beam
        # Nothing for now.
        if char == b"F":
            self.reward_this_turn += 0

    def get_done(self):
        return self.is_done

    def consume(self, char):
        """Defines how an agent interacts with the char it is standing on"""
        if char == b"d":
            self.reward_this_turn += 1
            self.is_done = True
            return b" "
        else:
            return char


text:
import datetime
import os
from functools import partial
from pathlib import Path
from typing import Dict, Optional, Set

import imageio
import numpy as np
from audio_utils import float_to_wav
from image_utils import stack_images_vertically
from text_utils.types import Speaker
from tqdm import tqdm
from tts_preparation import (PreparedData, get_merged_dir, get_prep_dir,
                             load_testset, load_valset)
from tts_preparation.app.prepare import load_totalset
from waveglow.app.defaults import (DEFAULT_DENOISER_STRENGTH, DEFAULT_SEED,
                                   DEFAULT_SIGMA)
from waveglow.app.io import (_get_validation_root_dir, get_checkpoints_dir,
                             get_train_dir, load_prep_settings)
from waveglow.core import (CheckpointWaveglow, ValidationEntries,
                           ValidationEntryOutput)
from waveglow.core import validate as validate_core
from waveglow.core.validation import get_df
from waveglow.utils import (get_all_checkpoint_iterations, get_checkpoint,
                            get_last_checkpoint, prepare_logger)

def get_repr_entries(entry_ids: Optional[Set[int]]) -> str:
  if entry_ids is None:
    return "none"
  if len(entry_ids) == 0:
    return "empty"
  return ",".join(list(sorted(map(str, entry_ids))))

def get_repr_speaker(speaker: Optional[Speaker]) -> Speaker:
  if speaker is None:
    return "none"
  return speaker

def get_val_dir(train_dir: Path, ds: str, iterations: Set[int], full_run: bool, entry_ids: Optional[Set[int]]) -> Path:
  if len(iterations) > 3:
    its = ",".join(str(x) for x in sorted(iterations)[:3]) + ",..."
  else:
    its = ",".join(str(x) for x in sorted(iterations))

  subdir_name = f"{datetime.datetime.now():%d.%m.%Y__%H-%M-%S}__ds={ds}__entries={get_repr_entries(entry_ids)}__its={its}__full={full_run}"
  return _get_validation_root_dir(train_dir) / subdir_name

def get_val_entry_dir(val_dir: Path, entry: PreparedData, iteration: int) -> Path:
  return val_dir / f"it={iteration}_id={entry.entry_id}"

def save_stats(val_dir: Path, validation_entries: ValidationEntries) -> None:
  path = val_dir / "total.csv"
  df = get_df(validation_entries)
  df.to_csv(path, sep="\t", header=True, index=False)

def save_results(entry: PreparedData, output: ValidationEntryOutput, val_dir: Path, iteration: int) -> None:
  dest_dir = get_val_entry_dir(val_dir, entry, iteration)
  dest_dir.mkdir(parents=True, exist_ok=True)
  imageio.imsave(dest_dir / "original.png", output.mel_orig_img)
  imageio.imsave(dest_dir / "inferred_denoised.png", output.mel_inferred_denoised_img)
  imageio.imsave(dest_dir / "diff.png", output.mel_denoised_diff_img)
  np.save(dest_dir / "original.mel.npy", output.mel_orig)
  np.save(dest_dir / "inferred_denoised.mel.npy", output.mel_inferred_denoised)
  float_to_wav(output.wav_orig, dest_dir / "original.wav", sample_rate=output.orig_sr)

  float_to_wav(output.wav_inferred_denoised, dest_dir /
               "inferred_denoised.wav", sample_rate=output.inferred_sr)

  float_to_wav(output.wav_inferred, dest_dir / "inferred.wav", sample_rate=output.inferred_sr)

  stack_images_vertically(
    list_im=[
      dest_dir / "original.png",
      dest_dir / "inferred_denoised.png",
      dest_dir / "diff.png",
    ],
    out_path=dest_dir / "comparison.png"
  )

def validate_generic(base_dir: Path, ttsp_dir: Path, merge_name: str, prep_name: str, train_name: str, ds: str = "val", entry_ids: Optional[Set[int]] = None, custom_checkpoints: Optional[Set[int]] = None, sigma: float = DEFAULT_SIGMA, denoiser_strength: float = DEFAULT_DENOISER_STRENGTH, custom_hparams: Optional[Dict[str, str]] = None, full_run: bool = False, seed: int = DEFAULT_SEED) -> None:
  train_dir = get_train_dir(base_dir, train_name)
  assert train_dir.is_dir()

  merge_dir = get_merged_dir(ttsp_dir, merge_name)
  prep_dir = get_prep_dir(merge_dir, prep_name)

  _validate(
    train_dir=train_dir,
    train_name=train_name,
    prep_dir=prep_dir,
    entry_ids=entry_ids,
    custom_checkpoints=custom_checkpoints,
    sigma=sigma,
    denoiser_strength=denoiser_strength,
    custom_hparams=custom_hparams,
    full_run=full_run,
    ds=ds,
    seed=seed,
  )

def validate(base_dir: Path, train_name: str, entry_ids: Optional[Set[int]] = None, ds: str = "val", custom_checkpoints: Optional[Set[int]] = None, sigma: float = DEFAULT_SIGMA, denoiser_strength: float = DEFAULT_DENOISER_STRENGTH, custom_hparams: Optional[Dict[str, str]] = None, full_run: bool = False, seed: int = DEFAULT_SEED) -> None:
  train_dir = get_train_dir(base_dir, train_name)
  assert train_dir.is_dir()

  ttsp_dir, merge_name, prep_name = load_prep_settings(train_dir)
  merge_dir = get_merged_dir(ttsp_dir, merge_name)
  prep_dir = get_prep_dir(merge_dir, prep_name)

  _validate(
    train_dir=train_dir,
    train_name=train_name,
    prep_dir=prep_dir,
    entry_ids=entry_ids,
    custom_checkpoints=custom_checkpoints,
    sigma=sigma,
    denoiser_strength=denoiser_strength,
    custom_hparams=custom_hparams,
    full_run=full_run,
    ds=ds,
    seed=seed,
  )

def _validate(train_dir, train_name: str, prep_dir: Path, entry_ids: Optional[Set[int]], custom_checkpoints: Optional[Set[int]], sigma: float, denoiser_strength: float, custom_hparams: Optional[Dict[str, str]], full_run: bool, ds: str, seed: int) -> None:
  if ds == "val":
    data = load_valset(prep_dir)
  elif ds == "test":
    data = load_testset(prep_dir)
  elif ds == "total":
    data = load_totalset(prep_dir)
  else:
    raise Exception()

  iterations: Set[int] = set()
  checkpoint_dir = get_checkpoints_dir(train_dir)

  if custom_checkpoints is None:
    _, last_it = get_last_checkpoint(checkpoint_dir)
    iterations.add(last_it)
  else:
    if len(custom_checkpoints) == 0:
      iterations = set(get_all_checkpoint_iterations(checkpoint_dir))
    else:
      iterations = custom_checkpoints

  val_dir = get_val_dir(
    train_dir=train_dir,
    ds=ds,
    entry_ids=entry_ids,
    full_run=full_run,
    iterations=iterations,
  )

  val_dir.mkdir(parents=True, exist_ok=True)
  val_log_path = val_dir / "log.txt"
  logger = prepare_logger(val_log_path)
  logger.info("Validating...")
  logger.info(f"Checkpoints: {','.join(str(x) for x in sorted(iterations))}")

  result = ValidationEntries()

  for iteration in tqdm(sorted(iterations)):
    logger.info(f"Current checkpoint: {iteration}")
    checkpoint_path = get_checkpoint(checkpoint_dir, iteration)
    taco_checkpoint = CheckpointWaveglow.load(checkpoint_path, logger)
    save_callback = partial(save_results, val_dir=val_dir, iteration=iteration)

    validation_entries = validate_core(
      checkpoint=taco_checkpoint,
      data=data,
      custom_hparams=custom_hparams,
      entry_ids=entry_ids,
      full_run=full_run,
      train_name=train_name,
      save_callback=save_callback,
      logger=logger,
      sigma=sigma,
      denoiser_strength=denoiser_strength,
      seed=seed,
    )

    result.extend(validation_entries)

  if len(result) > 0:
    save_stats(val_dir, result)
    logger.info(f"Saved output to: {val_dir}")

ext: py

max_stars_repo_name: stefantaubert/waveglow

max_issues_repo_name: stefantaubert/waveglow

max_forks_repo_name: stefantaubert/waveglow

reason: import numpy


text:
import argparse
from typing import Dict, List, Optional, Union

import matplotlib.pyplot as plt
import mlflow
import numpy as np
import yaml

import air_quality_uci_dataset
import air_quality_uci_rnn

def train_and_evaluate(this_type: Optional[Union[List[str], str]], train_size: float, time_steps: int,
                       n_units: Dict[str, int], activations: Dict[str, str], optimizer: str, loss: str, metrics: List[str], batch_size: int, epochs: int, shuffle: bool) -> object:
    """Train an rnn model on air quality uci dataset and evaluate it.

    :param Optional[Union[List[str], str]] this_type: string or list of strings to define elements of this dataset, defaults to None
    :param float train_size: ratio of training dataset size
    :param int time_steps: the length of data to predict
    :param Dict[str, int] n_units:  the numbers of units on an RNN layer and a hidden layer
    :param Dict[str, str] activations: activation function names on an RNN layer and a hidden layer
    :param str optimizer: optimizer
    :param str loss: loss function
    :param List[str] metrics: metrics
    :param int batch_size: batch size
    :param int epochs: the number of epochs
    :param bool shuffle: whether the dataset shuffles or not
    :return object: trained rnn model on air quality uci dataset
    """
    # Load the air quality uci dataset
    air_quality_uci_data = air_quality_uci_dataset.AirQualityUciDataset(this_type=this_type)

    # Create an RNN for air quality uci
    model = air_quality_uci_rnn.AirQualityUciRNN()

    # Prepare to train
    model.prepare_to_train(air_quality_uci_data=air_quality_uci_data, train_size=train_size, time_steps=time_steps)

    # Build a model
    model.build_rnn(n_units=n_units, activations=activations)

    # Train the model
    model.train(optimizer=optimizer, loss=loss, metrics=metrics, batch_size=batch_size, epochs=epochs, shuffle=shuffle)

    # Evaluate the model
    model.evaluate()

    return model

def save_evaluation_result(trained_model: object, output_path: str) -> None:
    """Save a given information.

    :param object trained_model: a trained model that has already been evaluated
    :param str output_path: a path to an output file
    """
    evaluated_data = trained_model.evaluated_data
    test_y = trained_model.test_y

    size = len(test_y[0])
    num_columns = 4
    num_rows = int(size // num_columns + 1)
    f, axs = plt.subplots(nrows=num_rows, ncols=num_columns, sharex=True, sharey=True)
    len_data = len(test_y)
    row_index = 0
    col_index = 0
    for _ in range(size):
        pred_color = "red"
        true_color = "blue"

        axs[row_index, col_index].plot(range(len_data), evaluated_data[:, row_index + col_index], linewidth=0.5, label="prediction", color=pred_color)
        axs[row_index, col_index].plot(range(len_data), test_y[:, row_index + col_index], linewidth=0.5, label="true value", color=true_color)
        all_values = [evaluated_data[:, row_index + col_index], test_y[:, row_index + col_index]]
        min_y = np.min(all_values)
        max_y = np.max(all_values)
        axs[row_index, col_index].set_ylim([min_y - 0.1, max_y + 0.1])
        plt.xlabel(f"prediction: {pred_color}, true: {true_color}")
        if col_index != num_columns - 1:
            col_index += 1
        else:
            col_index = 0
            row_index += 1

    plt.savefig(output_path, dpi=300)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Train and evaluate RNN for air quality uci dataset.")

    parser.add_argument("-c", "--config_yaml_path", required=False, type=str, default="./config_air_quality_uci.yaml")
    args = parser.parse_args()

    # Load configs
    with open(args.config_yaml_path, "r") as yaml_f:
        config = yaml.safe_load(yaml_f)
    config_mlflow = config["mlflow"]

    # Start training and evaluating whilist logging the information
    mlflow.set_experiment(config_mlflow["experiment_name"])
    with mlflow.start_run(run_name=config_mlflow["run_name"]):
        mlflow.keras.autolog()

        config_dataset = config["dataset"]
        config_rnn = config["rnn"]
        config_train = config["train"]
        trained_model = train_and_evaluate(**config_dataset, **config_rnn, **config_train)

        config_save = config["save"]
        save_evaluation_result(trained_model=trained_model, **config_save)

        mlflow.log_artifact(args.config_yaml_path)
        mlflow.log_artifact(config_save["output_path"])


text:
#!/usr/bin/env python3
# encoding: utf-8

import time
import numpy as np
from keras.models import Sequential
from keras.layers.recurrent import LSTM
from keras.callbacks import EarlyStopping
from keras.layers.core import Dense, Activation, Dropout
from utils import read_dataset, split_dataset
from nn_common import plot_result, store_model, load_model
from evaluation import mase

def compile_model(nneurons, loss_fn, dropout=(0.0, 0.0)):
    model = Sequential()
    model.add(LSTM(nneurons[0], input_dim=1, return_sequences=True))
    if dropout[0] > 0:
        model.add(Dropout(dropout[0]))
    model.add(LSTM(nneurons[1], return_sequences=False))
    if dropout[1] > 0:
        model.add(Dropout(dropout[1]))
    model.add(Dense(1))
    model.add(Activation('linear'))
    model.compile(optimizer='rmsprop', loss=loss_fn)
    return model

def run_network(window, model=None, show_plot=False, save_model=False):
    start_time = time.time()

    print('loading and prepare data set...')
    data = read_dataset('../datasets/internet-traffic-data-5minutes.csv')
    X_train, y_train, X_test, y_test, mean, std = split_dataset(
        data, window, ratio=0.90, standardize=True)

    # reshape s.t. the data has the form (#examples, #values in sequences,
    # dim. of each value in the sequence)
    X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
    X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))

    if model is None:
        print('initialize model...')
        model = compile_model(
            nneurons=(19, 5), dropout=(0.0, 0.0), loss_fn='mse')
        print('model ', model.summary())
        print('train model...')
        early_stopping = EarlyStopping(monitor='val_loss', patience=2)
        model.fit(X_train, y_train, nb_epoch=100, validation_split=0.10,
                  callbacks=[early_stopping])

    print('make predictions...')
    prediction = model.predict(X_test).flatten()

    if show_plot:
        plot_result(prediction, y_test, mean, std)
        print('mase = ', mase(y_train, y_test, prediction))

    if save_model:
        store_model(model)

    print('totoal duration: {:.2f} seconds'.format(time.time() - start_time))

def hyper_parameter_search(max_evals=200):
    from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL

    data = read_dataset('../datasets/internet-traffic-data-5minutes.csv')
    space = {
        'nneurons1': hp.randint('nneurons1', 15),
        'nneurons2': hp.randint('nneurons2', 15),
        'window': hp.randint('window', 15)
    }

    def objective(params):
        nneurons1 = params['nneurons1']
        nneurons2 = params['nneurons2']
        window = params['window']

        if nneurons1 < 1 or nneurons2 < 1 or window < 1:
            return {'status': STATUS_FAIL}

        X_train, y_train, *_ = split_dataset(
            data, [1] * window, ratio=0.90, standardize=True)
        X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
        model = compile_model(
            (nneurons1, nneurons2), loss_fn='mse', dropout=(0.0, 0.0))
        hist = model.fit(
            X_train, y_train, nb_epoch=100, validation_split=0.10,
            callbacks=[EarlyStopping(monitor='val_loss', patience=2)],
            verbose=0)

        return {'loss': hist.history['val_loss'][-1], 'status': STATUS_OK}

    return fmin(objective, space=space, algo=tpe.suggest, max_evals=max_evals)

if __name__ == '__main__':
    print('run hyper param search')
    print(hyper_parameter_search(100))


text:
import json

import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
from iteration_utilities import flatten

from define_collection_wave import folder
from helpers import create_folder, cleanhtml

path_mcdonalds = create_folder('1_McDonalds', folder)

def parse_json(dat):
    fileName = path_mcdonalds + '/item_' + str(item_id) + '.json'
    with open(fileName, 'w') as f:
        json.dump(dat, f)
    item_type = dat.get('item').get('item_type')
    nutrients = dat.get('item').get('nutrient_facts').get('nutrient')
    nutrient_dic = {}
    for i in range(len(nutrients)):
        nutrientName = nutrients[i]['nutrient_name_id']
        nutrientQuantity = nutrients[i]['value']
        nurteintHundred_g_per_product = nutrients[i]['hundred_g_per_product']
        nutrient_uom = nutrients[i]['uom']
        nutrient_adultDV = nutrients[i]['adult_dv']
        nutrient_childDV = nutrients[i]['child_dv']
        nutrient_dic.update({nutrientName + '_Quantity': nutrientQuantity,
                             nutrientName + '_100_g_per_product': nurteintHundred_g_per_product,
                             nutrientName + '_uom': nutrient_uom, nutrientName + '_adultDV': nutrient_adultDV,
                             nutrientName + '_childDV': nutrient_childDV})
    dat_dict = {
        'Product_Name': dat.get('item').get('item_name'),
        'Product_Title': dat.get('item').get('item_meta_title'),
        'Product_Description': dat.get('item').get('description'),
        'Product_Ingredients': cleanhtml(dat.get('item').get('item_ingredient_statement')),
        'Product_Category': dat.get('item').get('default_category').get('category').get('name') if dat.get('item').get(
            'default_category') else 'NA',
        'item_type': item_type
    }
    dat_dict.update(nutrient_dic)
    return dat_dict

def parse_combo(dat):
    # combo meals --> get individual nutrient information and add them together
    fileName = path_mcdonalds + '/item_' + str(item_id) + '.json'
    with open(fileName, 'w') as f:
        json.dump(dat, f)
    item_type = dat.get('item').get('item_type')
    components = dat.get('item').get('components').get('component')
    components_list = []
    for component in components:
        component_id = component.get('id')
        url_component = f'https://www.mcdonalds.com/wws/json/getItemDetails.htm?country=UK&language=en&showLiveData=true&item={component_id}'
        dat_component = requests.get(url_component).json()
        parse_component = parse_json(dat_component)
        if float(parse_component.get('energy_kcal_100_g_per_product'))*100 !=0:
            parse_component.update({'servingweight': float(parse_component.get('energy_kcal_Quantity'))/float(parse_component.get('energy_kcal_100_g_per_product'))*100})
        else:
            parse_component.update({'servingweight': None})
        components_list.append(parse_component)
    # add the quantities (absolute energy) and % DV
    nutrients = {}
    for var in [key for key in components_list[0].keys() if 'Quantity' in key or 'DV' in key]:
        try:
            nutrients[var] = sum([float(component.get(var)) for component in components_list])
        except:
            nutrients[var] = None
    # weighted density
    for var in [key for key in components_list[0].keys() if '100_g_per_product'in key]:
        try:
            nutrients[var] = np.average([float(component.get(var)) for component in components_list], weights=[float(component.get('servingweight')) for component in components_list])
        except:
            pass
    # keep the same unit of measure
    for var in [key for key in components_list[0].keys() if 'uom'in key]:
        nutrients[var] = components_list[0].get(var)
    dat_dict = {
        'Product_Name': dat.get('item').get('item_name'),
        'Product_Title': dat.get('item').get('item_meta_title'),
        'Product_Description': ', '.join([component.get('Product_Name') for component in components_list]),
        'Product_Ingredients': cleanhtml(dat.get('item').get('item_ingredient_statement')),
        'Product_Category': dat.get('item').get('default_category').get('category').get('name') if dat.get('item').get(
            'default_category') else 'NA',
        'item_type': item_type
    }
    dat_dict.update(nutrients)
    return(dat_dict)

mcdonalds_url = 'https://www.mcdonalds.com/gb/en-gb/menu.html'
r = requests.get(mcdonalds_url)
soup = BeautifulSoup(r.text, 'html.parser')

# categories
categories = soup.find_all('a',{"class":"link key-arrow-move"})
category_links = ['https://www.mcdonalds.com' + category.get('href') for category in categories]
category_links = set(category_links)
item_ids = []
for category_link in category_links:
    #print(category_link)
    r = requests.get(category_link)
    soup = BeautifulSoup(r.text,'html.parser')
    items = soup.find_all('a', {"class": 'categories-item-link'}) #the new path
    # old path
    if len(items) == 0:
        items = soup.find_all('a', {"class": 'mcd-category-page__item-link'})
    ids = [item['data-at'].split(':')[3] for item in items[:-1]]
    #print(len(ids))
    item_ids.append(ids)

item_ids = list(flatten(item_ids))
item_ids = [item_id.replace('a','') for item_id in item_ids]
item_ids = set(item_ids)

mcdonalds_list = []
additional_items = []
for item_id in item_ids:
    url = f'https://www.mcdonalds.com/wws/json/getItemDetails.htm?country=UK&language=en&showLiveData=true&item={item_id}'
    dat = requests.get(url).json()
    if dat.get('item').get('relation_types'):
        related_items = dat.get('item').get('relation_types').get('relation_type')[0].get('related_items').get(
            'related_item')
        related_itemids = [related_item['id'] for related_item in related_items if related_item['id'] != int(item_id)]
        additional_items.extend(related_itemids)
    try:
        dict_temp = parse_json(dat)
    except:
        # likely a combo meal (e.g., happy meal with different components)
        dict_temp = parse_combo(dat)
    dict_temp.update({'item_id': item_id})
    mcdonalds_list.append(dict_temp)

for item_id in additional_items:
    url = f'https://www.mcdonalds.com/wws/json/getItemDetails.htm?country=UK&language=en&showLiveData=true&item={item_id}'
    dat = requests.get(url).json()
    dict_temp = parse_json(dat)
    dict_temp.update({'item_id': item_id})
    mcdonalds_list.append(dict_temp)

mcdonalds = pd.DataFrame(mcdonalds_list)
mcdonalds.to_csv(path_mcdonalds + '/mcdonalds_items.csv')

max_stars_repo_name: YuruHuang/MenuTracker

max_issues_repo_name: YuruHuang/MenuTracker

max_forks_repo_name: YuruHuang/MenuTracker


text:
"""
Modified version of librosa's mel function and supporting functions to not require hard install on pi (we dont need numba here)
https://github.com/librosa/librosa
"""

import numpy as np
import time

def fft_frequencies(sr=22050, n_fft=2048):
    return np.linspace(0, float(sr) / 2, int(1 + n_fft // 2), endpoint=True)

def mel_to_hz(mels, htk=False):
    mels = np.asanyarray(mels)

    if htk:
        return 700.0 * (10.0 ** (mels / 2595.0) - 1.0)

    # Fill in the linear scale
    f_min = 0.0
    f_sp = 200.0 / 3
    freqs = f_min + f_sp * mels

    # And now the nonlinear scale
    min_log_hz = 1000.0  # beginning of log region (Hz)
    min_log_mel = (min_log_hz - f_min) / f_sp  # same (Mels)
    logstep = np.log(6.4) / 27.0  # step size for log region

    if mels.ndim:
        # If we have vector data, vectorize
        log_t = mels >= min_log_mel
        freqs[log_t] = min_log_hz * np.exp(logstep * (mels[log_t] - min_log_mel))
    elif mels >= min_log_mel:
        # If we have scalar data, check directly
        freqs = min_log_hz * np.exp(logstep * (mels - min_log_mel))

    return freqs

def hz_to_mel(frequencies, htk=False):
    frequencies = np.asanyarray(frequencies)

    if htk:
        return 2595.0 * np.log10(1.0 + frequencies / 700.0)

    # Fill in the linear part
    f_min = 0.0
    f_sp = 200.0 / 3

    mels = (frequencies - f_min) / f_sp

    # Fill in the log-scale part

    min_log_hz = 1000.0  # beginning of log region (Hz)
    min_log_mel = (min_log_hz - f_min) / f_sp  # same (Mels)
    logstep = np.log(6.4) / 27.0  # step size for log region

    if frequencies.ndim:
        # If we have array data, vectorize
        log_t = frequencies >= min_log_hz
        mels[log_t] = min_log_mel + np.log(frequencies[log_t] / min_log_hz) / logstep
    elif frequencies >= min_log_hz:
        # If we have scalar data, heck directly
        mels = min_log_mel + np.log(frequencies / min_log_hz) / logstep

    return mels

def mel_frequencies(n_mels=128, fmin=0.0, fmax=11025.0, htk=False):

    # 'Center freqs' of mel bands - uniformly spaced between limits
    min_mel = hz_to_mel(fmin, htk=htk)
    max_mel = hz_to_mel(fmax, htk=htk)

    mels = np.linspace(min_mel, max_mel, n_mels)

    return mel_to_hz(mels, htk=htk)

def mel(
    sr,
    n_fft,
    n_mels=128,
    fmin=0.0,
    fmax=None,
    htk=False,
    dtype=np.float32,
):

    if fmax is None:
        fmax = float(sr) / 2

    # Initialize the weights
    n_mels = int(n_mels)
    weights = np.zeros((n_mels, int(1 + n_fft // 2)), dtype=dtype)

    # Center freqs of each FFT bin
    fftfreqs = fft_frequencies(sr=sr, n_fft=n_fft)

    # 'Center freqs' of mel bands - uniformly spaced between limits
    mel_f = mel_frequencies(n_mels + 2, fmin=fmin, fmax=fmax, htk=htk)
    fdiff = np.diff(mel_f)
    ramps = np.subtract.outer(mel_f, fftfreqs)

    for i in range(n_mels):
        # lower and upper slopes for all bins
        lower = -ramps[i] / fdiff[i]
        upper = ramps[i + 2] / fdiff[i + 1]

        # .. then intersect them with each other and zero
        weights[i] = np.maximum(0, np.minimum(lower, upper))

    for i in range(n_mels):
        # lower and upper slopes for all bins
        lower = -ramps[i] / fdiff[i]
        upper = ramps[i + 2] / fdiff[i + 1]

        # .. then intersect them with each other and zero
        weights[i] = np.maximum(0, np.minimum(lower, upper))

    # Slaney-style mel is scaled to be approx constant energy per channel
    enorm = 2.0 / (mel_f[2 : n_mels + 2] - mel_f[:n_mels])
    weights *= enorm[:, np.newaxis]

    return weights

max_stars_repo_name: SpyGuyIan/UDPixelViz

max_issues_repo_name: SpyGuyIan/UDPixelViz

max_forks_repo_name: SpyGuyIan/UDPixelViz


text:
# Copyright (c) 2020, Huawei Technologies.All rights reserved.
#
# Licensed under the BSD 3-Clause License  (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import torch
import numpy as np
from common_device_type import dtypes, instantiate_device_type_tests
from util_test import create_common_tensor
from common_utils import TestCase, run_tests

class TestSlice(TestCase):
   def npu_op_exec(self, input1, offset, sizes):
        output = torch.npu_slice(input1, offset, sizes)
        output = output.to("cpu")
        output = output.numpy()
        return output

   def test_slice_int32(self, device):
        input    = torch.tensor([[1,2,3,4,5], [6,7,8,9,10]]).npu()
        exoutput = torch.tensor([[1,2],[6,7]])
        output   = self.npu_op_exec(input, [0, 0], [2, 2])
        self.assertRtolEqual(exoutput.numpy(), output)

instantiate_device_type_tests(TestSlice, globals(), except_for="cpu")
if __name__ == "__main__":
    run_tests()

max_stars_repo_name: Ascend/pytorch

max_issues_repo_name: Ascend/pytorch

max_forks_repo_name: Ascend/pytorch


text:
# ==============================================================================
# ConvLSTM in seq2seq mode
#
# (1) simple:
# (2) teacher forcing:
#
# author: Lu Li
# email: lilu83@mail.sysu.edu.cn
# ==============================================================================

import numpy as np
import tensorflow as tf
from tensorflow.keras import Model, layers, Input
from tensorflow.python.keras.layers.convolutional import Conv2D
from tensorflow.python.keras.layers.convolutional_recurrent import \
    ConvLSTM2DCell

from tensorflow.keras.layers import ConvLSTM2D, BatchNormalization, Dense

#TODO: 把训练方式作为形参设置，整合iterative/teacher forcing/standard
class IterativeConvLSTM2D(layers.Layer):
    def __init__(self,
                 filters,
                 kernel_size,
                 padding='valid',
                 strides=1,
                 return_sequences=True,
                 return_states=True):
        self.filters=filters
        self.kernel_size = kernel_size
        self.padding = padding
        self.strides = strides
        self.return_sequences = return_sequences
        self.return_states = return_states
        super().__init__()

    def build(self, input_shape):

        self.t = input_shape[1]
        print(self.t)

        self.cell_list = []
        for i in range(self.t):
            self.cell_list.append(ConvLSTM2DCell(filters=self.filters,
                       kernel_size=self.kernel_size,
                       padding=self.padding,
                       strides=self.strides))
        self.built = True

    def call(self, inputs, initial_state):

        if initial_state is not None: h, c = initial_state

        h_list = []
        for i in range(self.t):
            h, [h, c] = self.cell_list[i](h, states=[h, c])
            h_list.append(h)

        return tf.stack(h_list, axis=1)

class __Seq2seqConvLSTM(layers.Layer):
    def __init__(self,
                 n_filters_factor,
                 filter_size,
                 dec_len,
                 train_mode=None,
                 *args,
                 **kwargs):
        super().__init__(*args, **kwargs)
        self.dec_len = dec_len
        self.train_mode = 'teacher'

        # encode convlstm
        self.enc_convlstm = layers.ConvLSTM2D(np.int(16 * n_filters_factor),
                                              kernel_size=filter_size,
                                              return_sequences=True,
                                              return_state=True,
                                              padding='same',
                                              kernel_initializer='he_normal')
        self.enc_bn = layers.BatchNormalization(axis=-1)

        # decode convlstm
        self.dec_convlstm = layers.ConvLSTM2D(np.int(16 * n_filters_factor),
                                              kernel_size=filter_size,
                                              return_sequences=True,
                                              padding='same',
                                              kernel_initializer='he_normal')
        self.dec_bn = layers.BatchNormalization(axis=-1)

        #
        self.dec_convlstm_cell = ConvLSTM2DCell(np.int(16 * n_filters_factor),
                                                kernel_size=filter_size,
                                                padding='same',
                                                kernel_initializer='he_normal')

    def call(self, x_encoder, x_decoder, training=None, mask=None):
        print(x_encoder.shape)
        print(x_decoder.shape)

        enc_out, state = self.encoder(x_encoder=x_encoder)
        print(enc_out.shape)

        if self.train_mode == 'teacher':
            if x_decoder:
                raise KeyError('teacher forcing training needs decoder inputs')
            dec_out = self.decoder(x_decoder, initial_state=state)
            print(dec_out.shape)
        else:
            dec_out = self.decoder(enc_out[:, -1], initial_state=state)
        return dec_out

    def encoder(self, x_encoder):
        enc_out, state_h, state_c = self.enc_convlstm(x_encoder)
        return enc_out, [state_h, state_c]

    def decoder(self, initial_x_decoder, initial_state=None):
        """decoder follow standard propagation method."""
        print('1')
        print(initial_x_decoder.shape)

        h, c = initial_state
        #x = initial_x_decoder
        print(h.shape)
        x_stack, h_stack, c_stack = [], [], []
        for i in range(self.dec_len):
            print(i)
            x = initial_x_decoder[:,i]
            print(x.shape)
            x, [h, c] = self.dec_convlstm_cell(initial_x_decoder=x,
                                               initial_state=[h, c])
            x_stack.append(x)

        return x

class Seq2seqConvLSTM(Model):
    def __init__(self,
                 filters,
                 kernel_size,
                 padding='same',
                 kernel_initializer='he_normal',
                 mode='train'):
        super().__init__()

        self.filters = filters
        self.kernel_size = kernel_size
        self.padding = padding
        self.kernel_initializer = kernel_initializer

        # encoder convlstm
        self.encoder_convlstm = ConvLSTM2D(filters=self.filters,
                                           kernel_size=self.kernel_size,
                                           return_sequences=True,
                                           return_state=True,
                                           padding=self.padding,
                                           kernel_initializer=self.kernel_initializer)
        self.encoder_bn = BatchNormalization()

        # decoder convlstm
        self.decoder_convlstm = ConvLSTM2D(filters=self.filters,
                                           kernel_size=self.kernel_size,
                                           return_sequences=True,
                                           return_state=True,
                                           padding=self.padding,
                                           kernel_initializer=self.kernel_initializer)
        self.decoder_bn = BatchNormalization()

        # decoder convlstm
        self.decoder_convlstm_cell = ConvLSTM2DCell(filters=self.filters,
                                                    kernel_size=self.kernel_size,
                                                    padding=self.padding,
                                                    kernel_initializer=self.kernel_initializer)
        self.decoder_bn = BatchNormalization()

        # dense
        self.dense = Dense(1)

    def fit(self, X, y, batch_size, epochs, validation_data, callbacks):
        mdl = self.train_model()
        mdl.fit(X, y, batch_size, epochs, validation_data, callbacks)
        return self

    def train_model(self):
        X_encoder = Input((7, 112, 112, 1))
        X_decoder = Input((7, 112, 112, 1))

        encoder_outputs, encoder_states = self.encoder_convlstm(X_encoder)
        decoder_outputs = []
        for i in range(7):
            decoder_output, encoder_states = self.decoder_convlstm_cell(
                X_decoder[:,i], states=encoder_states)
            decoder_outputs.append(decoder_output)
        decoder_outputs = tf.stack(decoder_outputs, axis=1)
        outputs = tf.concat([encoder_outputs, decoder_outputs], axis=1)
        outputs = self.dense(outputs)
        mdl = Model([X_encoder, X_decoder], outputs)
        return mdl

    def predict(self, X_encoder):
        y_pred = []
        encoder_outputs, encoder_states = self.encoder_convlstm(X_encoder)
        decoder_input = encoder_states[-1]

        for _ in range(7):
            decoder_inputs, encoder_states = self.decoder_convlstm_cell(
                decoder_input, states=encoder_states)
            y_pred.append(decoder_inputs)

        return tf.stack(y_pred, axis=1)

class Seq2seqConvLSTM(layers.Layer):
    def __init__(self,
                 n_filters_factor,
                 filter_size,
                 dec_len,
                 train_mode=None,
                 *args,
                 **kwargs):
        super().__init__(*args, **kwargs)
        self.dec_len = dec_len
        self.train_mode = 'teacher'

        # encode convlstm
        self.enc_convlstm = layers.ConvLSTM2D(np.int(16 * n_filters_factor),
                                              kernel_size=filter_size,
                                              return_sequences=True,
                                              return_state=True,
                                              padding='same',
                                              kernel_initializer='he_normal')
        self.enc_bn = layers.BatchNormalization(axis=-1)

        # decode convlstm
        self.dec_convlstm = layers.ConvLSTM2D(np.int(16 * n_filters_factor),
                                              kernel_size=filter_size,
                                              return_sequences=True,
                                              padding='same',
                                              kernel_initializer='he_normal')
        self.dec_bn = layers.BatchNormalization(axis=-1)

        #
        self.dec_convlstm_cell = ConvLSTM2DCell(np.int(16 * n_filters_factor),
                                                kernel_size=filter_size,
                                                padding='same',
                                                kernel_initializer='he_normal')

    def call(self, x_encoder, x_decoder, training=None, mask=None):
        print(x_encoder.shape)
        print(x_decoder.shape)

        enc_out, state = self.encoder(x_encoder=x_encoder)
        print(enc_out.shape)

        if self.train_mode == 'teacher':
            if x_decoder:
                raise KeyError('teacher forcing training needs decoder inputs')
            dec_out = self.decoder(x_decoder, initial_state=state)
            print(dec_out.shape)
        else:
            dec_out = self.decoder(enc_out[:, -1], initial_state=state)
        return dec_out

    def encoder(self, x_encoder):
        enc_out, state_h, state_c = self.enc_convlstm(x_encoder)
        return enc_out, [state_h, state_c]

    def decoder(self, initial_x_decoder, initial_state=None):
        """decoder follow standard propagation method."""
        print('1')
        print(initial_x_decoder.shape)

        h, c = initial_state
        #x = initial_x_decoder
        print(h.shape)
        x_stack, h_stack, c_stack = [], [], []
        for i in range(self.dec_len):
            print(i)
            x = initial_x_decoder[:,i]
            print(x.shape)
            x, [h, c] = self.dec_convlstm_cell(initial_x_decoder=x,
                                               initial_state=[h, c])
            x_stack.append(x)

        return x

max_stars_repo_name: leelew/MetReg

max_issues_repo_name: leelew/MetReg

max_forks_repo_name: leelew/MetReg


text:
import os
import glob
import torch
import torch.nn as nn
import time
import datetime
import logging
import json
import numpy as np
from torch.utils.data import DataLoader

def init_log(output_dir, time, mode='train'):
    logging.basicConfig(level=logging.INFO,
                        format='%(asctime)s-%(filename)s[l:%(lineno)d]: %(message)s',
                        datefmt='%Y%m%d-%H:%M:%S',
                        filename=os.path.join(output_dir, '{}_log_{}.log'.format(mode, time)),
                        filemode='a')
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    logging.getLogger('').addHandler(console)
    return logging

class DataPrefetcher():
    """
       direct to:
       https://github.com/NVIDIA/apex/blob/f5cd5ae937f168c763985f627bbf850648ea5f3f/examples/imagenet/main_amp.py#L256
    """
    def __init__(self, loader):
        self.loader = iter(loader)
        self.stream = torch.cuda.Stream()
        self.preload()

    def preload(self):
        try:
            self.next_data = next(self.loader)
        except StopIteration:
            self.next_data = None
            return
        except TypeError:
            self.next_data = None
            return
        with torch.cuda.stream(self.stream):
            self.next_data = [x.cuda(non_blocking=True) for x in self.next_data]

    def next(self):
        torch.cuda.current_stream().wait_stream(self.stream)
        data = self.next_data
        self.preload()
        return data

class Trainer(object):
    """Base trainer class. Contains functions for training and saving/loading chackpoints.
    Trainer classes should inherit from this one and overload the train_epoch function."""

    def __init__(self, net, datasets, optimizer, scheduler, criterion,
                 batch_size, batch_size_val, max_epochs, threads, eval_freq,
                 use_gpu, resume, mode, sets, workdir, logdir, resdir):
        self.net = net
        self.datasets = datasets
        self.optimizer = optimizer
        self.scheduler = scheduler
        self.criterion = criterion
        self.batch_size = batch_size
        self.batch_size_val = batch_size_val
        self.max_epochs = max_epochs
        self.threads = threads
        self.eval_freq = eval_freq  # use 0 to disable test during training
        self.use_gpu = use_gpu
        self.resume = resume
        self.mode = mode
        self.sets = sets
        self.workdir = workdir
        self.logdir = logdir
        self.resdir = resdir
        # Init
        self.best_epoch = 0
        self.best_acc = 0
        self.start_epoch = 1
        self.global_step = 1
        #self.stats = {}
        # trainloader
        if 'train' in self.sets and self.datasets['train'] is not None:
            self.trainset = self.datasets[self.sets[0]]
            self.trainloader = DataLoader(self.trainset, self.batch_size,
                                      shuffle=True, num_workers=self.threads,
                                      pin_memory=True, drop_last=False,
                                      worker_init_fn=lambda work_id:np.random.seed(work_id))
        elif self.params.mode != 'test':
            raise Exception("train set not found!")
        # valloader
        if 'train' in self.sets and self.datasets['val'] is not None:
            self.valset = self.datasets[self.sets[1]]
            self.valloader = DataLoader(self.valset, self.batch_size_val,
                                      shuffle=False, num_workers=self.threads,
                                      pin_memory=Trainer, drop_last=False,
                                      worker_init_fn=lambda work_id:np.random.seed(work_id))
        elif self.params.mode != 'test':
            raise Exception("Val set not found!")
        # testloader
        if 'test' in self.sets and self.datasets['test'] is not None:
            self.testset = self.datasets[self.sets[2]]
            self.testloader = DataLoader(self.testset, 1, shuffle=False)
        # Workspace dir
        if self.workdir is not None:
            if not os.path.exists(self.workdir):
                os.makedirs(self.workdir)
        else:
            raise Exception("Workspace directory doesn't exist!")
        # log dir
        if self.logdir is not None:
            if not os.path.exists(self.logdir):
                os.mkdir(self.logdir)
            logging = init_log(self.logdir, self.time, self.params.mode)
            self.print = logging.info
        else:
            self.print = print
        # result dir
        if self.resdir is not None:
            self.resdir = '{}_{}'.format(self.resdir, self.time)
            if not os.path.exists(self.resdir):
                os.makedirs(self.resdir)

    def train(self):
        """Do training, you can overload this function according to your need."""
        self.print("Log dir: {}".format(self.logdir))
        # Calculate total step
        self.n_train = len(self.trainset)
        self.steps_per_epoch = np.ceil(
            self.n_train / self.batch_size).astype(np.int32)
        self.verbose = min(self.verbose, self.steps_per_epoch)
        self.n_steps = self.max_epochs * self.steps_per_epoch
        # calculate model parameters memory
        para = sum([np.prod(list(p.size())) for p in self.net.parameters()])
        memory = para * 4 / 1000 / 1000
        self.print('Model {} : params: {:4f}M'.format(self.net._get_name(), memory))
        self.print('###### Experiment Parameters ######')
        for k, v in self.params.items():
            self.print('{0:<22s} : {1:}'.format(k, v))
        self.print("{0:<22s} : {1:} ".format('trainset sample', self.n_train))
        # GO!!!!!!!!!
        start_time = time.time()
        self.train_total_time = 0
        self.time_sofar = 0
        for epoch in range(self.start_epoch, self.max_epochs + 1):
            # Decay Learning Rate
            self.scheduler.step()
            # Train one epoch
            total_loss = self.train_epoch(epoch)
            torch.cuda.empty_cache()
            # Evaluate the model
            if self.eval_freq and epoch % self.eval_freq == 0:
                acc = self.eval(epoch)
                torch.cuda.empty_cache()
        self.print("Finished training! Best epoch {} best acc {}".format(self.best_epoch, self.best_acc))
        self.print("Spend time: {:.2f}h".format((time.time() - start_time) / 3600))

    def train_epoch(self, epoch):
        """Train one epoch, you can overload this function according to your need."""
        device = torch.device('cuda:0' if self.use_gpu else 'cpu')
        self.net.to(device)
        self.criterion.to(device)
        self.net.train()
        # Iterate over data.
        prefetcher = DataPrefetcher(self.trainloader)
        image, label = prefetcher.next()
        step = 0
        while image is not None:
            image, label = data[0].to(device), data[1].to(device)
            before_op_time = time.time()
            self.optimizer.zero_grad()
            output = self.net(image)
            total_loss = self.criterion(output, label)
            total_loss.backward()
            self.optimizer.step()
            fps = image.shape[0] / (time.time() - before_op_time)
            time_sofar = self.train_total_time / 3600
            time_left = (self.n_steps / self.global_step - 1.0) * time_sofar
            if self.verbose > 0 and (step + 1) % (self.steps_per_epoch // self.verbose) == 0:
                print_str = 'Epoch [{:>3}/{:>3}] | Step [{:>3}/{:>3}] | fps {:4.2f} | Loss: {:7.3f} | Time elapsed {:.2f}h | Time left {:.2f}h'. \
                    format(epoch, self.max_epochs, step + 1, self.steps_per_epoch,
                           fps, total_loss, time_sofar, time_left)
                self.print(print_str)
            self.global_step += 1
            self.train_total_time += time.time() - before_op_time
        image, label = prefetcher.next()
        step += 1
        return total_loss

    def eval(self, epoch):
        """Do evaluating, you can overload this function according to your need."""
        torch.backends.cudnn.benchmark = True
        self.n_val = len(self.valset)
        self.print("{0:<22s} : {1:} ".format('valset sample', self.n_val))
        self.print("<-------------Evaluate the model-------------->")
        # Evaluate one epoch
        acc = self.eval_epoch()
        self.print('The {}th epoch | acc: {:.4f}%'.format(epoch, acc * 100))
        # Save the checkpoint
        if self.logdir:
            self.save_checkpoint(epoch, acc)
            self.print('=> Checkpoint was saved successfully!')
        else:
            if acc >= self.best_acc:
                self.best_epoch, self.acc = epoch, acc
        return acc

    def eval_epoch(self):
        """Evaluate one epoch, you can overload this function according to your need."""
        if self.eval_freq:
            raise NotImplementedError

    def save(self, epoch, acc):
        self._save_checkpoint(epoch, acc)
        self.print('=> Checkpoint was saved successfully!')
        self._update_best_epoch(epoch, acc)

    def _update_best_epoch(self, epoch, acc):
        net_type = type(self.net).__name__
        if acc >= self.best_acc:
            last_best = os.path.join(self.logdir, '{}_{:03d}.pkl'.format(net_type, self.best_epoch))
            if os.path.isfile(last_best):
                os.remove(last_best)
            self.best_epoch = epoch
            self.best_acc = acc

    def _save_checkpoint(self, epoch, acc):
        """
        Saves a checkpoint of the network and other variables.
        Only save the best and latest epoch.
        """
        net_type = type(self.net).__name__
        if epoch - self.eval_freq != self.best_epoch:
            pre_save = os.path.join(self.logdir, '{}_{:03d}.pkl'.format(net_type, epoch - self.eval_freq))
            if os.path.isfile(pre_save):
                os.remove(pre_save)
        cur_save = os.path.join(self.logdir, '{}_{:03d}.pkl'.format(net_type, epoch))
        state = {
            'epoch': epoch,
            'acc': acc,
            'net_type': net_type,
            'net': self.net.state_dict(),
            'optimizer': self.optimizer.state_dict(),
            #'scheduler': self.scheduler.state_dict(),
            'use_gpu': self.use_gpu,
            'save_time': datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
        }
        torch.save(state, cur_save)
        return True

    def reload(self, resume, mode='finetune'):
        if resume:
            if self._load_checkpoint(resume, mode):
                self.print('=> Checkpoint was loaded successfully!')
        else:
            raise Exception("Resume doesn't exist!")

    def _load_checkpoint(self, checkpoint=None, mode='finetune'):
        """
        Parameters
        ----------
        checkpoint:
            Loads a network checkpoint file. Can be called in three different ways:
                _load_checkpoint(epoch_num):
                    Loads the network at the given epoch number (int).
                _load_checkpoint(path_to_checkpoint):
                    Loads the file from the given absolute path (str).
                _load_checkpoint(net):
                    Loads the parameters from the given net (nn.Module).
        mode:
            'fintune': load the pretrained model and start training from epoch 1,
            'retain': reload the model and restart training from the breakpoint.
            'test': just testing the model.
        """
        net_type = type(self.net).__name__
        if isinstance(checkpoint, nn.Module):
            old_model_dict = checkpoint.state_dict()
        else:
            if isinstance(checkpoint, int):
                # Checkpoint is the epoch number
                if self.logdir is not None:
                    checkpoint_path = os.path.join(self.logdir, '{}_{:03d}.pkl'.format(net_type, checkpoint))
                else:
                    raise Exception("Log dir doesn't exist!")
            elif isinstance(checkpoint, str):
                # checkpoint is the path
                checkpoint_path = os.path.expanduser(checkpoint)
            else:
                raise TypeError
            checkpoint_dict = torch.load(checkpoint_path, map_location={'cuda:1': 'cuda:0'})
            old_model_dict = checkpoint_dict['net']
        #assert net_type == checkpoint_dict['net_type'], 'Network is not of correct type.'
        if mode == 'finetune':
            net_state_dict = self.net.state_dict()
            new_model_dict = {key: value for key, value in old_model_dict.items()
                              if key in net_state_dict and value.shape == net_state_dict[key].shape}
            net_state_dict.update(new_model_dict)
            start_epoch = 1
            best_acc = 0
            use_gpu = self.use_gpu
        elif mode == 'retain':
            net_state_dict = checkpoint_dict['net']
            start_epoch = checkpoint_dict['epoch'] + 1
            best_acc = checkpoint_dict['acc']
            self.optimizer.load_state_dict(checkpoint_dict['optimizer'])
            if 'scheduler' in checkpoint_dict:
                self.scheduler.load_state_dict(checkpoint_dict['scheduler'])
                self.scheduler.last_epoch = start_epoch - 1
            use_gpu = checkpoint_dict['use_gpu']
        elif mode == 'test':
            net_state_dict = checkpoint_dict['net']
            use_gpu = checkpoint_dict['use_gpu']
            start_epoch = checkpoint_dict['epoch']
            best_acc = checkpoint_dict['acc']
        self.start_epoch = start_epoch
        self.best_acc = best_acc
        self.net.load_state_dict(net_state_dict)
        self.use_gpu = use_gpu
        return True

if __name__ == '__main__':
    pass

max_stars_repo_name: ShawnCheung/Attention-depth

max_issues_repo_name: ShawnCheung/Attention-depth

max_forks_repo_name: ShawnCheung/Attention-depth


text:
#
# Copyright 1993-2018 NVIDIA Corporation.  All rights reserved.
#
# NOTICE TO LICENSEE:
#
# This source code and/or documentation ("Licensed Deliverables") are
# subject to NVIDIA intellectual property rights under U.S. and
# international Copyright laws.
#
# These Licensed Deliverables contained herein is PROPRIETARY and
# CONFIDENTIAL to NVIDIA and is being provided under the terms and
# conditions of a form of NVIDIA software license agreement by and
# between NVIDIA and Licensee ("License Agreement") or electronically
# accepted by Licensee.  Notwithstanding any terms or conditions to
# the contrary in the License Agreement, reproduction or disclosure
# of the Licensed Deliverables to any third party without the express
# written consent of NVIDIA is prohibited.
#
# NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
# LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
# SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
# PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
# NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
# DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
# NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
# NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
# LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
# SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
# OF THESE LICENSED DELIVERABLES.
#
# U.S. Government End Users.  These Licensed Deliverables are a
# "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
# 1995), consisting of "commercial computer software" and "commercial
# computer software documentation" as such terms are used in 48
# C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
# only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
# 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
# U.S. Government End Users acquire the Licensed Deliverables with
# only those rights set forth herein.
#
# Any use of the Licensed Deliverables in individual and commercial
# software must include, in the user documentation and internal
# comments to the code, the above Disclaimer and U.S. Government End
# Users Notice.
#

#!/usr/bin/python
import os
from random import randint
import numpy as np

try:
    import pycuda.driver as cuda
    import pycuda.gpuarray as gpuarray
    import pycuda.autoinit
except ImportError as err:
    raise ImportError("""ERROR: Failed to import module({})
Please make sure you have pycuda and the example dependencies installed.
sudo apt-get install python(3)-pycuda
pip install tensorrt[examples]""".format(err))

try:
    from PIL import Image
except ImportError as err:
    raise ImportError("""ERROR: Failed to import module ({})
Please make sure you have Pillow installed.
For installation instructions, see:
http://pillow.readthedocs.io/en/stable/installation.html""".format(err))

import mnist

try:
    import torch
except ImportError as err:
    raise ImportError("""ERROR: Failed to import module ({})
Please make sure you have PyTorch installed.
For installation instructions, see:
http://pytorch.org/""".format(err))

# TensorRT must be imported after any frameworks in the case where
# the framework has incorrect dependencies setup and is not updated
# to use the versions of libraries that TensorRT imports.
try:
    import tensorrt as trt
except ImportError as err:
    raise ImportError("""ERROR: Failed to import module ({})
Please make sure you have the TensorRT Library installed
and accessible in your LD_LIBRARY_PATH""".format(err))

G_LOGGER = trt.infer.ConsoleLogger(trt.infer.LogSeverity.INFO)

ITERATIONS = 10
INPUT_LAYERS = ["data"]
OUTPUT_LAYERS = ['prob']
INPUT_H = 28
INPUT_W = 28
OUTPUT_SIZE = 10

def create_pytorch_engine(max_batch_size, builder, dt, model):
    network = builder.create_network()

    data = network.add_input(INPUT_LAYERS[0], dt, (1, INPUT_H, INPUT_W))
    assert(data)

    #-------------
    conv1_w = model['conv1.weight'].cpu().numpy().reshape(-1)
    conv1_b = model['conv1.bias'].cpu().numpy().reshape(-1)
    conv1 = network.add_convolution(data, 20, (5,5),  conv1_w, conv1_b)
    assert(conv1)
    conv1.set_stride((1,1))

    #-------------
    pool1 = network.add_pooling(conv1.get_output(0), trt.infer.PoolingType.MAX, (2,2))
    assert(pool1)
    pool1.set_stride((2,2))

    #-------------
    conv2_w = model['conv2.weight'].cpu().numpy().reshape(-1)
    conv2_b = model['conv2.bias'].cpu().numpy().reshape(-1)
    conv2 = network.add_convolution(pool1.get_output(0), 50, (5,5), conv2_w, conv2_b)
    assert(conv2)
    conv2.set_stride((1,1))

    #-------------
    pool2 = network.add_pooling(conv2.get_output(0), trt.infer.PoolingType.MAX, (2,2))
    assert(pool2)
    pool2.set_stride((2,2))

    #-------------
    fc1_w = model['fc1.weight'].cpu().numpy().reshape(-1)
    fc1_b = model['fc1.bias'].cpu().numpy().reshape(-1)
    fc1 = network.add_fully_connected(pool2.get_output(0), 500, fc1_w, fc1_b)
    assert(fc1)

    #-------------
    relu1 = network.add_activation(fc1.get_output(0), trt.infer.ActivationType.RELU)
    assert(relu1)

    #-------------
    fc2_w = model['fc2.weight'].cpu().numpy().reshape(-1)
    fc2_b = model['fc2.bias'].cpu().numpy().reshape(-1)
    fc2 = network.add_fully_connected(relu1.get_output(0), OUTPUT_SIZE, fc2_w, fc2_b)
    assert(fc2)

    #-------------
    # Using log_softmax in training, cutting out log softmax here since no log softmax in TRT
    fc2.get_output(0).set_name(OUTPUT_LAYERS[0])
    network.mark_output(fc2.get_output(0))

    builder.set_max_batch_size(max_batch_size)
    builder.set_max_workspace_size(1 << 20)

    #builder.set_fp16_mode(True)

    engine = builder.build_cuda_engine(network)
    network.destroy()

    return engine

def model_to_engine(model, max_batch_size):
    builder = trt.infer.create_infer_builder(G_LOGGER)
    engine = create_pytorch_engine(max_batch_size, builder, trt.infer.DataType.FLOAT, model)
    assert(engine)

    modelstream = engine.serialize()
    engine.destroy()
    builder.destroy()
    return modelstream

# Run inference on device
def infer(context, input_img, output_size, batch_size):
    # Load engine
    engine = context.get_engine()
    assert(engine.get_nb_bindings() == 2)
    # Convert input data to Float32
    input_img = input_img.astype(np.float32)
    # Create output array to receive data
    output = np.empty(output_size, dtype = np.float32)

    # Allocate device memory
    d_input = cuda.mem_alloc(batch_size * input_img.nbytes)
    d_output = cuda.mem_alloc(batch_size * output.nbytes)

    bindings = [int(d_input), int(d_output)]

    stream = cuda.Stream()

    # Transfer input data to device
    cuda.memcpy_htod_async(d_input, input_img, stream)
    # Execute model
    context.enqueue(batch_size, bindings, stream.handle, None)
    # Transfer predictions back
    cuda.memcpy_dtoh_async(output, d_output, stream)

    # Return predictions
    return output

def main():
    path = dir_path = os.path.dirname(os.path.realpath(__file__))

    # The mnist package is a simple PyTorch mnist example. mnist.learn() trains a network for
    # PyTorch's provided mnist dataset. mnist.get_trained_model() returns the state dictionary
    # of the trained model. We use this to demonstrate the full training to inference pipeline
    mnist.learn()
    model = mnist.get_trained_model()

    # Typically training and inference are seperated so using torch.save() and saving the
    # model's state dictionary and then using torch.load() to load the state dictionary
    #
    # e.g:
    # model = torch.load(path + "/trained_mnist.pyt")
    modelstream = model_to_engine(model, 1)

    runtime = trt.infer.create_infer_runtime(G_LOGGER)
    engine = runtime.deserialize_cuda_engine(modelstream.data(), modelstream.size(), None)

    if modelstream:
        modelstream.destroy()

    img, target = mnist.get_testcase()
    img = img.numpy()
    target = target.numpy()
    print("\n| TEST CASE | PREDICTION |")
    for i in range(ITERATIONS):
        img_in = img[i].ravel()
        target_in = target[i]
        context = engine.create_execution_context()
        out = infer(context, img_in, OUTPUT_SIZE, 1)
        print("|-----------|------------|")
        print("|     " + str(target_in) + "     |      " + str(np.argmax(out)) + "     |")

    print('')
    context.destroy()
    engine.destroy()
    runtime.destroy()

if __name__ == "__main__":
    main()

max_stars_repo_name: aimuch/AIEnvConfig

max_issues_repo_name: aimuch/AIEnvConfig

max_forks_repo_name: aimuch/AIEnvConfig

reason: import numpy,import pycuda


text:
import numpy as np
from IO_utils import *
import matplotlib.pyplot as plt
import matplotlib
from scipy import ndimage
from skimage import measure, morphology, segmentation
from ete3 import Tree, TreeStyle, NodeStyle, faces, AttrFace, CircleFace

class Bubble(measure._regionprops._RegionProperties):
	def centroid(self):
		return tuple(self.coords.mean(axis=0))
	def ionization_center(self):
		return

def bubbleprops(label_image, intensity_image=None, cache=True, areasort=True):
    label_image = np.squeeze(label_image)
    if label_image.ndim not in (2, 3):
        raise TypeError('Only 2-D and 3-D images supported.')
    if not np.issubdtype(label_image.dtype, np.integer):
        raise TypeError('Label image must be of integral type.')
    bubbles = []
    objects = ndimage.find_objects(label_image)
    for i, sl in enumerate(objects):
        if sl is None:
            continue
        label = i + 1
        props = Bubble(sl, label, label_image, intensity_image, cache)
        bubbles.append(props)
    if areasort:
    	bubbles = sorted(bubbles, key=lambda r: r.area, reverse=True)
    return bubbles

def bbox_merit(b1L, b2L, b1R, b2R):
	bb = np.where(b1R<b2R, b1R, b2R) - np.where(b1L>b2L, b1L, b2L)
	vol1 = np.product(b1R-b1L)
	vol2 = np.product(b2R-b2L)
	if False:
		return float(np.product(bb))/max(vol1, vol2)
	else:
		return float(np.product(bb))**2/np.product(b1R-b1L)/np.product(b2R-b2L)

def get_merit(coords1, coords2):
	N1, N2 = coords1.shape[0], coords2.shape[0]
	N12 = float(len( set([tuple(row) for row in coords1]) & set([tuple(row) for row in coords2])))
	if False:
		#print 'N12, N1, N2', N12, N1, N2, N12/max(N1, N2)
		return N12/max(N1, N2)
	else:
		return N12**2/N1/N2

def get_bubbles(file, connectivity=3):
	img = np.load(file)['labels']
	labeled = measure.label(img, connectivity=connectivity)
	#mask = labeled > 0
	#labeled  = labeled *morphology.remove_small_objects(mask, 30)

	return bubbleprops(labeled)

class MergerTree:
	def __init__(self, files, maxnodes=1, min_merit=0.03):
		self.Tree = Tree()
		self.TreeDepth = len(files)
		self.maxnodes = maxnodes
		self.files = files
		self.min_merit = min_merit
		print files

	def build(self):
		Rup = None
		for n in xrange(self.TreeDepth):
			file = self.files[n]
			if n == 0:
				R = get_bubbles(file)
				R = [R[0]]  #only one root
				for r in R:
					name = str(n)+'_'+str(r.label)
					print 'Root:', name, r.area, r.centroid()
					self.Tree.add_child(name=name)
				Rup = R
			else:
				if len(Rup) == 0: break
				R = get_bubbles(file)
				newRup = []
				root = self.Tree.get_tree_root()
				for i, leaf in enumerate(self.Tree.get_leaves()):

					level = int(leaf.name.split('_')[0])
					#assert level == int(root.get_distance(leaf)-1)
					#print level, n, int(root.get_distance(leaf)-1)
					if level != n-1: continue  #make sure we're operating on the lower level leaves only
					print leaf
					ind = int(leaf.name.split('_')[1])
					rup = None
					for r in Rup:
						if r.label == ind:
							rup = r
					if rup is None:
						print ind, "not found!"
						continue
					leaf.add_features(vol=rup.area)

					assert ind == int(rup.label)
					# R, merits = self.find_merger(rup, R, returnmerit=True)
					# print merits
					R, merits = self.find_merger(rup, R, returnmerit=True)
					for i, r in enumerate(R):
						if merits[i] < self.min_merit: continue
						name = str(n)+'_'+str(r.label)
						leaf.add_child(name=name)
					newRup += R
				Rup = newRup

	def find_merger(self, r1, R2, returnmerit=False):
		print len(R2)
		R2 = self.overlap_bbox(r1, R2)
		print len(R2)
		R2 = sorted(R2, key=lambda r: get_merit(r1.coords, r.coords), reverse=True)
		if len(R2)> self.maxnodes:
			R2 = R2[:self.maxnodes]
		if returnmerit:
			merits = []
			for r2 in R2:
				merits.append(get_merit(r1.coords, r2.coords))
			return R2, merits
		return R2

	def overlap_bbox(self, r1, R2, keep=20):
		"""keep the top keep number of regions with overlapping bbox merit"""
		b1L = np.asarray(r1.bbox[:3])  #lower bounds
		b1R = np.asarray(r1.bbox[3:])  #upper bounds
		realR2 = []; BMerit = []
		for r2 in R2:
			b2L = np.asarray(r2.bbox[:3])
			b2R = np.asarray(r2.bbox[3:])

			if ( (b1L<b2R).all() and (b1R>b2L).all() ):
				BMerit.append(bbox_merit(b1L, b2L, b1R, b2R))
				realR2.append(r2)

		keepind = [ind for (merit,ind) in sorted(zip(BMerit,range(len(BMerit))),reverse=True)]
		if len(keepind)> keep:
			keepind = keepind[:keep]

		return [realR2[ind] for ind in keepind]

def layout(node):
    if node.is_leaf():
        # Add node name to laef nodes
        N = AttrFace("name", fsize=14, fgcolor="black")
        faces.add_face_to_node(N, node, 0)
    if "vol" in node.features:
        # Creates a sphere face whose size is proportional to node's
        # feature "weight"
        print node.vol
        C = CircleFace(radius=float(node.vol)/5000, color="RoyalBlue", style="sphere")
        # Let's make the sphere transparent
        C.opacity = 0.3
        # And place as a float face over the tree
        faces.add_face_to_node(C, node, 0, position="float")

def get_style():
	# Create an empty TreeStyle
    ts = TreeStyle()
    # Set our custom layout function
    ts.layout_fn = layout
    # Draw a tree
    ts.mode = "c"
    # We will add node names manually
    ts.show_leaf_name = True
    # Show branch data
    ts.show_branch_length = True
    ts.show_branch_support = True
    return ts
def parse_name(name):
	return [int(num) for num in name.split('_')]

def get_top_dict(T):
	"""return dictionary of the descendants of the top bubble"""
	cnt = 0
	dic = {}
	for node in T.Tree.traverse('preorder'):
		if cnt >T.TreeDepth: break
		cnt += 1; print node.name
		for child in node.get_children():
			lvl, lab = parse_name(child.name)
			dic[lvl] = dic.get(lvl, []) + [lab]
	return dic

def get_all_dict(T):
	"""return dictionary of all nodes keyed by level"""
	dic = {}
	for node in T.Tree.traverse():
		try:
			lvl, lab = parse_name(node.name)
		except:
			print 'Skipping', node.name
			continue
		dic[lvl] = dic.get(lvl, []) + [lab]
	return dic

def _merger_volume(node, bubblemask, file):
	print file
	children = node.get_children()
	if len(children) == 0: return 0
	mergermask = np.zeros_like(bubblemask)
	data = np.load(file)['labels']
	data = measure.label(data)
	for child in children:
		lvl, lab = parse_name(child.name)
		mergermask = mergermask | (data == lab)
	merge_vol= np.sum((bubblemask & mergermask))
	return merge_vol

def get_volume_history(T, files):
	"""volumes of top progenitors at each redshift, and how much of that volume was present
	at earlier redshift (i.e. due to merger)"""
	prog_dict = get_top_dict(T)
	prog_vol = []
	cnt = 0
	for node in T.Tree.traverse('preorder'):
		if cnt >=T.TreeDepth: break
		print '{}/{}'.format(cnt,T.TreeDepth), node.name
		try:
			lvl, lab = parse_name(node.name)
		except:
			print 'Skipping', cnt, node.name
			continue
		data = np.load(files[cnt])['labels']
		data = measure.label(data)
		bubblemask = data == lab
		if cnt != T.TreeDepth-1:
			merge_vol = _merger_volume(node, bubblemask, files[cnt+1])
		else:
			merge_vol = 0
		tot_vol = np.sum(bubblemask)
		prog_vol.append((tot_vol, merge_vol))
		cnt += 1;
	return prog_vol

if __name__=='__main__':
	#DIR = '/data2/lin0_logz10-15_zeta40/Boxes/'
	#DIR = '/home/yunfanz/Data/21cmFast/Boxes/'
	DIR = './NPZ/'
	files = find_files(DIR, pattern='watershed_z*')
	T = MergerTree(files, maxnodes=3)
	T.build()
	print get_volume_history(T, files)

	import IPython; IPython.embed()
	# ts = get_style()
	# print T.Tree
	# dic = get_all_dict(T)
	# print dic
	# save_obj(dic, 'allnodeTree')
	# #T.Tree.show(tree_style=ts)
	# import IPython; IPython.embed()

max_stars_repo_name: yunfanz/ReionBub

max_issues_repo_name: yunfanz/ReionBub

max_forks_repo_name: yunfanz/ReionBub

reason: import numpy,from scipy


text:
import sys
import os
import glob
import imageio
import re
import numpy as np
import time

def main():
    folder_path = input('Enter path of folder with images to convert:\n')
    out_path = f'output/timelapse_{time.time()}.mp4'
    image_paths = []
    fps = 60.0
    out = imageio.get_writer(out_path, fps=fps)

    status = 'Converting images to video...'
    print_percent(status)

    #save the starting path, and move to the path of the images
    #load all the images (and apply a 90 deg rotation b/c my camera is sideways) and write to video
    # original_path = os.getcwd()
    # os.chdir(folder_path)
    # for file in glob.glob('*.png'):
    #     image_paths.append(file)
    # os.chdir(original_path)
    image_paths = get_image_paths(folder_path)
    image_paths = natural_sort(image_paths)

    for i, image_path in enumerate(image_paths):
        try:
            image = imageio.imread(image_path)
        except ValueError as e:
            print(f'\nError loading image {image_path}')
            print(e)
            continue
        image = np.rot90(image)
        out.append_data(image)

        print_percent(status, i / len(image_paths) * 100)
    out.close()
    print_percent(status)
    print('Done     ')

def get_image_paths(folder_path):
    image_paths = []
    for file in glob.glob(os.path.join(folder_path, '*.png')):
        image_paths.append(file)
    return image_paths

def natural_sort(l):
    convert = lambda text: int(text) if text.isdigit() else text.lower()
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(l, key = alphanum_key)

def print_percent(string, percent=None):
    sys.stdout.write(f'\r{string}')
    if percent is not None:
        sys.stdout.write(f'[{percent:.2f}%]')
    sys.stdout.flush()
    pass

if __name__ == '__main__':
    main()

max_stars_repo_name: david-andrew/timelapse

max_issues_repo_name: david-andrew/timelapse

max_forks_repo_name: david-andrew/timelapse


text:
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import six
import toolz
import collections
from enum import Enum
from dateparser.date import DateDataParser

from . import schema
from .util import force_encode

class AccountType(Enum):
    CHECKING = 'Checking'
    SAVINGS = 'Savings'
    CREDIT_CARD = 'CreditCard'
    CASH = 'Cash'
    LINE_OF_CREDIT = 'LineofCredit'
    PAYPAL = 'Paypal'
    MERCHANT_ACCOUNT = 'MerchantAccount'
    INVESTMENT_ACCOUNT = 'InvestmentAccount'
    MORTGAGE = 'Mortgage'
    OTHER_ASSET = 'OtherAsset'
    OTHER_LIABILITY = 'OtherLiability'

class CategoryType(Enum):
    OUTFLOW = 'OUTFLOW'

class TransactionStatus(Enum):
    CLEARED = 'Cleared'
    RECONCILED = 'Reconciled'
    UNCLEARED = 'Uncleared'

class Model(object):
    _entity_type = None

    def __init__(self, ynab, entity):
        self._ynab = ynab
        self._entity = entity

    @classmethod
    @toolz.curry
    def _from_flat(cls, ynab, data):
        return cls(ynab, cls._entity_type(data, strict=False))

    @property
    def id(self):
        return self._entity.entityId

    @property
    def is_valid(self):
        return not self._entity.isTombstone

class Account(Model):
    _entity_type = schema.Account

    @force_encode
    def __repr__(self):
        return '<Account: {}>'.format(self.name)

    @property
    def name(self):
        return self._entity.accountName

    @property
    def type(self):
        return AccountType(self._entity.accountType)

    @property
    def on_budget(self):
        return self._entity.onBudget

    @property
    def last_reconciled_date(self):
        return self._entity.lastReconciledDate

    @property
    def last_reconciled_balance(self):
        return self._entity.lastReconciledBalance

    @property
    def last_reconciled_check_number(self):
        return self._entity.lastReconciledCheckNumber

    @property
    def hidden(self):
        return self._entity.hidden

    @property
    def payees(self):
        return self._ynab.payees.filter('target_account', self)

    @property
    def transactions(self):
        return self._ynab.transactions.filter('account', self)

    @property
    def inbound_transactions(self):
        return self._ynab.transactions.filter('target_account', self)

    @property
    def balance(self):
        return round(sum(self.transactions.amount), self._ynab.precision)

    @property
    def cleared_balance(self):
        return round(sum(self.transactions.filter('cleared').amount), self._ynab.precision)

    @property
    def reconciled_balance(self):
        return round(sum(self.transactions.filter('reconciled').amount), self._ynab.precision)

    @property
    def note(self):
        return self._entity.note

    @property
    def sortable_index(self):
        return self._entity.sortableIndex

class Payee(Model):
    _entity_type = schema.Payee

    @force_encode
    def __repr__(self):
        return '<Payee: {}>'.format(self.name)

    @property
    def name(self):
        return self._entity.name

    @property
    def target_account(self):
        return self._ynab.accounts.by_id(self._entity.targetAccountId)

    @property
    def enabled(self):
        return self._entity.enabled

    @property
    def transactions(self):
        return self._ynab.transactions.filter('payee', self)

class CategoryModel(Model):
    @property
    def name(self):
        return self._entity.name

    @property
    def type(self):
        return CategoryType(self._entity.type)

class Category(CategoryModel):
    _entity_type = schema.SubCategory

    @force_encode
    def __repr__(self):
        return '<Category: {}>'.format(self.full_name)

    @property
    def cached_balance(self):
        return self._entity.cachedBalance

    @property
    def master_category(self):
        return self._ynab.master_categories.by_id(self._entity.masterCategoryId)

    @property
    def has_unresolved_conflicts(self):
        return not self._entity.isResolvedConflict

    @property
    def note(self):
        return self._entity.note

    @property
    def full_name(self):
        return '{}/{}'.format(self.master_category.name, self.name)

    @property
    def transactions(self):
        return self._ynab.transactions.filter('category', self)

class MasterCategory(CategoryModel):
    _entity_type = schema.MasterCategory

    def __init__(self, ynab, entity):
        super(MasterCategory, self).__init__(ynab, entity)
        self._categories = Categories(
            Category(ynab, category) for category in self._entity.subCategories or [])

    @force_encode
    def __repr__(self):
        return '<MasterCategory: {}>'.format(self.name)

    @property
    def categories(self):
        return self._categories

    def __iter__(self):
        return iter(self._categories)

class TransactionModel(Model):
    @property
    def memo(self):
        return self._entity.memo

    @property
    def amount(self):
        return round(float(self._entity.amount or 0.), self._ynab.precision)

    @property
    def category(self):
        return self._ynab.categories.by_id(self._entity.categoryId)

    @property
    def target_account(self):
        return self._ynab.accounts.by_id(self._entity.targetAccountId)

    @property
    def transfer_transaction(self):
        return self._ynab.transactions.by_id(self._entity.transferTransactionId)

    @property
    def has_unresolved_conflicts(self):
        return not self._entity.isResolvedConflict

class SubTransaction(TransactionModel):
    _entity_type = schema.SubTransaction

    @force_encode
    def __repr__(self):
        return '<SubTransaction: {:.2f} ({})>'.format(
            self.amount, self.category.name if self.category else 'no category')

    @property
    def parent(self):
        return self._ynab.transactions.by_id(self._entity.parentTransactionId)

class Transaction(TransactionModel):
    _entity_type = schema.Transaction

    def __init__(self, ynab, entity):
        super(Transaction, self).__init__(ynab, entity)
        self._sub_transactions = SubTransactions(
            SubTransaction(ynab, t) for t in self._entity.subTransactions or [])

    @force_encode
    def __repr__(self):
        info = ''
        if self.category:
            info += ' ({})'.format(self.category.name)
        if self.payee:
            info += ' [{}]'.format(self.payee.name)
        return '<Transaction: [{:%d/%m/%y}]: {}: {:.2f}{}>'.format(
            self.date or 'no date', self.account.name if self.account else 'no account',
            self.amount, info)

    @property
    def date(self):
        return self._entity.date

    @property
    def status(self):
        return TransactionStatus(self._entity.cleared)

    @property
    def cleared(self):
        return self.status in (TransactionStatus.CLEARED, TransactionStatus.RECONCILED)

    @property
    def reconciled(self):
        return self.status == TransactionStatus.RECONCILED

    @property
    def accepted(self):
        return self._entity.accepted

    @property
    def account(self):
        return self._ynab.accounts.by_id(self._entity.accountId)

    @property
    def payee(self):
        return self._ynab.payees.by_id(self._entity.payeeId)

    @property
    def date_entered_from_schedule(self):
        return self._entity.dateEnteredFromSchedule

    @property
    def sub_transactions(self):
        return self._sub_transactions

class ModelCollection(collections.Sequence):
    _model_type = None
    _index_key = None
    _NO_VALUE = object()

    def __init__(self, elements):
        self._elements = list(e for e in elements if e.is_valid)

        # keep a reverse index for faster id indexing
        self._index = {element.id: element for element in self._elements}

    @classmethod
    @toolz.curry
    def _from_flat(cls, ynab, data):
        return cls(map(cls._model_type._from_flat(ynab), data))

    def __len__(self):
        return len(self._elements)

    def __getitem__(self, key):
        # try behave both like a list and like a dict with string keys
        if isinstance(key, six.string_types):
            # _index_key defines the attribute name that will be matched
            if self._index_key is not None:
                for element in self:
                    if getattr(element, self._index_key) == key:
                        return element
            raise KeyError(key)
        else:
            return self._elements[key]

    def __getattr__(self, key):
        # if the attribute is not found, propagate it to children
        return [getattr(element, key) for element in self]

    def __repr__(self):
        # list(self) due to py2/py3 unicode problems
        return repr(list(self))

    def __str__(self):
        # list(self) due to py2/py3 unicode problems
        return str(list(self))

    def by_id(self, id):
        """
        Retrieve an element by entity ID. Returns None if not found.

        Parameters
        ----------
        id : str
        """
        return self._index.get(id, None)

    def sort_by(self, field):
        """
        In-place sort by a specified field.

        Parameters
        ----------
        field : string
        """
        self._elements = sorted(self._elements, key=lambda element: getattr(element, field))

    def filter(self, field, value=_NO_VALUE):
        """
        Filters the collection by field value of child elements.

        Parameters
        ----------
        field : str
            Name of the attribute to be matched.
        value : object (optional)
            If specified, the values will be matched to this value by equality. Otherwise,
            the values will be converted to booleans and matched to True.
        Returns
        -------
        collection : ModelCollection
            The return value is always of the same type as the original object.
        """
        return type(self)(element for element in self
                          if (value is not self._NO_VALUE and getattr(element, field) == value) or
                          (value is self._NO_VALUE and getattr(element, field)))

class Accounts(ModelCollection):
    _model_type = Account
    _index_key = 'name'

class Payees(ModelCollection):
    _model_type = Payee
    _index_key = 'name'

class MasterCategories(ModelCollection):
    _model_type = MasterCategory
    _index_key = 'name'

class Categories(ModelCollection):
    _model_type = Category
    _index_key = 'full_name'

class Transactions(ModelCollection):
    _model_type = Transaction

    @property
    def amount(self):
        amount = [t.amount for t in self]
        try:
            # try to return a numpy array if possible
            import numpy as np
            return np.round(np.array(amount, dtype=np.float64), self._ynab.precision)
        except ImportError:
            # return a simple list otherwise
            return amount

    def _parse_date(self, string):
        parser = DateDataParser()
        date = parser.get_date_data(string)['date_obj']
        if date is None:
            raise RuntimeError('Unable to parse date: {!r}'.format(string))
        return date.date()

    def between(self, start=None, end=None):
        """
        Select all transactions between the specified dates.

        The dates may be specified as date objects, standard date strings ('2015-01-15') or
        human-readable strings ('two weeks ago').

        Parameters
        ----------
        start : date or str (optional)
        end : date or str (optional)

        Returns
        -------
        transactions : Transactions
        """
        transactions = list(self)
        if start is not None:
            transactions = [t for t in transactions if t.date >= self._parse_date(start)]
        if end is not None:
            transactions = [t for t in transactions if t.date <= self._parse_date(end)]
        return type(self)(transactions)

    def since(self, date):
        """
        Select all transactions since the specified date.

        The date may be specified as date object, standard date string ('2015-01-15') or
        a human-readable string ('two weeks ago').

        Parameters
        ----------
        start : date or str

        Returns
        -------
        transactions : Transactions
        """
        return self.between(start=date)

    def till(self, date):
        """
        Select all transactions before and including the specified date.

        The date may be specified as date object, standard date string ('2015-01-15') or
        a human-readable string ('two weeks ago').

        Parameters
        ----------
        start : date or str

        Returns
        -------
        transactions : Transactions
        """
        return self.between(end=date)

class SubTransactions(ModelCollection):
    _model_type = SubTransaction

class BudgetMetaData(Model):
    _entity_type = schema.BudgetMetaData

    _PRECISIONS = {
        'ar_BH': 3,
        'ar_EG': 3,
        'ar_JO': 3,
        'ar_KW': 3,
        'ar_TN': 3,
        'id_ID': 0,
        'is_IS': 0,
        'ja_JP': 0,
        'ko_KR': 0,
        'uz_Latn_UZ': 0,
    }

    @property
    def is_valid(self):
        # BudgetMetaData is not deleteable, so it is always valid.
        return True

    @property
    def currency_locale(self):
        return self._entity.currencyLocale

    @property
    def date_locale(self):
        return self._entity.dateLocale

    @property
    def budget_type(self):
        return self._entity.budgetType

    @property
    def is_strict(self):
        # Is this used?
        return self._entity.strictBudget

    @property
    def precision(self):
        return self._PRECISIONS.get(self.currency_locale, 2)

class FileMetaData(Model):
    _entity_type = schema.FileMetaData

    @property
    def current_knowledge(self):
        return self._entity.currentKnowledge

    @property
    def id(self):
        return None

    @property
    def is_valid(self):
        return True

max_stars_repo_name: joyinsky/pynab

max_issues_repo_name: joyinsky/pynab

max_forks_repo_name: joyinsky/pynab


text:
from typing import Type, TYPE_CHECKING, Tuple, Union, List, NewType  # , Any, NewType, TypeVar
from PIL import Image
import numpy as np
from typing_extensions import Literal

if TYPE_CHECKING:  # for mypy
    class ndarray(np.ndarray): ...
    imageType = NewType('imageType', np._ArrayLike[np.ndarray[np.ndarray[int]]])
    videoType = NewType('videoType', np._ArrayLike[imageType])
    contourType = NewType('contourType', np._ArrayLike[np.ndarray[np.ndarray[int]]])
    intArray = np.ndarray[int]
    floatArray = np.ndarray[float]
else:  # python doesnt allow ndarray subscriptable yet
    imageType = np.ndarray
    videoType = np.ndarray
    contourType = np.ndarray
    intArray = np.ndarray
    floatArray = np.ndarray

class PILImage(Image.Image): ...

colorHSV = Tuple[float, float, float]
colorRGB = Tuple[int, int, int]
colorType = Union[colorHSV, colorRGB]

RGBLit = Literal["RGB", "rgb"]
HSVLit = Literal["HSV", "hsv"]
colorSystemLit = Union[RGBLit, HSVLit]

pointType = Tuple[int, int]

max_stars_repo_name: Yobmod/dmlvis

max_issues_repo_name: Yobmod/dmlvis

max_forks_repo_name: Yobmod/dmlvis