text:
"""
To run:

$ python lstm_frag.py --data_path=path/to/train.list

"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import sys
sys.path.insert(0, "../src/")

from utils.vector_manager import VectorManager

import numpy as np
import tensorflow as tf

import subprocess
import inspect
import time

flags = tf.flags
logging = tf.logging

flags.DEFINE_string(
    "model", "small",
    "A type of model. Possible options are: small, medium, large.")

flags.DEFINE_string(
    "tasks", "all",
    "Tasks to be performed. Possible options are: all, train, test, valid")

flags.DEFINE_string(
    "word_to_id_path", "../models/eos/word2id_1000.pklz",
    "A type of model. Possible options are: small, medium, large.")

flags.DEFINE_string(
    "embeddings", "../models/eos/idWordVec_",
    "Embeddings path")

flags.DEFINE_string("data_path", None,
                    "Where the training/test data is stored.")
flags.DEFINE_string("save_path", None,
                    "Model output directory.")
flags.DEFINE_bool("use_fp16", False,
                  "Train using 16-bit floats instead of 32bit floats")

FLAGS = flags.FLAGS

def data_type():
    return tf.float16 if FLAGS.use_fp16 else tf.float32

def get_vocab_size():
    word_to_id = VectorManager.read_vector(FLAGS.word_to_id_path)
    size = len(word_to_id)
    print("Vocabulary size: %s" % size)
    return size

def generate_arrays_from_list(name, files, embeddings, num_steps=35, batch_size=20, embedding_size=200):

    debug = False
    while 1:
        for file_name in files:
            print("Generating from file %s for %s" % (file_name, name))
            raw_list = VectorManager.parse_into_list(open(file_name).read())

            n_words = len(raw_list)
            batch_len = n_words // batch_size
            data = np.reshape(raw_list[0:batch_size*batch_len], [batch_size, batch_len])

            for i in range(0, n_words - num_steps, 1):

                x = data[0:batch_size, i * num_steps:(i + 1) * num_steps]
                x = [[embeddings[int(elem)][2] for elem in l] for l in x]
                y = data[0:batch_size, i * num_steps + 1:(i + 1) * num_steps + 1]

                if len(x[0]) < num_steps or len(y[0]) < num_steps:
                    break
                if debug:
                    print("Batch size %s\nNum steps %s\nEmbedding size %s" % (batch_size, num_steps, embedding_size
                                                                              ))
                    print("Len(x): %s\n Len(x[0] %s\n Len(x[0][0] %s" % (len(x), len(x[0]), len(x[0][0])))
                    print("Len(y): %s\n Len(y[0] %s" % (len(y), len(y[0])))
                x = np.reshape(x, newshape=(batch_size, num_steps, embedding_size))

                y = np.reshape(y, newshape=(batch_size, num_steps))

                yield x, y

class WPModel(object):
    """Word Prediction model."""

    def __init__(self, is_training, config):

        self.config = config
        batch_size = config.batch_size
        num_steps = config.num_steps
        size = config.hidden_size
        vocab_size = config.vocab_size
        embedding_size = config.embedding_size

        def lstm_cell():
            # With the latest TensorFlow source code (as of Mar 27, 2017),
            # the BasicLSTMCell will need a reuse parameter which is unfortunately not
            # defined in TensorFlow 1.0. To maintain backwards compatibility, we add
            # an argument check here:
            # if 'reuse' in inspect.getargspec(
            #     tf.contrib.rnn.BasicLSTMCell.__init__).args:
            #   return tf.contrib.rnn.BasicLSTMCell(
            #       size, forget_bias=0.0, state_is_tuple=True,
            #       reuse=tf.get_variable_scope().reuse)
            # else:
            return tf.contrib.rnn.BasicLSTMCell(
                size, forget_bias=0.0, state_is_tuple=True)

        attn_cell = lstm_cell
        if is_training and config.keep_prob < 1:
            def attn_cell():
                return tf.contrib.rnn.DropoutWrapper(
                    lstm_cell(), output_keep_prob=config.keep_prob)

        cell = tf.contrib.rnn.MultiRNNCell(
            [attn_cell() for _ in range(config.num_layers)], state_is_tuple=True)

        self._initial_state = cell.zero_state(batch_size, data_type())

        with tf.device("/cpu:0"):
            self.inputs = tf.placeholder(dtype=data_type(), shape=(batch_size, num_steps, embedding_size))
            self.targets = tf.placeholder(dtype=tf.int32, shape=(batch_size, num_steps))

        if is_training and config.keep_prob < 1:
            # Dropout allows to use the net for train and testing
            # See: https://stackoverflow.com/questions/34597316/why-input-is-scaled-in-tf-nn-dropout-in-tensorflow
            # and: http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf
            inputs = tf.nn.dropout(self.inputs, config.keep_prob)
        else:
            inputs = self.inputs

        inputs = tf.unstack(inputs, num=num_steps, axis=1)

        outputs, state = tf.contrib.rnn.static_rnn(
            cell, inputs, initial_state=self._initial_state)

        output = tf.reshape(tf.stack(axis=1, values=outputs), [-1, size])
        softmax_w = tf.get_variable(
            "softmax_w", [size, vocab_size], dtype=data_type())
        softmax_b = tf.get_variable("softmax_b", [vocab_size], dtype=data_type())
        logits = tf.matmul(output, softmax_w) + softmax_b
        loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(
            [logits],
            [tf.reshape(self.targets, [-1])],
            [tf.ones([batch_size * num_steps], dtype=data_type())])
        self._cost = cost = tf.reduce_sum(loss) / batch_size
        self._final_state = state

        if not is_training:
            return

        self._lr = tf.Variable(0.0, trainable=False)
        tvars = tf.trainable_variables()
        grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
                                          config.max_grad_norm)
        optimizer = tf.train.GradientDescentOptimizer(self._lr)
        self._train_op = optimizer.apply_gradients(
            zip(grads, tvars),
            global_step=tf.contrib.framework.get_or_create_global_step())

        self._new_lr = tf.placeholder(
            tf.float32, shape=[], name="new_learning_rate")
        self._lr_update = tf.assign(self._lr, self._new_lr)

    def assign_lr(self, session, lr_value):
        session.run(self._lr_update, feed_dict={self._new_lr: lr_value})

    @property
    def input(self):
        return self._input

    @property
    def initial_state(self):
        return self._initial_state

    @property
    def cost(self):
        return self._cost

    @property
    def final_state(self):
        return self._final_state

    @property
    def lr(self):
        return self._lr

    @property
    def train_op(self):
        return self._train_op

class SmallConfig(object):
    """Small config."""
    init_scale = 0.1
    learning_rate = 1.0
    max_grad_norm = 5
    num_layers = 1
    num_steps = 20
    hidden_size = 200
    max_epoch = 2
    max_max_epoch = 13
    keep_prob = 1.0
    lr_decay = 0.5
    batch_size = 20
    vocab_size = 126930
    embedding_size = 200
    epoch_size = 1

class MediumConfig(object):
    """Medium config."""
    init_scale = 0.05
    learning_rate = 1.0
    max_grad_norm = 5
    num_layers = 1
    num_steps = 35
    hidden_size = 512
    max_epoch = 6
    max_max_epoch = 39
    keep_prob = 0.5
    lr_decay = 0.8
    batch_size = 20
    vocab_size = 126930
    embedding_size = 200
    epoch_size = 1

class LargeConfig(object):
    """Large config."""
    init_scale = 0.04
    learning_rate = 1.0
    max_grad_norm = 10
    num_layers = 1
    num_steps = 35
    hidden_size = 1024
    max_epoch = 14
    max_max_epoch = 55
    keep_prob = 0.35
    lr_decay = 1 / 1.15
    batch_size = 20
    vocab_size = 126930
    embedding_size = 1000
    epoch_size = 1

class TestConfig(object):
    """Tiny config, for testing."""
    init_scale = 0.1
    learning_rate = 1.0
    max_grad_norm = 1
    num_layers = 1
    num_steps = 2
    hidden_size = 2
    max_epoch = 1
    max_max_epoch = 1
    keep_prob = 1.0
    lr_decay = 0.5
    batch_size = 10
    vocab_size = 126930
    embedding_size = 200
    epoch_size = 1

def run_epoch(session, generator, model, eval_op=None, verbose=False):
    """Runs the model on the given data."""
    start_time = time.time()
    costs = 0.0
    iters = 0
    config = model.config
    state = session.run(model.initial_state)

    fetches = {
        "cost": model.cost,
        "final_state": model.final_state,
    }
    if eval_op is not None:
        fetches["eval_op"] = eval_op

    print("Epoch size starting training %s" % config.epoch_size)
    for step in range(config.epoch_size):
        x, y = next(generator)
        feed_dict = {}
        for i, (c, h) in enumerate(model.initial_state):
            feed_dict[c] = state[i].c
            feed_dict[h] = state[i].h
        feed_dict[model.inputs] = x
        feed_dict[model.targets] = y

        vals = session.run(fetches, feed_dict)
        cost = vals["cost"]
        state = vals["final_state"]

        costs += cost
        iters += config.num_steps

        # if verbose and step % 100 == 0:
        print("%.3f perplexity: %.3f speed: %.0f wps" %
              (step * 1.0 / config.epoch_size, np.exp(costs / iters),
               iters * config.batch_size / (time.time() - start_time)))
        sys.stdout.flush()

    return np.exp(costs / iters)

def get_config():
    if FLAGS.model == "small":
        return SmallConfig()
    elif FLAGS.model == "medium":
        return MediumConfig()
    elif FLAGS.model == "large":
        return LargeConfig()
    elif FLAGS.model == "test":
        return TestConfig()
    else:
        raise ValueError("Invalid model: %s", FLAGS.model)

def get_epoch_size(files, config):
    total = 0
    for file in files:
        file_words = subprocess.check_output(['wc', '-w', file])
        number = file_words.split()[0]
        words = int(number)
        total += words - (words % (config.batch_size * config.num_steps))
    print("Total words: %s, Batch size: %s, Num steps: %s" % (total, config.batch_size, config.num_steps))
    sys.stdout.flush()
    epoch_size = ((total // config.batch_size) - 1) // config.num_steps

    return epoch_size

def main(_):
    if not FLAGS.data_path:
        raise ValueError("Must set --data_path to wiki data directory list")

    vocab_size = 126930

    config = get_config()
    config.vocab_size = vocab_size

    valid_config = get_config()
    config.vocab_size = vocab_size

    eval_config = get_config()
    eval_config.batch_size = 1
    eval_config.num_steps = 1
    eval_config.vocab_size = vocab_size

    print("Embeddings path: {}".format(FLAGS.embeddings))
    embeddings = VectorManager.read_vector(FLAGS.embeddings)
    files = open(FLAGS.data_path).read().split()

    training_list = files[0:int(0.8 * len(files))]
    validation_list = files[int(0.8 * len(files)):int(0.9 * len(files))]
    testing_list = files[int(0.9 * len(files)):len(files)]

    config.epoch_size = get_epoch_size(training_list, config)
    valid_config.epoch_size = get_epoch_size(validation_list, valid_config)
    eval_config.epoch_size = get_epoch_size(testing_list, eval_config)

    gen_train = generate_arrays_from_list("Train", training_list, embeddings, batch_size=config.batch_size,
                                          embedding_size=config.embedding_size, num_steps=config.num_steps)
    gen_valid = generate_arrays_from_list("Validation", validation_list, embeddings, batch_size=valid_config.batch_size,
                                          embedding_size=valid_config.embedding_size, num_steps=valid_config.num_steps)
    gen_test = generate_arrays_from_list("Test", testing_list, embeddings, batch_size=eval_config.batch_size,
                                         embedding_size=eval_config.embedding_size, num_steps=eval_config.num_steps)

    print("Epoch sizes\n * Training: %s\n * Validation: %s\n * Testing: %s" %
          (config.epoch_size, valid_config.epoch_size, eval_config.epoch_size))
    sys.stdout.flush()
    with tf.Graph().as_default():
        # Args: [minval, maxval]
        initializer = tf.random_uniform_initializer(-config.init_scale,
                                                    config.init_scale)

        with tf.name_scope("Train"):
            with tf.variable_scope("Model", reuse=None, initializer=initializer):
                m = WPModel(is_training=True, config=config)
            tf.summary.scalar("Training Loss", m.cost)
            tf.summary.scalar("Learning Rate", m.lr)

        with tf.name_scope("Valid"):
            with tf.variable_scope("Model", reuse=True, initializer=initializer):
                mvalid = WPModel(is_training=False, config=valid_config)
            tf.summary.scalar("Validation Loss", mvalid.cost)

        with tf.name_scope("Test"):
            with tf.variable_scope("Model", reuse=True, initializer=initializer):
                mtest = WPModel(is_training=False, config=eval_config)

        sv = tf.train.Supervisor(logdir=FLAGS.save_path)
        with sv.managed_session() as session:
            for i in range(config.max_max_epoch):
                lr_decay = config.lr_decay ** max(i + 1 - config.max_epoch, 0.0)
                m.assign_lr(session, config.learning_rate * lr_decay)

                print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
                train_perplexity = run_epoch(session, generator=gen_train, model=m, eval_op=m.train_op,
                                             verbose=True)
                print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
                valid_perplexity = run_epoch(session, generator=gen_valid, model=mvalid)
                print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))

            test_perplexity = run_epoch(session, generator=gen_test, model=mtest)
            print("Test Perplexity: %.3f" % test_perplexity)

            if FLAGS.save_path:
                print("Saving model to %s." % FLAGS.save_path)
                sv.saver.save(session, FLAGS.save_path, global_step=sv.global_step)

if __name__ == "__main__":
    tf.app.run()

max_stars_repo_name: kafkasl/contextualLSTM

max_issues_repo_name: kafkasl/contextualLSTM

max_forks_repo_name: kafkasl/contextualLSTM


text:
#!/usr/bin/env python3
#
#  smarties
#  Copyright (c) 2018 CSE-Lab, ETH Zurich, Switzerland. All rights reserved.
#  Distributed under the terms of the MIT license.
#
#  Created by Guido Novati (novatig@ethz.ch).
#
# SIMPLE PYTHON SCRIPT TO PLOT .raw GRADIENT FILES. Input is the path to the
# file, which is $(network_naem)_grads.raw (eg. net_grads.raw)

import sys
import numpy as np
import matplotlib.pyplot as plt
FILE=    sys.argv[1]

#np.savetxt(sys.stdout, np.fromfile(sys.argv[1], dtype='i4').reshape(2,10).transpose())
DATA = np.fromfile(FILE, dtype=np.float32)
NOUTS = int(DATA[0])
DATA = DATA[1:]
DATA = DATA.reshape(DATA.size // (2*NOUTS), 2*NOUTS)

EPS = 1e-5
for ind in range(0, NOUTS):
  plt.subplot(121)
  #plt.plot(DATA[:,ind]/DATA[:,NOUTS+ind], label=str(ind))
  #plt.semilogy(abs(DATA[:,ind]),label=str(ind))
  plt.semilogy(abs(DATA[:,ind])/(DATA[:,NOUTS+ind]+1e-16)+EPS, label=str(ind))
  plt.subplot(122)
  plt.semilogy(abs(DATA[:,NOUTS+ind])+EPS,'--',  label=str(ind))

plt.legend(loc=5, bbox_to_anchor=(1.2, 0.5))
#plt.savefig('prova.png', dpi=100)
plt.show()

ext: py

reason: import numpy


text:
import logging
import numpy
import pandas

from . import ourworldindata
from .. import preprocessing

_log = logging.getLogger(__file__)

def forecast_NL(df: pandas.DataFrame):
    """ Applies testcount interpolation/extrapolation.

    Currently this assumes the OWID data, which only has an "all" region.
    In the future, this should be replaced with more fine graned data loading!
    """
    # only weekly total_tests are available -> interpolate the totals, then diff to get daily new_tests
    df.at[("all", "2020-01-01"), 'total_tests'] = 0
    df["new_tests"] = df.total_tests.interpolate("linear").diff()
    df["new_tests"][df.new_tests == 0] = numpy.nan
    # forecast with existing data
    df['predicted_new_tests'], results = preprocessing.predict_testcounts_all_regions(df, 'NL')
    return df, results

from .. import data
data.set_country_support(
    country_alpha2="NL",
    compute_zone=data.Zone.Europe,
    region_name={
        "all": "Netherlands",
    },
    region_population={
        "all": 17_290_688,
    },
    fn_load=ourworldindata.create_loader_function("NL"),
    fn_process=forecast_NL,
)

max_stars_repo_name: michaelosthege/covid-model

max_issues_repo_name: michaelosthege/covid-model

max_forks_repo_name: michaelosthege/covid-model


text:
# Copyright 2020 The TensorFlow Probability Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Tests for tensorflow_probability.spinoffs.oryx.experimental.nn.function."""

from absl.testing import absltest
import jax
from jax import random
from jax import tree_util
import jax.numpy as jnp
import numpy as np

from oryx.core import state
from oryx.experimental import nn
from oryx.internal import test_util

class AddOne(nn.Layer):

  @classmethod
  def initialize(cls, rng, in_spec):
    return nn.LayerParams()

  @classmethod
  def spec(cls, in_spec):
    return in_spec

  def _call(self, x):
    return x + 1

class ScalarMul(nn.Layer):

  @classmethod
  def initialize(cls, rng, in_spec, weight):
    weight = jnp.array(weight)
    return nn.LayerParams(params=weight)

  @classmethod
  def spec(cls, in_spec, weight):
    return in_spec

  def _call(self, x):
    return self.params * x

class Counter(nn.Layer):

  @classmethod
  def initialize(cls, rng, in_spec, initial_state):
    return nn.LayerParams(state=initial_state)

  @classmethod
  def spec(cls, in_spec, initial_state):
    return in_spec

  def _call(self, x):
    return self.state + x

  def _update(self, x):
    return self.replace(state=self.state + 1)

class IsTraining(nn.Layer):

  @classmethod
  def initialize(cls, rng, in_spec):
    return nn.LayerParams()

  @classmethod
  def spec(cls, in_spec):
    return in_spec

  def _call(self, x, training=True):
    if training:
      return jnp.ones_like(x)
    else:
      return jnp.zeros_like(x)

class Sampler(nn.Layer):

  @classmethod
  def initialize(cls, rng, in_spec):
    return nn.LayerParams()

  @classmethod
  def spec(cls, in_spec):
    return in_spec

  def _call(self, x, rng=None):
    assert rng is not None, 'Layer needs valid RNG'
    return random.normal(rng, x.shape)

class FunctionTest(test_util.TestCase):

  def setUp(self):
    super().setUp()
    self._seed = random.PRNGKey(0)

  def test_identity(self):
    def identity(x):
      return x
    in_spec = state.Shape(50)
    out_spec = state.spec(identity)(in_spec)
    self.assertEqual(out_spec, in_spec)

    net = state.init(identity)(self._seed, jnp.ones(50))
    np.testing.assert_array_equal(net(jnp.arange(5)), jnp.arange(5))

  def test_add(self):
    def add(x):
      return jnp.add(x, x)
    in_spec = state.Shape(50)
    out_spec = state.spec(add)(in_spec)
    self.assertEqual(out_spec, in_spec)

    in_spec = state.Shape((100, 20))
    out_spec = state.spec(add)(in_spec)
    self.assertEqual(out_spec, in_spec)

    in_spec = state.Shape(2)
    net = state.init(add)(self._seed, jnp.ones(2))
    np.testing.assert_array_equal(net(jnp.arange(5)), 2 * jnp.arange(5))

  def test_multiple_input_single_output(self):
    def add(x, y):
      return jnp.add(x, y)

    in_spec = (state.Shape((100, 20)), state.Shape(50))
    with self.assertRaises(ValueError):
      out_spec = state.spec(add)(*in_spec)

    in_spec = (state.Shape(50), state.Shape(50))
    out_spec = state.spec(add)(*in_spec)
    self.assertEqual(out_spec, in_spec[0])

    net = state.init(add)(self._seed, jnp.ones(50), jnp.ones(50))
    np.testing.assert_array_equal(net(jnp.arange(5), jnp.arange(5)),
                                  2 * jnp.arange(5))

  def test_single_input_multiple_output(self):
    def dup(x):
      return x, x

    in_spec = state.Shape(50)
    out_spec = state.spec(dup)(in_spec)
    self.assertEqual(out_spec, (in_spec, in_spec))

    net = state.init(dup)(self._seed, jnp.ones(50))
    for x1, x2 in zip(net(jnp.arange(5)),
                      (jnp.arange(5), jnp.arange(5))):
      np.testing.assert_array_equal(x1, x2)

  def test_multiple_input_multiple_output(self):
    def swap(x, y):
      return y, x

    in_spec = (state.Shape(50), state.Shape(20))
    out_spec = state.spec(swap)(*in_spec)
    self.assertEqual(out_spec, (in_spec[1], in_spec[0]))

    net = state.init(swap)(self._seed, jnp.ones(50), jnp.ones(20))
    for x1, x2 in zip(net(jnp.zeros(50), jnp.ones(20)),
                      (jnp.ones(20), jnp.zeros(50))):
      np.testing.assert_array_equal(x1, x2)

  def test_nested_add(self):
    def add(x):
      return (lambda x: jnp.add(x, x))(x)
    in_spec = state.Shape(50)
    out_spec = state.spec(add)(in_spec)
    self.assertEqual(out_spec, in_spec)

    in_spec = state.Shape((100, 20))
    out_spec = state.spec(add)(in_spec)
    self.assertEqual(out_spec, in_spec)

    in_spec = state.Shape(2)
    net = state.init(add)(self._seed, jnp.ones(2))
    np.testing.assert_array_equal(net(jnp.arange(5)), 2 * jnp.arange(5))

  def test_dense_function(self):
    def dense_no_rng(x):
      return nn.Dense(20)(x, name='dense')

    with self.assertRaises(ValueError):
      out_spec = state.spec(dense_no_rng)(state.Shape(50))

    with self.assertRaises(ValueError):
      net = state.init(dense_no_rng)(self._seed, state.Shape(2))

    def dense(x, init_key=None):
      return nn.Dense(20)(x, init_key=init_key, name='dense')

    out_spec = state.spec(dense)(state.Shape(2))
    self.assertEqual(out_spec, state.Shape(20))

    net = state.init(dense)(self._seed, jnp.ones(2))
    self.assertTupleEqual(net(jnp.ones(2)).shape, (20,))
    np.testing.assert_allclose(net(jnp.ones(2)),
                               dense(jnp.ones(2), init_key=self._seed),
                               rtol=1e-5)

  def test_dense_combinator(self):
    def dense(x, init_key=None):
      return (nn.Dense(50) >> nn.Dense(20))(x, init_key=init_key, name='dense')
    in_spec = state.Shape(50)
    out_spec = state.spec(dense)(in_spec)
    self.assertEqual(out_spec, state.Shape(20, dtype=in_spec.dtype))

    net = state.init(dense)(self._seed, jnp.ones(2))
    self.assertTupleEqual(net(jnp.ones(2)).shape, out_spec.shape)
    np.testing.assert_allclose(
        dense(jnp.ones(2), init_key=self._seed),
        net(jnp.ones(2)), rtol=1e-5)

  def test_add_one_combinator(self):
    def add_two(x, init_key=None):
      return (AddOne() >> AddOne())(x, name='add_one', init_key=init_key)

    in_spec = state.Shape(20)
    out_spec = state.spec(add_two)(in_spec)
    self.assertEqual(out_spec, in_spec)

    in_spec = state.Shape((5, 50))
    out_spec = state.spec(add_two)(in_spec)
    self.assertEqual(out_spec, in_spec)

    net = state.init(add_two)(self._seed, jnp.ones(2))
    np.testing.assert_allclose(
        net(jnp.ones(2)),
        3 * jnp.ones(2))
    np.testing.assert_array_equal(
        net(jnp.ones(2)),
        add_two(jnp.ones(2), init_key=self._seed))

  def test_add_one_imperative(self):
    def add_two(x, init_key=None):
      k1, k2 = random.split(init_key)
      x = AddOne()(x, name='add_one_1', init_key=k1)
      x = AddOne()(x, name='add_one_2', init_key=k2)
      return x
    in_spec = state.Shape(20)
    out_spec = state.spec(add_two)(in_spec)
    self.assertEqual(out_spec, in_spec)

    in_spec = state.Shape((5, 50))
    out_spec = state.spec(add_two)(in_spec)
    self.assertEqual(out_spec, in_spec)

    net = state.init(add_two)(self._seed, jnp.ones(2))
    np.testing.assert_array_equal(
        net(jnp.ones(2)),
        3 * jnp.ones(2))
    np.testing.assert_array_equal(
        net(jnp.ones(2)),
        add_two(jnp.ones(2), init_key=self._seed))

  def test_dense_imperative(self):
    def dense(x, init_key=None):
      key, subkey = random.split(init_key)
      x = nn.Dense(50)(x, init_key=key, name='dense1')
      x = nn.Dense(20)(x, init_key=subkey, name='dense2')
      return x
    in_spec = state.Shape(50)
    out_spec = state.spec(dense)(in_spec)
    self.assertEqual(out_spec, state.Shape(20, dtype=in_spec.dtype))

    net = state.init(dense)(self._seed, jnp.ones(2))
    self.assertTupleEqual(net(jnp.ones(2)).shape, out_spec.shape)
    np.testing.assert_allclose(
        dense(jnp.ones(2), init_key=self._seed),
        net(jnp.ones(2)), rtol=1e-5)

  def test_function_in_combinator(self):
    def add_one(x):
      return x + 1

    template = AddOne() >> add_one >> AddOne()

    net = state.init(template)(self._seed, jnp.ones(2))
    np.testing.assert_array_equal(net(jnp.zeros(2)), 3 * jnp.ones(2))

  def test_function_in_combinator_in_function(self):
    def add_one(x):
      return x + 1
    def template(x, init_key=None):
      return (AddOne() >> add_one >> AddOne())(x, init_key=init_key)

    net = state.init(template)(self._seed, jnp.ones(2))
    np.testing.assert_array_equal(net(jnp.zeros(2), init_key=self._seed),
                                  3 * jnp.ones(2))

  def test_grad_of_function_with_literal(self):
    def template(x, init_key=None):
      # 1.0 behaves like a literal when tracing
      return ScalarMul(1.0)(x, init_key=init_key, name='scalar_mul')
    net = state.init(template)(self._seed, jnp.ones(5))
    def loss(net, x):
      return net(x).sum()
    g = jax.grad(loss)(net, jnp.ones(5))
    def add(x, y):
      return x + y
    net = tree_util.tree_multimap(add, net, g)
    # w_new = w_old + 5
    np.testing.assert_array_equal(net(jnp.ones(5)), 6 * jnp.ones(5))

  def test_grad_of_function_constant(self):
    def template(x):
      return x + jnp.ones_like(x)
    net = state.init(template)(self._seed, jnp.ones(5))
    def loss(net, x):
      return net(x).sum()
    g = jax.grad(loss)(net, jnp.ones(5))
    def add(x, y):
      return x + y
    net = tree_util.tree_multimap(add, net, g)
    # w_new = w_old + 5
    np.testing.assert_array_equal(net(jnp.ones(5)), 2 * jnp.ones(5))

  def test_grad_of_function(self):
    def template(x, init_key=None):
      # jnp.ones(1) does not behave like a literal when tracing
      return ScalarMul(jnp.ones(1))(x, init_key=init_key, name='scalar_mul')
    net = state.init(template)(self._seed, jnp.ones(5))
    def loss(net, x):
      return net(x).sum()
    g = jax.grad(loss)(net, jnp.ones(5))
    def add(x, y):
      return x + y
    net = tree_util.tree_multimap(add, net, g)
    # w_new = w_old + 5
    np.testing.assert_array_equal(net(jnp.ones(5)), 6 * jnp.ones(5))

  def test_grad_of_stateful_function(self):
    def template(x, init_key=None):
      x = ScalarMul(jnp.ones(1))(x, init_key=init_key, name='scalar_mul')
      x = Counter(jnp.zeros(1))(x, init_key=init_key, name='counter')
      return x
    net = state.init(template)(self._seed, jnp.ones(5))
    def loss(net, x):
      return net(x).sum()
    g = jax.grad(loss)(net, jnp.ones(5))
    def add(x, y):
      return x + y
    net = tree_util.tree_multimap(add, net, g)
    # w_new = w_old + 5
    np.testing.assert_array_equal(net(jnp.ones(5)), 6 * jnp.ones(5))
    net = net.update(jnp.ones(5))
    np.testing.assert_array_equal(net(jnp.ones(5)), 7 * jnp.ones(5))

    g = jax.grad(loss)(net, jnp.ones(5))
    net = tree_util.tree_multimap(add, net, g)
    # w_new = w_old + 5
    np.testing.assert_array_equal(net(jnp.ones(5)), 12 * jnp.ones(5))

  def test_shared_layer(self):
    def template(x, init_key=None):
      layer = state.init(ScalarMul(2 * jnp.ones(1)), name='scalar_mul')(
          init_key, x)
      return layer(layer(x))
    net = state.init(template)(self._seed, jnp.ones(5))
    np.testing.assert_array_equal(net(jnp.ones(5)), 4 * jnp.ones(5))

  def test_grad_of_shared_layer(self):
    def template(x, init_key=None):
      layer = state.init(ScalarMul(2 * jnp.ones(1)), name='scalar_mul')(
          init_key, x)
      return layer(layer(x)).sum()
    net = state.init(template)(self._seed, jnp.ones(()))

    def loss(net, x):
      return net(x)
    g = jax.grad(loss)(net, jnp.ones(()))
    def add(x, y):
      return x + y
    net = tree_util.tree_multimap(add, net, g)
    np.testing.assert_array_equal(net(jnp.ones(())), 36.)

  def test_update(self):
    def template(x, init_key=None):
      return Counter(jnp.zeros(()))(x, init_key=init_key, name='counter')
    net = state.init(template)(self._seed, jnp.ones(()))
    self.assertEqual(net(jnp.ones(())), 1.)

    net2 = state.update(net, jnp.ones(()))
    self.assertEqual(net2(jnp.ones(())), 2.)

    net2 = net.update(jnp.ones(()))
    self.assertEqual(net2(jnp.ones(())), 2.)

  def test_update_in_combinator(self):
    def template(x, init_key=None):
      def increment(x, init_key=None):
        return Counter(jnp.zeros(()))(x, init_key=init_key, name='counter')
      return nn.Serial([increment, increment])(x, init_key=init_key,
                                               name='increment')
    net = state.init(template)(self._seed, jnp.ones(()))
    self.assertEqual(net(jnp.ones(())), 1.)
    net = state.update(net, jnp.ones(()))
    self.assertEqual(net(jnp.ones(())), 3.)

  def test_kwargs_training(self):
    def template(x, training=False, init_key=None):
      return IsTraining()(x, name='training', training=training,
                          init_key=init_key)
    net = state.init(template)(self._seed, jnp.ones(()))
    self.assertEqual(net(jnp.ones(()), training=True), 1.)
    self.assertEqual(net(jnp.ones(()), training=False), 0.)

    def template1(x, training=True, init_key=None):
      return IsTraining()(x, training=training, name='training',
                          init_key=init_key)
    net = state.init(template1)(self._seed, jnp.ones(()))
    self.assertEqual(net(jnp.ones(()), training=True), 1.)
    self.assertEqual(net(jnp.ones(()), training=False), 0.)

    def template2(x, init_key=None):
      return IsTraining()(x, name='training', init_key=init_key) + 1
    net = state.init(template2)(self._seed, jnp.ones(()))
    self.assertEqual(net(jnp.ones(()), training=True), 2.)
    self.assertEqual(net(jnp.ones(()), training=False), 1.)

  def test_kwargs_rng(self):
    def template(x, init_key=None):
      return Sampler()(x, name='sampler', init_key=init_key)
    with self.assertRaises(AssertionError):
      net = state.init(template)(self._seed, jnp.ones(()))
    def template1(x, rng, init_key=None):
      return Sampler()(x, rng=rng, init_key=init_key)
    net = state.init(template1)(self._seed, jnp.ones(()),
                                jnp.ones(2, dtype=jnp.uint32))
    x1 = net(jnp.ones(()), random.PRNGKey(0))
    x2 = net(jnp.ones(()), random.PRNGKey(1))
    self.assertNotEqual(x1, x2)

  def test_kwargs_training_rng(self):
    def template(x, rng, training=True, init_key=None):
      k1, k2 = random.split(init_key)
      x = Sampler()(x, rng=rng, name='sampler', init_key=k1)
      return (IsTraining()(x, training=training, name='training', init_key=k2)
              + x)

    net = state.init(template)(
        self._seed, jnp.ones(()), random.PRNGKey(0))
    x0n = net(jnp.ones(()), random.PRNGKey(0), training=False)
    x0t = net(jnp.ones(()), random.PRNGKey(0), training=True)
    x1n = net(jnp.ones(()), random.PRNGKey(1), training=False)
    x1t = net(jnp.ones(()), random.PRNGKey(1), training=True)
    # Different seeds generate different results
    # Same seed generates offset based on training flag
    self.assertNotEqual(x0n, x1n)
    self.assertNotEqual(x0t, x1t)
    np.testing.assert_allclose(x0n, x0t - 1, rtol=1e-6)
    np.testing.assert_allclose(x1n, x1t - 1, rtol=1e-6)

  def test_call_tuple(self):
    def template(x, init_key=None):
      return state.call((Counter(0.), AddOne()), x, init_key=init_key,
                        name='counter_add_one')
    layer = state.init(template)(self._seed, jnp.ones(()))
    self.assertTupleEqual(layer(jnp.zeros(())), (0, 1))

    layer = layer.update(jnp.zeros(()))
    self.assertTupleEqual(layer(jnp.zeros(())), (1, 1))

  def test_call_list(self):
    def template(x, init_key=None):
      return state.call([Counter(0.), AddOne()], x, init_key=init_key,
                        name='counter_add_one')
    layer = state.init(template)(self._seed, jnp.ones(()))
    self.assertEqual(layer(jnp.zeros(())), 1)

    layer = layer.update(jnp.zeros(()))
    self.assertEqual(layer(jnp.zeros(())), 2)

  def test_duplicate_names(self):
    def template(x, init_key=None):
      k1, k2 = random.split(init_key)
      layer1 = state.init(nn.Dense(20), name='dense')(k1, x)
      layer2 = state.init(nn.Dense(20), name='dense')(k2, x)
      return layer1(x) + layer2(x)
    with self.assertRaises(ValueError):
      state.init(template)(self._seed, jnp.ones(5))

if __name__ == '__main__':
  absltest.main()

reason: import numpy,import jax,from jax


text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow import keras
import cv2
import random
import numpy as np

# In[2]:

(x_train,y_train),(x_test,y_test) = keras.datasets.mnist.load_data()

# In[3]:

plt.imshow(x_train[1],cmap="gray")

# In[4]:

x_train = x_train/255.0
x_test = x_test/255.0

# In[5]:

x_train[0].shape

# In[6]:

def get_model(level):
    encoder_input = keras.Input(shape=(28, 28, 1), name='img')
    x = keras.layers.Flatten()(encoder_input)

    encoder_output = keras.layers.Dense(level*level, activation="relu")(x)

    encoder = keras.Model(encoder_input, encoder_output, name='encoder')

    decoder_input = keras.layers.Dense(level*level, activation="relu")(encoder_output)
    x = keras.layers.Dense(784, activation="relu")(decoder_input)
    decoder_output = keras.layers.Reshape((28, 28, 1))(x)

    opt = tf.keras.optimizers.Adam(lr=0.001, decay=1e-6)

    autoencoder = keras.Model(encoder_input, decoder_output, name='autoencoder')
    autoencoder.summary()

    autoencoder.compile(opt, loss='mse')
    return (autoencoder,encoder)

# In[7]:

autoencoder,encoder = get_model(8)

# In[8]:

with tf.device('/GPU:0'):
    autoencoder.fit(x_train,x_train,epochs=3,batch_size=32,validation_split=0.1)

# In[9]:

example = autoencoder.predict([x_test[0].reshape(-1,28,28,1)])

# In[10]:

example.shape

# In[11]:

plt.imshow(example.reshape(28,28),cmap="gray")
plt.axis("off")

# In[12]:

plt.imshow(x_test[0],cmap="gray")
plt.axis("off")

# In[13]:

decoded_img = encoder.predict([x_test[0].reshape(-1,28,28,1)])[0]
plt.imshow(decoded_img.reshape((8,8)),cmap="gray")
plt.axis("off")

# # Denoising

# In[14]:

def add_noise(img,random_chance=5):
    noisy = []
    for row in img:
        new_row = []
        for pix in row:
            if(random.choice(range(100))<= random_chance):
                new_val = random.uniform(0,1)
                new_row.append(new_val)
            else:
                new_row.append(pix)

        noisy.append(new_row)
    return np.array(noisy)

# In[15]:

noisy=[]
for i in range(10):
    noised = add_noise(x_test[i],10)
    noisy.append(noised)

# In[16]:

fig = plt.figure(figsize=(10,10))
columns=3
rows=3
t=0;
for i in range(1,columns*rows+1):
    fig.add_subplot(rows,columns,i)
    plt.imshow(noisy[t],cmap="gray")
    plt.axis("off")
    t+=1
plt.show()

# In[17]:

denoised = []
for i in range(10):
    denoised.append(autoencoder.predict([noisy[i].reshape(-1,28,28,1)])[0])

# In[18]:

fig = plt.figure(figsize=(10,10))
columns=3
rows=3
t=0;
for i in range(1,columns*rows+1):
    fig.add_subplot(rows,columns,i)
    plt.imshow(denoised[t].reshape((28,28)),cmap="gray")
    plt.axis("off")
    t+=1
plt.show()

# In[ ]:

max_stars_repo_name: vedantgoswami/Autoencoder

max_issues_repo_name: vedantgoswami/Autoencoder

max_forks_repo_name: vedantgoswami/Autoencoder


text:
import sys

import scipy.io
import numpy as np
import glob
import time
import dislib as ds
from dislib.classification import CascadeSVM
from dislib.data.array import Array
from dislib.model_selection import GridSearchCV
from sklearn.datasets import make_classification
import pandas as pd
from pycompss.api.api import compss_wait_on, compss_barrier
from scipy import signal
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
from sklearn.tree._tree import Tree as SklearnTree
from sklearn.svm import SVC as SklearnSVC
from sklearn.tree import DecisionTreeClassifier as SklearnDTClassifier
from sklearn.tree import DecisionTreeRegressor as SklearnDTRegressor
import pickle
from scipy.sparse import csr_matrix
from scipy import sparse as sp
from collections import Counter
import random
import json
import os

def zero_pad(data, length):
    extended = np.zeros(length)
    signal_length = np.min([length, data.shape[0]])
    extended[:signal_length] = data[:signal_length]
    return extended

def spectrogram(data, fs=300, nperseg=64, noverlap=32):
    f, t, Sxx = signal.spectrogram(data, fs=fs, nperseg=nperseg, noverlap=noverlap)
    Sxx = np.transpose(Sxx, [0, 2, 1])
    Sxx = np.abs(Sxx)
    mask = Sxx > 0
    Sxx[mask] = np.log(Sxx[mask])
    return f, t, Sxx

def load_n_preprocess(dataDir):
    max_length = 61
    freq = 300

    ## Loading labels and time serie signals (A and N)
    import csv
    csvfile = list(csv.reader(open(dataDir + 'REFERENCE.csv')))

    files = [dataDir + i[0] + ".mat" for i in csvfile]
    dataset = np.zeros((len(files), 18810))
    count = 0
    for f in files:
        mat_val = zero_pad(scipy.io.loadmat(f)['val'][0], length=max_length * freq)
        sx = spectrogram(np.expand_dims(mat_val, axis=0))[2]  # generate spectrogram
        sx_norm = (sx - np.mean(sx)) / np.std(sx)  # normalize the spectrogram
        dataset[count,] = sx_norm.flatten()
        count += 1

    labels = np.zeros((dataset.shape[0], 1))
    classes = ['A', 'N', 'O', '~']
    for row in range(len(csvfile)):
        labels[row, 0] = 0 if classes.index(csvfile[row][1]) == 0 else 1 if classes.index(
            csvfile[row][1]) == 1 else 2 if classes.index(csvfile[row][1]) == 2 else 3

    return (dataset, labels)

if __name__ == "__main__":
    args = sys.argv[1:]
    start_time = time.time()
    dataset_to_use = args[0]
    block_size_x = (int(args[1]), int(args[2]))
    block_size_y = int(args[3])
    seed = 1234
    parameters = {'kernel': ('rbf', 'linear'),
                  'c': (0.75, 1.0, 2.0),
                  'tol': (1e-3, 1e-4, 1e-5)}
    csvm = CascadeSVM()
    # csvm = CascadeSVC(fold_size=500)

    X_train, y_train = load_n_preprocess(dataset_to_use)
    print([X_train.shape, y_train.shape])
    print(Counter(y_train.flatten()))
    idx = random.sample(list(np.where(y_train == 1.0)[0]),
                        Counter(y_train.flatten())[1.0] - Counter(y_train.flatten())[0.0])
    y_train = np.delete(y_train, idx, axis=0)
    X_train = np.delete(X_train, idx, axis=0)
    print(Counter(y_train.flatten()))
    load_time = time.time()

    x = ds.array(X_train, block_size=block_size_x)
    y = ds.array(y_train, block_size=(block_size_y, 1))

    searcher = GridSearchCV(csvm, parameters, cv=5)
    np.random.seed(0)
    searcher.fit(x, y)
    print(searcher.cv_results_['params'])
    print(searcher.cv_results_['mean_test_score'])
    pd_df = pd.DataFrame.from_dict(searcher.cv_results_)
    print(pd_df[['params', 'mean_test_score']])
    with pd.option_context('display.max_rows', None,
                           'display.max_columns', None):
        print(pd_df)

    print('best_estimator')
    print(searcher.best_estimator_)
    print(searcher.best_score_)
    print(searcher.best_params_)
    print(searcher.best_index_)
    print(searcher.scorer_)
    print(searcher.n_splits_)

max_stars_repo_name: lezzidan/aisprint

max_issues_repo_name: lezzidan/aisprint

max_forks_repo_name: lezzidan/aisprint

reason: import numpy,import scipy,from scipy


text:
import argparse
import collections

import numpy as np
import pandas as pd
from sklearn import metrics
from sklearn.preprocessing import label_binarize
from sklearn.metrics import f1_score

def vlsp_2018_PRF(y_true, y_pred, domain):
    """
    Calculate "Micro P R F" of aspect detection task of VLSP-2018.
    """
    s_all=0
    g_all=0
    s_g_all=0

    total_available_aspects = 12 if domain == 'restaurant' else 34

    for i in range(len(y_pred)//total_available_aspects):
        s=set()
        g=set()
        for j in range(total_available_aspects):
            if y_pred[i*total_available_aspects+j]!=3:
                s.add(j)
            if y_true[i*total_available_aspects+j]!=3:
                g.add(j)
        if len(g)==0:continue
        s_g=s.intersection(g)
        s_all+=len(s)
        g_all+=len(g)
        s_g_all+=len(s_g)

    p=s_g_all/s_all
    r=s_g_all/g_all
    f=2*p*r/(p+r)

    return p,r,f

def get_y_true(task_name, true_data_dir):
    """
    Read file to obtain y_true.
    All of five tasks of VLSP 2018 use the test set of task-BERT-pair-NLI-M to get true labels.
    """

    df = pd.read_csv(true_data_dir,sep='\t',header=None).values
    y_true=[]
    for i in range(len(df)):
        label = df[i][1]
        assert label in ['tích_cực', 'trung_lập', 'tiêu_cực', 'không_có'], "error!"
        if label == 'tích_cực':
            n = 0
        elif label == 'trung_lập':
            n = 1
        elif label == 'tiêu_cực':
            n = 2
        elif label == 'không_có':
            n = 3
        y_true.append(n)

    return y_true

def get_y_pred(task_name, pred_data_dir):
    """
    Read file to obtain y_pred and scores.
    """
    pred=[]
    score=[]
    if task_name in ["vlsp_2018_NLI_M", "vlsp_2018_QA_M"]:
        with open(pred_data_dir,"r",encoding="utf-8") as f:
            s=f.readline().strip().split()
            while s:
                pred.append(int(s[0]))
                score.append([float(s[1]), float(s[2]), float(s[3]), float(s[4])])
                s = f.readline().strip().split()
    elif task_name in ["vlsp_2018_NLI_B", "vlsp_2018_QA_B"]:
        count = 0
        tmp = []
        with open(pred_data_dir, "r", encoding="utf-8") as f:
            s = f.readline().strip().split()
            while s:
                tmp.append([float(s[2])])
                count += 1
                if count % 4 == 0:
                    tmp_sum = np.sum(tmp)
                    t = []
                    for i in range(4):
                        t.append(tmp[i] / tmp_sum)
                    score.append(t)
                    if t[0] >= t[1] and t[0] >= t[2] and t[0]>=t[3] and t[0]>=t[4]:
                        pred.append(0)
                    elif t[1] >= t[0] and t[1] >= t[2] and t[1]>=t[3] and t[1]>=t[4]:
                        pred.append(1)
                    elif t[2] >= t[0] and t[2] >= t[1] and t[2]>=t[3] and t[2]>=t[4]:
                        pred.append(2)
                    else:
                        pred.append(3)
                    tmp = []
                s = f.readline().strip().split()
    else:
        print('Has not implement for single choice method')

    return pred, score

def vlsp_2018_Acc(y_true, y_pred, score, classes=3):
    """
    Calculate "Acc" of sentiment classification task of VLSP 2018.
    """
    assert classes in [2, 3], "classes must be 2 or 3."

    if classes == 3:
        total=0
        total_right=0
        for i in range(len(y_true)):
            if y_true[i]==3:continue
            total+=1
            tmp=y_pred[i]
            if tmp>=3:
                if score[i][0]>=score[i][1] and score[i][0]>=score[i][2]:
                    tmp=0
                elif score[i][1]>=score[i][0] and score[i][1]>=score[i][2]:
                    tmp=1
                else:
                    tmp=2
            if y_true[i]==tmp:
                total_right+=1
        sentiment_Acc = total_right/total
    else:
        total=0
        total_right=0
        for i in range(len(y_true)):
            if y_true[i]>=3 or y_true[i]==1:continue
            total+=1
            tmp=y_pred[i]
            if tmp>=3 or tmp==1:
                if score[i][0]>=score[i][2]:
                    tmp=0
                else:
                    tmp=2
            if y_true[i]==tmp:
                total_right+=1
        sentiment_Acc = total_right/total

    return sentiment_Acc

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--task_name",
                        default=None,
                        type=str,
                        required=True,
                        choices=["vlsp_2018_single", \
                                "vlsp_2018_NLI_M", "vlsp_2018_QA_M", "vlsp_2018_NLI_B", "vlsp_2018_QA_B"],
                        help="The name of the task to evalution.")

    parser.add_argument("--true_data_dir",
                        default=None,
                        type=str,
                        required=True,
                        help="The pred data dir.")

    parser.add_argument("--pred_data_dir",
                        default=None,
                        type=str,
                        required=True,
                        help="The pred data dir.")

    parser.add_argument("--domain",
                        default=None,
                        type=str,
                        required=True,
                        choices=['restaurant', 'hotel'],
                        help="domain")
    args = parser.parse_args()

    result = collections.OrderedDict()

    y_true = get_y_true(args.task_name, args.true_data_dir)
    y_pred, score = get_y_pred(args.task_name, args.pred_data_dir)
    aspect_P, aspect_R, aspect_F = vlsp_2018_PRF(y_true, y_pred, args.domain)
    sentiment_Acc_3_classes = vlsp_2018_Acc(y_true, y_pred, score, 3)
    f1_micro = f1_score(y_true, y_pred, average='micro')

    result = {
            'aspect_P': aspect_P,
            'aspect_R': aspect_R,
            'aspect_F': aspect_F,
            'sentiment_accuracy': sentiment_Acc_3_classes,
            'f1': f1_micro
    }

    for key in result.keys():
        print(key, "=",str(result[key]))

if __name__ == "__main__":
    main()


text:
# Copyright (c) 2016 The UUV Simulator Authors.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import rospy
import numpy as np
from copy import deepcopy
from uuv_waypoints import Waypoint, WaypointSet
from ..trajectory_point import TrajectoryPoint
from tf.transformations import quaternion_multiply, quaternion_about_axis, \
    quaternion_from_euler, rotation_matrix, quaternion_from_matrix
from visualization_msgs.msg import MarkerArray

class PathGenerator(object):
    """
    Abstract class to be inherited by custom path generator to interpolate
    waypoints
    """
    LABEL = ''

    def __init__(self, full_dof=False):
        # Waypoint set
        self._waypoints = None

        # True if the path is generated for all degrees of freedom, otherwise
        # the path will be generated for (x, y, z, yaw) only
        self._is_full_dof = full_dof

        # The parametric variable to use as input for the interpolator
        self._s = list()
        self._segment_to_wp_map = list()
        self._cur_s = 0
        self._s_step = 0.0001

        self._start_time = None
        self._duration = None

        self._termination_by_time = True

        self._final_pos_tolerance = 0.1

        self._init_rot = quaternion_about_axis(0.0, [0, 0, 1])
        self._last_rot = quaternion_about_axis(0.0, [0, 0, 1])

        self._markers_msg = MarkerArray()
        self._marker_id = 0

    @staticmethod
    def get_generator(name, *args, **kwargs):
        for gen in PathGenerator.__subclasses__():
            if name == gen.LABEL:
                return gen(*args, **kwargs)
        rospy.ROSException('Invalid path generator method')

    @staticmethod
    def get_all_generators():
        generators = list()
        for gen in PathGenerator.__subclasses__():
            generators.append(gen())
        return generators

    @property
    def waypoints(self):
        return self._waypoints

    @property
    def max_time(self):
        return self._duration + self._start_time

    @property
    def duration(self):
        return self._duration

    @duration.setter
    def duration(self, t):
        assert t > 0, 'Duration must be a positive value'
        self._duration = t

    @property
    def start_time(self):
        return self._start_time

    @start_time.setter
    def start_time(self, time):
        assert time >= 0, 'Invalid negative time'
        self._start_time = time

    @property
    def closest_waypoint(self):
        """Return the closest waypoint to the current position on the path."""
        return self._waypoints.get_waypoint(self.closest_waypoint_idx)

    @property
    def closest_waypoint_idx(self):
        """
        Return the index of the closest waypoint to the current position on the
        path.
        """

        if self._cur_s == 0:
            return 0
        if self._cur_s == 1:
            return len(self._s) - 1
        v = np.array(self._s - self._cur_s)
        idx = np.argmin(v)
        return idx

    @property
    def s_step(self):
        return self._s_step

    @s_step.setter
    def s_step(self, step):
        assert 0 < step < 1
        self._s_step = step

    @property
    def termination_by_time(self):
        return self._termination_by_time

    def reset(self):
        self._s = list()
        self._segment_to_wp_map = list()
        self._cur_s = 0
        self._s_step = 0.0001

        self._start_time = None
        self._duration = None

    def get_segment_idx(self, s):
        if len(self._s) == 0:
            return 0
        # Ensure the parameter s is 0 <= s <= 1
        s = max(0, s)
        s = min(s, 1)

        if s == 1:
            idx = self._s.size - 1
        else:
            idx = (self._s - s >= 0).nonzero()[0][0]
        return idx

    def get_remaining_waypoints_idx(self, s):
        idx = self.get_segment_idx(s)
        try:
            wps = self._segment_to_wp_map[idx::]
            return np.unique(wps)
        except:
            print 'Invalid index'
            return None

    def is_full_dof(self):
        return self._is_full_dof

    def set_full_dof(self, flag):
        self._is_full_dof = flag

    def get_label(self):
        return self.LABEL

    def init_interpolator(self):
        raise NotImplementedError()

    def get_samples(self, max_time, step=0.005):
        raise NotImplementedError()

    def get_visual_markers(self):
        return self._markers_msg

    def add_waypoint(self, waypoint, add_to_beginning=False):
        """Add waypoint to the existing waypoint set. If no waypoint set has
        been initialized, create new waypoint set structure and add the given
        waypoint."""
        if self._waypoints is None:
            self._waypoints = WaypointSet()
        self._waypoints.add_waypoint(waypoint, add_to_beginning)
        return self.init_interpolator()

    def init_waypoints(self, waypoints=None, init_rot=np.array([0, 0, 0, 1])):
        if waypoints is not None:
            self._waypoints = deepcopy(waypoints)

        if self._waypoints is None:
            print 'Waypoint list has not been initialized'
            return False

        self._init_rot = init_rot
        return True

    def interpolate(self, tag, s):
        return self._interp_fcns[tag](s)

    def is_finished(self, t):
        if self._termination_by_time:
            return t > self.max_time
        else:
            return True

    def has_started(self, t):
        if self._termination_by_time:
            return t - self.start_time > 0
        else:
            return True

    def generate_pnt(self, s):
        raise NotImplementedError()

    def generate_pos(self, s):
        raise NotImplementedError()

    def generate_quat(self, s):
        raise NotImplementedError()

    def set_parameters(self, params):
        raise NotImplementedError()

    def _compute_rot_quat(self, dx, dy, dz):
        heading = np.arctan2(dy, dx)
        rotq = quaternion_about_axis(heading, [0, 0, 1])

        if self._is_full_dof:
            rote = quaternion_about_axis(
                -1 * np.arctan2(dz, np.sqrt(dx**2 + dy**2)),
                [0, 1, 0])
            rotq = quaternion_multiply(rotq, rote)
        return rotq


text:
# Importing Libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
import statsmodels.formula.api as sm

# Importing dataset
dataset = pd.read_csv('50_Startups.csv')

# Printing first five rows
print(dataset.head())

# Separating the Independent variables and Dependent variables
X = dataset.iloc[:,:-1].values
y = dataset.iloc[:,4].values

print("X contains the following values: \n",X)
print("y contains the following values: \n",y)

# Checking the shape of X and y
print(X.shape,y.shape)

# Now we will convert the categories into numbers from 1st and last column of the Dataset
labelEncoder_x = LabelEncoder()
# labelEncoder_x is a object of LabelEncoder class which converts text to numbers
X[:, 3] = labelEncoder_x.fit_transform(X[:,3])
# Now we will use one hot encoder to expand the columns corresponding to the number of categories
oneHotEncoder = OneHotEncoder(categorical_features= [3])
X = oneHotEncoder.fit_transform(X).toarray()
print(X)

## Now we will avoid the Dummy Variable Trap
X = X[:,1:]

print(X)

## Now we need to split the dataset into training set and testing set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

regressor = LinearRegression()
regressor.fit(X_train,y_train)

# Predicting the test set results
y_pred = regressor.predict(X_test)
print(y_pred)

## ****************Backward Elimination Technique*******************
## Building an optimal model using backward elimination i.e we will remove all those columns which are not giving us any accurate
## Predictions
X = np.append(arr=np.ones((50,1)).astype(int),values=X,axis=1)
print(X)

## Now we will remove all the independent variables whose value is greater than the significance value
X_opt = X[:,[0,1,2,3,4,5]]
regressor_OLS = sm.OLS(endog=y,exog=X_opt).fit()
regressor_OLS.summary()

X_opt = X[:,[0,1,3,4,5]]
regressor_OLS = sm.OLS(endog=y,exog=X_opt).fit()
regressor_OLS.summary()

X_opt = X[:,[0,3,4,5]]
regressor_OLS = sm.OLS(endog=y,exog=X_opt).fit()
regressor_OLS.summary()

X_opt = X[:,[0,3,5]]
regressor_OLS = sm.OLS(endog=y,exog=X_opt).fit()
regressor_OLS.summary()

X_opt = X[:,[0,3]]
regressor_OLS = sm.OLS(endog=y,exog=X_opt).fit()
regressor_OLS.summary()

max_stars_repo_path: Day 3 - Multiple Linear Regression/Multiple_Linear_Regression_%26_Backward_Elimination.py

max_stars_repo_name: Satyam-Bhalla/Machine-Learning-Practice

max_issues_repo_name: Satyam-Bhalla/Machine-Learning-Practice

max_forks_repo_name: Satyam-Bhalla/Machine-Learning-Practice

reason: import numpy,import statsmodels


text:
# %%
import argparse
import datetime
from pathlib import Path

import astroplan as ap
import astropy.units as u
import numpy as np
import pandas as pd
import pytz
from astroplan import plots as aplt
from astropy.coordinates import EarthLocation, get_body, SkyCoord
from astropy.time import Time
from matplotlib import pyplot as plt
from matplotlib import rcParams
import warnings
warnings.filterwarnings('ignore', append=True)

try:
    from rich import print
    from rich.traceback import install
    install()
except ImportError:
    print("You may want to install `rich` by $ pip install rich")
    pass

plt.style.use('default')
rcParams.update({
    'font.family': 'Times', 'font.size': 12, 'mathtext.fontset': 'stix',
    'axes.formatter.use_mathtext': True, 'axes.formatter.limits': (-4, 4),
    'axes.grid': True, 'grid.color': 'gray', 'grid.linewidth': 0.5,
    'xtick.top': True, 'ytick.right': True,
    'xtick.direction': 'inout', 'ytick.direction': 'inout',
    'xtick.minor.size': 4.0, 'ytick.minor.size': 4.0,  # default 2.0
    'xtick.major.size': 8.0, 'ytick.major.size': 8.0,  # default 3.5
    'xtick.minor.visible': True, 'ytick.minor.visible': True
})

PLANETS = {
    'Mercury': "gray",
    'Venus': "brown",
    'Mars': "red",
    'Jupiter': "orange",
    'Saturn': "olive",
    'Uranus': "lightseagreen",
    'Neptune': "blue"
}

COLS2DROP = ["Other ID", "Distance (kly)", "Constellation", "RA", "DEC"]

PLOTKW = {
    'gal': dict(ls="-", alpha=0.7, lw=2),
    'Neb': dict(ls="--", alpha=1, lw=2),
    "cl": dict(ls="-.", alpha=1, lw=2),
}
PLOTKW_OTHERS = dict(ls=":", alpha=1, lw=2)

INFOSTR = """
Abbreviations for "Type" column
* cl: Cluster
  - cl-O: open cluster
  - cl-G: globular cluster
* gal: Galaxy
  - gal-SB: Barred spiral galaxy
  - gal-S: Spiral galaxy
  - gal-S0: Lenticular galaxy
  - gal-E: Elliptical galaxy
  - gal-dE: Dwarf elliptical galaxy
  - gal-dSph: Dwarf Spheroidal Galaxy
  - gal-cD: Supergiant Elliptical Galaxy
  - gal-Irr: Irregular galaxy
  - gal-IrrB: Barred irregular galaxy
  - gal-Inter: Interacting galaxy
* Neb: Nebula
  - Neb-P: Planetary nebula
  - Neb-HII: H II Region Nebula
  - Neb-SN: Supernova remnant
* MW: Milky Way

Original of Messier data from https://en.wikipedia.org/wiki/Messier_object
Original of Caldwell data from https://en.wikipedia.org/wiki/Caldwell_catalogue
"""

p = argparse.ArgumentParser(description='')

# TODO: let user choose which to use first y-axis (airmass, altitude)
p.add_argument("YYYY", nargs='?', type=int,
               help="Year; If not given, current time is used")
p.add_argument("MM", nargs='?', default=1, type=int,
               help="Month   (1 <= month <= 12)")
p.add_argument("DD", nargs='?', default=1, type=int,
               help="Date    (1 <= day <= number of days in the given month and year)")
p.add_argument("HH", nargs='?', default=0, type=int,
               help="Hour    (0 <= hour < 24)")
p.add_argument("mm", nargs='?', default=0, type=int,
               help="Minutes (0 <= minute < 60)")
p.add_argument("ss", nargs='?', default=0, type=int,
               help="Seconds (0 <= second < 60)")

p.add_argument("-U", "--UTC", action="store_true",
               help="Time is in UTC instead of local time")
p.add_argument("-M", "--Messier", action="store_false",
               help="Do NOT use Messier objects")
p.add_argument("-C", "--Caldwell", action="store_false",
               help="Do NOT use Caldwell objects")
p.add_argument("-X", "--airmass", action="store_true",
               help="Whether to use the primary y-axis as airmass (`X`)")
p.add_argument("-A", "--always-visible", action="store_true",
               help="Discard objects that are not *always* visible during the timespan?")
p.add_argument("-N", "--nickname", action="store_true",
               help="Use only those that have common nick name.")
p.add_argument("-E", "--type-exclude", action="store_true",
               help="Whether the -T (--types) is for exclusion")

p.add_argument("-c", "--currentlocation", action="store_true",
               help=("Use current location and timezone automatically "
                     + "(using http://ip-api.com/json/)"))
p.add_argument("-v", "--verbose", action="store_true",
               help="Print miscellaneous information")

p.add_argument("-d", "--duration", default=2., type=float,
               help="Duration of the observing run (+- this number) [hour]")
p.add_argument("-l", "--location", nargs=2, default=[127, 37.5],
               help="(longitude, latitude) [˚]; Used in astropy.coordinates.EarthLocation")
p.add_argument("-z", "--timezone", default="Asia/Seoul", type=str,
               help="Time zone")

p.add_argument("-a", "--min-alt", default=30., type=float,
               help="Minimum altitude of the object to be drawn [deg]")
p.add_argument("-p", "--min-alt-pl", default=25., type=float,
               help="Minimum altitude of the planetary objects to be drawn [deg]")
p.add_argument("-T", "--types", default=None, nargs='+',
               help="Target types to be drawn or excluded (exclusion by -E)")
p.add_argument("-t", "--targets", nargs='+', default=None,
               help="Target names to be drawn")
p.add_argument("-o", "--output", default=None, help="HTML name to save DataFrame")

args = p.parse_args()

def check_observable(min_alt, observer, targets, times, always):
    consts = [ap.AltitudeConstraint(min_alt*u.deg)]
    # , ap.AtNightConstraint(max_solar_altitude=0*u.deg)
    mask_fun = ap.is_always_observable if always else ap.is_observable
    mask = mask_fun(
        constraints=consts,
        observer=observer,
        targets=targets.tolist(),
        times=times
    )
    # coo_up = []
    # for _m, _t in zip(mask, targets):
    #     if _m:
    #         coo_up.append(_t)
    return mask

def get_geoloc(use_current_location, verbose):
    ''' Find geological location information from ip-api.com (lon, lat, timezone).
    '''
    if use_current_location:
        # TODO: Save these as cache files
        from requests import get
        ip = get('https://api.ipify.org').content.decode('utf8')
        response = get("http://ip-api.com/json/" + ip).json()
        # {'status': 'success', 'country': 'South Korea', 'countryCode': 'KR',
        # 'region': '11', 'regionName': 'Seoul', 'city': 'Gwanak-gu', 'zip': '08841',
        # 'lat': 37.4625, 'lon': 126.9438, 'timezone': 'Asia/Seoul', 'isp': 'SNU',
        # 'org': '', 'as': 'AS9488 Seoul National University', 'query': '147.46.135.73'}
        lon = float(response['lon'])*u.deg
        lat = float(response['lat'])*u.deg
        tz = pytz.timezone(response['timezone'])
        if verbose:
            print(response)
    else:
        lon = float(args.location[0])*u.deg
        lat = float(args.location[1])*u.deg
        tz = pytz.timezone(args.timezone)
    return lon, lat, tz

def get_time(YYYY, MM, DD, HH, mm, ss, in_utc, tz):
    ''' Parse the time infromation
    '''
    if YYYY is None:
        _obstime = datetime.datetime.utcnow()
        _localnow = datetime.datetime.now()
        args.YYYY = _localnow.year
        args.MM = _localnow.month
        args.DD = _localnow.day
        args.HH = _localnow.hour
        args.mm = _localnow.minute
    else:
        YYYY = int(YYYY)
        MM = int(MM)
        DD = int(DD)
        HH = int(HH)
        mm = int(mm)
        ss = int(ss)
        if in_utc:
            _obstime = datetime.datetime(YYYY, MM, DD, HH, mm, ss)
        else:
            _obstime = datetime.datetime(YYYY, MM, DD, HH, mm, ss).astimezone(tz)
    return _obstime

def add2kw(kw, cat, coo, mask, cmap):
    count = np.sum(mask)
    kw["df"] = cat[mask]
    kw["coo"] = coo[mask]
    kw["colors"] = cmap(np.linspace(0, 1, count))

def parseID(catid):
    if catid.startswith("M"):
        catname = "Messier"
    elif catid.startswith("C"):
        catname = "Caldwell"
    else:
        catname = None
    return catname

def mk_wikilink(catid):
    catname = parseID(catid)
    url = f"https://en.wikipedia.org/wiki/{catname}_{catid[1:]}"
    return f'
{catid:s}
'

def alt_color(alt):
    altval = float(alt)
    if altval < 0:
        color = "red"
    elif altval < args.min_alt:
        color = "orange"
    else:
        color = "limegreen"

    return f'
{alt:.1f}˚
'

if __name__ == "__main__":
    if args.verbose:
        print(args)

    TOP = Path(__file__).parent
    OUTPUT = Path(args.output) if args.output else Path("output.html")
    FIGDIR = (TOP/"figs")#.relative_to(OUTPUT.parent)
    FIGDIR.mkdir(exist_ok=True, parents=True)

    # == Get location and time information =============================================== #
    lon, lat, tz = get_geoloc(args.currentlocation, args.verbose)
    _obstime = get_time(args.YYYY, args.MM, args.DD,
                        args.HH, args.mm, args.ss,
                        args.UTC, tz)

    print(f"Date & Time : {_obstime} ({tz})")
    print(f"lon , lat  : {lon.value:.2f}˚, {lat.value:.2f}˚")

    OBSTIME = Time(_obstime)  # in UTC
    dt = args.duration*u.hour
    OBSTIME_RANGE = Time([OBSTIME - dt, OBSTIME, OBSTIME + dt])
    OBSTIMES = OBSTIME + np.linspace(-dt, dt, max(6, int(args.duration*12)))
    # around once per 5 minutes

    # NOTE: Default elevation of the observatory is set to 500m. Only small
    #   offset will be added, and that is insignificant for the purpose (maybe
    #   about the size of line width...).
    loc = EarthLocation.from_geodetic(lon, lat, 500*u.m)
    obs = ap.Observer(location=loc, timezone=tz)

    # == Prepare catalog ================================================================= #
    cat = pd.read_csv(TOP/"amastro_catalog_radec.csv", delimiter=',', comment="#")
    if args.targets is not None:
        cat = cat[cat["ID"].isin(args.targets)]
        if args.verbose:
            print(f"Choosing only thses: {args.targets}")
    else:
        if not args.Messier:
            if args.verbose:
                print("Messier objects are ignored.")
            cat = cat[~cat["ID"].str.startswith("M")]
        if not args.Caldwell:
            if args.verbose:
                print("Caldwell objects are ignored.")
            cat = cat[~cat["ID"].str.startswith("C")]
        if args.types is not None:
            if args.type_exclude:
                cat = cat[~cat["Type"].isin(args.types)]
                if args.verbose:
                    print(f"Following types will be ignored: {args.types}")
            else:
                cat = cat[cat["Type"].isin(args.types)]
                if args.verbose:
                    print(f"Only take the following types: {args.types}")

        if args.nickname:
            if args.verbose:
                print("Only those with common nicknames are selected.")
            cat = cat[cat["Name"].notna()]

    if args.verbose:
        print(f"{len(cat)} objects are selected by the user.")

    cat.sort_values(by="DEC", ascending=False, ignore_index=True, inplace=True)

    # plotting order will anyway be based on Type.
    coo = np.array(
        [ap.FixedTarget(SkyCoord(ra=_a*u.deg, dec=_d*u.deg), name=f"{_id} ({_t})")
         for _a, _d, _id, _t in zip(cat["RA"], cat["DEC"], cat["ID"], cat["Type"])]
    )

    # == Find coordinates of planets ===================================================== #
    coo_pl = []
    for planet in PLANETS.keys():
        _coo = get_body(planet, time=OBSTIME, location=loc)
        _coo.name = planet
        coo_pl.append(_coo)
    # RA/DEC only at the middle of the time.
    coo_pl = np.array(coo_pl)

    # style_ini = dict(style_kwargs=dict(marker=".", color="r"))
    # style_mid = dict(style_kwargs=dict(marker="x", color="r"))
    # style_end = dict(style_kwargs=dict(marker="o", color="r"))
    # OBSTIMES_5 = OBSTIME + np.linspace(-1, 1, 5)*u.hour
    # aplt.plot_sky(target=cat["coord"][0], observer=obs, time=OBSTIMES_5[:-1], **style_ini)
    # aplt.plot_sky(target=cat["coord"][0], observer=obs, time=OBSTIMES_5[2], **style_mid)
    # aplt.plot_sky(target=cat["coord"][0], observer=obs, time=OBSTIMES_5[-1], **style_end)

    # def find_coo_up(coords, generous=False):
    #     times = OBSTIMES if generous else OBSTIME
    #     horizon = 25*u.deg if generous else 30*u.deg
    #     coo_up = [_coo for _coo in coords
    #               if np.any(observer.target_is_up(times, _coo, horizon=horizon))]
    #     return coo_up

    # == Set plotting style ============================================================== #
    observable_kw = dict(observer=obs, times=OBSTIMES, always=args.always_visible)
    upmask = check_observable(args.min_alt, targets=coo, **observable_kw)
    coo_up = coo[upmask]
    cat_up = cat[upmask]

    fullmask = np.ones(len(cat_up))
    for typ, kwdict in PLOTKW.items():
        typmask = cat_up["Type"].str.startswith(typ)
        add2kw(kwdict, cat_up, coo_up, typmask, plt.cm.viridis)
        fullmask -= typmask
    add2kw(PLOTKW_OTHERS, cat_up, coo_up, fullmask.astype(bool), plt.cm.viridis)
    PLOTKW["others"] = PLOTKW_OTHERS

    cat_up["lowres"] = cat_up["ID"].apply(
        lambda x: f''
    )
    cat_up["DSS"] = cat_up["ID"].apply(
        lambda x: f''
    )
    cat_up["DSS-zscale"] = cat_up["ID"].apply(
        lambda x: f''
    )
    if args.verbose:
        print(f"{len(cat_up)} objects are visible by the user's criteria.")

    # == Plot ============================================================================ #
    fig, axs = plt.subplots(1, 1, figsize=(9, 9))
    alts_beg = []
    alts_mid = []
    alts_end = []

    for typ, kw in PLOTKW.items():
        for _coo, _color in zip(kw["coo"], kw["colors"]):
            aplt.plot_altitude(
                ax=axs,
                targets=_coo,
                observer=obs,
                time=OBSTIMES,
                min_altitude=args.min_alt,
                style_kwargs=dict(
                    linestyle=kw["ls"],
                    color=_color,
                    alpha=kw["alpha"],
                    linewidth=kw["lw"]
                )
            )
            alts = obs.altaz(OBSTIME_RANGE, target=_coo).alt
            alts_beg.append(alts[0].value)
            alts_mid.append(alts[1].value)
            alts_end.append(alts[2].value)

        if args.verbose:
            print(f"{typ:>6s}: {len(kw['coo']):02d} objects")

    _radec = cat_up["RA"].astype(str) + "
" + cat_up["DEC"].astype(str)
    cat_up.sort_values(by=["Type", "DEC"], ascending=False, ignore_index=True, inplace=True)
    cat_up.insert(loc=3, column="RADEC[˚]", value=_radec.values)
    cat_up["ID"] = cat_up["ID"].apply(mk_wikilink)  # Add wiki links.
    cat_up["ID"] = "" + cat_up["ID"] + "

" + cat_up["Other ID"]

    colnames = [
        f"- {dt.value:.0f} hr
altitude",
        (f"{str(args.YYYY)[-2:]}-{args.MM:02d}-{args.DD:02d}
"
         + f"{args.HH:02d}:{args.mm:02d}
altitude"),
        f"+ {dt.value:.0f} hr
altitude"
    ]
    for i, (alts, col) in enumerate(zip([alts_beg, alts_mid, alts_end], colnames)):
        cat_up.insert(loc=i+3, column=col, value=alts)
        cat_up[col] = cat_up[col].apply(alt_color)

    cat_up.drop(columns=COLS2DROP, inplace=True)

    # == Convert to HTML ================================================================= #
    html_str = cat_up.to_html(None, index=False, escape=False)
    html_str = "
" + html_str + "
"
    with open(OUTPUT, "w+") as output:
        output.write(html_str)

    if args.verbose:
        print(f"* Catalog saved to {OUTPUT}")
    # ------------------------------------------------------------------------------------ #

    # for i, (_coo, ls) in enumerate(zip(coo_up, lss)):
    #     aplt.plot_altitude(
    #         targets=_coo, observer=obs, time=OBSTIMES, ax=axs, min_altitude=args.min_alt,
    #         style_kwargs=dict(linestyle=ls, color=colors[i], alpha=0.7, linewidth=2)
    #     )

    moon_phase = obs.moon_phase(OBSTIME).to(u.deg).value
    # in radian, phase=pi is “new”, phase=0 is “full”.
    alt_moon = obs.moon_altaz(OBSTIMES).alt
    axs.plot_date(OBSTIMES.plot_date, alt_moon, '-', color='k', linewidth=6, alpha=0.4,
                  label=f'Moon (θ_full={moon_phase:.0f}˚)')

    upmask_pl = check_observable(args.min_alt_pl, targets=coo_pl, **observable_kw)
    coo_pl_up = coo_pl[upmask_pl]
    if args.verbose:
        print(f"{len(coo_pl_up)} planets are visible under the user's criteria.")
        print(INFOSTR)

    if len(coo_pl_up) > 0:
        for _coo in coo_pl_up:
            aplt.plot_altitude(
                ax=axs,
                targets=_coo,
                observer=obs,
                time=OBSTIMES,
                min_altitude=args.min_alt,
                style_kwargs=dict(
                    color=PLANETS[_coo.name],
                    linewidth=6,
                    alpha=0.3
                )
            )

    fake_coo = coo_pl[0].copy()
    fake_coo.name = None
    aplt.plot_altitude(
        ax=axs,
        targets=fake_coo,
        observer=obs,
        time=OBSTIMES,
        min_altitude=args.min_alt,
        airmass_yaxis=True,
        brightness_shading=True,
        style_kwargs=dict(linestyle='')
    )

    axs.axhline(30, color='k', linestyle='-')
    axs.legend(ncol=4, bbox_to_anchor=(0.5, -0.15), loc=9,
               prop={"family": "monospace", "size": 10},
               title="<Catalog ID> (<Type>). Use `-v` for detailed explanation for Type.")
    plt.tight_layout()
    plt.show()

reason: import numpy,import astropy,from astropy


text:
import rospy
from styx_msgs.msg import TrafficLight
import tensorflow as tf
from PIL import Image
import cv2
import numpy as np
from glob import glob
import os
from keras import backend as K
from keras.models import load_model
from keras.utils import plot_model
import time

class TLClassifier(object):
    def __init__(self, is_site):
        self.state = 0
        self.out = 0
        self.done= 0
        cwd = os.path.dirname(os.path.realpath(__file__))
        print "path" , cwd

        self.class_graph = tf.get_default_graph()

        # detection graph
        self.dg = tf.Graph()
        # classification graph
        self.cl = tf.Graph()

        with self.cl.as_default():
            #open keras classification model
            sess= tf.Session()
            K.set_session(sess)
            model = 'carla_aug.h5' if is_site else 'model.h5'
            self.class_model =load_model(cwd+'/models/'+model)
            self.session_cl = tf.Session(graph=self.cl )

        with self.dg.as_default():
            gdef = tf.GraphDef()
            with open(cwd + "/models/frozen_inference_graph.pb", 'rb') as f:
                gdef.ParseFromString( f.read() )
                tf.import_graph_def( gdef, name="" )

            self.session_dg = tf.Session(graph=self.dg )
            self.image_tensor = self.dg.get_tensor_by_name('image_tensor:0')
            self.detection_boxes =  self.dg.get_tensor_by_name('detection_boxes:0')
            self.detection_scores = self.dg.get_tensor_by_name('detection_scores:0')
            self.detection_classes = self.dg.get_tensor_by_name('detection_classes:0')
            self.num_detections    = self.dg.get_tensor_by_name('num_detections:0')

        self.tlclasses = [ TrafficLight.RED, TrafficLight.YELLOW, TrafficLight.GREEN ]
        self.tlclasses_d = { TrafficLight.RED : "RED", TrafficLight.YELLOW:"YELLOW", TrafficLight.GREEN:"GREEN", TrafficLight.UNKNOWN:"UNKNOWN" }

    def get_classification(self, image, st):
        """Determines the color of the traffic light in the image

        Args:
            image (cv::Mat): image containing the traffic light. OpenCV is BGR by default.

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """
        box = self.localize_lights( image )
        if box is None:
            return TrafficLight.UNKNOWN
        class_image = cv2.resize( image[box[0]:box[2], box[1]:box[3]], (32,32) )
        if self.out:
            cv2.imwrite('/home/carkyo/imageration.jpg',class_image)
            self.done=1

        light_state =  self.classify_lights( class_image )
        #debug comments
        colors = ['red','yellow','green']
        rospy.loginfo('Detected light is classified as '+colors[light_state])
        return light_state

    def classify_lights_old(self, image):

        """ Given a 32x32x3 image classifies it as red, greed or yellow
            Expects images in BGR format. Important otherwide won't classify correctly

        status = TrafficLight.UNKNOWN
        img_resize = np.expand_dims(image, axis=0).astype('float32')
        with self.class_graph.as_default():
            predict = self.class_model.predict(img_resize)
            status  = self.tlclasses[ np.argmax(predict) ]

        return status
        """

        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        image = image.reshape((image.shape[0] * image.shape[1], 3))
        r=0.0;g=0.0;y=0.0
        for i in range (len(image)):
            red=image[i][0]
            green=image[i][1]
            blue=image[i][2]
            if ( (red > green+100) and (red > blue+100) and (red > 190) ):
                r+=1
            elif ( (green > red+100) and (green > blue+100) and (green > 190) ):
                g+=1
            elif ( (red > 200) and (green > 200) and (blue < 100) ):
                y+=1

        if (r > max(g,y) ):
            self.state=0
        elif (g > max(y,r) ):
            self.state=2
        elif (y > max(g,r) ):
            self.state=1

        rospy.loginfo (str(self.state))
        return self.state

    def classify_lights(self,image):
        with self.cl.as_default():
            #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            #rospy.loginfo(type(image))
            #rospy.loginfo(str(image.shape))
            #rospy.loginfo(image.dtype)
            state = self.class_model.predict(np.array([image]))
            return np.argmax(state)

    def localize_lights(self, image):
        """ Localizes bounding boxes for lights using pretrained TF model
            expects BGR8 image
        """
        with self.dg.as_default():
            #switch from BGR to RGB. Important otherwise detection won't work
            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

            tf_image_input = np.expand_dims(image,axis=0)
            #run detection model
            (detection_boxes, detection_scores, detection_classes, num_detections) = self.session_dg.run(
                    [self.detection_boxes, self.detection_scores, self.detection_classes, self.num_detections],
                    feed_dict={self.image_tensor: tf_image_input})

            detection_boxes   = np.squeeze(detection_boxes)
            detection_classes = np.squeeze(detection_classes)
            detection_scores  = np.squeeze(detection_scores)

            ret = None
            detection_threshold = 0.4

            # Find first detection of signal. It's labeled with number 10
            idx = -1
            for i, cl in enumerate(detection_classes.tolist()):
                if cl == 10:
                    idx = i;
                    break;

            if idx == -1:
                pass  # no signals detected
            elif detection_scores[idx] < detection_threshold:
                pass # we are not confident of detection
            else:
                dim = image.shape[0:2]
                box = self.from_normalized_dims__to_pixel(detection_boxes[idx], dim)
                box_h, box_w  = (box[2] - box[0], box[3]-box[1])
                if (box_h < 20) or (box_w < 20):
                    rospy.logwarn("Box too small")
                    pass    # box too small
                elif ( box_h/box_w < 1.6):
                    rospy.logwarn("Box wrong ratio: "+str(box))
                    self.out=1
#                    pass    # wrong ratio
                    ret = box
                else:
                    if self.done==1:
                        self.out=0
                    rospy.loginfo('detected bounding box: {} conf: {}'.format(box, detection_scores[idx]))
                    ret = box

        return ret

    def from_normalized_dims__to_pixel(self, box, dim):
            height, width = dim[0], dim[1]
            box_pixel = [int(box[0]*height), int(box[1]*width), int(box[2]*height), int(box[3]*width)]
            return np.array(box_pixel)

    def draw_box(self, img, box):
        cv2.rectangle(img, (box[1],box[0]), (box[3],box[2]), (255,0,0), 5)

        return img

max_stars_repo_name: EmilePapillon/CarND-Capstone

max_issues_repo_name: EmilePapillon/Udacity-SDC-Capstone

max_forks_repo_name: EmilePapillon/Udacity-SDC-Capstone


text:
"""Contains data generator for orgnaizing various audio data preprocessing
pipeline and offering data reader interface of PaddlePaddle requirements.
"""

import random
import numpy as np
import paddle
import paddle.fluid as fluid
from threading import local
from data_utils.utility import read_manifest
from data_utils.augmentor.augmentation import AugmentationPipeline
from data_utils.featurizer.speech_featurizer import SpeechFeaturizer
from data_utils.speech import SpeechSegment
from data_utils.normalizer import FeatureNormalizer

class DataGenerator(object):
    """
    DataGenerator provides basic audio data preprocessing pipeline, and offers
    data reader interfaces of PaddlePaddle requirements.

    :param vocab_filepath: Vocabulary filepath for indexing tokenized
                           transcripts.
    :type vocab_filepath: str
    :param mean_std_filepath: File containing the pre-computed mean and stddev.
    :type mean_std_filepath: None|str
    :param augmentation_config: Augmentation configuration in json string.
                                Details see AugmentationPipeline.__doc__.
    :type augmentation_config: str
    :param max_duration: Audio with duration (in seconds) greater than
                         this will be discarded.
    :type max_duration: float
    :param min_duration: Audio with duration (in seconds) smaller than
                         this will be discarded.
    :type min_duration: float
    :param stride_ms: Striding size (in milliseconds) for generating frames.
    :type stride_ms: float
    :param window_ms: Window size (in milliseconds) for generating frames.
    :type window_ms: float
    :param use_dB_normalization: Whether to normalize the audio to -20 dB
                                before extracting the features.
    :type use_dB_normalization: bool
    :param random_seed: Random seed.
    :type random_seed: int
    :param keep_transcription_text: If set to True, transcription text will
                                    be passed forward directly without
                                    converting to index sequence.
    :type keep_transcription_text: bool
    :param place: The place to run the program.
    :type place: CPUPlace or CUDAPlace
    :param is_training: If set to True, generate text data for training,
                        otherwise,  generate text data for infer.
    :type is_training: bool
    """

    def __init__(self,
                 vocab_filepath,
                 mean_std_filepath,
                 augmentation_config='{}',
                 max_duration=float('inf'),
                 min_duration=0.0,
                 stride_ms=10.0,
                 window_ms=20.0,
                 use_dB_normalization=True,
                 random_seed=0,
                 keep_transcription_text=False,
                 place=paddle.CPUPlace(),
                 is_training=True):
        self._max_duration = max_duration
        self._min_duration = min_duration
        self._normalizer = FeatureNormalizer(mean_std_filepath)
        self._augmentation_pipeline = AugmentationPipeline(augmentation_config=augmentation_config,
                                                           random_seed=random_seed)
        self._speech_featurizer = SpeechFeaturizer(vocab_filepath=vocab_filepath,
                                                   stride_ms=stride_ms,
                                                   window_ms=window_ms,
                                                   use_dB_normalization=use_dB_normalization)
        self._rng = random.Random(random_seed)
        self._keep_transcription_text = keep_transcription_text
        self._epoch = 0
        self._is_training = is_training
        # for caching tar files info
        self._local_data = local()
        self._local_data.tar2info = {}
        self._local_data.tar2object = {}
        self._place = place

    def process_utterance(self, audio_file, transcript):
        """对语音数据加载、扩充、特征化和归一化

        :param audio_file: 音频文件的文件路径或文件对象
        :type audio_file: str | file
        :param transcript: 音频对应的文本
        :type transcript: str
        :return: 经过归一化等预处理的音频数据，音频文件对应文本的ID
        :rtype: tuple of (2darray, list)
        """
        speech_segment = SpeechSegment.from_file(audio_file, transcript)
        self._augmentation_pipeline.transform_audio(speech_segment)
        specgram, transcript_part = self._speech_featurizer.featurize(speech_segment, self._keep_transcription_text)
        specgram = self._normalizer.apply(specgram)
        return specgram, transcript_part

    def batch_reader_creator(self,
                             manifest_path,
                             batch_size,
                             padding_to=-1,
                             flatten=False,
                             shuffle_method="batch_shuffle"):
        """
        Batch data reader creator for audio data. Return a callable generator
        function to produce batches of data.

        Audio features within one batch will be padded with zeros to have the
        same shape, or a user-defined shape.

        :param manifest_path: Filepath of manifest for audio files.
        :type manifest_path: str
        :param batch_size: Number of instances in a batch.
        :type batch_size: int
        :param padding_to:  If set -1, the maximun shape in the batch
                            will be used as the target shape for padding.
                            Otherwise, `padding_to` will be the target shape.
        :type padding_to: int
        :param flatten: If set True, audio features will be flatten to 1darray.
        :type flatten: bool
        :param shuffle_method: Shuffle method. Options:
                                '' or None: no shuffle.
                                'instance_shuffle': instance-wise shuffle.
                                'batch_shuffle': similarly-sized instances are
                                                 put into batches, and then
                                                 batch-wise shuffle the batches.
                                                 For more details, please see
                                                 ``_batch_shuffle.__doc__``.
                                'batch_shuffle_clipped': 'batch_shuffle' with
                                                         head shift and tail
                                                         clipping. For more
                                                         details, please see
                                                         ``_batch_shuffle``.
                              If sortagrad is True, shuffle is disabled
                              for the first epoch.
        :type shuffle_method: None|str
        :return: Batch reader function, producing batches of data when called.
        :rtype: callable
        """

        def batch_reader():
            # 读取数据列表
            manifest = read_manifest(manifest_path=manifest_path,
                                     max_duration=self._max_duration,
                                     min_duration=self._min_duration)
            # 将数据列表长到短排序
            if self._epoch == 0:
                manifest.sort(key=lambda x: x["duration"])
                manifest.reverse()
            else:
                if shuffle_method == "batch_shuffle":
                    manifest = self._batch_shuffle(manifest, batch_size, clipped=False)
                elif shuffle_method == "batch_shuffle_clipped":
                    manifest = self._batch_shuffle(manifest, batch_size, clipped=True)
                elif shuffle_method == "instance_shuffle":
                    self._rng.shuffle(manifest)
                elif shuffle_method is None:
                    pass
                else:
                    raise ValueError("Unknown shuffle method %s." % shuffle_method)
            # 准备批量数据
            batch = []
            instance_reader = self._instance_reader_creator(manifest)

            for instance in instance_reader():
                batch.append(instance)
                if len(batch) == batch_size:
                    yield self._padding_batch(batch, padding_to, flatten)
                    batch = []
            if len(batch) >= 1:
                yield self._padding_batch(batch, padding_to, flatten)
            self._epoch += 1

        return batch_reader

    @property
    def feeding(self):
        """返回数据读取器的exe读取字典

        :return: 数据读取字典
        :rtype: dict
        """
        feeding_dict = {"audio_spectrogram": 0, "transcript_text": 1}
        return feeding_dict

    @property
    def vocab_size(self):
        """返回词汇表大小

        :return: 词汇表大小
        :rtype: int
        """
        return self._speech_featurizer.vocab_size

    @property
    def vocab_list(self):
        """返回词汇表列表

        :return: 词汇表列表
        :rtype: list
        """
        return self._speech_featurizer.vocab_list

    def _instance_reader_creator(self, manifest):
        """
        创建一个数据生成器reader

        Instance: 生成器得到的数据是一个元组，包含了经过预处理音频数据和音频对应文本的ID
        """

        def reader():
            for instance in manifest:
                inst = self.process_utterance(instance["audio_filepath"], instance["text"])
                yield inst

        return reader

    def _padding_batch(self, batch, padding_to=-1, flatten=False):
        """
        用零填充音频功能，使它们在同一个batch具有相同的形状(或一个用户定义的形状)

        如果padding_to为-1，则批处理中的最大形状将被使用 作为填充的目标形状。
        否则，' padding_to '将是目标形状(仅指第二轴)。

        如果“flatten”为True，特征将被flatten为一维数据
        """
        # 获取目标形状
        max_length = max([audio.shape[1] for audio, text in batch])
        if padding_to != -1:
            if padding_to < max_length:
                raise ValueError("如果padding_to不是-1，它应该大于批处理中任何实例的形状")
            max_length = padding_to
        # 填充操作
        padded_audios = []
        texts, text_lens = [], []
        audio_lens = []
        masks = []
        for audio, text in batch:
            padded_audio = np.zeros([audio.shape[0], max_length])
            padded_audio[:, :audio.shape[1]] = audio
            if flatten:
                padded_audio = padded_audio.flatten()
            padded_audios.append(padded_audio)
            if self._is_training:
                texts += text
            else:
                texts.append(text)
            text_lens.append(len(text))
            audio_lens.append(audio.shape[1])
            mask_shape0 = (audio.shape[0] - 1) // 2 + 1
            mask_shape1 = (audio.shape[1] - 1) // 3 + 1
            mask_max_len = (max_length - 1) // 3 + 1
            mask_ones = np.ones((mask_shape0, mask_shape1))
            mask_zeros = np.zeros((mask_shape0, mask_max_len - mask_shape1))
            mask = np.repeat(
                np.reshape(np.concatenate((mask_ones, mask_zeros), axis=1),
                           (1, mask_shape0, mask_max_len)), 32, axis=0)
            masks.append(mask)
        padded_audios = np.array(padded_audios).astype('float32')
        if self._is_training:
            texts = np.expand_dims(np.array(texts).astype('int32'), axis=-1)
            texts = fluid.create_lod_tensor(texts, recursive_seq_lens=[text_lens], place=self._place)
        audio_lens = np.array(audio_lens).astype('int64').reshape([-1, 1])
        masks = np.array(masks).astype('float32')
        return padded_audios, texts, audio_lens, masks

    def _batch_shuffle(self, manifest, batch_size, clipped=False):
        """将大小相似的实例放入小批量中可以提高效率，并进行批量打乱

        1. 按持续时间对音频剪辑进行排序
        2. 生成一个随机数k， k的范围[0,batch_size)
        3. 随机移动k实例，为不同的epoch训练创建不同的批次
        4. 打乱minibatches.

        :param manifest: 数据列表
        :type manifest: list
        :param batch_size: 批量大小。这个大小还用于为批量洗牌生成一个随机数。
        :type batch_size: int
        :param clipped: 是否剪辑头部(小移位)和尾部(不完整批处理)实例。
        :type clipped: bool
        :return: Batch shuffled mainifest.
        :rtype: list
        """
        manifest.sort(key=lambda x: x["duration"])
        shift_len = self._rng.randint(0, batch_size - 1)
        batch_manifest = list(zip(*[iter(manifest[shift_len:])] * batch_size))
        self._rng.shuffle(batch_manifest)
        batch_manifest = [item for batch in batch_manifest for item in batch]
        if not clipped:
            res_len = len(manifest) - shift_len - len(batch_manifest)
            batch_manifest.extend(manifest[-res_len:])
            batch_manifest.extend(manifest[0:shift_len])
        return batch_manifest

max_stars_repo_name: WinderWL/PaddlePaddle-DeepSpeech

max_issues_repo_name: WinderWL/PaddlePaddle-DeepSpeech

max_forks_repo_name: WinderWL/PaddlePaddle-DeepSpeech


text:
#
# This is application demonstrates how you can use cvui when your project has
# multiple files that use cvui.
#
# Copyright (c) 2018 Fernando Bevilacqua <dovyski@gmail.com>
# Licensed under the MIT license.
#

import numpy as np
import cv2
import cvui

# Import the two dummy external files
from Class1 import Class1
from Class2 import Class2

WINDOW_NAME = 'CVUI Multiple files'

def main():
	frame = np.zeros((300, 500, 3), np.uint8)

	c1 = Class1()
	c2 = Class2()

	# Init cvui and tell it to create a OpenCV window, i.e. cv2.namedWindow(WINDOW_NAME).
	cvui.init(WINDOW_NAME)

	while (True):
		# Fill the frame with a nice color
		frame[:] = (49, 52, 49)

		c1.renderInfo(frame)
		c2.renderMessage(frame)

		# This function must be called *AFTER* all UI components. It does
		# all the behind the scenes magic to handle mouse clicks, etc.
		cvui.update()

		# Show everything on the screen
		cv2.imshow(WINDOW_NAME, frame)

		# Check if ESC key was pressed
		if cv2.waitKey(20) == 27:
			break

if __name__ == '__main__':
    main()