text:
from comet_ml import Experiment, OfflineExperiment

import copy
import glob
import os
import time
from collections import deque

import gym
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from a2c_ppo_acktr import algo
from a2c_ppo_acktr.arguments import get_args
from a2c_ppo_acktr.envs import make_vec_envs
from a2c_ppo_acktr.model import Policy
from a2c_ppo_acktr.storage import RolloutStorage
from a2c_ppo_acktr.utils import get_vec_normalize, update_linear_schedule
from a2c_ppo_acktr.visualize import visdom_plot

import pickle

import collections

args = get_args()

assert args.algo in ['a2c', 'ppo', 'acktr']
if args.recurrent_policy:
    assert args.algo in ['a2c', 'ppo'], \
        'Recurrent policy is not implemented for ACKTR'

if args.comet == "offline":
    experiment = OfflineExperiment(project_name="recurrent-filter", workspace="nishanthvanand",
    disabled=args.disable_log, offline_directory="/scratch/nish127/comet_offline",
    parse_args=False)
elif args.comet == "online":
    experiment = experiment = Experiment(api_key="tSACzCGFcetSBTapGBKETFARf",
                        project_name="recurrent-filter", workspace="nishanthvanand",disabled=args.disable_log,
                        parse_args=False)

experiment.log_parameters(vars(args))

num_updates = int(args.num_env_steps) // args.num_steps // args.num_processes

torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)

if args.cuda and torch.cuda.is_available() and args.cuda_deterministic:
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

try:
    os.makedirs(args.log_dir)
except OSError:
    files = glob.glob(os.path.join(args.log_dir, '*.monitor.csv'))
    for f in files:
        os.remove(f)

eval_log_dir = args.log_dir + "_eval"

try:
    os.makedirs(eval_log_dir)
except OSError:
    files = glob.glob(os.path.join(eval_log_dir, '*.monitor.csv'))
    for f in files:
        os.remove(f)

def main():
    torch.set_num_threads(1)
    device = torch.device("cuda:0" if args.cuda else "cpu")

    if args.vis:
        from visdom import Visdom
        viz = Visdom(port=args.port)
        win = None

    envs, is_minigrid = make_vec_envs(args.env_name, args.seed, args.num_processes,
                        args.gamma, args.log_dir, args.add_timestep, device, False, args.num_frame_stack)

    actor_critic = Policy(envs.observation_space.shape, envs.action_space,
        base_kwargs={'recurrent': args.recurrent_policy, 'filter_mem':args.filter_memory},
        is_minigrid=is_minigrid)
    actor_critic.to(device)

    '''
    passing the size of latent representation here
    '''
    if len(envs.observation_space.shape)==3 and not is_minigrid:
        hidden_size = 512
    if len(envs.observation_space.shape)==3 and is_minigrid:
        hidden_size = 64
    elif len(envs.observation_space.shape)==1:
        hidden_size = 64
    else:
        NotImplementedError

    if args.algo == 'a2c':
        agent = algo.A2C_ACKTR(actor_critic, args.value_loss_coef,
                               args.entropy_coef, lr=args.lr, filter_mem=args.filter_memory,
                               eps=args.eps, alpha=args.alpha,
                               max_grad_norm=args.max_grad_norm)
    elif args.algo == 'ppo':
        agent = algo.PPO(actor_critic, args.clip_param, args.ppo_epoch, args.num_mini_batch,
                         args.value_loss_coef, args.entropy_coef, lr=args.lr,
                               eps=args.eps,
                               max_grad_norm=args.max_grad_norm)
    elif args.algo == 'acktr':
        agent = algo.A2C_ACKTR(actor_critic, args.value_loss_coef,
                               args.entropy_coef, acktr=True)

    rollouts = RolloutStorage(args.num_steps, args.num_processes,
                        envs.observation_space.shape,
                        hidden_size, (1, args.filter_memory),
                        envs.action_space,
                        actor_critic.recurrent_hidden_state_size)

    obs = envs.reset()
    obs = obs + torch.randn_like(obs) * args.noise_obs

    rollouts.obs[0].copy_(obs)
    rollouts.to(device)

    episode_rewards = deque(maxlen=10)

    start = time.time()

    filter_coeff_list = []

    all_rewards_local = deque()
    all_frame_local = deque()
    all_filter_mean_local = deque()
    all_filter_std_local = deque()

    for j in range(num_updates):

        value_prev = collections.deque([torch.zeros(args.num_processes, 1).to(device) for i in range(args.filter_memory)], maxlen=args.filter_memory)
        value_prev_eval = collections.deque([torch.zeros(args.num_processes, 1).to(device) for i in range(args.filter_memory)], maxlen=args.filter_memory)
        filter_mem_latent_eval = collections.deque([torch.zeros(args.num_processes, hidden_size).to(device) for i in range(args.filter_memory)], maxlen=args.filter_memory)

        if args.filter_type == "IIR":
            raise NotImplementedError

        if args.use_linear_lr_decay:
            # decrease learning rate linearly
            if args.algo == "acktr":
                # use optimizer's learning rate since it's hard-coded in kfac.py
                update_linear_schedule(agent.optimizer, j, num_updates, agent.optimizer.lr)
            else:
                update_linear_schedule(agent.optimizer, j, num_updates, args.lr)

        if args.algo == 'ppo' and args.use_linear_clip_decay:
            agent.clip_param = args.clip_param  * (1 - j / float(num_updates))

        for step in range(args.num_steps):
            # Sample actions
            with torch.no_grad():
                value, action, action_log_prob, recurrent_hidden_states = actor_critic.act(
                        rollouts.obs[step],
                        rollouts.recurrent_hidden_states[step],
                        rollouts.masks[step],
                        value_prev=value_prev,
                        filter_type=args.filter_type)

            # Obser reward and next obs
            obs, reward, done, infos = envs.step(action)
            obs = obs + torch.randn_like(obs) * args.noise_obs

            for info in infos:
                if 'episode' in info.keys():
                    episode_rewards.append(info['episode']['r'])

            # If done then clean the history of observations.
            masks = torch.FloatTensor([[0.0] if done_ else [1.0]
                                       for done_ in done])

            rollouts.insert(obs, recurrent_hidden_states, action, action_log_prob, value, reward, masks)

        with torch.no_grad():
            next_value, next_latent = actor_critic.get_value(rollouts.obs[-1],
                                                rollouts.recurrent_hidden_states[-1],
                                                rollouts.masks[-1])

        next_value = next_value.detach()
        next_latent = next_latent.detach()

        rollouts.compute_returns(next_value, next_latent, args.use_gae, args.gamma, args.tau)

        value_loss, action_loss, dist_entropy, value_prev_eval, filter_mem_latent_eval, att_list = agent.update(rollouts, value_prev_eval, filter_mem_latent_eval, filter_type=args.filter_type, filter_mem=args.filter_memory)

        filter_coeff_list.append(att_list)
        rollouts.after_update()

        # save for every interval-th episode or for the last epoch
        if (j % args.save_interval == 0 or j == num_updates - 1) and args.save_dir != "":
            save_path = os.path.join(args.save_dir, args.algo)
            try:
                os.makedirs(save_path)
            except OSError:
                pass

            # A really ugly way to save a model to CPU
            save_model = actor_critic
            if args.cuda:
                save_model = copy.deepcopy(actor_critic).cpu()

            save_model = [save_model,
                          getattr(get_vec_normalize(envs), 'ob_rms', None)]

            torch.save(save_model, os.path.join(save_path, args.env_name +"_seed_"+str(args.seed)+"_filter_mem_"+str(args.filter_memory)+"_entropy_"+str(args.entropy_coef)+"_steps_"+str(args.num_steps)+"_noise_obs_"+str(args.noise_obs)+".pt"))

        total_num_steps = (j + 1) * args.num_processes * args.num_steps

        if j % args.log_interval == 0 and len(episode_rewards) > 1:
            end = time.time()
            print("Updates {}, num timesteps {}, FPS {} \n Last {} training episodes: mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}\n".
                format(j, total_num_steps,
                       int(total_num_steps / (end - start)),
                       len(episode_rewards),
                       np.mean(episode_rewards),
                       np.median(episode_rewards),
                       np.min(episode_rewards),
                       np.max(episode_rewards), dist_entropy,
                       value_loss, action_loss))

            filter_coeff_mean = {"coeff_mean_"+str(i):0 for i in range(args.filter_memory)}
            filter_coeff_std = {"coeff_std_"+str(i):0 for i in range(args.filter_memory)}

            filter_list = []
            for batches in range(len(filter_coeff_list)):
                    if not args.cuda:
                        filter_np = filter_coeff_list[batches].numpy()
                    else:
                        filter_np = filter_coeff_list[batches].cpu().numpy()
                    filter_list.append(filter_np.mean(1).ravel())
            filter_numpy = np.array(filter_list)
            filter_mean = filter_numpy.mean(0)
            filter_std = filter_numpy.std(0)

            for idx, (m,s) in enumerate(zip(filter_mean, filter_std)):
                filter_coeff_mean["coeff_mean_"+str(idx)] = m
                filter_coeff_std["coeff_std_"+str(idx)] = s

            experiment.log_metrics({"mean reward": np.mean(episode_rewards),
                                 "Value loss": value_loss, "Action Loss": action_loss},
                                 step=j * args.num_steps * args.num_processes)

            experiment.log_metrics(filter_coeff_mean, step=j * args.num_steps * args.num_processes)
            experiment.log_metrics(filter_coeff_std, step=j * args.num_steps * args.num_processes)

            all_rewards_local.append(np.mean(episode_rewards))
            all_frame_local.append(total_num_steps)
            all_filter_mean_local.append(filter_mean)
            all_filter_std_local.append(filter_std)

            del filter_coeff_list[:]

        if (args.eval_interval is not None
                and len(episode_rewards) > 1
                and j % args.eval_interval == 0):
            eval_envs = make_vec_envs(
                args.env_name, args.seed + args.num_processes, args.num_processes,
                args.gamma, eval_log_dir, args.add_timestep, device, True)

            vec_norm = get_vec_normalize(eval_envs)
            if vec_norm is not None:
                vec_norm.eval()
                vec_norm.ob_rms = get_vec_normalize(envs).ob_rms

            eval_episode_rewards = []

            obs = eval_envs.reset()
            eval_recurrent_hidden_states = torch.zeros(args.num_processes,
                            actor_critic.recurrent_hidden_state_size, device=device)
            eval_masks = torch.zeros(args.num_processes, 1, device=device)

            while len(eval_episode_rewards) < 10:
                with torch.no_grad():
                    _, action, _, eval_recurrent_hidden_states = actor_critic.act(
                        obs, eval_recurrent_hidden_states, eval_masks, deterministic=True)

                # Obser reward and next obs
                obs, reward, done, infos = eval_envs.step(action)

                eval_masks = torch.tensor([[0.0] if done_ else [1.0]
                                           for done_ in done],
                                           dtype=torch.float32,
                                           device=device)

                for info in infos:
                    if 'episode' in info.keys():
                        eval_episode_rewards.append(info['episode']['r'])

            eval_envs.close()

            print(" Evaluation using {} episodes: mean reward {:.5f}\n".
                format(len(eval_episode_rewards),
                       np.mean(eval_episode_rewards)))

        if args.vis and j % args.vis_interval == 0:
            try:
                # Sometimes monitor doesn't properly flush the outputs
                win = visdom_plot(viz, win, args.log_dir, args.env_name,
                                  args.algo, args.num_env_steps)
            except IOError:
                pass

    with open(args.save_local_dir+"Rewards_"+str(args.env_name)+"_seed_"+str(args.seed)+"_filter_mem_"+str(args.filter_memory)+"_entropy_"+str(args.entropy_coef)+"_steps_"+str(args.num_steps)+"_noise_obs_"+str(args.noise_obs)+".pkl", 'wb') as f:
        pickle.dump(all_rewards_local, f)

    with open(args.save_local_dir+"Frames_"+str(args.env_name)+"_seed_"+str(args.seed)+"_filter_mem_"+str(args.filter_memory)+"_entropy_"+str(args.entropy_coef)+"_steps_"+str(args.num_steps)+"_noise_obs_"+str(args.noise_obs)+".pkl", 'wb') as f:
        pickle.dump(all_frame_local, f)

    with open(args.save_local_dir+"filter_mean_"+str(args.env_name)+"_seed_"+str(args.seed)+"_filter_mem_"+str(args.filter_memory)+"_entropy_"+str(args.entropy_coef)+"_steps_"+str(args.num_steps)+"_noise_obs_"+str(args.noise_obs)+".pkl", 'wb') as f:
        pickle.dump(all_filter_mean_local, f)

    with open(args.save_local_dir+"filter_std_"+str(args.env_name)+"_seed_"+str(args.seed)+"_filter_mem_"+str(args.filter_memory)+"_entropy_"+str(args.entropy_coef)+"_steps_"+str(args.num_steps)+"_noise_obs_"+str(args.noise_obs)+".pkl", 'wb') as f:
        pickle.dump(all_filter_std_local, f)

if __name__ == "__main__":
    main()


text:
import cPickle
import os
import tarfile
import urllib
import pickle

import numpy as np

datadir = os.path.expanduser('~/repos/drmad/data/cifar10')

def datapath(fname):
    return os.path.join(datadir, fname)

def cifar10(cifar10_datadir):
    datadir = cifar10_datadir

    if not os.path.isdir(datadir):
        os.mkdir(datadir)

    cifar_download_url = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
    cifar_download_savepath = os.path.join(datadir, os.path.basename(cifar_download_url))

    if not os.path.exists(cifar_download_savepath):
        print('Downloading cifar-10 from {}'.format(cifar_download_url))
        urllib.urlretrieve(cifar_download_url, cifar_download_savepath)
        print('File saved to {}'.format(cifar_download_savepath))

    train_samples_count = 50000

    X_train = np.zeros((train_samples_count, 3, 32, 32), dtype="uint8")
    y_train = np.zeros((train_samples_count,), dtype="uint8")
    X_test = None
    y_test = None

    with tarfile.open(cifar_download_savepath, 'r:gz') as tar_fp:
        for tar_item in tar_fp.getmembers():
            fp = tar_fp.extractfile(tar_item)
            if fp is not None:
                if 'test_batch' not in tar_item.name and 'data_batch' not in tar_item.name:
                    continue

                pickle_data = cPickle.loads(fp.read())
                data = pickle_data['data']
                data = data.reshape(data.shape[0], 3, 32, 32)
                labels = pickle_data['labels']

                if 'data_batch' in tar_item.name:
                    batch_num = int(tar_item.name.strip()[-1])
                    X_train[(batch_num - 1) * 10000: batch_num * 10000, :, :, :] = data
                    y_train[(batch_num - 1) * 10000: batch_num * 10000] = labels
                elif 'test_batch' in tar_item.name:
                    X_test = data
                    y_test = np.asarray(labels)

    y_train = np.reshape(y_train, (len(y_train), 1))
    y_test = np.reshape(y_test, (len(y_test), 1))

    return X_train, y_train, X_test, y_test

def load_data(normalize=False):
    with open(datapath("cifar10_data.pkl")) as f:
        train_images, train_labels, test_images, test_labels = pickle.load(f)

    one_hot = lambda x, K : np.array(x[:,None] == np.arange(K)[None, :], dtype=int)
    partial_flatten = lambda x : np.reshape(x, (x.shape[0], np.prod(x.shape[1:])))
    train_images = partial_flatten(train_images) / 255.0
    test_images  = partial_flatten(test_images)  / 255.0
    train_labels = one_hot(train_labels, 10)
    test_labels = one_hot(test_labels, 10)
    N_data = train_images.shape[0]

    train_labels = train_labels.reshape(-1, 10)
    test_labels = test_labels.reshape(-1, 10)
    if normalize:
        train_mean = np.mean(train_images, axis=0)
        train_images = train_images - train_mean
        test_images = test_images - train_mean
    return train_images, train_labels, test_images, test_labels, N_data

def gz_to_pickle():
    data = cifar10(datadir);
    with open(datapath("cifar10_data.pkl"), "w") as f:
        pickle.dump(data, f, 1)

def load_data_as_dict(normalize=True):
    X, T = load_data(normalize)[:2]
    return {'X' : X, 'T' : T}

    return [{"X" : dat[0], "T" : dat[1]} for dat in datapairs]
if __name__ == '__main__':
    X_train, y_train, X_test, y_test = cifar10(datadir)
    gz_to_pickle();
    print('X_train: {}, y_train: {}', X_train.shape, y_train.shape)
    print('X_test: {}, y_test: {}', X_test.shape, y_test.shape)

ext: py

max_stars_repo_name: bigaidream-projects/drmad

max_issues_repo_name: LinZichuan/drmad

max_forks_repo_name: LinZichuan/drmad

reason: import numpy


text:
from bs4 import BeautifulSoup
from scipy.spatial.distance import cosine
import string
import re
import os
import numpy as np

from bert_helper import *
from wordnet_helper import *

def get_test_data(dir_name):
    sentence = ''
    target_index = []
    target_words = []
    target_gold_keys = []
    with open('../data/eval_data/{0}/{0}.data.xml'.format(dir_name)) as ef:
        soup = BeautifulSoup(ef, 'lxml')
        sentences = soup.find_all('sentence',{'id':'d000.s003'})

        for sent in sentences:
            for child in sent.children:
                sentence += child.string.strip()+ ' '
                if child.name == 'instance':
                    target_words.append(child.string.strip())
                    target_gold_keys.extend(child.get_attribute_list('id'))
            break

    sentence = sentence.translate(str.maketrans('', '', string.punctuation))

    sentence = re.sub(r'\b\s{2,}\b', ' ', sentence)

    sentence = sentence.strip()

    words = sentence.split(' ')
    for tw in target_words:
        if tw in words:
            target_index.append(words.index(tw))
        else:
            target_index.append(-1)

    print(sentence)
    print(target_words)
    # print(target_index)
    # print(target_gold_keys)

    test_data = {
        'sentence': sentence,
        'target_words': target_words,
        'target_index': target_index,
        'target_gold_keys': target_gold_keys
    }

    return test_data

def get_test_gold_key(target_gold_key, dir_name):

    with open('../data/eval_data/{0}/{0}.gold.key.txt'.format(dir_name)) as f:
        lines = f.readlines()

        for line in lines:
            target_sense = line.split(' ')
            if target_sense[0] == target_gold_key:
                return target_sense[1:][0].strip()

def find_candidate_embedding(target_word):
    all_synset_lex_ids = []
    candidate_embeddings = []
    for ss in wn.synsets(target_word):
        for lemma in ss.lemmas():
            all_synset_lex_ids.append(lemma.key())

    for root, dirs, files in os.walk('../data/embeddings'):
        for file in files:
            if offsetpos_to_lexid(file.split('.')[0]) in all_synset_lex_ids:
                candidate_embeddings.append(file)

    return candidate_embeddings

def find_most_similar_embedding(candidate_embedding, target_embedding):
    max_sim = -1
    most_sim = ''
    for curr_embedding in candidate_embedding:
        with open('../data/embeddings/{}'.format(curr_embedding)) as f:
            temp_vec = np.loadtxt(f, skiprows=1)
            diff_sense = 1 - cosine(temp_vec, target_embedding)
            print(offsetpos_to_lexid(curr_embedding.split('.')[0]))
            if diff_sense > max_sim:
                max_sim = diff_sense
                most_sim = curr_embedding

    return most_sim.split('.')[0]

def main():
    model, tokenizer = load_bert()

    test_data = get_test_data('senseval2')

    # current_target = 'sound'
    for target_word in test_data['target_words']:
        # if target_word != current_target:
        #     continue

        target_word_index = test_data['target_words'].index(target_word)
        target_word_sent_pos = test_data['target_index'][test_data['target_words'].index(target_word)]
        print(target_word)

        # generate BERT vector for target_word
        tokenized, seg_ids = prepare_data(test_data['sentence'], tokenizer)
        bert_vec = get_feeaturs(tokenized, seg_ids, model, target_word_sent_pos).detach().cpu().numpy()

        # generate BERT vectors for all synset with synset.name == target_word
        candidate_embeddings = find_candidate_embedding(target_word)
        most_sim = find_most_similar_embedding(candidate_embeddings, bert_vec)

        if len(most_sim) > 0:
            print('Predicted sense:lex_id: ',offsetpos_to_lexid(most_sim))
            print('Gold sense:lex_id:', get_test_gold_key(test_data['target_gold_keys'][target_word_index], 'senseval2'))

if __name__ == '__main__':
    main()

reason: import numpy,from scipy


text:
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for ragged_util."""

from absl.testing import parameterized
import numpy as np

from tensorflow.python.framework import constant_op
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops.ragged import ragged_util
from tensorflow.python.platform import googletest

# Example 3d tensor for test cases.  Has shape [4, 2, 3].
TENSOR_3D = [[[('%d%d%d' % (i, j, k)).encode('utf-8')
               for k in range(3)]
              for j in range(2)]
             for i in range(4)]

# Example 4d tensor for test cases.  Has shape [4, 2, 3, 5].
TENSOR_4D = [[[[('%d%d%d%d' % (i, j, k, l)).encode('utf-8')
                for l in range(5)]
               for k in range(3)]
              for j in range(2)]
             for i in range(4)]

@test_util.run_all_in_graph_and_eager_modes
class RaggedUtilTest(test_util.TensorFlowTestCase,
                     parameterized.TestCase):

  @parameterized.parameters([
      # Docstring examples
      dict(
          data=['a', 'b', 'c'],
          repeats=[3, 0, 2],
          axis=0,
          expected=[b'a', b'a', b'a', b'c', b'c']),
      dict(
          data=[[1, 2], [3, 4]],
          repeats=[2, 3],
          axis=0,
          expected=[[1, 2], [1, 2], [3, 4], [3, 4], [3, 4]]),
      dict(
          data=[[1, 2], [3, 4]],
          repeats=[2, 3],
          axis=1,
          expected=[[1, 1, 2, 2, 2], [3, 3, 4, 4, 4]]),

      # Scalar repeats value
      dict(
          data=['a', 'b', 'c'],
          repeats=2,
          axis=0,
          expected=[b'a', b'a', b'b', b'b', b'c', b'c']),
      dict(
          data=[[1, 2], [3, 4]],
          repeats=2,
          axis=0,
          expected=[[1, 2], [1, 2], [3, 4], [3, 4]]),
      dict(
          data=[[1, 2], [3, 4]],
          repeats=2,
          axis=1,
          expected=[[1, 1, 2, 2], [3, 3, 4, 4]]),

      # data & repeats are broadcast to have at least one dimension,
      # so these are all equivalent:
      dict(data=3, repeats=4, axis=0, expected=[3, 3, 3, 3]),
      dict(data=[3], repeats=4, axis=0, expected=[3, 3, 3, 3]),
      dict(data=3, repeats=[4], axis=0, expected=[3, 3, 3, 3]),
      dict(data=[3], repeats=[4], axis=0, expected=[3, 3, 3, 3]),
      # Empty tensor
      dict(data=[], repeats=[], axis=0, expected=[]),
  ])
  def testRepeat(self, data, repeats, expected, axis=None):
    result = ragged_util.repeat(data, repeats, axis)
    self.assertAllEqual(result, expected)

  @parameterized.parameters([
      dict(mode=mode, **args)
      for mode in ['constant', 'dynamic', 'unknown_shape']
      for args in [
          # data & repeats are broadcast to have at least one dimension,
          # so these are all equivalent:
          dict(data=3, repeats=4, axis=0),
          dict(data=[3], repeats=4, axis=0),
          dict(data=3, repeats=[4], axis=0),
          dict(data=[3], repeats=[4], axis=0),

          # 1-dimensional data tensor.
          dict(data=[], repeats=5, axis=0),
          dict(data=[1, 2, 3], repeats=5, axis=0),
          dict(data=[1, 2, 3], repeats=[3, 0, 2], axis=0),
          dict(data=[1, 2, 3], repeats=[3, 0, 2], axis=-1),
          dict(data=[b'a', b'b', b'c'], repeats=[3, 0, 2], axis=0),

          # 2-dimensional data tensor.
          dict(data=[[1, 2, 3], [4, 5, 6]], repeats=3, axis=0),
          dict(data=[[1, 2, 3], [4, 5, 6]], repeats=3, axis=1),
          dict(data=[[1, 2, 3], [4, 5, 6]], repeats=[3, 5], axis=0),
          dict(data=[[1, 2, 3], [4, 5, 6]], repeats=[3, 5, 7], axis=1),

          # 3-dimensional data tensor: shape=[4, 2, 3].
          dict(data=TENSOR_3D, repeats=2, axis=0),
          dict(data=TENSOR_3D, repeats=2, axis=1),
          dict(data=TENSOR_3D, repeats=2, axis=2),
          dict(data=TENSOR_3D, repeats=[2, 0, 4, 1], axis=0),
          dict(data=TENSOR_3D, repeats=[3, 2], axis=1),
          dict(data=TENSOR_3D, repeats=[1, 3, 1], axis=2),

          # 4-dimensional data tensor: shape=[4, 2, 3, 5].
          dict(data=TENSOR_4D, repeats=2, axis=0),
          dict(data=TENSOR_4D, repeats=2, axis=1),
          dict(data=TENSOR_4D, repeats=2, axis=2),
          dict(data=TENSOR_4D, repeats=2, axis=3),
          dict(data=TENSOR_4D, repeats=[2, 0, 4, 1], axis=0),
          dict(data=TENSOR_4D, repeats=[3, 2], axis=1),
          dict(data=TENSOR_4D, repeats=[1, 3, 1], axis=2),
          dict(data=TENSOR_4D, repeats=[1, 3, 0, 0, 2], axis=3),
      ]
  ])
  def testValuesMatchesNumpy(self, mode, data, repeats, axis):
    # Exception: we can't handle negative axis if data.ndims is unknown.
    if axis < 0 and mode == 'unknown_shape':
      return

    expected = np.repeat(data, repeats, axis)

    if mode == 'constant':
      data = constant_op.constant(data)
      repeats = constant_op.constant(repeats)
    elif mode == 'dynamic':
      data = constant_op.constant(data)
      repeats = constant_op.constant(repeats)
      data = array_ops.placeholder_with_default(data, data.shape)
      repeats = array_ops.placeholder_with_default(repeats, repeats.shape)
    elif mode == 'unknown_shape':
      data = array_ops.placeholder_with_default(data, None)
      repeats = array_ops.placeholder_with_default(repeats, None)

    result = ragged_util.repeat(data, repeats, axis)
    self.assertAllEqual(result, expected)

  @parameterized.parameters([
      dict(
          descr='axis >= rank(data)',
          mode='dynamic',
          data=[1, 2, 3],
          repeats=[3, 0, 2],
          axis=1,
          error='axis=1 out of bounds: expected -1<=axis<1'),
      dict(
          descr='axis < -rank(data)',
          mode='dynamic',
          data=[1, 2, 3],
          repeats=[3, 0, 2],
          axis=-2,
          error='axis=-2 out of bounds: expected -1<=axis<1'),
      dict(
          descr='len(repeats) != data.shape[axis]',
          mode='dynamic',
          data=[[1, 2, 3], [4, 5, 6]],
          repeats=[2, 3],
          axis=1,
          error='Dimensions 3 and 2 are not compatible'),
      dict(
          descr='rank(repeats) > 1',
          mode='dynamic',
          data=[[1, 2, 3], [4, 5, 6]],
          repeats=[[3], [5]],
          axis=1,
          error=r'Shape \(2, 1\) must have rank at most 1'),
      dict(
          descr='non-integer axis',
          mode='constant',
          data=[1, 2, 3],
          repeats=2,
          axis='foo',
          exception=TypeError,
          error='`axis` must be an int'),
  ])
  def testError(self,
                descr,
                mode,
                data,
                repeats,
                axis,
                exception=ValueError,
                error=None):
    # Make sure that this is also an error case for numpy.
    with self.assertRaises(exception):
      np.repeat(data, repeats, axis)

    if mode == 'constant':
      data = constant_op.constant(data)
      repeats = constant_op.constant(repeats)
    elif mode == 'dynamic':
      data = constant_op.constant(data)
      repeats = constant_op.constant(repeats)
      data = array_ops.placeholder_with_default(data, data.shape)
      repeats = array_ops.placeholder_with_default(repeats, repeats.shape)
    elif mode == 'unknown_shape':
      data = array_ops.placeholder_with_default(data, None)
      repeats = array_ops.placeholder_with_default(repeats, None)

    with self.assertRaisesRegex(exception, error):
      ragged_util.repeat(data, repeats, axis)

if __name__ == '__main__':
  googletest.main()

max_stars_repo_name: EricRemmerswaal/tensorflow

max_issues_repo_name: EricRemmerswaal/tensorflow

max_forks_repo_name: EricRemmerswaal/tensorflow


text:
import networkx as nx
from . import utils

# ===== asexual lineage metrics =====

def get_asexual_lineage_length(lineage):
    """Get asexual lineage length.

    Will check that given lineage is an asexual lineage.

    Args:
        lineage (networkx.DiGraph): an asexual lineage

    Returns:
        length (int) of given lineage
    """
    if not utils.is_asexual_lineage(lineage): raise Exception("the given lineage is not an asexual lineage")
    return len(lineage.nodes)

def get_asexual_lineage_num_discrete_state_changes(lineage, attribute_list):
    """Get the number of discrete state changes from an asexual lineage.

    State is described by the aggregation of all attributes give by attribute list.

    Args:
        lineage (networkx.DiGraph): an asexual lineage
        attribute_list (list): list of attributes (strings) to use when defining
            a state

    Returns:
        Returns the number of discrete states along the lineage.
    """
    # Check that lineage is an asexual lineage.
    if not utils.is_asexual_lineage(lineage): raise Exception("the given lineage is not an asexual lineage")
    # Check that all nodes have all given attributes in the attribute list
    if not utils.all_taxa_have_attributes(lineage, attribute_list): raise Exception("given attributes are not universal among all taxa along the lineage")
    # get the first state (root node)
    lineage_id = utils.get_root_ids(lineage)[0]
    num_states = 1
    cur_state = [lineage.nodes[lineage_id][attr] for attr in attribute_list]
    # count the number of state changes moving down the lineage
    while True:
        successor_ids = list(lineage.successors(lineage_id))
        if len(successor_ids) == 0: break # We've hit the last thing!
        lineage_id = successor_ids[0]
        state = [lineage.nodes[lineage_id][attr] for attr in attribute_list]
        if cur_state != state:
            cur_state = state
            num_states += 1
    return num_states

def get_asexual_lineage_num_discrete_unique_states(lineage, attribute_list):
    """Get the number of discrete unique states along a lineage where what it
    means to be a state is defined by attribute_list.

    Args:
        lineage (networkx.DiGraph): an asexual lineage
        attribute_list (list): list of attributes (strings) to use when defining
            a state

    Returns:
        The number of discrete unique states found along the lineage.
    """
    # Check that lineage is an asexual lineage.
    if not utils.is_asexual_lineage(lineage): raise Exception("the given lineage is not an asexual lineage")
    # Check that all nodes have all given attributes in the attribute list
    if not utils.all_taxa_have_attributes(lineage, attribute_list): raise Exception("given attributes are not universal among all taxa along the lineage")
    # get the first state (root node)
    lineage_id = utils.get_root_ids(lineage)[0]
    unique_states = set()
    unique_states.add(tuple([lineage.nodes[lineage_id][attr] for attr in attribute_list]))
    while True:
        successor_ids = list(lineage.successors(lineage_id))
        if len(successor_ids) == 0: break # We've hit the last thing!
        lineage_id = successor_ids[0]
        unique_states.add(tuple([lineage.nodes[lineage_id][attr] for attr in attribute_list]))
    return len(unique_states)

def get_asexual_lineage_mutation_accumulation(lineage, mutation_attributes, skip_root=False):
    """Get the distribution of mutation type accumulations over an asexual lineage.

    Args:
        lineage (networkx.DiGraph): an asexual lineage
        mutation_attributes (list of str): what are the mutation count attributes
            that we should accumulate over the lineage?
        skip_root (bool): Should we include root node mutation count values in
            our accumlation? Defaults to false.

    Returns:
        A dictionary indexed by mutation types (mutation_attributes) where each
        value in the dictionary is the sum of that type of mutation along the lineage.
    """
    # Check that lineage is an asexual lineage.
    if not utils.is_asexual_lineage(lineage): raise Exception("the given lineage is not an asexual lineage")
    # Check that all nodes have all given attributes in the attribute list
    if not utils.all_taxa_have_attributes(lineage, mutation_attributes): raise Exception("given mutation attributes are not universal among all taxa along the lineage")
    # initialize
    mut_accumulators = {mut_attr:0 for mut_attr in mutation_attributes}
    # get the root node
    lineage_id = utils.get_root_ids(lineage)[0]
    if not skip_root:
        for mut_attr in mutation_attributes:
            mut_accumulators[mut_attr] += lineage.nodes[lineage_id][mut_attr]
    while True:
        successor_ids = list(lineage.successors(lineage_id))
        if len(successor_ids) == 0: break # We've hit the last thing!
        # Is this a new state or a member of the current state?
        lineage_id = successor_ids[0]
        for mut_attr in mutation_attributes:
            mut_accumulators[mut_attr] += lineage.nodes[lineage_id][mut_attr]
    return mut_accumulators

# ===== asexual phylogeny metrics =====

def get_mrca_tree_depth_asexual(phylogeny, ids=None):
    """Get the tree depth of the most recent common ancestor shared by the specified
    taxa ids (ids) in an asexual phylogeny (phylogeny).
    """
    # Get the id of the most recent common ancestor
    mrca_id = utils.get_mrca_id_asexual(phylogeny, ids)
    if mrca_id == -1: raise Exception("phylogeny has no common ancestor")
    # Calculate distance from root to mrca
    cur_id = mrca_id
    depth = 0
    while True:
        ancestor_ids = list(phylogeny.predecessors(cur_id))
        if len(ancestor_ids) == 0: break
        depth+=1
        cur_id = ancestor_ids[0]
    return depth

# ===== phylogenetic richness =====

def calc_phylogenetic_diversity_asexual(phylogeny, ids=None):
    """Calculate phylogenetic diversity (i.e., the number of nodes in the minimum
    spanning tree from the MRCA to all extant taxa). Currently only for asexual
    phylogenies.

    (Faith, 1992)

    ids gives the set we want to calculate phylogenetic diversity on. i.e.,
    we'll get the mrca for those ids and compute the minimum spanning tree

    none defaults to including all leaf nodes
    """
    # if given no ids, default to leaf taxa; otherwise, validate given ids
    if ids == None:
        # Find MRCA on leaf nodes
        ids = utils.get_leaf_taxa_ids(phylogeny)
    # (1) get the mrca
    mrca_id = utils.get_mrca_id_asexual(phylogeny, ids)
    if mrca_id == -1: raise Exception("given ids have no common ancestor")
    # (2) collect paths from each id to mrca
    canopy = set([i for i in ids] + [mrca_id])
    for i in ids:
        cur_id = i
        while True:
            ancestor_ids = list(phylogeny.predecessors(cur_id))
            if len(ancestor_ids) == 0: break
            cur_id = ancestor_ids[0]
            # If we've encountered this path before, we can skip the rest because
            # we're guaranteed an asexual phylogeny.
            if cur_id in canopy: break
            canopy.add(cur_id)
    # Build a subgraph with only the canopy
    canopy_phylo = nx.subgraph(phylogeny, list(canopy))
    # Okay, now we can compute the minimum spanning tree.
    return len(nx.minimum_spanning_tree(canopy_phylo.to_undirected()).nodes)

max_stars_repo_name: alackles/alife-std-dev-python

max_issues_repo_name: alackles/alife-std-dev-python

max_forks_repo_name: alackles/alife-std-dev-python

reason: import networkx


text:
#-*- coding: utf-8 -*-
#! python

# Installed Libs
import numpy as np

# Pvin[Pa]
# A [m2]
# mjont [kg]

def IDEE_CEM_calc(BxBx,A,Pvin,mjout,tf):

    final_brix = BxBx[len(BxBx)-1]*100
    init_brix = BxBx[0]*100
    Pvin = Pvin*0.0001450377 # Pa -> psi
    tf = tf/3600 # s -> h
    mjout = mjout/(1000*tf) # ton/h

    print("A: ",A)
    print("init brix: ",init_brix)
    print("final brix: ",final_brix)
    print("Pvin: ",Pvin)
    print("tf: ",tf)
    print("Mjout: ",mjout)

    IDEE = ((mjout/tf)*(final_brix-init_brix))/(A*Pvin)
    CEM = ((mjout/tf)*(final_brix-init_brix)/final_brix)/(A*Pvin)

    return IDEE, CEM

max_stars_repo_path: Versiones viejas/02_26_21/IDEE_CEM.py

max_issues_repo_path: Versiones viejas/02_26_21/IDEE_CEM.py

max_forks_repo_path: Versiones viejas/02_26_21/IDEE_CEM.py


text:
from astropy.io import fits as pyfits
from builtins import input

from .logger import get_logger
from .safe_save import safe_save

reason: from astropy


text:
from typing import Any, Union, Callable

import biorbd_casadi as biorbd
from casadi import horzcat, Function, MX, SX
import numpy as np

from .penalty_node import PenaltyNodeList
from ..misc.enums import Node, PlotType, ControlType
from ..misc.mapping import Mapping
from ..misc.options import OptionGeneric

class PenaltyOption(OptionGeneric):
    """
    A placeholder for a penalty

    Attributes
    ----------
    node: Node
        The node within a phase on which the penalty is acting on
    quadratic: bool
        If the penalty is quadratic
    rows: Union[list, tuple, range, np.ndarray]
        The index of the rows in the penalty to keep
    cols: Union[list, tuple, range, np.ndarray]
        The index of the columns in the penalty to keep
    expand: bool
        If the penalty should be expanded or not
    target: np.array(target)
        A target to track for the penalty
    target_plot_name: str
        The plot name of the target
    target_to_plot: np.ndarray
        The subset of the target to plot
    plot_target: bool
        If the target should be plotted
    custom_function: Callable
        A user defined function to call to get the penalty
    node_idx: Union[list, tuple, Node]
        The index in nlp to apply the penalty to
    dt: float
        The delta time
    function: Function
        The casadi function of the penalty
    weighted_function: Function
        The casadi function of the penalty weighted
    derivative: bool
        If the minimization is applied on the numerical derivative of the state [f(t+1) - f(t)]
    explicit_derivative: bool
        If the minimization is applied to derivative of the penalty [f(t, t+1)]
    transition: bool
        If the penalty is a transition
    phase_pre_idx: int
        The index of the nlp of pre when penalty is transition
    phase_post_idx: int
        The index of the nlp of post when penalty is transition
    is_internal: bool
        If the penalty is from the user or from bioptim
    multi_thread: bool
        If the penalty is multithreaded

    Methods
    -------
    set_penalty(self, penalty: Union[MX, SX], all_pn: PenaltyNodeList)
        Prepare the dimension and index of the penalty (including the target)
    _set_dim_idx(self, dim: Union[list, tuple, range, np.ndarray], n_rows: int)
        Checks if the variable index is consistent with the requested variable.
    _check_target_dimensions(self, all_pn: PenaltyNodeList, n_time_expected: int)
        Checks if the variable index is consistent with the requested variable.
        If the function returns, all is okay
    _set_penalty_function(self, all_pn: Union[PenaltyNodeList, list, tuple], fcn: Union[MX, SX])
        Finalize the preparation of the penalty (setting function and weighted_function)
    add_target_to_plot(self, all_pn: PenaltyNodeList, combine_to: str)
        Interface to the plot so it can be properly added to the proper plot
    _finish_add_target_to_plot(self, all_pn: PenaltyNodeList)
        Internal interface to add (after having check the target dimensions) the target to the plot if needed
    add_or_replace_to_penalty_pool(self, ocp, nlp)
        Doing some configuration on the penalty and add it to the list of penalty
    _add_penalty_to_pool(self, all_pn: PenaltyNodeList)
        Return the penalty pool for the specified penalty (abstract)
    clear_penalty(self, ocp, nlp)
        Resets a penalty. A negative penalty index creates a new empty penalty (abstract)
    _get_penalty_node_list(self, ocp, nlp) -> PenaltyNodeList
        Get the actual node (time, X and U) specified in the penalty
    """

    def __init__(
        self,
        penalty: Any,
        phase: int = 0,
        node: Union[Node, list, tuple] = Node.DEFAULT,
        target: np.ndarray = None,
        quadratic: bool = None,
        weight: float = 1,
        derivative: bool = False,
        explicit_derivative: bool = False,
        integrate: bool = False,
        index: list = None,
        rows: Union[list, tuple, range, np.ndarray] = None,
        cols: Union[list, tuple, range, np.ndarray] = None,
        custom_function: Callable = None,
        is_internal: bool = False,
        multi_thread: bool = None,
        expand: bool = False,
        **params: Any,
    ):
        """
        Parameters
        ----------
        penalty: PenaltyType
            The actual penalty
        phase: int
            The phase the penalty is acting on
        node: Union[Node, list, tuple]
            The node within a phase on which the penalty is acting on
        target: np.ndarray
            A target to track for the penalty
        quadratic: bool
            If the penalty is quadratic
        weight: float
            The weighting applied to this specific penalty
        derivative: bool
            If the function should be evaluated at X and X+1
        explicit_derivative: bool
            If the function should be evaluated at [X, X+1]
        index: int
            The component index the penalty is acting on
        custom_function: Callable
            A user defined function to call to get the penalty
        is_internal: bool
            If the penalty is internally defined [True] or by the user
        **params: dict
            Generic parameters for the penalty
        """

        super(PenaltyOption, self).__init__(phase=phase, type=penalty, **params)
        self.node: Union[Node, list, tuple] = node
        self.quadratic = quadratic

        if index is not None and rows is not None:
            raise ValueError("rows and index cannot be defined simultaneously since they are the same variable")
        self.rows = rows if rows is not None else index
        self.cols = cols
        self.expand = expand

        self.target = None
        if target is not None:
            self.target = np.array(target)
            if len(self.target.shape) == 0:
                self.target = self.target[np.newaxis]
            if len(self.target.shape) == 1:
                self.target = self.target[:, np.newaxis]
        self.target_plot_name = None
        self.target_to_plot = None
        self.plot_target = True

        self.custom_function = custom_function

        self.node_idx = []
        self.dt = 0
        self.weight = weight
        self.function: Union[Function, None] = None
        self.weighted_function: Union[Function, None] = None
        self.derivative = derivative
        self.explicit_derivative = explicit_derivative
        self.integrate = integrate
        self.transition = False
        self.phase_pre_idx = None
        self.phase_post_idx = None
        if self.derivative and self.explicit_derivative:
            raise ValueError("derivative and explicit_derivative cannot be both True")
        self.is_internal = is_internal

        self.multi_thread = multi_thread

    def set_penalty(self, penalty: Union[MX, SX], all_pn: PenaltyNodeList):
        """
        Prepare the dimension and index of the penalty (including the target)

        Parameters
        ----------
        penalty: Union[MX, SX],
            The actual penalty function
        all_pn: PenaltyNodeList
            The penalty node elements
        """

        self.rows = self._set_dim_idx(self.rows, penalty.rows())
        self.cols = self._set_dim_idx(self.cols, penalty.columns())
        if self.target is not None:
            self._check_target_dimensions(all_pn, len(all_pn.t))
            if self.plot_target:
                self._finish_add_target_to_plot(all_pn)
        self._set_penalty_function(all_pn, penalty)
        self._add_penalty_to_pool(all_pn)

    def _set_dim_idx(self, dim: Union[list, tuple, range, np.ndarray], n_rows: int):
        """
        Checks if the variable index is consistent with the requested variable.

        Parameters
        ----------
        dim: Union[list, tuple, range]
            The dimension to set
        n_rows: int
            The expected row shape

        Returns
        -------
        The formatted indices
        """

        if dim is None:
            dim = range(n_rows)
        else:
            if isinstance(dim, int):
                dim = [dim]
            if max(dim) > n_rows:
                raise RuntimeError(f"{self.name} index cannot be higher than nx ({n_rows})")
        dim = np.array(dim)
        if not np.issubdtype(dim.dtype, np.integer):
            raise RuntimeError(f"{self.name} index must be a list of integer")
        return dim

    def _check_target_dimensions(self, all_pn: PenaltyNodeList, n_time_expected: int):
        """
        Checks if the variable index is consistent with the requested variable.
        If the function returns, all is okay

        Parameters
        ----------
        all_pn: PenaltyNodeList
            The penalty node elements
        n_time_expected: Union[list, tuple]
            The expected shape (n_rows, ns) of the data to track
        """

        n_dim = len(self.target.shape)
        if n_dim != 2 and n_dim != 3:
            raise RuntimeError(f"target cannot be a vector (it can be a matrix with time dimension equals to 1 though)")
        if self.target.shape[-1] == 1:
            self.target = np.repeat(self.target, n_time_expected, axis=-1)

        shape = (len(self.rows), n_time_expected) if n_dim == 2 else (len(self.rows), len(self.cols), n_time_expected)
        if self.target.shape != shape:
            raise RuntimeError(
                f"target {self.target.shape} does not correspond to expected size {shape} for penalty {self.name}"
            )

        # If the target is on controls and control is constant, there will be one value missing
        if all_pn is not None:
            if (
                all_pn.nlp.control_type == ControlType.CONSTANT
                and all_pn.nlp.ns in all_pn.t
                and self.target.shape[-1] == all_pn.nlp.ns
            ):
                if all_pn.t[-1] != all_pn.nlp.ns:
                    raise NotImplementedError("Modifying target for END not being last is not implemented yet")
                self.target = np.concatenate((self.target, np.nan * np.zeros((self.target.shape[0], 1))), axis=1)

    def _set_penalty_function(self, all_pn: Union[PenaltyNodeList, list, tuple], fcn: Union[MX, SX]):
        """
        Finalize the preparation of the penalty (setting function and weighted_function)

        Parameters
        ----------
        all_pn: PenaltyNodeList
            The nodes
        fcn: Union[MX, SX]
            The value of the penalty function
        """

        # Sanity checks
        if self.transition and self.explicit_derivative:
            raise ValueError("transition and explicit_derivative cannot be true simultaneously")
        if self.transition and self.derivative:
            raise ValueError("transition and derivative cannot be true simultaneously")
        if self.derivative and self.explicit_derivative:
            raise ValueError("derivative and explicit_derivative cannot be true simultaneously")

        if self.transition:
            ocp = all_pn[0].ocp
            nlp = all_pn[0].nlp
            nlp_post = all_pn[1].nlp
            name = self.name.replace("->", "_").replace(" ", "_")
            state_cx = horzcat(nlp.states.cx_end, nlp_post.states.cx)
            control_cx = horzcat(nlp.controls.cx_end, nlp_post.controls.cx)

        else:
            ocp = all_pn.ocp
            nlp = all_pn.nlp
            name = self.name
            if self.integrate:
                state_cx = horzcat(*([all_pn.nlp.states.cx] + all_pn.nlp.states.cx_intermediates_list))
                control_cx = all_pn.nlp.controls.cx
            else:
                state_cx = all_pn.nlp.states.cx
                control_cx = all_pn.nlp.controls.cx
            if self.explicit_derivative:
                if self.derivative:
                    raise RuntimeError("derivative and explicit_derivative cannot be simultaneously true")
                state_cx = horzcat(state_cx, all_pn.nlp.states.cx_end)
                control_cx = horzcat(control_cx, all_pn.nlp.controls.cx_end)

        param_cx = nlp.cx(nlp.parameters.cx)

        # Do not use nlp.add_casadi_func because all functions must be registered
        self.function = biorbd.to_casadi_func(
            name, fcn[self.rows, self.cols], state_cx, control_cx, param_cx, expand=self.expand
        )
        if self.derivative:
            state_cx = horzcat(all_pn.nlp.states.cx_end, all_pn.nlp.states.cx)
            control_cx = horzcat(all_pn.nlp.controls.cx_end, all_pn.nlp.controls.cx)
            self.function = biorbd.to_casadi_func(
                f"{name}",
                self.function(all_pn.nlp.states.cx_end, all_pn.nlp.controls.cx_end, param_cx)
                - self.function(all_pn.nlp.states.cx, all_pn.nlp.controls.cx, param_cx),
                state_cx,
                control_cx,
                param_cx,
            )

        modified_fcn = self.function(state_cx, control_cx, param_cx)

        dt_cx = nlp.cx.sym("dt", 1, 1)
        weight_cx = nlp.cx.sym("weight", 1, 1)
        target_cx = nlp.cx.sym("target", modified_fcn.shape)
        modified_fcn = modified_fcn - target_cx

        if self.weight:
            modified_fcn = modified_fcn ** 2 if self.quadratic else modified_fcn
            modified_fcn = weight_cx * modified_fcn * dt_cx
        else:
            modified_fcn = modified_fcn * dt_cx

        # Do not use nlp.add_casadi_func because all of them must be registered
        self.weighted_function = Function(
            name, [state_cx, control_cx, param_cx, weight_cx, target_cx, dt_cx], [modified_fcn]
        )

        if ocp.n_threads > 1 and self.multi_thread and len(self.node_idx) > 1:
            self.function = self.function.map(len(self.node_idx), "thread", ocp.n_threads)
            self.weighted_function = self.weighted_function.map(len(self.node_idx), "thread", ocp.n_threads)
        else:
            self.multi_thread = False  # Override the multi_threading, since only one node is optimized

        if self.expand:
            self.function = self.function.expand()
            self.weighted_function = self.weighted_function.expand()

    def add_target_to_plot(self, all_pn: PenaltyNodeList, combine_to: str):
        """
        Interface to the plot so it can be properly added to the proper plot

        Parameters
        ----------
        all_pn: PenaltyNodeList
            The penalty node elements
        combine_to: str
            The name of the underlying plot to combine the tracking data to
        """

        if self.target is None or combine_to is None:
            return

        self.target_plot_name = combine_to
        if self.target.shape[1] == all_pn.nlp.ns:
            self.target_to_plot = np.concatenate((self.target, np.nan * np.ndarray((self.target.shape[0], 1))), axis=1)
        else:
            self.target_to_plot = self.target

    def _finish_add_target_to_plot(self, all_pn: PenaltyNodeList):
        """
        Internal interface to add (after having check the target dimensions) the target to the plot if needed

        Parameters
        ----------
        all_pn: PenaltyNodeList
            The penalty node elements

        """

        if self.target_to_plot is not None:
            all_pn.ocp.add_plot(
                self.target_plot_name,
                lambda x, u, p: self.target_to_plot,
                color="tab:red",
                linestyle=".-",
                plot_type=PlotType.STEP,
                phase=all_pn.nlp.phase_idx,
                axes_idx=Mapping(self.rows),
            )

    def add_or_replace_to_penalty_pool(self, ocp, nlp):
        """
        Doing some configuration on the penalty and add it to the list of penalty

        Parameters
        ----------
        ocp: OptimalControlProgram
            A reference to the ocp
        nlp: NonLinearProgram
            A reference to the current phase of the ocp
        """
        if not self.name:
            if self.type.name == "CUSTOM":
                self.name = self.custom_function.__name__
            else:
                self.name = self.type.name

        penalty_type = self.type.get_type()
        if self.node == Node.TRANSITION:
            all_pn = []

            # Make sure the penalty behave like a PhaseTransition, even though it may be an Objective or Constraint
            self.node = Node.END
            self.node_idx = [0]
            self.transition = True
            self.dt = 1
            self.phase_pre_idx = nlp.phase_idx
            self.phase_post_idx = (nlp.phase_idx + 1) % ocp.n_phases

            all_pn.append(self._get_penalty_node_list(ocp, nlp))
            all_pn[0].u = [nlp.U[-1]]  # Make an exception to the fact that U is not available for the last node

            nlp = ocp.nlp[(nlp.phase_idx + 1) % ocp.n_phases]
            self.node = Node.START
            all_pn.append(self._get_penalty_node_list(ocp, nlp))

            self.node = Node.TRANSITION

            penalty_type.validate_penalty_time_index(self, all_pn[0])
            penalty_type.validate_penalty_time_index(self, all_pn[1])
            self.clear_penalty(ocp, all_pn[0].nlp)

        else:
            all_pn = self._get_penalty_node_list(ocp, nlp)
            penalty_type.validate_penalty_time_index(self, all_pn)
            self.clear_penalty(all_pn.ocp, all_pn.nlp)
            self.dt = penalty_type.get_dt(all_pn.nlp)
            self.node_idx = all_pn.t

        penalty_function = self.type.value[0](self, all_pn, **self.params)
        self.set_penalty(penalty_function, all_pn)

    def _add_penalty_to_pool(self, all_pn: PenaltyNodeList):
        """
        Return the penalty pool for the specified penalty (abstract)

        Parameters
        ----------
        all_pn: PenaltyNodeList
            The penalty node elements
        """

        raise RuntimeError("get_dt cannot be called from an abstract class")

    def clear_penalty(self, ocp, nlp):
        """
        Resets a penalty. A negative penalty index creates a new empty penalty (abstract)

        Parameters
        ----------
        ocp: OptimalControlProgram
            A reference to the ocp
        nlp: NonLinearProgram
            A reference to the current phase of the ocp
        """

        raise RuntimeError("_reset_penalty cannot be called from an abstract class")

    def _get_penalty_node_list(self, ocp, nlp) -> PenaltyNodeList:
        """
        Get the actual node (time, X and U) specified in the penalty

        Parameters
        ----------
        ocp: OptimalControlProgram
            A reference to the ocp
        nlp: NonLinearProgram
            A reference to the current phase of the ocp

        Returns
        -------
        The actual node (time, X and U) specified in the penalty
        """

        if not isinstance(self.node, (list, tuple)):
            self.node = (self.node,)

        t = []
        for node in self.node:
            if isinstance(node, int):
                if node < 0 or node > nlp.ns:
                    raise RuntimeError(f"Invalid node, {node} must be between 0 and {nlp.ns}")
                t.append(node)
            elif node == Node.START:
                t.append(0)
            elif node == Node.MID:
                if nlp.ns % 2 == 1:
                    raise (ValueError("Number of shooting points must be even to use MID"))
                t.append(nlp.ns // 2)
            elif node == Node.INTERMEDIATES:
                t.extend(list(i for i in range(1, nlp.ns - 1)))
            elif node == Node.PENULTIMATE:
                if nlp.ns < 2:
                    raise (ValueError("Number of shooting points must be greater than 1"))
                t.append(nlp.ns - 1)
            elif node == Node.END:
                t.append(nlp.ns)
            elif node == Node.ALL_SHOOTING:
                t.extend(range(nlp.ns))
            elif node == Node.ALL:
                t.extend(range(nlp.ns + 1))
            else:
                raise RuntimeError(" is not a valid node")

        x = [nlp.X[idx] for idx in t]
        u = [nlp.U[idx] for idx in t if idx != nlp.ns]
        return PenaltyNodeList(ocp, nlp, t, x, u, nlp.parameters.cx)

max_stars_repo_name: Naassila/bioptim

max_issues_repo_name: Naassila/bioptim

max_forks_repo_name: Naassila/bioptim


text:
import theano
import theano.tensor as T
from theano.ifelse import ifelse
import numpy as np

from neupy.core.properties import (ChoiceProperty, NumberProperty,
                                   WithdrawProperty)
from neupy.algorithms.gd import StepSelectionBuiltIn
from neupy.algorithms.utils import parameter_values, setup_parameter_updates
from neupy.optimizations.wolfe import line_search
from neupy.layers.utils import count_parameters, iter_parameters
from neupy.utils import asfloat
from .base import GradientDescent

__all__ = ('QuasiNewton',)

def bfgs(inverse_hessian, weight_delta, gradient_delta, maxrho=1e4):
    ident_matrix = T.eye(inverse_hessian.shape[0])

    maxrho = asfloat(maxrho)
    rho = asfloat(1.) / gradient_delta.dot(weight_delta)

    rho = ifelse(
        T.isinf(rho),
        maxrho * T.sgn(rho),
        rho,
    )

    param1 = ident_matrix - T.outer(weight_delta, gradient_delta) * rho
    param2 = ident_matrix - T.outer(gradient_delta, weight_delta) * rho
    param3 = rho * T.outer(weight_delta, weight_delta)

    return param1.dot(inverse_hessian).dot(param2) + param3

def dfp(inverse_hessian, weight_delta, gradient_delta, maxnum=1e5):
    maxnum = asfloat(maxnum)
    quasi_dot_gradient = inverse_hessian.dot(gradient_delta)

    param1 = (
        T.outer(weight_delta, weight_delta)
    ) / (
        T.dot(gradient_delta, weight_delta)
    )
    param2_numerator = T.clip(
        T.outer(quasi_dot_gradient, gradient_delta) * inverse_hessian,
        -maxnum, maxnum
    )
    param2_denominator = gradient_delta.dot(quasi_dot_gradient)
    param2 = param2_numerator / param2_denominator

    return inverse_hessian + param1 - param2

def psb(inverse_hessian, weight_delta, gradient_delta, **options):
    gradient_delta_t = gradient_delta.T
    param = weight_delta - inverse_hessian.dot(gradient_delta)

    devider = (1. / T.dot(gradient_delta, gradient_delta))
    param1 = T.outer(param, gradient_delta) + T.outer(gradient_delta, param)
    param2 = (
        T.dot(gradient_delta, param) *
        T.outer(gradient_delta, gradient_delta_t)
    )

    return inverse_hessian + param1 * devider - param2 * devider ** 2

def sr1(inverse_hessian, weight_delta, gradient_delta, epsilon=1e-8):
    epsilon = asfloat(epsilon)
    param = weight_delta - inverse_hessian.dot(gradient_delta)
    denominator = T.dot(param, gradient_delta)

    return ifelse(
        T.lt(
            T.abs_(denominator),
            epsilon * param.norm(L=2) * gradient_delta.norm(L=2)
        ),
        inverse_hessian,
        inverse_hessian + T.outer(param, param) / denominator
    )

class QuasiNewton(StepSelectionBuiltIn, GradientDescent):
    """
    Quasi-Newton algorithm optimization.

    Parameters
    ----------
    update_function : {{'bfgs', 'dfp', 'psb', 'sr1'}}
        Update function. Defaults to ``bfgs``.

    h0_scale : float
        Default Hessian matrix is an identity matrix. The
        ``h0_scale`` parameter scales identity matrix.
        Defaults to ``1``.

    {GradientDescent.connection}

    {GradientDescent.error}

    {GradientDescent.show_epoch}

    {GradientDescent.shuffle_data}

    {GradientDescent.epoch_end_signal}

    {GradientDescent.train_end_signal}

    {GradientDescent.verbose}

    {GradientDescent.addons}

    Attributes
    ----------
    {GradientDescent.Attributes}

    Methods
    -------
    {GradientDescent.Methods}

    Examples
    --------
    >>> import numpy as np
    >>> from neupy import algorithms
    >>>
    >>> x_train = np.array([[1, 2], [3, 4]])
    >>> y_train = np.array([[1], [0]])
    >>>
    >>> qnnet = algorithms.QuasiNewton(
    ...     (2, 3, 1),
    ...     update_function='bfgs'
    ... )
    >>> qnnet.train(x_train, y_train, epochs=10)

    See Also
    --------
    :network:`GradientDescent` : GradientDescent algorithm.
    """
    update_function = ChoiceProperty(
        default='bfgs',
        choices={
            'bfgs': bfgs,
            'dfp': dfp,
            'psb': psb,
            'sr1': sr1,
        }
    )
    h0_scale = NumberProperty(default=1, minval=0)

    step = WithdrawProperty()

    def init_variables(self):
        super(QuasiNewton, self).init_variables()
        n_params = count_parameters(self.connection)
        self.variables.update(
            inv_hessian=theano.shared(
                name='algo:quasi-newton/matrix:inv-hessian',
                value=asfloat(self.h0_scale * np.eye(int(n_params))),
            ),
            prev_params=theano.shared(
                name='algo:quasi-newton/vector:prev-params',
                value=asfloat(np.zeros(n_params)),
            ),
            prev_full_gradient=theano.shared(
                name='algo:quasi-newton/vector:prev-full-gradient',
                value=asfloat(np.zeros(n_params)),
            ),
        )

    def init_train_updates(self):
        network_inputs = self.variables.network_inputs
        network_output = self.variables.network_output
        inv_hessian = self.variables.inv_hessian
        prev_params = self.variables.prev_params
        prev_full_gradient = self.variables.prev_full_gradient

        params = parameter_values(self.connection)
        param_vector = T.concatenate([param.flatten() for param in params])

        gradients = T.grad(self.variables.error_func, wrt=params)
        full_gradient = T.concatenate([grad.flatten() for grad in gradients])

        new_inv_hessian = ifelse(
            T.eq(self.variables.epoch, 1),
            inv_hessian,
            self.update_function(inv_hessian,
                                 param_vector - prev_params,
                                 full_gradient - prev_full_gradient)
        )
        param_delta = -new_inv_hessian.dot(full_gradient)
        layers_and_parameters = list(iter_parameters(self.layers))

        def prediction(step):
            updated_params = param_vector + step * param_delta

            # This trick allow us to replace shared variables
            # with theano variables and get output from the network
            start_pos = 0
            for layer, attrname, param in layers_and_parameters:
                end_pos = start_pos + param.size
                updated_param_value = T.reshape(
                    updated_params[start_pos:end_pos],
                    param.shape
                )
                setattr(layer, attrname, updated_param_value)
                start_pos = end_pos

            output = self.connection.output(*network_inputs)

            # Restore previous parameters
            for layer, attrname, param in layers_and_parameters:
                setattr(layer, attrname, param)

            return output

        def phi(step):
            return self.error(network_output, prediction(step))

        def derphi(step):
            error_func = self.error(network_output, prediction(step))
            return T.grad(error_func, wrt=step)

        step = asfloat(line_search(phi, derphi))
        updated_params = param_vector + step * param_delta
        updates = setup_parameter_updates(params, updated_params)

        updates.extend([
            (inv_hessian, new_inv_hessian),
            (prev_params, param_vector),
            (prev_full_gradient, full_gradient),
        ])

        return updates

max_stars_repo_name: vishalbelsare/neupy

max_issues_repo_name: vishalbelsare/neupy

max_forks_repo_name: vishalbelsare/neupy

reason: import numpy,import theano,from theano


text:
import numpy as np
from numpy.testing import assert_allclose
from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score

from pygbm.binning import BinMapper
from pygbm.grower import TreeGrower

def test_boston_dataset():
    boston = load_boston()
    X_train, X_test, y_train, y_test = train_test_split(
        boston.data, boston.target, random_state=42)

    mapper = BinMapper(random_state=42)
    X_train_binned = mapper.fit_transform(X_train)
    X_test_binned = mapper.transform(X_test)

    gradients = y_train.astype(np.float32)
    hessians = np.ones(1, dtype=np.float32)

    grower = TreeGrower(X_train_binned, gradients, hessians, max_leaf_nodes=31)
    grower.grow()
    predictor = grower.make_predictor(bin_thresholds=mapper.bin_thresholds_)

    assert r2_score(y_train, predictor.predict_binned(X_train_binned)) > 0.9
    assert r2_score(y_test, predictor.predict_binned(X_test_binned)) > 0.65

    assert_allclose(predictor.predict(X_train),
                    predictor.predict_binned(X_train_binned))

    assert_allclose(predictor.predict(X_test),
                    predictor.predict_binned(X_test_binned))

    assert r2_score(y_train, predictor.predict(X_train)) > 0.9
    assert r2_score(y_test, predictor.predict(X_test)) > 0.65

reason: import numpy,from numpy


text:
"""Test cases for the coffee environment."""

import numpy as np
import pytest

from predicators.src import utils
from predicators.src.envs.coffee import CoffeeEnv
from predicators.src.structs import Action, GroundAtom

def test_coffee():
    """Tests for CoffeeEnv()."""
    utils.reset_config({
        "env": "coffee",
        "coffee_num_cups_test": [4],  # used to assure 4 cups in custom state
        "video_fps": 10,  # for faster debugging videos
    })
    env = CoffeeEnv()
    for task in env.get_train_tasks():
        for obj in task.init:
            assert len(obj.type.feature_names) == len(task.init[obj])
    for task in env.get_test_tasks():
        for obj in task.init:
            assert len(obj.type.feature_names) == len(task.init[obj])
    assert len(env.predicates) == 12
    assert len(env.goal_predicates) == 1
    pred_name_to_pred = {p.name: p for p in env.predicates}
    CupFilled = pred_name_to_pred["CupFilled"]
    JugInMachine = pred_name_to_pred["JugInMachine"]
    OnTable = pred_name_to_pred["OnTable"]
    NotAboveCup = pred_name_to_pred["NotAboveCup"]
    assert len(env.options) == 6
    option_name_to_option = {o.name: o for o in env.options}
    assert len(env.types) == 4
    type_name_to_type = {t.name: t for t in env.types}
    cup_type = type_name_to_type["cup"]
    jug_type = type_name_to_type["jug"]
    machine_type = type_name_to_type["machine"]
    robot_type = type_name_to_type["robot"]
    assert env.action_space.shape == (6, )
    # Create a custom initial state, with cups positions at the extremes of
    # their possible initial positions.
    state = env.get_test_tasks()[0].init.copy()
    robot, = state.get_objects(robot_type)
    jug, = state.get_objects(jug_type)
    machine, = state.get_objects(machine_type)
    cups = state.get_objects(cup_type)
    assert len(cups) == 4
    # Reposition the cups.
    cup_corners = [
        (env.cup_init_x_lb, env.cup_init_y_lb),
        (env.cup_init_x_lb, env.cup_init_y_ub),
        (env.cup_init_x_ub, env.cup_init_y_ub),
        (env.cup_init_x_ub, env.cup_init_y_lb),
    ]
    for cup, (x, y) in zip(cups, cup_corners):
        state.set(cup, "x", x)
        state.set(cup, "y", y)
    # Reposition the jug just so we know exactly where it is.
    state.set(jug, "x", env.jug_init_x_ub)
    state.set(jug, "y", env.jug_init_y_ub)
    state.set(jug, "rot", 0.0)
    env.render_state(state, task)

    ## Test simulate ##

    # Helper function to compute a sequence of actions that moves the robot
    # from an initial position to a target position in a straight line.
    def _get_position_action_arrs(init_x, init_y, init_z, final_x, final_y,
                                  final_z):
        action_arrs = []
        current_pos = np.array([init_x, init_y, init_z])
        delta = np.subtract((final_x, final_y, final_z), current_pos)
        pos_norm = float(np.linalg.norm(delta))
        if pos_norm > env.max_position_vel:
            delta = env.max_position_vel * (delta / pos_norm)
            num_max_steps = int(np.floor(pos_norm / env.max_position_vel))
            for _ in range(num_max_steps):
                dx, dy, dz = delta / env.max_position_vel
                action_arrs.append(
                    np.array([dx, dy, dz, 0.0, 0.0, 0.0], dtype=np.float32))
                current_pos = current_pos + delta
            delta = np.subtract((final_x, final_y, final_z), current_pos)
            pos_norm = float(np.linalg.norm(delta))
        if pos_norm > 0:
            delta = delta / env.max_position_vel
            dx, dy, dz = delta
            action_arrs.append(
                np.array([dx, dy, dz, 0.0, 0.0, 0.0], dtype=np.float32))
        return action_arrs

    # Test twisting the jug.
    target_x = state.get(jug, "x")
    target_y = state.get(jug, "y")
    target_z = env.jug_height
    action_arrs = _get_position_action_arrs(state.get(robot, "x"),
                                            state.get(robot, "y"),
                                            state.get(robot, "z"), target_x,
                                            target_y, target_z)
    num_twists = 2
    twist_act_arr = np.array([0.0, 0.0, 0.0, 0.0, 1.0, 0.0], dtype=np.float32)
    action_arrs.extend([twist_act_arr for _ in range(num_twists)])
    policy = utils.action_arrs_to_policy(action_arrs)
    traj = utils.run_policy_with_simulator(policy,
                                           env.simulate,
                                           state,
                                           lambda _: False,
                                           max_num_steps=len(action_arrs))
    twist_amt = num_twists * env.max_angular_vel
    assert abs(traj.states[-1].get(jug, "rot") - twist_amt) < 1e-6
    s = traj.states[-1]

    # The jug is too twisted now, so picking it up should fail.
    target_x, target_y, target_z = env._get_jug_handle_grasp(s, jug)  # pylint: disable=protected-access
    move_action_arrs = _get_position_action_arrs(s.get(robot, "x"),
                                                 s.get(robot, "y"),
                                                 s.get(robot, "z"), target_x,
                                                 target_y, target_z)
    action_arrs.extend(move_action_arrs)
    pick_act_arr = np.array([0.0, 0.0, 0.0, 0.0, 0.0, -1.0], dtype=np.float32)
    action_arrs.append(pick_act_arr)

    policy = utils.action_arrs_to_policy(action_arrs)
    traj = utils.run_policy_with_simulator(policy,
                                           env.simulate,
                                           state,
                                           lambda _: False,
                                           max_num_steps=len(action_arrs))
    assert traj.states[-1].get(jug, "is_held") < 0.5

    # Test picking up the jug.
    target_x, target_y, target_z = env._get_jug_handle_grasp(state, jug)  # pylint: disable=protected-access
    action_arrs = _get_position_action_arrs(state.get(robot, "x"),
                                            state.get(robot, "y"),
                                            state.get(robot, "z"), target_x,
                                            target_y, target_z)
    pick_act_arr = np.array([0.0, 0.0, 0.0, 0.0, 0.0, -1.0], dtype=np.float32)
    action_arrs.append(pick_act_arr)

    policy = utils.action_arrs_to_policy(action_arrs)
    traj = utils.run_policy_with_simulator(policy,
                                           env.simulate,
                                           state,
                                           lambda _: False,
                                           max_num_steps=len(action_arrs))
    assert traj.states[-2].get(jug, "is_held") < 0.5
    assert traj.states[-1].get(jug, "is_held") > 0.5
    assert GroundAtom(OnTable, [jug]).holds(traj.states[-2])
    assert not GroundAtom(OnTable, [jug]).holds(traj.states[-1])
    s = traj.states[-1]

    # Test moving and placing the jug at the machine.
    # Offset based on the grasp.
    target_x = env.dispense_area_x - (s.get(jug, "x") - s.get(robot, "x"))
    target_y = env.dispense_area_y - (s.get(jug, "y") - s.get(robot, "y"))
    target_z = s.get(robot, "z")
    move_jug_act_arrs = _get_position_action_arrs(s.get(robot, "x"),
                                                  s.get(robot, "y"),
                                                  s.get(robot, "z"), target_x,
                                                  target_y, target_z)
    action_arrs.extend(move_jug_act_arrs)
    # Drop the jug.
    place_act_arr = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 1.0], dtype=np.float32)
    action_arrs.append(place_act_arr)

    policy = utils.action_arrs_to_policy(action_arrs)
    traj = utils.run_policy_with_simulator(policy,
                                           env.simulate,
                                           state,
                                           lambda _: False,
                                           max_num_steps=len(action_arrs))
    assert traj.states[-2].get(jug, "is_held") > 0.5
    assert traj.states[-1].get(jug, "is_held") < 0.5
    s = traj.states[-1]

    # Test pressing the machine button.
    # First move to above the button.
    move_to_above_button_act_arrs = _get_position_action_arrs(
        s.get(robot, "x"), s.get(robot, "y"), s.get(robot, "z"), env.button_x,
        env.button_y + 1.0, env.button_z)
    action_arrs.extend(move_to_above_button_act_arrs)
    # Move forward to press the button.
    move_to_press_button_act_arrs = _get_position_action_arrs(
        env.button_x,
        env.button_y + 1.0,
        env.button_z,
        env.button_x,
        env.button_y,
        env.button_z,
    )
    action_arrs.extend(move_to_press_button_act_arrs)

    policy = utils.action_arrs_to_policy(action_arrs)
    traj = utils.run_policy_with_simulator(policy,
                                           env.simulate,
                                           state,
                                           lambda _: False,
                                           max_num_steps=len(action_arrs))
    assert traj.states[-2].get(machine, "is_on") < 0.5
    assert traj.states[-1].get(machine, "is_on") > 0.5
    # The jug should also now be filled.
    assert traj.states[-2].get(jug, "is_filled") < 0.5
    assert traj.states[-1].get(jug, "is_filled") > 0.5
    s = traj.states[-1]

    # Test picking up the filled jug.
    target_x, target_y, target_z = env._get_jug_handle_grasp(s, jug)  # pylint: disable=protected-access
    move_to_pick_act_arrs = _get_position_action_arrs(s.get(robot, "x"),
                                                      s.get(robot, "y"),
                                                      s.get(robot,
                                                            "z"), target_x,
                                                      target_y, target_z)
    action_arrs.extend(move_to_pick_act_arrs)
    pick_act_arr = np.array([0.0, 0.0, 0.0, 0.0, 0.0, -1.0], dtype=np.float32)
    action_arrs.append(pick_act_arr)

    policy = utils.action_arrs_to_policy(action_arrs)
    traj = utils.run_policy_with_simulator(policy,
                                           env.simulate,
                                           state,
                                           lambda _: False,
                                           max_num_steps=len(action_arrs))
    assert traj.states[-2].get(jug, "is_held") < 0.5
    assert traj.states[-1].get(jug, "is_held") > 0.5
    assert GroundAtom(NotAboveCup, [robot, jug]).holds(traj.states[-1])
    s = traj.states[-1]

    # Check that an EnvironmentFailure is raised when pouring into nothing.
    pour_act_lst = [0.0, 0.0, 0.0, 1.0, 0.0, 0.0]
    pour_act_arr = np.array(pour_act_lst, dtype=np.float32)
    with pytest.raises(utils.EnvironmentFailure) as e:
        env.simulate(s, Action(pour_act_arr))
    assert "Spilled" in str(e)

    # Test pouring in each of the cups.
    for cup in cups:
        jug_target_x, jug_target_y, jug_target_z = env._get_pour_position(  # pylint: disable=protected-access
            s, cup)
        target_x = jug_target_x - (s.get(jug, "x") - s.get(robot, "x"))
        target_y = jug_target_y - (s.get(jug, "y") - s.get(robot, "y"))
        target_z = jug_target_z + env.jug_handle_height
        move_to_pour_act_arrs = _get_position_action_arrs(
            s.get(robot, "x"), s.get(robot, "y"), s.get(robot, "z"), target_x,
            target_y, target_z)
        action_arrs.extend(move_to_pour_act_arrs)
        target_liquid = state.get(cup, "target_liquid")
        num_pour_steps = int(np.ceil(target_liquid / env.pour_velocity))
        # Start pouring.
        action_arrs.append(pour_act_arr)
        # Keep pouring.
        action_arrs.extend([pour_act_arr for _ in range(num_pour_steps - 1)])
        # Stop pouring.
        action_arrs.append(-1 * pour_act_arr)

        policy = utils.action_arrs_to_policy(action_arrs)
        traj = utils.run_policy_with_simulator(policy,
                                               env.simulate,
                                               state,
                                               lambda _: False,
                                               max_num_steps=len(action_arrs))
        assert not GroundAtom(CupFilled, [cup]).holds(traj.states[-3])
        assert GroundAtom(CupFilled, [cup]).holds(traj.states[-1])
        assert not GroundAtom(NotAboveCup, [robot, jug]).holds(traj.states[-1])
        s = traj.states[-1]
        # Render a state where we are in the process of pouring.
        env.render_state(traj.states[-2], task)

    # Check that an EnvironmentFailure is raised when overfilling a cup.
    with pytest.raises(utils.EnvironmentFailure) as e:
        overfill_state = s.copy()
        for _ in range(10):
            overfill_state = env.simulate(overfill_state, Action(pour_act_arr))
    assert "Overfilled cup" in str(e)

    # Uncomment for debugging.
    # policy = utils.action_arrs_to_policy(action_arrs)
    # monitor = utils.SimulateVideoMonitor(task, env.render_state)
    # traj = utils.run_policy_with_simulator(policy,
    #                                        env.simulate,
    #                                        state,
    #                                        lambda _: False,
    #                                        max_num_steps=len(action_arrs),
    #                                        monitor=monitor)
    # video = monitor.get_video()
    # outfile = "hardcoded_actions_coffee.mp4"
    # utils.save_video(outfile, video)

    ## Test options ##

    # Test MoveToTwistJug and TwistJug.
    MoveToTwistJug = option_name_to_option["MoveToTwistJug"]
    TwistJug = option_name_to_option["TwistJug"]
    option1 = MoveToTwistJug.ground([robot, jug], [])
    option2 = TwistJug.ground([robot, jug], np.array([1.0], dtype=np.float32))
    option_plan = [option1, option2]
    policy = utils.option_plan_to_policy(option_plan)
    traj = utils.run_policy_with_simulator(
        policy,
        env.simulate,
        state,
        lambda _: False,
        max_num_steps=1000,
        exceptions_to_break_on={utils.OptionExecutionFailure})
    expected_rot = env.jug_init_rot_ub
    assert abs(traj.states[-1].get(jug, "rot") - expected_rot) < 1e-6

    # Test PickJug.
    PickJug = option_name_to_option["PickJug"]
    option = PickJug.ground([robot, jug], [])
    option_plan = [option]

    policy = utils.option_plan_to_policy(option_plan)
    traj = utils.run_policy_with_simulator(
        policy,
        env.simulate,
        state,
        lambda _: False,
        max_num_steps=1000,
        exceptions_to_break_on={utils.OptionExecutionFailure})
    assert traj.states[-2].get(jug, "is_held") < 0.5
    assert traj.states[-1].get(jug, "is_held") > 0.5

    # Test PlaceJugInMachine.
    PlaceJugInMachine = option_name_to_option["PlaceJugInMachine"]
    option = PlaceJugInMachine.ground([robot, jug, machine], [])
    option_plan.append(option)

    policy = utils.option_plan_to_policy(option_plan)
    traj = utils.run_policy_with_simulator(
        policy,
        env.simulate,
        state,
        lambda _: False,
        max_num_steps=1000,
        exceptions_to_break_on={utils.OptionExecutionFailure})
    assert traj.states[-2].get(jug, "is_held") > 0.5
    assert traj.states[-1].get(jug, "is_held") < 0.5
    assert GroundAtom(JugInMachine, [jug, machine]).holds(traj.states[-1])

    # Test TurnMachineOn.
    TurnMachineOn = option_name_to_option["TurnMachineOn"]
    option = TurnMachineOn.ground([robot, machine], [])
    option_plan.append(option)

    policy = utils.option_plan_to_policy(option_plan)
    traj = utils.run_policy_with_simulator(
        policy,
        env.simulate,
        state,
        lambda _: False,
        max_num_steps=1000,
        exceptions_to_break_on={utils.OptionExecutionFailure})
    assert traj.states[-2].get(machine, "is_on") < 0.5
    assert traj.states[-1].get(machine, "is_on") > 0.5

    # Test PickJug from the dispense area.
    option = PickJug.ground([robot, jug], [])
    option_plan.append(option)

    policy = utils.option_plan_to_policy(option_plan)
    traj = utils.run_policy_with_simulator(
        policy,
        env.simulate,
        state,
        lambda _: False,
        max_num_steps=1000,
        exceptions_to_break_on={utils.OptionExecutionFailure})
    assert traj.states[-2].get(jug, "is_held") < 0.5
    assert traj.states[-1].get(jug, "is_held") > 0.5

    # Test Pour into each of the cups.
    Pour = option_name_to_option["Pour"]
    for cup in cups:
        option = Pour.ground([robot, jug, cup], [])
        option_plan.append(option)
        policy = utils.option_plan_to_policy(option_plan)
        traj = utils.run_policy_with_simulator(
            policy,
            env.simulate,
            state,
            lambda _: False,
            max_num_steps=1000,
            exceptions_to_break_on={utils.OptionExecutionFailure})
        assert not GroundAtom(CupFilled, [cup]).holds(traj.states[-3])
        assert GroundAtom(CupFilled, [cup]).holds(traj.states[-1])
        s = traj.states[-1]

    # Uncomment for debugging.
    # policy = utils.option_plan_to_policy(option_plan)
    # monitor = utils.SimulateVideoMonitor(task, env.render_state)
    # traj = utils.run_policy_with_simulator(
    #     policy,
    #     env.simulate,
    #     state,
    #     lambda _: False,
    #     max_num_steps=1000,
    #     exceptions_to_break_on={utils.OptionExecutionFailure},
    #     monitor=monitor)
    # video = monitor.get_video()
    # outfile = "hardcoded_options_coffee.mp4"
    # utils.save_video(outfile, video)


text:
import os
import tempfile
import pickle

from sympycore.core import Expr
from sympycore.utils import SYMBOL

class MyExpr(Expr):
    # see comment in sympycore/core.py for pickling pure Expr
    # instances.
    pass

def test_pickle():

    fn = tempfile.mktemp()
    obj = MyExpr(SYMBOL, 'x')

    f = open(fn, 'wb')
    pickle.dump(obj, f)
    f.close()

    f = open(fn, 'rb')
    obj2 = pickle.load(f)
    f.close()
    os.remove(fn)

    assert obj.head == obj2.head,`obj.head, obj2.head`
    assert obj.head is obj2.head,`obj.head, obj2.head, id(obj.head), id(obj2.head)`
    assert obj.data == obj2.data,`obj.data, obj2.data`
    assert obj.pair == obj2.pair,`obj.pair, obj2.pair`
    assert obj==obj2,`obj,obj2`

max_stars_repo_name: radovankavicky/pymaclab

reason: from sympy


text:
# main imports
import numpy as np
import pandas as pd
import sys, os, argparse
import math

# image processing
from PIL import Image
from ipfml import utils
from ipfml.processing import transform, segmentation

import matplotlib.pyplot as plt

# modules and config imports
sys.path.insert(0, '') # trick to enable import of main folder module

import custom_config as cfg

# utils information
zone_width, zone_height = (200, 200)
scene_width, scene_height = (800, 800)
nb_x_parts = math.floor(scene_width / zone_width)
zones_indices  = cfg.zones_indices

def extract_thresholds_from_file(filename):
    # extract thresholds
    thresholds_dict = {}
    with open(filename) as f:
        thresholds_line = f.readlines()

        for line in thresholds_line:
            data = line.split(';')
            del data[-1] # remove unused last element `\n`
            current_scene = data[0]
            thresholds_scene = data[1:]

            thresholds_dict[current_scene] = [ int(threshold) for threshold in  thresholds_scene ]

    return thresholds_dict

def reconstruct_image(images_paths, thresholds, output, step):

    # to avoid issue
    images_paths = sorted(images_paths)

    # 1. compute zone start index
    zones_coordinates = []
    for zone_index in cfg.zones_indices:
        x_zone = (zone_index % nb_x_parts) * zone_width
        y_zone = (math.floor(zone_index / nb_x_parts)) * zone_height

        zones_coordinates.append((x_zone, y_zone))

    images_zones = []
    line_images_zones = []

    # 2. get image using threshold by zone
    for id, t in enumerate(thresholds):

        t_index = int(t / step) - 1 # -1 because index start at 0 in image_paths list
        image_path = images_paths[t_index]
        selected_image = Image.open(image_path)

        x_zone, y_zone = zones_coordinates[id]
        zone_image = np.array(selected_image)[y_zone:y_zone+zone_height, x_zone:x_zone+zone_width]
        line_images_zones.append(zone_image)

        if int(id + 1) % int(scene_width / zone_width) == 0:
            images_zones.append(np.concatenate(line_images_zones, axis=1))
            line_images_zones = []

    # 3. reconstructed the image using these zones
    reconstructed_image = np.concatenate(images_zones, axis=0)

    # 4. Save the image with generated name based on scene
    reconstructed_pil_img = Image.fromarray(reconstructed_image)

    reconstructed_pil_img.save(output)

def main():

    parser = argparse.ArgumentParser(description="Read and display simulation from multiple models")

    parser.add_argument('--folder', type=str, help='dataset path with scenes images files', required=True)
    parser.add_argument('--data', type=str, help='folder with obtained thresholds model data to compare', required=True)
    parser.add_argument('--thresholds', type=str, help='file with specific thresholds (using only scene from this file', required=True)
    parser.add_argument('--labels', type=str, help='labels list to display in figure', required=True)
    parser.add_argument('--common', type=int, help='common only with human', default=0, choices=[0, 1])
    parser.add_argument('--output', type=str, help='output folder with figures for each scene', required=True)

    args = parser.parse_args()

    p_folder = args.folder
    p_data = args.data
    p_thresholds = args.thresholds
    p_labels = args.labels.split(',')
    p_common = bool(args.common)
    p_output = args.output

    if not os.path.exists(p_output):
        os.makedirs(p_output)

    # get y lim
    y_lim = 0, 10000

    # 1. retrieve human_thresholds
    human_thresholds = extract_thresholds_from_file(p_thresholds)

    # 3. extract models thresholds
    models_thresholds = {}

    for i, simu in enumerate(sorted(os.listdir(p_data))):
        method_label = p_labels[i]
        simu_path = os.path.join(p_data, simu)
        models_thresholds[method_label] = extract_thresholds_from_file(simu_path)

    # 4. extract information for each expected scene

    for method_name in p_labels + ['human']:
        method_output = os.path.join(p_output, method_name)

        if not os.path.exists(method_output):
            os.makedirs(method_output)

    for scene in sorted(os.listdir(p_folder)):

        # get all estimated thresholds for this scene
        estimated_thresholds = {}
        scene_path = os.path.join(p_folder, scene)
        images_path = sorted([ os.path.join(scene_path, img) for img in os.listdir(scene_path) ])

        # get only necessary thresholds
        if scene in human_thresholds:
            current_human_thresholds = human_thresholds[scene]
            estimated_thresholds['human'] = current_human_thresholds

        for method_name in p_labels:
            estimated_thresholds[method_name] = models_thresholds[method_name][scene]

        if p_common:

            # do it only if human is also available
            if 'human' in estimated_thresholds:
                # create figure with these information
                for name, thresholds in estimated_thresholds.items():
                    output_image = os.path.join(p_output, name, scene + '.png')
                    reconstruct_image(images_path, thresholds, output_image, step=20)
                    print('Image reconstructed save at {0}'.format(output_image))
        else:
            # create figure with these information
            for name, thresholds in estimated_thresholds.items():
                output_image = os.path.join(p_output, name, scene + '.png')
                reconstruct_image(images_path, thresholds, output_image, step=20)
                print('Image reconstructed save at {0}'.format(output_image))

if __name__== "__main__":
    main()


text:
import numpy as np
import cv2
from keras.models import Model
from keras.layers import Input, Conv2D, Reshape, Lambda
from keras.applications.mobilenet import MobileNet
from box_utils import decode_netout, compute_overlap, compute_ap

class MobileNetFeatureExtractor:
    """
    A lightweight CNN for object detection, developed to run on resource constrained devices -
    MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications
    Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam
    April 2017 [arXiv:1704.04861v1 [cs.CV]]
    https://arxiv.org/abs/1704.04861

    Implementation as a feature extractor adapted from:
    https://github.com/experiencor/keras-yolo2/blob/master/backend.py
    """

    def __init__(self, input_size):
        input_image = Input(shape=(input_size, input_size, 3))

        mobilenet = MobileNet(input_shape=(224,224,3), include_top=False)
        mobilenet.load_weights("data/mobilenet_backend.h5")
        x = mobilenet(input_image)

        self.feature_extractor = Model(input_image, x)

    def normalize(self, image):
        image = image / 255.
        image = image - 0.5
        image = image * 2.
        return image

    def get_output_shape(self):
        return self.feature_extractor.get_output_shape_at(-1)[1:3]

    def extract(self, input_image):
        return self.feature_extractor(input_image)

class ObjectDetection(object):
    def __init__(self, backend,
                       input_size,
                       labels,
                       max_box_per_image,
                       anchors):

        self.input_size = input_size
        self.labels   = list(labels)
        self.nb_class = len(self.labels)
        self.nb_box   = len(anchors)//2
        self.class_wt = np.ones(self.nb_class, dtype='float32')
        self.anchors  = anchors
        self.max_box_per_image = max_box_per_image

        # =======================================
        # Two inputs:
        #   1. image
        #   2. bounding box (training phase only)
        # =======================================

        input_image     = Input(shape=(self.input_size, self.input_size, 3))
        self.true_boxes = Input(shape=(1, 1, 1, max_box_per_image , 4))

        # ========================
        # Feature extraction layer
        # ========================

        self.feature_extractor = MobileNetFeatureExtractor(self.input_size)

        self.grid_h, self.grid_w = self.feature_extractor.get_output_shape()

        print("Output from feature extractor has shape:", self.grid_h, ",", self.grid_w)

        features = self.feature_extractor.extract(input_image)

        # ======================
        # Object detection layer
        # ======================

        MULT = 3 # increase the number of Convolutional filters, if the object classes prove harder to detect

        # tensor shape: (1, 7, 7, 1024)
        # intepretation: 7*7 grid of feature vectors describing each grid segment
        output = Conv2D(MULT * self.nb_box * (4 + 1 + self.nb_class), # 105 filters if MULT == 3
                        (1,1), strides=(1,1),
                        padding='same',
                        name='DetectionLayer',
                        kernel_initializer='lecun_normal')(features)

        # tensor shape: (1, 7, 7, 105)
        # intepretation: 7*7 grid of bounding box predictions for each grid segment
        output = Reshape((self.grid_h, self.grid_w, self.nb_box, MULT * (4 + 1 + self.nb_class)))(output)

        # tensor shape: (1, 7, 7, 5, 21)
        output = Lambda(lambda args: args[0])([output, self.true_boxes]) # dummy layer (workaround for Keras "Exception: Layer is not connected")

        self.model = Model([input_image, self.true_boxes], output)

        # print a summary of the whole model
        self.model.summary()

    def load_weights(self, weight_path):
        self.model.load_weights(weight_path)

    def save_weights(self, weight_path):
        self.model.save_weights(weight_path)

    def save(self, path):
        self.model.save(path)

    def to_json(self, path):
        return self.model.to_json()

    def predict(self, image):
        image_h, image_w, _ = image.shape
        image = self.feature_extractor.normalize(image)

        input_image = image[:,:,::-1]
        input_image = np.expand_dims(input_image, 0)
        dummy_array = np.zeros((1,1,1,1,self.max_box_per_image,4))

        netout = self.model.predict([input_image, dummy_array])[0]
        boxes  = decode_netout(netout, self.anchors, self.nb_class)

        return boxes

max_stars_repo_name: OpenSUTD/SmartBin

max_issues_repo_name: OpenSUTD/SmartBin

max_forks_repo_name: OpenSUTD/SmartBin