text:
# -*- coding: utf-8 -*-

# **Exercise: Animate the timeseries plot**

# In[1]:

# Imports
from threading import Thread
import datetime
import logging
import time

import numpy as np
import netCDF4
import pandas as pd

from bokeh.plotting import vplot, hplot, cursession, curdoc, output_server, show
from bokeh.models.widgets import Button, Icon

from viz import climate_map, timeseries, legend, get_slice

# In[2]:

# Data
data = netCDF4.Dataset('data/Land_and_Ocean_LatLong1.nc')
t = data.variables['temperature']
df = pd.read_csv('data/Land_Ocean_Monthly_Anomaly_Average.csv', parse_dates=[0])

# In[3]:

# Output option
output_server("climate")

# In[4]:

from bokeh.plotting import figure

# Data
year = 1850
month = 1

years = [str(x) for x in np.arange(1850, 2015, 1)]

months = [str(x) for x in np.arange(1, 13, 1)]
months_str = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']

month_str = months_str[month-1]

date = datetime.date(year, month, 1)
df['moving_average'] = pd.rolling_mean(df['anomaly'], 12)
df = df.fillna(0)

# New text Plot
title = figure(width=1200, height=100, x_range=(0, 1200), y_range=(0, 100), toolbar_location=None,
        x_axis_type=None, y_axis_type=None, outline_line_color="#FFFFFF", tools="", min_border=0)

title.text(x=500, y=5, text=[month_str], text_font_size='36pt', text_color='black', name="month", text_font="Georgia")
title.text(x=350, y=5, text=[str(year)], text_font_size='36pt', text_color='black', name="year",text_font="Georgia")

# In[5]:

# Plots
climate_map = climate_map()
timeseries = timeseries()
legend = legend()

# ADD WIDGETS
play = True

def play_handler():
    print("button_handler: start click")
    global play
    play = True

def stop_handler():
    print("button_handler: stop click")
    global play
    play = False

button_start = Button(label="Start", type="success")
button_start.on_click(play_handler)

button_stop = Button(label="Stop", type="danger")
button_stop.on_click(stop_handler)

controls = hplot(button_start, button_stop)

# In[6]:

# New circle in timeseries plot
timeseries.circle(x=[date], y=[df[df.datetime == date].moving_average], size=8, name="circle")

# In[7]:

# Create layout
map_legend = hplot(climate_map, legend)
layout = vplot(controls, title, map_legend, timeseries)

# In[8]:

# Show
show(layout)

# In[9]:

# Select data source for climate_map and month and year
renderer = climate_map.select(dict(name="image"))
ds = renderer[0].data_source

month_renderer = title.select(dict(name="month"))
month_ds = month_renderer[0].data_source

year_renderer = title.select(dict(name="year"))
year_ds = year_renderer[0].data_source

# Select data source for timeseries data
timeseries_renderer = timeseries.select(dict(name="circle"))
timeseries_ds = timeseries_renderer[0].data_source

def should_play():
    """
    Return true if we should play animation, otherwise block
    """
    global play
    while True:
        if play:
            return True
        else:
            time.sleep(0.05)

def background_thread(ds, year_ds, month_ds, timeseries_ds):
    """Plot animation, update data if play is True, otherwise stop"""
    try:
        while True:
            for year_index in np.arange(2000, 2015, 1):
                    year_ds.data["text"] = [str(year_index)]
                    for month_index in np.arange(1, 13, 1):
                        if should_play():
                            month_ds.data["text"] = [months_str[month_index-1]]
                            image = get_slice(t, year_index, month_index)
                            date = datetime.date(year_index, month_index, 1)
                            timeseries_ds.data["x"] = [date]
                            timeseries_ds.data["y"] = [df[df.datetime == date].moving_average]
                            ds.data["image"] = [image]
                            cursession().store_objects(ds, year_ds, month_ds, timeseries_ds)
                            time.sleep(0.5)
                        time.sleep(0.5)
    except:
        logger.exception("An error occurred")
        raise

# spin up a background thread
Thread(target=background_thread, args=(ds, year_ds, month_ds, timeseries_ds)).start()

# endlessly poll
cursession().poll_document(curdoc(), 0.04)


text:
# vim: fdm=marker
'''
author:     Fabio Zanini
date:       23/10/13
content:    Description module for HIV samples from patients.

            Note: it is possible to read the Excel XLSX table with xlrd. It also
            works ok, but it's painful and error-prone code, and it assumes that
            table is up-to-date. Better write down the machine-relevant info
            manually down here (until otherwise specified).
'''
# Modules
import numpy as np
import pandas as pd

from hivwholeseq.sequencing.filenames import table_filename

# Globals
_sequencing_runs = None

# Classes
class SampleSeq(pd.Series):
    '''A sequenced sample (if something has been sequenced twice, they are separate)'''

    _sequencing_run = None
    _sample_pat = None

    def __init__(self, *args, **kwargs):
        '''Initialize a sequenced sample'''
        super(SampleSeq, self).__init__(*args, **kwargs)

        from hivwholeseq.sequencing.filenames import get_seqrun_foldername
        from hivwholeseq.sequencing.adapter_info import foldername_adapter
        seq_run = self.loc['seq run']
        adaID = self.loc['adapter']
        self['folder'] = str(get_seqrun_foldername(seq_run)+foldername_adapter(adaID))
        self['seqrun_folder'] = str(get_seqrun_foldername(seq_run))

    @property
    def _constructor(self):
        return SampleSeq

    @property
    def sequencing_run(self):
        '''The sequencing run object of this sample'''
        if self._sequencing_run is None:
            self._sequencing_run = load_sequencing_run(self.loc['seq run'])
        return self._sequencing_run

    @property
    def sample_pat(self):
        '''Patient sample of this sequencing sample'''
        if self._sample_pat is None:
            from hivwholeseq.patients.patients import load_samples_sequenced as lssp
            from hivwholeseq.patients.patients import SamplePat
            self._sample_pat = SamplePat(lssp(include_wrong=True).loc[self['patient sample']])
        return self._sample_pat

    @property
    def patientname(self):
        '''Name of the patient this sample belongs to'''
        return self.sample_pat.patient

    @property
    def regions_complete(self):
        '''Get the complete regions, e.g. F5ao'''
        if self.PCR == 1:
            PCR_suffix = 'o'
        elif self.PCR == 2:
            PCR_suffix = 'i'
        else:
            PCR_suffix = ''

        regions = self.regions.split(' ')
        if len(regions):
            return ['F'+fr+PCR_suffix for fr in regions]
        else:
            return []

    @property
    def regions_generic(self):
        '''Get the complete regions, e.g. F5ao'''
        regions = self.regions.split(' ')
        if len(regions):
            return ['F'+fr[0] for fr in regions]
        else:
            return []

    def convert_region(self, fragment):
        '''Get the complete region of a generic or vice versa'''
        if fragment in self.regions_generic:
            return self.regions_complete[self.regions_generic.index(fragment)]
        elif fragment in self.regions_complete:
            return self.regions_generic[self.regions_complete.index(fragment)]
        else:
            raise IndexError('Region not in complete nor in generic regions!')

    def get_read_filenames(self, **kwargs):
        '''Get the filenames of the demultiplexed reads'''
        from hivwholeseq.sequencing.filenames import get_read_filenames as gfn
        return gfn(self.folder, **kwargs)

    def get_reference_premap_filename(self, **kwargs):
        '''Get the filename of the premapping refernce'''
        from hivwholeseq.sequencing.filenames import get_reference_premap_filename as gfn
        return gfn(self.seqrun_folder, self.adapter, **kwargs)

    def get_premapped_filename(self, **kwargs):
        '''Get the filename of the readed premapped to reference'''
        from hivwholeseq.sequencing.filenames import get_premapped_filename as gfn
        return gfn(self.folder, **kwargs)

    def get_divided_filename(self, fragment, **kwargs):
        '''Get the filename of the divided and trimmed reads'''
        from hivwholeseq.sequencing.filenames import get_divided_filename as gfn
        return gfn(self.folder, adaID=None, fragment=fragment, **kwargs)

    def get_consensus_filename(self, fragment, **kwargs):
        '''Get the filename of the consensus'''
        from hivwholeseq.sequencing.filenames import get_consensus_filename as gfn
        from hivwholeseq.sequencing.filenames import get_merged_consensus_filename as gfn2
        if fragment != 'genomewide':
            return gfn(self.folder, adaID=None, fragment=fragment, **kwargs)
        else:
            return gfn2(self.folder, adaID=None, **kwargs)

    def get_mapped_filename(self, fragment, **kwargs):
        '''Get the filename of the mapped reads'''
        from hivwholeseq.sequencing.filenames import get_mapped_filename as gfn
        return gfn(self.folder, adaID=None, fragment=fragment, **kwargs)

    def get_mapped_to_initial_filename(self, fragment, **kwargs):
        '''Get the filename of the reads mapped to patient initial reference'''
        from hivwholeseq.patients.filenames import get_mapped_to_initial_filename as gfn
        pname = self.patientname
        samplename_pat = self['patient sample']
        PCR = int(self.PCR)
        return gfn(pname, samplename_pat, self.name, fragment, PCR=PCR, **kwargs)

    def get_mapped_filtered_filename(self, fragment, **kwargs):
        '''Get the filename of the reads mapped to patient initial reference'''
        from hivwholeseq.patients.filenames import get_mapped_filtered_filename as gfn
        pname = self.patientname
        samplename_pat = self['patient sample']
        PCR = int(self.PCR)
        return gfn(pname, samplename_pat, fragment, PCR=PCR, **kwargs)

    def get_consensus(self, fragment):
        '''Get consensus sequence for mapping'''
        from Bio import SeqIO
        return SeqIO.read(self.get_consensus_filename(fragment), 'fasta')

    def get_allele_counts_filename(self, fragment):
        '''Get the filename with the allele counts'''
        from .filenames import get_allele_counts_filename
        return get_allele_counts_filename(self.seqrun_folder, self.adapter, fragment)

    def get_allele_counts(self, fragment, merge_read_types=False):
        '''Get the allele counts'''
        import numpy as np
        counts = np.load(self.get_allele_counts_filename(fragment))
        if merge_read_types:
            counts = counts.sum(axis=0)
        return counts

    def get_insert_size_distribution(self, fragment, VERBOSE=0, **kwargs):
        '''Get insert size distribution for a fragment'''
        from .check_insert_distribution import get_insert_size_distribution as gisd
        return gisd(self.seqrun_folder, self.adapter, fragment, VERBOSE=VERBOSE,
                    **kwargs)

class SamplesSeq(pd.DataFrame):
    '''Table of sequenced samples'''

    @property
    def _constructor(self):
        return SamplesSeq

    def filter_seq_run(self, run):
        '''Get only the samples from one sequencing run'''
        return self.loc[self.run == run]

    def filter_patient(self, patient_id, exclude_reps=True):
        '''Get only the samples from a specific patient'''
        samples = self.loc[self.patient == patient_id]
        if exclude_reps:
            ind = [(len(sn) < 2) or (sn[-2:] not in ('-2', '-3')) for sn in samples.name.tolist()]
            samples = samples[ind]
        return samples

class SequencingRun(pd.Series):
    '''Sequencing run'''

    def __init__(self, *args, **kwargs):
        '''Initialize a sequencing run'''
        super(SequencingRun, self).__init__(*args, **kwargs)

        from hivwholeseq.sequencing.filenames import get_seqrun_foldername
        self['folder'] = str(get_seqrun_foldername(self.name))

        self['samples'] = load_samples_sequenced(seq_runs=[self.name])

    @property
    def _constructor(self):
        return SequencingRun

    def itersamples(self):
        '''Generator for samples in this run, each with extended attributes'''
        for samplename, sample in self.samples.iterrows():
            yield SampleSeq(sample)

    @property
    def adapters(self):
        '''The adapters used in the sequencing run'''
        return self.samples.adapter

    def discard_nondivided_samples(self):
        '''Discard samples that have no divided reads (e.g. SA, random hexamers)'''
        import os
        from hivwholeseq.sequencing.filenames import get_divided_filename
        ind = []
        for sample in self.itersamples():
            frag = sample.regions_complete[0]
            div = os.path.isfile(get_divided_filename(self.folder, sample.adapter, frag))
            ind.append(div)
        self.samples = self.samples.loc[ind]

# Functions
def load_samples_sequenced(seq_runs=None):
    '''Load samples sequenced from general table'''
    sample_table = pd.read_excel(table_filename, 'Samples sequenced',
                                 index_col=0)
    sample_table.index = pd.Index(map(str, sample_table.index))
    sample_table.loc[:, 'patient sample'] = map(str, sample_table.loc[:, 'patient sample'])
    sample_table.loc[:, 'regions'] = map(str, sample_table.loc[:, 'regions'])
    sample_table.loc[sample_table.loc[:, 'regions'] == 'nan', 'regions'] = ''

    if seq_runs is not None:
        sample_table = sample_table.loc[sample_table.loc[:, 'seq run'].isin(seq_runs)]

    return SamplesSeq(sample_table)

def load_sample_sequenced(samplename):
    '''Load a sequenced sample from the general table'''
    return SampleSeq(load_samples_sequenced().loc[samplename])

def load_sequencing_runs(seq_runs=None):
    '''Load sequencing runs from general table'''
    global _sequencing_runs
    if _sequencing_runs is None:
        seq_runs_in = pd.read_excel(table_filename, 'Sequencing runs',
                                 index_col=0)
        _sequencing_runs = seq_runs_in

    if seq_runs is not None:
        seq_runs = _sequencing_runs.loc[_sequencing_runs.index.isin(seq_runs)]
        return seq_runs
    else:
        return _sequencing_runs

def load_sequencing_run(seq_run):
    '''Load a single sequencing run (with extended attributes)'''
    seq_runs = load_sequencing_runs()
    seq_run_obj = SequencingRun(seq_runs.loc[seq_run])
    return seq_run_obj

ext: py

max_stars_repo_name: iosonofabio/hivwholeseq

max_issues_repo_name: iosonofabio/hivwholeseq

max_forks_repo_name: iosonofabio/hivwholeseq

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
.. module:: hindex
    :synopsis: Calculate the hindex.

.. moduleauthor:: Alex Gates <ajgates42@gmail.com>
 """

import sys

import pandas as pd
import numpy as np

# determine if we are loading from a jupyter notebook (to make pretty progress bars)
if 'ipykernel' in sys.modules:
    from tqdm.notebook import tqdm
else:
    from tqdm import tqdm

from pyscisci.utils import zip2dict

## Q-factor
def qfactor(a):
    """
    This function calculates the Q-factor for an author.  See :cite:`Sinatra2016individual` for details.

    """

    return np.exp(np.mean(np.log(a[a>0])))

def compute_qfactor(df, colgroupby, colcountby, show_progress=False):
    """
    Calculate the q factor for each group in the DataFrame.  See :cite:`Sinatra2016individual` for the definition.

    The algorithmic implementation for each author can be found in :py:func:`citationanalysis.author_qfactor`.

    Parameters
    ----------
    :param df : DataFrame
        A DataFrame with the citation information for each Author.

    :param colgroupby : str
        The DataFrame column with Author Ids.

    :param colcountby : str
        The DataFrame column with Citation counts for each publication.

    Returns
    -------
    DataFrame
        DataFrame with 2 columns: colgroupby, 'Hindex'

        """
    # register our pandas apply with tqdm for a progress bar
    tqdm.pandas(desc='Qfactor', disable= not show_progress)

    newname_dict = zip2dict([str(colcountby), '0'], [str(colgroupby)+'Qfactor']*2)
    return df.groupby(colgroupby, sort=False)[colcountby].progress_apply(qfactor).to_frame().reset_index().rename(columns=newname_dict)

max_stars_repo_name: kishorevasan/pyscisci

max_issues_repo_name: kishorevasan/pyscisci

max_forks_repo_name: kishorevasan/pyscisci


text:
# Example file that finds the groundstate of a condensate in a rotating frame.
# Takes quite some time to run so you can just stop it when you run out of
# patience and run the plotting script.

# Run with 'mpirun -n <N CPUs> python run.py'

from __future__ import division, print_function
import sys
sys.path.insert(0, '../..') # The location of the modules we need to import

import numpy as np

from parPDE import Simulator2D, LAPLACIAN, GRADX, GRADY
from BEC2D import BEC2D

def get_number_and_trap(rhomax, R):
    """Gives the 2D normalisation constant and trap frequency required for the
    specified maximum density and radius of a single-component condensate in
    the Thomas-Fermi approximation"""
    N = pi * rhomax * R**2 / 2
    omega = np.sqrt(2 * g * rhomax / (m * R**2))
    return N, omega

# Constants:
pi = np.pi
hbar = 1.054571726e-34                        # Reduced Planck's constant
a_0  = 5.29177209e-11                         # Bohr radius
u    = 1.660539e-27                           # unified atomic mass unit
m  = 86.909180*u                              # 87Rb atomic mass
a  = 98.98*a_0                                # 87Rb |2,2> scattering length
g  = 4*pi*hbar**2*a/m                         # 87Rb self interaction constant

rhomax = 2.5e14 * 1e6                         # Desired peak condensate density
R = 9e-6                                      # Desired condensate radius
mu = g* rhomax                                # Approximate chemical potential for desired max density
                                              # (assuming all population is in in mF=+1 or mF=-1)
N_2D, omega = get_number_and_trap(rhomax, R)  # 2D normalisation constant and trap frequency
                                              # required for specified radius and peak density

# Rotation rate:
Omega = 2*omega

# Space:
nx_global = ny_global = 256
x_max_global = y_max_global = 10e-6

simulator = Simulator2D(-x_max_global, x_max_global, -y_max_global, y_max_global, nx_global, ny_global,
                        periodic_x=False, periodic_y=False, operator_order=6)
bec2d = BEC2D(simulator, natural_units=False, use_ffts=False)

x = simulator.x
y = simulator.y
dx = simulator.dx
dy = simulator.dy

r2 = x**2.0 + y**2.0
r  = np.sqrt(r2)

# A harmonic trap to exactly cancel out the centrifugal force:
alpha = 2
V = 0.5 * m * Omega**2 * R**2.0 * (r/R)**alpha

# A high order polynomial trap as a hard wall potential:
alpha = 4
V += 0.5 * m * omega**2 * R**2.0 * (r/R)**alpha

import IPython
IPython.embed()

# The kinetic and rotation terms of the Hamiltonian:
K = -hbar**2/(2*m)*LAPLACIAN - 1j*hbar*Omega*(y * GRADX - x * GRADY)

def H(t, psi):
    """The Hamiltonian for single-component wavefunction psi. Returns the
    kinetic term as an OperatorSum instance, and the local terms separately."""
    H_local_lin = V
    H_local_nonlin = g * abs(psi)**2
    return K, H_local_lin, H_local_nonlin

if __name__ == '__main__':
    # The initial Thomas-Fermi guess:
    # psi = rhomax * (1 - (x**2 + y**2) / R**2)
    # psi[psi < 0] = 0
    # psi = np.sqrt(psi)

    # psi = np.array(psi, dtype=complex)

    import h5py
    with h5py.File('initial_16.h5', 'r') as f:
        psi = f['psi'][:]

    # # Some random vortices to give the relaxation something to start with:
    # np.random.seed(42) # must be seeded so that all MPI processes agree
    # for i in range(100):
    #     x_vortex = np.random.normal(0, scale=R)
    #     y_vortex = np.random.normal(0, scale=R)
    #     psi[:] *= np.exp(1j*np.arctan2(x - y_vortex, y - x_vortex))

    # Find the groundstate:
    psi = bec2d.find_groundstate(H, mu, psi, relaxation_parameter=1.9, convergence=1e-6,
                                 output_interval=1000, output_directory='groundstate_4', convergence_check_interval=10)

max_stars_repo_name: chrisjbillington/parpde

max_issues_repo_name: chrisjbillington/parpde

max_forks_repo_name: chrisjbillington/parpde


text:
# !/usr/bin/python
# coding: utf-8

import numpy as np
import matplotlib.pyplot as plt

np.random.seed(100)

# 生成一组随机数，数量为1000
data = np.random.normal(size = (1000, ), loc = 0, scale= 1)

# sym 调整好异常值的点的形状
# whis 默认是1.5， 通过调整它的竖直来设置异常值显示的数量，
# 如果想显示尽可能多的异常值，whis设置很小，否则很大
plt.boxplot(data, sym ="o", whis = 1.5)
# plt.boxplot(data, sym ="o", whis = 0.01)
# plt.boxplot(data, sym ="o", whis = 999)
plt.show()


text:
import matplotlib.pyplot as pyplot
import matplotlib.image as mpimg
import numpy as np
img = mpimg.imread(r'src\matplotlib\stinkbug.png')

print(len(img), type(img))

imgplot = pyplot.imshow(img)
pyplot.show()

ar = [np.random.randn(1000), np.random.randn(10000), np.random.randn(10000)]
lum_img =  img[:, :, 0]
pyplot.imshow(lum_img)

pyplot.show()
pyplot.imshow(lum_img, cmap="hot")
pyplot.colorbar()

pyplot.show()

pyplot.hist(lum_img.ravel(), bins=256, range=(0.0, 1.0), fc='k', ec='k')
pyplot.show()

from PIL import Image

img = Image.open(r'src\matplotlib\stinkbug.png')
img.thumbnail((64, 64), Image.ANTIALIAS)  # resizes image in-place
imgplot = pyplot.imshow(img)
pyplot.show()

imgplot = pyplot.imshow(img, interpolation="nearest")


text:
import numpy as np
from gym.envs.registration import register
from .cartpole import CartPoleEnv

register(
    id='CartPole-v2',
    entry_point='envs:CartPoleEnv',
)


text:
"""Classes of networks

Author: Xiaodong Nian
Time: November 2019
"""

import numpy as np
import tensorflow as tf
from collections import deque

from networks import Actor, Critic, Critic0, ActorCritic, ActorCriticER
# NN configs.
from config_agent import IM_SIZE, BLACK_AND_WHITE
# from diverse_mem import DiverseMemory
from diverse_mem_attentive import AttentiveMemoryBuffer, MemoryBuffer
from history import History
from utils import *

class DCRACAgent:
    def __init__(self,
                 env,
                 gamma=0.98,
                 weights=None,
                 timesteps=5,
                 batch_size=32,
                 replay_type='STD',
                 buffer_size=10000,
                 buffer_a=2,
                 buffer_e=0.01,
                 memnn_size=9,
                 start_annealing=0.05,
                 max_episode_length=500,
                 start_e=1.,
                 end_e=0.05,
                 im_size=(IM_SIZE, IM_SIZE),
                 grayscale=BLACK_AND_WHITE,
                 net_type='R',
                 obj_func='a',
                 lr=1e-3,
                 lr_2=1e-3,
                 frame_skip=4,
                 update_interval=4,
                 clipnorm=1,
                 clipvalue=1,
                 nesterov=True,
                 momentum=0.9,
                 dup=False,
                 min_buf_size=0.1,
                 action_conc=True,
                 feature_embd=True,
                 extra=None,
                 gpu_setting='1',
                 attentive=True):

        self.env = env
        self.nb_action = self.env.action_space.shape[0]
        self.observation_shape = self.env.observation_space.shape
        self.nb_objective = self.env.obj_cnt
        self.discount = gamma
        self.qvalue_dim = self.nb_objective

        self.timesteps = timesteps
        self.batch_size = batch_size
        self.replay_type = replay_type
        self.buffer_size = buffer_size
        self.buffer_a = buffer_a
        self.buffer_e = buffer_e
        self.max_episode_length = max_episode_length

        self.fill_history = False
        self.stoch_policy = True
        self.use_prob = False

        self.memnn_size = memnn_size

        self.weight_history = []
        if weights is not None:
            self.set_weights(np.array(weights))

        self.start_e = start_e
        self.end_e = end_e
        self.start_annealing = start_annealing

        self.im_size = im_size
        channel = 1 if grayscale else 3
        self.im_shape = im_size + (channel, )

        self.lr_2 = lr_2
        self.lr = lr
        self.frame_skip = frame_skip

        self.net_type = net_type
        self.action_conc = action_conc
        self.feature_embd = feature_embd

        # self.mask_empty = True if self.action_conc and self.net_type in ('R', 'M') else False
        self.mask_empty = True

        self.dup = dup
        self.obj_func = obj_func.lower()
        self.min_buf_size = min_buf_size
        self.update_interval = update_interval
        self.actor_update_interval = 1

        self.nesterov = nesterov
        self.clipnorm = clipnorm
        self.clipvalue = clipvalue
        self.momentum = momentum

        self.direct_update = True
        self.is_first_update = True
        self.max_error = .0
        self.trace_values={}
        self.recent_experiences = []

        self.start_lambda = 3
        self.end_lambda = 1
        self.alpha = 1
        self.attentive = attentive

        self.extra = extra
        self.gpu_setting = gpu_setting

        # Tensorflow GPU optimization.
        config = tf.compat.v1.ConfigProto()
        config.gpu_options.allow_growth = True
        sess = tf.compat.v1.Session(config=config)
        tf.compat.v1.keras.backend.set_session(sess)

        # config = tf.ConfigProto()
        # config.gpu_options.allow_growth = True
        # sess = tf.Session(config=config)
        # from keras import backend as K
        # K.set_session(sess)

        # Initialize the history
        self.history = History(self.timesteps, self.im_shape, self.nb_action)

        # Make neural networks
        self.build_models()

        # Initialize replay buffer.
        self.initialize_memory()

    def build_models(self):
        # Make actor network (creates target model internally).
        self.actor = Actor(self.nb_action, self.im_shape, self.nb_objective, self.qvalue_dim,
            timesteps=self.timesteps, net_type=self.net_type, memnn_size=self.memnn_size,
            lr=self.lr_2, action_conc=self.action_conc, action_embd=self.feature_embd,
            weight_embd=self.feature_embd, backendtype=self.gpu_setting)

        # Make critic network (creates target model internally).
        self.critic = Critic(self.nb_action, self.im_shape, self.nb_objective, self.qvalue_dim,
            timesteps=self.timesteps, net_type=self.net_type, memnn_size=self.memnn_size,
            lr=self.lr, action_conc=self.action_conc, action_embd=self.feature_embd,
            weight_embd=self.feature_embd, backendtype=self.gpu_setting)

        # log the settings
        with open("output/logs/info_"+self.extra+".log", "a") as f:
            f.write(time.asctime()+'\n\n')
            f.write('ACTORRRRRRRRRRRRRRRRRRRRRRR\n')
            f.write(str(self.actor.model.get_config())+'\n\n')
            f.write('CRITICCCCCCCCCCCCCCCCCCCCCC\n')
            f.write(str(self.critic.model.get_config())+'\n\n')
            f.write(str(vars(self))+'\n\n')
        print(vars(self))

    # Train or run for some number of episodes.
    def train(self, log_file, learning_steps, weights,
              per_weight_steps, total_steps, log_game_step=False):

        self.learning_steps = learning_steps
        self.epsilon = self.start_e
        per_weight_steps = per_weight_steps

        weight_index = 0
        self.steps = 0
        self.total_steps = int(total_steps)
        half_total = int(total_steps/2)

        self.log = Log(log_file)
        self.log_game_step = log_game_step

        self.set_weights(weights[weight_index])

        episodes = 1
        episode_steps = 0
        pred_idx = None

        current_state_raw = self.env.reset()
        self.current_state, last_action = self.history.reset_with_raw_frame(current_state_raw, fill=self.fill_history)

        for step in range(self.total_steps):

            self.steps = step
            episode_steps += 1

            # pick an action following an epsilon-greedy strategy
            action, acts_prob = self.pick_action(self.current_state, last_action)

            # perform the action
            next_state_raw, reward, terminal, info = self.env.step(action, self.frame_skip)
            next_state, next_last_action = self.history.add_raw_frame(next_state_raw, action)

            if self.log_game_step:
                print("Taking action", action, "under prob", acts_prob, "at", info["position"], "with reward", reward)

            # memorize the experienced transition
            pred_idx = self.memorize(
                self.current_state,
                action,
                reward,
                next_state,
                terminal,
                last_action,
                acts_prob,
                trace_id=episodes if self.replay_type == "DER" else step,
                pred_idx=pred_idx)

            # update the networks and exploration rate
            self.update_lambda(step)
            loss = self.perform_updates(step)
            self.update_epsilon(step)

            self.log.log_step(episodes, step, loss, reward,
                              terminal or episode_steps > self.max_episode_length,
                              self.weights, self.discount, episode_steps,
                              self.epsilon, self.frame_skip, action)

            self.current_state = next_state
            current_state_raw = next_state_raw
            last_action = next_last_action

            if terminal or episode_steps > self.max_episode_length:
                current_state_raw = self.env.reset()
                self.current_state, last_action = self.history.reset_with_raw_frame(current_state_raw, fill=self.fill_history)
                pred_idx = None

                is_weight_change = int(
                    (step + 1) / per_weight_steps) != weight_index

                if per_weight_steps == 1:
                    weight_index += 1
                else:
                    weight_index = int((step + 1) / per_weight_steps)

                if per_weight_steps == 1 or is_weight_change:
                    self.set_weights(weights[weight_index])

                episodes += 1
                episode_steps = 0

            if (step + 1) % 10000 == 0:
                # self.save_weights()
                self.save_model()

            if step == half_total:
                self.use_prob = True

    def set_weights(self, weights):
        """Set current weight vector

        Arguments:
            weights {np.array} -- Weight vector of size N
        """

        self.weights = np.array(weights)
        self.weight_history.append(self.weights)

    def pick_action(self, trace_o, trace_a, weights=None):
        """Given a observation trace and weights, compute the next action, following an
            epsilon-greedy strategy

        Arguments:
            trace_o {np.array} -- The trace of observations o_t in which to act
            trace_a {np.array} -- The trace of actions a_{t-1}, in the same order as trace_o

        Keyword Arguments:
            weights {np.array} -- The weights on which to act (default: self.weights)

        Returns:
            int -- The selected action's index
        """
        # np.random.seed(self.steps)
        if np.random.random() < self.epsilon:
            return np.random.choice(self.env.action_space), (np.ones(self.nb_action)/self.nb_action)

        trace_a = self._get_onehot_action_trace(trace_a)
        weights = self.weights if weights is None else weights

        action, acts_prob = self.actor.choose_action(
            weights[np.newaxis, ], trace_o[np.newaxis, ], trace_a[np.newaxis, ],
            stochastic=self.stoch_policy, return_probs=True)

        return action, acts_prob

    def _get_onehot_action_trace(self, actions):
        """Translate the numerical actions array as one-hot array.

        Args:
            actions ([int]): integer action list

        Returns:
            np.array: one-hot action list
        """
        length = len(actions)
        onehot_trace = np.zeros((length, self.nb_action))
        for i, a in enumerate(actions):
            if a >= 0:
                onehot_trace[i][a] = 1
            # if mask_empty, set all action slots as -1 to be masked by the network.
            if self.mask_empty and a == -1:
                onehot_trace[i] = -1
        return onehot_trace

    # Given a bunch of experienced histories, update our models.
    def perform_updates(self, steps):
        loss = 0
        if steps> self.min_buf_size * self.learning_steps:

            if steps % self.update_interval == 0:

                if self.is_first_update:
                    # Compute buffer's priorities all at once before the first update
                    self.recent_experiences = []
                    self.update_all_priorities()
                    self.is_first_update = False

                loss = self.policy_update(update_actor=(steps%(self.actor_update_interval*self.update_interval) == 0))
                self.update_target(soft_update=True)

        return loss

    def update_target(self, **kwargs):
        self.actor.update_target(**kwargs)
        self.critic.update_target(**kwargs)

    def policy_update(self, update_actor=True):
        # np.random.seed(self.steps)
        # ids, batch, _ = self.buffer.sample(self.batch_size)
        if self.attentive:
            ids, batch, _ = self.buffer.sample(self.batch_size, self.k, self.steps, self.weights, self.current_state)
        else:
            ids, batch, _ = self.buffer.sample(self.batch_size)

        if self.direct_update:
            # Add recent experiences to the priority update batch
            batch = np.concatenate((batch, self.buffer.get(self.recent_experiences)), axis=0)
            ids = np.concatenate((ids, self.recent_experiences)).astype(int)

        # if self.dup is True, we train each sample in the batch on two
        # weight vectors, hence we duplicate the batch data
        if self.dup:
            batch = np.repeat(batch, 2, axis=0)
            ids = np.repeat(ids, 2, axis=0)

        cur_batch_size = len(batch)

        # q_predict, q_target, p_predict, p_target, w_batch, o_batch, a_last_batch
        q_predict, q_target, p_predict, p_target, w_batch, o_batch, a_last_batch = self._get_training_data(batch)

        q_true = np.copy(q_predict)
        q_mask = np.zeros((cur_batch_size, self.nb_action, self.qvalue_dim), dtype=float)
        action_objective = np.zeros((cur_batch_size, self.nb_action), dtype=float)

        for i, (_, action, reward, _, done, _, _, _) in enumerate(batch):
            q_true[i][action] = np.copy(reward)
            q_mask[i][action] = 1

            if not done:
                if self.stoch_policy:
                    # # stochastic policy (sample an action 'a_next' with target action_probs)
                    # a_next = np.random.choice(range(self.nb_action), p=p_target[i])
                    # q_true[i][action] += self.discount * q_target[i][a_next]

                    # Expectation for stochastic policy
                    q_true[i][action] += self.discount * np.dot(p_target[i], q_target[i])
                else:
                    # deterministic policy (choose the 'a_next' with largest target action_probs)
                    a_next = np.argmax(p_target[i])
                    q_true[i][action] += self.discount * q_target[i][a_next]

            if update_actor:
                action_objective[i][action] = self._get_action_objective(q_true[i], q_predict[i], p_predict[i], action, w_batch[i])

        a_batch = self._get_onehot_action_trace([b[1] for b in batch])
        p_old_batch = np.array([b[6] for b in batch])

        self.train_on_batch(w_batch, o_batch, a_last_batch, q_true, action_objective, a_batch, p_old_batch, q_mask)

        loss = self.update_priorities(batch, ids)
        self.recent_experiences = []

        return loss

    def train_on_batch(self, w_batch, o_batch, a_last_batch, q_true, action_objective, a_batch, p_old_batch, q_mask=None):
        self.critic.model.train_on_batch([w_batch, o_batch, a_last_batch], q_true)
        self.actor.model.train_on_batch([w_batch, o_batch, a_last_batch], action_objective)

    def _get_training_data(self, batch):

        o_batch, o_next_batch, a_last_batch, a_last_next_batch = self._get_full_states(batch)

        w_batch = self.get_training_weights(batch)

        # Predict both the model and target q-values
        predict_input = [w_batch, o_batch, a_last_batch]
        target_input = [w_batch, o_next_batch, a_last_next_batch]

        # Predict both the model and target q-values
        q_predict = self.critic.model.predict(predict_input)
        q_target = self.critic.target_model.predict(target_input)

        # Predict both the model and target action probs
        p_predict = self.actor.model.predict(predict_input)
        p_target = self.actor.target_model.predict(target_input)

        return q_predict, q_target, p_predict, p_target, w_batch, o_batch, a_last_batch

    def _get_full_states(self, batch):

        o_batch = np.zeros((len(batch),) + self.history.history_o.shape, dtype=np.uint8)
        o_next_batch = np.zeros((len(batch),) + self.history.history_o.shape, dtype=np.uint8)

        # one-hot for predict
        a_last_batch = np.zeros((len(batch), self.history.length, self.nb_action), dtype=float)
        a_last_next_batch = np.zeros((len(batch), self.history.length, self.nb_action), dtype=float)

        for i, b in enumerate(batch):
            o_batch[i] = b[0]
            o_next_batch[i][:-1] = b[0][1:]
            o_next_batch[i][-1] = b[3]

            a_last_batch[i] = self._get_onehot_action_trace(b[5])
            a_last_next_batch[i] = np.roll(a_last_batch[i], -1, axis=0)
            a_last_next_batch[i][-1][b[1]] = 1

        return o_batch, o_next_batch, a_last_batch, a_last_next_batch

    def _get_action_objective(self, q_true, q_predict, p_predict, action, weight):
        if self.obj_func == 'a':
            objective = q_true[action] - q_predict.mean(axis=0)
        elif self.obj_func == 'ap':
            # based on advantages
            objective = q_true[action] - np.average(q_predict, weights=p_predict, axis=0)
        elif self.obj_func == 'td':
            # based on td error
            objective = q_true[action] - q_predict[action]
        elif self.obj_func == 'y':
            # based on y values
            objective = q_true[action]
        else:
            # based on q values
            objective = q_predict[action]

        return np.dot(objective, weight)

    def get_training_weights(self, batch):
        """Given a batch of transitions, this method generates a batch of
        weights to train on

        Arguments:
            batch {list} -- batch of transitions

        Returns:
            list -- batch of weights
        """
        w_batch = np.repeat([self.weights], len(batch), axis=0)
        if self.dup and len(self.weight_history) > 1:
            idx = np.random.randint(len(self.weight_history)-1 , size=int(len(batch)))
            w_batch[::] = np.array(self.weight_history)[idx]
            w_batch[::2] = self.weights
        return w_batch

    def update_priorities(self, batch, ids, ignore_dup=False, pr=False):
        """Given a batch of transitions, this method computes each transition's
        error and uses that error to update its priority in the replay buffer

        Arguments:
            batch {list} -- list of transitions
            ids {list} -- list of identifiers of each transition in the replay
                        buffer

        Returns:
            float -- The batch's mean loss
        """
        # q_predict, q_target, p_predict, p_target, w_batch, o_batch, a_last_batch
        q_predict, q_target, _, p_target, _, _, _ = self._get_training_data(batch)

        errors = np.zeros(len(batch))

        for i, (_, action, reward, _, terminal, _, _, _) in enumerate(batch):

            target = np.copy(reward)

            if not terminal:
                if self.stoch_policy:
                    # # stochastic policy (sample an action 'a_next' with target action_probs)
                    # a_next = np.random.choice(range(self.nb_action), p=p_target[i])
                    # target += self.discount * q_target[i][a_next]

                    # Expectation for stochastic policy
                    target += self.discount * np.dot(p_target[i], q_target[i])
                else:
                    # deterministic policy (choose the 'a_next' with largest target action_probs)
                    a_next = np.argmax(p_target[i])
                    target += self.discount * q_target[i][a_next]

            error = mae(q_predict[i][action], target)
            errors[i] = error

            # When dup is True, we train each sample on two weight vectors
            # Hence, there are two TD-errors per sample, we use the mean of
            # both errors to update the priorities
            if self.dup:
                if i % 2 == 0:
                    continue
                error = (error + errors[i - 1]) / 2
            self.buffer.update(ids[i], error)
            self.max_error = max(error, self.max_error)

        return np.mean(errors)

    def update_all_priorities(self):
        """Updates all priorities of the replay buffer
        """
        data = self.buffer.get_data(True)

        chunk_size = 100
        for i in range(0, len(data[0]), chunk_size):
            chunk_data = np.array(data[1][i:i + chunk_size])
            chunk_ids = data[0][i:i + chunk_size]

            if self.dup:
                chunk_data = np.repeat(chunk_data, 2, axis=0)
                chunk_ids = np.repeat(chunk_ids, 2, axis=0)

    def update_epsilon(self, steps):
        """Update exploration rate

        Arguments:
            steps {int} -- Elapsed number of steps
        """

        start_steps = self.learning_steps * self.start_annealing
        annealing_steps = self.learning_steps * (1 - self.start_annealing)

        self.epsilon = linear_anneal(steps, annealing_steps, self.start_e, self.end_e, start_steps)

    def update_lambda(self, steps):
        start_steps = self.learning_steps * self.start_annealing
        annealing_steps = self.total_steps * self.alpha

        self.k = self.linear_anneal_lambda(steps, annealing_steps, self.start_lambda, self.end_lambda, start_steps)

    def linear_anneal_lambda(self, steps, annealing_steps, start_lambda, end_lambda, start_steps):
        t = max(0, steps - start_steps)
        return max(end_lambda, (annealing_steps-t) * (start_lambda - end_lambda) / annealing_steps + end_lambda)

    def initialize_memory(self):
        """Initialize the replay buffer, with a secondary diverse buffer and/or
            a secondary tree to store prediction errors
        """
        if self.replay_type == "DER":
            main_capacity = sec_capacity = self.buffer_size // 2
        else:
            main_capacity, sec_capacity = self.buffer_size, 0

        def der_trace_value(trace, trace_id, memory_indices):
            """Computes a trace's value as its return

            Arguments:
                trace {list} -- list of transitions
                trace_id {object} -- the trace's id
                memory_indices {list} -- list of the trace's indexes in memory

            Returns:
                np.array -- the trace's value
            """

            if trace_id in self.trace_values:
                return self.trace_values[trace_id]
            I_REWARD = 2
            value = np.copy(trace[0][I_REWARD])
            for i, v in enumerate(trace[1:]):
                value += v[I_REWARD] * self.discount**(i + 1)
            if type(value) == float:
                value = np.array([value])
            self.trace_values[trace_id] = value
            return value

        if self.attentive:
            self.buffer = AttentiveMemoryBuffer(main_capacity=main_capacity, sec_capacity=sec_capacity,
                value_function=der_trace_value, trace_diversity=True, a=self.buffer_a, e=self.buffer_e)
        else:
            self.buffer = MemoryBuffer(main_capacity=main_capacity, sec_capacity=sec_capacity,
                value_function=der_trace_value, trace_diversity=True, a=self.buffer_a, e=self.buffer_e)

    def memorize(self, state, action, reward, next_state, terminal, action_prev, acts_prob,
        initial_error=0, trace_id=None, pred_idx=None):
        """Memorizes a transition into the replay, if no error is provided, the
        transition is saved with the lowest priority possible, and should be
        updated accordingly later on.

        Arguments:
            state {object} -- s_t
            action {int} -- a_t
            reward {np.array} -- r_t
            next_state {object} -- s_{t+1}
            terminal {bool} -- wether s_{t+1} is terminal
            action_prev {object} -- a_{t-1}
            acts_prob {np.array} -- sample probability

        Keyword Arguments:
            initial_error {float} -- The initial error of the transition (default: {0})
            trace_id {object} -- The trace's identifier, if None, the transition is treated as
                                an individual trace. (default: {None})
        """
        if initial_error == 0 and not self.direct_update:
            initial_error = self.max_error

        extra = np.array([self.weights])
        transition = np.array((state, action, reward, next_state[-1], terminal, action_prev, acts_prob, extra))

        # Add transition to replay buffer
        idx = self.buffer.add(initial_error, transition, pred_idx=pred_idx, trace_id=trace_id)
        self.recent_experiences.append(idx)

        return idx

    @property
    def name(self):
        return self.extra

    def save_weights(self):
        """Saves the networks' weights to files identified by the agent's name
        and the current weight vector
        """
        self.critic.model.save_weights("output/networks/{}_critic.weights".format(self.name))
        self.actor.model.save_weights("output/networks/{}_actor.weights".format(self.name))

    def save_model(self):
        """Saves the networks to files identified by the agent's name
        and the current weight vector
        """
        self.critic.model.save("output/networks/{}_critic_model.h5".format(self.name))
        self.actor.model.save("output/networks/{}_actor_model.h5".format(self.name))

class DCRACSAgent(DCRACAgent):
    def build_models(self):
        # Make only critic network (creates target model internally).
        self.actorcritic = ActorCritic(self.nb_action, self.im_shape, self.nb_objective, self.qvalue_dim,
            timesteps=self.timesteps, net_type=self.net_type, memnn_size=self.memnn_size,
            lr=self.lr, action_conc=self.action_conc, action_embd=self.feature_embd,
            weight_embd=self.feature_embd, actor_lr=self.lr_2, backendtype=self.gpu_setting)

        with open("output/logs/info_"+self.extra+".log", "a") as f:
            f.write(time.asctime()+'\n\n')
            f.write(str(self.actorcritic.model.get_config())+'\n\n')
            f.write(str(vars(self))+'\n\n')
        print(vars(self))

    def pick_action(self, trace_o, trace_a, weights=None):
        # np.random.seed(self.steps)
        if np.random.random() < self.epsilon:
            return np.random.choice(self.env.action_space), (np.ones(self.nb_action)/self.nb_action)

        trace_a = self._get_onehot_action_trace(trace_a)
        weights = self.weights if weights is None else weights

        action, acts_prob = self.actorcritic.choose_action(
            weights[np.newaxis, ], trace_o[np.newaxis, ], trace_a[np.newaxis, ],
            stochastic=self.stoch_policy, use_prob=self.use_prob, return_probs=True)

        return action, acts_prob

    def update_target(self, **kwargs):
        self.actorcritic.update_target(**kwargs)

    def train_on_batch(self, w_batch, o_batch, a_last_batch, q_true, action_objective, a_batch, p_old_batch, q_mask=None):
        self.actorcritic.model.train_on_batch([w_batch, o_batch, a_last_batch], [q_true, action_objective])

    def _get_training_data(self, batch):

        o_batch, o_next_batch, a_last_batch, a_last_next_batch = self._get_full_states(batch)

        w_batch = self.get_training_weights(batch)

        # Predict both the model and target q-values
        predict_input = [w_batch, o_batch, a_last_batch]
        target_input = [w_batch, o_next_batch, a_last_next_batch]

        # Predict both the model and target q-values
        q_predict, p_predict = self.actorcritic.model.predict(predict_input)
        q_target, p_target = self.actorcritic.target_model.predict(target_input)

        return q_predict, q_target, p_predict, p_target, w_batch, o_batch, a_last_batch

    def save_weights(self):
        """Saves the networks' weights to files identified by the agent's name
        and the current weight vector
        """
        self.actorcritic.model.save_weights("output/networks/{}_actorcritic_share.weights".format(self.name))

    def save_model(self):
        """Saves the networks to files identified by the agent's name
        and the current weight vector
        """
        self.actorcritic.model.save("output/networks/{}_critic_model.h5".format(self.name))

class DCRACSEAgent(DCRACSAgent):
    def build_models(self):
        # Make only critic network (creates target model internally).
        self.actorcritic = ActorCriticER(self.nb_action, self.im_shape, self.nb_objective, self.qvalue_dim,
            timesteps=self.timesteps, net_type=self.net_type, memnn_size=self.memnn_size,
            lr=self.lr, action_conc=self.action_conc, action_embd=self.feature_embd,
            weight_embd=self.feature_embd, backendtype=self.gpu_setting)

        with open("output/logs/info_"+self.extra+".log", "a") as f:
            f.write(time.asctime()+'\n\n')
            f.write(str(self.actorcritic.model.get_config())+'\n\n')
            f.write(str(vars(self))+'\n\n')
        print(vars(self))

    def train_on_batch(self, w_batch, o_batch, a_last_batch, q_true, action_objective, a_batch, p_old_batch, q_mask=None):
        action_objective = np.sum(action_objective, axis=1)
        self.actorcritic.trainable_model.train_on_batch([w_batch, o_batch, a_last_batch, action_objective, p_old_batch], [q_true, a_batch])

class DCRAC0Agent(DCRACAgent):
    def build_models(self):
        # Make actor network (creates target model internally).
        self.actor = Actor(self.nb_action, self.im_shape, self.nb_objective, self.qvalue_dim,
            timesteps=self.timesteps, net_type=self.net_type, memnn_size=self.memnn_size,
            lr=self.lr_2, action_conc=self.action_conc, action_embd=self.feature_embd,
            weight_embd=self.feature_embd, backendtype=self.gpu_setting)

        # Make critic network (creates target model internally).
        self.critic = Critic0(self.nb_action, self.im_shape, self.nb_objective, self.qvalue_dim,
            timesteps=self.timesteps, net_type=self.net_type, memnn_size=self.memnn_size,
            lr=self.lr, action_conc=self.action_conc, action_embd=self.feature_embd,
            weight_embd=self.feature_embd, backendtype=self.gpu_setting)

        # log the settings
        with open("output/logs/info_"+self.extra+".log", "a") as f:
            f.write(time.asctime()+'\n\n')
            f.write('ACTORRRRRRRRRRRRRRRRRRRRRRR\n')
            f.write(str(self.actor.model.get_config())+'\n\n')
            f.write('CRITICCCCCCCCCCCCCCCCCCCCCC\n')
            f.write(str(self.critic.model.get_config())+'\n\n')
            f.write(str(vars(self))+'\n\n')
        print(vars(self))

    def train_on_batch(self, w_batch, o_batch, a_last_batch, q_true, action_objective, a_batch, p_old_batch, q_mask=None):
        dummy = q_true[:, 0, :]
        self.critic.trainable_model.train_on_batch([w_batch, o_batch, a_last_batch, q_true, q_mask], [dummy, q_true])
        self.actor.model.train_on_batch([w_batch, o_batch, a_last_batch], action_objective)

class CNAgent(DCRACAgent):
    def build_models(self):
        # Make only critic network (creates target model internally).
        self.critic = Critic(self.nb_action, self.im_shape, self.nb_objective, self.qvalue_dim,
            timesteps=self.timesteps, net_type=self.net_type, memnn_size=self.memnn_size,
            lr=self.lr, action_conc=self.action_conc, action_embd=self.feature_embd,
            weight_embd=self.feature_embd, backendtype=self.gpu_setting)

        with open("output/logs/info_"+self.extra+".log", "a") as f:
            f.write(time.asctime()+'\n\n')
            f.write(str(self.critic.model.get_config())+'\n\n')
            f.write(str(vars(self))+'\n\n')
        print(vars(self))

    def pick_action(self, trace_o, trace_a, weights=None):
        """Given a observation trace and weights, compute the next action, following an
            epsilon-greedy strategy

        Arguments:
            trace_o {np.array} -- The trace of observations o_t in which to act
            trace_a {np.array} -- The trace of actions a_{t-1}, in the same order as trace_o

        Keyword Arguments:
            weights {np.array} -- The weights on which to act (default: self.weights)

        Returns:
            int -- The selected action's index
        """
        # np.random.seed(self.steps)
        if np.random.random() < self.epsilon:
            return np.random.choice(self.env.action_space), (np.ones(self.nb_action)/self.nb_action)

        trace_a = self._get_onehot_action_trace(trace_a)
        weights = self.weights if weights is None else weights

        q_value = self.critic.model.predict([weights[np.newaxis, ], trace_o[np.newaxis, ], trace_a[np.newaxis, ]])

        scal_q = np.dot(q_value[0], weights)
        action = np.argmax(scal_q)
        return action, scal_q

    # Given a bunch of experienced histories, update our models.
    def update_target(self, **kwargs):
        self.critic.update_target(**kwargs)

    def policy_update(self, update_actor=False):
        # np.random.seed(self.steps)
        ids, batch, _ = self.buffer.sample(self.batch_size)

        if self.direct_update:
            # Add recent experiences to the priority update batch
            batch = np.concatenate((batch, self.buffer.get(self.recent_experiences)), axis=0)
            ids = np.concatenate((ids, self.recent_experiences)).astype(int)

        # if self.dup is True, we train each sample in the batch on two
        # weight vectors, hence we duplicate the batch data
        if self.dup:
            batch = np.repeat(batch, 2, axis=0)
            ids = np.repeat(ids, 2, axis=0)

        # q_predict, q_target, w_batch, o_batch, a_last_batch
        q_predict, q_target, w_batch, o_batch, a_last_batch = self._get_training_data(batch)

        q_true = np.copy(q_predict)
        masks = np.zeros((len(batch), self.nb_action, self.qvalue_dim), dtype=float)

        for i, (_, action, reward, _, done, _, _) in enumerate(batch):
            q_true[i][action] = np.copy(reward)
            masks[i][action] = 1

            if not done:
                a_next = np.argmax(np.dot(q_target[i], w_batch[i]))
                q_true[i][action] += self.discount * q_target[i][a_next]

        dummy = q_true[:, 0, :]
        self.train_on_batch(w_batch, o_batch, a_last_batch, q_true, dummy, masks)

        loss = self.update_priorities(batch, ids)
        self.recent_experiences = []

        return loss

    def train_on_batch(self, w_batch, o_batch, a_last_batch, q_true, dummy=None, q_mask=None):
        self.critic.model.train_on_batch([w_batch, o_batch, a_last_batch], q_true)

    def _get_training_data(self, batch):

        o_batch, o_next_batch, a_last_batch, a_last_next_batch = self._get_full_states(batch)

        w_batch = self.get_training_weights(batch)

        # Predict both the model and target q-values
        predict_input = [w_batch, o_batch, a_last_batch]
        target_input = [w_batch, o_next_batch, a_last_next_batch]

        # Predict both the model and target q-values
        q_predict = self.critic.model.predict(predict_input)
        q_target = self.critic.target_model.predict(target_input)

        return q_predict, q_target, w_batch, o_batch, a_last_batch

    def update_priorities(self, batch, ids, ignore_dup=False, pr=False):
        """Given a batch of transitions, this method computes each transition's
        error and uses that error to update its priority in the replay buffer

        Arguments:
            batch {list} -- list of transitions
            ids {list} -- list of identifiers of each transition in the replay
                        buffer

        Returns:
            float -- The batch's mean loss
        """
        q_predict, q_target, w_batch, _, _ = self._get_training_data(batch)

        errors = np.zeros(len(batch))

        for i, (_, action, reward, _, terminal, _, _) in enumerate(batch):
            target = np.copy(reward)

            if not terminal:
                a_next = np.argmax(np.dot(q_target[i], w_batch[i]))
                target += self.discount * q_target[i][a_next]

            error = mae(q_predict[i][action], target)
            errors[i] = error

            # When dup is True, we train each sample on two weight vectors
            # Hence, there are two TD-errors per sample, we use the mean of
            # both errors to update the priorities
            if self.dup:
                if i % 2 == 0:
                    continue
                error = (error + errors[i - 1]) / 2
            self.buffer.update(ids[i], error)
            self.max_error = max(error, self.max_error)

        return np.mean(errors)

    def save_weights(self):
        """Saves the networks' weights to files identified by the agent's name
        and the current weight vector
        """
        self.critic.model.save_weights("output/networks/{}_dqn.weights".format(self.name))

    def save_model(self):
        """Saves the networks to files identified by the agent's name
        and the current weight vector
        """
        self.critic.model.save("output/networks/{}_dqn_model.h5".format(self.name))

class CN0Agent(CNAgent):
    def build_models(self):
        # Make only critic network (creates target model internally).
        self.critic = Critic0(self.nb_action, self.im_shape, self.nb_objective, self.qvalue_dim,
            timesteps=self.timesteps, net_type=self.net_type, memnn_size=self.memnn_size,
            lr=self.lr, action_conc=self.action_conc, action_embd=self.feature_embd,
            weight_embd=self.feature_embd, backendtype=self.gpu_setting)

        with open("output/logs/info_"+self.extra+".log", "a") as f:
            f.write(time.asctime()+'\n\n')
            f.write(str(self.critic.model.get_config())+'\n\n')
            f.write(str(vars(self))+'\n\n')
        print(vars(self))

    def train_on_batch(self, w_batch, o_batch, a_last_batch, q_true, dummy=None, q_mask=None):
        self.critic.trainable_model.train_on_batch([w_batch, o_batch, a_last_batch, q_true, q_mask], [dummy, q_true])


text:
#!/usr/bin/env python

from __future__ import print_function

import itertools

from sympy import symbols, simplify_logic

TMPL_ADD = '''def {name}({args}):
    if {body}
    else:
        raise Exception('{name}: Unhandled case "{{}}"'.format({args}))'''

def _sym_to_py(expr):
    try:
        if expr.is_Symbol:
            return expr.name
        elif expr.is_Function:
            name = str(expr.func)
            if name == 'And':
                return '(' + ' & '.join(sym_to_py(a) for a in expr.args) + ')'
            elif name == 'Xor':
                return '(' + ' ^ '.join(sym_to_py(a) for a in expr.args) + ')'
            elif name == 'Or':
                return '(' + ' | '.join(sym_to_py(a) for a in expr.args) + ')'
            elif name == 'Not':
                assert len(expr.args) == 1
                return '(~{})'.format(sym_to_py(expr.args[0]))
            else:
                raise Exception('Operator "{}" missing'.format(name))
        else:
            return str(bool(expr))
    except Exception as e:
        print(e)
        import IPython; IPython.embed()

def sym_to_py(expr):
    expr_simp = simplify_logic(expr)
    # Stupid heuristics
    if expr.count_ops() > expr_simp.count_ops():
        expr = expr_simp
    return _sym_to_py(expr).replace('True', '_one').replace('False', '_zero')

def bool_to_cond(b):
    if b is None:
        return 'X'
    elif b:
        return 'I'
    else:
        return 'O'

TMPL_COND = '''{cond}:
        return {stmts}'''

def mk_funk(funk, vars, exprs_tmpls):
    nn = ['ss[{}]'.format(i) for i in range(vars)]
    ii = ['b{}'.format(i) for i in range(vars)]
    ss = symbols(' '.join(ii))

    conds = []
    body = []

    exprs = []
    for e in exprs_tmpls:
        exprs.append(eval(e.format(*nn)))

    for vv in itertools.product((False, True, None), repeat=3):
        s = dict((si, vi if vi is not None else si) for si, vi in zip(ss, vv))
        cond = ' and '.join('{}.{}'.format(n, bool_to_cond(v)) for n, v in zip(ii, vv))
        conds.append(cond)
        body.append(tuple(sym_to_py(e.subs(s)) for e in exprs))

    stmts = [TMPL_COND.format(cond=cond, stmts=', '.join(stmts)) for cond, stmts in zip(conds, body)]
    stmts = '\n    elif '.join(stmts)

    return TMPL_ADD.format(
        name=funk,
        args=', '.join(ii),
        body=stmts,
    )

def main():
    print('Generating the functions...')
    defs = (
        ('_bit_add', 3, ['{0} ^ {1} ^ {2}', '({0} & {1}) | ({2} & ({0} ^ {1}))']),
    )
    funks = []
    for d in defs:
        print('[+] Making "{}"'.format(d[0]))
        funks.append(mk_funk(*d))
    src = 'from ._tbits import _zero, _one\n\n\n' + '\n\n'.join(funks) + '\n'

    print('Writing to file...')
    with open('./tbits/_gen_tables.py', 'w') as f:
        f.write(src)

    print('End of this giant hack :D')

if __name__ == '__main__':
    main()

max_stars_repo_name: thebabush/tbits

max_issues_repo_name: thebabush/tbits

max_forks_repo_name: thebabush/tbits

reason: from sympy