text:
from __future__ import absolute_import
from __future__ import print_function

import os
import sys
import functools
import math
import numpy as np

# the next line can be removed after installation
sys.path.insert(0, os.path.dirname(os.path.dirname(
    os.path.dirname(os.path.abspath(__file__)))))

import nngen as ng

from veriloggen import *
import veriloggen.thread as vthread
import veriloggen.types.axi as axi

def run(act_shape=(1, 7, 7, 15),
        weight1_shape=(7, 3, 3, 15), bias1_shape=None, scale1_shape=None,
        weight2_shape=(9, 3, 3, 7), bias2_shape=None, scale2_shape=None,
        weight3_shape=(9, 3, 3, 9), bias3_shape=None, scale3_shape=None,
        act_dtype=ng.int32,
        weight1_dtype=ng.int32, bias1_dtype=ng.int32, scale1_dtype=ng.int32,
        weight2_dtype=ng.int32, bias2_dtype=ng.int32, scale2_dtype=ng.int32,
        weight3_dtype=ng.int32, bias3_dtype=ng.int32, scale3_dtype=ng.int32,
        tmp_dtype=ng.int32,
        out_dtype=ng.int32,
        stride1=(1, 1, 1, 1), stride2=(1, 1, 1, 1), stride3=(1, 1, 1, 1),
        rshift_mul1=None, rshift_sum1=None, rshift_out1=None,
        rshift_mul2=None, rshift_sum2=None, rshift_out2=None,
        rshift_mul3=None, rshift_sum3=None, rshift_out3=None,
        act_func1=None, act_func2=None, act_func3=None,
        par_ich1=1, par_och1=1, par_col1=1, par_row1=1,
        concur_och1=None, stationary1='filter',
        par_ich2=1, par_och2=1, par_col2=1, par_row2=1,
        concur_och2=None, stationary2='filter',
        par_ich3=1, par_och3=1, par_col3=1, par_row3=1,
        concur_och3=None, stationary3='filter',
        input_ram_size1=None, filter_ram_size1=None,
        bias_ram_size1=None, scale_ram_size1=None,
        out_ram_size1=None,
        input_ram_size2=None, filter_ram_size2=None,
        bias_ram_size2=None, scale_ram_size2=None,
        out_ram_size2=None,
        input_ram_size3=None, filter_ram_size3=None,
        bias_ram_size3=None, scale_ram_size3=None,
        out_ram_size3=None,
        axi_datawidth=32, silent=False,
        filename=None, simtype='iverilog', outputfile=None):

    # create target hardware
    act = ng.placeholder(act_dtype, shape=act_shape, name='act')

    weight1 = ng.variable(weight1_dtype, shape=weight1_shape,
                          name='weight1')

    if bias1_shape is not None:
        bias1 = ng.variable(bias1_dtype, bias1_shape, name='bias1')
    else:
        bias1 = None

    if scale1_shape is not None:
        scale1 = ng.variable(scale1_dtype, scale1_shape, name='scale1')
    else:
        scale1 = None

    weight2 = ng.variable(weight2_dtype, shape=weight2_shape,
                          name='weight2')

    if bias2_shape is not None:
        bias2 = ng.variable(bias2_dtype, bias2_shape, name='bias2')
    else:
        bias2 = None

    if scale2_shape is not None:
        scale2 = ng.variable(scale2_dtype, scale2_shape, name='scale2')
    else:
        scale2 = None

    weight3 = ng.variable(weight3_dtype, shape=weight3_shape,
                          name='weight3')

    if bias3_shape is not None:
        bias3 = ng.variable(bias3_dtype, bias3_shape, name='bias3')
    else:
        bias3 = None

    if scale3_shape is not None:
        scale3 = ng.variable(scale3_dtype, scale3_shape, name='scale3')
    else:
        scale3 = None

    tmp = ng.conv2d(act, weight1, stride1,
                    bias1, scale1,
                    rshift_mul1, rshift_sum1, rshift_out1,
                    act_func1, 'SAME', False,
                    tmp_dtype, ng.int32, ng.int32,
                    'conv2d_1',
                    par_ich1, par_och1, par_col1, par_row1,
                    concur_och1, stationary1,
                    input_ram_size1, filter_ram_size1,
                    bias_ram_size1, scale_ram_size1,
                    None, None, None,
                    out_ram_size1)

    tmp = ng.conv2d(tmp, weight2, stride2,
                    bias2, scale2,
                    rshift_mul2, rshift_sum2, rshift_out2,
                    act_func2, 'SAME', False,
                    out_dtype, ng.int32, ng.int32,
                    'conv2d_2',
                    par_ich2, par_och2, par_col2, par_row2,
                    concur_och2, stationary2,
                    input_ram_size2, filter_ram_size2,
                    bias_ram_size2, scale_ram_size2,
                    None, None, None,
                    out_ram_size2)

    out = ng.conv2d(tmp, weight3, stride3,
                    bias3, scale3,
                    rshift_mul3, rshift_sum3, rshift_out3,
                    act_func3, 'SAME', False,
                    out_dtype, ng.int32, ng.int32,
                    'conv2d_3',
                    par_ich3, par_och3, par_col3, par_row3,
                    concur_och3, stationary3,
                    input_ram_size3, filter_ram_size3,
                    bias_ram_size3, scale_ram_size3,
                    None, None, None,
                    out_ram_size3)

    targ = ng.to_veriloggen([out], 'matrix_conv2d_conv2d_conv2d', silent=silent,
                            config={'maxi_datawidth': axi_datawidth})

    # verification data
    vact = np.arange(act.length, dtype=np.int64).reshape(act.shape) % [16]

    vweight1 = np.arange(weight1.length,
                         dtype=np.int64).reshape(weight1_shape) % [32] - [16]

    if bias1 is not None:
        vbias1 = np.arange(bias1.length,
                           dtype=np.int64).reshape(bias1.shape) % [16]
    else:
        vbias1 = None

    if scale1 is not None:
        vscale1 = np.arange(scale1.length,
                            dtype=np.int64).reshape(scale1.shape) % [8]
    else:
        vscale1 = None

    vweight2 = np.arange(weight2.length,
                         dtype=np.int64).reshape(weight2_shape) % [32] - [16]

    if bias2 is not None:
        vbias2 = np.arange(bias2.length,
                           dtype=np.int64).reshape(bias2.shape) % [16]
    else:
        vbias2 = None

    if scale2 is not None:
        vscale2 = np.arange(scale2.length,
                            dtype=np.int64).reshape(scale2.shape) % [8]
    else:
        vscale2 = None

    vweight3 = np.arange(weight3.length,
                         dtype=np.int64).reshape(weight3_shape) % [32] - [16]

    if bias3 is not None:
        vbias3 = np.arange(bias3.length,
                           dtype=np.int64).reshape(bias3.shape) % [16]
    else:
        vbias3 = None

    if scale3 is not None:
        vscale3 = np.arange(scale3.length,
                            dtype=np.int64).reshape(scale3.shape) % [8]
    else:
        vscale3 = None

    eval_outs = ng.eval([out], act=vact, weight1=vweight1, bias1=vbias1, scale1=vscale1,
                        weight2=vweight2, bias2=vbias2, scale2=vscale2,
                        weight3=vweight3, bias3=vbias3, scale3=vscale3)
    vout = eval_outs[0]

    # to memory image
    size_max = int(math.ceil(max(act.memory_size, weight1.memory_size,
                                 bias1.memory_size if bias1 is not None else 0,
                                 scale1.memory_size if scale1 is not None else 0,
                                 weight2.memory_size,
                                 bias2.memory_size if bias2 is not None else 0,
                                 scale2.memory_size if scale2 is not None else 0,
                                 weight3.memory_size,
                                 bias3.memory_size if bias3 is not None else 0,
                                 scale3.memory_size if scale3 is not None else 0,
                                 out.memory_size) / 4096)) * 4096
    check_addr = max(act.addr, weight1.addr,
                     bias1.addr if bias1 is not None else -1,
                     scale1.addr if scale1 is not None else -1,
                     weight2.addr,
                     bias2.addr if bias2 is not None else -1,
                     scale2.addr if scale2 is not None else -1,
                     weight3.addr,
                     bias3.addr if bias3 is not None else -1,
                     scale3.addr if scale3 is not None else -1,
                     out.addr) + size_max
    size_check = size_max
    tmp_addr = check_addr + size_check

    memimg_datawidth = 32
    mem = np.zeros([1024 * 1024 * 8 // (memimg_datawidth // 8)], dtype=np.int64)
    mem = mem + [100]

    axi.set_memory(mem, vact, memimg_datawidth,
                   act_dtype.width, act.addr,
                   max(int(math.ceil(axi_datawidth / act_dtype.width)), par_ich1))

    axi.set_memory(mem, vweight1, memimg_datawidth,
                   weight1_dtype.width, weight1.addr,
                   max(int(math.ceil(axi_datawidth / weight1_dtype.width)), par_ich1))
    if bias1_shape is not None:
        axi.set_memory(mem, vbias1, memimg_datawidth,
                       bias1_dtype.width, bias1.addr,
                       max(int(math.ceil(axi_datawidth / bias1_dtype.width)), par_och1))
    if scale1_shape is not None:
        axi.set_memory(mem, vscale1, memimg_datawidth,
                       scale1_dtype.width, scale1.addr,
                       max(int(math.ceil(axi_datawidth / scale1_dtype.width)), par_och1))

    axi.set_memory(mem, vweight2, memimg_datawidth,
                   weight2_dtype.width, weight2.addr,
                   max(int(math.ceil(axi_datawidth / weight2_dtype.width)), par_ich2))
    if bias2_shape is not None:
        axi.set_memory(mem, vbias2, memimg_datawidth,
                       bias2_dtype.width, bias2.addr,
                       max(int(math.ceil(axi_datawidth / bias2_dtype.width)), par_och2))
    if scale2_shape is not None:
        axi.set_memory(mem, vscale2, memimg_datawidth,
                       scale2_dtype.width, scale2.addr,
                       max(int(math.ceil(axi_datawidth / scale2_dtype.width)), par_och2))

    axi.set_memory(mem, vweight3, memimg_datawidth,
                   weight3_dtype.width, weight3.addr,
                   max(int(math.ceil(axi_datawidth / weight3_dtype.width)), par_ich3))
    if bias3_shape is not None:
        axi.set_memory(mem, vbias3, memimg_datawidth,
                       bias3_dtype.width, bias3.addr,
                       max(int(math.ceil(axi_datawidth / bias3_dtype.width)), par_och3))
    if scale3_shape is not None:
        axi.set_memory(mem, vscale3, memimg_datawidth,
                       scale3_dtype.width, scale3.addr,
                       max(int(math.ceil(axi_datawidth / scale3_dtype.width)), par_och3))

    axi.set_memory(mem, vout, memimg_datawidth,
                   out_dtype.width, check_addr,
                   max(int(math.ceil(axi_datawidth / out_dtype.width)), par_och3))

    # test controller
    m = Module('test')
    params = m.copy_params(targ)
    ports = m.copy_sim_ports(targ)
    clk = ports['CLK']
    resetn = ports['RESETN']
    rst = m.Wire('RST')
    rst.assign(Not(resetn))

    # AXI memory model
    if outputfile is None:
        outputfile = os.path.splitext(os.path.basename(__file__))[0] + '.out'

    memimg_name = 'memimg_' + outputfile

    memory = axi.AxiMemoryModel(m, 'memory', clk, rst,
                                datawidth=axi_datawidth,
                                memimg=mem, memimg_name=memimg_name,
                                memimg_datawidth=memimg_datawidth)
    memory.connect(ports, 'maxi')

    # AXI-Slave controller
    _saxi = vthread.AXIMLite(m, '_saxi', clk, rst, noio=True)
    _saxi.connect(ports, 'saxi')

    # timer
    time_counter = m.Reg('time_counter', 32, initval=0)
    seq = Seq(m, 'seq', clk, rst)
    seq(
        time_counter.inc()
    )

    def ctrl():
        for i in range(100):
            pass

        ng.sim.set_global_addrs(_saxi, tmp_addr)

        start_time = time_counter.value
        ng.sim.start(_saxi)

        print('# start')

        ng.sim.wait(_saxi)
        end_time = time_counter.value

        print('# end')
        print('# execution cycles: %d' % (end_time - start_time))

        # verify
        ok = True
        for bat in range(out.shape[0]):
            for y in range(out.shape[1]):
                for x in range(out.shape[2]):
                    for ch in range(out.shape[3]):
                        orig = memory.read_word(
                            bat * out.aligned_shape[1] * out.aligned_shape[2] * out.aligned_shape[3] +
                            y * out.aligned_shape[2] * out.aligned_shape[3] +
                            x * out.aligned_shape[3] + ch,
                            out.addr, out_dtype.width)
                        check = memory.read_word(
                            bat * out.aligned_shape[1] * out.aligned_shape[2] * out.aligned_shape[3] +
                            y * out.aligned_shape[2] * out.aligned_shape[3] +
                            x * out.aligned_shape[3] + ch,
                            check_addr, out_dtype.width)
                        if vthread.verilog.NotEql(orig, check):
                            print('NG (', bat, y, x, ch,
                                  ') orig: ', orig, ' check: ', check)
                            ok = False
                        # else:
                        #    print('OK (', bat, y, x, ch,
                        #          ') orig: ', orig, ' check: ', check)

        if ok:
            print('# verify: PASSED')
        else:
            print('# verify: FAILED')

        vthread.finish()

    th = vthread.Thread(m, 'th_ctrl', clk, rst, ctrl)
    fsm = th.start()

    uut = m.Instance(targ, 'uut',
                     params=m.connect_params(targ),
                     ports=m.connect_ports(targ))

    # simulation.setup_waveform(m, uut)
    simulation.setup_clock(m, clk, hperiod=5)
    init = simulation.setup_reset(m, resetn, m.make_reset(), period=100, polarity='low')

    init.add(
        Delay(10000000),
        Systask('finish'),
    )

    # output source code
    if filename is not None:
        m.to_verilog(filename)

    # run simulation
    sim = simulation.Simulator(m, sim=simtype)
    rslt = sim.run(outputfile=outputfile)
    lines = rslt.splitlines()
    if simtype == 'verilator' and lines[-1].startswith('-'):
        rslt = '\n'.join(lines[:-1])
    return rslt

if __name__ == '__main__':
    rslt = run(silent=False, filename='tmp.v')
    print(rslt)

max_stars_repo_name: RyusukeYamano/nngen

max_issues_repo_name: RyusukeYamano/nngen

max_forks_repo_name: RyusukeYamano/nngen


text:
# Copyright 2018 The TensorFlow Probability Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Tests for Gradient."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# Dependency imports
import numpy as np

import tensorflow.compat.v2 as tf
import tensorflow_probability as tfp

from tensorflow_probability.python.internal import test_util
from tensorflow_probability.python.math.gradient import batch_jacobian
from tensorflow_probability.python.math.gradient import value_and_gradient_with_auto_expansion

tfd = tfp.distributions
tfm = tfp.math

@test_util.test_all_tf_execution_regimes
class GradientTest(test_util.TestCase):

  @test_util.numpy_disable_gradient_test
  def test_non_list(self):
    f = lambda x: x**2 / 2
    g = lambda x: x
    x = np.concatenate([np.linspace(-100, 100, int(1e1)), [0]], axis=0)
    y, dydx = self.evaluate(tfm.value_and_gradient(f, x))
    self.assertAllClose(f(x), y, atol=1e-6, rtol=1e-6)
    self.assertAllClose(g(x), dydx, atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  def test_list(self):
    f = lambda x, y: x * y
    g = lambda x, y: [y, x]
    args = [np.linspace(0, 100, int(1e1)),
            np.linspace(-100, 0, int(1e1))]
    y, dydx = self.evaluate(tfm.value_and_gradient(f, args))
    self.assertAllClose(f(*args), y, atol=1e-6, rtol=1e-6)
    self.assertAllClose(g(*args), dydx, atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  def test_nested(self):
    f = lambda value: value['x'] * value['y']
    g = lambda value: {'x': value['y'], 'y': value['x']}
    args = {'x': np.linspace(0, 100, int(1e1)),
            'y': np.linspace(-100, 0, int(1e1))}
    y, dydx = self.evaluate(tfm.value_and_gradient(f, args))
    self.assertAllClose(f(args), y, atol=1e-6, rtol=1e-6)
    self.assertAllCloseNested(g(args), dydx, atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  def test_output_list(self):
    f = lambda x, y: [x, x * y]
    g = lambda x, y: [y + 1., x]
    args = [np.linspace(0, 100, int(1e1)),
            np.linspace(-100, 0, int(1e1))]
    y, dydx = self.evaluate(tfm.value_and_gradient(f, args))
    self.assertAllClose(f(*args), y, atol=1e-6, rtol=1e-6)
    self.assertAllClose(g(*args), dydx, atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  @test_util.jax_disable_test_missing_functionality('value_and_gradient')
  def test_multi_input_old_style(self):
    arg0 = [2., 3., 4.]
    arg1 = [5., 6., 7.]
    f_actual = lambda x, y: x * np.log(y)
    g_actual = lambda x, y: (np.log(y), x / np.array(y))
    y, dydx = self.evaluate(
        tfm.value_and_gradient(lambda x, y: x * tf.math.log(y), [arg0, arg1]))
    self.assertAllClose(f_actual(arg0, arg1), y, atol=1e-6, rtol=1e-6)
    self.assertAllClose(g_actual(arg0, arg1), dydx, atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  @test_util.jax_disable_test_missing_functionality('value_and_gradient')
  def test_multi_input_no_auto_unpack(self):
    arg0 = [2., 3., 4.]
    arg1 = [5., 6., 7.]
    f_actual = lambda x, y: x * np.log(y)
    g_actual = lambda x, y: (np.log(y), x / np.array(y))

    # This is how users would typically write things.
    y, dydx = self.evaluate(
        tfm.value_and_gradient(lambda x, y: x * tf.math.log(y), arg0, arg1))
    self.assertAllClose(f_actual(arg0, arg1), y, atol=1e-6, rtol=1e-6)
    self.assertAllClose(g_actual(arg0, arg1), dydx, atol=1e-6, rtol=1e-6)

    # This is uncommon but possible and unambigous under new style.
    y, dydx = self.evaluate(tfm.value_and_gradient(
        lambda x: x[0] * tf.math.log(x[1]), [arg0, arg1],
        auto_unpack_single_arg=False))
    self.assertAllClose(f_actual(arg0, arg1), y, atol=1e-6, rtol=1e-6)
    self.assertAllClose(g_actual(arg0, arg1), dydx, atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  @test_util.jax_disable_test_missing_functionality('value_and_gradient')
  def test_simple_input_no_auto_unpack(self):
    x = [1., 2., 3.]
    y, dydx = self.evaluate(tfm.value_and_gradient(
        tf.math.log, x, auto_unpack_single_arg=False))
    self.assertAllClose(np.log(x), y, atol=1e-6, rtol=1e-6)
    self.assertAllClose(1. / np.array(x), dydx, atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  @test_util.jax_disable_test_missing_functionality('value_and_gradient')
  def test_variable_and_constant_identical(self):
    expected = (2. * np.log(2.), [1. + np.log(2.), 1. + np.log(2.)])
    x = tf.constant(2.)
    self.assertAllClose(
        expected,
        self.evaluate(tfp.math.value_and_gradient(
            lambda a, b: a * tf.math.log(x), x, x)),
        atol=1e-6, rtol=1e-6)
    x = tf.Variable(2.)
    self.evaluate(x.initializer)
    self.assertAllClose(
        expected,
        self.evaluate(tfp.math.value_and_gradient(
            lambda a, b: a*tf.math.log(x), x, x)),
        atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  @test_util.jax_disable_test_missing_functionality('value_and_gradient')
  def test_docstring_examples(self):
    # Case 1: argless `f`.
    x = tf.constant(2.)
    self.assertAllClose(
        [np.log(2.), 0.5],
        self.evaluate(tfm.value_and_gradient(lambda: tf.math.log(x), x)),
        atol=1e-6, rtol=1e-6)

    # Case 2: packed arguments.
    self.assertAllClose(
        [2. * np.log(3.), (np.log(3.), 2. / 3)],
        self.evaluate(tfm.value_and_gradient(
            lambda x, y: x * tf.math.log(y), [2., 3.])),
        atol=1e-6, rtol=1e-6)

    # Case 3: default.
    x = np.array([1., 2, 3])
    self.assertAllClose(
        (np.log(x), 1. / x),
        self.evaluate(tfm.value_and_gradient(
            tf.math.log, [1., 2., 3.], auto_unpack_single_arg=False)),
        atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  @test_util.jax_disable_test_missing_functionality('value_and_gradient')
  def test_variable_tracking(self):
    value_and_gradient = value_and_gradient_with_auto_expansion
    q = tfd.Normal(tf.Variable(1.), tf.Variable(1., trainable=False))
    r = tfd.Normal(0., tf.Variable(1.))
    self.evaluate([v.initializer for v in q.variables + r.variables])

    y, dydx = self.evaluate(value_and_gradient(tfd.kl_divergence, q, r))
    self.assertAllClose(0.5, y, atol=1e-6, rtol=1e-6)
    self.assertAllClose([[1.], [-1.]], dydx, atol=1e-6, rtol=1e-6)

    y, dydx = self.evaluate(value_and_gradient(
        lambda: tfd.kl_divergence(q, r), q, r))
    self.assertAllClose(0.5, y, atol=1e-6, rtol=1e-6)
    self.assertAllClose([[1.], [-1.]], dydx, atol=1e-6, rtol=1e-6)

    y, dydx = self.evaluate(value_and_gradient(
        lambda: tfd.kl_divergence(q, r), *q.trainable_variables, r))
    self.assertAllClose(0.5, y, atol=1e-6, rtol=1e-6)
    self.assertAllClose([1., [-1.]], dydx, atol=1e-6, rtol=1e-6)

    y, dydx = self.evaluate(value_and_gradient(
        lambda *_: tfd.kl_divergence(q, r), q, r))
    self.assertAllClose(0.5, y, atol=1e-6, rtol=1e-6)
    self.assertAllClose([[1.], [-1.]], dydx, atol=1e-6, rtol=1e-6)

    y, dydx = self.evaluate(value_and_gradient(
        lambda **kw: tfd.kl_divergence(tfd.Normal(kw['loc_q'], 1),  # pylint: disable=g-long-lambda
                                       tfd.Normal(0, kw['scale_r'])),
        loc_q=1., scale_r=1.))
    self.assertAllClose(0.5, y, atol=1e-6, rtol=1e-6)
    self.assertAllClose({'loc_q': 1., 'scale_r': -1.}, dydx,
                        atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  def test_output_gradients(self):
    jacobian = np.float32([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]])
    f = lambda x: tf.squeeze(tf.matmul(jacobian, x[:, tf.newaxis]))
    x = np.ones([3], dtype=np.float32)
    output_gradients = np.float32([1., 2., 3.])
    y, dydx = self.evaluate(
        tfm.value_and_gradient(f, x, output_gradients=output_gradients))
    self.assertAllClose(f(x), y, atol=1e-6, rtol=1e-6)
    self.assertAllClose(
        np.dot(output_gradients, jacobian), dydx, atol=1e-6, rtol=1e-6)

  @test_util.numpy_disable_gradient_test
  def test_batch_jacobian(self):
    w = tf.reshape(tf.range(12.) + 1., (4, 3))
    def f(x):
      return tf.math.cumsum(w * x, axis=-1)

    self.assertAllEqual(
        tf.convert_to_tensor([
            [[1., 0., 0.], [1., 2., 0.], [1., 2., 3.]],
            [[4., 0., 0.], [4., 5., 0.], [4., 5., 6.]],
            [[7., 0., 0.], [7., 8., 0.], [7., 8., 9.]],
            [[10., 0., 0.], [10., 11., 0.], [10., 11., 12.]]]),
        batch_jacobian(f, tf.ones((4, 3))))

  @test_util.numpy_disable_gradient_test
  def test_batch_jacobian_larger_rank_and_dtype(self):
    w1 = tf.reshape(tf.range(24., dtype=tf.float64) + 1., (4, 2, 3))
    w2 = tf.reshape(tf.range(24., dtype=tf.float32) * 0.5 - 6., (4, 2, 1, 3))
    def f(x, y):  # [4, 2, 3], [4, 2, 1, 3] -> [4, 3, 2]
      return tf.transpose(
          tf.cast(tf.math.cumsum(w1 * x, axis=-1), dtype=tf.float32)
          + tf.square(tf.reverse(w2 * y, axis=[-3]))[..., 0, :],
          perm=[0, 2, 1])

    x = tf.cast(np.random.uniform(size=(4, 2, 3)), dtype=tf.float64)
    y = tf.cast(np.random.uniform(size=(4, 2, 1, 3)), dtype=tf.float32)
    jac = batch_jacobian(f, [x, y])

    # Check shapes.
    self.assertLen(jac, 2)
    self.assertAllEqual([4, 3, 2, 2, 3], jac[0].shape)
    self.assertAllEqual([4, 3, 2, 2, 1, 3], jac[1].shape)
    self.assertEqual(tf.float64, jac[0].dtype)
    self.assertEqual(tf.float32, jac[1].dtype)

    # Check results against `value_and_gradient`.
    out_shape = f(x, y).shape[1:]
    for i in range(np.prod(out_shape)):
      idx = (slice(None),) + np.unravel_index(i, out_shape)
      # pylint: disable=cell-var-from-loop
      _, grad = tfm.value_and_gradient(lambda x, y: f(x, y)[idx], [x, y])
      print(grad[0].shape, jac[0].shape, jac[0][idx].shape)
      self.assertAllClose(grad[0], jac[0][idx])
      self.assertAllClose(grad[1], jac[1][idx])

  @test_util.numpy_disable_gradient_test
  def test_aux(self):
    x = tf.constant([[2.]])

    def f(x):
      return x**2, x

    (y, aux), dx = tfm.value_and_gradient(f, x, has_aux=True)

    self.assertAllClose(x**2, y)
    self.assertAllClose(2 * x, dx)
    self.assertAllClose(x, aux)

    dx, aux = batch_jacobian(f, x, has_aux=True)

    self.assertAllClose((2 * x)[..., tf.newaxis], dx)
    self.assertAllClose(x, aux)

  @test_util.numpy_disable_gradient_test
  def test_aux_multi_arg(self):
    x = tf.constant([[2.]])
    z = tf.constant([[3.]])

    def f(x, z):
      return x**2 + z**2, (x, z)

    (y, aux), (dx, dz) = tfm.value_and_gradient(f, (x, z), has_aux=True)

    self.assertAllClose(x**2 + z**2, y)
    self.assertAllClose(2 * x, dx)
    self.assertAllClose(2 * z, dz)
    self.assertAllClose(x, aux[0])
    self.assertAllClose(z, aux[1])

    (dx, dz), aux = batch_jacobian(f, (x, z), has_aux=True)

    self.assertAllClose((2 * x)[..., tf.newaxis], dx)
    self.assertAllClose((2 * z)[..., tf.newaxis], dz)
    self.assertAllClose(x, aux[0])
    self.assertAllClose(z, aux[1])

  @test_util.numpy_disable_gradient_test
  def test_aux_grads(self):
    """Tests that gradients flow through the auxiliary output."""
    x = tf.constant([[2.]])

    def f(x):
      return x**2, x**2

    def f2(x):
      (_, aux), _ = tfm.value_and_gradient(f, x, has_aux=True)
      return aux

    y, dx = tfm.value_and_gradient(f2, x)
    self.assertAllClose(x**2, y)
    self.assertAllClose(2 * x, dx)

if __name__ == '__main__':
  test_util.main()

ext: py

max_stars_repo_name: NeelGhoshal/probability

max_issues_repo_name: gregorystrubel/probability

max_forks_repo_name: gregorystrubel/probability

reason: import numpy


text:
from numpy.lib.function_base import select
from nlp.nlpModel import NlpModel, TAG_NER_TYPE, CLASSIFIER_TYPE
import spacy
from spacy.util import minibatch, compounding
import random
from spacy.training import Example

class SpacyNERModel(NlpModel):
    def createModel(self):
        """
        Create Ner Tagger model
        return: NONE
        """
        nlpBlank=spacy.blank("en")
        nlpBlank.add_pipe(TAG_NER_TYPE)
        nlpBlank.to_disk(self.path)

    def trainModel(self, trainData):
        """
        Train NER model
        trainData (list): List of train data in below foramt
        [("Pizza is a common fast food.", {"entities": [(0, 5, "FOOD")]})]

        after train it will save mode at
        Location: path
        """
        TRAIN_DATA = trainData
        nlp=spacy.load(self.path)
        nlp.begin_training()
        # Resume training
        optimizer = nlp.resume_training()
        # List of pipes you want to train
        pipe_exceptions = [TAG_NER_TYPE, "trf_wordpiecer", "trf_tok2vec"]
        # List of pipes which should remain unaffected in training
        other_pipes = [pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions]
        # Begin training by disabling other pipeline components
        with nlp.disable_pipes(*other_pipes) :
            sizes = compounding(1.0, 4.0, 1.001)
            # Training for 30 iterations
            for itn in range(30):
                # shuffle examples before training
                random.shuffle(TRAIN_DATA)
                # batch up the examples using spaCy's minibatch
                batches = minibatch(TRAIN_DATA, size=sizes)
                # ictionary to store losses
                losses = {}
                for batch in batches:
                    texts, annotations = batch[0]
                    doc = nlp.make_doc(texts)
                    example = Example.from_dict(doc, annotations)
                    # Calling update() over the iteration
                    nlp.update([example], sgd=optimizer, drop=0.35, losses=losses)

        ## Save model
        nlp.to_disk(self.path)

    def predict(self, text):
        """
        Predict model output for given text
        text (str): Any text
        return : text with label
        [{'text', 'label'}]
        """
        nlp = spacy.load(self.path)
        doc = nlp(text)
        list = []
        for ent in doc.ents:
            list.append({ent.text, ent.label_})
        return list

class SpacyClassifierModel(NlpModel):
    def createModel(self, categories=[]):
        """
        Create Classifier model with categories
        categories (list): List of categories

        return: List of Labels
        """
        nlpBlank=spacy.blank("en")
        nlpBlank.add_pipe(CLASSIFIER_TYPE)
        # Adding the labels to textcat
        textcat = nlpBlank.get_pipe(CLASSIFIER_TYPE)
        for cat in categories:
            textcat.add_label(cat)
        nlpBlank.to_disk(self.path)
        return textcat.labels

    def getCateogories(self, data):
        return set([item['label'] for item in data])

    def prepareData(self, trainData, allCats):
        data = []
        for item in trainData:
            cat = item['label']
            value = {}
            for lb in allCats:
                if (cat == lb):
                    value[lb] = 1
                else:
                    value[lb] = 0
            item  = (item['text'], {'cats': value})
            data.append(item)
        return data

    def trainModel(self, trainData):
        """
        Train Classifier model
        trainData (list): List of train data in below foramt
        expecting train data in this format:
        [{"label": "CRIME", "text": "Any Text"}]
        further convert it in below foramt
        [('Review or Text or sentence',{'cats': {'Category1': 0, 'Category2': 1}})]

        after train it will save mode at
        Location: path
        """
        allCats = self.getCateogories(trainData)
        TRAIN_DATA = self.prepareData(trainData, allCats)
        nlp=spacy.load(self.path)
        # Adding the labels to textcat
        textcat = nlp.get_pipe(CLASSIFIER_TYPE)
        for cat in allCats:
            textcat.add_label(cat)

        # Disabling other components
        other_pipes = [pipe for pipe in nlp.pipe_names if pipe != CLASSIFIER_TYPE]
        with nlp.disable_pipes(*other_pipes):  # only train textcat
            optimizer = nlp.begin_training()
            # Performing training
            for i in range(30):
                losses = {}
                batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
                for batch in batches:
                    texts, annotations = batch[0]
                    doc = nlp.make_doc(texts)
                    example = Example.from_dict(doc, annotations)
                    nlp.update([example], sgd=optimizer, drop=0.2,
                            losses=losses)
        ## Save model
        nlp.to_disk(self.path)

    def predict(self, text):
        """
        Predict model output for given text
        text (str): Any text
        return : categories with confindecs score
        {'Cat1': 0.9, 'Cat2': 0.2}
        """
        nlp = spacy.load(self.path)
        doc = nlp(text)
        return doc.cats

max_stars_repo_name: bombaybrew/oyster

max_issues_repo_name: bombaybrew/oyster

max_forks_repo_name: bombaybrew/oyster

reason: from numpy


text:
import logging

import numpy

import scipy.integrate

from repro.hpo.dispatcher.dispatcher import HPODispatcher

logger = logging.getLogger(__name__)

class DPF(HPODispatcher):
    def __init__(self, space, configurator_config, max_trials, seed,
                 steps_ratio=0.5, asynchronicity=0.5,
                 initial_population=20, final_population=1,
                 window_size=11, max_epochs=10, min_population=3):

        super(DPF, self).__init__(space, configurator_config, max_trials, seed)

        # Hints for start
        self._initial_population = initial_population
        self._max_epochs = max_epochs

        # Static values
        self._steps_ratio = steps_ratio
        self.final_population = final_population
        self.window_size = window_size
        self.padding = int(window_size / 2)
        self.asynchronicity = asynchronicity

        self.metrics = numpy.ones((int(initial_population * 2), max_epochs + 1)) * -1
        self.smooth_metrics = numpy.ones((int(initial_population * 2), max_epochs + 1)) * -1
        self.suspension_matrix = numpy.ones((int(initial_population * 2), max_epochs + 1))

        self.trials = {}

    def increase_metrics(self, axis):
        for name in ['metrics', 'smooth_metrics']:
            setattr(self, name, self._increase_metrics(getattr(self, name), axis))

    def _increase_metrics(self, metrics, axis):
        shape = metrics.shape
        new_metrics = numpy.ones((shape[0] * (2 if axis == 0 else 1),
                                  shape[1] + (10 if axis == 1 else 0))) * -1
        new_metrics[:shape[0], :shape[1]] = metrics
        return new_metrics

    def _observe(self, trial_id, params, step, objective, finished=False, **kwargs):

        super(DPF, self)._observe(trial_id, params, step, objective, finished, **kwargs)

        self.add_trial(trial_id)
        self.add_metric(trial_id, step, objective)
        self.suspension_matrix = self.compute_suspension_matrix(self.get_decision_steps())

    def is_completed(self):
        return len(self.finished) >= self.final_population  # and self.trials_count >= self.max_trials

    def add_trial(self, trial_id):
        added = trial_id not in self.trials

        if added:
            self.trials[trial_id] = len(self.trials)

        return added

    def add_metric(self, trial_id, step, value):
        metric_key = self.trials[trial_id]

        if metric_key >= self.metrics.shape[0]:
            self.increase_metrics(0)
        elif step >= self.metrics.shape[1]:
            self.increase_metrics(1)

        if step < 0:
            return

        self.metrics[metric_key][step] = value

        # NOTE: this may fail if some steps are missing in metrics and have default -1
        if step > 1:
            metrics = self.metrics[metric_key][1:step + 1]
            self.smooth_metrics[metric_key][step] = metrics[numpy.where(metrics > 0)].min()
        else:
            self.smooth_metrics[metric_key][step] = self.metrics[metric_key][step]

    @property
    def trials_count(self):
        return len(self.trials)

    @property
    def population(self):
        return max(self._initial_population, len(self.trials))

    @property
    def max_epochs(self):
        observed = (self.metrics >= 0).sum(1).max()
        return max(self._max_epochs, observed)

    @property
    def n_steps(self):
        return int(self._steps_ratio * self.max_epochs + 0.5)

    @property
    def ratio(self):
        return (self.final_population / self.population) ** (1 / self.n_steps)

    @property
    def percentile(self):
        return int(self.ratio * 100 + 0.5)

    def get_decision_steps(self):
        return self._get_decision_steps()

        # steps = self._get_decision_steps()
        # # Create a mask based on steps
        # suspension = self.compute_suspension_matrix(steps)

        # mask = (self.metrics >= 0)
        # for i, step in enumerate(steps):
        #     mask[:self.trials_count, step:] *= (suspension[:, i] < 1)[:, None]

        # return self._get_decision_steps(mask)

    def _get_decision_steps(self, mask=None):
        """Compute at which steps suspension should occur based on metric history"""
        # Compute variance of performances at each epoch across all trials so far.
        if mask is None:
            mask = (self.metrics >= 0)

        n_points = mask.sum(axis=0)
        # # Ignore epochs where there is less trials than final_population
        mask *= (n_points >= self.final_population)

        max_metrics = numpy.max(self.smooth_metrics * mask, axis=0)
        min_metrics = numpy.min(self.smooth_metrics * mask + (1 - mask) * max_metrics, axis=0)

        diff = max_metrics - min_metrics

        steps = numpy.where(diff > 0)[0]
        if steps.shape[0] > 0:
            first_step = steps[0]
        else:
            return list(range(2, self.n_steps + 2))

        # Compute the cummulative integral of variance over epochs
        # and pick a list of epochs so that all intervals contain the same
        # total amount of variance.
        dcum = scipy.integrate.cumtrapz(diff[first_step:], initial=0)
        block = dcum[-1] / (self.n_steps - first_step)
        steps = []
        j = 1
        for i, accum in enumerate(dcum):
            if accum > block * j:
                j += 1
                steps.append(i + first_step)

        if not steps:
            steps.append(1)

        while len(steps) < self.n_steps and steps[-1] + 1 < self.max_epochs:
            steps.append(steps[-1] + 1)

        return steps

    def compute_suspension_matrix(self, steps):

        suspend = numpy.zeros((self.trials_count, len(steps)))
        populations = []
        pop = []
        for i, step in enumerate(steps):
            mask = self.smooth_metrics[:, step] >= 0
            reached_step = numpy.where(mask)[0]
            population = self.get_population(step, mask)
            populations.append([i, step, population, self.get_min_population(i)])
            if population:
                pop.append((step, population))
            if self.get_population(step, mask) < self.get_min_population(i):
                suspend[reached_step, i] = 1
                continue

            values = self.smooth_metrics[reached_step, step]
            if len(values) < 2:
                continue

            percentile = numpy.percentile(values, self.percentile)

            populations[-1].append(percentile)

            suspend[reached_step, i:] = (values >= percentile)[:, None]

        return suspend

    def get_min_population(self, ith_step):
        return self.population * (self.ratio) ** (ith_step + 1) * (1 - self.asynchronicity)

    def get_population(self, step, mask):
        return mask.sum()

    def should_suspend(self, trial_id):
        row_id = self.trials[trial_id]
        return bool(self.suspension_matrix[row_id].sum() > 0)

    def should_resume(self, trial_id):
        suspend = self.should_suspend(trial_id)
        last_step, obj = self.get_objective(trial_id)
        return not suspend

def build(space, configurator_config, max_trials, seed, **kwargs):
    return DPF(space, configurator_config, max_trials, seed, **kwargs)

max_stars_repo_name: bouthilx/dpd

max_issues_repo_name: bouthilx/dpd

max_forks_repo_name: bouthilx/dpd

reason: import numpy,import scipy


text:
import cmath
import numpy as np

def lag(x0, n, f, fp, fpp, eps):
    while abs(f(x0)) > eps:
        G = fp(x0)/f(x0)
        H = G**2 - fpp(x0)/f(x0)
        st = cmath.sqrt((n-1)*(n*H-G**2))
        b = G + st if abs(G+st) > abs(G-st) else G - st
        a = n/b
        x0 = x0-a
    return x0

def sr(A, b, n=25):
    L = np.tril(A)
    U = A - L
    x = np.zeros(b.shape)
    for i in range(n):
        x = np.dot(np.linalg.inv(L), b-np.dot(U, x))
    return x

# print(lag(0.1, 5, lambda x: x**2 + 1, lambda x: 2*x, lambda x: 2, 0.000001))

A = np.array([[4.0, -2.0, 1.0], [1.0, -3.0, 2.0], [-1.0, 2.0, 6.0]])
b = np.array([1.0, 2.0, 3.0])

print(np.linalg.solve(A, b))
print(sr(A, b))


text:
import theano
import theano.tensor as tensor
from util import ortho_weight, norm_weight, tanh, rectifier, linear
import numpy
from utils import _p

# LSTM layer
def param_init_lstm(options, params, prefix='lstm', nin=None, dim=None):
    if nin is None:
        nin = options['dim_proj']
    if dim is None:
        dim = options['dim_proj']
    """
     Stack the weight matricies for all the gates
     for much cleaner code and slightly faster dot-prods
    """
    # input weights
    W = numpy.concatenate([norm_weight(nin,dim),
                           norm_weight(nin,dim),
                           norm_weight(nin,dim),
                           norm_weight(nin,dim)], axis=1)
    params[_p(prefix,'W')] = W
    # for the previous hidden activation
    U = numpy.concatenate([ortho_weight(dim),
                           ortho_weight(dim),
                           ortho_weight(dim),
                           ortho_weight(dim)], axis=1)
    params[_p(prefix,'U')] = U
    params[_p(prefix,'b')] = numpy.zeros((4 * dim,)).astype('float32')

    return params

# This function implements the lstm forward propagation
def lstm_layer(tparams, state_below, options, prefix='lstm', mask=None, **kwargs):
    nsteps = state_below.shape[0]
    dim = tparams[_p(prefix,'U')].shape[0]

    # if we are dealing with a mini-batch
    if state_below.ndim == 3:
        n_samples = state_below.shape[1]
        init_state = tensor.alloc(0., n_samples, dim)
        init_memory = tensor.alloc(0., n_samples, dim)
    # during sampling
    else:
        n_samples = 1
        init_state = tensor.alloc(0., dim)
        init_memory = tensor.alloc(0., dim)

    # if we have no mask, we assume all the inputs are valid
    if mask == None:
        mask = tensor.alloc(1., state_below.shape[0], 1)

    # use the slice to calculate all the different gates
    def _slice(_x, n, dim):
        if _x.ndim == 3:
            return _x[:, :, n*dim:(n+1)*dim]
        elif _x.ndim == 2:
            return _x[:, n*dim:(n+1)*dim]
        return _x[n*dim:(n+1)*dim]

    # one time step of the lstm
    def _step(m_, x_, h_, c_):
        preact = tensor.dot(h_, tparams[_p(prefix, 'U')])
        preact += x_

        i = tensor.nnet.sigmoid(_slice(preact, 0, dim))
        f = tensor.nnet.sigmoid(_slice(preact, 1, dim))
        o = tensor.nnet.sigmoid(_slice(preact, 2, dim))
        c = tensor.tanh(_slice(preact, 3, dim))

        c = f * c_ + i * c
        h = o * tensor.tanh(c)

        return h, c, i, f, o, preact

    state_below = tensor.dot(state_below, tparams[_p(prefix, 'W')]) + tparams[_p(prefix, 'b')]

    rval, updates = theano.scan(_step,
                                sequences=[mask, state_below],
                                outputs_info=[init_state, init_memory, None, None, None, None],
                                name=_p(prefix, '_layers'),
                                n_steps=nsteps, profile=False)
    return rval

max_stars_repo_name: wenhuchen/ethz-bootstrapped-captioner

max_issues_repo_name: wenhuchen/ethz-bootstrapped-captioner

max_forks_repo_name: wenhuchen/ethz-bootstrapped-captioner

reason: import numpy,import theano


text:
#! -*- coding:utf-8 -*-

import json
import numpy as np
from random import choice
from keras_bert import load_trained_model_from_checkpoint, Tokenizer
import re, os
import codecs

mode = 0
maxlen = 100
learning_rate = 5e-5
min_learning_rate = 1e-5

config_path = './chinese_L-12_H-768_A-12/bert_config.json'
checkpoint_path = './chinese_L-12_H-768_A-12/bert_model.ckpt'
dict_path = './chinese_L-12_H-768_A-12/vocab.txt'

token_dict = {}

with codecs.open(dict_path, 'r', 'utf8') as reader:
    for line in reader:
        token = line.strip()
        token_dict[token] = len(token_dict)

# 重写tokenizer
class OurTokenizer(Tokenizer):
    def _tokenize(self, text):
        R = []
        for c in text:
            if c in self._token_dict:
                R.append(c)
            elif self._is_space(c):
                R.append('[unused1]')  # space类用未经训练的[unused1]表示
            else:
                R.append('[UNK]')  # 剩余的字符是[UNK]
        return R

tokenizer = OurTokenizer(token_dict)

train_data = json.load(open('./data/train_data_real.json'))
id2predicate, predicate2id = json.load(open('./data/all_schemas_me_real.json'))
id2predicate = {int(i): j for i, j in id2predicate.items()}
num_classes = len(id2predicate)

total_data = []
total_data.extend(train_data)

if not os.path.exists('./data/random_order_train.json'):
    random_order = list(range(len(total_data)))
    np.random.shuffle(random_order)
    json.dump(
        random_order,
        open('./data/random_order_train.json', 'w'),
        indent=4
    )
else:
    random_order = json.load(open('./data/random_order_train.json'))

train_data = [total_data[j] for i, j in enumerate(random_order) if i % 8 != mode]
dev_data = [total_data[j] for i, j in enumerate(random_order) if i % 8 == mode]

predicates = {}

for d in train_data:
    for sp in d['spo_list']:
        if sp[2] not in predicates:
            predicates[sp[2]] = []
        predicates[sp[2]].append(sp)

def seq_padding(X, padding=0):
    L = [len(x) for x in X]
    ML = max(L)
    return np.array(
        [
            np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X
        ]
    )

def list_find(list1, list2):
    n_list2 = len(list2)
    for i in list(range(len(list1))):
        if list1[i: i + n_list2] == list2:
            return i
    return -1

class data_generator:
    def __init__(self, data, batch_size=32):
        self.data = data
        self.batch_size = batch_size
        self.steps = len(self.data) // self.batch_size
        if len(self.data) % self.batch_size != 0:
            self.steps += 1

    def __len__(self):
        return self.steps

    def __iter__(self):
        while True:
            idxs = list(range(len(self.data)))
            np.random.shuffle(idxs)
            T1, T2, S1, S2, S3, S4, K1, K2, K3, K4, O1, O2, O3, O4 = [], [], [], [], [], [], [], [], [], [], [], [], [], []
            for i in idxs:
                d = self.data[i]
                text = d['text'][:maxlen]
                tokens = tokenizer.tokenize(text)
                items = {}
                for sp in d['spo_list']:
                    sp = (tokenizer.tokenize(sp[0])[1:-1], tokenizer.tokenize(sp[1])[1:-1], sp[2],
                          tokenizer.tokenize(sp[3])[1:-1], tokenizer.tokenize(sp[4])[1:-1])
                    subjectid = list_find(tokens, sp[0])
                    ssubjectid = list_find(tokens, sp[1])
                    objectid = list_find(tokens, sp[3])
                    oobjectid = list_find(tokens, sp[4])
                    if subjectid != -1 and objectid != -1:
                        key = (subjectid, subjectid + len(sp[0]), ssubjectid, ssubjectid + len(sp[1]))
                        if key not in items:
                            items[key] = []
                        items[key].append((objectid,
                                           objectid + len(sp[3]),
                                           oobjectid,
                                           oobjectid + len(sp[4]),
                                           predicate2id[sp[2]]))
                if items:
                    t1, t2 = tokenizer.encode(first=text)
                    T1.append(t1)
                    T2.append(t2)
                    s1, s2, s3, s4 = np.zeros(len(tokens)), np.zeros(len(tokens)), np.zeros(len(tokens)), np.zeros(len(tokens))
                    # 追加sub_Task
                    for j in items:
                        s1[j[0]] = 1
                        s2[j[1] - 1] = 1
                        s3[j[2]] = 1
                        s4[j[3] - 1] = 1
                    k1, k2, k3, k4 = np.array(list(items.keys())).T
                    k1 = choice(k1)
                    k2 = choice(k2[k2 >= k1])
                    k3 = choice(k3)
                    k4 = choice(k4[k4 >= k3])
                    o1, o2, o3, o4 = np.zeros((len(tokens), num_classes)), np.zeros((len(tokens), num_classes)), \
                                     np.zeros((len(tokens), num_classes)), np.zeros((len(tokens), num_classes))
                    # 追加object_Task
                    for j in items.get((k1, k2), []):
                        o1[j[0]][j[4]] = 1
                        o2[j[1] - 1][j[4]] = 1
                        o3[j[2]][j[4]] = 1
                        o4[j[3] - 1][j[4]] = 1
                    S1.append(s1)
                    S2.append(s2)
                    S3.append(s3)
                    S4.append(s4)
                    K1.append([k1])
                    K2.append([k2 - 1])
                    K3.append([k3])
                    K4.append([k4 - 1])
                    O1.append(o1)
                    O2.append(o2)
                    O3.append(o3)
                    O4.append(o4)
                    if len(T1) == self.batch_size or i == idxs[-1]:
                        T1 = seq_padding(T1)
                        T2 = seq_padding(T2)
                        S1 = seq_padding(S1)
                        S2 = seq_padding(S2)
                        S3 = seq_padding(S3)
                        S4 = seq_padding(S4)
                        O1 = seq_padding(O1, np.zeros(num_classes))
                        O2 = seq_padding(O2, np.zeros(num_classes))
                        O3 = seq_padding(O3, np.zeros(num_classes))
                        O4 = seq_padding(O4, np.zeros(num_classes))
                        K1, K2, K3, K4 = np.array(K1), np.array(K2), np.array(K3), np.array(K4)
                        yield [T1, T2, S1, S2, S3, S4, K1, K2, K3, K4, O1, O2, O3, O4], None
                        T1, T2, S1, S2, S3, S4, K1, K2, K3, K4, O1, O2, O3, O4 = [], [], [], [], [], [], [], [], [], [], [], [], [], []

# Bert预训练模型开始

from keras.layers import *
from keras.models import Model
import keras.backend as K
import tensorflow as tf
from keras.callbacks import Callback
from keras.optimizers import Adam

def seq_gather(x):
    # seq是[none,seq_len,s_size]的格式，idxs是[None,1]的格式
    # 在seq的第i个序列中选出第i个向量，最终输出[None,s_size]的向量
    seq, idxs = x
    idxs = K.cast(idxs, 'int32')
    batch_idxs = K.arange(0, K.shape(seq)[0])
    batch_idxs = K.expand_dims(batch_idxs, 1)
    idxs = K.concatenate([batch_idxs, idxs], 1)
    return tf.gather_nd(seq, idxs)

bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None)

for l in bert_model.layers:
    l.trainable = True

t1_in = Input(shape=(None,))
t2_in = Input(shape=(None,))
s1_in = Input(shape=(None,))
s2_in = Input(shape=(None,))
s3_in = Input(shape=(None,))
s4_in = Input(shape=(None,))
k1_in = Input(shape=(1,))
k2_in = Input(shape=(1,))
k3_in = Input(shape=(1,))
k4_in = Input(shape=(1,))
o1_in = Input(shape=(None, num_classes))
o2_in = Input(shape=(None, num_classes))
o3_in = Input(shape=(None, num_classes))
o4_in = Input(shape=(None, num_classes))

t1, t2, s1, s2, s3, s4, k1, k2, k3, k4, o1, o2, o3, o4 = t1_in, t2_in, s1_in, s2_in, s3_in, s4_in, k1_in, k2_in, k3_in, k4_in, o1_in, o2_in, o3_in, o4_in

mask = Lambda(lambda x: K.cast(K.greater(K.expand_dims(x, 2), 0), 'float32'))(t1)

t = bert_model([t1, t2])
ps1 = Dense(1, activation='sigmoid')(t)
ps2 = Dense(1, activation='sigmoid')(t)
ps3 = Dense(1, activation='sigmoid')(t)
ps4 = Dense(1, activation='sigmoid')(t)

subject_model = Model([t1_in, t2_in], [ps1, ps2, ps3, ps4])

k1v = Lambda(seq_gather)([t, k1])
k2v = Lambda(seq_gather)([t, k2])
k3v = Lambda(seq_gather)([t, k3])
k4v = Lambda(seq_gather)([t, k4])

kv = Average()([k1v, k2v, k3v, k4v])
t = Add()([t, kv])
po1 = Dense(num_classes, activation='sigmoid')(t)
po2 = Dense(num_classes, activation='sigmoid')(t)
po3 = Dense(num_classes, activation='sigmoid')(t)
po4 = Dense(num_classes, activation='sigmoid')(t)

object_model = Model([t1_in, t2_in, k1_in, k2_in, k3_in, k4_in], [po1, po2, po3, po4])

train_model = Model([t1_in, t2_in, s1_in, s2_in, s3_in, s4_in, k1_in, k2_in, k3_in, k4_in, o1_in, o2_in, o3_in, o4_in],
                    [ps1, ps2, ps3, ps4, po1, po2, po3, po4])

s1 = K.expand_dims(s1, 2)
s2 = K.expand_dims(s2, 2)
s3 = K.expand_dims(s3, 2)
s4 = K.expand_dims(s4, 2)

s1_loss = K.binary_crossentropy(s1, ps1)
s1_loss = K.sum(s1_loss * mask) / K.sum(mask)
s2_loss = K.binary_crossentropy(s2, ps2)
s2_loss = K.sum(s2_loss * mask) / K.sum(mask)
s3_loss = K.binary_crossentropy(s3, ps3)
s3_loss = K.sum(s3_loss * mask) / K.sum(mask)
s4_loss = K.binary_crossentropy(s4, ps4)
s4_loss = K.sum(s4_loss * mask) / K.sum(mask)

o1_loss = K.sum(K.binary_crossentropy(o1, po1), 2, keepdims=True)
o1_loss = K.sum(o1_loss * mask) / K.sum(mask)
o2_loss = K.sum(K.binary_crossentropy(o2, po2), 2, keepdims=True)
o2_loss = K.sum(o2_loss * mask) / K.sum(mask)
o3_loss = K.sum(K.binary_crossentropy(o3, po3), 2, keepdims=True)
o3_loss = K.sum(o3_loss * mask) / K.sum(mask)
o4_loss = K.sum(K.binary_crossentropy(o1, po1), 2, keepdims=True)
o4_loss = K.sum(o4_loss * mask) / K.sum(mask)

loss = (s1_loss + s2_loss + s3_loss + s4_loss) + (o1_loss + o2_loss + o3_loss + o4_loss)

train_model.add_loss(loss)
train_model.compile(optimizer=Adam(learning_rate))
train_model.summary()

class Evaluate(Callback):
    def __init__(self):
        self.F1 = []
        self.best = 0.
        self.passed = 0
        self.stage = 0

    def on_batch_begin(self, batch, logs=None):
        if self.passed < self.params['steps']:
            lr = (self.passed + 1.) / self.params['steps'] * learning_rate
            K.set_value(self.model.optimizer.lr, lr)
            self.passed += 1
        elif self.params['steps'] <= self.passed < self.params['steps'] * 2:
            lr = (2 - (self.passed + 1.) / self.params['steps']) * (learning_rate - min_learning_rate)
            lr += min_learning_rate
            K.set_value(self.model.optimizer.lr, lr)
            self.passed += 1

train_D = data_generator(train_data)
evaluator = Evaluate()

if __name__ == '__main__':
    train_model.fit_generator(train_D.__iter__(),
                              steps_per_epoch=1000,
                              epochs=40,
                              callbacks=[evaluator]
                              )
else:
    train_model.load_weights('best_model.weights')


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import math
import os
import random
import zipfile

import numpy as np

import tensorflow as tf

from nltk.tokenize import RegexpTokenizer

data_index = 0

def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):
  assert low_dim_embs.shape[0] >= len(labels), "More labels than embeddings"
  plt.figure(figsize=(18, 18))  # in inches
  for i, label in enumerate(labels):
    x, y = low_dim_embs[i, :]
    plt.scatter(x, y)
    plt.annotate(label,
                 xy=(x, y),
                 xytext=(5, 2),
                 textcoords='offset points',
                 ha='right',
                 va='bottom')

  plt.savefig(filename)

def matplotlib_to_plotly(cmap, pl_entries):
    h = 1.0/(pl_entries-1)
    pl_colorscale = []
    for k in range(pl_entries):
        C = map(uint8, array(cmap(k*h)[:3])*255)
        pl_colorscale.append([k*h, 'rgb'+str((C[0], C[1], C[2]))])

    return pl_colorscale

def plot_n_save(data, positions, thecolors, name):
    my_cmap = cm.get_cmap('jet')
    cscale = matplotlib_to_plotly(my_cmap, 255)
    index = arange(0,data.shape[0],1);
    traces = [];
    for i in unique(thecolors):
        trace = go.Scatter3d(
                             x=positions[thecolors == i,0],
                             y=positions[thecolors == i,1],
                             z=positions[thecolors == i,2],
                             mode='markers',
                             name=i,
                             text=data['title'][thecolors == i],
                             #text=["%s"%(i,j) for j, i in zip(data['Document Title'][thecolors == i],data[u'PDF Link'][thecolors == i])],
                             marker=dict(
                                         #color=thecolors[thecolors == i],
                                         colorscale=cscale,            # choose a colorscale
                                         size=8,
                                         opacity=1
                                         )
                             )
        traces.append(trace);
    plot(traces,filename='%s.html'%(name))

def plot2d_n_save(data, positions, thecolors, name):
    from numpy import zeros
    my_cmap = cm.get_cmap('jet')
    cscale = matplotlib_to_plotly(my_cmap, 255)
    index = arange(0,data.shape[0],1);
    traces = [];
    for i in unique(thecolors):
        trace = go.Scatter3d(
                             x=positions[thecolors == i,0],
                             z=positions[thecolors == i,1],
                             y=zeros(positions[thecolors == i].shape[0]),
                             mode='markers',
                             name=i,
                             text=data['title'][thecolors == i],
                             #text=["%s"%(i,j) for j, i in zip(data['Document Title'][thecolors == i],data[u'PDF Link'][thecolors == i])],
                             marker=dict(
                                         #color=thecolors[thecolors == i],
                                         colorscale=cscale,            # choose a colorscale
                                         size=8,
                                         opacity=1
                                         )
                             )
        traces.append(trace);
    plot(traces,filename='%s.html'%(name))

wordtokenizer = RegexpTokenizer(r"(?u)\b\w\w+\b")
sentencetokenizer = RegexpTokenizer(r'[.!?]', gaps = True)

'''
def minimain:
    try:
      from sklearn.manifold import TSNE
      import matplotlib.pyplot as plt

      tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
      plot_only = 500

      #data1, count1, dictionary1, reverse_dictionary1 = build_dataset(data['abstract'][0].split(), vocabulary_size)
      #val0 = pd.DataFrame([low_dim_embs[dictionary[i]] for i in "the robots are coming".split()])
      #val1 = pd.DataFrame([low_dim_embs[dictionary[i]] for i in "the robots are not coming".split()])
      #val2 = pd.DataFrame([low_dim_embs[dictionary[i]] for i in "the previous robot is good".split()])
      #val3 = pd.DataFrame([low_dim_embs[dictionary[i]] for i in "good is the robot previous".split()])

      viscap="Automatically  describing  the  content  of  an  image  is  a fundamental problem in artificial intelligence that connects computer vision and natural language processing.  In this paper, we present a generative model based on a deep recurrent architecture that combines recent advances in computer vision and machine translation and that can be used to  generate  natural  sentences  describing  an  image.   The model is trained to maximize the likelihood of the target description sentence given the training image.   Experiments on several datasets show the accuracy of the model and the fluency of the language it learns solely from image descriptions.  Our model is often quite accurate, which we verify both  qualitatively  and  quantitatively.   For  instance,  while the  current  state-of-the-art  BLEU-1  score  (the  higher  the better) on the Pascal dataset is 25, our approach yields 59, to be compared to human performance around 69. We also show BLEU-1 score improvements on Flickr30k, from 56 to 66, and on SBU, from 19 to 28. Lastly, on the newly released COCO dataset, we achieve a BLEU-4 of 27.7, which is the current state-of-the-art."
      vals=[]
      for j in xrange(4):
        vals.append(pd.DataFrame([low_dim_embs[dictionary[i]] for i in mydata['abstract'][j].split()]))
      vals.append(pd.DataFrame([low_dim_embs[dictionary[i]] for i in viscap.split() if i in dictionary]))
      for dat in [val0, val1, val2, val3]: plt.plot(dat[0])
      for dat in vals: plt.plot(dat[0],'.')
      plt.legend(["the robots are coming","the robots are not coming","the previous robot is good","good is the robot previous"])

      low_dim_embs = tsne.fit_transform(final_embeddings)
      labels = [reverse_dictionary[i] for i in xrange(plot_only)]
      plot_with_labels(low_dim_embs, labels)

    except ImportError:
      print("Please install sklearn, matplotlib, and scipy to visualize embeddings.")
'''

def build_dataset(words, vocabulary_size):
      count = [['UNK', -1]]
      count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
      print(len(count))
      dictionary = dict()
      for word, _ in count:
        dictionary[word] = len(dictionary)
      data = list()
      unk_count = 0
      for word in words:
        if word in dictionary:
          index = dictionary[word]
        else:
          index = 0  # dictionary['UNK']
          unk_count += 1
        data.append(index)
      count[0][1] = unk_count
      #print(count)
      reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
      return data, count, dictionary, reverse_dictionary

    # Step 3: Function to generate a training batch for the skip-gram model.
def generate_batch(data, batch_size, num_skips, skip_window):
      global data_index
      assert batch_size % num_skips == 0
      assert num_skips <= 2 * skip_window
      batch = np.ndarray(shape=(batch_size), dtype=np.int32)
      labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
      span = 2 * skip_window + 1  # [ skip_window target skip_window ]
      buffer = collections.deque(maxlen=span)
      for _ in range(span):
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
      for i in range(batch_size // num_skips):
        target = skip_window  # target label at the center of the buffer
        targets_to_avoid = [skip_window]
        for j in range(num_skips):
          while target in targets_to_avoid:
            target = random.randint(0, span - 1)
          targets_to_avoid.append(target)
          batch[i * num_skips + j] = buffer[skip_window]
          labels[i * num_skips + j, 0] = buffer[target]
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
      # Backtrack a little bit to avoid skipping words in the end of a batch
      data_index = (data_index + len(data) - span) % len(data)
      return batch, labels

def transform(alldata):
    from sys import exc_info
    global data_index

    mysentences = sentencetokenizer.tokenize("".join(alldata))
    mywords = wordtokenizer.tokenize("".join(alldata))
    for i in range(len(mywords)):
        mywords[i] = mywords[i].lower()
    vocabulary_size = len(unique(mywords)) #4712 #4652

    data, count, dictionary, reverse_dictionary = build_dataset(mywords, vocabulary_size)
    data_index = 0

    # Step 4: Build and train a skip-gram model.

    batch_size = 128
    embedding_size = 128  # Dimension of the embedding vector.
    skip_window = 1       # How many words to consider left and right.
    num_skips = 2         # How many times to reuse an input to generate a label.

    # We pick a random validation set to sample nearest neighbors. Here we limit the
    # validation samples to the words that have a low numeric ID, which by
    # construction are also the most frequent.
    valid_size = 50     # Random set of words to evaluate similarity on.
    valid_window = 100  # Only pick dev samples in the head of the distribution.
    valid_examples = np.random.choice(valid_window, valid_size, replace=False)
    num_sampled = 64    # Number of negative examples to sample.

    graph = tf.Graph()

    with graph.as_default():

      # Input data.
      train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
      train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
      valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

      # Ops and variables pinned to the CPU because of missing GPU implementation
      with tf.device('/cpu:0'):
        # Look up embeddings for inputs.
        embeddings = tf.Variable(
            tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        embed = tf.nn.embedding_lookup(embeddings, train_inputs)

        # Construct the variables for the NCE loss
        nce_weights = tf.Variable(
            tf.truncated_normal([vocabulary_size, embedding_size],
                                stddev=1.0 / math.sqrt(embedding_size)))
        nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

      # Compute the average NCE loss for the batch.
      # tf.nce_loss automatically draws a new sample of the negative labels each
      # time we evaluate the loss.
      loss = tf.reduce_mean(
          tf.nn.nce_loss(weights=nce_weights,
                         biases=nce_biases,
                         labels=train_labels,
                         inputs=embed,
                         num_sampled=num_sampled,
                         num_classes=vocabulary_size))

      # Construct the SGD optimizer using a learning rate of 1.0.
      optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)

      # Compute the cosine similarity between minibatch examples and all embeddings.
      norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
      normalized_embeddings = embeddings / norm
      valid_embeddings = tf.nn.embedding_lookup(
          normalized_embeddings, valid_dataset)
      similarity = tf.matmul(
          valid_embeddings, normalized_embeddings, transpose_b=True)

      # Add variable initializer.
      init = tf.global_variables_initializer()

    # Step 5: Begin training.
    num_steps = 50001#100001

    with tf.Session(graph=graph) as session:
      # We must initialize all variables before we use them.
      init.run()
      print("Initialized")

      average_loss = 0
      for step in xrange(num_steps):
        batch_inputs, batch_labels = generate_batch( data,
            batch_size, num_skips, skip_window)
        feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}

        # We perform one update step by evaluating the optimizer op (including it
        # in the list of returned values for session.run()
        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
        average_loss += loss_val

        if step % 2000 == 0:
          if step > 0:
            average_loss /= 2000
          # The average loss is an estimate of the loss over the last 2000 batches.
          print("Average loss at step ", step, ": ", average_loss)
          average_loss = 0

        # Note that this is expensive (~20% slowdown if computed every 500 steps)
        if step % 10000 == 0:
          sim = similarity.eval()
          for i in xrange(valid_size):
            valid_word = reverse_dictionary[valid_examples[i]]
            top_k = 8  # number of nearest neighbors
            nearest = (-sim[i, :]).argsort()[1:top_k + 1]
            log_str = "Nearest to %s:" % valid_word
            for k in xrange(top_k):
              try:
                close_word = reverse_dictionary[nearest[k]]
              except:
                print(exc_info()[0], k, i, nearest, log_str)
              log_str = "%s %s," % (log_str, close_word)
            print(log_str)
      final_embeddings = normalized_embeddings.eval()
    return final_embeddings, dictionary, reverse_dictionary, count

def transform_abstracts(alldata, final_embeddings, dictionary, reverse_dictionary):
    from sys import exc_info
    # Step 6: Visualize the embeddings.
    pca=PCA(n_components=2, random_state=1);
    plotlen = 500;#len(reverse_dictionary)
    low_dim_embs = pca.fit_transform(final_embeddings[:plotlen])
    labels = [reverse_dictionary[i].encode('utf-8') for i in xrange(plotlen)]
    plot_with_labels(low_dim_embs, labels)

    embabs = []
    for abstract in alldata:
        mysentences = sentencetokenizer.tokenize(abstract)
        embsentences =  []
        for l in range(len(mysentences)):
                sentence = mysentences[l]
                try:
                    embsentences.append(array([low_dim_embs[dictionary[i.lower()]] for i in wordtokenizer.tokenize(sentence)]).flatten())
                except:
                    print(exc_info(), sentence)
        embabs.append(array(embsentences))

    embabstracts = array(embabs)
    senences = sentencetokenizer.tokenize("".join(mydata['abstract']))
    words = wordtokenizer.tokenize("".join(mydata['abstract']))

    return embabstracts;

'''

[low_dim_embs[dictionary[i.lower()]] for sentence in sentences for i in sentence.split()]
[[i.lower()] for sentence in sentences for i in sentence.split()]
[i for sentence in sentences for i in sentence.split()]

for j in xrange(4):
    vals.append(
                pd.DataFrame([low_dim_embs[dictionary[i]] for i in j.split() for j in sentences])
                )

data, count, dictionary, reverse_dictionary = build_dataset(mywords, vocabulary_size)

pdata, pcount, pdictionary, preverse_dictionary = build_dataset(array(embabstracts).flatten(), vocabulary_size)

data_index = 0

'''

max_stars_repo_name: slremy/cloudroboticsreferences

max_issues_repo_name: slremy/cloudroboticsreferences

max_forks_repo_name: slremy/cloudroboticsreferences

reason: import numpy,from numpy


text:
"""
Created on 9 Mar, 2018 from mapclientplugins.meshgeneratorstep.

@author: Richard Christie
"""

import string

from opencmiss.zinc.field import Field
from opencmiss.zinc.glyph import Glyph
from opencmiss.zinc.graphics import Graphics
from opencmiss.zinc.node import Node
from scaffoldmaker.scaffoldmaker import Scaffoldmaker
from scaffoldmaker.utils.zinc_utils import *
import numpy as np

from mapclientplugins.meshgeneratorstep.model.meshalignmentmodel import MeshAlignmentModel

STRING_FLOAT_FORMAT = '{:.8g}'

class MeshGeneratorModel(MeshAlignmentModel):
    """
    Framework for generating meshes of a number of types, with mesh type specific options
    """

    def __init__(self, region, material_module):
        super(MeshGeneratorModel, self).__init__()  # is this not a super of the class we are in?
        self._region_name = "generated_mesh"
        self._parent_region = region
        self._materialmodule = material_module
        self._region = None
        self._sceneChangeCallback = None
        self._deleteElementRanges = []
        self._scale = [ 1.0, 1.0, 1.0 ]
        self._settings = {
            'meshTypeName' : '',
            'meshTypeOptions' : { },
            'deleteElementRanges' : '',
            'scale' : '*'.join(STRING_FLOAT_FORMAT.format(value) for value in self._scale),
            'displayAxes' : True,
            'displayElementNumbers' : True,
            'displayLines' : True,
            'displayNodeDerivatives' : False,
            'displayNodeNumbers' : True,
            'displaySurfaces' : True,
            'displaySurfacesExterior' : True,
            'displaySurfacesTranslucent' : True,
            'displaySurfacesWireframe' : False,
            'displayXiAxes' : False
        }
        self._discoverAllMeshTypes()

    def getRegion(self):
        return self._region

    def _discoverAllMeshTypes(self):
        scaffoldmaker = Scaffoldmaker()
        self._meshTypes = scaffoldmaker.getMeshTypes()
        self._currentMeshType = scaffoldmaker.getDefaultMeshType()
        self._settings['meshTypeName'] = self._currentMeshType.getName()
        self._settings['meshTypeOptions'] = self._currentMeshType.getDefaultOptions()

    def getAllMeshTypeNames(self):
        meshTypeNames = []
        for meshType in self._meshTypes:
            meshTypeNames.append(meshType.getName())
        return meshTypeNames

    def getMeshTypeName(self):
        return self._settings['meshTypeName']

    def _getMeshTypeByName(self, name):
        for meshType in self._meshTypes:
            if meshType.getName() == name:
                return meshType
        return None

    def setMeshTypeByName(self, name):
        meshType = self._getMeshTypeByName(name)
        if meshType is not None:
            if meshType != self._currentMeshType:
                self._currentMeshType = meshType
                self._settings['meshTypeName'] = self._currentMeshType.getName()
                self._settings['meshTypeOptions'] = self._currentMeshType.getDefaultOptions()
                self._generateMesh()

    def getMeshTypeOrderedOptionNames(self):
        return self._currentMeshType.getOrderedOptionNames()

    def getMeshTypeOption(self, key):
        return self._settings['meshTypeOptions'][key]

    def setMeshTypeOption(self, key, value):
        oldValue = self._settings['meshTypeOptions'][key]
        # print('setMeshTypeOption: key ', key, ' value ', str(value))
        newValue = None
        try:
            if type(oldValue) is bool:
                newValue = bool(value)
            elif type(oldValue) is int:
                newValue = int(value)
            elif type(oldValue) is float:
                newValue = float(value)
            elif type(oldValue) is str:
                newValue = str(value)
            else:
                newValue = value
        except:
            print('setMeshTypeOption: Invalid value')
            return
        self._settings['meshTypeOptions'][key] = newValue
        self._currentMeshType.checkOptions(self._settings['meshTypeOptions'])
        # print('final value = ', self._settings['meshTypeOptions'][key])
        if self._settings['meshTypeOptions'][key] != oldValue:
            self._generateMesh()

    def getDeleteElementsRangesText(self):
        return self._settings['deleteElementRanges']

    def _parseDeleteElementsRangesText(self, elementRangesTextIn):
        """
        :return: True if ranges changed, otherwise False
        """
        elementRanges = []
        for elementRangeText in elementRangesTextIn.split(','):
            try:
                elementRangeEnds = elementRangeText.split('-')
                # remove trailing non-numeric characters, workaround for select 's' key ending up there
                for e in range(len(elementRangeEnds)):
                    size = len(elementRangeEnds[e])
                    for i in range(size):
                        if elementRangeEnds[e][size - i - 1] in string.digits:
                            break;
                    if i > 0:
                        elementRangeEnds[e] = elementRangeEnds[e][:(size - i)]
                elementRangeStart = int(elementRangeEnds[0])
                if len(elementRangeEnds) > 1:
                    elementRangeStop = int(elementRangeEnds[1])
                else:
                    elementRangeStop = elementRangeStart
                if elementRangeStop >= elementRangeStart:
                    elementRanges.append([elementRangeStart, elementRangeStop])
                else:
                    elementRanges.append([elementRangeStop, elementRangeStart])
            except:
                pass
        elementRanges.sort()
        elementRangesText = ''
        first = True
        for elementRange in elementRanges:
            if first:
                first = False
            else:
                elementRangesText += ','
            elementRangesText += str(elementRange[0])
            if elementRange[1] != elementRange[0]:
                elementRangesText += '-' + str(elementRange[1])
        changed = self._deleteElementRanges != elementRanges
        self._deleteElementRanges = elementRanges
        self._settings['deleteElementRanges'] = elementRangesText
        return changed

    def setDeleteElementsRangesText(self, elementRangesTextIn):
        if self._parseDeleteElementsRangesText(elementRangesTextIn):
            self._generateMesh()

    def getScaleText(self):
        return self._settings['scale']

    def _parseScaleText(self, scaleTextIn):
        """
        :return: True if scale changed, otherwise False
        """
        scale = []
        for valueText in scaleTextIn.split('*'):
            try:
                scale.append(float(valueText))
            except:
                scale.append(1.0)
        for i in range(3 - len(scale)):
            scale.append(scale[-1])
        if len(scale) > 3:
            scale = scale[:3]
        scaleText = '*'.join(STRING_FLOAT_FORMAT.format(value) for value in scale)
        changed = self._scale != scale
        self._scale = scale
        self._settings['scale'] = scaleText
        return changed

    def setScaleText(self, scaleTextIn):
        if self._parseScaleText(scaleTextIn):
            self._generateMesh()

    def registerSceneChangeCallback(self, sceneChangeCallback):
        self._sceneChangeCallback = sceneChangeCallback

    def _getVisibility(self, graphicsName):
        return self._settings[graphicsName]

    def _setVisibility(self, graphicsName, show):
        self._settings[graphicsName] = show
        graphics = self._region.getScene().findGraphicsByName(graphicsName)
        graphics.setVisibilityFlag(show)

    def isDisplayAxes(self):
        return self._getVisibility('displayAxes')

    def setDisplayAxes(self, show):
        self._setVisibility('displayAxes', show)

    def isDisplayElementNumbers(self):
        return self._getVisibility('displayElementNumbers')

    def setDisplayElementNumbers(self, show):
        self._setVisibility('displayElementNumbers', show)

    def isDisplayLines(self):
        return self._getVisibility('displayLines')

    def setDisplayLines(self, show):
        self._setVisibility('displayLines', show)

    def isDisplayNodeDerivatives(self):
        return self._getVisibility('displayNodeDerivatives')

    def setDisplayNodeDerivatives(self, show):
        graphicsName = 'displayNodeDerivatives'
        self._settings[graphicsName] = show
        scene = self._region.getScene()
        graphics = scene.getFirstGraphics()
        while graphics.isValid():
            if graphics.getName() == graphicsName:
                graphics.setVisibilityFlag(show)
            graphics = scene.getNextGraphics(graphics)

    def isDisplayNodeNumbers(self):
        return self._getVisibility('displayNodeNumbers')

    def setDisplayNodeNumbers(self, show):
        self._setVisibility('displayNodeNumbers', show)

    def isDisplaySurfaces(self):
        return self._getVisibility('displaySurfaces')

    def setDisplaySurfaces(self, show):
        self._setVisibility('displaySurfaces', show)

    def isDisplaySurfacesExterior(self):
        return self._settings['displaySurfacesExterior']

    def setDisplaySurfacesExterior(self, isExterior):
        self._settings['displaySurfacesExterior'] = isExterior
        surfaces = self._region.getScene().findGraphicsByName('displaySurfaces')
        surfaces.setExterior(self.isDisplaySurfacesExterior() if (self.getMeshDimension() == 3) else False)

    def isDisplaySurfacesTranslucent(self):
        return self._settings['displaySurfacesTranslucent']

    def setDisplaySurfacesTranslucent(self, isTranslucent):
        self._settings['displaySurfacesTranslucent'] = isTranslucent
        surfaces = self._region.getScene().findGraphicsByName('displaySurfaces')
        surfacesMaterial = self._materialmodule.findMaterialByName('trans_blue' if isTranslucent else 'solid_blue')
        surfaces.setMaterial(surfacesMaterial)

    def isDisplaySurfacesWireframe(self):
        return self._settings['displaySurfacesWireframe']

    def setDisplaySurfacesWireframe(self, isWireframe):
        self._settings['displaySurfacesWireframe'] = isWireframe
        surfaces = self._region.getScene().findGraphicsByName('displaySurfaces')
        surfaces.setRenderPolygonMode(Graphics.RENDER_POLYGON_MODE_WIREFRAME if isWireframe else Graphics.RENDER_POLYGON_MODE_SHADED)

    def isDisplayXiAxes(self):
        return self._getVisibility('displayXiAxes')

    def setDisplayXiAxes(self, show):
        self._setVisibility('displayXiAxes', show)

    def needPerturbLines(self):
        """
        Return if solid surfaces are drawn with lines, requiring perturb lines to be activated.
        """
        if self._region is None:
            return False
        mesh2d = self._region.getFieldmodule().findMeshByDimension(2)
        if mesh2d.getSize() == 0:
            return False
        return self.isDisplayLines() and self.isDisplaySurfaces() and not self.isDisplaySurfacesTranslucent()

    def _getMesh(self):
        fm = self._region.getFieldmodule()
        for dimension in range(3,0,-1):
            mesh = fm.findMeshByDimension(dimension)
            if mesh.getSize() > 0:
                break
        if mesh.getSize() == 0:
            mesh = fm.findMeshByDimension(3)
        return mesh

    def getMeshDimension(self):
        return self._getMesh().getDimension()

    def getSettings(self):
        return self._settings

    def setSettings(self, settings):
        self._settings.update(settings)
        self._currentMeshType = self._getMeshTypeByName(self._settings['meshTypeName'])
        self._parseDeleteElementsRangesText(self._settings['deleteElementRanges'])
        # merge any new options for this generator
        savedMeshTypeOptions = self._settings['meshTypeOptions']
        self._settings['meshTypeOptions'] = self._currentMeshType.getDefaultOptions()
        self._settings['meshTypeOptions'].update(savedMeshTypeOptions)
        self._parseScaleText(self._settings['scale'])
        self._generateMesh()

    def _generateMesh(self):
        if self._region:
            self._parent_region.removeChild(self._region)
        self._region = self._parent_region.createChild(self._region_name)
        self._scene = self._region.getScene()
        fm = self._region.getFieldmodule()
        fm.beginChange()
        # logger = self._context.getLogger()
        annotationGroups = self._currentMeshType.generateMesh(self._region, self._settings['meshTypeOptions'])
        # loggerMessageCount = logger.getNumberOfMessages()
        # if loggerMessageCount > 0:
        #     for i in range(1, loggerMessageCount + 1):
        #         print(logger.getMessageTypeAtIndex(i), logger.getMessageTextAtIndex(i))
        #     logger.removeAllMessages()
        mesh = self._getMesh()
        # meshDimension = mesh.getDimension()
        nodes = fm.findNodesetByFieldDomainType(Field.DOMAIN_TYPE_NODES)
        if len(self._deleteElementRanges) > 0:
            deleteElementIdentifiers = []
            elementIter = mesh.createElementiterator()
            element = elementIter.next()
            while element.isValid():
                identifier = element.getIdentifier()
                for deleteElementRange in self._deleteElementRanges:
                    if (identifier >= deleteElementRange[0]) and (identifier <= deleteElementRange[1]):
                        deleteElementIdentifiers.append(identifier)
                element = elementIter.next()
            #print('delete elements ', deleteElementIdentifiers)
            for identifier in deleteElementIdentifiers:
                element = mesh.findElementByIdentifier(identifier)
                mesh.destroyElement(element)
            del element
            # destroy all orphaned nodes
            #size1 = nodes.getSize()
            nodes.destroyAllNodes()
            #size2 = nodes.getSize()
            #print('deleted', size1 - size2, 'nodes')
        fm.defineAllFaces()
        if annotationGroups is not None:
            for annotationGroup in annotationGroups:
                annotationGroup.addSubelements()
        if self._settings['scale'] != '1*1*1':
            coordinates = fm.findFieldByName('coordinates').castFiniteElement()
            scale = fm.createFieldConstant(self._scale)
            newCoordinates = fm.createFieldMultiply(coordinates, scale)
            fieldassignment = coordinates.createFieldassignment(newCoordinates)
            fieldassignment.assign()
            del newCoordinates
            del scale
        fm.endChange()
        self._createGraphics(self._region)
        if self._sceneChangeCallback is not None:
            self._sceneChangeCallback()

        name = mesh.getName()
        print(name)

    def deleteAll(self):
        self._scene = self._region.getScene()
        fm = self._region.getFieldmodule()
        fm.beginChange()
        # logger = self._context.getLogger()
        # annotationGroups = self._currentMeshType.generateMesh(self._region, self._settings['meshTypeOptions'])
        #         # loggerMessageCount = logger.getNumberOfMessages()
        # if loggerMessageCount > 0:
        #     for i in range(1, loggerMessageCount + 1):
        #         print(logger.getMessageTypeAtIndex(i), logger.getMessageTextAtIndex(i))
        #     logger.removeAllMessages()
        mesh = self._getMesh()
        # meshDimension = mesh.getDimension()
        nodes = fm.findNodesetByFieldDomainType(Field.DOMAIN_TYPE_NODES)
        if len(self._deleteElementRanges) > 0:
            deleteElementIdentifiers = []
            elementIter = mesh.createElementiterator()
            element = elementIter.next()
            while element.isValid():
                identifier = element.getIdentifier()
                for deleteElementRange in self._deleteElementRanges:
                    if (identifier >= deleteElementRange[0]) and (identifier <= deleteElementRange[1]):
                        deleteElementIdentifiers.append(identifier)
                element = elementIter.next()
            #print('delete elements ', deleteElementIdentifiers)
            for identifier in deleteElementIdentifiers:
                element = mesh.findElementByIdentifier(identifier)
                mesh.destroyElement(element)
            del element
            # destroy all orphaned nodes
            #size1 = nodes.getSize()
            nodes.destroyAllNodes()
            #size2 = nodes.getSize()
            #print('deleted', size1 - size2, 'nodes')
        fm.defineAllFaces()
        fm.endChange()

    def initialiseTimeSequences(self, data):
        fm = self._region.getFieldmodule()
        cache = fm.createFieldcache()
        colour = fm.findFieldByName('colour')

    def _createGraphics(self, region):
        # Node numbers are generated here
        fm = region.getFieldmodule()
        meshDimension = self.getMeshDimension()
        coordinates = fm.findFieldByName('coordinates')
        nodeDerivativeFields = [
            fm.createFieldNodeValue(coordinates, Node.VALUE_LABEL_D_DS1, 1),
            fm.createFieldNodeValue(coordinates, Node.VALUE_LABEL_D_DS2, 1),
            fm.createFieldNodeValue(coordinates, Node.VALUE_LABEL_D_DS3, 1),
        ]
        elementDerivativeFields = []
        for d in range(meshDimension):
            elementDerivativeFields.append(fm.createFieldDerivative(coordinates, d + 1))
        elementDerivativesField = fm.createFieldConcatenate(elementDerivativeFields)
        cmiss_number = fm.findFieldByName('cmiss_number')
        # make graphics
        scene = region.getScene()
        scene.beginChange()
        axes = scene.createGraphicsPoints()
        pointattr = axes.getGraphicspointattributes()
        pointattr.setGlyphShapeType(Glyph.SHAPE_TYPE_AXES_XYZ)
        pointattr.setBaseSize([1.0, 1.0, 1.0])
        axes.setMaterial(self._materialmodule.findMaterialByName('grey50'))
        axes.setName('displayAxes')
        axes.setVisibilityFlag(self.isDisplayAxes())
        lines = scene.createGraphicsLines()
        lines.setCoordinateField(coordinates)
        lines.setName('displayLines')
        lines.setVisibilityFlag(self.isDisplayLines())

        nodeNumbers = scene.createGraphicsPoints()
        nodeNumbers.setFieldDomainType(Field.DOMAIN_TYPE_NODES)
        nodeNumbers.setCoordinateField(coordinates)
        pointattr = nodeNumbers.getGraphicspointattributes()
        pointattr.setLabelField(cmiss_number)
        pointattr.setGlyphShapeType(Glyph.SHAPE_TYPE_SPHERE)
        pointattr.setBaseSize([0.02,0.02,0.02])
        nodeNumbers.setVisibilityFlag(self.isDisplayNodeNumbers())
        nodeNumbers.setMaterial(self._materialmodule.findMaterialByName('white'))
        nodeNumbers.setName('displayNodeNumbers')

        elementNumbers = scene.createGraphicsPoints()
        elementNumbers.setFieldDomainType(Field.DOMAIN_TYPE_MESH_HIGHEST_DIMENSION)
        elementNumbers.setCoordinateField(coordinates)
        pointattr = elementNumbers.getGraphicspointattributes()
        pointattr.setLabelField(cmiss_number)
        pointattr.setGlyphShapeType(Glyph.SHAPE_TYPE_NONE)
        elementNumbers.setMaterial(self._materialmodule.findMaterialByName('cyan'))
        elementNumbers.setName('displayElementNumbers')
        elementNumbers.setVisibilityFlag(self.isDisplayElementNumbers())
        surfaces = scene.createGraphicsSurfaces()
        surfaces.setCoordinateField(coordinates)
        surfaces.setRenderPolygonMode(Graphics.RENDER_POLYGON_MODE_WIREFRAME if self.isDisplaySurfacesWireframe() else Graphics.RENDER_POLYGON_MODE_SHADED)
        surfaces.setExterior(self.isDisplaySurfacesExterior() if (meshDimension == 3) else False)
        surfacesMaterial = self._materialmodule.findMaterialByName('trans_blue' if self.isDisplaySurfacesTranslucent() else 'solid_blue')
        surfaces.setMaterial(surfacesMaterial)

        colour = fm.findFieldByName('colour2')
        colour = colour.castFiniteElement()

        # Add Spectrum
        scene = region.getScene()
        spcmod = scene.getSpectrummodule()
        spec = spcmod.getDefaultSpectrum()
        spec.setName('eegColourSpectrum')

        spcc = spec.getFirstSpectrumcomponent()
        spcc.setRangeMaximum(1000)
        spcc.setRangeMinimum(-8000)

        # Set attributes for our mesh
        surfaces.setSpectrum(spec)
        surfaces.setDataField(colour)

        nodeNumbers.setSpectrum(spec)
        nodeNumbers.setDataField(colour)

        surfaces.setName('displaySurfaces')
        surfaces.setVisibilityFlag(self.isDisplaySurfaces())

        # derivative arrow width is based on shortest non-zero side
        minScale = 1.0
        first = True
        for i in range(coordinates.getNumberOfComponents()):
            absScale = abs(self._scale[i])
            if absScale > 0.0:
                if first or (absScale < minScale):
                    minScale = absScale
                    first = False
        width = 0.01*minScale

        nodeDerivativeMaterialNames = [ 'gold', 'silver', 'green' ]
        for i in range(meshDimension):
            nodeDerivatives = scene.createGraphicsPoints()
            nodeDerivatives.setFieldDomainType(Field.DOMAIN_TYPE_NODES)
            nodeDerivatives.setCoordinateField(coordinates)
            pointattr = nodeDerivatives.getGraphicspointattributes()
            pointattr.setGlyphShapeType(Glyph.SHAPE_TYPE_ARROW_SOLID)
            pointattr.setOrientationScaleField(nodeDerivativeFields[i])
            pointattr.setBaseSize([0.0, width, width])
            pointattr.setScaleFactors([1.0, 0.0, 0.0])
            nodeDerivatives.setMaterial(self._materialmodule.findMaterialByName(nodeDerivativeMaterialNames[i]))
            nodeDerivatives.setName('displayNodeDerivatives')
            nodeDerivatives.setVisibilityFlag(self.isDisplayNodeDerivatives())

        xiAxes = scene.createGraphicsPoints()
        xiAxes.setFieldDomainType(Field.DOMAIN_TYPE_MESH_HIGHEST_DIMENSION)
        xiAxes.setCoordinateField(coordinates)
        pointattr = xiAxes.getGraphicspointattributes()
        pointattr.setGlyphShapeType(Glyph.SHAPE_TYPE_AXES_123)
        pointattr.setOrientationScaleField(elementDerivativesField)
        if meshDimension == 1:
            pointattr.setBaseSize([0.0, 2*width, 2*width])
            pointattr.setScaleFactors([0.25, 0.0, 0.0])
        elif meshDimension == 2:
            pointattr.setBaseSize([0.0, 0.0, 2*width])
            pointattr.setScaleFactors([0.25, 0.25, 0.0])
        else:
            pointattr.setBaseSize([0.0, 0.0, 0.0])
            pointattr.setScaleFactors([0.25, 0.25, 0.25])
        xiAxes.setMaterial(self._materialmodule.findMaterialByName('yellow'))
        xiAxes.setName('displayXiAxes')
        xiAxes.setVisibilityFlag(self.isDisplayXiAxes())
        self.applyAlignment()
        scene.endChange()

    def updateEEGnodeColours(self, values):

        fm = self._region.getFieldmodule()
        cache = fm.createFieldcache()
        colour = fm.findFieldByName('colour2')
        colour = colour.castFiniteElement()
        nodeset = fm.findNodesetByName('nodes')
        for i in range(1, 64):
            node = nodeset.findNodeByIdentifier(i)
            cache.setNode(node)
            colour.setNodeParameters(cache, -1, Node.VALUE_LABEL_VALUE, 1, values[(i % (len(values) - 1))])
        fm.endChange()

    def writeModel(self, file_name):
        self._region.writeFile(file_name)


text:
from pydnameth.infrastucture.path import get_cache_path
from pydnameth.config.annotations.conditions import check_conditions
from pydnameth.config.annotations.types import AnnotationKey
import os.path
import pickle
import numpy as np

def subset_annotations(config):
    aux_data_fn = get_cache_path(config) + '/' + 'aux_data.pkl'

    if os.path.isfile(aux_data_fn):
        f = open(aux_data_fn, 'rb')
        aux_data = pickle.load(f)
        f.close()
        config.cpg_list = aux_data['cpg_list']
        config.cpg_gene_dict = aux_data['cpg_gene_dict']
        config.cpg_bop_dict = aux_data['cpg_bop_dict']
        config.gene_cpg_dict = aux_data['gene_cpg_dict']
        config.gene_bop_dict = aux_data['gene_bop_dict']
        config.bop_cpg_dict = aux_data['bop_cpg_dict']
        config.bop_gene_dict = aux_data['bop_gene_dict']
    else:
        config.cpg_list = []
        config.cpg_gene_dict = {}
        config.cpg_bop_dict = {}
        config.gene_cpg_dict = {}
        config.gene_bop_dict = {}
        config.bop_cpg_dict = {}
        config.bop_gene_dict = {}

        cpgs = config.annotations_dict[AnnotationKey.cpg.value]
        genes = config.annotations_dict[AnnotationKey.gene.value]
        bops = config.annotations_dict[AnnotationKey.bop.value]
        map_infos = config.annotations_dict[AnnotationKey.map_info.value]
        for id in range(0, len(cpgs)):

            if id % 10000 == 0:
                print('id: ' + str(id))

            curr_ann_dict = {}
            for key in config.annotations_dict:
                curr_ann_dict[key] = config.annotations_dict[key][id]

            if check_conditions(config, curr_ann_dict):

                cpg = cpgs[id]
                gene_raw = genes[id]
                curr_genes = list(set(gene_raw.split(';')))
                bop = bops[id]

                config.cpg_list.append(cpg)

                config.cpg_gene_dict[cpg] = curr_genes

                config.cpg_bop_dict[cpg] = bop

                for gene in curr_genes:
                    if gene in config.gene_cpg_dict:
                        config.gene_cpg_dict[gene].append(cpg)
                    else:
                        config.gene_cpg_dict[gene] = [cpg]

                for gene in curr_genes:
                    if gene in config.gene_bop_dict:
                        config.gene_bop_dict[gene].append(bop)
                    else:
                        config.gene_bop_dict[gene] = [bop]

                if len(bop) > 0:
                    if bop in config.bop_cpg_dict:
                        config.bop_cpg_dict[bop].append(cpg)
                    else:
                        config.bop_cpg_dict[bop] = [cpg]

                config.bop_gene_dict[bop] = curr_genes

        # Sorting cpgs by map_info in gene dict
        for curr_gene, curr_cpgs in config.gene_cpg_dict.items():
            curr_map_infos = []
            for curr_cpg in curr_cpgs:
                cpg_index = cpgs.index(curr_cpg)
                curr_map_infos.append(int(map_infos[cpg_index]))
            order = np.argsort(curr_map_infos)
            curr_cpgs_sorted = list(np.array(curr_cpgs)[order])
            config.gene_cpg_dict[curr_gene] = curr_cpgs_sorted

        # Sorting cpgs by map_info in bop dict
        for curr_bop, curr_cpgs in config.bop_cpg_dict.items():
            curr_map_infos = []
            for curr_cpg in curr_cpgs:
                cpg_index = cpgs.index(curr_cpg)
                curr_map_infos.append(int(map_infos[cpg_index]))
            order = np.argsort(curr_map_infos)
            curr_cpgs_sorted = list(np.array(curr_cpgs)[order])
            config.bop_cpg_dict[curr_bop] = curr_cpgs_sorted

        aux_data = {
            'cpg_list': config.cpg_list,
            'cpg_gene_dict': config.cpg_gene_dict,
            'cpg_bop_dict': config.cpg_bop_dict,
            'gene_cpg_dict': config.gene_cpg_dict,
            'gene_bop_dict': config.gene_bop_dict,
            'bop_cpg_dict': config.bop_cpg_dict,
            'bop_gene_dict': config.bop_gene_dict,
        }

        f = open(aux_data_fn, 'wb')
        pickle.dump(aux_data, f, pickle.HIGHEST_PROTOCOL)
        f.close()

max_stars_repo_name: GillianGrayson/pydnameth

max_issues_repo_name: GillianGrayson/pydnameth

max_forks_repo_name: GillianGrayson/pydnameth