text:
import math
import numpy as np
import tensorflow as tf

from tensorflow.python.util import nest
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_ops
from tensorflow.python.ops import clip_ops

from tensorflow.contrib.layers import l2_regularizer
from tensorflow.contrib.layers import variance_scaling_initializer

from functools import reduce
from operator import mul

initializer = lambda : variance_scaling_initializer(factor=1.0, mode='FAN_AVG', uniform=True, dtype=tf.float32)
initializer_relu = lambda : variance_scaling_initializer(factor=2.0, mode='FAN_IN', uniform=False, dtype=tf.float32)
regularizer = l2_regularizer(scale=3e-7)

def glu(x):
    """Gated Linear Units
    Args:
        x: [batch_size, seq_len, 2 * hidden_size]

    Returns:
        glu(x): [batch_size, seq_len, hidden_size]
    """
    x, x_h = tf.split(x, 2, axis=-1)
    return tf.sigmoid(x) * x_h

def noam_norm(x, epsilon=1.0, scope=None, reuse=None):
    """One version of layer normalization.

    apply normalization along the last dimension.
    """
    with tf.name_scope(scope, default_name='noam_norm', values=[x]):
        shape = x.get_shape()
        ndims = len(shape)
        return tf.nn.l2_normalize(x, ndims - 1, epsilon=epsilon) \
               * tf.sqrt(tf.to_float(shape[-1]))

def layer_norm_compute_python(x, epsilon, scale, bias):
    """Layer norm raw computation."""
    mean = tf.reduce_mean(x, axis=[-1], keepdims=True)
    variance = tf.reduce_mean(tf.square(x - mean), axis=[-1], keepdims=True)
    norm_x = (x - mean) * tf.rsqrt(variance + epsilon)
    return norm_x * scale + bias

def layer_norm(x, filters=None, epsilon=1e-6, scope=None, reuse=None):
    """Layer normalize the tensor x, averaging over the last dimension."""
    if filters is None:
        filters = x.get_shape()[-1]
    with tf.variable_scope(scope, default_name='layer_norm', values=[x], reuse=reuse):
        scale = tf.get_variable(
            name='layer_norm_scale', shape=[filters], regularizer=regularizer, initializer=tf.ones_initializer())
        bias = tf.get_variable(
            name='layer_norm_bias', shape=[filters], regularizer=regularizer, initializer=tf.zeros_initializer())
        result = layer_norm_compute_python(x, epsilon, scale, bias)
        return result

norm_fn = layer_norm

def highway(x, size=None, activation=None, num_layers=2, scope='highway', dropout=0.0, reuse=None):
    """Use two convolution layers to implement highway"""
    with tf.variable_scope(scope, reuse):
        if size is None:
            size = x.shape.as_list()[-1]
        else:
            x = conv(x, size, name='input_projection', reuse=reuse)

        for i in range(num_layers):
            T = conv(x, size, bias=True, activation=tf.sigmoid, name='gate_%d'%i, reuse=reuse)
            H = conv(x, size, bias=True, activation=activation, name='activation_%d'%i, reuse=reuse)

            H = tf.nn.dropout(H, 1.0 - dropout)

            x = T * H + (1.0 - T) * x

        return x

def layer_dropout(inputs, residual, dropout):
    """implement stochastic residual connection"""
    pred = tf.random_uniform([]) < dropout
    return tf.cond(pred,
                   lambda : residual,
                   lambda : tf.nn.dropout(inputs, 1.0 - dropout) + residual)

def residual_block(inputs, num_blocks, num_conv_layers, kernel_size,
                   mask=None, num_filters=128, input_projection=False, num_heads=8,
                   seq_len=None, scope='res_block', is_training=True, reuse=None, bias=True, dropout=0.0):
    with tf.variable_scope(scope, reuse=reuse):
        if input_projection:
            inputs = conv(inputs, num_filters, name='input_projection', reuse=reuse)

        outputs = inputs
        sublayer = 1
        total_sublayers = (num_conv_layers + 2) * num_blocks
        for i in range(num_blocks):
            outputs = add_timing_signal_1d(outputs) # add 1d time signal to every block's inputs

            outputs, sublayer = conv_block(outputs,
                                           num_conv_layers,
                                           kernel_size,
                                           num_filters,
                                           seq_len=seq_len,
                                           scope='encoder_block_%d'%i,
                                           reuse=reuse,
                                           bias=bias,
                                           dropout=dropout,
                                           sublayers=(sublayer, total_sublayers))
            outputs, sublayer = self_attention_block(outputs,
                                                     num_filters,
                                                     seq_len,
                                                     mask=mask,
                                                     num_heads=num_heads,
                                                     scope='self_attention_layer_%d'%i,
                                                     reuse=reuse,
                                                     is_training=is_training,
                                                     bias=bias,
                                                     dropout=dropout,
                                                     sublayers=(sublayer, total_sublayers))
        return outputs

def conv_block(inputs, num_conv_block, kernel_size, num_filters, seq_len=None, scope='conv_block', is_training=True,
               reuse=None, bias=True, dropout=0.0, sublayers=(1, 1)):
    with tf.variable_scope(scope, reuse=reuse):
        outputs = tf.expand_dims(inputs, axis=2)
        l, L = sublayers
        for i in range(num_conv_block):
            residual = outputs  # the input to the current layer
            outputs = norm_fn(outputs, scope='layer_norm_%d'%i, reuse=reuse)    # norm layer
            if i % 2 == 0:
                outputs = tf.nn.dropout(outputs, 1.0 - dropout) # apply dropout at even layer
            output = depthwise_separable_convolution(
                outputs, kernel_size=(kernel_size, 1), num_filters=num_filters,
                scope='depthwise_conv_layers_%d'%i, is_training=is_training, reuse=reuse)
            outputs = layer_dropout(outputs, residual, dropout * float(l) / L)  # according to the layer num
            l += 1
        return tf.squeeze(outputs, axis=2), l

def self_attention_block(inputs, num_filters, seq_len, mask=None, num_heads=8,
                         scope='self_attention_ffn', reuse=None, is_training=True,
                         bias=True, dropout=0.0, sublayers=(1, 1)):
    with tf.variable_scope(scope, reuse=reuse):
        l, L = sublayers

        # Self attention
        outputs = norm_fn(inputs, scope='layer_norm_1', reuse=reuse)
        outputs = tf.nn.dropout(outputs, 1.0 - dropout)
        outputs = multi_head_attention(
            outputs, num_filters, num_heads=num_heads, seq_len=seq_len, reuse=reuse,
            mask=mask, is_training=is_training, bias=bias, dropout=dropout)
        residual = layer_dropout(outputs, inputs, dropout * float(l) / L)
        l += 1

        # Feed forward layer
        outputs = norm_fn(residual, scope='layer_norm_2', reuse=reuse)
        outputs = tf.nn.dropout(outputs, 1.0 - dropout)
        outputs = conv(outputs, num_filters, True, tf.nn.relu, name='FFN_1', reuse=reuse)
        outputs = conv(outputs, num_filters, True, None, name='FFN_2', reuse=reuse)
        outputs = layer_dropout(outputs, residual, dropout * float(l) / L)
        l += 1
        return outputs, l

def multi_head_attention(queries, units, num_heads,
                         memory=None,
                         seq_len=None,
                         scope='Multi_Head_Attention',
                         reuse=None,
                         mask=None,
                         is_training=True,
                         bias=True,
                         dropout=0.0):
    with tf.variable_scope(scope, reuse=reuse):
        # Self Attention
        if memory is None:
            memory = queries

        # Linear projection use convolution nets
        memory = conv(memory, 2 * units, name='memory_projection', reuse=reuse)
        query = conv(queries, units, name='query_projection', reuse=reuse)

        # Turn to multi head
        Q = split_last_dimension(query, num_heads)
        K, V = [split_last_dimension(tensor, num_heads) for tensor in tf.split(memory, num_or_size_splits=2, axis=2)]

        key_depth_per_head = units / num_heads
        Q *= key_depth_per_head ** -0.5 # scale
        x = dot_product_attention(Q, K, V,
                                  bias=bias,
                                  seq_len=seq_len,
                                  mask=mask,
                                  is_training=is_training,
                                  scope='dot_production_attention',
                                  reuse=reuse,
                                  dropout=dropout)
        return combine_last_two_dimensions(tf.transpose(x, [0, 2, 1, 3]))

def conv(inputs, output_size, bias=None,
         activation=None, kernel_size=1, name='conv', reuse=None):
    """inputs shape can be 3D or 4D.

        [batch_size, seq_len, hidden_size] or [batch_size, 1, seq_len, hidden_size]
        depend on the inputs shape, we choose to use tf.nn.conv1d or tf.nn.conv2d.

    """
    with tf.variable_scope(name, reuse=reuse):
        shapes = inputs.shape.as_list()
        if len(shapes) > 4:
            raise NotImplementedError
        elif len(shapes) == 4:  # [batch_size, 1, seq_len, hidden_size]
            filter_shape = [1, kernel_size, shapes[-1], output_size]
            bias_shape = [1, 1, 1, output_size]
            strides = [1, 1, 1, 1]

        else:                   # [batch_size, seq_len, hidden_size]
            filter_shape = [kernel_size, shapes[-1], output_size]
            bias_shape = [1, 1, output_size]
            strides = 1

        conv_func = tf.nn.conv1d if len(shapes) == 3 else tf.nn.conv2d
        kernel_ = tf.get_variable(
            name='kernel_', shape=filter_shape, dtype=tf.float32,
            regularizer=regularizer,
            initializer=initializer_relu() if activation is not None else initializer())
        outputs = conv_func(inputs, kernel_, strides, 'VALID')  # valid or same ?
        if bias:
            outputs += tf.get_variable(
                name='bias_', shape=bias_shape,
                regularizer=regularizer,
                initializer=tf.zeros_initializer())
        if activation is not None:
            return activation(outputs)
        else:
            return outputs

def mask_logits(inputs, mask, mask_value=-1e30):
    """inputs and mask have same shape"""
    shapes = inputs.shape.as_list()
    mask = tf.cast(mask, tf.float32)
    return inputs + mask_value * (1 - mask)

def depthwise_separable_convolution(inputs, kernel_size, num_filters,
                                    scope='depthwise_separable_convolution',
                                    bias=True, is_training=True, reuse=None):
    with tf.variable_scope(scope, reuse=reuse):
        shapes = inputs.shape.as_list()
        depthwise_filter = tf.get_variable(
            name='depthwise_filter',
            shape=[kernel_size[0], kernel_size[1], shapes[-1], 1], # output_size == 1
            dtype=tf.float32,
            regularizer=regularizer,
            initializer=initializer_relu())
        pointwise_filter = tf.get_variable(
            name='pointwise_filter',
            shape=[1, 1, shapes[-1], num_filters],  # kernel_size == 1
            dtype=tf.float32,
            regularizer=regularizer,
            initializer=initializer_relu())

        outputs = tf.nn.separable_conv2d(
            input=inputs,
            depthwise_filter=depthwise_filter,
            pointwise_filter=pointwise_filter,
            strides=[1, 1, 1, 1],
            padding='SAME')

        if bias:
            b = tf.get_variable(
                name='bias', shape=outputs.shape[-1],
                regularizer=regularizer,
                initializer=tf.zeros_initializer())
            outputs += b
        outputs = tf.nn.relu(outputs)
        return outputs

def split_last_dimension(x, n):
    """Reshape x so that the last dimension becomes two dimension.
    The first of these two dimensions is n.

    Two steps: 1) Reshape 2) Transpose

    Args:
        x: a Tensor with shape [..., m]
        n: an integer.
    Returns:
        a Tensor with shape [..., n, m/n]
    """
    old_shape = x.get_shape().dims
    last = old_shape[-1]    # m
    new_shape = old_shape[: -1] + [n] + [last // n if last else None]
    ret = tf.reshape(x, tf.concat([tf.shape(x)[: -1], [n, -1]], axis=0))
    ret.set_shape(new_shape)
    return tf.transpose(ret, [0, 2, 1, 3])

def dot_product_attention(q, k, v,
                          bias,
                          seq_len=None,
                          mask=None,
                          is_training=True,
                          scope=None,
                          reuse=None,
                          dropout=0.0):
    """Dot product attention.

    Args:
        q: [batch, heads, length_q, depth_k]
        k: [batch, heads, length_kv, depth_k]
        v: [batch, heads, length_kv, depth_v]
        bias: attention_bias()
        is_training: a bool of training
        scope: an optional string
    Returns:
        A Tensor.
    """
    with tf.variable_scope(scope, default_name='dot_product_attention', reuse=reuse):
        # [batch, num_heads, query_length, memory_length]
        logits = tf.matmul(q, k, transpose_b=True)
        if bias:
            b = tf.get_variable(
                name='bias',
                shape=logits.shape[-1],
                regularizer=regularizer,
                initializer=tf.zeros_initializer())
            logits += b
        if mask is not None:
            shapes = [x if x != None else -1 for x in logits.shape.as_list()]
            mask = tf.reshape(mask, shape=[shapes[0], 1, 1, shapes[-1]])
            logits = mask_logits(logits, mask)

        weights = tf.nn.softmax(logits, name='attention_weights')

        # dropping out the attention links for each of the heads
        weights = tf.nn.dropout(weights, 1.0 - dropout)

        return tf.matmul(weights, v)

def combine_last_two_dimensions(x):
    """Reshape x so that the last two dimension become one.
    Args:
        x: a Tensor with shape [..., a, b]
    Returns:
        a Tensor with shape [..., ab]
    """
    old_shape = x.get_shape().dims
    a, b = old_shape[-2: ]
    new_shape = old_shape[: -2] + [a * b if a and b else None]
    ret = tf.reshape(x, tf.concat([tf.shape(x)[: -2], [-1]], axis=0))
    ret.set_shape(new_shape)
    return ret

def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
    """Adds a bunch of sinusoids of different frequencies to a Tensor.
    Args:
        x: Tensor with shape of [batch, length, channels]
        min_timescale: float
        max_timescale: float

    Returns:
        a Tensor with the same shape as x.
    """
    length = tf.shape(x)[1]
    channels = tf.shape(x)[-1]
    signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
    return x + signal

def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4):
    """Gets a bunch of sinusoids of different frequencies.
    Args:
        length: scalar, length of timing signal sequence.
        channels: scalar, size of timing emebddings to create.
            The number of different timescales is equal to channels / 2
        min_timescale: float
        max_timescale: float

    Returns:
        a Tensor of timing signals [1, length, channels]
    """
    position = tf.to_float(tf.range(length))
    num_timescales = channels // 2
    log_timescale_increment = (
        math.log(float(max_timescale) / float(min_timescale)) /
        (tf.to_float(num_timescales) - 1)
    )
    inv_timescales = min_timescale * tf.exp(
        tf.to_float(tf.range(num_timescales)) * -log_timescale_increment)
    scaled_time = tf.expand_dims(position, axis=1) * tf.expand_dims(inv_timescales, axis=0)
    signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)
    signal = tf.pad(signal, [[0, 0], [0, tf.mod(channels, 2)]])
    signal = tf.reshape(signal, [1, length, channels])
    return signal

def ndim(x):
    """Returns the number of exes in a tensor, as an integer."""
    dims = x.get_shape()._dims
    if dims is not None:
        return len(dims)
    return None

def dot(x, y):
    """"""
    if ndim(x) is not None and (ndim(x) > 2 or ndim(y) > 2):
        x_shape = []
        for i, s in zip(x.get_shape().as_list(), tf.unstack(tf.shape(x))):
            if i is not None:
                x_shape.append(i)
            else:
                x_shape.append(s)

        x_shape = tuple(x_shape)
        y_shape = []
        for i, s in zip(y.get_shape().as_list(), tf.unstack(tf.shape(y))):
            if i is not None:
                y_shape.append(i)
            else:
                y_shape.append(s)

        y_shape = tuple(y_shape)

        y_permute_dim = list(range(ndim(y)))
        y_permute_dim = [y_permute_dim.pop(-2)] + y_permute_dim
        xt = tf.reshape(x, shape=[-1, x_shape[-1]])
        yt = tf.reshape(tf.transpose(y, perm=y_permute_dim),
                        shape=[y_shape[-2], -1])

        return tf.reshape(tf.matmul(xt, yt),
                          x_shape[: -1] + y_shape[: -2] + y_shape[-1: ])

    if isinstance(x, tf.SparseTensor):
        out = tf.sparse_tensor_dense_matmul(x, y)
    else:
        out = tf.matmul(x, y)
    return out

def batch_dot(x, y, axes=None):
    """Batch-wise dot product.
    keras==2.0.6

    'batch_dot' is used to compute dot product of 'x' and 'y' when
    'x' and 'y' are data in batch, i.e. in a shape of '(batch_size, :)'.

    'batch_dot' results in a tensor or variable with less dimensions than the input.
    If the number of dimensions is reduced to 1, we use 'expand_dims' to make sure
    that ndim is at least 2.

    Args:
        x: A Tensor or variable with 'ndim >= 2'
        y: A Tensor of variable with 'ndim >= 2'
        axes: list of (or single) in with target dimensions.
            The lengths of 'axes[0]' and 'axes[1]' should be the same.

    Returns:
        A Tensor with shape equal to the concatenation of 'x''s shape
        (less the dimension that was summed over) and 'y''s shape
        (less the batch dimension and the dimension that was summed over).
        If the final rank is 1, we reshape it to '(batch_size, 1)'
    """
    if isinstance(axes, int):
        axes = (axes, axes)

    x_ndim = ndim(x)
    y_ndim = ndim(y)

    if x_ndim > y_ndim:
        diff = x_ndim - y_ndim
        y = tf.reshape(y, tf.concat([tf.shape(y), [1] * (diff)], axis=0))
    elif y_ndim > x_ndim:
        diff = y_ndim - x_ndim
        x = tf.reshape(x, tf.concat([tf.shape(x), [1] * (diff)], axis=0))
    else:
        diff = 0

    if ndim(x) == 2 and ndim(y) == 2:
        if axes[0] == axes[1]:
            out = tf.reduce_sum(tf.multiply(x, y), axes=[0])
        else:
            out = tf.reduce_sum(tf.multiply(tf.transpose(x, [1, 0]), y), axis=[1])
    else:
        if axes is not None:
            adj_x = None if axes[0] == ndim(x) - 1 else True
            adj_y = True if axes[1] == ndim(y) - 1 else None
        else:
            adj_x = None
            adj_y = None
        out = tf.matmul(x, y, adjoint_a=adj_x, adjoint_b=adj_y)
    if diff:
        if x_ndim > y_ndim:
            idx = x_ndim + y_ndim - 3
        else:
            idx = x_ndim - 1
        out = tf.squeeze(out, list(range(idx, idx + diff)))
    if ndim(out) == 1:
        out = tf.expand_dims(out, axis=1)
    return out

def optimized_trilinear_for_attention(args,
                                      c_maxlen,
                                      q_maxlen,
                                      input_keep_prob=1.0,
                                      scope = 'efficient_trilinear',
                                      bias_initializer=tf.zeros_initializer(),
                                      kernel_initializer=initializer()):
    """trilinear for attention.

    c, q = args : [batch_size, {c/q}_len, depth]

    """
    assert len(args) == 2, 'just use for computing attention with two input'
    arg0_shape = args[0].get_shape().as_list()
    arg1_shape = args[1].get_shape().as_list()

    if len(arg0_shape) != 3 or len(arg1_shape) != 3:
        raise ValueError("'args' must be 3 dims (batch_size, len, dimension)")
    if arg0_shape[2] != arg1_shape[2]:
        raise ValueError("The last dimension of 'args' must be equal")

    arg_size = arg0_shape[2]
    dtype = args[0].dtype
    print('dtype: {}'.format(dtype))
    droped_args = [tf.nn.dropout(arg, input_keep_prob) for arg in args]
    with tf.variable_scope(scope):
        weights4arg0 = tf.get_variable(
            name='linear_kernel4arg0',
            shape=[arg_size, 1],
            dtype=dtype,
            regularizer=regularizer,
            initializer=kernel_initializer)
        weights4arg1 = tf.get_variable(
            name='linear_kernel4arg1',
            shape=[arg_size, 1],
            dtype=dtype,
            regularizer=regularizer,
            initializer=kernel_initializer)
        weights4mul = tf.get_variable(
            name='linear_kernel4mul',
            shape=[1, 1, arg_size],
            dtype=dtype,
            regularizer=regularizer,
            initializer=kernel_initializer)
        biases = tf.get_variable(
            name='linear_bias',
            shape=[1],
            dtype=dtype,
            regularizer=regularizer,
            initializer=bias_initializer)

        subres0 = tf.tile(dot(droped_args[0], weights4arg0), [1, 1, q_maxlen])
        subres1 = tf.tile(tf.transpose(dot(droped_args[1], weights4arg1), perm=[0, 2, 1]), [1, c_maxlen, 1])
        subres2 = batch_dot(droped_args[0] * weights4mul, tf.transpose(droped_args[1], perm=[0, 2, 1]))
        res = subres0 + subres1 + subres2
        # res = nn_ops.bias_add(res, biases)  # [batch_size, c_maxlen, q_maxlen]
        return res

def trilinear(args,
              output_size=1,
              bias=True,
              squeeze=False,
              wd=0.0,
              input_keep_prob=1.0,
              scope='trilinear'):
    with tf.variable_scope(scope):
        flat_args = [flatten(arg, 1) for arg in args]
        flat_out = _linear(flat_args, output_size, bias, scope=scope)
        out = reconstruct(flat_out, args[0], 1)
        return tf.squeeze(out, -1)

def flatten(tensor, keep):
    fixed_shape = tensor.get_shape().as_list()
    start = len(fixed_shape) - keep
    left = reduce(mul, [fixed_shape[i] or tf.shape(tensor)[i] for i in range(start)])
    out_shape = [left] + [fixed_shape[i] or tf.shape(tensor)[i] for i in range(start, len(fixed_shape))]
    flat = tf.reshape(tensor, out_shape)
    return flat

def reconstruct(tensor, ref, keep):
    ref_shape = ref.get_shape().as_list()
    tensor_shape = tensor.get_shape().as_list()

    ref_stop = len(ref_shape) - keep
    tensor_start = len(tensor_shape) - keep

    pre_shape = [ref_shape[i] or tf.shape(ref)[i] for i in range(ref_stop)]
    keep_shape = [tensor_shape[i] or tf.shape(tensor)[i] for i in range(tensor_start, len(tensor_shape))]

    target_shape = pre_shape + keep_shape
    out = tf.reshape(tensor, target_shape)
    return out

def _linear(args,
            output_size,
            bias,
            bias_initializer=tf.zeros_initializer(),
            kernel_initializer=initializer(),
            scope=None,
            reuse=None):
    """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.

    Args:
        args: a 2D Tensor or a list of 2D, batch x n, Tensors.
        output_size: int, second dimension of W[i].
        bias: boolean, whether to add a bias term or not.
        bias_initializer: starting value to initialize the bias
            (default is all zeros).
        kernel_initializer: starting value to initialize the weight.
    Returns:
        A 2D Tensor with shape [batch x output_size] equal to
        sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
    Raises:
        ValueError: If some of the arguments has unspecified or wrong shape.
    """
    if args is None or (nest.is_sequence(args) and not args):
        raise ValueError("'args' must be specified")
    if not nest.is_sequence(args):
        args = [args]

    # Calculate the total size of arguments on dimension 1.
    total_arg_size = 0
    shapes = [a.get_shape() for a in args]
    for shape in shapes:
        if shape.ndims != 2:
            raise ValueError('linear is expecting 2D arguments: %s' % shapes)
        if shape[1].value is None:
            raise ValueError('linear expects shape[1] to be provided for shape %s, '
                             'but saw %s' % (shape, shape[1]))
        else:
            total_arg_size += shape[1].value

    dtype = [a.dtype for a in args][0]

    # Now to computatin.
    with tf.variable_scope(scope, reuse=reuse) as outer_scope:
        weights = tf.get_variable(
            name='linear_kernel',
            shape=[total_arg_size, output_size],
            dtype=dtype,
            regularizer=regularizer,
            initializer=initializer)
        if len(args) == 1:
            res = math_ops.matmul(args[0], weights)
        else:
            res = math_ops.matmul(array_ops.concat(args, axis=1), weights)
        if not bias:
            return res

        with tf.variable_scope(outer_scope) as inner_scope:
            inner_scope.set_partitioner(None)
            biases = tf.get_variable(
                name='linear_bias',
                shape=[output_size],
                dtype=dtype,
                regularizer=regularizer,
                initializer=initializer)
            return nn_ops.bias_add(res, biases)

def total_params():
    total_parameters = 0
    for variable in tf.trainable_variables():
        shape = variable.get_shape()
        variable_parameters = 1
        for dim in shape:
            variable_parameters *= dim.value
        total_parameters += variable_parameters
    print('Total number of trainable parameters: {}'.format(total_parameters))

#####################################################################################################
from tensorflow.contrib.rnn import MultiRNNCell
from tensorflow.python.ops.rnn import bidirectional_dynamic_rnn
from tensorflow.contrib.rnn import LSTMCell, GRUCell

class BiRNNEncoder(object):
    def __init__(self, hidden_size, num_layers, name):
        self._cell = GRUCell(num_units=hidden_size)
        self._hidden_size = hidden_size
        self._num_layers = num_layers
        self.name = name

    def __call__(self, inputs, sequence_length, dropout_prob=0.3, is_last_states=False):
        with tf.variable_scope(self.name, initializer=tf.orthogonal_initializer(), reuse=tf.AUTO_REUSE):

            # cell_fw = MultiRNNCell(cells=[self._cell for _ in range(self._num_layers)])
            # cell_bw = MultiRNNCell(cells=[self._cell for _ in range(self._num_layers)])
            cell_fw = self._cell
            cell_bw = self._cell
            outputs, last_states = bidirectional_dynamic_rnn(cell_bw=cell_bw,
                                                             cell_fw=cell_fw,
                                                             dtype="float32",
                                                             sequence_length=sequence_length,
                                                             inputs=inputs,
                                                             swap_memory=True,
                                                             scope=self.name)
            # print('{} ouputs: {}'.format(self.name, outputs))
            # print('{} last_states: {}'.format(self.name, last_states))

            if is_last_states:
                # outputs shape: (None, hidden_size * 2)
                outputs = tf.concat(last_states, axis=-1)
            else:
                # outputs shape: (None, max_length, hidden_size * 2)
                outputs = tf.concat(outputs, axis=-1)

            # dropout
            if dropout_prob:
                outputs = tf.layers.dropout(outputs, dropout_prob)

            return outputs

max_stars_repo_name: libertatis/mrc-cbt

max_issues_repo_name: libertatis/mrc-cbt

max_forks_repo_name: libertatis/mrc-cbt


text:
import numpy as np
import os, collections, time, json, natsort
import matplotlib.pyplot as plt
from keras.preprocessing import image
from keras.layers import GlobalAveragePooling2D, Dense, Dropout,Activation,Flatten
from kerasfactory.imagenet_utils import preprocess_input
from keras.layers import Input
from keras.models import Model
from keras.utils import np_utils
from keras.callbacks import TensorBoard, EarlyStopping
from tensorboard.plugins.pr_curve import summary as pr_summary
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from kerasfactory.convnetfactory import ConvNetFactory

class TrainService():
	def __init__(self, config_path):
		self.__config = json.loads(open(config_path).read())
		self.__image_input = Input(shape=(224, 224, 3))
		self.__num_classes = 4
		self.__hist = None

	def configure_model(self):
		train_network_params = self.__config['network']
		image_input = self.__image_input

		network_name = train_network_params['name']
		include_top = train_network_params['include_top']

		model = ConvNetFactory.build(network_name, include_top, image_input)
		return model

	def fine_tune_model(self, model):
		train_network_params = self.__config['network']
		activation = train_network_params['activation']
		loss_type = train_network_params['loss_type']
		optimizer = train_network_params['optimizer']
		metrics = train_network_params['metrics']
		image_input = self.__image_input
		num_classes = self.__num_classes

		last_layer = model.get_layer('fc2').output
		out = Dense(num_classes, activation=activation, name='output')(last_layer)
		custom_model = Model(image_input, out)
		custom_model.summary()

		for layer in custom_model.layers[:-1]:
			layer.trainable = False

		custom_model.layers[3].trainable
		print("[INFO] compiling model...")
		custom_model.compile(loss=loss_type,optimizer=optimizer,metrics=[metrics])

		return custom_model

	def train_model(self, custom_model, X_train, y_train, X_test, y_test):
		train_network_params = self.__config['network']
		batch_size_training = train_network_params['batch_size_training']
		batch_size_evaluate = train_network_params['batch_size_evaluate']
		epochs = train_network_params['epochs']
		verbose = train_network_params['verbose']
		tensorboard_params = self.__config['tensorboard']
		name_runs = tensorboard_params["name_runs"]

		t=time.time()
		tensorboard_name = "{0}-{1}".format(name_runs, t)
		tensorBoard = [VisualizeTensorBoard(log_dir="logs/{}".format(tensorboard_name), histogram_freq=1,
          					write_graph=True, write_images=True), EarlyStopping(monitor='val_loss', patience=3)]

		print("[INFO] starting training...")
		print('Run `tensorboard --logdir="./logs"` to see the results')
		hist = custom_model.fit(X_train, y_train, batch_size=batch_size_training, epochs=epochs,
							verbose=verbose, validation_data=(X_test, y_test), callbacks=tensorBoard)
		self.__hist = hist
		print('Training time: %s' % (t - time.time()))

		print("[INFO] Evaluate model...")
		(loss, accuracy) = custom_model.evaluate(X_test, y_test, batch_size=batch_size_evaluate, verbose=verbose)
		print("[INFO] loss={:.4f}, accuracy: {:.4f}%".format(loss,accuracy * 100))

		return custom_model

	def save_model(self, model):
		train_network_params = self.__config['network']
		model_save_path = train_network_params["model_save_path"]
		print("[INFO] dumping architecture and weights to file...")
		model.save(model_save_path)

	def get_data(self):
		image_path = self.__config['data_path']
		data_path = image_path['path']
		num_classes = self.__num_classes

		img_data_list=[]

		img_list=os.listdir(data_path)
		img_list = natsort.natsorted(img_list,reverse=False)

		for img in img_list:
			print(img)
			img_path = data_path + '/'+ img
			img = image.load_img(img_path, target_size=(224, 224))
			x = image.img_to_array(img)
			x = np.expand_dims(x, axis=0)
			x = preprocess_input(x)
			print('Input image shape:', x.shape)
			img_data_list.append(x)

		img_data = np.array(img_data_list)
		print (img_data.shape)
		img_data=np.rollaxis(img_data,1,0)
		print (img_data.shape)
		img_data=img_data[0]
		print (img_data.shape)

		num_of_samples = img_data.shape[0]
		labels = np.ones((num_of_samples,),dtype='int64')

		# labels[:24]=0
		# labels[24:48]=1
		# labels[48:72]=2
		# labels[72:96]=3

		labels[:50]=0
		labels[50:100]=1
		labels[100:150]=2

		print(collections.Counter(labels))

		Y = np_utils.to_categorical(labels, num_classes)
		x,y = shuffle(img_data,Y, random_state=2)
		X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=2)

		return X_train, X_test, y_train, y_test

	def visualize_graphs_metrics(self):

		hist = self.__hist

		train_loss=hist.history['loss']
		val_loss=hist.history['val_loss']
		train_acc=hist.history['acc']
		val_acc=hist.history['val_acc']
		xc=range(12)

		plt.figure(1,figsize=(7,5))
		plt.plot(xc,train_loss)
		plt.plot(xc,val_loss)
		plt.xlabel('num of Epochs')
		plt.ylabel('loss')
		plt.title('train_loss vs val_loss')
		plt.grid(True)
		plt.legend(['train','val'])
		plt.style.use(['classic'])

		plt.figure(2,figsize=(7,5))
		plt.plot(xc,train_acc)
		plt.plot(xc,val_acc)
		plt.xlabel('num of Epochs')
		plt.ylabel('accuracy')
		plt.title('train_acc vs val_acc')
		plt.grid(True)
		plt.legend(['train','val'],loc=4)
		plt.style.use(['classic'])

		plt.show()

class VisualizeTensorBoard(TensorBoard):
    def __init__(self, *args, **kwargs):
        self.pr_curve = kwargs.pop('pr_curve', True)
        super(VisualizeTensorBoard, self).__init__(*args, **kwargs)

        global tf
        import tensorflow as tf

    def set_model(self, model):
        super(VisualizeTensorBoard, self).set_model(model)

        if self.pr_curve:
            predictions = self.model._feed_outputs[0]
            labels = tf.cast(self.model._feed_targets[0], tf.bool)
            self.pr_summary = pr_summary.op(name='pr_curve',
                                            predictions=predictions,
                                            labels=labels,
                                            display_name='Precision-Recall Curve')

    def on_epoch_end(self, epoch, logs=None):
        super(VisualizeTensorBoard, self).on_epoch_end(epoch, logs)

        if self.pr_curve and self.validation_data:
            tensors = self.model._feed_targets + self.model._feed_outputs
            predictions = self.model.predict(self.validation_data[:-2])
            val_data = [self.validation_data[-2], predictions]
            feed_dict = dict(zip(tensors, val_data))
            result = self.sess.run([self.pr_summary], feed_dict=feed_dict)
            self.writer.add_summary(result[0], epoch)
        self.writer.flush()

ext: py

max_stars_repo_name: alexandergg/Keras-Factory

max_issues_repo_name: alexandergg/Keras-Factory

max_forks_repo_name: alexandergg/Keras-Factory

reason: import numpy


text:
import numpy as np
from sklearn.exceptions import NotFittedError
from sklearn.utils import check_array
from sklearn.metrics import pairwise_distances_argmin_min

def debug_print(s, debug=True):
    if debug:
        print(s)

def compute_cost(X, cluster_centers, cost_func, remove_outliers=None):
    """
    :param X: array,
        data set
    :param remove_outliers: None or int, default None
        whether to remove outliers when computing the cost on X
    :return: float,
        actual cost
    """
    if cluster_centers is None:
        raise NotFittedError("Model hasn't been fitted yet\n")
    X = check_array(X, ensure_2d=True)
    _, dists = pairwise_distances_argmin_min(X, cluster_centers, axis=1)
    dist_idxs = np.argsort(dists)
    if remove_outliers is not None:
        assert remove_outliers >= 0
        dist_idxs = dist_idxs if remove_outliers == 0 else dist_idxs[:-int(remove_outliers)]

        return cost_func(X[dist_idxs], cluster_centers)
    else:
        return cost_func(X, cluster_centers)

max_stars_repo_name: xyguo/OnlineKZMedian

max_issues_repo_name: xyguo/OnlineKZMedian

max_forks_repo_name: xyguo/OnlineKZMedian


text:
# -*- coding:utf-8 -*-

"""
@file:sas_dataset.py
@author:DrLC(Zhang Huangzhao)
@email:zhang_hz@pku.edu.cn
@time:2018/3/5 21:22
"""

import pickle, gzip
import random
import numpy

class SAS(object):
    '''
    The simple add sequence dataset

    Load the SAS dataset and split original train-set into train-set and
    valid-set.
    This class generate batch randomly, by calling "minibatch", "test_batch"
    or "valid_batch", and convert batch back to sequence data pair by calling
    "batch2raw".

    Parameters:

        path: path of the generated dataset, "./sas.pkl.gz" as default.
        valid_ratio: ratio of the valid-set in the original train-set.
        rand_seed: random seed for numpy.random and random.
    '''

    def __init__(self, path,
                 valid_ratio=0.2,
                 rand_seed=1234):

        self.set_rand_seed(rand_seed)

        if valid_ratio > 1:
            valid_ratio = 1
        elif valid_ratio < 0:
            valid_ratio = 0

        f = gzip.open(path, "rb")
        d = pickle.load(f)
        f.close()

        valid_size = int(valid_ratio * len(d["train"]))

        self.__test = d["test"]
        self.__valid = d["train"][:valid_size]
        self.__train = d["train"][valid_size:]
        self.__test_size = len(self.__test)
        self.__valid_size = len(self.__valid)
        self.__train_size = len(self.__train)

        self.__epoch = random.sample(list(range(self.__train_size)),
                                     self.__train_size)
        self.__dictionary = ["1", "2", "3", "4", "5",
                             "6", "7", "8", "9", "0",
                             " "]

    def minibatch(self, batch_size, max_len=15):

        '''
        Generate a minibatch from the current epoch. If the current epoch
        is smaller than batch size, generate a new epoch and extract a
        minibatch from the new epoch.
        The minibatch includes sequences in the form of a numpy.2darray,
        labels in the form of a numpy.1darray, lengths of each sequence in
        the form of a numpy.1darray.

        Parameter:

            batch_size: size of the minibatch.
            max_len: the final length of the sequences, with padding. Of
                course, max_len >= max(len(any sequence)). If the length
                of a sequence is less than max_len, pad blank at the end
                until the length reaches max_len.

        Returns:

            A batch, which is a list of [X, Y, L]
            X: sequence array, a numpy.2darray with the shape of
                [batch size, max length]. Each row is a padded sequence.
            Y: label vector, a numpy.1darray with the shape of [batch size].
                Each element is a label.
            L: length vector, a numpy.1darray with the shape of [batch size].
                Each element is the valid length (no padding length) of the
                corresponding sequence.
        '''

        if batch_size > self.__train_size:
            batch_size = self.__train_size
        if batch_size > len(self.__epoch):
            self.__epoch = random.sample(list(range(self.__train_size)),
                                         self.__train_size)

        idx = self.__epoch[:batch_size]
        self.__epoch = self.__epoch[batch_size:]

        X = []
        Y = []
        L = []
        for i in idx:

            if self.__train[i][1] == "GE":
                Y.append(0)
            elif self.__train[i][1] == "LT":
                Y.append(1)
            else:
                assert False

            assert len(self.__train[i][0]) <= max_len
            L.append(len(self.__train[i][0]))

            tmp_seq = []
            for t in self.__train[i][0]:
                assert t in self.__dictionary
                tmp_seq.append(self.__dictionary.index(t))
            tmp_seq += [10 for iii in range(max_len - L[-1])]
            X.append(tmp_seq)

        X = numpy.asarray(X, dtype=numpy.int32)
        Y = numpy.asarray(Y, dtype=numpy.int32)
        L = numpy.asarray(L, dtype=numpy.int32)

        return X, Y, L

    def test_batch(self, batch_size, max_len=15):

        '''
        Generate a batch from test-set. Similar to minibatch, but there is
        no such thing like "epoch" in test_batch.

        Parameters:

            batch_size: size of the test batch.
            max_len: length of the sequences after padding.

        Returns:

            Same as minibatch
        '''

        if batch_size > self.__test_size:
            batch_size = self.__test_size

        idx = random.sample(list(range(self.__test_size)),
                            batch_size)

        X = []
        Y = []
        L = []
        for i in idx:

            if self.__test[i][1] == "GE":
                Y.append(0)
            elif self.__test[i][1] == "LT":
                Y.append(1)
            else:
                assert False

            assert len(self.__test[i][0]) <= max_len
            L.append(len(self.__test[i][0]))

            tmp_seq = []
            for t in self.__test[i][0]:
                assert t in self.__dictionary
                tmp_seq.append(self.__dictionary.index(t))
            tmp_seq += [10 for iii in range(max_len - L[-1])]
            X.append(tmp_seq)

        X = numpy.asarray(X, dtype=numpy.int32)
        Y = numpy.asarray(Y, dtype=numpy.int32)
        L = numpy.asarray(L, dtype=numpy.int32)

        return X, Y, L

    def valid_batch(self, batch_size, max_len=15):

        '''
        Generate a batch from valid-set. Same as test_batch

        Parameters:

            batch_size: size of the test batch.
            max_len: length of the sequences after padding.

        Returns:

            Same as test_batch
        '''

        if batch_size > self.__valid_size:
            batch_size = self.__valid_size

        idx = random.sample(list(range(self.__valid_size)),
                            batch_size)

        X = []
        Y = []
        L = []
        for i in idx:

            if self.__valid[i][1] == "GE":
                Y.append(0)
            elif self.__valid[i][1] == "LT":
                Y.append(1)
            else:
                assert False

            assert len(self.__valid[i][0]) <= max_len
            L.append(len(self.__valid[i][0]))

            tmp_seq = []
            for t in self.__valid[i][0]:
                assert t in self.__dictionary
                tmp_seq.append(self.__dictionary.index(t))
            tmp_seq += [10 for iii in range(max_len - L[-1])]
            X.append(tmp_seq)

        X = numpy.asarray(X, dtype=numpy.int32)
        Y = numpy.asarray(Y, dtype=numpy.int32)
        L = numpy.asarray(L, dtype=numpy.int32)

        return X, Y, L

    def batch2raw(self, batch):

        '''
        Convert a batch back to raw sequence data., for debugging usage.

        Parameters:

            batch: a batch generated from minibatch, test_batch or
                valid_batch.

        Returns:

            A sequence data pair, as a list of [sequences, labels].
        '''

        x, y, l = batch

        seq = []
        for i in range(len(l)):
            tmp_seq = ""
            for j in range(l[i]):
                tmp_seq += self.__dictionary[x[i][j]]
            seq.append(tmp_seq)

        label = []
        for i in y:
            if i == 0:
                label.append("GE")
            elif i == 1:
                label.append("LT")
            else:
                assert False

        return seq, label

    def set_rand_seed(self, rand_seed):

        '''
        Set random seed for numpy.random and random.

        Parameters:

            rand_seed: random seed.

        Returns:

            None
        '''

        random.seed(rand_seed)
        numpy.random.seed(rand_seed)

    def get_test_size(self):

        '''
        Get test-set size

        Parameters:

            None

        Returns:

            Size of test-set.
        '''

        return self.__test_size

    def get_train_size(self):

        '''
        Get train-set size

        Parameters:

            None

        Returns:

            Size of train-set.
        '''

        return self.__train_size

    def get_valid_size(self):

        '''
        Get valid-set size

        Parameters:

            None

        Returns:

            Size of valid-set.
        '''

        return self.__valid_size

class generator(object):
    '''
    The simple add sequence generator

    Randomly generate pieces of data for sas GE/LT classification task.
    Each piece of data has an input string and an output label.

    The input string has 4 integer numbers, seperated by a blank. There
    are two candidate output labels -- "GE" and "LT".
    Denote the 4 integer as a, b, c, ans (in string sequential order).
    If a+b-c >= ans, the output label should be "GE"; otherwise, the
    output label should be "LT".

    eg. Input: "65 235 328 905"; output: "LT".
        Input: "744 25 3 376";   output: 'GE'.
    '''

    def __init__(self, num_lim=[0, 1000],
                 seq_num=120000,
                 test_num=20000,
                 train_num=100000,
                 rand_seed=1234):

        # set parameters
        self.set_param(num_lim, seq_num, test_num, train_num, rand_seed)
        # generate sequences
        self.generate()

    def set_param(self, num_lim=[0, 1000],
                  seq_num=100000,
                  test_num=20000,
                  train_num=100000,
                  rand_seed=1234):

        '''
        Set the parameters, including boundary of a, b, c and ans, sequence
        number, and random seed.

        Parameters:

            num_lim: boundary of a, b, c and ans. The first element is the
                lower bound, and the second is the upper bound.
            seq_num: sequence number to be generated.
            test_num: sequence number for test set.
            train_num: sequence number for train set.
            rand_seed: random seed for numpy.random

        Returns:

            None
        '''

        # notice that sequence number must eqaul to
        # the sum of test set size and train set size
        assert test_num + train_num == seq_num

        # set random seed
        numpy.random.seed(rand_seed)
        # set the parameters
        self.__min_num = num_lim[0]
        self.__max_num = num_lim[1]
        self.__seq_num = seq_num
        self.__test_num = test_num
        self.__train_num = train_num

    def generate(self):

        '''
        Generate data pairs (input string and output label), and do some
        statistics on the generated dataset.

        Parameters:

            None

        Returns:

            None
        '''

        self.__seq = []
        for cnt in range(self.__seq_num):
            # generate a, b, c, ans randomly
            a, b, c, ans = self.__generate_random_number_set()
            tmp_seq = str(a) + " " + str(b) + " " + str(c) + " " + str(ans)
            # compare a+b-c with ans
            if a + b - c >= ans:
                tmp_label = "GE"  # greater or equal
            else:
                tmp_label = "LT"  # less than
            self.__seq.append([tmp_seq, tmp_label])

        self.test = self.__seq[:self.__test_num]
        self.train = self.__seq[self.__test_num:]

        # do some statistics works
        self.__stat_seq_len()
        self.__stat_label()

    def __generate_random_number_set(self):

        '''
        Generate the four numbers for each data pair.

        Parameters:

            None

        Returns:

            a, b, c and ans for one input string sequence
        '''

        tmp_arr = numpy.random.uniform(self.__min_num,
                                       self.__max_num,
                                       size=[4])
        a, b, c, d = numpy.asarray(tmp_arr, dtype=numpy.int32)
        return a, b, c, d

    def __stat_seq_len(self):

        '''
        Statistics. Get the distribution of sequence lengths.

        Parameters:

            None

        Returns:

            None
        '''

        self.seq_len = {}
        for d in self.__seq:
            tmp_l = len(d[0])
            if tmp_l not in self.seq_len.keys():
                self.seq_len[tmp_l] = 1
            else:
                self.seq_len[tmp_l] += 1

    def __stat_label(self):

        '''
        Statistics. Get the distribution of GE/LT labels.

        Parameters:

            None

        Returns:

            None
        '''

        self.label_cnt = {"GE": 0, "LT": 0}
        for d in self.__seq:
            self.label_cnt[d[1]] += 1

    def save_sas(self, path):

        f = gzip.open(path, "wb")
        pickle.dump({"train": self.train,
                     "test": self.test}, f)
        f.close()

    def get_sas(self):

        return {"train": self.train,
                "test": self.test}


text:
import sys
import os
sys.path.insert(1, os.path.join(sys.path[0], '..'))
from core.ConstraintNetwork import ConstraintNetwork
import numpy as np
import torch

torch.manual_seed(2021)
np.random.seed(2021)

def main():

    # Settings
    abs_path = os.path.dirname(os.path.abspath(__file__)) + '/'
    datasets_dir = abs_path + '../data/'
    output_dir   = abs_path + '../data/constraint_networks_MADDPG/'

    # Training Settings
    EPOCHS = 100
    BATCH_SIZE = 256
    VAL_RATIO = 0.1

    # Check if output directory exists
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    # Import Datasets
    state = np.genfromtxt(datasets_dir + "D_state_decentralized.csv", delimiter = ',')
    action = np.genfromtxt(datasets_dir + "D_action_decentralized.csv", delimiter = ',')
    constraint_diff = np.genfromtxt(datasets_dir + "D_constraint_decentralized.csv", delimiter = ',')

    # Remove Indices
    state  = state[:, 1:]
    action = action[:, 1:]
    constraint_diff = constraint_diff[:, 1:]

    # Number of networks
    N = constraint_diff.shape[1]

    # Train one network for each constraint
    for i in range(N):

        # Define Network for constraint i
        net = ConstraintNetwork(state_dim = state.shape[1], act_dim = action.shape[1])
        net.train(state, action, constraint_diff[:, i], EPOCHS, BATCH_SIZE, VAL_RATIO)

        # Store
        torch.save(net.state_dict(), output_dir + "constraint_net_" + str(i) + ".pkl")

if __name__ == "__main__":
    main()

max_stars_repo_name: zisikons/deep-rl

max_issues_repo_name: zisikons/deep-rl

max_forks_repo_name: zisikons/deep-rl


text:
from tkinter import W
import cv2
import numpy as np

img = cv2.imread('output.png')
img = cv2.cvtColor(img, cv2.COLOR_BGR2HLS)

l = (10,50,150)
u = (255,255,255)

mask = cv2.inRange(img, l, u)
erode = cv2.erode(mask, None, iterations=1)
# dilate = cv2.dilate(erode, None, iterations=3)

cv2.imshow('frame', erode)
cv2.waitKey()


text:
#import numpy as np
import matplotlib.pyplot as plt

from wordcloud import WordCloud

f1 = open("rajdeep.txt",'r')
f2 = open("test.txt",'w')

def preprocess(sentence):
    #CASE FOLDING [NOT COMPLETE]
    sentence = sentence.lower()

    #DATA CLEANING
    sentence = sentence.replace('https','')
    sentence = sentence.replace('t.co','')
    sentence = sentence.replace('sardesairajdeep','')
    sentence = sentence.replace('abhijtmajumder','')
    sentence = sentence.replace('anupamPkher','')
    sentence = sentence.replace('@','')
    sentence = sentence.replace('[#]?','')
    sentence = sentence.replace('rt','')
    sentence = sentence.replace(',','')
    sentence = sentence.replace('!','')
    sentence = sentence.replace('?','')
    sentence = sentence.replace('.','')
    sentence = sentence.replace('\'','')
    sentence = sentence.replace('\"','')
    sentence = sentence.replace(':','')
    sentence = sentence.replace('indiatoday','')
    sentence = sentence.replace('today','')
    sentence = sentence.replace('new','')
    #REMOVE REPEATED CHARS
    #sentence = re.sub(r'(\w)\1+', r'\1', sentence)

    return sentence

for eachline in f1:
    f2.write(preprocess(eachline))
f2.close()
f1.close()
# Read the whole text.
text = open('test.txt').read()

# read the mask image
# taken from
# http://www.stencilry.org/stencils/movies/alice%20in%20wonderland/255fk.jpg
#alice_mask = np.array(Image.open(path.join(d, "alice_mask.png")))

wc = WordCloud(background_color="white", max_words=2000)
# generate word cloud
wc.generate(text)

# store to file
#wc.to_file(path.join(d, "alice.png"))

# show
plt.imshow(wc)
plt.axis("off")
plt.figure()
#plt.imshow(alice_mask, cmap=plt.cm.gray)
#plt.axis("off")
plt.show()

max_stars_repo_name: suraj-jayakumar/sentiment-analysis

max_issues_repo_name: sjayakum/sentiment-analysis

max_forks_repo_name: sjayakum/sentiment-analysis


text:
import pandas as pd
import numpy as np

# Read the dataset into a data table using Pandas
df = pd.read_csv("movie_ratings_data_set.csv")

# Convert the running list of user ratings into a matrix using the 'pivot table' function
ratings_df = pd.pivot_table(df, index='user_id', columns='movie_id', aggfunc=np.max)

# Create a csv file of the data for easy viewing
ratings_df.to_csv("review_matrix.csv", na_rep="")


text:
import numpy as np
from functools import wraps
import pandas as pd
from numba import jit
from array import array
"""
Miscellaneous functions.
"""

def sort_data(func):
    """
    Initial datasets are rarely sorted by obmt. This is trivial to fix
    but needs to be fixed often. As per DRY, it is packaged here for use
    as a decorator.
    """
    @wraps(func)
    def sort(df, *args, **kwargs):
        if isinstance(df, pd.DataFrame) and 'obmt' in df.columns.values:
            sorted_df = df.sort_values('obmt')
        else:
            sorted_df = df
        return func(sorted_df, *args, **kwargs)
    return sort

def s2o(secs):
    """Convert seconds to obmt."""
    return secs/21600

def o2s(obmt):
    """Convert obmt to seconds."""
    return obmt*21600

@jit(nopython=True)
def isolate_true(data):
    """Turn all Trues except the first into Falses in a run of Trues."""
    data_backwards = data[::-1]
    x = []
    for i in range(len(data) - 1):
        if data_backwards[i] and data_backwards[i+1]:
            x.append(0)
        else:
            x.append(data_backwards[i])
    x.append(data[0])
    return x[::-1]

def isolate_hit_df(df):
    """
    Isolate events by removing multiple True readings for the same hit.
    """
    df['hits'] = isolate_true(list(df['anomaly']))
    df['hits'] = df['hits'].astype('bool')
    return df

def hit_start_end_df(df):
    """Label hit start and end times."""
    df['start'] = isolate_true(list(df['anomaly']))
    df['end'] = isolate_true(list(df['anomaly'])[::-1])[::-1]
    df['start'] = df['start'].astype('bool')
    df['end'] = df['end'].astype('bool')
    return df

max_stars_repo_name: bombrun/GaiaLab

max_issues_repo_name: MaraBucur/GaiaLab

max_forks_repo_name: MaraBucur/GaiaLab

reason: import numpy,from numba


text:
"""
Demo of a line plot on a polar axis.
"""
import numpy as np
import matplotlib.pyplot as plt

r = np.arange(0, 3.0, 0.01)
theta = 2 * np.pi * r

ax = plt.subplot(111, polar=True)
ax.plot(theta, r, color='r', linewidth=3)
ax.set_rmax(2.0)
ax.grid(True)

ax.set_title("A line plot on a polar axis", va='bottom')
plt.show()

max_stars_repo_name: jbbrokaw/matplotlib

max_issues_repo_name: yingkailiang/matplotlib

max_forks_repo_name: yingkailiang/matplotlib


text:
import marshmallow as ma
from marshmallow import pre_load, ValidationError
import numpy as np

class HHLAlgorithmRequest:
    def __init__(self, matrix, vector):
        self.matrix = matrix
        self.vector = vector

class HHLAlgorithmRequestSchema(ma.Schema):
    matrix = ma.fields.List(ma.fields.List(ma.fields.Float()))
    vector = ma.fields.List(ma.fields.Float())

class QAOAAlgorithmRequest:
    def __init__(self, pauli_op_string, gammas, betas):
        self.pauli_op_string = pauli_op_string
        self.reps = reps
        self.gammas = gammas
        self.betas = betas

class QAOAAlgorithmRequestSchema(ma.Schema):
    pauli_op_string = ma.fields.String()
    reps = ma.fields.Int()
    gammas = ma.fields.List(ma.fields.Float())
    betas = ma.fields.List(ma.fields.Float())

max_stars_repo_name: UST-QuAntiL/Quokka

max_issues_repo_name: UST-QuAntiL/Quokka

max_forks_repo_name: UST-QuAntiL/Quokka


text:
import torch
import numpy as np
import umap
from torch import autograd
from tqdm import tqdm
from ot import emd2, sinkhorn2
from scipy.spatial.distance import cdist

def predict(model, dataloader):
    prediction_list = []
    with torch.no_grad():
        for i, batch in enumerate(dataloader):
            pred = model(batch[0])
            prediction_list.append(pred.cpu())
    return torch.cat(prediction_list, 0).numpy()

def score_clf(model, dataloader, domain=None):
    prediction_list = []
    labels = []
    with torch.no_grad():
        for i, batch in enumerate(dataloader):
            x, y, _ = batch
            pred = model(x, y, 0)[1]
            prediction_list.append(pred.cpu())
            labels.append(y.cpu())

    score = np.mean(
        np.argmax(torch.cat(prediction_list, 0).numpy(), 1)
        == np.argmax(torch.cat(labels, 0).numpy(), 1)
    )
    return score

def score_dis(model, dataloader1, dataloader2):
    prediction_list = []
    with torch.no_grad():
        for i, batch in enumerate(dataloader1):
            x, y, _ = batch
            pred = model(x, y, 0)[2]
            prediction_list.append(pred.cpu() <= 0.5)
        score1 = torch.cat(prediction_list, 0)
        prediction_list = []
        for i, batch in enumerate(dataloader2):
            x, y, _ = batch
            pred = model(x, y, 0)[2]
            prediction_list.append(pred.cpu() > 0.5)
        score2 = torch.cat(prediction_list, 0)
    return torch.true_divide(
        torch.sum(torch.cat((score1, score2), 0)), (score1.shape[0] + score2.shape[0])
    )

class TensorArray:
    def __init__(self, batch_size, number_classes, *args):
        self.array = [np.random.rand(1, number_classes, *args) for _ in range(batch_size)]
        self.array = list(
            [
                torch.Tensor(x / np.sum(x, axis=1, keepdims=True)).cuda()
                for x in self.array
            ]
        )
        self.batch_size = batch_size
        self.number_classes = number_classes

    def __call__(self):
        return torch.cat(self.array)

    def append(self, element):
        length = element.shape[0]
        if length > 0:
            self.array = self.array[length:]
            # make all except the new tensors detached from the graph
            #self.array = list([x.detach() for x in self.array])
            # make a list of tensors and add them to the list
            list_elements = list(torch.chunk(element, length, dim=0))
            self.array = self.array + list_elements

def get_n_params(model):
    pp=0
    for p in list(model.parameters()):
        nn=1
        for s in list(p.size()):
            nn = nn*s
        pp += nn
    return pp

def get_feature_matrix(net, dataloader1, dataloader2=None, with_labels=False):
    d1_feats = []
    d2_feats = []

    if not with_labels:
        with torch.no_grad():
            for data in tqdm(dataloader1):
                x, _, _ = data
                feat = net.embedder(x)
                d1_feats.append(feat)
            d1_feats = torch.cat(d1_feats).detach().cpu().numpy()
            if dataloader2 is not None:
                for data in tqdm(dataloader2):
                    x, _, _ = data
                    feat = net.embedder(x)
                    d2_feats.append(feat)
                d2_feats = torch.cat(d2_feats).detach().cpu().numpy()
                return d1_feats, d2_feats
            else:
                return d1_feats
    else:
        with torch.no_grad():
            for data in tqdm(dataloader1):
                x, y, _ = data
                feat, _, _, ymix = net(x, y, 1)
                d1_feats.append(torch.hstack((feat, ymix)))
            d1_feats = torch.cat(d1_feats).detach().cpu().numpy()
            if dataloader2 is not None:
                for data in tqdm(dataloader2):
                    x, y, _ = data
                    feat, _, _, ymix = net(x, y, 1)
                    d2_feats.append(torch.hstack((feat, ymix)))
                d2_feats = torch.cat(d2_feats).detach().cpu().numpy()
                return d1_feats, d2_feats
            else:
                return d1_feats

def wasserstein_distance(feats1, feats2, sinkhorn=False):
    C_feats = cdist(feats1, feats2)
    if sinkhorn:
        out_feats = sinkhorn2([], [], C_feats, log=True, processes=36)
    else:
        out_feats = emd2([], [], C_feats, log=True, processes=36)
    return out_feats[0]

def get_umap_features(feats1, feats2):
    all_feats = np.concatenate([feats1, feats2])
    stop = len(feats1)
    ubeds = umap.UMAP(verbose=True).fit_transform(all_feats)
    ubeds1 = ubeds[:stop]
    ubeds2 = ubeds[stop:]
    return ubeds1, ubeds2

max_stars_repo_name: leoandeol/ldir

max_issues_repo_name: leoandeol/ldir

max_forks_repo_name: leoandeol/ldir

reason: import numpy,from scipy


text:
import h5py
import numpy as np

from neurokernel.LPU.InputProcessors.BaseInputProcessor import BaseInputProcessor

class BU_InputProcessor(BaseInputProcessor):
    def __init__(self, shape, dt, dur, name, video_config, rf_config, neurons,
                 record_file = None, record_interval = 1):
        video_cls = Video_factory(video_config.get('type', 'moving_bar_l2r'))
        self.video = video_cls(shape, dt, dur, video_config.get('bar_width', 50),
                               record_file = video_config.get('record', None),
                               record_interval = video_config.get('record_interval', 1))
        uids = list(neurons.keys())
        neuron_names = [neurons[n]['name'] for n in uids]
        neuron_ids = np.array([int(name.split('/')[1][1:]) for name in neuron_names])
        neuron_side = set([name.split('/')[1][0] for name in neuron_names])
        if len(neuron_side) > 1:
            raise ValueError('BU neurons must be on one side')
        else:
            self.hemisphere = neuron_side.pop()

        self.fc = CircularGaussianFilterBank(
                        (shape[0], shape[1]),
                        rf_config.get('sigma', 0.05), 10,
                        hemisphere = self.hemisphere)

        self.index = neuron_ids - 1
        var_list = [('I', uids)]
        self.name = name
        # self.n_inputs = 80
        #self.filter_filename = '{}_filters.h5'.format(self.name)
        super(BU_InputProcessor, self).__init__(var_list,
                                                sensory_file = self.video.record_file,
                                                sensory_interval = self.video.record_interval,
                                                input_file = record_file,
                                                input_interval = record_interval)

    def pre_run(self):
        self.video.pre_run()
        self.fc.create_filters()
        # self.file = h5py.File('{}_inputs.h5'.format(self.name), 'w')
        # self.file.create_dataset('I',
        #                          (0, self.n_inputs),
        #                          dtype = np.double,
        #                          maxshape=(None, self.n_inputs))

    def is_input_available(self):
        return True

    def update_input(self):
        frame = self.video.run_step()
        BU_input = (3*self.fc.apply_filters(frame)).reshape(-1)
        self.variables['I']['input'] = BU_input[self.index]
        # self.record_frame(BU_input)

    def record_frame(self, input):
        self.file['I'].resize((self.file['I'].shape[0]+1, self.n_inputs))
        self.file['I'][-1,:] = input

    def __del__(self):
        try:
            self.close_file()
        except:
            pass

class PB_InputProcessor(BaseInputProcessor):
    def __init__(self, shape, dt, dur, name, video_config, rf_config, neurons,
                 record_file = None, record_interval = 1):
        video_cls = Video_factory(video_config.get('type', 'moving_bar_l2r'))
        self.video = video_cls(shape, dt, dur, video_config.get('bar_width', 50),
                               record_file = video_config.get('record', None),
                               record_interval = video_config.get('record_interval', 1))
        num_glomeruli = rf_config.get('num_glomeruli', 18)
        self.fr = RectangularFilterBank(shape, num_glomeruli)
        uids = list(neurons.keys())
        neuron_names = [neurons[n]['name'] for n in uids]
        neuron_side = [name.split('/')[1][0] for name in neuron_names]
        neuron_ids = np.array([(num_glomeruli//2+1 - int(name.split('/')[1][1:])) \
                     if neuron_side[i] == 'L' else\
                      int(name.split('/')[1][1:]) + num_glomeruli//2 \
                      for i, name in enumerate(neuron_names)])

        self.index = neuron_ids - 1
        var_list = [('I', uids)]
        self.name = name
        # self.n_inputs = 18
        super(PB_InputProcessor, self).__init__(var_list,
                                                sensory_file = self.video.record_file,
                                                sensory_interval = self.video.record_interval,
                                                input_file = record_file,
                                                input_interval = record_interval)

    def pre_run(self):
        self.video.pre_run()
        self.fr.create_filters()
        # self.file = h5py.File('{}_inputs.h5'.format(self.name), 'w')
        # self.file.create_dataset('I',
        #                          (0, self.n_inputs),
        #                          dtype = np.double,
        #                          maxshape=(None, self.n_inputs))

    def is_input_available(self):
        return True

    def update_input(self):
        frame = self.video.run_step()
        PB_input = 3*self.fr.apply_filters(frame)

        self.variables['I']['input'] = PB_input[self.index]
        # self.record_frame(PB_input)

    def record_frame(self, input):
        self.file['I'].resize((self.file['I'].shape[0]+1, self.n_inputs))
        self.file['I'][-1,:] = input

    def __del__(self):
        try:
            self.close_file()
        except:
            pass

class CX_Video(object):
    """
    Create a test video signal.
    """
    def __init__(self, shape, dt, dur, record_file = None, record_interval = 1):
        self.shape = shape
        self.dt = dt
        self.dur = dur
        self.N_t = int(self.dur/self.dt)
        self.frame = 0
        self.record_file = record_file
        self.record_interval = record_interval
        self.record_count = 0

    def run_step(self):
        frame = self.generate_frame()
        self.frame += 1
        if self.record:
            if self.record_count == 0:
                self.record_frame()
            self.record_count = (self.record_count+1)%self.record_interval
        return frame

    def pre_run(self):
        if self.record_file is not None:
            self.file = h5py.File(self.record_file, 'w')
            self.file.create_dataset('sensory',
                                     (0, self.shape[0], self.shape[1]),
                                     dtype = np.double,
                                     maxshape=(None, self.shape[0],
                                               self.shape[1]))
            self.record = True
        else:
            self.record = False
        self.data = np.empty(self.shape, np.double)

    def record_frame(self):
        self.file['sensory'].resize((self.file['sensory'].shape[0]+1,
                                         self.shape[0], self.shape[1]))
        self.file['sensory'][-1,:,:] = self.data

    def generate_frame(self):
        pass

    def close_file(self):
        if self.record:
            self.file.close()

    def __del__(self):
        try:
            self.close_file()
        except:
            pass

class moving_bar_l2r(CX_Video):
    def __init__(self, shape, dt, dur, bar_width,
                 record_file = None, record_interval = 1):
        super(moving_bar_l2r, self).__init__(shape, dt, dur,
                                             record_file = record_file,
                                             record_interval = record_interval)
        self.bar_width = bar_width

    def generate_frame(self):
        start = int(np.ceil(self.frame*(self.shape[1]-self.bar_width)/float(self.N_t)))
        self.data.fill(0)
        self.data[:, start:start+self.bar_width] = 1.0
        return self.data

class moving_bar_r2l(CX_Video):
    def __init__(self, shape, dt, dur, bar_width,
                 record_file = None, record_interval = 1):
        super(moving_bar_r2l, self).__init__(shape, dt, dur,
                                             record_file = record_file,
                                             record_interval = record_interval)
        self.bar_width = bar_width

    def generate_frame(self):
        start = int(np.ceil(self.frame*(self.shape[1]-self.bar_width)/float(self.N_t)))
        self.data.fill(0)
        self.data[:, self.shape[1]-self.bar_width-start:-start] = 1.0
        return self.data

def Video_factory(video_class_name):
    all_video_cls = CX_Video.__subclasses__()
    all_video_names = [cls.__name__ for cls in all_video_cls]
    try:
        video_cls = all_video_cls[all_video_names.index(video_class_name)]
    except ValueError:
        print('Invalid Video subclass name: {}'.format(video_class_name))
        print('Valid names: {}'.format(all_video_names))
        return None
    return video_cls

class CircularGaussianFilterBank(object):
    """
    Create a bank of circular 2D Gaussian filters.

    Parameters
    ----------
    shape : tuple
        Image dimensions.
    sigma : float
        Parameter of Gaussian.
    n : int
        How many blocks should occupy the x-axis.
    """

    def __init__(self, shape, sigma, n, hemisphere = 'L'):
        self.shape = shape
        self.sigma = sigma
        self.n = n
        self.hemisphere = hemisphere

        # Compute maximal and minimal response of a centered filter to use for
        # normalization:
        self.norm_min = np.inner(np.zeros(np.prod(shape)),
                                 self.gaussian_mat(shape, sigma, 0, 0, n).reshape(-1))
        self.norm_max = np.inner(np.ones(np.prod(shape)),
                                 self.gaussian_mat(shape, sigma, 0, 0, n).reshape(-1))

    def normalize_output(self, output):
        """
        Normalize filter output against range of responses to a centered RF.
        """

        return output/(self.norm_max-self.norm_min)

    @classmethod
    def func_gaussian(cls, x, y, sigma):
        """
        2D Gaussian function.
        """

        return (1.0/(1*np.pi*(sigma**2)))*np.exp(-(1.0/(2*(sigma**2)))*(x**2+y**2))

    @classmethod
    def gaussian_mat(cls, shape, sigma, n_x_offset, n_y_offset, n):
        """
        Compute offset circular 2D Gaussian.
        """

        # Image dimensions in pixels:
        N_y, N_x = shape

        # Normalized image width and height:
        x_max = 1.0
        y_max = N_y/float(N_x)

        X, Y = np.meshgrid(np.linspace(-x_max/2, x_max/2, N_x)-(n_x_offset/float(n)),
                           np.linspace(-y_max/2, y_max/2, N_y)-(n_y_offset/float(n)/2))
        return cls.func_gaussian(X, Y, sigma)

    def create_filters(self, filename = None):
        """
        Create filter bank as order-4 tensor.
        """

        N_y, N_x = self.shape

        # Normalized image width and height:
        x_max = 1.0
        y_max = N_y/float(N_x)

        # Compute how many blocks to use along the y-axis:
        n = self.n
        m = n*2*N_y//N_x

        # Construct filters offset by the blocks:
        # n_x_offsets = np.linspace(np.ceil(-n/2.0), np.floor(n/2.0), n)
        if self.hemisphere == 'L':
            n_x_offsets = np.linspace((-n+0.5)/2, -0.5/2, n)
        else:
            n_x_offsets = np.linspace(0.5/2, (n-0.5)/2, n)[::-1].copy()
        n_y_offsets = np.linspace(np.ceil(-m/2.0), np.floor(m/2.0), m)
        filters = np.empty((m, n, N_y, N_x), np.float64)
        for j, n_x_offset in enumerate(n_x_offsets):
            for i, n_y_offset in enumerate(n_y_offsets):
                filters[i, j] = self.gaussian_mat(self.shape, self.sigma,
                                                 n_x_offset, n_y_offset, n)
        self.filters = filters
        if filename is not None:
            file = h5py.File(filename,'w')
            file.create_dataset('filter', filters.shape, filters.dtype, data=filters)
            file.close()

    def apply_filters(self, frame, normalize=True):
        """
        Compute inner products of computed filters and a video frame.
        """

        result = np.tensordot(self.filters, frame)
        if normalize:
            return self.normalize_output(result)
        else:
            return result

class RectangularFilterBank(object):
    """
    Create a bank of 2D rectangular filters that tile the x-axis.
    """

    def __init__(self, shape, n):
        self.shape = shape
        self.n = n

        # Compute maximal and minimal response of a centered filter to use for
        # normalization:
        self.norm_min = np.inner(np.zeros(np.prod(shape)),
                                 self.rect_mat(shape, 0, n).reshape(-1))
        self.norm_max = np.inner(np.ones(np.prod(shape)),
                                 self.rect_mat(shape, 0, n).reshape(-1))

    def normalize_output(self, output):
        """
        Normalize filter output against range of responses to a centered RF.
        """

        return output/(self.norm_max-self.norm_min)

    @classmethod
    def func_rect(cls, x, y, width):
        return np.logical_and(x > -width/2.0, x <= width/2.0).astype(np.float64)

    @classmethod
    def rect_mat(cls, shape, n_x_offset, n):
        N_y, N_x = shape

        x_max = 1.0
        y_max = N_y/float(N_x)

        X, Y = np.meshgrid(np.linspace(-x_max/2, x_max/2, N_x)-(n_x_offset/float(n)),
                           np.linspace(-y_max/2, y_max/2, N_y))
        return cls.func_rect(X, Y, 1.0/n)

    def create_filters(self):
        N_y, N_x = self.shape

        # Normalized image width and height:
        x_max = 1.0
        y_max = N_y/float(N_x)

        # Construct filters offset by the blocks:
        n_x_offsets = np.linspace(np.ceil(-self.n/2.0), np.floor(self.n/2.0), self.n)
        filters = np.empty((self.n, N_y, N_x), np.float64)

        for j, n_x_offset in enumerate(n_x_offsets):
            filters[j] = self.rect_mat(self.shape, n_x_offset, self.n)
        self.filters = filters
        file = h5py.File('PB_filters.h5','w')
        file.create_dataset('filter', filters.shape, filters.dtype, data=filters)
        file.close()

    def apply_filters(self, frame, normalize=True):
        """
        Compute inner products of computed filters and a video frame.
        """

        result = np.tensordot(self.filters, frame)
        if normalize:
            return self.normalize_output(result)
        else:
            return result


text:
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# graph_tool -- a general graph manipulation python module
#
# Copyright (C) 2006-2018 Tiago de Paula Peixoto <tiago@skewed.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import division, absolute_import, print_function
import sys
if sys.version_info < (3,):
    range = xrange

import numpy
from . util import *
from . mcmc import *
from . bisection import *
from . blockmodel import *
from . overlap_blockmodel import *
from . layered_blockmodel import *
from . nested_blockmodel import *

def default_args(mcmc_args={}, anneal_args={}, mcmc_equilibrate_args={},
                 shrink_args={}, mcmc_multilevel_args={}, overlap=False):
    mcmc_args = dict(dict(beta=numpy.inf, c=0, niter=5, allow_vacate=False,
                          entropy_args=dict(dl=True)), **mcmc_args)
    if overlap:
        mcmc_args = dict(mcmc_args, bundled=True)
    mcmc_equilibrate_args = dict(dict(wait=1, nbreaks=1, epsilon=1e-4,
                                      mcmc_args=mcmc_args),
                                 **dmask(mcmc_equilibrate_args,
                                         ["mcmc_args"]))
    shrink_entropy_args = dict(dict(mcmc_args["entropy_args"]),
                               **shrink_args.get("entropy_args", {}))
    if not shrink_entropy_args.get("dense", False):
         shrink_entropy_args["multigraph"] = False
    shrink_args = dict(dict(entropy_args=shrink_entropy_args, niter=20),
                       **dmask(shrink_args, ["entropy_args"]))
    mcmc_multilevel_args = \
            dict(dict(r=1.3, anneal=False,
                      shrink_args=shrink_args,
                      mcmc_equilibrate_args=mcmc_equilibrate_args,
                      anneal_args=anneal_args),
                 **dmask(mcmc_multilevel_args,
                         ["shrink_args", "mcmc_equilibrate_args",
                          "anneal_args"]))
    return mcmc_multilevel_args

def get_states(g, B_min=None, B_max=None, b_min=None, b_max=None, deg_corr=True,
               overlap=False, nonoverlap_init=True, layers=False, clabel=None,
               state_args={}, mcmc_multilevel_args={}):

    if B_min is None:
        if clabel is None:
            B_min = 1
        else:
            B_min = len(set(clabel.fa))
            if b_min is None:
                b_min = clabel

    _B_max = g.num_vertices()
    if overlap and not nonoverlap_init:
        _B_max = 2 * g.num_edges()

    if B_max is None:
        B_max = _B_max
        if overlap and nonoverlap_init and b_max is None:
            b_max = g.vertex_index.copy("int")

    if layers:
        State = LayeredBlockState
        state_args = dict(state_args, overlap=overlap)
    elif overlap:
        State = OverlapBlockState
    else:
        State = BlockState

    if b_max is not None:
        max_state = State(g, b=b_max, deg_corr=deg_corr, clabel=clabel,
                          **dmask(state_args, ["B", "b", "deg_corr", "clabel"]))
    else:
        max_state = State(g, B=_B_max, deg_corr=deg_corr, clabel=clabel,
                          **dmask(state_args,["B", "b", "deg_corr", "clabel"]))

    if max_state.get_nonempty_B() > B_max:
        max_state = mcmc_multilevel(max_state, B=B_max, **mcmc_multilevel_args)

    if b_min is not None:
        min_state = State(g, B=B_min, b=b_min, deg_corr=deg_corr, clabel=clabel,
                          **dmask(state_args, ["B", "b", "deg_corr", "clabel"]))
    elif B_min == 1:
        min_state = State(g, B=1, deg_corr=deg_corr, clabel=clabel,
                          **dmask(state_args, ["B", "b", "deg_corr", "clabel"]))
    else:
        min_state = max_state.copy()

    if min_state.get_nonempty_B() > B_min:
        min_state = mcmc_multilevel(min_state, B=B_min, **mcmc_multilevel_args)

    return min_state, max_state

def minimize_blockmodel_dl(g, B_min=None, B_max=None, b_min=None, b_max=None,
                           deg_corr=True, overlap=False, nonoverlap_init=True,
                           layers=False, state_args={}, bisection_args={},
                           mcmc_args={}, anneal_args={},
                           mcmc_equilibrate_args={}, shrink_args={},
                           mcmc_multilevel_args={}, verbose=False):
    """Fit the stochastic block model, by minimizing its description length using an
    agglomerative heuristic.

    Parameters
    ----------
    g : :class:`~graph_tool.Graph`
        The graph.
    B_min : ``int`` (optional, default: ``None``)
        The minimum number of blocks.
    B_max : ``int`` (optional, default: ``None``)
        The maximum number of blocks.
    b_min : :class:`~graph_tool.PropertyMap` (optional, default: ``None``)
        The partition to be used with the minimum number of blocks.
    b_max : :class:`~graph_tool.PropertyMap` (optional, default: ``None``)
        The partition to be used with the maximum number of blocks.
    deg_corr : ``bool`` (optional, default: ``True``)
        If ``True``, the degree-corrected version of the model will be used.
    overlap : ``bool`` (optional, default: ``False``)
        If ``True``, the overlapping version of the model will be used.
    nonoverlap_init : ``bool`` (optional, default: ``True``)
        If ``True``, and ``overlap == True`` a non-overlapping initial state
        will be used.
    layers : ``bool`` (optional, default: ``False``)
        If ``True``, the layered version of the model will be used.
    state_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to appropriate state constructor (e.g.
        :class:`~graph_tool.inference.blockmodel.BlockState`,
        :class:`~graph_tool.inference.overlap_blockmodel.OverlapBlockState` or
        :class:`~graph_tool.inference.layered_blockmodel.LayeredBlockState`)
    bisection_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :func:`~graph_tool.inference.bisection.bisection_minimize`.
    mcmc_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :meth:`graph_tool.inference.blockmodel.BlockState.mcmc_sweep`,
        :meth:`graph_tool.inference.overlap_blockmodel.OverlapBlockState.mcmc_sweep` or
        :meth:`graph_tool.inference.layered_blockmodel.LayeredBlockState.mcmc_sweep`.
    mcmc_equilibrate_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :func:`~graph_tool.inference.mcmc.mcmc_equilibrate`.
    shrink_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :meth:`graph_tool.inference.blockmodel.BlockState.shrink`,
        :meth:`graph_tool.inference.overlap_blockmodel.OverlapBlockState.shrink` or
        :meth:`graph_tool.inference.layered_blockmodel.LayeredBlockState.shrink`.
    mcmc_multilevel_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :func:`~graph_tool.inference.mcmc.mcmc_multilevel`.
    verbose : ``bool`` or ``tuple`` (optional, default: ``False``)
        If ``True``, progress information will be shown. Optionally, this
        accepts arguments of the type ``tuple`` of the form ``(level, prefix)``
        where ``level`` is a positive integer that specifies the level of
        detail, and ``prefix`` is a string that is prepended to the all output
        messages.

    Returns
    -------
    min_state : :class:`~graph_tool.inference.blockmodel.BlockState` or  :class:`~graph_tool.inference.overlap_blockmodel.OverlapBlockState` or  :class:`~graph_tool.inference.layered_blockmodel.LayeredBlockState`
        State with minimal description length.

    Notes
    -----

    This function is a convenience wrapper around
    :func:`~graph_tool.inference.bisection.bisection_minimize`.

    See [peixoto-efficient-2014]_ for details on the algorithm.

    This algorithm has a complexity of :math:`O(V \ln^2 V)`, where :math:`V` is
    the number of nodes in the network.

    Examples
    --------

    .. testsetup:: mdl

       gt.seed_rng(43)
       np.random.seed(43)

    .. doctest:: mdl

       >>> g = gt.collection.data["polbooks"]
       >>> state = gt.minimize_blockmodel_dl(g)
       >>> state.draw(pos=g.vp["pos"], vertex_shape=state.get_blocks(),
       ...            output="polbooks_blocks_mdl.pdf")
       <...>

    .. testcleanup:: mdl

       state.draw(pos=g.vp["pos"], vertex_shape=state.get_blocks(),
                  output="polbooks_blocks_mdl.png")

    .. figure:: polbooks_blocks_mdl.*
       :align: center

       Block partition of a political books network, which minimizes the
       description length of the network according to the degree-corrected
       stochastic blockmodel.

    .. testsetup:: mdl_overlap

       gt.seed_rng(42)
       np.random.seed(42)

    .. doctest:: mdl_overlap

       >>> g = gt.collection.data["polbooks"]
       >>> state = gt.minimize_blockmodel_dl(g, overlap=True)
       >>> state.draw(pos=g.vp["pos"], output="polbooks_overlap_blocks_mdl.pdf")
       <...>

    .. testcleanup:: mdl_overlap

       state.draw(pos=g.vp["pos"], output="polbooks_overlap_blocks_mdl.png")

    .. figure:: polbooks_overlap_blocks_mdl.*
       :align: center

       Overlapping partition of a political books network, which minimizes the
       description length of the network according to the overlapping
       degree-corrected stochastic blockmodel.

    References
    ----------
    .. [holland-stochastic-1983] Paul W. Holland, Kathryn Blackmond Laskey,
       Samuel Leinhardt, "Stochastic blockmodels: First steps",
       Carnegie-Mellon University, Pittsburgh, PA 15213, U.S.A.,
       :doi:`10.1016/0378-8733(83)90021-7`.
    .. [faust-blockmodels-1992] Katherine Faust, and Stanley
       Wasserman. "Blockmodels: Interpretation and Evaluation." Social Networks
       14, no. 1-2 (1992): 5-61, :doi:`10.1016/0378-8733(92)90013-W`.
    .. [karrer-stochastic-2011] Brian Karrer, and M. E. J. Newman. "Stochastic
       Blockmodels and Community Structure in Networks." Physical Review E 83,
       no. 1 (2011): 016107, :doi:`10.1103/PhysRevE.83.016107`.
    .. [peixoto-entropy-2012] Tiago P. Peixoto "Entropy of Stochastic Blockmodel
       Ensembles." Physical Review E 85, no. 5 (2012): 056122,
       :doi:`10.1103/PhysRevE.85.056122`, :arxiv:`1112.6028`.
    .. [peixoto-parsimonious-2013] Tiago P. Peixoto, "Parsimonious module
       inference in large networks", Phys. Rev. Lett. 110, 148701 (2013),
       :doi:`10.1103/PhysRevLett.110.148701`, :arxiv:`1212.4794`.
    .. [peixoto-efficient-2014] Tiago P. Peixoto, "Efficient Monte Carlo and greedy
       heuristic for the inference of stochastic block models", Phys. Rev. E 89,
       012804 (2014), :doi:`10.1103/PhysRevE.89.012804`, :arxiv:`1310.4378`.
    .. [peixoto-model-2016] Tiago P. Peixoto, "Model selection and hypothesis
       testing for large-scale network models with overlapping groups",
       Phys. Rev. X 5, 011033 (2016), :doi:`10.1103/PhysRevX.5.011033`,
       :arxiv:`1409.3059`.
    .. [peixoto-inferring-2016] Tiago P. Peixoto, "Inferring the mesoscale
       structure of layered, edge-valued and time-varying networks",
       Phys. Rev. E 92, 042807 (2015), :doi:`10.1103/PhysRevE.92.042807`,
       :arXiv:`1504.02381`.

    """

    b_cache = {} # keep a global cache

    mcmc_multilevel_args = \
        default_args(mcmc_args=mcmc_args,
                     anneal_args=anneal_args,
                     mcmc_equilibrate_args=mcmc_equilibrate_args,
                     shrink_args=shrink_args,
                     mcmc_multilevel_args=dict(mcmc_multilevel_args,
                                               b_cache=b_cache),
                     overlap=overlap)

    bisection_args = dict(dict(mcmc_multilevel_args=mcmc_multilevel_args,
                               random_bisection=False),
                          **bisection_args)

    clabel = state_args.get("clabel", None)
    if clabel is None:
        clabel = state_args.get("pclabel", None)

    min_state, max_state = get_states(g, B_min=B_min, B_max=B_max, b_min=b_min,
                                      b_max=b_max, deg_corr=deg_corr,
                                      overlap=overlap,
                                      nonoverlap_init=nonoverlap_init,
                                      layers=layers, clabel=clabel,
                                      state_args=state_args,
                                      mcmc_multilevel_args=mcmc_multilevel_args)

    if B_min is None:
        B_min = 1
    if B_max is None:
        B_max = numpy.inf

    Bs = list(b_cache.keys())
    for B in Bs:
        if B > B_max or B < B_min:
            del b_cache[B]

    state = bisection_minimize([min_state, max_state], verbose=verbose,
                               **bisection_args)

    return state

def minimize_nested_blockmodel_dl(g, B_min=None, B_max=None, b_min=None,
                                  b_max=None, Bs=None, bs=None, deg_corr=True,
                                  overlap=False, nonoverlap_init=True,
                                  layers=False, hierarchy_minimize_args={},
                                  state_args={}, bisection_args={},
                                  mcmc_args={}, anneal_args={},
                                  mcmc_equilibrate_args={}, shrink_args={},
                                  mcmc_multilevel_args={}, verbose=False):
    """Fit the nested stochastic block model, by minimizing its description length
    using an agglomerative heuristic.

    Parameters
    ----------
    g : :class:`~graph_tool.Graph`
        The graph.
    B_min : ``int`` (optional, default: ``None``)
        The minimum number of blocks.
    B_max : ``int`` (optional, default: ``None``)
        The maximum number of blocks.
    b_min : :class:`~graph_tool.PropertyMap` (optional, default: ``None``)
        The partition to be used with the minimum number of blocks.
    b_max : :class:`~graph_tool.PropertyMap` (optional, default: ``None``)
        The partition to be used with the maximum number of blocks.
    Bs : ``list`` of ints (optional, default: ``None``)
        If provided, it will correspond to the sizes of the initial hierarchy.
    bs : ``list`` of integer-valued :class:`numpy.ndarray` objects (optional, default: ``None``)
        If provided, it will correspond to the initial hierarchical partition.
    deg_corr : ``bool`` (optional, default: ``True``)
        If ``True``, the degree-corrected version of the model will be used.
    overlap : ``bool`` (optional, default: ``False``)
        If ``True``, the overlapping version of the model will be used.
    nonoverlap_init : ``bool`` (optional, default: ``True``)
        If ``True``, and ``overlap == True`` a non-overlapping initial state
        will be used.
    layers : ``bool`` (optional, default: ``False``)
        If ``True``, the layered version of the model will be used.
    hierarchy_minimize_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :func:`~graph_tool.inference.nested_blockmodel.hierarchy_minimize`.
    state_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to appropriate state constructor (e.g.
        :class:`~graph_tool.inference.blockmodel.BlockState`,
        :class:`~graph_tool.inference.overlap_blockmodel.OverlapBlockState` or
        :class:`~graph_tool.inference.layered_blockmodel.LayeredBlockState`)
    bisection_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :func:`~graph_tool.inference.bisection.bisection_minimize`.
    mcmc_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :meth:`graph_tool.inference.blockmodel.BlockState.mcmc_sweep`,
        :meth:`graph_tool.inference.overlap_blockmodel.OverlapBlockState.mcmc_sweep` or
        :meth:`graph_tool.inference.layered_blockmodel.LayeredBlockState.mcmc_sweep`.
    mcmc_equilibrate_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :func:`~graph_tool.inference.mcmc.mcmc_equilibrate`.
    shrink_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :meth:`graph_tool.inference.blockmodel.BlockState.shrink`,
        :meth:`graph_tool.inference.overlap_blockmodel.OverlapBlockState.shrink` or
        :meth:`graph_tool.inference.layered_blockmodel.LayeredBlockState.shrink`.
    mcmc_multilevel_args : ``dict`` (optional, default: ``{}``)
        Arguments to be passed to :func:`~graph_tool.inference.mcmc.mcmc_multilevel`.
    verbose : ``bool`` or ``tuple`` (optional, default: ``False``)
        If ``True``, progress information will be shown. Optionally, this
        accepts arguments of the type ``tuple`` of the form ``(level, prefix)``
        where ``level`` is a positive integer that specifies the level of
        detail, and ``prefix`` is a string that is prepended to the all output
        messages.

    Returns
    -------
    min_state : :class:`~graph_tool.inference.nested_blockmodel.NestedBlockState`
        Nested state with minimal description length.

    Notes
    -----

    This function is a convenience wrapper around
    :func:`~graph_tool.inference.nested_blockmodel.hierarchy_minimize`.

    See [peixoto-hierarchical-2014]_ for details on the algorithm.

    This algorithm has a complexity of :math:`O(V \ln^2 V)`, where :math:`V` is
    the number of nodes in the network.

    Examples
    --------
    .. testsetup:: nested_mdl

       gt.seed_rng(43)
       np.random.seed(43)

    .. doctest:: nested_mdl

       >>> g = gt.collection.data["power"]
       >>> state = gt.minimize_nested_blockmodel_dl(g, deg_corr=True)
       >>> state.draw(output="power_nested_mdl.pdf")
       (...)

    .. testcleanup:: nested_mdl

       state.draw(output="power_nested_mdl.png")

    .. figure:: power_nested_mdl.*
       :align: center

       Hierarchical Block partition of a power-grid network, which minimizes
       the description length of the network according to the nested
       (degree-corrected) stochastic blockmodel.

    .. doctest:: nested_mdl_overlap

       >>> g = gt.collection.data["celegansneural"]
       >>> state = gt.minimize_nested_blockmodel_dl(g, deg_corr=True, overlap=True)
       >>> state.draw(output="celegans_nested_mdl_overlap.pdf")
       (...)

    .. testcleanup:: nested_mdl_overlap

       state.draw(output="celegans_nested_mdl_overlap.png")

    .. figure:: celegans_nested_mdl_overlap.*
       :align: center

       Overlapping block partition of the *C. elegans* neural network, which
       minimizes the description length of the network according to the nested
       overlapping degree-corrected stochastic blockmodel.

    References
    ----------

    .. [peixoto-hierarchical-2014] Tiago P. Peixoto, "Hierarchical block
       structures and high-resolution model selection in large networks ",
       Phys. Rev. X 4, 011047 (2014), :doi:`10.1103/PhysRevX.4.011047`,
       :arxiv:`1310.4377`.
    .. [peixoto-efficient-2014] Tiago P. Peixoto, "Efficient Monte Carlo and
       greedy heuristic for the inference of stochastic block models",
       Phys. Rev. E 89, 012804 (2014), :doi:`10.1103/PhysRevE.89.012804`,
       :arxiv:`1310.4378`.
    .. [peixoto-model-2016] Tiago P. Peixoto, "Model selection and hypothesis
       testing for large-scale network models with overlapping groups",
       Phys. Rev. X 5, 011033 (2016), :doi:`10.1103/PhysRevX.5.011033`,
       :arxiv:`1409.3059`.
    .. [peixoto-inferring-2016] Tiago P. Peixoto, "Inferring the mesoscale
       structure of layered, edge-valued and time-varying networks",
       Phys. Rev. E 92, 042807 (2015), :doi:`10.1103/PhysRevE.92.042807`,
       :arXiv:`1504.02381`.

    """

    mcmc_multilevel_args = \
            default_args(mcmc_args=mcmc_args,
                         anneal_args=anneal_args,
                         mcmc_equilibrate_args=mcmc_equilibrate_args,
                         shrink_args=shrink_args,
                         mcmc_multilevel_args=mcmc_multilevel_args,
                         overlap=overlap)

    if bs is None:
        clabel = state_args.get("clabel", None)
        if clabel is None:
            clabel = state_args.get("pclabel", None)
        min_state, max_state = get_states(g, B_min=B_min, B_max=B_max,
                                          b_min=b_min, b_max=b_max,
                                          deg_corr=deg_corr, overlap=overlap,
                                          nonoverlap_init=nonoverlap_init,
                                          layers=layers, clabel=clabel,
                                          state_args=dmask(state_args,
                                                           ["hstate_args",
                                                            "hentropy_args"]),
                                          mcmc_multilevel_args=mcmc_multilevel_args)
        if b_max is None:
            b_max = max_state.b.fa
        if b_min is None:
            b_min = min_state.b.fa
        if Bs is None:
            bs = [min_state.b.fa, zeros(min_state.b.fa.max() + 1, dtype="int")]
        else:
            bs = []
            bstate = max_state
            for B in Bs:
                bstate = mcmc_multilevel(bstate, B=B,
                                         verbose=verbose_push(verbose,
                                                              ("l = %d " %
                                                               len(bs))),
                                         **mcmc_multilevel_args)
                bs.append(bstate.b.a)
                bstate = bstate.get_block_state()
        State = type(min_state)
    else:
        if layers:
            State = LayeredBlockState
            state_args = dict(state_args, overlap=overlap)
        elif overlap:
            State = OverlapBlockState
        else:
            State = BlockState

    if layers:
        state_args = dict(state_args, overlap=overlap)

    state = NestedBlockState(g, bs=bs,
                             base_type=State,
                             deg_corr=deg_corr,
                             **dmask(state_args, ["deg_corr"]))

    bisection_args = dict(dict(mcmc_multilevel_args=mcmc_multilevel_args,
                               random_bisection=False),
                          **bisection_args)

    hierarchy_minimize(state, B_max=B_max, B_min=B_min, b_max=b_max,
                       b_min=b_min, bisection_args=bisection_args,
                       verbose=verbose,
                       **dmask(hierarchy_minimize_args,
                               ["B_max", "B_min", "bisection_args", "verbose"]))

    return state