text:
### tensorflow==2.3.0

### https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html
### https://google.github.io/mediapipe/solutions/pose

### https://www.tensorflow.org/api_docs/python/tf/keras/Model
### https://www.tensorflow.org/lite/guide/ops_compatibility

### https://www.tensorflow.org/api_docs/python/tf/keras/layers/Conv2D
### https://www.tensorflow.org/api_docs/python/tf/keras/layers/DepthwiseConv2D
### https://www.tensorflow.org/api_docs/python/tf/keras/layers/Add
### https://www.tensorflow.org/api_docs/python/tf/keras/layers/ReLU
### https://www.tensorflow.org/api_docs/python/tf/keras/layers/MaxPool2D
### https://www.tensorflow.org/api_docs/python/tf/keras/layers/Reshape
### https://www.tensorflow.org/api_docs/python/tf/keras/layers/Concatenate
### https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer

### How to initialize a convolution layer with an arbitrary kernel in Keras? https://stackoverrun.com/ja/q/12269118

###  saved_model_cli show --dir saved_model_hand_landmark_new/ --tag_set serve --signature_def serving_default

import tensorflow as tf
import tensorflow_datasets as tfds
from tensorflow.keras import Model, Input
from tensorflow.keras.layers import Conv2D, DepthwiseConv2D, Add, ReLU, MaxPool2D, Reshape, Concatenate, Layer
from tensorflow.keras.initializers import Constant
from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2
import numpy as np
import sys

# tmp = np.load('weights/depthwise_conv2d_Kernel')
# print(tmp.shape)
# print(tmp)

# def init_f(shape, dtype=None):
#        ker = np.load('weights/depthwise_conv2d_Kernel')
#        print(shape)
#        return ker

# sys.exit(0)

inputs = Input(shape=(256, 256, 3), name='input')

# Block_01
conv1_1 = Conv2D(filters=24, kernel_size=[3, 3], strides=[2, 2], padding="same", dilation_rate=[1, 1], activation='relu',
                 kernel_initializer=Constant(np.load('weights_new/conv2d_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_Bias')))(inputs)
depthconv1_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_Bias')))(conv1_1)
conv1_2 = Conv2D(filters=24, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_1_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_1_Bias')))(depthconv1_1)
add1_1 = Add()([conv1_1, conv1_2])
relu1_1 = ReLU()(add1_1)

# Block_02
depthconv2_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_1_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_1_Bias')))(relu1_1)
conv2_1 = Conv2D(filters=24, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_2_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_2_Bias')))(depthconv2_1)
add2_1 = Add()([relu1_1, conv2_1])
relu2_1 = ReLU()(add2_1)

# Block_03
depthconv3_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[2, 2], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_2_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_2_Bias')))(relu2_1)
conv3_1 = Conv2D(filters=48, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_3_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_3_Bias')))(depthconv3_1)
maxpool3_1 = MaxPool2D(pool_size=[2, 2], strides=[2, 2], padding='valid')(relu2_1)
pad3_1 = tf.pad(maxpool3_1, paddings=tf.constant(np.load('weights_new/channel_padding_Paddings')))

add3_1 = Add()([conv3_1, pad3_1])
relu3_1 = ReLU()(add3_1)

# Block_04
depthconv4_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_3_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_3_Bias')))(relu3_1)
conv4_1 = Conv2D(filters=48, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_4_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_4_Bias')))(depthconv4_1)
add4_1 = Add()([conv4_1, relu3_1])
relu4_1 = ReLU()(add4_1)

# Block_05
depthconv5_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_4_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_4_Bias')))(relu4_1)
conv5_1 = Conv2D(filters=48, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_5_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_5_Bias')))(depthconv5_1)
add5_1 = Add()([relu4_1, conv5_1])
relu5_1 = ReLU()(add5_1)

# Block_06
depthconv6_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[2, 2], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_5_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_5_Bias')))(relu5_1)
conv6_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_6_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_6_Bias')))(depthconv6_1)
maxpool6_1 = MaxPool2D(pool_size=[2, 2], strides=[2, 2], padding='valid')(relu5_1)
pad6_1 = tf.pad(maxpool6_1, paddings=tf.constant(np.load('weights_new/channel_padding_1_Paddings')))
add6_1 = Add()([conv6_1, pad6_1])
relu6_1 = ReLU()(add6_1)

# Block_07
depthconv7_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_6_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_6_Bias')))(relu6_1)
conv7_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_7_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_7_Bias')))(depthconv7_1)
add7_1 = Add()([conv7_1, relu6_1])
relu7_1 = ReLU()(add7_1)

# Block_08
depthconv8_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_7_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_7_Bias')))(relu7_1)
conv8_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_8_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_8_Bias')))(depthconv8_1)
add8_1 = Add()([conv8_1, relu7_1])
relu8_1 = ReLU()(add8_1)

# Block_09
depthconv9_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[2, 2], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_8_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_8_Bias')))(relu8_1)
conv9_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_9_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_9_Bias')))(depthconv9_1)
maxpool9_1 = MaxPool2D(pool_size=[2, 2], strides=[2, 2], padding='valid')(relu8_1)
add9_1 = Add()([conv9_1, maxpool9_1])
relu9_1 = ReLU()(add9_1)

# Block_10
depthconv10_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_9_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_9_Bias')))(relu9_1)
conv10_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_10_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_10_Bias')))(depthconv10_1)
add10_1 = Add()([conv10_1, relu9_1])
relu10_1 = ReLU()(add10_1)

# Block_11
depthconv11_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_10_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_10_Bias')))(relu10_1)
conv11_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_11_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_11_Bias')))(depthconv11_1)
add11_1 = Add()([conv11_1, relu10_1])
relu11_1 = ReLU()(add11_1)

# Block_12
depthconv12_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[2, 2], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_11_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_11_Bias')))(relu11_1)
conv12_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_12_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_12_Bias')))(depthconv12_1)
maxpool12_1 = MaxPool2D(pool_size=[2, 2], strides=[2, 2], padding='valid')(relu11_1)
add12_1 = Add()([conv12_1, maxpool12_1])
relu12_1 = ReLU()(add12_1)
resize12_1 = tf.image.resize(relu12_1, np.load('weights_new/up_sampling2d_Size'))
add12_2 = Add()([relu9_1, resize12_1])

# Block_13
depthconv13_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_12_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_12_Bias')))(add12_2)
conv13_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_13_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_13_Bias')))(depthconv13_1)
add13_1 = Add()([add12_2, conv13_1])
relu13_1 = ReLU()(add13_1)
resize13_1 = tf.image.resize(relu13_1, np.load('weights_new/up_sampling2d_1_Size'))
add13_2 = Add()([relu6_1, resize13_1])

# Block_14
depthconv14_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_13_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_13_Bias')))(add13_2)
conv14_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_14_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_14_Bias')))(depthconv14_1)
add14_1 = Add()([add13_2, conv14_1])
relu14_1 = ReLU()(add14_1)
conv14_2 = Conv2D(filters=48, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_15_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_15_Bias')))(relu14_1)
resize14_1 = tf.image.resize(conv14_2, np.load('weights_new/up_sampling2d_2_Size'))
relu14_2 = ReLU()(relu3_1)
add14_1 = Add()([relu14_2, resize14_1])

# Block_15
depthconv15_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_14_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_14_Bias')))(add14_1)
conv15_1 = Conv2D(filters=48, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_16_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_16_Bias')))(depthconv15_1)
add15_1 = Add()([add14_1, conv15_1])
relu15_1 = ReLU()(add15_1)

# Block_16
depthconv16_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_15_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_15_Bias')))(relu15_1)
conv16_1 = Conv2D(filters=48, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_17_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_17_Bias')))(depthconv16_1)
add16_1 = Add()([conv16_1, relu15_1])
relu16_1 = ReLU()(add16_1)

# Block_17
depthconv17_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_16_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_16_Bias')))(relu16_1)
conv17_1 = Conv2D(filters=48, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_18_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_18_Bias')))(depthconv17_1)
add17_1 = Add()([conv17_1, relu16_1])
relu17_1 = ReLU()(add17_1)

# Block_18
depthconv18_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_17_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_17_Bias')))(relu17_1)
conv18_1 = Conv2D(filters=48, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_19_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_19_Bias')))(depthconv18_1)
add18_1 = Add()([conv18_1, relu17_1])
relu18_1 = ReLU()(add18_1)

# Block_19
depthconv19_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[2, 2], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_18_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_18_Bias')))(relu18_1)
conv19_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_20_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_20_Bias')))(depthconv19_1)
maxpool19_1 = MaxPool2D(pool_size=[2, 2], strides=[2, 2], padding='valid')(relu18_1)
pad19_1 = tf.pad(maxpool19_1, paddings=tf.constant(np.load('weights_new/channel_padding_2_Paddings')))
add19_1 = Add()([conv19_1, pad19_1])
relu19_1 = ReLU()(add19_1)

# Block_20
depthconv20_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_19_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_19_Bias')))(relu19_1)
conv20_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_21_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_21_Bias')))(depthconv20_1)
add20_1 = Add()([relu19_1, conv20_1])
relu20_1 = ReLU()(add20_1)

# Block_21
depthconv21_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_20_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_20_Bias')))(relu20_1)
conv21_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_22_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_22_Bias')))(depthconv21_1)
add21_1 = Add()([relu20_1, conv21_1])
relu21_1 = ReLU()(add21_1)

# Block_22
depthconv22_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_21_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_21_Bias')))(relu21_1)
conv22_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_23_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_23_Bias')))(depthconv22_1)
add22_1 = Add()([conv22_1, relu21_1])
relu22_1 = ReLU()(add22_1)

# Block_23
depthconv23_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_22_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_22_Bias')))(relu22_1)
conv23_1 = Conv2D(filters=96, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_24_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_24_Bias')))(depthconv23_1)
add23_1 = Add()([conv23_1, relu22_1])
relu23_1 = ReLU()(add23_1)

# Block_24
depthconv24_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[2, 2], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_23_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_23_Bias')))(relu23_1)
conv24_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_25_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_25_Bias')))(depthconv24_1)
maxpool24_1 = MaxPool2D(pool_size=[2, 2], strides=[2, 2], padding='valid')(relu23_1)
pad24_1 = tf.pad(maxpool24_1, paddings=tf.constant(np.load('weights_new/channel_padding_3_Paddings')))
add24_1 = Add()([conv24_1, pad24_1])
relu24_1 = ReLU()(add24_1)

# Block_25
depthconv25_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_24_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_24_Bias')))(relu24_1)
conv25_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_26_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_26_Bias')))(depthconv25_1)
add25_1 = Add()([relu24_1, conv25_1])
relu25_1 = ReLU()(add25_1)

# Block_26
depthconv26_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_25_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_25_Bias')))(relu25_1)
conv26_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_27_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_27_Bias')))(depthconv26_1)
add26_1 = Add()([relu25_1, conv26_1])
relu26_1 = ReLU()(add26_1)

# Block_27
depthconv27_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_26_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_26_Bias')))(relu26_1)
conv27_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_28_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_28_Bias')))(depthconv27_1)
add27_1 = Add()([conv27_1, relu26_1])
relu27_1 = ReLU()(add27_1)

# Block_28
depthconv28_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_27_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_27_Bias')))(relu27_1)
conv28_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_29_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_29_Bias')))(depthconv28_1)
add28_1 = Add()([conv28_1, relu27_1])
relu28_1 = ReLU()(add28_1)

# Block_29
depthconv29_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[2, 2], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_28_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_28_Bias')))(relu28_1)
conv29_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_30_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_30_Bias')))(depthconv29_1)
maxpool29_1 = MaxPool2D(pool_size=[2, 2], strides=[2, 2], padding='valid')(relu28_1)
add29_1 = Add()([conv29_1, maxpool29_1])
relu29_1 = ReLU()(add29_1)

# Block_30
depthconv30_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_29_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_29_Bias')))(relu29_1)
conv30_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_31_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_31_Bias')))(depthconv30_1)
add30_1 = Add()([conv30_1, relu29_1])
relu30_1 = ReLU()(add30_1)

# Block_31
depthconv31_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_30_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_30_Bias')))(relu30_1)
conv31_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_32_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_32_Bias')))(depthconv31_1)
add31_1 = Add()([conv31_1, relu30_1])
relu31_1 = ReLU()(add31_1)

# Block_32
depthconv32_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_31_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_31_Bias')))(relu31_1)
conv32_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_33_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_33_Bias')))(depthconv32_1)
add32_1 = Add()([conv32_1, relu31_1])
relu32_1 = ReLU()(add32_1)

# Block_33
depthconv33_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_32_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_32_Bias')))(relu32_1)
conv33_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_34_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_34_Bias')))(depthconv33_1)
add33_1 = Add()([conv33_1, relu32_1])
relu33_1 = ReLU()(add33_1)

# Block_34
depthconv34_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[2, 2], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_33_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_33_Bias')))(relu33_1)
conv34_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_35_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_35_Bias')))(depthconv34_1)
maxpool34_1 = MaxPool2D(pool_size=[2, 2], strides=[2, 2], padding='valid')(relu33_1)
add34_1 = Add()([conv34_1, maxpool34_1])
relu34_1 = ReLU()(add34_1)

# Block_35
depthconv35_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_34_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_34_Bias')))(relu34_1)
conv35_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_36_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_36_Bias')))(depthconv35_1)
add35_1 = Add()([conv35_1, relu34_1])
relu35_1 = ReLU()(add35_1)

# Block_36
depthconv36_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_35_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_35_Bias')))(relu35_1)
conv36_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_37_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_37_Bias')))(depthconv36_1)
add36_1 = Add()([conv36_1, relu35_1])
relu36_1 = ReLU()(add36_1)

# Block_37
depthconv37_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_36_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_36_Bias')))(relu36_1)
conv37_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_38_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_38_Bias')))(depthconv37_1)
add37_1 = Add()([conv37_1, relu36_1])
relu37_1 = ReLU()(add37_1)

# Block_38
depthconv38_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_37_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_37_Bias')))(relu37_1)
conv38_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_39_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_39_Bias')))(depthconv38_1)
add38_1 = Add()([conv38_1, relu37_1])
relu38_1 = ReLU()(add38_1)

# Block_39
depthconv39_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[2, 2], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_38_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_38_Bias')))(relu38_1)
conv39_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_40_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_40_Bias')))(depthconv39_1)
maxpool39_1 = MaxPool2D(pool_size=[2, 2], strides=[2, 2], padding='valid')(relu38_1)
add39_1 = Add()([conv39_1, maxpool39_1])
relu39_1 = ReLU()(add39_1)

# Block_40
depthconv40_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_39_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_39_Bias')))(relu39_1)
conv40_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_41_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_41_Bias')))(depthconv40_1)
add40_1 = Add()([conv40_1, relu39_1])
relu40_1 = ReLU()(add40_1)

# Block_41
depthconv41_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_40_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_40_Bias')))(relu40_1)
conv41_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_42_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_42_Bias')))(depthconv41_1)
add41_1 = Add()([conv41_1, relu40_1])
relu41_1 = ReLU()(add41_1)

# Block_42
depthconv42_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_41_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_41_Bias')))(relu41_1)
conv42_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_43_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_43_Bias')))(depthconv42_1)
add42_1 = Add()([conv42_1, relu41_1])
relu42_1 = ReLU()(add42_1)

# Block_43
depthconv43_1 = DepthwiseConv2D(kernel_size=[5, 5], strides=[1, 1], padding="same", depth_multiplier=1, dilation_rate=[1, 1],
                 depthwise_initializer=Constant(np.load('weights_new/depthwise_conv2d_42_Kernel')),
                 bias_initializer=Constant(np.load('weights_new/depthwise_conv2d_42_Bias')))(relu42_1)
conv43_1 = Conv2D(filters=288, kernel_size=[1, 1], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv2d_44_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv2d_44_Bias')))(depthconv43_1)
add43_1 = Add()([conv43_1, relu42_1])
relu43_1 = ReLU()(add43_1)

# Final Block_99
conv99_1 = Conv2D(filters=1, kernel_size=[2, 2], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv_handflag_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv_handflag_Bias')))(relu43_1)
sigm99_1 = tf.math.sigmoid(conv99_1, name='output_handflag')
# reshape99_1 = tf.reshape(sigm99_1, (1, -1), name='output_handflag')

conv99_2 = Conv2D(filters=1, kernel_size=[2, 2], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/conv_handedness_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/conv_handedness_Bias')))(relu43_1)
sigm99_2 = tf.math.sigmoid(conv99_2, name='output_handedness')
# reshape99_2 = tf.reshape(sigm99_2, (1, -1), name='output_handedness')

conv99_3 = Conv2D(filters=63, kernel_size=[2, 2], strides=[1, 1], padding="valid", dilation_rate=[1, 1],
                 kernel_initializer=Constant(np.load('weights_new/convld_21_3d_Kernel').transpose(1,2,3,0)),
                 bias_initializer=Constant(np.load('weights_new/convld_21_3d_Bias')))(relu43_1)
reshape99_3 = tf.reshape(conv99_3, (1, 63), name='ld_21_3d')

model = Model(inputs=inputs, outputs=[sigm99_1, sigm99_2, reshape99_3])
# model = Model(inputs=inputs, outputs=[reshape99_1, reshape99_2, reshape99_3])

model.summary()

tf.saved_model.save(model, 'saved_model_hand_landmark_new')
model.save('hand_landmark_new.h5')

full_model = tf.function(lambda inputs: model(inputs))
full_model = full_model.get_concrete_function(inputs = (tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype)))
frozen_func = convert_variables_to_constants_v2(full_model, lower_control_flow=False)
frozen_func.graph.as_graph_def()
tf.io.write_graph(graph_or_graph_def=frozen_func.graph,
                    logdir=".",
                    name="hand_landmark_new_256x256_float32.pb",
                    as_text=False)

# No Quantization - Input/Output=float32
converter = tf.lite.TFLiteConverter.from_keras_model(model)
tflite_model = converter.convert()
with open('hand_landmark_new_256x256_float32.tflite', 'wb') as w:
    w.write(tflite_model)
print("tflite convert complete! - hand_landmark_new_256x256_float32.tflite")

# Weight Quantization - Input/Output=float32
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE]
tflite_model = converter.convert()
with open('hand_landmark_new_256x256_weight_quant.tflite', 'wb') as w:
    w.write(tflite_model)
print("Weight Quantization complete! - hand_landmark_new_256x256_weight_quant.tflite")

def representative_dataset_gen():
  for data in raw_test_data.take(100):
    image = data['image'].numpy()
    image = tf.image.resize(image, (256, 256))
    image = image[np.newaxis,:,:,:]
    image = image - 127.5
    image = image * 0.007843
    yield [image]

raw_test_data, info = tfds.load(name="the300w_lp", with_info=True, split="train", data_dir="~/TFDS", download=True)

# Integer Quantization - Input/Output=float32
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.representative_dataset = representative_dataset_gen
tflite_quant_model = converter.convert()
with open('hand_landmark_new_256x256_integer_quant.tflite', 'wb') as w:
    w.write(tflite_quant_model)
print("Integer Quantization complete! - hand_landmark_new_256x256_integer_quant.tflite")

# # Full Integer Quantization - Input/Output=int8
# converter = tf.lite.TFLiteConverter.from_keras_model(model)
# converter.optimizations = [tf.lite.Optimize.DEFAULT]
# converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
# converter.inference_input_type = tf.uint8
# converter.inference_output_type = tf.uint8
# converter.representative_dataset = representative_dataset_gen
# tflite_quant_model = converter.convert()
# with open('hand_landmark_new_256x256_full_integer_quant.tflite', 'wb') as w:
#     w.write(tflite_quant_model)
# print("Full Integer Quantization complete! - hand_landmark_new_256x256_full_integer_quant.tflite")

# Float16 Quantization - Input/Output=float32
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.target_spec.supported_types = [tf.float16]
tflite_quant_model = converter.convert()
with open('hand_landmark_new_256x256_float16_quant.tflite', 'wb') as w:
    w.write(tflite_quant_model)
print("Float16 Quantization complete! - hand_landmark_new_256x256_float16_quant.tflite")

# # EdgeTPU
# import subprocess
# result = subprocess.check_output(["edgetpu_compiler", "-s", "hand_landmark_new_256x256_full_integer_quant.tflite"])
# print(result)


text:
# product-key memory
# translated from https://github.com/facebookresearch/XLM/blob/master/PKM-layer.ipynb
# reference https://github.com/zackchase/mxnet-the-straight-dope/blob/master/cheatsheets/pytorch_gluon.md
#           https://gist.github.com/zhanghang1989/3d646f71d60c17048cf8ad582393ac6c

import mxnet as mx
from mxnet import gluon, nd
from mxnet.gluon import Block, loss, nn

import math
import numpy as np

class AttrDict(dict):
    def __init__(self, *args, **kwargs):
        super(AttrDict, self).__init__(*args, **kwargs)
        self.__dict__ = self

def get_uniform_keys(n_keys, dim, seed):
    """
    Generate random uniform keys (same initialization as nn.Linear).
    """
    rng = np.random.RandomState(seed)
    bound = 1 / math.sqrt(dim)
    keys = rng.uniform(-bound, bound, (n_keys, dim))
    return keys.astype(np.float32)

class HashingMemory(Block):
    r'''
    A simple implementation of the product-key memory block
    '''
    def __init__(self, input_dim, output_dim, ctx, params):
        super(HashingMemory, self).__init__()
        # global parameters
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.k_dim = params.k_dim
        self.v_dim = output_dim
        self.n_keys = params.n_keys
        self.size = self.n_keys ** 2
        self.heads = params.heads
        self.knn = params.knn
        self.ctx = ctx
        assert self.k_dim >= 2 and self.k_dim % 2 == 0
        # dropout
        self.input_dropout = params.input_dropout
        self.query_dropout = params.query_dropout
        self.value_dropout = params.value_dropout
        # initialize keys / values
        self.initialize_keys()
        # no embedding-bag available, thus need to sum it up latter
        self.values = nn.Embedding(self.size, self.v_dim)
        # query network
        self.query_proj = nn.HybridSequential()
        with self.name_scope():
            self.query_proj.add(nn.Dense(units=self.heads * self.k_dim,
                                        in_units=self.input_dim,
                                        use_bias=True))
            if params.query_batchnorm:
                self.query_proj.add(nn.BatchNorm(center=True))

        if params.query_batchnorm:
            print("WARNING: Applying batch normalization to queries improves the performance "
                  "and memory usage. But if you use it, be sure that you use batches of "
                  "sentences with the same size at training time (i.e. without padding). "
                  "Otherwise, the padding token will result in incorrect mean/variance "
                  "estimations in the BatchNorm layer.\n")

    def initialize_keys(self):
        """
        Create two subkey sets per head.
        `self.keys` is of shape (heads, 2, n_keys, k_dim // 2)
        """
        half = self.k_dim // 2
        keys = nd.array(np.array([
                get_uniform_keys(self.n_keys, half, seed=(2 * i + j))
                for i in range(self.heads)
                for j in range(2)
            ])).reshape(self.heads, 2, self.n_keys, half)
        self.keys = gluon.Parameter("keys", shape=keys.shape, init=mx.init.Zero())
        self.keys.initialize(ctx=self.ctx)
        self.keys.set_data(keys)

    def _get_indices(self, query, subkeys):
        """
        Generate scores and indices for a specific head.
        """
        assert len(query.shape) == 2 and query.shape[1] == self.k_dim
        bs = query.shape[0]
        knn = self.knn
        half = self.k_dim // 2
        n_keys = len(subkeys[0])
        # split query for product quantization
        q1 = query[:, :half]                                            # (bs,half)
        q2 = query[:, half:]                                            # (bs,half)

        #q1 = nd.ones(q1.shape)
        #q2 = nd.ones(q2.shape)
        # compute indices with associated scores
        scores1 = nd.dot(q1, subkeys[0].T)                              # (bs,n_keys)
        scores2 = nd.dot(q2, subkeys[1].T)                              # (bs,n_keys)
        indices1 = scores1.topk(k=knn, axis=1)                          # (bs,knn)
        indices2 = scores2.topk(k=knn, axis=1)                          # (bs,knn)
        scores1 = nd.array([scores1[i][indices1[i]].asnumpy() for i in range(bs)])
        scores2 = nd.array([scores2[i][indices2[i]].asnumpy() for i in range(bs)])

        # cartesian product on best candidate keys
        all_scores = (
            scores1.reshape(bs, knn, 1).broadcast_to([bs, knn, knn]) +
            scores2.reshape(bs, 1, knn).broadcast_to([bs, knn, knn])
        ).reshape(bs, -1)                                                # (bs,knn**2)
        all_indices = (
            indices1.reshape(bs, knn, 1).broadcast_to([bs, knn, knn]) * n_keys +
            indices2.reshape(bs, 1, knn).broadcast_to([bs, knn, knn])
        ).reshape(bs, -1)                                                # (bs,knn**2)

        # select best scores with associated indices
        best_indices = all_scores.topk(k=knn, axis=1)                    # (bs,knn)
        scores = all_scores[best_indices]
        scores = nd.array([all_scores[i][best_indices[i]].asnumpy() for i in range(bs)])
        indices = nd.array([all_indices[i][best_indices[i]].asnumpy() for i in range(bs)])    # (bs,knn)

        assert scores.shape == indices.shape == (bs, knn)
        return scores, indices

    def get_indices(self, query):
        """
        Generate scores and indices.
        """
        assert len(query.shape) == 2 and query.shape[1] == self.k_dim
        query = query.reshape(-1, self.heads, self.k_dim)
        bs = len(query)
        outputs = [self._get_indices(query[:, i], self.keys.data(self.ctx)[i]) for i in range(self.heads)] # 4
        s = mx.ndarray.concat(*[s.reshape(bs, 1, self.knn) for s, _ in outputs], dim=1)  # (bs,heads,knn)
        i = mx.ndarray.concat(*[i.reshape(bs, 1, self.knn) for _, i in outputs], dim=1)  # (bs,heads,knn)
        return s.reshape(-1, self.knn), i.reshape(-1, self.knn)

    def forward(self, _input):
        """
        Read from the memory.
        """
        # input dimensions
        assert _input.shape[-1] == self.input_dim
        prefix_shape = _input.shape[:-1]
        bs = np.prod(prefix_shape)
        # compute query
        _input = mx.ndarray.Dropout(_input, p=self.input_dropout)       # (...,i_dim)
        query = self.query_proj(_input.reshape(-1, self.input_dim))     # (bs,heads*k_dim)
        query = query.reshape(bs * self.heads, self.k_dim)              # (bs*heads,k_dim)
        query = mx.ndarray.Dropout(query, p=self.query_dropout)         # (bs*heads,k_dim)
        assert query.shape == (bs * self.heads, self.k_dim)
        # retrieve indices and scores
        scores, indices = self.get_indices(query)                       # (bs*heads,knn)
        scores = mx.ndarray.softmax(scores, axis=-1)                    # (bs*heads,knn)
        # merge heads / knn (since we sum heads)
        indices = indices.reshape(bs, self.heads * self.knn)                       # (bs,heads*knn)
        scores = scores.reshape(bs, self.heads * self.knn)                         # (bs,heads*knn)
        # weighted sum of values
        output_raw = self.values(indices)                                          # (bs,knn,v_dim)
        # print(self.values.weight.data())
        output = mx.ndarray.squeeze(mx.ndarray.batch_dot(mx.ndarray.expand_dims(scores, 1), output_raw), axis=1)
        output = mx.ndarray.Dropout(output, p=self.value_dropout)                   # (bs,v_dim)
        # reshape output
        if len(prefix_shape) >= 2:
            output = output.reshape(prefix_shape + (self.v_dim,))                  # (...,v_dim)
        return output

params = AttrDict({
    "sparse": False,
    "k_dim": 128,
    "heads": 4,
    "knn": 32,
    "n_keys": 512,  # the memory will have (n_keys ** 2) values
    "query_batchnorm": True,
    "input_dropout": 0,
    "query_dropout": 0,
    "value_dropout": 0,
})

if __name__ == "__main__":
    ctx = mx.gpu(0) if mx.context.num_gpus() else mx.cpu()
    input_dim = 50
    output_dim = 100
    memory = HashingMemory(input_dim, output_dim, ctx, params)
    # net.collect_params()
    memory.initialize(init=mx.init.Normal(output_dim ** -0.5), ctx=ctx)
    memory.hybridize()
    print(memory)
    # the input
    x = mx.ndarray.random.randn(2, 3, 4, input_dim).as_in_context(ctx)
    output = memory(x)
    print(output.sum().asscalar())
    print(output.shape)

ext: py

max_stars_repo_name: PatriciaXiao/gluon-nlp

max_issues_repo_name: PatriciaXiao/gluon-nlp

max_forks_repo_name: PatriciaXiao/gluon-nlp

reason: import numpy


text:
"""
Speed tests of Medi. To prove that certain things don't take longer than they
should.
"""

import time
import functools

from .helpers import get_example_dir
import medi

def _check_speed(time_per_run, number=4, run_warm=True):
    """ Speed checks should typically be very tolerant. Some machines are
    faster than others, but the tests should still pass. These tests are
    here to assure that certain effects that kill medi performance are not
    reintroduced to Medi."""
    def decorated(func):
        @functools.wraps(func)
        def wrapper(Script, **kwargs):
            if run_warm:
                func(Script=Script, **kwargs)
            first = time.time()
            for i in range(number):
                func(Script=Script, **kwargs)
            single_time = (time.time() - first) / number
            message = 'speed issue %s, %s' % (func, single_time)
            assert single_time < time_per_run, message
        return wrapper
    return decorated

@_check_speed(0.15)
def test_scipy_speed(Script):
    s = 'import scipy.weave; scipy.weave.inline('
    script = Script(s, path='')
    script.get_signatures(1, len(s))

@_check_speed(0.8)
def test_precedence_slowdown(Script):
    """
    Precedence calculation can slow down things significantly in edge
    cases. Having strange recursion structures increases the problem.
    """
    path = get_example_dir('speed', 'precedence.py')
    with open(path) as f:
        line = len(f.read().splitlines())
    assert Script(path=path).infer(line=line)

max_stars_repo_name: yuan-xy/medi

max_issues_repo_name: yuan-xy/medi

max_forks_repo_name: yuan-xy/medi

reason: import scipy


text:
import pickle
from gem.evaluation import metrics
from gem.utils import evaluation_util, graph_util
import networkx as nx
import numpy as np

def evaluateStaticGraphReconstruction(digraph, graph_embedding,
                                      X_stat, node_l=None, file_suffix=None,
                                      sample_ratio_e=None, is_undirected=True,
                                      is_weighted=False):
    node_num = len(digraph.nodes)
    # evaluation
    if sample_ratio_e:
        eval_edge_pairs = evaluation_util.get_random_edge_pairs(
            node_num,
            sample_ratio_e,
            is_undirected
        )
    else:
        eval_edge_pairs = None
    if file_suffix is None:
        estimated_adj = graph_embedding.get_reconstructed_adj(X_stat, node_l)
    else:
        estimated_adj = graph_embedding.get_reconstructed_adj(
            X_stat,
            file_suffix,
            node_l
        )
    predicted_edge_list = evaluation_util.get_edge_list_from_adj_mtrx(
        estimated_adj,
        is_undirected=is_undirected,
        edge_pairs=eval_edge_pairs
    )
    MAP = metrics.computeMAP(predicted_edge_list, digraph, is_undirected=is_undirected)
    prec_curv, _ = metrics.computePrecisionCurve(predicted_edge_list, digraph)
    # If weighted, compute the error in reconstructed weights of observed edges
    if is_weighted:
        digraph_adj = nx.to_numpy_matrix(digraph)
        estimated_adj[digraph_adj == 0] = 0
        err = np.linalg.norm(digraph_adj - estimated_adj)
        err_baseline = np.linalg.norm(digraph_adj)
    else:
        err = None
        err_baseline = None
    return MAP, prec_curv, err, err_baseline

reason: import numpy,import networkx


text:
#!/usr/bin/env python

"""
Copyright 2020, Benjamin L. Dowdell

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at:

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import numpy as np
from bokeh.plotting import figure
from bokeh.models import ColumnDataSource, LinearAxis, Range1d, Span

def plot_tuning_curve(z, amp, z_apparent, z_tuning, z_onset):
    """

    Parameters
    ----------
    z : ndarray
        wedge thickness in milliseconds
    amp : ndarray
        amplitude along top of wedge
    z_apparent : ndarray
        apparent wedge thickness in milliseconds
    z_tuning : float
        measured tuning thickness
    z_onset : float
        measured onset of tuning

    Returns
    -------

    """
    min_amp = np.min(np.abs(amp))
    max_amp = np.max(np.max(amp))
    source = ColumnDataSource(data=dict(x=z, y=amp, z=z_apparent))
    TOOLTIPS = [
        ("Amplitude", "$y{1.111}"),
        ("TWT thickness", "$x{1.1} ms")
    ]
    tools = "crosshair, pan, reset, save, wheel_zoom, box_zoom"
    plot = figure(
        plot_height=300, plot_width=400,
        tooltips=TOOLTIPS, title="Tuning Curve", tools=tools,
        x_axis_label="TWT thickness (ms)", y_axis_label="Abs(Amplitude)",
        x_range=Range1d(-0.01, 100), y_range=Range1d(min_amp, max_amp + max_amp*0.1)
    )
    plot.line('x', 'y', source=source, line_width=3)
    # add wedge true & measured thickness to plot
    plot.extra_y_ranges = {"thickness": Range1d(start=0, end=100)}
    plot.add_layout(LinearAxis(y_range_name="thickness", axis_label="TWT thickness (ms)"), "left")
    plot.line('x', 'x', source=source, line_width=2, line_alpha=0.6, line_color="green", y_range_name="thickness")
    plot.line('x', 'z', source=source, line_width=2, line_alpha=0.6, line_color="red", y_range_name="thickness")
    z_tuning_vline = Span(location=z_tuning, dimension="height", line_color="black", line_width=2)
    plot.add_layout(z_tuning_vline)
    z_onset_vline = Span(location=z_onset, dimension="height", line_color="black", line_dash="dashed", line_width=2)
    plot.add_layout(z_onset_vline)
    plot.toolbar.logo = None

    return plot


text:
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np

def plot_histogram(betas):
    fig, ax = plt.subplots(figsize = (12,8))

    ax.hist(betas[:,0], alpha = 0.3, label = r'$\beta_1$', bins = 100)
    ax.hist(betas[:,1], alpha = 0.3, label = r'$\beta_2$', bins = 100)
    ax.hist(betas[:,2], alpha = 0.3, label = r'$\beta_3$', bins = 100)
    ax.legend(fontsize = 18)

    ax.grid(which='major', alpha=0.8, linestyle='dotted')
    ax.set_xlabel(r'$\beta_k$', fontsize = 18)
    ax.set_ylabel('count', fontsize = 18)
    ax.set_title("Histogram of budget shares", fontsize = 18)
    return fig

def plot_excess_demand(pp1, pp2, dd1, dd2, dd3):
    fig = plt.figure(figsize=(2*12,2*4))
    ax1 = fig.add_subplot(131, projection='3d')
    ax2 = fig.add_subplot(132, projection='3d')
    ax3 = fig.add_subplot(133, projection='3d')

    ax1.plot_surface(pp1.reshape(30,30), pp2.reshape(30,30), dd1.reshape(30,30), cmap = plt.cm.viridis, alpha = .8)
    ax2.plot_surface(pp1.reshape(30,30), pp2.reshape(30,30), dd2.reshape(30,30), cmap = plt.cm.viridis, alpha = .8)
    ax3.plot_surface(pp1.reshape(30,30), pp2.reshape(30,30), dd3.reshape(30,30), cmap = plt.cm.viridis, alpha = .8)

    ax1.set_title("$z_1$", fontsize = 18)
    ax2.set_title("$z_2$", fontsize = 18)
    ax3.set_title("$z_3$", fontsize = 18)

    ax1.set_xlabel('$p_1$', fontsize = 18)
    ax1.set_ylabel('$p_2$', fontsize = 18)
    ax2.set_xlabel('$p_1$', fontsize = 18)
    ax2.set_ylabel('$p_2$', fontsize = 18)
    ax3.set_xlabel('$p_1$', fontsize = 18)
    ax3.set_ylabel('$p_2$', fontsize = 18)

    ax1.set_ylim(10,1)
    ax2.set_ylim(10,1)
    ax3.set_ylim(10,1)

    ax1.view_init(30, 360-30)
    ax2.view_init(30, 360-30)
    ax3.view_init(30, 360-30)
    return fig

def plot_utility_distribution(utilities):
    fig, ax = plt.subplots(figsize = (12,8))
    ax.hist(utilities, alpha = 0.6, bins = 100)

    ax.set_xlabel('$u^{j*}$', fontsize = 18)
    ax.grid(which='major', alpha=0.8, linestyle='dotted')
    ax.set_title("Distribution of $u^j$ in equilibrium", fontsize = 18)

    return fig

def plot_mean_variance(g_space, mean_util, var_util):
    fig, ax = plt.subplots(figsize = (12,8))

    ax.plot(g_space, mean_util, color = 'red', label = 'mean($u$)')
    ax.plot(g_space, var_util, color = 'blue', label = 'var($u$)')

    ax.set_xlabel('$\gamma$', fontsize = 18)
    ax.grid(which='major', alpha=0.8, linestyle='dotted')
    ax.legend()

    return fig


text:
import sys
import time
import click
import qtree
import networkx as nx
import numpy as np
from tqdm import tqdm

import qtree.operators as ops
import qensor.optimisation as qop
from qensor.FeynmanSimulator import FeynmanSimulator

from qensor.optimisation.TensorNet import QtreeTensorNet
from qensor.optimisation.Optimizer import OrderingOptimizer, TamakiOptimizer, WithoutOptimizer
from qensor import QtreeQAOAComposer
from qensor import PerfNumpyBackend
from qensor.ProcessingFrameworks import CMKLExtendedBackend, PerfBackend

@click.group()
def cli():
    pass

@cli.command()
@click.argument('filename', nargs=-1)
@click.option('-p','--num-processes', default=1)
@click.option('-P','--profile', default=False, is_flag=True)
@click.option('-t','--target-tw', default=25)
def sim_file(filename, profile=False, num_processes=1, target_tw=25):
    if not filename:
        stream = sys.stdin
    else:
        stream = open(filename[0],'r')

    n_qubits, circuit = ops.read_circuit_stream(stream)
    kwargs = dict(
        n_processes=num_processes
        ,target_tw=target_tw
        , pool_type='thread'
    )
    if profile:
        class PerfMKLBackend(PerfBackend):
            Backend = CMKLExtendedBackend
        backend = PerfMKLBackend(print=False)
        # backend = PerfNumpyBackend(print=False)
        kwargs['bucket_backend'] = backend
    sim = FeynmanSimulator(**kwargs)
    circuit = sum(circuit, [])
    result = sim.simulate(circuit, batch_vars=4, tw_bias=0)
    print(result)

    if profile:
        print('Profiling results')
        backend.gen_report()

@cli.command()
@click.argument('filename')
@click.option('-t', '--tamaki-time', default=15)
@click.option('-T', '--max-tw', default=32)
@click.option('-s', '--slice-step', default=None, type=int)
@click.option('-C', '--cost-type', default='length')
def opt_file(filename, tamaki_time, max_tw, slice_step, cost_type):
    tn = qop.TensorNet.QtreeTensorNet.from_qsim_file(filename)
    fopt = qop.Optimizer.TamakiTrimSlicing()
    fopt.max_tw = max_tw
    fopt.par_var_step = slice_step
    fopt.cost_type = cost_type
    fopt.tw_bias = 0
    try:
        peo, par_vars, tn = fopt.optimize(tn)
        #print('peo', peo)
    except Exception as e:
        print(e)

    hist = fopt._slice_hist
    sep = '\t'

    print(sep.join(['p_vars','tw']))
    for x in hist:
        print(sep.join(str(n) for n in x))

@cli.command()
@click.argument('filename')
def tw_exact(filename):
    tn = qop.TensorNet.QtreeTensorNet.from_qsim_file(filename)
    graph = tn.get_line_graph()
    peo, tw = qtree.graph_model.peo_calculation.get_peo(graph)
    print(peo)
    print(tw)

@cli.command()
@click.argument('filename')
@click.option('-t','--tamaki_time', default=15)
def tw_heuristic(filename, tamaki_time):
    tn = qop.TensorNet.QtreeTensorNet.from_qsim_file(filename)
    fopt = qop.Optimizer.TamakiOptimizer()
    fopt.wait_time = tamaki_time
    try:
        peo, tn = fopt.optimize(tn)
        data = {'treewidth': fopt.treewidth}
        print(data)
        #print('peo', peo)
    except Exception as e:
        print(e)

@cli.command()
@click.option('-s','--seed', default=42)
@click.option('-d','--degree', default=3)
@click.option('-n','--nodes', default=10)
@click.option('-p','--p', default=1)
@click.option('-G','--graph-type', default='random_regular')
def generate_qaoa_ansatz_circuit(seed, degree, nodes, p, graph_type):
    np.random.seed(seed)
    if graph_type=='random_regular':
        G = nx.random_regular_graph(degree, nodes)
    elif graph_type=='erdos_renyi':
        G = nx.erdos_renyi_graph(nodes, degree/(nodes-1))
    elif graph_type=='erdos_renyi_core':
        G = nx.erdos_renyi_graph(nodes, degree/(nodes-1))
        G = nx.algorithms.core.k_core(G, k=degree)
    else:
        raise Exception('Unsupported graph type')
    gamma, beta = [0.1]*p, [0.2]*p
    composer = QtreeQAOAComposer(G, beta=beta, gamma=gamma)
    composer.ansatz_state()
    txt = qtree.operators.circuit_to_text([composer.circuit], nodes)
    print(txt)

@cli.command()
@click.option('-s','--seed', default=42)
@click.option('-d','--degree', default=3)
@click.option('-n','--nodes', default=10)
@click.option('-p','--p', default=1)
@click.option('-G','--graph-type', default='random_regular')
@click.option('-E','--edge-index', default=0)
def generate_qaoa_energy_circuit(seed, degree, nodes, p, graph_type, edge_index):
    np.random.seed(seed)
    if graph_type=='random_regular':
        G = nx.random_regular_graph(degree, nodes)
    elif graph_type=='erdos_renyi':
        G = nx.erdos_renyi_graph(nodes, degree/(nodes-1))
    elif graph_type=='erdos_renyi_core':
        G = nx.erdos_renyi_graph(nodes, degree/(nodes-1))
        G = nx.algorithms.core.k_core(G, k=degree)
    else:
        raise Exception('Unsupported graph type')
    gamma, beta = [0.1]*p, [0.2]*p
    edge = list(G.edges())[edge_index]
    composer = QtreeQAOAComposer(G, beta=beta, gamma=gamma)
    composer.energy_expectation_lightcone(edge)
    txt = qtree.operators.circuit_to_text([composer.circuit], nodes)
    print(txt)

@cli.command()
@click.option('-s','--seed', default=42)
@click.option('-d','--degree', default=3)
@click.option('-n','--nodes', default=10)
@click.option('-p','--p', default=1)
@click.option('-G','--graph-type', default='random_regular')
@click.option('-T','--max-time', default=0, help='Max time for every evaluation')
@click.option('-O','--ordering-algo', default='greedy', help='Algorithm for elimination order')
def qaoa_energy_tw(nodes, seed, degree, p, graph_type, max_time, ordering_algo):
    np.random.seed(seed)
    if graph_type=='random_regular':
        G = nx.random_regular_graph(degree, nodes)
    elif graph_type=='erdos_renyi':
        G = nx.erdos_renyi_graph(nodes, degree/(nodes-1))
    elif graph_type=='erdos_renyi_core':
        G = nx.erdos_renyi_graph(nodes, degree/(nodes-1))
        print('degrees', list(G.degree))
        G = nx.algorithms.core.k_core(G, k=degree)
        print('nodes', G.number_of_nodes())
    else:
        raise Exception('Unsupported graph type')
    gamma, beta = [0]*p, [0]*p

    def get_tw(circ):

        tn = QtreeTensorNet.from_qtree_gates(circ)

        if ordering_algo=='greedy':
            opt = OrderingOptimizer()
        elif ordering_algo=='tamaki':
            opt = TamakiOptimizer(wait_time=45)
        elif ordering_algo=='without':
            opt = WithoutOptimizer()
        else:
            raise ValueError("Ordering algorithm not supported")
        peo, tn = opt.optimize(tn)
        treewidth = opt.treewidth
        return treewidth

    twidths = []
    if max_time:
        start = time.time()
    else:
        start = np.inf
    for edge in tqdm(G.edges()):
        composer = QtreeQAOAComposer(G, beta=beta, gamma=gamma)
        composer.energy_expectation_lightcone(edge)
        tw = get_tw(composer.circuit)
        twidths.append(tw)
        if time.time() - start > max_time:
            break
    print(f'med={np.median(twidths)} mean={round(np.mean(twidths), 2)} max={np.max(twidths)}')

cli()

max_stars_repo_name: Argonne-QIS/QTensor

max_issues_repo_name: Argonne-QIS/QTensor

max_forks_repo_name: Argonne-QIS/QTensor


text:
import datetime
import numpy as np
from music21 import stream
from rnn_model import *

note2int = NOTE_TO_INT()
int2note = INT_TO_NOTE()

def from_scratch(midi_num=MIDI_NUM):
    """Creates empty encoded songs for generating music from scratch.

    :param midi_num (int): Number of MIDI files

    :return data (list): Empty initial music sequences
    :return filenames (list): Filenames generated based on the date time
    """

    # encoded songs and their file names
    data = []
    filenames = []

    # read current time
    nowTime = datetime.datetime.utcnow().strftime('%Y-%m-%d-%H-%M-%S')

    # create empty data with names
    for midi_index in range(midi_num):

        data.append([])
        filenames.append(nowTime+'-'+str(midi_index+1))

    return data, filenames

def sample(prediction, temperature=TEMPERATURE):
    """Sampling the probability vector output by RNN model.

    :param prediction (ndarray): Probability vector output by RNN model
    :param temperature (float): 0 is equivalent to argmax/max and inf is equivalent to uniform sampling

    :return index (int): Index of the sampled result
    """

    # change the distribution of probability
    prediction = np.log(prediction) / temperature
    probabilites = np.exp(prediction) / np.sum(np.exp(prediction))

    # random sampling
    index = np.random.choice(range(len(probabilites)), p=probabilites)

    return index

def generate_notes(model, data, filenames, max_notes=MAX_BARS*16):
    """Generates notes with RNN model.

    :param model: RNN model
    :param data (list): Encoded songs
    :param filenames (list): File names of the encoded songs
    :param max_notes (int): Maximum number of notes to be generated

    :return:
    """

    # process each song in data
    for index in range(len(data)):

        # append fillers to the song
        song = ['*']*SEGMENT_LENGTH + data[index]

        # map each element in the song to int
        try:

            input_notes = [note2int[note] for note in song[-SEGMENT_LENGTH:]]

        except:

            print("Warning: Unknown notes found in \"%s\"" %filenames[index])
            continue

        midi_path = OUTPUTS_PATH+ '\\' + filenames[index] + '.mid'
        print("Processing \"%s\"" %midi_path)

        output_note = None
        num_notes = 0

        # generate note if no '*' is sampled or the number of notes does not exceed the limit
        while (num_notes<max_notes or max_notes==0) and output_note!='*':

            # one-hot vectorize input and add an axis to it
            one_hot_input = to_categorical(input_notes, num_classes=len(NOTE_TO_INT()))
            one_hot_input = one_hot_input[np.newaxis, ...]

            # predict the next note
            prediction = model.predict(one_hot_input)[0]
            note_index = sample(prediction)
            output_note = int2note[note_index]

            # update the input sequence
            input_notes = input_notes[1:]
            input_notes.append(note_index)

            num_notes += 1
            song.append(output_note)

        convert_midi(song[SEGMENT_LENGTH:], midi_path)

def convert_midi(song, midi_path):
    """Converts the encoded song to a midi file.

    :param song (list): Encoded song
    :param midi_path (str): Path to the midi file

    :return:
    """

    # initialization
    midi_notes = []
    pre_element = None
    duration = 0.0
    offset = 0.0

    # decode the song
    for element in song:

        if element!='-':

            # create new note
            if pre_element!=None:

                if pre_element=='0':

                    new_note = note.Rest()

                else:

                    new_note = note.Note(int(pre_element))

                new_note.quarterLength = duration
                new_note.offset = offset
                midi_notes.append(new_note)

            # update offset and save current note
            offset += duration
            pre_element = element
            duration = 0.25

        else:

            # update duration
            duration += 0.25

    # save as midi
    score_stream = stream.Stream(midi_notes)
    score_stream.write('mid', fp=midi_path)

if __name__ == "__main__":

    # generate music from scratch or based on existing file
    if os.listdir(INPUTS_PATH):

        data, filenames = encode_data(INPUTS_PATH)

    else:

        data, filenames = from_scratch()

    model = build_model(WEIGHTS_PATH)
    generate_notes(model, data, filenames)


text:
#!/usr/bin/env python

import matplotlib.pyplot as plt
import numpy as np
import paths as p
import matplotlib.ticker as ticker

import pylab as plb

import nessy_spec
import pltaux

#wvl_icl, flu_icl = nessy_spec.read(p.it3f + 'TST_ICL', 1005., 9995.)
#wvl_keg, flu_keg = nessy_spec.read(p.it3f + 'TST_KEG', 1005., 9995.)

#wvl_icl = wvl_icl / 10.0
#wvl_keg = wvl_keg / 10.0

#np.savez(p.npz + 'icl_keg_spec', wvl_icl = wvl_icl, flu_icl = flu_icl, \
#                                 wvl_keg = wvl_keg, flu_keg = flu_keg)

#icl_keg_spec = np.load(p.npz + 'icl_keg_spec.npz')

#wvl_icl = icl_keg_spec['wvl_icl']
#flu_icl = icl_keg_spec['flu_icl']
#wvl_keg = icl_keg_spec['wvl_keg']
#flu_keg = icl_keg_spec['flu_keg']

#wvls_icl, flus_icl = nessy_spec.atl_con(wvl_icl * 10.0, flu_icl, 0.6 * 3.5)
#wvls_keg, flus_keg = nessy_spec.atl_con(wvl_keg * 10.0, flu_keg, 0.6 * 3.5)

#wvls_icl = wvls_icl / 10.0
#wvls_keg = wvls_keg / 10.0

#np.savez(p.npz + 'icl_keg_spec_atl_con', wvls_icl = wvls_icl, flus_icl = flus_icl, \
#                                         wvls_keg = wvls_keg, flus_keg = flus_keg)

icl_keg_spec_atl_con = np.load(p.npz + 'icl_keg_spec_atl_con.npz')

wvls_icl = icl_keg_spec_atl_con['wvls_icl']
flus_icl = icl_keg_spec_atl_con['flus_icl']
wvls_keg = icl_keg_spec_atl_con['wvls_keg']
flus_keg = icl_keg_spec_atl_con['flus_keg']

plb.rcParams['xtick.major.pad'] = 15
plb.rcParams['ytick.major.pad'] = 10

#plt.rcParams['font.size'] = 20
#plt.rcParams['font.family'] = 'serif'
#plt.rcParams['font.serif'] = 'Times'
#plt.rcParams['font.monospace'] = 'Ubuntu Mono'

plt.rc('font', size = 20)
plt.rc('font', family = 'serif')
#plt.rc('font', serif = 'Times New Roman')

fig, ax = plt.subplots(nrows = 5, ncols = 1, figsize = (10.0, 14.5))

fig.tight_layout()

x1 = wvls_keg
x2 = wvls_icl

xmin = [125, 160, 240, 320, 400]
xmax = [160, 240, 320, 400, 480]

ymax = [5.00, 0.08, 1.00, 2.00, 2.50]

for i in range(len(ax)):

    if i == 0:

       y1 = 1e+4 * flus_keg
       y2 = 1e+4 * flus_icl

    if i != 0:

       y1 = flus_keg
       y2 = flus_icl

    ax[i].plot(x2, y2, 'r', linewidth = 2.5, label = 'imperial')
    ax[i].plot(x1, y1, 'k', linewidth = 0.5, label = 'pmod/wrc')

    if i == 0: ax[i].set_ylabel(r'$\mathrm{Flux}\times 10^{4}, [\mathrm{W} / \mathrm{m}^2 / \mathrm{nm}]$', \
                                labelpad = 20, \
                                fontsize = 15)

    if i != 0: ax[i].set_ylabel(r'$\mathrm{Flux}, [\mathrm{W} / \mathrm{m}^2 / \mathrm{nm}]$', \
                                labelpad = 20, \
                                fontsize = 20)

    ax[i].set_xlim(xmin[i], xmax[i])
    ax[i].set_ylim(0.0,     ymax[i])

    if i == 0: minloc_x = ticker.AutoMinorLocator(5)
    if i != 0: minloc_x = ticker.AutoMinorLocator(4)

    if i <= 2: minloc_y = ticker.AutoMinorLocator(4)
    if i >  2: minloc_y = ticker.AutoMinorLocator(5)

    ax[i].xaxis.set_minor_locator(minloc_x)
    ax[i].yaxis.set_minor_locator(minloc_y)

    if i != 0: ax[i].xaxis.set_major_locator(ticker.MultipleLocator(20))
    if i == 1: ax[i].yaxis.set_major_locator(ticker.MultipleLocator(0.02))

    ax[i].tick_params(axis='x', length = 10,  which = 'major')
    ax[i].tick_params(axis='x', length = 5,   which = 'minor')
    ax[i].tick_params(axis='y', length = 9,   which = 'major')
    ax[i].tick_params(axis='y', length = 4.5, which = 'minor')

    if i == 4: ax[i].set_xlabel('Wavelength, [nm]', labelpad = 20)

leg = ax[0].legend(framealpha = 0, loc = 'best', handlelength = 0, handletextpad=0, prop={'size': 20})

for handle in leg.legendHandles:
    handle.set_visible(False)

text_icl, text_pmo = leg.get_texts()

plb.setp(text_icl, color = 'r')
plb.setp(text_pmo, color = 'k')

pltaux.savepdf(p.figdir, 'icl_keg_spec_IT3_1')

fig, ax = plt.subplots(nrows = 5, ncols = 1, figsize = (10.0, 14.5))

fig.tight_layout()

xmin = [480, 560, 640, 720, 800]
xmax = [560, 640, 720, 800, 880]

ymax = [2.5, 2.0, 2.0, 1.5, 1.2]

y1 = flus_keg
y2 = flus_icl

for i in range(len(ax)):

    ax[i].plot(x2, y2, 'r', linewidth = 2.5, label = 'imperial')
    ax[i].plot(x1, y1, 'k', linewidth = 0.5, label = 'pmod/wrc')

    ax[i].set_ylabel(r'$\mathrm{Flux}, [\mathrm{W} / \mathrm{m}^2 / \mathrm{nm}]$', \
                     labelpad = 20, \
                     fontsize = 20)

    ax[i].set_xlim(xmin[i], xmax[i])
    ax[i].set_ylim(0.0,     ymax[i])

    minloc_x = ticker.AutoMinorLocator(4)

    if i != 4: minloc_y = ticker.AutoMinorLocator(5)
    if i == 4: minloc_y = ticker.AutoMinorLocator(4)

    ax[i].xaxis.set_minor_locator(minloc_x)
    ax[i].yaxis.set_minor_locator(minloc_y)

    ax[i].xaxis.set_major_locator(ticker.MultipleLocator(20))
    if i == 3: ax[i].yaxis.set_major_locator(ticker.MultipleLocator(0.5))

    ax[i].tick_params(axis='x', length = 10,  which = 'major')
    ax[i].tick_params(axis='x', length = 5,   which = 'minor')
    ax[i].tick_params(axis='y', length = 9,   which = 'major')
    ax[i].tick_params(axis='y', length = 4.5, which = 'minor')

    if i == 4: ax[i].set_xlabel('Wavelength, [nm]', labelpad = 20)

leg = ax[0].legend(framealpha = 0, loc = 'best', handlelength = 0, handletextpad=0, prop={'size': 20})

for handle in leg.legendHandles:
    handle.set_visible(False)

text_icl, text_pmo = leg.get_texts()

plb.setp(text_icl, color = 'r')
plb.setp(text_pmo, color = 'k')

pltaux.savepdf(p.figdir, 'icl_keg_spec_IT3_2')


text:
import numpy as np
import glob, os
import json
import time

training_list = []
test_list = []

with open('ncRNA_s.train.txt') as f:
    lines = f.readlines()  # list containing lines of file
    columns = []  # To store column names
    # missing_lines = []
    i = 1
    for line in lines:
        # line = line.strip()  # remove leading/trailing white spaces
        if line:
            # line_added = False
            # if i == 1:
            d = {}
            columns = [item.strip() for item in line.split(' ')]
            d['value'] = columns[0]
            d['attributes'] = []
            # d['attributes'] = np.zeros(8)
            prev_value = 1
            for indx, column in enumerate(columns[1:]):
                if column:
                    data = [item.strip() for item in column.split(':')]
                    attribute_val = int(data[0])
                    if attribute_val == prev_value:
                        # d['attributes'][int(data[0]) - 1] = float(data[1])
                        d['attributes'].append(float(data[1]))
                    else:
                        # if not line_added:
                            # missing_lines.append(i)
                        # line_added = True
                        while prev_value != attribute_val:
                            d['attributes'].append(0.0)
                            prev_value += 1
                        d['attributes'].append(float(data[1]))
                    prev_value += 1
            # for indx, data in enumerate(columns):
            #     if indx == 1:
            i = i + 1
            training_list.append(d)

with open('ncRNA_s.test.txt') as f:
    lines = f.readlines()  # list containing lines of file
    columns = []  # To store column names
    # missing_lines = []
    i = 1
    for line in lines:
        # line = line.strip()  # remove leading/trailing white spaces
        if line:
            # line_added = False
            # if i == 1:
            d = {}
            columns = [item.strip() for item in line.split(' ')]
            d['value'] = columns[0]
            d['attributes'] = []
            # d['attributes'] = np.zeros(8)
            prev_value = 1
            for indx, column in enumerate(columns[1:]):
                if column:
                    data = [item.strip() for item in column.split(':')]
                    attribute_val = int(data[0])
                    if attribute_val == prev_value:
                        # d['attributes'][int(data[0]) - 1] = float(data[1])
                        d['attributes'].append(float(data[1]))
                    else:
                        # if not line_added:
                            # missing_lines.append(i)
                        # line_added = True
                        while prev_value != attribute_val:
                            d['attributes'].append(0.0)
                            prev_value += 1
                        d['attributes'].append(float(data[1]))
                    prev_value += 1
            # for indx, data in enumerate(columns):
            #     if indx == 1:

            i = i + 1
            test_list.append(d)

### TRAINING SESSION ###

for c in range(-4, 9):
    C = 2 ** c

max_stars_repo_name: mertkoc/Simple-SVM

max_issues_repo_name: mertkoc/Simple-SVM

max_forks_repo_name: mertkoc/Simple-SVM