text:
import sys
import os
import pickle
import copy
import numpy as np
import pandas as pd
from absl import flags

from sklearn import linear_model
from scipy.stats import f as fdist
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
from sklearn.model_selection import KFold

from DogR import *
from represent import *

flags.DEFINE_string('dataset', 'Tract_Data_AFF_reduced_5', 'Name of the dataset located in Data/ folder')
flags.DEFINE_string('y_var', 'meanV', 'Y variable of the dataset')

FLAGS = flags.FLAGS
FLAGS(sys.argv)

def get_AIC_BIC(df, name_of_variables, name_of_variables_including_y, components_range, pickle_dir):
  X = np.array(df[name_of_variables].values)
  Y = np.array(df[Y_var].values)
  XY = np.array(df[name_of_variables_including_y].values)
  D = len(name_of_variables)

  BIC = []
  AIC = []
  for number_of_components in components_range:
      X = np.array(df[name_of_variables_including_y].values)

      dogr = DogR(number_of_components)
      dogr.fit(X)
      dogr.save("{}/fdwr_params_{}.p".format(pickle_dir, number_of_components))
      number_of_params = number_of_components * (D + 1 + D + D * D + 1 + 1)
      BIC.append(np.log(len(df)) * number_of_params - 2 * dogr.ll)
      AIC.append(2 * number_of_params - 2 * dogr.ll)

      print ("# of comp = {0}: AIC={1:.2f}, BIC={2:.2f}".format(number_of_components, AIC[-1], BIC[-1]))

  return AIC, BIC

if __name__ == '__main__':
  file_name =  "Data/" + FLAGS.dataset + ".csv"
  figs_dir = 'Figures/{}'.format(FLAGS.dataset)
  pickle_dir = 'Pickels/{}'.format(FLAGS.dataset)

  Y_var = FLAGS.y_var

  if not os.path.exists(figs_dir):
      os.makedirs(figs_dir)

  if not os.path.exists(pickle_dir):
      os.makedirs(pickle_dir)

  df = pd.read_csv(file_name)

  name_of_variables = set(df.columns.values)
  if Y_var not in name_of_variables:
      print ("Y_var is not in name of variables!")
      exit()

  name_of_variables.remove(Y_var)
  name_of_variables = list(sorted(name_of_variables))
  name_of_variables_including_y = []
  name_of_variables_including_y.extend(name_of_variables)
  name_of_variables_including_y.append(Y_var)
  components_range = range(1, 11)

  print ("Variables: \n", name_of_variables)
  # ***************************************
  # ********         AIC BIC       ********
  # ***************************************

  AIC, BIC = get_AIC_BIC(df, name_of_variables, name_of_variables_including_y, components_range, pickle_dir)
  pickle.dump([AIC, BIC], open("{}/AIC_BIC.p".format(pickle_dir), "wb"))

  # ***********************************
  # ********       Show        ********
  # ***********************************
  AIC_BIC_plot(components_range, AIC, BIC, figs_dir)

max_stars_repo_name: ninoch/DoGR

max_issues_repo_name: ninoch/DoGR

max_forks_repo_name: ninoch/DoGR


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Nov 16 09:50:56 2017

@author: 212612902
"""

import numpy as np

from numpy import sqrt, sin, cos

# =============================================================================
# <Function: calculate the symmetrical components (Fortescue)>
# =============================================================================
def cal_symm(a, b, c):

    # 120 degree rotator
    ALPHA = np.exp(1j * 2/3 * np.pi)

    # positive sequence
    a_pos = 1/3 * ( a + b * ALPHA + c * (ALPHA ** 2) )

    b_pos = 1/3 * ( a * (ALPHA ** 2) + b + c * ALPHA )

    c_pos = 1/3 * ( a * ALPHA + b * (ALPHA ** 2) + c )

    # negative sequence
    a_neg = 1/3 * ( a + b * (ALPHA ** 2) + c * ALPHA )

    b_neg = 1/3 * ( a * ALPHA + b + c * (ALPHA ** 2) )

    c_neg = 1/3 * ( a * (ALPHA ** 2) + b * ALPHA + c )

    # zero sequence
    zero = 1/3 * (a + b + c)

    return a_pos, b_pos, c_pos, a_neg, b_neg, c_neg, zero

# =============================================================================
# </Function: calculate the symmetrical components (Fortescue)>
# =============================================================================

# =============================================================================
# <Function: calculate the amplitude invariant Clarke Transform>
# =============================================================================
def cal_clarke(a, b, c):

    alpha = 2/3 * ( a - 1/2 * (b + c) )

    beta = 2/3 * ( sqrt(3)/2 * (b - c) )

    zero = 2/3 * 1/2 * (a + b + c)

    return alpha, beta, zero
# =============================================================================
# </Function: calculate the amplitude invariant Clarke Transform>
# =============================================================================

# =============================================================================
# <Function: calculate the symmetrical components for the amplitude invariant Clarke Transform>
# =============================================================================
def cal_clarke_symm(a, b, c):

    # based on the DSOGI

    QUAD = np.exp(-1j * np.pi/2)

    # calculate the Clarke components
    alpha, beta, zero = cal_clarke(a, b, c)

    # positive alpha and beta
    alpha_pos = 1/2 * ( alpha - beta * QUAD )

    beta_pos = 1/2 * ( alpha * QUAD + beta )

    # negative alpha and beta
    alpha_neg = 1/2 * ( alpha + beta * QUAD )

    beta_neg = 1/2 ( -1 * alpha * QUAD + beta )

    return alpha_pos, beta_pos, alpha_neg, beta_neg, zero
# =============================================================================
# </Function: calculate the symmetrical components for the amplitude invariant Clarke Transform>
# =============================================================================

# =============================================================================
# <Function: calculate the Park Transform>
# =============================================================================
def cal_park(theta, alpha, beta, zero):

    # Park transform

    length_theta = len(theta)

    length_alpha = len(alpha)

    length_beta = len(beta)

    length_zero = len(zero)

    if all( x == length_theta for x in (length_theta, length_alpha, length_beta, length_zero) ):

        pass

    else:

        raise ValueError('Element length mismatch.'
                         + 'The length of theta, alpha, beta and zero must be all the same')

    d = cos(theta) * alpha + sin(theta) * beta

    q = -sin(theta) * alpha + cos(theta) * beta

    zero = zero

    return d, q, zero
# =============================================================================
# </Function: calculate the Park Transform>
# =============================================================================

# =============================================================================
# def draw_clarke(dbl_har_order, dbl_base_freq=50, dbl_cycle=4,
#                 mag_a=1, mag_b=1, mag_c=1,
#                 bool_savefig=False, int_dpi=600, str_fig_path=''):
#
#     # import modules
#     import matplotlib as mpl
#     import matplotlib.pyplot as plt
#     import numpy as np
#     import os
#
#     from numpy import pi, cos
#     from gsyDqLib import date_time_now
#
#     # matplotlib setup
#     mpl.rcParams['font.family'] = 'serif'
#     mpl.rcParams['text.usetex'] = True
#     mpl.rcParams['text.latex.preview'] = True
#     mpl.rcParams['text.latex.preamble'] = [r'\boldmath']
#     mpl.rcParams['font.weight'] = 'bold'
#
#
#     # =============================================================================
#     # <Process input arguments>
#     # =============================================================================
#     # process input harmonic order
#     try:
#
#         dbl_har_order = float(dbl_har_order)
#
#         if dbl_har_order != 0:
#
#             dbl_har_order = abs(dbl_har_order)
#
#         else:
#
#             dbl_har_order = 1
#
#             print(date_time_now() + ' Invalid harmoinc order. Harmonic order set to 1')
#
#     except:
#
#         dbl_har_order = 1
#
#         print(date_time_now() + ' Invalid harmoinc order. Harmonic order set to 1')
#
#         pass
#
#     # process base frequency
#     try:
#
#         dbl_base_freq = float(dbl_base_freq)
#
#         if dbl_base_freq != 0:
#
#             dbl_base_freq = abs(dbl_base_freq)
#
#         else:
#
#             dbl_base_freq = 50.0
#
#             print(date_time_now() + ' Invalid base frequency. Base frequency set to 50')
#
#     except:
#
#         dbl_base_freq = 50.0
#
#         print(date_time_now() + ' Invalid base frequency. Base frequency set to 50')
#
#         pass
#
#     # process how many cycles to display
#     try:
#
#         dbl_cycle = float(dbl_cycle)
#
#         if dbl_cycle != 0:
#
#             dbl_cycle = abs(dbl_cycle)
#
#         else:
#
#             dbl_cycle = 4
#
#             print(date_time_now() + ' Invalid display cycles. Set display cycles to 4')
#
#     except:
#
#         dbl_cycle = 4
#
#         print(date_time_now() + ' Invalid display cycles. Set display cycles to 4')
#
#         pass
#
#     # process whether to save the figure
#     try:
#
#         bool_savefig = bool(bool_savefig)
#
#     except:
#
#         bool_savefig = False
#
#         pass
#
#     # process dpi
#     try:
#
#         int_dpi = int(int_dpi)
#
#         if int_dpi != 0:
#
#             int_dpi = int_dpi
#
#         else:
#
#             int_dpi = 600
#
#             print(date_time_now() + ' Invalid dpi. Set dpi to 600')
#
#     except:
#
#         int_dpi = 600
#
#         print(date_time_now() + ' Invalid dpi. Set dpi to 600')
#
#         pass
#
#     # process figure path
#     try:
#
#         str_fig_path = str(str_fig_path)
#
#         if bool_savefig == True:
#
#             if len(str_fig_path) == 0:
#
#                 str_fig_path = 'Figure_' + str( int(np.random.rand() * 1e6) ) + '.png'
#
#                 str_fig_path = os.path.join(os.getcwd(), str_fig_path)
#
#             else:
#
#                 pass
#
#         else:
#
#             pass
#
#     except:
#
#         str_fig_path = 'Figure_' + str( int(np.random.rand() * 1e6) ) + '.png'
#
#         str_fig_path = os.path.join(os.getcwd(), str_fig_path)
#
#         print(date_time_now() + ' Invalid figure path. Set figure path to "' + str_fig_path + '"')
#
#         pass
#     # =============================================================================
#     # </Process input arguments>
#     # =============================================================================
#
#
#     # =============================================================================
#     # <Make data, titles, legends>
#     # =============================================================================
#     int_remainder = np.mod(dbl_har_order, 3)
#
#     dbl_base_period = 1 / dbl_base_freq
#
#     time_end = dbl_base_period / dbl_har_order * dbl_cycle
#
#     # time vector
#     time = np.linspace( 0, time_end, (10 ** 5) )
#
#     # angular freq
#     omega = 2 * np.pi * dbl_base_freq
#
#     # base phases of the 3-phase inputs, note, base phases only
#     phase_a = omega * time
#     phase_b = omega * time - (2 / 3 * pi)
#     phase_c = omega * time + (2 / 3 * pi)
#
#     # 3-phase inputs
#     input_a = mag_a * cos(dbl_har_order * phase_a)
#     input_b = mag_b * cos(dbl_har_order * phase_b)
#     input_c = mag_c * cos(dbl_har_order * phase_c)
#
#     # amplitude invariant Clarke transform
#     alpha, beta, zero = cal_clarke(input_a, input_b, input_c)
#
#     # legend labels for the 3-phase inputs
#     str_a_lbl = r'$a = cos(' + str(dbl_har_order) + r'\cdot \omega t)$'
#     str_b_lbl = r'$b = cos[' + str(dbl_har_order) + r'\cdot (\omega t - \frac{2}{3}\pi)]$'
#     str_c_lbl = r'$c = cos[' + str(dbl_har_order) + r'\cdot (\omega t + \frac{2}{3}\pi)]$'
#
#     # legend labels for the Clarke transform
#     str_alpha_lbl = r'$\alpha = \frac{2}{3} (a - \frac{1}{2} b - \frac{1}{2} c)$'
#
#     str_beta_lbl = r'$\beta = \frac{2}{3} ( 0 + \frac{\sqrt{3}}{2} b - \frac{\sqrt{3}}{2} c)$'
#
#     str_zero_lbl = r'$Zero = \frac{2}{3} (\frac{1}{2} a + \frac{1}{2} b + \frac{1}{2} c)$'
#
#     # condition, coordinate 1 title
#     if all( x == mag_a for x in (mag_a, mag_b, mag_c) ):
#
#         if int_remainder == 1:
#
#             str_ax1_title = ( r'$\textbf{Three-Phase Inputs, } \omega =2 \pi \times'
#                              + str(dbl_har_order) + r'\times' + str(dbl_base_freq)
#                              + r'\textbf{ (positive sequence)}$' )
#
#         elif int_remainder == 2:
#
#             str_ax1_title = ( r'$\textbf{Three-Phase Inputs, } \omega =2 \pi \times'
#                              + str(dbl_har_order) + r'\times' + str(dbl_base_freq)
#                              + r'\textbf{ (negative sequence)}$' )
#
#         elif int_remainder == 0:
#
#             str_ax1_title = ( r'$\textbf{Three-Phase Inputs, } \omega =2 \pi \times'
#                              + str(dbl_har_order) + r'\times' + str(dbl_base_freq)
#                              + r'\textbf{ (zero sequence)}$' )
#
#         else:
#
#             str_ax1_title = ( r'$\textbf{Three-Phase Inputs, } \omega =2 \pi \times'
#                              + str(dbl_har_order) + r'\times' + str(dbl_base_freq)
#                              + r'$' )
#
#     else:
#
#         str_ax1_title = ( r'$\textbf{Three-Phase Inputs, } \omega =2 \pi \times'
#                              + str(dbl_har_order) + r'\times' + str(dbl_base_freq)
#                              + r'$' )
#
#     # coordinate 2 title
#     str_ax2_title = r'$\textbf{Outputs of the General Amplitude Invariant Clarke Transform}$'
#     # =============================================================================
#     # </Make data, titles, legends>
#     # =============================================================================
#
#
#     # =============================================================================
#     # <Main figure setup>
#     # =============================================================================
#     # make main figure
#     fig = plt.figure(figsize=(900.0/100.0, 600.0/100.0), dpi = 100.0)
#
#     # adjust spacing between subplots
#     fig.subplots_adjust(hspace=0.5)
#
#     # <coordnate 1> =============================================================================
#     # make coordinate 1
#     ax1 = plt.subplot(2, 1, 1)
#
#     # set coordinate 1 title
#     ax1.set_title(str_ax1_title, fontsize=14, fontweight='bold', y=1.2)
#
#     # set coordinate 1 horizontal line
#     ax1.axhline(y=0, color='k', lw=3)
#
#     # plot 3-phase inputs
#     ax1_input_a, = plt.plot(time, input_a, color='r', lw=2, label=str_a_lbl)
#     ax1_input_b, = plt.plot(time, input_b, color='g', lw=2, label=str_b_lbl)
#     ax1_input_c, = plt.plot(time, input_c, color='b', lw=2, label=str_c_lbl)
#
#     # get automatic y limits
#     y_min, y_max = ax1.get_ylim()
#
#     # set limits and grid lines
#     plt.xlim([0, time_end])
#     plt.ylim([y_min, y_max])
#     plt.grid(True)
#
#     # range arguments for ploting period helping lines
#     rng_start = dbl_base_period / dbl_har_order
#     rng_stop = time_end + rng_start
#     rng_step = rng_start
#
#     # plot period helping lines
#     for item in np.arange(rng_start, rng_stop, rng_step):
#
#         plt.plot([item, item], [y_min, y_max],
#                  linestyle='--', linewidth=2, color='k')
#
#     # set legend
#     ax1_lgd = plt.legend(handles=[ax1_input_a, ax1_input_b, ax1_input_c],
#                          loc='upper center', fontsize=11, bbox_to_anchor=(0.5, 1.25),
#                          shadow=False, fancybox=True, ncol=3)
#
#     # set legend transparence
#     ax1_lgd.get_frame().set_alpha(1.0)
#
#     # set y label
#     ax1.set_ylabel(r'\textbf{Amplitude}', fontweight='bold', fontsize=12)
#     # </coordnate 1> =============================================================================
#
#     # <coordnate 2> =============================================================================
#     # make coordinate 2
#     ax2 = plt.subplot(2, 1, 2)
#
#     # set coordinate 2 title
#     ax2.set_title(str_ax2_title, fontsize=14, fontweight='bold', y=1.23)
#
#     # plot coordinate 2 horizontal line
#     ax2.axhline(y=0, color='k', lw=3)
#
#     # plot Clarke transform components
#     ax2_alpha, = plt.plot(time, alpha, color='r', lw=2, label=str_alpha_lbl)
#     ax2_beta, = plt.plot(time, beta, color='g', lw=2, label=str_beta_lbl)
#     ax2_zero, = plt.plot(time, zero, color='b', lw=2, label=str_zero_lbl)
#
#     # get automatic y limits
# #    y_min, y_max = ax2.get_ylim()
#
#     # set coordinate 2 limits
#     plt.xlim([0, time_end])
#     plt.ylim([y_min, y_max])
#     plt.grid(True)
#
#     # plot period helping lines
#     for item in np.arange(rng_start, rng_stop, rng_step):
#
#         plt.plot([item, item], [y_min, y_max], linestyle='--', linewidth=2, color='k')
#
#     # set coordinate 2 legend
#     ax2_lgd = plt.legend(handles=[ax2_alpha, ax2_beta, ax2_zero], loc='upper center',
#                          fontsize=11, bbox_to_anchor=(0.5, 1.27),
#                          shadow=False, fancybox=True, ncol=3)
#
#     # set legend transparence
#     ax2_lgd.get_frame().set_alpha(1.0)
#
#     # set labels
#     ax2.set_xlabel(r'\textbf{Time (s)}', fontweight='bold', fontsize=12)
#     ax2.set_ylabel(r'\textbf{Amplitude}', fontweight='bold', fontsize=12)
#     # </coordnate 2> =============================================================================
#
#     plt.show()
#     # =============================================================================
#     # </Main figure setup>
#     # =============================================================================
#
#
#     # =============================================================================
#     # <Condition, save figure>
#     # =============================================================================
#     if bool_savefig == True:
#
#         plt.tight_layout()
#
#         fig.savefig(str_fig_path, dpi=int_dpi)
#
#         plt.tight_layout()
#
#         print( date_time_now() + 'Figure saved as:"' + str_fig_path + '"' )
#
#         plt.close()
#
#
#     else:
#
#         pass
#     # =============================================================================
#     # </Condition, save figure>
#     # =============================================================================
#
# draw_clarke(1, mag_a=0.7, mag_b=1.2, mag_c=0.6)
# =============================================================================

# =============================================================================
#
# import numpy as np
# import os
#
# for item in (1.7, 2.4, 5.6):
#
#     path = r'J:\_Clarke & Park\Figures\Clarke_unbalanced'
#
#     path = os.path.join(path, str(item) + '.png')
#
#     draw_clarke(item, bool_savefig=True, str_fig_path=path)
# =============================================================================

ext: py

reason: import numpy,from numpy


text:
import collections

import numpy as np
import tensorflow as tf

from model.seq_crf import *
from model.att_crf import *
from model.seq_softmax import *
from util.data_util import *

__all__ = ["TrainModel", "EvalModel", "OnlineModel",
           "create_train_model", "create_eval_model", "create_online_model",
           "init_model", "load_model"]

class TrainModel(collections.namedtuple("TrainModel",
    ("graph", "model", "data_pipeline", "word_embedding", "input_data", "input_text", "input_label"))):
    pass

class EvalModel(collections.namedtuple("EvalModel",
    ("graph", "model", "data_pipeline", "word_embedding", "input_data", "input_text", "input_label"))):
    pass

class OnlineModel(collections.namedtuple("OnlineModel", ("model", "data_pipeline"))):
    pass

def create_train_model(logger,
                       hyperparams):
    graph = tf.Graph()
    with graph.as_default():
        logger.log_print("# prepare train data")
        (input_data, input_text_data, input_label_data, input_ext_data,
            word_embed_data, word_vocab_size, word_vocab_index, word_vocab_inverted_index,
            char_vocab_size, char_vocab_index, char_vocab_inverted_index,
            label_vocab_size, label_vocab_index, label_vocab_inverted_index) = prepare_sequence_data(logger,
            hyperparams.data_train_sequence_file, hyperparams.data_train_sequence_file_type, hyperparams.data_word_vocab_file,
            hyperparams.data_word_vocab_size, hyperparams.data_word_vocab_threshold, hyperparams.model_word_embed_dim,
            hyperparams.data_embedding_file, hyperparams.data_full_embedding_file, hyperparams.data_word_unk, hyperparams.data_word_pad,
            hyperparams.model_word_feat_enable, hyperparams.model_word_embed_pretrained, hyperparams.data_char_vocab_file,
            hyperparams.data_char_vocab_size, hyperparams.data_char_vocab_threshold, hyperparams.data_char_unk, hyperparams.data_char_pad,
            hyperparams.model_char_feat_enable, hyperparams.data_label_vocab_file, hyperparams.data_label_vocab_size,
            hyperparams.data_label_unk, hyperparams.data_label_pad)

        external_data = {}
        if hyperparams.data_pipeline_mode == "dynamic":
            logger.log_print("# create train text dataset")
            text_placeholder = tf.placeholder(shape=[None], dtype=tf.string)
            text_dataset = tf.data.Dataset.from_tensor_slices(text_placeholder)
            input_text_word_dataset, input_text_char_dataset = create_text_dataset(text_dataset,
                word_vocab_index, hyperparams.data_text_word_size, hyperparams.data_word_pad,
                hyperparams.model_word_feat_enable, char_vocab_index, hyperparams.data_text_char_size,
                hyperparams.data_char_pad, hyperparams.model_char_feat_enable, hyperparams.data_num_parallel)

            logger.log_print("# create train label dataset")
            label_placeholder = tf.placeholder(shape=[None], dtype=tf.string)
            label_dataset = tf.data.Dataset.from_tensor_slices(label_placeholder)
            input_label_dataset = create_label_dataset(label_dataset, label_vocab_index,
                hyperparams.data_label_size, hyperparams.data_label_pad, hyperparams.data_num_parallel)

            logger.log_print("# create train extended dataset")
            ext_placeholder = tf.placeholder(shape=[None, None, None], dtype=tf.float32)
            ext_dataset = tf.data.Dataset.from_tensor_slices(ext_placeholder)
            default_ext_pad = tf.constant(0.0, shape=[], dtype=tf.float32)
            input_ext_dataset = create_ext_dataset(ext_dataset,
                hyperparams.data_ext_max_size, hyperparams.model_ext_embed_dim, default_ext_pad,
                hyperparams.model_ext_feat_enable, hyperparams.data_num_parallel)

            logger.log_print("# create train data pipeline")
            data_size_placeholder = tf.placeholder(shape=[], dtype=tf.int64)
            batch_size_placeholder = tf.placeholder(shape=[], dtype=tf.int64)
            data_pipeline = create_dynamic_pipeline(input_text_word_dataset,
                input_text_char_dataset, input_label_dataset, input_ext_dataset,
                word_vocab_size, word_vocab_index, hyperparams.data_word_pad, hyperparams.model_word_feat_enable,
                char_vocab_size, char_vocab_index, hyperparams.data_char_pad, hyperparams.model_char_feat_enable,
                label_vocab_index, label_vocab_inverted_index, hyperparams.data_label_pad, hyperparams.model_ext_feat_enable,
                hyperparams.train_random_seed, hyperparams.train_enable_shuffle, hyperparams.train_shuffle_buffer_size,
                text_placeholder, label_placeholder, ext_placeholder, data_size_placeholder, batch_size_placeholder)
        else:
            if word_embed_data is not None:
                external_data["word_embedding"] = word_embed_data

            logger.log_print("# create train text dataset")
            text_dataset = tf.data.Dataset.from_tensor_slices(input_text_data)
            input_text_word_dataset, input_text_char_dataset = create_text_dataset(text_dataset,
                word_vocab_index, hyperparams.data_text_word_size, hyperparams.data_word_pad,
                hyperparams.model_word_feat_enable, char_vocab_index, hyperparams.data_text_char_size,
                hyperparams.data_char_pad, hyperparams.model_char_feat_enable, hyperparams.data_num_parallel)

            logger.log_print("# create train label dataset")
            label_dataset = tf.data.Dataset.from_tensor_slices(input_label_data)
            input_label_dataset = create_label_dataset(label_dataset, label_vocab_index,
                hyperparams.data_label_size, hyperparams.data_label_pad, hyperparams.data_num_parallel)

            logger.log_print("# create train extended dataset")
            ext_dataset = tf.data.Dataset.from_tensor_slices(input_ext_data)
            default_ext_pad = tf.constant(0.0, shape=[], dtype=tf.float32)
            input_ext_dataset = create_ext_dataset(ext_dataset,
                hyperparams.data_ext_max_size, hyperparams.model_ext_embed_dim, default_ext_pad,
                hyperparams.model_ext_feat_enable, hyperparams.data_num_parallel)

            logger.log_print("# create train data pipeline")
            data_pipeline = create_data_pipeline(input_text_word_dataset,
                input_text_char_dataset, input_label_dataset, input_ext_dataset,
                word_vocab_size, word_vocab_index, hyperparams.data_word_pad, hyperparams.model_word_feat_enable,
                char_vocab_size, char_vocab_index, hyperparams.data_char_pad, hyperparams.model_char_feat_enable,
                label_vocab_index, label_vocab_inverted_index, hyperparams.data_label_pad, hyperparams.model_ext_feat_enable,
                hyperparams.train_random_seed, hyperparams.train_enable_shuffle, hyperparams.train_shuffle_buffer_size,
                len(input_data), hyperparams.train_batch_size)

        model_creator = get_model_creator(hyperparams.model_type)
        model = model_creator(logger=logger, hyperparams=hyperparams, data_pipeline=data_pipeline,
            external_data=external_data, mode="train", scope=hyperparams.model_scope)

        return TrainModel(graph=graph, model=model, data_pipeline=data_pipeline,
            word_embedding=word_embed_data, input_data=input_data, input_text=input_text_data, input_label=input_label_data)

def create_eval_model(logger,
                      hyperparams):
    graph = tf.Graph()
    with graph.as_default():
        logger.log_print("# prepare eval data")
        (input_data, input_text_data, input_label_data, input_ext_data,
            word_embed_data, word_vocab_size, word_vocab_index, word_vocab_inverted_index,
            char_vocab_size, char_vocab_index, char_vocab_inverted_index,
            label_vocab_size, label_vocab_index, label_vocab_inverted_index) = prepare_sequence_data(logger,
            hyperparams.data_eval_sequence_file, hyperparams.data_eval_sequence_file_type, hyperparams.data_word_vocab_file,
            hyperparams.data_word_vocab_size, hyperparams.data_word_vocab_threshold, hyperparams.model_word_embed_dim,
            hyperparams.data_embedding_file, hyperparams.data_full_embedding_file, hyperparams.data_word_unk, hyperparams.data_word_pad,
            hyperparams.model_word_feat_enable, hyperparams.model_word_embed_pretrained, hyperparams.data_char_vocab_file,
            hyperparams.data_char_vocab_size, hyperparams.data_char_vocab_threshold, hyperparams.data_char_unk, hyperparams.data_char_pad,
            hyperparams.model_char_feat_enable, hyperparams.data_label_vocab_file, hyperparams.data_label_vocab_size,
            hyperparams.data_label_unk, hyperparams.data_label_pad)

        external_data = {}
        if hyperparams.data_pipeline_mode == "dynamic":
            logger.log_print("# create eval text dataset")
            text_placeholder = tf.placeholder(shape=[None], dtype=tf.string)
            text_dataset = tf.data.Dataset.from_tensor_slices(text_placeholder)
            input_text_word_dataset, input_text_char_dataset = create_text_dataset(text_dataset,
                word_vocab_index, hyperparams.data_text_word_size, hyperparams.data_word_pad,
                hyperparams.model_word_feat_enable, char_vocab_index, hyperparams.data_text_char_size,
                hyperparams.data_char_pad, hyperparams.model_char_feat_enable, hyperparams.data_num_parallel)

            logger.log_print("# create eval label dataset")
            label_placeholder = tf.placeholder(shape=[None], dtype=tf.string)
            label_dataset = tf.data.Dataset.from_tensor_slices(label_placeholder)
            input_label_dataset = create_label_dataset(label_dataset, label_vocab_index,
                hyperparams.data_label_size, hyperparams.data_label_pad, hyperparams.data_num_parallel)

            logger.log_print("# create eval extended dataset")
            ext_placeholder = tf.placeholder(shape=[None, None, None], dtype=tf.float32)
            ext_dataset = tf.data.Dataset.from_tensor_slices(ext_placeholder)
            default_ext_pad = tf.constant(0.0, shape=[], dtype=tf.float32)
            input_ext_dataset = create_ext_dataset(ext_dataset,
                hyperparams.data_ext_max_size, hyperparams.model_ext_embed_dim, default_ext_pad,
                hyperparams.model_ext_feat_enable, hyperparams.data_num_parallel)

            logger.log_print("# create eval data pipeline")
            data_size_placeholder = tf.placeholder(shape=[], dtype=tf.int64)
            batch_size_placeholder = tf.placeholder(shape=[], dtype=tf.int64)
            data_pipeline = create_dynamic_pipeline(input_text_word_dataset,
                input_text_char_dataset, input_label_dataset, input_ext_dataset,
                word_vocab_size, word_vocab_index, hyperparams.data_word_pad, hyperparams.model_word_feat_enable,
                char_vocab_size, char_vocab_index, hyperparams.data_char_pad, hyperparams.model_char_feat_enable,
                label_vocab_index, label_vocab_inverted_index, hyperparams.data_label_pad, hyperparams.model_ext_feat_enable,
                None, False, 0, text_placeholder, label_placeholder, ext_placeholder, data_size_placeholder, batch_size_placeholder)
        else:
            if word_embed_data is not None:
                external_data["word_embedding"] = word_embed_data

            logger.log_print("# create eval text dataset")
            text_dataset = tf.data.Dataset.from_tensor_slices(input_text_data)
            input_text_word_dataset, input_text_char_dataset = create_text_dataset(text_dataset,
                word_vocab_index, hyperparams.data_text_word_size, hyperparams.data_word_pad,
                hyperparams.model_word_feat_enable, char_vocab_index, hyperparams.data_text_char_size,
                hyperparams.data_char_pad, hyperparams.model_char_feat_enable, hyperparams.data_num_parallel)

            logger.log_print("# create eval label dataset")
            label_dataset = tf.data.Dataset.from_tensor_slices(input_label_data)
            input_label_dataset = create_label_dataset(label_dataset, label_vocab_index,
                hyperparams.data_label_size, hyperparams.data_label_pad, hyperparams.data_num_parallel)

            logger.log_print("# create eval extended dataset")
            ext_dataset = tf.data.Dataset.from_tensor_slices(input_ext_data)
            default_ext_pad = tf.constant(0.0, shape=[], dtype=tf.float32)
            input_ext_dataset = create_ext_dataset(ext_dataset,
                hyperparams.data_ext_max_size, hyperparams.model_ext_embed_dim, default_ext_pad,
                hyperparams.model_ext_feat_enable, hyperparams.data_num_parallel)

            logger.log_print("# create eval data pipeline")
            data_pipeline = create_data_pipeline(input_text_word_dataset,
                input_text_char_dataset, input_label_dataset, input_ext_dataset,
                word_vocab_size, word_vocab_index, hyperparams.data_word_pad, hyperparams.model_word_feat_enable,
                char_vocab_size, char_vocab_index, hyperparams.data_char_pad, hyperparams.model_char_feat_enable,
                label_vocab_index, label_vocab_inverted_index, hyperparams.data_label_pad, hyperparams.model_ext_feat_enable,
                None, False, 0, len(input_data), hyperparams.train_eval_batch_size)

        model_creator = get_model_creator(hyperparams.model_type)
        model = model_creator(logger=logger, hyperparams=hyperparams, data_pipeline=data_pipeline,
            external_data=external_data, mode="eval", scope=hyperparams.model_scope)

        return EvalModel(graph=graph, model=model, data_pipeline=data_pipeline,
            word_embedding=word_embed_data, input_data=input_data, input_text=input_text_data, input_label=input_label_data)

def create_online_model(logger,
                        hyperparams):
    logger.log_print("# prepare online data")
    (word_embed_data, word_vocab_size, word_vocab_index,
        word_vocab_inverted_index, char_vocab_size, char_vocab_index,
        char_vocab_inverted_index) = prepare_text_data(logger, None, hyperparams.data_word_vocab_file,
        hyperparams.data_word_vocab_size, hyperparams.data_word_vocab_threshold, hyperparams.model_word_embed_dim,
        hyperparams.data_embedding_file, hyperparams.data_full_embedding_file, hyperparams.data_word_unk,
        hyperparams.data_word_pad, hyperparams.model_word_feat_enable, hyperparams.model_word_embed_pretrained,
        hyperparams.data_char_vocab_file, hyperparams.data_char_vocab_size, hyperparams.data_char_vocab_threshold,
        hyperparams.data_char_unk, hyperparams.data_char_pad, hyperparams.model_char_feat_enable)

    label_vocab_size, label_vocab_index, label_vocab_inverted_index = prepare_label_data(logger, None,
        hyperparams.data_label_vocab_file, hyperparams.data_label_vocab_size, hyperparams.data_label_unk, hyperparams.data_label_pad)

    external_data={}
    if word_embed_data is not None:
        external_data["word_embedding"] = word_embed_data

    logger.log_print("# create online data pipeline")
    data_pipeline = create_online_pipeline(hyperparams.data_external_index_enable,
        word_vocab_size, word_vocab_index, hyperparams.data_text_word_size, hyperparams.data_word_pad,
        hyperparams.model_word_feat_enable, char_vocab_size, char_vocab_index, hyperparams.data_text_char_size,
        hyperparams.data_char_pad, hyperparams.model_char_feat_enable, label_vocab_inverted_index,
        hyperparams.data_ext_max_size, hyperparams.model_ext_embed_dim, hyperparams.model_ext_feat_enable)

    model_creator = get_model_creator(hyperparams.model_type)
    model = model_creator(logger=logger, hyperparams=hyperparams, data_pipeline=data_pipeline,
        external_data=external_data, mode="online", scope=hyperparams.model_scope)

    return OnlineModel(model=model, data_pipeline=data_pipeline)

def get_model_creator(model_type):
    if model_type == "seq_softmax":
        model_creator = SequenceSoftmax
    elif model_type == "seq_crf":
        model_creator = SequenceCRF
    elif model_type == "att_crf":
        model_creator = AttentionCRF
    else:
        raise ValueError("can not create model with unsupported model type {0}".format(model_type))

    return model_creator

def init_model(sess,
               model):
    with model.graph.as_default():
        sess.run(tf.global_variables_initializer())
        sess.run(tf.tables_initializer())

def load_model(sess,
               model,
               ckpt_file,
               ckpt_type):
    with model.graph.as_default():
        model.model.restore(sess, ckpt_file, ckpt_type)

reason: import numpy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 11 18:38:15 2019

@author: juangabriel
"""

# SVR

# Cómo importar las librerías
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importar el data set
dataset = pd.read_csv('Position_Salaries.csv')
X = dataset.iloc[:, 1:2].values
y = dataset.iloc[:, 2].values

# Dividir el data set en conjunto de entrenamiento y conjunto de testing
"""
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
"""

# Escalado de variables utilizamos reshepe -1 y 1 por sugerencia py
from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
sc_y = StandardScaler()
X = sc_X.fit_transform(X)
y = sc_y.fit_transform(y.reshape(-1,1))

# Ajustar la regresión con el dataset
#kernel puede (es por defecto "rbf"), "linear", "poly", "sigmoid", "precomputed"
from sklearn.svm import SVR
regression = SVR(kernel = "rbf")
regression.fit(X, y)

# Predicción de nuestros modelos con SVR
y_pred = sc_y.inverse_transform(regression.predict(sc_X.transform(np.array([[6.5]]))))

# Visualización de los resultados del SVR
X_grid = np.arange(min(X), max(X), 0.1)
X_grid = X_grid.reshape(len(X_grid), 1)
plt.scatter(X, y, color = "red")
plt.plot(X_grid, regression.predict(X_grid), color = "blue")
plt.title("Modelo de Regresión (SVR)")
plt.xlabel("Posición del empleado")
plt.ylabel("Sueldo (en $)")
plt.show()

# Visualización de los resultados del Modelo luego de hacer la inversa de la transformacion
#Regresar los datos a sus valores originales

X_orig = sc_X.inverse_transform(X)
y_orig = sc_y.inverse_transform(y)

#Graficar
plt.scatter(X_orig,y_orig,color="red")
plt.plot(X_orig,sc_y.inverse_transform(regression.predict(X)),color="blue")
plt.title("Modelo de regresión SVR")
plt.xlabel("Posición del empleado")
plt.ylabel("Sueldo")
plt.show()

#otra opcion para desescalar
# Visualización de los resultados del Modelo

X_grid = np.arange(min(X), max(X), 0.1)
X_grid = X_grid.reshape(len(X_grid), 1)
vector = sc_y.inverse_transform(regression.predict(X_grid))
X_grid = sc_X.inverse_transform(X_grid)
X = sc_X.inverse_transform(X)
y = sc_y.inverse_transform(y)
plt.scatter(X, y, color = "red")
plt.plot(X_grid, vector, color = "blue")
plt.title("Modelo de Regresión")
plt.xlabel("Posición del empleado")
plt.ylabel("Sueldo (en $)")
plt.show()

max_stars_repo_path: datasets/Part 2 - Regression/Section 7 - Support Vector Regression (SVR)/svr.py

max_stars_repo_name: pepealza/Curso-ML-A-a-Z

max_issues_repo_name: pepealza/Curso-ML-A-a-Z

max_forks_repo_name: pepealza/Curso-ML-A-a-Z


text:
import errno
import os
import sys
from collections import defaultdict

import numpy as np
from torch.utils.data import DataLoader, Dataset
from torchvision.datasets.cifar import CIFAR10, CIFAR100
from torchvision.datasets.folder import pil_loader, make_dataset, IMG_EXTENSIONS
from torchvision.transforms import transforms

class Caltech101(Dataset):
    link = 'http://www.vision.caltech.edu/Image_Datasets/Caltech101/101_ObjectCategories.tar.gz'
    filename = '101_ObjectCategories.tar.gz'
    foldername = '101_ObjectCategories'

    def _find_classes(self, dir):
        """
        Finds the class folders in a dataset.

        Args:
            dir (string): Root directory path.

        Returns:
            tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary.

        Ensures:
            No class is a subdirectory of another.
        """
        if sys.version_info >= (3, 5):
            # Faster and available in Python 3.5 and above
            classes = [d.name for d in os.scandir(dir) if d.is_dir()]
        else:
            classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))]
        classes.sort()
        classes.remove('BACKGROUND_Google')

        class_to_idx = {classes[i]: i for i in range(len(classes))}
        return classes, class_to_idx

    def __init__(self, root, train=True, transform=None, download=True):
        self.root = root
        root = os.path.join(root, self.foldername)

        if download:
            self.download()

        classes, class_to_idx = self._find_classes(root)
        samples = make_dataset(root, class_to_idx, IMG_EXTENSIONS)

        datapaths = defaultdict(list)

        for path, target in samples:
            datapaths[target].append(path)

        for target in datapaths.keys():
            if train:
                datapaths[target] = datapaths[target][:int(0.8 * len(datapaths[target]))]
            else:
                datapaths[target] = datapaths[target][int(0.8 * len(datapaths[target])):]

        newdatapaths = []
        labels = []
        for target in datapaths.keys():
            for path in datapaths[target]:
                newdatapaths.append(path)
                labels.append(target)

        self.train = train
        self.transform = transform
        self.labels = labels
        self.datapaths = newdatapaths
        self.cache = {}

    def __getitem__(self, index):
        target = self.labels[index]
        if index in self.cache:
            img = self.cache[index]
        else:
            path = self.datapaths[index]
            img = pil_loader(path)
            self.cache[index] = img

        if self.transform is not None:
            img = self.transform(img)

        return img, target

    def download(self):
        import tarfile

        if os.path.exists(os.path.join(self.root, self.filename)):
            print('Files already downloaded and verified')
            return

        self.download_url(self.link)

        # extract file
        cwd = os.getcwd()
        mode = 'r:gz' if self.filename.endswith('.gz') else 'r'
        tar = tarfile.open(os.path.join(self.root, self.filename), mode)
        os.chdir(self.root)
        tar.extractall()
        tar.close()
        os.chdir(cwd)

    def download_url(self, link):
        from six.moves import urllib

        root = os.path.expanduser(self.root)
        fpath = os.path.join(root, self.filename)

        try:
            os.makedirs(root)
        except OSError as e:
            if e.errno == errno.EEXIST:
                pass
            else:
                raise

        # downloads file
        try:
            print('Downloading ' + link + ' to ' + fpath)
            urllib.request.urlretrieve(link, fpath)
        except:
            if link[:5] == 'https':
                url = link.replace('https:', 'http:')
                print('Failed download. Trying https -> http instead.'
                      ' Downloading ' + url + ' to ' + fpath)
                urllib.request.urlretrieve(url, fpath)

    def __len__(self):
        return len(self.datapaths)

class Caltech256(Caltech101):
    link = 'http://www.vision.caltech.edu/Image_Datasets/Caltech256/256_ObjectCategories.tar'
    filename = '256_ObjectCategories.tar'
    foldername = '256_ObjectCategories'

class WMDataset(Dataset):
    def __init__(self, root, labelpath, transform):
        self.root = root

        self.datapaths = [os.path.join(self.root, fn) for fn in os.listdir(self.root)]
        self.labelpath = labelpath
        self.labels = np.loadtxt(self.labelpath)
        self.transform = transform
        self.cache = {}

    def __getitem__(self, index):
        target = self.labels[index]
        if index in self.cache:
            img = self.cache[index]
        else:
            path = self.datapaths[index]
            img = pil_loader(path)
            img = self.transform(img)  # transform is fixed CenterCrop + ToTensor
            self.cache[index] = img

        return img, int(target)

    def __len__(self):
        return len(self.datapaths)

def prepare_wm(datapath='data/trigger_set/pics',shuffle=True):
    triggerroot = datapath
    labelpath = 'data/trigger_set/labels-cifar.txt'

    mean, std = (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)

    transform_list = [
        transforms.CenterCrop(32),
        transforms.ToTensor()
    ]

    transform_list.append(transforms.Normalize(mean, std))

    wm_transform = transforms.Compose(transform_list)

    dataset = WMDataset(triggerroot, labelpath, wm_transform)

    loader = DataLoader(dataset,
                        batch_size=2,
                        shuffle=shuffle,
                        drop_last=True)

    return loader

def prepare_dataset(args):
    is_tl = args['transfer_learning']
    tl_ds = args['tl_dataset']
    ds = args['dataset'] if not is_tl else tl_ds

    ##### shortcut ######
    is_cifar = 'cifar' in ds
    root = f'data/{ds}'
    print('Loading dataset: ' + ds)

    DATASET = {
        'cifar10': CIFAR10,
        'cifar100': CIFAR100,
        'caltech-101': Caltech101,
        'caltech-256': Caltech256
    }[ds]

    mean, std = (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)

    ##### train transform #####
    if not is_cifar:
        transform_list = [
            transforms.Resize(32),
            transforms.CenterCrop(32)
        ]
    else:
        transform_list = []

    transform_list.extend([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize(mean, std)
    ])

    train_transforms = transforms.Compose(transform_list)

    ##### test transform #####
    if not is_cifar:
        transform_list = [
            transforms.Resize(32),
            transforms.CenterCrop(32)
        ]
    else:
        transform_list = []

    transform_list.extend([
        transforms.ToTensor(),
        transforms.Normalize(mean, std)
    ])

    test_transforms = transforms.Compose(transform_list)

    ##### dataset and loader #####
    train_dataset = DATASET(root,
                            train=True,
                            transform=train_transforms,
                            download=True)
    test_dataset = DATASET(root,
                           train=False,
                           transform=test_transforms)

    train_loader = DataLoader(train_dataset,
                              batch_size=args['batch_size'],
                              shuffle=True,
                              num_workers=4,
                              drop_last=True)
    test_loader = DataLoader(test_dataset,
                             batch_size=args['batch_size'] * 2,
                             shuffle=False,
                             num_workers=4)

    return train_loader, test_loader

max_stars_repo_name: ZJZAC/Passport-aware-Normalization

max_issues_repo_name: ZJZAC/Passport-aware-Normalization

max_forks_repo_name: ZJZAC/Passport-aware-Normalization


text:
import tensorflow as tf

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
tf.enable_eager_execution(config=config)

import numpy as np
import cv2
from dataset.Datasets import Datasets, Datasets_512
from model.PaintsTensorFlow import *

class PaintsTensorflowTest:

    def __init__(self):
        def model_init(model, size):
            zero_0 = np.zeros(shape=[1, size, size, 1], dtype=np.float32)
            zero_1 = np.zeros(shape=[1, size, size, 3], dtype=np.float32)
            model(zero_0, zero_1, False)

        pred_model_path = "./ckpt/PaintsTensorFlow/PredPaintsTensorFlow.h5"
        model_path = "./ckpt/PaintsTensorFlow/PaintsTensorFlow.h5"

        self.__pre_generator = Generator(resize=True, name="predPaintsTensorFlow")
        self.__generator = Generator(name="PaintsTensorFlow", convertUint8=True)

        model_init(self.__pre_generator, 128)
        model_init(self.__generator, 512)

        self.__pre_generator.load_weights(pred_model_path)
        self.__generator.load_weights(model_path)

    def test(self):
        dataSets = Datasets_512(batch_size=hp.batch_size)
        train_sets, test_sets = dataSets.buildDataSets()

        for line_128, hint_128, image, line, hint in test_sets.take(2):
            hint = hint.numpy()
            hint = np.ones_like(hint_128)
            hint += 1
            pred = self.__pre_generator(line_128, hint, False)
            outputs = self.__generator(line, pred, training=False)

            for img in outputs.numpy():
                cv2.imshow("", cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
                cv2.waitKey(0)

        self.__pre_generator.summary()
        self.__generator.summary()

if __name__ == '__main__':
    test = PaintsTensorflowTest()
    test.test()

max_stars_repo_name: seekerzz/PaintsTensorFlow

max_issues_repo_name: seekerzz/PaintsTensorFlow

max_forks_repo_name: seekerzz/PaintsTensorFlow


text:
from pathlib import Path, PurePosixPath
from sys import argv
import os, sys, time
from math import radians, cos
import numpy as np
import yaml
from collections import Counter
from .utils import space_group_lib

platform = sys.platform

spglib = space_group_lib()

threshold = 2.0

def parse_xds_ascii(fn):
    d = {"xds_ascii": fn.absolute()}
    with open(fn, "r") as f:
        for line in f:
            if not line.startswith("!"):
                break
            if "UNIT_CELL_CONSTANTS" in line:
                inp = line.split()
                cell = [float(val) for val in inp[-6:]]
            if "SPACE_GROUP_NUMBER" in line:
                inp = line.strip().split("=")
                spgr = int(inp[-1])

        d["space_group"] = spgr
        d["unit_cell"] = cell

    return d

def get_xds_ascii_names(lst):
    ret = []
    for d in lst:
        if "xds_ascii" in d:
            ret.append(d["xds_ascii"])
        else:
            ret.append(Path(d["directory"]) / "XDS_ASCII.HKL")
    return ret

def write_xscale_inp(fns, unit_cell, space_group, resolution):
    cwd = Path(".").resolve()

    cell_str = " ".join((f"{val:.3f}" for val in unit_cell))

    print("\nUsing:")
    print(f"  SPACE_GROUP_NUMBER= {space_group}")
    print(f"  UNIT_CELL_CONSTANTS= {cell_str}")
    print()

    with open("XSCALE.INP", "w") as f:

        print("SNRC= 2", file=f)
        print("SAVE_CORRECTION_IMAGES= FALSE", file=f)  # prevent local directory being littered with .cbf files
        print(f"SPACE_GROUP_NUMBER= {space_group}", file=f)
        print(f"UNIT_CELL_CONSTANTS= {cell_str}", file=f)

        print(file=f)
        print("OUTPUT_FILE= MERGED.HKL", file=f)
        print(file=f)

        for i, fn in enumerate(fns):
            fn = fn.absolute()
            try:
                fn = fn.relative_to(cwd).as_posix()
            except ValueError:
                if platform == "win32":
                    drive = fn.drive
                    drive_letter = drive.lower()[0]
                    fn = fn.as_posix().replace(f"{drive}", f"/mnt/{drive_letter}")

            print(f"    INPUT_FILE= {fn}", file=f)
            print(f"    INCLUDE_RESOLUTION_RANGE= {resolution[0]} {resolution[1]}", file=f)
            print(file=f)

    print(f"Wrote file {f.name}")

def write_xdsconv_inp(resolution):
    with open("XDSCONV.INP", "w") as f:
        print(f"""
INPUT_FILE= MERGED.HKL
INCLUDE_RESOLUTION_RANGE= {resolution[0]} {resolution[1]} ! optional
OUTPUT_FILE= shelx.hkl  SHELX    ! Warning: do _not_ name this file "temp.mtz" !
FRIEDEL'S_LAW= FALSE             ! default is FRIEDEL'S_LAW=TRUE""", file=f)

    print(f"Wrote file {f.name}")

def main():
    import argparse

    description = "Program to make an input file for XSCALE."
    parser = argparse.ArgumentParser(description=description,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("args",
                        type=str, nargs="*", metavar="FILE",
                        help="Path to a cells.yaml / XDS_ASCII.HKL files")

    parser.add_argument("-s","--spgr",
                        action="store", type=int, dest="spgr",
                        help="Space group number (default: most common one)")

    parser.add_argument("-c","--cell",
                        action="store", type=float, nargs=6, dest="cell",
                        help="Override the unit cell parameters (default: mean unit cell)")

    parser.add_argument("-r","--resolution",
                        action="store", type=float, nargs=2, dest="resolution",
                        help="Override the resolution (default: mean unit cell)")

    parser.set_defaults(cell=None,
                        spgr=None,
                        resolution=(20, 0.8))

    options = parser.parse_args()
    spgr = options.spgr
    cell = options.cell
    resolution = options.resolution
    args = options.args

    if not args:  # attempt to populate args
        if os.path.exists("cells.yaml"):
            args = ["cells.yaml"]
        else:
            args = list(Path(".").glob("*XDS_ASCII.HKL"))

    if not args:
        exit()
    else:
        lst = []
        for arg in args:
            fn = Path(arg)
            extension = fn.suffix.lower()
            if extension == ".yaml":
                d = yaml.load(open(fn, "r"), Loader=yaml.Loader)
                lst.extend(d)
            if extension == ".hkl":
                lst.append(parse_xds_ascii(fn))

    print(f"Loaded {len(lst)} cells")

    fns = get_xds_ascii_names(lst)

    cells = np.array([d["unit_cell"] for d in lst])

    if not cell:
        cell = np.mean(cells, axis=0)

    if not spgr:
        c = Counter([spglib[d["space_group"]]["laue_symmetry"] for d in lst])
        for key, count in c.most_common(10):
            d = spglib[key]
            lattice = d["lattice"]
            laue_symm = d["laue_symmetry"]
            print(f"Lattice type `{lattice}` was found {count:3} times (space group number: {laue_symm})")
        spgr =  spglib[c.most_common()[0][0]]["laue_symmetry"]

    else:
        d = spglib[spgr]
        lattice = d["lattice"]
        laue_symm = d["laue_symmetry"]
        print(f"Lowest possible symmetry for {spgr} ({lattice}): {laue_symm}")

    write_xscale_inp(fns, unit_cell=cell, space_group=spgr, resolution=resolution)
    write_xdsconv_inp(resolution=resolution)

if __name__ == '__main__':
    main()

max_stars_repo_name: stefsmeets/scedtools

max_issues_repo_name: stefsmeets/scedtools

max_forks_repo_name: stefsmeets/scedtools


text:
"""
leveesim is an environment for modeling flood risk under deep uncertainty
"""

import numpy as np
from gym.envs.registration import register

from .env import LeveeEnv

register(
    id="leveesim-v0", entry_point="leveesim.env:LeveeEnv", kwargs=dict(),
)

max_stars_repo_name: jdossgollin/leveesim

max_issues_repo_name: jdossgollin/leveesim

max_forks_repo_name: jdossgollin/leveesim


text:
from enum import Enum

import hal
import numpy as np
import wpilib
from networktables import NetworkTables
from wpilib.geometry import Pose2d, Rotation2d
from wpilib.kinematics import (ChassisSpeeds, DifferentialDriveKinematics,
                               DifferentialDriveOdometry,
                               DifferentialDriveWheelSpeeds)

from utils import lazypigeonimu, lazytalonfx, units
from controls import motorstate

class WheelState:
    def __init__(self, left=0, right=0):
        self.left = left
        self.right = right

    def norm(self, value):
        max_output = np.max(np.abs([self.left, self.right]))
        if max_output == 0:
            return
        scale = value / max_output
        self.left *= scale
        self.right *= scale

    def __str__(self):
        return f"({self.left}, {self.right})"

class Chassis:

    # chassis physical constants
    WHEEL_DIAMETER = 6 * units.inches
    WHEEL_RADIUS = WHEEL_DIAMETER / 2
    WHEEL_CIRCUMFERENCE = 2 * np.pi * WHEEL_RADIUS
    GEAR_RATIO = (48 / 14) * (50 / 16)  # 10.7142861

    BUMPER_WIDTH = 3.25 * units.inches
    ROBOT_WIDTH = 30 * units.inches + BUMPER_WIDTH
    ROBOT_LENGTH = 30 * units.inches + BUMPER_WIDTH
    ROBOT_MASS = 100 * units.pounds

    TRACK_WIDTH = 24 * units.inches
    TRACK_RADIUS = TRACK_WIDTH / 2

    # conversions
    RADIANS_PER_METER = (2 * np.pi * GEAR_RATIO) / WHEEL_CIRCUMFERENCE
    METERS_PER_RADIAN = WHEEL_CIRCUMFERENCE / (2 * np.pi * GEAR_RATIO)

    # motor config
    LEFT_INVERTED = True
    RIGHT_INVERTED = False

    # motor coefs
    KS = 0.149 * units.volts
    KV = 2.4 * (units.volts / units.seconds)
    KA = 0.234 * (units.volts / units.seconds / units.seconds)

    # velocity pidf gains
    VL_KP = 0.000363
    VL_KI = 0
    VL_KD = 0
    VL_KF = 0

    VR_KP = 0.000363
    VR_KI = 0
    VR_KD = 0
    VR_KF = 0

    # joystick
    MAX_JOYSTICK_OUTPUT = 1

    # required devices
    dm_l: lazytalonfx.LazyTalonFX
    dm_r: lazytalonfx.LazyTalonFX
    ds_l: lazytalonfx.LazyTalonFX
    ds_r: lazytalonfx.LazyTalonFX

    imu: lazypigeonimu.LazyPigeonIMU

    # constraints
    MAX_VELOCITY = 3 * (units.meters / units.seconds)

    class _Mode(Enum):
        Idle = 0
        PercentOutput = 1
        Velocity = 2

    def __init__(self):
        self.mode = self._Mode.Idle

        self.odometry = DifferentialDriveOdometry(Rotation2d.fromDegrees(0))

        self.desired_output = WheelState()
        self.desired_velocity = WheelState()

        self.feedforward = WheelState()

        self.wheel_left = motorstate.MotorState()
        self.wheel_right = motorstate.MotorState()

        self.nt = NetworkTables.getTable(f"/components/chassis")

    def setup(self):
        self.dm_l.setInverted(self.LEFT_INVERTED)
        self.dm_r.setInverted(self.RIGHT_INVERTED)

        self.dm_l.setRadiansPerUnit(self.RADIANS_PER_METER)
        self.dm_r.setRadiansPerUnit(self.RADIANS_PER_METER)

        self.dm_l.setPIDF(
            0, self.VL_KP, self.VL_KI, self.VL_KD, self.VL_KF,
        )
        self.dm_r.setPIDF(
            0, self.VR_KP, self.VR_KI, self.VR_KD, self.VR_KF,
        )

    def on_enable(self):
        pass

    def on_disable(self):
        self.stop()

    def stop(self) -> None:
        self.mode = self._Mode.Idle
        if wpilib.RobotBase.isSimulation():
            self._setSimulationOutput(1, 0)
            self._setSimulationOutput(3, 0)

    def setOutput(self, output_l: float, output_r: float) -> None:
        self.mode = self._Mode.PercentOutput
        self.desired_output.left = output_l
        self.desired_output.right = output_r

    def setWheelVelocity(self, velocity_l: float, velocity_r: float) -> None:
        self.mode = self._Mode.Velocity
        self.desired_velocity.left = velocity_l
        self.desired_velocity.right = velocity_r

    def setChassisVelocity(
        self, velocity_x: float, velocity_y: float, velocity_omega
    ) -> None:
        state = ChassisSpeeds(velocity_x, velocity_y, -velocity_omega)
        velocity = self.kinematics.toWheelSpeeds(state)
        self.setWheelVelocity(velocity.left, velocity.right)

    def setTankDrive(self, throttle, rotation):
        self.mode = self._Mode.PercentOutput
        self.desired_output.left = throttle + rotation
        self.desired_output.right = throttle - rotation
        # self.desired_output.norm(self.MAX_JOYSTICK_OUTPUT)

    def setForzaDrive(self, throttle, reverse, rotation):
        if throttle <= 0:
            throttle = -reverse
        self.setTankDrive(throttle, rotation)

    def ntPutLeftRight(self, key, value):
        self.nt.putNumber(f"{key}_left", value.left)
        self.nt.putNumber(f"{key}_right", value.right)

    def updateNetworkTables(self):
        """Update network table values related to component."""
        self.wheel_left.putNT(self.nt, "wheel_left")
        self.wheel_right.putNT(self.nt, "wheel_right")
        self.ntPutLeftRight("desired_output", self.desired_output)
        self.ntPutLeftRight("desired_velocity", self.desired_velocity)
        self.ntPutLeftRight("feedforward", self.feedforward)

    def getHeading(self):
        return self.getPose().rotation().radians()

    def getPose(self):
        if wpilib.RobotBase.isSimulation():
            x = (
                wpilib.simulation.SimDeviceSim("Field2D").getDouble("x").get()
                * units.meters
            )
            y = (
                wpilib.simulation.SimDeviceSim("Field2D").getDouble("y").get()
                * units.meters
            )
            rot = (
                wpilib.simulation.SimDeviceSim("Field2D").getDouble("rot").get()
                * units.degrees
            )
            rot = units.angle_range(rot)
            return Pose2d(x, y, rot)
        else:
            return self.odometry.getPose()

    def _setSimulationOutput(self, id, output):
        wpilib.simulation.SimDeviceSim(f"Talon FX[{id}]").getDouble("Motor Output").set(
            output
        )

    def _getSimulationPosition(self, id):
        return (
            wpilib.simulation.SimDeviceSim(f"Custom Talon FX[{id}]")
            .getDouble("Position")
            .get()
        )

    def execute(self):
        dt = 0.02
        self.wheel_left.update(self.dm_l.getPosition(), dt)
        self.wheel_right.update(self.dm_r.getPosition(), dt)
        if wpilib.RobotBase.isSimulation():
            self.wheel_left.position = self._getSimulationPosition(1)
            self.wheel_right.position = self._getSimulationPosition(3)

        self.odometry.update(
            Rotation2d(self.getHeading()),
            self.wheel_left.position,
            self.wheel_right.position,
        )

        if self.mode == self._Mode.Idle:
            self.dm_l.setOutput(0)
            self.dm_r.setOutput(0)
        elif self.mode == self._Mode.PercentOutput:
            self.dm_l.setOutput(self.desired_output.left)
            self.dm_r.setOutput(self.desired_output.right)
        elif self.mode == self._Mode.Velocity:
            self.feedforward.left = (
                self.feedforward_l.calculate(self.desired_velocity.left, dt) / 12
            )

            self.feedforward.right = (
                self.feedforward_r.calculate(self.desired_velocity.right, dt) / 12
            )
            if not wpilib.RobotBase.isSimulation():
                self.dm_l.setVelocity(
                    self.desired_velocity.left, self.feedforward.left,
                )
                self.dm_r.setVelocity(
                    self.desired_velocity.right, self.feedforward.right,
                )
            else:
                self.dm_l.setOutput(self.desired_velocity.left / self.MAX_VELOCITY)
                self.dm_r.setOutput(self.desired_velocity.right / self.MAX_VELOCITY)
        self.updateNetworkTables()


text:
from typing import Union

import numpy as np

from .exceptions import NumericException

def closest_index(array: np.ndarray, value: Union[float, int]) -> int:
    """
    get closest index in numpy array, or -1 if empty

    >>> closest_index([1,2,3,4,5], 1.2)
    0
    >>> closest_index([1,2,3,4,5], -1)
    0
    >>> closest_index([1,2,3,4,5], 1.6)
    1
    >>> closest_index([1,2,3,4,5], 999)
    4
    >>> closest_index([1,2,3,4,5], 2.4)
    1
    >>> closest_index([1,2,3,4,5], 4.9)
    4
    """

    if array.ndim != 1:
        raise NumericException(f"expected 1-dimensional array, got: {array.ndim}")

    if not array.shape[0]:
        return -1

    return int(abs(array - value).argmin())

max_stars_repo_name: joeledwardson/joelsutilities

max_issues_repo_name: joeledwardson/joelsutilities

max_forks_repo_name: joeledwardson/joelsutilities


text:
import numpy as np
import time
import pprint
import csv
from collections import OrderedDict

import tensorflow as tf

import os

if not os.path.exists('weights/'):
    os.makedirs('weights/')

if not os.path.exists('logs/'):
    os.makedirs('logs/')

class StateSpace:
    '''
    State Space manager

    Provides utilit functions for holding "states" / "actions" that the controller
    must use to train and predict.

    Also provides a more convenient way to define the search space
    '''
    def __init__(self, B, operators,
                 input_lookback_depth=-1,
                 input_lookforward_depth=None):
        '''
        Constructs a search space which models the NAS and PNAS papers

        A single block consists of the 4-tuple:
        (input 1, operation 1, input 2, operation 2)

        The merge operation can be a sum or a concat as required.

        The input operations are used for adding up intermediate values
        inside the same cell. See the NASNet and P-NASNet models to see
        how intermediate blocks connect based on input values.

        The default operation values are based on the P-NAS paper. They
        should be changed as needed.

        # Note:
        This only provides a convenient mechanism to train the networks.
        It is upto the model designer to interpret this block tuple
        information and construct the final model.

        # Args:
            B: Maximum number of blocks

            operators: a list of operations (can be anything, must be
                interpreted by the model designer when constructing the
                actual model. Usually a list of strings.

            input_lookback_depth: should be a negative number or 0.
                Describes how many cells the input should look behind.
                Can be used to tensor information from 0 or more cells from
                the current cell.

                The negative number describes how many cells to look back.
                Set to 0 if the lookback feature is not needed (flat cells).

            input_lookforward_depth: sets a limit on input depth that can be looked forward.
                This is useful for scenarios where "flat" models are preferred,
                wherein each cell is flat, though it may take input from deeper
                layers (if the designer so chooses)

                The default searches over cells which can have inter-connections.
                Setting it to 0 limits this to just the current input for that cell (flat cells).
        '''
        self.states = OrderedDict()
        self.state_count_ = 0

        self.children = None
        self.intermediate_children = None

        self.B = B

        if operators is None:
            self.operators = ['identity', '3x3 dconv', '5x5 dconv', '7x7 dconv',
                              '1x7-7x1 conv', '3x3 conv', '3x3 maxpool', '3x3 avgpool']
        else:
            self.operators = operators

        self.input_lookback_depth = input_lookback_depth
        self.input_lookforward_depth = input_lookforward_depth

        input_values = list(range(input_lookback_depth, self.B))  # -1 = Hc-1, 0-(B-1) = Hci

        self.inputs_embedding_max = len(input_values)
        self.operator_embedding_max = len(np.unique(operators))

        self._add_state('inputs', values=input_values)
        self._add_state('ops', values=self.operators)
        self.prepare_initial_children()

    def _add_state(self, name, values):
        '''
        Adds a "state" to the state manager, along with some metadata for efficient
        packing and unpacking of information required by the RNN ControllerManager.

        Stores metadata such as:
        -   Global ID
        -   Name
        -   Valid Values
        -   Number of valid values possible
        -   Map from value ID to state value
        -   Map from state value to value ID

        # Args:
            name: name of the state / action
            values: valid values that this state can take

        # Returns:
            Global ID of the state. Can be used to refer to this state later.
        '''
        index_map = {}
        for i, val in enumerate(values):
            index_map[i] = val

        value_map = {}
        for i, val in enumerate(values):
            value_map[val] = i

        metadata = {
            'id': self.state_count_,
            'name': name,
            'values': values,
            'size': len(values),
            'index_map_': index_map,
            'value_map_': value_map,
        }
        self.states[self.state_count_] = metadata
        self.state_count_ += 1

        return self.state_count_ - 1

    def embedding_encode(self, id, value):
        '''
        Embedding index encode the specific state value

        # Args:
            id: global id of the state
            value: state value

        # Returns:
            embedding encoded representation of the state value
        '''
        state = self[id]
        value_map = state['value_map_']
        value_idx = value_map[value]

        encoding = np.zeros((1, 1), dtype=np.float32)
        encoding[0, 0] = value_idx
        return encoding

    def get_state_value(self, id, index):
        '''
        Retrieves the state value from the state value ID

        # Args:
            id: global id of the state
            index: index of the state value (usually from argmax)

        # Returns:
            The actual state value at given value index
        '''
        state = self[id]
        index_map = state['index_map_']
        value = index_map[index]
        return value

    def parse_state_space_list(self, state_list):
        '''
        Parses a list of one hot encoded states to retrieve a list of state values

        # Args:
            state_list: list of one hot encoded states

        # Returns:
            list of state values
        '''
        state_values = []
        for id, state_value in enumerate(state_list):
            state_val_idx = state_value[0, 0]
            value = self.get_state_value(id % 2, state_val_idx)
            state_values.append(value)

        return state_values

    def entity_encode_child(self, child):
        '''
        Perform encoding for all blocks in a cell

        # Args:
            child: a list of blocks (which forms one cell / layer)

        # Returns:
            list of entity encoded blocks of the cell
        '''
        encoded_child = []
        for i, val in enumerate(child):
            encoded_child.append(self.embedding_encode(i % 2, val))

        return encoded_child

    def prepare_initial_children(self):
        '''
        Prepare the initial set of child models which must
        all be trained to obtain the initial set of scores
        '''
        # set of all operations
        ops = list(range(len(self.operators)))
        inputs = list(range(self.input_lookback_depth, 0))

        # if input_lookback_depth == 0, then we need to adjust to have at least
        # one input (generally 0)
        if len(inputs) == 0:
            inputs = [0]

        print()
        print("Obtaining search space for b = 1")
        print("Search space size : ", (len(inputs) * (len(self.operators) ** 2)))

        search_space = [inputs, ops, inputs, ops]
        self.children = list(self._construct_permutations(search_space))

    def prepare_intermediate_children(self, new_b):
        '''
        Generates the intermediate product of the previous children
        and the current generation of children.

        # Args:
            new_b: the number of blocks in current stage

        # Returns:
            a generator that produces a joint of the previous and current
            child models
        '''
        if self.input_lookforward_depth is not None:
            new_b_dash = min(self.input_lookforward_depth, new_b)
        else:
            new_b_dash = new_b

        new_ip_values = list(range(self.input_lookback_depth, new_b_dash))
        ops = list(range(len(self.operators)))

        # if input_lookback_depth == 0, then we need to adjust to have at least
        # one input (generally 0)
        if len(new_ip_values) == 0:
            new_ip_values = [0]

        new_child_count = ((len(new_ip_values)) ** 2) * (len(self.operators) ** 2)
        print()
        print("Obtaining search space for b = %d" % new_b)
        print("Search space size : ", new_child_count)

        print()
        print("Total models to evaluate : ", (len(self.children) * new_child_count))
        print()

        search_space = [new_ip_values, ops, new_ip_values, ops]
        new_search_space = list(self._construct_permutations(search_space))

        for i, child in enumerate(self.children):
            for permutation in new_search_space:
                temp_child = list(child)
                temp_child.extend(permutation)
                yield temp_child

    def _construct_permutations(self, search_space):
        ''' state space is a 4-tuple (ip1, op1, ip2, op2) '''
        for input1 in search_space[0]:
            for operation1 in search_space[1]:
                for input2 in search_space[2]:
                    for operation2 in search_space[3]:
                        yield (input1, self.operators[operation1], input2, self.operators[operation2])

    def print_state_space(self):
        ''' Pretty print the state space '''
        print('*' * 40, 'STATE SPACE', '*' * 40)

        pp = pprint.PrettyPrinter(indent=2, width=100)
        for id, state in self.states.items():
            pp.pprint(state)
            print()

    def print_actions(self, actions):
        ''' Print the action space properly '''
        print('Actions :')

        for id, action in enumerate(actions):
            state = self[id]
            name = state['name']
            vals = [(self.get_state_value(id % 2, p), p) for n, p in zip(state['values'], *action)]
            print("%s : " % name, vals)
        print()

    def update_children(self, children):
        self.children = children

    def __getitem__(self, id):
        return self.states[id % self.size]

    @property
    def size(self):
        return self.state_count_

    def print_total_models(self, K):
        ''' Compute the total number of models to generate and train '''
        num_inputs = 1 if self.input_lookback_depth == 0 else abs(self.input_lookback_depth)
        level1 = (num_inputs ** 2) * (len(self.operators) ** 2)
        remainder = (self.B - 1) * K
        total = level1 + remainder

        print("Total number of models : ", total)
        print()
        return total

class Controller(tf.keras.Model):

    def __init__(self, controller_cells, embedding_dim,
                 input_embedding_max, operator_embedding_max):
        '''
        LSTM Controller model which accepts encoded sequence describing the
        architecture of the model and predicts a singular value describing
        its probably validation accuracy.

        # Args:
            controller_cells: number of cells of the Controller LSTM.
            embedding_dim: size of the embedding dimension.
            input_embedding_max: maximum input dimension of the input embedding.
            operator_embedding_max: maximum input dimension of the operator encoding.
        '''
        super(Controller, self).__init__(name='EncoderRNN')

        self.controller_cells = controller_cells
        self.embedding_dim = embedding_dim
        self.input_embedding_max = input_embedding_max
        self.operator_embedding_max = operator_embedding_max

        # Layers
        self.input_embedding = tf.keras.layers.Embedding(input_embedding_max + 1, embedding_dim)
        self.operators_embedding = tf.keras.layers.Embedding(operator_embedding_max + 1, embedding_dim)

        if tf.test.is_gpu_available():
            self.rnn = tf.keras.layers.CuDNNLSTM(controller_cells, return_state=True)
        else:
            self.rnn = tf.keras.layers.LSTM(controller_cells, return_state=True)

        self.rnn_score = tf.keras.layers.Dense(1, activation='sigmoid')

    def call(self, inputs_operators, states=None, training=None, mask=None):
        inputs, operators = self._get_inputs_and_operators(inputs_operators)  # extract the data

        if states is None:  # initialize the state vectors
            states = self.rnn.get_initial_state(inputs)
            states = [tf.to_float(state) for state in states]

        # map the sparse inputs and operators into dense embeddings
        embed_inputs = self.input_embedding(inputs)
        embed_ops = self.operators_embedding(operators)

        # concatenate the embeddings
        embed = tf.concat([embed_inputs, embed_ops], axis=-1)  # concatenate the embeddings

        # run over the LSTM
        out = self.rnn(embed, initial_state=states)
        out, h, c = out  # unpack the outputs and states

        # get the predicted validation accuracy
        score = self.rnn_score(out)

        return [score, [h, c]]

    def _get_inputs_and_operators(self, inputs_operators):
        '''
        Splits the joint inputs and operators into seperate inputs
        and operators list for convenience of the SearchSpace.

        # Args:
            inputs_operators: interleaved [input; operator] pairs.

        # Returns:
            list of inputs and list of operators.
        '''
        inputs = inputs_operators[:, 0::2]  # even place data
        operators = inputs_operators[:, 1::2]  # odd place data

        return inputs, operators

class ControllerManager:
    '''
    Utility class to manage the RNN Controller.

    Tasked with maintaining the state of the training schedule,
    keep track of the children models generated from cross-products,
    cull non-optimal children model configurations and resume
    training.
    '''
    def __init__(self, state_space,
                 B=5, K=256,
                 train_iterations=10,
                 reg_param=0.001,
                 controller_cells=32,
                 embedding_dim=20,
                 input_B=None,
                 restore_controller=False):
        '''
        Manages the Controller network training and prediction process.

        # Args:
            state_space: completely defined search space.
            B: depth of progression.
            K: maximum number of children model trained per level of depth.
            train_iterations: number of training epochs for the RNN per depth level.
            reg_param: strength of the L2 regularization loss.
            controller_cells: number of cells in the Controller LSTM.
            embedding_dim: embedding dimension for inputs and operators.
            input_B: override value of B, used only when we are restoring the controller.
                Determing the maximum input connectivity allowed to the RNN Controller,
                to maintain backward compatibility with trained models.

                Use it alongside `restore_controller` to evaluate model settings
                with larger depth `B` than allowed at training time.
            restore_controller: flag whether to restore a pre-trained RNN controller
                upon construction.
        '''

        self.state_space = state_space  # type: StateSpace
        self.state_size = self.state_space.size

        self.b_ = 1
        self.B = B
        self.K = K
        self.embedding_dim = embedding_dim

        self.train_iterations = train_iterations
        self.controller_cells = controller_cells
        self.reg_strength = reg_param
        self.input_B = input_B
        self.restore_controller = restore_controller

        self.children_history = None
        self.score_history = None

        # No support for summary statistics yet
        timestr = time.strftime("%Y-%m-%d-%H-%M-%S")
        self.logdir = 'logs/%s' % timestr
        # self.summary_writer = tf.contrib.summary.create_file_writer(self.logdir)
        # self.summary_write.set_as_default()

        self.build_policy_network()

    def get_actions(self, top_k=None):
        '''
        Gets a one hot encoded action list, either from random sampling or from
        the ControllerManager RNN

        # Args:
            top_k: Number of models to return

        # Returns:
            A one hot encoded action list
        '''
        models = self.state_space.children

        if top_k is not None:
            models = models[:top_k]

        actions = []
        for model in models:
            encoded_model = self.state_space.entity_encode_child(model)
            actions.append(encoded_model)

        return actions

    def build_policy_network(self):
        '''
        Construct the RNN controller network with the provided settings.

        Also constructs saver and restorer to the RNN controller if required.
        '''
        if tf.test.is_gpu_available():
            device = '/gpu:0'
        else:
            device = '/cpu:0'
        self.device = device

        if self.restore_controller and self.input_B is not None:
            input_B = self.input_B
        else:
            input_B = self.state_space.inputs_embedding_max

        self.global_step = tf.train.get_or_create_global_step()
        learning_rate = tf.train.exponential_decay(0.001, self.global_step,
                                                   500, 0.98, staircase=True)

        with tf.device(device):
            self.controller = Controller(self.controller_cells,
                                         self.embedding_dim,
                                         input_B,
                                         self.state_space.operator_embedding_max)

            self.optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)

        self.saver = tf.train.Checkpoint(controller=self.controller,
                                         optimizer=self.optimizer,
                                         global_step=self.global_step)

        if self.restore_controller:
            path = tf.train.latest_checkpoint('weights/')

            if path is not None and tf.train.checkpoint_exists(path):
                print("Loading Controller Checkpoint !")
                self.saver.restore(path)

    def loss(self, real_acc, rnn_scores):
        '''
        Computes the surrogate losses to train the controller.

        - rnn score loss is the MSE between the real validation acc and the
        predicted acc of the rnn.

        - reg loss is the L2 regularization loss on the parameters of the controller.

        # Args:
            real_acc: actual validation accuracy obtained by child models.
            rnn_scores: predicted validation accuracy obtained by child models.

        # Returns:
            weighted sum of rnn score loss + reg loss.
        '''
        # RNN predicted accuracies
        rnn_score_loss = tf.losses.mean_squared_error(real_acc, rnn_scores)

        # Regularization of model
        params = self.controller.trainable_variables
        reg_loss = tf.reduce_sum([tf.nn.l2_loss(x) for x in params])

        total_loss = rnn_score_loss + self.reg_strength * reg_loss

        return total_loss

    def train_step(self, rewards):
        '''
        Perform a single train step on the Controller RNN

        # Returns:
            final training loss
        '''
        children = np.array(self.state_space.children, dtype=np.object)  # take all the children
        rewards = np.array(rewards, dtype=np.float32)
        loss = 0

        if self.children_history is None:
            self.children_history = [children]
            self.score_history = [rewards]
            batchsize = rewards.shape[0]
        else:
            self.children_history.append(children)
            self.score_history.append(rewards)
            batchsize = sum([data.shape[0] for data in self.score_history])

        train_size = batchsize * self.train_iterations
        print("Controller: Number of training steps required for this stage : %d" % (train_size))
        print()

        for current_epoch in range(self.train_iterations):
            print("Controller: Begin training epoch %d" % (current_epoch + 1))
            print()

            for dataset_id in range(self.b_):
                children = self.children_history[dataset_id]
                scores = self.score_history[dataset_id]

                ids = np.array(list(range(len(scores))))
                np.random.shuffle(ids)

                print("Controller: Begin training - B = %d" % (dataset_id + 1))

                for id, (child, score) in enumerate(zip(children[ids], scores[ids])):
                    child = child.tolist()
                    state_list = self.state_space.entity_encode_child(child)
                    state_list = np.concatenate(state_list, axis=-1).astype('int32')

                    with tf.device(self.device):
                        state_list = tf.convert_to_tensor(state_list)

                        with tf.GradientTape() as tape:
                            rnn_scores, states = self.controller(state_list, states=None)
                            acc_scores = score.reshape((1, 1))

                            total_loss = self.loss(acc_scores, rnn_scores)

                        grads = tape.gradient(total_loss, self.controller.trainable_variables)
                        grad_vars = zip(grads, self.controller.trainable_variables)

                        self.optimizer.apply_gradients(grad_vars, self.global_step)

                    loss += total_loss.numpy().sum()

                print("Controller: Finished training epoch %d / %d of B = %d / %d" % (
                    current_epoch + 1, self.train_iterations,
                    dataset_id + 1, self.b_))

        # save weights
        self.saver.save('weights/controller.ckpt')

        return loss.mean()

    def update_step(self):
        '''
        Updates the children from the intermediate products for the next generation
        of larger number of blocks in each cell
        '''
        if self.b_ + 1 <= self.B:

            self.b_ += 1
            models_scores = []

            # iterate through all the intermediate children
            for i, intermediate_child in enumerate(self.state_space.prepare_intermediate_children(self.b_)):
                state_list = self.state_space.entity_encode_child(intermediate_child)
                state_list = np.concatenate(state_list, axis=-1).astype('int32')

                state_list = tf.convert_to_tensor(state_list)

                # score the child
                score, _ = self.controller(state_list, states=None)
                score = score[0, 0].numpy()

                # preserve the child and its score
                models_scores.append([intermediate_child, score])

                with open('scores_%d.csv' % (self.b_), mode='a+', newline='') as f:
                    writer = csv.writer(f)
                    data = [score]
                    data.extend(intermediate_child)
                    writer.writerow(data)

                if (i + 1) % 500 == 0:
                    print("Scored %d models. Current model score = %0.4f" % (i + 1, score))

            # sort the children according to their score
            models_scores = sorted(models_scores, key=lambda x: x[1], reverse=True)

            # account for case where there are fewer children than K
            if self.K is not None:
                children_count = min(self.K, len(models_scores))
            else:
                children_count = len(models_scores)

            # take only the K highest scoring children for next iteration
            children = []
            for i in range(children_count):
                children.append(models_scores[i][0])

            # save these children for next round
            self.state_space.update_children(children)
        else:
            print()
            print("No more updates necessary as max B has been reached !")


text:
# Author: bbsipingsoft
# Project: DCCA_demo
# Date: 4/24/19
# Time: 12:18 PM
# File: dcca_iris.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import pandas as pd
import torch
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
from torch.autograd import Variable
from torch.optim import Adam
from torch.optim.lr_scheduler import ExponentialLR

from models import ViewModel
from objective import dcca_objective
from utils import construct_iris_views, create_view_loader, visualize, svm_classify, process_evaluation_data, evaluate

torch.manual_seed(0)
np.random.seed(0)
batch_sz = 90

def train(view1, view2, dloader, lr, reg, n_iters):
    """
    Trains the models for correlation maximization

    :param view1: Model 1
    :param view2: Model 2
    :param v1_data: data loaders for view 1
    :param v2_data: data loaders for view 2
    :param lr: learning rate
    :param reg: regularization parameter
    :param n_iters: number of total iterations
    :return: the correlation history
    """
    n_epochs = n_iters / float(len(dloader))
    n_epochs = int(n_epochs)
    corr_hist = []
    counter = 0

    optimizer1 = Adam(view1.parameters())
    optimizer2 = Adam(view2.parameters())

    # sch1 = ExponentialLR(optimizer1, gamma=0.001)
    # sch2 = ExponentialLR(optimizer2, gamma=0.001)

    for epoch in range(n_epochs):
        for Xs, _ in dloader:
            x_v1, x_v2 = Xs
            x_v1 = Variable(x_v1, requires_grad=False)
            x_v2 = Variable(x_v2, requires_grad=False)

            pred1 = view1(x_v1)
            pred2 = view2(x_v2)

            corr, dH1, dH2, A1, A2 = dcca_objective(pred1, pred2, reg)
            corr_hist.append(corr)
            view1.canonical_weight = A1
            view2.canonical_weight = A2

            optimizer1.zero_grad()
            optimizer2.zero_grad()

            pred1.backward(-dH1)
            pred2.backward(-dH2)

            optimizer1.step()
            optimizer2.step()

            if counter % 100 == 0:
                print("Epoch={}/{}, iteration={}, corr={}".format(epoch + 1, n_epochs, counter, corr))

            counter += 1

        # sch1.step()
        # sch2.step()
    return corr_hist

if __name__ == '__main__':
    df = pd.read_csv('iris.csv', header=0)
    df = df[:100][:]
    labels = list(set(df['Species']))
    species = [labels.index(sp) for sp in df['Species']]
    df['Species'] = species

    shuffle(df)

    v1_data, v2_data = construct_iris_views(df)

    # view 1
    v1_data_xtrain, v1_data_xvalid, v1_data_ytrain, v1_data_yvalid = train_test_split(v1_data[:, 0:2], v1_data[:, 2],
                                                                                test_size=0.1)
    v1_data = ([v1_data_xtrain, v1_data_ytrain], [v1_data_xvalid, v1_data_yvalid])

    # view 2
    v2_data_xtrain, v2_data_xvalid, v2_data_ytrain, v2_data_yvalid = train_test_split(v2_data[:, 0:2], v2_data[:, 2],
                                                                                test_size=0.1)
    v2_data = ([v2_data_xtrain, v2_data_ytrain], [v2_data_xvalid, v2_data_yvalid])

    tr_ldr = create_view_loader(((v1_data_xtrain, v1_data_ytrain), (v2_data_xtrain, v2_data_ytrain)), batch_sz,
                                shuffle=True)
    val_ldr = create_view_loader(((v1_data_xvalid, v1_data_yvalid), (v2_data_xvalid, v2_data_yvalid)), 10,
                                 shuffle=False)

    latent_dim = 2
    view1 = ViewModel(input_dim=2, out_dim=latent_dim, h_layers=(100, 100))
    view2 = ViewModel(input_dim=2, out_dim=latent_dim, h_layers=(100, 100))
    corr_hist = train(view1, view2, tr_ldr, lr=1e-2, reg=1e-4, n_iters=1000)
    vis_series = {'corr_hist': corr_hist}

    # DCCA evaluation
    sanity_check, test_accuracy = evaluate(tr_ldr=tr_ldr, val_ldr=val_ldr, tt_ldr=None, model=view1,
                                           view_idx=0, has_test=False, dim=latent_dim)
    print("DCCA: training set accuracy={}, test set accuracy={}".format(sanity_check, test_accuracy))

    # Linear SVM baseline
    data_x = df[['SepalLengthCm', 'SepalWidthCm', 'PetalLengthCm', 'PetalWidthCm']].values
    data_y = df['Species'].values
    train_x, valid_x, train_y, valid_y = train_test_split(data_x, data_y, test_size=0.1)
    tr_ldr = create_view_loader(((train_x, train_y),), batch_sz, shuffle=True)
    val_ldr = create_view_loader(((valid_x, valid_y),), 10, shuffle=False)
    sanity_check, test_accuracy = evaluate(tr_ldr=tr_ldr, val_ldr=val_ldr, tt_ldr=None, model=None, view_idx=0, dim=4,
                                           has_test=False)
    print("Linear SVC (baseline): training set accuracy={}, test set accuracy={}".format(sanity_check, test_accuracy))

    visualize('corr_iris.png', vis_series)


text:
import pandas as pd
import numpy as np
import preproc as pre
import detrend as dtr
import networkx as nx
import dcor

# TODO: develop distance correlation network

def dist_corr():
    df = dtr.detrend()
    df.dropna(inplace=True)

    # store the column names as a list
    col_names = df.columns.tolist()

    df_dcor = pd.DataFrame(index=col_names, columns=col_names)

    k = 0
    for i in col_names:

        v1 = df.loc[:, i].values
        for j in col_names[k:]:

            v2 = df.loc[:, j].values
            rez = dcor.distance_correlation(v1, v2)
            df_dcor.at[i, j] = rez
            df_dcor.at[j, i] = rez
        k += 1

    return df_dcor

if __name__ == "__main__":
    dist_corr()

max_stars_repo_name: mayabenowitz/hedging-the-stock-market

max_issues_repo_name: mayabenowitz/hedging-the-stock-market

max_forks_repo_name: mayabenowitz/hedging-the-stock-market

reason: import numpy,import networkx