text:
# Copyright (c) 2021 Jacob F. Valdez. Released under the MIT license.
"""Interactive Connect-N game."""

from __future__ import annotations

import datetime
import pickle
from typing import Tuple

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

import jnumpy as jnp
import jnumpy.nn as jnn
import jnumpy.rl as jrl

from common import BoardEnv

default_hparams = dict(
    hidden_size=64,  # hidden layer size for RealDQN
    categorical_hidden_size=32,  # hidden layer size for CategoricalDQN
    activation="Relu",  # activation function for networks
    optimizer=None,  # optimizer for networks (filled in later)
    learning_rate=0.001,  # learning rate for optimizer
    epsilon_start=1.0,  # Starting value for epsilon
    epsilon_decay=0.9,  # Decay rate for epsilon per epoch
    min_epsilon=0.01,  # Final value for epsilon
    discount=0.9,  # Discount factor
    epoch=0,  # Current epoch (different for each agent)
    batch_size=8,  # Number of samples per training batch
    train_batch_size=32,  # Batch size for training
    board_size=10,  # Board size
    train_win_length=4,  # Number of pieces in a row needed to win in training
    test_win_length=5,  # Number of pieces in a row needed to win in testing
    min_steps_per_epoch=64,  # Minimum number of steps per epoch
    num_steps_replay_coef=0.5,  # How much to upweight longer episodes
    success_replay_coef=1.5,  # How much to upweight successful experience
    age_replay_coef=0.5,  # How much to downweight older trajectories
)

encoder_pool = list()  # [Encoder] encoders can be shared across agents
agent_pool = list()  # [Agent], not shared by definition
hparam_pool = dict()  # {name: hparams}  not shared. Updated in `Trainer.train`

encoder_pool.append(
    jnn.Sequential(
        [
            jnn.Conv2D(8, 3, 1, "same", jnp.Relu),
            jnn.Conv2D(16, 3, 1, "same", jnp.Relu),
            jnn.GlobalMaxPooling(1),
        ],
        name="simple_encoder",
    )
)  # [B, H, W, 2] -> [B, W, d_enc]

def maybe_input(prompt: str, default: str) -> str:
    answer = input(f"{prompt} (default: {default}): ").strip().lower()
    if answer == "":
        return default
    return answer

def get_activation_fn(name: str) -> jnn.Activation:
    name = name.strip().lower()
    if name == "relu":
        return jnp.Relu
    elif name == "sigm":
        return jnp.Sigm
    elif name == "tanh":
        return jnp.Tanh
    elif name == "linear":
        return jnp.Linear
    else:
        raise ValueError(f"Invalid activation: {name}")

def new_encoder() -> jnn.Sequential:
    print("\nNew encoder:")
    layers = []
    num_layers = int(maybe_input("Enter number of conv layers", 2))
    for i in range(num_layers):
        print(f"\nConv Layer {i}:")
        filters = int(maybe_input("Enter number of filters", 8))
        kernel_size = eval(maybe_input("Enter kernel size (odd, int or 2-tuple)", "3"))
        stride = eval(maybe_input("Enter stride (int or 2-tuple)", "1"))
        activation = get_activation_fn(
            maybe_input(
                "Enter activation function ('relu', 'sigm', 'tanh', or 'linear')",
                "linear",
            )
        )
        layers.append(jnn.Conv2D(filters, kernel_size, stride, "same", activation))
    layers.append(jnn.GlobalMaxPooling(1))
    encoder_name_candidate = input("\nPlease name this encoder: ").strip()
    encoder = jnn.Sequential(layers, name=encoder_name_candidate)
    encoder_pool.append(encoder)
    return encoder

def get_encoder() -> jnn.Sequential:
    print("\nAll encoders:")
    for encoder in encoder_pool:
        print(f" - {encoder.name}: conv_layers={len(encoder.layers)-1}")
    encoder_name = input('Enter encoder name (or enter "new"): ').strip().lower()
    if encoder_name == "new":
        encoder = new_encoder()
    else:
        encoder = [enc for enc in encoder_pool if enc.name == encoder_name][0]
    return encoder

def get_hparams(agent_name: str) -> dict:
    print(f"\nHyperameters for {agent_name}:")
    hparams = dict()
    hparams.update(default_hparams)
    for key, value in default_hparams.items():
        if isinstance(value, float):
            hparams[key] = float(maybe_input(f"Enter {key}", value))
        elif isinstance(value, int):
            hparams[key] = int(maybe_input(f"Enter {key}", value))
        elif isinstance(value, str):
            hparams[key] = maybe_input(f"Enter {key}", value)
        else:
            continue
    hparams["optimizer"] = jnp.SGD(hparams["learning_rate"])
    hparams["activation"] = get_activation_fn(hparams["activation"])
    return hparams

def new_agent() -> jrl.Agent:
    print("\nNew agent:")
    print("Agent types:")
    print(" - random: takes random actions at each step")
    print(" - dqn: standard deep Q-network; a* = arg_a max dqn(o, a)")
    print(
        " - categorical-dqn: categorical deep Q-network; <q0,q1,..,qn> = dqn(o); a* = arg_i max qi"
    )
    agent_type = input("Select an agent type: ").strip().lower()
    while agent_type not in ["random", "dqn", "categorical-dqn"]:
        agent_type = input("Invalid agent type; try again: ").strip().lower()
    agent_name_candidate = input("Please name this agent: ").strip().lower()
    if agent_type == "random":
        num_actions = int(input("Enter num_actions (int): ").strip())
        agent = jrl.agents.RandomAgent(
            num_actions=num_actions, name=agent_name_candidate
        )
        hparam_pool[agent.name] = dict(
            epoch=0,  # Current epoch (not used for random agent)
            min_steps_per_epoch=64,  # Minimum number of steps per epoch
            num_steps_replay_coef=0.5,  # How much to upweight longer episodes
            success_replay_coef=1.5,  # How much to upweight successful experience
            age_replay_coef=0.5,  # How much to downweight older trajectories
        )
    elif agent_type == "dqn":
        num_actions = int(input("Enter num_actions (int): ").strip())
        encoder = get_encoder()
        hparams = get_hparams(agent_name=agent_name_candidate)
        agent = jrl.agents.RealDQN(
            num_actions=num_actions,
            encoder=encoder,
            hparams=hparams,
            name=agent_name_candidate,
        )
        hparam_pool[agent.name] = hparams
    elif agent_type == "categorical-dqn":
        num_actions = int(input("Enter num_actions (int): ").strip())
        encoder = get_encoder()
        hparams = get_hparams(agent_name=agent_name_candidate)
        agent = jrl.agents.CategoricalDQN(
            num_actions=num_actions,
            encoder=encoder,
            hparams=hparams,
            name=agent_name_candidate,
        )
        hparam_pool[agent.name] = hparams
    agent_pool.append(agent)
    return agent

def get_agent() -> jrl.Agent:
    print("\nAll agents:")
    for agent in agent_pool:
        print(
            f" - {agent.name}: type={type(agent).__name__} num_actions={agent.num_actions} epoch={hparam_pool[agent.name]['epoch']}"
        )
    agent_name = input("Enter agent name (or enter 'new'): ").strip().lower()
    while agent_name not in [agent.name for agent in agent_pool] + ["new"]:
        print(f"Invalid agent name: {agent_name}")
        agent_name = input("Enter agent name (or enter 'new'): ").strip().lower()
    if agent_name == "new":
        return new_agent()
    else:
        return [agent for agent in agent_pool if agent.name == agent_name][0]

def get_env(env_type) -> Tuple[str, dict]:
    print(f"\n{env_type}:")

    board_size = int(maybe_input("Enter board size (int)", 8))
    win_length = int(maybe_input("Enter win length (int)", 4))
    reward_mode = maybe_input(
        "Enter reward mode ('sparse', 'dense_stateless', or 'dense_advantage')",
        "sparse",
    ).lower()
    batch_size = int(maybe_input("Enter environment batch size (int)", 4))

    return jrl.ParallelEnv(
        batch_size=batch_size,
        env_init_fn=lambda: BoardEnv(
            board_size=board_size,
            win_length=win_length,
            reward_mode=reward_mode,
        ),
    )

def train():

    train_env = get_env("Training environment")
    test_env = get_env("Test environment")

    while train_env.env.board_size != test_env.env.board_size:
        print(
            f"Board sizes must match: {train_env.env.board_size} vs {test_env.env.board_size}"
        )
        print(
            f"Please choose a test environment with a board size of {train_env.env.board_size}"
        )
        test_env = get_env("Test environment")

    agent1 = get_agent()
    agent2 = get_agent()

    while agent1.num_actions != agent2.num_actions:
        print(
            "Agents must have the same number of actions: "
            f"{agent1.name} has {agent1.num_actions} actions but "
            f"{agent2.name} has {agent2.num_actions} actions."
        )
        agent1 = get_agent()
        agent2 = get_agent()

    trainer = jrl.ParallelTrainer(
        callbacks=[
            jrl.QEvalCallback(
                eval_on_collect=True, eval_on_train=True, eval_on_test=True
            ),
            jrl.PrintCallback(
                keys=[
                    "epoch",
                    "agent",
                    "collect_reward",
                    "train_reward",
                    "test_reward",
                    "q_collect",
                    "q_train",
                    "q_test",
                ],
            ),
        ],
    )
    global hparam_pool
    training_epochs = int(maybe_input("\nEnter number of training epochs (int)", 10))
    train_start_time = datetime.datetime.now()
    print(f"\nTraining {agent1.name} with {agent2.name}...")
    hparam_pool = trainer.train(
        agents=[agent1, agent2],
        all_hparams=hparam_pool,
        env=train_env,
        test_env=test_env,
        training_epochs=training_epochs,
    )
    train_finish_time = datetime.datetime.now()
    print(f"Training finished. Elapsed time = {train_finish_time-train_start_time}")

def play():
    def is_human_first(human_name, agent_name) -> bool:
        first_agent = input(f"Who goes first ({human_name}, {agent_name})? ").strip()
        if first_agent == human_name:
            return True
        elif first_agent == agent_name:
            return False
        else:
            print(f"{first_agent} is not a valid option.")
            return is_human_first(human_name, agent_name)

    human_name = input("Enter your name: ").strip()
    agent = get_agent()
    human_first = is_human_first(human_name, agent.name)

    board_size = int(maybe_input("Enter board size (int)", 8))
    win_length = int(maybe_input("Enter win length (int)", 4))
    reward_mode = maybe_input(
        "Enter reward mode ('sparse', 'dense_stateless', or 'dense_advantage')",
        "sparse",
    ).lower()
    env = BoardEnv(
        board_size=board_size,
        win_length=win_length,
        reward_mode=reward_mode,
    )
    batch_env = jrl.Batch1Env(env)  # to handle executing single actions

    def get_move():
        env.render()
        action = input(f"Enter move (0-{env.board_size}): ").strip()
        while action not in [str(i) for i in range(env.board_size)]:
            print(f"{action} is not a valid move.")
            action = input(f"Enter move (0-{env.board_size}): ").strip()
        action = np.eye(env.board_size)[int(action)][None, :]

    step = batch_env.reset()
    if human_first:
        action = get_move()
        step = batch_env.step(action)
    while True:
        action = agent.forward(step)
        step = batch_env.step(action)
        if step.done:
            winner = agent.name if step.reward > 0 else human_name
            break
        action = get_move()
        step = batch_env.step(action)
        if step.done:
            winner = human_name if step.reward > 0 else agent.name
            break

    # announce winner
    print(f"{winner} wins!\n")

    # play again?
    if input("Play again? (y/N):").strip().lower() == "y":
        play()

def main_menu():
    print("1. Train")
    print("2. Play")
    print("3. Exit")
    print("\n")

    choice = input("Enter your choice: ")
    if choice == "1":
        train()
    elif choice == "2":
        play()
    elif choice == "3":
        exit()
    else:
        print("Invalid choice")

    _ = input("\nPress enter to continue...")

def main():
    print("CSE 4309 Machine Learning Project 7++")
    print("Copyright 2021 Jacob Valdez. Released under the MIT License")
    print("Code at https://github.com/JacobFV/jnumpy")
    print()
    while True:
        main_menu()
        print(128 * "\n")

        continue

        try:
            main_menu()
        except Exception as e:
            print(e)
            print("\n")
            print("An error occurred. Please try again.")
            print(
                "Please report issues at https://github.com/JacobFV/jnumpy/issues/new"
            )
            print("\n")

main()

max_stars_repo_name: JacobFV/jumpy

max_issues_repo_name: JacobFV/jumpy

max_forks_repo_name: JacobFV/jumpy


text:
# /usr/bin/env python3.5
# -*- mode: python -*-
# =============================================================================
#  @@-COPYRIGHT-START-@@
#
#  Copyright (c) 2018, Qualcomm Innovation Center, Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are met:
#
#  1. Redistributions of source code must retain the above copyright notice,
#     this list of conditions and the following disclaimer.
#
#  2. Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#
#  3. Neither the name of the copyright holder nor the names of its contributors
#     may be used to endorse or promote products derived from this software
#     without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
#  POSSIBILITY OF SUCH DAMAGE.
#
#  SPDX-License-Identifier: BSD-3-Clause
#
#  @@-COPYRIGHT-END-@@
# =============================================================================

""" Sub-sample data for weight reconstruction for channel pruning feature """

from typing import Iterator, Callable
import math
import numpy as np

import torch
import torch.utils.data
import torch.nn

# Import AIMET specific modules
from aimet_common.utils import AimetLogger
from aimet_common.input_match_search import InputMatchSearch
from aimet_torch import utils

logger = AimetLogger.get_area_logger(AimetLogger.LogAreas.ChannelPruning)

class StopForwardException(Exception):
    """ Dummy exception to early-terminate forward-pass """

class DataSubSampler:
    """ Utilities to sub-sample data for weight reconstruction """

    @staticmethod
    def _forward_pass(model: torch.nn.Module, batch: torch.Tensor):
        """
        forward pass depending model allocation on CPU / GPU till StopForwardException
        :param model: model
        :param batch: batch
        :return: Nothing
        """
        model.eval()

        # first check if the model is on GPU or not
        if utils.is_model_on_gpu(model):
            batch = batch.cuda()

        try:
            with torch.no_grad():
                _ = model(batch)
        except StopForwardException:
            pass

    @staticmethod
    def _register_fwd_hook_for_layer(layer: torch.nn.Module, hook: Callable) -> torch.utils.hooks.RemovableHandle:
        """
        register forward hook for given layer
        :param layer: layer
        :param hook: hook function
        :return: hook handle
        """
        hook_handle = layer.register_forward_hook(hook)
        return hook_handle

    @classmethod
    def get_sub_sampled_data(cls, orig_layer: torch.nn.Conv2d, pruned_layer: torch.nn.Conv2d,
                             orig_model: torch.nn.Module, comp_model: torch.nn.Module, data_loader: Iterator,
                             num_reconstruction_samples: int) -> (np.ndarray, np.ndarray):
        # pylint: disable=too-many-locals
        """
        Get all the input data from pruned model and output data from original model

        :param orig_layer: original layer
        :param pruned_layer: pruned layer
        :param orig_model: original model, un-pruned, used to provide the actual outputs
        :param comp_model: comp. model, this is potentially already pruned in the upstreams layers of given layer name
        :param data_loader: data loader
        :param num_reconstruction_samples: The number of reconstruction samples
        :return: input_data, output_data
        """

        def _hook_to_collect_input_data(module, inp_data, _):
            """
            hook to collect input data
            """
            inp_data = utils.to_numpy(inp_data[0])
            pruned_layer_inp_data.append(inp_data)
            raise StopForwardException

        def _hook_to_collect_output_data(module, _, out_data):
            """
            hook to collect output data
            """
            out_data = utils.to_numpy(out_data)
            orig_layer_out_data.append(out_data)
            raise StopForwardException

        assert isinstance(orig_layer, torch.nn.Conv2d)

        assert isinstance(pruned_layer, torch.nn.Conv2d)

        assert orig_layer.dilation == (1, 1), 'No Conv2d layers supported for dilation other than (1, 1)'

        assert pruned_layer.dilation == (1, 1), 'No Conv2d layers supported for dilation other than (1, 1)'

        # hard coded value
        samples_per_image = 10

        total_num_of_images = int(num_reconstruction_samples / samples_per_image)

        # number of possible batches - round up
        num_of_batches = math.ceil(total_num_of_images / data_loader.batch_size)

        # Todo - I am not sure if checking the length of a data loader is a great idea.
        if num_of_batches > len(data_loader):
            raise ValueError("There are insufficient batches of data in the provided data loader for the "
                             "purpose of weight reconstruction!")

        hook_handles = list()

        orig_layer_out_data = list()
        pruned_layer_inp_data = list()

        all_sub_sampled_inp_data = list()
        all_sub_sampled_out_data = list()

        # register forward hooks
        hook_handles.append(cls._register_fwd_hook_for_layer(orig_layer, _hook_to_collect_output_data))

        hook_handles.append(cls._register_fwd_hook_for_layer(pruned_layer, _hook_to_collect_input_data))

        # forward pass for given number of batches for both original model and compressed model
        for batch, (images_in_one_batch, _) in enumerate(data_loader):

            DataSubSampler._forward_pass(orig_model, images_in_one_batch)
            DataSubSampler._forward_pass(comp_model, images_in_one_batch)

            input_data = np.vstack(pruned_layer_inp_data)
            output_data = np.vstack(orig_layer_out_data)

            # delete list entries used for hooks
            del pruned_layer_inp_data[:]
            del orig_layer_out_data[:]

            layer_attributes = (orig_layer.kernel_size, orig_layer.stride, orig_layer.padding)

            # get the sub sampled input and output data
            sub_sampled_inp_data, sub_sampled_out_data = InputMatchSearch.subsample_data(layer_attributes, input_data,
                                                                                output_data, samples_per_image)

            all_sub_sampled_inp_data.append(sub_sampled_inp_data)
            all_sub_sampled_out_data.append(sub_sampled_out_data)

            if batch == num_of_batches - 1:
                logger.info("batch index : %s reached number of batches: %s", batch + 1, num_of_batches)
                break

        # remove hook handles
        for hook_handle in hook_handles:
            hook_handle.remove()

        # accumulate total sub sampled input and output data

        return np.vstack(all_sub_sampled_inp_data), np.vstack(all_sub_sampled_out_data)

ext: py

max_stars_repo_name: quic-ssiddego/aimet

max_issues_repo_name: quic-ssiddego/aimet

max_forks_repo_name: quic-ssiddego/aimet

reason: import numpy


text:
import time
import types
import typing

from trainer.srwarp import warptrainer
from misc import image_utils

import numpy as np
import torch
import cv2

from srwarp import transform
from srwarp import warp
from srwarp import wtypes

_parent_class = warptrainer.SRWarpTrainer

class CV2Predictor(_parent_class):

    def __init__(
            self,
            *args,
            scale: int=4,
            naive: bool=False,
            interpolation: str='bicubic',
            is_demo: bool=False,
            **kwargs) -> None:

        super().__init__(*args, **kwargs)
        self.model.fill = -255
        self.scale = scale
        self.naive = naive
        self.interpolation = interpolation
        self.is_demo = is_demo
        self.time_acc = 0
        self.count = 0
        return

    @staticmethod
    def get_kwargs(cfg: types.SimpleNamespace) -> dict:
        kwargs = _parent_class.get_kwargs(cfg)
        kwargs['scale'] = cfg.scale
        kwargs['naive'] = cfg.cv2_naive
        kwargs['interpolation'] = cfg.cv2_interpolation
        kwargs['is_demo'] = ('srwarp.demo' in cfg.dtest)
        return kwargs

    def forward(self, **samples) -> typing.Tuple[torch.Tensor, float]:
        if self.is_demo:
            hr = samples['gt']
            lr_crop = samples['img']
            m = samples['m'][0].cpu()
        else:
            hr = samples['img']
            m_inv = samples['m'][0].cpu()
            lr_crop, m = self.get_input(hr, m_inv)

        sizes = (hr.size(-1), hr.size(-2))

        if self.naive:
            sr = lr_crop
        else:
            sr = self.model(lr_crop)

        m = transform.compensate_scale(m, 1 / self.scale)
        if self.interpolation == 'bicubic':
            flags = cv2.INTER_CUBIC
        elif self.interpolation == 'area':
            flags = cv2.INTER_AREA
        elif self.interpolation == 'lanczos':
            flags = cv2.INTER_LANCZOS4
        elif self.interpolation == 'layer':
            flags = 'layer'
        elif self.interpolation == 'adaptive':
            flags = 'adaptive'

        fill = -255
        if self.interpolation in ('bicubic', 'area', 'lanczos'):
            sr_np = sr.squeeze(0).cpu().numpy()
            sr_np = np.transpose(sr_np, (1, 2, 0))
            sr_np = np.ascontiguousarray(sr_np)
            m_np = m.cpu().numpy()
            sr_cv2_np = cv2.warpPerspective(
                sr_np,
                m_np,
                sizes,
                flags=flags,
                borderMode=cv2.BORDER_REFLECT,
            )
            sr_cv2_np = np.transpose(sr_cv2_np, (2, 0, 1))
            sr_cv2_np = np.ascontiguousarray(sr_cv2_np)

            ref = warp.warp_by_function(
                sr,
                m,
                f_inverse=False,
                sizes=(sizes[1], sizes[0]),
                fill=fill,
            )
            mask = (ref != fill).float()
            sr_cv2 = torch.from_numpy(sr_cv2_np).to(sr.device).unsqueeze(0)
        else:
            adaptive_grid = (flags == 'adaptive')
            sr_cv2 = warp.warp_by_function(
                sr,
                m,
                f_inverse=False,
                sizes=(sizes[1], sizes[0]),
                adaptive_grid=adaptive_grid,
                fill=fill,
            )
            mask = (sr_cv2 != fill).float()

        sr_cv2 = image_utils.quantize(sr_cv2)
        sr_cv2 = mask * (sr_cv2 + 1) - 1
        loss = self.loss(sr=sr_cv2, hr=hr, mask=mask)
        return loss, {'sr': sr_cv2, 'mask': 2 * mask - 1}

max_stars_repo_name: sanghyun-son/srwarp

max_issues_repo_name: sanghyun-son/srwarp

max_forks_repo_name: sanghyun-son/srwarp


text:
"""neural network model."""
import math

import tensorflow as tf
import numpy as np

import inputs

#########################################
# FLAGS
#########################################
FLAGS = tf.app.flags.FLAGS
tf.app.flags.DEFINE_boolean('top_unlabel', True,
                            "whether to use unlabeled feature in top rnn.")
tf.app.flags.DEFINE_boolean('top_label', True,
                            "whether to use labeled feature in top rnn.")

# parameters applied for both train.py and eval.py will be kept here.
# currently don't put any flag here
# only in_top_k for both train and eval

#########################################
# global variables
#########################################
# FEATURE_NUM = 256

# Constants describing the training process.
# MOVING_AVERAGE_DECAY = 0.9999  # The decay to use for the moving average.

# If a model is trained with multiple GPUs, prefix all Op names with tower_name
# to differentiate the operations. Note that this prefix is removed from the
# names of the summaries when visualizing a model.
# TOWER_NAME = 'tower'

#########################################
# functions
#########################################
class Model(object):
    """super class for neural network models."""

    def __init__(self, is_train=False):
        self.num_cats = FLAGS.num_cats
        self.NUM_EPOCHS_PER_DECAY = FLAGS.num_epochs_per_decay
        self.LEARNING_RATE_DECAY_FACTOR = FLAGS.lr_decay_factor
        self.INITIAL_LEARNING_RATE = FLAGS.initial_lr
        self.min_lr = 0.1**FLAGS.min_lr

        if is_train:
            # build training graph
            self.dropout_keep_prob = FLAGS.dropout_keep_prob

            self.global_step = tf.get_variable(
                "global_step",
                initializer=tf.zeros_initializer(
                    [],
                    dtype=tf.int64),
                trainable=False)

            # get input data
            page_batch, label_batch = self.inputs_train()
            if FLAGS.debug:
                label_batch = tf.Print(label_batch,
                                       [label_batch],
                                       message='\nlabel_batch:',
                                       summarize=128)

            # Build a Graph that computes the logits predictions from the
            self.logits = self.inference(page_batch)

            # Calculate predictions.
            self.top_k_op = tf.nn.in_top_k(self.logits, label_batch,
                                           FLAGS.in_top_k)
            tf.scalar_summary("accuracy",
                              tf.reduce_mean(tf.cast(self.top_k_op, "float")))

            # Calculate loss.
            self.loss = self.loss(self.logits, label_batch)

            # Build a Graph that trains the model with one batch of examples and
            # updates the model parameters.
            self.train_op = self.training(self.loss, self.global_step)
        else:
            # build eval graph
            self.dropout_keep_prob = 1

            page_batch_eval, label_batch_eval = self.inputs_eval()
            self.logits_eval = self.inference(page_batch_eval)
            # Calculate predictions.
            self.top_k_op_eval = tf.nn.in_top_k(
                self.logits_eval, label_batch_eval, FLAGS.in_top_k_eval)
            tf.scalar_summary(
                "accuracy_eval (batch)",
                tf.reduce_mean(tf.cast(self.top_k_op_eval, "float")))

    def _activation_summary(self, x):
        """Helper to create summaries for activations.
        Creates a summary that provides a histogram of activations.
        Creates a summary that measure the sparsity of activations.
        Args:
            x: Tensor
        Returns:
            nothing
        """
        # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
        # session. This helps the clarity of presentation on tensorboard.
        # Error: these summaries cause high classifier error!!!
        # All inputs to node MergeSummary/MergeSummary must be from the same frame.

        # tensor_name = re.sub('%s_[0-9]*/' % "tower", '', x.op.name)
        # tf.histogram_summary(tensor_name + '/activations', x)
        # tf.scalar_summary(tensor_name + '/sparsity', tf.nn.zero_fraction(x))

    def _variable_on_cpu(self, name, shape, initializer):
        """Helper to create a Variable stored on CPU memory.
        Args:
            name: name of the variable
            shape: list of ints
            initializer: initializer for Variable
        Returns:
            Variable Tensor
        """
        with tf.device('/cpu:0'):
            var = tf.get_variable(name, shape, initializer=initializer)
        return var

    def _variable_with_weight_decay(self, name, shape, stddev, wd=None):
        """Helper to create an initialized Variable with weight decay.
        Note that the Variable is initialized with a truncated normal distribution.
        A weight decay is added only if one is specified.
        Args:
            name: name of the variable
            shape: list of ints
            stddev: standard deviation of a truncated Gaussian
            wd: add L2Loss weight decay multiplied by this float. If None, weight
                decay is not added for this Variable.
        Returns:
            Variable Tensor
        """
        var = self._variable_on_cpu(
            name,
            shape,
            tf.truncated_normal_initializer(stddev=stddev))
        if wd is not None:
            # weight_decay = tf.mul(tf.constant(0.1), wd, name='weight_loss')
            weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
            tf.add_to_collection('losses', weight_decay)
            # tf.add_to_collection('losses', wd)
        return var

    def inputs_train(self):
        """Construct input examples for training process.
        Returns:
            sequences: 4D tensor of [batch_size, 1, input_length, alphabet_length] size.
            labels: Labels. 1D tensor of [batch_size] size.
        """
        return inputs.inputs("train",
                             FLAGS.batch_size,
                             FLAGS.num_epochs,
                             min_shuffle=1000)

    def inputs_eval(self):
        """Construct input examples for evaluations.
        similar to inputs_train
        """
        # don't shuffle
        return inputs.inputs("test", FLAGS.batch_size, None, min_shuffle=1)

    def inference(self, page_batch):
        raise NotImplementedError("Should have implemented this")
        return

    def high_classifier(self, page_batch, low_classifier):
        """high level classifier.
        category vectors are fed into RNN by the order:
            unlabeled_rel -> labeld_rel -> target
        two options to concat variable-length relatives:
            1. concat (un_rel, la_rel, target) by tf.scatter_update??
                or tf.slice, complex
            2. concat multiple dynamic_rnn, by passing the previouse final state
        """
        target_batch, un_batch, un_len, la_batch, la_len = page_batch

        def add_mark(rnn_inputs, mark_types, mark):
            """add a mark to the end of cat vectory,
            to distinguish la_rel, un_rel, target.
            """
            m = [0] * mark_types
            m[mark] = 1
            m = tf.constant(m, dtype=tf.float32)
            m = tf.expand_dims(m, 0)
            m = tf.expand_dims(m, 0)
            # sh = rnn_inputs.get_shape()
            sha = tf.shape(rnn_inputs)
            # [1, 1, mark_types] -> [batch_size, N, mark_types]
            m = tf.tile(m, [sha[0], sha[1], 1])
            ret = tf.concat(2, [rnn_inputs, m])
            ret = tf.reshape(ret, [sha[0], sha[1], FLAGS.num_cats + mark_types
                                   ])
            return ret

        with tf.variable_scope("low_classifier"):
            # [batch_size, html_len, we_dim] -> [batch_size, num_cats]
            target_cats = low_classifier(target_batch)
            target_cats = tf.nn.softmax(target_cats)
            # [batch_size, 1, num_cats]
            target_cats = tf.expand_dims(target_cats, 1)
            target_len = tf.ones([FLAGS.batch_size], dtype=tf.int32)
            if FLAGS.debug:
                target_cats = tf.Print(target_cats,
                                       [target_cats],
                                       message='\ntarget_cats):',
                                       summarize=FLAGS.debug_len)
        # high-level classifier - RNN
        with tf.variable_scope("dynamic_rnn"):
            cell = tf.nn.rnn_cell.GRUCell(num_units=FLAGS.num_cats)
            state = None
            if FLAGS.add_mark:
                target_cats = add_mark(target_cats, 3, 2)
                if FLAGS.debug:
                    target_cats = tf.Print(target_cats,
                                           [target_cats],
                                           message='\nadd_mark):',
                                           summarize=FLAGS.debug_len)
            outputs, state = tf.nn.dynamic_rnn(cell=cell,
                                               inputs=target_cats,
                                               sequence_length=target_len,
                                               initial_state=state,
                                               dtype=tf.float32)

        if FLAGS.top_unlabel:
            with tf.variable_scope("low_classifier", reuse=True):
                un_rel = tf.sparse_tensor_to_dense(un_batch)
                un_rel = tf.reshape(un_rel, [FLAGS.batch_size, -1,
                                             FLAGS.html_len, FLAGS.we_dim])
                # call low_classifier to classify relatives
                # all relatives of one target composed of one batch
                # ?? variable scope, init problem of low_classifier ???????
                # not efficient!!! even un_len = 0, un_rel will still be classified.
                # hope sth like: low_classifier(un_rel, un_len),
                # map_fn(low_classifier, [un_rel, un_len])
                # low_classifier slice input, classify and then pad output
                un_rel = tf.transpose(un_rel, perm=[1, 0, 2, 3])
                un_rel = tf.map_fn(low_classifier, un_rel, name="map_fn_low")
                # -> [batch_size, un_len(max???), num_cats]
                # dynamic_rnn set to time_major=True, do not need to transpose back
                # un_rel = tf.transpose(un_rel, perm=[1, 0, 2])
                un_rel = tf.map_fn(tf.nn.softmax, un_rel, name="map_fn_sm")
                if FLAGS.debug:
                    un_rel = tf.Print(un_rel,
                                      [un_len, un_rel],
                                      message='\nun_rel):',
                                      summarize=FLAGS.debug_len)
            # high-level classifier - RNN
            with tf.variable_scope("dynamic_rnn", reuse=True):
                if FLAGS.add_mark:
                    un_rel = add_mark(un_rel, 3, 1)
                    if FLAGS.debug:
                        target_cats = tf.Print(target_cats,
                                               [un_rel],
                                               message='\nadd_mark):',
                                               summarize=FLAGS.debug_len)
                outputs, state = tf.nn.dynamic_rnn(cell=cell,
                                                   time_major=True,
                                                   inputs=un_rel,
                                                   sequence_length=un_len,
                                                   initial_state=state,
                                                   dtype=tf.float32)

        if FLAGS.top_label:
            # labeled relatives
            la_rel = tf.sparse_tensor_to_dense(la_batch)
            la_rel = tf.reshape(la_rel, [FLAGS.batch_size, -1, FLAGS.num_cats])
            if FLAGS.debug:
                la_rel = tf.Print(la_rel,
                                  [la_len, la_rel],
                                  message='\nla_rel:',
                                  summarize=FLAGS.debug_len)

            # high-level classifier - RNN
            with tf.variable_scope("dynamic_rnn", reuse=True):
                if FLAGS.add_mark:
                    la_rel = add_mark(la_rel, 3, 0)
                    if FLAGS.debug:
                        target_cats = tf.Print(target_cats,
                                               [la_rel],
                                               message='\nadd_mark):',
                                               summarize=FLAGS.debug_len)
                outputs, state = tf.nn.dynamic_rnn(cell=cell,
                                                   inputs=la_rel,
                                                   sequence_length=la_len,
                                                   initial_state=state,
                                                   dtype=tf.float32)

        # [batch_size, num_cats]
        # outputs = tf.reduce_mean(outputs, 1)
        # return outputs
        return state

    def c_high_classifier(self, page_batch, low_classifier):
        """high level classifier.
        category vectors are fed into RNN by the order:
            unlabeled_rel -> labeld_rel -> target
        two options to concat variable-length relatives:
            1. concat (un_rel, la_rel, target) by tf.scatter_update??
                or tf.slice, complex
            2. concat multiple dynamic_rnn, by passing the previouse final state
        """
        target_batch, un_batch, un_len, la_batch, la_len = page_batch

        with tf.variable_scope("low_classifier") as low_scope:
            # [batch_size, html_len, we_dim] -> [batch_size, num_cats]
            target_cats = low_classifier(target_batch)
            target_cats = tf.nn.softmax(target_cats)
            # [batch_size, 1, num_cats]
            target_cats = tf.expand_dims(target_cats, 1)
            target_len = tf.ones([FLAGS.batch_size], dtype=tf.int32)
            if FLAGS.debug:
                target_cats = tf.Print(target_cats,
                                       [target_cats],
                                       message='\ntarget_cats):',
                                       summarize=FLAGS.debug_len)

            # reuse parameters for low_classifier
            low_scope.reuse_variables()

            un_rel = tf.sparse_tensor_to_dense(un_batch)
            un_rel = tf.reshape(un_rel, [FLAGS.batch_size, -1, FLAGS.html_len,
                                         FLAGS.we_dim])
            # call low_classifier to classify relatives
            # all relatives of one target composed of one batch
            # ?? variable scope, init problem of low_classifier ???????
            # not efficient!!! even un_len = 0, un_rel will still be classified.
            # hope sth like: low_classifier(un_rel, un_len),
            # map_fn(low_classifier, [un_rel, un_len])
            # low_classifier slice input, classify and then pad output
            # -> [batch_size, un_len(max???), num_cats]
            un_rel = tf.map_fn(low_classifier, un_rel, name="map_fn_low")
            un_rel = tf.map_fn(tf.nn.softmax, un_rel, name="map_fn_sm")
            if FLAGS.debug:
                un_rel = tf.Print(un_rel,
                                  [un_len, un_rel],
                                  message='\nun_rel):',
                                  summarize=FLAGS.debug_len)

        # labeled relatives
        la_rel = tf.sparse_tensor_to_dense(la_batch)
        la_rel = tf.reshape(la_rel, [FLAGS.batch_size, -1, FLAGS.num_cats])
        if FLAGS.debug:
            la_rel = tf.Print(la_rel,
                              [la_len, la_rel],
                              message='\nla_rel:',
                              summarize=FLAGS.debug_len)

        def add_mark(rnn_inputs, mark_types, mark):
            """add a mark to the end of cat vectory,
            to distinguish la_rel, un_rel, target.
            """
            m = [0] * mark_types
            m[mark] = 1
            m = tf.constant(m, dtype=tf.float32)
            m = tf.expand_dims(m, 0)
            m = tf.expand_dims(m, 0)
            # sh = rnn_inputs.get_shape()
            sha = tf.shape(rnn_inputs)
            # [1, 1, mark_types] -> [batch_size, N, mark_types]
            m = tf.tile(m, [sha[0], sha[1], 1])
            ret = tf.concat(2, [rnn_inputs, m])
            ret = tf.reshape(ret, [sha[0], sha[1], FLAGS.num_cats + mark_types
                                   ])
            return ret

        # high-level classifier - RNN
        with tf.variable_scope("dynamic_rnn") as rnn_scope:
            cell = tf.nn.rnn_cell.GRUCell(num_units=FLAGS.num_cats)
            state = None
            # outputs, state = tf.nn.dynamic_rnn(cell=cell,
            #                                    inputs=concat_inputs,
            #                                    sequence_length=num_pages,
            #                                    dtype=tf.float32)
            if FLAGS.add_mark:
                la_rel = add_mark(la_rel, 3, 0)
                un_rel = add_mark(un_rel, 3, 1)
                target_cats = add_mark(target_cats, 3, 2)
                if FLAGS.debug:
                    target_cats = tf.Print(target_cats,
                                           [la_rel, un_rel, target_cats],
                                           message='\nadd_mark):',
                                           summarize=FLAGS.debug_len)
            outputs, state = tf.nn.dynamic_rnn(cell=cell,
                                               inputs=la_rel,
                                               sequence_length=la_len,
                                               initial_state=state,
                                               dtype=tf.float32)
            # reuse parameters for dynamic_rnn
            rnn_scope.reuse_variables()

            outputs, state = tf.nn.dynamic_rnn(cell=cell,
                                               inputs=un_rel,
                                               sequence_length=un_len,
                                               initial_state=state,
                                               dtype=tf.float32)
            outputs, state = tf.nn.dynamic_rnn(cell=cell,
                                               inputs=target_cats,
                                               sequence_length=target_len,
                                               initial_state=state,
                                               dtype=tf.float32)

        # [batch_size, num_cats]
        # outputs = tf.reduce_mean(outputs, 1)
        # return outputs
        return state

    def b_high_classifier(self, page_batch, low_classifier):
        """high level classifier.
        category vectors are fed into RNN by the order:
            unlabeled_rel -> labeld_rel -> target
        two options to concat variable-length relatives:
            1. concat (un_rel, la_rel, target) by tf.scatter_update??
                or tf.slice, complex
            2. concat multiple dynamic_rnn, by passing the previouse final state

        """
        target_batch, un_batch, un_len, la_batch, la_len = page_batch

        # with tf.variable_scope("low_classifier") as low_scope:
        with tf.variable_scope("low_classifier"):
            # [batch_size, 1, html_len, we_dim]
            target_exp = tf.expand_dims(target_batch, 1)

            un_rel = tf.sparse_tensor_to_dense(un_batch)
            un_rel = tf.reshape(un_rel, [FLAGS.batch_size, -1, FLAGS.html_len,
                                         FLAGS.we_dim])
            # concat: unlabeled + target
            un_and_target = tf.concat(1, [target_exp])
            # un_and_target = tf.concat(1, [un_rel, target_exp])

            # call low_classifier to classify relatives
            # all relatives of one target composed of one batch
            # ?? variable scope, init problem of low_classifier ???????
            # [batch_size, num_len(variant) + 1, num_cats]
            un_and_target = tf.map_fn(low_classifier,
                                      un_and_target,
                                      name="map_fn")
            # un_and_target = low_classifier(target_batch)
            un_and_target = tf.squeeze(un_and_target, squeeze_dims=[1])
            # tmp = un_and_target
            un_and_target = tf.nn.softmax(un_and_target)
            un_and_target = tf.expand_dims(un_and_target, 1)

        # labeled relatives
        la_rel = tf.sparse_tensor_to_dense(la_batch)
        la_rel = tf.reshape(la_rel, [FLAGS.batch_size, -1, FLAGS.num_cats])
        if FLAGS.debug:
            la_rel = tf.Print(la_rel,
                              [la_len, la_rel],
                              message='\nla_rel:',
                              summarize=100)

        # concat all inputs for high-level classifier RNN
        # concat_inputs = tf.concat(1, [un_and_target])
        # concat_inputs = tf.concat(1, [la_rel, un_and_target])
        # concat_inputs = tf.concat(1, [un_and_target, la_rel])
        # if FLAGS.debug:
        #     concat_inputs = tf.Print(concat_inputs, [la_len, concat_inputs], message='\nconcat:', summarize=100)

        # number of pages for each target
        # num_pages = tf.ones([FLAGS.batch_size],
        #                     dtype=tf.int32)
        # num_pages = tf.add(
        #     # tf.add(un_len, la_len),
        #     la_len,
        #     tf.ones(
        #         [FLAGS.batch_size],
        #         dtype=tf.int32))
        target_len = tf.ones([FLAGS.batch_size], dtype=tf.int32)

        # high-level classifier - RNN
        with tf.variable_scope("dynamic_rnn") as rnn_scope:
            cell = tf.nn.rnn_cell.GRUCell(num_units=FLAGS.num_cats)
            state = None
            # outputs, state = tf.nn.dynamic_rnn(cell=cell,
            #                                    inputs=concat_inputs,
            #                                    sequence_length=num_pages,
            #                                    dtype=tf.float32)
            outputs, state = tf.nn.dynamic_rnn(cell=cell,
                                               inputs=la_rel,
                                               sequence_length=la_len,
                                               initial_state=state,
                                               dtype=tf.float32)
            # reuse parameters for low_classifier
            rnn_scope.reuse_variables()

            outputs, state = tf.nn.dynamic_rnn(cell=cell,
                                               inputs=un_and_target,
                                               sequence_length=target_len,
                                               initial_state=state,
                                               dtype=tf.float32)

        # [batch_size, num_cats]
        # outputs = tf.reduce_mean(outputs, 1)
        # return outputs
        return state
        # return tmp

    def a_high_classifier(self, page_batch, low_classifier):
        """high level classifier."""
        target_batch, un_batch, un_len, la_batch, la_len = page_batch

        with tf.variable_scope("low_classifier") as low_scope:
            # [batch_size, 1, html_len, we_dim]
            target_exp = tf.expand_dims(target_batch, 1)
            # [batch_size, 1, num_cats]
            target_logits = tf.map_fn(low_classifier,
                                      target_exp,
                                      name="map_fn")

            # reuse parameters for low_classifier
            low_scope.reuse_variables()

            un_rel = tf.sparse_tensor_to_dense(un_batch)
            un_rel = tf.reshape(un_rel, [FLAGS.batch_size, -1, FLAGS.html_len,
                                         FLAGS.we_dim])
            # call low_classifier to classify relatives
            # all relatives of one target composed of one batch
            # [batch_size, num_len(variant), num_cats]
            un_rel = tf.map_fn(low_classifier, un_rel, name="map_fn")

        # labeled relatives
        la_rel = tf.sparse_tensor_to_dense(la_batch)
        la_rel = tf.reshape(la_rel, [FLAGS.batch_size, -1, FLAGS.num_cats])

        # concat all inputs for high-level classifier RNN
        # concat_inputs = tf.concat(1, [un_rel, target_logits])
        concat_inputs = tf.concat(1, [un_rel, la_rel, target_logits])

        # number of pages for each target
        num_pages = tf.add(
            tf.add(un_len, la_len),
            tf.ones(
                [FLAGS.batch_size],
                dtype=tf.int32))

        # high-level classifier - RNN
        with tf.variable_scope("dynamic_rnn"):
            cell = tf.nn.rnn_cell.GRUCell(num_units=FLAGS.num_cats)
            outputs, state = tf.nn.dynamic_rnn(cell,
                                               inputs=concat_inputs,
                                               sequence_length=num_pages,
                                               dtype=tf.float32)

        return state

    def loss(self, logits, labels):
        """Add L2Loss to all the trainable variables.
        Add summary for "Loss" and "Loss/avg".
        Args:
            logits: Logits from inference().
            labels: Labels from distorted_inputs or inputs(). 1-D tensor
                    of shape [batch_size]
        Returns:
            Loss tensor of type float.
        """
        # Calculate the average cross entropy loss across the batch.
        labels = tf.cast(labels, tf.int64)
        cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits,
            labels,
            name='cross_entropy_per_example')
        cross_entropy_mean = tf.reduce_mean(cross_entropy,
                                            name='cross_entropy')
        # from tensorflow.python.ops import variables
        # added to the collection GraphKeys.REGULARIZATION_LOSSES and can be used for regularization.
        tf.add_to_collection('REGULARIZATION_LOSSES', cross_entropy_mean)

        # The total loss is defined as the cross entropy loss plus all of the weight
        # decay terms (L2 loss).
        total_loss = tf.add_n(
            tf.get_collection('REGULARIZATION_LOSSES'),
            name='total_loss')
        self._add_loss_summaries(total_loss)
        return total_loss

    def _add_loss_summaries(self, total_loss):
        """Add summaries for losses in CNN model.
        Generates moving average for all losses and associated summaries for
        visualizing the performance of the network.
        Args:
            total_loss: Total loss from loss().
        Returns:
            loss_averages_op: op for generating moving averages of losses.
        """
        # # Compute the moving average of all individual losses and the total loss.
        # loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
        # losses = tf.get_collection('losses')
        # loss_averages_op = loss_averages.apply(losses + [total_loss])
        #
        # # Attach a scalar summary to all individual losses and the total loss; do the
        # # same for the averaged version of the losses.
        # for l in losses + [total_loss]:
        #     # Name each loss as '(raw)' and name the moving average version of the loss
        #     # as the original loss name.
        #     tf.scalar_summary(l.op.name + ' (raw)', l)
        #     tf.scalar_summary(l.op.name, loss_averages.average(l))

        # losses = tf.get_collection('REGULARIZATION_LOSSES')
        # all_losses = losses + [total_loss]
        all_losses = [total_loss]
        # is it necessary to add all REGULARIZATION_LOSSES ?????
        for l in all_losses:
            tf.scalar_summary(l.op.name, l)

    def training(self, total_loss, global_step):
        """Train CNN model.
        Create an optimizer and apply to all trainable variables. Add moving
        average for all trainable variables.
        Args:
            total_loss: Total loss from loss().
            global_step: Integer Variable counting the number of training steps
            processed.
        Returns:
            train_op: op for training.
        """
        # Variables that affect learning rate.
        num_batches_per_epoch = FLAGS.num_train_examples / FLAGS.batch_size
        print("num_batches_per_epoch: {}".format(num_batches_per_epoch))
        decay_steps = int(num_batches_per_epoch * self.NUM_EPOCHS_PER_DECAY)
        print("decay_steps: {}".format(decay_steps))

        # Decay the learning rate exponentially based on the number of steps.
        lr_decay = tf.train.exponential_decay(self.INITIAL_LEARNING_RATE,
                                              global_step,
                                              decay_steps,
                                              self.LEARNING_RATE_DECAY_FACTOR,
                                              staircase=True)
        # compare with 0.01 * 0.5^10
        lr = tf.maximum(lr_decay, self.min_lr)
        tf.scalar_summary('learning_rate', lr)

        # optimizer = tf.train.AdamOptimizer(lr)
        optimizer = tf.train.MomentumOptimizer(lr, 0.9)
        grads_and_vars = optimizer.compute_gradients(total_loss)
        train_op = optimizer.apply_gradients(grads_and_vars,
                                             global_step=global_step)

        # Add histograms for trainable variables.
        for var in tf.trainable_variables():
            tf.histogram_summary(var.op.name, var)

        # Add histograms for gradients.
        for grad, var in grads_and_vars:
            if grad is not None:
                tf.histogram_summary(var.op.name + '/gradients', grad)

        return train_op

    def train_step(self, sess):
        """run one step on one batch trainning examples."""
        step, _, loss_value, top_k = sess.run([self.global_step, self.train_op,
                                               self.loss, self.top_k_op])
        # step, loss_value, top_k = sess.run([self.global_step,
        #                                        self.loss, self.top_k_op])
        # step, l = sess.run([self.global_step, self.logits])
        return step, loss_value, top_k
        # return step, l, 2

    def eval_once(self, sess):
        # it's better to divide exactly with no remainder
        num_iter = int(math.ceil(FLAGS.num_test_examples / FLAGS.batch_size))
        true_count = 0  # counts the number of correct predictions.
        total_sample_count = num_iter * FLAGS.batch_size
        eval_step = 0
        while eval_step < num_iter:
            predictions = sess.run([self.top_k_op_eval])
            true_count += np.sum(predictions)
            eval_step += 1

        # compute precision @ 1.
        precision = true_count / total_sample_count
        return precision


text:
"""
Script that caches train data for future training

Based on Kaggle Competition

Modified to take 3-channel data
"""

from __future__ import division

import os
import pandas as pd
import extra_functions
from tqdm import tqdm
import h5py
import numpy as np

data_path = '../data'

train_wkt = pd.read_csv(os.path.join(data_path, 'train_wkt_v4.csv'))
gs = pd.read_csv(os.path.join(data_path, 'grid_sizes.csv'), names=['ImageId', 'Xmax', 'Ymin'], skiprows=1)

shapes = pd.read_csv(os.path.join(data_path, '3_shapes.csv'))

def cache_train_16():
    print('num_train_images =', train_wkt['ImageId'].nunique())

    train_shapes = shapes[shapes['image_id'].isin(train_wkt['ImageId'].unique())]

    min_train_height = train_shapes['height'].min()
    min_train_width = train_shapes['width'].min()

    num_train = train_shapes.shape[0]

    image_rows = min_train_height
    image_cols = min_train_width

    num_channels = 3

    num_mask_channels = 10

    f = h5py.File(os.path.join(data_path, 'train_3.h5'), 'w')

    imgs = f.create_dataset('train', (num_train, num_channels, image_rows, image_cols), dtype=np.float16)
    imgs_mask = f.create_dataset('train_mask', (num_train, num_mask_channels, image_rows, image_cols), dtype=np.uint8)

    ids = []

    i = 0
    for image_id in tqdm(sorted(train_wkt['ImageId'].unique())):
        image = extra_functions.read_image_16(image_id)
        _, height, width = image.shape

        imgs[i] = image[:, :min_train_height, :min_train_width]
        imgs_mask[i] = extra_functions.generate_mask(image_id,
                                                     height,
                                                     width,
                                                     num_mask_channels=num_mask_channels,
                                                     train=train_wkt)[:, :min_train_height, :min_train_width]

        ids += [image_id]
        i += 1

    # fix from there: https://github.com/h5py/h5py/issues/441
    f['train_ids'] = np.array(ids).astype('|S9')

    f.close()

if __name__ == '__main__':
    cache_train_16()


text:
import torch
import os
import pandas as pd
import matchzoo as mz
import numpy as np
from sklearn.model_selection import train_test_split

print('MatchZoo version: ', mz.__version__)
os.environ['CUDA_LAUNCH_BLOCKING'] = "1"

import nltk
nltk.download('punkt')
# nltk.set_proxy('SYSTEM PROXY')

TYPE = 'classification'

classification_task = mz.tasks.Classification(num_classes=3)
classification_task.metrics = ['acc']
# print(classification_task.num_classes)
# print(classification_task.output_shape)
# print(classification_task.output_dtype)
# print(classification_task)

annot = pd.read_csv('../annotations/annotations_merged.csv')
# print(annot.dtypes)
annot.sort_values('PMCID').head(10)

"""Prompt file"""
prompt = pd.read_csv('../annotations/prompts_merged.csv')
# print(prompt.dtypes)
# prompt.sort_values('PMCID').head(10)

"""Concatenate prompt for input"""
prompt['Prompt'] = [str(triplet)[1:-1] for triplet in zip(prompt['Outcome'].values,
                                             prompt['Intervention'].values,
                                             prompt['Comparator'].values)]
# print(type(prompt['Prompt'][0]), type(prompt['Outcome'][0]))
# print(prompt['Intervention'][0])
prompt.head()

"""Process txt file for Articles input"""
TXT_PATH = '../annotations/txt_files/'
TAR_PATH = '../annotations/processed_txt_files/'
look_up = {}

if not os.path.exists(TAR_PATH):
    os.mkdir(TAR_PATH)
# else: os.removedirs(TAR_PATH)
for file in os.listdir(TXT_PATH):
    fname = file[3:]
    look_up[str(file[3:-4])] = fname
    with open(TXT_PATH+file, 'r', encoding='utf-8') as f, open(TAR_PATH+fname, 'w', encoding='utf-8') as t:
        for line in f.readlines():
            t.write(line.strip())

"""Articles Map"""
left_articles = []
for file in os.listdir(TAR_PATH):
    with open(TAR_PATH+file, 'r', encoding='utf-8') as f:
        left_articles.append([str(file[:-4]), f.readlines()[0]])
# print(left_articles[0:5])

"""Prompts Map"""
right_prompts = [list(pair) for pair in zip(annot['PromptID'].values, annot['Annotations'].values)]
# print(right_prompts[0:5])

"""Articles <-> Prompts Map"""
article_prompt_relations = [list(triplet) for triplet in zip(annot['PMCID'].values,
                                                             annot['PromptID'].values,
                                                             annot['Label Code'].values)]
# print(article_prompt_relations[0:5])

"""Create Data-pack"""
left = pd.DataFrame(left_articles, columns=['id_left', 'text_left'])
right = pd.DataFrame(right_prompts, columns=['id_right', 'text_right'])
relation = pd.DataFrame(article_prompt_relations, columns=['id_left', 'id_right', 'label'])
dp = mz.DataPack(
    relation = relation,
    left = left,
    right = right
)

""" Read article by name """
def read_article(id: str) -> str:
    with open(TAR_PATH+look_up[id], 'r', encoding='utf-8') as f:
        return f.readlines()[0]

text_left = [read_article(str(id)) for id in annot['PMCID']]
# print(text_left[0:5])

df = pd.DataFrame(data={
    'id_left': annot['PMCID'],
    'text_left': text_left,
    'id_right': annot['PromptID'],
    'text_right': prompt['Prompt'],
    'label': annot['Label Code']+1
})

""" Split data pack into train/valid """
train, valid = train_test_split(df, test_size=0.2)
train_pack = mz.pack(train, task=classification_task)
valid_pack = mz.pack(valid, task=classification_task)
train_pack.frame().head(10) # DataFrame

dp = mz.pack(df, task=classification_task)
# print(type(dp.frame))
frame_slice = dp.frame[0:5]
# print(list(frame_slice.columns))
full_frame = dp.frame()
assert len(full_frame) == len(dp)

""" MODEL & Train """
preprocessor = mz.models.DIIN.get_default_preprocessor(
    # truncated_length_left=30,
    # truncated_length_right=30,
    # ngram_size=1
)
train_processed = preprocessor.fit_transform(train_pack)
valid_processed = preprocessor.transform(valid_pack)

# print(preprocessor.context)

""" TODO : Initialize embedding with Biobert """
# glove_embedding = mz.datasets.embeddings.load_glove_embedding(dimension=100)
# term_index = preprocessor.context['vocab_unit'].state['term_index']
# embedding_matrix = glove_embedding.build_matrix(term_index)
# l2_norm = np.sqrt((embedding_matrix * embedding_matrix).sum(axis=1))
# embedding_matrix = embedding_matrix / l2_norm[:, np.newaxis]

ngram_callback = mz.dataloader.callbacks.Ngram(preprocessor, mode='index')

trainset = mz.dataloader.Dataset(
    data_pack=train_processed,
    mode='pair',
    batch_size=16,
    num_dup=1,
    num_neg=4,
    resample=True,
    callbacks=[ngram_callback]
)
validset = mz.dataloader.Dataset(
    data_pack=valid_processed,
    mode='point',
    batch_size=16,
    resample=False,
    callbacks=[ngram_callback]
)

padding_callback = mz.models.DIIN.get_default_padding_callback(
    # fixed_length_left=30,
    # fixed_length_right=30,
    # fixed_ngram_length=15
)

trainloader = mz.dataloader.DataLoader(
    dataset=trainset,
    stage='train',
    callback=padding_callback
)
validloader = mz.dataloader.DataLoader(
    dataset=validset,
    stage='dev',
    callback=padding_callback
)

model = mz.models.DIIN()
model.params['task'] = classification_task
# model.params['embedding'] = embedding_matrix
model.params['embedding_output_dim'] = 100
model.params['embedding_input_dim'] = preprocessor.context['embedding_input_dim']
model.params['char_embedding_input_dim'] = preprocessor.context['ngram_vocab_size']
model.params['dropout_rate'] = 0.2
model.guess_and_fill_missing_params()
model.build()

# print(model)
print('Trainable params: ', sum(p.numel() for p in model.parameters() if p.requires_grad))

optimizer = torch.optim.Adam(model.parameters())

trainer = mz.trainers.Trainer(
    model=model,
    optimizer=optimizer,
    trainloader=trainloader,
    validloader=validloader,
    epochs=10
)

trainer.run()

max_stars_repo_name: freesinger/evidence-inference

max_issues_repo_name: freesinger/evidence-inference

max_forks_repo_name: freesinger/evidence-inference


text:
from numpy.core.fromnumeric import size
import tensorflow as tf
# import tensorflow.compat.v1 as tf
import numpy as np

from layers import FCLayer, ReLULayer, ConvLayer, MaxPoolingLayer, ReshapeLayer

input_dim=1
channels=1
kernel_size=2
height=4
batch=3

inputs=np.random.randint(9,size=(batch,height,height,input_dim)).astype(np.float32)
# conv=tf.keras.layers.Conv2D(self.channels, (self.kernel_size, self.kernel_size), strides=(1, 1), input_shape=(self.height, self.width, 1), padding='same', activation='relu', kernel_initializer='uniform')
# offital_conv=tf.keras.layers.Conv2D(channels, (kernel_size, kernel_size), strides=(1, 1), input_shape=(height, height, input_dim), padding='same', activation='relu', kernel_initializer='uniform')

own_pool=(MaxPoolingLayer(2, 0))

# shape-(batch_size, filters, output_height, output_width)
height_o=int(height / kernel_size)
			# self.width_o=int(self.width / self.kernel_size)
delta=np.random.randint(9,size=(batch,channels,height_o,height_o)).astype(np.float32)
print('own')
res=own_pool.forward(inputs.transpose(0,3,1,2))
# for input in inputs:
print(res)

print(inputs.shape)
print(res.shape)
new_delta=own_pool.backward(delta)
# # print(res2.shape)
print(new_delta)

max_stars_repo_name: easilylazy/pattern-recognition

max_issues_repo_name: easilylazy/pattern-recognition

max_forks_repo_name: easilylazy/pattern-recognition

reason: import numpy,from numpy


text:
import hashlib
import json
import os
import pickle
import numpy as np
import iisignature
from scipy.optimize import brentq as brentq

# Adapted from the drone_identification repository in the DataSig GitHub

# This is how to define a decorator function in Python.
# See https://wiki.python.org/moin/PythonDecorators.
# We use this function to cache the results of calls to
# compute_signature().
def cache_result(function_to_cache, cache_directory='cached_signatures'):
    """
    Cache the result of calling function_to_cache().
    """
    if not os.path.isdir(cache_directory):
        os.mkdir(cache_directory)

    def _read_result(parameter_hash):
        cache_file = os.path.join(cache_directory, str(parameter_hash))
        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as file:
                return pickle.load(file)
        return None

    def _write_result(parameter_hash, result):
        cache_file = os.path.join(cache_directory, str(parameter_hash))
        with open(cache_file, 'wb') as file:
            pickle.dump(result, file)

    def wrapper_function(*args, **kwargs):
        # Aggregate all parameters passed to function_to_cache.
        # Note that args[0] will contain an instance of ExpectedSignatureCalculator(), which
        # we subsequently convert to a string. In this way, we cache results, while
        # considering values of constants and the truncation level.
        parameters = args + tuple(kwargs[k] for k in sorted(kwargs.keys()))
        # Map aggregated parameters to an MD5 hash.
        parameter_hash = hashlib.md5(json.dumps(parameters, sort_keys=True,
                                                default=str).encode('utf-8')).hexdigest()

        result = _read_result(parameter_hash)
        if result is None:
            # Call function_to_cache if no cached result is available
            result = function_to_cache(*args, **kwargs)
            _write_result(parameter_hash, result)

        return result

    return wrapper_function

class SignatureCalculator():
    """
    Class for computing the path signatures.
    """

    def __init__(self, dataset, dataset_type, truncation_level=5, add_time=False,
                 lead_lag_transformation=False, Harald_rescaling=-1):
        """
        Parameters
        ----------
        dataset : string
            Name of the dataset
        dataset_type: string
            Whether it is the train, validation or the test set
        truncation_level : int
            Path signature trucation level.
        add_time: bool
            Whether add time was applied
        lead_lag_transformation : bool
            Whether lead-lag was applied.
        """

        assert dataset_type in ['train', 'val', 'test'], "dataset_type should be 'train', 'val' or 'test'"

        self.dataset = dataset
        self.dataset_type = dataset_type
        self.truncation_level = truncation_level
        self.add_time = add_time
        self.lead_lag_transformation = lead_lag_transformation
        self.Harald_rescaling = Harald_rescaling

    @cache_result
    def compute_signature(self, data):
        if self.Harald_rescaling!=-1:
            data = scale_paths(X=data,M=self.Harald_rescaling,a=1,level_sig=self.truncation_level)
            return [iisignature.sig(data, self.truncation_level), data]
        return iisignature.sig(data, self.truncation_level)

    def __str__(self):
        """
        Convert object to string. Used in conjunction with cache_result().
        """
        return str((self.dataset,
                    self.dataset_type,
                    self.truncation_level,
                    self.add_time,
                    self.lead_lag_transformation,
                    self.Harald_rescaling))

''' MAIN FUNCTIONS USED FOR PATH RE-SCALING '''

def scale_paths(X,M,a,level_sig):
    '''
        This function computes a single scaling factor \theta_x (path-dependent) per path and rescales each path x
        , i.e. return x_new: t -> \theta_x*x_t, such that ||S_{<level_sig}(x_new)|| < M(1+1/a).

        Inputs:
            - X: an array (N,L,D) representing N paths where L is the length and D is the state-space dimension
            - (int) level_sig: the truncation level to compute the norm of S_{<level_sig}(x)
            - (int) M: the first parameter used to define the maximum norm allowed
            - (float) a: the second parameter used to define the maximum norm allowed
        Outputs:
            - X_new: the rescaled paths
    '''
    N = X.shape[0]
    D = X.shape[2]

    maxi = M*(1.+1./a)

    Sig = iisignature.sig(X, level_sig) # computes the signatures of the paths in X

    norms_levels = get_norms_level_sig(Sig, D, level_sig) # (N,n) each row contains the (squared) norm of each tensor in S(x_i) truncated at level level_sig=n

    norms = np.sum(norms_levels,axis=1) # (N,1) gets the (squared) norms of S(x) truncated at level level_sig

    psi = [psi_tilde(norms[i],M,a) for i in range(N)]  # computes an intermediary number which is used to find the scale

    thetas = [brentq(poly,0, 10000, args=(psi[i], norms_levels[i],level_sig)) for i in range(N)]# computes the scale

    return np.array(thetas)[:,None,None]*X

''' UTILS FUNCTIONS '''

def get_norms_level_sig(Sig, d, level_sig):
    '''
        This function computes the norm of each tensor in the truncated signature in input
        INPUTS:
            - (array) Sig: flat signatures (N,M)
            - (int) d: the original state-space dimension of the paths
            - (int) level_sig: the truncation level of the signatures
        OUTPUT:
            - (list) norms: a list containing the norms of each level of the truncated signature in input
    '''
    norms = np.ones((Sig.shape[0],level_sig+1))

    for k in range(1,level_sig+1):

        start = int(((1 - d ** k) / (1 - d)) - 1)
        end = int((1 - d ** (k + 1)) / (1 - d) - 1)

        norms[:,k] = np.sum(Sig[:,start:end]**2,axis=1) # (1,N)

    return norms # (N,n)

def psi_tilde(x, M, a):
    '''
        psi(\sqrt{x}) =  x if x<=M; M+M^{1+a}(M^{-a}-x^{-a})/a otherwise
        this returns psi(||S_M(x)||) with ||S_M(x)||^2 given in input
    '''
    if x <= M:
        return x
    else:
        return M + pow(M, 1. + a) * (pow(M, -a) - pow(x, -a)) / a

def poly(x,psi,coef,level_sig):
    return np.sum([coef[i]*x**(2*i) for i in range(level_sig+1)])-psi

reason: import numpy,from scipy


text:
"""
Generic functions to manipulate categorical features in pandas data frame.

This library include the functions:
    1. remove_categories(database, column_name, categories_to_drop):
        Remove all indexes in the database that contain the categories from the categories_to_drop.
        Before removing the indexes will print a message to the user with the number of indexes that
        will be remove and the percent of the database that will be lost.
        the user will input 'y'(yes) to pressed and 'n'(no) to cancel the action.
        If the user choose 'yes' the method will continue to drop the indexes and will
        plot the new database shape.

    2. fill_na_by_ratio(database, column_name):
        Fill all missing values in the given column by the ratio of the categories in the column.
        Because the ratio sum is not a perfect one - the extra missing values will
        be filled with the most common category in the column.

    3. combine_categories(database, column_name, category_name="other", threshold=0.01):
        Receives a threshold that is the minimum relative part of the category within the column.
        all categories that are less than this threshold will be combined under the same category
        under the name 'category_name'.
        the method will return a list with all the name of categories that were combined under 'category_name'
        with this list the user will be able to make the same action on the test set (assuming that the data
        was already splitted to train and test sets)

    4. categories_not_in_common(train, test, column_name):
        To avoid different shapes of train and test data sets after creating dummies, the user is able to
        check if one categories is missing in the data sets.
        It will check all categories name of the two data sets and returns the name of the categories not
        in common for every data set.
        The information from this method will be returned as list of tuples:
            [(exists only in the first data set),(exists only in the second data set)]
        will also print every list as a message to the user.

    5. category_ratio(database, columns_to_check=None, num_categories=5):
        Returns a data base with categories and their ratio of appearance in the column.
        The user can input a list of columns to check or the method will return all the columns categories ratio.
        The user can choose how many *top* categories will be returned.
        Categories with value under 5% or over 90% will be marked in red to raise a flag that the data
        is imbalanced.

        6. label_encoder_features(database, features_to_encode):
            Encode features in the giving database using LabelEncoder() from sklearn.
            Will preform encoding on all giving features and returns a dictionary with all the different
            encoders where the ket is the feature name and the value is the LabelEncoder instance fitted
            to that feature. Thus, the user is able to un-encode the feature when necessary.

        7.  OHE(database, features_list=None):
            Encode features in the giving database using get_dummies() from pandas.
            Will preform one hot encoding on all giving features or if non were given will preform on all non-numeric
            features in the database.
            Will check with the user that agree to add num of features
            Return the encoded database and the name of the original features that was encoded

Create by: Nir Barazida
Good luck!
"""

from NBprocessing.categorical._input_check_categorical import _InputCheckCategorical

from NBprocessing.categorical._general_functions_categorical import color_imbalanced
from NBprocessing.src import constance_object

import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder

class NBcategorical(object):
    """
    Generic functions to manipulate categorical features in pandas data frame.

    This library include the functions:
        1. remove_categories(database, column_name, categories_to_drop):
            Remove all indexes in the database that contain the categories from the categories_to_drop.
            Before removing the indexes will print a message to the user with the number of indexes that
            will be remove and the percent of the database that will be lost.
            the user will input 'y'(yes) to pressed and 'n'(no) to cancel the action.
            If the user choose 'yes' the method will continue to drop the indexes and will
            plot the new database shape.

        2. fill_na_by_ratio(database, column_name):
            Fill all missing values in the given column by the ratio of the categories in the column.
            Because the ratio sum is not a perfect one - the extra missing values will
            be filled with the most common category in the column.

        3. combine_categories(database, column_name, category_name="other", threshold=0.01):
            Receives a threshold that is the minimum relative part of the category within the column.
            all categories that are less than this threshold will be combined under the same category
            under the name 'category_name'.
            the method will return a list with all the name of categories that were combined under 'category_name'
            with this list the user will be able to make the same action on the test set (assuming that the data
            was already splitted to train and test sets)

        4. categories_not_in_common(train, test, column_name):
            To avoid different shapes of train and test data sets after creating dummies, the user is able to
            check if one categories is missing in the data sets.
            It will check all categories name of the two data sets and returns the name of the categories not
            in common for every data set.
            The information from this method will be returned as list of tuples:
                [(exists only in the first data set),(exists only in the second data set)]
            will also print every list as a message to the user.

        5. category_ratio(database, columns_to_check=None, num_categories=5):
            Returns a data base with categories and their ratio of appearance in the column.
            The user can input a list of columns to check or the method will return all the columns categories ratio.
            The user can choose how many *top* categories will be returned.
            Categories with value over 90% will be marked in red to raise a flag that the data
            is imbalanced.

        6. label_encoder_features(database, features_to_encode):
            Encode features in the giving database using LabelEncoder() from sklearn.
            Will preform encoding on all giving features and returns a dictionary with all the different
            encoders where the ket is the feature name and the value is the LabelEncoder instance fitted
            to that feature. Thus, the user is able to un-encode the feature when necessary.

        7.  OHE(database, features_list=None):
            Encode features in the giving database using get_dummies() from pandas.
            Will preform one hot encoding on all giving features or if non were given will preform on all non-numeric
            features in the database.
            Will check with the user that agree to add num of features
            Return the encoded database and the name of the original features that was encoded

    Create by: Nir Barazida
    Good luck!

    """

    @staticmethod
    @_InputCheckCategorical._remove_categories_checker
    def remove_categories(database, column_name, categories_to_drop):
        """
        General Information
        ----------
        Remove all indexes in the database that contain the categories from the categories_to_drop.
        Before removing the indexes will print a message to the user with the number of indexes that
        will be remove and the percent of the database that will be lost.
        the user will input 'y'(yes) to pressed and 'n'(no) to cancel the action.
        If the user choose 'yes' the method will continue to drop the indexes and will
        plot the new database shape.

        Parameters
        ----------
        :param database: pandas Data Frame
        the database must contain the column that was sent to the method 'column_name'

        :param column_name: string
        The name of the column where the values are and from which we will be parsing the categories.
        the column_name must be identical to the column name in the database.
        This column must be a categorical column.

        :param categories_to_drop: list or tuple of string int of float
        The name of the categories the user wish to drop.

        Returns
        -------
        None
        This method prints information to help the user decide rather to drop the categories or not.
        eventually returns None

        Raises
        ------
        ValueError : If input value not as mentioned above.
        """

        remove_df = database[database[column_name].isin(categories_to_drop)]

        user_input = input(
            constance_object.USER_INPUT.format(len(remove_df), round(len(remove_df) * 100 / len(database), 2)))

        while user_input != 'y' and user_input != 'n':
            user_input = input(
                constance_object.USER_INPUT.format(len(remove_df), round(len(remove_df) * 100 / len(database), 2)))

        if user_input == 'y':
            for value in categories_to_drop:
                database.drop(database.loc[database[column_name] == value].index, inplace=True)
            print(constance_object.DATABASE_SHAPE.format(database.shape))

    @staticmethod
    @_InputCheckCategorical._fill_na_by_ratio_checker
    def fill_na_by_ratio(database, column_name):
        """
        General Information
        ----------
        Fill all missing values in the given column by the ratio of the categories in the column.
        Because the ratio sum is not a perfect one - the extra missing values will
        be filled with the most common category in the column.

        Parameters
        ----------
        :param database: pandas Data Frame
        the database must contain the column that was sent to the method 'column_name'

        :param column_name:  string
        The name of the column where method will fill the missing values in.
        This column must be a categorical column

        Returns
        -------
        None
        fill the missing values in the column inplace.

        Raises
        ------
        ValueError : If input value not as mentioned above.
        """
        try:
            categories_names = list(database[column_name].value_counts(normalize=True, dropna=True).index)
            categories_ratio = list(database[column_name].value_counts(normalize=True, dropna=True).values)

            database[column_name] = database[column_name].fillna(
                pd.Series(np.random.choice(categories_names, p=categories_ratio, size=len(database))))

            # Fill the extra missing values
            database[column_name].fillna(database[column_name].value_counts().index[0], inplace=True)
        except NameError as e:
            print(e)

    @staticmethod
    @_InputCheckCategorical._combine_categories_checker
    def combine_categories(database, column_name, category_name="other", threshold=0.01):
        """
        General Information
        ----------
        Receives a threshold that is the minimum relative part of the category within the column.
        all categories that are less than this threshold will be combined under the same category
        under the name 'category_name'.
        the method will return a list with all the name of categories that were combined under 'category_name'
        with this list the user will be able to make the same action on the test set (assuming that the data
        was already splitted to train and test sets)

        Parameters
        ----------
        :param database: pandas Data Frame
        the database must contain the column that was sent to the method 'column_name'

        :param column_name: string
        The name of the column where method will fill the missing values in.
        This column must be a categorical column

        :param category_name: string
         The name of the new category of the combined categories

        :param threshold: float - 0 < threshold_percentage < 1
        Threshold that represent the ratio value the categories under it will be combined

        Returns
        -------
        the method will return a list with all the name of categories that were combined under 'category_name'
        with this list the user will be able to make the same action on the test set (assuming that the data
        was already splitted to train and test sets)

        for exemple: (LIST - the returned list from the function)
        X_test["column_name"].replace(LIST, category_name, inplace=True)

        Raises
        ------
        ValueError : If input value not as mentioned above.
        """

        values_to_combine = database[column_name].value_counts()[
            database[column_name].value_counts(normalize=True) < threshold].index
        database[column_name].replace(values_to_combine, category_name, inplace=True)
        return values_to_combine

    @staticmethod
    @_InputCheckCategorical._categories_not_in_common_checker
    def categories_not_in_common(train, test, column_name):
        """
        General Information
        ----------
        To avoid different shapes of train and test data sets after creating dummies, the user is able to
        check if one categories is missing in the data sets.
        It will check all categories name of the two data sets and returns the name of the categories not
        in common for every data set.
        The information from this method will be returned as list of tuples:
            [(exists only in the first data set),(exists only in the second data set)]
        will also print every list as a message to the user.

        Parameters
        ----------
        :param train: pandas Data Frame
        First data set to check columns names from

        :param test: pandas Data Frame
        Second data set to check columns names from

        :param column_name: string
        The name of the column to preform the check.
        This column must be a categorical column

        :return:
        The information from this method will be returned as list of tuples:
            [(exists only in the first data set),(exists only in the second data set)]

        Raises
        ------
        ValueError : If input value not as mentioned above.
        """

        in_test = set(test[column_name]) - set(train[column_name])
        in_train = set(train[column_name]) - set(test[column_name])
        print(constance_object.FIRST.format(in_test))
        print(constance_object.SECOND.format(in_train))

    @staticmethod
    @_InputCheckCategorical._category_ratio_checker
    def category_ratio(database, columns_to_check=None, num_categories=5):
        """
        General Information
        ----------
        Returns a data base with categories and their ratio of appearance in the column.
        The user can input a list of columns to check or the method will return all the columns categories ratio.
        The user can choose how many *top* categories will be returned.
        Categories with value over 90% will be marked in red to raise a flag that the data
        is imbalanced.

        Parameters
        ----------
        :param database: pandas Data Frame
        First data set to check columns names from

        :param columns_to_check: list or tuple of strings/int/float
        The column name that the user wishes to check the categories ratio in.
        All column names must be in the database.

        :param num_categories: int
        The number of top categories that will be plotted.
        If the number is higher than the number of categories than will insert '0.00'

        :return:
        The information from this method will be returned as list of tuples:
            [(exists only in the first data set),(exists only in the second data set)]

        Raises
        ------
        ValueError : If input value not as mentioned above.
        """

        category_ratio_df = pd.DataFrame()
        if not columns_to_check:
            columns_to_check = database.columns

        for column_name in columns_to_check:
            name = database[column_name].value_counts(normalize=True).index
            values = database[column_name].value_counts(normalize=True).values
            val_count_temp = [f"{category} : {round(frequent * 100, 2)}%" for category, frequent in zip(name, values)]
            if len(val_count_temp) < num_categories:
                val_count = ['0.00'] * num_categories
                val_count[:len(val_count_temp)] = val_count_temp
            else:
                val_count = val_count_temp[:num_categories]
            category_ratio_df[column_name] = val_count
        return category_ratio_df.transpose().style.applymap(color_imbalanced)

    # todo: debug
    @staticmethod
    @_InputCheckCategorical._label_encoder_features_checker
    def label_encoder_features(database, features_to_encode):
        """
        General Information
        ----------
        Encode features in the giving database using LabelEncoder() from sklearn.
        Will preform encoding on all giving features and returns a dictionary with all the different
        encoders where the ket is the feature name and the value is the LabelEncoder instance fitted
        to that feature. Thus, the user is able to un-encode the feature when necessary.

        Parameters
        ----------
        :param database: pandas Data Frame
        First data set to check columns names from

        :param features_to_encode: list or tuple of strings/int/float
        The column names that the user wishes to encode.
        All column names must be in the database and categorical/

        :return:
        returns a dictionary with all the different
        encoders where the ket is the feature name and the value is the LabelEncoder instance fitted
        to that feature. Thus, the user is able to un-encode the feature when necessary.

        Raises
        ------
        ValueError : If input value not as mentioned above.
        """

        label_encoder_dict = {}

        for feat in features_to_encode:
            le = LabelEncoder()

            # fit the feature that we want to encode and tresform to numeric categories
            database[feat] = le.fit_transform(database[feat])

            print(f"for feature {feat} the categories are:\n", database[feat].value_counts().index)

            label_encoder_dict[feat] = le

        return label_encoder_dict

    @staticmethod
    @_InputCheckCategorical._OHE_checker
    def OHE(database, features_list=None):
        """
        General Information
        ----------
        Encode features in the giving database using get_dummies() from pandas.
        Will preform one hot encoding on all giving features or if non were given will preform on all non-numeric
        features in the database.
        Will check with the user that agree to add num of features
        Return the encoded database and the name of the original features that was encoded

        Parameters
        ----------
        :param database: pandas Data Frame
        First data set to check columns names from

        :param features_list: list or tuple of strings/int/float
        The column names that the user wishes to encode.
        All column names must be in the database and categorical/

        :return:
        the encoded database and the name of the original features that was encoded

        Raises
        ------
        ValueError : If input value not as mentioned above.
        """
        total_cat = 0
        user_input = None
        # check if one of the columns is numeric
        if features_list and database[features_list]._get_numeric_data().columns.tolist():
            return print("features list contains numeric features please check again")
        # all the non-numeric column will be OHE
        elif not features_list:
            numeric_col = list(database._get_numeric_data().columns)
            features_list = [col for col in database.columns if col not in numeric_col]

        # Number of new features
        for feat in features_list: total_cat += database[feat].nunique()
        # Make sure that the user aware to the num of added features
        while user_input != 'n' and user_input != 'y':
            user_input = input(
                f"By OHE you will add {total_cat - len(features_list)} "
                f"features to the database. Do you wish to continue[y/n]:")
        if user_input == 'n':
            return database, features_list
        else:
            database = pd.get_dummies(database, columns=features_list)
            return database, features_list

max_stars_repo_name: nirbarazida/NBprocessing

max_issues_repo_name: nirbarazida/NBprocessing

max_forks_repo_name: nirbarazida/NBprocessing