text:
import itertools
import math
import json
from collections import Counter

from os.path import join, abspath, dirname, isfile

import pandas as pd
import numpy as np
from nltk import word_tokenize, sent_tokenize
from sklearn.utils import check_random_state
import torch
from torch.utils.data import TensorDataset, Dataset

DATASET_ROOT_PATH = join(dirname(abspath(__file__)), "..", "datasets")
TRAIN_FILE = "train.csv"
TEST_FILE = "test.csv"
CLASSES_FILE = "classes.txt"

TRAIN_TOKEN_FILE = "train_token.pkl"
TEST_TOKEN_FILE = "test_token.pkl"

STR_TO_REPLACE = [("\\n", " "), ("\\\"", "\""), ("<br />", " "), ("<", "<"), (">", ">"), ("\\", " "), ("/", " "), (".", ". ")]

class TextDatasetBuilder(object):

    def __init__(self, name, word_vectors="glove", vector_size=50, random_state=42, build_inv_vocab=False):
        self.name = name
        self.word_vectors = word_vectors
        self.vector_size = vector_size
        self.rng = check_random_state(random_state)
        self.build_inv_vocab = build_inv_vocab
        self._load_dataset()
        self.classes = self._load_classes()

    def _load_classes(self):
        classes_path = join(DATASET_ROOT_PATH, self.name, CLASSES_FILE)
        if isfile(classes_path):
            classes = [line.rstrip() for line in open(classes_path)]
        else:
            classes = sorted(list(np.unique(self.train_labels)))
        return classes

    def _load_dataset(self):
        train_token_path = join(DATASET_ROOT_PATH, self.name, TRAIN_TOKEN_FILE)
        test_token_path = join(DATASET_ROOT_PATH, self.name, TEST_TOKEN_FILE)

        if not(isfile(train_token_path) and isfile(test_token_path)):
            print("Tokenized dataset not found -> Launching Tokenization...")
            self._tokenize_source()

        train = pd.read_pickle(train_token_path)
        self.train_corpus = train['text'].values
        self.train_labels = train['label'].values

        test = pd.read_pickle(test_token_path)
        self.test_corpus = test['text'].values
        self.test_labels = test['label'].values

    def _tokenize_source(self):
        for source, token in [(TRAIN_FILE, TRAIN_TOKEN_FILE), (TEST_FILE, TEST_TOKEN_FILE)]:
            df = pd.read_csv(join(DATASET_ROOT_PATH, self.name, source), header=None)
            text_index = 1
            if self.name in ["yahoo_answers"]:
                df[4] = df[1].astype(str) + " " + df[2].astype(str) + " " + df[3].astype(str)
                text_index = 4
            if self.name in ["amazon_review_polarity", "amazon_review_full", "ag_news", "dbpedia", "sogou_news"]:
                df[3] = df[1].astype(str) + ". " + df[2].astype(str)
                text_index = 3

            corpus = df[text_index].values
            labels = df[0].values - 1
            corpus = [self.tokenize(self._clean_text(text)) for text in corpus]
            df = pd.DataFrame.from_items([('label', labels), ('text', corpus)])
            df.to_pickle(join(DATASET_ROOT_PATH, self.name, token))

    def _clean_text(self, text):
        for old, new in STR_TO_REPLACE:
            text = text.replace(old, new)
        return text

    def build_vocab(self, corpus, min_freq=1):
        self.vocab = sorted(Counter(itertools.chain(*list(corpus))).items())
        self.vocab = [word for word, freq in self.vocab if freq >= min_freq and word != 'unk']
        self.vocab = {token:(i+2) for i, token in enumerate(self.vocab)}
        self.vocab['PAD'] = 0
        self.vocab['unk'] = 1

        if self.build_inv_vocab:
            self.inv_vocab = {i: token for token, i in self.vocab.items()}

    def tokenize(self, text):
        return [token.lower() for sent in sent_tokenize(text) for token in word_tokenize(sent)]

    def get_max_text_len(self, corpus):
        return np.amax([len(text) for text in corpus])

    def get_texts_len_distrib(self, corpus):
        return [len(text) for text in corpus]

    def vectorize(self, corpus, max_len=None):
        vcorpus = []
        for text in corpus:
            vtext = np.array([self.vocab[word] if word in self.vocab else self.vocab['unk'] for word in text])
            if max_len is not None:
                vtext = vtext[:max_len]
            vcorpus.append(vtext)
        return vcorpus

    def _load_glove_vectors(self):
        embeddings = {}
        path = join(DATASET_ROOT_PATH, "glove", "glove.6B." + str(self.vector_size) + "d.txt")
        with open(path, 'r', encoding='utf8') as embeddings_file:
            for line in embeddings_file:
                if len(line) > 50:
                    fields = line.strip().split(' ')
                    word = fields[0]
                    vector = np.asarray(fields[1:], dtype='float32')
                    embeddings[word] = vector
        return embeddings

    def _load_custom_vectors(self):
        embeddings = {}
        path = join(DATASET_ROOT_PATH, "custom_embeddings")
        vocab_to_int = json.load(open(join(path, f"{self.name}_vocab_to_int.json")))
        embedding_vectors = np.load(join(path, f"{self.name}_{self.vector_size}_embeddings.npy"))
        for word in vocab_to_int.keys():
            embeddings[word] = embedding_vectors[vocab_to_int[word]]
        return embeddings

    def build_embeddings(self):
        embeddings = np.random.rand(len(self.vocab), self.vector_size)

        if self.word_vectors != 'random':

            if self.word_vectors == 'glove':
                word_vectors = self._load_glove_vectors()
            elif self.word_vectors == 'custom':
                word_vectors = self._load_custom_vectors()

            for word, index in self.vocab.items():
                if word in word_vectors:
                    embeddings[index] = word_vectors[word]

        return embeddings
    def n_vocab_in_word_vectors(self):
        word_vectors = {}
        if self.word_vectors == 'glove':
            word_vectors = self._load_glove_vectors()
        elif self.word_vectors == 'custom':
            word_vectors = self._load_custom_vectors()
        count = 0
        for word in self.vocab.keys():
            if word in word_vectors:
                count += 1
        return count

    def pre_process(self, min_freq=1, max_len=None):
        self.build_vocab(self.train_corpus, min_freq)
        self.train_corpus = self.vectorize(self.train_corpus, max_len)
        self.test_corpus = self.vectorize(self.test_corpus, max_len)

    def _split_train_valid(self, ratio):
        n_train = int(math.ceil(ratio*self.train_labels.shape[0]))
        data = list(zip(self.train_corpus, self.train_labels))
        self.rng.shuffle(data)
        self.train_corpus, self.train_labels = list(zip(*data))
        self.train_corpus = list(self.train_corpus)
        self.train_labels = np.array(self.train_labels)
        return self.train_corpus[:n_train],\
            self.train_labels[:n_train],\
            self.train_corpus[n_train:],\
            self.train_labels[n_train:]

    def _build_tensor_dataset(self, corpus, labels):
        return TensorDataset(torch.from_numpy(corpus).long(), torch.from_numpy(labels))

    def get_train_valid_test(self, ratio=0.8):
        train_corpus, train_labels, valid_corpus, valid_labels = self._split_train_valid(ratio)
        train = TextDataset(train_corpus, train_labels)
        valid = TextDataset(valid_corpus, valid_labels)
        test = TextDataset(self.test_corpus, self.test_labels)
        return train, valid, test

class TextDataset(Dataset):

    def __init__(self, corpus, labels,):
        self.corpus = corpus
        self.labels = labels

    def __len__(self):
        return self.labels.shape[0]

    def __getitem__(self, index):
        return (self.corpus[index], self.labels[index])

def collate_padding(samples):
    batch_texts, labels = list(zip(*samples))
    text_lens = [text.shape[0] for text in batch_texts]
    text_tensor = torch.zeros(max(text_lens), len(samples)).long()
    text_mask = torch.zeros(len(samples), max(text_lens)).bool()
    for i, (text, text_len) in enumerate(zip(batch_texts, text_lens)):
        text_tensor[:text_len, i] = torch.LongTensor(text)
        text_mask[i, text_len:] = 1
    labels = torch.LongTensor(np.array(labels))
    return (text_tensor, text_mask), labels

max_stars_repo_name: gletarte/sanet

max_issues_repo_name: gletarte/sanet

max_forks_repo_name: gletarte/sanet


text:
import numpy as np
np.random.seed(42)

from nn.layers import Layer

class ReLU(Layer):
    """ Simply applies elementwise rectified linear unit to all inputs. """

    def __init__(self):
        self.type = 'activation'

    def forward(self, inputs):
        """ Forward pass of the ReLU non-linearity.

        Args:
            inputs (numpy.ndarray): the linear transformation from dense layer.
        """
        return np.maximum(0, inputs)

    def backward(self, inputs, gradients, **kwargs):
        """ Backward pass of the ReLU non-linearity.
        Computes gradient of loss w.r.t. ReLU input.

        Args:
            inputs (numpy.ndarray): the inputs to the ReLU layer to compute the gradients.
            gradients (numpy.ndarray): the gradients w.r.t. loss propagated back from following layers.
        Returns:
            numpy.ndarray: the gradient of loss w.r.t. ReLU inputs.
        """
        grad_relu = inputs > 0
        return gradients * grad_relu

ext: py

reason: import numpy


text:
import sys
import os
import shutil
import time
import logging
import argparse
#import boto3
import torch
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.nn.functional as F
from torch import optim
from torch.optim import lr_scheduler
from torch.utils.data import DataLoader, ConcatDataset
import torchvision.transforms as transforms
import numpy as np

from LASC18.data import DatasetGenerator, LocatorDataGenerator
from LASC18.utils import *
from LASC18.model import VNet

#os.environ["AWS_DEFAULT_REGION"] = 'eu-wes'
#os.environ["AWS_ACCESS_KEY_ID"] = 'AKIA4FKPSDX'
#os.environ["AWS_SECRET_ACCESS_KEY"] = 'xzi+z0Ct+YfATLsbtgy+xRII/sm'

#s3 = boto3.resource(service_name='s3')

#s3_bucket_name = 'lasc18'

logger = get_logger('VNetAttnHybrid_R4_D4-2')
#logger = get_logger('VNetAttnDice_R4_D4-2')

def weights_init(net):
    classname = net.__class__.__name__
    if classname.find('Conv3d') != -1:
        nn.init.xavier_normal_(net.weight)
        nn.init.zeros_(net.bias)
    elif classname.find('PRelu') != -1:
        nn.init.xavier_normal_(net.weight)

if __name__ == '__main__':

        parser = argparse.ArgumentParser()

        #parser.add_argument('--root_dir', dest='root_dir', default='/home/ubuntu/LASC18', type=str,
        #                  help='path to the LASC18 dataset')
        parser.add_argument('--root_dir', dest='root_dir', default='/home/xyz/LASC18', type=str,
                          help='path to the LASC18 dataset')

        #parser.add_argument('--s3_bucket_name', dest='s3_bucket_name', default='lasc18', type=str,
        #                  help='AWS S3 bucket to use for checkpoints')

        parser.add_argument('--max_epochs', dest='max_epochs', default=201, type=int,
                          help='number of epochs')

        parser.add_argument('--batch_size', dest='batch_size', default=3, type=int,
                          help='batch size for the segmentation model')

        parser.add_argument('--locator_batch_size', dest='locator_batch_size', default=5, type=int,
                          help='batch size for the locator model')

        parser.add_argument('--scale_factor', dest='scale_factor', default=(0.5, 0.25, 0.25), type=tuple,
                          help='scale down factor(D,H,W) for locator model training')

        parser.add_argument('--padding', dest='padding', default=(35,35,35), type=tuple,
                          help='padding along each axis for segmentation model inputs')

        parser.add_argument('--locator_learning_rate', dest='locator_lr', default=0.0001, type=float,
                          help='optimizer learning rate for the locator model')

        parser.add_argument('--learning_rate', dest='lr', default=0.00015, type=float,
                          help='optimizer learning rate for the segmentation model')

        parser.add_argument('--loss_criterion', dest='loss_criterion', default='Dice', type=str,
                          help='loss function to be used for the segmentation model')

        parser.add_argument('--dir_checkpoints', dest='dir_checkpoints', default='/home/xyz/LASC18/Checkpoints',
                          type=str, help='file path to save segmentation state checkpoints')

        parser.add_argument('--dir_locator_checkpoints', dest='dir_locator_checkpoints', default='/home/xyz/LASC18/Checkpoints/Locate',
                          type=str, help='file path to save locator state checkpoints')

        parser.add_argument('--best_checkpoints', dest='best_checkpoints', default='/home/xyz/LASC18/Best_Checkpoints',
                          type=str, help='file path to save best performing segmentation state checkpoints')

        parser.add_argument('--best_locator_checkpoints', dest='best_locator_checkpoints', default='/home/xyz/LASC18/Best_Checkpoints/Locate',
                          type=str, help='file path to save best performing locator state checkpoints')

        parser.add_argument('--save_after_epochs', dest='save_after_epochs', default=100, type=int,
                          help='number of epochs after which state is saved by default')

        parser.add_argument('--validate_after_epochs', dest='validate_after_epochs', default=1, type=int,
                          help='number of epochs after which validation occurs')

        parser.add_argument('--resume', dest='resume', default=None, type=str,
                          help='file path of the stored checkpoint state to resume segmentation training')

        parser.add_argument('--locator_resume', dest='locator_resume', default=None, type=str,
                          help='file path of the stored locator checkpoint state to resume locator training')

        parser.add_argument('--best_locator', dest='best_locator', default='/home/xyz/LASC18/Checkpoints/Locate/24_10 [23:34:57].pt', type=str,
                          help='file path of the best locator checkpoint state to use before segmentation')

        parser.add_argument('--seed', dest='seed', default=123, type=int,
                          help='seed for RNG')

        parser.add_argument('--gpu', action='store_true', dest='gpu', default=True,
                          help='use cuda')

        parser.add_argument('--num_layers', dest='num_layers', default=1, type=int,
                          help='Number of convolution layers in addition to default layers at each level for both models')

        parser.add_argument('--attention_module', dest='attention_module', default=False,
                          help='Use attention mechanism for Segmentation model')

        parser.add_argument('--dilation', dest='dilation', default=1, type=int,
                          help='Dilation in convolution layers for Segmentation model')

        parser.add_argument('--patience', dest='patience', default=7, type=int,
                          help='LR Scheduler patience')

        parser.add_argument('--reduce', dest='reduce', default=0.8, type=float,
                          help='LRScheduler learning_rate reduction factor ')

        options = parser.parse_args()

        #set RNG for both CPU and CUDA
        torch.manual_seed(options.seed)

        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False

        root_dir = options.root_dir
        #s3_bucket_name = options.s3_bucket_name
        max_epochs = options.max_epochs
        start_epoch = 0
        locator_batch_size = options.locator_batch_size
        locator_lr_reduce_factor = 0.85
        locator_scheduler_patience = 15
        locator_loss_function = Dice()
        locator_training_error = {'dice_loss': ([],[])}
        locator_validation_error = {'dice_loss': ([],[])}

        best_locator_validation_score = 0
        dir_locator_checkpoints = options.dir_locator_checkpoints
        best_locator_checkpoints = options.best_locator_checkpoints
        use_cuda = torch.cuda.is_available() and options.gpu == True
        device = torch.device("cuda" if use_cuda else "cpu")

        if not os.path.exists(dir_locator_checkpoints):
            os.mkdir(dir_locator_checkpoints)

        if not os.path.exists(best_locator_checkpoints):
            os.mkdir(best_locator_checkpoints)

        logger.info(f'''Provided locator checkpoint path: '{dir_locator_checkpoints}'
                                    and best locator checkpoint path: '{best_locator_checkpoints}'
                        ''')

        net_locator = VNet(num_layers = 1, dilation = 1)
        logger.info(f'Initialised locator model.')

        locator_optimizer = optim.Adam(net_locator.parameters(), lr = options.locator_lr)

        locator_scheduler = lr_scheduler.ReduceLROnPlateau(locator_optimizer, mode = 'min', factor = locator_lr_reduce_factor,
                                                            patience = locator_scheduler_patience, verbose = True)

        if options.best_locator is not None:
            assert os.path.isfile(options.best_locator), "Best locator load path provided doesn't exist!"
            best_checkpoint = torch.load(options.best_locator, map_location = 'cpu')

            net_locator.load_state_dict(best_checkpoint['locator_model_state'])

            logger.info(f'Loaded best locator model weights.')

        elif options.best_locator is None and options.locator_resume is not None:
            #s3.Bucket(s3_bucket_name).download_file(Key = '/s3_checkpoints/' + options.resume, Filename = options.resume)
            assert os.path.isfile(options.locator_resume), "Locator resume file path provided doesn't exist!"
            checkpoint = torch.load(options.locator_resume, map_location = 'cpu')

            logger.info(f"Loading checkpoint locator model state from resume path: '~/{options.locator_resume}'")

            start_epoch = int(checkpoint['epoch']) + 1
            max_epochs = int(checkpoint['max_epochs'])
            locator_optimizer.load_state_dict(checkpoint['locator_optimizer_state'])
            net_locator.load_state_dict(checkpoint['locator_model_state'])
            best_locator_validation_score = float(checkpoint['best_locator_validation_score'])
            locator_training_error = checkpoint['locator_training_error']
            locator_validation_error =  checkpoint['locator_validation_error']

        elif options.best_locator is None and options.locator_resume is None:
            net_locator.apply(weights_init)

            logger.info(f'Initialised model weights.')

        logger.info(f'Number of Trainable parameters for locator model: {number_of_parameters(net_locator)}')

        if use_cuda:
            net_locator.to(device)
            logger.info(f'Training locator with {device} using {torch.cuda.device_count()} GPUs.')

        if options.best_locator is None:
            locator_train_set = LocatorDataGenerator(root_dir = root_dir, scale_factor = options.scale_factor, mode = 'train')
            locator_validation_set = LocatorDataGenerator(root_dir = root_dir, scale_factor = options.scale_factor, mode = 'validate')

            logger.info(f'''Created locator dataset generator objects.
                        ''')

            locator_train_set_loader = DataLoader(locator_train_set, batch_size = locator_batch_size, shuffle = True, num_workers = 4)
            locator_validation_set_loader = DataLoader(locator_validation_set, batch_size = locator_batch_size, shuffle = True, num_workers = 4)

            logger.info(f'''Loaded locator training and validation datasets from '{root_dir}'
                                      Length of locator train set: {len(locator_train_set)} and validation set: {len(locator_validation_set)}
                                      Batch Size: {locator_batch_size}
                                      -----------------------------------------------------------------------------
                                      Beginning locator model training from epoch: {start_epoch} / {max_epochs - 1}
                                      Best locator validation score: {best_locator_validation_score}
                                      Adam optimiser with lr: {'{:.7f}'.format(locator_optimizer.param_groups[0]['lr'])}
                                      Scheduler ReduceLROnPlateau with mode: 'min', factor: {lr_reduce_factor}, patience: {scheduler_patience}
                                      Loss Criterion: 'Dice'
                                      -----------------------------------------------------------------------------
                        ''')

            for epoch in range(start_epoch, max_epochs):
                net_locator.train()
                locator_training_score = {'dice_loss': torch.zeros(1, requires_grad = False, dtype = torch.float32)}
                locator_validation_score = {'dice_loss': torch.zeros(1, requires_grad = False, dtype = torch.float32)}
                logger.info(f"""-------Locator training Epoch: [{epoch} / {max_epochs - 1}]-------""")
                for iteration, data in enumerate(locator_train_set_loader):

                    raw_image, label = data
                    raw_image = raw_image.to(device)
                    label = label.to(device)

                    output = net_locator(raw_image)

                    dice_loss = locator_loss_function(output, label)

                    locator_optimizer.zero_grad()
                    dice_loss.backward()
                    locator_optimizer.step()

                    locator_training_score['dice_loss'] = torch.cat((locator_training_score['dice_loss'], dice_loss.detach())) if iteration > 0 \
                                                                     else dice_loss.detach()

                locator_train_dice_error = torch.mean(locator_training_score['dice_loss'])
                locator_training_error['dice_loss'][0].append(locator_train_dice_error)
                locator_training_error['dice_loss'][1].append(epoch)

                logger.info(f"Locator training dice error for epoch {epoch} / {max_epochs - 1}:  {locator_train_dice_error}")

                if epoch % options.validate_after_epochs == 0:
                    net_locator.eval()
                    logger.info(f"""-------Performing Validation for locator--------""")

                    with torch.no_grad():
                        for iter, val_data in enumerate(locator_validation_set_loader):

                            val_raw_image, val_label = val_data
                            val_label = val_label.to(device)
                            val_raw_image = val_raw_image.to(device)

                            val_output = net_locator(val_raw_image)

                            dice_loss = locator_loss_function(val_output, val_label)

                            locator_validation_score['dice_loss'] = torch.cat((locator_validation_score['dice_loss'].cuda(), dice_loss.detach())) if iteration > 0 \
                                                                                else dice_loss.detach()

                    locator_validation_dice_error = torch.mean(locator_validation_score['dice_loss'])
                    locator_validation_error['dice_loss'][0].append(locator_validation_dice_error)
                    locator_validation_error['dice_loss'][1].append(epoch)

                    logger.info(f"""Val. Dice for Epoch {epoch}:  {locator_validation_dice_error}
                                """)

                    locator_scheduler.step(locator_validation_dice_error)

                best_locator_model = True if best_locator_validation_score < (1 - locator_validation_dice_error) else False

                if best_locator_model:
                    best_locator_validation_score = (1 - locator_validation_dice_error)

                if best_locator_model or epoch % options.save_after_epochs == 0:
                    if isinstance(net_locator, nn.DataParallel):
                        model_state = net_locator.module.state_dict()
                    else:
                        model_state = net_locator.state_dict()

                    locator_state = {'epoch': epoch,
                                     'max_epochs': max_epochs,
                                     'locator_optimizer_state': locator_optimizer.state_dict(),
                                     'locator_model_state': model_state,
                                     'best_validation_score': best_locator_validation_score,
                                     'locator_training_error': locator_training_error,
                                     'locator_validation_error': locator_validation_error,
                                     'scale_factor': options.scale_factor
                                    }

                    t = time.strftime("%d_%m [%H:%M:%S]", time.localtime())
                    checkpoint_path = os.path.join(dir_locator_checkpoints, t + '.pt')
                    torch.save(locator_state, checkpoint_path)

                    logger.info(f"""Saving locator model state to '{checkpoint_path}'
                                        Locator Training error: {locator_train_dice_error}
                                        Locator Validation error: {locator_validation_dice_error}
                                        Optimizer Learning Rate: {'{:.10f}'.format(locator_optimizer.param_groups[0]['lr'])}
                                        Is Best Locator model: {best_locator_model}
                                """)

                    if best_locator_model:
                        best_checkpoint_path = os.path.join(best_locator_checkpoints, t + '.pt')
                        shutil.copyfile(checkpoint_path, best_checkpoint_path)

        start_epoch = 0
        batch_size = options.batch_size
        lr_reduce_factor = options.reduce
        scheduler_patience = options.patience
        training_error = {'dice_loss': ([],[]), 'focal_loss': ([],[]), 'hybrid_loss': ([],[])}
        validation_error = {'dice_loss': ([],[]), 'focal_loss': ([],[]), 'hybrid_loss': ([],[])}
        best_validation_score = 0
        dir_checkpoints = options.dir_checkpoints
        best_checkpoints = options.best_checkpoints

        loss_criterion = options.loss_criterion
        alpha = 1

        assert options.loss_criterion in ['FocalLoss', 'Dice', 'HybridLoss'], 'The specified loss function is not supported!'
        if options.loss_criterion == 'FocalLoss':
            loss_function = FocalLoss()
        elif options.loss_criterion == 'Dice':
            loss_function = Dice()
        elif options.loss_criterion == 'HybridLoss':
            loss_function = HybridLoss()

        if not os.path.exists(dir_checkpoints):
            os.mkdir(dir_checkpoints)

        if not os.path.exists(best_checkpoints):
            os.mkdir(best_checkpoints)

        logger.info(f"""Provided checkpoint path: '{dir_checkpoints}'
                                       and best checkpoint path: '{best_checkpoints}""")

        net = VNet(num_layers = options.num_layers, dilation = options.dilation, attention_module = options.attention_module)
        logger.info(f'Initialised segmentation model.')

        optimizer = optim.Adam(net.parameters(), lr = options.lr)

        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode = 'min', factor = lr_reduce_factor, patience = scheduler_patience, verbose = True)

        if options.resume is not None:
                assert os.path.isfile(options.resume), "Resume file name provided for segmentation model doesn't exist!"
                checkpoint = torch.load(options.resume, map_location = 'cpu')

                start_epoch = int(checkpoint['epoch']) + 1
                max_epochs = int(checkpoint['max_epochs'])
                optimizer.load_state_dict(checkpoint['optimizer_state'])
                net.load_state_dict(checkpoint['model_state'])
                alpha = int(checkpoint['alpha'])
                best_validation_score = float(checkpoint['best_validation_score'])
                training_error = checkpoint['training_error']
                validation_error =  checkpoint['validation_error']

                logger.info(f"Checkpoint segmentation model state loaded from resume path: '~/{options.resume}'")
        else:
            net.apply(weights_init)
            logger.info(f'Initialised segmentation model weights.')

        logger.info(f'Number of Trainable parameters for segmentation model: {number_of_parameters(net)}')

        if use_cuda:
            net.to(device)
            logger.info(f'Training with {device} using {torch.cuda.device_count()} GPUs.')

        train_set_builder = LocatorDataGenerator(root_dir = root_dir, mode = 'paths_train', scale_factor = options.scale_factor)
        validation_set_builder = LocatorDataGenerator(root_dir = root_dir, mode = 'paths_validate', scale_factor = options.scale_factor)
        train_builder_inputs = []
        validation_builder_inputs = []

        net_locator.eval()
        with torch.no_grad():
            for idx in range(len(train_set_builder)):
                train_raw_data, train_label_data = train_set_builder[idx]
                raw_file_name, raw_image = train_raw_data
                raw_image = torch.unsqueeze(raw_image, dim = 0).to(device)

                train_output = net_locator(raw_image)

                train_builder_inputs.append(((raw_file_name, train_output), (train_label_data)))

            for idx in range(len(validation_set_builder)):
                validate_raw_data, validate_label_data = validation_set_builder[idx]
                raw_file_name, raw_image = validate_raw_data
                raw_image = torch.unsqueeze(raw_image, dim = 0).to(device)

                validate_output = net_locator(raw_image)

                validation_builder_inputs.append(((raw_file_name, validate_output), (validate_label_data)))

        train_set1 = DatasetGenerator(mode = 'train', inputs = train_builder_inputs, pad = options.padding,
                                     scale_factor = options.scale_factor, loss_criterion = options.loss_criterion)

        dataset_mean = train_set1.mean
        dataset_std = train_set1.std
        patch_size = train_set1.patch_size

        # Resuming training from checkpoint will affect reproducibility for random rotate and translate sets
        train_set_rotate = DatasetGenerator(mode = 'train', inputs = train_builder_inputs, pad = options.padding,
                                            scale_factor = options.scale_factor, loss_criterion = options.loss_criterion, seed = options.seed,
                                            transform = 'random_rotate', mean = dataset_mean, std = dataset_std, patch_dims = patch_size)

        train_set_translate = DatasetGenerator(mode = 'train', inputs = train_builder_inputs, pad = options.padding,
                                                scale_factor = options.scale_factor, loss_criterion = options.loss_criterion, seed = options.seed,
                                                transform = 'random_translate', mean = dataset_mean, std = dataset_std, patch_dims = patch_size)

        train_set_translate2 = DatasetGenerator(mode = 'train', inputs = train_builder_inputs, pad = options.padding,
                                                scale_factor = options.scale_factor, loss_criterion = options.loss_criterion, seed = options.seed*2,
                                                transform = 'random_translate', mean = dataset_mean, std = dataset_std, patch_dims = patch_size)

        train_set = ConcatDataset([train_set1, train_set_rotate, train_set_translate, train_set_translate2])

        logger.info(f"""Created segmentation train dataset generator objects.
                                        Dataset mean: {dataset_mean}
                                        Dataset std: {dataset_std}
                                        Patch size: {patch_size}""")

        validation_set =  DatasetGenerator(mode = 'validate', inputs = validation_builder_inputs, pad = options.padding,
                                             scale_factor = options.scale_factor, loss_criterion = options.loss_criterion,
                                             mean = dataset_mean, std = dataset_std, patch_dims = patch_size)

        logger.info(f'Created segmentation validation dataset generator object.')

        train_set_loader = DataLoader(train_set, batch_size = batch_size, shuffle = True, num_workers = 8)

        validation_set_loader = DataLoader(validation_set, batch_size = batch_size, shuffle = True, num_workers = 8)

        logger.info(f"""Length of train set: {len(train_set)} and validation set: {len(validation_set)}
                                      Batch Size: {batch_size}
                                      -----------------------------------------------------------------------------
                                      Beginning model training from epoch: {start_epoch} / {max_epochs - 1}
                                      Best validation score: {best_validation_score}
                                      Adam optimiser with lr: {'{:.7f}'.format(optimizer.param_groups[0]['lr'])}
                                      Scheduler ReduceLROnPlateau with mode: 'min', factor: {lr_reduce_factor}, patience: {scheduler_patience}
                                      Loss Criterion: '{options.loss_criterion}'
                                      -----------------------------------------------------------------------------
                        """)

        for epoch in range(start_epoch, max_epochs):
            net.train()
            a = {'surface_avg': torch.zeros(1, requires_grad = False, dtype = torch.float32)}
            training_score = {'dice_loss': torch.zeros(1, requires_grad = False, dtype = torch.float32),
                              'focal_loss': torch.zeros(1, requires_grad = False, dtype = torch.float32),
                              'hybrid_loss':torch.zeros(1, requires_grad = False, dtype = torch.float32)}

            validation_score = {'dice_loss': torch.zeros(1, requires_grad = False, dtype = torch.float32),
                                'focal_loss': torch.zeros(1, requires_grad = False, dtype = torch.float32),
                                'hybrid_loss':torch.zeros(1, requires_grad = False, dtype = torch.float32)}

            logger.info(f"""-------Segmentation training Epoch: [{epoch} / {max_epochs - 1}]-------""")

            for iteration, data in enumerate(train_set_loader):
                raw_image_patches, label_patches, dist_maps = data
                raw_image_patches = raw_image_patches.to(device)
                label_patches = label_patches.to(device)

                output_patches = net(raw_image_patches)

                if loss_criterion == 'HybridLoss':
                    dice_loss, loss = loss_function(output_patches, label_patches, dist_maps.to(device), alpha)
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()
                    training_score['hybrid_loss'] = torch.cat((training_score['hybrid_loss'], loss.detach())) if iteration > 0 else loss.detach()

                elif loss_criterion == 'FocalLoss':
                    dice_loss, loss = loss_function(output_patches, label_patches)
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()
                    training_score['focal_loss'] = torch.cat((training_score['focal_loss'], loss.detach())) if iteration > 0 else loss.detach()

                else:
                    dice_loss = loss_function(output_patches, label_patches)
                    optimizer.zero_grad()
                    dice_loss.backward()
                    optimizer.step()

                training_score['dice_loss'] = torch.cat((training_score['dice_loss'], dice_loss.detach())) if iteration > 0 else dice_loss.detach()

            train_dice_error = torch.mean(training_score['dice_loss'])
            train_focal_error = torch.mean(training_score['focal_loss'])
            train_hybrid_error = torch.mean(training_score['hybrid_loss'])

            training_error['dice_loss'][0].append(train_dice_error)
            training_error['focal_loss'][0].append(train_focal_error)
            training_error['hybrid_loss'][0].append(train_hybrid_error)

            for loss_name in training_error.keys(): training_error[loss_name][1].append(epoch)

            logger.info(f"""Training dice error for epoch {epoch} / {max_epochs - 1}:  {train_dice_error}""")

            net.eval()
            logger.info(f"""-------Performing Validation--------""")##

            with torch.no_grad():#
                for iteration, val_data in enumerate(validation_set_loader):
                    val_raw_image_patches, val_label_patches, val_dist_maps = val_data
                    val_label_patches = val_label_patches.to(device)
                    val_raw_image_patches = val_raw_image_patches.to(device)

                    val_output_patches = net(val_raw_image_patches)

                    if loss_criterion == 'HybridLoss':
                       dice_loss, loss = loss_function(val_output_patches, val_label_patches, val_dist_maps.to(device), alpha)
                       validation_score['hybrid_loss'] = torch.cat((validation_score['hybrid_loss'], loss.detach())) if iteration > 0 else loss.detach()

                    elif loss_criterion == 'FocalLoss':
                         dice_loss, loss = loss_function(val_output_patches, val_label_patches)
                         validation_score['focal_loss'] = torch.cat((validation_score['focal_loss'], loss.detach())) if iteration > 0 else loss.detach()

                    else:
                        dice_loss = loss_function(val_output_patches, val_label_patches)

                    validation_score['dice_loss'] = torch.cat((validation_score['dice_loss'], dice_loss.detach())) if iteration > 0 else dice_loss.detach()####

                validation_hybrid_error = torch.mean(validation_score['hybrid_loss'])
                validation_focal_error = torch.mean(validation_score['focal_loss'])
                validation_dice_error = torch.mean(validation_score['dice_loss'])

                logger.info(f'''Validation dice- epoch {epoch}:  {validation_dice_error}
                            ''')

                scheduler.step(validation_dice_error)

                validation_error['dice_loss'][0].append(validation_dice_error)
                validation_error['focal_loss'][0].append(validation_focal_error)
                validation_error['hybrid_loss'][0].append(validation_hybrid_error)

                for loss_name in validation_error.keys(): validation_error[loss_name][1].append(epoch)

            best_model = True if best_validation_score < (1 - validation_dice_error) else False

            if best_model:
                best_validation_score = (1 - validation_dice_error)

            if best_model or epoch % options.save_after_epochs == 0:

                if isinstance(net, nn.DataParallel):
                    model_state = net.module.state_dict()
                else:
                    model_state = net.state_dict()

                state = {'epoch': epoch,
                         'max_epochs': max_epochs,
                         'optimizer_state': optimizer.state_dict(),
                         'model_state': model_state,
                         'alpha': alpha,
                         'best_validation_score': best_validation_score,
                         'training_error': training_error,
                         'validation_error': validation_error,
                         'patch_size': patch_size,
                         'scale_factor': options.scale_factor,
                         'mean': dataset_mean,
                         'std': dataset_std
                        }

                t = time.strftime("%d_%m [%H:%M:%S]", time.localtime())
                checkpoint_path = os.path.join(dir_checkpoints, t + '.pt')
                torch.save(state, checkpoint_path)

                logger.info(f'''Saving model state to '{checkpoint_path}'
                                    Training error: {train_dice_error}
                                    Validation error: {validation_dice_error}
                                    Optimizer Learning Rate: {'{:.10f}'.format(optimizer.param_groups[0]['lr'])}
                                    Is Best model: {best_model}
                            ''')

                if best_model:
                    best_checkpoint_path = os.path.join(best_checkpoints, t + '.pt')
                    shutil.copyfile(checkpoint_path, best_checkpoint_path)

            if options.loss_criterion == 'HybridLoss' and  (alpha-0.1) >= 0.05:
                    alpha -= 0.01
                    logger.info(f"""
                                        Hybrid Loss alpha value reduced to: {alpha}
                                    """)

        logger.info(f"""
                                      ____________________________________
                                      Finished segmentation model training
                                      ====================================
                         """)


text:
# coding: utf-8
"""
    Test the special potentials...
"""

from __future__ import absolute_import, unicode_literals, division, print_function

# Third-party
import astropy.units as u

# This project
from .helpers import CompositePotentialTestBase
from ..builtin.special import LM10Potential, MilkyWayPotential

class TestLM10Potential(CompositePotentialTestBase):
    potential = LM10Potential()
    w0 = [8.,0.,0.,0.,0.22,0.1]

class TestLM10Potential2(CompositePotentialTestBase):
    potential = LM10Potential(disk={'m': 5E10*u.Msun}, bulge={'m': 5E10*u.Msun})
    w0 = [8.,0.,0.,0.,0.22,0.1]

class TestMilkyWayPotential(CompositePotentialTestBase):
    potential = MilkyWayPotential()
    w0 = [8.,0.,0.,0.,0.22,0.1]

max_stars_repo_name: ltlancas/gala

max_issues_repo_name: ltlancas/gala

max_forks_repo_name: ltlancas/gala

reason: import astropy


text:
import numpy as np
import os
import argparse
from original_otc import ObstacleTowerEnv, ActionFlattener

from stable_baselines.common.policies import CnnPolicy
from stable_baselines.common.vec_env import SubprocVecEnv, DummyVecEnv
from stable_baselines.bench import Monitor
from stable_baselines import PPO2

def run_episode(env, model):
    done = False
    episode_reward = 0.0
    obs = env.reset()

    while not done:
        action, _states = model.predict(obs)
        obs, reward, done, info = env.step(action)
        episode_reward += reward
    return episode_reward

def run_evaluation(env, model):
    while not env.done_grading():
        run_episode(env, model)
        env.reset()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('environment_filename', default='./ObstacleTower/obstacletower', nargs='?')
    parser.add_argument('--docker_training', action='store_true')
    parser.set_defaults(docker_training=False)
    args = parser.parse_args()

    env = ObstacleTowerEnv(args.environment_filename, docker_training=args.docker_training, worker_id=np.random.randint(0, 100),
                           retro=True, config={'tower-seed':5}, greyscale=True, timeout_wait=600, realtime_mode=False)
    env._flattener = ActionFlattener([2, 3, 2, 1])
    env._action_space = env._flattener.action_space

    multienv = SubprocVecEnv([lambda: env])
    multimodel = PPO2(CnnPolicy, multienv, verbose=1, gamma=.999, learning_rate=.0000625)
    multimodel = multimodel.load('models/ppo/multimodel_running', multienv)

    running_reward = 0
    running_count = 0
    five_reward = np.array([0, 0, 0, 0, 0])
    if env.is_grading():
        episode_reward = run_evaluation(env, multimodel)
    else:
        while True:
            episode_reward = run_episode(env, multimodel)
            running_reward += episode_reward
            running_count += 1

            five_reward = np.append(five_reward, episode_reward)[1:]

            print("Episode reward: " + str(episode_reward),
                  "Running reward: " + str(running_reward / running_count),
                  "Five reward: " + str(five_reward.mean()))
            env.reset()

    env.close()


text:
import os
from collections import OrderedDict
import time

import tensorflow as tf
import numpy as np
from tqdm import tqdm
from prettytable import PrettyTable

from modules.tf_ops import MySaver, zeros_from_arrays, add_arrays, mul_const_by_arrays

from modules.utils import get_minibatch_indices

class NNBase(object):
    name = 'nnbase'
    scopes = ['base']
    scope_suffixes = ['_b']
    dtype = tf.float32

    def __init__(self, hypparams, model_dir, x, y=None, z=None, _run=None):
        os.makedirs(model_dir, exist_ok=True)
        self.dir = model_dir
        self._run = _run
        self.x = x
        self.y = y
        self.z = z
        self.hypparams = hypparams

        # build graph and initialize
        self.losses, self.optimizers, self.vars, self.lrs, self.reg_losses, self.bare_losses = self.build(x=x, y=y, z=z)
        for scope, var in self.vars.items():
            print('scope: {}'.format(scope))
            self.print_variable_info(var)

        # initialize
        self.init = tf.global_variables_initializer()

        self.sess = self.create_gpu_session()
        self.sess.run(self.init)

        # get saver
        self.saver = MySaver(tf.trainable_variables(), self.sess)

    def create_gpu_session(self):
        config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
        return tf.Session(config=config)

    def build(self, x, y, z):
        '''
        make computational graph

        Args:
            x (Tensor): input
            y (Tensor): output, None for Generator
            z (Tensor): latent variable, None for Classifier

        Returns:
            losses (list): list of loss tensors passed to the optimizers
            optimizers (list): list of optimizer ops
            vars (list): list of the list of Variables
            lrs (list): list of learning rates tensors
            reg_losses (list): list of regularization loss tensors
            bare_losses (list): list of non-regularized loss tensors
        '''
        raise NotImplementedError

    def init_stored_info(self):
        '''
        initialize self.stored_info used for the counterfactual training

        Returns: None

        '''
        raise NotImplementedError

    def print_variable_info(self, variables):
        # print out variables information
        nparams_total = 0
        pt_param = PrettyTable(['index', 'param', 'shape', 'size', 'total'])
        for i, w in enumerate(variables):
            param_shape = w.shape.as_list()
            nparams = np.prod(param_shape).astype(np.int32)
            nparams_total += nparams
            pt_param.add_row([i, w.name, param_shape, nparams, nparams_total])
        print(pt_param)

    def sgd_train(self, dataset_train, nepochs, dataset_valid=None, metrics=None,
                  quiet=False, mode='vanilla', eval_interval=1, j=-1, metric_kwargs={}):
        '''
        run ASGD training.

        Args:
            dataset_train (modules.dataset.MyDataset): dataset used for the training
            nepochs: number of training epochs
            dataset_valid (modules.dataset.MyDataset): dataset used for the validation
            metrics: name of ops or tensor that are evaluated by self.eval_metric at the end of the epoch
            quiet: when True, stop printing
            mode:
                'vanilla': starts training from 0 epoch with newly sampled minibatch indices. no save of the information
                'store': same as vanilla trainig but save the information (e.g., parameters, latent variables learning rates, epoch num) after the epoch specified in self.hypparams['retrace_after'] for counterfactual training
                'counterfactual': restart training using the stored information. it can restarts from intermediate epoch if the the epoch num starts from intermidiate epoch)
            eval_interval (int): interval of epochs how frequently self.eval_metric is ran
            j (array_like): indices of removed instances from minibatches. used only if mode == 'counterfactual'.
            metric_kwargs: kwargs passed to self.eval_metric

        Returns: None

        '''

        if mode in ['vanilla']:
            epoch_start = 0
            minibatch_indices_of_epochs = []
            for epoch in range(epoch_start, nepochs):
                minibatch_indices_of_epochs.append(get_minibatch_indices(dataset_train.sample_size,
                                                                         self.hypparams['batch_size'],
                                                                         append_remainder=False))
            get_opt_scopes = self.get_opt_scopes
            get_minibatch = self.sample_minibatch
            gen_sample_images = self.gen_sample_images

        elif mode in ['store']:
            epoch_start = 0
            minibatch_indices_of_epochs = []
            for epoch in range(epoch_start, nepochs):
                minibatch_indices_of_epochs.append(get_minibatch_indices(dataset_train.sample_size,
                                                                         self.hypparams['batch_size'],
                                                                         append_remainder=False))
            get_opt_scopes = self.get_opt_scopes
            get_minibatch = self.sample_minibatch_by_storing_info
            gen_sample_images = self.gen_sample_images

        elif mode == 'counterfactual':
            self.saver.restore(self.stored_info['params'][0])
            epoch_start = self.stored_info['epoch'][0]
            nepochs_retrace = nepochs - epoch_start
            minibatch_indices_of_steps = np.asarray(self.stored_info['indices'])
            assert minibatch_indices_of_steps.shape[1] == (self.hypparams['batch_size'])
            minibatch_indices_of_epochs = np.reshape(minibatch_indices_of_steps, [nepochs_retrace, -1, self.hypparams['batch_size']])
            get_opt_scopes = self.restore_opt_scopes
            get_minibatch = self.restore_minibatch
            gen_sample_images = lambda *args, **kwargs: None

        else:
            raise ValueError('invalid mode: {}'.format(mode))

        # refresh feed_dict indice
        start_time = time.time()
        step = 0

        for epoch, minibatch_indices_of_epoch, in zip(range(epoch_start, nepochs), minibatch_indices_of_epochs):
            # mini batch train
            with tqdm(minibatch_indices_of_epoch, disable=quiet, ascii=True) as pbar:
                for minibatch_indices in pbar:
                    pbar.set_description('Epoch: [{}/{}]'.format(epoch, nepochs))

                    # update feed dict
                    scopes = get_opt_scopes(step)
                    feed_dict = get_minibatch(dataset_train, minibatch_indices, step, scopes, epoch)

                    # param update
                    if mode == 'counterfactual':
                        if len(np.intersect1d(j, minibatch_indices)) > 0:
                            feed_dict[self.x] = feed_dict[self.x][np.logical_not(np.isin(minibatch_indices, j))]

                    optimizers = self.get_optimizers(scopes, epoch)
                    self.sess.run(optimizers, feed_dict=feed_dict)

                    step += 1

            cal_time = time.time() - start_time

            if metrics is not None and dataset_valid is not None:
                if epoch % eval_interval == 0:
                    dic = OrderedDict()
                    for metric in metrics:
                        if 'lr'in metric:
                            val = self.sess.run(self.get_metric(metric), feed_dict)
                        else:
                            val = self.eval_metric(metric, dataset_valid[:], **metric_kwargs)
                        if self._run is not None:
                            self._run.log_scalar(metric, float(val), step=epoch)
                        dic[metric] = val

                    if cal_time is not None:
                        dic['duration'] = cal_time

                    print(', '.join(['{}={}'.format(k, v) for k, v in dic.items()]))

            if dataset_valid is not None:
                gen_sample_images(dataset_valid, basename='step_{}.png'.format(epoch))

    def get_lr_feed_dict(self, step, scopes):
        if self.hypparams['lr_scheduler'] == 'ghadimi':
            return {self.lrs[scope]: self.hypparams['lrs'][scope] * (step+1) ** -0.5 for scope in scopes}
        elif self.hypparams['lr_scheduler'] == 'constant':
            return {self.lrs[scope]: self.hypparams['lrs'][scope] for scope in scopes}
        else:
            raise ValueError(self.hypparams['lr_scheduler'])

    def get_optimizers(self, scopes, epoch):
        return [self.optimizers[scope] for scope in scopes]

    def store_sgd_info(self, feed_dict, indices, step, optimizer_index, epoch):
        '''
        Store information for the counterfactual training

        Args:
            feed_dict: feed_dict of mini-batch
            indices: indices of mini-batch instances in the dataset
            step: number of the current training step
            optimizer_index: index of the optimizer used for the training
            epoch: number of the current epoch

        Returns: None

        '''
        raise NotImplementedError

    def get_metric(self, name, **kwargs):
        '''
        return tensor defined in the global score.
        name string is used to find the target tensor.

        Args:
            name: name of pre-defined tensor to be returned
            **kwargs:

        Returns: Tensor

        '''
        return tf.get_default_graph().get_tensor_by_name("{}:0".format(name))

    def eval_metric(self, name, feed_dict, **kwargs):
        '''
        get metric tensor using name string.
        then evaluate the tensor with given feed_dict to get numpy result

        Args:
            name: name of the tensor to be evaluated
            feed_dict: feed dict used for the evaluation
            **kwargs:

        Returns: Numpy scalar value

        '''
        metric_tensor = self.get_metric(name, **kwargs)
        return self.sess.run(metric_tensor, feed_dict)

    def add_asgd_influence_ops(self):
        raise NotImplementedError

    def get_classifier_ops(self, x, classifier_conf=None):
        '''
        append classifier ops to get logits or features of input x.

        Args:
            x: tensor of the input
            classifier_conf: dict of the hyper-parameters of classifier

        Returns:

        '''

        raise NotImplementedError

    def restore_opt_scopes(self, step):
        '''
        get list of names of scopes of original training at the given step

        Args:
            step: number of the current step

        Returns: scopes (list)

        '''
        return self.stored_info['scopes'][step]

    def get_opt_scopes(self, step):
        '''
        returns the list of names of the scopes of which parameters are trained

        Args:
            step: current number of step

        Returns: scopes (list)

        '''
        return self.scopes

    def gen_sample_images(self, dataset, basename, batch_size=64, **kwargs):
        '''
        generate and save samples.
        Generator type models overwrite method and Classifier type models use this to pass the generation.

        Args:
            dataset (modules.dataset.MyDataset): dataset that contains latent variables
            basename: basename of the saved image.
            batch_size: batch size of the generation
            **kwargs:

        Returns: None

        '''

        pass

    def sample_minibatch(self, dataset, minibatch_indices, step, scopes, *args, **kwargs):
        '''
        get minibatch feed dict given dataset and indices
        also sample latent variables and include them to feed dict
        learning rates are also determined according to the number of steps and included in the feed dict

        Args:
            dataset (modules.dataset.MyDataset): the whole training dataset
            minibatch_indices: indices in the dataset which construct the minibatch
            step: number of the current step
            scopes: scopes of optimizers to train
            *args:
            **kwargs:

        Returns: feed_dict (dict)

        '''
        feed_dict = dataset[minibatch_indices]
        feed_dict.update(self.get_lr_feed_dict(step, scopes))
        return feed_dict

    def sample_minibatch_by_storing_info(self, dataset, minibatch_indices, step, scopes, epoch):
        '''
        get minibatch feed dict and run self.store_sgd_info method

        Args:
            dataset (modules.dataset.MyDataset): the whole training dataset
            minibatch_indices: indices in the dataset which construct the minibatch
            step: number of the current step
            scopes: scopes of optimizers to train
            epoch: number of the current epoch

        Returns: feed_dict (dict)

        '''
        raise NotImplementedError

    def restore_minibatch(self, dataset, minibatch_indices, step, scopes, epoch):
        '''
        Get_items of instances from dataset given indices of the mini-batch to make a mini-bathch feed_dict.
        Then update feed_dict by stored latent variables (if saved) and learning rates

        Args:
            dataset (modules.dataset.MyDataset): the whole training dataset
            minibatch_indices: indices in the dataset which construct the minibatch
            step: number of the current step
            scopes: scopes of optimizers to train
            epoch: number of the current epoch

        Returns: feed_dict (dict)

        '''
        raise NotImplementedError

    def run_with_batches(self, fetch, dataset, batch_size=None):
        '''
        For memory efficiency, run fetch tensor with separated dataset.
        Then concatenate or average result.

        Args:
            fetch: tensor of which sess.run is computed
            dataset: dataset with which fetch tensor is computed
            batch_size: batch_size of a singel run

        Returns:
            numpy result of Session.run of fetch

        '''
        if batch_size is None or dataset.sample_size <= batch_size:
            return self.sess.run(fetch, dataset[:])

        else:
            assert dataset.sample_size % batch_size == 0
            if type(fetch) is list:
                out = zeros_from_arrays(fetch)
                update_func = lambda old, new, coef: add_arrays(old, mul_const_by_arrays(new, coef))
            else:
                shape = fetch._shape_as_list()
                if shape is None:
                    shape = [None]
                if None in shape:  # concatenate mode
                    assert shape[0] is None
                    out = None

                    def update_func(old, new, coef):
                        if old is None:
                            out = new
                        else:
                            out = np.concatenate([old, new])
                        return out
                else:  # update mode
                    out = np.zeros(shape)
                    update_func = lambda old, new, coef: out + new * coef

            idxs_of_minibatches = get_minibatch_indices(dataset.sample_size, batch_size, append_remainder=False, original_order=True)
            for idxs in idxs_of_minibatches:
                # calculate coefficient
                coef = batch_size / dataset.sample_size
                # evaluate
                tmp = self.sess.run(fetch, dataset[idxs])
                # update
                out = update_func(out, tmp, coef)

            return out


text:
import random
import numpy
import itertools
from my_code.block_designer import BlockDesigner

import pdb

class Sampler(BlockDesigner):
    """
    Inherits __init__ so that it can be initialized with
    a block from a BlockDesigner
    """
    def __init__(self, source, K=5, seed=None):
        super(Sampler, self).__init__(source, K=5, seed=None)
        all_klasses = self.reservoir.keys()
        self.loopy_klass_examples = [itertools.cycle(self.reservoir[k_]) for k_ in all_klasses]

    def uniform_klass_counts(self, batch_size):
        avg_num_batch_klass_examples = int(batch_size / self.K)
        num_extras_needed = batch_size - avg_num_batch_klass_examples * self.K
        extra_pad = numpy.zeros(self.K, dtype=int)
        extra_pad[:num_extras_needed] = 1 # take 1 extra of most prevalent classes
        all_num_feed_klass_examples = numpy.array([avg_num_batch_klass_examples] * self.K, dtype=int) + extra_pad
        return all_num_feed_klass_examples

    def custom_distribution(self, k, batch_size, requested_distribution=None):
        """
        loops through all the examples in class k filling in with other
        classes as neccessary. Assuming the default uniform distribution:
        classes greater than k will be oversampled (empirically leads to
        overfitting) and classes less than k will be undersampled
        """
        all_num_feed_klass_examples = requested_distribution or self.uniform_klass_counts(batch_size)
        assert(len(all_num_feed_klass_examples) == self.K)
        assert(sum(all_num_feed_klass_examples) == batch_size)

        num_feed_klass_examples = all_num_feed_klass_examples[k]
        num_batches_to_yield = int(len(self.reservoir[k]) / all_num_feed_klass_examples[k])

        X = []
        y = []
        for batch_i in range(num_batches_to_yield):
            X_batch = []
            y_batch = []

            for klass, num_examples in enumerate(all_num_feed_klass_examples):
                for _ in range(num_examples):
                    X_batch.append(self.loopy_klass_examples[klass].next())
                y_batch.extend((klass*numpy.ones(num_examples, dtype=int)).tolist())

            X.extend(X_batch)
            y.extend(y_batch)
        return(X, y)

max_stars_repo_name: ilyakava/kaggle-dr


text:
import argparse
import math
import h5py
import numpy as np
import tensorflow as tf
import socket

import os
import sys
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = os.path.dirname(BASE_DIR)
sys.path.append(BASE_DIR)
sys.path.append(ROOT_DIR)
sys.path.append(os.path.join(ROOT_DIR, 'utils'))
import provider
import tf_util
from model import *

parser = argparse.ArgumentParser()
parser.add_argument('--path_data', help='folder with train val data')
parser.add_argument('--cls', type=int, help='number of classes')
parser.add_argument('--num_gpu', type=int, default=1, help='the number of GPUs to use [default: 2]')
parser.add_argument('--log_dir', default='log', help='Log dir [default: log]')
parser.add_argument('--num_point', type=int, default=4096, help='Point number [default: 4096]')
parser.add_argument('--max_epoch', type=int, default=100, help='Epoch to run [default: 100]')
parser.add_argument('--batch_size', type=int, default=32, help='Batch Size during training for each GPU [default: 32]')
parser.add_argument('--learning_rate', type=float, default=0.001, help='Initial learning rate [default: 0.001]')
parser.add_argument('--momentum', type=float, default=0.9, help='Initial learning rate [default: 0.9]')
parser.add_argument('--optimizer', default='adam', help='adam or momentum [default: adam]')
parser.add_argument('--decay_step', type=int, default=300000, help='Decay step for lr decay [default: 300000]')
parser.add_argument('--decay_rate', type=float, default=0.5, help='Decay rate for lr decay [default: 0.5]')
parsed_args = parser.parse_args()

TOWER_NAME = 'tower'

path_data = parsed_args.path_data
cls = parsed_args.cls
BATCH_SIZE = parsed_args.batch_size
NUM_POINT = parsed_args.num_point
MAX_EPOCH = parsed_args.max_epoch
NUM_POINT = parsed_args.num_point
BASE_LEARNING_RATE = parsed_args.learning_rate
MOMENTUM = parsed_args.momentum
OPTIMIZER = parsed_args.optimizer
DECAY_STEP = parsed_args.decay_step
DECAY_RATE = parsed_args.decay_rate

LOG_DIR = parsed_args.log_dir
if not os.path.exists(LOG_DIR): os.mkdir(LOG_DIR)
os.system('cp model.py %s' % (LOG_DIR))
os.system('cp train.py %s' % (LOG_DIR))
LOG_FOUT = open(os.path.join(LOG_DIR, 'log_train.txt'), 'w')
LOG_FOUT.write(str(parsed_args)+'\n')

NUM_CLASSES = cls
BN_INIT_DECAY = 0.5
BN_DECAY_DECAY_RATE = 0.5
BN_DECAY_DECAY_STEP = float(DECAY_STEP)
BN_DECAY_CLIP = 0.99

HOSTNAME = socket.gethostname()

path_train = os.path.join(path_data, 'train')
files_train = provider.getDataFiles(os.path.join(path_train, 'files.txt'))

data_batch_list = []
label_batch_list = []

for h5_filename in files_train:
    file_path = os.path.join(path_train, h5_filename)
    data_batch, label_batch = provider.loadDataFile(file_path)
    data_batch_list.append(data_batch)
    label_batch_list.append(label_batch)
train_data = np.concatenate(data_batch_list, 0)
train_label = np.concatenate(label_batch_list, 0)
print(train_data.shape, train_label.shape)

path_val = os.path.join(path_data, 'val')
files_val = provider.getDataFiles(os.path.join(path_val, 'files.txt'))

data_batch_list = []
label_batch_list = []

for h5_filename in files_val:
    file_path = os.path.join(path_val, h5_filename)
    data_batch, label_batch = provider.loadDataFile(file_path)
    data_batch_list.append(data_batch)
    label_batch_list.append(label_batch)
val_data = np.concatenate(data_batch_list, 0)
val_label = np.concatenate(label_batch_list, 0)
print(val_data.shape, val_label.shape)

def log_string(out_str):
  LOG_FOUT.write(out_str+'\n')
  LOG_FOUT.flush()
  print(out_str)

def get_learning_rate(batch):
  learning_rate = tf.train.exponential_decay(
            BASE_LEARNING_RATE,  # Base learning rate.
            batch * BATCH_SIZE,  # Current index into the dataset.
            DECAY_STEP,          # Decay step.
            DECAY_RATE,          # Decay rate.
            staircase=True)
  learning_rate = tf.maximum(learning_rate, 0.00001) # CLIP THE LEARNING RATE!!
  return learning_rate

def get_bn_decay(batch):
  bn_momentum = tf.train.exponential_decay(
            BN_INIT_DECAY,
            batch*BATCH_SIZE,
            BN_DECAY_DECAY_STEP,
            BN_DECAY_DECAY_RATE,
            staircase=True)
  bn_decay = tf.minimum(BN_DECAY_CLIP, 1 - bn_momentum)
  return bn_decay

def average_gradients(tower_grads):
  """Calculate average gradient for each shared variable across all towers.

  Note that this function provides a synchronization point across all towers.

  Args:
    tower_grads: List of lists of (gradient, variable) tuples. The outer list
    is over individual gradients. The inner list is over the gradient
    calculation for each tower.
  Returns:
     List of pairs of (gradient, variable) where the gradient has been
     averaged across all towers.
  """
  average_grads = []
  for grad_and_vars in zip(*tower_grads):
    # Note that each grad_and_vars looks like the following:
    #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
    grads = []
    for g, _ in grad_and_vars:
      expanded_g = tf.expand_dims(g, 0)
      grads.append(expanded_g)

    # Average over the 'tower' dimension.
    grad = tf.concat(grads, 0)
    grad = tf.reduce_mean(grad, 0)

    # Keep in mind that the Variables are redundant because they are shared
    # across towers. So .. we will just return the first tower's pointer to
    # the Variable.
    v = grad_and_vars[0][1]
    grad_and_var = (grad, v)
    average_grads.append(grad_and_var)
  return average_grads

def train():
  with tf.Graph().as_default(), tf.device('/gpu:0'):
    batch = tf.Variable(0, trainable=False)

    bn_decay = get_bn_decay(batch)
    tf.summary.scalar('bn_decay', bn_decay)

    learning_rate = get_learning_rate(batch)
    tf.summary.scalar('learning_rate', learning_rate)

    trainer = tf.train.AdamOptimizer(learning_rate)

    tower_grads = []
    pointclouds_phs = []
    labels_phs = []
    is_training_phs =[]

    with tf.variable_scope(tf.get_variable_scope()):
      for i in range(parsed_args.num_gpu):
        with tf.device('/gpu:%d' % i):
          with tf.name_scope('%s_%d' % (TOWER_NAME, i)) as scope:

            pointclouds_pl, labels_pl = placeholder_inputs(BATCH_SIZE, NUM_POINT)
            is_training_pl = tf.placeholder(tf.bool, shape=())

            pointclouds_phs.append(pointclouds_pl)
            labels_phs.append(labels_pl)
            is_training_phs.append(is_training_pl)

            pred = get_model(pointclouds_phs[-1], is_training_phs[-1], bn_decay=bn_decay)
            loss = get_loss(pred, labels_phs[-1])
            tf.summary.scalar('loss', loss)

            correct = tf.equal(tf.argmax(pred, 2), tf.to_int64(labels_phs[-1]))
            accuracy = tf.reduce_sum(tf.cast(correct, tf.float32)) / float(BATCH_SIZE*NUM_POINT)
            tf.summary.scalar('accuracy', accuracy)

            tf.get_variable_scope().reuse_variables()

            grads = trainer.compute_gradients(loss)

            tower_grads.append(grads)

    grads = average_gradients(tower_grads)

    train_op = trainer.apply_gradients(grads, global_step=batch)

    saver = tf.train.Saver(tf.global_variables(), sharded=True, max_to_keep=10)

    # Create a session
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    config.allow_soft_placement = True
    sess = tf.Session(config=config)

    # Add summary writers
    merged = tf.summary.merge_all()
    train_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'train'),
                  sess.graph)
    val_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'val'))

    # Init variables for two GPUs
    init = tf.group(tf.global_variables_initializer(),
             tf.local_variables_initializer())
    sess.run(init)

    ops = {'pointclouds_phs': pointclouds_phs,
         'labels_phs': labels_phs,
         'is_training_phs': is_training_phs,
         'pred': pred,
         'loss': loss,
         'train_op': train_op,
         'merged': merged,
         'step': batch}

    loss_t_list = list()
    loss_val_list = list()
    acc_t_list = list()
    acc_val_list = list()

    for epoch in range(MAX_EPOCH):
      log_string('**** EPOCH %03d ****' % (epoch))
      sys.stdout.flush()

      loss_t, acc_t = train_one_epoch(sess, ops, train_writer)
      loss_t_list.append(loss_t)
      acc_t_list.append(acc_t)

      loss_val, acc_val = eval_one_epoch(sess, ops, val_writer)
      loss_val_list.append(loss_val)
      acc_val_list.append(acc_val)

      if loss_val == min(loss_val_list):
          best_sess = sess
          best_epoch = epoch

      stop = early_stopping(loss_t_list, loss_val_list, 1)
      if stop:
          log_string('early stopping')
          break

  log_string('save session at epoch %03d' % (best_epoch))
  # Save the variables to disk.
  save_path = saver.save(best_sess, os.path.join(LOG_DIR, "model.ckpt"))
  log_string("Model saved in file: %s" % save_path)

def early_stopping(t_loss, v_loss, thr):

    stop = False

    if len(t_loss) > 9:

        a = 100 * ((v_loss[-1] / min(v_loss)) - 1)
        b = 1000 * ((sum(t_loss[-10:]) / (10 * min(t_loss[-10:]))) - 1)
        ab = a / b

        if ab > thr:
            stop = True

    return stop

def train_one_epoch(sess, ops, train_writer):
  """ ops: dict mapping from string to tf ops """
  is_training = True

  log_string('----')
  current_data, current_label, _ = provider.shuffle_data(train_data[:,0:NUM_POINT,:], train_label)

  file_size = current_data.shape[0]
  num_batches = file_size // (parsed_args.num_gpu * BATCH_SIZE)

  total_correct = 0
  total_seen = 0
  loss_sum = 0

  for batch_idx in range(num_batches):
    if batch_idx % 100 == 0:
      print('Current batch/total batch num: %d/%d'%(batch_idx,num_batches))
    start_idx_0 = batch_idx * BATCH_SIZE
    end_idx_0 = (batch_idx+1) * BATCH_SIZE

    feed_dict = {ops['pointclouds_phs'][0]: current_data[start_idx_0:end_idx_0, :, :],
                 ops['labels_phs'][0]: current_label[start_idx_0:end_idx_0],
                 ops['is_training_phs'][0]: is_training}
    summary, step, _, loss_val, pred_val = sess.run([ops['merged'], ops['step'], ops['train_op'], ops['loss'], ops['pred']],
                     feed_dict=feed_dict)
    train_writer.add_summary(summary, step)
    pred_val = np.argmax(pred_val, 2)
    correct = np.sum(pred_val == current_label[start_idx_0:end_idx_0])
    total_correct += correct
    total_seen += (BATCH_SIZE*NUM_POINT)
    loss_sum += loss_val

  mean_loss = loss_sum / float(num_batches)
  accuracy = total_correct / float(total_seen)

  log_string('mean loss: %f' % (mean_loss))
  log_string('accuracy: %f' % (accuracy))

  return mean_loss, accuracy

def eval_one_epoch(sess, ops, val_writer):
  """ ops: dict mapping from string to tf ops """
  is_training = False
  total_correct = 0
  total_seen = 0
  loss_sum = 0
  total_seen_class = [0 for _ in range(NUM_CLASSES)]
  total_correct_class = [0 for _ in range(NUM_CLASSES)]

  log_string('----')
  current_data = val_data[:,0:NUM_POINT,:]
  current_label = np.squeeze(val_label)

  file_size = current_data.shape[0]
  num_batches = file_size // BATCH_SIZE

  for batch_idx in range(num_batches):
    start_idx = batch_idx * BATCH_SIZE
    end_idx = (batch_idx+1) * BATCH_SIZE

    feed_dict = {ops['pointclouds_phs'][0]: current_data[start_idx:end_idx, :, :],
                  ops['labels_phs'][0]: current_label[start_idx:end_idx],
                  ops['is_training_phs'][0]: is_training}
    summary, step, loss_val, pred_val = sess.run([ops['merged'], ops['step'], ops['loss'], ops['pred']], feed_dict=feed_dict)
    val_writer.add_summary(summary, step)
    pred_val = np.argmax(pred_val, 2)
    correct = np.sum(pred_val == current_label[start_idx:end_idx])
    total_correct += correct
    total_seen += (BATCH_SIZE*NUM_POINT)
    loss_sum += (loss_val*BATCH_SIZE)
    for i in range(start_idx, end_idx):
      for j in range(NUM_POINT):
        l = current_label[i, j]
        total_seen_class[l] += 1
        total_correct_class[l] += (pred_val[i-start_idx, j] == l)

  mean_loss = loss_sum / float(total_seen/NUM_POINT)
  accuracy = total_correct / float(total_seen)

  log_string('eval mean loss: %f' % (mean_loss))
  log_string('eval accuracy: %f'% (accuracy))
  log_string('eval class acc: ' + str((np.array(total_correct_class)/np.array(total_seen_class, dtype=np.float))))
  log_string('eval avg class acc: %f' % (np.mean(np.array(total_correct_class)/np.array(total_seen_class, dtype=np.float))))

  return mean_loss, accuracy

if __name__ == "__main__":
  train()
  LOG_FOUT.close()

max_stars_repo_name: mmabadal/dgcnn

max_issues_repo_name: mmabadal/dgcnn

max_forks_repo_name: mmabadal/dgcnn


text:
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-

"""
Naturalunit system.

The natural system comes from "setting c = 1, hbar = 1". From the computer
point of view it means that we use velocity and action instead of length and
time. Moreover instead of mass we use energy.
"""

from __future__ import division

from sympy.physics.units.definitions import eV, hbar, c

from sympy.physics.units.dimensions import DimensionSystem
from sympy.physics.units.dimensions import length, mass, time, momentum,\
    force, energy, power, frequency, action, velocity
from sympy.physics.units.unitsystem import UnitSystem
from sympy.physics.units.prefixes import PREFIXES, prefix_unit

dims = (length, mass, time, momentum, force, energy, power, frequency)

# dimension system
_natural_dim = DimensionSystem(base=(action, energy, velocity), dims=dims,
                              name="Natural system")

units = prefix_unit(eV, PREFIXES)

# unit system
natural = UnitSystem(base=(hbar, eV, c), units=units, name="Natural system")

reason: from sympy