text:
import argparse
from utils.utils import load_data, define_logger, tokenize_gen, evaluate_gpt2, gpt2_evaluate, compute_ppl
import random
import numpy as np
import torch
from model.generatively_model import gpt2_generate, bart_generate
from transformers import AdamW, GPT2LMHeadModel, GPT2Tokenizer
import sys
import torch.nn as nn
import os
from torch.utils.data import TensorDataset, DataLoader
from tqdm import trange
import datetime
import logging
import pdb
from rouge import Rouge
from nltk import bleu
import csv
import copy
import torch.nn.functional as F

class gpt2_multi_task(nn.Module):
    def __init__(self, hps):
        super(gpt2_multi_task, self).__init__()
        self.hps = hps
        self.model = GPT2LMHeadModel.from_pretrained(hps.model_dir)
        self.linear = nn.Linear(self.model.config.hidden_size, 1)

    def forward(self, input_ids, attention_mask, pos, mode='train', token_type_ids=None):
        # if mode == 'train':

        if mode == 'train':
            # pdb.set_trace()
            outputs = self.model(input_ids, attention_mask=attention_mask, output_hidden_states=True, token_type_ids=token_type_ids)
            hidden_state = outputs.hidden_states[-1]
            # pos = pos.squeeze().unsqueeze(0)
            hidden_state = hidden_state[range(hidden_state.shape[0]), pos, :]
            logits = self.linear(hidden_state).squeeze(-1)
            gen_logits = outputs.logits
            return logits, gen_logits

        if mode == 'test':
            output_dict = self.model.generate(input_ids=input_ids,
                                              attention_mask=attention_mask,
                                              max_length=self.hps.length + input_ids.shape[1],
                                              num_beams=self.hps.beam_size,
                                              early_stopping=self.hps.early_stopping,
                                              # do_sample=self.hps.do_sample,
                                              no_repeat_ngram_size=self.hps.no_repeat_ngram_size,
                                              repetition_penalty=self.hps.repetition_penalty,
                                              output_hidden_states=True,
                                              return_dict_in_generate=True,
                                              output_scores=True
                                              )

            if self.hps.mode == 'generate_discriminate':
                # pdb.set_trace()
                scores = output_dict.scores
                hiddens = output_dict.hidden_states

                hiddens = [h[-1][:, -1, :].view(input_ids.shape[0], self.hps.beam_size, -1) for h in hiddens]
                scores = [s.view(input_ids.shape[0], self.hps.beam_size, -1) for s in scores]
                index = [s.argmax(-1) for s in scores]

                generated_text = output_dict.sequences[:, input_ids.shape[1]:]
                index = [index[i].eq(generated_text[:, i].unsqueeze(1)) for i in range(len(index))]
                selected = None
                # pdb.set_trace()
                for i in range(len(index)):
                    ttmp = None
                    for j in range(index[i].shape[0]):
                        tmp = hiddens[i][j][index[i][j]]
                        if tmp.shape[0] == 0:
                            tmp = hiddens[i][j][0]
                        else:
                            tmp = tmp[0]
                        ttmp = tmp.unsqueeze(0) if ttmp is None else torch.cat((ttmp, tmp.unsqueeze(0)), 0)
                    selected = ttmp.unsqueeze(1) if selected is None else torch.cat((selected, ttmp.unsqueeze(1)), 1)

                # pdb.set_trace()
                # eos_mask = generated_text.eq(self.model.config.eos_token_id)
                eos_mask = generated_text.eq(13)

                gen_ids = generated_text
                for i in range(eos_mask.shape[0]):
                    flag = False
                    if eos_mask[i].sum().item() == 1:
                        continue
                    elif eos_mask[i].sum().item() == 0:
                        eos_mask[i][-1] = True
                        continue
                    else:
                        for j, t in enumerate(eos_mask[i]):
                            if t.item():
                                if flag:
                                    eos_mask[i][j] = False
                                flag = True
                            else:
                                continue

                # pdb.set_trace()
                # selected = selected.view(eos_mask.shape[0], eos_mask.shape[1], -1)[eos_mask, :]
                selected = selected[eos_mask, :]
                logits = self.linear(selected).squeeze()
                gen_ids = generated_text

            else:
                # pdb.set_trace()
                outputs = self.model(input_ids, attention_mask=attention_mask, output_hidden_states=True)
                hidden_state = outputs.hidden_states[-1]
                # pos = pos.squeeze().unsqueeze(0)
                hidden_state = hidden_state[range(hidden_state.shape[0]), pos, :]
                logits = self.linear(hidden_state).squeeze(-1)
                # logits = torch.cat((logits[::2].unsqueeze(1), logits[1::2].unsqueeze(1)), 1)
                gen_ids = output_dict.sequences[:, input_ids.shape[1]:]

            return logits, gen_ids

def tokenization(data, hps):
    tokenizer = GPT2Tokenizer.from_pretrained(hps.model_dir)
    tokenizer.pad_token = tokenizer.eos_token

    inputs = []
    labels = []
    pos1 = []
    pos2 = []
    loss_label = []
    truth = []
    # token_type_ids = []

    for example in data:
        if example['ask-for'] == 'cause':
            inputs.append([example['alternative1'], example['premise']])
            inputs.append([example['alternative2'], example['premise']])
        else:
            inputs.append([example['premise'], example['alternative1']])
            inputs.append([example['premise'], example['alternative2']])
        truth += [example['general_truth']] * 2
        labels += [0, 1] if example['label'] == 1 else [1, 0]

    outputs = tokenizer(inputs, return_length=True)
    input_ids = outputs['input_ids']
    attention_mask = outputs['attention_mask']
    length = outputs['length']
    # max_length = max(length)

    truth_output = tokenizer(truth, return_length=True)
    truth_ids = truth_output['input_ids']
    truth_mask = truth_output['attention_mask']
    truth_length = truth_output['length']

    premise_ids = []
    premise_mask = []

    for i in range(len(input_ids)):
        pos1.append(len(input_ids[i]))
        input_ids[i] += [50256]
        attention_mask[i] += [1]
        loss_label.append([-100 for _ in range(len(input_ids[i]))])
        # token_type_ids += [0 for _ in range(len(input_ids[i]))]

        gap1 = max(length) + 1 - len(input_ids[i])
        premise_ids.append(input_ids[i] + [50256 for _ in range(gap1)])
        premise_mask.append(attention_mask[i] + [0 for _ in range(gap1)])

        input_ids[i] += truth_ids[i]
        attention_mask[i] += truth_mask[i]
        # token_type_ids += [1 for _ in range(len(truth_ids[i]))]
        pos2.append(len(input_ids[i])-1)

        loss_label[i] += truth_ids[i]

        gap2 = max(truth_length) - len(truth_ids[i])
        truth_ids[i] += [50256 for _ in range(gap2)]

    truth_ids = torch.LongTensor(truth_ids)

    max_length = max([length[k]+truth_length[k] for k in range(len(length))]) + 2
    for i in range(len(input_ids)):
        gap = max_length - len(input_ids[i])
        input_ids[i] += [50256 for _ in range(gap)]
        attention_mask[i] += [0 for _ in range(gap)]
        # token_type_ids += [0 for _ in range(gap)]
        loss_label[i] += [-100 for _ in range(gap)]

    # premise_outputs = tokenizer(inputs, padding=True)
    premise_ids = torch.LongTensor(premise_ids)
    premise_mask = torch.LongTensor(premise_mask)

    if hps.mode == 'discriminate_generate':
        pos = pos1
    else:
        pos = pos2
    return torch.LongTensor(input_ids), torch.LongTensor(attention_mask), torch.LongTensor(pos), torch.LongTensor(
        labels), torch.LongTensor(loss_label), premise_ids, premise_mask, truth_ids

def evaluate(hps, model, dataloader, loss_function, loss_function2, optimizer):

    tokenizer = GPT2Tokenizer.from_pretrained(hps.model_dir)
    # predictions = []
    labels = []
    predict_labels, attack_predict_labels = [], []
    loss = 0
    attack_loss = 0

    bleu1, bleu2, bleu3, bleu4 = 0, 0, 0, 0
    rouge1p, rouge1r, rouge1f, rouge2p, rouge2r, rouge2f, rougelp, rougelr, rougelf = 0, 0, 0, 0, 0, 0, 0, 0, 0
    rouge = Rouge()
    output_text = []

    # model.eval()
    # pdb.set_trace()
    for batch in dataloader:
        attack_model = copy.deepcopy(model)

        # attack_model.eval()
        optimizer.zero_grad()
        if hps.cuda:
            batch = tuple(term.cuda() for term in batch)

        ids, mask, pos, tmp_labels, loss_label, input_ids, attention_mask, truth = batch
        dis_logits, gen_logits = model(ids, attention_mask=mask, pos=pos, mode='train')

        # predictions += logits.cpu().tolist()
        tmp_loss = loss_function(dis_logits, tmp_labels.float())
        loss += tmp_loss.cpu().item()

        t_1 = tmp_labels[::2].unsqueeze(1)
        t_2 = tmp_labels[1::2].unsqueeze(1)
        t_ = torch.cat((t_1, t_2), 1)
        t_index = t_.argmax(1)

        g1 = gen_logits[::2].unsqueeze(1)
        g2 = gen_logits[1::2].unsqueeze(1)
        g = torch.cat((g1, g2), 1)

        l1 = loss_label[::2].unsqueeze(1)
        l2 = loss_label[1::2].unsqueeze(1)
        l = torch.cat((l1, l2), 1)

        g = g[range(g.shape[0]), t_index, :]
        l = l[range(l.shape[0]), t_index, :]

        # pdb.set_trace()

        shift_logits = g[..., :-1, :].contiguous()
        shift_labels = l[..., 1:].contiguous()

        loss_gen = loss_function2(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
        total_loss = hps.alpha * tmp_loss + (1 - hps.alpha) * loss_gen
        total_loss.backward()
        embedding_grad = optimizer.param_groups[0]['params'][0].grad
        # attack

        # optimizer.zero_grad()
        with torch.no_grad():
            # pdb.set_trace()
            model.eval()
            state_dict = attack_model.state_dict()
            logits, gen_ids = model(input_ids, attention_mask=attention_mask, pos=pos, mode='test')
            # embedding_grad = F.softmax(embedding_grad, 1)
            attack_embedding = torch.sum(embedding_grad * embedding_grad, -1)
            attack_embedding = attack_embedding.pow(0.5).unsqueeze(-1)
            attack_embedding = attack_embedding.pow(-1) * embedding_grad
            state_dict['model.transformer.wte.weight'] += hps.attack_rate * attack_embedding
            attack_model.load_state_dict(state_dict)
            attack_model.eval()
            attack_logits, _ = attack_model(input_ids, attention_mask=attention_mask, pos=pos, mode='test')

            attack_loss += loss_function(attack_logits, tmp_labels.float()).item()
            labels += tmp_labels.cpu().numpy().tolist()

        # generated = gen_ids[:, input_ids.shape[1]:]
            tmp_predict = logits.cpu().tolist()
            a1 = torch.FloatTensor(tmp_predict[::2]).unsqueeze(1)
            a2 = torch.FloatTensor(tmp_predict[1::2]).unsqueeze(1)
            a = torch.cat((a1, a2), dim=1)
            predict_label = torch.argmax(a, 1)
            predict_labels += predict_label.cpu().tolist()

            attack_tmp_predict = attack_logits.cpu().tolist()
            a_t1 = torch.FloatTensor(attack_tmp_predict[::2]).unsqueeze(1)
            a_t2 = torch.FloatTensor(attack_tmp_predict[1::2]).unsqueeze(1)
            a_t = torch.cat((a_t1, a_t2), 1)
            attack_predict_label = torch.argmax(a_t, 1)
            attack_predict_labels += attack_predict_label.cpu().tolist()

            g1 = gen_ids[::2].unsqueeze(1)
            g2 = gen_ids[1::2].unsqueeze(1)
            g = torch.cat((g1, g2), 1)

            t_1 = tmp_labels[::2].unsqueeze(1)
            t_2 = tmp_labels[1::2].unsqueeze(1)
            t = torch.cat((t_1, t_2), 1)

        # generated = g[range(g.shape[0]), predict_label]
            generated = g[range(g.shape[0]), t.argmax(1)]

            generated_text = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in
                              generated.cpu().tolist()]

            gold_text = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in
                         truth[::2].cpu().tolist()]
            # input_text = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in input_ids]
            output_text += [[gold_text[i], generated_text[i].split('.')[0] + '.'] for i in range(len(gold_text))]
            for i in range(generated.shape[0]):
                bleu1 += bleu([gold_text[i]], generated_text[i].split('.')[0] + '.', [1, 0, 0, 0])
                bleu2 += bleu([gold_text[i]], generated_text[i].split('.')[0] + '.', [0, 1, 0, 0])
                bleu3 += bleu([gold_text[i]], generated_text[i].split('.')[0] + '.', [0, 0, 1, 0])
                bleu4 += bleu([gold_text[i]], generated_text[i].split('.')[0] + '.', [0, 0, 0, 1])

                scores = rouge.get_scores(generated_text[i], gold_text[i])
                rouge1 = scores[0]['rouge-1']
                rouge1f += rouge1['f']
                rougelp += rouge1['p']
                rouge1r += rouge1['r']

                rouge2 = scores[0]['rouge-2']
                rouge2f += rouge2['f']
                rouge1p += rouge2['p']
                rouge2r += rouge2['r']

                rougel = scores[0]['rouge-l']
                rougelf += rougel['f']
                rougelp += rougel['p']
                rougelr += rougel['r']

    # predict_labels = predict_labels.cpu().tolist()
    # pdb.set_trace()
    t_a1 = torch.FloatTensor(labels[::2]).unsqueeze(1)
    t_a2 = torch.FloatTensor(labels[1::2]).unsqueeze(1)
    t_a = torch.cat((t_a1, t_a2), dim=1)
    true_labels = torch.argmax(t_a, 1).tolist()
    count = 0
    attack_count = 0
    for i in range(len(predict_labels)):
        if predict_labels[i] == true_labels[i] and attack_predict_labels[i] == true_labels[i]:
            count += 1
            attack_count += 1
        elif predict_labels[i] == true_labels[i]:
            count += 1
        elif attack_predict_labels[i] == true_labels[i]:
            attack_count += 1
        else:
            continue
    # pdb.set_trace()
    nowtime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
    fo = open(hps.output_dir + '/gpt2_predict_' + nowtime + '.csv', 'w', encoding='utf-8')
    writer = csv.writer(fo)
    writer.writerows(output_text)

    # num_instances = int(((len(data_loader)-1) * hps.batch_size + truth_ids.shape[0]) / 2)

    return count / len(true_labels), bleu1 / len(true_labels), bleu2 / len(true_labels), bleu3 / len(
        true_labels), bleu4 / len(true_labels), rouge1r / len(true_labels), rouge2r / len(true_labels), rougelr / len(
        true_labels), loss, attack_count / len(true_labels), attack_loss

def compute_ppl(hps, model, data):
    # pdb.set_trace()
    tokenizer = GPT2Tokenizer.from_pretrained(hps.model_dir)
    lls = []
    total_length = 0
    for example in data:
        if example['ask-for'] == 'cause':
            input_text = (example['alternative1'] + ' ' + example['premise']) if example['label'] == 0 else (example['alternative2'] + ' ' + example['premise'])
        else:
            input_text = (example['premise'] + ' ' + example['alternative1']) if example['label'] == 0 else (example['premise'] + ' ' + example['alternative2'])
        truth = example['general_truth']
        inputs = tokenizer(input_text)
        input_ids = torch.LongTensor(inputs['input_ids']+[50256]).unsqueeze(0).cuda()
        attention_mask = torch.LongTensor(inputs['attention_mask']+[1]).unsqueeze(0).cuda()
        label_inputs = tokenizer(truth)
        label_ids = torch.LongTensor(label_inputs['input_ids']).unsqueeze(0).cuda()
        length = label_ids.shape[1]
        total_length += length

        # label_mask = torch.LongTensor(label_inputs['attention_mask']).unsqueeze(0).cuda()
        attention_mask = torch.cat((attention_mask, torch.ones(1, label_ids.shape[1]).long().cuda()), 1)
        label_ids = torch.cat((torch.LongTensor([-100]*input_ids.shape[1]).unsqueeze(0).cuda(), label_ids), 1)
        input_ids = torch.cat((input_ids, label_ids[:, input_ids.shape[1]:]), 1)
        with torch.no_grad():
            loss = model.model(input_ids, attention_mask=attention_mask, labels=label_ids)[0]
            lls.append(loss * length)
    # pdb.set_trace()
    ppl = torch.exp(torch.stack(lls).sum() / total_length)

    return ppl.item()

def main():
    parser = argparse.ArgumentParser(description='xCAR')

    # Data Paths
    parser.add_argument('--data_dir', type=str, default='./data/', help='The dataset directory')
    parser.add_argument('--model_dir', type=str, default='../../huggingface_transformers/gpt2/',
                        help='The pretrained model directory')
    parser.add_argument('--save_dir', type=str, default='./output/saved_model', help='The model saving directory')
    parser.add_argument('--log_dir', type=str, default='./output/log', help='The training log directory')
    parser.add_argument('--apex_dir', type=str, default='./output/log', help='The apex directory')

    # Data names
    parser.add_argument('--train', type=str, default='train.pkl', help='The train data directory')
    parser.add_argument('--dev', type=str, default='dev.pkl', help='The dev data directory')
    parser.add_argument('--test', type=str, default='test.pkl', help='The test data directory')

    # Model Settings
    parser.add_argument('--model_name', type=str, default='gpt2', help='Pretrained model name')
    parser.add_argument('--cuda', type=bool, default=True, help='Whether to use gpu for training')
    parser.add_argument('--gpu', type=str, default='0', help='Gpu ids for training')
    # parser.add_argument('--apex', type=bool, default=False, help='Whether to use half precision')
    parser.add_argument('--batch_size', type=int, default=16, help='batch_size for training and evaluation')
    parser.add_argument('--shuffle', type=bool, default=False, help='whether to shuffle training data')
    parser.add_argument('--epochs', type=int, default=200, help='training iterations')
    parser.add_argument('--evaluation_step', type=int, default=2,
                        help='when training for some steps, start evaluation')
    parser.add_argument('--lr', type=float, default=1e-5, help='the learning rate of training')
    parser.add_argument('--set_seed', type=bool, default=True, help='Whether to fix the random seed')
    parser.add_argument('--seed', type=int, default=1024, help='fix the random seed for reproducible')
    parser.add_argument('--patient', type=int, default=10, help='the patient of early-stopping')
    parser.add_argument('--length', type=int, default=20, help='the max length of generated text')
    parser.add_argument('--output_dir', type=str, default='./output/output_examples')
    parser.add_argument('--hyp_only', type=bool, default=False)
    parser.add_argument('--alpha', type=float, default=0)
    parser.add_argument('--mode', type=str, default='discriminate_generate')
    # parser.add_argument('--length', type=int, default=22)
    parser.add_argument('--beam_size', type=int, default=5)
    parser.add_argument('--no_repeat_ngram_size', type=int, default=3)
    parser.add_argument('--repetition_penalty', type=float, default=1.5)
    parser.add_argument('--early_stopping', type=bool, default=False)
    parser.add_argument('--do_sample', type=bool, default=True)
    parser.add_argument('--attack_rate', type=float, default=0.015)

    # parsing the hyper-parameters from command line and define logger
    hps = parser.parse_args()
    logger, formatter = define_logger()
    # nowtime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
    log_path = os.path.join(hps.log_dir, hps.mode + '_' + hps.model_name + '.txt')

    file_handler = logging.FileHandler(log_path)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)

    # fix random seed
    if hps.set_seed:
        random.seed(hps.seed)
        np.random.seed(hps.seed)
        torch.manual_seed(hps.seed)
        torch.cuda.manual_seed(hps.seed)

    # load data
    # logger.info("[Pytorch] %s", torch.)
    logger.info("[INFO] Loading Data")
    logger.info("[INFO] Mode:\t{}".format(hps.mode))
    train_data = load_data(os.path.join(hps.data_dir, hps.train))
    dev_data = load_data(os.path.join(hps.data_dir, hps.dev))
    test_data = load_data(os.path.join(hps.data_dir, hps.test))

    # Tokenization
    logger.info("[INFO] Tokenization and Padding for Data")
    train_ids, train_mask, train_pos, train_labels, train_loss_labels, _, _, _ = tokenization(train_data, hps)
    dev, dev_mask, dev_pos, dev_labels, dev_loss_labels, dev_premise_ids, dev_premise_mask, dev_truth = tokenization(dev_data, hps)
    test, test_mask, test_pos, test_labels, test_loss_labels, test_premise_ids, test_premise_mask, test_truth = tokenization(test_data, hps)

    # Dataset and DataLoader
    logger.info("[INFO] Creating Dataset and splitting batch for data")
    TRAIN = TensorDataset(train_ids, train_mask, train_pos, train_labels, train_loss_labels)
    DEV = TensorDataset(dev, dev_mask, dev_pos, dev_labels, dev_loss_labels, dev_premise_ids, dev_premise_mask, dev_truth)
    TEST = TensorDataset(test, test_mask, test_pos, test_labels, test_loss_labels, test_premise_ids, test_premise_mask, test_truth)
    train_dataloader = DataLoader(TRAIN, batch_size=hps.batch_size, shuffle=hps.shuffle, drop_last=False)
    dev_dataloader = DataLoader(DEV, batch_size=hps.batch_size, shuffle=hps.shuffle, drop_last=False)
    test_dataloader = DataLoader(TEST, batch_size=hps.batch_size, shuffle=hps.shuffle, drop_last=False)

    # initialize model, optimizer, loss_function
    logger.info('[INFO] Loading pretrained model, setting optimizer and loss function')

    model = gpt2_multi_task(hps)

    optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=hps.lr)
    loss_function = nn.BCEWithLogitsLoss(reduction='mean')
    loss_function2 = nn.CrossEntropyLoss(reduction='mean', ignore_index=-100)

    # Multi-Gpu training
    if hps.cuda:
        gpu_ids = [int(x) for x in hps.gpu.split(' ')]
        model.cuda(gpu_ids[0])
        if len(gpu_ids) > 1:
            model = nn.DataParallel(model, device_ids=gpu_ids)

    # training
    logger.info("[INFO] Start Training")
    step = 0
    patient = 0
    best_accuracy = 0
    stop_train = False

    for epoch in range(hps.epochs):
        logger.info('[Epoch] {}'.format(epoch))
        t = trange(len(train_dataloader))
        epoch_step = 0
        total_loss = 0
        for i, batch in zip(t, train_dataloader):
            optimizer.zero_grad()
            model.train()
            if hps.cuda:
                batch = tuple(term.cuda() for term in batch)

            input_ids, input_mask, pos, label, loss_label = batch

            dis_logits, gen_logits = model(input_ids, attention_mask=input_mask, pos=pos)
            # pdb.set_trace()
            loss_dis = loss_function(dis_logits, label.float())

            # a1 = dis_logits[::2].unsqueeze(1)
            # a2 = dis_logits[1::2].unsqueeze(1)
            # a = torch.cat((a1, a2), 1)
            # index = a.argmax(1)

            t_1 = label[::2].unsqueeze(1)
            t_2 = label[1::2].unsqueeze(1)
            t_ = torch.cat((t_1, t_2), 1)
            t_index = t_.argmax(1)

            g1 = gen_logits[::2].unsqueeze(1)
            g2 = gen_logits[1::2].unsqueeze(1)
            g = torch.cat((g1, g2), 1)

            l1 = loss_label[::2].unsqueeze(1)
            l2 = loss_label[1::2].unsqueeze(1)
            l = torch.cat((l1, l2), 1)

            g = g[range(g.shape[0]), t_index, :]
            l = l[range(l.shape[0]), t_index, :]

            # pdb.set_trace()

            shift_logits = g[..., :-1, :].contiguous()
            shift_labels = l[..., 1:].contiguous()

            loss_gen = loss_function2(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
            loss = hps.alpha * loss_dis + (1 - hps.alpha) * loss_gen

            total_loss += loss.item()
            t.set_postfix(avg_loss='{}'.format(total_loss / (epoch_step + 1)))
            epoch_step += 1

            loss.backward()
            optimizer.step()

            if step % hps.evaluation_step == 0 and step != 0:
                model.eval()

                # with torch.no_grad():
                print('\n')
                logger.info("[Dev Evaluation] Start Evaluation on Dev Set")
                evaluate_output = evaluate(hps, model, dev_dataloader, loss_function, loss_function2, optimizer)
                dev_ppl = compute_ppl(hps, model, dev_data)
                logger.info("[Dev Metrics] Dev Perplexity: \t{}".format(dev_ppl))
                print('\n')
                logger.info("[Dev Metrics] Dev Accuracy: \t{}".format(evaluate_output[0]))
                logger.info("[DEV Metrics] Dev Attack Accuracy: \t{}".format(evaluate_output[-2]))
                logger.info(
                    "[Dev Metrics] Dev BLEU:\t({}, {}, {}, {})".format(evaluate_output[1], evaluate_output[2],
                                                                       evaluate_output[3], evaluate_output[4]))
                logger.info("[Dev Metrics] Dev Discriminate Loss: \t{}".format(evaluate_output[-3]))
                logger.info("[Dev Metrics] Dev Discriminate Attack Loss: \t{}".format(evaluate_output[-1]))
                logger.info(
                    "[Dev Metrics] Dev Rouge Recall:\t({}, {}, {})".format(evaluate_output[5], evaluate_output[6],
                                                                           evaluate_output[7]))

                if evaluate_output[0] >= best_accuracy:
                    patient = 0
                    best_accuracy = evaluate_output[0]
                    logger.info("[Saving] Saving Model to {}".format(hps.save_dir))
                    # torch.save(model, os.path.join(hps.save_dir, '{}_{}'.format('generated', hps.model_name)))
                    logger.info("[Test Evaluation] Start Evaluation on Test Set")

                    test_output = evaluate(hps, model, test_dataloader, loss_function, loss_function2, optimizer)
                    test_ppl = compute_ppl(hps, model, test_data)
                    logger.info("[Test Metrics] Test Perplexity: \t{}".format(test_ppl))

                    print('\n')
                    logger.info("[Test Metrics] Test Accuracy: \t{}".format(test_output[0]))
                    logger.info("[Test Metrics] Test Attack Accuracy: \t{}".format(test_output[-2]))
                    logger.info("[Test Metrics] Test BLEU:\t({}, {}, {}, {})".format(test_output[1], test_output[2],
                                                                                test_output[3],
                                                                                test_output[4]))
                    logger.info("[Test Metrics] Test Discriminate Loss: \t{}".format(test_output[-3]))
                    logger.info("[Test Metrics] Test Discriminate Attack Loss: \t{}".format(test_output[-1]))
                    logger.info(
                        "[Test Metrics] Test Rouge Recall:\t({}, {}, {})".format(test_output[5], test_output[6],
                                                                                test_output[7]))
                else:
                    patient += 1

                logger.info("[Patient] {}".format(patient))

                if patient >= hps.patient:
                    logger.info("[INFO] Stopping Training by Early Stopping")
                    stop_train = True
                    break
            step += 1

        if stop_train:
            break

if __name__ == '__main__':
    main()

max_stars_repo_name: Waste-Wood/e-CARE

max_issues_repo_name: Waste-Wood/e-CARE

max_forks_repo_name: Waste-Wood/e-CARE


text:
import numpy as np
from shapely.geometry import Polygon
import pyclipper

from vkit.label.type import VPolygon

def remove_duplicated_xy_pairs(xy_pairs):
    xy_pairs = tuple(map(tuple, xy_pairs))
    unique_xy_pairs = []

    idx = 0
    while idx < len(xy_pairs):
        unique_xy_pairs.append(xy_pairs[idx])

        next_idx = idx + 1
        while next_idx < len(xy_pairs) and xy_pairs[idx] == xy_pairs[next_idx]:
            next_idx += 1
        idx = next_idx

    # Check head & tail.
    if len(unique_xy_pairs) > 1 and unique_xy_pairs[0] == unique_xy_pairs[-1]:
        unique_xy_pairs.pop()

    assert len(unique_xy_pairs) >= 3
    return unique_xy_pairs

def vatti_clip(polygon: VPolygon, ratio, shrink):
    xy_pairs = polygon.to_xy_pairs()

    shapely_polygon = Polygon(xy_pairs)
    assert shapely_polygon.area > 0

    distance = shapely_polygon.area * (1 - np.power(ratio, 2)) / shapely_polygon.length
    if shrink:
        distance *= -1

    clipper = pyclipper.PyclipperOffset()  # type: ignore
    clipper.AddPath(xy_pairs, pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON)  # type: ignore

    clipped_paths = clipper.Execute(distance)
    assert clipped_paths
    clipped_path = clipped_paths[0]

    clipped_xy_pairs = remove_duplicated_xy_pairs(clipped_path)
    clipped_polygon = VPolygon.from_xy_pairs(clipped_xy_pairs)

    return clipped_polygon, distance

def shrink_polygon(polygon, ratio):
    return vatti_clip(polygon, ratio, True)

def dilate_polygon(polygon, ratio):
    return vatti_clip(polygon, ratio, False)

ext: py

max_stars_repo_name: Yuri-x/vkit

max_issues_repo_name: Yuri-x/vkit

max_forks_repo_name: Yuri-x/vkit

reason: import numpy


text:
import math
import scipy
import pandas as pd
from rcmh import config
from sklearn import linear_model
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import cross_val_score

def count_partitions(n,d,k):
    nCd =  math.factorial(n)/(math.factorial(d)*math.factorial(n-d))
    return nCd*(k-1)**d

def get_uniform_transition_prob(n,d,k):
    count = 0
    for j in range(0,d+1):
        count += count_partitions(n,d,k)
    prob = 1/count
    return prob

def get_poisson_transition_prob(n,d,k,j):
    poisson = scipy.stats.poisson(j).pmf(d)
    count = count_partitions(n,d,k)
    prob = (1/count)*poisson
    return prob

def get_alpha(loss_actual,loss_candidate,t):
    print('loss_actual',loss_actual)
    print('loss_candidate',loss_candidate)
    pi_actual = math.exp(-1*(loss_actual-710000)/t)
    print('pi_actual',pi_actual)
    pi_candidate = math.exp(-1*(loss_candidate-710000)/t)
    print('pi_candidate',pi_candidate)
    print('div',pi_candidate/pi_actual)
    alpha = min(1,pi_candidate/pi_actual)
    return alpha

def get_loss(group):
    train_sample_x, train_sample_y = get_sample(group,'train')
    test_sample_x, test_sample_y = get_sample(group,'test')
    regr = linear_model.LinearRegression(n_jobs=4)
    regr.fit(train_sample_x,train_sample_y)
    test_hat_y = regr.predict(test_sample_x)
    mae = mean_absolute_error(test_hat_y,test_sample_y)
    return mae

def get_loss_upper(group):
    train_sample_x, train_sample_y = get_sample(group,'train')
    test_sample_x, test_sample_y = get_sample(group,'test_upper')
    regr = linear_model.LinearRegression()
    regr.fit(train_sample_x,train_sample_y)
    test_hat_y = regr.predict(test_sample_x)
    mae = mean_absolute_error(test_hat_y,test_sample_y)
    return mae

def get_loss_lower(group):
    train_sample_x, train_sample_y = get_sample(group,'train')
    test_sample_x, test_sample_y = get_sample(group,'test_lower')
    regr = linear_model.LinearRegression()
    regr.fit(train_sample_x,train_sample_y)
    test_hat_y = regr.predict(test_sample_x)
    mae = mean_absolute_error(test_hat_y,test_sample_y)
    return mae

def get_features_from_diag(sample,group):
    #df = pd.DataFrame(0,index=sample.index,columns=group.groups)
    #for column in config.diag_columns:
    #    df.loc[sample[column]==1,group.grouping[column]] = 1
    #return df
    a = sample[config.diag_columns].groupby(by=group.grouping,axis=1)
    return a.any()

def get_sample(group,type):
    #sample = config.diag_csv.sample(config.sample_size)
    if type == 'train':
        sample = config.train
    elif type == 'test_upper':
        sample = config.test[config.test['VALOR_TOT_2011'] >= config.test['VALOR_TOT_2011'].quantile(q=0.90)]
    elif type == 'test_lower':
        sample = config.test[config.test['VALOR_TOT_2011'] <= config.test['VALOR_TOT_2011'].quantile(q=0.10)]
    else:
        sample = config.test
    #sample_x = pd.concat([sample[config.feature_columns],get_features_from_diag(sample,group)],axis=1)
    sample_x = sample[config.feature_columns]
    sample_y = sample[config.target_columns]
    return sample_x,sample_y

def get_loss_cv(group):
    train_sample_x, train_sample_y = get_sample(group,'train')
    test_sample_x, test_sample_y = get_sample(group,'test')
    regr = linear_model.LinearRegression(n_jobs=4)
    #regr.fit(train_sample_x,train_sample_y)
    #test_hat_y = regr.predict(test_sample_x)
    #mae = mean_absolute_error(test_hat_y,test_sample_y)
    mae = cross_val_score(estimator=regr,X=train_sample_x,y=train_sample_y,cv=5,scoring='neg_mean_absolute_error')
    return mae

def get_loss_upper_cv(group):
    train_sample_x, train_sample_y = get_sample(group,'train')
    test_sample_x, test_sample_y = get_sample(group,'test_upper')
    regr = linear_model.LinearRegression()
    regr.fit(train_sample_x,train_sample_y)
    test_hat_y = regr.predict(test_sample_x)
    mae = mean_absolute_error(test_hat_y,test_sample_y)
    return mae

def get_loss_lower_cv(group):
    train_sample_x, train_sample_y = get_sample(group,'train')
    test_sample_x, test_sample_y = get_sample(group,'test_lower')
    regr = linear_model.LinearRegression()
    regr.fit(train_sample_x,train_sample_y)
    test_hat_y = regr.predict(test_sample_x)
    mae = mean_absolute_error(test_hat_y,test_sample_y)
    return mae

max_stars_repo_name: RamirezAmayaS/rcmh

max_issues_repo_name: RamirezAmayaS/rcmh

max_forks_repo_name: RamirezAmayaS/rcmh

reason: import scipy


text:
"""
This module provides uncertainty-aware functions that generalize some
of the functions from numpy.linalg.

(c) 2010-2016 by Eric O. LEBIGOT (EOL) <eric.lebigot@normalesup.org>.
"""

from uncertainties import __author__
from uncertainties.unumpy.core import inv, pinv

# This module cannot import unumpy because unumpy imports this module.

__all__ = ['inv', 'pinv']

max_stars_repo_name: AdityaSavara/uncertainties

max_issues_repo_name: AdityaSavara/uncertainties

max_forks_repo_name: AdityaSavara/uncertainties

reason: from numpy


text:
import numpy as np
import pickle
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
from mvn.utils.op import translate_quaternion_to_euler
angle_names = [
    'knee_angle_r',
    'hip_flexion_r',
    'hip_adduction_r',
    'hip_rotation_r',
    'hip_flexion_l',
    'hip_adduction_l',
    'hip_rotation_l',
    'knee_angle_l',
    'elbow_flexion_r',
    'shoulder_flex_r',
    'shoulder_add_r',
    'shoulder_rot_r',
    'shoulder_flex_l',
    'shoulder_add_l',
    'shoulder_rot_l',
    'elbow_flexion_l'
]

angle_names_overview = [
    'knee_angle',
    'hip_flexion',
    'hip_adduction',
    'hip_rotation',
    'shoulder_flex',
    'shoulder_add',
    'shoulder_rot',
    'elbow_flexion'
]

angle_pairs = [
    (0, 7), (1, 4), (2, 5), (3, 6),
    (9, 12), (10, 13), (11, 14), (8, 15)
]

def main(gt_angles_trajs, pred_angles_trajs):
    if pred_angles_trajs.shape[1] == 32:
        eulers = []
        for a in pred_angles_trajs:
            eulers.append(translate_quaternion_to_euler(list(a)))
        pred_angles_trajs = np.array(eulers)
    print('<Angle>: <MSE> | <MAE>')
    maes = []
    mses = []

    for idx, an in enumerate(angle_names):
        mae = mean_absolute_error(gt_angles_trajs[:, idx], pred_angles_trajs[:, idx])
        mse = mean_squared_error(gt_angles_trajs[:, idx], pred_angles_trajs[:, idx])
        maes.append(mae)
        mses.append(mse)
        print(f'{an}: {mse:.3f} rad | {mae:.3f} rad ({np.rad2deg(mae):.3f} deg)')

    print('-------------------')
    print('Overview')
    for idx, p in enumerate(angle_pairs):
        avg_mae = (maes[p[0]] + maes[p[1]]) / 2
        avg_mse = (mses[p[0]] + mses[p[1]]) / 2
        print(f'{angle_names_overview[idx]}: {avg_mse:.3f} rad | {avg_mae:.3f} rad ({np.rad2deg(avg_mae):.3f} deg)')

    print('-------------------')
    print('Average:')
    mae = mean_absolute_error(gt_angles_trajs, pred_angles_trajs)
    mse = mean_squared_error(gt_angles_trajs, pred_angles_trajs)
    r2 = r2_score(gt_angles_trajs, pred_angles_trajs)
    print(f'MAE: {mae} rad ({np.rad2deg(mae):.3f} deg)')
    print(f'MSE: {mse} rad')
    print(f'R2: {r2}')

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='Save predictions for easier analysis pipeline')
    parser.add_argument('--records', '-r', required=True, type=str, help='prediction records file')
    args = parser.parse_args()

    labels = np.load('../baseline-angles/roofing-multiview-v2.npy', allow_pickle=True).item()

    test_subjects = ['S08', 'S10']
    test_subjects  = list(labels['subject_names'].index(x) for x in test_subjects)

    indices = []

    mask = np.isin(labels['table']['subject_idx'], test_subjects, assume_unique=True)
    indices.append(np.nonzero(mask)[0][::1])

    labels['table'] = labels['table'][np.concatenate(indices)]

    angles_gt = np.deg2rad(labels['table']['angles'][:, :16])

    with open(args.records, 'rb') as infile:
        data = pickle.load(infile)
    angles_pred = data['angles']

    main(angles_gt, angles_pred)

max_stars_repo_name: Nyquixt/learnable-triangulation-pytorch

max_issues_repo_name: Nyquixt/learnable-triangulation-pytorch

max_forks_repo_name: Nyquixt/learnable-triangulation-pytorch


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Feb 12 18:01:14 2016

@author: lei

test sdp.model.wave.DispersionRelation
"""

import numpy as np

import sdp.model.wave.disprel as dr
import sdp.plasma.dielectensor as dt
import sdp.plasma.analytic.testparameter as tp
from sdp.settings.unitsystem import cgs

c = cgs['c']
m_e = cgs['m_e']
e = cgs['e']
keV = cgs['keV']
pi = np.pi

tp.set_parameter1D(ne_shape='uniform', ne_0=6.0e13, Te_shape='uniform',
                   Te_0=15*keV, B_0=3.0e4)

p1d = tp.create_profile1D()
p1d.setup_interps()
c_dielect = dt.ColdElectronColdIon(p1d)
r_dielect_1 = dt.RelElectronColdIon(p1d, max_harmonic=1, max_power=5)
r_dielect_2 = dt.RelElectronColdIon(p1d, max_harmonic=2, max_power=5)
r_dielect_3 = dt.RelElectronColdIon(p1d, max_harmonic=3, max_power=5)
r_dielect_4 = dt.RelElectronColdIon(p1d, max_harmonic=4, max_power=5)
r_dielect_5 = dt.RelElectronColdIon(p1d, max_harmonic=5, max_power=5)
r_dielect_1_1 = dt.RelElectronColdIon(p1d, max_harmonic=1, max_power=1)

omega = 124e9 * 2*pi
k_para = 0
n_perp_r = np.linspace(-0.1, 1, 100)
n_perp_i = np.linspace(-0.3, 0.3, 50)
k_perp_r = omega/c*n_perp_r
k_perp_i = omega/c*n_perp_i

k_perp = k_perp_r[np.newaxis, :] + 1j*k_perp_i[:, np.newaxis]

coordinates = [tp.Parameter1D['R_0']]

c_Lambd = dr.Lambda(omega, k_para, k_perp, c_dielect, coordinates)
r_Lambd1 = dr.Lambda(omega, k_para, k_perp, r_dielect_1, coordinates)
r_Lambd2 = dr.Lambda(omega, k_para, k_perp, r_dielect_2, coordinates)
r_Lambd3 = dr.Lambda(omega, k_para, k_perp, r_dielect_3, coordinates)
r_Lambd4 = dr.Lambda(omega, k_para, k_perp, r_dielect_4, coordinates)
r_Lambd5 = dr.Lambda(omega, k_para, k_perp, r_dielect_5, coordinates)
r_Lambd11 = dr.Lambda(omega, k_para, k_perp, r_dielect_1_1, coordinates)

max_stars_repo_name: LeiShi/Synthetic-Diagnostics-Platform


text:
import os
from cov.coverage import Coverage
import numpy as np

def fuzzing_analyzer(dtmc_wrapper_f, states_data_path):
    BSCov, BTCov = 0, 0
    states = np.load(states_data_path)
    coverage_handlers = []

    for criteria, k_step in [("state", 0), ("transition", 0)]:  # , ("k-step", 3), ("k-step", 6)
        cov = Coverage(dtmc_wrapper_f, criteria, k_step)
        coverage_handlers.append(cov)

    for i, coverage_handler in enumerate(coverage_handlers):
        cov = coverage_handler.get_coverage_criteria(states)
        total = coverage_handler.get_total()
        if i == 0:
            print("Basic State Coverage(BSCov):", len(cov) / total)
            BSCov = len(cov) / total
        if i == 1:
            print("Basic Transition Coverage(BTCov):", len(cov) / total)
            BTCov = len(cov) / total
        if coverage_handler.mode != "k-step":  # to printout the weighted coverage metrics
            weight_dic = coverage_handler.get_weight_dic()
            # print(sum([weight_dic[e] for e in cov]))
            rev_weight_dic = coverage_handler.get_weight_dic(reverse=True)
            # print(sum([rev_weight_dic[e] for e in cov]))
    return BSCov, BTCov

def get_BSCov_BTCov(wrapper, states_data):
    return fuzzing_analyzer(wrapper, states_data)

if __name__ == '__main__':
    wrapper = "../save_profile/nlu-model/snips_abst_train_model/wrapper_lstm_nlu_3_10.pkl"
    states_data = "../save_profile/nlu-model/snips_test_states/out_0_batch_0.npy"
    BSCov, BTCov = get_BSCov_BTCov(wrapper, states_data)
    print("BSCov: ", BSCov, "BTCov: ", BTCov)

max_stars_repo_name: cicilzx/DialTest

max_issues_repo_name: cicilzx/DialTest

max_forks_repo_name: cicilzx/DialTest


text:
# -*- coding: utf-8 -*-
import glob
import time
import datetime
import subprocess
import numpy as np
import requests
import supply
import logging
import os, sys
if sys.platform.lower() == "win32":
    os.system('color')

if not supply.isTest:
    try:
        import RPi.GPIO as GPIO
    except ImportError as e:
        raise type(e)(
            str(e) +
            "\ntry to test without GPIO by setting supply.isTest = True")

shouldPostToLocalhost = False  # the host server will set True when it starts

class PrintStyle():
    BLACK = lambda x: '\033[30m' + str(x)
    RED = lambda x: '\033[31m' + str(x)
    GREEN = lambda x: '\033[32m' + str(x)
    YELLOW = lambda x: '\033[33m' + str(x)
    BLUE = lambda x: '\033[34m' + str(x)
    MAGENTA = lambda x: '\033[35m' + str(x)
    CYAN = lambda x: '\033[36m' + str(x)
    WHITE = lambda x: '\033[37m' + str(x)
    UNDERLINE = lambda x: '\033[4m' + str(x)
    RESET = lambda x: '\033[0m' + str(x)

def init_devices():
    if supply.isTest:
        for name in supply.outPins:
            pin = supply.outPins[name]['pin']
            if pin is not None:
                state = supply.outPins[name]['state']
                if state is None:
                    supply.outPins[name]['state'] = 0
        return

    GPIO.setwarnings(False)
    GPIO.setmode(GPIO.BCM)
    for name in supply.outPins:
        pin = supply.outPins[name]['pin']
        if pin is not None:
            GPIO.setup(pin, GPIO.OUT)
            state = supply.outPins[name]['state']
            if state is None:
                state = int(GPIO.input(pin) != GPIO.HIGH)
                supply.outPins[name]['state'] = state
            set_pin(name, state)

    if supply.temperatureSource['device'].startswith('rasp'):  # raspberry Pi
        # if you prefer a physical pul-up, remove it here:
        GPIO.setup(supply.temperatureSource['pin'], GPIO.IN,
                   # pull_up_down=GPIO.PUD_UP
                   )
        w1Dir = '/sys/bus/w1/devices/'
        deviceFolders = glob.glob(w1Dir + '28*')
        supply.deviceFiles = [df + '/w1_slave' for df in deviceFolders]
        assert len(deviceFolders) == len(supply.temperatures)

    for name in supply.inPinsFromRaspberry:
        pin = supply.inPinsFromRaspberry[name]['pin']
        GPIO.setup(pin, GPIO.IN)

    if True:
        supply.lenTemperaturesArd = len(supply.temperatures) \
            if supply.temperatureSource['device'].startswith('ardu') else 0
        supply.lenSensorsArd = len(supply.sensorsFromArduino)
        supply.lenInPinsArd = len(supply.inPinsFromArduino)
    else:  # to test with GPIO but without Arduino:
        supply.lenTemperaturesArd = 0
        supply.lenSensorsArd = 0
        supply.lenInPinsArd = 0
        supply.deviceFiles = []
    if (supply.lenTemperaturesArd + supply.lenSensorsArd +
            supply.lenInPinsArd > 0):
        import serial
        supply.serial = serial
        arduinoFolder = glob.glob(supply.portArduino)[0]
        try:
            supply.arduinoSerial = serial.Serial(
                arduinoFolder, supply.portArduinoBaudRate,
                timeout=supply.arduinoBootDelay)
            supply.arduinoSerial.reset_input_buffer()
            supply.lastArduinoRes = []
            supply.lastArduinoTime = datetime.datetime.now()
        except serial.serialutil.SerialException as e:
            if supply.wantEmailAlarms:
                supply.arduino_alarm("Arduino is not connected!")
                raise e
            else:
                raise IOError("Cannot read from Arduino!")

    spiSensorsFromRaspberry = [
        sensor['kind'][1] for sensor in supply.sensorsFromRaspberry.values()
        if sensor['kind'][0].startswith('spi')]
    if len(spiSensorsFromRaspberry) > 0:
        import spidev
        supply.spi = spidev.SpiDev()
        supply.spi.open(0, 0)

    counterSensorsFromRaspberry = [
        sensor['kind'][1] for sensor in supply.sensorsFromRaspberry.values()
        if sensor['kind'][0].startswith('cou')]
    if len(counterSensorsFromRaspberry) > 0:
        pulseCounter = \
            {c: [0, 0, time.time()] for c in counterSensorsFromRaspberry}

        def _count(channel):
            pulseCounter[channel][0] += 1

        for sensor in counterSensorsFromRaspberry:
            GPIO.setup(sensor, GPIO.IN, pull_up_down=GPIO.PUD_UP)
            GPIO.add_event_detect(sensor, GPIO.FALLING, callback=_count)

def init_supply():
    for dd in list(supply.outPins.values()) + [supply.thermostat]:
        color = dd['color'][1:]  # without the leading '#'
        cr, cg, cb = [int(int(color[i:i+2], 16) * 0.75)
                      for i in range(0, len(color), 2)]
        dd['dimmedColor'] = '#' + ''.join(hex(c)[2:] for c in (cr, cg, cb))
    now = datetime.datetime.now()
    supply.thermostat['lastSwitch'] = now - supply.thermostat['delay']
    supply.lastGoodArduinoTime = now
    supply.lastGoodTemperatureTime = now
    supply.lastGoodMainsTime = now
    supply.lastGoodBLETime = now
    supply.lastEmailSentTime = now
    supply.aveT = None
    supply.feedStartTime = None

    if supply.wantEmailAlarms:
        import alarm
        assert alarm.check_set()
        supply.arduino_alarm = alarm.send_email
    else:
        supply.arduino_alarm = None

def set_pin(name, state):
    if state == 'switch':
        supply.outPins[name]['state'] ^= 1
    else:
        supply.outPins[name]['state'] = 1 if state in ('on', 'yes', 1) else 0

    pin = supply.outPins[name]['pin']
    if pin is None or supply.isTest:
        return
    # GPIO.HIGH means "off" for relays
    GPIO.output(pin, GPIO.LOW if supply.outPins[name]['state'] else GPIO.HIGH)

def get_pins():
    if supply.isTest:
        return

    for name in supply.outPins:
        pin = supply.outPins[name]['pin']
        if pin is None:
            continue
        state = int(GPIO.input(pin) == GPIO.HIGH)
        supply.outPins[name]['state'] = int(not state)
    for name in supply.inPinsFromRaspberry:
        pin = supply.inPinsFromRaspberry[name]['pin']
        state = int(GPIO.input(pin) == GPIO.HIGH)
        supply.outPins[name]['state'] = int(state)

    if supply.feedStartTime is not None:
        now = datetime.datetime.now()
        if now - supply.feedStartTime > supply.outPins['feeding']['delay']:
            for nameF in supply.outPins['feeding']['toSwitchOnAfter']:
                set_pin(nameF, 'on')
            set_pin('feeding', 'off')
            supply.feedStartTime = None
            dictToSend = {"pin": 'feeding', "state": 0,
                          "how": "Timer-switched", "skipset": 1}
            if shouldPostToLocalhost:
                try:
                    requests.post('http://localhost:80', json=dictToSend)
                except:  # noqa
                    pass
            try:
                requests.get('http://localhost:80')
            except:  # noqa
                pass

def _get_one_temperature_raspberry(fname):
    catdata = subprocess.Popen(
        ['cat', fname], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, _ = catdata.communicate()
    lines = out.decode('utf-8').split('\n')
    if lines[0].strip()[-3:] != 'YES':
        time.sleep(0.5)
        _get_one_temperature_raspberry(fname)
    equals_pos = lines[1].find('t=')
    if equals_pos == -1:
        return
    temp_string = lines[1][equals_pos+2:]
    res = float(temp_string) / 1000.0  # in C
    if "F" in supply.temperatureUnit:
        res = res*1.8 + 32
    return res

def _email_body():
    try:
        return "Cannot read from Arduino!\n" + \
            "Last succsessful reading on {0[0]} at {0[1]}:\n"\
            .format(str(supply.lastArduinoTime).split('.')[0].split(' ')) +\
            ", ".join([str(r) for r in supply.lastArduinoRes])
    except AttributeError:
        return

def _read_from_arduino(insist=True, attempt=0):
    now = datetime.datetime.now()
    lenExpected = supply.lenTemperaturesArd + supply.lenSensorsArd + \
        supply.lenInPinsArd
    try:
        ra = 0
        while supply.arduinoSerial.in_waiting <= 0:
            ra += 1
            time.sleep(supply.arduinoReadDelay)
            if ra >= supply.arduinoReadAttempts:
                raise ValueError('Arduino not ready')
        buf = supply.arduinoSerial.readline()
        try:
            res = [float(v) for v in buf.split()]
        except ValueError:
            bufOut = buf[:-2].decode()  # trailing \r\n
            try:
                if "temperature device" in bufOut:
                    bufS = bufOut.split()
                    PColor = PrintStyle.RED if int(bufS[1]) < \
                        supply.lenTemperaturesArd else PrintStyle.GREEN
                    print(PrintStyle.RESET(bufOut[:5]), PColor(bufOut[5:8]),
                          PrintStyle.RESET(bufOut[8:]))
                else:
                    raise Exception
            except Exception:
                print(bufOut)
            return _read_from_arduino(attempt=attempt)
        if len(res) != lenExpected:
            out = 'Wrong length={0} of arduino response, expected length={1}'\
                  '\ngot: {2}'.format(len(res), lenExpected, res)
            print(out)
            logging.warning(out)
            raise ValueError(out)
        supply.arduinoSerial.reset_input_buffer()
        supply.arduinoSerial.write("alive".encode())
        supply.lastGoodArduinoTime = now
        return res
    except (supply.serial.serialutil.SerialException, ValueError,
            OSError) as e:
        if not insist:
            return []
        try:
            supply.arduinoSerial.close()
            arduinoFolder = glob.glob(supply.portArduino)[0]
            print('Arduino rebooting attempt #{0}'.format(attempt+1))
            supply.arduinoSerial = supply.serial.Serial(
                arduinoFolder, supply.portArduinoBaudRate,
                timeout=supply.arduinoBootDelay)
            time.sleep(supply.arduinoBootDelay)
        except Exception as e:  # noqa
            logging.warning(e)
        if attempt >= supply.arduinoReadAttempts:
            if supply.wantEmailAlarms:
                if ((now - supply.lastGoodArduinoTime) >
                        supply.delayUntilArduinoAlarm):
                    supply.arduino_alarm(_email_body())
                    supply.lastEmailSentTime = now
                    return []
            else:
                out = "Cannot read from Arduino!"
                logging.error(out)
                raise IOError(out)
        return _read_from_arduino(attempt=attempt+1)

def _get_spi_raspberry(adcnum):
    supply.spi.max_speed_hz = 1350000
    raw = supply.spi.xfer2([1, (8+adcnum) << 4, 0])
    data = ((raw[1] & 3) << 8) + raw[2]
    return data

def get_sensors(insist=True):
    if supply.isTest:
        allSensors = (list(supply.temperatures.values()) +
                      list(supply.sensorsFromArduino.values()) +
                      list(supply.sensorsFromRaspberry.values()))
        rr = 0.1 * (np.random.random(len(allSensors)) - 0.5)
        for v, r in zip(allSensors, rr):
            v['value'] = round(v['value'] + r, 3)
    else:
        if (supply.lenTemperaturesArd + supply.lenSensorsArd +
                supply.lenInPinsArd > 0):
            arduinoRes = _read_from_arduino(insist)
            if arduinoRes:
                supply.lastArduinoRes = arduinoRes
                supply.lastArduinoTime = datetime.datetime.now()
        else:
            arduinoRes = []

        if supply.lenTemperaturesArd > 0:  # temperatures from arduino
            for v, ta in zip(supply.temperatures.values(), arduinoRes):
                v['value'] = ta
        else:  # temperatures from raspberry
            for v, df in zip(supply.temperatures.values(), supply.deviceFiles):
                v['value'] = _get_one_temperature_raspberry(df)

        for v, sa in zip(supply.sensorsFromArduino.values(),
                         arduinoRes[supply.lenTemperaturesArd:]):
            v['value'] = sa

        for v, pa in zip(supply.inPinsFromArduino.values(), arduinoRes[
                supply.lenTemperaturesArd+supply.lenSensorsArd:]):
            v['state'] = int(pa)

        for key, sensor in supply.sensorsFromRaspberry.items():
            channel = sensor['kind'][1]
            if sensor['kind'][0].startswith('cou'):
                now = time.time()
                ct = pulseCounter[channel][0]
                ct0 = pulseCounter[channel][1]
                then = pulseCounter[channel][2]
                rate = (ct - ct0) / (now - then)
                pulseCounter[channel][1] = ct
                pulseCounter[channel][2] = now
                if key == 'flow':
                    sensor['value'] = rate / 5.5 * 60  # L/h
            if sensor['kind'][0].startswith('spi'):
                channel = sensor['kind'][1]
                volt = _get_spi_raspberry(channel) * 5.0 / 1023.
                if key == 'turbidity':
                    sensor['value'] = -1120.4*volt**2 + 5742.3*volt - 4352.9
                if key == 'pH':
                    offset_pH = 7.0 - 6.88  # as measured
                    sensor['value'] = 3.5*volt + offset_pH
                if key == 'EC':
                    sensor['value'] = 1000 * volt / 820.0 / 200.0

    if supply.thermostat['active']:
        thermostat()

    check_alarms()

def _set_thermostat_pin(name, cond, now):
    state = int(cond)
    if supply.outPins[name]['state'] == state:  # do nothing
        return
    set_pin(name, state)
    supply.thermostat['lastSwitch'] = now
    dictToSend = {"pin": name, "state": state, "how": "Auto-switched",
                  "skipset": 1}
    if shouldPostToLocalhost:
        try:
            requests.post('http://localhost:80', json=dictToSend)
        except:  # noqa
            pass

def thermostat():
    now = datetime.datetime.now()
    if (now - supply.thermostat['lastSwitch']) < supply.thermostat['delay']:
        return
    allT = [v['value'] for v in supply.temperatures.values() if
            supply.temperatureOutlierLimits[0] < v['value'] <
            supply.temperatureOutlierLimits[1]]
    if len(allT) == 0:
        supply.aveT = None
        return
    supply.aveT = sum(allT) / len(allT)

    for tpn in supply.thermostat['heatingPins']:
        _set_thermostat_pin(
            tpn, supply.aveT < supply.thermostat['limits'][0], now)
    for tpn in supply.thermostat['coolingPins']:
        _set_thermostat_pin(
            tpn, supply.aveT > supply.thermostat['limits'][1], now)

def check_alarms():
    if not supply.wantEmailAlarms:
        return
    now = datetime.datetime.now()
    alarmTxt = ""
    interval = supply.intervalBetweenEmails

    if 'mains' in supply.wantEmailAlarms:
        if 'mains' in supply.inPinsFromRaspberry:
            mainsDict = supply.inPinsFromRaspberry['mains']
        elif 'mains' in supply.inPinsFromArduino:
            mainsDict = supply.inPinsFromArduino['mains']
        else:
            mainsDict = None
        if mainsDict is not None:
            if mainsDict['state'] == 0:
                if ((now - supply.lastGoodMainsTime) >
                        supply.delayUntilMainsAlarm):
                    alarmTxt += "The power supply is off!\n"
            else:
                supply.lastGoodMainsTime = now

    if 'BLE' in supply.wantEmailAlarms:
        if 'BLE' in supply.inPinsFromRaspberry:
            bleDict = supply.inPinsFromRaspberry['BLE']
        elif 'BLE' in supply.inPinsFromArduino:
            bleDict = supply.inPinsFromArduino['BLE']
        else:
            bleDict = None
        if bleDict is not None:
            if bleDict['state'] == 0:
                if ((now - supply.lastGoodBLETime) >
                        supply.delayUntilBLEAlarm):
                    alarmTxt += "BLE is not connected!\n"
            else:
                supply.lastGoodBLETime = now
        interval = supply.intervalBetweenEmailsBLE

    if 'temperature' in supply.wantEmailAlarms:
        if supply.aveT is None:
            if ((now - supply.lastGoodTemperatureTime) >
                    supply.delayUntilTemperatureAlarm):
                alarmTxt += u"All temperature readings are bad!"
        else:
            if (supply.temperatureAlarmLimits[0] < supply.aveT <
                    supply.temperatureAlarmLimits[1]):
                supply.lastGoodTemperatureTime = now
            else:
                if ((now - supply.lastGoodTemperatureTime) >
                        supply.delayUntilTemperatureAlarm):
                    alarmTxt += u"Dangerous temperature {0:.3f}{1}!".format(
                        supply.aveT, supply.temperatureUnit)

    if alarmTxt and ((now - supply.lastEmailSentTime) > interval):
        try:
            supply.arduino_alarm(alarmTxt)
            supply.lastEmailSentTime = now
        except Exception as e:
            logging.error(e)

if __name__ == "__main__":
    time.sleep(1)
    get_pins()
    get_sensors()
    print('pins = {0}'.format([v['state'] for v in supply.outPins.values()]))
    print('temperatures = {0}'.format(
        [v['value'] for v in supply.temperatures.values()]))
    for key, sensor in supply.sensorsFromRaspberry.items():
        print('{0} = {1:.3f} {2}'.format(key, sensor['value'], sensor['unit']))
    for key, sensor in supply.sensorsFromArduino.items():
        print('{0} = {1:.3f} {2}'.format(key, sensor['value'], sensor['unit']))

max_stars_repo_name: kklmn/ArdquariumPi

max_issues_repo_name: kklmn/ArdquariumPi

max_forks_repo_name: kklmn/ArdquariumPi


text:
#    Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
import shutil
import nnunet.utilities.shutil_sol as shutil_sol
from multiprocessing.pool import Pool

import numpy as np
from batchgenerators.utilities.file_and_folder_operations import *
from nnunet.configuration import default_num_threads
from nnunet.evaluation.evaluator import aggregate_scores
from nnunet.inference.segmentation_export import save_segmentation_nifti_from_softmax
from nnunet.paths import network_training_output_dir, preprocessing_output_dir
from nnunet.postprocessing.connected_components import determine_postprocessing

def merge(args):
    file1, file2, properties_file, out_file = args
    if not isfile(out_file):
        res1 = np.load(file1)['softmax']
        res2 = np.load(file2)['softmax']
        props = load_pickle(properties_file)
        mn = np.mean((res1, res2), 0)
        # Softmax probabilities are already at target spacing so this will not do any resampling (resampling parameters
        # don't matter here)
        save_segmentation_nifti_from_softmax(mn, out_file, props, 3, None, None, None, force_separate_z=None,
                                             interpolation_order_z=0)

def ensemble(training_output_folder1, training_output_folder2, output_folder, task, validation_folder, folds, allow_ensembling: bool = True):
    print("\nEnsembling folders\n", training_output_folder1, "\n", training_output_folder2)

    output_folder_base = output_folder
    output_folder = join(output_folder_base, "ensembled_raw")

    # only_keep_largest_connected_component is the same for all stages
    dataset_directory = join(preprocessing_output_dir, task)
    plans = load_pickle(join(training_output_folder1, "plans.pkl"))  # we need this only for the labels

    files1 = []
    files2 = []
    property_files = []
    out_files = []
    gt_segmentations = []

    folder_with_gt_segs = join(dataset_directory, "gt_segmentations")
    # in the correct shape and we need the original geometry to restore the niftis

    for f in folds:
        validation_folder_net1 = join(training_output_folder1, "fold_%d" % f, validation_folder)
        validation_folder_net2 = join(training_output_folder2, "fold_%d" % f, validation_folder)

        if not isdir(validation_folder_net1):
            raise AssertionError("Validation directory missing: %s. Please rerun validation with `nnUNet_train CONFIG TRAINER TASK FOLD -val --npz`" % validation_folder_net1)
        if not isdir(validation_folder_net2):
            raise AssertionError("Validation directory missing: %s. Please rerun validation with `nnUNet_train CONFIG TRAINER TASK FOLD -val --npz`" % validation_folder_net2)

        # we need to ensure the validation was successful. We can verify this via the presence of the summary.json file
        if not isfile(join(validation_folder_net1, 'summary.json')):
            raise AssertionError("Validation directory incomplete: %s. Please rerun validation with `nnUNet_train CONFIG TRAINER TASK FOLD -val --npz`" % validation_folder_net1)
        if not isfile(join(validation_folder_net2, 'summary.json')):
            raise AssertionError("Validation directory missing: %s. Please rerun validation with `nnUNet_train CONFIG TRAINER TASK FOLD -val --npz`" % validation_folder_net2)

        patient_identifiers1_npz = [i[:-4] for i in subfiles(validation_folder_net1, False, None, 'npz', True)]
        patient_identifiers2_npz = [i[:-4] for i in subfiles(validation_folder_net2, False, None, 'npz', True)]

        # we don't do postprocessing anymore so there should not be any of that noPostProcess
        patient_identifiers1_nii = [i[:-7] for i in subfiles(validation_folder_net1, False, None, suffix='nii.gz', sort=True) if not i.endswith("noPostProcess.nii.gz") and not i.endswith('_postprocessed.nii.gz')]
        patient_identifiers2_nii = [i[:-7] for i in subfiles(validation_folder_net2, False, None, suffix='nii.gz', sort=True) if not i.endswith("noPostProcess.nii.gz") and not i.endswith('_postprocessed.nii.gz')]

        if not all([i in patient_identifiers1_npz for i in patient_identifiers1_nii]):
            raise AssertionError("Missing npz files in folder %s. Please run the validation for all models and folds with the '--npz' flag." % (validation_folder_net1))
        if not all([i in patient_identifiers2_npz for i in patient_identifiers2_nii]):
            raise AssertionError("Missing npz files in folder %s. Please run the validation for all models and folds with the '--npz' flag." % (validation_folder_net2))

        patient_identifiers1_npz.sort()
        patient_identifiers2_npz.sort()

        assert all([i == j for i, j in zip(patient_identifiers1_npz, patient_identifiers2_npz)]), "npz filenames do not match. This should not happen."

        maybe_mkdir_p(output_folder)

        for p in patient_identifiers1_npz:
            files1.append(join(validation_folder_net1, p + '.npz'))
            files2.append(join(validation_folder_net2, p + '.npz'))
            property_files.append(join(validation_folder_net1, p) + ".pkl")
            out_files.append(join(output_folder, p + ".nii.gz"))
            gt_segmentations.append(join(folder_with_gt_segs, p + ".nii.gz"))

    p = Pool(default_num_threads)
    p.map(merge, zip(files1, files2, property_files, out_files))
    p.close()
    p.join()

    if not isfile(join(output_folder, "summary.json")) and len(out_files) > 0:
        aggregate_scores(tuple(zip(out_files, gt_segmentations)), labels=plans['all_classes'],
                     json_output_file=join(output_folder, "summary.json"), json_task=task,
                     json_name=task + "__" + output_folder_base.split("/")[-1], num_threads=default_num_threads)

    if allow_ensembling and not isfile(join(output_folder_base, "postprocessing.json")):
        # now lets also look at postprocessing. We cannot just take what we determined in cross-validation and apply it
        # here because things may have changed and may also be too inconsistent between the two networks
        determine_postprocessing(output_folder_base, folder_with_gt_segs, "ensembled_raw", "temp",
                                 "ensembled_postprocessed", default_num_threads, dice_threshold=0)

        out_dir_all_json = join(network_training_output_dir, "summary_jsons")
        json_out = load_json(join(output_folder_base, "ensembled_postprocessed", "summary.json"))

        json_out["experiment_name"] = output_folder_base.split("/")[-1]
        save_json(json_out, join(output_folder_base, "ensembled_postprocessed", "summary.json"))

        maybe_mkdir_p(out_dir_all_json)
        shutil_sol.copyfile(join(output_folder_base, "ensembled_postprocessed", "summary.json"),
                    join(out_dir_all_json, "%s__%s.json" % (task, output_folder_base.split("/")[-1])))

max_stars_repo_name: DIAGNijmegen/nnUNet

max_issues_repo_name: DIAGNijmegen/nnUNet

max_forks_repo_name: DIAGNijmegen/nnUNet


text:
import argparse
import os

import numpy as np
import onnxruntime
import torchaudio
import torch

SAMPLING_RATE = 16000
LENGTH_SECONDS = 4

def preprocess_audio(audio, total_samples):
    # total_samples = audio.shape[1]
    segment_length = SAMPLING_RATE * LENGTH_SECONDS
    n_segments = int(np.ceil(audio.shape[1] / segment_length))

    # Apply zero padding to the end if necessary
    samples_to_pad = n_segments * segment_length - total_samples
    if samples_to_pad > 0:
        zeros = torch.zeros(1, samples_to_pad)
        audio = torch.cat([audio, zeros], dim=0)

    # Reshape as batch [n_segments, 1, SAMPLING_RATE * LENGTH_SECONDS]
    audio = torch.reshape(audio, (n_segments, 1, segment_length))
    return audio

def postprocess_audio(audio, total_samples):
    n_segments = audio.shape[0]
    segment_length = audio.shape[2]

    # Select only the clean speech (channel 0)
    audio = audio[:, 0:1, :]

    # Apply a reshape to convert back to continuous audio
    audio = torch.reshape(audio, (1, segment_length * n_segments))
    audio = audio[:, 0:total_samples]
    return audio

if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("-i", "--input", required=True)
    ap.add_argument("-o", "--output", required=True)
    ap.add_argument("-m", "--model", required=True)
    args = ap.parse_args()

    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"  # see issue #152
    os.environ["CUDA_VISIBLE_DEVICES"] = ''

    assert os.path.isfile(args.model) and args.model.endswith(
        ".onnx"
    ), "ERROR: The provided ONNX argument is incorrect"

    ort_session = onnxruntime.InferenceSession(args.model)

    def to_numpy(tensor):
        return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()

    # Load and preprocess the audio
    audio_tensor, _ = torchaudio.load(args.input)
    total_samples = audio_tensor.shape[1]
    print(audio_tensor.shape)
    audio_tensor = preprocess_audio(audio_tensor, total_samples)
    print(audio_tensor.shape)

    # Compute ONNX Runtime output prediction
    ort_inputs = {ort_session.get_inputs()[0].name: to_numpy(audio_tensor)}
    ort_outs = ort_session.run(None, ort_inputs)
    ort_outs = ort_outs[0]

    print(ort_outs)

    # Apply postprocessing and save
    ort_outs = torch.from_numpy(ort_outs)
    print(ort_outs.shape)
    output_tensor = postprocess_audio(ort_outs, total_samples)

    output_tensor /= output_tensor.abs().max()
    print(output_tensor.shape)
    torchaudio.save(args.output, output_tensor, SAMPLING_RATE)

max_stars_repo_name: hmartelb/speech-denoising

max_issues_repo_name: hmartelb/speech-denoising

max_forks_repo_name: hmartelb/speech-denoising


text:
import sys
import numpy as np
import struct as strct
import string

class SwrBinaryStatements:
    integer = np.int32
    real = np.float64
    character = np.uint8
    integerbyte = 4
    realbyte = 8
    textbyte = 4

    def read_integer(self):
        intvalue = strct.unpack('i', self.file.read(1 * SwrBinaryStatements.integerbyte))[0]
        return intvalue

    def read_real(self):
        realvalue = strct.unpack('d', self.file.read(1 * SwrBinaryStatements.realbyte))[0]
        return realvalue

    def read_text(self):
        textvalue = np.fromfile(file=self.file, dtype=SwrBinaryStatements.character, count=16).tostring()
        return textvalue

    def read_obs_text(self, nchar=20):
        textvalue = np.fromfile(file=self.file, dtype=SwrBinaryStatements.character, count=nchar).tostring()
        return textvalue

    def read_record(self):
        if self.skip == True:
            lpos = self.file.tell() + ( self.nrecord * self.items * SwrBinaryStatements.realbyte )
            self.file.seek(lpos)
            x = np.zeros((self.nrecord * self.items), SwrBinaryStatements.real)
        else:
            x = np.fromfile(file=self.file, dtype=SwrBinaryStatements.real, count=self.nrecord * self.items)
        x.resize(self.nrecord, self.items)
        return x

    def read_items(self):
        if self.skip == True:
            lpos = self.file.tell() + ( self.items * SwrBinaryStatements.realbyte )
            self.file.seek(lpos)
            x = np.zeros((self.items), SwrBinaryStatements.real)
        else:
            x = np.fromfile(file=self.file, dtype=SwrBinaryStatements.real, count=self.items)
        return x

    def read_1dintegerarray(self):
        i = np.fromfile(file=self.file, dtype=SwrBinaryStatements.integer, count=self.nrecord)
        return i

class SwrObs(SwrBinaryStatements):
    'Reads binary SWR observations output from MODFLOW SWR Process binary observation files'

    def __init__(self, filename, verbose=False):
        # initialize class information
        self.skip = False
        self.verbose = verbose
        # open binary head file
        self.file = open(filename, 'rb')
        # NOBS
        self.nobs = self.read_integer()
        self.v = np.empty((self.nobs), dtype='float')
        self.v.fill(1.0E+32)
        # read obsnames
        obsnames = []
        for idx in range(0, self.nobs):
            cid = self.read_obs_text()
            obsnames.append(cid)
        self.obsnames = np.array(obsnames)
        # set position
        self.datastart = self.file.tell()
        #get times
        self.times = self.time_list()

    def get_time_list(self):
        return self.times

    def get_num_items(self):
        return self.nobs

    def get_obs_labels(self):
        return self.obsnames

    def rewind_file(self):
        self.file.seek(self.datastart)
        return True

    def time_list(self):
        self.skip = True
        self.file.seek(self.datastart)
        times = []
        while True:
            current_position = self.file.tell()
            totim, v, success = next(self)
            if success == True:
                times.append([totim, current_position])
            else:
                self.file.seek(self.datastart)
                times = np.array(times)
                self.skip = False
                return times

    def __iter__(self):
        return self

    def read_header(self):
        try:
            totim = self.read_real()
            return totim, True
        except:
            return -999., False

    def __next__(self):
        totim, success = self.read_header()
        if (success):
            for idx in range(0, self.nobs):
                self.v[idx] = self.read_real()
        else:
            if self.verbose == True:
                print('_BinaryObs object.next() reached end of file.')
            self.v.fill(1.0E+32)
        return totim, self.v, success

    def get_values(self, idx):
        iposition = int(self.times[idx, 1])
        self.file.seek(iposition)
        totim, v, success = next(self)
        if success == True:
            return totim, v, True
        else:
            self.v.fill(1.0E+32)
            return 0.0, self.v, False

    def get_time_gage(self, record):
        idx = -1
        try:
            idx = int(record) - 1
            if self.verbose == True:
                print('retrieving SWR observation record [{0}]'.format(idx + 1))
        except:
            for icnt, cid in enumerate(self.obsnames):
                if record.strip().lower() == cid.strip().lower():
                    idx = icnt
                    if self.verbose == True:
                        print('retrieving SWR observation record [{0}] {1}'.format(idx + 1, record.strip().lower()))
                    break
        gage_record = np.zeros((2))  # tottime plus observation
        if idx != -1 and idx < self.nobs:
            # --find offset to position
            ilen = self.get_point_offset(idx)
            # get data
            for time_data in self.times:
                self.file.seek(int(time_data[1]) + ilen)
                v = self.read_real()
                this_entry = np.array([float(time_data[0])])
                this_entry = np.hstack((this_entry, v))
                gage_record = np.vstack((gage_record, this_entry))
            #delete the first 'zeros' element
            gage_record = np.delete(gage_record, 0, axis=0)
        return gage_record

    def get_point_offset(self, ipos):
        self.file.seek(self.datastart)
        lpos0 = self.file.tell()
        point_offset = int(0)
        totim, success = self.read_header()
        idx = (ipos)
        lpos1 = self.file.tell() + idx * SwrBinaryStatements.realbyte
        self.file.seek(lpos1)
        point_offset = self.file.tell() - lpos0
        return point_offset

class SwrFile(SwrBinaryStatements):
    def __init__(self, swrtype, filename, verbose=False):
        # --swrtype =  0 = stage record
        # swrtype = -1 = reach group record
        # swrtype = -2 = reach group connection velocity record
        # swrtype >  0 = aq-reach exchange record type = nlay
        self.file = open(filename, 'rb')
        self.type = None
        try:
            ctype = swrtype.lower()
        except:
            ctype = None
            pass
        if ctype is not None:
            self.type = ctype
        else:
            try:
                itype = int(swrtype)
            except:
                print('SWR data type not defined')
                raise
            if itype == 0:
                self.type = 'stage'
            elif itype == -1:
                self.type = 'reachgroup'
            elif itype == -2:
                self.type = 'qm'
            elif itype > 0:
                self.type = 'qaq'
        if self.type is None:
            print('undefined SWR data type')
            raise

        self.verbose = verbose
        self.nrgout = 0
        if self.type == 'qm':
            self.nrgout = self.read_integer()
        self.nrecord = self.read_integer()
        self.items = self.get_num_items()
        self.null_record = np.zeros((self.nrecord, self.items)) + 1.0E+32
        #
        self.missingData = -9999.9
        self.dataAvailable = True
        self.skip = False
        #read connectivity for velocity data if necessary
        if self.type == 'qm':
            self.connectivity = self.read_connectivity()
            if self.verbose == True:
                print(self.connectivity)
        # initialize reachlayers and nqaqentries for qaq data
        if self.type == 'qaq':
            self.reachlayers = np.zeros((self.nrecord), np.int)
            self.nqaqentries = 0
        self.qaq_dtype = np.dtype([('layer', 'i4'),
                                   ('bottom', 'f8'), ('stage', 'f8'),
                                   ('depth', 'f8'), ('head', 'f8'),
                                   ('wetper', 'f8'), ('cond', 'f8'),
                                   ('headdiff', 'f8'), ('qaq', 'f8')])

        self.datastart = self.file.tell()
        #get times
        self.times = self.time_list()

    def get_nrecords(self):
        return self.nrgout, self.nrecord

    def get_time_list(self):
        return self.times

    def read_connectivity(self):
        conn = np.zeros((self.nrecord, 3), np.int)
        icount = 0
        for nrg in range(0, self.nrgout):
            nconn = self.read_integer()
            for ic in range(0, nconn):
                conn[icount, 0] = nrg
                conn[icount, 1] = self.read_integer()
                conn[icount, 2] = self.read_integer()
                icount += 1
        return conn

    def get_connectivity(self):
        if self.type == 'qm':
            return self.connectivity
        else:
            return None

    def get_num_items(self):
        if self.type == 'stage':
            return 1
        elif self.type == 'reachgroup':
            return 14
        elif self.type == 'qm':
            return 2
        elif self.type == 'qaq':
            return 10
        else:
            return -1

    def get_header_items(self):
        return ['totim', 'dt', 'kper', 'kstp', 'swrstp', 'success_flag']

    def get_item_list(self):
        if self.type == 'stage':
            list = ['stage']
        if self.type == 'reachgroup':
            list = ['stage', 'qsflow', 'qlatflow', 'quzflow', 'rain', 'evap',
                    'qbflow', 'qeflow', 'qexflow', 'qbcflow', 'qcrflow', 'dv', 'inf-out', 'volume']
        if self.type == 'qm':
            list = ['flow', 'velocity']
        if self.type == 'qaq':
            list = ['reach', 'layer', 'bottom', 'stage', 'depth', 'head',
                    'wetper', 'cond', 'headdiff', 'qaq']
        return list

    def get_temporal_list(self):
        list = ['totim', 'dt', 'kper', 'kstp', 'swrstp', 'success']
        return list

    def get_item_number(self, value, isTimeSeriesOutput=True):
        l = self.get_item_list()
        ioff = 6
        if isTimeSeriesOutput == False:
            ioff = 0
        try:
            i = l.index(value.lower())
            i += ioff
        except ValueError:
            l = self.get_temporal_list()
            try:
                i = l.index(value.lower())
            except ValueError:
                i = -1  # -no match
                print('no match to: ', value.lower())
        return i

    def return_gage_item_from_list(self, r, citem, scale=1.0):
        ipos = self.get_item_number(citem)
        n = r.shape[0]
        if n < 1:
            return self.null_record
        v = np.zeros((n), np.float)
        for i in range(0, n):
            v[i] = r[i, ipos] * scale
        return v

    def read_header(self):
        if self.type == 'qaq':
            try:
                self.nqaqentries = 0
                for i in range(0, self.nrecord):
                    self.reachlayers[i] = self.read_integer()
                    self.nqaqentries += self.reachlayers[i]
                    # print i+1, self.reachlayers[i]
                    #print self.nqaqentries
            except:
                if self.verbose == True:
                    sys.stdout.write('\nCould not read reachlayers')
                return 0.0, 0.0, 0, 0, 0, False
        try:
            totim = self.read_real()
            dt = self.read_real()
            kper = self.read_integer()
            kstp = self.read_integer()
            swrstp = self.read_integer()
            return totim, dt, kper, kstp, swrstp, True
        except:
            return 0.0, 0.0, 0, 0, 0, False

    def get_record(self, *args):
        # --pass a tuple of timestep,stress period
        try:
            kkspt = args[0]
            kkper = args[1]
            while True:
                totim, dt, kper, kstp, swrstp, success, r = next(self)
                if success == True:
                    if kkspt == kstp and kkper == kper:
                        if self.verbose == True:
                            print(totim, dt, kper, kstp, swrstp, True)
                        return totim, dt, kper, kstp, swrstp, True, r
                else:
                    return 0.0, 0.0, 0, 0, 0, False, self.null_record
        except:
            # pass a scalar of target totim -
            # returns either a match or the first
            # record that exceeds target totim
            try:
                ttotim = float(args[0])
                while True:
                    totim, dt, kper, kstp, swrstp, r, success = next(self)
                    if success == True:
                        if ttotim <= totim:
                            return totim, dt, kper, kstp, swrstp, True, r
                    else:
                        return 0.0, 0.0, 0, 0, 0, False, self.null_record
            except:
                # get the last successful record
                previous = next(self)
                while True:
                    this_record = next(self)
                    if this_record[-2] == False:
                        return previous
                    else:
                        previous = this_record

    def get_gage(self, rec_num=0, iconn=0, rec_lay=1):
        if self.type == 'qaq':
            gage_record = np.zeros((self.items + 6))  # items plus 6 header values, reach number, and layer value
        else:
            gage_record = np.zeros((self.items + 6))  # items plus 6 header values
        while True:
            totim, dt, kper, kstp, swrstp, success, r = next(self)
            if success == True:
                this_entry = np.array([totim, dt, kper, kstp, swrstp, success])
                irec = rec_num - 1
                # find correct entry for record and layer
                if self.type == 'qaq':
                    ifound = 0
                    ilay = rec_lay
                    ilen = np.shape(r)[0]
                    #print np.shape(r)
                    for i in range(0, ilen):
                        ir = int(r[i, 0])
                        il = int(r[i, 1])
                        if ir == rec_num and il == ilay:
                            ifound = 1
                            irec = i
                            break
                    if ifound < 1:
                        r[irec, :] = 0.0
                elif self.type == 'qm':
                    ifound = 0
                    for i in range(0, self.nrecord):
                        inode = self.connectivity[i, 1]
                        ic = self.connectivity[i, 2]
                        if rec_num == inode and ic == iconn:
                            ifound = 1
                            irec = i
                            break
                    if ifound < 1:
                        r[irec, :] = 0.0

                this_entry = np.hstack((this_entry, r[irec]))
                gage_record = np.vstack((gage_record, this_entry))

            else:
                gage_record = np.delete(gage_record, 0, axis=0)  # delete the first 'zeros' element
                return gage_record

    def __next__(self):
        totim, dt, kper, kstp, swrstp, success = self.read_header()
        if success == False:
            if self.verbose == True:
                print('SWR_Record.next() object reached end of file')
            return 0.0, 0.0, 0, 0, 0, False, self.null_record
        else:
            if self.type == 'qaq':
                r = self.read_qaq()
                return totim, dt, kper, kstp, swrstp, True, r
            else:
                r = self.read_record()
        return totim, dt, kper, kstp, swrstp, True, r

    def read_qaq(self):
        x = np.zeros((self.nqaqentries, self.items), SwrBinaryStatements.real)
        if self.skip == True:
            bytes = self.nqaqentries * (SwrBinaryStatements.integerbyte + 8 * SwrBinaryStatements.realbyte)
            lpos = self.file.tell() + ( bytes )
            self.file.seek(lpos)
        else:
            qaq_list = self.get_item_list()
            bd = np.fromfile(self.file, dtype=self.qaq_dtype, count=self.nqaqentries)
            ientry = 0
            for irch in range(self.nrecord):
                klay = self.reachlayers[irch]
                for k in range(klay):
                    x[ientry, 0] = irch + 1
                    ientry += 1
            for idx, k in enumerate(qaq_list[1:]):
                x[:, idx + 1] = bd[k]
        # print 'shape x: {}'.format(x.shape)
        return x

    def rewind_file(self):
        self.file.seek(self.datastart)
        return True

    def time_list(self):
        self.skip = True
        self.file.seek(self.datastart)
        idx = 0
        sys.stdout.write('Generating SWR binary data time list\n')
        times = []
        while True:
            # --output something to screen so it is possible to determine
            #  that the time list is being created
            idx += 1
            v = divmod(float(idx), 100.)
            if v[1] == 0.0:
                sys.stdout.write('.')
            # get current position
            current_position = self.file.tell()
            totim, dt, kper, kstp, swrstp, success, r = next(self)
            if success == True:
                times.append([totim, dt, kper, kstp, swrstp, current_position])
            else:
                self.file.seek(self.datastart)
                times = np.array(times)
                self.skip = False
                sys.stdout.write('\n')
                return times

    def get_time_record(self, time_index=0):
        self.file.seek(int(self.times[time_index][5]))
        totim, dt, kper, kstp, swrstp, success, r = next(self)
        if success == True:
            if self.verbose == True:
                print(totim, dt, kper, kstp, swrstp, True)
            return totim, dt, kper, kstp, swrstp, True, r
        else:
            return 0.0, 0.0, 0, 0, 0, False, self.null_record

    def get_point_offset(self, rec_num, iconn):
        self.file.seek(self.datastart)
        lpos0 = self.file.tell()
        point_offset = int(0)
        totim, dt, kper, kstp, swrstp, success = self.read_header()
        # --qaq terms
        if self.type == 'qaq':
            sys.stdout.write('MFBinaryClass::get_point_offset can not be used to extract QAQ data')
            sys.exit(1)
        # stage and reach group terms
        elif self.type == 'stage' or self.type == 'reachgroup':
            idx = (rec_num - 1) * self.items
            lpos1 = self.file.tell() + idx * SwrBinaryStatements.realbyte
            self.file.seek(lpos1)
            point_offset = self.file.tell() - lpos0
        # connection flux and velocity terms
        elif self.type == 'qm':
            frec = -999
            for i in range(0, self.nrecord):
                inode = self.connectivity[i, 1]
                ic = self.connectivity[i, 2]
                if rec_num == inode and ic == iconn:
                    frec = i
                    break
            if frec == -999:
                self.dataAvailable = False
            else:
                self.dataAvailable = True
                idx = (frec) * self.items
                lpos1 = self.file.tell() + idx * SwrBinaryStatements.realbyte
                self.file.seek(lpos1)
                point_offset = self.file.tell() - lpos0
        return point_offset

    def get_time_gage(self, rec_num=0, iconn=0):
        if self.type == 'qaq':
            sys.stdout.write('MFBinaryClass::get_time_gage can not be used to extract QAQ data\n')
            sys.exit(1)
        num_records = self.items + 6  # items plus 6 header values
        gage_record = np.zeros((num_records), np.float)
        # --find offset to position
        ilen = int(0)
        if rec_num > 0:
            ilen = self.get_point_offset(rec_num, iconn)
        else:
            self.dataAvailable = False
        if self.dataAvailable == False:
            sys.stdout.write('  Error: data is not available for reach {0} '.format(rec_num))
            if self.type == 'qm':
                sys.stdout.write('connected to reach {0}'.format(iconn))
            sys.stdout.write('\n')
        # get data
        if len(self.times) > 0:
            for time_data in self.times:
                totim = time_data[0]
                dt = time_data[1]
                kper = time_data[2]
                kstp = time_data[3]
                swrstp = time_data[4]
                success = True
                # get data
                if self.dataAvailable == True:
                    self.file.seek(int(time_data[5]) + ilen)
                    r = self.read_items()
                else:
                    r = np.empty((self.items), np.float)
                    r.fill(self.missingData)
                # push the data to the data structure
                this_entry = np.array([totim, dt, kper, kstp, swrstp, success])
                # update this_entry and current gage_record
                this_entry = np.hstack((this_entry, r))
                gage_record = np.vstack((gage_record, this_entry))
        # delete first empty entry and return gage_record
        gage_record = np.delete(gage_record, 0, axis=0)  #delete the first 'zeros' element
        return gage_record

max_stars_repo_name: langevin/flopy

max_issues_repo_name: langevin/flopy

max_forks_repo_name: langevin/flopy


text:
import sys
import os
import numpy.testing as npt
import numpy.testing.decorators as dec

import pytest

import nitime.lazyimports as l

# The next test requires nitime.lazyimports.disable_lazy_imports to have been
# set to false, otherwise the lazy import machinery is disabled and all imports
# happen at l.LazyImport calls which become equivalent to regular import
# statements
@pytest.mark.skipif(l.disable_lazy_imports, reason="Lazy imports disabled")
def test_lazy():
    mlab = l.LazyImport('matplotlib.mlab')
    # repr for mlab should be <module 'matplotlib.mlab' will be lazily loaded>
    assert 'lazily loaded' in repr(mlab)
    # accessing mlab's attribute will cause an import of mlab
    npt.assert_equal(mlab.dist(1969,2011), 42.0)
    # now mlab should be of class LoadedLazyImport an repr(mlab) should be
    # <module 'matplotlib.mlab' from # '.../matplotlib/mlab.pyc>
    assert 'lazily loaded' not in repr(mlab)

# A known limitation of our lazy loading implementation is that, when it it is
# enabled, reloading the module raises an ImportError, and it also does not
# actually perform a reload, as demonstrated by this test.
@pytest.mark.skipif(l.disable_lazy_imports, reason="Lazy imports disabled")
def test_lazy_noreload():
    "Reloading of lazy modules causes ImportError"
    mod = l.LazyImport('sys')
    # accessing module dictionary will trigger an import
    len(mod.__dict__)
    # do not use named tuple feature for Python 2.6 compatibility
    major, minor = sys.version_info[:2]
    if major == 2:
        with pytest.raises(ImportError) as e_info:
            reload(mod)
    elif major == 3:
        import imp
        with pytest.raises(ImportError) as e_info:
            imp.reload(mod)


text:
#encoding=utf8
import numpy as np
import re
import codecs
from tensorflow.contrib import learn

from config import Config

config = Config()

def preprocess():
    # Data Preparation
    print("Loading data...")
    x_text, y = load_data_and_labels(config.positive_data_file, config.negative_data_file)
    # print('x_text, y',x_text, y)

    # Build vocabulary
    print('Build vocabulary...')
    max_document_length = max([len(x.split(" ")) for x in x_text])#获取最长的句子长度
    vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length)
    x = np.array(list(vocab_processor.fit_transform(x_text)))#获取训练语料词序列标签
    #print('vocab_processor.vocabulary_',len(vocab_processor.vocabulary_))

    # Randomly shuffle data
    np.random.seed(10)#设置种子确保每次随机筛选的数字相同
    shuffle_indices = np.random.permutation(np.arange(len(y))) #生成随机序列
    x_shuffled = x[shuffle_indices]
    y_shuffled = y[shuffle_indices]

    # Split train/test set
    # TODO: This is very crude, should use cross-validation
    dev_sample_index = -1 * int(config.dev_sample_percentage * float(len(y)))
    x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]
    y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]
    # print('dev_sample_index',dev_sample_index)
    del x, y, x_shuffled, y_shuffled
    print('x_dev',x_dev,x_dev.shape)
    print('y_dev',y_dev,y_dev.shape)
    print('max_document_length',max_document_length)
    print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_)))
    print("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev)))
    return x_train, y_train, vocab_processor, x_dev, y_dev

def clean_str(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

def load_data_and_labels(positive_data_file, negative_data_file):
    """
    Loads MR polarity data from files, splits the data into words and generates labels.
    Returns split sentences and labels.
    """
    # Load data from files
    positive_examples = list(codecs.open(positive_data_file, "r", encoding='utf-8').readlines())
    positive_examples = [s.strip() for s in positive_examples]
    negative_examples = list(codecs.open(negative_data_file, "r", encoding='utf-8').readlines())
    negative_examples = [s.strip() for s in negative_examples]
    # Split by words
    x_text = positive_examples + negative_examples
    x_text = [clean_str(sent) for sent in x_text]
    # Generate labels
    positive_labels = [[0, 1] for _ in positive_examples]
    negative_labels = [[1, 0] for _ in negative_examples]
    y = np.concatenate([positive_labels, negative_labels], 0)
    return [x_text, y]

def batch_iter(data, batch_size, num_epochs, shuffle=True):
    """
    Generates a batch iterator for a dataset.
    """
    data = np.array(data)
    data_size = len(data)
    print('data_size',data_size)
    num_batches_per_epoch = int((len(data)-1)/batch_size) + 1
    print('num_batches_per_epoch ',num_batches_per_epoch)
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        if shuffle:
            shuffle_indices = np.random.permutation(np.arange(data_size))
            shuffled_data = data[shuffle_indices]
        else:
            shuffled_data = data
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]