text:
import sys
import contextlib
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from seqeval.metrics import classification_report as class_report
import copy

def run_full_code_char(device, model_output_file, data_dir, glove_embeddings_file, vocabulary_output_file):
    path = data_dir
    if path.endswith("/") == False:
        path += "/"

    def load_glove_emb(path_glove):
        word2vectors, word2id = {}, {}
        count = 0
        with open(f'{path_glove}', 'rb') as f:
            for l in f:
                line = l.decode().split()
                word = line[0]
                word2vectors[word] = np.array(line[1:]).astype(np.float)
                word2id[word] = count
                count +=1
        return word2vectors, word2id

    word2vectors, word2id = load_glove_emb(glove_embeddings_file)

    def read_file(path):
        with open(path) as f:
            l = f.readlines()
        l = [x.strip() for x in l]
        while len(l[0]) == 0:
            l = l[1:]
        token = []
        label = []
        temp = []
        for i in range(len(l)):
            if len(l[i]) == 0:
                label.append([])
                token.append([])
                for x in temp:
                    x = x.split(" ")
                    token[-1].append(x[0])
                    label[-1].append(x[3])
                temp = []
            else:
                temp.append(l[i])
        return token, label

    # Using readlines()
    file1 = open(path + 'train.txt', 'r')
    Lines = file1.readlines()
    count = 0
    counttag = 0
    countchar = 0
    # Strips the newline character
    vocabtoidx = {}
    idxtovocab = {}
    countvocab = {}
    chartoidx = {}
    idxtochar = {}
    tagtoidx = {}
    idxtotag = {}
    for line in Lines:
        temp = line.strip().split(' ')
        if len(temp[0]) == 0:
            continue
        if (temp[0] not in countvocab):
            countvocab[temp[0]] = 1
        else:
            countvocab[temp[0]] += 1
    threshold = 2
    for line in Lines:
        temp = line.strip().split(' ')
        if(len(temp[0]) == 0 ):
            continue
        if((temp[0] not in vocabtoidx) and countvocab[temp[0]] >= threshold):
            vocabtoidx[temp[0]] = count
            idxtovocab[count] = temp[0]
            for i in range(0, len(temp[0])):
                if(temp[0][i] not in chartoidx):
                    chartoidx[temp[0][i]] = countchar
                    idxtochar[countchar] = temp[0]
                    countchar += 1
            count += 1
        if((temp[3] not in tagtoidx)):
            tagtoidx[temp[3]] = counttag
            idxtotag[counttag] = temp[3]
            counttag += 1

    pad_char_idx = countchar
    countchar += 1
    unknown_char_idx = countchar
    countchar += 1
    charvocabsize = countchar
    pad_idx = count
    count += 1
    unknown_idx = count
    count += 1
    vocab_size = count

    # dump dicts to vocab_file
    def dict2string(d):
        s = ""
        for x in d:
            s += x + " " + str(d[x]) + " "
        return s

    with open(vocabulary_output_file, "w") as f:
        f.write(dict2string(tagtoidx) + "\n")
        f.write(dict2string(vocabtoidx) + "\n")
        f.write(dict2string(chartoidx) + "\n")

    def prepare_dataset(path):
        data, label = read_file(path)
        char_data = copy.deepcopy(data)
        for i in range(len(data)):
            for j in range(len(data[i])):
                char_list = []
                for k in range(len(data[i][j])):
                    if data[i][j][k] in chartoidx:
                        char_list.append(chartoidx[data[i][j][k]])
                    else:
                        char_list.append(unknown_char_idx)
                if data[i][j] in vocabtoidx:
                    data[i][j] = vocabtoidx[data[i][j]]
                else:
                    data[i][j] = unknown_idx
                char_data[i][j] = char_list
                label[i][j] = tagtoidx[label[i][j]]
        return data , label, char_data

    train_data, train_label, train_char_data = prepare_dataset(path + "train.txt")
    val_data, val_label, val_char_data = prepare_dataset(path + "dev.txt")

    np.random.seed(0)
    embedding = np.random.uniform(low = -1, high = 1, size=(vocab_size, 100))
    for i in range(vocab_size-2):
        if idxtovocab[i] in word2vectors:
            embedding[i] = word2vectors[idxtovocab[i]]

    class CustomDataset(Dataset):
        def __init__(self, data, char_data, label):
            self.data = data
            self.label = label
            self.char_data = char_data
        def __len__(self):
            return len(self.data)
        def __getitem__(self, idx):
            return self.data[idx], self.char_data[idx], self.label[idx]

    def collate_fn_padd(batch):
        sentences = []
        labels = []
        char_words = []
        max_len = 0
        for x in batch:
            sent, char_sen, lab = x
            sentences.append(torch.Tensor(sent).to(device))
            labels.append(torch.Tensor(lab).to(device))
        sentences = torch.nn.utils.rnn.pad_sequence(sentences, padding_value = pad_idx, batch_first = True)
        labels = torch.nn.utils.rnn.pad_sequence(labels, padding_value = 0, batch_first = True)
        pad_word = []
        if 'p' in chartoidx :
            pad_word.append(chartoidx['p'])
        else:
            pad_word.append(unknown_char_idx)
        if 'a' in chartoidx :
            pad_word.append(chartoidx['a'])
        else:
            pad_word.append(unknown_char_idx)
        if 'd' in chartoidx :
            pad_word.append(chartoidx['d'])
        else:
            pad_word.append(unknown_char_idx)
        for i in range(0, sentences.shape[0]):
            sen, char_sen, lab = batch[i]
            for j in range(0, sentences.shape[1]):
                if(sentences[i][j] == pad_idx):
                    char_words.append(torch.Tensor(pad_word).to(device))
                else:
                    char_words.append(torch.Tensor(char_sen[j]).to(device))
        char_words = torch.nn.utils.rnn.pad_sequence(char_words, padding_value = pad_char_idx, batch_first = True)
        return sentences.to(torch.int64),char_words.to(torch.int64),  labels.to(torch.int64)

    train_dataset = CustomDataset(train_data, train_char_data, train_label)
    val_dataset = CustomDataset(val_data, val_char_data, val_label)
    train_data_loader = DataLoader(train_dataset, batch_size = 128, shuffle=True, collate_fn = collate_fn_padd)
    val_data_loader = DataLoader(val_dataset, batch_size = 128, shuffle=True, collate_fn = collate_fn_padd)

    class char_word_model(nn.Module):
        def __init__(self, vocab_size, embedding_dim = 50, hidden_dim= 25):
            super(char_word_model, self).__init__()
            self.embedding = nn.Embedding(vocab_size, embedding_dim)
            self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True, bidirectional = True)
        def forward(self, x):
            x = x.to(torch.int64)
            x = x.to(device)
            x = self.embedding(x)
            _,(h, _) = self.lstm(x)
            h = torch.cat((h[0,:,:], h[1,:,:]), dim = 1)
            return h

    class lstm_Model(nn.Module):
        def __init__(self, vocab_size, embedding_dim, hidden_dim, number_of_tags, embedding_matrix, total_char):
            super(lstm_Model, self).__init__()
            self.char_embedding = char_word_model(total_char, 50, 25)
            self.embedding = nn.Embedding(vocab_size, embedding_dim)
            self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix,dtype=torch.float32))
            self.dropout = nn.Dropout(p = 0.5)
            self.lstm = nn.LSTM(150, hidden_dim, batch_first=True, bidirectional = True)
            self.fc = nn.Linear(2*hidden_dim, number_of_tags)

        def forward(self, x, char_x):
            x = x.to(torch.int64)
            x = x.to(device)
            char_x = char_x.to(torch.int64)
            char_x = char_x.to(device)
            a = x.shape[0]
            b = x.shape[1]
            char_x = self.char_embedding(char_x)
            x = self.embedding(x)
            x = x.reshape((a*b, 100))
            x = torch.cat((x, char_x), dim = 1)
            x = x.reshape((a, b, 150))
            x = self.dropout(x)
            x, _ = self.lstm(x)
            x = x.contiguous()
            x = x.view(-1, 200)
            x = self.fc(x)
            return x

    model = lstm_Model(vocab_size, 100, 100, 17, embedding, charvocabsize)
    model = model.to(device)

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum = 0.9, weight_decay = 0.0001)

    train_loss = []
    def train(epoch_number):
        model.train()
        tloss = 0
        for s,b in enumerate(train_data_loader):
            X_train, char_X_train, Y_train = b
            Y_train = Y_train.reshape(Y_train.shape[0]*Y_train.shape[1])
            optimizer.zero_grad()
            Yp = model(X_train, char_X_train)

            loss = criterion(Yp, Y_train)
            loss.backward()
            tloss = tloss + loss.item()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
            optimizer.step()
        avgloss = tloss / len(train_data_loader)
        train_loss.append(avgloss)
        return avgloss

    val_loss = []
    def evaluate():
        model.eval()
        total_loss, total_accuracy = 0, 0
        correct = 0
        total = 0
        for step,b in enumerate(val_data_loader):
            X_val, char_X_val, Y_val = b
            Y_val = Y_val.reshape(Y_val.shape[0]*Y_val.shape[1])
            with  torch.no_grad():
                Yp = model(X_val, char_X_val)
                loss = criterion(Yp, Y_val)
                total_loss = total_loss + loss.item()
                Yp = Yp.argmax(axis = 1)
                Yp = Yp.reshape(Yp.shape[0])
                temp = ((Yp) == Y_val)
                correct += temp.sum().float()
                total += temp.shape[0]
        avg_loss = total_loss / len(val_data_loader)
        accuracy = correct/(total)
        return avg_loss, accuracy

    best_val_loss = float('inf')
    nepochs = 20
    for epoch in range(nepochs):
        print('\n Epoch {:} / {:}'.format(epoch + 1, nepochs))
        tloss = train(epoch)
        vloss, valreport = evaluate()
        if vloss < best_val_loss:
            best_val_loss = vloss
            torch.save(model.state_dict(), model_output_file)
        print(f'\nTraining Loss: {tloss:.3f}')
        print(f'Validation Loss: {vloss:.3f}')
        print(f'Validation Report: {valreport}')


text:
import pandas as pd
import numpy as np
import scipy
import sklearn
import matplotlib

ext: py

reason: import numpy,import scipy


text:
import numpy as np
import pandas as pd
import scipy as spy
import random as rndm
import math

def rPlotGen():
    i=0
    lidar=[289,391,298,797,699,400,504,508,509]
    np_lidar = np.array(lidar)
    for each in np_lidar:
        print(random.choice(np_lidar))
    rPlotGen()
    print(rPlotGen)

def pyLidar(file, *):
    for each max[] in file:
    return max(height), X, Y


text:
#!/usr/bin/env python3
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import pandas as pd
from scipy import stats
import sys
from matplotlib import cm

barwidth = 0.5

# make hatches less annoyingly thick
mpl.rcParams['hatch.linewidth'] = 0.5

mpl.rcParams["errorbar.capsize"] = 2

BIGGER_SIZE = 19
plt.rc('font', size=BIGGER_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=BIGGER_SIZE)     # fontsize of the axes title
plt.rc('axes', labelsize=BIGGER_SIZE)     # fontsize of the x and y labels
plt.rc('xtick', labelsize=BIGGER_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=BIGGER_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=BIGGER_SIZE)

fname     = sys.argv[2]

data_dir  = sys.argv[1]
files = ['http_baseline_lat.csv', 'http_virtine_lat.csv', 'http_virtine_snapshot_lat.csv']

bar_pos  = np.arange(3)

y = [pd.read_csv(data_dir + "/" + x, comment='#', names=['trial', 'microseconds'])['microseconds'].values for x in files]
y_means = [np.mean(x) for x in y]
y_std = [np.std(x) for x in y]

fig, ax = plt.subplots(1, figsize=(5,5))

hatches = ['/', 'o', '-']

color = cm.viridis(np.linspace(0.3, 0.9, 3))

labels=['native', 'virtine', 'snapshot']

plt.bar(bar_pos[0], y_means[0], align='edge', hatch=hatches[0]*3, yerr=y_std[0], zorder=3, capsize=8, color=color[0], width=barwidth, label='native', linewidth=0.25, edgecolor='black')
plt.bar(bar_pos[1], y_means[1], align='edge', hatch=hatches[1]*3, yerr=y_std[1], zorder=3, capsize=8, color=color[1], width=barwidth, label='virtine', linewidth=0.25, edgecolor='black')
plt.bar(bar_pos[2], y_means[2], align='edge', hatch=hatches[2]*3, yerr=y_std[2], zorder=3, capsize=8, color=color[2], width=barwidth, label='virtine SP', linewidth=0.25, edgecolor='black')

ax.set_xticks([r + barwidth/2 for r in range(0, 3)])

#ax.legend(loc='upper left', bbox_to_anchor=(-0.4, -0.02), fontsize=BIGGER_SIZE-3, ncol=3)
ax.set_ylabel('Request latency ($\mu$s)')
ax.set_xticklabels(labels)
ax.grid(alpha=0.5, zorder=0, which='major', axis='y')
plt.tight_layout()
plt.savefig(fname)

max_stars_repo_name: virtines/wasp

max_issues_repo_name: virtines/wasp

max_forks_repo_name: virtines/wasp

reason: import numpy,from scipy


text:
import os
import re
import time
import random
import itertools

import numpy as np

from rdkit import Chem
from rdkit.Chem import AllChem
from rdkit.Chem import Draw

limit_ = 200

def set_avoid_ring(_smiles):
    avoid_ring = []
    ring_tmp = set(re.findall(r"\d", _smiles))
    for j in ring_tmp:
        tmp = [i for i, val in enumerate(_smiles) if val == j]
        while tmp:
            avoid_ring += [j for j in range(tmp.pop(0), tmp.pop(0) + 1)]
    return set(avoid_ring)

def prepare_rigid_crossover(_smiles, side, consider_ring=True, _minimum_len=4):
    """ 1 point crossover
    :param _smiles: SMILES (str)
    :param side: Left SMILES or Right SMILES ['L'|'R'] (str)
    :param consider_ring: consider about avoiding ring (bool)
    :param _minimum_len: minimum cut size (int)
    :return:
    """

    if side not in ["L", "R"]:
        raise Exception("You must choice in L(Left) or R(Right)")

    _smiles_len = len(_smiles)
    _smi = None

    if consider_ring:  # ring 부분을 피해서 자르도록 고려.
        avoid_ring_list = set_avoid_ring(_smiles)

    p = 0
    _start = None
    _end = None
    _gate = False
    while not _gate:  # 원하는 형태의 smiles piece가 나올 때 까지 반복
        if p == limit_:
            raise ValueError(f"main_gate fail ({side}): {_smiles}")

        if consider_ring:
            j = 0
            ring_gate = False
            if side == "L":
                while not ring_gate:
                    if j == 30:
                        raise ValueError(f"ring_gate fail (L): {_smiles}")
                    _end = np.random.randint(_minimum_len, _smiles_len + 1)
                    if _end not in avoid_ring_list:
                        ring_gate = True
                    j += 1
            elif side == "R":
                while not ring_gate:
                    if j == 30:
                        raise ValueError(f"ring_gate fail (R): {_smiles}")
                    _start = np.random.randint(0, _smiles_len - _minimum_len)
                    if _start not in avoid_ring_list:
                        ring_gate = True
                    j += 1
            _smi = _smiles[_start:_end]
        else:
            if side == "L":
                _end = np.random.randint(_minimum_len, _smiles_len)
            elif side == "R":
                _start = np.random.randint(0, _smiles_len - _minimum_len)

            _smi = _smiles[_start:_end]
            chk_ring = re.findall(r"\d", _smi)
            i = 0
            for i in set(chk_ring):
                list_ring = [_ for _, val in enumerate(_smi) if val == i]
                if (len(list_ring) % 2) == 1:
                    b = random.sample(list_ring, 1)
                    _smi = _smi[:b[0]] + _smi[b[0] + 1:]
                    # print(f'@ {_smi} // {_smiles}')

        p += 1

        if "." in _smi:  # 이온은 패스한다.
            continue

        n_chk = 0
        for j in _smi:  # [] 닫혀 있는지 확인.
            if j == "[":
                n_chk += 1
            if j == "]":
                n_chk -= 1
        if n_chk == 0:
            _gate = True

    return _smi

def chk_branch(_smi, side=None):

    if side not in ["L", "R", None]:
        raise Exception("You must choice in L(Left) or R(Right)")

    branch_list = []
    n_branch = 0
    min_branch = 0
    for i, b in enumerate(_smi):  # () 닫혀 있는지 확인.
        if b == "(":
            n_branch += 1  # 0 == (
        if b in ")":
            n_branch -= 1  # 1 == )
        if side == "L":
            if n_branch < min_branch:
                min_branch = n_branch
                branch_list.append(i)
        elif side == "R":  # track max_value
            if n_branch > min_branch:
                min_branch = n_branch
                branch_list.append(i)
    if side == None:
        return n_branch
    return np.asarray(branch_list), min_branch

def get_open_branch(_smi):
    return [i for i, e in enumerate(_smi) if e == "("]

def get_close_branch(_smi):
    return [i for i, e in enumerate(_smi) if e == ")"]

def tight_rm_branch(_smi_l, _smi_r):
    # tmp = time.time()

    _new_smi = _smi_l + _smi_r

    open_branch = get_open_branch(_new_smi)
    close_branch = get_close_branch(_new_smi)

    b = None
    n_branch = chk_branch(_new_smi)

    q = len(_smi_l)
    while n_branch > 0:  # over opened-branch
        _smi_l_open_branch = get_open_branch(_smi_l)
        _smi_r_open_branch = get_open_branch(_smi_r)
        open_branch = get_open_branch(_smi_l + _smi_r)
        avoid_tokens = [
            i for i, e in enumerate(_smi_l + _smi_r)
            if e in ["=", "#", "@", "1", "2", "3", "4", "5", "6", "7", "8"]
        ]

        if len(_smi_r_open_branch) == 0:  # open branch 가 없을 경우
            _smi_r_open_branch.append(len(_smi_r))
        if len(_smi_l_open_branch) == 0:
            _smi_l_open_branch.append(0)

        n = np.random.rand()  # 임의적으로 close branch 를 추가하거나 제거한다.
        if n > 0.5:  # 추가
            branch_gate = False
            j = 0
            while not branch_gate:  # Ring 부분을 피해서 자름
                if j == limit_:
                    raise ValueError
                b = np.random.randint(_smi_l_open_branch[-1] + 1,
                                      _smi_r_open_branch[-1] + q)
                j += 1
                if b not in avoid_tokens:
                    branch_gate = True
            n_branch -= 1
            if b <= len(_smi_l
                        ):  # SMILES 길이를 고려하여 자른다. 좌측 SMILES의 open branch를 cut!
                _smi_l = _smi_l[:b] + ")" + _smi_l[b:]
                q += 1
            else:  # 좌측 SMILES 길이를 제외한 수가 우측 SMILES 문자의 위치를 의미한다.
                b -= len(_smi_l)
                _smi_r = _smi_r[:b] + ")" + _smi_r[b:]
        else:  # 제거
            b = _smi_l_open_branch[-1]  # (Random으로도 가능함. 과한 부분만 Cut!)
            n_branch -= 1
            q -= 1
            _smi_l = _smi_l[:b] + _smi_l[b + 1:]

    while n_branch < 0:  # over closed-branch
        _smi_l_close_branch = get_close_branch(_smi_l)
        _smi_r_close_branch = get_close_branch(_smi_r)
        close_branch = get_close_branch(_smi_l + _smi_r)
        avoid_tokens = [
            i for i, e in enumerate(_smi_l + _smi_r)
            if e in ["=", "#", "@", "1", "2", "3", "4", "5", "6", "7", "8"]
        ]

        if len(_smi_r_close_branch) == 0:
            _smi_r_close_branch.append(len(_smi_r))
        if len(_smi_l_close_branch) == 0:
            _smi_l_close_branch.append(0)

        n = np.random.rand()
        if n > 0.5:
            branch_gate = False
            j = 0
            while not branch_gate:  # Ring 부분을 피해서 자름
                b = np.random.randint(_smi_l_close_branch[-1] + 1,
                                      _smi_r_close_branch[0] + q + 1)
                j += 1
                if b not in (close_branch + avoid_tokens):
                    branch_gate = True
                if j == limit_:
                    raise ValueError
            n_branch += 1
            if b < len(_smi_l):
                _smi_l = _smi_l[:b] + "(" + _smi_l[b:]
                q += 1
            else:
                b -= len(_smi_l)
                _smi_r = _smi_r[:b] + "(" + _smi_r[b:]
        else:
            b = _smi_r_close_branch[0]
            n_branch += 1
            # print(f'{_smi_r[b]}')
            _smi_r = _smi_r[:b] + _smi_r[b + 1:]

    # time_.append(time.time() - tmp)

    return _smi_l + _smi_r

def replace_atom(_smi):

    #                    C /B  N  P / O  S / F  Cl  Br  I
    replace_atom_list = [6, 5, 7, 15, 8, 16, 9, 17, 35, 53]
    #                         C  N  P / O  S
    replace_arom_atom_list = [6, 7, 15, 8, 16]

    # print(f"before: {_smi}")

    mol_ = Chem.MolFromSmiles(_smi)
    max_len = mol_.GetNumAtoms()

    mw = Chem.RWMol(mol_)
    # Chem.SanitizeMol(mw)

    p = 0
    gate_ = False
    while not gate_:
        if p == 30:
            # raise Exception
            raise PermissionError

        rnd_atom = np.random.randint(0, max_len)

        valence = mw.GetAtomWithIdx(rnd_atom).GetExplicitValence()
        if mw.GetAtomWithIdx(rnd_atom).GetIsAromatic():
            if valence == 3:
                mw.ReplaceAtom(
                    rnd_atom,
                    Chem.Atom(replace_arom_atom_list[np.random.randint(0, 3)]))
            elif valence == 2:
                mw.ReplaceAtom(
                    rnd_atom,
                    Chem.Atom(replace_arom_atom_list[np.random.randint(1, 5)]))
            else:
                continue
            mw.GetAtomWithIdx(rnd_atom).SetIsAromatic(True)
        else:
            if valence == 4:
                mw.ReplaceAtom(
                    rnd_atom,
                    Chem.Atom(replace_atom_list[np.random.randint(0, 1)]))
            elif valence == 3:
                mw.ReplaceAtom(
                    rnd_atom,
                    Chem.Atom(replace_atom_list[np.random.randint(0, 4)]))
            elif valence == 2:
                mw.ReplaceAtom(
                    rnd_atom,
                    Chem.Atom(replace_atom_list[np.random.randint(0, 6)]))
            elif valence == 1:
                mw.ReplaceAtom(
                    rnd_atom,
                    Chem.Atom(replace_atom_list[np.random.randint(0, 10)]))

        p += 1
        # print(f"after: {Chem.MolToSmiles(mw)}")
        try:
            Chem.SanitizeMol(mw)
            gate_ = True
        except Chem.rdchem.KekulizeException:
            # print(f"{_smi} {Chem.MolToSmiles(mw, kekuleSmiles=False)} {Chem.MolToSmiles(mol_, kekuleSmiles=False)}")
            # raise Exception
            pass

    Chem.Kekulize(mw)
    # print(f"after_San: {Chem.MolToSmiles(mw)}")

    return Chem.MolToSmiles(mw, kekuleSmiles=True), mw

def delete_atom(_smi):
    """
    Aromatic ring 을 제외하고 삭제함.
    :param _smi:
    :return:
    """
    max_len = len(_smi)
    mol_ = False
    _new_smi = None

    p = 0
    while not mol_:
        p += 1
        if p == 30:
            # raise Exception
            raise PermissionError

        rnd_insert = np.random.randint(max_len)
        _new_smi = _smi[:rnd_insert] + _smi[rnd_insert + 1:]
        mol_ = Chem.MolFromSmiles(_new_smi)

    return _new_smi, mol_

def add_atom(_smi):
    list_atom = ["C", "B", "N", "P", "O", "S", "Cl", "Br"]

    max_len = len(_smi)
    mol_ = False
    _new_smi = None

    p = 0
    while not mol_:
        p += 1
        if p == 30:
            # raise Exception
            raise PermissionError

        rnd_insert = np.random.randint(max_len)
        _new_smi = _smi[:rnd_insert] + random.sample(list_atom,
                                                     1)[0] + _smi[rnd_insert:]
        mol_ = Chem.MolFromSmiles(_new_smi)

    return _new_smi, mol_

def cut_smi(smi1, smi2, func, ring_bool):

    l_smi = None
    r_smi = None

    try:
        l_smi = func(smi1, "L", ring_bool, 4)
        r_smi = func(smi2, "R", ring_bool, 4)
    except (IndexError, ValueError):
        # fail_f.write(f"{l_smi},{r_smi},piece\n")
        raise PermissionError

    return l_smi, r_smi

def crossover_smiles(smi1, smi2, func, ring_bool):
    new_smi = None
    mol = None
    l_smi = None
    r_smi = None

    gate = 0
    while not (l_smi and r_smi):
        gate += 1
        if gate > 10:
            # fail_f.write(f"{l_smi},{r_smi},np\n")
            raise PermissionError
        try:
            l_smi, r_smi = cut_smi(smi1, smi2, func, ring_bool)
        except:
            pass

    gate = 0
    while not mol:
        gate += 1
        if gate > 5:
            break
        try:
            new_smi = tight_rm_branch(l_smi, r_smi)
        except ValueError:
            continue
        mol = Chem.MolFromSmiles(new_smi)

    return new_smi, mol

if __name__ == "__main__":

    from multiprocessing import Pool
    import datetime
    from rkdit import rdBase

    rdBase.DisableLog('rdApp.*')

    _minimum_len = 4

    def get_smi(_smi):
        mol = Chem.MolFromSmiles(_smi[:-1])
        Chem.Kekulize(mol)
        return Chem.MolToSmiles(mol, kekuleSmiles=True, isomericSmiles=False)

    with open(
            '/home/yongbeom/Git/duaibeom/gen_smiles/data/guacamol_v1_test.smiles'
    ) as f:
        with Pool(8) as pool:
            smiles = pool.map(get_smi, f)

    # smiles = smiles[:1000]
    with open('init.smi', 'w') as f:
        for smi in smiles:
            f.write(f'{smi}\n')

    def just_cut_left_smi(_smi):
        len_smi = len(_smi)
        _end = np.random.randint(_minimum_len, len_smi + 1)
        return _smi[:_end]

    def just_cut_right_smi(_smi):
        len_smi = len(_smi)
        _start = np.random.randint(0, len_smi - _minimum_len)
        return _smi[_start:]

    def simple_cut_smi(smi1, smi2):

        l_smi = None
        r_smi = None

        l_smi = just_cut_left_smi(smi1)
        r_smi = just_cut_right_smi(smi2)

        return l_smi + r_smi

    start_time = time.time()
    print("-- start -- simple cut with swap")
    n_valid = 0
    n_fail = 0

    smi_list = []
    for _ in range(100000):
        smi1, smi2 = random.sample(smiles, 2)
        gate = 0
        mol = None
        _smi = None

        while not mol:
            gate += 1
            if gate > 5:
                break
            try:
                _smi = simple_cut_smi(smi1, smi2)
                mol = Chem.MolFromSmiles(_smi)
            except:
                mol = None

        if not mol:
            gate = 0
            while not mol:
                gate += 1
                if gate > 5:
                    break
                try:
                    _smi = simple_cut_smi(smi2, smi1)
                    mol = Chem.MolFromSmiles(_smi)
                except:
                    mol = None

        if mol:
            smi_list.append(_smi)
            n_valid += 1
        else:
            n_fail += 1

        # if _ % 100000 == 0:
        #     print(f"Success rate: {n_valid/(n_valid + n_fail)*100:.3f}% @{_} Time: {(time.time() - start_time)/60:.3f} min")
    print(f"Final Success rate: {n_valid/(n_valid + n_fail)*100:.3f}%")
    print(f"Cost time: {(time.time() - start_time)/60:.3f} min")
    # with open('crossover_nothing_swap.smi', 'w') as f:
    #     for smi in smi_list:
    #         f.write(f'{smi}\n')

    start_time = time.time()
    print("-- start -- simple cut")
    n_valid = 0
    n_fail = 0

    smi_list = []
    for _ in range(100000):
        smi1, smi2 = random.sample(smiles, 2)
        gate = 0
        mol = None
        _smi = None

        while not mol:
            gate += 1
            if gate > 5:
                break
            try:
                _smi = simple_cut_smi(smi1, smi2)
                mol = Chem.MolFromSmiles(_smi)
            except:
                mol = None

        if mol:
            smi_list.append(_smi)
            n_valid += 1
        else:
            n_fail += 1

        # if _ % 100000 == 0:
        #     print(f"Success rate: {n_valid/(n_valid + n_fail)*100:.3f}% @{_} Time: {(time.time() - start_time)/60:.3f} min")
    print(f"Final Success rate: {n_valid/(n_valid + n_fail)*100:.3f}%")
    print(f"Cost time: {(time.time() - start_time)/60:.3f} min")
    # with open('crossover_nothing.smi', 'w') as f:
    #     for smi in smi_list:
    #         f.write(f'{smi}\n')

    start_time = time.time()
    print("-- start -- our crossover with swap")
    n_valid = 0
    n_fail = 0

    smi_list = []
    for _ in range(100000):
        smi1, smi2 = random.sample(smiles, 2)
        gate = 0
        mol = None
        new_smi = None

        try:
            new_smi, mol = crossover_smiles(smi1, smi2,
                                            prepare_rigid_crossover, True)
        except PermissionError:
            pass
        if not mol:
            try:
                new_smi, mol = crossover_smiles(smi2, smi1,
                                                prepare_rigid_crossover, True)
            except PermissionError:
                pass

        if mol:
            smi_list.append(new_smi)
            n_valid += 1
        else:
            n_fail += 1

        # if _ % 100000 == 0:
        #     print(f"Success rate: {n_valid/(n_valid + n_fail)*100:.3f}% @{_} Time: {(time.time() - start_time)/60:.3f} min")
    print(f"Final Success rate: {n_valid/(n_valid + n_fail)*100:.3f}%")
    print(f"Cost time: {(time.time() - start_time)/60:.3f} min")
    # with open('crossover_our_swap.smi', 'w') as f:
    #     for smi in smi_list:
    #         f.write(f'{smi}\n')

    start_time = time.time()
    print("-- start -- our crossover")
    n_valid = 0
    n_fail = 0

    smi_list = []
    for _ in range(100000):
        smi1, smi2 = random.sample(smiles, 2)
        gate = 0
        mol = None
        new_smi = None

        try:
            new_smi, mol = crossover_smiles(smi1, smi2,
                                            prepare_rigid_crossover, True)
        except PermissionError:
            pass

        if mol:
            smi_list.append(new_smi)
            n_valid += 1
        else:
            n_fail += 1

        # if _ % 100000 == 0:
        #     print(f"Success rate: {n_valid/(n_valid + n_fail)*100:.3f}% @{_} Time: {(time.time() - start_time)/60:.3f} min")
    print(f"Final Success rate: {n_valid/(n_valid + n_fail)*100:.3f}%")
    print(f"Cost time: {(time.time() - start_time)/60:.3f} min")
    # with open('crossover_our.smi', 'w') as f:
    #     for smi in smi_list:
    #         f.write(f'{smi}\n')

    start_time = time.time()
    print("-- start -- replace atom !")
    n_valid = 0
    n_fail = 0

    for _ in range(100000):
        smi1 = random.sample(smiles, 1)[0]
        try:
            _smi, _mol = replace_atom(smi1)
            mol = Chem.MolFromSmiles(_smi)
        except:
            mol = None

        if mol:
            n_valid += 1
        else:
            n_fail += 1

        # if _ % 100000 == 0:
        #     print(f"Success rate: {n_valid/(n_valid + n_fail)*100:.3f}% @{_} Time: {(time.time() - start_time)/60:.3f} min")
    print(f"Final Success rate: {n_valid/(n_valid + n_fail)*100:.3f}%")
    print(f"Cost time: {(time.time() - start_time)/60:.3f} min")

    start_time = time.time()
    print("-- start -- delete atom !")
    n_valid = 0
    n_fail = 0

    for _ in range(100000):
        smi1 = random.sample(smiles, 1)[0]
        try:
            _smi, _mol = delete_atom(smi1)
            mol = Chem.MolFromSmiles(_smi)
        except:
            mol = None

        if mol:
            n_valid += 1
        else:
            n_fail += 1

        # if _ % 100000 == 0:
        #     print(f"Success rate: {n_valid/(n_valid + n_fail)*100:.3f}% @{_} Time: {(time.time() - start_time)/60:.3f} min")
    print(f"Final Success rate: {n_valid/(n_valid + n_fail)*100:.3f}%")
    print(f"Cost time: {(time.time() - start_time)/60:.3f} min")

    start_time = time.time()
    print("-- start -- add atom !")
    n_valid = 0
    n_fail = 0

    for _ in range(100000):
        smi1 = random.sample(smiles, 1)[0]
        try:
            _smi, _mol = add_atom(smi1)
            mol = Chem.MolFromSmiles(_smi)
        except:
            mol = None

        if mol:
            n_valid += 1
        else:
            n_fail += 1

        # if _ % 100000 == 0:
        #     print(f"Success rate: {n_valid/(n_valid + n_fail)*100:.3f}% @{_} Time: {(time.time() - start_time)/60:.3f} min")
    print(f"Final Success rate: {n_valid/(n_valid + n_fail)*100:.3f}%")
    print(f"Cost time: {(time.time() - start_time)/60:.3f} min")

reason: import numpy


text:
# Copyright 2019 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

""" Base metaoptimizer class """
import importlib
import os
import subprocess

from mtl.config.opts.parser import mkdir_p
from mtl.util import session_manager
import numpy as np
import torch

class MetaOptimizer(session_manager.SessionManager):

  def __init__(self, opt, ds=None, dataloaders=None):
    super().__init__(opt)

    self.cfg = importlib.import_module('mtl.config.' + opt.config)
    self.optim = None
    self.curr_params = None
    self.results = []
    self.exp_count = 0
    self.worst_score = opt.worst_score
    self.best_score = self.worst_score
    self.best_exp = None
    self.best_params = None
    self.extra_child_args = []

    self.parameter_setup()
    self.restore(opt.restore_session)

  def checkpoint_ref_setup(self):
    super().checkpoint_ref_setup()
    self.checkpoint_ref['best'] = ['best_exp', 'best_score', 'best_params']
    self.checkpoint_ref['results'] = ['results', 'exp_count']

  def parameter_setup(self):
    opt = self.opt
    if opt.param:
      param = importlib.import_module('mtl.meta.param.' + opt.param)
      self.param = param.Metaparam
      opt.search = opt.search.split('-')

      test_sample = self.init_sample()
      self.curr_params = test_sample
      self.best_params = test_sample
      self.n_params = test_sample.get_params(opt.search).nelement()

    else:
      # Not optimizing any parameters
      self.param = None
      self.n_params = 0

  def init_sample(self):
    return self.param(self.opt)

  def copy_sample(self, p, k=None):
    p_new = self.init_sample()
    p_new.update_params(k, p.get_params(k))
    return p_new

  def submit_cmd(self,
                 cmd_queue,
                 exp_id,
                 param=None,
                 extra_args=None,
                 extra_child_args=[],
                 worker_mode=None):
    if worker_mode is None:
      worker_mode = self.opt.worker_mode
    sub_exp_id = '%s/%s' % (self.opt.exp_id, exp_id)
    extra_child_args = self.extra_child_args + extra_child_args

    if worker_mode == 'cmd':
      tmp_cmd = ['python', 'main.py', '-e', sub_exp_id] + self.base_cmd
      if extra_args is not None:
        tmp_cmd += extra_args

      # Initialize sub experiment directory
      tmp_dir = self.exp_root_dir + '/' + sub_exp_id
      mkdir_p(tmp_dir)

      if param is not None:
        if param.is_command:
          # Add extra arguments to new experiment
          param_file = '%s/params.txt' % self.exp_dir
          param_cmd = param.arg.get_cmd()
          with open(param_file, 'a') as f:
            f.write('%d %s\n' % (self.exp_count, ' '.join(param_cmd)))
          tmp_cmd += param_cmd

        else:
          # Point new experiment to metaparameters to load
          param_dir = '%s/params/%d/%d' % (self.opt.exp_id,
                                           self.exp_count // 100,
                                           self.exp_count % 100)
          param_path = '%s/%s' % (self.exp_root_dir, param_dir)
          mkdir_p(param_path)
          torch.save({'metaparams': param.state_dict()},
                     '%s/snapshot_meta' % param_path)
          tmp_cmd += ['--metaparam_load', param_dir]

    elif worker_mode == 'debug':
      tmp_cmd = [sub_exp_id, param.state_dict()]

    else:
      raise ValueError('Undefined worker mode: %s' % worker_mode)

    cmd_queue.put((self.exp_count, worker_mode, tmp_cmd, extra_child_args))
    self.exp_count += 1

  def collect_batch_results(self, n, result_queue):
    opt = self.opt
    scores = np.zeros(n)
    result_idx_ref = np.zeros(n, int)

    for i in range(n):
      result = result_queue.get()
      self.results += [result]
      sample_id = int(result[0].split('/')[-1])
      result_idx_ref[sample_id] = len(self.results) - 1
      if result[1] is not None:
        scores[sample_id] = result[1]['score']
      else:
        scores[sample_id] = self.worst_score

    if not opt.maximize:
      scores = -scores
    return result_idx_ref, scores

  def run(self, cmd_queue, result_queue):
    return

max_stars_repo_name: google/multi-task-architecture-search


text:
import numpy as np
import torch
from pycurt.classifier.preprocessing import run_preprocessing
from pycurt.classifier.dataloader import BpClassDataLoader2D
from collections import Counter
import matplotlib.pyplot as plot
from mrclass_resnet.MRClassiferDataset import MRClassifierDataset
from torch.utils.data import DataLoader
from collections import defaultdict
from mrclass_resnet.utils import load_checkpoint

def extract_slices_inference(patient_data, directions):

    toinfer = []
    if 1 in patient_data.shape[1:]:
        indx = np.where(np.asarray(patient_data.shape[1:])==1)[0][0]
        directions = [indx+1]
    for direction in directions:
        if patient_data.shape[direction] > 1:
            slices = np.arange(np.ceil(np.percentile(np.arange(patient_data.shape[direction]), 25)),
                               np.ceil(np.percentile(np.arange(patient_data.shape[direction]), 95)))
        else:
            slices = [0]
        for slice_idx in slices:
            slice_idx = int(slice_idx)
            if direction == 1:
                toinfer.append(patient_data[:, slice_idx, :, :])
            elif direction == 2:
                toinfer.append(patient_data[:, :, slice_idx, :])
            elif direction == 3:
                toinfer.append(patient_data[:, :, :, slice_idx])

    return toinfer

def load_checkpoint_bpclass(filepath):

    if torch.cuda.is_available():
        map_location=lambda storage, loc: storage.cuda()
    else:
        map_location='cpu'
    checkpoint = torch.load(filepath, map_location=map_location)

    model = checkpoint['model']
    model.load_state_dict(checkpoint['state_dict'])
    for parameter in model.parameters():
        parameter.requires_grad = False

    model.eval()

    return model

def run_inference_bpclass(for_inference, filepaths, modality='ct', body_parts=[], th=None):

    if modality == 'ct':
        directions = [1, 2, 3]
        if th is None:
            th = 0.33
    elif modality == 'mr':
        directions = [3]
        if th is None:
            th = 0.5

    patch_size = (400, 400)
    class_names = filepaths.keys()

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    model_dict = {}
    for key in filepaths:
        model_dict[key] = load_checkpoint_bpclass(filepaths[key])

    result_dict = {}
    for el, image in enumerate(for_inference):
        print('Process # {}, image: {}'.format(el+1, image))
        try:
            preproc_images = run_preprocessing([image], modality)
        except:
            print('Pre-processing failed! Image will be ignored!')
            continue
        if preproc_images is None:
            print('Image has only one slice, it will be ignored!')
            continue
        result_dict[image] = {}
        slices_all = extract_slices_inference(preproc_images, directions)
        slice_labels = {}
        labs = []
        dataloader = BpClassDataLoader2D(slices_all, 1, patch_size, 1,
                                         shuffle=False, crop=False)

        for step, data in enumerate(dataloader):
            slice_labels[step] = {}
            inputs = data['data']
            inputs = torch.from_numpy(np.float32(inputs))
            inputs = inputs.to(device)
            for key in model_dict.keys():
                model = model_dict[key]
                output = model(inputs)
                prob = output.data.cpu().numpy()
                actRange = abs(prob[0][0])+abs(prob[0][1])
                index = output.data.cpu().numpy().argmax()
                if index == 1:
                    slice_labels[step][key] = prob[0, 1]
                else:
                    slice_labels[step]["other"] = prob[0, 0]

        val = {}
        for k in list(class_names):
            val[k] = []
            for slice_id in slice_labels.keys():
                labs = list(slice_labels[slice_id].keys())
                if k in labs:
                    try:
                        val[k].append(slice_labels[slice_id][k])
                    except:
                        print()
                else:
                    val[k].append(0)

        labels = []
        for slice_id in slice_labels.keys():
            labs = list(slice_labels[slice_id].keys())
            if "other" in labs:
                labs.remove("other")
            if labs and len(labs) == 1:
                labels.append(labs[0])
            elif labs and len(labs) > 1:
                probs = [slice_labels[slice_id][x] for x in labs]
                max_index = np.asarray(probs).argmax()
                labels.append(labs[max_index])
        if not labels:
            result_dict[image] = ["other", 0]
        else:
            c = Counter(labels)
            cn = list(c.keys())
            tot_predicted = sum([c[x] for x in c])
            if len(cn) > 1:
                temp = [[x, np.round(c[x]/tot_predicted, 2)] for x in cn
                         if np.round(c[x]/tot_predicted, 2) >= th]
                probs = [x[1] for x in temp]
                cn = [x[0] for x in temp]
                if cn:
                    bp_intersection = [[x, probs[i]] for i, x in enumerate(cn)
                                       if x in body_parts]
                    if bp_intersection and len(bp_intersection) == 1:
                        result_dict[image] = bp_intersection[0]
                    elif bp_intersection and len(bp_intersection) > 1:
                        probs = [x[1] for x in bp_intersection]
                        max_index = np.asarray(probs).argmax()
                        result_dict[image] = bp_intersection[max_index]
                    else:
                        result_dict[image] = ["other", 0]
                else:
                    result_dict[image] = ["other", 0]
            else:
                result_dict[image] = [cn[0], 1]

        print('Image has been classified as "{}"  with probability of {}'.format(
            result_dict[image][0], result_dict[image][1]))

    return result_dict

def run_inference_mrclass(for_inference, checkpoints, sub_checkpoints):

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    # iteration through the different models
    labeled = defaultdict(list)
    for cl in checkpoints.keys():
        model,class_names,scan = load_checkpoint(checkpoints[cl])
        print('Classifying {0} MR scans'.format(class_names[0]))
        class_names[1] = '@lL'

        test_dataset = MRClassifierDataset(list_images = for_inference, class_names = class_names,
                                           scan = scan,infer = True,remove_corrupt= True)
        test_dataloader = DataLoader(test_dataset, batch_size = 1, shuffle=False, num_workers=8)
        for step, data in enumerate(test_dataloader):
            inputs = data['image']
            img_name = data['fn']
            inputs = inputs.to(device)
            output = model(inputs)
            prob = output.data.cpu().numpy()
            actRange = abs(prob[0][0])+abs(prob[0][1])
            index = output.data.cpu().numpy().argmax()
            if index == 1:
                labeled[img_name[0]].append([cl,actRange])

#check double classification and compare the activation value of class 0
    labeled_cleaned = defaultdict(list)
    for key in labeled.keys():
        r = 0
        j = 0
        for i in range(len(labeled[key])):
            if labeled[key][i][1] > r:
                r = labeled[key][i][1]
                j = i
        labeled_cleaned[key] = labeled[key][j]

    labeled_images = defaultdict(list)
    for key in labeled_cleaned.keys():
        labeled_images[labeled_cleaned[key][0]].append(key)

    # subclassification
    labeled_sub= defaultdict(list)
    for cl in sub_checkpoints.keys():

        model,class_names,scan = load_checkpoint(sub_checkpoints[cl])
        test_dataset = MRClassifierDataset(list_images = labeled_images[cl],
                                           class_names = class_names, scan = scan,
                                           subclasses = True,infer = True)
        print('Checking if contrast agent was administered for T1w scans' )
        test_dataloader = DataLoader(test_dataset, batch_size = 1, shuffle=False, num_workers=8)
        for step, data in enumerate(test_dataloader):
            inputs = data['image']
            img_name = data['fn']
            inputs = inputs.to(device)
            output = model(inputs)
            prob = output.data.cpu().numpy()
            actRange = abs(prob[0][0])+abs(prob[0][1])
            index = output.data.cpu().numpy().argmax()
            if index == 1:
                c = '-CA'
            else:
                c = ''
            labeled_sub[img_name[0]].append([cl+c,actRange])

    for key in labeled_sub.keys():
        labeled_cleaned[key] = labeled_sub[key][0]

    # check for the unlabeled images
    not_labeled = list(set(for_inference) - set(list(labeled_cleaned.keys())))
    for img in not_labeled:
        labeled_cleaned[img] = ['other','NA']

    return labeled_cleaned

max_stars_repo_name: TransRadOnc-HIT/PyCURT

max_issues_repo_name: TransRadOnc-HIT/PyCURT

max_forks_repo_name: TransRadOnc-HIT/PyCURT


text:
"""
DD simulates a Distributed Detector LDAR program.
"""

from DetectionModules.abstract_detection_method import DetectionMethod
from GeneralClassesFunctions.simulation_functions import sample_wr
from GeneralClassesFunctions.simulation_functions import set_kwargs_attrs
from GeneralClassesFunctions import simulation_functions
import numpy as np
from DetectionModules import helper_functions

class DD(DetectionMethod):
    def __init__(self, time, gas_field, **kwargs):
        """ Inputs:
       gas_field    a gas_field object (Defined in feast_classes)
       time         a time object (Defined in feast_classes)
       kwargs       optional input dicitionary that will override default parameters
        """
        DetectionMethod.__init__(self, time, gas_field)
        # -------------- Hardware properties --------------
        # Sensitivity is the trigger sensitivity for the sniffer
        self.sensitivity = .01  # g/m^3 (multiply by 1500 to get ppm)
        # x, y, and z define the positions of the sensors
        # x is distance East from the center of the well pad
        # y is distance North from the center of the well pad
        self.x = [-3.5, 0, 3.5, 0]
        self.y = [-10, -10, -10, 10]
        self.z = [2, 3, 4, 4]
        self.lifetime = 5*365  # days

        # -------------- Process variables --------------
        self.repair_interval = 50  # days
        self.survey_speed = 500  # component/person-hour
        self.drive_speed = 15  # m/s
        self.setup_time = 0.5  # hours
        self.work_time = 5  # hours/day on average...5 hours per day on average corresponds to 35 hours per week.

        # -------------- Financial --------------
        self.sniffer_cost = 500  # $ per sniffer
        self.camera_capital = 90000  # $
        self.truck_capital = 30000  # $
        self.labor = 100  # $/hour
        self.maintenance_factor = 0.1

        # -------------- Override default parameters with kwargs --------------
        set_kwargs_attrs(self, kwargs)

        # Calculated parameters
        self.n_sniff = len(self.x)  # sniffers per well
        self.sniffer_capital = self.sniffer_cost*self.n_sniff  # $ per well
        self.location_time = (gas_field.components_per_site/self.survey_speed/self.n_sniff+gas_field.site_spacing /
                              self.drive_speed/3600+self.setup_time)/self.n_sniff  # hours to locate one leak
        self.time_factor = self.location_time*gas_field.site_count/(self.repair_interval*self.work_time)
        self.capital_0 = self.sniffer_capital*gas_field.site_count+self.time_factor * (self.camera_capital +
                                                                                self.truck_capital)  # $
        self.maintenance0 = self.capital_0*self.maintenance_factor
        self.maintenance = [self.maintenance0*time.delta_t/365]*time.n_timesteps  # $/year
        self.location_cost = self.labor*self.location_time  # $/leak

        #  Calculate capital costs vector
        self.capital = np.zeros(time.n_timesteps)  # dollars
        helper_functions.replacement_cap(time, self)

        #  The finding np.costs must be calculated as leaks are found. The vector is
        #  initialized here.
        self.find_cost = [0]*time.n_timesteps

    def detection(self, time, gas_field, atm):
        """
        Determines which leaks are detected at each timestep
        Inputs:
        time:        object of type Time defining the time variables in the simulation
        gas_field:  object of type GasField defining gas field parameters
        atm:        object of type Atmosphere defining wind speed, direction and atmospheric stability
        """
        self.null_detection(time, gas_field)
        # Periodically repair all detected leaks
        if time.current_time % self.repair_interval < time.delta_t and time.current_time != 0:
            # countFound is a temporary variable to ease notation
            count_found = sum(self.leaks.leaks_detected)
            # account for the np.cost of locating the leaks
            self.find_cost[time.time_index] = self.location_cost*count_found
            # calculate the repair np.cost
            self.repair_cost[time.time_index] += sum(sample_wr(gas_field.repair_cost_dist.repair_costs,
                                                               int(count_found)))
            # update leaksFound and leakStruct
            if count_found > 0:
                # Identify indexes of leaks to delete
                ind = np.where(self.leaks.leaks_detected == 1)[0]
                self.leaks_found.extend(self.leaks.flux[ind])
                self.leaks.delete_leaks([ind])
        # Convert coordinates so that x is measured in the direction of the wind
        wind_dir_rad = np.radians(atm.wind_direction[time.time_index])
        detector_x, detector_y = helper_functions.coord_transform(wind_dir_rad, self.x, self.y)
        x_save, y_save = helper_functions.coord_transform(wind_dir_rad, self.leaks.x, self.leaks.y)
        # At each time step, iterate through each leak to see if it can be detected
        indexes = np.where(self.leaks.leaks_detected == 0)[0]
        # check if each detector detects the leak
        phi = simulation_functions.gauss_leak_model(self.x, self.y, self.z, self.leaks, atm, time.time_index,
                                                    index=indexes)
        for ind in range(0, self.n_sniff):
            found_indexes = np.where(phi[ind, :] > self.sensitivity)[0]
            self.leaks.leaks_detected[indexes[found_indexes]] = 1
        self.leaks.x, self.leaks.y, self.x, self.y = x_save, y_save, detector_x, detector_y

max_stars_repo_name: EAOgroup/FEAST

max_issues_repo_name: EAOgroup/FEAST

max_forks_repo_name: EAOgroup/FEAST


text:
import json
import numpy
import readchar

DATA_FILE='data.json'
NUMBER_OF_ROUNDS=10

# Loading data
with open(DATA_FILE,'r') as data_file:
    data = json.load(data_file)

keys = numpy.random.choice(
    data["keys"],
    NUMBER_OF_ROUNDS,
)

count_success = 0
print ("="*70)
print ("DOTA2 HOTKEY TRAINER")
print ("TOTAL ROUNDS = %s" % NUMBER_OF_ROUNDS)
print ("ENTER '.' to quit.")
print ("="*70)
for i, key in enumerate(keys):
    print (("%s. PRESS '%s' KEY") % (i+1, key['string']))
    user_input = readchar.readkey()
    if user_input == '.':
        break
    if user_input == key['key']:
        count_success =  1 + count_success
        print ("CORRECT!")
    else:
        print (("WRONG! YOU PRESSED '%s'. THE CORRECT KEY"
            " IS '%s'.") % (user_input, key['key'])
        )

print ("="*70)
print (("SCORE %s of %s.") % (count_success, NUMBER_OF_ROUNDS))
print ("="*70)


text:
"""
Created on Wed Oct 31 15:22:24 2018

@author: ryanwinfree
"""

# Convolutional Neural Network

# Installing Theano
# pip install --upgrade --no-deps git+git://github.com/Theano/Theano.git

# Installing Tensorflow
# pip install tensorflow

# Installing Keras
# pip install --upgrade keras

# Part 1 - Building the CNN

# Importing the libraries
from keras.models import Sequential
#from keras.layers import Convolution2D
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense

#Initializing the CNN
classifier = Sequential()

#Step 1 - Convolution Layer
#classifier.add(Convolution2D(32, kernel_size=(3,3), data_format='channels_last', input_shape=(64, 64, 3), activation='relu'))
classifier.add(Conv2D(32, (3,3), data_format='channels_last', input_shape=(64, 64, 3), activation='relu'))

#Step 2 - Pooling
classifier.add(MaxPooling2D(pool_size=(2,2)))

#Adding a 2nd convolution layer
classifier.add(Conv2D(32, (3,3), data_format='channels_last', activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2,2)))

#Step 3 - Flattening
classifier.add(Flatten())

#Step 4 - Fully Connection Layer
classifier.add(Dense(units=128, activation = 'relu'))
#Using the sigmoid because we have a binary outcome (cats or dogs)
#This is the layer that gives us the final result. So we get 1 result, cat or dog
#Thus we have 1 unit (layer)
classifier.add(Dense(units=1, activation = 'sigmoid'))

#Compiling the CNN
#using the binary_crossentropy due to our classificaion and because we have a binary outcome
#If more than binary then we would need to use categorical crossentropy
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

##############################################################
#Part 2 - Fitting the CNN to our images
from keras.preprocessing.image import ImageDataGenerator

#pixals take value between 1-255 so by rescaling by 1/255 it will make all our
#rescaled values between 0-1
#shear range = applying random transfomrations
#zoom range = applies random zooms
#Horizontal flip will apply a flip to some images
train_datagen = ImageDataGenerator(
        rescale=1./255,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True)

test_datagen = ImageDataGenerator(rescale=1./255)

train_set = train_datagen.flow_from_directory('dataset/training_set',
                                                 target_size = (64, 64),
                                                 batch_size = 32,
                                                 class_mode = 'binary')

validation_set = test_datagen.flow_from_directory('dataset/test_set',
                                                 target_size = (64, 64),
                                                 batch_size = 32,
                                                 class_mode = 'binary')

classifier.fit_generator(train_set,
        steps_per_epoch=8000,
        epochs=10,
        validation_data=validation_set,
        validation_steps=200)

# Homework Assignment
import numpy as np
from keras.preprocessing import image

#Loading the specific image we want to predict
test_image = image.load_img('dataset/single_prediction/cat_or_dog_2.jpg', target_size = (64,64))
#must now convert the image to an array to match the format we are passing into the 1st layer (input_shape)
test_image = image.img_to_array(test_image)

#Must add a batch dimension (4th dimension) because the predict method requires 4 dimensions even
#if we are using a single input...confusing
test_image = np.expand_dims(test_image, axis = 0)

#Run the model to predict on the test image
result = classifier.predict(test_image)

#Now need to map the result to the corresponding values for cats and dogs
train_set.class_indices

if result[0][0] == 1:
    prediction = 'dog'
else:
   prediction = 'cat'

max_stars_repo_path: Deep_Learning_A_Z/Volume_1-Supervised_Deep_Learning/Part 2 - Convolutional Neural Networks (CNN)/Section 8 - Building a CNN/Convolutional_Neural_Networks/cnn-rw.py


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 30 00:18:12 2020

We want to assess the loss of a credit portfolio under stressed conditions.
The portfolio is made of a large number of loans from a large number of
borrowers.

@author: anneg
"""

import numpy as np
import pandas as pds
from scipy.stats import norm

from cerm.settings import D_N_D, F, G, K, N_MC, SEED, T, W
from cerm import data

def cumulate_matrix(matrix):
    """ Cumulates a matrix (for QA).
        Shape (K, K)"""
    cum_matrix = np.flipud(np.transpose(matrix))
    cum_matrix = np.cumsum(cum_matrix, axis=0)
    cum_matrix = np.transpose(np.flipud(cum_matrix))
    return cum_matrix

def estimate_matrix(matrix, t):
    """ Computes a migration matrix at t h.
        Shape (K, K)"""
    m_m = np.linalg.matrix_power(matrix, t)
    return m_m

def cum_mat_t(matrix):
    """ Computes a migration matrix up to t h.
        Shape (G, T, K, K)"""
    cum_mat = np.zeros([G, T, K, K])
    for g in range(G):
        for h in range(T):
            mat = matrix[g, h, :, :]
            cum_mat[g, h, :, :] = np.linalg.matrix_power(mat, h+1)
    return cum_mat

def validate_matrix(matrix):
    """ Validates that a transition matrix has the right properties.
        Shape (K,K)"""
    if matrix.shape[0] != matrix.shape[1]:
        print("Matrix is not square.", matrix.shape)
    if np.amin(matrix) < 0:
        print("Negative Probabilities")
    if np.amax(matrix) > 1:
        print("Probabilities Larger than.")
        # checking row sums of matrix
    if np.amax(np.sum(matrix)-1) > 0:
        print("Row sum not equal to one", np.amax(np.sum(matrix)-1))

# Unconditional risk measures.

def expected_loss_g_i_t(u_m, lgd, ead, new):
    """ Expected loss that occur at time t for an initial rating i.
        Shape (G, K, T+1)"""
    el = np.zeros([G, K, T+1])
    e_l = np.zeros([K, K])
    i_m = u_m[:, 0, :, :]  # initial state as matrix is constant
    m_m_new = data.mig_matrix_new_loans(new)
    m_m_new = m_m_new[:, 0, :, :]
    for group in range(G):
        for t in range(1, T+1):
            if D_N_D == 0:  # with migration matrices
                if t == 1:
                    el[group, :, 1] = i_m[group, :, K-1] * ead[group, :] * lgd[group, :]
                elif t >= 2:
                    i_m_w = (1-W) * i_m + W * m_m_new
                    m = estimate_matrix(i_m_w[group, :, :], t-1)
                    e_l = np.einsum('ij,j->ij', m, i_m_w[group, :, K-1])
                    e_l[:, K-1] = 0
                    el[group, :, t] = np.sum(e_l, axis=1) * ead[group, :] * lgd[group, :]
            elif D_N_D == 1:  # simplified default/no default model
                el[group, :, t] = i_m[group, :, 0] * ead[group, :] * lgd[group, :]
    return el

def expected_loss_t(u_m, lgd, ead, new):
    """ Total expected loss.
        Shape (K,)"""
    el = np.sum(expected_loss_g_i_t(u_m, lgd, ead, new), axis=0)
    el = np.sum(el, axis=0)
    return el

def expected_loss_i(u_m, lgd, ead, new):
    """ Total expected loss.
        Shape (K,)"""
    el = np.sum(expected_loss_g_i_t(u_m, lgd, ead, new), axis=2)
    el = np.sum(el, axis=0)
    return el

def expected_loss(u_m, average_lgd, ead, new):
    """ Total expected loss.
        Shape 1"""
    el = np.sum(expected_loss_g_i_t(u_m, average_lgd, ead, new))
    return el

# Conditional risk measures.

def thresholds_reg(u_m):
    """ Unconditional thresholds values.
        Shape (K,K)
        Can be simplified using th[th > 9.00] = 9.00
        """

    th = u_m.copy()[0, 0, :, :]
    nine = np.ones([K]) * 9.00
    cum_u_m = cumulate_matrix(u_m[0, 0, :, :])
    for n in range(0, K):
        th[n, :] = np.fmin(norm.ppf(cum_u_m[n, :]), nine)
        th[n, :] = np.fmax(th[n, :], -nine)
    return th

def thresholds_t(u_m, c, a_ti):
    """ Generates time dependent thresholds.
        Shape (G, T, K, K)"""
    th = thresholds_reg(u_m)
    prism = np.repeat(th[np.newaxis, :, :], T, axis=0)
    prism = np.repeat(prism[np.newaxis, :, :, :], G, axis=0)
    a_reg = data.loadings_a1(u_m, c, a_ti)
    b = data.c_a3(u_m, c, a_ti)
    rect1 = data.norm_c_2D(b, c)
    vect2 = data.norm_c_2D(a_reg, c)[:, :, 0]
    rect2 = np.repeat(vect2[:, :, np.newaxis], T, axis=-1)
    rect = 1 / np.sqrt(1 + rect1 - rect2)
    th_t = np.einsum('gtij, git-> gtij', prism, rect)
    return th_t

def uncond_mig_matrix_t(u_m, c, a_ti, new):
    """ Conditional migration matrices.
        Shape (G, T, K, K) for u_m_t and (G, T, K, K) for u_m_t_new."""
    u_m_t = u_m.copy()
    a3_th_t = thresholds_t(u_m, c, a_ti)
    for g in range(G):
        for j in range(0, K):
            if D_N_D == 0:
                if j == 0:
                    u_m_t[g, :, :, j] = 1-norm.cdf(a3_th_t[g, :, :, 1])
                elif (j >= 1) & (j <= K-2):
                    u_m_t[g, :, :, j] = norm.cdf(a3_th_t[g, :, :, j])- norm.cdf(a3_th_t[g, :, :, j+1])
                elif j == K-1:
                    u_m_t[g, :, :, j] = norm.cdf(a3_th_t[g, :, :, K-1])
            elif D_N_D == 1:
                u_m_t[g, :, :, 0] = norm.cdf(a3_th_t[g, :, :, 0])
    return u_m_t

def expected_loss_a3_g_i_t(u_m_t, lgd, ead, new):
    """ Expected loss at time t with approach 3.
        Shape (G, K, T+1)"""
    el = np.zeros([G, K, T+1])
    e_l = np.zeros([K, K])
    m_m_new = data.mig_matrix_new_loans(new)
    for g in range(G):
        i_m = u_m_t[g, 0, :, :]
        m = np.eye(K)
        for t in range(1, T+1):
            if D_N_D == 0:
                if t == 1:
                    el[g, :, 1] = i_m[:, K-1] * ead[g, :] * lgd[g, :]
                elif t >= 2:
                    u_m_t_new = (1-W) * u_m_t + W * m_m_new
                    m = np.einsum('ij, jk->ik', m, u_m_t_new[g, t-2, :, :])
                    e_l = np.einsum('ij, j->ij', m, u_m_t_new[g, t-1, :, K-1])
                    e_l[:, K-1] = 0
                    el[g, :, t] = np.sum(e_l, axis=1) * ead[g, :] * lgd[g, :]
            elif D_N_D == 1:
                el[g, :, t] = u_m_t[g, t-1, :, 0] * ead[g, :] * lgd[g, :]
    return el

def pandas_expected_loss_a3_g_i_t(u_m_t, lgd, ead, new):
    """ Distribution of losses at time t.
        long format (G, K, T)."""
    mat_3D = expected_loss_a3_g_i_t(u_m_t, lgd, ead, new)
    names = ['group_ID', 'credit_rating_rank', 'time_steps']
    index = pds.MultiIndex.from_product([range(s)for s in mat_3D.shape], names=names)
    df = pds.DataFrame({'expected_loss_g_i_t': mat_3D.flatten()}, index=index)['expected_loss_g_i_t']
    df = pds.Series.to_frame(df)
    df['expected_loss_g_i_t_ID'] = np.arange(len(df))
    df.insert(0, 'portfolio_ID', 'pilot 1 Bank A', allow_duplicates=False)
    return df

def expected_loss_a3_t(u_m_t, lgd, ead, new):
    """ Total expected loss.
        Shape (T,)"""
    el = np.sum(expected_loss_a3_g_i_t(u_m_t, lgd, ead, new), axis=0)
    el = np.sum(el, axis=0)
    return el

def expected_loss_a3_i(u_m_t, lgd, ead, new):
    """ Distribution of total losses.
        Shape K"""
    el = expected_loss_a3_g_i_t(u_m_t, lgd, ead, new)
    el = np.sum(el, axis=2)
    el = np.sum(el, axis=0)
    return el

def expected_loss_a3(u_m_t, lgd, ead, new):
    """ Distribution of total losses.
        Shape 1"""
    el = np.sum(expected_loss_a3_g_i_t(u_m_t, lgd, ead, new))
    return el

def hazards(c):
    """ Monte Carlo simulations.
        Shape (N_MC, T, F)"""
    mean = np.zeros([F])
    np.random.seed(SEED)
    z = np.random.multivariate_normal(mean, c, (N_MC, T))
    return z

def conditional_mig_matrix(u_m, c, a, new):
    """ Conditional migration matrices."""
    c_m = np.zeros([N_MC, G, T, K, K])
    c_th = thresholds_reg(u_m)
    z = hazards(c)
    div = np.sqrt(1-data.norm_c_2D(a, c))
    div = np.swapaxes(div, 1, 2)

    for p in range(N_MC):
        if p == N_MC-1:
            print('Conditional migration matrix')
        z_path = z[p, :, :]
        a_z = np.einsum('gitf,tf->gti', a, z_path)
        c_th_new = np.repeat(c_th[np.newaxis, :, :], T, axis=0)
        c_th_new = np.repeat(c_th_new[np.newaxis, :, :, :], G, axis=0)
        if D_N_D == 0:
            for j in range(0, K):
                if j == 0:
                    c_m[p, :, :, :, j] = 1-norm.cdf((c_th_new[:, :, :, 1]
                                                  - a_z)/div)
                elif (j >= 1) & (j <= K-2):
                    c_m[p, :, :, :, j] = norm.cdf((c_th_new[:, :, :, j]
                                               - a_z)/div) - norm.cdf((c_th_new[:, :, :, j+1] - a_z)/div)
                elif j == K-1:
                    c_m[p, :, :, :, j] = norm.cdf((c_th_new[:, :, :, K-1]
                                                - a_z)/div)
        elif D_N_D == 1:
            c_m[p, :, :, :, 0] = norm.cdf((c_th_new[:, :, :, 0] - a_z)/div)
    return c_m

def conditional_mig_matrix_t(u_m, c, a_ti, new):
    """ Conditional migration matrices."""
    c_m = np.zeros([N_MC, G, T, K, K])
    b = data.loadings_a3(u_m, c, a_ti)
    c_th_t = thresholds_t(u_m, c, a_ti)
    z = hazards(c)
    div = np.sqrt(1-data.norm_c_2D(b, c))
    div = np.swapaxes(div, 1, 2)
    for p in range(N_MC):
        if p == N_MC-1:
            print('Conditional_migration_matrix_t')
        z_path = z[p, :, :]
        a_z = np.einsum('gitf,tf->gti', b, z_path)
        if D_N_D == 0:
            for j in range(0, K):
                if j == 0:
                    c_m[p, :, :, :, j] = 1-norm.cdf((c_th_t[:, :, :, 1]
                                                  - a_z)/div)
                elif (j >= 1) & (j <= K-2):
                    c_m[p, :, :, :, j] = norm.cdf((c_th_t[:, :, :, j]
                                               - a_z)/div) - norm.cdf((c_th_t[:, :, :, j+1] - a_z)/div)
                elif j == K-1:
                    c_m[p, :, :, :, j] = norm.cdf((c_th_t[:, :, :, K-1]
                                                - a_z)/div)
        elif D_N_D == 1:
            c_m[p, :, :, :, 0] = norm.cdf((c_th_t[:, :, :, 0] - a_z) / div)
    return c_m

def conditional_lgd(lgd):
    """Conditional LGD.
       Shape (N_MC, K, T), (K,) at this stage"""
    c_lgd = lgd.copy()
    return c_lgd

def loss_g_i_t(c_m, lgd, ead, new):
    """ Distribution of losses at time t.
        Shape (N_MC, G, K, T)."""
    c_m_new = np.zeros([N_MC, G, T, K, K])
    m_m_new = data.mig_matrix_new_loans(new)
    lo_g_t_i = np.zeros([N_MC, G, K, T+1])

    for p in range(N_MC):
        if p == N_MC-1:
            print('Loss')
        for g in range(G):
            i_m = c_m[p, g, 0, :, :]
            m = np.eye(K)
            for t in range(T+1):
                if t == 1:
                    lo_g_t_i[p, g, :, 1] = i_m[:, K-1] * ead[g, :] * conditional_lgd(lgd)[g, :]
                elif t >= 2:
                    c_m_new[p, g, :, :, :] = (1-W) * c_m[p, g, :, :, :] + W * m_m_new[g, :, :, :]
                    m = np.einsum('ij, jk->ik', m, c_m_new[p, g, t-2, :, :])
                    l_o = np.einsum('ij, j->ij', m, c_m_new[p, g, t-1, :, K-1])
                    l_o[:, K-1] = 0
                    lo_g_t_i[p, g, :, t] = np.sum(l_o, axis=1) * ead[g, :] * conditional_lgd(lgd) [g, :]
    return lo_g_t_i

def pandas_loss_p_g_i_t(c_m, lgd, ead, new):
    """ Distribution of losses at time t.
        long format (N_MC, G, K, T)."""
    mat_4D = loss_g_i_t(c_m, lgd, ead, new)
    names = ['paths', 'group_ID', 'credit_rating_rank', 'time_steps']
    index = pds.MultiIndex.from_product([range(s)for s in mat_4D.shape], names=names)
    df = pds.DataFrame({'loss_p_g_i_t': mat_4D.flatten()}, index=index)['loss_p_g_i_t']
    df = pds.Series.to_frame(df)
    df['loss_p_g_i_t_ID'] = np.arange(len(df))
    df.insert(0, 'portfolio_ID', 'pilot 1 Bank A', allow_duplicates=False)
    return df

def loss_t(c_m, lgd, ead, new):
    """ Distribution of losses at time t.
        Shape (N_MC, T)."""
    lo_t = np.sum(loss_g_i_t(c_m, lgd, ead, new), axis=2)
    lo_t = np.sum(lo_t, axis=1)
    return lo_t

def loss_g_i(c_m, lgd, ead, new):
    """ Distribution of losses at time t.
        Shape (N_MC, G, K)."""
    lo_g_i = np.sum(loss_g_i_t(c_m, lgd, ead, new), axis=2)
    return lo_g_i

def loss_i(c_m, lgd, ead, new):
    """ Distribution of losses for rating i.
        Shape (N_MC, K)."""
    lo_i = np.sum(loss_g_i_t(c_m, lgd, ead, new), axis=3)
    lo_i = np.sum(lo_i, axis=1)
    return lo_i

def loss(c_m, lgd, ead, new):
    """ Distribution of total losses."""
    lo = np.zeros([N_MC])
    lo = np.sum(loss_i(c_m, lgd, ead, new), axis=1)
    return lo

def pandas_loss(c_m, lgd, ead, new):
    """ Distribution of losses at time t.
        long format (N_MC)."""
    vect = loss(c_m, lgd, ead, new)
    df = pds.DataFrame(data=vect, columns=['loss'])
    df['path_ID'] = np.arange(len(df))
    df.insert(0, 'portfolio_ID', 'pilot 1 Bank A', allow_duplicates=False)
    return df

max_stars_repo_name: Iggaak/climate-extended-risk-model

max_issues_repo_name: Iggaak/climate-extended-risk-model

max_forks_repo_name: Iggaak/climate-extended-risk-model


text:
import numpy as np
import pytest
import tensorflow as tf

from doctr.io import DocumentFile
from doctr.models import detection
from doctr.models.detection.predictor import DetectionPredictor
from doctr.models.preprocessor import PreProcessor

@pytest.mark.parametrize(
    "arch_name, input_shape, output_size, out_prob",
    [
        ["db_resnet50", (1024, 1024, 3), (1024, 1024, 1), True],
        ["db_mobilenet_v3_large", (1024, 1024, 3), (1024, 1024, 1), True],
        ["linknet16", (1024, 1024, 3), (1024, 1024, 1), False],
    ],
)
def test_detection_models(arch_name, input_shape, output_size, out_prob):
    batch_size = 2
    model = detection.__dict__[arch_name](pretrained=True)
    assert isinstance(model, tf.keras.Model)
    input_tensor = tf.random.uniform(shape=[batch_size, *input_shape], minval=0, maxval=1)
    target = [
        np.array([[.5, .5, 1, 1], [0.5, 0.5, .8, .8]], dtype=np.float32),
        np.array([[.5, .5, 1, 1], [0.5, 0.5, .8, .9]], dtype=np.float32),
    ]
    # test training model
    out = model(input_tensor, target, return_model_output=True, return_boxes=True, training=True)
    assert isinstance(out, dict)
    assert len(out) == 3
    # Check proba map
    assert isinstance(out['out_map'], tf.Tensor)
    assert out['out_map'].dtype == tf.float32
    seg_map = out['out_map'].numpy()
    assert seg_map.shape == (batch_size, *output_size)
    if out_prob:
        assert np.all(np.logical_and(seg_map >= 0, seg_map <= 1))
    # Check boxes
    for boxes in out['preds']:
        assert boxes.shape[1] == 5
        assert np.all(boxes[:, :2] < boxes[:, 2:4])
        assert np.all(boxes[:, :4] >= 0) and np.all(boxes[:, :4] <= 1)
    # Check loss
    assert isinstance(out['loss'], tf.Tensor)
    # Target checks
    target = [
        np.array([[0, 0, 1, 1]], dtype=np.uint8),
        np.array([[0, 0, 1, 1]], dtype=np.uint8),
    ]
    with pytest.raises(AssertionError):
        out = model(input_tensor, target, training=True)

    target = [
        np.array([[0, 0, 1.5, 1.5]], dtype=np.float32),
        np.array([[-.2, -.3, 1, 1]], dtype=np.float32),
    ]
    with pytest.raises(ValueError):
        out = model(input_tensor, target, training=True)

@pytest.fixture(scope="session")
def test_detectionpredictor(mock_pdf):  # noqa: F811

    batch_size = 4
    predictor = DetectionPredictor(
        PreProcessor(output_size=(512, 512), batch_size=batch_size),
        detection.db_resnet50(input_shape=(512, 512, 3))
    )

    pages = DocumentFile.from_pdf(mock_pdf).as_images()
    out = predictor(pages)
    # The input PDF has 8 pages
    assert len(out) == 8

    # Dimension check
    with pytest.raises(ValueError):
        input_page = (255 * np.random.rand(1, 256, 512, 3)).astype(np.uint8)
        _ = predictor([input_page])

    return predictor

@pytest.fixture(scope="session")
def test_rotated_detectionpredictor(mock_pdf):  # noqa: F811

    batch_size = 4
    predictor = DetectionPredictor(
        PreProcessor(output_size=(512, 512), batch_size=batch_size),
        detection.db_resnet50(assume_straight_pages=False, input_shape=(512, 512, 3))
    )

    pages = DocumentFile.from_pdf(mock_pdf).as_images()
    out = predictor(pages)

    # The input PDF has 8 pages
    assert len(out) == 8

    # Dimension check
    with pytest.raises(ValueError):
        input_page = (255 * np.random.rand(1, 256, 512, 3)).astype(np.uint8)
        _ = predictor([input_page])

    return predictor

@pytest.mark.parametrize(
    "arch_name",
    [
        "db_resnet50",
        "db_mobilenet_v3_large",
        "linknet16",
    ],
)
def test_detection_zoo(arch_name):
    # Model
    predictor = detection.zoo.detection_predictor(arch_name, pretrained=False)
    # object check
    assert isinstance(predictor, DetectionPredictor)
    input_tensor = tf.random.uniform(shape=[2, 1024, 1024, 3], minval=0, maxval=1)
    out = predictor(input_tensor)
    assert all(isinstance(boxes, np.ndarray) and boxes.shape[1] == 5 for boxes in out)

def test_detection_zoo_error():
    with pytest.raises(ValueError):
        _ = detection.zoo.detection_predictor("my_fancy_model", pretrained=False)

def test_linknet_focal_loss():
    batch_size = 2
    input_shape = (1024, 1024, 3)
    model = detection.linknet16(pretrained=True)
    input_tensor = tf.random.uniform(shape=[batch_size, *input_shape], minval=0, maxval=1)
    target = [
        np.array([[.5, .5, 1, 1], [0.5, 0.5, .8, .8]], dtype=np.float32),
        np.array([[.5, .5, 1, 1], [0.5, 0.5, .8, .9]], dtype=np.float32),
    ]
    # test focal loss
    out = model(input_tensor, target, return_model_output=True, return_boxes=True, training=True, focal_loss=True)
    assert isinstance(out['loss'], tf.Tensor)

max_stars_repo_name: elejke/doctr

max_issues_repo_name: elejke/doctr

max_forks_repo_name: elejke/doctr


text:
print('{:*^80}'.format(" WALKING IN "))
print("AUTHOR: BU5DR1V3R , INSPIRED BY ULTRA GUO.")
print("Version -0.03, 2020/12/14")
print("Make sure that your pic is in the same folder of this .exe")

try:
    from PIL import Image, ImageDraw, ImageFont
except:
    print("There is no PIL found in your environment.Please install it first.")
    input("Press <enter>")
    quit()
import os,sys
try:
    import numpy as np
    from matplotlib import cm
    import matplotlib.pyplot as plt
except:
    print("There is no numpy or matplotlib found in your environment.Please install it first.")
    input("Press <enter>")
    quit()
from queue import Queue

max_row = 999
max_col = 999
q = Queue()
flag = np.full((1,3),0)
new_array = []
max_step = 0
output = np.full((1,3),0)

pt = os.path.dirname(sys.argv[0])
if pt!= '':
    os.chdir(os.path.dirname(sys.argv[0]))
os.getcwd()

pathLoad = input("Please input the name of your picture:(e.g. test.jpg, default:1.jpg)")
if len(pathLoad)<=4:
    pathLoad = "1.jpg"
try:
    img = Image.open(pathLoad).convert('RGB')
except:
    print("Wrong path.Try again!")
    input("Press <enter>")
    quit()
pathSave = input("Please input the name of the output picture:(e.g. out.jpg, default:1-out.jpg)")
if len(pathSave)<=4:
    pathSave = "1-out.jpg"
try:
    print("*When you see the picture is 1234X5678, the 5678 is ROW and 1234 is COL")
    rowEnterPoint = int(input("Please input the ROW enter point:(e.g.123,default:0)"))
    colEnterPoint = int(input("Please input the COLUMN enter point:(e.g.456,default:0)"))
except:
    rowEnterPoint = 0
    colEnterPoint = 0

def change(row,col,point):
    new_point = [(row,col),point,0]
    return new_point

def findEntry(array):
    global rowEnterPoint
    global colEnterPoint
    if array[rowEnterPoint][colEnterPoint][1] == [255,255,255]:
        print(array[rowEnterPoint][colEnterPoint])
        for row in array:
            for col in row:
                if col[1][0]!=[255,255,255]:
                    rowEnterPoint = col[0][0]
                    colEnterPoint = col[0][1]
                    return
        print("ERROR:ALL WHITE.")
        input("Press <enter>")
        quit()
    return

def inq(x,y,step):
    global q
    global flag
    global new_array
    global max_col
    global max_row
    global output
    if x<max_row and x>=0 and y<max_col and y>=0:
        if flag[x][y] == 0:
            flag[x][y] = 1
            if new_array[x][y][1] != [255,255,255]:
                new_array[x][y][2] = step + 1
                q.put(new_array[x][y])
    return

def walk():
    global q
    global flag
    global new_array
    global max_step
    ele = q.get()
    loc_x,loc_y = ele[0]
    step = ele[2]
    inq(loc_x+1,loc_y,step)
    inq(loc_x-1,loc_y,step)
    inq(loc_x,loc_y+1,step)
    inq(loc_x,loc_y-1,step)
    if step>max_step:
        max_step = step
    return

def main(pathLoad,pathSave):
    global flag
    global q
    global max_col
    global max_row
    global new_array
    global output
    global max_step
    img = Image.open(pathLoad).convert('RGB')
    array = np.array(img)
    array = array.tolist()
    while True:
        if array[rowEnterPoint][colEnterPoint] == [255,255,255]:
            print("You have chosen a white pixel,please try again.")
            input("Press <enter>")
            quit()
        else:
            break
    max_col = len(array[0])
    max_row = len(array)
    if rowEnterPoint<0 or rowEnterPoint>=max_row or colEnterPoint<0 or colEnterPoint>=max_col:
        print("row or col out of range, please check your input!")
        input("Press <enter>")
        quit()
    output = np.array(img)
    flag = np.full((max_row,max_col),0)
    for i in range(max_row):
        temp_row = []
        for j in range(max_col):
            temp_row.append(change(i,j,array[i][j]))
        new_array.append(temp_row)
    findEntry(new_array)
    q.put(new_array[rowEnterPoint][colEnterPoint])
    print("Drawing the route...")
    while not q.empty():
        walk()
    print("Drawing done. Now outputing pic.")
    cmap = cm.get_cmap('plasma')
    color = cmap([int(i/(max_step+1)*255) for i in range(max_step+1)])
    new_color = []
    for i in color:
        new_color.append([int(i[0]*255),int(i[1]*255),int(i[2]*255)])
    new_color = np.array(new_color)
    for i in range(max_row):
        for j in range(max_col):
            if output[i][j][0]!=255 and output[i][j][1]!=255 and output[i][j][2]!=255:
                output[i][j][0] = new_color[new_array[i][j][2]][0]
                output[i][j][1] = new_color[new_array[i][j][2]][1]
                output[i][j][2] = new_color[new_array[i][j][2]][2]
    img_tr = Image.fromarray(output).convert('RGB')
    img_tr.save(pathSave)
    print("done")
    input("Press <enter>")
    quit()
    return

main(pathLoad,pathSave)


text:
import pickle as pkl
import json

import numpy as np

def main():
    caption_pkl = pkl.load(open('data/dstc10/dstc10_data.caption.pickle', 'rb'))
    test = json.load(open('data/dstc10/test_set4DSTC7-AVSD.json'))
    features_pkl = pkl.load(open('data/dstc10/dstc10_videos_features_all.pickle', 'rb'))
    for d in test['dialogs']:
        image_id = d['image_id']
        caption_pkl[image_id] = dict()
        summary = d['summary']
        caption = d['caption']
        text = '%s | %s' % (summary, caption)
        caption_pkl[image_id]['text'] = np.array([text]).reshape(-1)
        caption_pkl[image_id]['transcript'] = np.array(['none']).reshape(-1)
        caption_pkl[image_id]['start'] = np.array([0]).reshape(-1)
        caption_pkl[image_id]['end'] = np.array([features_pkl[image_id].shape[0]]).reshape(-1)
    pkl.dump(caption_pkl, open('data/dstc10/dstc10_data_all.caption.pickle', 'wb'))
    print('done')

if __name__ == '__main__':
    main()

max_stars_repo_name: hxssgaa/UniVL

max_issues_repo_name: hxssgaa/UniVL

max_forks_repo_name: hxssgaa/UniVL


text:
import sys
import numpy
from io import StringIO

def get_html(items):
    s = "
"
    for values in items:
        s += "
"
        for val in values:
            if val == 1:
                s += 'a	'
            else:
                s += 'a	'

        s += "
"
    s += "
"
    return s

if len(sys.argv) != 2:
    print("usage: %s [filename]" % (sys.argv[0]))
    sys.exit(0)

fp = open(sys.argv[1])

line = fp.readline()

if line == "":
    sys.exit()

items = line.split(',')

pre_values = []

bitmap = []
for i in range(32):
    bitmap.append([0] * 128)

for i in range(len(items)):
    if i == 0:
        pre_values.append(float(items[i]))
    else:
        pre_values.append(int(items[i]))

index = 0
counter = 0
counter_clock = 0
file_index = 0
wfp = open("%04d.html" % file_index, "w")

while line:
    items = line.split(',')
    current_values = []
    for i in range(len(items)):
        if i == 0:
            current_values.append(float(items[i]))
        else:
            current_values.append(int(items[i]))

    if current_values[4] == 0 and pre_values[4] == 1:
        output = StringIO()
        numpy.savetxt(output, bitmap, delimiter="", fmt="%d")
        print(output.getvalue())
        sys.stdout.flush()
        print("renew")
        s = get_html(bitmap)
        wfp.write(s)
        wfp.close()
        file_index = file_index + 1
        wfp = open("%04d.html" % file_index, "w")

        bitmap = []
        for i in range(32):
            bitmap.append([0] * 128)
        counter = 0
        counter_clock = 0

    if current_values[1] == 1 and pre_values[1] == 0:
        counter_clock = 0
        counter = counter + 1

    if current_values[3] == 1 and pre_values[3] == 0:
        bitmap[counter][counter_clock] = (current_values[12] or current_values[13])

        counter_clock = counter_clock + 1

    pre_values = current_values
    line = fp.readline()

max_stars_repo_name: wani-hackase/wani-writeup

max_issues_repo_name: wani-hackase/wani-writeup

max_forks_repo_name: wani-hackase/wani-writeup


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 16 13:53:49 2020

@author: 7ml
"""

import numpy as np
import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.models import model_from_json
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

#meshes = np.loadtxt('data_set/initial_meshes.txt', skiprows=0)
#gradients = np.loadtxt('data_set/gradients.txt', skiprows=0)
#final_deltas = np.loadtxt('data_set/final_deltas.txt', skiprows=0)
solutions = np.loadtxt('data_set/solutions.txt', skiprows=0)
adapted_meshes = np.loadtxt('data_set/adapted_meshes.txt', skiprows=0)

input_dim = solutions.shape[1]
output_dim = adapted_meshes.shape[1]
training_size = adapted_meshes.shape[0]

#Split between train and validaiton set
x_train, x_test, y_train, y_test = train_test_split(solutions, adapted_meshes, test_size=0.2, random_state=42)

#build neural network
model = Sequential()
model.add(Dense(50, input_dim=input_dim, activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(output_dim, activation=None))

#set-up optimizer
opt = keras.optimizers.Nadam(learning_rate=0.00001, beta_1=0.9, beta_2=0.999)

# compile the keras model
model.compile(loss='mse', optimizer=opt, metrics=['accuracy'])

# fit the keras model on the dataset
history = model.fit(x_train, y_train, epochs=10000, batch_size=100, validation_split = 0.2)

#Save the trained model in a .json file
# serialize model to JSON
model_json = model.to_json()
with open("model.json", "w") as json_file:
    json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("model.h5")
print("Saved model to disk")

# later...

# load json and create model
json_file = open('model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("model.h5")
print("Loaded model from disk")

results = model.evaluate(x_test, y_test, batch_size=128)
print('test loss, test acc:', results)

preds = model.predict(x_test)

num_pred = 10

uniform = np.linspace(0,1,201)

plt.show()
fig = plt.figure()
ax = plt.subplot(111)
ax.plot(uniform, preds[num_pred], label = 'MLP - deep learning')
ax.plot(uniform, y_test[num_pred], label = 'Adaptive zoning')
plt.xlabel('index of the array coordinate')
plt.ylabel('Coordinates in final mesh zoning')
ax.legend()

plt.show()
fig = plt.figure()
ax = plt.subplot(111)
ax.plot(uniform, x_test[num_pred], label = 'Solution to ADR problem')
plt.xlabel('index of the array coordinate')
plt.ylabel('Coordinates in final mesh zoning')
ax.legend()

loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1,len(loss)+1)
plt.figure()
plt.plot(epochs, loss, 'b', label = 'training loss')
plt.plot(epochs, val_loss, 'r', label = 'validation loss')
plt.yscale('log')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.draw()
plt.savefig('loss_function_plot')