text:
"""Debug IAF and MAF by fitting to the moons dataset."""
import numpy as np
import torch
import torch.utils.data
import torch.nn as nn
from torch.nn import functional as F
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import seaborn as sns
import math
import yaml
import nomen
import random

import network
import moons
import flow

config = """
batch_size: 100
test_batch_size: 1000
learning_rate: 0.0001
flow_type: realnvp # masked_autoregressive
num_blocks: 5
num_hidden: 64
cuda: true
log_interval: 240  # one epoch for 24k datapoints with 100 batch size
max_iteration: 10000
out_dir: $LOG/debug
rho: 0.
use_tanh: true
seed: 582848
"""

def main(cfg):
  np.random.seed(cfg.seed)
  torch.manual_seed(cfg.seed)
  random.seed(48283)
  device = torch.device("cuda:0" if cfg.cuda else "cpu")
  dataset = moons.MOONS()
  kwargs = {'num_workers': 4, 'pin_memory': True} if cfg.cuda else {}
  train_tensor = torch.from_numpy(dataset.trn.x)
  train_dataset = torch.utils.data.TensorDataset(train_tensor)

  valid_tensor = torch.from_numpy(dataset.val.x)
  valid_dataset = torch.utils.data.TensorDataset(valid_tensor)

  test_tensor = torch.from_numpy(dataset.tst.x)
  test_dataset = torch.utils.data.TensorDataset(test_tensor)

  train_loader = torch.utils.data.DataLoader(
      train_dataset, batch_size=cfg.batch_size, shuffle=True, **kwargs)

  valid_loader = torch.utils.data.DataLoader(
      valid_dataset,
      batch_size=cfg.test_batch_size,
      shuffle=False,
      drop_last=False,
      **kwargs)

  test_loader = torch.utils.data.DataLoader(
      test_dataset,
      batch_size=cfg.test_batch_size,
      shuffle=False,
      drop_last=False,
      **kwargs)

  # plot real data
  fig, ax = plt.subplots()
  ax.plot(dataset.val.x[:,0], dataset.val.x[:,1], '.')
  ax.set_title('Real data')
  plt.savefig(cfg.out_dir / 'data.png')

  modules = []
  mask = network.mask.checkerboard((1, 2))
  base_mask = torch.from_numpy(mask)
  for flow_num in range(cfg.num_blocks):
    if cfg.flow_type == 'realnvp':
      if flow_num % 2 == 0:  # invert mask opposite to prior
        mask = 1 - base_mask
      else:
        mask = base_mask
      modules.append(flow.RealNVPCoupling(input_shape=(1, 2),
                                                 hidden_size=10,
                                                 mask=mask,
                                                 kernel_size=1))
    elif cfg.flow_type == 'maf':
      modules.append(flow.MaskedAutoregressiveFlow(dataset.n_dims,
                                                   cfg.num_hidden,
                                                   use_context=False,
                                                   use_tanh=cfg.use_tanh))
      modules.append(flow.BatchNormalization(dataset.n_dims))
      modules.append(flow.Reverse(dataset.n_dims))
  model = flow.FlowSequential(*modules)

  # orthogonal initialization helps
  for module in model.modules():
    if isinstance(module, nn.Linear):
      nn.init.orthogonal_(module.weight)
      module.bias.data.fill_(0)
  model.to(device)

  optimizer = torch.optim.Adam(model.parameters(),
                               lr=cfg.learning_rate,
                               weight_decay=1e-6)

  p_u = torch.distributions.Normal(loc=torch.zeros(dataset.n_dims, device=device),
                                   scale=torch.ones(dataset.n_dims, device=device))

  train_iter = iter(train_loader)
  for step in range(cfg.max_iteration):
    try:
      data = next(train_iter)
    except StopIteration:
      train_iter = iter(train_loader)
      data = next(train_iter)
    data = data[0].to(device)
    optimizer.zero_grad()
    log_prob = log_prob_fn(model, p_u, data).sum() / data.shape[0]
    loss = -log_prob
    loss.backward()
    optimizer.step()
    if step % cfg.log_interval == 0:
      if np.isnan(loss.item()):
        raise ValueError("Loss hit nan!")
      print(f'epoch: {step * cfg.batch_size // len(train_dataset)}')
      print(f"step: {step}\tlog_lik: {log_prob.item():.2f}")
      for module in model.modules():
        if isinstance(module, flow.BatchNormalization):
          module.momentum = 0.

      # initialize the moving averages with the full dataset
      all_data = train_loader.dataset.tensors[0].to(data.device)
      with torch.no_grad():
        model(all_data)

      # update momentum for proper eval
      for module in model.modules():
        if isinstance(module, flow.BatchNormalization):
          module.momentum = 1.0
      valid_log_lik = evaluate(model, p_u, log_prob_fn, valid_loader, device)
      print(f"\tvalid log-lik: {valid_log_lik:.10f}")
      model.eval()
      plot(step, cfg.out_dir, model, dataset, device)
      model.train()

def log_prob_fn(model, p_u, data):
  u, log_prob_model = model(data)
  # u is shape (batch_size, latent_size)
  log_prob_u = torch.sum(p_u.log_prob(u), dim=-1, keepdim=True)
  return log_prob_u + log_prob_model.sum(-1, keepdim=True)

def evaluate(model, p_u, log_prob_fn, loader, device):
  model.eval()
  total_log_prob = 0.0
  for data in loader:
    data = data[0].to(device)
    with torch.no_grad():
      log_prob = log_prob_fn(model, p_u, data)
      total_log_prob += log_prob.sum().item()
  return total_log_prob / len(loader.dataset)

def plot(step, out_dir, model, dataset, device):
  # generate some examples
  np_x = np.linspace(-1.1, 2.1, 50).astype(np.float32)
  np_y = np.linspace(-0.75, 1.25, 50).astype(np.float32)
  xx, yy = np.meshgrid(np_x, np_y)
  np_u = np.stack([xx.flatten(), yy.flatten()]).T
  u_tens = torch.from_numpy(np_u).to(device)
  _, log_prob = model(u_tens)
  log_prob = log_prob.sum(-1).detach().cpu().numpy()
  log_prob_grid = log_prob.reshape((len(xx), len(xx)))
  xlabels = ['%.1f' % x for x in np_x]
  ylabels = ['%.1f' % x for x in np_y]
  ax = sns.heatmap(log_prob_grid[::-1], xticklabels=xlabels, yticklabels=ylabels[::-1])
  ax.set_title('Heatmap of log likelihood of the flow')
  name = "log-lik_maf_%d.png" % step
  plt.savefig(out_dir / name)
  plt.close()

if __name__ == "__main__":
  cfg = nomen.Config(dictionary=yaml.load(config))
  main(cfg)


text:
import autograd as ag
import autograd.numpy as np
from scipy import linalg, sparse

# define the matrix of unknowns each comma represents a new equation, x represents the vector of unknowns
from scipy.sparse.linalg import spsolve

def f(x, rhs):
    output = np.array([x[0] + (2 * x[1]) - rhs[0], x[0] ** 2 + 4 * x[1] ** 2 - rhs[1]])
    return output

guess = np.array([1, 2], dtype=float)  # provide an initial guess
rhs = np.array([2, 4], dtype=float)  # provide a vector of right hand sides

newton_tol = 1e-6
err = np.inf
while err > newton_tol:
    val = -f(guess, rhs)
    jac = ag.jacobian(f)(guess, rhs)
    dx = linalg.solve(jac, val)
    guess = guess + dx
    err = np.sqrt(np.sum(val ** 2))

# using a sparse matrix for the Jacobian. Other linear problem solution methods can be found here
# https://docs.scipy.org/doc/scipy/reference/sparse.linalg.html
newton_tol = 1e-6
err = np.inf
iteration_count = 0
while err > newton_tol:
    val = -f(guess, rhs)
    iteration_count = iteration_count + 1
    jac = ag.jacobian(f)(guess, rhs)
    jac = sparse.csr_matrix(jac)
    dx = spsolve(jac, val)
    guess = guess + dx
    err = np.sqrt(np.sum(val ** 2))

# provides a more generalized function
def general_function(x, rhs, coeff1, coeff2, power1, power2):
    output = np.array([coeff1[0] * x[0] ** power1[0] + (coeff2[0] * x[1] ** power2[0]) - rhs[0],
                       coeff1[1] * x[0] ** power1[1] + coeff2[1] * x[1] ** power2[1] - rhs[1]])
    return output

guess = np.array([1, 2], dtype=float)
rhs = np.array([-2, 0], dtype=float)
coeff1 = np.array([3, 2], dtype=float)
coeff2 = np.array([-1, -1], dtype=float)
power1 = np.array([1, 2], dtype=float)
power2 = np.array([1, 1], dtype=float)

newton_tol = 1e-6
err = np.inf
iteration_count = 0
while err > newton_tol:
    val = -general_function(guess, rhs, coeff1, coeff2, power1, power2)
    iteration_count = iteration_count + 1
    jac = ag.jacobian(general_function)(guess, rhs, coeff1, coeff2, power1, power2)
    jac = sparse.csr_matrix(jac)
    dx = spsolve(jac, val)
    guess = guess + dx
    print(err)
    err = np.sqrt(np.sum(val ** 2))

def solve(equations):
    """
    the constants of a system of linear equations are stored in a list for each equation in the system
    for example the system below:
         2x+9y-3z+7w+8=0
         7x-2y+6z-1w-10=0
         -8x-3y+2z+5w+4=0
         0x+2y+z+w+0=0
    is expressed as the list:
         [[2,9,-3,7,8],[7,-2,6,-1,-10],[-8,-3,2,5,4],[0,2,1,1,0]]
    """
    for i in equations:
        if len(i) != (len(equations) + 1):
            raise ValueError("your equation system has not a valid format")
    lists = []  # I failed to name it meaningfully
    for eq in range(len(equations)):
        # print "equations 1", equations
        # find an equation whose first element is not zero and call it index
        index = -1
        for i in range(len(equations)):
            if equations[i][0] != 0:
                index = i
                break
        if index == -1:
            raise ValueError("your equation system can not be solved")
        # print "index "+str(eq)+": ",index
        # for the equation[index] calc the lists next item  as follows
        lists.append([-1.0 * i / equations[index][0] for i in equations[index][1:]])
        # print "list"+str(eq)+": ", lists[-1]
        # remove equation[index] and modify the others
        equations.pop(index)
        for i in equations:
            for j in range(len(lists[-1])):
                i[j + 1] += i[0] * lists[-1][j]
            i.pop(0)
    lists.reverse()
    answers = [lists[0][0]]
    for i in range(1, len(lists)):
        tmpans = lists[i][-1]
        for j in range(len(lists[i]) - 1):
            tmpans += lists[i][j] * answers[-1 - j]
        answers.append(tmpans)
    answers.reverse()
    return answers

answers = solve([[2, 9, -3, 7, 8], [7, -2, 6, -1, -10], [-8, -3, 2, 5, 4], [0, 2, 1, 1, 0]])

ext: py

reason: from scipy


text:
'''
This code is part of QuTIpy.

(c) Copyright Sumeet Khatri, 2021

This code is licensed under the Apache License, Version 2.0. You may
obtain a copy of this license in the LICENSE.txt file in the root directory
of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.

Any modifications or derivative works of this code must retain this
copyright notice, and modified files need to carry a notice indicating
that they have been altered from the originals.
'''

import itertools
import numpy as np

from qutipy.Pauli import generate_nQubit_Pauli
from qutipy.general_functions import dag,Tr

def nQubit_Pauli_coeff(X,n,return_dict=False):

    '''
    Generates the coefficients of the matrix X in the n-qubit Pauli basis.
    The coefficients c_{alpha} are such that

    X=(1/2^n)\sum_{alpha} c_alpha \sigma_alpha

    The coefficients are returned in lexicographical ordering.
    '''

    indices=list(itertools.product(*[range(0,4)]*n))

    if return_dict:
        C={}
    else:
        C=[]

    for index in indices:
        sigma_i=generate_nQubit_Pauli(index)
        if return_dict:
            C[index]=Tr(dag(sigma_i)@X)
        else:
            C.append(Tr(dag(sigma_i)@X))

    return C

max_stars_repo_name: sumeetkhatri/QuTIPy

max_issues_repo_name: sumeetkhatri/QuTIPy

max_forks_repo_name: sumeetkhatri/QuTIPy

reason: import numpy


text:
import numpy as np
from collections import Iterable

from .base import Attack

class ContrastReductionAttack(Attack):
    """Reduces the contrast of the image until it is misclassified.

    """

    def _apply(self, a, epsilons=1000):
        image = a.original_image
        min_, max_ = a.bounds()
        target = (max_ + min_) / 2

        if not isinstance(epsilons, Iterable):
            epsilons = np.linspace(0, 1, num=epsilons + 1)[1:]

        for epsilon in epsilons:
            perturbed = (1 - epsilon) * image + epsilon * target

            _, is_adversarial = a.predictions(perturbed)
            if is_adversarial:
                return


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

" Train from the replay files through python tensor (pt) file"

import os
import gc

import sys
import time
import traceback
import argparse
import datetime

import numpy as np
import torch
import torch.nn as nn

from torch.utils.data import DataLoader, ConcatDataset
from torch.optim import Adam, RMSprop

from tensorboardX import SummaryWriter

from absl import flags
from absl import app
from tqdm import tqdm

from alphastarmini.core.arch.arch_model import ArchModel

from alphastarmini.core.sl.feature import Feature
from alphastarmini.core.sl.label import Label
from alphastarmini.core.sl import sl_loss_multi_gpu as Loss
from alphastarmini.core.sl.dataset import ReplayTensorDataset
from alphastarmini.core.sl import sl_utils as SU

from alphastarmini.lib.utils import load_latest_model, initial_model_state_dict
from alphastarmini.lib.hyper_parameters import Arch_Hyper_Parameters as AHP
from alphastarmini.lib.hyper_parameters import SL_Training_Hyper_Parameters as SLTHP

import param as P

__author__ = "Ruo-Ze Liu"

debug = False

parser = argparse.ArgumentParser()
parser.add_argument("-p", "--path", default="./data/replay_data_tensor_new/", help="The path where data stored")
parser.add_argument("-m", "--model", choices=["sl", "rl"], default="sl", help="Choose model type")
parser.add_argument("-r", "--restore", action="store_true", default=True, help="whether to restore model or not")
parser.add_argument('--num_workers', type=int, default=1, help='')

args = parser.parse_args()

# training paramerters
PATH = args.path
MODEL = args.model
RESTORE = args.restore
NUM_WORKERS = args.num_workers

MODEL_PATH = "./model/"
if not os.path.exists(MODEL_PATH):
    os.mkdir(MODEL_PATH)
RESTORE_PATH = MODEL_PATH + 'sl_21-11-29_19-35-12.pth'

SIMPLE_TEST = not P.on_server
if SIMPLE_TEST:
    TRAIN_FROM = 0
    TRAIN_NUM = 1

    VAL_FROM = 1
    VAL_NUM = 1
else:
    TRAIN_FROM = 0
    TRAIN_NUM = 12

    VAL_FROM = 12
    VAL_NUM = 3

# hyper paramerters
BATCH_SIZE = AHP.batch_size
print('BATCH_SIZE:', BATCH_SIZE) if debug else None
SEQ_LEN = AHP.sequence_length
print('SEQ_LEN:', SEQ_LEN) if debug else None

NUM_EPOCHS = 100  # SLTHP.num_epochs
LEARNING_RATE = 1e-3  # SLTHP.learning_rate
WEIGHT_DECAY = 1e-5  # SLTHP.weight_decay
CLIP = 0.5  # SLTHP.clip

NUM_ITERS = 100  # 100

# use too many Files may cause the following problem:
# ERROR: Unexpected bus error encountered in worker. This might be caused by insufficient shared memory (shm).
FILE_SIZE = 25  # 100

EVAL_INTERFEVL = 200
EVAL_NUM = 50

# set random seed
# is is actually effective
torch.manual_seed(SLTHP.seed)
np.random.seed(SLTHP.seed)

def getReplayData(path, replay_files, from_index=0, end_index=None):
    td_list = []
    for i, replay_file in enumerate(tqdm(replay_files)):
        try:
            replay_path = path + replay_file
            print('replay_path:', replay_path) if 1 else None

            do_write = False
            if i >= from_index:
                if end_index is None:
                    do_write = True
                elif end_index is not None and i < end_index:
                    do_write = True

            if not do_write:
                continue

            features, labels = torch.load(replay_path)
            print('features.shape:', features.shape) if debug else None
            print('labels.shape::', labels.shape) if debug else None

            td_list.append(ReplayTensorDataset(features, labels))

        except Exception as e:
            traceback.print_exc()

    return td_list

def main_worker(device):

    print('==> Making model..')
    net = ArchModel()

    if RESTORE:
        # use state dict to restore
        net.load_state_dict(torch.load(RESTORE_PATH, map_location=device), strict=False)

    net = net.to(device)

    num_params = sum(p.numel() for p in net.parameters() if p.requires_grad)
    print('The number of parameters of model is', num_params)

    print('==> Preparing data..')

    replay_files = os.listdir(PATH)
    print('length of replay_files:', len(replay_files)) if debug else None
    replay_files.sort()

    train_list = getReplayData(PATH, replay_files, from_index=TRAIN_FROM, end_index=TRAIN_FROM + TRAIN_NUM)
    val_list = getReplayData(PATH, replay_files, from_index=VAL_FROM, end_index=VAL_FROM + VAL_NUM)

    train_set = ConcatDataset(train_list)
    val_set = ConcatDataset(val_list)

    print('len(train_set)', len(train_set))
    print('len(val_set)', len(val_set))

    train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True,
                              num_workers=NUM_WORKERS, pin_memory=False)

    val_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=False,
                            num_workers=NUM_WORKERS, pin_memory=False)

    print('len(train_loader)', len(train_loader))
    print('len(val_loader)', len(val_loader))

    optimizer = Adam(net.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)

    train(net, optimizer, train_set, train_loader, device, val_set, val_loader)

def train(net, optimizer, train_set, train_loader, device, val_set, val_loader=None):

    now = datetime.datetime.now()
    summary_path = "./log/" + now.strftime("%Y%m%d-%H%M%S") + "/"
    writer = SummaryWriter(summary_path)

    # model path
    SAVE_PATH = os.path.join(MODEL_PATH, MODEL + "_" + time.strftime("%y-%m-%d_%H-%M-%S", time.localtime()))

    epoch_start = time.time()
    batch_iter = 0

    for epoch in range(NUM_EPOCHS):

        loss_sum = 0

        for batch_idx, (features, labels) in enumerate(train_loader):
            # put model in train mode
            net.train()
            start = time.time()

            feature_tensor = features.to(device).float()
            labels_tensor = labels.to(device).float()
            del features, labels

            loss, loss_list, \
                acc_num_list = Loss.get_sl_loss_for_tensor(feature_tensor,
                                                           labels_tensor, net,
                                                           decrease_smart_opertaion=True,
                                                           return_important=True,
                                                           only_consider_small=False
                                                           )
            del feature_tensor, labels_tensor

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            # add a grad clip
            parameters = [p for p in net.parameters() if p is not None and p.requires_grad]
            torch.nn.utils.clip_grad_norm_(parameters, CLIP)

            loss_value = float(loss.item())

            loss_sum += loss_value
            del loss

            action_accuracy = acc_num_list[0] / (acc_num_list[1] + 1e-9)
            move_camera_accuracy = acc_num_list[2] / (acc_num_list[3] + 1e-9)
            non_camera_accuracy = acc_num_list[4] / (acc_num_list[5] + 1e-9)
            short_important_accuracy = acc_num_list[6] / (acc_num_list[7] + 1e-9)

            location_accuracy = acc_num_list[8] / (acc_num_list[9] + 1e-9)
            location_distance = acc_num_list[11] / (acc_num_list[9] + 1e-9)

            selected_units_accuracy = acc_num_list[12] / (acc_num_list[13] + 1e-9)
            selected_units_coverage = acc_num_list[14] / (acc_num_list[13] + 1e-9)

            target_unit_accuracy = acc_num_list[16] / (acc_num_list[17] + 1e-9)

            batch_time = time.time() - start

            batch_iter += 1
            print('batch_iter', batch_iter)

            if batch_iter % EVAL_INTERFEVL == 0:
                print('Epoch: [{}/{}]| loss: {:.3f} | acc: {:.3f} | batch time: {:.3f}s '.format(
                    batch_iter, epoch, loss_sum / (batch_iter + 1), action_accuracy, batch_time))

                # torch.save(net, SAVE_PATH + "" + ".pkl")
                # we use new save ways, only save the state_dict, and the extension changes to pt
                torch.save(net.state_dict(), SAVE_PATH + "" + ".pth")

            write(writer, loss_value, loss_list, action_accuracy,
                  move_camera_accuracy, non_camera_accuracy, short_important_accuracy,
                  location_accuracy, location_distance, selected_units_accuracy,
                  selected_units_coverage, target_unit_accuracy,
                  batch_iter)

            gc.collect()

        if True:
            print('eval begin')
            val_loss, val_acc = eval(net, val_set, val_loader, device)
            print('eval end')

            print("Val loss: {:.6f}.".format(val_loss))
            writer.add_scalar('Val/Loss', val_loss, batch_iter)

            # for accuracy of actions in val
            print("Val action acc: {:.6f}.".format(val_acc[0]))
            writer.add_scalar('Val/action Acc', val_acc[0], batch_iter)
            print("Val move_camera acc: {:.6f}.".format(val_acc[1]))
            writer.add_scalar('Val/move_camera Acc', val_acc[1], batch_iter)
            print("Val non_camera acc: {:.6f}.".format(val_acc[2]))
            writer.add_scalar('Val/non_camera Acc', val_acc[2], batch_iter)
            print("Val short_important acc: {:.6f}.".format(val_acc[3]))
            writer.add_scalar('Val/short_important Acc', val_acc[3], batch_iter)
            print("Val location_accuracy acc: {:.6f}.".format(val_acc[4]))
            writer.add_scalar('Val/location_accuracy Acc', val_acc[4], batch_iter)
            print("Val location_distance acc: {:.6f}.".format(val_acc[5]))
            writer.add_scalar('Val/location_distance Acc', val_acc[5], batch_iter)
            print("Val selected_units_accuracy acc: {:.6f}.".format(val_acc[6]))
            writer.add_scalar('Val/selected_units_accuracy Acc', val_acc[6], batch_iter)
            print("Val selected_units_coverage acc: {:.6f}.".format(val_acc[7]))
            writer.add_scalar('Val/selected_units_coverage Acc', val_acc[7], batch_iter)
            print("Val target_unit_accuracy acc: {:.6f}.".format(val_acc[8]))
            writer.add_scalar('Val/target_unit_accuracy Acc', val_acc[8], batch_iter)

            del val_loss, val_acc

            gc.collect()

    elapse_time = time.time() - epoch_start
    elapse_time = datetime.timedelta(seconds=elapse_time)
    print("Training time {}".format(elapse_time))

def eval(model, val_set, val_loader, device):
    model.eval()

    # should we use it ？
    # model.core.lstm.train()

    loss_sum = 0.0
    i = 0

    action_acc_num = 0.
    action_all_num = 0.

    move_camera_action_acc_num = 0.
    move_camera_action_all_num = 0.

    non_camera_action_acc_num = 0.
    non_camera_action_all_num = 0.

    short_important_action_acc_num = 0.
    short_important_action_all_num = 0.

    location_acc_num = 0.
    location_effect_num = 0.
    location_dist = 0.

    selected_units_correct_num, selected_units_gt_num, selected_units_pred_num = 0, 0, 0

    target_unit_correct_num, target_unit_all_num = 0, 0

    for i, (features, labels) in enumerate(val_loader):

        if False and i > EVAL_NUM:
            break

        feature_tensor = features.to(device).float()
        labels_tensor = labels.to(device).float()
        del features, labels

        with torch.no_grad():
            loss, _, acc_num_list = Loss.get_sl_loss_for_tensor(feature_tensor,
                                                                labels_tensor, model,
                                                                decrease_smart_opertaion=True,
                                                                return_important=True,
                                                                only_consider_small=False)
            del feature_tensor, labels_tensor

        print('eval i', i, 'loss', loss) if debug else None

        loss_sum += loss.item()

        action_acc_num += acc_num_list[0]
        action_all_num += acc_num_list[1]

        move_camera_action_acc_num += acc_num_list[2]
        move_camera_action_all_num += acc_num_list[3]

        non_camera_action_acc_num += acc_num_list[4]
        non_camera_action_all_num += acc_num_list[5]

        short_important_action_acc_num += acc_num_list[6]
        short_important_action_all_num += acc_num_list[7]

        location_acc_num += acc_num_list[8]
        location_effect_num += acc_num_list[9]
        location_dist += acc_num_list[11]

        selected_units_correct_num += acc_num_list[12]
        selected_units_gt_num += acc_num_list[13]
        selected_units_pred_num += acc_num_list[14]

        target_unit_correct_num += acc_num_list[16]
        target_unit_all_num += acc_num_list[17]

        del loss, acc_num_list

        gc.collect()

    val_loss = loss_sum / (i + 1e-9)

    action_accuracy = action_acc_num / (action_all_num + 1e-9)
    move_camera_accuracy = move_camera_action_acc_num / (move_camera_action_all_num + 1e-9)
    non_camera_accuracy = non_camera_action_acc_num / (non_camera_action_all_num + 1e-9)
    short_important_accuracy = short_important_action_acc_num / (short_important_action_all_num + 1e-9)

    location_accuracy = location_acc_num / (location_effect_num + 1e-9)
    location_distance = location_dist / (location_effect_num + 1e-9)

    selected_units_accuracy = selected_units_correct_num / (selected_units_gt_num + 1e-9)
    selected_units_coverage = selected_units_pred_num / (selected_units_gt_num + 1e-9)

    target_unit_accuracy = target_unit_correct_num / (target_unit_all_num + 1e-9)

    return val_loss, [action_accuracy, move_camera_accuracy, non_camera_accuracy,
                      short_important_accuracy, location_accuracy, location_distance,
                      selected_units_accuracy, selected_units_coverage, target_unit_accuracy]

def write(writer, loss, loss_list, action_accuracy, move_camera_accuracy,
          non_camera_accuracy, short_important_accuracy,
          location_accuracy, location_distance,
          selected_units_accuracy, selected_units_coverage, target_unit_accuracy,
          batch_iter):

    print("One batch loss: {:.6f}.".format(loss))
    writer.add_scalar('OneBatch/Loss', loss, batch_iter)

    if True:
        print("One batch action_type_loss loss: {:.6f}.".format(loss_list[0]))
        writer.add_scalar('OneBatch/action_type_loss', loss_list[0], batch_iter)

        print("One batch delay_loss loss: {:.6f}.".format(loss_list[1]))
        writer.add_scalar('OneBatch/delay_loss', loss_list[1], batch_iter)

        print("One batch queue_loss loss: {:.6f}.".format(loss_list[2]))
        writer.add_scalar('OneBatch/queue_loss', loss_list[2], batch_iter)

        print("One batch units_loss loss: {:.6f}.".format(loss_list[3]))
        writer.add_scalar('OneBatch/units_loss', loss_list[3], batch_iter)

        print("One batch target_unit_loss loss: {:.6f}.".format(loss_list[4]))
        writer.add_scalar('OneBatch/target_unit_loss', loss_list[4], batch_iter)

        print("One batch target_location_loss loss: {:.6f}.".format(loss_list[5]))
        writer.add_scalar('OneBatch/target_location_loss', loss_list[5], batch_iter)

        print("One batch action_accuracy: {:.6f}.".format(action_accuracy))
        writer.add_scalar('OneBatch/action_accuracy', action_accuracy, batch_iter)

        print("One batch move_camera_accuracy: {:.6f}.".format(move_camera_accuracy))
        writer.add_scalar('OneBatch/move_camera_accuracy', move_camera_accuracy, batch_iter)

        print("One batch non_camera_accuracy: {:.6f}.".format(non_camera_accuracy))
        writer.add_scalar('OneBatch/non_camera_accuracy', non_camera_accuracy, batch_iter)

        print("One batch short_important_accuracy: {:.6f}.".format(short_important_accuracy))
        writer.add_scalar('OneBatch/short_important_accuracy', short_important_accuracy, batch_iter)

        print("One batch location_accuracy: {:.6f}.".format(location_accuracy))
        writer.add_scalar('OneBatch/location_accuracy', location_accuracy, batch_iter)

        print("One batch location_distance: {:.6f}.".format(location_distance))
        writer.add_scalar('OneBatch/location_distance', location_distance, batch_iter)

        print("One batch selected_units_accuracy: {:.6f}.".format(selected_units_accuracy))
        writer.add_scalar('OneBatch/selected_units_accuracy', selected_units_accuracy, batch_iter)

        print("One batch selected_units_coverage: {:.6f}.".format(selected_units_coverage))
        writer.add_scalar('OneBatch/selected_units_coverage', selected_units_coverage, batch_iter)

        print("One batch target_unit_accuracy: {:.6f}.".format(target_unit_accuracy))
        writer.add_scalar('OneBatch/target_unit_accuracy', target_unit_accuracy, batch_iter)

def test(on_server):
    # gpu setting
    ON_GPU = torch.cuda.is_available()
    DEVICE = torch.device("cuda:0" if ON_GPU else "cpu")

    if ON_GPU:
        if torch.backends.cudnn.is_available():
            print('cudnn available')
            print('cudnn version', torch.backends.cudnn.version())
            torch.backends.cudnn.enabled = True
            torch.backends.cudnn.benchmark = True

    main_worker(DEVICE)

max_stars_repo_name: cloneniu/mini-AlphaStar

max_issues_repo_name: cloneniu/mini-AlphaStar

max_forks_repo_name: cloneniu/mini-AlphaStar


text:
import numpy as np
import random

#randomly shuts of neurons from each hidden layer (i.e., forces them to output zero) to combat overfitting.
def dropout(a, z, dropout_rate, neurons):
    turn_off = int(np.floor(dropout_rate*neurons)) #determines how many neurons from the layer to shut off
    picks = random.sample(range(neurons), turn_off) #randomly picks that number of neurons
    for i in picks: a[i] = 0; z[i] = 0 #zeros out the activations we chose
    return a, z

max_stars_repo_name: lstefanello/perceptron

max_issues_repo_name: lstefanello/perceptron

max_forks_repo_name: lstefanello/perceptron


text:
import numpy as np
import h5py
import matplotlib.pyplot as plt

plt.rcParams['figure.figsize'] = (5.0, 4.0)
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

np.random.seed(1)

def zero_pad(X, pad):

    '''
    Pad with zeros all images of the dataset X. The padding is applied to the height and width of an image,
    as illustrated in Figure 1.
    :param X:python numpy array of shape (m, n_H, n_W, n_C) representing a batch of m images
    :param pad:integer, amount of padding around each image on vertical and horizontal dimensions
    :return:
    X_pad:padded image of shape (m, n_H + 2*pad, n_W + 2*pad, n_C)
    '''

    X_pad = np.pad(X, ((0, 0), (pad, pad), (pad, pad), (0, 0)), 'constant', constant_values=0)

    return X_pad
'''
np.random.seed(1)
x = np.random.randn(4, 3, 3, 2)
x_pad = zero_pad(x, 2)
print('x.shape = ', x.shape)
print('x_pad.shape', x_pad.shape)
print('x[1, 1] = ', x[1, 1])
print('x_pad[1, 1] = ', x_pad[1, 1])

fig, axarr = plt.subplots(1, 2)
axarr[0].set_title('x')
axarr[0].imshow(x[0, :, :, 0])
axarr[1].set_title('x_pad')
axarr[1].imshow(x_pad[0, :, :, 0])
fig.show()
'''

def conv_single_step(a_slice_prev, W, b):

    '''
    Apply one filter defined by parameters W on a single slice (a_slice_prev) of the output activation
    of the previous layer.
    :param a_slice_prev:slice of input data of shape (f, f, n_C_prev)
    :param W: Weight parameters contained in a window - matrix of shape (f, f, n_C_prev)
    :param b:Bias parameters contained in a window - matrix of shape (1, 1, 1)
    :return:
    Z:a scalar value, result of convolving the sliding window (W, b) on a slice x of the input data
    '''

    s = np.multiply(a_slice_prev, W) + b
    Z = np.sum(s)

    return Z

'''
np.random.seed(1)
a_slice_prev= np.random.randn(4, 4, 3)
W = np.random.randn(4, 4, 3)
b = np.random.randn(1, 1, 1)
Z = conv_single_step(a_slice_prev, W, b)
print('Z=', Z)
'''

def conv_forward(A_prev, W, b, h_paramaters):

    '''
    Implements the forward propagation for a convolution function
    :param A_prev:output activations of the previous layer, numpy array of shape (m, n_H_prev, n_W_prev, n_C_prev)
    :param W:Weights, numpy array of shape (f, f, n_C_prev, n_C)
    :param b:Biases, numpy array of shape (1, 1, 1, n_C)
    :param h_paramaters:python dictionary containing "stride" and "pad"
    :return:
        Z: conv output, numpy array of shape (m, n_H, n_W, n_C)
        cache: cache of values needed for the conv_backward() function
    '''

    (m, n_H_prev, n_W_prev, n_C_prev) = A_prev.shape
    (f, f, n_C_prev, n_C) = W.shape
    stride = h_paramaters['stride']
    pad = h_paramaters['pad']

    # 纵向卷积次数
    n_H = int((n_H_prev + 2 * pad - f) / stride) + 1
    # 横向卷积次数
    n_W = int((n_W_prev + 2 * pad - f) / stride) + 1

    Z = np.zeros((m, n_H, n_W, n_C))

    A_prev_pad = zero_pad(A_prev, pad)

    for i in range(m):
        a_prev_pad = A_prev_pad[i]
        for h in range(n_H):
            for w in range(n_W):
                for c in range(n_C):

                    # 计算卷积区域
                    vert_start = h * stride
                    vert_end = vert_start + f
                    horiz_start = w * stride
                    horiz_end = horiz_start + f

                    a_slice_prev = a_prev_pad[vert_start: vert_end, horiz_start: horiz_end, :]

                    Z[i, h, w, c] = conv_single_step(a_slice_prev, W[..., c], b[..., c])
    assert(Z.shape == (m, n_H, n_W, n_C))
    cache = (A_prev, W, b, h_paramaters)
    return Z, cache

'''
np.random.seed(1)
A_prev = np.random.randn(10, 4, 4, 3)
W = np.random.randn(2, 2, 3, 8)
b = np.random.randn(1, 1, 1, 8)
h_parameters = {'pad': 2,
                'stride': 1}

Z, conv_cache = conv_forward(A_prev, W, b, h_parameters)
print("Z's mean = ", np.mean(Z))
print('cache_conv[0][1][2][3] = ', conv_cache[0][1][2][3])
'''

def pool_forward(A_prev, h_parameters, mode='max'):

    '''
    Implements the forward pass of the pooling layer
    :param A_prev: Input data, numpy array of shape (m, n_H_prev, n_W_prev, n_C_prev)
    :param h_parameters:python dictionary containing "f" and "stride"
    :param mode:the pooling mode you would like to use, defined as a string ("max" or "average")
    :return:
        A:output of the pool layer, a numpy array of shape (m, n_H, n_W, n_C)
        cache:cache used in the backward pass of the pooling layer, contains the input and hparameters
    '''

    (m, n_H_prev, n_W_prev, n_C_prev) = A_prev.shape

    f = h_parameters['f']
    stride = h_parameters['stride']

    n_H = int((n_H_prev - f) / stride) + 1
    n_W = int((n_W_prev - f) / stride) + 1
    n_C = n_C_prev

    A = np.zeros((m, n_H, n_W, n_C))

    for i in range(m):
        for h in range(n_H):
            for w in range(n_W):
                for c in range(n_C):
                    # 计算池化区域
                    vert_start = h * stride
                    vert_end = vert_start + f
                    horiz_start = w * stride
                    horiz_end = horiz_start + f

                    a_prev_slice = A_prev[i, vert_start: vert_end, horiz_start: horiz_end, c]

                    if mode == 'max':
                        A[i, h, w, c] = np.max(a_prev_slice)
                    elif mode == 'average':
                        A[i, h, w, c] = np.mean(a_prev_slice)

    cache = (A_prev, h_parameters)

    assert(A.shape == (m, n_H, n_W, n_C))
    return A, cache

'''
np.random.seed(1)
A_prev = np.random.randn(2, 4, 4, 3)
h_parameters = {'stride': 1, 'f': 4}

A, cache = pool_forward(A_prev, h_parameters)
print('mode =  max')
print('A = ', A)
print()
print('mode = average')
print('A = ', A)
'''

max_stars_repo_name: LiuNingtao/Python-

max_issues_repo_name: LiuNingtao/Python-

max_forks_repo_name: LiuNingtao/Python-


text:
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

import numpy as np
import pandas as pd
from network import Simple_Net

class DQN:
    def __init__(self,
                 n_states,
                 n_actions,
                 memory_size,
                 replace_target_iter,
                 epsilon=0.15,
                 ep_decay=1e-7,
                 gamma=0.9,
                 lr=0.05,
                 batch_size=32,
                 graph=True
                 ):
        self.device = torch.device(
            "cuda" if torch.cuda.is_available() else "cpu")
        # env
        self.n_states = n_states
        self.n_actions = n_actions
        self.gamma = gamma

        # memory(s,a,r,s_)
        self.memory_size = memory_size
        self.memory = pd.DataFrame(np.zeros((self.memory_size, n_states*2+2)))

        # greedy
        self.epsilon = epsilon
        self.ep_decay = ep_decay

        # net
        self.target_net, self.eval_net = self.build_net()

        # opt
        self.lr = lr
        self.loss = nn.MSELoss()
        self.optim = optim.Adam(self.eval_net.parameters(), lr=lr)
        self.batch_size = batch_size

        # counter
        self.replace_target_iter = replace_target_iter
        self.learn_step_counter = 0

        # record
        self.cost_history = []

    def build_net(self):
        target_net = Simple_Net(self.n_states, self.n_actions, 20)

        eval_net = Simple_Net(self.n_states, self.n_actions, 20)

        return target_net.to(self.device), eval_net.to(self.device)

    def save_to_memory(self, s, a, r, s_):
        if not hasattr(self, 'memory_counter'):
            self.memory_counter = 0
        transition = np.hstack((s, [a, r], s_))

        # replace old memory to new memory
        index = self.memory_counter % self.memory_size
        self.memory.iloc[index, :] = transition
        self.memory_counter += 1

    def choose_action(self, s):
        # to 2 dimension
        s = torch.tensor(s[np.newaxis, :], dtype=torch.float32).to(self.device)
        # exploitation
        if np.random.uniform() > self.epsilon:
            action_value = self.eval_net(s)
            action = int(action_value.argmax())
        # exploration
        else:
            action = np.random.randint(0, self.n_actions)
        return action

    def replace_target_params(self):
        self.target_net.load_state_dict(self.eval_net.state_dict())

    def learn(self):
        # check replace target
        if self.learn_step_counter % self.replace_target_iter == 0:
            self.replace_target_params()
            print('target_net update !')

        # random sampling
        if self.memory_counter > self.memory_size:
            # 記憶超過memoy size 直接作 random sampling
            batch_memory = self.memory.sample(self.batch_size)
        else:
            # 如果記憶次數不及batch_size,重複抽取
            batch_memory = self.memory.iloc[:self.memory_counter].sample(
                self.batch_size, replace=True)
        # catch
        s_ = torch.tensor(
            batch_memory.iloc[:, -self.n_states:].values, dtype=torch.float32)
        s = torch.tensor(
            batch_memory.iloc[:, :self.n_states].values, dtype=torch.float32)
        a = batch_memory.iloc[:, self.n_states:self.n_states+1].values
        r = torch.tensor(
            batch_memory.iloc[:, self.n_states+1:self.n_states+2].values, dtype=torch.float32)

        s, r, s_ = s.to(self.device), r.to(self.device), s_.to(self.device)

        # q
        idx = a.reshape(a.shape[0],)

        q_next = self.target_net(s_).detach()
        q_eval = self.eval_net(s)

        q_esti = q_eval[torch.arange(len(idx)), idx]
        q_target = r.squeeze()+q_next.max(1)[0]

        # cost function
        cost = self.loss(q_target, q_esti)
        self.cost_history.append(cost)

        # backpropagation
        self.optim.zero_grad()
        cost.backward()
        self.optim.step()

        # epsilon decay
        self.epsilon -= self.ep_decay
        self.learn_step_counter += 1

    def plot_cost(self):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(len(self.cost_history)), self.cost_history)
        plt.show()


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jun 11 20:21:34 2020

@author: nickcostanzino
"""

def NN_structure(layers, perceptrons):

    A = list()
    for n in range(layers):
        A.append(perceptrons)
    return tuple(A)

def NN_structures(layers, perceptrons):

    A = list()

    for i in range(1, layers):
        for j in range(1, perceptrons):
            A.append(NN_structure(i, j))

    A = array(A)
    A = list(A)
    return A

def MSE(prediction, true):

    from sklearn.metrics import mean_squared_error
    mse = mean_squared_error(prediction, true)
    return mse

def process_simulator(f, sigma_X, sigma_e, N):

    import pandas as pd
    import numpy as np
    from scipy.optimize import fsolve
    from sklearn.linear_model import LinearRegression
    from sklearn.neural_network import MLPRegressor

    f = 'np.' + f

    e = np.random.normal(0,sigma_e,N)
    X = np.random.normal(0,sigma_X,N)
    Y = eval(f) + e

    df = pd.DataFrame()
    df['X'] = X
    df['Y'] = Y
    df['e'] = e

    return df

def performance_analyzer(func, sigma_X, sigma_e, N, number_of_partitions, number_of_simulations, output_folder):

    import pandas as pd
    import numpy as np
    from sklearn.metrics import r2_score, mean_squared_error
    from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV, TimeSeriesSplit
    from sklearn.ensemble import RandomForestRegressor
    from sklearn.linear_model import LinearRegression
    from sklearn.neural_network import MLPRegressor

    COLUMNS = ['training_data_points', 'testing_data_points', 'LR_intercept', 'LR_slope', 'NN_layers', 'NN_perceptrons',
               'NN_activation', 'NN_alpha', 'LR-In-Sample-R2', 'NN-In-Sample-R2', 'Best-Possible-In-Sample-R2',
               'LR-Out-Sample-R2', 'NN-Out-Sample-R2', 'Best-Possible-Out-Sample-R2']

    full_results = pd.DataFrame(columns = COLUMNS)

    for k in range(number_of_simulations):

        S = process_simulator(func, sigma_X, sigma_e, N)

        X = pd.DataFrame(S.X)
        Y = pd.DataFrame(S.Y)
        e = pd.DataFrame(S.e)

        L = len(X)

        for l in range(1, number_of_partitions):

            results = pd.DataFrame()
            print(l)

            test_L = int(L/number_of_partitions)
            train_L  = int(L/number_of_partitions*l)
            train_start = L- test_L-train_L
            train_end = L- test_L

            max_layers = 2
            max_perceptrons = 8

            structures = NN_structures(max_layers, max_perceptrons)

            print("NN_Structures:")
            print(structures)

            X_train = pd.DataFrame(X[train_start:train_end])
            Y_train = pd.DataFrame(Y[train_start:train_end])
            e_train = pd.DataFrame(e[train_start:train_end])

            X_test = pd.DataFrame(X[train_end +1: L])
            Y_test = pd.DataFrame(Y[train_end +1: L])
            e_test = pd.DataFrame(e[train_end +1: L])

            LR_regressor = LinearRegression(fit_intercept=True)
            LR_regressor.fit(X_train, Y_train)

            print("Fitted LR")

            NN_regressor = MLPRegressor()

            # default max_iter = 10000
            param_grid = {'hidden_layer_sizes': structures,
                               'activation': ['identity', 'relu'],
                               'alpha': [0.01, 0.001, 0.0001, 0.00001],
                               'learning_rate': ['adaptive'],
                               'solver': ['adam'],
                               'random_state': [0],
                               'early_stopping': [True],
                               'max_iter': [10000],
                               'warm_start': [True]}

            tscv = TimeSeriesSplit(n_splits=4)
            NN_gridsearch = GridSearchCV(estimator=NN_regressor, param_grid=param_grid, n_jobs=-1, verbose=False, cv= tscv)

            print("Performing grid search for NN")
            NN_gridsearch.fit(X_train, Y_train)
            print("Finished grid search for NN")

            NN_params = NN_gridsearch.best_params_
            NN_model = NN_gridsearch.best_estimator_

            LR_params = np.append([LR_regressor.intercept_], LR_regressor.coef_)

            pred_LR = LR_regressor.predict(X_test)
            pred_NN = NN_model.predict(X_test)

            insample_LR = LR_regressor.predict(X_train)
            insample_NN = NN_model.predict(X_train)

            results.loc[l, COLUMNS[0]] = train_L
            results.loc[l, COLUMNS[1]] = test_L
            results.loc[l, COLUMNS[2]] = LR_params[0]
            results.loc[l, COLUMNS[3]] = LR_params[1]
            results.loc[l, COLUMNS[4]] = len(NN_params['hidden_layer_sizes'])
            results.loc[l, COLUMNS[5]] = sum(NN_params['hidden_layer_sizes'])
            results.loc[l, COLUMNS[6]] = str(NN_params['activation'])
            results.loc[l, COLUMNS[7]] = str(NN_params['alpha'])

            results.loc[l, COLUMNS[8]] = LR_regressor.score(X_train,Y_train)
            results.loc[l, COLUMNS[9]] = NN_model.score(X_train,Y_train)
            results.loc[l, COLUMNS[10]] = 1 - (e_train*e_train).sum().values[0]/(((Y_train-Y_train.mean())*(Y_train-Y_train.mean())).sum().values[0])

            results.loc[l, COLUMNS[11]] = LR_regressor.score(X_test,Y_test)
            results.loc[l, COLUMNS[12]] = NN_model.score(X_test,Y_test)
            results.loc[l, COLUMNS[13]] = 1 - (e_test*e_test).sum().values[0]/(((Y_test-Y_test.mean())*(Y_test-Y_test.mean())).sum().values[0])

            full_results = full_results.append(results, ignore_index=False)

    full_results.to_csv(output_folder + '/' + str(func) + '_' + str(N) + '_' + str(number_of_partitions) + '_.csv')

def main():

    # default values
    # func = 'power(X,3)'
    # sigma_X=2
    # N=10000
    # number_of_partitions = 20
    # number_of_simulations=1000
    # output_folder = 'results'
    func = 'power(X,3)'
    sigma_X = 2
    N = 1000
    number_of_partitions = 2
    number_of_simulations = 1
    output_folder = 'results'

    for i in range(1):
        performance_analyzer(func, sigma_X, 0.5 * i, N, number_of_partitions, number_of_simulations, output_folder)

if __name__ == "__main__":
    main()

max_stars_repo_name: AretasSensorNetworks/APIExamples

max_issues_repo_name: AretasSensorNetworks/APIExamples

max_forks_repo_name: AretasSensorNetworks/APIExamples

reason: import numpy,from scipy


text:
#!/usr/bin/env python
"""
Telluric mask file format:
    - Columns: (0) wavelength [A], (1) "flux".
    - Every region (line or set of lines) that should be blocked because it is affected by tellurics is represented by 0110 in the flux column.
"""
from __future__ import division
from __future__ import print_function

import matplotlib.pyplot as plt
import numpy as np
from scipy.interpolate import interp1d

# Constants
C_MS = 2.99792458*1.e8  # Light speed [m/s]
C_KMS = 2.99792458*1.e5  # Light speed [km/s]

###############################################################################

def read_mask(filin):
    w, f = np.loadtxt(filin, usecols=[0, 1], unpack=True)
    return w, f

def mask2wlimits(w, f):
    """Get wavelength limits of the telluric lines masked.
    If a line is given by:
        wa 0
        wb 1
        wc 1
        wd 0
    then get wb as the first limit (store it in `w1`) and wc as the second limit (store it in `w2`).
    """
    w1, w2 = [], []
    for i in range(len(w)-1):
        # Normal line 0110
        if f[i-1] == 0 and f[i] == 1 and f[i+1] == 1 and f[i+2] == 0:
            w1.append(w[i])
            w2.append(w[i+1])
        # Blended lines 01110
        elif f[i-1] == 0 and f[i] == 1 and f[i+1] == 1 and f[i+2] == 1 and f[i+3] == 0:
            w1.append(w[i])
            w2.append(w[i+2])
        # Blended lines 011110
        elif f[i-1] == 0 and f[i] == 1 and f[i+1] == 1 and f[i+2] == 1 and f[i+3] == 1 and f[i+4] == 0:
            w1.append(w[i])
            w2.append(w[i+3])
    w1, w2 = np.asarray(w1), np.asarray(w2)
    return w1, w2

def wlimits2mask(w1, w2, dw=0.001):
    w, f = [], []
    for i in range(len(w1)):  # for each line
        w += [w1[i]-dw, w1[i], w2[i], w2[i]+dw]
        f += [0., 1., 1., 0.]
    w, f = np.asarray(w), np.asarray(f)
    return w, f

def interp_mask_inverse(w, f, kind='linear'):
    """Interpolation function of the telluric mask inverted to be used to mask telluric regions in the data.

    Example
    -------

    >>> # wt, ft : Telluric mask
    >>> # ws, fs : Spectrum
    >>> # Make the inverted mask function
    >>> MaskInv, _ = interp_mask_inverse(wt, ft) # function
    >>> # Make the spectrum flux 0 where there are tellurics
    >>> fs_clean = MaskInv(ws)
    >>> # Plot
    >>> plt.plot(ws, fs, label='spec')
    >>> plt.plot(ws, fs_clean, label='spec clean')
    >>> plt.fill_between(wt, 0, ft, label='tell mask')
    """
    f_inv = np.array(~np.array(f, dtype=bool), dtype=float)
    MaskInv = interp1d(w, f_inv, kind=kind)
    return MaskInv, f_inv

def broaden_mask(w, f, dv):
    """Broaden mask by a velocity `dv` [m/s].

    w_broaden = w*(1+dv/(2.99792458*10**8))

    Parameters
    ----------
    w, f : array
    dv : float
        Velocity to add/subtract to each telluric line, in  m/s.

    Returns
    -------
    w_broaden : array
    """

    C_MS = 2.99792458e8  # [m/s]

    dv = abs(dv)

    w_broaden = [[]]*len(w)
    idone = np.zeros(len(w))
    i = 0
    while i < len(w):
        if idone[i] == 0 and i < len(w)-1:
            # Line (where flux = 1)
            if f[i] == 1 and f[i+1] == 1:
                w_broaden[i] = w[i]*(1-dv/C_MS)
                w_broaden[i+1] = w[i+1]*(1+dv/C_MS)
                idone[i], idone[i+1] = 1, 1
                i = i+2
            # Base of the lines (where flux = 0)
            elif f[i] == 0 and f[i+1] == 0:
                w_broaden[i] = w[i]*(1+dv/C_MS)
                w_broaden[i+1] = w[i+1]*(1-dv/C_MS)
                idone[i], idone[i+1] = 1, 1
                i = i+2
            # Half base (can only happend at the beginning)
            elif f[i] == 0 and f[i+1] == 1:
                w_broaden[i] = w[i]
                i = i+1

        # Last value
        elif idone[i] == 0 and i == len(w)-1:
            if f[i] == 1:  # ...01
                w_broaden[i] = w[i]*(1-dv/C_MS)
            if f[i] == 0:  # ...10
                w_broaden[i] = w[i]*(1+dv/C_MS)
            i = i+1
    w_broaden = np.asarray(w_broaden)
    return w_broaden

def broaden_wlimits(w1, w2, dv):
    """Same as `broaden_mask`, but takes as input the wavelength limits of each mask line, instead of the wavelength and flux (0110).
    """
    C_MS = 2.99792458e8  # [m/s]

    dv = abs(dv)
    w1_broaden = np.asarray(w1)*(1-dv/C_MS)
    w2_broaden = np.asarray(w2)*(1+dv/C_MS)
    return w1_broaden, w2_broaden

def is_there_overlap_wlimits(w1, w2):
    """Check if there is overlap between 2 consecutive telluric regions."""
    ret = False
    for i in range(len(w1)-1):
        if w2[i] >= w1[i+1]:
            ret = True
            break
    return ret

def join_overlap_wlimits_once(w1, w2):
    """Join consecutive telluric regions if there is overlap.
    Only check once.
    """
    w1new, w2new = [], []
    iused = -1
    for i in range(len(w1)-1):  # for each telluric region
        if i == iused:
            continue
        w1new.append(w1[i])
        if w2[i] >= w1[i+1]:  # Join lines
            # w1new.append(w1[i])
            w2new.append(w2[i+1])
            iused = i+1
        else:
            w2new.append(w2[i])

    return w1new, w2new

def join_overlap_wlimits(w1, w2):
    """Join consecutive telluric regions if there is overlap.
    Join until there is no overlap.
    """
    while is_there_overlap_wlimits(w1, w2):
        w1, w2 = join_overlap_wlimits_once(w1, w2)
    return w1, w2

# Plots

def plot_mask(w, f, ax=None, fscale=1., zorder=0, xlab='', ylab='', leglab='', title='', **kwargs):
    if ax is None: ax = plt.gca()
    ax.fill_between(w, f, zorder=zorder, label=leglab, **kwargs)
    ax.set_title(title)
    ax.set_xlabel(xlab)
    ax.set_ylabel(ylab)
    return ax

max_stars_repo_name: mlafarga/raccoon

max_issues_repo_name: mlafarga/raccoon

max_forks_repo_name: mlafarga/raccoon


text:
import numpy as np

import config

class IrDataCsvReader:
    def __init__(self, file_path):
        self._file_path = file_path
        self._frames, self._raw_frame_data = self.get_frames_from_file(file_path)

    @staticmethod
    def get_frames_from_file(file_path):
        frames = []
        raw_frame_data = []
        with open(file_path, 'r') as file:
            raw_lines = file.readlines()
        lines = [x.strip() for x in raw_lines]
        frame_lines = lines[1:]
        for frame_line in frame_lines:
            raw_frame_data.append(frame_line)
            line_parts = frame_line.split(',')
            frame_data_str = line_parts[2:]
            frame_data_1d = [float(x) for x in frame_data_str]
            frame_2d = np.reshape(frame_data_1d, config.IR_CAMERA_RESOLUTION)
            frames.append(frame_2d)
        return frames, raw_frame_data

    def get_number_of_frames(self):
        return len(self._frames)

    def get_frame(self, n):
        return self._frames[n]

    def get_raw_frame_data(self, n) -> str:
        return self._raw_frame_data[n]

max_stars_repo_name: PUTvision/thermo-presence

max_issues_repo_name: PUTvision/thermo-presence

max_forks_repo_name: PUTvision/thermo-presence


text:
###############################################################################
# Find ideal global transformation for point cloud allignment between
# prediction and ground truth pcl
# Perform the optimization on validation set and use the same parameters for
# test set - Need to first save the val point clouds using save_pcl.py
###############################################################################

import os, sys
from os.path import join
import glob
import random
import pdb

import numpy as np
import tensorflow as tf

sys.path.append('../')
sys.path.append('../chamfer_utils/')
sys.path.append('../utils/')
from get_losses import get_3d_loss
from shapenet_taxonomy import shapenet_category_to_id

# Change category id as required
categ = 'chair'
categ = shapenet_category_to_id[categ]
N_PTS = 1024
N_ITERS = 5000
batch_size = 1
lr = 1e-3
print_n = 100
save_outputs = True
# Change flag to use custom ground truth.
use_custom_gt = True

gt_pcl_filename = 'pointcloud_1024.npy' if not use_custom_gt else 'gt_pointcloud_1024.npy'

# Change experiment directory as required
exp_dir = '../../expts_chair/expts/_temp'

## this is hard coded...again...
## log_proj_pcl_$MODE is the regex for it, MODE is set in save_pcl flags
pred_pcl_dir = join(exp_dir, 'log_proj_pcl_disp_test')
print("pred_pcl_dir is", pred_pcl_dir)

gt_pcl_dir = '../../data/ShapeNet_v1/%s'%categ
out_dir = join(exp_dir, 'log_proj_pcl_test_rot' if not use_custom_gt else 'log_proj_pcl_test_rot_gt')
if not os.path.exists(out_dir):
    os.makedirs(out_dir)

# @Rosie: This line doesn't work.
models_list = sorted(glob.glob(join(pred_pcl_dir, '*.npy')))
N_MODELS = len(models_list)
random.shuffle(models_list)

def get_feed_dict(cnt, models_list):
    pred_pcl_list = []; gt_pcl_list = [];
    for i in range(batch_size):
        idx = (cnt+i)%len(models_list)
        # print("index is", idx)
        fullPath = models_list[idx].split('/')
        # print("fullPath in components:", fullPath)

        model_name = models_list[idx].split('/')[-1].split('_')[1]
        _pred_pcl = np.load(models_list[idx])[:,:3]

        # print("model name is", model_name)
        # print("model_list first element is", models_list[0])
        # print("pred_pcl_dir is", pred_pcl_dir)
        _gt_pcl = np.load(join(gt_pcl_dir, model_name, gt_pcl_filename))
        gt_pcl_list.append(_gt_pcl)
        pred_pcl_list.append(_pred_pcl)
    pred_pcl_list = np.stack(pred_pcl_list, 0)
    gt_pcl_list = np.stack(gt_pcl_list, 0)

    feed_dict = {gt_pcl: gt_pcl_list, pred_pcl: pred_pcl_list}
    return feed_dict

def get_rotmat(angles, use_tilt=True):
    angles = tf.tile(angles, [batch_size])
    az, el, tilt = tf.split(angles, 3, -1)
    rotmat_az=[
                    [tf.ones_like(az),tf.zeros_like(az),tf.zeros_like(az)],
                    [tf.zeros_like(az),tf.cos(az),-tf.sin(az)],
                    [tf.zeros_like(az),tf.sin(az),tf.cos(az)]
                    ]

    rotmat_el=[
                    [tf.cos(el),tf.zeros_like(az), tf.sin(el)],
                    [tf.zeros_like(az),tf.ones_like(az),tf.zeros_like(az)],
                    [-tf.sin(el),tf.zeros_like(az), tf.cos(el)]
                    ]
    rotmat_tilt =[
                    [tf.cos(tilt),-tf.sin(tilt),tf.zeros_like(az)],
                    [tf.sin(tilt), tf.cos(tilt),tf.zeros_like(az)],
                    [tf.zeros_like(az),tf.zeros_like(az),tf.ones_like(az)]
                    ]

    rotmat_az = tf.transpose(tf.stack(rotmat_az, 0), [2,0,1])
    rotmat_el = tf.transpose(tf.stack(rotmat_el, 0), [2,0,1])
    rotmat_tilt = tf.transpose(tf.stack(rotmat_tilt, 0), [2,0,1])

    if use_tilt:
        rotmat = tf.matmul(rotmat_tilt, tf.matmul(rotmat_el, rotmat_az))
    else:
        rotmat = tf.matmul(rotmat_el, rotmat_az)
    return rotmat

pred_pcl = tf.placeholder(tf.float32, (batch_size, N_PTS, 3),
        'predicted_pcl')
gt_pcl = tf.placeholder(tf.float32, (batch_size, N_PTS, 3),
        'gt_pcl')
angles = tf.Variable([29.*np.pi/180., 0.*np.pi/180., 270.*np.pi/180.],
                      tf.float32, name='transformation_matrix')
angles_gt = tf.constant([np.pi/3., np.pi/6., np.pi/2.], tf.float32,
                         name='gt_transformation_matrix')

# Verify code by rotating gt by known angle and predict the same
rotmat_gt = get_rotmat(angles_gt, use_tilt=False)
gt_pcl_rot = tf.transpose(tf.matmul(
                            rotmat_gt, tf.transpose(gt_pcl, [0,2,1])), [0,2,1])
rot_mat = get_rotmat(angles)

pred_rot = tf.transpose(tf.matmul(
                            rot_mat, tf.transpose(pred_pcl, [0,2,1])), [0,2,1])
loss = get_3d_loss(gt_pcl, pred_rot, 'chamfer')

optim = tf.train.AdamOptimizer(lr).minimize(1e4*loss, var_list=angles)
saver = tf.train.Saver()

config = tf.ConfigProto()
config.gpu_options.allow_growth = True

with tf.Session(config=config) as sess:
    sess.run(tf.global_variables_initializer())

    # Optimize for angles using point cloud based loss (Chamfer distance)
    for iters in range(N_ITERS):
        feed_dict = get_feed_dict(iters, models_list)
        _loss, _angles, _ = sess.run([loss, angles, optim], feed_dict)

        if iters % print_n == 0:
            print _loss
            print _angles * 180. / np.pi

    if save_outputs:
        for iters in range(N_MODELS):
            if iters % 100 == 0:
                print iters, '/', N_MODELS
            feed_dict = get_feed_dict(iters, models_list)
            model_name = models_list[iters].split('/')[-1].split('_')[1]
            _pred_pcl = sess.run(pred_rot[0], feed_dict)
            np.savetxt(join(out_dir, model_name + '_pred.xyz'), _pred_pcl)
            np.savetxt(join(out_dir, model_name + '_gt.xyz'),
                    feed_dict[gt_pcl][0])
            print("saving output to", join(out_dir, model_name+'_pred.npy'))
            np.save(join(out_dir, model_name+'_pred.npy'), _pred_pcl)

        _rotmat, _angles = sess.run([rot_mat, angles], feed_dict)
        np.savetxt(join(exp_dir, 'angles.txt' if not use_custom_gt else 'angles_gt.txt'), _angles * 180. / np.pi)
        np.save(join(exp_dir, 'angles.npy' if not use_custom_gt else 'angles_gt.npy'), _angles * 180. / np.pi)
        print sess.run(rotmat_gt)
    print sess.run(rot_mat, feed_dict)


text:
import os
import random
import struct

import numpy as np
import tensorflow as tf
import trimesh

def _float_feature(value):
  return tf.train.Feature(float_list=tf.train.FloatList(value=value))

def _bytes_feature(value):
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _int_feature(value):
  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

def get_octree_from_points(points, octree_depth=8):
  #get representative points for building the groundtruth octree
  assert(points.shape[1] == 6 or points.shape[1] == 3)
  points_pos = points[:,:3]
  octree_node_length = 2.0 / (2**octree_depth)
  octree_node_index = np.unique(np.clip(np.floor((points_pos + 1) / octree_node_length).astype(np.int32), a_min = 0, a_max = (2**octree_depth) - 1), axis=0)

  return octree_node_index

def write_data_to_tfrecords(sample_list, tfrecords_filename):
  #options = tf.python_io.TFRecordOptions(compression_type=tf.python_io.TFRecordCompressionType.ZLIB)
  writer = tf.python_io.TFRecordWriter(tfrecords_filename)#, options=options
  n_data = len(sample_list)
  for i in range(n_data):
    if i % 100 == 0 or i == n_data-1:
      print('=========================\n')
      print('data loaded: {} / {}\n'.format(i+1, n_data))
      print('=========================\n')
    sample_category = sample_list[i][0]
    sample_id = sample_list[i][2]
    #print("{}:{}".format(sample_category, sample_id))
    signature = "{}:{}".format(sample_category, sample_id)

    #points_files contains the 100k points sampled from groundtruth surface, 3k input points to the network will be drawn from this set
    points_file = os.path.join("occupancy_networks-master\\data\\ShapeNet.build", sample_category, "4_pointcloud", sample_id+".npz")
    assert(os.path.exists(points_file))
    points_dict = np.load(points_file)
    points_position = points_dict['points']
    points_normal = points_dict['normals']
    assert(points_position.shape == points_normal.shape)
    print("points in bbox:{} {}".format(points_position.min(), points_position.max()))
    assert(points_position.min() >= -1 and points_position.max() <= 1)
    points = np.concatenate((points_position, points_normal), axis=-1).astype(np.float32)
    assert(points.shape[0] == 100000 and points.shape[1] ==6)

    mesh_file = sample_list[i][1] + ".obj"
    assert(os.path.exists(mesh_file))
    #prepare ground truth octree for decoder supervision
    #sample 5M points to build octree as gt octree for decode shape
    mesh = trimesh.load(mesh_file)
    sampled_points = trimesh.sample.sample_surface(mesh, 5000000)[0]
    #scale to [-1,1] with 5% padding
    #sampled_points *= 1.9
    assert(sampled_points.min() >= -1 and sampled_points.max() <= 1)
    octree = get_octree_from_points(sampled_points, 8)
    assert(octree.min() >=0 and octree.max() <= 255)
    octree_points = (octree*np.array([256*256, 256, 1])).sum(axis=-1)
    assert(octree_points.dtype == np.int)
    assert(octree_points.min() >=0 and octree_points.max() <= 16777215)
    #octree_points_file = sample_list[i][1] + "_octreep.npz"
    #np.savez_compressed(octree_points_file, octree_points=octree_points)
    #print("{} octree points".format(octree_points.shape))

    octree_points_seq_length = 2000
    #pad octree_points
    octree_points = np.reshape(octree_points, [-1])
    octree_points_pad_elements = (octree_points_seq_length - (octree_points.shape[0] % octree_points_seq_length)) % octree_points_seq_length
    octree_points = np.reshape(np.pad(octree_points, (0, octree_points_pad_elements), mode='constant', constant_values=octree_points[0]), [-1, octree_points_seq_length])

    sdf_samples_file = sample_list[i][1] + ".mat.bin"
    if(not os.path.exists(sdf_samples_file)):
      os.system("vdb_tsdf {} {}".format(mesh_file, sdf_samples_file))
    assert(os.path.exists(sdf_samples_file))

    sdf_samples = np.reshape(np.fromfile(sdf_samples_file, dtype=np.float32), [-1, 7])
    assert(sdf_samples.shape[1] == 7)
    sdf_points = sdf_samples[:,:3]
    assert(sdf_points.min() >= -1 and sdf_points.max() <= 1)

    #first get samples on 64^3 grid
    x = sdf_samples[:,0]
    y = sdf_samples[:,1]
    z = sdf_samples[:,2]
    coordx = np.round((x+1)*128).astype(int)
    coordy = np.round((y+1)*128).astype(int)
    coordz = np.round((z+1)*128).astype(int)
    depth6_ind = np.where(np.logical_and(np.logical_and(coordx % 4 == 0, coordy % 4 == 0), coordz % 4 == 0))
    depth6_sdf_samples = sdf_samples[depth6_ind]
    #print("{} samples in depth 6".format(depth6_sdf_samples.shape))

    #filter sdf samples with sdf > 2/32
    sdf_samples = sdf_samples[np.where(np.abs(sdf_samples[:,3]) < 2.0/32)]
    x = sdf_samples[:,0]
    y = sdf_samples[:,1]
    z = sdf_samples[:,2]
    coordx = np.round((x+1)*128).astype(int)
    coordy = np.round((y+1)*128).astype(int)
    coordz = np.round((z+1)*128).astype(int)
    depth7_ind = np.where(np.logical_and(np.logical_and(coordx % 2 == 0, coordy % 2 == 0), coordz % 2 == 0))
    depth7_sdf_samples = sdf_samples[depth7_ind]
    #print("{} samples in depth 7".format(depth7_sdf_samples.shape))

    def efficient_padding(sdf_samples):
      xyz_coord = np.round((sdf_samples[:,:3] + 1)*128).astype(np.int32)
      assert(xyz_coord.min() >= 0 and xyz_coord.max() <= 256)
      xyz_id = (xyz_coord*np.array([257*257, 257, 1])).sum(axis=-1)
      sdf_pad_elements = (1000 - (xyz_id.shape[0] % 1000)) % 1000

      xyz_id = np.pad(xyz_id, (0, sdf_pad_elements), mode='constant', constant_values=-100) #should be carefully dealt when parsing
      sdf_values = np.pad(np.reshape(sdf_samples[:,3:], [-1]), (0, 4*sdf_pad_elements), mode='constant', constant_values=-100)
      assert(xyz_id.shape[0] % 1000 == 0 and xyz_id.dtype == np.int32)
      assert(sdf_values.shape[0] == xyz_id.shape[0]*4 and sdf_values.dtype==np.float32)
      return np.reshape(xyz_id, [-1,1000]), np.reshape(sdf_values, [-1, 4000])

    depth7_coord, depth7_sdf_samples = efficient_padding(depth7_sdf_samples)
    depth6_coord, depth6_sdf_samples = efficient_padding(depth6_sdf_samples)
    assert(depth7_coord.dtype == np.int32 and depth6_coord.dtype == np.int32)
    assert(depth7_sdf_samples.dtype == np.float32 and depth6_sdf_samples.dtype == np.float32)

    sequence_feature = {"depth7_coord":tf.train.FeatureList(feature=[_int_feature(coord_row.tolist()) for coord_row in depth7_coord]),
                      "depth6_coord":tf.train.FeatureList(feature=[_int_feature(coord_row.tolist()) for coord_row in depth6_coord]),
                      "sdf_samples_depth7":tf.train.FeatureList(feature=[_float_feature(sdf_row.tolist()) for sdf_row in depth7_sdf_samples]),
                      "sdf_samples_depth6":tf.train.FeatureList(feature=[_float_feature(sdf_row.tolist()) for sdf_row in depth6_sdf_samples]),
                      "octree_points_idx":tf.train.FeatureList(feature=[_int_feature(idx_row.tolist()) for idx_row in octree_points])}
    feature = {'points': _bytes_feature(np.reshape(np.transpose(points), [-1]).astype(np.float16).tobytes()),
               "filename":_bytes_feature(signature.encode('utf8')),
               }
    example = tf.train.SequenceExample(context=tf.train.Features(feature=feature),
              feature_lists=tf.train.FeatureLists(feature_list=sequence_feature))
    writer.write(example.SerializeToString())
  writer.close()

def main():
  cat = ['02691156', '02828884', '02933112', '02958343', '03001627', '03211117', '03636649', '03691459', '04090263', '04256520', '04379243', '04401088', '04530566']
  rootpc = "occupancy_networks-master\\data\\ShapeNet.build\\"

  list=[]
  for item in cat:
    print("working on {}".format(item))
    dir_point = os.path.join(rootpc, item, 'watertight_normalized')
    assert(os.path.exists(dir_point))

    #training data split file follow occupancy networks
    split_file = os.path.join("convolutional_occupancy_networks-master\\data\\ShapeNet", item, "train.lst")
    assert(os.path.exists(split_file))
    split_file = open(split_file, "r")
    fns = [line.replace("\r", "").replace("\n", "") for line in split_file]
    split_file.close()

    assert(len(fns) != 0)
    for fn in fns:
      list.append((item, os.path.join(dir_point, fn), fn))

  print("{} samples total".format(len(list)))
  random.shuffle(list)

  tfrecords_filename = 'ShapeNet_points_{}_w_octree_grid_occupancy_compress.tfrecords'.format("all_train")
  write_data_to_tfrecords(list, tfrecords_filename)

if __name__ == '__main__':
  main()

max_issues_repo_name: SLAMWang/DeepMLS

max_forks_repo_name: SLAMWang/DeepMLS


text:
from PyTorch.ResNetDropoutSource import resnet50dropout, resnet18
from PyTorch.vggish.vggish import VGGish
from torch.utils.data import TensorDataset, DataLoader
import torch.nn.functional as F
import torch.nn as nn
import torch
import torch.optim as optim
import numpy as np
from sklearn.metrics import accuracy_score
import config
from PyTorch import config_pytorch
from datetime import datetime
import os

class Resnet50DropoutFull(nn.Module):
    def __init__(self, dropout=0.2):
#     def __init__(self):
        super(Resnet50DropoutFull, self).__init__()
        self.resnet = resnet50dropout(pretrained=config_pytorch.pretrained, dropout_p=0.2)
        # self.resnet = resnet18(pretrained=config_pytorch.pretrained)
        self.dropout = dropout
        self.n_channels = 3  # For building data correctly with dataloaders. Check if 1 works with pretrained=False
        ##Remove final linear layer
        self.resnet = nn.Sequential(*(list(self.resnet.children())[:-1]))
        self.fc1 = nn.Linear(2048, 1)  # 512 for resnet18, resnet34, 2048 for resnet50. Determine from x.shape() before fc1 layer
#         self.apply(_weights_init)
    def forward(self, x):
        x = self.resnet(x).squeeze()
#         x = self.fc1(x)
        # print(x.shape)
        x = self.fc1(F.dropout(x, p=self.dropout))
        x = torch.sigmoid(x)
        return x

class Resnet18DropoutFull(nn.Module):
    def __init__(self, dropout=0.2):
#     def __init__(self):
        super(Resnet18DropoutFull, self).__init__()
        # self.resnet = resnet50dropout(pretrained=config_pytorch.pretrained, dropout_p=0.2)
        self.resnet = resnet18(pretrained=config_pytorch.pretrained)
        self.dropout = dropout
        self.n_channels = 3  # For building data correctly with dataloaders. Check if 1 works with pretrained=False
        ##Remove final linear layer
        self.resnet = nn.Sequential(*(list(self.resnet.children())[:-1]))
        self.fc1 = nn.Linear(512, 1)  # 512 for resnet18, resnet34, 2048 for resnet50. Determine from x.shape() before fc1 layer
#         self.apply(_weights_init)
    def forward(self, x):
        x = self.resnet(x).squeeze()
#         x = self.fc1(x)
        # print(x.shape)
        x = self.fc1(F.dropout(x, p=self.dropout))
        x = torch.sigmoid(x)
        return x

class VGGishDropout(nn.Module):
    def __init__(self, preprocess=False, dropout=0.2):
        super(VGGishDropout, self).__init__()
        self.model_urls = config_pytorch.vggish_model_urls
        self.vggish = VGGish(self.model_urls, postprocess=False, preprocess=preprocess)
        self.dropout = dropout
        self.n_channels = 1  # For building data correctly with dataloaders
        self.fc1 = nn.Linear(128, 1)
    def forward(self, x):
        n_segments = x.shape[1]
        ##(Batch, Segments, C, H, W) -> (Batch*Segments, C, H, W)
        x = x.view(-1, 1, 96, 64)
        x = self.vggish.forward(x)
        x = self.fc1(F.dropout(x, p=self.dropout))
        x = torch.sigmoid(x)
        return x

class VGGishDropoutFeatB(nn.Module):
    def __init__(self, preprocess=False, dropout=0.2):
        super(VGGishDropoutFeatB, self).__init__()
        self.model_urls = config_pytorch.vggish_model_urls
        self.vggish = VGGish(self.model_urls, pretrained=config_pytorch.pretrained, postprocess=False, preprocess=preprocess)
        # self.vggish = nn.Sequential(*(list(self.vggish.children())[2:])) # skip layers
        self.vggish.embeddings = nn.Sequential(*(list(self.vggish.embeddings.children())[2:])) # skip layers
        self.dropout = dropout
        self.n_channels = 1  # For building data correctly with dataloaders
        self.fc1 = nn.Linear(128, 1)  # for multiclass
        # For application to embeddings, see:
        #https://github.com/tensorflow/models/blob/master/research/audioset/vggish/vggish_train_demo.py
    def forward(self, x):
        n_segments = x.shape[1]
        x = x.view(-1, 1, 30, 128) # Feat B
        x = self.vggish.forward(x)
        x = self.fc1(F.dropout(x, p=self.dropout))
        x = torch.sigmoid(x)
        return x

def build_dataloader(x_train, y_train, x_val=None, y_val=None, shuffle=True, n_channels=1):
    x_train = torch.tensor(x_train).float()
    if n_channels == 3:
        x_train = x_train.repeat(1,3,1,1)  # Repeat across 3 channels to match ResNet pre-trained model expectation
    y_train = torch.tensor(y_train).float()
    train_dataset = TensorDataset(x_train, y_train)
    train_loader = DataLoader(train_dataset, batch_size=config_pytorch.batch_size, shuffle=shuffle)

    if x_val is not None:
        x_val = torch.tensor(x_val).float()
        if n_channels == 3:
            x_val = x_val.repeat(1,3,1,1)
        y_val = torch.tensor(y_val).float()
        val_dataset = TensorDataset(x_val, y_val)
        val_loader = DataLoader(val_dataset, batch_size=config_pytorch.batch_size, shuffle=shuffle)

        return train_loader, val_loader
    return train_loader

def train_model(x_train, y_train, x_val=None, y_val=None, model = Resnet18DropoutFull()):
    if x_val is not None:  # TODO: check dimensions when supplying validation data.
        train_loader, val_loader = build_dataloader(x_train, y_train, x_val, y_val, n_channels = model.n_channels)

    else:
        train_loader = build_dataloader(x_train, y_train, n_channels = model.n_channels)

    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    print(f'Training on {device}')

    if torch.cuda.device_count() > 1:
        print("Using data parallel")
        model = nn.DataParallel(model, device_ids=list(range(torch.cuda.device_count())))

    model = model.to(device)

    # Change compatibility to other loss function, cross-test with main.
    criterion = nn.BCELoss()
    optimiser = optim.Adam(model.parameters(), lr=config_pytorch.lr)

    all_train_loss = []
    all_train_acc = []
    all_val_loss = []
    all_val_acc = []
    best_val_loss = np.inf
    best_val_acc = -np.inf

    # best_train_loss = np.inf
    best_train_acc = -np.inf

    best_epoch = -1
    checkpoint_name = None
    overrun_counter = 0

    for e in range(config_pytorch.epochs):
        train_loss = 0.0
        model.train()

        all_y = []
        all_y_pred = []
        for batch_i, inputs in enumerate(train_loader):

            ##Necessary in order to handle single and multi input feature spaces
            x = [xi.to(device).detach() for xi in inputs[:-1]]
            y = inputs[-1].to(device).detach().view(-1,1)

            if len(x) == 1:
                x = x[0]

            optimiser.zero_grad()
            y_pred = model(x)
            loss = criterion(y_pred, y)

            loss.backward()
            optimiser.step()

            train_loss += loss.item()
            all_y.append(y.cpu().detach())
            all_y_pred.append(y_pred.cpu().detach())

            del x
            del y

        all_train_loss.append(train_loss/len(train_loader))

        all_y = torch.cat(all_y)
        all_y_pred = torch.cat(all_y_pred)
        train_acc = accuracy_score(all_y.numpy(), (all_y_pred.numpy() > 0.5).astype(float))
        all_train_acc.append(train_acc)

        # Can add more conditions to support loss instead of accuracy. Use *-1 for loss inequality instead of acc
        if x_val is not None:
            val_loss, val_acc = test_model(model, val_loader, criterion, 0.5, device=device)
            all_val_loss.append(val_loss)
            all_val_acc.append(val_acc)

            acc_metric = val_acc
            best_acc_metric = best_val_acc
        else:
            acc_metric = train_acc
            best_acc_metric = best_train_acc
        if acc_metric > best_acc_metric:
            # if checkpoint_name is not None:
                # os.path.join(os.path.pardir, 'models', 'pytorch', checkpoint_name)

            checkpoint_name = f'model_e{e}_{datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}.pth'

            torch.save(model.state_dict(), os.path.join(config.model_dir, 'pytorch', checkpoint_name))
            print('Saving model to:', os.path.join(config.model_dir, 'pytorch', checkpoint_name))
            best_epoch = e
            best_train_acc = train_acc
            best_train_loss = train_loss
            if x_val is not None:
                best_val_acc = val_acc
                best_val_loss = val_loss
            overrun_counter = -1

        overrun_counter += 1
        if x_val is not None:
            print('Epoch: %d, Train Loss: %.8f, Train Acc: %.8f, Val Loss: %.8f, Val Acc: %.8f, overrun_counter %i' % (e, train_loss/len(train_loader), train_acc, val_loss/len(val_loader), val_acc,  overrun_counter))
        else:
            print('Epoch: %d, Train Loss: %.8f, Train Acc: %.8f, overrun_counter %i' % (e, train_loss/len(train_loader), train_acc, overrun_counter))
        if overrun_counter > config_pytorch.max_overrun:
            break
    return model

def test_model(model, test_loader, criterion, class_threshold=0.5, device=None):
    with torch.no_grad():
        if device is None:
            torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

        test_loss = 0.0
        model.eval()

        all_y = []
        all_y_pred = []
        counter = 1
        for x, y in test_loader:
            x, y = x.to(device), y.to(device)

            y_pred = model(x).squeeze()

            loss = criterion(y_pred, y)

            test_loss += loss.item()

            all_y.append(y.cpu().detach())
            all_y_pred.append(y_pred.cpu().detach())

            del x
            del y
            del y_pred

            counter +=1

        all_y = torch.cat(all_y)
        all_y_pred = torch.cat(all_y_pred)

        test_loss = test_loss/len(test_loader)
        test_acc = accuracy_score(all_y.numpy(), (all_y_pred.numpy() > class_threshold).astype(float))

    return test_loss, test_acc

def load_model(filepath, model=Resnet18DropoutFull()):
    # Instantiate model to inspect
    device = torch.device('cuda:0' if torch.cuda.is_available() else torch.device("cpu"))
    print(f'Training on {device}')

    if torch.cuda.device_count() > 1:
        print("Using data parallel")
        model = nn.DataParallel(model, device_ids=list(range(torch.cuda.device_count())))
    model = model.to(device)
    # Load trained parameters from checkpoint (may need to download from S3 first)

    if torch.cuda.is_available():
        map_location=lambda storage, loc: storage.cuda()
    else:
        map_location='cpu'

    checkpoint = model.load_state_dict(torch.load(filepath))

    return model

def evaluate_model(model, X_test, y_test, n_samples):
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    print(f'Evaluating on {device}')

    x_test = torch.tensor(X_test).float()
    if model.n_channels == 3:
        x_test = x_test.repeat(1,3,1,1)

    y_test = torch.tensor(y_test).float()
    test_dataset = TensorDataset(x_test, y_test)
    test_loader = DataLoader(test_dataset, batch_size=128, shuffle=False)

    y_preds_all = np.zeros([n_samples, len(y_test), 2])
    model.eval() # Important to not leak info from batch norm layers and cause other issues

    for n in range(n_samples):
        all_y_pred = []
        all_y = []
        for x, y in test_loader:
            x, y = x.to(device), y.to(device)

            y_pred = model(x).squeeze()
            all_y.append(y.cpu().detach())

            all_y_pred.append(y_pred.cpu().detach())

            del x
            del y
            del y_pred

        all_y_pred = torch.cat(all_y_pred)
        all_y = torch.cat(all_y)

        y_preds_all[n,:,1] = np.array(all_y_pred)
        y_preds_all[n,:,0] = 1-np.array(all_y_pred) # Check ordering of classes (yes/no)
        test_acc = accuracy_score(all_y.numpy(), (all_y_pred.numpy() > 0.5).astype(float))
        # print(test_acc)
    return y_preds_all

def evaluate_model_aggregated(model, X_test, y_test, n_samples):
    ''' Generate predictions for VGGish features (Feat. A) rescaled to time window of 1.92 second features (Feat. B)'''
    preds_aggregated_by_mean = []
    y_aggregated_prediction_by_mean = []
    y_target_aggregated = []

    for idx, recording in enumerate(X_test):
        n_target_windows = len(recording)//2  # Calculate expected length: discard edge
        y_target = np.repeat(y_test[idx],n_target_windows) # Create y array of correct length
        preds = evaluate_model(model, recording, np.repeat(y_test[idx],len(recording)),n_samples) # Sample BNN
#         preds = np.mean(preds, axis=0) # Average across BNN samples
#         print(np.shape(preds))
        preds = preds[:,:n_target_windows*2,:] # Discard edge case
#         print(np.shape(preds))
#         print('reshaping')
        preds = np.mean(preds.reshape(len(preds),-1,2,2), axis=2) # Average every 2 elements, keep samples in first dim
#         print(np.shape(preds))
        preds_y = np.argmax(preds)  # Append argmax prediction (label output)
        y_aggregated_prediction_by_mean.append(preds_y)
        preds_aggregated_by_mean.append(preds)  # Append prob (or log-prob/other space)
        y_target_aggregated.append(y_target)  # Append y_target
#     return preds_aggregated_by_mean, y_aggregated_prediction_by_mean, y_target_aggregated
    return np.hstack(preds_aggregated_by_mean), np.concatenate(y_target_aggregated)

max_stars_repo_name: HumBug-Mosquito/HumBugDB

max_issues_repo_name: HumBug-Mosquito/HumBugDB

max_forks_repo_name: HumBug-Mosquito/HumBugDB


text:
import os
from copy import deepcopy

import numpy as np
import pytest

from eolearn.core import EOPatch, FeatureType

from hiector.tasks.reference import MergeBBoxesTask, PrepareMRRTask
from hiector.utils.metrics import intersection_over_smallest

@pytest.fixture(name="test_eop", scope="module")
def test_eop(input_folder):
    eop = EOPatch.load(os.path.join(input_folder, "test-eop"))
    return eop

@pytest.fixture(name="test_eop_prepared", scope="module")
def test_eop_prepared(input_folder):
    eop = EOPatch.load(os.path.join(input_folder, "test-eop-prepared"))
    return eop

def test_prepare_mmr(test_eop):

    task = PrepareMRRTask(
        input_feature=(FeatureType.VECTOR_TIMELESS, "BUILDINGS"),
        output_feature=(FeatureType.VECTOR_TIMELESS, "BUILDINGS"),
    )
    prepared = task.execute(deepcopy(test_eop))

    vec_in = test_eop.vector_timeless["BUILDINGS"]
    vec_out = prepared.vector_timeless["BUILDINGS"]

    assert len(vec_in) >= len(vec_out)
    for _id, geom in vec_in[["_ID_", "geometry"]].values:
        assert np.array([x.buffer(1e-1).contains(geom) for x in vec_out.geometry.values], dtype=bool).sum() > 0, _id

def test_merge_bbox(test_eop_prepared):
    merge_task = MergeBBoxesTask(
        input_feature=(FeatureType.VECTOR_TIMELESS, "BUILDINGS_MRR"),
        output_feature=(FeatureType.VECTOR_TIMELESS, "BUILDINGS_MRR"),
        iou_method=intersection_over_smallest,
        iou_thr=0.4,
        sorting_col="area",
    )

    eop_merged = merge_task.execute(deepcopy(test_eop_prepared))

    vec_in = test_eop_prepared.vector_timeless["BUILDINGS_MRR"]
    vec_out = eop_merged.vector_timeless["BUILDINGS_MRR"]

    assert set(vec_out._ID_.values) == {7417907, 8063303, 8375354, 8376000}

max_stars_repo_name: sentinel-hub/hiector

max_issues_repo_name: sentinel-hub/hiector

max_forks_repo_name: sentinel-hub/hiector


text:
import sys
sys.path.append('../code')
from tools import load_data_tools as ldt
from tools import emg_tools as et
from AfferentInput import AfferentInput
import matplotlib.pyplot as plt
import numpy as np
import pickle

""" get gait cycles """
kinFile = "./generateForSimInputs/data/rat/#208_100903_P0_BIP_TM9_4_KIN.csv"
footHeight = ldt.readGlabRecordedAnalogData(kinFile,['height'],["RTip_Z"])
footHeight['dt'] = 0.005
footHeight['time'] = np.arange(len(footHeight['height']))*footHeight['dt']

fig1,ax = plt.subplots(1,figsize=(12,6))
ax.plot(footHeight['time']-footHeight['time'][0],footHeight['height'])

fig1.suptitle("Please select all heel strikes")
plt.draw()
plt.pause(1)
temp = fig1.ginput(n=0,timeout=240)
if len(temp)==0: raise(Exception("no heel strikes detected..."))
else: heelStrikes = np.atleast_1d([ float(x) for x,_ in temp])
print heelStrikes

fig1.suptitle("Please select all foot offs")
plt.draw()
plt.pause(1)
temp = fig1.ginput(n=0,timeout=240)
if len(temp)==0: raise(Exception("no foot offs detected..."))
else: footOffs = np.atleast_1d([ float(x) for x,_ in temp])
print footOffs

plt.draw()

gaitCyclesFileName = "./generateForSimInputs/output/ratGaitCycles.p"
with open(gaitCyclesFileName, 'w') as pickle_file:
    pickle.dump(heelStrikes, pickle_file)
    pickle.dump(footOffs, pickle_file)

max_stars_repo_name: neurorestore/neuraldynamicplatform

max_issues_repo_name: neurorestore/neuraldynamicplatform

max_forks_repo_name: neurorestore/neuraldynamicplatform


text:
"""
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Show all four cameras and their device numbers
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""
import cv2
import numpy as np
import yaml
from imutils.video import VideoStream

# usb devices of the four cameras
camera_devices = [0, 1, 2, 3]
# intrinsic parameter files of the cameras
camera_files = ["./yaml/front.yaml",
                "./yaml/back.yaml",
                "./yaml/left.yaml",
                "./yaml/right.yaml"]
# resolution of the video stream
W, H = 640, 480

def main():
    print("[INFO] Preparing camera devices ...")
    captures = [VideoStream(src=k, resolution=(W, H)).start() for k in camera_devices]

    print("[INFO] loading camera intrinsic files ...")
    undistort_maps = []
    for conf in camera_files:
        with open(conf, "r") as f:
            data = yaml.load(f)

        K = np.array(data["K"])
        D = np.array(data["D"])
        map1, map2 = cv2.fisheye.initUndistortRectifyMap(
            K,
            D,
            np.eye(3),
            K,
            (W, H),
            cv2.CV_16SC2
        )
        undistort_maps.append((map1, map2))

    # put all frames in one window
    corrected = np.zeros((2*H, 2*W, 3), dtype=np.uint8)
    region = {0: corrected[:H, :W],
              1: corrected[:H, W:],
              2: corrected[H:, :W],
              3: corrected[H:, W:]}

    while True:
        print("[INFO] processing camera data ...")
        frames = []
        for device, cap, (map1, map2) in zip(camera_devices, captures, undistort_maps):
            frame = cap.read()
            frame = cv2.remap(frame, map1, map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)
            cv2.putText(frame, "device: {}".format(device), (50, 50), cv2.FONT_HERSHEY_COMPLEX, 2, (0, 0, 255), 2)
            frames.append(frame)

        for i, frame in enumerate(frames):
            region[i][...] = frame

        cv2.imshow("corrected", corrected)

        key = cv2.waitKey(1) & 0xFF
        if key == ord("q"):
            break

    for cap in captures:
        cap.stop()

if __name__ == "__main__":
    main()


text:
from terraingrid import TerrainGrid
import numpy as np
import cv2 as cv
import matplotlib.pyplot as plt
import numpy.ma as ma
import random as rng
import time as time
import tsp

def erodilate(val, kernel, iterate):
    img_erosion = cv.erode(val, np.ones((kernel, kernel), np.uint8), iterations = iterate)
    img_errdil = cv.dilate(np.array(img_erosion), np.ones((kernel, kernel), np.uint8), iterations = iterate)
    return img_errdil

rd0 = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_37ez2.tif"
rd1 = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_37fz1.tif"
rasdf = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_37hn2.tif"
ehamr = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_25dn1.tif"
r2 = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_37fz2.tif"
path = r"C:\Users\siddh\Documents\DSMS\R_25GN1\r_25gn1.tif"

a = TerrainGrid((rd0), (1,1), 1)
a.show(-5, 50)
a.arrayValues = a.arrayValues[11000:12000, 5000:6000]
a.show(-5, 50)

c = cv.threshold(a.arrayValues, 2.4, 200, cv.THRESH_BINARY)[1].astype('uint8')
c = cv.erode(c, np.ones((2,2), np.uint8), iterations = 1)

n_labels, labels, stats, centroids = cv.connectedComponentsWithStats(c, connectivity=4)
print(n_labels)

variance = []

buildings = []
vegetation = []

inbuildings = []
invegetation = []
histogram = []
above = []
unique = np.delete(np.unique(labels), 0)

incount = 0
bray = np.gradient(a.arrayValues)[0]
plt.imshow(bray, vmin = -2, vmax = 2)
plt.show()

blda = np.zeros(labels.shape, np.uint8)
vega = np.zeros(labels.shape, np.uint8)
start = time.time()
for i in unique:
    print(i)
    if (stats[i, 4] > 600):

        org = ma.masked_not_equal(labels, i) / i
        var = np.std(org * bray)
        variance.append(var)
        histogram.append(var)
        if (var > 1.75):
            vegetation.append(i)
            invegetation.append(incount)
            incount +=1
            vega = np.add(vega, org.filled(0))
        else:
            if (var > 1):
                above.append((var, i, incount))
            buildings.append(i)
            inbuildings.append(incount)
            incount +=1
            blda = np.add(blda,org.filled(0))

print(len(buildings), len(vegetation), len(variance))
end = time.time()
print(end - start ," seconds elapsed cuh.")

plt.imshow(a.arrayValues)
plt.imshow(ma.masked_values(blda, 0), cmap = 'gist_gray', vmin = 0, vmax = 1)
plt.imshow(ma.masked_values(vega, 0), cmap = 'winter', vmin = 0, vmax = 1)
plt.show()

histogram2 = []
for i in histogram:
    if (i < 50):
        histogram2.append(i)

n, bins, patches = plt.hist(histogram2, 250, facecolor='blue', alpha=0.5)
plt.show()

count = 0

while (count < len(buildings)):
    temp = (ma.masked_not_equal(labels, buildings[count]) / buildings[count]).astype('uint8')

    temp = temp.filled()
    temp = erodilate(temp, 5, 15)
    temp[:, 0] = 63
    temp[:, -1] = 63
    temp[0, :] = 63
    temp[-1, :] = 63

    corners = cv.goodFeaturesToTrack(temp, 11, 0.1, 10)
    print(corners)
    trueindices = tsp.tsp(corners)[1]
    truecorners = []
    for i in trueindices:
        truecorners.append(corners[i])

    for i in truecorners:
        x,y = i.ravel()
        cv.circle(temp,(x,y),3,140,-1)

    plt.imshow(a.arrayValues)
    plt.imshow(ma.masked_values(temp, 63), cmap = 'jet', alpha = 0.5)
    print("Real index of: ", buildings[count], " and relative index of: ", inbuildings[count], " with variance of: ", variance[inbuildings[count]], " (B)")
    print(stats[buildings[count], 4])

    plt.show()
    count +=1
count = 0
while (count < len(vegetation)):
    plt.imshow((ma.masked_not_equal(labels, vegetation[count]) / vegetation[count]) * a.arrayValues)
    print("Real index of: ", vegetation[count], " and relative index of: ", invegetation[count], " with variance of: ", variance[invegetation[count]], " (V)")
    plt.show()
    count +=1