text:
from PIL import Image
from math import sqrt
import torch
import os
import csv
import numpy as np
import argparse
import h5py
from torchvision import transforms
from torch.autograd import Variable

def _read_data_handle(_filename):
	f = open(_filename, 'r')
	csv_file_handle = list(csv.reader(f))
	_n_lines = len(csv_file_handle)

	_data = []
	_line_idx = 1
	while _line_idx < _n_lines:

		dic = {};
		dic['img_filename'] = csv_file_handle[_line_idx][0]
		dic['n_point'] = int(csv_file_handle[_line_idx][2])

		dic['y_A'] = []
		dic['y_B'] = []
		dic['x_A'] = []
		dic['x_B'] = []
		dic['ordianl_relation'] = []

		_line_idx+=1

		for point_idx in range(0,dic['n_point']):
			dic['y_A'].append(int(csv_file_handle[_line_idx][0])-1)
			dic['x_A'].append(int(csv_file_handle[_line_idx][1])-1)
			dic['y_B'].append(int(csv_file_handle[_line_idx][2])-1)
			dic['x_B'].append(int(csv_file_handle[_line_idx][3])-1)

			if csv_file_handle[_line_idx][4] == '>':
				dic['ordianl_relation'].append(1)
			elif csv_file_handle[_line_idx][4] == '<':
				dic['ordianl_relation'].append(-1)
			elif csv_file_handle[_line_idx][4] == '=':
				dic['ordianl_relation'].append(0)

			_line_idx+=1

		_data.append(dic)

	# print(_data[0], _data[1])

	return len(_data), _data

def _evaluate_correctness_out(_batch_output, _batch_target, WKDR, WKDR_eq, WKDR_neq):
	# print(_batch_output.size())

	# print(_batch_output)

	n_gt_correct = torch.zeros(n_thresh)
	n_gt = 0

	n_lt_correct = torch.zeros(n_thresh)
	n_lt = 0

	n_eq_correct = torch.zeros(n_thresh)
	n_eq = 0

	for point_idx in range(0,_batch_target['n_point']):
		x_A = _batch_target['x_A'][point_idx]
		y_A = _batch_target['y_A'][point_idx]
		x_B = _batch_target['x_B'][point_idx]
		y_B = _batch_target['y_B'][point_idx]

		z_A = _batch_output[0,0,y_A,x_A]
		z_B = _batch_output[0,0,y_B,x_B]

		ground_truth = _batch_target['ordianl_relation'][point_idx]

		z_A_z_B = (z_A - z_B)

		# print(z_A_z_B, y_A,x_A, y_B,x_B, ground_truth)

		for thresh_idx in range(0,n_thresh):
			if z_A_z_B > thresh[thresh_idx]:
				_classify_res = 1
			elif z_A_z_B < -thresh[thresh_idx]:
				_classify_res = -1
			else:
				_classify_res = 0

			if _classify_res == 0 and ground_truth == 0:
				n_eq_correct[thresh_idx]+=1
			elif _classify_res == 1 and ground_truth == 1:
				n_gt_correct[thresh_idx]+=1
			elif _classify_res == -1 and ground_truth == -1:
				n_lt_correct[thresh_idx]+=1

		if ground_truth > 0:
			n_gt += 1
		elif ground_truth < 0:
			n_lt+=1
		elif ground_truth == 0:
			n_eq+=1

	for i in range(0,n_thresh):
		WKDR[i] = (1-(n_eq_correct[i]+n_lt_correct[i]+n_gt_correct[i])/(n_eq+n_lt+n_gt))
		WKDR_eq[i] = (1 - n_eq_correct[i]/n_eq)
		WKDR_neq[i] = (1 - (n_lt_correct[i] + n_gt_correct[i])/(n_lt+n_gt))

def inpaint_pad_output_our(output, img_original_width, img_original_height):
	crop = cmd_params.crop
	resize_height = img_original_height - 2*crop
	resize_width = img_original_width -2*crop
	output_min = torch.min(output)
	output_max = torch.max(output)-output_min
	output-=output_min
	output/=output_max
	scale = transforms.Compose([
		transforms.ToPILImage(),
		transforms.Scale((resize_width, resize_height)),
		transforms.ToTensor()
		])
	resize_output = scale(torch.Tensor(output[0]))
	resize_output*=output_max
	resize_output+=output_min
	padded_output = torch.Tensor(1,img_original_height, img_original_width)

	padded_output[0,crop:img_original_height-crop, crop:img_original_width-crop].copy_(resize_output)

	for i in range(0,crop):
		# print(resize_output[0, :,0])
		padded_output[0, crop:img_original_height-crop, i].copy_(resize_output[0, :,0])
		# print(resize_output[0,:,resize_width-1])
		padded_output[0, crop:img_original_height-crop, img_original_width-1-i].copy_(resize_output[0,:,resize_width-1])

	for i in range(0,crop):
		# print(padded_output[0,crop,:])
		padded_output[0, i, :].copy_(padded_output[0,crop,:])
		# print(padded_output[0, resize_height+crop-1 ,:])
		padded_output[0, img_original_height-1-i, :].copy_(padded_output[0, resize_height+crop-1 ,:])

	return padded_output

def metric_error(gtz, z):
	gtz = torch.Tensor(gtz).cuda()
	z = z.cuda()
	fmse = torch.mean(torch.pow(gtz-z,2))
	fmselog = torch.mean(torch.pow(torch.log(gtz)-torch.log(z),2))
	flsi = torch.mean(torch.pow(torch.log(z)-torch.log(gtz) + torch.mean(torch.log(gtz)-torch.log(z)), 2))
	fabsrel = torch.mean(torch.abs(z-gtz)/gtz)
	fsqrrel = torch.mean(torch.pow(z-gtz, 2)/gtz)

	return fmse, fmselog, flsi, fabsrel, fsqrrel

def normalize_output_depth_with_NYU_mean_std(input):
	std_of_NYU_training = 0.6148231626
	mean_of_NYU_training = 2.8424594402

	transformed_z = input.clone()
	transformed_z -= torch.mean(transformed_z)
	transformed_z /= torch.std(transformed_z)
	transformed_z *= std_of_NYU_training
	transformed_z += mean_of_NYU_training

	if torch.sum(transformed_z<0) > 0:
		transformed_z = (transformed_z<0).float()*(torch.min(transformed_z>0)+0.00001)+transformed_z*(1- transformed_z<0).float()

	return transformed_z

### main entry ###

parser = argparse.ArgumentParser()
parser.add_argument('-num_iter', default=1, type = int, help ='number of testing iteration')
parser.add_argument('-prev_model_file', required=True, help='Absolute/relative path to the previous model file. Resume training from this file')
parser.add_argument('-vis', default = False, type = bool, help='visualize output')
parser.add_argument('-output_folder', default = './output_imgs', help = 'image output folder')
parser.add_argument('-mode', default='validate', help = 'mode: test or validate')
parser.add_argument('-valid_set', default='45_validate_from_795_NYU_MITpaper_train_imgs_800_points_resize_240_320.csv', help = 'validation file name')
parser.add_argument('-test_set', default = '654_NYU_MITpaper_test_imgs_orig_size_points.csv', help = 'test file name')
parser.add_argument('-crop', default = 16, type = int, help = 'cropping size')
parser.add_argument('-thresh', default = -1, help = 'threshold for determining WKDR. Obtained from validations set.')

cmd_params = parser.parse_args()

if cmd_params.mode == 'test':
	csv_file_name = os.path.join('../../data/',cmd_params.test_set)
elif cmd_params.mode  == 'validate':
	csv_file_name = os.path.join('../../data/',cmd_params.valid_set)
preload_pt_filename = csv_file_name.replace('.csv','.pt')

if not os.path.exists(cmd_params.output_folder):
	os.mkdir(cmd_params.output_folder)

try:
	f = open(preload_pt_filename, 'r')
	f.close()
	print('loading preload pt file...')
	data_handle = torch.load(preload_pt_filename)
	n_sample = len(data_handle)
except Exception as e:
	print('loading csv file...')
	n_sample, data_handle = _read_data_handle(csv_file_name)
	torch.save(data_handle, preload_pt_filename)

print('Hyper params: ')
print('csv_file_name:', csv_file_name)
print('N test samples:', n_sample)
n_iter = min(n_sample, cmd_params.num_iter)
print('n_iter = {}'.format(n_iter))

prev_model_file = cmd_params.prev_model_file
model = torch.load(prev_model_file)
print('Model file:', prev_model_file)

from common.NYU_params import g_input_height as network_input_height
from common.NYU_params import g_input_width as network_input_width
_batch_input_cpu = torch.Tensor(1,3,network_input_height,network_input_width)

n_thresh = 140
thresh = torch.Tensor(n_thresh)
for i in range(0,n_thresh):
	thresh[i] = 0.1 + (i+1) * 0.01

WKDR = torch.zeros(n_iter, n_thresh)
WKDR_eq = torch.zeros(n_iter,n_thresh)
WKDR_neq = torch.zeros(n_iter,n_thresh)
fmse = torch.zeros(n_iter)
fmselog = torch.zeros(n_iter)
flsi = torch.zeros(n_iter)
fabsrel = torch.zeros(n_iter)
fsqrrel = torch.zeros(n_iter)

t = transforms.Compose([
	transforms.Scale((network_input_width, network_input_height)),
	transforms.ToTensor(),
	# transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))#may not need this
	])

for i in range(0, n_iter):
	print(i)
	img = Image.open(data_handle[i]['img_filename']).convert('RGB')
	img_original_width, img_original_height = img.size

	crop = transforms.CenterCrop((img_original_height- 2*cmd_params.crop, img_original_width - 2*cmd_params.crop ))

	if cmd_params.mode == 'test':
		_batch_input_cpu[0] = t(crop(img))
	elif cmd_params.mode == 'validate':
		_batch_input_cpu[0] = t(img)

	_single_data = []
	_single_data.append(data_handle[i])

	batch_output = model(Variable(_batch_input_cpu).cuda())

	# temp = batch_output

	batch_output = batch_output.cpu().data #now a tensor
	batch_output_min = torch.min(batch_output)
	batch_output_max = torch.max(batch_output)-batch_output_min

	original_size_output = torch.Tensor(1,1,img_original_height, img_original_width)

	if cmd_params.mode == 'test':
		original_size_output[0].copy_(inpaint_pad_output_our(batch_output, img_original_width, img_original_height))

		_evaluate_correctness_out(original_size_output, _single_data[0], WKDR[i], WKDR_eq[i], WKDR_neq[i])

		gtz_h5_handle = h5py.File(os.path.join(os.path.dirname(data_handle[i]['img_filename']),str(i+1)+'_depth.h5'), 'r') #todo
		gtz = gtz_h5_handle['/depth']
		# print(gtz)
		assert(gtz.shape[0] == 480)
		assert(gtz.shape[1] == 640)

		transformed_z_orig_size = normalize_output_depth_with_NYU_mean_std(original_size_output[0,0,:,:])

		metric_test_crop = 16
		transformed_z_orig_size = transformed_z_orig_size[metric_test_crop:(img_original_height- metric_test_crop), metric_test_crop:(img_original_width- metric_test_crop)]
		gtz = gtz[metric_test_crop:(img_original_height- metric_test_crop), metric_test_crop:(img_original_width- metric_test_crop)]
		# print(gtz)

		fmse[i], fmselog[i], flsi[i], fabsrel[i], fsqrrel[i] = metric_error(gtz,transformed_z_orig_size)

		gtz_h5_handle.close()

	elif cmd_params.mode == 'validate':
		scale = transforms.Compose([
			transforms.ToPILImage(),
			transforms.Scale((img_original_width, img_original_height)),
			transforms.ToTensor()
			])#temporal solution
		batch_output -= batch_output_min
		batch_output /= batch_output_max
		temp = scale(batch_output[0])
		temp *= batch_output_max
		temp += batch_output_min
		original_size_output[0].copy_(temp)
		_evaluate_correctness_out(original_size_output, _single_data[0], WKDR[i], WKDR_eq[i], WKDR_neq[i])

	if cmd_params.vis:
		orig_size_output = original_size_output[0]
		orig_size_output = orig_size_output - torch.min(orig_size_output)
		orig_size_output = orig_size_output / torch.max(orig_size_output)
		t_back = transforms.ToPILImage()
		orig_size_output = t_back(orig_size_output)

		new_image = Image.new('RGB', (img_original_width*2, img_original_height))
		new_image.paste(img, (0,0))
		new_image.paste(orig_size_output, (img_original_width, 0))

		new_image.save(os.path.join(cmd_params.output_folder, str(i+1)+'.png'))

WKDR = torch.mean(WKDR, 0)
WKDR_eq = torch.mean(WKDR_eq, 0)
WKDR_neq = torch.mean(WKDR_neq, 0)
overall_summary = torch.Tensor(n_thresh, 4)

min_max = 100
min_max_i = 0
min_WKDR = 100
min_WKDR_i = 0

for i in range(0,n_thresh):
	overall_summary[i,0] = thresh[i]
	overall_summary[i,1] = WKDR[0,i]#-TODO: check the dimension
	overall_summary[i,2] = WKDR_eq[0,i]
	overall_summary[i,3] = WKDR_neq[0,i]
	if max(WKDR_eq[0,i], WKDR_neq[0,i])<min_max:
		min_max = max(WKDR_eq[0,i], WKDR_neq[0,i])
		min_max_i = i
	if WKDR[0,i] < min_WKDR:
		min_WKDR = WKDR[0,i]
		min_WKDR_i = i

if cmd_params.thresh < 0:
	print(overall_summary)
	print("====================================================================")
	if min_max_i > 0:
		if min_max_i < n_thresh-1:
			print(overall_summary[min_max_i-1:min_max_i+2, :])
	print(overall_summary[min_WKDR_i])
else:
	print('Result:\n')
	for i in range(0,n_thresh):
		if overall_summary[i,0] == cmd_params.thresh:
			print(' Thresh\tWKDR\tWKDR_eq\tWKDR_neq')
			print(overall_summary[i])

if cmd_params.mode == 'test':
	print("====================================================================")
	print("rmse:\t{}".format(sqrt(torch.mean(fmse))))
	print("rmselog:{}".format(sqrt(torch.mean(fmselog))))
	print("lsi:\t{}".format(sqrt(torch.mean(flsi))))
	print("absrel:\t{}".format(torch.mean(fabsrel)))
	print("sqrrel:\t{}".format(torch.mean(fsqrrel)))

max_stars_repo_name: XLEric/relative-depth

max_issues_repo_name: XLEric/relative-depth

max_forks_repo_name: XLEric/relative-depth


text:
from latte.functional.interpolatability import utils
import numpy as np
import pytest

class TestShape:
    def test_bad_samples(self):

        with pytest.raises(AssertionError):
            utils._validate_za_shape(np.random.randn(16, 32), np.random.randn(15, 32))

    def test_bad_interp(self):

        with pytest.raises(AssertionError):
            utils._validate_za_shape(
                np.random.randn(16, 1, 32), np.random.randn(16, 1, 2)
            )

    def test_bad_features(self):

        with pytest.raises(AssertionError):
            utils._validate_za_shape(
                np.random.randn(16, 1, 32), np.random.randn(16, 3, 2)
            )

    def test_bad_min_size(self):

        with pytest.raises(AssertionError):
            utils._validate_za_shape(
                np.random.randn(16, 1, 2), np.random.randn(16, 1, 2), min_size=3
            )

    def test_bad_regdim_shape(self):

        with pytest.raises(AssertionError):
            utils._validate_za_shape(
                np.random.randn(16, 8, 32),
                np.random.randn(16, 3, 32),
                reg_dim=[2, 0, 3, 4],
            )

    def test_bad_regdim_neg(self):

        with pytest.raises(AssertionError):
            utils._validate_za_shape(
                np.random.randn(16, 8, 32),
                np.random.randn(16, 3, 32),
                reg_dim=[2, -1, 3],
            )

    def test_bad_regdim_over(self):

        with pytest.raises(AssertionError):
            utils._validate_za_shape(
                np.random.randn(16, 8, 32),
                np.random.randn(16, 3, 32),
                reg_dim=[2, 8, 3],
            )

    def test_regdim_slice(self):
        zin = np.random.randn(16, 8, 32)
        z, _ = utils._validate_za_shape(
            zin, np.random.randn(16, 3, 32), reg_dim=[3, 4, 5]
        )

        np.testing.assert_equal(zin[:, [3, 4, 5], :], z)

    def test_regdim_auto(self):
        zin = np.random.randn(16, 8, 32)
        z, _ = utils._validate_za_shape(zin, np.random.randn(16, 3, 32), reg_dim=None)

        np.testing.assert_equal(zin[:, :3, :], z)

    def test_regdim_auto_eq(self):
        zin = np.random.randn(16, 3, 32)
        z, _ = utils._validate_za_shape(zin, np.random.randn(16, 3, 32))

        np.testing.assert_equal(zin, z)

    def test_auto_expand(self):
        z, a = utils._validate_za_shape(
            np.random.randn(16, 32), np.random.randn(16, 32)
        )

        assert z.shape == (16, 1, 32)
        assert a.shape == (16, 1, 32)

    def test_bad_za_shapes(self):
        with pytest.raises(AssertionError):
            utils._validate_za_shape(np.random.randn(16,), np.random.randn(16, 32))

        with pytest.raises(AssertionError):
            utils._validate_za_shape(
                np.random.randn(16, 1, 4, 5), np.random.randn(16, 32)
            )

        with pytest.raises(AssertionError):
            utils._validate_za_shape(
                np.random.randn(16, 1), np.random.randn(16, 32, 1, 3)
            )

        with pytest.raises(AssertionError):
            utils._validate_za_shape(np.random.randn(16, 1), np.random.randn(16,))

class TestFiniteDiff:
    def test_first_order(self):
        z = np.repeat(np.linspace(0.0, 1.0, 10)[None, :], 8, axis=0)
        a = np.repeat(np.linspace(0.0, 0.5, 10)[None, :], 8, axis=0)

        dadz, zv = utils.finite_diff(z, a, order=1, mode="forward", return_list=False)

        np.testing.assert_allclose(dadz, 0.5 * np.ones(shape=(8, 9)))
        np.testing.assert_allclose(zv, 0.5 * (z[:, 1:] + z[:, :-1]))

    def test_second_order(self):
        z = np.repeat(np.linspace(0.0, 1.0, 10)[None, :], 8, axis=0)
        a = np.repeat(np.linspace(0.0, 0.5, 10)[None, :], 8, axis=0)

        dadz, zv = utils.finite_diff(z, a, order=2, mode="forward", return_list=False)

        np.testing.assert_allclose(dadz, np.zeros(shape=(8, 8)))
        z1 = 0.5 * (z[:, 1:] + z[:, :-1])
        z2 = 0.5 * (z1[:, 1:] + z1[:, :-1])
        np.testing.assert_allclose(zv, z2)

    def test_list_return(self):
        z = np.repeat(np.linspace(0.0, 1.0, 10)[None, :], 8, axis=0)
        a = np.repeat(np.linspace(0.0, 0.5, 10)[None, :], 8, axis=0)

        rets = utils.finite_diff(z, a, order=2, mode="forward", return_list=True)

        assert len(rets) == 2

        dadz, zv = rets[0]
        d2adz2, z2v = rets[1]

        np.testing.assert_allclose(dadz, 0.5 * np.ones(shape=(8, 9)))
        np.testing.assert_allclose(zv, 0.5 * (z[:, 1:] + z[:, :-1]))
        np.testing.assert_allclose(d2adz2, np.zeros(shape=(8, 8)))
        z1 = 0.5 * (z[:, 1:] + z[:, :-1])
        z2 = 0.5 * (z1[:, 1:] + z1[:, :-1])
        np.testing.assert_allclose(z2v, z2)

    def test_list_return3(self):
        z = np.repeat(np.linspace(0.0, 1.0, 10)[None, :], 8, axis=0)
        a = np.repeat(np.linspace(0.0, 0.5, 10)[None, :], 8, axis=0)

        rets = utils.finite_diff(z, a, order=3, mode="forward", return_list=True)

        assert len(rets) == 3

    def test_mode(self):
        z = np.repeat(np.linspace(0.0, 1.0, 10)[None, :], 8, axis=0)
        a = np.repeat(np.linspace(0.0, 0.5, 10)[None, :], 8, axis=0)

        with pytest.raises(NotImplementedError):
            utils.finite_diff(z, a, order=1, mode="central", return_list=False)

class TestLiad:
    def test_liad_fd_agree(self):
        z = np.random.rand(8, 16)
        a = np.random.rand(8, 16)

        d1, z1 = utils.finite_diff(z, a, order=1, mode="forward", return_list=False)
        d2, z2 = utils.liad(z, a, order=1, mode="forward", return_list=False)

        np.testing.assert_allclose(d1, d2)
        np.testing.assert_allclose(z1, z2)

    def test_liad2_fd_agree(self):
        z = np.random.rand(8, 16)
        a = np.random.rand(8, 16)

        d1, z1 = utils.finite_diff(z, a, order=2, mode="forward", return_list=False)
        d2, z2 = utils.liad(z, a, order=2, mode="forward", return_list=False)

        np.testing.assert_allclose(d1, d2)
        np.testing.assert_allclose(z1, z2)

class TestLehmerMean:
    def test_p1(self):
        x = np.random.rand(8, 16)

        m = utils.lehmer_mean(x, p=1.0)
        np.testing.assert_allclose(m, np.mean(x, axis=-1))

    def test_p2(self):
        x = np.random.rand(8, 16)

        m = utils.lehmer_mean(x, p=2.0)
        np.testing.assert_allclose(
            m, np.sum(np.square(x), axis=-1) / np.sum(x, axis=-1)
        )

    def test_p0(self):
        x = np.random.rand(8, 16)

        m = utils.lehmer_mean(x, p=0.0)
        np.testing.assert_allclose(
            m, np.sum(np.ones_like(x), axis=-1) / np.sum(1.0 / x, axis=-1)
        )

ext: py

reason: import numpy


text:
"""Evaluation metrics."""

import numpy as np
from sklearn.metrics import matthews_corrcoef  # noqa
from sklearn.metrics import recall_score  # noqa
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import r2_score  # noqa
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import precision_score  # noqa
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import auc
from sklearn.metrics import jaccard_score
from sklearn.metrics import f1_score
from sklearn.metrics import roc_auc_score  # noqa
from sklearn.metrics import accuracy_score  # noqa
from sklearn.metrics import balanced_accuracy_score  # noqa
from scipy.stats import pearsonr

# kappa_score is an alias for `sklearn.metrics.cohen_kappa_score`
kappa_score = cohen_kappa_score

def pearson_r2_score(y: np.ndarray, y_pred: np.ndarray) -> float:
  """Computes Pearson R^2 (square of Pearson correlation).

  Parameters
  ----------
  y: np.ndarray
    ground truth array
  y_pred: np.ndarray
    predicted array

  Returns
  -------
  float
    The Pearson-R^2 score.
  """
  return pearsonr(y, y_pred)[0]**2

def jaccard_index(y: np.ndarray, y_pred: np.ndarray) -> float:
  """Computes Jaccard Index which is the Intersection Over Union metric
  which is commonly used in image segmentation tasks.

  DEPRECATED: WILL BE REMOVED IN A FUTURE VERSION OF DEEEPCHEM. USE `jaccard_score` instead.

  Parameters
  ----------
  y: np.ndarray
    ground truth array
  y_pred: np.ndarray
    predicted array

  Returns
  -------
  score: float
    The jaccard index. A number between 0 and 1.
  """
  return jaccard_score(y, y_pred)

def pixel_error(y: np.ndarray, y_pred: np.ndarray) -> float:
  """An error metric in case y, y_pred are images.

  Defined as 1 - the maximal F-score of pixel similarity, or squared
  Euclidean distance between the original and the result labels.

  Parameters
  ----------
  y: np.ndarray
    ground truth array
  y_pred: np.ndarray
    predicted array

  Returns
  -------
  score: float
    The pixel-error. A number between 0 and 1.
  """
  return 1 - f1_score(y, y_pred)

def prc_auc_score(y: np.ndarray, y_pred: np.ndarray) -> float:
  """Compute area under precision-recall curve

  Parameters
  ----------
  y: np.ndarray
    A numpy array of shape `(N, n_classes)` or `(N,)` with true labels
  y_pred: np.ndarray
    Of shape `(N, n_classes)` with class probabilities.

  Returns
  -------
  float
    The area under the precision-recall curve. A number between 0 and 1.
  """
  precision, recall, _ = precision_recall_curve(y[:, 1], y_pred[:, 1])
  return auc(recall, precision)

def rms_score(y_true: np.ndarray, y_pred: np.ndarray) -> float:
  """Computes RMS error."""
  return np.sqrt(mean_squared_error(y_true, y_pred))

def mae_score(y_true: np.ndarray, y_pred: np.ndarray) -> float:
  """Computes MAE."""
  return mean_absolute_error(y_true, y_pred)

def bedroc_score(y_true: np.ndarray, y_pred: np.ndarray, alpha: float = 20.0):
  """Compute BEDROC metric.

  BEDROC metric implemented according to Truchon and Bayley that modifies
  the ROC score by allowing for a factor of early recognition.
  Please confirm details from [1]_.

  Parameters
  ----------
  y_true: np.ndarray
    Binary class labels. 1 for positive class, 0 otherwise
  y_pred: np.ndarray
    Predicted labels
  alpha: float, default 20.0
    Early recognition parameter

  Returns
  -------
  float
    Value in [0, 1] that indicates the degree of early recognition

  Notes
  -----
  This function requires RDKit to be installed.

  References
  ----------
  .. [1] Truchon et al. "Evaluating virtual screening methods: good and bad metrics
     for the “early recognition” problem." Journal of chemical information and modeling
     47.2 (2007): 488-508.
  """
  try:
    from rdkit.ML.Scoring.Scoring import CalcBEDROC
  except ModuleNotFoundError:
    raise ValueError("This function requires RDKit to be installed.")

  # validation
  assert len(y_true) == len(y_pred), 'Number of examples do not match'
  assert np.array_equal(
      np.unique(y_true).astype(int),
      [0, 1]), ('Class labels must be binary: %s' % np.unique(y_true))

  yt = np.asarray(y_true)
  yp = np.asarray(y_pred)

  yt = yt.flatten()
  yp = yp[:, 1].flatten()  # Index 1 because one_hot predictions

  scores = list(zip(yt, yp))
  scores = sorted(scores, key=lambda pair: pair[1], reverse=True)

  return CalcBEDROC(scores, 0, alpha)

max_issues_repo_name: swpper/deepchem

max_forks_repo_name: swpper/deepchem

reason: import numpy,from scipy


text:
import headers
import pgn_move_processing
import networkx as nx
import matplotlib.pyplot as plt
from networkx.drawing.nx_pydot import graphviz_layout

"""
Game class. A game is a header object and and list of half_move objects
"""
class Game:
    def __init__(self, header, moves):
        self.header = header
        self.moves = moves
        self.pgn_graph = nx.Graph()
        self.label_dict = {}
        self.color_dict = {}

    """
    Print all object : headers and then moves
    """
    def print_all(self):
        self.header.print_headers()
        print("\n")

        for move in self.moves:
            move.print_full_object()
            print("\n")

    """
    Create graph of game by recursively process move and adding them to the graph
    """
    def make_graph(self):
        self.add_move(self.find_first_move())

    """
    Get pgn_graph element setup colors and label and show it
    """
    def display_graph(self):
        print(nx.get_node_attributes(self.pgn_graph, self.find_first_move()))
        values = [self.color_dict.get(node, 0.25) for node in self.pgn_graph.nodes()]
        print(values)
        pos = graphviz_layout(self.pgn_graph, prog="dot")
        nx.draw(self.pgn_graph, pos, labels=self.label_dict, with_labels=True, node_color=values, cmap=plt.cm.Greys)
        plt.show()

    """
    Find first move of the game in the list of moves by looking which move has no parent
    """
    def find_first_move(self):
        for move in self.moves:
            if move.parent is None:
                move.print_full_object()
                return move

    """
    Add move and recursively add its children to the graph with correct edges
    """
    def add_move(self, move):
        self.pgn_graph.add_node(move)
        self.label_dict[move] = move.move_name
        if move.white_move:
            self.color_dict[move] = 0.0
        else:
            self.color_dict[move] = 2.0

        if move.parent is not None:
            self.pgn_graph.add_edge(move.parent, move)

        for child in move.children:
            self.add_move(child)

    """
    Return a Game object from a path to a pgn file
    """
    @staticmethod
    def game_from_pgn(path):
        head = headers.Headers.get_headers_from_pgn(path)
        moves = pgn_move_processing.PGNManipulator(path)
        moves.process_file()

        return Game(head, moves.move_list)

reason: import networkx,from networkx


text:
import numpy as np
import scipy as sp

import matplotlib
import matplotlib.pyplot as plt

import ss_sim_3dof as dyn

def main():

    obstacle_state = [] #update obstacle state list
    sc4 = [np.array([0.3,1,0,0,0,0]),0.5]
    obstacle_state.append(sc4)

    dt = 0.25
    T = 100
    num_states = 6
    num_control = 8

    mass = 10
    inertia = 1.62
    l = 0.4
    b = 0.4

    param = np.array([mass,inertia,l,b])

    gain = np.array([6,3])

    u_min = 0
    u_max = 0.45

    _, ax1 = plt.subplots(6,1,figsize=(3,5))
    _, ax2 = plt.subplots(8,1,figsize=(3,5))

    desired_state = np.load('x_scp1.npy')
    desired_control = np.load('u_scp1.npy')

    # ax1.plot(desired_state[:,0],desired_state[:,1],color='black',linestyle='dashed',linewidth=2,label='Desired\nTrajectory')

    for i in range(8):
        ax2[i].plot(np.linspace(0,T-2,T-1),desired_control[:,i],color='black')
    x_desired = np.zeros((6,T))
    u_desired = np.zeros((8,T-1))
    for tt in range(T):
        for oo in range(6):
            x_desired[oo,tt] = desired_state[tt,oo]
    for tt in range(T-1):
        for uu in range(8):
            u_desired[uu,tt] = desired_control[tt,uu]

    # trying different controllers

    x1 = np.zeros((num_states,T))
    u1 = np.zeros((num_control,T-1))

    x2 = np.zeros((num_states,T))
    u2 = np.zeros((num_control,T-1))

    x3 = np.zeros((num_states,T))
    u3 = np.zeros((num_control,T-1))

    # setup initial condition
    x1[:,0] = x_desired[:,0] + np.array([0.2,0.0,0.0,0,0,0])
    x2[:,0] = x1[:,0]#desired[oo,0] + 0.01*np.random.rand(1)
    x3[:,0] = x1[:,0]#desired[oo,0] + 0.01*np.random.rand(1)

    u_old = np.zeros(8)
    actuator_fault = np.eye(8)
    actuator_fault[2,2] = 1
    actuator_fault[7,7] = 0
    actuator_fault[6,6] = 0

    for tt in range(T-1):
            # actuator_fault[3,3] = 0
        u1_force = dyn.ss_3dof_control(x1[:,tt].reshape((6,1)),x_desired[:,tt].reshape((6,1)),u_desired[:,tt].reshape((8,1)),param,gain)
        u1[:,tt] = dyn.psuedo_inverse_allocation(u1_force,x1[:,tt].reshape((6,1)),u_desired[:,tt],param,u_min,u_max,actuator_fault)
        u1[:,tt] = actuator_fault@u1[:,tt]
        print(u1[:,tt])
        x1[:,tt+1]  = dyn.ss_3dof_dyn(x1[:,tt],u1[:,tt].reshape([8,1]),param,dt) # For Markov Propagation

        u2_force = dyn.ss_3dof_control(x2[:,tt].reshape((6,1)),x_desired[:,tt].reshape((6,1)),u_desired[:,tt].reshape((8,1)),param,gain)
        u2[:,tt] = dyn.optimization_allocation(u2_force,x2[:,tt].reshape((6,1)),u_desired[:,tt],param,u_min,u_max,actuator_fault)
        u2[:,tt] = actuator_fault@u2[:,tt]
        x2[:,tt+1]  = dyn.ss_3dof_dyn(x2[:,tt],u2[:,tt].reshape([8,1]),param,dt) # For Markov Propagation

        # u2_force = dyn.ss_3dof_control(x2[:,tt].reshape((6,1)),x_desired[:,tt].reshape((6,1)),u_desired[:,tt].reshape((8,1)),param,gain)
        u3_force = dyn.ss_3dof_control(x3[:,tt].reshape((6,1)),x_desired[:,tt].reshape((6,1)),u_desired[:,tt].reshape((8,1)),param,gain)
        u3[:,tt] = dyn.optimization_stability_allocation(u3_force,x3[:,tt].reshape((6,1)),x_desired[:,tt].reshape((6,1)),u_desired[:,tt].reshape((8,1)),param,u_min,u_max,u_old,actuator_fault)
        u3[:,tt] = actuator_fault@u3[:,tt]
        x3[:,tt+1]  = dyn.ss_3dof_dyn(x3[:,tt],u3[:,tt].reshape([8,1]),param,dt) # For Markov Propagation
        u_old = u3[:,tt]

    state_name = ['x','y','\theta','\dot{x}','\dot{y}','\dot{\theta}']
    for i in range(6):
        ax1[i].plot(np.linspace(0,T-1,T),desired_state[:,i],color='black',linestyle='dashed',linewidth=2,label='desired')
        ax1[i].plot(np.linspace(0,T-1,T),x1[i,:],color='red',linestyle='dashed',linewidth=2,label='LinAlg')
        ax1[i].set_xlabel('time')
        ax1[i].set_ylabel(state_name[i])
        # state_name = ['x','y','\theta','\dot{x}','\dot{y}','\dot{\theta}']
    for i in range(6):
        ax1[i].plot(np.linspace(0,T-1,T),x2[i,:],color='green',linestyle='dashed',linewidth=2,label='Opt')
        # ax1.set_xlabel('time')
        # ax1.set_ylabel(state_name[i])
    for i in range(6):
        ax1[i].plot(np.linspace(0,T-1,T),x3[i,:],color='blue',linestyle='dashed',linewidth=2,label='Opt_Stable')
        # ax1.set_xlabel('time')
        # ax1.set_ylabel(state_name[i])

        # a.plot(x2[0,:],x2[1,:],color='green',linestyle='dashed',linewidth=2,label='Opt')
        # ax1.plot(x3[0,:],x3[1,:],color='blue',linestyle='dashed',linewidth=2,label='Opt_Stability')

    #ax1.set_ylim((100,130))
    # ax2.set_xlabel('Epoch')
    # ax2.set_ylabel('Differential\nEntropy (nats)')
    # # ax2.set_ylabel('Fisher Information')
    # #ax2.set_ylim((20,70))

    # ax3.set_xlabel('Epoch')
    # ax3.set_ylabel('Collisions Without\nSafety Filter')

    # ax1.grid(True,color='lightgrey')
    # ax2.grid(True,color='lightgrey')
    # ax3.grid(True,color='lightgrey')
    # ax1.legend(fontsize='7',loc='best')

    for i in range(8):
        ax2[i].plot(np.linspace(0,T-2,T-1),u1[i,:],color='red')
        ax2[i].set_xlabel('time')
        ax2[i].set_ylabel('T'+str(i))

    for i in range(8):
        ax2[i].plot(np.linspace(0,T-2,T-1),u2[i,:],color='green')

    for i in range(8):
        ax2[i].plot(np.linspace(0,T-2,T-1),u3[i,:],color='blue')

    # plt.tight_layout()
    plt.legend(fontsize='7',loc='best')

    plt.show()

    return 1

if __name__ == "__main__":

    main()

reason: import numpy,import scipy


text:
# -*- coding: utf-8 -*-
"""
@description: 森林火灾模拟
@author:XuMing
"""
from __future__ import print_function  # 兼容python3的print写法
from __future__ import unicode_literals  # 兼容python3的编码处理

# 1 对森林建模
# 2 随机生长
# 2.1 在原来的基础上,我们要先让树生长，即定义 grow_trees() 方法
# 2.2 定义方法之前，我们要先指定两个属性：
# 2.2.1 每个位置随机生长出树木的概率
# 2.2.2 每个位置随机被闪电击中的概率
# 2.3 为了方便，我们定义一个辅助函数来生成随机 bool 矩阵，大小与森林大小一致
import numpy as np

class Forest(object):
    """Forest can grow trees"""

    def __init__(self, size=(150, 150), p_sapling=0.0025, p_lightning=5.0e-6):
        self.size = size
        self.trees = np.zeros(self.size, dtype=bool)
        self.fires = np.zeros((self.size), dtype=bool)
        self.p_sapling = p_sapling
        self.p_lightning = p_lightning

    def __repr__(self):
        my_repr = "{}(size={})".format(self.__class__.__name__, self.size)
        return my_repr

    def __str__(self):
        return self.__class__.__name__

    @property
    def num_cells(self):
        """Number of cells available for grwoing trees"""
        return np.prod(self.size)

    @property
    def tree_fraction(self):
        """Fraction of trees"""
        num_trees = self.trees.sum()
        return float(num_trees) / self.num_cells

    @property
    def fire_fraction(self):
        """Fraction of fires"""
        num_fires = self.fires.sum()
        return float(num_fires) / self.num_cells

    def _rand_bool(self, p):
        """Random boolean distributed according to p, less than p will be True"""
        return np.random.uniform(size=self.trees.shape) < p

    def grow_trees(self):
        """Growing trees"""
        growth_sites = self._rand_bool(self.p_sapling)
        self.trees[growth_sites] = True

# Test
forest = Forest()
print(forest.tree_fraction)

forest.grow_trees()
print(forest.tree_fraction)

# 火灾模拟
# 1 定义 start_fires()：
# 按照给定的概率生成被闪电击中的位置
# 如果闪电击中的位置有树，那么将其设为着火点
# 2 定义 burn_trees()：
# 如果一棵树的上下左右有火，那么这棵树也会着火
# 3 定义 advance_one_step()：
# 进行一次生长，起火，燃烧
class FireForest(object):
    """ Forest can grow trees which eventually die."""

    def __init__(self, size=(150, 150), p_sapling=0.0025, p_lightning=5.0e-6):
        self.size = size
        self.trees = np.zeros(self.size, dtype=bool)
        self.fires = np.zeros((self.size), dtype=bool)
        self.p_sapling = p_sapling
        self.p_lightning = p_lightning

    def __repr__(self):
        my_repr = "{}(size={})".format(self.__class__.__name__, self.size)
        return my_repr

    def __str__(self):
        return self.__class__.__name__

    @property
    def num_cells(self):
        """Number of cells available for growing trees"""
        return np.prod(self.size)

    @property
    def tree_fraction(self):
        """
        Fraction of trees
        """
        num_trees = self.trees.sum()
        return float(num_trees) / self.num_cells

    @property
    def fire_fraction(self):
        """
        Fraction of fires
        """
        num_fires = self.fires.sum()
        return float(num_fires) / self.num_cells

    def _rand_bool(self, p):
        """
        Random boolean distributed according to p, less than p will be True
        """
        return np.random.uniform(size=self.trees.shape) < p

    def grow_trees(self):
        """
        Growing trees
        """
        growth_sites = self._rand_bool(self.p_sapling)
        self.trees[growth_sites] = True

    def start_fires(self):
        """
        Start of fire
        """
        lightning_strikes = (self._rand_bool(self.p_lightning) &
                             self.trees)
        self.fires[lightning_strikes] = True

    def burn_trees(self):
        """
        Burn trees
        """
        fires = np.zeros((self.size[0] + 2, self.size[1] + 2), dtype=bool)
        fires[1:-1, 1:-1] = self.fires
        north = fires[:-2, 1:-1]
        south = fires[2:, 1:-1]
        east = fires[1:-1, :-2]
        west = fires[1:-1, 2:]
        new_fires = (north | south | east | west) & self.trees
        self.trees[self.fires] = False
        self.fires = new_fires

    def advance_one_step(self):
        """
        Advance one step
        """
        self.grow_trees()
        self.start_fires()
        self.burn_trees()

fire = FireForest()
for i in range(100):
    fire.advance_one_step()

# 显示树木图像
import matplotlib.pyplot as plt
from matplotlib import cm

plt.matshow(fire.trees, cmap=cm.Greens)
plt.show()

# 查看不同着火概率下的森林覆盖率趋势变化：
forest = FireForest()
forest2 = FireForest(p_lightning=5e-4)
tree_fractions = []
for i in range(2000):
    forest.advance_one_step()
    forest2.advance_one_step()
    tree_fractions.append((forest.tree_fraction, forest2.tree_fraction))
plt.plot(tree_fractions)
plt.show()

max_stars_repo_name: KEVINYZY/python-tutorial


text:
import functools
import igraph as ig
import numpy as np

import torch

from . import tflite as tfl
from .base import ExtendedOperator
from .graph import CommonGraph

from tinynn.util.util import get_logger

log = get_logger(__name__)

class HybridQuantizer(object):
    graph: CommonGraph

    def __init__(self, graph, q_type) -> None:
        super().__init__()

        self.graph = graph
        self.q_type = q_type

    def quantize(self):
        self.quantize_pass()

    def quantize_pass(self):
        filtered_nodes = self.graph.graph.vs.select(is_quantizable_node)

        actions = []
        for node in filtered_nodes:
            weight_idx = 1
            new_weight = None
            weight_t = node['op'].inputs[weight_idx]
            name = weight_t.name
            weight_a = np.frombuffer(weight_t.buffer.data).view(np.float32).reshape(weight_t.shape)
            weight = torch.from_numpy(weight_a.copy())
            if weight_t.buffer is None or str(weight_t.dtype) != 'float32':
                continue
            if node['node_type'] == ExtendedOperator.CONV_2D:
                if self.q_type == np.uint8:
                    new_weight = quantize(name, weight, torch.quint8, torch.per_tensor_symmetric, q_type=self.q_type)
                else:
                    new_weight = quantize(name, weight, torch.qint8, torch.per_channel_symmetric, 0, q_type=self.q_type)
            elif node['node_type'] == ExtendedOperator.DEPTHWISE_CONV_2D:
                if self.q_type == np.uint8:
                    log.warning('DEPTHWISE_CONV_2D doesn\'t support u8 hybrid quantization')
                    continue
                else:
                    new_weight = quantize(name, weight, torch.qint8,
                                          torch.per_channel_symmetric, -1, q_type=self.q_type)
            elif node['node_type'] == ExtendedOperator.FULLY_CONNECTED:
                if self.q_type == np.uint8:
                    new_weight = quantize(name, weight, torch.quint8, torch.per_tensor_symmetric, q_type=self.q_type)
                else:
                    new_weight = quantize(name, weight, torch.qint8, torch.per_tensor_symmetric, q_type=self.q_type)

            actions.append((self.graph.replace_operator_input, (node, weight_idx, new_weight)))

        for func, args in actions:
            func(*args)

def is_quantizable_node(vertex: ig.Vertex):
    return vertex['node_type'] in (ExtendedOperator.CONV_2D,
                                   ExtendedOperator.DEPTHWISE_CONV_2D,
                                   ExtendedOperator.FULLY_CONNECTED)

def quantize(name, tensor, dtype, qscheme, axis=None, q_type=np.uint8):
    assert qscheme in (torch.per_tensor_symmetric, torch.per_channel_symmetric)

    new_name = f'{name}_hybrid_q'

    if dtype == torch.quint8:
        quant_min, quant_max = 0, 255
    else:
        quant_min, quant_max = -127, 127

    if axis is not None:
        if axis < 0:
            axis += tensor.ndim
        dim = [i for i in range(tensor.ndim) if i != axis]
    else:
        dim = None

    if hasattr(torch, 'amin') and hasattr(torch, 'amax'):
        min_val = torch.amin(tensor, dim)
        max_val = torch.amax(tensor, dim)
    else:
        if dim is None:
            min_val = torch.min(tensor)
            max_val = torch.max(tensor)
        else:
            orig_dim = tensor.size(axis)
            if axis != 0:
                perm = [axis] + dim
                tensor_perm = tensor.permute(perm)
            else:
                tensor_perm = tensor
            tensor_2d = tensor_perm.reshape(orig_dim, -1)
            min_val, _ = torch.min(tensor_2d, 1)
            max_val, _ = torch.max(tensor_2d, 1)

    min_val_neg = torch.min(min_val, torch.zeros_like(min_val))
    max_val_pos = torch.max(max_val, torch.zeros_like(max_val))

    scale = torch.ones(min_val_neg.size(), dtype=torch.float32)
    zero_point = torch.zeros(min_val_neg.size(), dtype=torch.int64)

    eps = torch.tensor(torch.finfo(torch.float32).eps)

    max_val_pos = torch.max(-min_val_neg, max_val_pos)
    scale = max_val_pos / (float(quant_max - quant_min) / 2)
    scale = torch.max(scale, eps)
    if dtype == torch.quint8:
        zero_point = zero_point.new_full(zero_point.size(), 128)

    if qscheme == torch.per_channel_symmetric:
        q_tensor = torch.quantize_per_channel(tensor, scale, zero_point, axis, dtype)
    else:
        q_tensor = torch.quantize_per_tensor(tensor, scale, zero_point, dtype)

    return tfl.Tensor(q_tensor, new_name, q_type=q_type)

max_stars_repo_name: kingkie/TinyNeuralNetwork

max_issues_repo_name: kingkie/TinyNeuralNetwork

max_forks_repo_name: kingkie/TinyNeuralNetwork


text:
import unittest

from cupy import testing

@testing.gpu
class TestConj(unittest.TestCase):

    _multiprocess_can_split_ = True

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_almost_equal()
    def test_conj(self, xp, dtype):
        x = testing.shaped_arange((2, 3), xp, dtype)
        return x.conj()

    @testing.for_all_dtypes(no_complex=True)
    @testing.numpy_cupy_array_almost_equal()
    def test_conj_pass(self, xp, dtype):
        x = testing.shaped_arange((2, 3), xp, dtype)
        y = x.conj()
        self.assertIs(x, y)
        return y

@testing.gpu
class TestAngle(unittest.TestCase):

    _multiprocess_can_split_ = True

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_almost_equal()
    def test_angle(self, xp, dtype):
        x = testing.shaped_arange((2, 3), xp, dtype)
        return xp.angle(x)

@testing.gpu
class TestRealImag(unittest.TestCase):

    _multiprocess_can_split_ = True

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_almost_equal(accept_error=False)
    def test_real(self, xp, dtype):
        x = testing.shaped_arange((2, 3), xp, dtype)
        return x.real

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_almost_equal(accept_error=False)
    def test_imag(self, xp, dtype):
        x = testing.shaped_arange((2, 3), xp, dtype)
        return x.imag

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_almost_equal(accept_error=False)
    def test_real_setter(self, xp, dtype):
        x = testing.shaped_arange((2, 3), xp, dtype)
        x.real = testing.shaped_reverse_arange((2, 3), xp, dtype).real
        return x

    @testing.for_dtypes('FD')
    @testing.numpy_cupy_array_almost_equal(accept_error=False)
    def test_imag_setter(self, xp, dtype):
        x = testing.shaped_arange((2, 3), xp, dtype)
        x.imag = testing.shaped_reverse_arange((2, 3), xp, dtype).real
        return x

    @testing.for_all_dtypes(no_complex=True)
    @testing.numpy_cupy_raises(accept_error=TypeError)
    def test_imag_setter_raise(self, xp, dtype):
        x = testing.shaped_arange((2, 3), xp, dtype)
        x.imag = testing.shaped_reverse_arange((2, 3), xp, dtype)
        return x

reason: from cupy


text:
'''
F-PointNet, http://openaccess.thecvf.com/content_cvpr_2018/html/Qi_Frustum_PointNets_for_CVPR_2018_paper.html
'''

import torch
import torch.nn as nn

from lib.backbones.pointnet import PointNet
from lib.backbones.pointnet import PointNet_SEG
from lib.helpers.fpointnet_helper import point_cloud_masking
from lib.helpers.fpointnet_helper import parse_outputs
from lib.helpers.misc_helper import init_weights

class FPointNet(nn.Module):
    def __init__(self, cfg, num_heading_bin, num_size_cluster, mean_size_arr):
        super().__init__()
        self.cfg = cfg
        self.mean_size_arr = mean_size_arr  # fixed anchor
        self.num_heading_bin = num_heading_bin
        self.num_size_cluster = num_size_cluster
        self.point_seg_backbone = PointNet_SEG(num_points=self.cfg['num_frustum_point'],
                                               input_channels=cfg['input_channel'])
        self.point_seg_head = nn.Sequential(nn.Dropout(0.5),
                                            nn.Conv1d(128, 2, 1))
        self.center_reg_backbone = PointNet(num_points=self.cfg['num_object_points'],
                                            input_channels=3,
                                            layer_cfg=[128, 128, 256],
                                            batch_norm=True)
        self.center_reg_head = nn.Sequential(nn.Linear(259, 256), nn.BatchNorm1d(256), nn.ReLU(inplace=True),
                                             nn.Linear(256, 128), nn.BatchNorm1d(128), nn.ReLU(inplace=True),
                                             nn.Linear(128, 3))
        self.box_est_backbone = PointNet(num_points=self.cfg['num_object_points'],
                                         input_channels=3,
                                         layer_cfg=[128, 128, 256, 512],
                                         batch_norm=True)
        self.box_est_head = nn.Sequential(nn.Linear(515, 512), nn.BatchNorm1d(512), nn.ReLU(inplace=True),
                                          nn.Linear(512, 256), nn.BatchNorm1d(256), nn.ReLU(inplace=True),
                                          nn.Linear(256, 3 + self.num_heading_bin*2 + self.num_size_cluster*4))

        init_weights(self, self.cfg['init'])

    def forward(self, point_cloud, one_hot_vec):
        output_dict = {}

        # background points segmentation
        seg_features = self.point_seg_backbone(point_cloud, one_hot_vec)
        seg_logits = self.point_seg_head(seg_features).transpose(1, 2).contiguous()    # B*C*N -> B*N*C
        output_dict['mask_logits'] = seg_logits

        # background points masking
        # select masked points and translate to masked points' centroid
        point_cloud = point_cloud.transpose(1, 2).contiguous()  # B*C*N -> B*N*C, meet the shape requirements of funcs
        object_point_cloud_xyz, mask_xyz_mean, output_dict = \
            point_cloud_masking(point_cloud, seg_logits, self.cfg['num_object_points'], output_dict)

        # T-Net and coordinate translation
        object_point_cloud_xyz = object_point_cloud_xyz.transpose(1, 2).contiguous()  # B*N*C -> B*C*N
        center_features = self.center_reg_backbone(object_point_cloud_xyz).view(-1, 256)  #  B*C*1 -> B*C
        center_features = torch.cat([center_features, one_hot_vec], 1)
        center_tnet = self.center_reg_head(center_features)
        stage1_center = center_tnet + mask_xyz_mean  # Bx3
        output_dict['stage1_center'] = stage1_center

        # Get object point cloud in object coordinate
        object_point_cloud_xyz_new = object_point_cloud_xyz - center_tnet.view(-1, 3, 1)

        # Amodel Box Estimation PointNet
        box_features = self.box_est_backbone(object_point_cloud_xyz_new).view(-1, 512)  # B*C*1 -> B*C
        box_features = torch.cat([box_features, one_hot_vec], 1)
        box_results = self.box_est_head(box_features)
        output_dict = parse_outputs(box_results, self.num_heading_bin, self.num_size_cluster, self.mean_size_arr, output_dict)
        output_dict['center'] = output_dict['center_boxnet'] + stage1_center  # Bx3

        return output_dict

if __name__ == '__main__':
    # only for debug
    import yaml
    import numpy as np
    cfg_file = '../../experiments/fpointnet/config_fpointnet.yaml'
    cfg = yaml.load(open(cfg_file, 'r'), Loader=yaml.Loader)['model']
    fpointnet = FPointNet(cfg, 12, 8, np.zeros((8, 3), dtype=np.float32))

    points = torch.Tensor(2, 4, 1024)
    one_hot_vec = torch.Tensor(2, 3)

    outputs = fpointnet(points, one_hot_vec)
    for key in outputs:
        print((key, outputs[key].shape))
    print(len(outputs.keys()))

max_stars_repo_name: atonderski/patchnet

max_issues_repo_name: atonderski/patchnet

max_forks_repo_name: atonderski/patchnet


text:
import sys
sys.path.append('.')

import tensorflow as tf
import numpy as np
import time
from lib.core.model.facebox.losses_and_ohem import localization_loss, ohem_loss
from lib.core.model.facebox.utils.box_utils import batch_decode
from lib.core.model.facebox.utils.nms import batch_non_max_suppression

from train_config import config as cfg

def batch_norm():
    return tf.keras.layers.BatchNormalization(fused=True,momentum=0.997,epsilon=1e-5)

class RDCL(tf.keras.Model):

    def __init__(self,
                 kernel_initializer='glorot_normal'):

        super(RDCL, self).__init__()

        self.conv1_1 = tf.keras.layers.Conv2D(filters=12,
                                              kernel_size=(7, 7),
                                              strides=2,
                                              padding='same',
                                              use_bias=False,
                                              kernel_initializer=kernel_initializer)
        self.bn1_1=batch_norm()
        self.conv1_2 = tf.keras.layers.Conv2D(filters=24,
                                              kernel_size=(3, 3),
                                              strides=2,
                                              padding='same',
                                              use_bias=False,
                                              kernel_initializer=kernel_initializer)
        self.bn1_2 = batch_norm()

        self.conv2_1 = tf.keras.layers.Conv2D(filters=32,
                                              kernel_size=(3, 3),
                                              strides=2,
                                              padding='same',
                                              use_bias=False,
                                              kernel_initializer=kernel_initializer)
        self.bn2_1 = batch_norm()
        self.conv2_2 = tf.keras.layers.Conv2D(filters=64,
                                              kernel_size=(3, 3),
                                              strides=2,
                                              padding='same',
                                              use_bias=False,
                                              kernel_initializer=kernel_initializer)
        self.bn2_2 = batch_norm()

    def __call__(self,x,training=False):
        x = tf.nn.relu(self.bn1_1(self.conv1_1(x),training=training))
        x = tf.nn.crelu(self.bn1_2(self.conv1_2(x),training=training))
        x = tf.nn.relu(self.bn2_1(self.conv2_1(x),training=training))
        x = tf.nn.crelu(self.bn2_2(self.conv2_2(x),training=training))

        return x

class Inception(tf.keras.Model):

    def __init__(self,
                 kernel_initializer='glorot_normal'):
        super(Inception, self).__init__()
        self.conv1 = tf.keras.layers.Conv2D(filters=32,
                                            kernel_size=(1, 1),
                                            strides=1,
                                            padding='same',
                                            use_bias=False,
                                            kernel_initializer=kernel_initializer)
        self.bn1 = batch_norm()

        self.avgpool=tf.keras.layers.AveragePooling2D(pool_size=(3,3),
                                                      strides=1,
                                                      padding='same')
        self.conv2 = tf.keras.layers.Conv2D(filters=32,
                                            kernel_size=(1, 1),
                                            strides=1,
                                            padding='same',
                                            use_bias=False,
                                            kernel_initializer=kernel_initializer)
        self.bn2 = batch_norm()

        self.conv3_1 = tf.keras.layers.Conv2D(filters=24,
                                              kernel_size=(1, 1),
                                              strides=1,
                                              padding='same',
                                              use_bias=False,
                                              kernel_initializer=kernel_initializer)
        self.bn3_1 = batch_norm()
        self.conv3_2 = tf.keras.layers.Conv2D(filters=32,
                                              kernel_size=(3, 3),
                                              strides=1,
                                              padding='same',
                                              use_bias=False,
                                              kernel_initializer=kernel_initializer)
        self.bn3_2 = batch_norm()

        self.conv4_1 = tf.keras.layers.Conv2D(filters=24,
                                              kernel_size=(1, 1),
                                              strides=1,
                                              padding='same',
                                              use_bias=False,
                                              kernel_initializer=kernel_initializer)
        self.bn4_1 = batch_norm()
        self.conv4_2 = tf.keras.layers.Conv2D(filters=32,
                                              kernel_size=(3, 3),
                                              strides=1,
                                              padding='same',
                                              use_bias=False,
                                              kernel_initializer=kernel_initializer)
        self.bn4_2 = batch_norm()
        self.conv4_3 = tf.keras.layers.Conv2D(filters=32,
                                              kernel_size=(3, 3),
                                              strides=1,
                                              padding='same',
                                              use_bias=False,
                                              kernel_initializer=kernel_initializer)
        self.bn4_3 = batch_norm()

    def __call__(self, x,training=False):

        path1=tf.nn.relu(self.bn1(self.conv1(x),training=training))

        path2_pool=self.avgpool(x)
        path2=tf.nn.relu(self.bn2(self.conv2(path2_pool),training=training))

        path3_1=tf.nn.relu(self.bn3_1(self.conv3_1(x),training=training))
        path3 = tf.nn.relu(self.bn3_2(self.conv3_2(path3_1),training=training))

        path4_1 = tf.nn.relu(self.bn4_1(self.conv4_1(x),training=training))
        path4_2 = tf.nn.relu(self.bn4_2(self.conv4_2(path4_1),training=training))
        path4 = tf.nn.relu(self.bn4_3(self.conv4_3(path4_2),training=training))

        return tf.concat([path1,path2,path3,path4],axis=3)

class DecreaseBlock(tf.keras.Model):

    def __init__(self,
                 kernel_initializer='glorot_normal'):
        super(DecreaseBlock, self).__init__()
        self.conv1 = tf.keras.layers.Conv2D(filters=128,
                                            kernel_size=(1, 1),
                                            strides=1,
                                            padding='same',
                                            use_bias=False,
                                            kernel_initializer=kernel_initializer)
        self.bn1 = batch_norm()

        self.conv2 = tf.keras.layers.Conv2D(filters=256,
                                            kernel_size=(3, 3),
                                            strides=2,
                                            padding='same',
                                            use_bias=False,
                                            kernel_initializer=kernel_initializer)
        self.bn2 = batch_norm()

    def __call__(self, x,training=False):

        x=tf.nn.relu(self.bn1(self.conv1(x),training=training))

        x = tf.nn.relu(self.bn2(self.conv2(x),training=training))

        return x

class FaceBoxesHead(tf.keras.Model):
    def __init__(self,
                 kernel_initializer='glorot_normal'):
        super(FaceBoxesHead, self).__init__()
        self.conv_reg_1 = tf.keras.layers.Conv2D(filters=4 * 21,
                                                 kernel_size=(3, 3),
                                                 strides=1,
                                                 padding='same',
                                                 use_bias=True,
                                                 kernel_initializer=kernel_initializer)
        self.conv_cls_1 = tf.keras.layers.Conv2D(filters=2 * 21,
                                                 kernel_size=(3, 3),
                                                 strides=1,
                                                 padding='same',
                                                 use_bias=True,
                                                 kernel_initializer=kernel_initializer)

        self.conv_reg_2 = tf.keras.layers.Conv2D(filters=4 * 1,
                                                 kernel_size=(3, 3),
                                                 strides=1,
                                                 padding='same',
                                                 use_bias=True,
                                                 kernel_initializer=kernel_initializer)
        self.conv_cls_2 = tf.keras.layers.Conv2D(filters=2 * 1,
                                                 kernel_size=(3, 3),
                                                 strides=1,
                                                 padding='same',
                                                 use_bias=True,
                                                 kernel_initializer=kernel_initializer)

        self.conv_reg_3 = tf.keras.layers.Conv2D(filters=4 * 1,
                                                 kernel_size=(3, 3),
                                                 strides=1,
                                                 padding='same',
                                                 use_bias=True,
                                                 kernel_initializer=kernel_initializer)
        self.conv_cls_3 = tf.keras.layers.Conv2D(filters=2 * 1,
                                                 kernel_size=(3, 3),
                                                 strides=1,
                                                 padding='same',
                                                 use_bias=True,
                                                 kernel_initializer=kernel_initializer)

    def __call__(self, fms, training=False):
        fm1,fm2,fm3=fms

        reg_1=self.conv_reg_1(fm1)
        reg_1 = tf.reshape(reg_1, ([-1, 32, 32, 21, 4]))
        reg_1 = tf.reshape(reg_1, ([-1, 32 * 32 * 21, 4]))

        reg_2 = self.conv_reg_2(fm2)
        reg_2 = tf.reshape(reg_2, ([-1, 16, 16, 1, 4]))
        reg_2 = tf.reshape(reg_2, ([-1, 16 * 16, 4]))

        reg_3 = self.conv_reg_3(fm3)
        reg_3 = tf.reshape(reg_3, ([-1, 8, 8, 1, 4]))
        reg_3 = tf.reshape(reg_3, ([-1, 8 * 8, 4]))

        reg = tf.concat([reg_1, reg_2, reg_3], axis=1)

        cls_1 = self.conv_cls_1(fm1)
        cls_1 = tf.reshape(cls_1, ([-1, 32, 32, 21, 2]))
        cls_1 = tf.reshape(cls_1, ([-1, 32 * 32 * 21, 2]))

        cls_2 = self.conv_cls_2(fm2)
        cls_2 = tf.reshape(cls_2, ([-1, 16, 16, 1, 2]))
        cls_2 = tf.reshape(cls_2, ([-1, 16 * 16, 2]))

        cls_3 = self.conv_cls_3(fm3)
        cls_3 = tf.reshape(cls_3, ([-1, 8, 8, 1, 2]))
        cls_3 = tf.reshape(cls_3, ([-1, 8 * 8, 2]))

        cls = tf.concat([cls_1, cls_2, cls_3], axis=1)

        return reg, cls

class FaceBoxes(tf.keras.Model):
    def __init__(self,
                 kernel_initializer='glorot_normal'):
        super(FaceBoxes, self).__init__()

        self.RDCL=RDCL(kernel_initializer=kernel_initializer)
        self.inception_blocks=[Inception(kernel_initializer=kernel_initializer) for i in range(3)]

        self.decrease_blocks=[DecreaseBlock(kernel_initializer=kernel_initializer) for i in range(2)]

        self.head=FaceBoxesHead(kernel_initializer=kernel_initializer)

    def call(self,images, training):

        fms=[]

        x=self.preprocess(images)

        x=self.RDCL(x,training=training)

        for inception in self.inception_blocks:
            x=inception(x,training=training)
        fms.append(x)

        for decrese in self.decrease_blocks:
            x=decrese(x,training=training)
            fms.append(x)

        loc_predict,cls_predict=self.head(fms,training=training)

        return loc_predict,cls_predict

    def preprocess(self, image):

        image = tf.cast(image, tf.float32)

        mean = cfg.DATA.PIXEL_MEAN
        # std = np.asarray(cfg.DATA.PIXEL_STD)

        image_mean = tf.constant(mean, dtype=tf.float32)
        # image_invstd = tf.constant(1.0 / std, dtype=tf.float32)
        image = (image - image_mean)  # *image_invstd

        return image

    def get_predictions(self,box_encodings,cla,anchors, score_threshold=cfg.TEST.score_threshold, iou_threshold=cfg.TEST.iou_threshold, max_boxes=cfg.TEST.max_boxes):
        """Postprocess outputs of the network.
        Returns:
            boxes: a float tensor with shape [batch_size, N, 4].
            scores: a float tensor with shape [batch_size, N].
            num_boxes: an int tensor with shape [batch_size], it
                represents the number of detections on an image.
            where N = max_boxes.
        """

        boxes = batch_decode(box_encodings, anchors)
        # if the images were padded we need to rescale predicted boxes:

        boxes = tf.clip_by_value(boxes, 0.0, 1.0)
        # it has shape [batch_size, num_anchors, 4]

        scores = tf.nn.softmax(cla, axis=2)[:, :, 1]
        # it has shape [batch_size, num_anchors],  background are ignored

        with tf.device('/cpu:0'):
            boxes, scores, num_detections = batch_non_max_suppression(
                boxes, scores, score_threshold, iou_threshold, max_boxes
            )

        boxes=tf.identity(boxes,name='boxes')
        scores = tf.identity(scores, name='scores')
        num_detections = tf.identity(num_detections, name='num_detections')

        return {'boxes': boxes, 'scores': scores, 'num_boxes': num_detections}

    @tf.function(input_signature=[tf.TensorSpec([None,cfg.MODEL.hin,cfg.MODEL.win,3], tf.float32)])
    def inference(self,images):

        reg_prediction,cls_prediction=self.call(images,False)
        res=self.get_predictions(reg_prediction,cls_prediction,anchors=cfg.MODEL.anchors)
        return res

@tf.function
def calculate_loss(reg_targets,matches,loc_predict,cls_predict):
    #### loss

    # whether anchor is matched
    is_matched = tf.greater_equal(matches, 0)

    weights = tf.cast(is_matched,dtype=tf.float32)
    # shape [batch_size, num_anchors]

    # we have binary classification for each anchor
    cls_targets = tf.cast(is_matched,dtype=tf.int32)

    cls_losses = ohem_loss(
        cls_predict,
        cls_targets,
        is_matched
    )

    location_losses = localization_loss(
        loc_predict,
        reg_targets, weights
    )
    # they have shape [batch_size, num_anchors]

    matches_per_image = tf.reduce_sum(weights, axis=1)  # shape [batch_size]
    num_matches = tf.reduce_sum(matches_per_image)  # shape []
    normalizer = tf.maximum(num_matches, 1.0)

    reg_loss = tf.reduce_sum(location_losses) / normalizer
    cla_loss = tf.reduce_sum(cls_losses) / normalizer

    loss=reg_loss+ cla_loss
    return loss

if __name__=='__main__':
    input=tf.zeros(shape=[1,512,512,3],)

    model=FaceBoxes(images=input)

    x=model(is_training=True)


text:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import scipy.stats

def stringify(lst):
    s = ""
    for el in lst:
        s += str(el) + '-'
    return s[:-1]

def clear():
    plt.ioff()
    plt.clf()
    plt.cla()
    plt.close()

def score(est, X, y):
    preds = est.predict(X)
    loss = np.sum(np.square(y - preds))
    res = np.sum(np.square(y - np.mean(y, axis=0)))
    return 1 - loss/res

def stringify(param):
    if isinstance(param, list):
        s = ""
        for el in param:
            s += str(el) + '-'
        return s[:-1]
    else:
        return str(param)

def trajs2data(trajs):
    for i in range(len(trajs)):
        traj = trajs[i]
        states, controls = zip(*traj)
        if i == 0:
            X = np.array(states)
            y = np.array(controls)
        else:
            X = np.vstack((X, np.array(states)))
            y = np.vstack((y, np.array(controls)))
    return X, y

PARENT_DIR = "/cluster/scratch/fpirovan/DART"
# PARENT_DIR = "/Users/fedepiro/Projects/DART"
import os

def generate_dir(title, sub_dir, input_params):
    d = 'results/' + sub_dir + '/' + input_params['envname'] + '.pkl/' + title + '/'
    d = os.path.join(PARENT_DIR, d)
    keys = sorted(input_params.keys())
    for key in keys:
        param = stringify(input_params[key])
        d += key + param + '_'
    return d[:-1]

def generate_plot_dir(title, sub_dir, input_params):
    d = generate_dir(title, sub_dir, input_params)
    return d + '_plot/'

def generate_data_dir(title, sub_dir, input_params):
    d = generate_dir(title, sub_dir, input_params)
    return d + '_data/'

def extract_data(params, iters, title, sub_dir, ptype):
    means, sems = [], []
    for it in iters:
        params['it'] = it

        path = generate_data_dir(title, sub_dir, params) + 'data.csv'
        data = pd.read_csv(path)
        arr = np.array(data[ptype])

        mean, sem = np.mean(arr), scipy.stats.sem(arr)
        means.append(mean)
        sems.append(sem)
    return np.array(means), np.array(sems)

def extract_time_data(params, iters, title, sub_dir):
    params['iters'] = iters
    path = generate_data_dir(title, sub_dir, params) + 'time_tests.txt'
    data = pd.read_csv(path)
    arr = np.array(data['time'])
    mean, sem = np.mean(arr), scipy.stats.sem(arr)
    return mean, sem

def filter_data(params, states, i_actions):
    """
        Filter the data, choosing only 50 samples
        as described in Ho and Ermon, 2016
    """
    T = params['t']
    k = np.random.randint(0, T/50)
    upper_limit = int(T/50)
    states, i_actions = states[k::upper_limit], i_actions[k::upper_limit]
    return states, i_actions

max_stars_repo_name: fpirovan/NoiseInjection

max_issues_repo_name: fpirovan/NoiseInjection

max_forks_repo_name: fpirovan/NoiseInjection


text:
"""
    Copyright 2018 Ashar <ashar786khan@gmail.com>

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""

import numpy as np

def get_one_hot(targets, nb_classes):
    res = np.eye(nb_classes)[np.array(targets).reshape(-1)]
    return res.reshape(list(targets.shape)+[nb_classes])

class FilePipeline():
    def __init__(self, file_name, encoding='utf-8', batch_size=100, epoch=10):
        self.file = file_name
        self.encoding = encoding
        self._data = None
        self.read_pointer = 0
        self.cur_epoch = 0
        self.tot_epoch = epoch
        self.batch = batch_size

        # Private calls
        self._read()
        self._index_list = self._get_index_list()
        self._char_dict = self._get_vocab_dict()
        self._unique_chars = self.get_distinct_chars()

        # Useful information for user
        print('Reading the file :', file_name)
        print('Encoding : ', encoding)
        print('Unique Characters : ', len(self._unique_chars))
        print('Total Characters :', len(list(self._data)))
        print('Epoch :', epoch)
        print('Batch Size : ', batch_size)
        print('Total Expected iteration : ',
              self.get_expected_total_iteration())

    def _read(self):
        res = None
        with open(self.file, mode='r', encoding=self.encoding) as f:
            res = f.read()
        self._data = res
        return res

    def get_distinct_chars(self):
        chars = set(list(self._data))
        return chars

    def _get_vocab_dict(self, vocabs=None):
        return {char: index for index, char in enumerate(sorted(self.get_distinct_chars()))}

    def _get_index_list(self):
        return [char for char in sorted(self.get_distinct_chars())]

    def _batch_to_one_hot(self, batch):
        index_rep = [self._get_vocab_dict()[char] for char in batch]
        return get_one_hot(np.array(index_rep), len(self._unique_chars))

    def next_data(self):
        data = self._data
        start = self.read_pointer
        stop = start + 1
        if stop+1 > len(data):
            self.read_pointer = 0
            self.cur_epoch += 1
            return (np.squeeze(self._batch_to_one_hot(data[0:1])), np.squeeze(self._batch_to_one_hot(data[1:2])), False)
        else:
            self.read_pointer = stop
            if self.cur_epoch > self.tot_epoch:
                return (self._batch_to_one_hot(data[start:stop]), self._batch_to_one_hot(data[start+1:stop+1]), True)
            return (np.squeeze(self._batch_to_one_hot(data[start:stop])), np.squeeze(self._batch_to_one_hot(data[start+1:stop+1])), False)

    def next_batch(self):
        data = (np.zeros((self.batch, self.get_distinct_char_count())),
                np.zeros((self.batch, self.get_distinct_char_count())))
        for _ in range(self.batch):
            x, y, end = self.next_data()
            data[0][_, :] = x
            data[1][_, :] = y
        return (data, end)

    def get_distinct_char_count(self):
        return len(self.get_distinct_chars())

    def get_expected_total_iteration(self):
        data_len = len(list(self._data))-1
        iteration = (self.tot_epoch*data_len + data_len)//self.batch
        return iteration+1