text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
This file contains the experiments in the active scenario of [Haddenhorst2021].
In order to rerun these, UNCOMMENT in "Part 4" below the corresponding lines and
then execute this file.

[Haddenhorst2021]: B. Haddenhorst, V. Bengs, J. Brandt and E.Hüllermeier, Testification of Condorcet Winners in Dueling Bandits, Proceedings of UAI, 2021
"""
import ReciprocalRelations as rr
import TestEnvironment as tenv
import TestingComponent as tc
import numpy as np
import DeterministicTestingComponent as dtc
import SELECT as select
import math
import matplotlib.pyplot as plt

nr_items, decimal_precision = 5,3
np.set_printoptions(precision=3)

###############################################################################
#   PART 1: Define 'find_CW_with_symm_test', 'SELECT_then_verify'
#           and 'buf_SPRT'
###############################################################################
def find_CW_with_symm_test(TE, h, gamma):
    """
    This is an implementation of the Algorithm NTS from the paper.
    """
    SymmTC = tc.Symmetric_TestingComponent(TE.N, TE.R, h=float(h), gamma=gamma)
    sampling_strategy = dtc.Optimal_Deterministic_CW_Tester(len(TE.N))
    for t in range(500000):
        [i,j] = sampling_strategy.getQuery()
        while not SymmTC.G.has_edge(i,j) and not SymmTC.G.has_edge(j,i):
            feedback = TE.pullArmPair(i,j)
            SymmTC.update(i,j,feedback)
            if SymmTC.TC():
                SymmTC.DC()
                return SymmTC.find_CW(), SymmTC.time
        sampling_strategy.giveFeedback(feedback)

def SELECT_then_verify(TE,h,gamma, variant = "Hoeffding"):
    """
    This is the implementation from SELECT-then-verify from the paper.
    The internal hypothesis test for verifying the output of SELECT can either
    be the non-sequential Hoeffding-bound test (with 'variant="Hoeffding"')
    or the corresponding SPRT (with 'variant="SPRT"').
    """
    assert variant=="Hoeffding" or variant=="SPRT", "'variant' has to be 'Hoeffding' or 'SPRT'"
    m = TE.P.m
    epsilon = -np.log(0.5*gamma)/np.log(np.log2(m))
    m_h = math.floor((1+epsilon)*math.log(2)/2*math.log(math.log(m,2),2)/(h*h))+1
    CW_M, itera = select.select(list(np.arange(m)), m_h, TE.pullArmPair)
    CW_M = int(CW_M)
    # print("Output of SELECT:",CW_M,"(after ",itera," iterations)")
    # TE.show()
    if variant=="Hoeffding":
        t0 = np.ceil(2/(h**2) * np.log(2*(m-1)/gamma))
        w = np.zeros(m)
        for j in range(0,m):
            if CW_M is not j:
                for t in range(0,int(t0)):
                    buf = TE.pullArmPair(CW_M,j)
                    w[j] += buf     #Increases by 1 if CW_M has won
                    if w[j]/t0 < 0.5:
                        # print("w",w)
                        return(False)
        return(CW_M)

    if variant=="SPRT":
        for j in range(0,m):
            if CW_M is not j:
                winner = buf_SPRT(TE,h,gamma/(2*(m-1)),CW_M,j)
                if winner != CW_M:
                    return(False)
    return(CW_M)

def buf_SPRT(TE,h,gamma,i,j):
    """
    This function conducts a SPRT (with parameters h,gamma) in order to decide
    whether the (i,j)-entry of TE.P is >1/2 or <1/2.
    """
    N = 1
    C = (1/(2*N)) * np.ceil(np.log( (1-gamma) / gamma ) / np.log( (0.5+h) / (0.5-h) ))
    w = TE.pullArmPair(i,j)
    while 0.5-C < w/N and w/N< 0.5+C:
        w += TE.pullArmPair(i,j)
        C = (1/(2*N)) * np.ceil(np.log( (1-gamma) / gamma ) / np.log( (0.5+h) / (0.5-h) ))
        N = N+1
    if w/N >= 0.5+C:
        return(i)
    else:
        return(j)

###############################################################################
#   PART 3: Define the evaluation functions 'experiments_one', 'experiment_two'
#           'experiment_three' as well as 'generate_main_figure'.
###############################################################################
def experiments_one(m,h,gamma,nr_iterations=100, real_h = 0.05, has_CW = "No",verify_variant="SPRT"):
    """
    This function compares NTS with SELECT-then-verify. It is required for the
    function "experiment_two".
    """
    assert has_CW=="No" or has_CW=="Yes" or has_CW=="Both", "'has_CW' has to be 'Yes','No' or 'Both'."
    results = dict()
    results["NTS_output"] =list()
    results["NTS_time"] = list()
    results["S_t_verify_output"] = list()
    results["S_t_verify_time"] = list()
    results["Truth"] = list()
    for iteration in range(0,nr_iterations):
        # Step 1: Sample a reciprocal relation, create a TE and a dictionary to save results.
        if has_CW == "No":
            P, buf = rr.sampleCW_boundedFromOneHalf(m,real_h,decimal_precision=3)
        elif has_CW == "Yes":
            P = rr.sampleNotCW_boundedFromOneHalf(m,real_h,max_tries=10000,decimal_precision=3)
        else:
            P = rr.sampleReciprocal(m,decimal_precision=3)
            P = rr.__EnforceBoundedFromOneHalf__(P,real_h)
        results["Truth"].append(rr.get_CW(P))

        # Step 2: Run and log our NTS
        TE = tenv.TestEnvironment(P)
        current_output = find_CW_with_symm_test(TE, h, gamma)
        results["NTS_output"].append(current_output[0])
        results["NTS_time"].append(TE.time)

        # Step 3: Run and log SELECT_then_verify
        TE = tenv.TestEnvironment(P)
        current_output = SELECT_then_verify(TE, h, gamma,variant=verify_variant)
        results["S_t_verify_output"].append(current_output)
        results["S_t_verify_time"].append(TE.time)

    # Step 4: Calculate the accuracy of both algorithms and return the results
    nr_correct_NTS ,nr_correct_S_t_verify = 0,0
    for i in range(0,nr_iterations):
        if results["Truth"][i] == results["NTS_output"][i]:
            nr_correct_NTS += 1
        if results["Truth"][i] == results["S_t_verify_output"][i]:
            nr_correct_S_t_verify += 1
    results["Acc_NTS"] = nr_correct_NTS / nr_iterations
    results["Acc_S_t_verify"] = nr_correct_S_t_verify / nr_iterations
    results["NTS_mean_time"] = np.mean(results["NTS_time"])
    results["S_t_verify_mean_time"] = np.mean(results["S_t_verify_time"])
    # print(results)
    # print("NTS: mean",np.mean(results["NTS_time"]),"\t std:",np.std(results["NTS_time"]), "\t Accuracy:", results["Acc_NTS"])
    # print("SELECT_then_verify: mean",np.mean(results["S_t_verify_time"]),"\t std:",np.std(results["S_t_verify_time"]), "\t Accuracy:", results["Acc_S_t_verify"])
    return(results)

def experiment_two(m=5, real_h=0.1, h=0.3, file_name ="plot",nr_iterations =25000,has_CW = "No",verify_variant="SPRT"):
    """
    This function compares NTS with SELECT-then-verify for the given parameters,
    saves the observed accuracies and averaged termination times and plots the
    results.
    """
    gammas = [0.001,0.005,0.01,0.015,0.02,0.03,0.05,0.075,0.1,0.125,0.15,0.2,0.25,0.35,0.45,0.5,0.55,0.6,0.65,0.7,0.75,0.7,0.95,0.99]
    x_NTS = np.zeros(len(gammas))
    y_NTS = np.zeros(len(gammas))
    x_StV = np.zeros(len(gammas))
    y_StV = np.zeros(len(gammas))
    print("Progress for "+str(file_name)+" (.. out of "+str(len(gammas))+"): ")
    for i in range(0,len(gammas)):
        print(i,end=",")
        buf = experiments_one(m,h,gammas[i],nr_iterations,real_h=real_h,has_CW = has_CW)
        x_NTS[i] = buf["NTS_mean_time"]
        x_StV[i] = buf["S_t_verify_mean_time"]
        y_NTS[i] = buf["Acc_NTS"]
        y_StV[i] = buf["Acc_S_t_verify"]
    plt.plot(x_NTS,y_NTS, marker = "^", label="NTS")
    plt.plot(x_StV, y_StV, marker = "o", label="SELECT-then-verify")
    plt.xlabel("Iterations")
    plt.ylabel("Success Rate")
    plt.legend()
    plt.title("h="+str(h))
    plt.savefig(str(file_name)+"_plot.png",dpi=300)
    plt.show()
    np.savetxt(str(file_name)+"_results.csv",np.asarray([x_NTS,y_NTS,x_StV,y_StV]),delimiter=",")
    # print(x_NTS,y_NTS)
    print("Done.")

def experiment_three_help(m,h,gamma,nr_iterations=100, real_h = 0.05, has_CW = "No",verify_variant="SPRT"):
    """
    This is an appropriate modification of 'experiments_one' from above
    Each line of code that has been modified is indicated with a comment '[CHANGE]',
    together with the original code.
    """
    assert has_CW=="No" or has_CW=="Yes" or has_CW=="Both", "'has_CW' has to be 'Yes','No' or 'Both'."
    results = dict()
    results["NTS_output"] =list()
    results["NTS_time"] = list()
    results["S_t_verify_output"] = list()
    results["S_t_verify_time"] = list()
    results["Truth"] = list()
    for iteration in range(0,nr_iterations):
        # Step 1: Sample a reciprocal relation, create a TE and a dictionary to save results.
        if has_CW == "No":
            #P, buf = rr.sampleCW_boundedFromOneHalf(m,real_h,decimal_precision=3)  [CHANGE]
            P, buf = rr.sampleCW_exactly_h(m,real_h,decimal_precision=3)
        elif has_CW == "Yes":
            # P = rr.sampleNotCW_boundedFromOneHalf(m,real_h,max_tries=10000,decimal_precision=3) [CHANGE]
            P = rr.sampleNotCW_exactly_h(m, real_h,max_tries = 100000,decimal_precision = 3)
        else:
            # P = rr.sampleReciprocal(m,decimal_precision=3) [CHANGE]
            # P = rr.__EnforceBoundedFromOneHalf__(P,real_h) [CHANGE]
            P = rr.sampleRecRel_exactly_h(m,real_h,decimal_precision = 3)
        results["Truth"].append(rr.get_CW(P))

        # Step 2: Run and log our NTS
        TE = tenv.TestEnvironment(P)
        current_output = find_CW_with_symm_test(TE, h, gamma)
        results["NTS_output"].append(current_output[0])
        results["NTS_time"].append(TE.time)

        # Step 3: Run and log SELECT_then_verify
        TE = tenv.TestEnvironment(P)
        current_output = SELECT_then_verify(TE, h, gamma,variant=verify_variant)
        results["S_t_verify_output"].append(current_output)
        results["S_t_verify_time"].append(TE.time)

    # Step 4: Calculate the accuracy of both algorithms and return the results
    nr_correct_NTS ,nr_correct_S_t_verify = 0,0
    for i in range(0,nr_iterations):
        if results["Truth"][i] == results["NTS_output"][i]:
            nr_correct_NTS += 1
        if results["Truth"][i] == results["S_t_verify_output"][i]:
            nr_correct_S_t_verify += 1
    results["Acc_NTS"] = nr_correct_NTS / nr_iterations
    results["Acc_S_t_verify"] = nr_correct_S_t_verify / nr_iterations
    results["NTS_mean_time"] = np.mean(results["NTS_time"])
    results["S_t_verify_mean_time"] = np.mean(results["S_t_verify_time"])
    return(results)

def experiment_three(m=5, gamma = 0.05, real_hs=[0.1,0.2,0.3,0.4], h=0.1, file_name ="plot",nr_iterations =25000,has_CW = "No",verify_variant="SPRT"):
    """
    This is a simple modification of experiment_two.
    -- instead of modifying gamma, we modify the value of real_h.
    """
    # gammas = [0.001,0.005,0.01,0.015,0.02,0.03,0.05,0.075,0.1,0.125,0.15,0.2,0.25,0.35,0.45,0.5,0.55,0.6,0.65,0.7,0.75,0.7,0.95,0.99]
    len_real_hs = len(real_hs)
    x_NTS = np.zeros(len_real_hs)
    y_NTS = np.zeros(len_real_hs)
    x_StV = np.zeros(len_real_hs)
    y_StV = np.zeros(len_real_hs)
    print("Progress for "+str(file_name)+" (.. out of "+str(len_real_hs)+"): ")
    for i in range(0,len_real_hs):
        print(i,end=",")
        buf = experiment_three_help(m,h,gamma,nr_iterations,real_h=real_hs[i],has_CW = has_CW)
        x_NTS[i] = buf["NTS_mean_time"]
        x_StV[i] = buf["S_t_verify_mean_time"]
        y_NTS[i] = buf["Acc_NTS"]
        y_StV[i] = buf["Acc_S_t_verify"]
    plt.plot(real_hs,x_NTS, marker = "^", label="NTS")
    plt.plot(real_hs, x_StV, marker = "o", label="SELECT-then-verify")
    plt.xlabel("real_h")
    plt.ylabel("Iterations")
    plt.legend()
    plt.title("h="+str(h))
    plt.savefig(str(file_name)+"_plot.png",dpi=300)
    plt.show()
    np.savetxt(str(file_name)+"_results.csv",np.asarray([real_hs,x_NTS,y_NTS,x_StV,y_StV]),delimiter=",")
    print("Done.")
    # OUTPUT THE RESULTS AS A TABLE:
    print("The results in form of [h, T A^NTS, Acc. A^NTS, T^StV, Acc StV] are:")
    for i in range(0,len_real_hs):
        print(real_hs[i],x_NTS[i],y_NTS[i],x_StV[i],y_StV[i])

##############################################################################
# The following function helps to create Figure 1  of our paper.
##############################################################################
def generate_main_figure():
    a = np.loadtxt("MAIN_h02_results.csv",delimiter=",")
    b = np.loadtxt("MAIN_h03_results.csv",delimiter=",")
    plt.figure(figsize=(12,5))
    plt.rcParams.update({'font.size': 14})
    plt.subplot(1,2,1)
    plt.subplots_adjust(left=0.07,
                    bottom=0.1,
                    right=0.99,
                    top=0.92,
                    wspace=0.1,
                    hspace=0.1)
    plt.plot(a[0], a[1], marker = "^", label="NTS")
    plt.plot(a[2], a[3], marker = "o", label="SELECT-then-verify")
    plt.ylabel("Success Rate")
    plt.xlabel("Iterations")
    plt.title("h=0.2")
    plt.legend(loc="lower right")
    plt.subplot(1,2,2)
    plt.xlim(left=0.5*min(min(b[0]),min(b[2])),right=2*min(max(b[0]),max(b[2])))
    plt.plot(b[0],b[1], marker = "^", label="NTS")
    plt.plot(b[2],b[3], marker = "o", label="SELECT-then-verify")
    plt.xlabel("Iterations")
    plt.title("h=0.3")
    plt.legend(loc="lower right")
    plt.savefig("MAIN_figure.png",dpi=600)
    # fig.tight_layout()
    plt.show()
    plt.rcParams.update(plt.rcParamsDefault)

###############################################################################
#   PART 4: Reconstruct the results from our paper.
###############################################################################

###############################################################################
#   PART 4.1: Reconstruct the results from Section 7.1
###############################################################################
NR_it = 25000
np.random.seed(1)
experiment_two(m=5, real_h=0.05, h=0.2, file_name ="MAIN_h02",nr_iterations =NR_it,has_CW = "Both",verify_variant="SPRT")
np.random.seed(2)
experiment_two(m=5, real_h=0.05, h=0.3, file_name ="MAIN_h03",nr_iterations =NR_it,has_CW = "Both",verify_variant="SPRT")
generate_main_figure()

###############################################################################
#   PART 4.2.1: Reconstruct the results from Section I.1, Figure 3
#               (Similar to those in Sec. 7.1, but with larger number of arms)
###############################################################################
NR_it = 100000
np.random.seed(3)
experiment_two(m=10, real_h=0.05, h=0.3, file_name ="SUPPL_LARGE10_h03",nr_iterations =NR_it,has_CW = "Both",verify_variant="SPRT")
np.random.seed(101)
experiment_two(m=8, real_h=0.05, h=0.3, file_name ="SUPPL_LARGE08_h03",nr_iterations =NR_it,has_CW = "Both",verify_variant="SPRT")

###############################################################################
#   PART 4.2.2: Reconstruct the results from Section I.1, Table 2
#               (Comparison of SELECT-then-Verify and NTS on \hat{Q}_{m}^{h})
###############################################################################
NR_it = 100
real_hs = [0.02,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45]
np.random.seed(3)
experiment_three(m=20, gamma = 0.05, real_hs=real_hs, h=0.05, file_name ="Exp3_m20_h05",nr_iterations =NR_it,has_CW = "Yes",verify_variant="SPRT")

###############################################################################
#   PART 4.2.2: Reconstruct the results from Section I.2, Figures 4 & 5
#               (Similar to those in Sec. 7.1, but with restriction to CW/Non-CW instances)
###############################################################################
NR_it = 25000
np.random.seed(5)
experiment_two(m=5, real_h=0.05, h=0.1, file_name ="SUPPL_CW_h01",nr_iterations =NR_it,has_CW = "Yes",verify_variant="SPRT")
np.random.seed(6)
experiment_two(m=5, real_h=0.05, h=0.2, file_name ="SUPPL_CW_h02",nr_iterations =NR_it,has_CW = "Yes",verify_variant="SPRT")
np.random.seed(7)
experiment_two(m=5, real_h=0.05, h=0.3, file_name ="SUPPL_CW_h03",nr_iterations =NR_it,has_CW = "Yes",verify_variant="SPRT")
np.random.seed(8)
experiment_two(m=5, real_h=0.05, h=0.1, file_name ="SUPPL_noCW_h01",nr_iterations =NR_it,has_CW = "No",verify_variant="SPRT")
np.random.seed(9)
experiment_two(m=5, real_h=0.05, h=0.2, file_name ="SUPPL_noCW_h02",nr_iterations =NR_it,has_CW = "No",verify_variant="SPRT")
np.random.seed(10)
experiment_two(m=5, real_h=0.05, h=0.3, file_name ="SUPPL_noCW_h03",nr_iterations =NR_it,has_CW = "No",verify_variant="SPRT")

max_stars_repo_name: bjoernhad/CondorcetWinnerTestification

max_issues_repo_name: bjoernhad/CondorcetWinnerTestification

max_forks_repo_name: bjoernhad/CondorcetWinnerTestification


text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""Testcode for iOS RPC.

To use it, start a rpc proxy with "python -m tvm.exec.rpc_proxy".
And configure the proxy host field as commented.
"""

import tvm
from tvm import te
import os
import re
import sys
from tvm import rpc
from tvm.contrib import utils, xcode
import numpy as np

# Set to be address of tvm proxy.
proxy_host = os.environ["TVM_IOS_RPC_PROXY_HOST"]
# Set your desination via env variable.
# Should in format "platform=iOS,id=<the test device uuid>"
destination = os.environ["TVM_IOS_RPC_DESTINATION"]

if not re.match(r"^platform=.*,id=.*$", destination):
    print("Bad format: {}".format(destination))
    print("Example of expected string: platform=iOS,id=1234567890abcabcabcabc1234567890abcabcab")
    sys.exit(1)

proxy_port = 9090
key = "iphone"

# Change target configuration, this is setting for iphone6s
arch = "arm64"
sdk = "iphoneos"
target = "llvm -mtriple=%s-apple-darwin" % arch

# override metal compiler to compile to iphone
@tvm.register_func("tvm_callback_metal_compile")
def compile_metal(src):
    return xcode.compile_metal(src, sdk=sdk)

def test_rpc_module():
    # graph
    n = tvm.runtime.convert(1024)
    A = te.placeholder((n,), name="A")
    B = te.compute(A.shape, lambda *i: A(*i) + 1.0, name="B")
    temp = utils.tempdir()
    s = te.create_schedule(B.op)
    xo, xi = s[B].split(B.op.axis[0], factor=64)
    s[B].bind(xi, te.thread_axis("threadIdx.x"))
    s[B].bind(xo, te.thread_axis("blockIdx.x"))
    # Build the dynamic lib.
    # If we don't want to do metal and only use cpu, just set target to be target
    f = tvm.build(s, [A, B], "metal", target_host=target, name="myadd")
    path_dso1 = temp.relpath("dev_lib.dylib")
    f.export_library(path_dso1, xcode.create_dylib, arch=arch, sdk=sdk)
    xcode.codesign(path_dso1)

    s = te.create_schedule(B.op)
    xo, xi = s[B].split(B.op.axis[0], factor=64)
    s[B].parallel(xi)
    s[B].pragma(xo, "parallel_launch_point")
    s[B].pragma(xi, "parallel_barrier_when_finish")
    f = tvm.build(s, [A, B], target, name="myadd_cpu")
    path_dso2 = temp.relpath("cpu_lib.dylib")
    f.export_library(path_dso2, xcode.create_dylib, arch=arch, sdk=sdk)
    xcode.codesign(path_dso2)

    # Start RPC test server that contains the compiled library.
    server = xcode.popen_test_rpc(
        proxy_host, proxy_port, key, destination=destination, libs=[path_dso1, path_dso2]
    )

    # connect to the proxy
    remote = rpc.connect(proxy_host, proxy_port, key=key)
    dev = remote.metal(0)
    f1 = remote.load_module("dev_lib.dylib")
    a_np = np.random.uniform(size=1024).astype(A.dtype)
    a = tvm.nd.array(a_np, dev)
    b = tvm.nd.array(np.zeros(1024, dtype=A.dtype), dev)
    time_f = f1.time_evaluator(f1.entry_name, dev, number=10)
    cost = time_f(a, b).mean
    print("%g secs/op" % cost)
    np.testing.assert_equal(b.asnumpy(), a.asnumpy() + 1)
    # CPU
    dev = remote.cpu(0)
    f2 = remote.load_module("cpu_lib.dylib")
    a_np = np.random.uniform(size=1024).astype(A.dtype)
    a = tvm.nd.array(a_np, dev)
    b = tvm.nd.array(np.zeros(1024, dtype=A.dtype), dev)
    time_f = f2.time_evaluator(f1.entry_name, dev, number=10)
    cost = time_f(a, b).mean
    print("%g secs/op" % cost)
    np.testing.assert_equal(b.asnumpy(), a.asnumpy() + 1)

test_rpc_module()

ext: py

max_issues_repo_name: were/tvm

max_forks_repo_name: were/tvm

reason: import numpy


text:
import numpy as np
import matplotlib.pyplot as plt
import dnnlib.tflib as tflib
import training.misc as misc
import os
from training import dataset
from tqdm import tqdm
import tensorflow as tf

tflib.init_tf()

fig, axes = plt.subplots(2, 4, subplot_kw=dict(projection='polar'), figsize=(16, 9))
fig.tight_layout(pad=3.0)

rotation_offset = 108
tfrecord_dir = '../../datasets/car_labels_new_rotation_oversample'
dset = dataset.TFRecordDataset(tfrecord_dir)
batch_size = 8
dset.configure(batch_size)

# network_dir = '../../results/00190-stylegan2-car_labels_new_rotation_oversample-2gpu-config-f-euclidean-interpolate-50-percent_continue_189-v7-256'
# network_dir = '../../results/00189-stylegan2-car_labels_new_rotation_oversample-2gpu-config-f-euclidean-v7-256-resume-00274'

network_dir = '../../results/00207-stylegan2-car_labels_v7_oversample_filter-2gpu-config-f-squared_euclidean_10_interpolate_50_percent_int_reg_remove_half_fl_fr_square-256'

file = sorted(os.listdir(network_dir))[-1]
G, D, Gs = misc.load_pkl(network_dir + '/' + file)
mean_dist_disc = []
mean_dist_gen = []

titles = [
    'Front Center',
    'Front Left',
    'Profile Left',
    'Rear Left',
    'Rear Center',
    'Rear Right',
    'Profile Right',
    'Front Right'
]

all_rotations = np.array([
    [1.0, 0.0],
    [0.7071, 0.7071],
    [0.0, 1.0],
    [-0.7071, 0.7071],
    [-1.0, 0.0],
    [-0.7071, -0.7071],
    [0.0, -1.0],
    [0.7071, -0.7071],
    [0., 0.]
],
    dtype=np.float32
)
all_rotations = np.tile(np.expand_dims(all_rotations, axis=0), [batch_size, 1, 1])
all_label_rotations = [[], [], [], [], [], [], [], [], []]
all_fake_rotations = [[], [], [], [], [], [], [], [], []]
all_pred_rotations = [[], [], [], [], [], [], [], [], []]

label_placeholder = tf.placeholder(tf.float32, shape=G.input_shapes[1])
image_placeholder = tf.placeholder(tf.float32, shape=D.input_shapes[0])

latents = tf.random_normal(shape=[batch_size] + G.input_shapes[0][1:])
fake_images = G.get_output_for(latents, label_placeholder, is_training=True)
real_scores = D.get_output_for(image_placeholder, label_placeholder, is_training=True)
fake_scores = D.get_output_for(fake_images, label_placeholder, is_training=True)

for i in tqdm(range(30)):
    images, labels = dset.get_minibatch_np(batch_size)
    images = misc.adjust_dynamic_range(images, [0, 255], [-1, 1])
    fake_images_out, real_scores_out, fake_scores_out = tflib.run([fake_images, real_scores, fake_scores],
                                                                  feed_dict={
                                                                      label_placeholder: labels,
                                                                      image_placeholder: images
                                                                  })

    labels_rotation = labels[:, rotation_offset:rotation_offset + 2]
    fake_rotations = fake_scores_out[:, rotation_offset:rotation_offset + 2]
    pred_rotations = real_scores_out[:, rotation_offset:rotation_offset + 2]

    labels_rotation_tile = np.tile(np.expand_dims(labels_rotation, axis=1), [1, 9, 1])
    distances = np.linalg.norm(all_rotations - labels_rotation_tile, axis=-1)
    rotation_index = np.argmin(distances, axis=-1)
    print(rotation_index)
    for j in range(batch_size):
        all_label_rotations[rotation_index[j]].append(labels_rotation[j])
        all_fake_rotations[rotation_index[j]].append(fake_rotations[j])
        all_pred_rotations[rotation_index[j]].append(pred_rotations[j])

index = 0

for i in range(2):
    for j in range(4):
        mean_disc = 0
        mean_gen = 0
        for k in range(len(all_label_rotations[index])):
            ax = axes[i, j]
            ax.set_rscale('linear')
            current_label = np.array(all_label_rotations[index][k])
            current_pred = np.array(all_pred_rotations[index][k])
            current_fake = np.array(all_fake_rotations[index][k])
            mean_disc += np.linalg.norm(current_label - current_pred, ord=2)
            mean_gen += np.linalg.norm(current_label - current_fake, ord=2)

            abs_label_real = [np.linalg.norm(current_label, ord=2), np.linalg.norm(current_pred, ord=2)]
            angle_label_real = [np.arctan2(current_label[1], current_label[0]),
                                np.arctan2(current_pred[1], current_pred[0])]
            ax.plot(angle_label_real, abs_label_real, c='green')
            ax.scatter(angle_label_real, abs_label_real, c='green')

            abs_label_fake = [np.linalg.norm(current_label, ord=2), np.linalg.norm(current_fake, ord=2)]
            angle_label_fake = [np.arctan2(current_label[1], current_label[0]),
                                np.arctan2(current_fake[1], current_fake[0])]
            ax.plot(angle_label_fake, abs_label_fake, c='red')
            ax.scatter(angle_label_fake, abs_label_fake, c='red')

            # abs_zero_fake = [0, np.linalg.norm(current_fake, ord=2)]
            # angle_zero_fake = [0, np.arctan2(current_fake[1], current_fake[0])]
            # ax.plot(angle_zero_fake, abs_zero_fake, c='orange')
            # ax.scatter(angle_zero_fake, abs_zero_fake, c='orange')
        mean_disc /= len(all_label_rotations[index])
        mean_gen /= len(all_label_rotations[index])
        mean_dist_disc.append(mean_disc)
        mean_dist_gen.append(mean_gen)
        index += 1

index = 0
for i in range(2):
    for j in range(4):
        ax = axes[i, j]
        ax.title.set_text(titles[index] + '\nmean_dist_disc: ' + str(np.round(mean_dist_disc[index], 4)) + '\nmean_dist_gen: ' + str(np.round(mean_dist_gen[index], 4)))
        index += 1

plt.savefig("plot_circle_multiple_207.png")
plt.show()


text:
"""
Converting TIFF images to PNG

Convert the original TIFF image containing a pyramid to single image
of a particular level (regarding the pyramid) using mosaic techniques

Be aware that this conversion takes lots of memory, for example
image of size 50k x 100k takes about 80GB in RAM

EXAMPLE
-------
>> python convert_tiff2png.py \
    -i "/datagrid/Medical/microscopy/histology/*/*.tiff" \
    -l 0 --level_offset 2 --nb_jobs 2

Copyright (C) 2016-2018 Jiri Borovec <jiri.borovec@fel.cvut.cz>
"""

import os
import sys
import glob
import time
import gc
import logging
import argparse
import multiprocessing as mproc
from functools import partial

import tqdm
import cv2 as cv
import numpy as np
from openslide import OpenSlide

sys.path += [os.path.abspath('.'), os.path.abspath('..')]  # Add path to root
from benchmark.utilities.experiments import wrap_execute_sequence

DEFAULT_LEVEL = 1
DEFAULT_LEVEL_OFFSET = 2
IMAGE_EXTENSION = '.png'
NB_THREADS = int(mproc.cpu_count() * .5)

def arg_parse_params():
    """ parse the input parameters
    :return dict: {str: any}
    """
    # SEE: https://docs.python.org/3/library/argparse.html
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--path_images', type=str, required=True,
                        help='path (pattern) to the input image')
    parser.add_argument('-l', '--level', type=int, required=False,
                        help='list of output scales', default=DEFAULT_LEVEL)
    parser.add_argument('--level_offset', type=int, required=False,
                        help='list of output scales', default=DEFAULT_LEVEL_OFFSET)
    parser.add_argument('--overwrite', action='store_true', required=False,
                        default=False, help='visualise the landmarks in images')
    parser.add_argument('--nb_jobs', type=int, required=False,
                        help='number of processes running in parallel',
                        default=NB_THREADS)
    args = vars(parser.parse_args())
    args['path_images'] = os.path.expanduser(args['path_images'])
    logging.info('ARGUMENTS: \n%s' % repr(args))
    return args

def convert_image(path_img, level=DEFAULT_LEVEL, level_shift=DEFAULT_LEVEL_OFFSET, overwrite=False):
    slide_img = OpenSlide(path_img)
    assert level < len(slide_img.level_dimensions), \
        'unsupported level %i' % level
    assert (level + level_shift) < len(slide_img.level_dimensions), \
        'unsupported shift %i for level %i' % (level_shift, level)

    path_img_new = os.path.splitext(path_img)[0] + IMAGE_EXTENSION
    if os.path.isfile(path_img_new) and not overwrite:
        logging.debug('existing "%s"', path_img_new)
        return

    tile_size = slide_img.level_dimensions[level + level_shift]
    img_tiles_d0 = []
    tqdm_bar = tqdm.tqdm(total=(2 ** level_shift) ** 2,
                         desc=os.path.basename(path_img))
    for i in range(2 ** level_shift):
        img_tiles_d1 = []
        for j in range(2 ** level_shift):
            loc = (i * tile_size[0] * 2 ** level,
                   j * tile_size[1] * 2 ** level)
            img = slide_img.read_region(loc, level, size=tile_size)
            img_tiles_d1.append(img)
            tqdm_bar.update()
        img_tiles_d0.append(np.vstack(img_tiles_d1))
    image = np.hstack(img_tiles_d0)
    if image.shape[2] == 4:
        image = cv.cvtColor(image, cv.COLOR_RGBA2RGB)

    logging.debug('save image: "%s"', path_img_new)
    cv.imwrite(path_img_new, image, params=(cv.IMWRITE_PNG_COMPRESSION, 9))
    gc.collect()
    time.sleep(1)

def main(path_images, level=DEFAULT_LEVEL, level_shift=DEFAULT_LEVEL_OFFSET,
         overwrite=False, nb_jobs=1):
    paths_img = sorted(glob.glob(path_images))

    _wrap_convert = partial(convert_image,
                            level=level,
                            level_shift=level_shift,
                            overwrite=overwrite)

    list(wrap_execute_sequence(_wrap_convert, paths_img,
                               desc='converting images', nb_jobs=nb_jobs))

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    arg_params = arg_parse_params()
    main(arg_params['path_images'], arg_params['level'], arg_params['level_offset'],
         arg_params['overwrite'], arg_params['nb_jobs'])

    logging.info('DONE')

max_stars_repo_name: esgomezm/BIRL

max_issues_repo_name: esgomezm/BIRL

max_forks_repo_name: esgomezm/BIRL


text:
"""Module to calculate document frequency of J-MFD words and moral foundations.
"""
import os
from pathlib import Path
from collections import Counter
from typing import NamedTuple
import errno

import numpy as np
import pandas as pd
import MeCab
import ipadic

from jmfdtk.utils import _display, _display_md
from jmfdtk.jmfdloader import load_jmfd
from jmfdtk.khcodegen import _generate
from jmfdtk import JMFD_PATH

# J-MFD words which are treated as compound.
# These words are compared with 2-grams and 3-grams.
_, _, JMFD_COMPOUND = _generate(jmfd_path=JMFD_PATH, exclude_general=True)

# Target POS.
# '助動詞' is for '母親らしい' in J-MFD and '助動詞' terms will be removed after morphological analysis.
TARGET_POS = ['名詞', '動詞', '形容詞', '副詞', '助動詞']

class CountData(NamedTuple):
    """Class to store counting results.
    """

    dfmfw: dict
    dfmf: pd.DataFrame
    cumfw: pd.DataFrame
    foundation: tuple

    def show_dfmfw(self):
        for foundation, df in self.dfmfw.items():
            _display_md('### Document frequency: {}'.format(foundation))
            _display(df)

    def show_dfmf(self):
        _display_md('### Document frequency of moral foundation')
        _display(self.dfmf)

    def describe_cumfw(self):
        _display_md('### Descriptibe statistics of unique moral foundation word count')
        _display(self.cumfw[self.foundation].describe())

def _load_jmfwords(jmfd_path, jmfd_excluded):
    """Loads moral foundation words and stems from J-MFD.
    """

    mfd, _ = load_jmfd(jmfd_path)

    # Removes MoralityGeneral foundation.
    mfd = mfd[~(mfd['foundation'] == 'MoralityGeneral')]

    # Removes specified words.
    if len(jmfd_excluded) > 0:
        mfd = mfd[~mfd['word'].isin(jmfd_excluded)]

    mfword = {}
    mfstem = {}
    _display_md('### Loaded JMFD words')
    for foundation, g in mfd.groupby('foundation'):
        mfstem[foundation] = g[g['stem']]['word'].tolist()
        mfword[foundation] = g[~g['stem']]['word'].tolist()
        print('{} ==> stem:{} word:{}'.format(foundation, len(mfstem[foundation]),
                                              len(mfword[foundation])))
    print()

    return mfword, mfstem

def _get_ngram(words, n):
    return [''.join(words[idx:idx + n]) for idx in range(len(words) - n + 1)]

def _find_compound(morphe, ngram, n):
    """Finds compound moral foundation words.
    """

    mfd_words = list(set(ngram).intersection(set(JMFD_COMPOUND)))

    if len(mfd_words) == 0:
        return morphe

    # Updates current morpheme list.
    for w in mfd_words:
        # Searches for indices of a word which matches n-gram words.
        indices = [i for i, x in enumerate(ngram) if w in x]

        for idx in indices:
            # Replaces the current word with a word from MFD.
            morphe[idx][0] = w

            for i in range(1, n):
                # Sets '*' for a morpheme which will be removed.
                morphe[idx+i][0] = '*'

    # Removes morphemes with '*'.
    return [m for m in morphe if m[0] != '*']

def _segment(m, texts, stop_list):
    """Morphological analysis.
    """

    low = []
    for t in texts:
        node = m.parseToNode(t)
        morphe = []
        while node:
            feature_split = node.feature.split(',')
            pos = feature_split[0]

            if pos in TARGET_POS:
                # When '*' is set as base form, the surface will be used instead.
                base_form = node.surface if feature_split[6] == '*' else feature_split[6]
                if base_form not in stop_list:
                    morphe.append([base_form, pos])

            node = node.next

        # Searches for moral foundation words which are segmented into multiple tokens.
        if len(morphe) >= 2:
            # Searches from bi-gram.
            words = [m[0] for m in morphe]
            bigram = _get_ngram(words, 2)
            morphe = _find_compound(morphe, bigram, 2)

            if len(morphe) >= 3:
                # Searches from tri-gram.
                trigram = _get_ngram(words, 3)
                morphe = _find_compound(morphe, trigram, 3)

        # Removes '助動詞'.
        words = [m[0] for m in morphe if m[1] != '助動詞']

        low.append(words)

    return low

def _get_mfword(words, mfs, stems):
    """Gets unique moral foundation words. Stems are treated as word.
    """

    # Moral foundation word list.
    mfl = mfs + stems
    mfw = list(set(words).intersection(set(mfl)))
    return Counter(mfw)

def _count(low, mfword, mfstem, texts):
    """Counts documents which contain moral foundation words.
    """

    foundations = mfword.keys()
    unm_foundation = len(mfword)

    # Document frequency of moral foundation words.
    dfmfw = {}
    for f in foundations:
        dfmfw[f] = Counter()

    # Appearance of moral foundation words.
    amfw = np.empty((0, unm_foundation), int)

    # Counts of unique moral foundation word in each document.
    cumfw = np.empty((0, unm_foundation), int)

    num_doc = len(low)
    for i, words in enumerate(low):
        print('{}/{}\r'.format(i, num_doc), end='')
        row = np.zeros(unm_foundation, int)
        row_prop = np.zeros(unm_foundation, int)
        for idx, (foundation, mfs, stems) in enumerate(zip(foundations, mfword.values(),
                                                       mfstem.values())):
            counter_res = _get_mfword(words, mfs, stems)
            dfmfw[foundation] += counter_res

            soc = sum(counter_res.values())
            row[idx] = 1 if soc > 0 else 0
            row_prop[idx] = soc

        amfw = np.append(amfw, row)
        cumfw = np.append(cumfw, row_prop)

    # Stores document frequency of moral foundation words in dict of DataFrame.
    dict_dfmfw = {}
    for foundation in foundations:
        # Sorts by key and then value
        dfmfw_sorted = dict(sorted(sorted(dfmfw[foundation].items()), key=lambda x: x[1],
                                   reverse=True))
        df = pd.DataFrame({
            'word': dfmfw_sorted.keys(),
            'count': dfmfw_sorted.values(),
            'percent': [v / num_doc * 100 for v in dfmfw_sorted.values()]})
        dict_dfmfw[foundation] = df

    # Stores document frequency of moral foundations in DataFrame.
    df_amfw = pd.DataFrame(amfw.reshape([-1, unm_foundation]), columns=foundations)
    sr_dfmf = df_amfw.sum()
    sr_dfmf.name = 'count'
    df_dfmf = sr_dfmf.to_frame()
    df_dfmf['percent'] = df_dfmf['count'] / num_doc * 100

    # Stores counts of unique moral foundation word and texts in DataFrame.
    df_cumfw = pd.DataFrame(cumfw.reshape([-1, unm_foundation]), columns=foundations)
    df_cumfw['sum'] = df_cumfw.sum(axis=1)
    df_cumfw['texts'] = texts

    # Removes data without moral foundation words.
    df_cumfw = df_cumfw[df_cumfw['sum'] > 0]
    df_cumfw.drop(columns=['sum'], inplace=True)

    print('Counting completed.')
    return CountData(dict_dfmfw, df_dfmf, df_cumfw, foundations)

def jmfcounter(texts, stop_list=[], jmfd_path=None, jmfd_excluded=[], dicdir=None):
    """
    Args:
        texts (list): List of list of word.
        stop_list (list, optional): Stop words. Defaults to [].
        jmfd_path (str, optional): Path of J-MFD file. Defaults to None.
        jmfd_excluded (list, optional): Excluded words in J-MFD. Defaults to [].
        dicdir (str, optional): Path of MeCab dictionary. Defaults to None.

    Returns:
        CountData containing results of J-MFD word counting.

    Raises:
        FileNotFoundError: Nonexistence of MeCab dictionary directory.
        FileNotFoundError: Nonexistent path of JMFD file.
    """

    print('Document number: {}'.format(len(texts)))
    if len(stop_list) > 0:
        _display_md('### Stop words')
        print(stop_list)
    if len(jmfd_excluded) > 0:
        _display_md('### Excluded moran foundation words')
        print(jmfd_excluded)

    if dicdir:
        dic = Path(dicdir)
        if not dic.is_dir():
            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), dic)

        m = MeCab.Tagger('-d ' + dicdir + '-r /dev/null')
    else:
        m = MeCab.Tagger(ipadic.MECAB_ARGS)

    m.parse('')

    if jmfd_path is None:
        jmfd_path = JMFD_PATH
    else:
        jmfd = Path(jmfd_path)
        if not jmfd.is_file():
            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), jmfd_path)

    mfword, mfstem = _load_jmfwords(jmfd_path, jmfd_excluded)
    print('Loading J-MFD completed.')

    low = _segment(m, texts, stop_list)
    print('Segmentation completed.')

    return _count(low, mfword, mfstem, texts)

max_stars_repo_name: irukafe/jmfdtk

max_issues_repo_name: irukafe/jmfdtk

max_forks_repo_name: irukafe/jmfdtk


text:
from math import gcd
def gcd_mult(numbers):
    numbers=set(numbers)
    if len(numbers)==1:
        return numbers.pop()
    elif len(numbers)==2:
        return gcd(*numbers)
    else:
        return gcd_mult(set(gcd(*pair) for pair in combinations(numbers,2)))

from itertools import combinations
def combinationsDif_meanVar(Usc,Xc):
    """
    Compares each _deep_row_ of array X,
    and returns mean variance in the difference of rows
    Xc has shape ( Tcriterions, time, components )
    """
    return np.array([(square(Usc)@(xi-xj).transpose()).var() for xi,xj in combinations(Xc,2)]).mean()

from scipy.signal import decimate
def downscale(X, outSize):
    """
    Downscales the columns of X,
    returns newX of shape (outSize, X.shape[1])
    """
    downFactor =  X.shape[0]/outSize
    assert downFactor == int(downFactor)
    return decimate(X.transpose(),int(downFactor),n=2).transpose()

def square(matrix):
    n = int(sqrt(len(matrix)))
    return matrix.reshape(n,n)

from numpy.linalg import slogdet
from numpy import sqrt
def log_det(flat_array):

    sign, value = slogdet(square(flat_array))
    return sign*value

from scipy.optimize import minimize
def minimize_differences(Xc):
    constrain = ({'type': 'eq',
                  'fun' : log_det,
                  })
    return minimize(lambda U: combinationsDif_meanVar(U,Xc), np.eye(Xc.shape[2]).ravel(),constraints=constrain,method='SLSQP',jac=False)

from sklearn.decomposition import PCA
class SCA():
    def __init__(self,n_components=3,new_size=None,pca_to_use=None):
        self.new_size = new_size
        self.n_components = n_components
        self.pca_to_use = n_components if pca_to_use is None else pca_to_use

    def _norm(self, X_list, down=True):
        pca = PCA(n_components=self.pca_to_use)
        if down:
            return [downscale(pca.fit_transform(X), self.new_size) for X in X_list]
        else:
            return [pca.fit_transform(X) for X in X_list]
    def fit(self, X_list, y=None):
        """
        Each X in X_list has the form (time, features)
        """
        if self.new_size is None:
            sizes = set(X.shape[0] for X in X_list)
            self.new_size = gcd_mult(sizes)

        res = square(minimize_differences(np.stack(self._norm(X_list) ) ).x )
        self.components_ = res[np.argsort((res**2).sum(axis=1))[::-1]][:self.n_components,:]
        return self

    def transform(self, X_list, y=None):
        return [(self.components_@X.transpose()).transpose() for X in self._norm(X_list,down=False)]

    def fit_transform(self, Xc, y=None):
        return self.fit(Xc).transform(Xc)

max_stars_repo_name: EstevaoVieira/spikelearn

max_issues_repo_name: EstevaoVieira/spikelearn

max_forks_repo_name: EstevaoVieira/spikelearn

reason: from numpy,from scipy


text:
"""
Layer definitions
"""

from initializers import ortho_weight, norm_weight
import tensorflow as tf
import numpy

def matmul3d(x3d, matrix):
    shape = tf.shape(x3d)
    mat_shape = tf.shape(matrix)
    x2d = tf.reshape(x3d, [shape[0]*shape[1], shape[2]])
    result2d = tf.matmul(x2d, matrix)
    result3d = tf.reshape(result2d, [shape[0], shape[1], mat_shape[1]])
    return result3d

def apply_dropout_mask(x, mask, input_is_3d=False):
    if mask == None:
        return x
    if input_is_3d:
        mask_3d = tf.expand_dims(mask, 0)
        mask_3d = tf.tile(mask_3d, [tf.shape(x)[0], 1, 1])
        return tf.multiply(x, mask_3d)
    else:
        return tf.multiply(x, mask)

class FeedForwardLayer(object):
    def __init__(self,
                 in_size,
                 out_size,
                 batch_size,
                 non_linearity=tf.nn.tanh,
                 W=None,
                 use_layer_norm=False,
                 dropout_input=None):
        if W is None:
            W = tf.Variable(norm_weight(in_size, out_size), name='W')
        self.W = W
        self.b = tf.Variable(numpy.zeros((out_size,)).astype('float32'), name='b')
        self.non_linearity = non_linearity
        self.use_layer_norm = use_layer_norm
        if use_layer_norm:
            self.layer_norm = LayerNormLayer(layer_size=out_size)
        # Create a dropout mask for input values (reused at every timestep).
        if dropout_input == None:
            self.dropout_mask = None
        else:
            ones = tf.ones([batch_size, in_size])
            self.dropout_mask = dropout_input(ones)

    def forward(self, x, input_is_3d=False):
        x = apply_dropout_mask(x, self.dropout_mask, input_is_3d)
        if input_is_3d:
            y = matmul3d(x, self.W) + self.b
        else:
            y = tf.matmul(x, self.W) + self.b
        if self.use_layer_norm:
            y = self.layer_norm.forward(y, input_is_3d=input_is_3d)
        y = self.non_linearity(y)
        return y

class EmbeddingLayer(object):
    def __init__(self,
                 vocabulary_size,
                 embedding_size):
        self.embeddings = tf.Variable(norm_weight(vocabulary_size, embedding_size),
                                      name='embeddings')

    def forward(self, x):
        embs = tf.nn.embedding_lookup(self.embeddings, x)
        return embs

    def get_embeddings(self):
        return self.embeddings

class EmbeddingLayerWithFactors(object):
    def __init__(self,
                 vocabulary_size,
                 dim_per_factor):
        self.embedding_matrices = [
            tf.Variable(norm_weight(vocabulary_size, dim), name='embeddings')
            for dim in dim_per_factor]

    def forward(self, x):
        # Assumes that x has shape: factors, ...
        embs = [tf.nn.embedding_lookup(matrix, x[i])
                for i, matrix in enumerate(self.embedding_matrices)]
        return tf.concat(embs, axis=-1)

class RecurrentLayer(object):
    def __init__(self,
                 initial_state,
                 step_fn):
        self.initial_state = initial_state
        self.step_fn = step_fn

    def forward(self, x):
        # Assumes that x has shape: time, batch, ...
        states = tf.scan(fn=self.step_fn,
                         elems=x,
                         initializer=self.initial_state)
        return states

class LayerNormLayer(object):
    def __init__(self,
                 layer_size,
                 eps=1e-5):
        #TODO: If nematus_compat is true, then eps must be 1e-5!
        new_mean = numpy.zeros(shape=[layer_size], dtype=numpy.float32)
        self.new_mean = tf.Variable(new_mean,
                                    dtype=tf.float32,
                                    name='new_mean')
        new_std = numpy.ones(shape=[layer_size], dtype=numpy.float32)
        self.new_std = tf.Variable(new_std,
                                   dtype=tf.float32,
                                   name='new_std')
        self.eps = eps
    def forward(self, x, input_is_3d=False):
        # NOTE: tf.nn.moments does not support axes=[-1] or axes=[tf.rank(x)-1] :-(
        # TODO: Actually, this is probably fixed now and should be tested with latest
        # TF version. See: https://github.com/tensorflow/tensorflow/issues/8101
        axis = 2 if input_is_3d else 1
        m, v = tf.nn.moments(x, axes=[axis], keep_dims=True)
        std = tf.sqrt(v + self.eps)
        norm_x = (x-m)/std
        new_x = norm_x*self.new_std + self.new_mean
        return new_x

class GRUStep(object):
    def __init__(self,
                 input_size,
                 state_size,
                 batch_size,
                 use_layer_norm=False,
                 nematus_compat=False,
                 dropout_input=None,
                 dropout_state=None):
        self.state_to_gates = tf.Variable(
                                numpy.concatenate(
                                    [ortho_weight(state_size),
                                     ortho_weight(state_size)],
                                    axis=1),
                                name='state_to_gates')
        self.input_to_gates = tf.Variable(
                                numpy.concatenate(
                                    [norm_weight(input_size, state_size),
                                     norm_weight(input_size, state_size)],
                                    axis=1),
                                name='input_to_gates')
        self.gates_bias = tf.Variable(
                            numpy.zeros((2*state_size,)).astype('float32'),
                            name='gates_bias')

        self.state_to_proposal = tf.Variable(
                                    ortho_weight(state_size),
                                    name = 'state_to_proposal')
        self.input_to_proposal = tf.Variable(
                                    norm_weight(input_size, state_size),
                                    name = 'input_to_proposal')
        self.proposal_bias = tf.Variable(
                                    numpy.zeros((state_size,)).astype('float32'),
                                    name='proposal_bias')
        self.nematus_compat = nematus_compat
        self.use_layer_norm = use_layer_norm

        if self.use_layer_norm:
            with tf.name_scope('gates_x_norm'):
                self.gates_x_norm = LayerNormLayer(2*state_size)
            with tf.name_scope('gates_state_norm'):
                self.gates_state_norm = LayerNormLayer(2*state_size)
            with tf.name_scope('proposal_x_norm'):
                self.proposal_x_norm = LayerNormLayer(state_size)
            with tf.name_scope('proposal_state_norm'):
                self.proposal_state_norm = LayerNormLayer(state_size)

        # Create dropout masks for input values (reused at every timestep).
        if dropout_input == None:
            self.dropout_mask_input_to_gates = None
            self.dropout_mask_input_to_proposal = None
        else:
            ones = tf.ones([batch_size, input_size])
            self.dropout_mask_input_to_gates = dropout_input(ones)
            self.dropout_mask_input_to_proposal = dropout_input(ones)

        # Create dropout masks for state values (reused at every timestep).
        if dropout_state == None:
            self.dropout_mask_state_to_gates = None
            self.dropout_mask_state_to_proposal = None
        else:
            ones = tf.ones([batch_size, state_size])
            self.dropout_mask_state_to_gates = dropout_state(ones)
            self.dropout_mask_state_to_proposal = dropout_state(ones)

    def _get_gates_x(self, x, input_is_3d=False):
        x = apply_dropout_mask(x, self.dropout_mask_input_to_gates, input_is_3d)
        if input_is_3d:
            gates_x = matmul3d(x, self.input_to_gates)
        else:
            gates_x = tf.matmul(x, self.input_to_gates)
        if not self.nematus_compat:
            gates_x += self.gates_bias
        if self.use_layer_norm:
            gates_x = self.gates_x_norm.forward(gates_x, input_is_3d=input_is_3d)
        return gates_x

    def _get_gates_state(self, prev_state):
        prev_state = apply_dropout_mask(prev_state,
                                        self.dropout_mask_state_to_gates)
        gates_state = tf.matmul(prev_state, self.state_to_gates)
        if self.nematus_compat:
            gates_state += self.gates_bias
        if self.use_layer_norm:
            gates_state = self.gates_state_norm.forward(gates_state)
        return gates_state

    def _get_proposal_x(self,x, input_is_3d=False):
        x = apply_dropout_mask(x, self.dropout_mask_input_to_proposal,
                               input_is_3d)
        if input_is_3d:
            proposal_x = matmul3d(x, self.input_to_proposal)
        else:
            proposal_x = tf.matmul(x, self.input_to_proposal)
        if not self.nematus_compat:
            proposal_x += self.proposal_bias
        if self.use_layer_norm:
            proposal_x = self.proposal_x_norm.forward(proposal_x, input_is_3d=input_is_3d)
        return proposal_x

    def _get_proposal_state(self, prev_state):
        prev_state = apply_dropout_mask(prev_state,
                                        self.dropout_mask_state_to_proposal)
        proposal_state = tf.matmul(prev_state, self.state_to_proposal)
        # placing the bias here is unorthodox, but we're keeping this behavior for compatibility with dl4mt-tutorial
        if self.nematus_compat:
            proposal_state += self.proposal_bias
        if self.use_layer_norm:
            proposal_state = self.proposal_state_norm.forward(proposal_state)
        return proposal_state

    def precompute_from_x(self, x):
        # compute gates_x and proposal_x in one big matrix multiply
        # if x is fully known upfront
        # this method exists only for efficiency reasons

        gates_x = self._get_gates_x(x, input_is_3d=True)
        proposal_x = self._get_proposal_x(x, input_is_3d=True)
        return gates_x, proposal_x

    def forward(self,
                prev_state,
                x=None,
                gates_x=None,
                gates_state=None,
                proposal_x=None,
                proposal_state=None):
        if gates_x is None:
            gates_x = self._get_gates_x(x)
        if proposal_x is None:
            proposal_x = self._get_proposal_x(x)
        if gates_state is None:
            gates_state = self._get_gates_state(prev_state)
        if proposal_state is None:
            proposal_state = self._get_proposal_state(prev_state)

        gates = gates_x + gates_state
        gates = tf.nn.sigmoid(gates)
        read_gate, update_gate = tf.split(gates,
                                          num_or_size_splits=2,
                                          axis=1)

        proposal = proposal_state*read_gate + proposal_x
        proposal = tf.tanh(proposal)
        new_state = update_gate*prev_state + (1-update_gate)*proposal

        return new_state

class AttentionStep(object):
    def __init__(self,
                 context,
                 context_state_size,
                 context_mask,
                 state_size,
                 hidden_size,
                 use_layer_norm=False,
                 dropout_context=None,
                 dropout_state=None):
        self.state_to_hidden = tf.Variable(
                                norm_weight(state_size, hidden_size),
                                name='state_to_hidden')
        self.context_to_hidden = tf.Variable( #TODO: Nematus uses ortho_weight here - important?
                                    norm_weight(context_state_size, hidden_size),
                                    name='context_to_hidden')
        self.hidden_bias = tf.Variable(
                            numpy.zeros((hidden_size,)).astype('float32'),
                            name='hidden_bias')
        self.hidden_to_score = tf.Variable(
                                norm_weight(hidden_size, 1),
                                name='hidden_to_score')
        self.use_layer_norm = use_layer_norm
        if self.use_layer_norm:
            with tf.name_scope('hidden_context_norm'):
                self.hidden_context_norm = LayerNormLayer(layer_size=hidden_size)
            with tf.name_scope('hidden_state_norm'):
                self.hidden_state_norm = LayerNormLayer(layer_size=hidden_size)
        self.context = context
        self.context_mask = context_mask

        batch_size = tf.shape(context)[1]

        # Create a dropout mask for context values (reused at every timestep).
        if dropout_context == None:
            self.dropout_mask_context_to_hidden = None
        else:
            ones = tf.ones([batch_size, context_state_size])
            self.dropout_mask_context_to_hidden = dropout_context(ones)

        # Create a dropout mask for state values (reused at every timestep).
        if dropout_state == None:
            self.dropout_mask_state_to_hidden = None
        else:
            ones = tf.ones([batch_size, state_size])
            self.dropout_mask_state_to_hidden = dropout_state(ones)

        # precompute these activations, they are the same at each step
        # Ideally the compiler would have figured out that too
        context = apply_dropout_mask(context,
                                     self.dropout_mask_context_to_hidden, True)
        self.hidden_from_context = matmul3d(context, self.context_to_hidden)
        self.hidden_from_context += self.hidden_bias
        if self.use_layer_norm:
            self.hidden_from_context = \
                self.hidden_context_norm.forward(self.hidden_from_context, input_is_3d=True)

    def forward(self, prev_state):
        prev_state = apply_dropout_mask(prev_state,
                                        self.dropout_mask_state_to_hidden)
        hidden_from_state = tf.matmul(prev_state, self.state_to_hidden)
        if self.use_layer_norm:
            hidden_from_state = \
                self.hidden_state_norm.forward(hidden_from_state, input_is_3d=False)
        hidden = self.hidden_from_context + hidden_from_state
        hidden = tf.nn.tanh(hidden)
        # context has shape seqLen x batch x context_state_size
        # mask has shape seqLen x batch

        scores = matmul3d(hidden, self.hidden_to_score) # seqLen x batch x 1
        scores = tf.squeeze(scores, axis=2)
        scores = scores - tf.reduce_max(scores, axis=0, keep_dims=True)
        scores = tf.exp(scores)
        scores *= self.context_mask
        scores = scores / tf.reduce_sum(scores, axis=0, keep_dims=True)

        attention_context = self.context * tf.expand_dims(scores, axis=2)
        attention_context = tf.reduce_sum(attention_context, axis=0, keep_dims=False)

        return attention_context

class Masked_cross_entropy_loss(object):
    def __init__(self,
                 y_true,
                 y_mask):
        self.y_true = y_true
        self.y_mask = y_mask

    def forward(self, logits):
        cost = tf.nn.sparse_softmax_cross_entropy_with_logits(
                labels=self.y_true,
                logits=logits)
        #cost has shape seqLen x batch
        cost *= self.y_mask
        cost = tf.reduce_sum(cost, axis=0, keep_dims=False)
        return cost

class PReLU(object):
    def __init__(self,
                 in_size,
                 initial_slope = 1.0):

        self.slope = tf.Variable(initial_slope * numpy.ones((in_size,)).astype('float32'), name='slope')

    def forward(self, x):
        pos = tf.nn.relu(x)
        neg = x - pos
        y = pos + self.slope * neg
        return y


text:
import pandas as pd
import numpy as np
import mayavi as mayax
from mayavi import mlab
from functools import partial
import tables as tb
import sys
sys.path.append("../../")
from focusadd.coils.CoilSet import CoilSet

def read_lhd_data():
	r_surf = np.load("../../focusadd/initFiles/lhd/lhd_r_surf.npy")
	nn = np.load("../../focusadd/initFiles/lhd/lhd_nn_surf.npy")
	sg = np.load("../../focusadd/initFiles/lhd/lhd_sg_surf.npy")
	fc_init = np.load("../../focusadd/initFiles/lhd/lhd_fc.npy")
	return r_surf, nn, sg, fc_init

def plot_surface(r):
	x = r[:,:,0]
	y = r[:,:,1]
	z = r[:,:,2]
	p = mlab.mesh(x,y,z,color=(0.8,0.0,0.0))
	return p
def get_coil_data():
	NC = 2
	NS = 96
	NFC = 10
	NFR = 3
	ln = 0.07
	lb = 0.07
	NNR = 2
	NBR = 2
	rc = 2.0
	NR = 0
	return NC, NS, NFC, NFR, ln, lb, NNR, NBR, rc, NR
def get_all_coil_data(filename):
	with tb.open_file(filename, "r") as f:
		coil_data = f.root.metadata[0]
		fc = np.asarray(f.root.coilSeries[:, :, :])
		fr = np.asarray(f.root.rotationSeries[:, :]) # NEEDS TO BE EDITED
		params = (fc, fr)
	return coil_data, params

def compute_r_centroid(NS, fc):
	theta = np.linspace(0, 2 * np.pi, NS + 1)
	NC = fc.shape[1]
	NF = fc.shape[2]
	xc = fc[0]
	yc = fc[1]
	zc = fc[2]
	xs = fc[3]
	ys = fc[4]
	zs = fc[5]
	x = np.zeros((NC, NS + 1))
	y = np.zeros((NC, NS + 1))
	z = np.zeros((NC, NS + 1))
	for m in range(NF):
		arg = m * theta
		carg = np.cos(arg)
		sarg = np.sin(arg)
		x += (
			xc[:, np.newaxis, m] * carg[np.newaxis, :]
			+ xs[:, np.newaxis, m] * sarg[np.newaxis, :]
		)
		y += (
			yc[:, np.newaxis, m] * carg[np.newaxis, :]
			+ ys[:, np.newaxis, m] * sarg[np.newaxis, :]
		)
		z += (
			zc[:, np.newaxis, m] * carg[np.newaxis, :]
			+ zs[:, np.newaxis, m] * sarg[np.newaxis, :]
		)
	return np.concatenate(
		(x[:, :, np.newaxis], y[:, :, np.newaxis], z[:, :, np.newaxis]), axis=2
	)

def plot_coils_centroid(r_coils, color=(0.0, 0.0, 0.8)):
	r_coils = np.concatenate((r_coils[:, :, :, :, :], r_coils[:, 0:1, :, :, :]),axis=1)
	for ic in range(r_coils.shape[0]):
		p = mlab.plot3d(r_coils[ic,:,0], r_coils[ic,:,1], r_coils[ic,:,2], tube_radius=0.02, color = color)#, line_width = 0.01,)
	return p
def plot_coils(r_coils, color=(0.0, 0.0, 0.8)):
	r_coils = np.concatenate((r_coils[:, :, :, :, :], r_coils[:, 0:1, :, :, :]),axis=1)
	for ic in range(r_coils.shape[0]):
		for n in range(r_coils.shape[2]):
			for b in range(r_coils.shape[3]):
				p = mlab.plot3d(r_coils[ic,:,n,b,0], r_coils[ic,:,n,b,1], r_coils[ic,:,n,b,2], tube_radius=0.02, color = color)#, line_width = 0.01,)
	return p

mlab.clf()
NS = 96
r_surf, nn, sg, fc_init = read_lhd_data()
coil_data, params = get_all_coil_data("../../tests/lhd/scan/lhd_l4.hdf5")
#coil_data_fb, params_fb = get_all_coil_data("fb_lhd_test.hdf5")
r_init = compute_r_centroid(NS, fc_init)
_, _, r_c, _ = CoilSet.get_outputs(coil_data, params)
#_, _, r_fb, _ = CoilSet.get_outputs(coil_data_fb, params_fb)
mlab.options.offscreen = True
mlab.figure(size=(2400,2400), bgcolor=(1,1,1))

p = plot_surface(r_surf)
#p = plot_coils_centroid(r_init,color=(0.0,0.0,0.8))
p = plot_coils(r_c,color=(0.0,0.0,0.8))
#p = plot_coils(r_fb,color=(0.0,0.0,0.8))
mlab.view(azimuth=0.0, elevation=180, distance=20.0, focalpoint=(0.0,0.0,0.0), roll=None, reset_roll=True, figure=None)
mlab.savefig('lhd_coils.png', figure=mlab.gcf())

max_stars_repo_name: smiet/FOCUSADD

max_issues_repo_name: smiet/FOCUSADD

max_forks_repo_name: smiet/FOCUSADD


text:
from torch.utils.data import Dataset,DataLoader
from data_classes import PointCloud, Box
from pyquaternion import Quaternion
import numpy as np
import pandas as pd
import os
import torch
import torch.nn.functional as F
from tqdm import tqdm
import kitty_utils as utils
from kitty_utils import *
from searchspace import KalmanFiltering
import logging
import time
import pickle
from voxel.voxel_grid import VoxelGrid
logger =logging.getLogger('global')

class kittiDataset():

    def __init__(self, path,which_dataset):
        self.which_dataset=which_dataset
        self.KITTI_Folder = path
        self.KITTI_velo = os.path.join(self.KITTI_Folder, "velodyne")
        self.KITTI_label = os.path.join(self.KITTI_Folder, "label_02")

    def getSceneID(self, split):
        if self.which_dataset.upper() == 'NUSCENES':
            if "TRAIN" in split.upper():  # Training SET
                if "TINY" in split.upper():
                    sceneID = [0]
                else:
                    sceneID = list(range(0, 350))
            elif "VALID" in split.upper():  # Validation Set
                if "TINY" in split.upper():
                    sceneID = [0]
                else:
                    sceneID = list(range(0, 10))
            elif "TEST" in split.upper():  # Testing Set
                if "TINY" in split.upper():
                    sceneID = [0]
                else:
                    sceneID = list(range(0, 150))
        elif self.which_dataset.upper() == 'KITTI':
            if "TRAIN" in split.upper():  # Training SET
                if "TINY" in split.upper():
                    sceneID = [12]
                else:
                    sceneID = list(range(0, 17))
            elif "VALID" in split.upper():  # Validation Set
                if "TINY" in split.upper():
                    sceneID = [18]
                else:
                    sceneID = list(range(17, 19))
            elif "TEST" in split.upper():  # Testing Set
                if "TINY" in split.upper():
                    sceneID = [19]
                else:
                    sceneID = list(range(19, 21))

            else:  # Full Dataset
                sceneID = list(range(21))
        return sceneID
    def getBBandPC(self, anno):
        calib_path = os.path.join(self.KITTI_Folder, 'calib',
                                  anno['scene'] + ".txt")
        calib = self.read_calib_file(calib_path)
        transf_mat = np.vstack((calib["Tr_velo_cam"], np.array([0, 0, 0, 1])))
        PC, box = self.getPCandBBfromPandas(anno, transf_mat)
        return PC, box

    def getListOfAnno(self, sceneID, category_name="Car"):
        list_of_scene = [
            path for path in os.listdir(self.KITTI_velo)
            if os.path.isdir(os.path.join(self.KITTI_velo, path)) and
            int(path) in sceneID
        ]
        list_of_scene=sorted(list_of_scene)
        print(list_of_scene)
        list_of_tracklet_anno = []
        for scene in list_of_scene:

            label_file = os.path.join(self.KITTI_label, scene + ".txt")
            if self.which_dataset.upper()=='NUSCENES':
                df = pd.read_csv(
                    label_file,
                    sep=' ',
                    names=[
                        "frame", "track_id", "type", "truncated", "occluded",
                        "alpha", "bbox_left", "bbox_top", "bbox_right",
                        "bbox_bottom", "height", "width", "length", "x", "y", "z",
                        "rotation_y","score",'num_lidar_pts','is_key_frame'
                    ])
            else:
                df = pd.read_csv(
                    label_file,
                    sep=' ',
                    names=[
                        "frame", "track_id", "type", "truncated", "occluded",
                        "alpha", "bbox_left", "bbox_top", "bbox_right",
                        "bbox_bottom", "height", "width", "length", "x", "y", "z",
                        "rotation_y", "score"
                    ])
            df = df[df["type"] == category_name]
            df.insert(loc=0, column="scene", value=scene)
            for track_id in df.track_id.unique():
                df_tracklet = df[df["track_id"] == track_id]
                df_tracklet = df_tracklet.reset_index(drop=True)

                tracklet_anno = [anno for index, anno in df_tracklet.iterrows()]
                list_of_tracklet_anno.append(tracklet_anno)

        return list_of_tracklet_anno

    def getPCandBBfromPandas(self, box, calib):
        center = [box["x"], box["y"]- box["height"] / 2 , box["z"]]
        size = [box["width"], box["length"], box["height"]]

        orientation = Quaternion(
            axis=[0, 1, 0], radians=box["rotation_y"]) * Quaternion(
                axis=[1, 0, 0], radians=np.pi / 2)
        BB = Box(center, size, orientation)

        try:
            # VELODYNE PointCloud
            velodyne_path = os.path.join(self.KITTI_velo, box["scene"],
                                         '{:06}.bin'.format(box["frame"]))
            PC = PointCloud(
                np.fromfile(velodyne_path, dtype=np.float32).reshape(-1, 4).T)
            PC.transform(calib)
        except :
            # in case the Point cloud is missing
            # (0001/[000177-000180].bin)
            PC = PointCloud(np.array([[0, 0, 0]]).T)

        return PC, BB

    def read_calib_file(self, filepath):
        """Read in a calibration file and parse into a dictionary."""
        data = {}
        with open(filepath, 'r') as f:
            for line in f.readlines():
                values = line.split()
                # The only non-float values in these files are dates, which
                # we don't care about anyway
                try:
                    ind = values[0].find(':')
                    if ind != -1:
                        data[values[0][:ind]] = np.array(
                            [float(x) for x in values[1:]]).reshape(3, 4)
                    else:
                        data[values[0]] = np.array(
                            [float(x) for x in values[1:]]).reshape(3, 4)
                except ValueError:
                    data[values[0]] = np.array(
                        [float(x) for x in values[1:]]).reshape(3, 3)
        return data

class SiameseDataset(Dataset):

    def __init__(self,
                 which_dataset,
                 input_size,
                 path,
                 split,
                 category_name="Car",
                 regress="GAUSSIAN",
                 offset_BB=0,
                 scale_BB=1.0,
                 ):

        self.dataset = kittiDataset(path=path,which_dataset=which_dataset)

        self.input_size = input_size
        self.split = split
        self.sceneID = self.dataset.getSceneID(split=split)
        self.getBBandPC = self.dataset.getBBandPC

        self.category_name = category_name
        self.regress = regress
        self.list_of_tracklet_anno = self.dataset.getListOfAnno(
            self.sceneID, category_name)
        self.list_of_anno = [
            anno for tracklet_anno in self.list_of_tracklet_anno
            for anno in tracklet_anno
        ]

    def isTiny(self):
        return ("TINY" in self.split.upper())

    def __getitem__(self, index):
        return self.getitem(index)

class SiameseTrain(SiameseDataset):

    def __init__(self,
                 which_dataset,
                 input_size,
                 path,
                 split="",
                 category_name="Car",
                 regress="GAUSSIAN",
                 sigma_Gaussian=1,
                 offset_BB=0,
                 scale_BB=1.0,
                 voxel_size=[0.2, 0.2, 0.2],
                 xy_size=[0.2, 0.2],
                 area_extents=[-5.6, 5.6, -3.6, 3.6, -2.4, 2.4],
                 xy_area_extents=[-5.6, 5.6, -3.6, 3.6],
                 downsample=1.0,
                 regress_radius=2,
                 num_T=10
                 ):
        super(SiameseTrain,self).__init__(
            which_dataset=which_dataset,
            input_size=input_size,
            path=path,
            split=split,
            category_name=category_name,
            regress=regress,
            offset_BB=offset_BB,
            scale_BB=scale_BB,
        )

        self.sigma_Gaussian = sigma_Gaussian
        self.offset_BB = offset_BB
        self.scale_BB = scale_BB
        self.voxel_size=np.array(voxel_size)
        self.xy_size=np.array(xy_size)*downsample
        self.area_extents = np.array(area_extents).reshape(3, 2)
        self.xy_area_extents = np.array(xy_area_extents).reshape(2, 2)
        self.regress_radius=regress_radius
        self.num_T=num_T
        voxel_extents_transpose = self.area_extents.copy().transpose()
        self.scene_ground = voxel_extents_transpose[0]
        self.voxel_grid_size = np.ceil(voxel_extents_transpose[1] / self.voxel_size) - np.floor(
            voxel_extents_transpose[0] / self.voxel_size)
        self.voxel_grid_size = self.voxel_grid_size.astype(np.int32)

        self.num_candidates_perframe = 4

        logging.info("preloading PC...")

        if which_dataset.upper()=="NUSCENES":
            dataset = 'val' if 'val' in self.split.lower() else 'train_track'
            preload_data_path = os.path.join('/opt/data/private_nas/nusc_kitti_all/', dataset,
                                         "preload_{}_{}_10.dat".format(self.category_name, self.split.lower()))
            print(preload_data_path)
        training_samples = []
        self.list_of_PCs = [None] * len(self.list_of_anno)
        self.list_of_BBs = [None] * len(self.list_of_anno)
        if which_dataset.upper() == "NUSCENES" and os.path.isfile(preload_data_path):
            print('loading from saved file {}.'.format(preload_data_path))
            with open(preload_data_path, 'rb') as f:
                training_samples = pickle.load(f)
            self.list_of_PCs = training_samples[0]
            self.list_of_BBs = training_samples[1]
        else:
            for index in tqdm(range(len(self.list_of_anno))):
                anno = self.list_of_anno[index]
                PC, box = self.getBBandPC(anno)
                new_PC = utils.cropPC(PC, box, offset=10)
                self.list_of_PCs[index] = new_PC
                self.list_of_BBs[index] = box

            if which_dataset.upper()=="NUSCENES":
                training_samples.append(self.list_of_PCs)
                training_samples.append(self.list_of_BBs)
                with open(preload_data_path, 'wb') as f:
                    print('saving loaded data to {}'.format(preload_data_path))
                    pickle.dump(training_samples, f)
        logging.info("PC preloaded!")

        logging.info("preloading Model..")
        self.model_PC = [None] * len(self.list_of_tracklet_anno)

        f=open('all_car_obeject_pc_num.txt','w')
        for i in tqdm(range(len(self.list_of_tracklet_anno))):
            list_of_anno = self.list_of_tracklet_anno[i]
            PCs = []
            BBs = []
            cnt = 0
            for anno in list_of_anno:
                this_PC, this_BB = self.getBBandPC(anno)

                PCs.append(this_PC)
                BBs.append(this_BB)

                anno["model_idx"] = i
                anno["relative_idx"] = cnt
                cnt += 1

            self.model_PC[i] = getModel(
                PCs, BBs, offset=self.offset_BB, scale=self.scale_BB)
            # pc=self.model_PC[i].points.copy()
            # pc_symmetry=pc.copy()
            # pc_symmetry[1,:]=-pc_symmetry[1,:]
            # pc=np.concatenate((pc,pc_symmetry),axis=1)
            # pc=self.unique(pc.T)
            # model_pc=PointCloud(pc)
            # self.model_PC_symmetry[i]=model_pc

        logging.info("Model preloaded!")
    #
    def unique_pc(self,pc):
        #pc:nx3
        order = np.lexsort(pc.T)
        pc = pc[order]
        diff = np.diff(pc, axis=0)
        ui = np.ones(len(pc), 'bool')
        ui[1:] = (diff != 0).any(axis=1)
        return pc[ui].T

    def __getitem__(self, index):
        return self.getitem(index)

    def getPCandBBfromIndex(self, anno_idx):
        this_PC = self.list_of_PCs[anno_idx]
        this_BB = self.list_of_BBs[anno_idx]
        return this_PC, this_BB

    def getitem(self, index):
        anno_idx = self.getAnnotationIndex(index)
        sample_idx = self.getSearchSpaceIndex(index)

        if sample_idx == 0:
            sample_offsets = np.zeros(4)
        else:
            gaussian = KalmanFiltering(bnd=[1, 1, 0.5, 5])
            sample_offsets = gaussian.sample(1)[0]

        this_anno = self.list_of_anno[anno_idx]
        model_idx=this_anno["model_idx"]
        this_PC, this_BB = self.getPCandBBfromIndex(anno_idx)
        sample_BB = utils.getOffsetBB(this_BB, sample_offsets)

        sample_PC, sample_label, sample_reg, new_box_gt,align_gt_PC = utils.cropAndCenterPC_label(
            this_PC,sample_BB, this_BB, sample_offsets, offset=self.offset_BB, scale=self.scale_BB,limit_area=self.area_extents)
        if sample_PC.nbr_points() <= 20:
            return self.getitem(np.random.randint(0, self.__len__()))

        sample_PC, sample_label, sample_reg,align_gt_PC,all_label = utils.regularizePCwithlabel(sample_PC,align_gt_PC,sample_label,sample_reg,self.input_size)

        if this_anno["relative_idx"] == 0:
            prev_idx = anno_idx
            fir_idx = anno_idx
        else:
            prev_idx = anno_idx - 1
            fir_idx = anno_idx - this_anno["relative_idx"]
        gt_PC_pre, gt_BB_pre = self.getPCandBBfromIndex(prev_idx)
        gt_PC_fir, gt_BB_fir = self.getPCandBBfromIndex(fir_idx)

        if sample_idx == 0:
            samplegt_offsets = np.zeros(4)
        else:
            samplegt_offsets = np.random.uniform(low=-0.3, high=0.3, size=4)
            samplegt_offsets[3] = samplegt_offsets[3]*5.0
        gt_BB_pre= utils.getOffsetBB(gt_BB_pre, samplegt_offsets)

        gt_PC = getModel([gt_PC_fir,gt_PC_pre], [gt_BB_fir,gt_BB_pre], offset=self.offset_BB, scale=self.scale_BB)

        if gt_PC.nbr_points() <= 20:
            return self.getitem(np.random.randint(0, self.__len__()))
        gt_PC = utils.regularizePC(gt_PC,self.input_size)

        completion_PC_part = utils.regularizePC(self.model_PC[model_idx],self.input_size*4)
        rot=Quaternion(axis=[0, 0, 1], angle=-sample_offsets[3] * np.pi / 180)
        #3 N
        completion_PC_part = completion_PC_part.transpose(0, 1).contiguous()
        completion_PC_part = completion_PC_part.numpy()
        completion_PC_part = np.dot(rot.rotation_matrix, completion_PC_part)
        completion_PC_part=torch.from_numpy(completion_PC_part).float()

        extents_transpose = self.xy_area_extents.transpose()
        if extents_transpose.shape != (2, 2):
            raise ValueError("Extents are the wrong shape {}".format(extents_transpose.shape))

        # Set image grid extents
        self.min_img_coord = np.floor(extents_transpose[0] / self.xy_size)
        self.max_img_coord = np.ceil(extents_transpose[1] / self.xy_size) - 1
        self.img_size=((self.max_img_coord - self.min_img_coord) + 1).astype(np.int32)
        output_h=self.img_size[1]
        output_w=self.img_size[0]
        #hm
        corners=new_box_gt.corners()[:2].T
        corners=corners[[0,2,4,6]]
        corners_int=np.floor((corners / self.xy_size) - self.min_img_coord).astype(np.int32)
        corners_int_ul=np.min(corners_int,axis=0)
        corners_int_br=np.max(corners_int,axis=0)
        x = np.arange(corners_int_ul[0], corners_int_br[0]+1, 1)
        y = np.arange(corners_int_ul[1], corners_int_br[1]+1, 1)
        xx,yy=np.meshgrid(x,y)
        hm_grid=np.concatenate([xx[:,:,np.newaxis],yy[:,:,np.newaxis]],axis=2)
        ct=sample_reg[0,:2].numpy()
        ct_image = (ct/self.xy_size)-self.min_img_coord
        ct_image_int=np.floor(ct_image).astype(np.int32)
        # (1,h,w)
        hm_grid=np.sqrt(np.sum((hm_grid-ct_image_int)**2,axis=2))
        hm_grid[hm_grid==0]=1e-6
        hm_grid=1.0/hm_grid

        hm = np.zeros((1,output_h, output_w), dtype=np.float32)
        if hm[0,corners_int_ul[1]:corners_int_br[1]+1,corners_int_ul[0]:corners_int_br[0]+1].shape!=(corners_int_br[1]+1-corners_int_ul[1],corners_int_br[0]+1-corners_int_ul[0]):
            print(hm_grid.shape)
            return self.getitem(np.random.randint(0, self.__len__()))
        hm[0,corners_int_ul[1]:corners_int_br[1]+1,corners_int_ul[0]:corners_int_br[0]+1]=hm_grid
        hm[0,ct_image_int[1],ct_image_int[0]]=1.0
        hm[0,[ct_image_int[1],ct_image_int[1],ct_image_int[1]+1,ct_image_int[1]-1],[ct_image_int[0]-1,ct_image_int[0]+1,ct_image_int[0],ct_image_int[0]]]=0.8

        # (2r+1,3)x,y,ry
        loc_offsets = np.zeros(((2*self.regress_radius+1)**2, 3), dtype=np.float32)
        # (1,3)
        hwl = np.array([[this_anno["height"],this_anno["width"], this_anno["length"]]],dtype=np.float32)
        # (1,1)
        z_axis = np.array([[sample_reg[0,2]]],dtype=np.float32)
        #center ind
        ind_ct = np.array([ct_image_int[1]*output_w+ct_image_int[0]], dtype=np.int64)
        #loc offsets ind
        ind_offsets = np.zeros(((2*self.regress_radius+1)**2,), dtype=np.int64)

        for i in range(-self.regress_radius,self.regress_radius+1):
            for j in range(-self.regress_radius,self.regress_radius+1):
                offsets=np.zeros((3,),dtype=np.float32)
                offsets[:2]= ct_image - ct_image_int-np.array([i,j])
                #ry
                offsets[2]=sample_reg[0,3]
                loc_offsets[(i+self.regress_radius)*(2*self.regress_radius+1)+(j+self.regress_radius)] = offsets
                ind_int=ct_image_int+np.array([i,j])
                ind_offsets[(i+self.regress_radius)*(2*self.regress_radius+1)+(j+self.regress_radius)]=ind_int[1]*output_w+ind_int[0]

        hm=torch.from_numpy(hm)
        ind_ct=torch.from_numpy(ind_ct)
        hwl=torch.from_numpy(hwl)
        z_axis=torch.from_numpy(z_axis)
        ind_offsets=torch.from_numpy(ind_offsets)
        loc_offsets=torch.from_numpy(loc_offsets)

        ret={'search_pc':sample_PC,
             'model_pc':gt_PC,
             'search_seeds_label':sample_label,
             'search_all_label':all_label,
             'completion_PC_part':completion_PC_part,
             'hm':hm,
             'ind_ct':ind_ct,
             'hwl':hwl,
             'z_axis':z_axis,
             'ind_offsets':ind_offsets,
             'loc_reg':loc_offsets}
        return ret

    def __len__(self):
        nb_anno = len(self.list_of_anno)
        return nb_anno * self.num_candidates_perframe

    def getAnnotationIndex(self, index):
        return int(index / (self.num_candidates_perframe))

    def getSearchSpaceIndex(self, index):
        return int(index % self.num_candidates_perframe)

    def _gather_feat(self,feat, ind, mask=None):
        dim = feat.size(2)
        ind = ind.unsqueeze(2).expand(ind.size(0), ind.size(1), dim)
        feat = feat.gather(1, ind)
        if mask is not None:
            mask = mask.unsqueeze(2).expand_as(feat)
            feat = feat[mask]
            feat = feat.view(-1, dim)
        return feat
    def _tranpose_and_gather_feat(self,feat, ind):
        feat = feat.permute(0, 2, 3, 1).contiguous()
        feat = feat.view(feat.size(0), -1, feat.size(3))
        feat = self._gather_feat(feat, ind)
        return feat

class SiameseTest(SiameseDataset):

    def __init__(self,
                 which_dataset,
                 input_size,
                 path,
                 split="",
                 category_name="Car",
                 regress="GAUSSIAN",
                 offset_BB=0,
                 scale_BB=1.0,
                 voxel_size=[0.2, 0.2, 0.2],
                 xy_size=[0.2, 0.2],
                 # area_extents=[-2.0, 2.0, -2.0, 2.0, -1.5, 1.5],
                 # xy_area_extents=[-2.0, 2.0, -2.0, 2.0],
                 area_extents=[-5.6, 5.6, -3.6, 3.6, -2.4, 2.4],
                 xy_area_extents=[-5.6, 5.6, -3.6, 3.6],
                 downsample=1.0,
                 ):
        super(SiameseTest,self).__init__(
            which_dataset=which_dataset,
            input_size=input_size,
            path=path,
            split=split,
            category_name=category_name,
            regress=regress,
            offset_BB=offset_BB,
            scale_BB=scale_BB)
        self.which_dataset=which_dataset
        self.split = split
        self.offset_BB = offset_BB
        self.scale_BB = scale_BB
        self.voxel_size = np.array(voxel_size)
        self.area_extents = np.array(area_extents).reshape(3, 2)
        self.xy_size = np.array(xy_size) * downsample
        self.xy_area_extents = np.array(xy_area_extents).reshape(2, 2)
        extents_transpose = np.array(self.xy_area_extents).transpose()
        if extents_transpose.shape != (2, 2):
            raise ValueError("Extents are the wrong shape {}".format(extents_transpose.shape))

        # Set image grid extents
        self.min_img_coord = np.floor(extents_transpose[0] / self.xy_size)
        voxel_extents_transpose = self.area_extents.transpose()
        self.scene_ground = voxel_extents_transpose[0]
        self.voxel_grid_size = np.ceil(voxel_extents_transpose[1] / self.voxel_size) - np.floor(
            voxel_extents_transpose[0] / self.voxel_size)
        self.voxel_grid_size = self.voxel_grid_size.astype(np.int32)

    def getitem(self, index):
        list_of_anno = self.list_of_tracklet_anno[index]
        PCs = []
        BBs = []
        for anno in list_of_anno:
            this_PC, this_BB = self.getBBandPC(anno)
            PCs.append(this_PC)
            BBs.append(this_BB)
        return PCs, BBs, list_of_anno

    def __len__(self):
        return len(self.list_of_tracklet_anno)


text:
"""
Reformat data for simulated SNRs by Pierre Cristofari from TXT to ECSV
"""
import numpy as np
import astropy.units as u
from astropy.table import Table, Column
from gammapy.utils.coordinates import galactic as compute_galactic_coordinates
import numpy as np

def read_txt_files(version):
    filename = 'test_roberta_september_6.txt'#.format(version)
    print('Reading {}'.format(filename))
    data = np.genfromtxt(filename, defaultfmt='%0.6f', names=True)

    # Trick to read the Energy grid
    data2 = np.genfromtxt(filename, defaultfmt='%0.6f')

    EGRID = data2[0]
    EGRID = EGRID[~np.isnan(EGRID)]
    size = len(data.dtype.names)
    number_of_SNRs = len(data)  # number of SNRs

    # Reformat spectral data into arrays
    test = [None] * number_of_SNRs
    for j in range(0, number_of_SNRs):
        test[j] = [data[j][0]]
        for i in range(1, size):
            test[j] = np.append(test[j], data[j][i])

    temp = test[0]
    for j in range(1, number_of_SNRs):
        temp = np.vstack([temp, test[j]])

    NAMES = data.dtype.names[0]
    for x in range(1, 10):
        NAMES = np.append(NAMES, data.dtype.names[x])
    for x in range(10, size):
        NAMES = np.append(NAMES, EGRID[x - 10])

    t = Table(temp, names=NAMES)

    # Parameters used for this simulation
    t.meta['simulation_parameters'] = [
        dict(name='alpha', value=4.1, description='Spectral index'),
        dict(name='Kep', value=1e-2, description='Electron to proton ratio'),
        dict(name='xi', value=0.1, description='CR efficiency'),
    ]

    t.meta['energy_array'] = EGRID.tolist()

    t[NAMES[0]].unit = ''
    t[NAMES[0]].format = '%i'
    t[NAMES[0]].description = 'Simulation number'

    t[NAMES[1]].unit = 'kpc'
    t[NAMES[1]].format = '%0.4f'
    t[NAMES[1]].description = 'X Position'

    t[NAMES[2]].unit = 'kpc'
    t[NAMES[2]].format = '%0.4f'
    t[NAMES[2]].description = 'Y Position'

    t[NAMES[3]].unit = 'kpc'
    t[NAMES[3]].format = '%0.4f'
    t[NAMES[3]].description = 'Z Position'

    t[NAMES[4]].unit = u.Unit('cm-3')
    t[NAMES[4]].format = '%0.4f'
    t[NAMES[4]].description = 'density'

    t[NAMES[5]].unit = ''
    t[NAMES[5]].format = '%i'
    t[NAMES[5]].description = 'type of progenitor'

    t[NAMES[6]].unit = 'kyear'
    t[NAMES[6]].format = '%0.4f'
    t[NAMES[6]].description = 'age of the SNR'

    t[NAMES[7]].unit = 'arcmin'
    t[NAMES[7]].format = '%0.4f'
    t[NAMES[7]].description = 'apparent size of the SNR'

    t[NAMES[8]].unit = 'pc'
    t[NAMES[8]].format = '%0.4f'
    t[NAMES[8]].description = 'SNR Radius'

    t[NAMES[9]].unit = 'cm-2 s-1'
    t[NAMES[9]].format = '%0.4e'
    t[NAMES[9]].description = 'Integral flux 1-10TeV'

    for x in range(10, size):
        t[NAMES[x]].unit = 'cm-2 s-1 TeV-1'
        t[NAMES[x]].description = 'Differential spectrum'

    return t

def add_extra_info(table):
    # Change Pierre's XYZ to the one used in Gammapy at the moment
    # This was checked to be correct in https://github.com/gammasky/cta-dc/issues/17
    table['galactocentric_x'] = Column(table['POS_Y'].data, unit='kpc', description='Galactocentric X', format='%0.5f')
    table['galactocentric_y'] = Column(-table['POS_X'].data, unit='kpc', description='Galactocentric Y', format='%0.5f')
    table['galactocentric_z'] = Column(table['POS_Z'].data, unit='kpc', description='Galactocentric Y', format='%0.5f')
    table.remove_columns(['POS_X', 'POS_Y', 'POS_Z'])
    table['flux_1_10'] = Column(table['flux_1_10'].data, unit='cm-2 s-1', description='Integral flux from 1 to 10 TeV',
                                format='%0.4e')

    table.rename_column('Radius', 'size_physical')
    table.rename_column('size', 'sigma')

    r = np.sqrt(table['galactocentric_x'] ** 2 + table['galactocentric_y'] ** 2)
    table['galactocentric_r'] = Column(r, unit='kpc', description='Galactocentric radius in the xy plan')

    distance, glon, glat = compute_galactic_coordinates(
        x=table['galactocentric_x'].quantity,
        y=table['galactocentric_y'].quantity,
        z=table['galactocentric_z'].quantity,
    )
    table['distance'] = Column(distance, unit='kpc', description='Distance from Earth')
    table['distance'].format = '%.5f'

    table['glon'] = Column(glon, unit='deg', description='Galactic longitude')
    table['glon'].format = '%.5f'
    table['glat'] = Column(glat, unit='deg', description='Galactic latitude')
    table['glat'].format = '%.5f'
    zarray = np.zeros(len(table['distance']))
    table['skip'] = Column(zarray, description='Skip boolean, 1 skip 0 keep')
    table['skip'].format = '%.d'

    #print(table['flux_1_10'])
    #print(table.info())
    return table

if __name__ == '__main__':
    for version in [1]:
        table = read_txt_files(version=version)
        table = add_extra_info(table)

        filename = 'ctadc_skymodel_gps_sources_snr_{}.ecsv'.format(version)
        print('Writing {}'.format(filename))
        table.write(filename, format='ascii.ecsv', overwrite=True)

max_stars_repo_name: gammasky/cta-dc

max_issues_repo_name: gammasky/cta-dc

max_forks_repo_name: gammasky/cta-dc

reason: import numpy,import astropy,from astropy


text:
# import packages
import cv2
import numpy as np

img1 = cv2.imread("image/road.jpg")
img2 = cv2.imread("image/car.jpg")

# convert image to gray
img2_gray = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
ret, mask = cv2.threshold(img2_gray, 240, 255, cv2.THRESH_BINARY)
cv2.imwrite("image/06.mask.png", mask)

mask_inv = cv2.bitwise_not(mask)

road = cv2.bitwise_and(img1, img1, mask=mask)
cv2.imwrite("image/06.road_mask.png", road)
car = cv2.bitwise_and(img2, img2, mask=mask_inv)
cv2.imwrite("image/06.car_mask.png", car)
result = cv2.add(road, car)
cv2.imwrite("image/06.result_add.png", result)

max_stars_repo_name: hainguyenvan/images-processing

max_issues_repo_name: hainguyenvan/images-processing

max_forks_repo_name: hainguyenvan/images-processing


text:
import os

import numpy as np
import tensorflow as tf
from scipy.misc import imread, imresize
from tqdm import trange

from hparams import hparams
from models import create_model
from util import audio

class Synthesizer:
    def load(self, checkpoint_path, vgg19_path, model_name='tacotron'):
        print('Constructing model: %s' % model_name)
        inputs = tf.placeholder(tf.float32, [1, hparams.image_dim, hparams.image_dim, 3], 'inputs')
        with tf.variable_scope('model') as _:
            self.model = create_model(model_name, hparams)
            self.model.initialize(inputs, vgg19_path)
            self.wav_output = audio.inv_spectrogram_tensorflow(self.model.linear_outputs[0])

        print('Loading checkpoint: %s' % checkpoint_path)
        self.session = tf.Session()
        self.session.run(tf.global_variables_initializer())
        checkpoint_saver = tf.train.import_meta_graph('%s.%s' % (checkpoint_path, 'meta'))
        checkpoint_saver.restore(self.session, checkpoint_path)

    def synthesize(self, images_dir, output_wav_dir):
        for path, _, filenames in os.walk(images_dir):
            for i in trange(len(filenames)):
                test_file = filenames[i]
                if str.endswith(test_file, '.png'):
                    base_file_name, _ = os.path.splitext(test_file)
                    raw_image = imread(os.path.join(path, test_file), mode='RGB')
                    processed_image = imresize(raw_image, (224, 224, 3))

                    feed_dict = {
                        self.model.inputs: [np.asarray(processed_image, dtype=np.float32)],
                    }
                    wav = self.session.run(self.wav_output, feed_dict=feed_dict)
                    wav = audio.inv_preemphasis(wav)
                    wav = wav[:audio.find_endpoint(wav)]
                    audio_out_path = os.path.join(output_wav_dir, 'eval-{}.wav'.format(base_file_name))
                    audio.save_wav(wav, audio_out_path)
                    print('Wav - {} generated successfully!'.format(audio_out_path))

max_stars_repo_name: nikhilsu/Mixed-modal-learning

max_issues_repo_name: nikhilsu/Mixed-modal-learning

max_forks_repo_name: nikhilsu/Mixed-modal-learning

reason: import numpy,from scipy