text:
import numpy as np
import pandas as pd
import os
import cplex
from cplex.exceptions import CplexSolverError
from cplex.callbacks import BranchCallback
from cplex.callbacks import NodeCallback

import pickle
import argparse
from collections import OrderedDict

import shared
import utilities

"""
Run as
>>> python data_run.py --instance air04.mps.gz --seed 20180329 --node_limit $NODELIM

Extract information from the B&B tree via BranchCB and NodeCB.

Callbacks are invoked by the solver at *every* branched node, and at each node selection, but we extract information
only up to a certain number of nodes (--node_limit), before stopping the run.
Node limit is computed from a full run, as the # of nodes processed up to rho% of a certain time_limit
(see full_run.py).

In particular, we trigger

    > BranchCB features: at every node in the partial tree, i.e., until optimization stops
    > NodeCB features: at few points in time, {25, 50, 75, 100}% of rho time stamp.

    (e.g., when rho=20, we collect NodeCB features at {5, 10, 15, 20}% of time_limit,
    where 20 = 100% of rho means just before stopping data collection)
    Note: there is no collection after the root node (the time-stamp is not mapped in TIME_NODES_DICT)

    In this way we can have more info about problems that are solved *before* tau,
    for which the observed tree is the whole tree.

The information is saved directly into a (global) list, for the entire (partial) run.
At the end of the run, the list is converted to a single array and saved it in DATA_PATH/NPY/.

NOTE: for instances that are fully solved before tau (`trivial' ones), the last node will *not* be mapped by callbacks.
      This happens naturally, because solve() ends, and there is no way that node is going to enter
      the BranchCB or the NodeCB.
      In particular, the last datum about the gap will not be 0.0, and we will not have NodeCB as last row of the data.
      This is why final_info are collected and stored (for analysis and checks), though the information about the
      end of the resolution process is *not explicit* in the collected data.

Now the order of the (31) basic extracted features is important, and fixed as follows:

    GENERAL
0:  'global_cb_count'
1:  'times_called'
2:  'get_time'
3:  'get_dettime'
4:  'elapsed'
5:  'det_elapsed'

    BRANCH_CB ONLY
6:  'index'
7:  'parent_id'
8:  'parent_index'
9:  'node'
10: 'nodes_left'
11: 'objective'
12: 'iinf'                  **expensive**
13: 'best_integer'
14: 'best_bound'
15: 'itCnt'
16: 'gap'
17: 'num_nodes'
18: 'depth'
19: 'num_fixed_vars'        **expensive**
20: 'is_integer_feasible'
21: 'has_incumbent'

    NODE_CB ONLY
22: 'open_nodes_len'
23: 'open_nodes_avg'
24: 'open_nodes_min'
25: 'open_nodes_max'
26: 'num_nodes_at_min'
27: 'num_nodes_at_max'
28: 'num_cuts'

    GENERAL
29: 'cb_time'
30: 'cb_dettime'

"""

"""
Parser definition.
"""

parser = argparse.ArgumentParser(description='Arg parser for data_run script.')

# Run identifiers: instance name, random seed, dataset
parser.add_argument(
    '--instance',
    type=str,
    required=True,
    help='Name of MIP instance to be processed (w/ full extension).'
)
parser.add_argument(
    '--seed',
    type=int,
    default=201610271,
    help='Random seed to be set in the solver. Default is 201610271 (CPLEX version 12.7.1).'
)
parser.add_argument(
    '--dataset',
    type=str,
    required=True,
    help='Dataset of origin identifier for the MILP instance (e.g., Benchmark78).'
)

# Node and time limit settings:
parser.add_argument(
    '--time_limit',
    type=float,
    default=7200,
    help='Sets the time limit (secs). Default is 7200.'
)
parser.add_argument(
    '--node_limit',
    type=int,
    required=True,
    help='Node limit to be used in data collection. Obtained from full_run stats.'
)
parser.add_argument(
    '--rho',
    type=int,
    required=True,
    help='Percentage of time limits to be considered.'
)
parser.add_argument(
    '--label_time',
    type=float,
    required=True,
    help='Time(limit) w.r.t. which label and node_limit were computed.'
)

# Additional (only to store data)
parser.add_argument(
    '--label',
    type=int,
    required=True,
    help='Label in {0, 1} assigned during full_run w.r.t. label_time.'
)
parser.add_argument(
    '--trivial',
    type=bool,
    required=True,
    help='Trivial flag assigned during full_run.'
)
parser.add_argument(
    '--tau',
    type=float,
    required=True,
    help='Time corresponding to rho % of label_time.'
)
parser.add_argument(
    '--sol_time',
    type=float,
    required=True,
    help='Solution time of (instance, seed) during full_run.'
)

parser.add_argument(
    '--data_path',
    type=str,
    default=shared.DATA_PATH,
    help='Absolute path to data archive. Can be set within shared.py.'
)
parser.add_argument(
    '--inst_path',
    type=str,
    default=shared.INST_PATH,
    help='Absolute path to instances. Can be set within shared.py.'
)

ARGS = parser.parse_args()

"""
Custom classes definition.
"""

class GlobalCBCount:
    """
    A class setting a global count on Callbacks calls.
    """
    __counter = 0  # global counter of callbacks calls

    def __init__(self):
        GlobalCBCount.__counter += 1
        self.__cb_count = GlobalCBCount.__counter

    def get_cb_count(self):
        return self.__cb_count

class UserNodeIndex:
    """
    A class setting a global counter as unique index to each created node.
    """
    __index = -1  # so that root has index 0

    def __init__(self):
        UserNodeIndex.__index += 1
        self.__node_index = UserNodeIndex.__index

    def get_node_index(self):
        return self.__node_index

class UserNodeData:
    """
    A class defining a custom data-handle for nodes in the B&B tree.
    """
    def __init__(self, user_index, user_depth, user_parent_id, user_parent_index):
        self.user_index = user_index  # UserNodeIndex counter
        self.user_depth = user_depth  # depth will be updated recursively from parent to children
        self.user_parent_id = user_parent_id  # id of parent node
        self.user_parent_index = user_parent_index  # index of parent node

"""
Global definitions.
"""

# global definition of list, which will contain lists of length 31, i.e., the rows of the final array.
GLOBAL_LIST = list()

# the correct dictionary RHO_TAU_NODES is name_seed.pickle in ARGS.data_path/TIME_NODES_DICT/dataset_seed/
dict_dir = ARGS.data_path + "/TIME_NODES_DICT/" + ARGS.dataset + "_" + str(ARGS.seed) + "/"
dict_name = ARGS.instance.split('.')[0] + "_" + str(ARGS.seed) + ".pkl"

with open(os.path.join(dict_dir, dict_name), "rb") as p:
    RHO_TAU_NODES_DICT = pickle.load(p)
    nodeCB_FLAGS = OrderedDict()
    for eta in RHO_TAU_NODES_DICT[ARGS.rho][ARGS.tau]:
        if eta:  # map only node marks that are not None
            nodeCB_FLAGS[eta] = False
p.close()

# additional last NodeCB call at the end of the data collection
# (not met if node_limit corresponds to final number of nodes)
nodeCB_FLAGS[ARGS.node_limit] = False

# global instance of UserNodeData for the root node
root = UserNodeIndex()
print("\tRoot index is: {}".format(root.get_node_index()))
root_user_data = UserNodeData(user_index=root.get_node_index(), user_depth=0, user_parent_id=-1,
                              user_parent_index=None)

"""
Callbacks definition.
"""

# NOTE: there is no 'real' time_limit for data_run (enforce at 2h, with node_limit being effectively used).
# So it makes no sense to measure time_to_end.
# Elapsed time is meaningful, but accounts for data collection overhead as well.
# A more deterministic metric should be given by the number of nodes.

class MyDataNode(NodeCallback):
    """
    Custom subclass of NodeCallback.
    This callback will be used *before CPLEX enters a node*.
    """
    def __init__(self, env):
        NodeCallback.__init__(self, env)
        self.times_called = 0
        self.count_data = 0

    def __call__(self):
        self.times_called += 1
        gcb = GlobalCBCount()
        global nodeCB_FLAGS
        global GLOBAL_LIST

        t_stamp = self.get_time()
        det_stamp = self.get_dettime()

        s_elapsed = self.get_time() - self.get_start_time()
        det_elapsed = self.get_dettime() - self.get_start_dettime()

        # measure overhead of the whole CB call
        cb_start = self.get_time()
        cb_det_start = self.get_dettime()

        # at appropriate node marks, extract data
        for n in nodeCB_FLAGS.keys():
            if (self.get_num_nodes() >= n) and not nodeCB_FLAGS[n]:
                # NOTE: last flag will not be hit if it corresponds to the final number of nodes to solve a problem
                # (in case of problems that are solved before tau)
                print("*** NodeCB data call at num_nodes {}".format(self.get_num_nodes()))
                nodeCB_FLAGS[n] = True
                self.count_data += 1

                # print(self.get_num_nodes(), self.times_called, "NodeCB_FLAG", nodeCB_FLAGS[t])

                node_cb_list = list()
                # times and counters
                node_cb_list.append(gcb.get_cb_count())
                node_cb_list.append(self.times_called)
                node_cb_list.extend([t_stamp, det_stamp, s_elapsed, det_elapsed])

                # None for BranchCB entries
                node_cb_list.extend([None] * 16)

                # open nodes list
                open_list_obj = [self.get_objective_value(node) for node in range(self.get_num_remaining_nodes())]
                open_list_obj = np.asarray(open_list_obj)
                node_cb_list.extend([
                    len(open_list_obj),
                    np.mean(open_list_obj),
                    np.min(open_list_obj),
                    np.max(open_list_obj),
                    len(np.argwhere(open_list_obj == np.min(open_list_obj))),
                    len(np.argwhere(open_list_obj == np.max(open_list_obj)))
                ])
                # cuts
                cut_codes = [108, 111, 110, 107, 115, 120, 119, 117, 112, 122, 126, 133, 134, 135, 136]
                node_cb_list.append(np.sum([self.get_num_cuts(k) for k in cut_codes]))

                # cb_time, overhead of the callback call
                node_cb_list.extend([self.get_time() - cb_start, self.get_dettime() - cb_det_start])

                GLOBAL_LIST.append(node_cb_list)

                break

class MyDataBranch(BranchCallback):
    """
    Custom subclass of BranchCallback.
    This callback will be used *prior to branching* at a node in the branch and cut tree.
    """
    def __init__(self, env):
        BranchCallback.__init__(self, env)
        self.times_called = 0
        self.count_data = 0

    def __call__(self):
        self.times_called += 1
        gcb = GlobalCBCount()
        global ARGS
        global GLOBAL_LIST

        t_stamp = self.get_time()
        det_stamp = self.get_dettime()

        s_elapsed = self.get_time() - self.get_start_time()
        det_elapsed = self.get_dettime() - self.get_start_dettime()

        # measure overhead of the whole CB call
        cb_start = self.get_time()
        cb_det_start = self.get_dettime()

        # print("{}: Node - {}".format(self.times_called, self.get_node_ID()))

        # set user handle on new nodes, if cplex would create them
        # root node case is special
        data = self.get_node_data()  # UserNodeData object, or None for the root node

        if data is None:
            data = root_user_data  # now UserNodeData object as initialized above (global)

        if self.get_num_branches() > 0:
            for k in range(self.get_num_branches()):
                idx = UserNodeIndex().get_node_index()
                self.make_cplex_branch(
                    which_branch=k,
                    node_data=UserNodeData(
                        user_index=idx,
                        user_depth=data.user_depth+1,
                        user_parent_id=self.get_node_ID(),
                        user_parent_index=data.user_index
                    )
                )

        # features extraction
        if self.get_num_nodes() <= ARGS.node_limit:  # should not be necessary with node_limit enforced
            # NOTE: last node of node_limit is not recorded if it corresponds to the last node of the resolution,
            # because it will not enter BranchCB (in case of problems that are solved before tau)
            self.count_data += 1
            branch_cb_list = list()

            # times, indices, counters
            branch_cb_list.append(gcb.get_cb_count())
            branch_cb_list.append(self.times_called)
            branch_cb_list.extend([
                t_stamp,
                det_stamp,
                s_elapsed,
                det_elapsed
            ])
            branch_cb_list.extend([
                data.user_index,
                data.user_parent_id,
                data.user_parent_index
            ])

            # log info
            branch_cb_list.extend([
                self.get_node_ID(),
                self.get_num_remaining_nodes(),
                self.get_objective_value(),
                self.get_feasibilities().count(1),  # np.asarray ?
                self.get_incumbent_objective_value(),
                self.get_best_objective_value(),
                self.get_num_iterations(),
                self.get_MIP_relative_gap()
            ])
            # additional
            branch_cb_list.extend([
                self.get_num_nodes(),
                data.user_depth,
                ((np.asarray(self.get_upper_bounds()) - np.asarray(self.get_lower_bounds())) == 0).sum(),
                self.is_integer_feasible(),
                self.has_incumbent()
            ])

            # None for NodeCB entries
            branch_cb_list.extend([None] * 7)

            # cb_time, overhead of the callback call
            branch_cb_list.extend([
                self.get_time() - cb_start,
                self.get_dettime() - cb_det_start
            ])

            GLOBAL_LIST.append(branch_cb_list)

if __name__ == "__main__":

    import sys
    import time
    import os

    name = ARGS.instance.split('.')[0]
    inst_name_info = name + '_' + str(ARGS.seed) + '_' + str(ARGS.label_time) + '_' + str(ARGS.tau) + '_' + str(ARGS.rho)

    # sys.stdout = None

    os.chdir(ARGS.inst_path)
    pb = cplex.Cplex(ARGS.instance)

    print("\nPb name and type: {} {}".format(pb.get_problem_name(), pb.problem_type[pb.get_problem_type()]))
    print("# Variables: {}".format(pb.variables.get_num()))
    print("# Constraints: {}".format(pb.linear_constraints.get_num()))

    print("\nNode limit = {}".format(ARGS.node_limit))
    print("(rho, tl) = ({}, {}), tau = {}".format(ARGS.rho, ARGS.label_time, ARGS.tau))
    print("Solution time = {}".format(ARGS.sol_time))  # solution time of the full run
    print("Label = {}".format(ARGS.label))
    print("Trivial = {}".format(ARGS.trivial))

    # set solver parameters
    utilities.set_solver_data_run_parameters(pb, ARGS)

    # register callback
    branch_instance = pb.register_callback(MyDataBranch)
    node_instance = pb.register_callback(MyDataNode)

    try:
        t0, det_t0 = pb.get_time(), pb.get_dettime()
        print("\nInitial time-stamps: {} {}\n".format(t0, det_t0))
        pb.solve()
        elapsed = pb.get_time() - t0
        elapsed_ticks = pb.get_dettime() - det_t0
        status = pb.solution.get_status()
        num_nodes = int(pb.solution.progress.get_num_nodes_processed())
        itCnt = int(pb.solution.progress.get_num_iterations())
        gap = pb.solution.MIP.get_mip_relative_gap()
        print("\nStatus: {}".format(pb.solution.get_status()))
    except CplexSolverError as exc:  # data will be all zeros if exception is raised
        print("Exception raised during solve: {}".format(exc.args[2]))
        elapsed = None
        elapsed_ticks = None
        status = exc.args[2]  # error code
        num_nodes = None
        itCnt = None
        gap = None

    final_info = [elapsed, elapsed_ticks, status, num_nodes, itCnt, gap]

    print("\nMyDataBranch was called {} times".format(branch_instance.times_called))
    print("MyDataBranch count data: {}".format(branch_instance.count_data))
    print("\nMyDataNode was called {} times".format(node_instance.times_called))
    print("MyDataNode count data: {}".format(node_instance.count_data))

    print("\nNodeCB flags: ")
    print(nodeCB_FLAGS)

    print("\nFinal info of data_run: {}".format(final_info))

    # conversion to array
    global_arr = np.asarray(GLOBAL_LIST)
    print("Array shape is: ", global_arr.shape)

    # use np.savez to save into inst_name_info:
    npy_rho_dir = ARGS.data_path + '/NPY_RHO_' + str(int(ARGS.rho)) + '/' + ARGS.dataset + '_' + str(ARGS.seed) + '/'
    try:
        os.mkdir(npy_rho_dir)
    except OSError:
        if not os.path.isdir(npy_rho_dir):
            raise

    os.chdir(npy_rho_dir)
    np.savez_compressed(
        inst_name_info,
        ft_matrix=global_arr,
        name=name,
        seed=ARGS.seed,
        dataset=ARGS.dataset,
        rho=ARGS.rho,
        tau=ARGS.tau,
        node_limit=ARGS.node_limit,
        label_time=ARGS.label_time,
        label=ARGS.label,
        trivial=ARGS.trivial,
        sol_time=ARGS.sol_time,
        data_final_info=final_info,
    )

    pb.end()

max_stars_repo_name: XinEDprob/milp-outcome

max_issues_repo_name: XinEDprob/milp-outcome

max_forks_repo_name: XinEDprob/milp-outcome


text:
# ***************************************************************
# Copyright (c) 2020 Jittor. Authors:
#     Guowei Yang <471184555@qq.com>
#     Dun Liang <randonlang@gmail.com>.
# All Rights Reserved.
# This file is subject to the terms and conditions defined in
# file 'LICENSE.txt', which is part of this source code package.
# ***************************************************************
import unittest
import jittor as jt
import jittor.attention as jtatt
import numpy as np

skip_this_test = False

try:
    jt.dirty_fix_pytorch_runtime_error()
    import torch
    import torch.nn as tnn
    import fairseq
except:
    torch = None
    tnn = None
    skip_this_test = True

def check_equal(q,k,v,tatt,jatt):
    tq=torch.from_numpy(q)
    jq=jt.array(q)
    tk=torch.from_numpy(k)
    jk=jt.array(k)
    tv=torch.from_numpy(v)
    jv=jt.array(v)

    jatt.load_parameters(tatt.state_dict())
    ty, tw = tatt(tq, tk, tv)
    jy, jw = jatt(jq, jk, jv)
    assert np.allclose(ty.detach().numpy(), jy.numpy(), rtol=1e-3)
    assert np.allclose(tw.detach().numpy(), jw.numpy(), rtol=1e-3)

@unittest.skipIf(skip_this_test, "No Torch found")
class TestAttention(unittest.TestCase):
    def test_attention(self):
        q=np.random.rand(4,8,16).astype(np.float32)
        k=np.random.rand(4,8,16).astype(np.float32)
        v=np.random.rand(4,8,16).astype(np.float32)

        tatt=fairseq.modules.multihead_attention.MultiheadAttention(16,1)
        jatt=jt.attention.MultiheadAttention(16,1)
        check_equal(q,k,v,tatt,jatt)

        tatt=fairseq.modules.multihead_attention.MultiheadAttention(16,4)
        jatt=jt.attention.MultiheadAttention(16,4)
        check_equal(q,k,v,tatt,jatt)

        tatt=fairseq.modules.multihead_attention.MultiheadAttention(16,1,self_attention=True)
        jatt=jt.attention.MultiheadAttention(16,1,self_attention=True)
        check_equal(q,q,q,tatt,jatt)

        tatt=fairseq.modules.multihead_attention.MultiheadAttention(16,4,self_attention=True)
        jatt=jt.attention.MultiheadAttention(16,4,self_attention=True)
        check_equal(q,q,q,tatt,jatt)

if __name__ == "__main__":
    unittest.main()

ext: py

reason: import numpy


text:
# python imports
import os
import glob
import sys
import random

import numpy as np
import scipy.io as sio
import utils
import subspacemodels
import kernels
import dists

if __name__ == '__main__':

    #load data
    #training_data=numpy array of size M,N where M is the data dimension an N is the number of samples. In our MICCAI paper, we use the SCT/JSRT data set. Please see the code files related to our Medical Image Analysis paper (https://imi.uni-luebeck.de/multi-resolution-multi-object-statistical-shape-models) on how to obtain and convert that data.

    #test_data= see training data

    M,N=training_data.shape
    #objects indices for the SCR/JSRT data when pre-processed as described above
    obj_indicator=np.zeros(M)
    obj_indicator[0:88]=0 #right lung
    obj_indicator[88:188]=1 #left lung
    obj_indicator[188:240]=2 #heart
    obj_indicator[240:286]=3 #right clavicle
    obj_indicator[286:332]=4 #left clavicle
    color_map=['b','k','r','g','y']
    spacing=0.35
    training_data=training_data*spacing
    test_data=test_data*spacing
    spec_samples=1000
    alpha_spec=5
    spec_mode='normalbouned'

    # Select a (small) training data set

    size_training_set=[5,10,15,20,30,40,70,120];
    size_test_set=test_data.shape[1]
    num_random_runs=[50,50,50,50,50,50,50,50]

    models=[]
    gen_results=[]
    spec_results=[]
    basis_sizes=[]

    num_levels=5

    #compute geodesic distance matrix only once on all training images to save time during training (of course, introduces small bias)

    #multi-object distance proposed in our MedIA paper
 distance_matrix_sp=utils.compute_multi_object_pseudo_euclidean_geodesic_shortest_path_2d_point_distance_matrix(np.mean(training_data,axis=1,keepdims=True),obj_indicator,eta=10,kappa=150)
    max_distance=np.max(distance_matrix_sp)
    distance_schedule=max_distance*np.power(0.5,np.array(range(0,num_levels)))

    #euclidean distance
    distance_matrix_eucl=utils.compute_euclidean_2d_point_distance_matrix(np.mean(training_data,axis=1,keepdims=True))
    max_distance_eucl=np.max(distance_matrix_eucl)
    distance_schedule_eucl=max_distance_eucl*np.power(0.5,np.array(range(0,num_levels)))

    #run main loop
    for curr_index in range(0,len(size_training_set)):
        curr_size_training_set=size_training_set[curr_index]
        curr_num_random_runs=num_random_runs[curr_index]

        curr_models=[[],[],[],[]]
        curr_results=[[],[],[],[]]
        curr_spec=[[],[],[],[]]
        curr_size_basis=[[],[],[],[]]

        print('\nSize training set: '+str(curr_size_training_set)+' size test set: '+str(size_test_set)+' runs: '+str(curr_num_random_runs))

        #run each random experiment
        for run in range(0,curr_num_random_runs):

            #(randomly) select the data entries for training and test
            training_idx=np.random.choice(training_data.shape[1],curr_size_training_set,replace=False)
            sample_data_matrix=training_data[:,training_idx];
            sample_test_data=test_data;

            #generate standard pca model
            variability_retained=0.95

            pca_model=subspacemodels.SubspaceModelGenerator.compute_pca_subspace(np.matrix(sample_data_matrix),variability_retained)
            approx,error=pca_model.approximate(sample_test_data,utils.mean_error_2d_contour)
            spec=pca_model.compute_specificity(sample_test_data,alpha_spec, utils.mean_error_2d_contour, num_samples=spec_samples,mode=spec_mode)

            curr_spec[0].append(np.mean(spec))
            curr_models[0].append(pca_model)
            curr_results[0].append(np.mean(error))
            curr_size_basis[0].append(pca_model.basis.shape[1])

            #generate LSSM model
           localized_model=subspacemodels.SubspaceModelGenerator.compute_localized_subspace_media(np.matrix(sample_data_matrix),variability_retained,distance_matrix_sp,distance_schedule,sample_test_data,merge_method=utils.merge_subspace_models_closest_rotation_decorr)
            approx,error=localized_model.approximate(sample_test_data,utils.mean_error_2d_contour)
            spec=localized_model.compute_specificity(sample_test_data,alpha_spec, utils.mean_error_2d_contour, num_samples=spec_samples,mode=spec_mode)

            curr_spec[1].append(np.mean(spec))
            curr_models[1].append(localized_model)
            curr_results[1].append(np.mean(error))
            curr_size_basis[1].append(localized_model.basis.shape[1])

            #generate KLSSM+Grass model
            contour_distance=dists.SimpleMatrixDist(distance_matrix_eucl)
            cov_kernel=kernels.CovKernel(1/(sample_data_matrix.shape[1]-1))
            gamma=1/(2*((2*distance_schedule_eucl)**2))
            exponent=2

            kernels_lvl_1=[(cov_kernel,None,'data',None,1)]
            kernels_lvl_2=[(cov_kernel,None,'data',None,1), (kernels.ExponentialKernel(gamma[1],exponent),np.multiply,'dist',contour_distance,1)]
            kernels_lvl_3=[(cov_kernel,None,'data',None,1), (kernels.ExponentialKernel(gamma[2],exponent),np.multiply,'dist',contour_distance,1)]
            kernels_lvl_4=[(cov_kernel,None,'data',None,1), (kernels.ExponentialKernel(gamma[3],exponent),np.multiply,'dist',contour_distance,1)]
            kernels_lvl_5=[(cov_kernel,None,'data',None,1), (kernels.ExponentialKernel(gamma[4],exponent),np.multiply,'dist',contour_distance,1)]

            kernel_list=[kernels_lvl_1,kernels_lvl_2,kernels_lvl_3,kernels_lvl_4,kernels_lvl_5]
            max_rank=331

            localized_model_kernel_eucl2=subspacemodels.SubspaceModelGenerator.compute_localized_subspace_kernel(np.matrix(sample_data_matrix),variability_retained,distance_schedule_eucl,kernel_list,max_rank,sample_test_data,debug=False,eig_method=lambda_eig_method,merge_method=utils.merge_subspace_models_closest_rotation_decorr_kernel)
            approx,error=localized_model_kernel_eucl2.approximate(sample_test_data,utils.mean_error_2d_contour)
            spec=localized_model_kernel_eucl2.compute_specificity(sample_test_data,alpha_spec, utils.mean_error_2d_contour, num_samples=spec_samples,mode=spec_mode)

            curr_spec[2].append(np.mean(spec))
            curr_models[2].append(localized_model_kernel_eucl2)
            curr_results[2].append(np.mean(error))
            curr_size_basis[2].append(localized_model_kernel_eucl2.basis.shape[1])

            #generate KLSSM+Kernel model
            cov_kernel=kernels.CovKernel(1/(sample_data_matrix.shape[1]-1))

            kernel_list=[[(kernels.OnesKernel(),None,'dist',contour_distance,.2), (kernels.ExponentialKernel(gamma[1],exponent),np.add,'dist',contour_distance,.2),(kernels.ExponentialKernel(gamma[2],exponent),np.add,'dist',contour_distance,.2), (kernels.ExponentialKernel(gamma[3],exponent),np.add,'dist',contour_distance,.2), (kernels.ExponentialKernel(gamma[4],exponent),np.add,'dist',contour_distance,.2), (cov_kernel,np.multiply,'data',None,1)]]

            localized_model_kernel_gpmm_eucl2=subspacemodels.SubspaceModelGenerator.compute_localized_subspace_kernel(np.matrix(sample_data_matrix),variability_retained,[0],kernel_list,max_rank,sample_test_data,eig_method=lambda_eig_method)
            approx,error=localized_model_kernel_gpmm_eucl2.approximate(sample_test_data,utils.mean_error_2d_contour)
            spec=localized_model_kernel_gpmm_eucl2.compute_specificity(sample_test_data,alpha_spec, utils.mean_error_2d_contour, num_samples=spec_samples,mode=spec_mode)

            curr_spec[3].append(np.mean(spec))
            curr_models[3].append(localized_model_kernel_gpmm_eucl2)
            curr_results[3].append(np.mean(error))
            curr_size_basis[3].append(localized_model_kernel_gpmm_eucl2.basis.shape[1])

            ## end random runs

        models.append(curr_models)
        gen_results.append(curr_results)
        basis_sizes.append(curr_size_basis)
        spec_results.append(curr_spec)

        print('\nSSM model: '+str(np.mean(curr_results[0]))+' size: '+str(np.mean(curr_size_basis[0]))+' spec: '+str(np.mean(curr_spec[0])))
        print('\nLSSM: '+str(np.mean(curr_results[1]))+' size: '+str(np.mean(curr_size_basis[1]))+' spec: '+str(np.mean(curr_spec[1])))
        print('\nKLSSM+Grass: '+str(np.mean(curr_results[2]))+' size: '+str(np.mean(curr_size_basis[2]))+' spec: '+str(np.mean(curr_spec[2])))
        print('\nKLSSM+Kernel: '+str(np.mean(curr_results[3]))+' size: '+str(np.mean(curr_size_basis[3]))+' spec: '+str(np.mean(curr_spec[3])))

    ##end training set size

    print('\nPress any key to continue')

    input()

max_stars_repo_name: wilmsm/localizedssm

max_issues_repo_name: wilmsm/localizedssm

max_forks_repo_name: wilmsm/localizedssm

reason: import numpy,import scipy


text:
# Written by: Nick Gerend, @dataoutsider
# Viz: "Good Read", enjoy!

import numpy as np
import pandas as pd
import os

df1 = pd.read_csv(os.path.dirname(__file__) + '/quill_cat.csv')
df2 = pd.read_csv(os.path.dirname(__file__) + '/thegreatestbooks_quill_cat.csv')
df3 = pd.read_csv(os.path.dirname(__file__) + '/1001bookreviews_quill_cat.csv')

df1['source'] = 'Most Recommended Books'
df2['source'] = 'The Greatest Books of All Time'
df3['source'] = '1,000 Books to Read Before You Die'

dfs = []
dfs.append(df1)
dfs.append(df2)
dfs.append(df3)
df_out = pd.concat(dfs, ignore_index=True)

df_out.to_csv(os.path.dirname(__file__) + '/quill_categories.csv', encoding='utf-8', index=False)

print('finished')

max_stars_repo_name: nickgerend/GoodRead

max_issues_repo_name: nickgerend/GoodRead

max_forks_repo_name: nickgerend/GoodRead


text:
import numpy as np
import cv2
import scipy.misc
import os
from Detection.MtcnnDetector import MtcnnDetector
from Detection.detector import Detector
from Detection.fcn_detector import FcnDetector
from train_models.mtcnn_model import P_Net, R_Net, O_Net

test_mode = "onet"
thresh = [0.9, 0.3, 0.3]
min_face_size = 24
stride = 2
slide_window = False
shuffle = False
detectors = [None, None, None]
prefix = ['data/MTCNN_model/PNet_landmark/PNet', 'data/MTCNN_model/RNet_landmark/RNet', 'data/MTCNN_model/ONet_landmark/ONet']
epoch = [18, 14, 16]
model_path = ['%s-%s' % (x, y) for x, y in zip(prefix, epoch)]
PNet = FcnDetector(P_Net, model_path[0])
detectors[0] = PNet
RNet = Detector(R_Net, 24, 1, model_path[1])
detectors[1] = RNet
ONet = Detector(O_Net, 48, 1, model_path[2])
detectors[2] = ONet
mtcnn_detector = MtcnnDetector(detectors=detectors, min_face_size=min_face_size,
                               stride=stride, threshold=thresh, slide_window=slide_window)

camera = cv2.VideoCapture(1)
name = 'test'
count = 0

path = os.getcwd() + '/data-face/' + name
if not os.path.exists(path):
    os.makedirs(path)
while True:
	_, frame = camera.read()
	gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
	boxes_c,_ = mtcnn_detector.detect(frame)
	for i in range(boxes_c.shape[0]):
		bbox = boxes_c[i, :4]
		score = boxes_c[i, 4]
		corpbbox = [int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])]
		cv2.rectangle(frame, (corpbbox[0], corpbbox[1]), (corpbbox[2], corpbbox[3]), (255, 0, 0), 1)
		cv2.putText(frame, '{:.3f}'.format(score), (corpbbox[0], corpbbox[1] - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
		try:
			scipy.misc.imsave(path + '/' + name + str(count) + '.png', gray[int(bbox[1]): int(bbox[3]), int(bbox[0]) : int(bbox[2])])
		except:
			continue
		count += 1
	cv2.imshow("test", frame)
	if cv2.waitKey(1) & 0xFF == ord('q'):
		break
camera.release()
cv2.destroyAllWindows()


text:
"""
AI denoising applied in 2D postprocessing stage.

Note: denoiser binaries are not installed by default, run::

   python -m plotoptix.install denoiser

to download denoiser.
"""

import numpy as np
from plotoptix import TkOptiX
from plotoptix.enums import DenoiserKind

def main():

    # make some data first:
    rx = (-1, 1.8)
    rz = (-1, 1.8)
    n = 18

    x = np.linspace(rx[0], rx[1], n)
    z = np.linspace(rz[0], rz[1], n)

    X, Z = np.meshgrid(x, z)
    Y = np.sqrt(X ** 2 + Z ** 2)

    # positions of blocks
    data = np.stack((X.flatten(), np.zeros(n*n), Z.flatten())).T
    # heights of blocks
    v = np.zeros(data.shape); v[:,1] = (Y.flatten() + 0.3 * np.random.rand(n*n))[:]

    optix = TkOptiX() # create and configure, show the window later

    optix.set_param(max_accumulation_frames=50)  # accumulate up to 50 frames
    optix.set_background(0)             # black background
    optix.set_ambient([0.1, 0.15, 0.2]) # cold ambient light

    # add plot geometry
    size_u = 0.98 * (rx[1] - rx[0]) / (n - 1)
    size_w = 0.98 * (rz[1] - rz[0]) / (n - 1)
    optix.set_data("blocks", pos=data,
                   u=[size_u, 0, 0], v=v, w=[0, 0, size_w],
                   c = np.random.rand(n*n),
                   geom="Parallelepipeds")

    # set camera and light position
    optix.setup_camera("cam1", cam_type="DoF",
                       eye=[-0.3, 2, -0.3], target=[1, 1, 1],
                       fov=60, focal_scale=0.85)
    optix.setup_light("light1", pos=[3, 5.5, 1], color=[6.5, 5, 4.5], radius=2)

    # AI denoiser includes exposure and gamma corection, configured with the
    # same variables as the Gamma postprocessing algorithm.
    optix.set_float("tonemap_exposure", 0.5)
    optix.set_float("tonemap_gamma", 2.2)

    # Denoiser blend allows for different mixing with the raw image. Its value
    # can be modified also during the ray tracing.
    # Note: denoising is applied when > 4 accumulation frames are completed.
    optix.set_float("denoiser_blend", 0.25)

    # Denoiser can use various inputs. By default it is raw RGB and surface
    # albedo, but not always it results with optimal quality at the output.
    # Try one of the below settings and find best configuration for your scene.
    #optix.set_int("denoiser_kind", DenoiserKind.Rgb.value)
    #optix.set_int("denoiser_kind", DenoiserKind.RgbAlbedo.value)
    #optix.set_int("denoiser_kind", DenoiserKind.RgbAlbedoNormal.value)

    optix.add_postproc("Denoiser")

    # Postprocessing stages are applied after AI denoiser (even if configured
    # in a different order).

    optix.set_float("levels_low_range", 0.1, 0.05, -0.05)
    optix.set_float("levels_high_range", 0.95, 0.85, 0.8)
    optix.add_postproc("Levels")

    optix.start()
    print("done")

if __name__ == '__main__':
    main()

max_stars_repo_name: robertsulej/plotoptix

max_stars_repo_licenses: libtiff MIT

max_issues_repo_name: robertsulej/plotoptix

max_issues_repo_licenses: libtiff MIT

max_forks_repo_name: robertsulej/plotoptix

max_forks_repo_licenses: libtiff MIT