text:
import ctypes
from collections import namedtuple
from itertools import chain

import numpy
from petsc4py.PETSc import IntType
from pyop3.obj.sets import DataSet, ExtrudedSet, Set
from pyop3.utils import cached_property, check_args

class IdentityMap(object):
    """Singleton marker for the identity map (direct args)"""
    _codegen_info_ = None
    _parloop_argtypes_ = ()
    _parloop_args_ = ()

    @staticmethod
    def validate(iterset, toset):
        return iterset == toset

IdentityMap = IdentityMap()

class Map(object):
    dtype = IntType
    offset = None
    Info = namedtuple("Info", ["shape", "offset", "dtype", "setinfo"])
    _parloop_argtypes_ = (ctypes.c_voidp, )

    def validator(self, iterset, toset, arity, values, offset=None):
        assert isinstance(iterset, (Set, ExtrudedSet))
        assert isinstance(toset, (Set, ExtrudedSet))
        values = numpy.asarray(values).astype(IntType)
        assert max(values) < toset.total_size
        if offset is not None:
            assert len(offset) == arity
            assert all(o >= 0 for o in offset)

    @check_args(validator)
    def __init__(self, iterset, toset, arity, values, offset=None):
        self.iterset = iterset
        self.toset = toset
        self.comm = toset.comm  # FIXME
        values = numpy.asarray(values).astype(IntType)
        self.values = values.reshape(iterset.total_size, arity)
        self.arity = arity
        self.shape = self.values.shape
        if offset is not None:
            self.offset = offset.reshape(self.arity)

    def validate(self, iterset, toset):
        return iterset == self.iterset and toset == self.toset

    @cached_property
    def _codegen_info_(self):
        return Map.Info(shape=self.shape, offset=self.offset, dtype=self.dtype,
                        setinfo=self.iterset._codegen_info_)

    @cached_property
    def _parloop_args_(self):
        return (self.values.ctypes.data, )

class MixedMap(object):

    def validator(self, maps):
        assert all(isinstance(m, (Map, type(None))) for m in maps)

    @check_args(validator)
    def __init__(self, maps):
        self.maps = tuple(maps)
        self.comm = maps[0].comm  # FIXME

    @cached_property
    def _parloop_argtypes_(self):
        return tuple(chain(*(m._parloop_argtypes_ for m in self.maps)))

    @cached_property
    def _parloop_args_(self):
        return tuple(chain(*(m._parloop_args_ for m in self.maps)))

class Sparsity(object):
    def validator(self, datasets, maps, *, block_sparse=True):
        assert all(isinstance(d, DataSet) for d in datasets)
        assert all(isinstance(m, Map) for m in chain(*maps))
        rset, cset = datasets
        for rmap, cmap in maps:
            assert rmap.iterset == cmap.iterset
            assert rmap.toset == rset.iterset
            assert cmap.toset == cset.iterset

    @check_args(validator)
    def __init__(self, datasets, maps, *, block_sparse=True):
        self.datasets = tuple(datasets)
        rowmaps, colmaps = zip(*maps)
        self.rowmaps = tuple(rowmaps)
        self.colmaps = tuple(colmaps)
        self.block_sparse = block_sparse
        self.shape = tuple(int(numpy.prod(d.shape, dtype=int)) for d in self.datasets)

class MixedSparsity(object):
    def validator(self, sparsities, shape):
        assert all(isinstance(s, Sparsity) for s in sparsities)
        assert len(sparsities) == int(numpy.prod(shape))

    @check_args(validator)
    def __init__(self, sparsities, shape):
        self.sparsities = numpy.asarray(sparsities, dtype=object).reshape(shape)


text:
"""
Copyright (c) 2018-2022 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import pytest

pytest.importorskip('openvino.tools.accuracy_checker.launcher.dlsdk_launcher')
import cv2
import numpy as np

from pathlib import Path
from unittest.mock import PropertyMock
from openvino.tools.accuracy_checker.config import ConfigError
from openvino.tools.accuracy_checker.launcher import DLSDKLauncher
from openvino.tools.accuracy_checker.launcher.dlsdk_launcher_config import DLSDKLauncherConfigValidator
from openvino.tools.accuracy_checker.launcher.launcher import create_launcher
from tests.common import update_dict
from openvino.tools.accuracy_checker.data_readers import DataRepresentation
from openvino.tools.accuracy_checker.utils import contains_all
try:
    import ngraph as ng
except ImportError:
    ng = None

def no_available_myriad():
    return True

def has_layers():
    try:
        from openvino.inference_engine import IENetwork
        return hasattr(IENetwork, 'layers')
    except Exception:
        return False

@pytest.fixture()
def mock_inference_engine(mocker):
    mocker.patch('openvino.inference_engine.IECore')
    mocker.patch('openvino.inference_engine.IENetwork')

@pytest.fixture()
def mock_inputs(mocker):
    mocker.patch(
        'openvino.tools.accuracy_checker.launcher.input_feeder.InputFeeder._parse_inputs_config',
        return_value=({}, ['data'], None)
    )

@pytest.fixture()
def mock_affinity_map_exists(mocker):
    mocker.patch('pathlib.Path.exists', return_value=True)
    mocker.patch('os.path.exists', return_value=True)
    mocker.patch('pathlib.Path.is_file', return_value=True)

    def side_effect(filename):
        if filename == './affinity_map.yml':
            return True
        else:
            Path.exists(filename)
    mocker.side_effect = side_effect

def get_dlsdk_test_model(models_dir, config_update=None):
    config = {
        'framework': 'dlsdk',
        'weights': str(models_dir / 'SampLeNet.bin'),
        'model': str(models_dir / 'SampLeNet.xml'),
        'device': 'CPU',
        'adapter': 'classification',
    }
    if config_update:
        config.update(config_update)

    return create_launcher(config, model_name='SampLeNet')

def get_dlsdk_test_blob(models_dir, config_update=None):
    config = {
        'framework': 'dlsdk',
        'model': str(models_dir / 'SampLeNet.blob'),
        'device': 'MYRIAD',
        'adapter': 'classification',
    }
    if config_update:
        config.update(config_update)

    return create_launcher(config)

def get_onnx_test_model(model_dir, config_update=None):
    config = {
        'framework': 'dlsdk',
        'model': str(model_dir / 'samplenet.onnx'),
        'device': 'CPU',
        'adapter': 'classification'
    }
    if config_update:
        config.update(config_update)

    return create_launcher(config)

def get_image(image_path, input_shape):
    _, _, h, w = input_shape
    img_raw = cv2.imread(str(image_path))

    return DataRepresentation(cv2.resize(img_raw, (w, h)))

class TestDLSDKLauncherInfer:
    def test_infer(self, data_dir, models_dir):
        dlsdk_test_model = get_dlsdk_test_model(models_dir)
        image = get_image(data_dir / '1.jpg', dlsdk_test_model.inputs['data'].shape)
        input_blob = np.transpose([image.data], (0, 3, 1, 2))
        result = dlsdk_test_model.predict([{'data': input_blob.astype(np.float32)}], [image.metadata])
        assert dlsdk_test_model.output_blob == 'fc3'

        assert np.argmax(result[0][dlsdk_test_model.output_blob]) == 7
        assert image.metadata['input_shape'] == {'data': [1, 3, 32, 32]}

    def test_launcher_creates(self, models_dir):
        assert get_dlsdk_test_model(models_dir).inputs['data'].shape == [1, 3, 32, 32]

    def test_infer_with_additional_outputs(self, models_dir):
        dlsdk_test_model = get_dlsdk_test_model(models_dir, {'outputs': ['fc1', 'fc2']})
        outputs = list(dlsdk_test_model.network.outputs.keys())

        assert contains_all(outputs, ['fc1', 'fc2', 'fc3'])
        assert dlsdk_test_model.output_blob == 'fc3'

    def test_dlsdk_launcher_set_batch_size(self, models_dir):
        dlsdk_test_model = get_dlsdk_test_model(models_dir, {'batch': 2})
        assert dlsdk_test_model.batch == 2

    @pytest.mark.skipif(no_available_myriad(), reason='no myriad device in the system')
    def test_dlsdk_launcher_import_network(self, data_dir, models_dir):
        dlsdk_test_model = get_dlsdk_test_blob(models_dir)
        image = get_image(data_dir / '1.jpg', dlsdk_test_model.inputs['data'].shape)
        input_blob = np.transpose([image.data], (0, 3, 1, 2))
        result = dlsdk_test_model.predict([{'data': input_blob.astype(np.float32)}], [image.metadata])
        assert dlsdk_test_model.output_blob == 'fc3'

        assert np.argmax(result[0][dlsdk_test_model.output_blob]) == 7
        assert image.metadata['input_shape'] == {'data': [1, 3, 32, 32]}

    def test_dlsdk_launcher_model_search(self, models_dir):
        config_update = {
            'model': str(models_dir),
            'weights': str(models_dir)
        }
        dlsdk_test_model = get_dlsdk_test_model(models_dir, config_update)
        assert dlsdk_test_model._model == models_dir / 'SampLeNet.xml'
        assert dlsdk_test_model._weights == models_dir / 'SampLeNet.bin'

    def test_dlsdk_onnx_import(self, data_dir, models_dir):
        dlsdk_test_model = get_onnx_test_model(models_dir)
        image = get_image(data_dir / '1.jpg', dlsdk_test_model.inputs['data'].shape)
        input_blob = np.transpose([image.data], (0, 3, 1, 2))
        dlsdk_test_model.predict([{'data': input_blob.astype(np.float32)}], [image.metadata])
        assert dlsdk_test_model.output_blob == 'fc3'

@pytest.mark.skipif(ng is None and not has_layers(), reason='no functionality to set affinity')
class TestDLSDKLauncherAffinity:
    @pytest.mark.usefixtures('mock_affinity_map_exists')
    def test_dlsdk_launcher_valid_affinity_map(self, mocker, models_dir):
        affinity_map = {'conv1': 'GPU'}
        if not has_layers():
            affinity_map.update({
                'conv1/Dims294/copy_const': 'GPU'
            })

        mocker.patch(
            'openvino.tools.accuracy_checker.launcher.dlsdk_launcher.read_yaml', return_value=affinity_map
        )

        dlsdk_test_model = get_dlsdk_test_model(models_dir, {
            'device': 'HETERO:CPU,GPU', 'affinity_map': './affinity_map.yml'
        })
        if has_layers():
            layers = dlsdk_test_model.network.layers
            for key, value in affinity_map.items():
                assert layers[key].affinity == value
        else:
            ng_function = ng.function_from_cnn(dlsdk_test_model.network)
            for node in ng_function.get_ordered_ops():
                if node.get_friendly_name() != 'conv1':
                    continue
                assert node.get_friendly_name() in affinity_map
                assert node.get_rt_info()['affinity'] == affinity_map[node.get_friendly_name()]

    @pytest.mark.usefixtures('mock_file_exists')
    def test_dlsdk_launcher_affinity_map_invalid_device(self, mocker, models_dir):
        affinity_map = {'conv1': 'GPU'}

        mocker.patch(
            'openvino.tools.accuracy_checker.launcher.dlsdk_launcher.read_yaml', return_value=affinity_map
        )

        with pytest.raises(ConfigError):
            get_dlsdk_test_model(models_dir, {'device': 'HETERO:CPU,CPU', 'affinity_map': './affinity_map.yml'})

    @pytest.mark.usefixtures('mock_file_exists')
    def test_dlsdk_launcher_affinity_map_invalid_layer(self, mocker, models_dir):
        affinity_map = {'none-existing-layer': 'CPU'}

        mocker.patch(
            'openvino.tools.accuracy_checker.launcher.dlsdk_launcher.read_yaml', return_value=affinity_map
        )

        with pytest.raises(ConfigError):
            get_dlsdk_test_model(models_dir, {'device': 'HETERO:CPU,CPU', 'affinity_map': './affinity_map.yml'})

@pytest.mark.usefixtures('mock_path_exists', 'mock_inference_engine', 'mock_inputs')
class TestDLSDKLauncher:
    def test_dlsdk_launcher_device_config_config_not_dict_like(self, models_dir):
        device_config = 'ENFORCE_BF16'

        with pytest.raises(ConfigError):
            get_dlsdk_test_model(models_dir, {'device_config': device_config})

    def test_dlsdk_launcher_device_config_device_unknown(self, models_dir):
        device_config = {'device': {'ENFORCE_BF16': 'NO'}}

        with pytest.warns(Warning):
            get_dlsdk_test_model(models_dir, {'device_config': device_config})

    def test_dlsdk_launcher_device_config_one_option_for_device_is_not_dict(self, models_dir):
        device_config = {'CPU': {'ENFORCE_BF16': 'NO'}, 'GPU': 'ENFORCE_BF16'}

        with pytest.warns(Warning):
            get_dlsdk_test_model(models_dir, {'device_config': device_config})

    def test_dlsdk_launcher_device_config_one_option_is_not_binding_to_device(self, models_dir):
        device_config = {'CPU': {'ENFORCE_BF16': 'NO'}, 'ENFORCE_BF16': 'NO'}

        with pytest.warns(Warning):
            get_dlsdk_test_model(models_dir, {'device_config': device_config})

@pytest.mark.usefixtures('mock_path_exists', 'mock_inputs', 'mock_inference_engine')
class TestDLSDKLauncherConfig:
    def setup(self):
        self.launcher = {
            'model': 'foo.xml',
            'weights': 'foo.bin',
            'device': 'CPU',
            'framework': 'dlsdk',
            'adapter': 'classification',
        }
        self.config = DLSDKLauncherConfigValidator('dlsdk_launcher', fields=DLSDKLauncher.parameters())

    def test_hetero_correct(self):
        self.config.validate(update_dict(self.launcher, device='HETERO:CPU'))
        self.config.validate(update_dict(self.launcher, device='HETERO:CPU,CPU'))

    def test_hetero_endswith_comma(self):
        with pytest.raises(ConfigError):
            self.config.validate(update_dict(self.launcher, device='HETERO:CPU,CPU,'))

    def test_normal_multiple_devices(self):
        with pytest.raises(ConfigError):
            self.config.validate(update_dict(self.launcher, device='CPU,CPU'))

    def test_hetero_empty(self):
        with pytest.raises(ConfigError):
            self.config.validate(update_dict(self.launcher, device='HETERO:'))

    def test_normal(self):
        self.config.validate(update_dict(self.launcher, device='CPU'))

    def test_missed_model_in_create_dlsdk_launcher_raises_config_error_exception(self):
        config = {'framework': 'dlsdk', 'weights': 'custom', 'adapter': 'classification', 'device': 'cpu'}

        with pytest.raises(ConfigError):
            create_launcher(config)

    def test_missed_adapter_in_create_dlsdk_launcher_raises_config_error_exception(self):
        launcher_config = {'framework': 'dlsdk', 'model': 'custom', 'weights': 'custom'}

        with pytest.raises(ConfigError):
            create_launcher(launcher_config)

    def test_undefined_str_adapter_in_create_dlsdk_launcher_raises_config_error_exception(self):
        launcher_config = {'framework': 'dlsdk', 'model': 'custom', 'weights': 'custom', 'adapter': 'undefined_str'}

        with pytest.raises(ConfigError):
            create_launcher(launcher_config)

    def test_empty_dir_adapter_in_create_dlsdk_launcher_raises_config_error_exception(self):
        launcher_config = {'framework': 'dlsdk', 'model': 'custom', 'weights': 'custom', 'adapter': {}}

        with pytest.raises(ConfigError):
            create_launcher(launcher_config)

    def test_missed_type_in_dir_adapter_in_create_dlsdk_launcher_raises_config_error_exception(self):
        launcher_config = {'framework': 'dlsdk', 'model': 'custom', 'weights': 'custom', 'adapter': {'key': 'val'}}

        with pytest.raises(ConfigError):
            create_launcher(launcher_config)

    def test_undefined_type_in_dir_adapter_in_create_dlsdk_launcher_raises_config_error_exception(self):
        launcher_config = {
            'framework': 'dlsdk',
            'model': 'custom',
            'weights': 'custom',
            'adapter': {'type': 'undefined'}
        }

        with pytest.raises(ConfigError):
            create_launcher(launcher_config)

    def test_dlsdk_launcher(self):
        launcher = {
            'framework': 'dlsdk', 'model': 'custom', 'weights': 'custom', 'adapter': 'ssd', 'device': 'cpu'
        }
        create_launcher(launcher, model_name='custom')

    def test_dlsdk_launcher_model_with_several_image_inputs_raise_value_error(self, mocker):
        launcher_config = {'framework': 'dlsdk', 'model': 'custom', 'weights': 'custom', 'adapter': {'key': 'val'}}

        with pytest.raises(ValueError):
            mocker.patch(
                'openvino.tools.accuracy_checker.launcher.dlsdk_launcher.DLSDKLauncher.inputs',
                new_callable=PropertyMock(return_value={'data1': [3, 227, 227], 'data2': [3, 227, 227]})
            )
            create_launcher(launcher_config)

    def test_dlsdk_launcher_model_no_image_inputs_raise_value_error(self):
        launcher_config = {'framework': 'dlsdk', 'model': 'custom', 'weights': 'custom', 'adapter': {'key': 'val'}}

        with pytest.raises(ValueError):
            create_launcher(launcher_config)

ext: py

reason: import numpy


text:
import torch,os
import paddle
import numpy as np
from resnet_paddle import ResNetV1C
from apcnet_paddle import APCHead
from fcnhead_paddle import FCNHead

def isIn(key, str):
    for ele in key:
        if ele in str:
            return True

def rename_state_dict(paddle_dict, torch_dict):
    result = {}
    _ = {}
    skip_params = ['batches_tracked', 'fc.weight', 'fc.bias']
    for k, v in torch_dict.items():
        if isIn(skip_params, k):
            print('---skip batches_tracked---')
        else:

            _[k] = v.cpu().detach().numpy()
    torch_dict = _

    assert len(torch_dict) == len(paddle_dict)
    for paddle_param, torch_param in zip(paddle_dict.items(), torch_dict.items()):
        k1, v1 = paddle_param
        k2, v2 = torch_param
        v1 = v1.numpy()
        v2 = v2
        print('{} shape {}, {} shape {}'.format(k1, v1.shape, k2, v2.shape))
        assert v1.shape == v2.shape
        result[k1] = v2
    print(len(torch_dict))
    print(len(paddle_dict))
    return result

def trans():
    path = './pretrained/resnet101_v1c-e67eebb6.pth'
    torch_dict = torch.load(path)
    paddle_dict = {}
    paddle_dict['state_dict'] = {}
    paddle_dict['meta'] = torch_dict['meta']

    model = ResNetV1C()
    paddle_dict['state_dict'] = rename_state_dict(model.state_dict(), torch_dict['state_dict'])

    paddle.save(paddle_dict, './pretrained/resnet101_aistudio.pdparams')
trans()

max_stars_repo_name: ImportPaddle/APCNet

max_issues_repo_name: ImportPaddle/APCNet

max_forks_repo_name: ImportPaddle/APCNet


text:
# TODO: variable length batching

import numpy as np

from openchem.utils.graph import Graph

from rdkit import Chem

from torch.utils.data import Dataset
from openchem.data.utils import read_smiles_property_file, sanitize_smiles

class GraphDataset(Dataset):
    def __init__(self, get_atomic_attributes, node_attributes, filename,
                 cols_to_read, delimiter=',', get_bond_attributes=None, edge_attributes=None):
        super(GraphDataset, self).__init__()
        assert (get_bond_attributes is None) == (edge_attributes is None)
        data_set = read_smiles_property_file(filename, cols_to_read,
                                             delimiter)
        data = data_set[0]
        target = data_set[1:]
        clean_smiles, clean_idx = sanitize_smiles(data)
        target = np.array(target).T
        max_size = 0
        for sm in clean_smiles:
            mol = Chem.MolFromSmiles(sm)
            if mol.GetNumAtoms() > max_size:
                max_size = mol.GetNumAtoms()
        self.target = target[clean_idx, :]
        self.graphs = []
        self.node_feature_matrix = []
        self.adj_matrix = []
        for sm in clean_smiles:
            graph = Graph(sm, max_size, get_atomic_attributes,
                          get_bond_attributes)
            self.node_feature_matrix.append(
                graph.get_node_feature_matrix(node_attributes, max_size))
            if get_bond_attributes is None:
                self.adj_matrix.append(graph.adj_matrix)
            else:
                self.adj_matrix.append(
                    graph.get_edge_attr_adj_matrix(edge_attributes, max_size)
                )
        self.num_features = self.node_feature_matrix[0].shape[1]

    def __len__(self):
        return len(self.target)

    def __getitem__(self, index):
        sample = {'adj_matrix': self.adj_matrix[index].astype('float32'),
                  'node_feature_matrix':
                      self.node_feature_matrix[index].astype('float32'),
                  'labels': self.target[index].astype('float32')}
        return sample


text:
from candiy_lemon import lemon
import sys, os
import numpy as np
import pandas as pd

# List of dictionaries to keep track of parts of the file

# Key: reference pdbid, Value: path to .mmtf file
pathDict = {}
# Key: reference pdbID, Value: chemical ID of ligand bound to reference protein for removal
referenceLigandDict = {}
# Key: reference pdbID, Value: list of sm or non-sm protein
referenceDict = {}
# Key: reference pdbID, Value: list of proteins to aling to reference (like in pinc)
alignProtDict = {}
# Key: align protein pdbID, Value: ligand chemical ID code for ligand removal
alignProtLigandDict = {}
# Key: pdbID, Value: chemical id for SM ligand
pdbIDSMDict = {}
# Key: pdbID, Value: tuple(resCode, chainID, residue ID)
pdbIDNonSMDict = {}
# Key: pdbID, Value: chemical id for SM ligand
noAlignSMDict = {}
# Key: pdbID, Value: tuple(resCode, chainID, residue ID)
noAlignNonSMDict = {}

# Maximum distance allowed between a residue and the ligand
maximumResidueDistance = 25.0

entries = set()

# Method for getting binding affinity information
# Input is list of tuple with each tuple -> (pdb_id,lig_code)
def get_bind_affinty(bind_tup_list):
    # We can get the csv file directly from a url using pandas
    # Use custom columns due to some of the columns not being needed
    binding_moad_url = "http://bindingmoad.org/files/csv/every_bind.csv"
    binding_moad_data = pd.read_csv(binding_moad_url,usecols=[0,2,3,4,5,7,8,9],header=None)

    # Set the columns manually because there is no header given
    binding_moad_data.columns = ["ec","pdb_id","lig_code","validity","affnty_type","affnty_val","affnty_units","SMILES"]

    # Create our dictionaries and varaibles here
    pdb_id_dict = {}
    lig_id_dict = {}
    cur_pdb_id = ""

    # Go through each of the row and add the data to the dictionaries
    for index,row in binding_moad_data.iterrows():
        if pd.isna(row["pdb_id"]) == False:
            cur_pdb_id = row["pdb_id"]
        elif (pd.isna(row["lig_code"]) == False) and (pd.isna(row["affnty_type"]) == False) and (pd.isna(row["affnty_val"]) == False) and (pd.isna(row["affnty_units"]) == False):
            simple_lig_code = row["lig_code"].split(":")[0]
            simple_lig_code = simple_lig_code.split(" ")

            for code in simple_lig_code:

                if pdb_id_dict.get(cur_pdb_id,0) == 0:
                    pdb_id_dict[cur_pdb_id] = [code]
                else:
                    if simple_lig_code not in pdb_id_dict[cur_pdb_id]:
                        pdb_id_dict[cur_pdb_id].append(code)

                if lig_id_dict.get((cur_pdb_id,code),0) == 0:
                    lig_id_dict[(cur_pdb_id,code)] = [[row["lig_code"],row["affnty_type"],row["affnty_val"],row["affnty_units"]]]
                else:
                    lig_id_dict[(cur_pdb_id,code)].append([row["lig_code"],row["affnty_type"],row["affnty_val"],row["affnty_units"]])

    #To access the ligand we need to first check to see if the ligand exists in pdb_id_dict
    # If this is one of the ligands availibe, then we access the tuple in the lig_id_dict with the binding affinity
    ret_dict = {}
    for pair in bind_tup_list:
        pdb_id = pair[0]
        lig_code = pair[1]

        if lig_id_dict.get((pdb_id,lig_code),0) == 0:
            ret_dict[(pdb_id,lig_code)] = "No Binding Information Found!"
        else:
            ret_dict[(pdb_id,lig_code)] = lig_id_dict[(pdb_id,lig_code)]

    # Return the final dict, key -> (pdb_id,lig_code) and value is information on that ligand interaction
    return ret_dict

# Method for parsing a formated input file
def parse_input_file(fname):
    # Open file and initialize flags to 0
    f = open(fname,"r")
    curRefPdbID = ""
    flag = 0

    global maximumResidueDistance
    global referenceLigandDict
    global referenceDict
    global alignProtDict

    for line in f:
        # Check to see if the line contains any of the tags
        # Set appropriate flags if it does
        if line.startswith("#"):
            continue
        elif line.startswith("@<reference>"):
            flag = 1
            continue
        elif line.startswith("@<align_prot>"):
            flag = 2
            continue
        elif line.startswith("@<align_sm_ligands>"):
            flag = 3
            continue
        elif line.startswith("@<align_non_sm_ligands>"):
            flag = 4
            continue
        elif line.startswith("@<no_align_sm_ligands>"):
            flag = 5
            continue
        elif line.startswith("@<no_align_non_sm_ligands>"):
            flag = 6
            continue
        elif line.startswith("@<maximum_residue_distance>"):
            flag = 7
            continue
        elif line.startswith("@<end>"):
            flag = 0
            continue
        elif line.startswith("@"):
            print("Invalid tag (@<>) detected, please check all tags are in the proper format")
            sys.exit()

        # If the line does not contain a flag
        # Add info to appropriate dictionary based of set flags
        if flag == 1:
            # Error Check
            if len(line.split(" ")) < 2:
                print("Invalid input under @<reference> tag, please check to ensure proper input")
                sys.exit(1)

            pdbID = line.split(" ")[0].strip().upper()
            path = line.split(" ")[1].strip()
            curRefPdbID = pdbID
            pathDict[pdbID] = path

            if len(line.split(" ")) >= 3:
                chemID = line.split(" ")[2].strip().upper()
                referenceLigandDict[pdbID] = chemID

        elif flag == 2:
            if len(line.split(" ")) < 2:
                print("Invalid input under @<align_prot> tag, please check to ensure proper input")
                sys.exit(1)

            pdbID = line.split(" ")[0].strip()
            chemID = line.split(" ")[1].strip()

            if alignProtDict.get(curRefPdbID,0) == 0:
                alignProtDict[curRefPdbID] = [pdbID]
            else:
                alignProtDict[curRefPdbID].append(pdbID)

            if alignProtLigandDict.get(pdbID, 0) == 0:
                alignProtLigandDict[pdbID] = [chemID]
            else:
                alignProtLigandDict[pdbID].append(chemID)

                entries.add(pdbID)

        elif flag == 3:
            if len(line.split(" ")) < 2:
                print("Invalid input under @<align_sm_ligands> tag, please check to ensure proper input")
                sys.exit(1)

            pdbID = line.split(" ")[0].strip()
            chemID = line.split(" ")[1].strip()

            if referenceDict.get(curRefPdbID,0) == 0:
                referenceDict[curRefPdbID] = [pdbID]
            else:
                referenceDict[curRefPdbID].append(pdbID)

            if pdbIDSMDict.get(pdbID,0) == 0:
                pdbIDSMDict[pdbID] = [chemID]
            else:
                pdbIDSMDict[pdbID].append(chemID)

            entries.add(pdbID)

        elif flag == 4:
            if len(line.split(" ")) < 3:
                print("Invalid input under @<align_non_sm_ligands> tag, please check to ensure proper input")
                sys.exit(1)

            pdbID = line.split(" ")[0].strip().upper()
            residueCode = line.split(" ")[1].split("-")[0].strip().upper()
            chainID = line.split(" ")[1].split("-")[1].strip().upper()
            residueID = line.split(" ")[1].split("-")[2].strip().upper()

            if referenceDict.get(curRefPdbID,0) == 0:
                referenceDict[curRefPdbID] = [pdbID]
            else:
                referenceDict[curRefPdbID].append(pdbID)

            if pdbIDNonSMDict.get(pdbID,0) == 0:
                pdbIDNonSMDict[pdbID] = [tuple([residueCode,chainID,residueID])]
            else:
                pdbIDNonSMDict[pdbID].append(tuple([residueCode,chainID,residueID]))

            entries.add(pdbID)

        elif flag == 5:
            if len(line.split(" ")) < 2:
                print("Invalid input under @<no_align_sm_ligands> tag, please check to ensure proper input")
                sys.exit(1)

            pdbID = line.split(" ")[0].strip().upper()
            chemID = line.split(" ")[1].strip().upper()

            if noAlignSMDict.get(pdbID,0) == 0:
                noAlignSMDict[pdbID] = [chemID]
            else:
                noAlignSMDict[pdbID].append(chemID)

            entries.add(pdbID)

        elif flag == 6:
            if len(line.split(" ")) < 3:
                print("Invalid input under @<no_align_non_sm_ligands> tag, please check to ensure proper input")
                sys.exit(1)

            pdbID = line.split(" ")[0].strip().upper()
            residueCode = line.split(" ")[1].split("-")[0].strip().upper()
            chainID = line.split(" ")[1].split("-")[1].strip()
            residueID = line.split(" ")[1].split("-")[2].strip()

            if noAlignNonSMDict.get(pdbID,0) == 0:
                noAlignNonSMDict[pdbID] = [tuple([residueCode,chainID,residueID])]
            else:
                noAlignNonSMDict[pdbID].append(tuple([residueCode,chainID,residueID]))

            entries.add(pdbID)

        elif flag == 7:
            maximumResidueDistance = float(line.strip())

# Get from the command line
# for testing we also can hard set the path
if len(sys.argv) > 4:
    input_file_path = sys.argv[1]
    hadoop_path = sys.argv[2]
    cores = int(sys.argv[3])
    outdir = sys.argv[4]
else:
    #TODO change this if needed for testing
    print("You must give the input file, path to RCSB hadoop files, number of cores, and working directory")
    sys.exit(1)

# Define Lemon workflow class
class MyWorkflow(lemon.Workflow):
    def __init__(self):
        lemon.Workflow.__init__(self)

        self.reference_structures = {}
        for key, value in pathDict.items():
            self.reference_structures[key] = lemon.open_file(value)

        self.noAlignSMDict = noAlignSMDict

        self.outdir = outdir

        self.write_all_proteins = False

        self.maximumResidueDistance = maximumResidueDistance

    def extact_ligand_write(self, entry, pdbid, ligand_code):
        ligand_name = ligand_code
        ligand_chain = ""
        ligand_altloc = "A"

        if ligand_code.find(":") != -1:
            ligand_split = ligand_code.split(":")
            ligand_chain = ligand_split[0]
            ligand_name = ligand_split[1]

        if ligand_name.find("_") != -1:
            ligand_split = ligand_name.split("_")
            ligand_name = ligand_split[0]
            ligand_altloc = ligand_split[1]

        ligand_ids = ""

        if ligand_name.startswith("+"):
            ids_to_use = set()
            for my_id in ligand_name.split("+"):
                if my_id.isdigit():
                    ids_to_use.add(int(my_id))

            ligand_ids = lemon.select_residue_ids(entry, ids_to_use)

            ligand_ids = lemon.prune_identical_residues(entry, ligand_ids)
            if ligand_chain:
                ligand_ids = lemon.has_property(entry, ligand_ids, "chainname", lemon.Property(ligand_chain))

            protein = lemon.Frame()
            ligand = lemon.Frame()

            if not os.path.isdir(self.outdir + "/" + pdbid):
                os.mkdir(self.outdir + "/" + pdbid)

            lemon.separate_protein_and_ligands(entry, ligand_ids, self.maximumResidueDistance, protein, ligand, ligand_altloc)
            out_base = self.outdir + "/" + pdbid + "/" + pdbid + "_" + ligand_name

            lemon.write_file(protein, out_base + ".pdb")
            lemon.write_file(ligand, out_base + ".sdf")

            return

        rns = set()
        rns.add(lemon.ResidueName(ligand_name))
        ligand_ids = lemon.select_specific_residues(entry, rns)

        if ligand_chain:
            ligand_ids = lemon.has_property(entry, ligand_ids, "chainname", lemon.Property(ligand_chain))

        ligand_ids = lemon.prune_identical_residues(entry, ligand_ids)

        for ligand_id in ligand_ids:
            protein = lemon.Frame()
            ligand = lemon.Frame()

            if not os.path.isdir(self.outdir + "/" + pdbid):
                os.mkdir(self.outdir + "/" + pdbid)

            lemon.separate_protein_and_ligand(entry, ligand_id, self.maximumResidueDistance, protein, ligand, ligand_altloc)
            out_base = self.outdir + "/" + pdbid + "/" + pdbid + "_" + ligand_name

            lemon.write_file(protein, out_base + ".pdb")
            lemon.write_file(ligand, out_base + ".sdf")

            if len(ligand_ids) > 1:
                print("WARNING:It appears that {} in {}".format(ligand_code, pdbid), file=sys.stderr)
                print("        is defined multiple times in the same biological assembly,", file=sys.stderr)
                print("        only keeping the first time it appears", file=sys.stderr)
                break

    def worker(self, entry, pdbid):
        # Define and assign the reference pdbid
        refpdbid = ""
        # mode is 0 unassigned, 1 for alignment for protein, 2 for alignment for ligand
        mode = 0

        # Check for pdbID as a protein to be aligned (like in PINC)
        for key, value in alignProtDict.items():
            if pdbid in value:
                refpdbid = key
                mode = 1

        # Check for protein-ligand pair for alignment
        for key, value in referenceDict.items():
            if pdbid in value:
                refpdbid = key
                mode = 2

        if mode == 0:
            for ligand_code in self.noAlignSMDict.get(pdbid, []):
                self.extact_ligand_write(entry, pdbid, ligand_code)
            return pdbid + " no alignment\n"

        elif mode == 1:
            # If we need to align to a protein (like in PINC)
            alignment = lemon.TMscore(entry, self.reference_structures[refpdbid])
            positions = entry.positions()
            lemon.align(positions, alignment.affine)

            lemon.write_file(entry, self.outdir + "/" + refpdbid + "_" + pdbid + ".pdb")

            return "Align Protein: " + refpdbid + "_" + pdbid + " to " + refpdbid + " with score of " + str(alignment.score) + "\n"

        elif mode == 2:
            # If we are doing ligand alignment, that can be done here
            # Get a list of the ligands associated with the protein we are trying to align
            SM_ligandList = pdbIDSMDict.get(pdbid, [])
            Non_SM_ligandList = pdbIDNonSMDict.get(pdbid, [])

            alignment = lemon.TMscore(entry, self.reference_structures[refpdbid])
            positions = entry.positions()
            lemon.align(positions, alignment.affine)

            if len(SM_ligandList) > 0:

                for ligand_code in SM_ligandList:
                    self.extact_ligand_write(entry, pdbid, ligand_code)
            return "Align Protein: " + pdbid + " to " + refpdbid + " with score of " + str(alignment.score) + "\n"

    def finalize(self):
        pass

# Parse the input file
parse_input_file(input_file_path)
print(pathDict)
# Initilize the workflow
wf = MyWorkflow()

# TODO Get these from the command-line or ask the user
lemon.launch(wf, hadoop_path, cores, entries)

max_stars_repo_name: frodofine/lemon

max_issues_repo_name: frodofine/lemon

max_forks_repo_name: frodofine/lemon


text:
import hashlib
import logging
import math
import time
from abc import ABC
from typing import List, Iterable, Union, Optional

import ConfigSpace as CS
import numpy as np
from ConfigSpace import hyperparameters as CSH

from pyPDP.utils.typing import SelectedHyperparameterType

class ConfigSpaceHolder(ABC):
    def __init__(self, config_space: CS.ConfigurationSpace, *, seed: Union[None, int, bool] = None):
        self.logger = logging.getLogger(self.__class__.__name__)
        self.logger.debug(f"{self.__class__.__name__}: Initial Seed = {seed}")

        if seed is True:
            # Use existing config_space
            self.logger.debug(f"{self.__class__.__name__}: Seed = Use existing")
            self.config_space = config_space
            return
        elif seed is None:
            # Use random seed
            seed = int(time.time() * 1000)
        else:
            # Use seed
            # Hash of class prevents using the exact same seed for every step (e.g. Sampler, ICE, ...)
            h = hashlib.md5(self.__class__.__name__.encode("latin"), usedforsecurity=False)
            seed = int.from_bytes(h.digest(), "big") + seed
        self.logger.debug(f"{self.__class__.__name__}: Final Seed = {seed % 2 ** 31}")
        self.config_space = copy_config_space(config_space, seed=seed % 2 ** 31)

    def sample_random_configuration(self, n: int) -> List[CS.Configuration]:
        return self.config_space.sample_configuration(n)

def config_list_to_array(X: Union[List[CS.Configuration], List[Iterable[float]], np.ndarray]) -> np.ndarray:
    if isinstance(X, (tuple, list)):
        if len(X) == 0:
            X = []
        elif isinstance(X[0], CS.Configuration):
            X = [config.get_array() for config in X]
    return np.asarray(X)

def get_selected_idx(selected_hyperparameter: Iterable[CSH.Hyperparameter],
                     config_space: CS.ConfigurationSpace) -> List[int]:
    return [
        config_space.get_idx_by_hyperparameter_name(hp.name)
        for hp in selected_hyperparameter
    ]

def scale_float(
        value: float,
        cs: CS.ConfigurationSpace,
        hp: CSH.NumericalHyperparameter
) -> float:
    cs_hp = cs.get_hyperparameter(hp.name)
    if cs_hp.log:
        log_lower = np.log(cs_hp.lower)
        log_upper = np.log(cs_hp.upper)
        value = np.log(value)
        normalized_value = (value - log_lower) / (log_upper - log_lower)
    else:
        normalized_value = (value - cs_hp.lower) / (cs_hp.upper - cs_hp.lower)
    normalized_value = np.minimum(1, normalized_value)
    normalized_value = np.maximum(0, normalized_value)
    return normalized_value

def unscale_float(
        normalized_value: float,
        cs: CS.ConfigurationSpace,
        hp: CSH.Hyperparameter
) -> float:
    cs_hp = cs.get_hyperparameter(hp.name)
    if cs_hp.log:
        log_lower = np.log(cs_hp.lower)
        log_upper = np.log(cs_hp.upper)
        value = normalized_value * (log_upper - log_lower) + log_lower
        value = math.exp(value)
    else:
        value = normalized_value * (cs_hp.upper - cs_hp.lower) + cs_hp.lower
    value = np.minimum(cs_hp.upper, np.maximum(cs_hp.lower, value))
    return value

def unscale(x: np.ndarray, cs: CS.ConfigurationSpace) -> np.ndarray:
    """
    assumes that the cs-features are located in the last dimension
    """
    x_copy = x.copy()
    num_dims = len(x.shape)
    for i, hp in enumerate(cs.get_hyperparameters()):
        if isinstance(hp, CSH.NumericalHyperparameter):
            if hp.log:
                unscaler = lambda values: \
                    np.minimum(hp.upper,
                               np.maximum(hp.lower,
                                          np.exp(values * (np.log(hp.upper) - np.log(hp.lower)) + np.log(hp.lower))))
            else:
                unscaler = lambda values: values * (hp.upper - hp.lower) + hp.lower
        elif isinstance(hp, CSH.CategoricalHyperparameter):
            unscaler = lambda values: np.asarray([hp.choices[k] for k in values])
            x_copy = x_copy.astype(object)
        else:
            raise TypeError(f"Currently not support hyperparameter-type {type(hp)}")

        if num_dims == 1:
            x_copy[i] = unscaler(x[i])
        elif num_dims == 2:
            x_copy[:, i] = unscaler(x[:, i])
        elif num_dims == 3:
            x_copy[:, :, i] = unscaler(x[:, :, i])
        else:
            raise Exception(f'Only up to three dimensions are supported in undo_normalize, but got {num_dims}')
    return x_copy

def get_stds(stds: Optional[np.ndarray] = None, variances: Optional[np.ndarray] = None) -> np.ndarray:
    if stds is None and variances is None or (stds is not None and variances is not None):
        raise RuntimeError("Requires either stds or variances")

    if variances is not None:
        stds = np.sqrt(variances)
    return stds

def get_hyperparameters(hyperparameters: Optional[SelectedHyperparameterType],
                        cs: CS.ConfigurationSpace) -> List[CSH.Hyperparameter]:
    if hyperparameters is None:
        # None -> All hyperparameters in cs
        return list(cs.get_hyperparameters())
    elif isinstance(hyperparameters, CSH.Hyperparameter):
        # Single Hyperparameter
        return [hyperparameters]
    elif isinstance(hyperparameters, str):
        # Single Hyperparameter name
        return [cs.get_hyperparameter(hyperparameters)]
    else:
        # Either list of names or list of Hyperparameters
        hps = []
        for hp in hyperparameters:
            if isinstance(hp, str):
                hps.append(cs.get_hyperparameter(hp))
            elif isinstance(hp, CSH.Hyperparameter):
                hps.append(hp)
            else:
                raise TypeError(f"Could not identify hyperparameter {hp} (Type: {type(hp)})")
        return hps

def get_uniform_distributed_ranges(
        cs: Union[CS.ConfigurationSpace, Iterable[CSH.NumericalHyperparameter]],
        samples_per_axis: int = 100,
        scaled=False
) -> np.ndarray:
    """
    :param cs: Configuration_space to sample from
    :param samples_per_axis: Number of samples per axis
    :param scaled: if scaled: Ranges normalized between 0 and 1, otherwise Ranges are as give n in configspace
    :return: Shape: (num_hyperparameters, num_samples_per_axis)
    """
    ranges = []
    if isinstance(cs, CS.ConfigurationSpace):
        cs = cs.get_hyperparameters()
    for parameter in cs:
        assert isinstance(parameter, CSH.NumericalHyperparameter)
        if scaled:
            ranges.append(np.linspace(0, 1, num=samples_per_axis))
        else:
            if parameter.log:
                ranges.append(np.logspace(np.log10(parameter.lower), np.log10(parameter.upper), num=samples_per_axis))
            else:
                ranges.append(np.linspace(parameter.lower, parameter.upper, num=samples_per_axis))

    res = np.asarray(ranges)
    assert len(res) == len(cs)
    return res

def median_distance_between_points(X: np.ndarray) -> float:
    X = np.expand_dims(X, axis=0)
    dif = X - X.transpose((1, 0, 2))
    dif_2 = np.sum(np.square(dif), axis=2)
    distances = np.sqrt(dif_2)
    median = np.median(distances[distances != 0]).item()
    return median

def calculate_log_delta(nll: float, nll_root: float) -> float:
    return (nll_root - nll) / np.absolute(nll_root)

def convert_hyperparameters(
        hyperparameters: Union[str, CSH.Hyperparameter, Iterable[Union[CSH.Hyperparameter, str]]],
        config_space: CS.ConfigurationSpace
) -> List[CSH.Hyperparameter]:
    """
    Converts either
        * a single hyperparameter (CSH.Hyperparameter)
        * a single hyperparameter name (str)
        * an iterable (list, tuple, etc.) of hyperparameters (CSH.Hyperparameter)
        * an iterable of hyperparameter names (str)
    to a list of hyperparameters (CSH.Hyperparameter)
    """
    if isinstance(hyperparameters, (str, CSH.Hyperparameter)):
        hyperparameters = (hyperparameters,)
    hps = []
    for hp in hyperparameters:
        if isinstance(hp, str):
            hps.append(config_space.get_hyperparameter(hp))
        elif isinstance(hp, CSH.Hyperparameter):
            hps.append(hp)
        else:
            raise TypeError(f'Could not interpret hp: {hp}')
    return hps

def copy_config_space(cs: CS.ConfigurationSpace, *, seed=None) -> CS.ConfigurationSpace:
    # copy cs
    hp_dic = {}
    for hp in cs.get_hyperparameters():
        if isinstance(hp, CSH.NumericalHyperparameter):
            new_hp = hp.__class__(hp.name, lower=hp.lower, upper=hp.upper, log=hp.log)
            hp_dic[hp.name] = new_hp
        elif isinstance(hp, CSH.CategoricalHyperparameter):
            new_hp = hp.__class__(hp.name, choices=hp.choices[:])  # Copy choices
            hp_dic[hp.name] = new_hp  # TODO: Test copy categorical hp and unscaler
        else:
            raise TypeError(f"Currently not support hyperparameter-type {type(hp)}")

    # add new hp to new cs
    cs_copy = CS.ConfigurationSpace(seed=seed)
    for hp in hp_dic.values():
        cs_copy.add_hyperparameter(hp)

    return cs_copy

class ProgressDummy:
    """ Small dummy class that does nothing except existing """

    def update(self, n=1):
        pass

    def close(self):
        pass

    def refresh(self, nolock=False, lock_args=None):
        pass

max_stars_repo_name: dwoiwode/py-pdp-partitioner

max_issues_repo_name: dwoiwode/py-pdp-partitioner

max_forks_repo_name: dwoiwode/py-pdp-partitioner


text:
# PyTorch 实现 Word2Vec https://wmathor.com/index.php/archives/1435/
import timeit

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as tud

from collections import Counter
import numpy as np
import random
import scipy
from sklearn.metrics.pairwise import cosine_similarity

C = 3  # context window
K = 15  # number of negative samples pre positive
EPOCHS = 1
MAX_VOCAB_SIZE = 10000
EMBEDDING_SIZE = 100

# 调整BATCH_SIZE和DATALOADER_WORKER_NUMS让数据集产生和消费达到一个平衡 效果最佳
BATCH_SIZE = 16  # 每个次迭代从数据集中取出的数据多少，决定迭代轮次，另外考虑dataloader加载效率
DATALOADER_WORKER_NUMS = 4  # 本方案数据集采样CPU密集型，GPU加成不足，采用多个worker提高效率。dataloader的并行机制是生产者消费者模式？

LR = 1e-3
SEED = 10086

random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# 读取文本 拆分并按照单词索引
def encoder_word(text):
    # 构造 单词到单词出现次数的dict
    vocab_dict = dict(
        Counter(text).most_common(MAX_VOCAB_SIZE - 1)
    )
    vocab_dict['<UNK>'] = len(text) - np.sum(list(vocab_dict.values()))  # 没有编号的词全部统计到UNK上

    # 映射关系

    word2idx = {word: i for i, word in enumerate(vocab_dict.keys())}
    idx2word = {i: word for i, word in enumerate(vocab_dict.keys())}

    word_counts = np.array([count for count in vocab_dict.values()], dtype=float)
    word_freqs = word_counts / np.sum(word_counts)  # 词频

    return word2idx, idx2word, word_freqs

# DataLoader
class WordEmbeddingDataset(tud.Dataset):
    def __init__(self,
                 text,
                 word2idx,
                 word_freqs):
        """
        text: 拆分成单词的语料
        word2idx: 单词到索引号（编号）的映射
        word_freqs: 词频
        """
        super(WordEmbeddingDataset, self).__init__()

        # 编码后的语料
        self.text_encoded = [word2idx.get(word, word2idx['<UNK>']) for word in text]  # 把语料库用索引号编码
        self.text_encoded = torch.LongTensor(self.text_encoded)

        # 单词索引
        self.word2idx = word2idx
        # 词频
        scaled_word_freqs = word_freqs ** (3. / 4)  # 论文里要求频率变为原来的0.75次方 , 猜测是非均衡缩放下频率这样抽样会稍微平均一点
        self.word_freqs = torch.Tensor(scaled_word_freqs)

    def __len__(self):
        return len(self.text_encoded)

    def __getitem__(self, idx):
        """
        返回：
        - 中心词
        - 相关词 positive word
        - 随机采样得到的 negative word  ( K倍的positive word )
        """
        center_word = self.text_encoded[idx]

        positive_indices = list(range(idx - C, idx)) + list(range(idx + 1, idx + C + 1))
        positive_indices = [i % len(self.text_encoded) for i in positive_indices]  # 避免越界，取个余数

        positive_words = self.text_encoded[positive_indices]  # positive_indices是个列表，返回的会是个tensor

        # 多项式分布抽样 K倍正向词数量
        negative_words = torch.multinomial(self.word_freqs, K * positive_words.shape[0], True)

        # While 循环确保正负向词无交集(GPU下用tensor算交集
        while len(
                set(negative_words.cpu().numpy().tolist()) & set(positive_words.cpu().numpy().tolist())  # 取交集
        ) > 0:
            negative_words = torch.multinomial(self.word_freqs, K * positive_words.shape[0], True)

        return center_word, positive_words, negative_words

# 定义模型
class EmbeddingModel(nn.Module):
    def __init__(self, vocab_size, embed_size):
        super(EmbeddingModel, self).__init__()

        self.vocab_size = vocab_size
        self.embed_size = embed_size

        self.input_embed = nn.Embedding(self.vocab_size, self.embed_size)
        self.output_embed = nn.Embedding(self.vocab_size, self.embed_size)

        self.to(device)

    def forward(self, input_labels, positive_labels, negative_labels):
        """
        input_labels: center words,    shape:  [batch_size]
        positive_labels: positive words,  shape: [batch_size, context window size * 2]   乘以二是因为有前文后文两部分
        negative_labels: negative words, shape: [batch_size, K * context window size * 2]

        return loss, shape:[batch_size]
        """
        input_embedding = self.input_embed(input_labels)  # shape: [batch_size,embed size]
        positive_embedding = self.output_embed(positive_labels)  # [batch_size, (window * 2), embed_size]
        negative_embedding = self.output_embed(negative_labels)  # [batch_size, (window * 2 * K), embed_size]

        input_embedding = input_embedding.unsqueeze(2)  # [batch_size, embed_size, 1]

        # bmm(a, b)，batch matrix multiply
        positive_dot = torch.bmm(positive_embedding, input_embedding)  # [batch_size, (window * 2), 1]
        negative_dot = torch.bmm(negative_embedding, -input_embedding)  # [batch_size, (window * 2 * K), 1]

        positive_dot = positive_dot.squeeze(2)  # [batch_size, (window * 2)]
        negative_dot = negative_dot.squeeze(2)  # batch_size, (window * 2 * K)]

        log_positive = F.logsigmoid(positive_dot).sum(1)
        log_negative = F.logsigmoid(negative_dot).sum(1)

        loss = log_positive + log_negative

        return -loss

    def input_embedding(self):
        return self.input_embed.weight.detach().cpu().numpy()

# 训练
def train(text, word2idx, word_freqs):
    dataset = WordEmbeddingDataset(text, word2idx, word_freqs)
    dataloader = tud.DataLoader(dataset, BATCH_SIZE, shuffle=True, num_workers=DATALOADER_WORKER_NUMS)
    print('dataset word size: ', len(dataset))

    model = EmbeddingModel(MAX_VOCAB_SIZE, EMBEDDING_SIZE)
    print('model on cuda : ', next(model.parameters()).is_cuda)
    optimizer = torch.optim.Adam(model.parameters(), lr=LR)

    for e in range(EPOCHS):
        print('EPOCH', EPOCHS, 'STARTED !')
        epoch_start = timeit.default_timer()
        calc_total_time = 0
        for i, (input_labels, positive_labels, negative_labels) in enumerate(dataloader):
            start = timeit.default_timer()
            input_labels = input_labels.long().to(device)
            positive_labels = positive_labels.long().to(device)
            negative_labels = negative_labels.long().to(device)

            optimizer.zero_grad()
            loss = model(input_labels, positive_labels, negative_labels).mean()
            loss.backward()
            optimizer.step()
            calc_use = timeit.default_timer() - start
            calc_total_time += calc_use
            if i % 100 == 0 and i > 0:
                used_time = timeit.default_timer() - epoch_start
                print(
                    # 轮次
                    'EPOCH: ', e + 1, '/', EPOCHS,
                    # 迭代进度
                    '| ITER: ', i, '/', len(dataloader),
                    # 迭代一次用于训练计算的时间和迭代一次用时（其他时间因为数据集采样消耗
                    '| TRAINING TIME PER ITER:', round(calc_total_time / i, 4), '/', round(used_time / i, 4),
                    # 估计剩余时间
                    ' , ETA :', round((used_time / i) * (len(dataloader) - i), 2),
                    # 误差
                    '| LOSS: ', loss.item(), )
    print('Training done.')
    return model

# 验证
def test(word2idx, idx2word, embedding_weights):
    def find_nearest(word):
        index = word2idx[word]
        embedding = embedding_weights[index]
        cos_dis = np.array([scipy.spatial.distance.cosine(e, embedding) for e in embedding_weights])
        return [idx2word[i] for i in cos_dis.argsort()[:10]]

    for w in ["two", "america", "computer"]:
        print(w, word2idx[w])
        print(w, find_nearest(w))

    """
    OUTPUT >>

    two ['two', 'three', 'four', 'five', 'zero', 'six', 'seven', 'one', 'eight', 'nine']
    america ['america', 'europe', 'africa', 'asia', 'australia', 'north', 'south', 'canada', 'korea', 'india']
    computer ['computer', 'computers', 'video', 'hardware', 'software', 'digital', 'computing', 'graphics', 'engineering', 'technology']

    """

if __name__ == '__main__':
    print('run on device : ', device)

    DATA_SET = 'dev'

    # 读取文本
    with open('data/text8.{}.txt'.format(DATA_SET)) as f:
        the_text = f.read()

    # 分词
    the_text = the_text.lower().split()
    print('Got', len(the_text), 'words of text.')

    # 编码单词
    the_word2idx, the_idx2word, the_word_freqs = encoder_word(the_text)

    # save word encoded
    torch.save((the_word2idx, the_idx2word, the_word_freqs),
               "word_mapping-dataset_{}-embedding_size_{}.th".format(DATA_SET, EMBEDDING_SIZE))
    print('encoded word saved!')

    # 加载 编码好的单词
    (the_word2idx, the_idx2word, the_word_freqs) = torch.load(
        "word_mapping-dataset_{}-embedding_size_{}.th".format(DATA_SET, EMBEDDING_SIZE))
    print('encoded word loaded!')

    '''
    训练
    '''
    trained_model = train(text=the_text, word2idx=the_word2idx, word_freqs=the_word_freqs)

    # 保存训练好的 embedding
    embedding_weights = trained_model.input_embedding()
    torch.save(trained_model.state_dict(),
               "trained_model-dataset_{}-embedding_size_{}.th".format(DATA_SET, EMBEDDING_SIZE))
    print('model saved!')

    # 加载训练好的 embedding
    model_dict = torch.load(
        "trained_model-dataset_{}-embedding_size_{}.th".format(DATA_SET, EMBEDDING_SIZE))
    model_to_load = EmbeddingModel(MAX_VOCAB_SIZE, EMBEDDING_SIZE)
    model_to_load.load_state_dict(model_dict)
    print('trained_model loaded!')

    '''
    测试
    '''
    test(the_word2idx, the_idx2word, model_to_load.input_embedding())

    print('Done.')

reason: import numpy,import scipy


text:
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""serving config for mindspore serving"""

import time
import os
import json
import numpy as np

from mindspore_serving.server import register
import mindspore.context as context
from mindspore.common.tensor import Tensor
from mindspore import load_checkpoint

from data.feature.feature_extraction import process_features
from data.tools.data_process import data_process
from commons.utils import compute_confidence
from commons.generate_pdb import to_pdb, from_prediction
from model import AlphaFold
from config import config, global_config
from fold_service.config import config as serving_config

context.set_context(mode=context.GRAPH_MODE,
                    device_target="Ascend",
                    variable_memory_max_size="31GB",
                    device_id=serving_config.device_id,
                    save_graphs=False)
model_name = "model_1"
model_config = config.model_config(model_name)
num_recycle = model_config.model.num_recycle
global_config = global_config.global_config(serving_config.seq_length)
extra_msa_length = global_config.extra_msa_length

fold_net = AlphaFold(model_config, global_config)
load_checkpoint(serving_config.ckpt_path, fold_net)

def fold_model(input_fasta_path):
    """defining fold model"""

    seq_files = os.listdir(input_fasta_path)
    for seq_file in seq_files:
        print(seq_file)
        t1 = time.time()
        seq_name = seq_file.split('.')[0]

        input_features = data_process(seq_name, serving_config)
        tensors, aatype, residue_index, ori_res_length = process_features(
            raw_features=input_features, config=model_config, global_config=global_config)
        prev_pos = Tensor(np.zeros([global_config.seq_length, 37, 3]).astype(np.float16))
        prev_msa_first_row = Tensor(np.zeros([global_config.seq_length, 256]).astype(np.float16))
        prev_pair = Tensor(np.zeros([global_config.seq_length, global_config.seq_length, 128]).astype(np.float16))

        t2 = time.time()
        for i in range(num_recycle+1):
            tensors_i = [tensor[i] for tensor in tensors]
            input_feats = [Tensor(tensor) for tensor in tensors_i]
            final_atom_positions, final_atom_mask, predicted_lddt_logits,\
                prev_pos, prev_msa_first_row, prev_pair = fold_net(*input_feats,
                                                                   prev_pos,
                                                                   prev_msa_first_row,
                                                                   prev_pair)

        t3 = time.time()

        final_atom_positions = final_atom_positions.asnumpy()[:ori_res_length]
        final_atom_mask = final_atom_mask.asnumpy()[:ori_res_length]
        predicted_lddt_logits = predicted_lddt_logits.asnumpy()[:ori_res_length]

        confidence = compute_confidence(predicted_lddt_logits)
        unrelaxed_protein = from_prediction(final_atom_mask, aatype[0], final_atom_positions, residue_index[0])
        pdb_file = to_pdb(unrelaxed_protein)

        seq_length = aatype.shape[-1]
        os.makedirs(f'./result/seq_{seq_name}_{seq_length}', exist_ok=True)

        with open(os.path.join(f'./result/seq_{seq_name}_{seq_length}/', f'unrelaxed_model_{seq_name}.pdb'), 'w') as f:
            f.write(pdb_file)
        t4 = time.time()
        timings = {"pre_process_time": round(t2 - t1, 2),
                   "model_time": round(t3 - t2, 2),
                   "pos_process_time": round(t4 - t3, 2),
                   "all_time": round(t4 - t1, 2),
                   "confidence": confidence}
        print(timings)
        with open(f'./result/seq_{seq_name}_{seq_length}/timings', 'w') as f:
            f.write(json.dumps(timings))
    return True

@register.register_method(output_names=["res"])
def folding(input_fasta_path):
    res = register.add_stage(fold_model, input_fasta_path, outputs_count=1)
    return res

max_stars_repo_name: mindspore-ai/mindscience

max_issues_repo_name: mindspore-ai/mindscience

max_forks_repo_name: mindspore-ai/mindscience


text:
#! /usr/bin/env python
import cv2 as cv
import numpy as np
import math
import rospy
from yaml_parser import Parser
from iros_vision.msg import ObjectCoords

MAIN_WINDOW_NAME = 'camera'
DEBUG = 'debug'

class Color_object:
    def __init__(self, config):
        self.debug = None
        self.config = config
        self.name = self.config['name']
        self.min_size = self.config['min_size']
        self.max_size = self.config['max_size']
        self.color = self.config['colour']
        self.hsv = None
        self.hsv_vals = {}

        self.pub = rospy.Publisher(
            '/iros_vision_node/' + self.name, ObjectCoords, queue_size=1)

    def publish_coords(self, x, y, width, height, area, angle):
        message = ObjectCoords()
        message.x = x
        message.y = y
        message.width = width
        message.height = height
        message.area = area
        message.angle = angle
        self.pub.publish(message)
        # print('{0}: x = {1}\ty = {2}\tarea = {3}\tangle = {4}'.format(self.name, x, y, area, angle))

class Vision:
    def __init__(self, file_path):
        self.parser = Parser(file_path)
        data = self.parser.data

        self.color_objects = []
        for d in data['object']:
            new_color = Color_object(d)

            col_lb = new_color.config['lower_bound']
            col_ub = new_color.config['upper_bound']

            new_color.hsv_vals = {'HL': col_lb[0], 'HU': col_ub[0], 'SL': col_lb[1], 'SU': col_ub[1], 'VL': col_lb[2], 'VU': col_ub[2]}

            self.color_objects.append(new_color)

        self.selected_object = self.color_objects[0]

        self.frame = None

        self.cap = cv.VideoCapture(0)

        cv.namedWindow(MAIN_WINDOW_NAME)
        cv.namedWindow(DEBUG)

        if len(self.color_objects) > 1:
            cv.createTrackbar('Key', DEBUG, 0, len(self.color_objects)-1, self.pick_object)
        cv.createTrackbar('H (lower)', DEBUG, 0, 255, lambda val: self.update_from_trackbars(val, 'HL'))
        cv.createTrackbar('H (upper)', DEBUG, 0, 255, lambda val: self.update_from_trackbars(val, 'HU'))
        cv.createTrackbar('S (lower)', DEBUG, 0, 255, lambda val: self.update_from_trackbars(val, 'SL'))
        cv.createTrackbar('S (upper)', DEBUG, 0, 255, lambda val: self.update_from_trackbars(val, 'SU'))
        cv.createTrackbar('V (lower)', DEBUG, 0, 255, lambda val: self.update_from_trackbars(val, 'VL'))
        cv.createTrackbar('V (upper)', DEBUG, 0, 255, lambda val: self.update_from_trackbars(val, 'VU'))

        col_lb = self.selected_object.config['lower_bound']
        col_ub = self.selected_object.config['upper_bound']

        self.update_trackbars(col_lb[0], col_ub[0], \
            col_lb[1], col_ub[1], \
                col_lb[2], col_ub[2])

        cv.setMouseCallback(MAIN_WINDOW_NAME, self.mouse_cb)

    def pick_object(self, val):
        self.selected_object = self.color_objects[val]
        lower_b = self.selected_object.config['lower_bound']
        upper_b = self.selected_object.config['upper_bound']

        self.update_trackbars(lower_b[0], upper_b[0], lower_b[1], upper_b[1], lower_b[2], upper_b[2])

    def update_from_trackbars(self, val, var):
        self.selected_object.hsv_vals[var] = val

        bound = None
        index = -1
        if var[0] == 'H':
            index = 0
        elif var[0] == 'S':
            index = 1
        elif var[0] == 'V':
            index = 2

        if var[1] == 'U':
            bound = 'upper_bound'
        elif var[1] == 'L':
            bound = 'lower_bound'

        self.parser.write_values(self.selected_object.config, bound, index, val)

    def update_trackbars(self, hl, hu, sl, su, vl, vu):
        cv.setTrackbarPos('H (lower)', DEBUG, hl)
        cv.setTrackbarPos('H (upper)', DEBUG, hu)

        cv.setTrackbarPos('S (lower)', DEBUG, sl)
        cv.setTrackbarPos('S (upper)', DEBUG, su)

        cv.setTrackbarPos('V (lower)', DEBUG, vl)
        cv.setTrackbarPos('V (upper)', DEBUG, vu)

    def create_hsv(self):
        blurred = cv.GaussianBlur(self.frame, (11,11),0)
        for color_obj in self.color_objects:
            color_obj.hsv = cv.cvtColor(blurred, cv.COLOR_BGR2HSV)

            lower_bound = (color_obj.hsv_vals['HL'], color_obj.hsv_vals['SL'], color_obj.hsv_vals['VL'])
            upper_bound = (color_obj.hsv_vals['HU'], color_obj.hsv_vals['SU'], color_obj.hsv_vals['VU'])

            color_obj.debug = cv.inRange(color_obj.hsv, lower_bound, upper_bound) # get binary
            color_obj.debug = cv.erode(color_obj.debug, None, iterations=2)
            color_obj.debug = cv.dilate(color_obj.debug, None, iterations=2)

    def read_camera(self):
        _, self.frame = self.cap.read()
        if not self.cap.isOpened():
            print('Cannot open camera')
            exit()

        self.frame = cv.resize(self.frame, (320, 240))

    def mouse_cb(self, event, x, y, flags, param):
        if event == cv.EVENT_LBUTTONDOWN:
            self.click_reg = [(x,y)]
        elif event == cv.EVENT_LBUTTONUP:
            self.click_reg.append((x,y))
            mean = cv.mean(self.selected_object.hsv[self.click_reg[0][1]:y, self.click_reg[0][0]:x])

            h_mean = int(math.floor(mean[0]))
            s_mean = int(math.floor(mean[1]))
            v_mean = int(math.floor(mean[2]))

            init_bound = 20

            self.update_trackbars(h_mean - init_bound, h_mean + init_bound, \
                s_mean - init_bound, s_mean + init_bound, \
                    v_mean - init_bound, v_mean + init_bound)

max_stars_repo_name: chris-mega/PianoPlayer

max_issues_repo_name: chris-mega/PianoPlayer

max_forks_repo_name: chris-mega/PianoPlayer


text:
from torch.utils.tensorboard import SummaryWriter
import torchvision
import torch
import numpy as np
import matplotlib.pyplot as plt

class Visualizer():
    """
        Defines the functions needed to write statistics and the output images to a tensorboard file.

    """
    def __init__(self, log_dir):
        """
            log_dir = folder where the tensorboard log should be stored
        """

        self.summary_writer = SummaryWriter(log_dir=log_dir)

    # Write the learning rate to the tensorboard file
    def write_lr(self, optim, globaliter):
        for i, param_group in enumerate(optim.param_groups):
            self.summary_writer.add_scalar('learning_rate/lr_'+ str(i) , param_group['lr'], globaliter)
        self.summary_writer.flush()

    # Write statistics in text form to the tensorboard file. Training time or maximum GPU memory usage.
    def write_text(self, value, statType):
        if statType == 'Time':
            self.summary_writer.add_text('Time','Total Training time: ' + value + ' seconds')
        elif statType == 'Memory':
            self.summary_writer.add_text('GPU', 'Maximum GPU usage: ' + value + ' MiB')
        self.summary_writer.flush()

    # Write the training loss to the tensorboard file
    def write_loss_train(self, value, globaliter):
        self.summary_writer.add_scalar('Loss/train', value, globaliter)
        self.summary_writer.flush()

    # Write the validation loss to the tensorboard file
    def write_loss_validation(self, value, globaliter, if_testtimes=False):
        if if_testtimes:
            postfix = '_testtimes'
        else:
            postfix = ''

        self.summary_writer.add_scalar('Loss/validation'+postfix, value, globaliter)
        self.summary_writer.flush()

    # Write the output images to the tensorboard file
    def write_image(self, images, epoch, if_predict=False, if_testtimes=False, includeHeading = False):
        """
            images: Prediction or ground truth in image form
            epoch: Current epoch of the training
            if_predict: Is it a prediction or a ground truth?
            if_testtimes: Is the validation happening on the testtimes only?
            includeHeading: Is the heading channel included in the training?
        """

        if if_testtimes:
            postfix = '_testtimes'
        else:
            postfix = ''

        # restructure the data, save volume, speed and heading separately
        if len(images.shape) == 4:
            _, _, row, col = images.shape
            vol_batch = torch.zeros((3, 1 , row, col))
            speed_batch = torch.zeros((3, 1 , row, col))
            if includeHeading:
                head_batch = torch.zeros((3, 1 , row, col))

            # volume
            vol_batch[0] = images[0,0,:,:]
            vol_batch[1] = images[0,3,:,:]
            vol_batch[2] = images[0,6,:,:]
            # speed
            speed_batch[0] = images[0,1,:,:]
            speed_batch[1] = images[0,4,:,:]
            speed_batch[2] = images[0,7,:,:]

            # heading
            if includeHeading:
                head_batch[0] = images[0,2,:,:]
                head_batch[1] = images[0,5,:,:]
                head_batch[2] = images[0,8,:,:]

        else:

            _, _, _, row, col = images.shape
            vol_batch = torch.zeros((3, 1 , row, col))
            speed_batch = torch.zeros((3, 1 , row, col))
            if includeHeading:
                head_batch = torch.zeros((3, 1 , row, col))

            # volume
            vol_batch[0] = images[0,0,0,:,:]
            vol_batch[1] = images[0,1,0,:,:]
            vol_batch[2] = images[0,2,0,:,:]

            # speed
            speed_batch[0] = images[0,0,1,:,:]
            speed_batch[1] = images[0,1,1,:,:]
            speed_batch[2] = images[0,2,1,:,:]
            # heading
            if includeHeading:
                head_batch[0] = images[0,0,2,:,:]
                head_batch[1] = images[0,1,2,:,:]
                head_batch[2] = images[0,2,2,:,:]

        # add images to the tensorboard file
        if if_predict:

            vol_batch = torchvision.utils.make_grid(vol_batch, normalize=True)
            self.summary_writer.add_image('prediction'+postfix+'/volume', vol_batch, epoch)

            speed_batch = torchvision.utils.make_grid(speed_batch, normalize=True)
            self.summary_writer.add_image('prediction'+postfix+'/speed', speed_batch, epoch)

            if includeHeading:
                head_batch = torchvision.utils.make_grid(head_batch, normalize=True)
                self.summary_writer.add_image('prediction'+postfix+'/heading', head_batch, epoch)

        else:
            vol_batch = torchvision.utils.make_grid(vol_batch, normalize=True)
            self.summary_writer.add_image('ground_truth'+postfix+'/volume', vol_batch, epoch)

            speed_batch = torchvision.utils.make_grid(speed_batch, normalize=True)
            self.summary_writer.add_image('ground_truth'+postfix+'/speed', speed_batch, epoch)

            if includeHeading:
                head_batch = torchvision.utils.make_grid(head_batch, normalize=True)
                self.summary_writer.add_image('ground_truth'+postfix+'/heading', head_batch, epoch)

        # Apply changes to tensorboard file
        self.summary_writer.flush()

    def close(self):
        self.summary_writer.close()


text:
import numpy as np
import trimesh
import open3d
from visualize_data import draw_scene, get_gripper_collision_geometry
from copy import deepcopy as copy

def get_gripper_finger_sweep_volume_mayavi(grasp):
    """
    This is just for the sawyer gripper
    """
    align = tra.euler_matrix(np.pi / 2, 0, 0)
    extents = [0.06, 0.02, 0.14]
    transform = np.eye(4)
    transform = np.matmul(align, transform)
    transform = np.matmul(grasp, transform)
    finger_sweep_volume = trimesh.primitives.Box(
        extents=extents, transform=transform)

    return finger_sweep_volume, extents, transform

def get_gripper_finger_sweep_volume(grasp):
    """
    This is just for the sawyer gripper
    """
    align = tra.euler_matrix(np.pi / 2, 0, 0)
    extents = [0.06, 0.02, 0.14]
    transform = np.eye(4)
    transform[0, 3] = -extents[0] / 2
    transform[1, 3] = -extents[1] / 2
    transform[2, 3] = -extents[2] / 2
    transform = np.matmul(align, transform)
    transform = np.matmul(grasp, transform)
    finger_sweep_volume = trimesh.primitives.Box(
        extents=extents, transform=transform)

    return finger_sweep_volume, extents, transform

def get_gripper_collision_geometry(grasp):
    """
    This is an approximation of the sawyer mesh
    """
    meshes = []

    align = tra.euler_matrix(np.pi / 2, 0, 0)
    extents = [0.02, 0.02, 0.14]
    transform = np.eye(4)
    transform[0, 3] = -extents[0]
    transform = np.matmul(align, transform)
    transform = np.matmul(grasp, transform)
    part1 = trimesh.primitives.Box(extents=extents, transform=transform)
    meshes.append((part1, extents, transform))

    extents = [0.06, 0.02, 0.02]
    transform = np.eye(4)
    transform[2, 3] = 0.07
    transform = np.matmul(align, transform)
    transform = np.matmul(grasp, transform)
    part2 = trimesh.primitives.Box(extents=extents, transform=transform)
    meshes.append((part2, extents, transform))

    extents = [0.06, 0.02, 0.02]
    transform = np.eye(4)
    transform[2, 3] = -0.07
    transform = np.matmul(align, transform)
    transform = np.matmul(grasp, transform)
    part3 = trimesh.primitives.Box(extents=extents, transform=transform)
    meshes.append((part3, extents, transform))

    extents = [0.07, 0.02, 0.02]
    transform = np.eye(4)
    transform[0, 3] = -extents[0]
    transform = np.matmul(align, transform)
    transform = np.matmul(grasp, transform)
    part4 = trimesh.primitives.Box(extents=extents, transform=transform)
    meshes.append((part4, extents, transform))

    return meshes

class CollisionManager(object):
    def __init__(self, voxel_size=0.005):
        self._manager = trimesh.collision.CollisionManager()
        self._collision_objects = []
        self._voxel_size = voxel_size
        self._voxel_extents = [
            self._voxel_size,
            self._voxel_size,
            self._voxel_size]
        self._pc = None

    def construct_occupancy_grid(self, pc, max_points=1000):
        """
        pc: (N, 3) array
        Assume point cloud is already mean centered
        """

        if len(self._collision_objects) > 0:
            raise ValueError('Occupancy grid already constucted')

        self._pc = copy(pc)
        n_points = self._pc.shape[0]

        if n_points > max_points:
            # TODO Do farthest point sampling instead of random sampling
            chosen_idx = np.random.choice(
                list(range(n_points)), max_points, replace=False)
            self._pc = self._pc[chosen_idx, :]
            n_points = self._pc.shape[0]

        # Construct collision objects
        for i in range(n_points):
            extents = self._voxel_extents
            transform = np.eye(4)
            transform[:3, 3] = self._pc[i, :3]
            voxel = trimesh.primitives.Box(
                extents=extents, transform=transform)
            self._collision_objects.append((voxel, extents, transform))

        # Add to collision manager
        for i, (voxel, _, _) in enumerate(self._collision_objects):
            self._manager.add_object("voxel_{}".format(i), voxel)

    def check_collisions(self, grasps):
        """
        grasps: (N, 4, 4)
        returns:
            (N, 1) True if colliding, False otherwise
        """
        result = []
        for grasp in grasps:
            gripper_mesh = get_gripper_collision_geometry(grasp)
            is_collision = np.array([self.check_collision_manager(
                elem[0]) for elem in gripper_mesh]).sum() > 0
            result.append(is_collision)
        return np.array(result)

    def check_free_space_grasp(self, grasps):
        """
        grasps: (N, 4, 4)
        returns:
            (N, 1) True if grasping free space, false otherwise
        """
        result = []
        for grasp in grasps:
            finger_sweep_volume, _, _ = get_gripper_finger_sweep_volume_mayavi(
                grasp)
            is_collision = self.check_collision_manager(finger_sweep_volume)
            result.append(is_collision)
        return np.array(result)

    def check_collision_manager(self, mesh):
        return self._manager.in_collision_single(mesh)

    def visualize_occupancy_grid(
            self,
            grasp=None,
            debug_mode=False):
        """
        grasp: (4,4)
        Assume grasp is pc mean centered
        Only one grasp at a time
        """

        meshes = copy(self._collision_objects)
        grasps = []
        if grasp is not None:
            grasps.append(grasp)
            finger_sweep_volume, extents, transform = get_gripper_finger_sweep_volume(
                grasp)
            meshes.append((finger_sweep_volume, extents, transform))

        draw_scene(
            pc=self._pc,
            grasps=grasps,
            meshes=meshes,
            subtract_pc_mean=False,
            debug_mode=debug_mode)


text:
from mido import MidiFile
import pandas as pd
import pickle
import numpy as np
import matplotlib.pyplot as plt
import sys
import time
from collections import deque
from smallbrain import toolbar_init, toolbar_tick, data_folder, typemap, data_split

# input data type is passed as argument
# default = 0 (basic notes extraction)
input_data_type=0
if len(sys.argv)>1 and sys.argv[1] in set(smallbrain.typemap):
    input_data_type=smallbrain.typemap.index(sys.argv[1])

files = pd.read_csv('maestro/maestro-v2.0.0.csv')[['midi_filename','split']]
data_raw = [list(files[files['split']==x]['midi_filename']) for x in smallbrain.data_split]

lowest_note = 21
highest_note = 87

delta_bins = [1,2,4,8,16,32,64,128,256,512,1024]
velocity_bins = [1,20,36,57,78,90,100]

def extract_notes_basic(x):
    return np.array([j.note-lowest_note for j in x if j.type=='note_on' and j.velocity], dtype=np.uint8)

def extract_notes_basic_velocity(x):
    return np.array([ [j.note-lowest_note, j.velocity] for j in x if j.type=='note_on' and j.velocity], dtype=np.uint8)

def extract_notes_delta(x):
    y = np.cumsum([j.time for j in x])
    #y = [i if i<200 else int(200+np.sqrt(i)) for i in y ] # retarded data smoothing with .9999th quantile onward
    z = [ [j.note-lowest_note,j.velocity,i] for j,i in zip(x,y) if j.type=='note_on' and j.velocity] # notes with their time
    return np.array([z[0]] + [ [z[i-1][0],z[i-1][1],z[i][2]-z[i-1][2] ] for i in range(1,len(z))]) # make deltas
    # 0 for first item because its delta=0

def extract_notes_duration(x):
    z = np.cumsum([j.time for j in x])
    notes = [deque() for i in range(88)]
    y = []
    for j,i in zip(x,z):
        if j.type=='note_on':
            note = j.note-lowest_note
            if j.velocity: # if real note on event
                notes[note].append([len(y),i]) # insert position in queue
                y.append([note,j.velocity])
            else: # if note off event
                try:
                    last=notes[note].popleft()
                    y[last[0]].append(i-last[1])
                except:
                    pass
    for i in notes:
        for j in i:
            del y[j[0]]

    return np.array(y, dtype=np.uint16)

def velocity_category(x):
    for i in range(len(velocity_bins)):
        if x < velocity_bins[i]:
            return i
    return len(velocity_bins)

def delta_category(x):
    return min(11, int(x).bit_length())
    #hack, basically what this one does:
    #for i in range(len(freq_bins)):
    #   if x < freq_bins[i]:
    #       return i
    #return len(freq_bins)

def extract_notes_events(x):
    y = np.cumsum([j.time for j in x])
    z = [ [j.note-lowest_note,velocity_category(j.velocity),i] for j,i in zip(x,y) if j.type=='note_on'] # notes with their time
    d = np.array([[z[0][0],z[0][1],delta_category(z[0][2])]] + [ [ z[i-1][0], z[i-1][1], delta_category(z[i][2]-z[i-1][2]) ] for i in range(1,len(z))])
    deltas_only = np.array([z[i][2]-z[i-1][2] for i in range(1,len(z))])
    min_steps = 500
    threshold = 2048 # ... took me some time
    # those guys  - data_parsed[0][379] to data_parsed[0][381]
    # Frédéric Chopin,"12 Etudes, Op. 25",train,2004,2004/MIDI-Unprocessed_SMF_05_R1_2004_01_ORIG_MID--AUDIO_05_R1_2004_02_Track02_wav.midi,2004/MIDI-Unprocessed_SMF_05_R1_2004_01_ORIG_MID--AUDIO_05_R1_2004_02_Track02_wav.wav,1409.28857763
    # Frédéric Chopin,"12 Etudes, Op. 25",train,2004,2004/MIDI-Unprocessed_SMF_05_R1_2004_01_ORIG_MID--AUDIO_05_R1_2004_03_Track03_wav.midi,2004/MIDI-Unprocessed_SMF_05_R1_2004_01_ORIG_MID--AUDIO_05_R1_2004_03_Track03_wav.wav,327.327025835
    # Frédéric Chopin,"24 Preludes, Op. 28",train,2004,2004/MIDI-Unprocessed_XP_06_R1_2004_01_ORIG_MID--AUDIO_06_R1_2004_01_Track01_wav.midi,2004/MIDI-Unprocessed_XP_06_R1_2004_01_ORIG_MID--AUDIO_06_R1_2004_01_Track01_wav.wav,2398.64035989
    # they have multiple pieces in the same file
    # Chopin is amazing and I like those performances very much, but, wtf
    sz = len(deltas_only)
    start = int((sz+9)/10)
    # ignore first and last 10 percent when doing the cuts
    # can't remember what i was doing here, but it works.
    # Dont touch!
    split_stuff = []
    last_split = 0
    for i in range(start,sz-start):
        if deltas_only[i]>threshold and (i-last_split) > min_steps:
            split_stuff.append(d[last_split:i])
            last_split=i

    split_stuff.append(d[last_split:len(d)])
    return split_stuff

def extract_notes(x,mode=0):
    if mode==0:
        return extract_notes_basic(x)
    elif mode==1:
        return extract_notes_basic_velocity(x)
    elif mode==2:
        return extract_notes_delta(x)
    elif mode==3:
        return extract_notes_duration(x)
    return extract_notes_events(x)

def parse(data):
    cnt = len(data)
    thing = []

    toolbar_cnt = toolbar_init(60,cnt)
    begin = time.time()

    for i in range(cnt):
        name = f'maestro/{data[i]}'
        x=MidiFile(name)
        toolbar_tick(i%toolbar_cnt)
        result = extract_notes(x.tracks[1],input_data_type)
        if input_data_type==4:
            #print (i, len(result))
            for songs_split in result:
                thing.append(songs_split)
        else:
            thing.append(result)
    sys.stdout.write("] Done in %s sec!\n" % ('{0:.2f}'.format(time.time()-begin))) # this ends the progress barprint("done!")
    return np.array(thing)

print ("Generating dataset for "+smallbrain.typemap[input_data_type])
data_parsed=[]
for i in range(len(smallbrain.data_split)):
    print(smallbrain.data_split[i])
    data_parsed.append(parse(data_raw[i]))
    pickle.dump(data_parsed[i],open(f'{smallbrain.data_folder}/rick-{smallbrain.typemap[input_data_type]}-{smallbrain.data_split[i]}','wb'))

## --------------------- EOF ------------------------


text:
import tensorflow as tf
import numpy as np

import random
import loader as l

import math
import matplotlib.pyplot as plt

from tensorflow.contrib import rnn
from tensorflow.python.ops import variable_scope
from tensorflow.python.framework import dtypes
import copy

X_train , y_train ,X_test , y_test , ratio = l.process()

batch_size = 32

ratio = [1.,0.88,1.35]

ratio = np.reshape(ratio , [-1,1])

train_loss = []
validation_loss = []

def generate_batches(batch_size , X_train , Y_train):

    num_batches = int(len(X_train)) // batch_size

    if(num_batches*batch_size< len(X_train)):
        num_batches+=0

    batch_indices = list(range(num_batches))

    random.shuffle(batch_indices)
    #print("batch_indices" , batch_indices)
    batches_X = []
    batches_Y = []

    for j in batch_indices:
        batch_X =  X_train[j * batch_size: (j + 1) * batch_size]
        batch_y =  Y_train[j * batch_size: (j + 1) * batch_size]

        batches_X.append(batch_X)
        batches_Y.append(batch_y)

    return batches_X , batches_Y

class RNNConfig():

    ## Parameters
    init_learning_rate = 0.0005
    lambda_l2_reg = 0.001
    max_epoch = 151
    learning_rate_decay = 1

    ## Network Parameters
    # length of input signals
    input_seq_len = l.NUM_STEPS

    # length of output signals
    output_seq_len = l.LEAD_TIME

    # size of LSTM Cell
    hidden_dim = 30

    dropout= [0]
    # num of input signals
    input_dim = l.INPUTS

    # num of output signals
    output_dim = 3

    # num of stacked lstm layers
    num_stacked_layers = 1

    alpha = 1 #Loss parameter

config = RNNConfig()

def step():
    global_step = tf.Variable(initial_value=0,name="global_step",trainable=False,
    collections=[tf.GraphKeys.GLOBAL_STEP, tf.GraphKeys.GLOBAL_VARIABLES])

    return global_step

def create_placeholders():
    enc_inp = [tf.placeholder(tf.float32, shape=(None, config.input_dim), name="inp_{}".format(t))for t in range(config.input_seq_len)]
    target_seq = [tf.placeholder(tf.float32, shape=(None, config.output_dim), name="y".format(t))for t in range(config.output_seq_len)]
    dec_inp = [ tf.zeros_like(target_seq[0], dtype=tf.float32, name="GO") ] + target_seq[:-1]
    learning_rate = tf.placeholder(dtype=tf.float32, shape=None)
    dropout = tf.placeholder(dtype=tf.float32 , shape=(config.num_stacked_layers))

    return enc_inp , target_seq , dec_inp , learning_rate , dropout

def weight_variable(shape):
    return (tf.Variable(tf.truncated_normal(shape=shape , stddev=0.1)))

def bias_variable(shape):
    return tf.Variable(tf.constant(0., shape=shape))

def encoder_network(dropout):
    cells = []
    for i in range(config.num_stacked_layers):

        lstm_cell = tf.contrib.rnn.LSTMCell(config.hidden_dim)
        if(i>1):
            lstm_cell = tf.contrib.DropoutWrapper(lstm_cell,0.7)
        cells.append(lstm_cell)

    cell = tf.contrib.rnn.MultiRNNCell(cells)

    return cell

def decoder_network(attention_mechanism):

    lstm_cell = tf.contrib.rnn.LSTMCell(config.hidden_dim)

    decoder_cell = tf.contrib.seq2seq.AttentionWrapper(
        lstm_cell, attention_mechanism=attention_mechanism,
        attention_layer_size=config.hidden_dim)

    return decoder_cell

def _rnn_decoder(decoder_inputs,initial_state, attention_mechanism , Why , by , loop_function=None,scope=None):

    outputs = []
    prev = None

    cell = decoder_network(attention_mechanism)

    state = cell.zero_state(batch_size=batch_size,dtype=tf.float32).clone(cell_state=initial_state[-1])

    for i, inp in enumerate(decoder_inputs):

        if loop_function is not None and prev is not None:
           inp = loop_function(prev, Why , by )

        if i > 0:
            variable_scope.get_variable_scope().reuse_variables()

        output,state = cell(inp, state=state)
        outputs.append(output)

        if loop_function is not None:
            prev = output

    return outputs, state

def _basic_rnn_seq2seq(encoder_inputs,decoder_inputs,cell,Why , by , feed_previous, dtype=dtypes.float32,scope=None):

    enc_cell = copy.deepcopy(cell)
    outputs, enc_state = rnn.static_rnn(enc_cell, encoder_inputs, dtype=dtype)

    enc_outputs = tf.stack(outputs)

    attention_states = tf.transpose(enc_outputs, [1, 0, 2])

    attention_mechanism = tf.contrib.seq2seq.LuongAttention(
                            num_units=config.hidden_dim, memory =attention_states)

    if feed_previous:
        return _rnn_decoder(decoder_inputs, enc_state, attention_mechanism , Why , by ,_loop_function)
    else:
        return _rnn_decoder(decoder_inputs, enc_state , attention_mechanism , Why , by)

def reshape(dec_outputs , Why , by):

    reshaped_outputs = []

    for j in range(config.output_seq_len):

        print("j",j)
        i = dec_outputs[j]

        temp = (tf.add(tf.matmul(i , Why),by))
        reshaped_outputs.append(temp)

        last_outputs = temp

    return reshaped_outputs , last_outputs

def compute_loss(reshaped_outputs , last_outputs,target_seq , learning_rate):

     # Training loss and optimizer
    with tf.variable_scope('Loss'):

        class_weight = tf.constant(ratio)
        class_weight = tf.cast(class_weight , tf.float32)

        # L2 loss
        output_loss = 0

        all_steps_cost = 0

        for y , Y in zip(reshaped_outputs , target_seq):
            weight_per_label = tf.transpose(tf.matmul(Y,class_weight))
            all_steps_cost += tf.reduce_mean(tf.multiply(weight_per_label,tf.nn.softmax_cross_entropy_with_logits(labels=Y,logits=y)))

            if(Y==target_seq[-1]):
                weight_per_label = tf.transpose(tf.matmul(Y,class_weight))
                last_step_cost = tf.reduce_mean(tf.multiply(weight_per_label,tf.nn.softmax_cross_entropy_with_logits(labels=Y,logits=y)))

        output_loss = config.alpha * all_steps_cost + (1-config.alpha) * last_step_cost

        # L2 regularization for weights and biases
        reg_loss = 0
        for tf_var in tf.trainable_variables():
            reg_loss += tf.reduce_mean(tf.nn.l2_loss(tf_var))

        loss = output_loss  + config.lambda_l2_reg* reg_loss

    with tf.variable_scope('Optimizer' , reuse=tf.AUTO_REUSE):
        optimizer = tf.train.AdamOptimizer(learning_rate)
        minimize = optimizer.minimize(loss)

    return loss , optimizer , minimize

def _loop_function(prev, Why , by):

    temp= tf.nn.softmax(tf.matmul(prev, Why) + by)
    one_hot = tf.one_hot(tf.argmax(temp, dimension = 1), depth = 3)
    return one_hot

def build_train_graph(feed_previous = False):

    print("Building graph")
    global_step = step()

    Why = weight_variable([config.hidden_dim , config.output_dim])
    by = bias_variable([config.output_dim])

    print("Weights initialised")

    enc_inp , target_seq , dec_inp , learning_rate , dropout = create_placeholders()
    print("Placeholders created")

    cell = encoder_network(dropout)
    print("Encoder created")

    dec_outputs, dec_memory = _basic_rnn_seq2seq(enc_inp, dec_inp, cell, Why , by , feed_previous=feed_previous)
    print("decoder computed")

    reshaped_outputs,last_outputs = reshape(dec_outputs , Why , by)
    print("Outputs computed")

    loss , optimizer , minimize = compute_loss(reshaped_outputs , last_outputs,target_seq , learning_rate)
    print("Loss computed")

    return dict(
        enc_inp = enc_inp,
        target_seq = target_seq,
        reshaped_outputs = reshaped_outputs,
        loss = loss,
        learning_rate=learning_rate,
        dropout=dropout,
        optimizer = optimizer,
        minimize = minimize,
        )

def train():

    total_epochs = config.max_epoch
    train_losses = []

    rnn_model = build_train_graph(feed_previous=True)

    saver = tf.train.Saver()

    init = tf.global_variables_initializer()
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
    with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:

        sess.run(init)

        checkpoint = tf.train.get_checkpoint_state("saved_networks")

        if checkpoint and checkpoint.model_checkpoint_path:
            saver.restore(sess, checkpoint.model_checkpoint_path)
            print("Loaded :", checkpoint.model_checkpoint_path)
            print("\n\n")
        else:
            print("Unable to find network weights")

        print("Training losses: ")

        learning_rates = [
        config.init_learning_rate * (
            math.pow(config.learning_rate_decay , (i))
        ) for i in range(config.max_epoch)]

        for epoch_step in range(total_epochs):
            total_loss = 0
            avg_loss = 0
            j=0
            batch_inputs , batch_outputs = generate_batches(batch_size,X_train , y_train)
            current_lr = learning_rates[j]
            for batch_input,batch_output in zip(batch_inputs , batch_outputs):

                feed_dict = {rnn_model['enc_inp'][t]: batch_input[:,t] for t in range(config.input_seq_len)}
                feed_dict.update({rnn_model['target_seq'][t]: batch_output[:,t] for t in range(config.output_seq_len)})
                feed_dict.update({rnn_model['dropout']:config.dropout})
                feed_dict.update({rnn_model['learning_rate']:current_lr})

                outputs = sess.run(rnn_model['reshaped_outputs'] , feed_dict)

                loss_t= sess.run(rnn_model['loss'] , feed_dict)
                _ = sess.run(rnn_model['minimize'] , feed_dict)
                total_loss += loss_t
                j+=1

                #print("Outputs\n" , outputs),
                #print("Actual\n" ,  batch_output)
                #print("loss" , loss_t)

            avg_loss = total_loss/j
            print("Epoch " + str(epoch_step))
            print("Average loss "),
            print(avg_loss)

            if(epoch_step%5==0):
                saver.save(sess, 'saved_networks/' , global_step = epoch_step)

                print("Checkpoint saved")

                train_loss.append(avg_loss)

                batch_inputs , batch_outputs = generate_batches(batch_size,X_test[:244] , y_test[:244])
                loss_v = 0
                j=0
                for batch_input,batch_output in zip(batch_inputs , batch_outputs):

                    feed_dict = {rnn_model['enc_inp'][t]: batch_input[:,t] for t in range(config.input_seq_len)}
                    feed_dict.update({rnn_model['target_seq'][t]: batch_output[:,t] for t in range(config.output_seq_len)})

                    dropout = [1 for i in range(config.num_stacked_layers)]
                    feed_dict.update({rnn_model['dropout']:dropout})
                    j+=1
                    loss_v += sess.run(rnn_model['loss'], feed_dict)

                print("Validation loss" , loss_v/j)

if __name__=='__main__':
    train()


text:
import networkx as nx
from sqlalchemy import Column, Integer, String, ForeignKey
from sqlalchemy.ext.declarative import declarative_base

from sqlalchemy_json import MutableJson

Base = declarative_base()

UNKNOWN = 0
PENDING = 1
RUNNING = 2
SUCCESS = 3
FAILED = 4

class ComputationalPipeline(Base):
    __tablename__ = 'comp_pipeline'

    pipeline_id = Column(Integer, primary_key=True)
    dag_adjacency_list = Column(MutableJson)
    state = Column(String, default=UNKNOWN)

    @property
    def execution_graph(self):
        d = self.dag_adjacency_list
        G = nx.DiGraph()

        for node in d.keys():
            nodes = d[node]
            if len(nodes) == 0:
                G.add_node(node)
                continue
            G.add_edges_from([(node, n) for n in nodes])
        return G

    def __repr__(self):
        return '<id {}>'.format(self.id)

class ComputationalTask(Base):
    __tablename__ = 'comp_tasks'
    # this task db id
    task_id = Column(Integer, primary_key=True)
    pipeline_id = Column(Integer, ForeignKey('comp_pipeline.pipeline_id'))
    # dag node id
    node_id = Column(String)
    # celery task id
    job_id = Column(String)
    # internal id (better for debugging, nodes from 1 to N)
    internal_id = Column(Integer)

    input = Column(MutableJson)
    output = Column(MutableJson)
    state = Column(Integer, default=UNKNOWN)

max_stars_repo_name: mguidon/simcore-comp-backend

max_issues_repo_name: mguidon/simcore-comp-backend

max_forks_repo_name: mguidon/simcore-comp-backend

reason: import networkx