text:
# -------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------

import numpy as np

from ..proto import onnx_proto
from ..common._apply_operation import (
    apply_abs, apply_add, apply_cast, apply_concat, apply_clip,
    apply_div, apply_exp, apply_mul, apply_reshape, apply_sub)
from ..common._topology import FloatTensorType
from ..common._registration import register_converter
from .._supported_operators import decision_function_classifiers
from .._supported_operators import sklearn_operator_name_map

def _handle_zeros(scope, container, concatenated_prob_name,
                  reduced_prob_name, n_classes):
    """
    This function replaces 0s in concatenated_prob_name with 1s and
    0s in reduced_prob_name with n_classes.
    """
    cast_prob_name = scope.get_unique_variable_name('cast_prob')
    bool_not_cast_prob_name = scope.get_unique_variable_name(
        'bool_not_cast_prob')
    mask_name = scope.get_unique_variable_name('mask')
    masked_concatenated_prob_name = scope.get_unique_variable_name(
        'masked_concatenated_prob')
    n_classes_name = scope.get_unique_variable_name('n_classes')
    reduced_prob_mask_name = scope.get_unique_variable_name(
        'reduced_prob_mask')
    masked_reduced_prob_name = scope.get_unique_variable_name(
        'masked_reduced_prob')

    container.add_initializer(n_classes_name, onnx_proto.TensorProto.FLOAT,
                              [], [n_classes])

    apply_cast(scope, reduced_prob_name, cast_prob_name, container,
               to=onnx_proto.TensorProto.BOOL)
    container.add_node('Not', cast_prob_name,
                       bool_not_cast_prob_name,
                       name=scope.get_unique_operator_name('Not'))
    apply_cast(scope, bool_not_cast_prob_name, mask_name, container,
               to=onnx_proto.TensorProto.FLOAT)
    apply_add(scope, [concatenated_prob_name, mask_name],
              masked_concatenated_prob_name, container, broadcast=1)
    apply_mul(scope, [mask_name, n_classes_name], reduced_prob_mask_name,
              container, broadcast=1)
    apply_add(scope, [reduced_prob_name, reduced_prob_mask_name],
              masked_reduced_prob_name, container, broadcast=0)
    return masked_concatenated_prob_name, masked_reduced_prob_name

def _transform_sigmoid(scope, container, model, df_col_name, k):
    """
    Sigmoid Calibration method
    """
    a_name = scope.get_unique_variable_name('a')
    b_name = scope.get_unique_variable_name('b')
    a_df_prod_name = scope.get_unique_variable_name('a_df_prod')
    exp_parameter_name = scope.get_unique_variable_name(
        'exp_parameter')
    exp_result_name = scope.get_unique_variable_name('exp_result')
    unity_name = scope.get_unique_variable_name('unity')
    denominator_name = scope.get_unique_variable_name('denominator')
    sigmoid_predict_result_name = scope.get_unique_variable_name(
        'sigmoid_predict_result')

    container.add_initializer(a_name, onnx_proto.TensorProto.FLOAT,
                              [], [model.calibrators_[k].a_])
    container.add_initializer(b_name, onnx_proto.TensorProto.FLOAT,
                              [], [model.calibrators_[k].b_])
    container.add_initializer(unity_name, onnx_proto.TensorProto.FLOAT,
                              [], [1])

    apply_mul(scope, [a_name, df_col_name], a_df_prod_name, container,
              broadcast=0)
    apply_add(scope, [a_df_prod_name, b_name], exp_parameter_name,
              container, broadcast=0)
    apply_exp(scope, exp_parameter_name, exp_result_name, container)
    apply_add(scope, [unity_name, exp_result_name], denominator_name,
              container, broadcast=0)
    apply_div(scope, [unity_name, denominator_name],
              sigmoid_predict_result_name, container, broadcast=0)
    return sigmoid_predict_result_name

def _transform_isotonic(scope, container, model, T, k):
    """
    Isotonic calibration method
    This function can only handle one instance at a time because
    ArrayFeatureExtractor can only extract based on the last axis,
    so we can't fetch different columns for different rows.
    """
    if model.calibrators_[k].out_of_bounds == 'clip':
        clipped_df_name = scope.get_unique_variable_name('clipped_df')

        apply_clip(scope, T, clipped_df_name, container,
                   operator_name=scope.get_unique_operator_name('Clip'),
                   max=model.calibrators_[k].X_max_,
                   min=model.calibrators_[k].X_min_)
        T = clipped_df_name

    reshaped_df_name = scope.get_unique_variable_name('reshaped_df')
    calibrator_x_name = scope.get_unique_variable_name('calibrator_x')
    calibrator_y_name = scope.get_unique_variable_name('calibrator_y')
    distance_name = scope.get_unique_variable_name('distance')
    absolute_distance_name = scope.get_unique_variable_name(
        'absolute_distance')
    nearest_x_index_name = scope.get_unique_variable_name(
        'nearest_x_index')
    nearest_y_name = scope.get_unique_variable_name('nearest_y')

    container.add_initializer(
        calibrator_x_name, onnx_proto.TensorProto.FLOAT,
        [len(model.calibrators_[k]._X_)], model.calibrators_[k]._X_)
    container.add_initializer(
        calibrator_y_name, onnx_proto.TensorProto.FLOAT,
        [len(model.calibrators_[k]._y_)], model.calibrators_[k]._y_)

    apply_reshape(scope, T, reshaped_df_name, container,
                  desired_shape=(-1, 1))
    apply_sub(scope, [reshaped_df_name, calibrator_x_name],
              distance_name, container, broadcast=1)
    apply_abs(scope, distance_name, absolute_distance_name, container)
    container.add_node('ArgMin', absolute_distance_name,
                       nearest_x_index_name, axis=1,
                       name=scope.get_unique_operator_name('ArgMin'))
    container.add_node(
        'ArrayFeatureExtractor',
        [calibrator_y_name, nearest_x_index_name],
        nearest_y_name, op_domain='ai.onnx.ml',
        name=scope.get_unique_operator_name('ArrayFeatureExtractor'))
    return nearest_y_name

def convert_calibrated_classifier_base_estimator(scope, operator, container,
                                                 model):
    # Computational graph:
    #
    # In the following graph, variable names are in lower case characters only
    # and operator names are in upper case characters. We borrow operator names
    # from the official ONNX spec:
    # https://github.com/onnx/onnx/blob/master/docs/Operators.md
    # All variables are followed by their shape in [].
    #
    # Symbols:
    # M: Number of instances
    # N: Number of features
    # C: Number of classes
    # CLASSIFIERCONVERTER: classifier converter corresponding to the op_type
    # a: slope in sigmoid model
    # b: intercept in sigmoid model
    # k: variable in the range [0, C)
    # input: input
    # class_prob_tensor: tensor with class probabilities(function output)
    #
    # Graph:
    #
    #   input [M, N] -> CLASSIFIERCONVERTER -> label [M]
    #                          |
    #                          V
    #                    probability_tensor [M, C]
    #                          |
    #         .----------------'---------.
    #         |                          |
    #         V                          V
    # ARRAYFEATUREEXTRACTOR <- k [1] -> ARRAYFEATUREEXTRACTOR
    #         |                          |
    #         V                          V
    #  transposed_df_col[M, 1] transposed_df_col[M, 1]
    #       |--------------------------|----------.--------------------------.
    #       |                          |          |                          |
    #       |if model.method='sigmoid' |          |if model.method='isotonic'|
    #       |                          |          |                          |
    #       V                          V          |if out_of_bounds='clip'   |
    #      MUL <-------- a -------->  MUL         V                          V
    #       |                          |          CLIP     ...             CLIP
    #       V                          V          |                          |
    #  a_df_prod [M, 1]  ... a_df_prod [M, 1]     V                          V
    #       |                          |  clipped_df [M, 1]...clipped_df [M, 1]
    #       V                          V          |                          |
    #      ADD <--------- b ---------> ADD        '-------------------.------'
    #         |                          |                            |
    #         V                          V                            |
    #  exp_parameter [M, 1] ...   exp_parameter [M, 1]                |
    #         |                          |                            |
    #         V                          V                            |
    #        EXP        ...             EXP                           |
    #         |                          |                            |
    #         V                          V                            |
    #  exp_result [M, 1]  ...    exp_result [M, 1]                    |
    #         |                          |                            |
    #         V                          V                            |
    #       ADD <------- unity -------> ADD                           |
    #         |                          |                            |
    #         V                          V                            |
    #  denominator [M, 1]  ...   denominator [M, 1]                   |
    #         |                          |                            |
    #         V                          V                            |
    #        DIV <------- unity ------> DIV                           |
    #         |                          |                            |
    #         V                          V                            |
    # sigmoid_predict_result [M, 1] ... sigmoid_predict_result [M, 1] |
    #         |                          |                            |
    #         '-----.--------------------'                            |
    #               |-------------------------------------------------'
    #               |
    #               V
    #            CONCAT -> concatenated_prob [M, C]
    #                          |
    #        if  C = 2         |  if C != 2
    #      .-------------------'---------------------------.---------.
    #      |                                               |         |
    #      V                                               |         V
    # ARRAYFEATUREEXTRACTOR <- col_number [1]              |    REDUCESUM
    #                   |                                  |         |
    #                   '--------------------------------. |         |
    # unit_float_tensor [1] -> SUB <- first_col [M, 1] <-' |         |
    #                           |                         /          |
    #                           V                        V           V
    #                         CONCAT                    DIV <- reduced_prob [M]
    #                           |                        |
    #                           V                        |
    #                        class_prob_tensor [M, C] <--'

    base_model = model.base_estimator
    op_type = sklearn_operator_name_map[type(base_model)]
    n_classes = len(model.classes_)
    prob_name = [None] * n_classes

    this_operator = scope.declare_local_operator(op_type)
    this_operator.raw_operator = base_model
    this_operator.inputs = operator.inputs
    label_name = scope.declare_local_variable('label')
    df_name = scope.declare_local_variable('probability_tensor',
                                           FloatTensorType())
    this_operator.outputs.append(label_name)
    this_operator.outputs.append(df_name)
    df_inp = df_name.full_name

    for k in range(n_classes):
        cur_k = k
        if n_classes == 2:
            cur_k += 1
            # In case of binary classification, SVMs only return
            # scores for the positive class. We concat the same
            # column twice as we just use the second column.
            if op_type in ('SklearnLinearSVC', 'SklearnSVC'):
                df_input_name = scope.get_unique_variable_name('df_input')
                merged_input_name = scope.get_unique_variable_name(
                    'merged_input')

                apply_reshape(scope, df_inp,
                              df_input_name, container,
                              desired_shape=(-1, 1))
                apply_concat(scope, [df_input_name, df_input_name],
                             merged_input_name, container, axis=1)
                df_inp = merged_input_name
        k_name = scope.get_unique_variable_name('k')
        df_col_name = scope.get_unique_variable_name('transposed_df_col')
        prob_name[k] = scope.get_unique_variable_name('prob_{}'.format(k))

        container.add_initializer(k_name, onnx_proto.TensorProto.INT64,
                                  [], [cur_k])

        container.add_node(
            'ArrayFeatureExtractor', [df_inp, k_name], df_col_name,
            name=scope.get_unique_operator_name('ArrayFeatureExtractor'),
            op_domain='ai.onnx.ml')
        T = (_transform_sigmoid(scope, container, model, df_col_name, k)
             if model.method == 'sigmoid' else
             _transform_isotonic(scope, container, model, df_col_name, k))

        prob_name[k] = T
        if n_classes == 2:
            break

    if n_classes == 2:
        zeroth_col_name = scope.get_unique_variable_name('zeroth_col')
        merged_prob_name = scope.get_unique_variable_name('merged_prob')
        unit_float_tensor_name = scope.get_unique_variable_name(
            'unit_float_tensor')

        container.add_initializer(unit_float_tensor_name,
                                  onnx_proto.TensorProto.FLOAT, [], [1.0])

        apply_sub(scope, [unit_float_tensor_name, prob_name[0]],
                  zeroth_col_name, container, broadcast=1)
        apply_concat(scope, [zeroth_col_name, prob_name[0]],
                     merged_prob_name, container, axis=1)
        class_prob_tensor_name = merged_prob_name
    else:
        concatenated_prob_name = scope.get_unique_variable_name(
            'concatenated_prob')
        reduced_prob_name = scope.get_unique_variable_name('reduced_prob')
        calc_prob_name = scope.get_unique_variable_name('calc_prob')

        apply_concat(scope, prob_name, concatenated_prob_name,
                     container, axis=1)
        container.add_node('ReduceSum', concatenated_prob_name,
                           reduced_prob_name, axes=[1],
                           name=scope.get_unique_operator_name('ReduceSum'))
        num, deno = _handle_zeros(scope, container, concatenated_prob_name,
                                  reduced_prob_name, n_classes)
        apply_div(scope, [num, deno],
                  calc_prob_name, container, broadcast=1)
        class_prob_tensor_name = calc_prob_name
    return class_prob_tensor_name

def convert_sklearn_calibrated_classifier_cv(scope, operator, container):
    # Computational graph:
    #
    # In the following graph, variable names are in lower case characters only
    # and operator names are in upper case characters. We borrow operator names
    # from the official ONNX spec:
    # https://github.com/onnx/onnx/blob/master/docs/Operators.md
    # All variables are followed by their shape in [].
    #
    # Symbols:
    # M: Number of instances
    # N: Number of features
    # C: Number of classes
    # CONVERT_BASE_ESTIMATOR: base estimator convert function defined above
    # clf_length: number of calibrated classifiers
    # input: input
    # output: output
    # class_prob: class probabilities
    #
    # Graph:
    #
    #                         input [M, N]
    #                               |
    #           .-------------------|--------------------------.
    #           |                   |                          |
    #           V                   V                          V
    # CONVERT_BASE_ESTIMATOR  CONVERT_BASE_ESTIMATOR ... CONVERT_BASE_ESTIMATOR
    #           |                   |                          |
    #           V                   V                          V
    #  prob_scores_0 [M, C] prob_scores_1 [M, C] ... prob_scores_(clf_length-1)
    #           |                   |                          |  [M, C]
    #           '-------------------|--------------------------'
    #                               V
    #       add_result [M, C] <--- SUM
    #           |
    #           '--> DIV <- clf_length [1]
    #                 |
    #                 V
    #            class_prob [M, C] -> ARGMAX -> argmax_output [M, 1]
    #                                                   |
    #             classes -> ARRAYFEATUREEXTRACTOR  <---'
    #                               |
    #                               V
    #                            output [1]

    op = operator.raw_operator
    classes = op.classes_
    output_shape = (-1,)
    class_type = onnx_proto.TensorProto.STRING

    if np.issubdtype(op.classes_.dtype, np.floating):
        class_type = onnx_proto.TensorProto.INT32
        classes = classes.astype(np.int32)
    elif np.issubdtype(op.classes_.dtype, np.signedinteger):
        class_type = onnx_proto.TensorProto.INT32
    else:
        classes = np.array([s.encode('utf-8') for s in classes])

    clf_length = len(op.calibrated_classifiers_)
    prob_scores_name = []

    clf_length_name = scope.get_unique_variable_name('clf_length')
    classes_name = scope.get_unique_variable_name('classes')
    reshaped_result_name = scope.get_unique_variable_name('reshaped_result')
    argmax_output_name = scope.get_unique_variable_name('argmax_output')
    array_feature_extractor_result_name = scope.get_unique_variable_name(
        'array_feature_extractor_result')
    add_result_name = scope.get_unique_variable_name('add_result')

    container.add_initializer(classes_name, class_type, classes.shape, classes)
    container.add_initializer(clf_length_name, onnx_proto.TensorProto.FLOAT,
                              [], [clf_length])

    for clf in op.calibrated_classifiers_:
        if (hasattr(clf.base_estimator, 'decision_function') and
                not isinstance(clf.base_estimator,
                               decision_function_classifiers)):
            raise NotImplementedError(
                "'{0}' is not supported with CalibratedClassifierCV yet. "
                "You may raise an issue at "
                "https://github.com/onnx/sklearn-onnx/issues"
                "".format(type(clf.base_estimator)))
        prob_scores_name.append(convert_calibrated_classifier_base_estimator(
            scope, operator, container, clf))

    container.add_node('Sum', [s for s in prob_scores_name],
                       add_result_name, op_version=7,
                       name=scope.get_unique_operator_name('Sum'))
    apply_div(scope, [add_result_name, clf_length_name],
              operator.outputs[1].full_name, container, broadcast=1)
    class_prob_name = operator.outputs[1].full_name
    container.add_node('ArgMax', class_prob_name,
                       argmax_output_name,
                       name=scope.get_unique_operator_name('ArgMax'), axis=1)
    container.add_node(
        'ArrayFeatureExtractor', [classes_name, argmax_output_name],
        array_feature_extractor_result_name, op_domain='ai.onnx.ml',
        name=scope.get_unique_operator_name('ArrayFeatureExtractor'))

    if class_type == onnx_proto.TensorProto.INT32:
        apply_reshape(scope, array_feature_extractor_result_name,
                      reshaped_result_name, container,
                      desired_shape=output_shape)
        apply_cast(scope, reshaped_result_name, operator.outputs[0].full_name,
                   container, to=onnx_proto.TensorProto.INT64)
    else:
        apply_reshape(scope, array_feature_extractor_result_name,
                      operator.outputs[0].full_name, container,
                      desired_shape=output_shape)

register_converter('SklearnCalibratedClassifierCV',
                   convert_sklearn_calibrated_classifier_cv)

max_stars_repo_name: MaxNoe/sklearn-onnx

max_issues_repo_name: MaxNoe/sklearn-onnx

max_forks_repo_name: MaxNoe/sklearn-onnx


text:
from .. import core, all_simulations, get_simulation, get_timestep, get_object
from ..log import logger
import six
import numpy.testing as npt
from sqlalchemy.orm import joinedload, object_session, undefer, Session
import numpy as np

class TangosDbDiff(object):
    """Class to compare two databases, used by the tangos diff command line tool"""

    def __init__(self, uri1, uri2, max_objects=10, ignore_keys=[]):
        if isinstance(uri1, Session):
            self.session1 = uri1
            uri1 = str(uri1.connection().engine)
        else:
            core.init_db(uri1)
            self.session1 = core.get_default_session()

        if isinstance(uri2, Session):
            self.session2 = uri2
            uri2 = str(uri2.connection().engine)
        else:
            core.init_db(uri2)
            self.session2 = core.get_default_session()

        self.test_simulations = True
        self.test_timesteps = True
        self.test_objects = True
        self.max_objects = max_objects
        self.test_properties = True
        self.ignore_keys = ignore_keys
        self.failed = False

        logger.info("Database 1 = %s",uri1)
        logger.info("Database 2 = %s",uri2)
        logger.info("For each timestep will check data in first %d objects of each type",max_objects)
        if len(ignore_keys)>0:
            logger.info("The value of properties with the names %s will be ignored.",", ".join(ignore_keys))
            logger.info("Only the existence and type of these properties will be checked.")

    def fail(self, message, *args):
        logger.error(message, *args)
        self.failed = True

    def compare(self):
        sims1 = set([s.basename for s in all_simulations(self.session1)])
        sims2 = set([s.basename for s in all_simulations(self.session2)])

        if len(sims1-sims2)>0:
            self.fail("Database 1 has additional simulations: %s", list(sims1-sims2))
        if len(sims2-sims1)>0:
            self.fail("Database 2 has additional simulations: %s", list(sims2-sims1))

        if self.test_simulations:
            for sim in sims1.intersection(sims2):
                self.compare_simulation(sim)

    def compare_simulation(self, sim):
        logger.info("Comparing simulation %s",sim)
        sim1 = get_simulation(sim, self.session1)
        sim2 = get_simulation(sim, self.session2)

        ts1 = set([ts.extension for ts in sim1.timesteps])
        ts2 = set([ts.extension for ts in sim2.timesteps])

        self._check_same_set(ts1, ts2, str(sim), 'timesteps')

        if self.test_timesteps:
            for ts in ts1.intersection(ts2):
                self.compare_timestep(sim+"/"+ts)

    def compare_timestep(self, ts):
        logger.info("Comparing timestep %s", ts)
        ts1 = get_timestep(str(ts), self.session1)
        ts2 = get_timestep(str(ts), self.session2)

        obj_filter = (core.SimulationObjectBase.halo_number < self.max_objects) | (core.SimulationObjectBase.object_typecode == core.SimulationObjectBase.object_typecode_from_tag('bh'))

        objects1 = dict([(o.path, o) for o in ts1.objects.filter(obj_filter).all()])
        objects2 = dict([(o.path, o) for o in ts2.objects.filter(obj_filter).all()])
        self._check_same_set(objects1.keys(), objects2.keys(), str(ts), 'objects')

        if self.test_objects:
            keys = set(objects1.keys()).intersection(objects2.keys())
            for k in keys:
                self._compare_objects(objects1[k], objects2[k])

    def _joined_links_load(self, parent):
        return object_session(parent).query(core.HaloLink).with_parent(parent, type(parent).all_links).\
            options(joinedload(core.HaloLink.halo_to))

    def _joined_properties_load(self, parent):
        return object_session(parent).query(core.HaloProperty).with_parent(parent, type(parent).all_properties).\
            options(undefer("*")).options(joinedload(core.HaloProperty.name))

    def compare_object(self, obj):
        obj1 = get_object(obj, self.session1)
        obj2 = get_object(obj, self.session2)
        self._compare_objects(obj1, obj2)

    def _compare_objects(self, obj1, obj2):
        properties1 = dict([(prop.name.text, prop.data_raw) for prop in self._joined_properties_load(obj1)])
        properties2 = dict([(prop.name.text, prop.data_raw) for prop in self._joined_properties_load(obj2)])

        self._check_dict_same(properties1, properties2, obj1.path)

        links1 = dict([(link.relation.text+"->"+link.halo_to.path, link.weight) for link in self._joined_links_load(obj1)])
        links2 = dict([(link.relation.text+"->"+link.halo_to.path, link.weight) for link in self._joined_links_load(obj2)])

        self._check_dict_same(links1, links2, obj1.path, 'links')

    def _check_dict_same(self, properties1, properties2, name_of_object, name_of_things='properties'):
        prop1_names = set(properties1.keys())
        prop2_names = set(properties2.keys())
        self._check_same_set(prop1_names, prop2_names, name_of_object, name_of_things)
        self._check_almost_equal(properties1, properties2, name_of_object)

    def _check_same_set(self, objects1, objects2, name_of_object, name_of_things):
        objects1 = set(objects1)
        objects2 = set(objects2)
        self._check_setdiff_null("Database 1", objects1, objects2, name_of_object, name_of_things)
        self._check_setdiff_null("Database 2",objects2, objects1, name_of_object, name_of_things)

    def _check_setdiff_null(self, name_of_db_1, objects1, objects2,  name_of_object, name_of_things):
        if len(objects1 - objects2) > 0:
            self.fail("When comparing %s, %s has %d additional %s (of total %d)", name_of_object, name_of_db_1,
                      len(objects1 - objects2), name_of_things, len(objects1))
            self._print_objects(objects1 - objects2)

    def _print_objects(self, objects):
        for o in objects:
            logger.info("  %s", o)

    def _check_almost_equal(self, dict1, dict2, name_of_object):
        for d in dict1.keys():
            if d in dict2:
                v1 = dict1[d]
                v2 = dict2[d]
                if d in self.ignore_keys:
                    self._check_properties_compatible(v1, v2, d, name_of_object)
                else:
                    self._check_properties_equal(v1, v2, d, name_of_object)

    def _check_properties_equal(self, v1, v2, d, name_of_object):
        try:
            self._assert_equal(v1, v2)
        except AssertionError as e:
            self.fail("When comparing %s, value mismatch for key %s", name_of_object, d)
            self.fail("%s", e)

    def _check_properties_compatible(self, v1, v2, name_of_property, name_of_object):
        if type(v1) != type(v2):
            self.fail("When comparing %s, types of key %s differ", name_of_object, name_of_property)
        else:
            if isinstance(v1, np.ndarray):
                if v1.shape != v2.shape:
                    self.fail("When comparing %s, array shapes of key %s differ", name_of_object, name_of_property)

    def _assert_equal(self, v1, v2):
        if v1 is None or v2 is None:
            assert v1 is None, "Value 2 is None but Value 1 is not None"
            assert v2 is None, "Value 1 is None but Value 2 is not None"
        elif isinstance(v1, six.string_types):
            assert v1 == v2
        elif isinstance(v1, tuple):
            for e1, e2 in zip(v1,v2):
                self._assert_equal(e1, e2)
        else:
            npt.assert_allclose(v1, v2, rtol=1e-3, atol=1e-3)

ext: py

max_stars_repo_name: j-davies-astro/tangos

max_issues_repo_name: j-davies-astro/tangos

max_forks_repo_name: j-davies-astro/tangos

reason: import numpy


text:
import os
import yaml
import tables as t
import numpy as np
import pylab as p
import pandas as pa

from jobmon import redisbackend as rb

def sumvals(x):
    vec = np.array(x)
    return vec.mean(), vec.std()

db = rb.RedisDataStore('localhost')
jobhash = db.select_jobfile()

resdir = os.path.join('/home/bana/largeresearch/results', jobhash)

res = pa.DataFrame()
#res = pa.DataFrame(columns=('iters', 'bayes', 'post', 'sampler_full', 'sampler_20'))

for exp in os.listdir(resdir):
    # Get param value
    iters = yaml.load(db.get_params(exp))['iters']
    for fname in os.listdir(os.path.join(resdir,exp)):
        fid = t.openFile(os.path.join(resdir,exp,fname))
        v = {}
        for item in fid.root.object.objfxn._v_attrs._f_list():
            v[item] = fid.root.object.objfxn._v_attrs[item] # Ugly
        v['iters'] = iters
        row = pa.DataFrame(v, index=[1])
        res = res.append(row, ignore_index=True)
        fid.close()

groups = res.groupby('iters')
means = groups.mean()
std = groups.std()

print means.columns
p.errorbar(means.index, means['bayes_error'], yerr=std['bayes_error'])
p.errorbar(means.index, means['posterior_error'], yerr=std['posterior_error'])
p.errorbar(means.index, means['sample_error_full'], yerr=std['sample_error_full'], label='Sampler full')
p.errorbar(means.index, means['sample_error_20'], yerr=std['sample_error_20'], label='Sampler20')
p.legend()
p.show()

max_stars_repo_name: binarybana/samcnet

max_issues_repo_name: binarybana/samcnet

max_forks_repo_name: binarybana/samcnet


text:
import itertools
import logging
from enum import Enum, auto
from typing import (
    Mapping,
    Tuple,
    Union,
    cast,
    List,
    AbstractSet,
    Dict,
    Iterable,
    Optional,
    Callable,
    Sequence,
)
from adam.language_specific.english import DETERMINERS

from attr import attrib, attrs
from attr.validators import instance_of, optional
from immutablecollections import immutabledict, immutableset, ImmutableSet
from networkx import (
    number_weakly_connected_components,
    DiGraph,
    weakly_connected_components,
)
from vistautils.span import Span

from adam.language import LinguisticDescription, TokenSequenceLinguisticDescription
from adam.learner import LearningExample, get_largest_matching_pattern
from adam.learner.alignments import LanguageConceptAlignment
from adam.learner.alignments import LanguagePerceptionSemanticAlignment
from adam.learner.language_mode import LanguageMode
from adam.learner.perception_graph_template import PerceptionGraphTemplate
from adam.learner.surface_templates import (
    STANDARD_SLOT_VARIABLES,
    SurfaceTemplate,
    SurfaceTemplateBoundToSemanticNodes,
)
from adam.ontology.ontology import Ontology
from adam.ontology.phase1_spatial_relations import Region
from adam.perception import PerceptualRepresentation, MatchMode, ObjectPerception
from adam.perception.developmental_primitive_perception import (
    DevelopmentalPrimitivePerceptionFrame,
    RgbColorPerception,
)
from adam.perception.perception_graph import (
    ObjectSemanticNodePerceptionPredicate,
    PerceptionGraphPatternMatch,
    PerceptionGraphPattern,
    IsPathPredicate,
    RegionPredicate,
    REFERENCE_OBJECT_LABEL,
    REFERENCE_OBJECT_DESTINATION_LABEL,
    REFERENCE_OBJECT_SOURCE_LABEL,
    NodePredicate,
    RelationTypeIsPredicate,
    PerceptionGraph,
    DebugCallableType,
    GraphLogger,
)
from adam.semantics import (
    Concept,
    ObjectSemanticNode,
    SemanticNode,
    SyntaxSemanticsVariable,
)
from adam.utils import networkx_utils
from adam.utils.networkx_utils import subgraph

def get_classifier_for_string(input_string: str) -> Optional[str]:
    if input_string in ["chwang2", "jr3", "jwo1 dz"]:
        return "yi1_jang1"
    elif input_string in ["shu1"]:
        return "yi1_ben3"
    elif input_string in ["wu1"]:
        return "yi1_jyan1"
    elif input_string in ["chi4 che1", "ka3 che1"]:
        return "yi1_lyang4"
    elif input_string in ["yi3 dz"]:
        return "yi1_ba3"
    elif input_string in ["shou3", "gou3", "mau1", "nyau3", "syung2"]:
        return "yi1_jr1"
    elif input_string in ["men2"]:
        return "yi1_shan4"
    elif input_string in ["mau4 dz"]:
        return "yi1_ding3"
    elif input_string in ["chyu1 chi2 bing3", "niu2 rou1"]:
        return "yi1_kwai4"
    elif input_string in ["niu2"]:
        return "yi1_tiao2"
    elif input_string in ["ji1"]:
        return "yi1_zhi1"
    # eliminate mass and proper nouns and use the default classifier if another one hasn't already been used
    elif input_string not in [
        "ba4 ba4",
        "ma1 ma1",
        "shwei3",
        "gwo3 jr1",
        "nyou2 nai3",
        "di4 myan4",
    ]:
        return "yi1_ge4"
    return None

def pattern_match_to_description(
    *,
    surface_template: SurfaceTemplate,
    pattern: PerceptionGraphTemplate,
    match: PerceptionGraphPatternMatch,
    matched_objects_to_names: Mapping[ObjectSemanticNode, Tuple[str, ...]],
) -> TokenSequenceLinguisticDescription:
    """
    Given a `SurfaceTemplate`, will fill it in using a *match* for a *pattern*.
    This requires a mapping from matched object nodes in the perception
    to the strings which should be used to name them.
    """

    try:
        return surface_template.instantiate(
            template_variable_to_filler=immutabledict(
                (
                    pattern.pattern_node_to_template_variable[pattern_node],
                    matched_objects_to_names[
                        # We know, but the type system does not,
                        # that if a MatchedObjectPerceptionPredicate matched,
                        # the graph node must be a MatchedObjectNode
                        cast(ObjectSemanticNode, matched_graph_node)
                    ],
                )
                for (
                    pattern_node,
                    matched_graph_node,
                ) in match.pattern_node_to_matched_graph_node.items()
                if isinstance(pattern_node, ObjectSemanticNodePerceptionPredicate)
                # There can sometimes be relevant matched object nodes which are not themselves
                # slots, like the addressed possessor for "your X".
                and pattern_node in pattern.pattern_node_to_template_variable
            )
        )
    except KeyError as e:
        logging.warning(str(e))
        raise

def pattern_match_to_semantic_node(
    *,
    concept: Concept,
    pattern: PerceptionGraphTemplate,
    match: PerceptionGraphPatternMatch,
) -> SemanticNode:
    template_variable_to_filler: Mapping[
        SyntaxSemanticsVariable, ObjectSemanticNode
    ] = immutabledict(
        (
            pattern.pattern_node_to_template_variable[pattern_node],
            # We know, but the type system does not,
            # that if a ObjectSemanticNodePerceptionPredicate matched,
            # the graph node must be a MatchedObjectNode
            cast(ObjectSemanticNode, matched_graph_node),
        )
        for (
            pattern_node,
            matched_graph_node,
        ) in match.pattern_node_to_matched_graph_node.items()
        if isinstance(pattern_node, ObjectSemanticNodePerceptionPredicate)
        # There can sometimes be relevant matched object nodes which are not themselves
        # slots, like the addressed possessor for "your X".
        and pattern_node in pattern.pattern_node_to_template_variable
    )

    return SemanticNode.for_concepts_and_arguments(
        concept, slots_to_fillers=template_variable_to_filler
    )

def assert_static_situation(
    to_check: Union[
        LearningExample[DevelopmentalPrimitivePerceptionFrame, LinguisticDescription],
        PerceptualRepresentation[DevelopmentalPrimitivePerceptionFrame],
    ]
):
    if isinstance(to_check, LearningExample):
        perception = to_check.perception
    else:
        perception = to_check

    if len(perception.frames) != 1:
        raise RuntimeError("Pursuit learner can only handle single frames for now")
    if not isinstance(perception.frames[0], DevelopmentalPrimitivePerceptionFrame):
        raise RuntimeError(f"Cannot process frame type: {type(perception.frames[0])}")

def pattern_remove_incomplete_region_or_spatial_path(
    perception_graph: PerceptionGraphPattern
) -> PerceptionGraphPattern:
    """
    Helper function to return a `PerceptionGraphPattern` verifying
    that region and spatial path perceptions contain a reference object.
    """
    graph = perception_graph.copy_as_digraph()
    region_and_path_nodes: ImmutableSet[NodePredicate] = immutableset(
        node
        for node in graph.nodes
        if isinstance(node, IsPathPredicate) or isinstance(node, RegionPredicate)
    )
    nodes_without_reference: List[NodePredicate] = []
    for node in region_and_path_nodes:
        has_reference_edge: bool = False
        for successor in graph.successors(node):
            predicate = graph.edges[node, successor]["predicate"]
            if isinstance(predicate, RelationTypeIsPredicate):
                if predicate.relation_type in [
                    REFERENCE_OBJECT_LABEL,
                    REFERENCE_OBJECT_DESTINATION_LABEL,
                    REFERENCE_OBJECT_SOURCE_LABEL,
                ]:
                    has_reference_edge = True
                    break
        if not has_reference_edge:
            nodes_without_reference.append(node)

    logging.info(
        f"Removing incomplete regions and paths. "
        f"Removing nodes: {nodes_without_reference}"
    )
    graph.remove_nodes_from(nodes_without_reference)

    def sort_by_num_nodes(g: DiGraph) -> int:
        return len(g.nodes)

    # We should maybe consider doing this a different way
    # As this approach just brute force solves the problem rather than being methodical about it
    if number_weakly_connected_components(graph) > 1:
        components = [
            component
            for component in [
                subgraph(graph, comp) for comp in weakly_connected_components(graph)
            ]
        ]
        components.sort(key=sort_by_num_nodes, reverse=True)
        computed_graph = subgraph(graph, components[0].nodes)
        removed_nodes: List[NodePredicate] = []
        for i in range(1, len(components)):
            removed_nodes.extend(components[i].nodes)
        logging.info(f"Cleanup disconnected elements. Removing: {removed_nodes}")
    else:
        computed_graph = graph

    return PerceptionGraphPattern(computed_graph, dynamic=perception_graph.dynamic)

def covers_entire_utterance(
    bound_surface_template: SurfaceTemplateBoundToSemanticNodes,
    language_concept_alignment: LanguageConceptAlignment,
    *,
    ignore_determiners: bool = False,
) -> bool:
    num_covered_tokens = 0
    for element in bound_surface_template.surface_template.elements:
        if isinstance(element, str):
            # this is a hack for the classifier learning experiment (the classifier will be ignored with sized so we must ignore it here too)
            if element[:3] != "yi1":
                num_covered_tokens += 1
        else:
            slot_for_element = language_concept_alignment.node_to_language_span[
                bound_surface_template.slot_to_semantic_node[element]
            ]
            aligned_strings_for_slot = language_concept_alignment.language[
                slot_for_element.start : slot_for_element.end
            ]
            # we need to check here that the determiners aren't getting aligned; otherwise it can mess up our count
            if ignore_determiners:
                num_covered_tokens += len(
                    [x for x in aligned_strings_for_slot if x not in DETERMINERS]
                )
            else:
                num_covered_tokens += len(aligned_strings_for_slot)

    # We may need to ignore counting english determiners in our comparison
    # to the template as the way we treat english determiners is currently
    # a hack. See: https://github.com/isi-vista/adam/issues/498
    sized_tokens = (
        len([token for token in language_concept_alignment.language.as_token_sequence()])
        if not ignore_determiners
        else len(
            [
                token
                for token in language_concept_alignment.language.as_token_sequence()
                if token not in DETERMINERS
            ]
        )
    )
    # This assumes the slots and the non-slot elements are non-overlapping,
    # which is true for how we construct them.
    return num_covered_tokens == sized_tokens

class AlignmentSlots(Enum):
    """An argument is a slot for an object, and a fixed string is something we wish to learn"""

    Argument = auto()
    FixedString = auto()

@attrs(frozen=True, slots=True)
class SemanticNodeWithSpan:
    """This is a tuple class currently used by our verb and relation learners"""

    node: SemanticNode = attrib(validator=instance_of(SemanticNode))
    span: Span = attrib(validator=instance_of(Span))

def candidate_templates(
    language_perception_semantic_alignment: LanguagePerceptionSemanticAlignment,
    max_length: int,
    language_mode: LanguageMode,
    candidate_templates_function: Callable[[], Iterable[Tuple[AlignmentSlots, ...]]],
) -> AbstractSet[SurfaceTemplateBoundToSemanticNodes]:
    ret = []
    language_concept_alignment = (
        language_perception_semantic_alignment.language_concept_alignment
    )
    # We make an assumption that the order of nouns in our sentence will be
    # in the same order as they appear in the sentence, left to right,
    # To avoid calculating this condition multiple times we do so once
    # For the number of arguments (nouns) our candidate template
    # desires and store the resulting possible aligments in this dict
    # for easy look up.
    num_arguments_to_alignments_sets: Dict[
        int, ImmutableSet[Tuple[SemanticNodeWithSpan, ...]]
    ] = {}
    sentence_tokens = language_concept_alignment.language.as_token_sequence()

    # Any recognized object is a potential verb argument.
    # This method does not properly handle arguments which themselves have complex structure.
    # Arguments are restricted to ONLY be ObjectSemanticNodes
    # See https://github.com/isi-vista/adam/issues/785

    # We currently do not handle verb arguments
    # which are dropped and/or expressed only via morphology:
    # https://github.com/isi-vista/adam/issues/786

    def in_left_to_right_order(semantic_nodes: Tuple[SemanticNodeWithSpan, ...]) -> bool:
        previous_node = semantic_nodes[0]
        for i in range(1, len(semantic_nodes)):
            if not previous_node.span.precedes(semantic_nodes[i].span):
                return False
            previous_node = semantic_nodes[i]
        return True

    # A sample case for invalid tokens spans is if the language of a situation is
    # "a dog pushes a box to a car" but we are given a candidate template
    # of AFA then we could generate "slot1 pushes a box to slot2"
    # which is an undesired output. So if we provide the spans where
    # aligned objects are we can invalidate templates like the problem one above
    def is_legal_template_span(
        candidate_token_span: Span, *, invalid_token_spans: ImmutableSet[Span]
    ) -> bool:
        # A template token span can't exceed the bounds of the utterance
        if candidate_token_span.start < 0:
            return False
        if candidate_token_span.end > len(sentence_tokens):
            return False
        # or be bigger than our maximum template size...
        if len(candidate_token_span) > max_length:
            return False

        for span in invalid_token_spans:
            if candidate_token_span.contains_span(span):
                return False

        # or we have already aligned any of the tokens in between the objects
        # to some other meaning.
        for token_index in range(candidate_token_span.start, candidate_token_span.end):
            if language_concept_alignment.token_index_is_aligned(token_index):
                return False

        return True

    def aligned_object_nodes(
        num_arguments: int,
        num_arguments_to_alignments_sets: Dict[
            int, ImmutableSet[Tuple[SemanticNodeWithSpan, ...]]
        ],
        language_concept_alignment: LanguageConceptAlignment,
    ) -> ImmutableSet[Tuple[SemanticNodeWithSpan, ...]]:
        if num_arguments not in num_arguments_to_alignments_sets.keys():
            # we haven't seen a request for this number of arguments before so we need to generate all the valid options
            semantic_nodes_with_spans = immutableset(
                SemanticNodeWithSpan(node=node, span=span)
                for (
                    node,
                    span,
                ) in language_concept_alignment.node_to_language_span.items()
                if isinstance(node, ObjectSemanticNode)
            )
            num_arguments_to_alignments_sets[num_arguments] = immutableset(
                ordered_semantic_nodes
                for ordered_semantic_nodes in itertools.product(
                    semantic_nodes_with_spans, repeat=num_arguments
                )
                if in_left_to_right_order(ordered_semantic_nodes)
            )
        return num_arguments_to_alignments_sets[num_arguments]

    def process_aligned_objects_with_template(
        candidate_template: Tuple[AlignmentSlots, ...],
        aligned_nodes: Tuple[SemanticNodeWithSpan, ...],
        *,
        invalid_token_spans: ImmutableSet[Span],
    ) -> Iterable[Optional[SurfaceTemplateBoundToSemanticNodes]]:

        aligned_node_index = 0
        template_elements: List[Union[str, SyntaxSemanticsVariable]] = []
        slot_to_semantic_node: List[Tuple[SyntaxSemanticsVariable, SemanticNode]] = []

        # We need to handle fixed strings that are pre or post fix to the rest of the
        # Sentence differently as they don't have a fixed length so we could generate
        # multiple options.
        prefix_string_end = None
        postfix_string_start = None
        # In the event we generate a candidate template like:
        # A, F, F, A then we want to compute this like A, F, A
        # So we keep track if the previous token was a FixedString indicator
        previous_node_was_string = False

        for token in candidate_template:
            # If the token in our template is an argument we need to assign it a
            # unique SyntaxSemanticsVariable, and map it to the SemanticNode
            if token == AlignmentSlots.Argument:
                slot_semantic_variable = STANDARD_SLOT_VARIABLES[aligned_node_index]
                template_elements.append(slot_semantic_variable)
                aligned_node = aligned_nodes[aligned_node_index].node
                if not isinstance(aligned_node, ObjectSemanticNode):
                    logging.debug(
                        f"Attempted to make template where an Argument is not an ObjectSemanticNode."
                        f"Invalid node: {aligned_node}"
                    )
                    # Log this failure and then ignore this attempt
                    yield None
                slot_to_semantic_node.append((slot_semantic_variable, aligned_node))
                aligned_node_index += 1
                previous_node_was_string = False
            else:
                # We ignore this case to process A, F, F, A like A, F, A
                if previous_node_was_string:
                    continue
                # We make a note of where the end of our prefix string can be
                # Then continue as we'll handle this case afterwards
                elif aligned_node_index == 0:
                    prefix_string_end = aligned_nodes[aligned_node_index].span.start
                # Similiarly to above, we instead mark the start of the postfix string
                elif aligned_node_index == len(aligned_nodes):
                    postfix_string_start = aligned_nodes[aligned_node_index - 1].span.end
                else:
                    # If our FixedString is flanked by two Arguments we just want to acquire all the tokens
                    # between them
                    if (
                        aligned_nodes[aligned_node_index - 1].span.end
                        != aligned_nodes[aligned_node_index].span.start
                    ):
                        candidate_token_span = Span(
                            aligned_nodes[aligned_node_index - 1].span.end,
                            aligned_nodes[aligned_node_index].span.start,
                        )
                        if not is_legal_template_span(
                            candidate_token_span, invalid_token_spans=invalid_token_spans
                        ):
                            # If not a valid span, ignore this attempt
                            continue
                        template_elements.extend(
                            sentence_tokens[
                                candidate_token_span.start : candidate_token_span.end
                            ]
                        )
                    previous_node_was_string = True
        # We need to handle searching before or after the aligned token
        # And we could generate multiple options of different lengths
        # between 1 and _MAXIMUM_ACTION_TEMPLATE_TOKEN_LENGTH
        if prefix_string_end and postfix_string_start:
            for max_token_length_for_template_prefix in range(1, max_length + 1):
                prefix_candidate_token_span = Span(
                    prefix_string_end - max_token_length_for_template_prefix,
                    prefix_string_end,
                )
                if is_legal_template_span(
                    prefix_candidate_token_span, invalid_token_spans=invalid_token_spans
                ):
                    for max_token_length_for_template_postfix in range(1, max_length + 1):
                        postfix_candidate_token_span = Span(
                            postfix_string_start,
                            postfix_string_start + max_token_length_for_template_postfix,
                        )
                        if is_legal_template_span(
                            postfix_candidate_token_span,
                            invalid_token_spans=invalid_token_spans,
                        ):
                            final_template_elements: List[
                                Union[str, SyntaxSemanticsVariable]
                            ] = list(
                                sentence_tokens[
                                    prefix_candidate_token_span.start : prefix_candidate_token_span.end
                                ]
                            )
                            final_template_elements.extend(template_elements)
                            final_template_elements.extend(
                                sentence_tokens[
                                    postfix_candidate_token_span.start : postfix_candidate_token_span.end
                                ]
                            )
                            yield SurfaceTemplateBoundToSemanticNodes(
                                surface_template=SurfaceTemplate(
                                    elements=final_template_elements,
                                    determiner_prefix_slots=[
                                        SLOT for (SLOT, _) in slot_to_semantic_node
                                    ],
                                    language_mode=language_mode,
                                ),
                                slot_to_semantic_node=slot_to_semantic_node,
                            )
        elif prefix_string_end:
            for max_token_length_for_template_prefix in range(1, max_length + 1):
                prefix_candidate_token_span = Span(
                    prefix_string_end - max_token_length_for_template_prefix,
                    prefix_string_end,
                )
                if is_legal_template_span(
                    prefix_candidate_token_span, invalid_token_spans=invalid_token_spans
                ):
                    final_template_elements = list(
                        sentence_tokens[
                            prefix_candidate_token_span.start : prefix_candidate_token_span.end
                        ]
                    )
                    final_template_elements.extend(template_elements)
                    yield SurfaceTemplateBoundToSemanticNodes(
                        surface_template=SurfaceTemplate(
                            elements=final_template_elements,
                            determiner_prefix_slots=[
                                SLOT for (SLOT, _) in slot_to_semantic_node
                            ],
                            language_mode=language_mode,
                        ),
                        slot_to_semantic_node=slot_to_semantic_node,
                    )
        elif postfix_string_start:
            for max_token_length_for_template_postfix in range(1, max_length + 1):
                postfix_candidate_token_span = Span(
                    postfix_string_start,
                    postfix_string_start + max_token_length_for_template_postfix,
                )
                if is_legal_template_span(
                    postfix_candidate_token_span, invalid_token_spans=invalid_token_spans
                ):
                    final_template_elements = list(template_elements)
                    final_template_elements.extend(
                        sentence_tokens[
                            postfix_candidate_token_span.start : postfix_candidate_token_span.end
                        ]
                    )
                    yield SurfaceTemplateBoundToSemanticNodes(
                        surface_template=SurfaceTemplate(
                            elements=final_template_elements,
                            determiner_prefix_slots=[
                                SLOT for (SLOT, _) in slot_to_semantic_node
                            ],
                            language_mode=language_mode,
                        ),
                        slot_to_semantic_node=slot_to_semantic_node,
                    )
        else:
            yield SurfaceTemplateBoundToSemanticNodes(
                surface_template=SurfaceTemplate(
                    elements=template_elements,
                    determiner_prefix_slots=[SLOT for (SLOT, _) in slot_to_semantic_node],
                    language_mode=language_mode,
                ),
                slot_to_semantic_node=slot_to_semantic_node,
            )

    # Generate all the possible verb template alignments
    for candidate_template in candidate_templates_function():
        for aligned_nodes in aligned_object_nodes(
            sum(1 for token in candidate_template if token == AlignmentSlots.Argument),
            num_arguments_to_alignments_sets,
            language_concept_alignment,
        ):
            # aligned_object_nodes is guaranteed to only give us alignments
            # Which the spans go from left most to right most
            # We also provide a set of invalid token spans
            # for fixed string positions.
            # see: https://github.com/isi-vista/adam/issues/867
            invalid_token_spans = immutableset(
                language_concept_alignment.node_to_language_span.values()
            )
            for (
                surface_template_bound_to_semantic_nodes
            ) in process_aligned_objects_with_template(
                candidate_template, aligned_nodes, invalid_token_spans=invalid_token_spans
            ):
                if surface_template_bound_to_semantic_nodes:
                    ret.append(surface_template_bound_to_semantic_nodes)

    def contains_str(bst: SurfaceTemplateBoundToSemanticNodes) -> bool:
        for element in bst.surface_template.elements:
            if isinstance(element, str):
                return True
        return False

    return immutableset(
        bound_surface_template
        for bound_surface_template in ret
        # For now, we require templates to account for the entire utterance.
        # See https://github.com/isi-vista/adam/issues/789
        if covers_entire_utterance(bound_surface_template, language_concept_alignment)
        and contains_str(bound_surface_template)
    )

def default_post_process_enrichment(
    perception_graph_after_matching: PerceptionGraph,
    immutable_new_nodes: AbstractSet[SemanticNode],
) -> Tuple[PerceptionGraph, AbstractSet[SemanticNode]]:
    return perception_graph_after_matching, immutable_new_nodes

@attrs(frozen=True, slots=True)
class PartialMatchRatio:
    """
    See `compute_match_ratio`
    """

    matching_subgraph: Optional[PerceptionGraphPattern] = attrib(
        validator=optional(instance_of(PerceptionGraphPattern))
    )
    num_nodes_matched: int = attrib(validator=instance_of(int), kw_only=True)
    num_nodes_in_pattern: int = attrib(validator=instance_of(int), kw_only=True)
    matched_exactly: bool = attrib(init=False)
    match_ratio: float = attrib(init=False)

    @matched_exactly.default
    def matched(self) -> bool:
        return self.num_nodes_in_pattern == self.num_nodes_matched

    @match_ratio.default
    def ratio(self) -> float:
        return self.num_nodes_matched / self.num_nodes_in_pattern

def compute_match_ratio(
    pattern: PerceptionGraphTemplate,
    graph: PerceptionGraph,
    ontology: Ontology,
    *,
    graph_logger: Optional[GraphLogger] = None,
    debug_callback: Optional[DebugCallableType] = None,
) -> PartialMatchRatio:
    """
    Computes the fraction of pattern graph nodes of *pattern* which match *graph*.
    """
    hypothesis_pattern_common_subgraph = get_largest_matching_pattern(
        pattern.graph_pattern,
        graph,
        debug_callback=debug_callback,
        graph_logger=graph_logger,
        ontology=ontology,
        match_mode=MatchMode.OBJECT,
    )

    return PartialMatchRatio(
        hypothesis_pattern_common_subgraph,
        num_nodes_matched=(
            len(hypothesis_pattern_common_subgraph.copy_as_digraph().nodes)
            if hypothesis_pattern_common_subgraph
            else 0
        ),
        num_nodes_in_pattern=len(pattern.graph_pattern),
    )

def get_objects_from_perception(
    observed_perception_graph: PerceptionGraph
) -> List[PerceptionGraph]:
    """
    Utility function to get a list of `PerceptionGraphs` which are independent objects in the scene
    """
    perception_as_digraph = observed_perception_graph.copy_as_digraph()
    perception_as_graph = perception_as_digraph.to_undirected()

    meanings = []

    # 1) Take all of the obj perc that dont have part of relationships with anything else
    root_object_percetion_nodes = []
    for node in perception_as_graph.nodes:
        if isinstance(node, ObjectPerception) and node.debug_handle != "the ground":
            if not any(
                [
                    u == node and str(data["label"]) == "partOf"
                    for u, v, data in perception_as_digraph.edges.data()
                ]
            ):
                root_object_percetion_nodes.append(node)

    # 2) for each of these, walk along the part of relationships backwards,
    # i.e find all of the subparts of the root object
    for root_object_perception_node in root_object_percetion_nodes:
        # Iteratively get all other object perceptions that connect to a root with a part of
        # relation
        all_object_perception_nodes = [root_object_perception_node]
        frontier = [root_object_perception_node]
        updated = True
        while updated:
            updated = False
            new_frontier = []
            for frontier_node in frontier:
                for node in perception_as_graph.neighbors(frontier_node):
                    edge_data = perception_as_digraph.get_edge_data(
                        node, frontier_node, default=-1
                    )
                    if edge_data != -1 and str(edge_data["label"]) == "partOf":
                        new_frontier.append(node)

            if new_frontier:
                all_object_perception_nodes.extend(new_frontier)
                updated = True
                frontier = new_frontier

        # Now we have a list of all perceptions that are connected
        # 3) For each of these objects including root object, get axes, properties,
        # and relations and regions which are between these internal object perceptions
        other_nodes = []
        for node in all_object_perception_nodes:
            for neighbor in perception_as_graph.neighbors(node):
                # Filter out regions that don't have a reference in all object perception nodes
                # TODO: We currently remove colors to achieve a match - otherwise finding
                #  patterns fails.
                if (
                    isinstance(neighbor, Region)
                    and neighbor.reference_object not in all_object_perception_nodes
                    or isinstance(neighbor, RgbColorPerception)
                ):
                    continue
                # Append all other none-object nodes to be kept in the subgraph
                if not isinstance(neighbor, ObjectPerception):
                    other_nodes.append(neighbor)

        generated_subgraph = networkx_utils.subgraph(
            perception_as_digraph, all_object_perception_nodes + other_nodes
        )
        meanings.append(PerceptionGraph(generated_subgraph))

    logging.info(f"Got {len(meanings)} candidate meanings")
    return meanings

def candidate_object_hypotheses(
    language_perception_semantic_alignment: LanguagePerceptionSemanticAlignment
) -> Sequence[PerceptionGraphTemplate]:
    """
    Given a learning input, returns all possible meaning hypotheses.
    """
    return [
        PerceptionGraphTemplate(
            graph_pattern=PerceptionGraphPattern.from_graph(
                object_
            ).perception_graph_pattern
        )
        for object_ in get_objects_from_perception(
            language_perception_semantic_alignment.perception_semantic_alignment.perception_graph
        )
    ]

def get_slot_from_semantic_node(
    object_concept: Concept, semantic_node: SemanticNode
) -> str:
    slot = ""
    for slot_var, object_node in semantic_node.slot_fillings.items():
        if object_node.concept == object_concept:
            return slot_var.name
    return slot

max_stars_repo_name: Tubbz-alt/adam

max_issues_repo_name: Tubbz-alt/adam

max_forks_repo_name: Tubbz-alt/adam

reason: import networkx,from networkx


text:
import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')
import numpy as np

class Plot_Figure():
    def __init__(self):
        pass

    def plot(self,label, Ypredict, Ygroundtruth):
        '''
        :param str label: string label of data plot
        :param np.array Ypredict: numpy array of value predicted from model
        :param np.array Ygroundtruth: numpy array of value rainfall in dataset
        '''
        fig = plt.figure(figsize=(12, 6), dpi=80)
        ax = plt.subplot(111)
        plt.ylabel('Rain fall')
        plt.xlabel(label)
        ax.plot(Ygroundtruth, label="GroundTruth")
        ax.plot(Ypredict, label = "Predict")
        ax.legend()

max_stars_repo_name: Haiduongcable/Artificial-Intelligence

max_issues_repo_name: Haiduongcable/Artificial-Intelligence

max_forks_repo_name: Haiduongcable/Artificial-Intelligence


text:
import networkx
import re

class RawTaggerReader:
    def __init__(self, file_path, tagger=None):
        if tagger:
            self.tagger = tagger
        else:
            from konlpy.tag import Komoran
            self.tagger = Komoran(userdic='./text_rank/dic.txt')
        self.file_path = file_path
        self.rgxSplitter = re.compile('([.!?:](?:["\']|(?![0-9])))')

    def __iter__(self):
        for line in open(self.file_path, encoding='utf-8'):
            ch = self.rgxSplitter.split(line)
            for s in map(lambda a, b: a + b, ch[::2], ch[1::2]):
                if not s:
                    continue
                yield self.tagger.pos(s)

class TextRank:
    def __init__(self, **kwargs):
        self.graph = None
        self.window = kwargs.get('window', 5)
        self.coefficient = kwargs.get('coefficient=', 1.0)
        self.threshold = kwargs.get('threshold', 0.005)
        self.dictCount = {}
        self.dictBiCount = {}
        self.dictNear = {}
        self.nTotal = 0

    def load(self, sentence_iter, word_filter=None):
        def insert_pair(a, b):
            if a > b:
                a, b = b, a
            elif a == b:
                return
            self.dictBiCount[a, b] = self.dictBiCount.get((a, b), 0) + 1

        def insert_near_pair(a, b):
            self.dictNear[a, b] = self.dictNear.get((a, b), 0) + 1

        for sent in sentence_iter:
            for i, word in enumerate(sent):
                if word_filter and not word_filter(word):
                    continue
                self.dictCount[word] = self.dictCount.get(word, 0) + 1
                self.nTotal += 1
                if i - 1 >= 0 and (not word_filter or word_filter(sent[i - 1])):
                    insert_near_pair(sent[i - 1], word)
                if i + 1 < len(sent) and (not word_filter or word_filter(sent[i + 1])):
                    insert_near_pair(word, sent[i + 1])
                for j in range(i + 1, min(i + self.window + 1, len(sent))):
                    if word_filter and not word_filter(sent[j]):
                        continue
                    if sent[j] != word:
                        insert_pair(word, sent[j])

    def get_pmi(self, a, b):
        import math
        co = self.dictNear.get((a, b), 0)
        if not co:
            return None
        return math.log(float(co) * self.nTotal / self.dictCount[a] / self.dictCount[b])

    def get_i(self, a):
        import math
        if a not in self.dictCount:
            return None
        return math.log(self.nTotal / self.dictCount[a])

    def build(self):
        self.graph = networkx.Graph()
        self.graph.add_nodes_from(self.dictCount.keys())
        for (a, b), n in self.dictBiCount.items():
            self.graph.add_edge(a, b, weight=n * self.coefficient + (1 - self.coefficient))

    def rank(self):
        return networkx.pagerank(self.graph, weight='weight')

    def extract(self, ratio=0.1):
        ranks = self.rank()
        cand = sorted(ranks, key=ranks.get, reverse=True)[:int(len(ranks) * ratio)]
        print(cand)
        pairness = {}
        start_of = {}
        tuples = {}
        for k in cand:
            tuples[(k,)] = self.get_i(k) * ranks[k]
            for l in cand:
                if k == l:
                    continue
                pmi = self.get_pmi(k, l)
                if pmi:
                    pairness[k, l] = pmi

        for (k, l) in sorted(pairness, key=pairness.get, reverse=True):
            if k not in start_of:
                start_of[k] = (k, l)

        for (k, l), v in pairness.items():
            pmis = v
            rs = ranks[k] * ranks[l]
            path = (k, l)
            tuples[path] = pmis / (len(path) - 1) * rs ** (1 / len(path)) * len(path)
            last = l
            while last in start_of and len(path) < 7:
                if last in path:
                    break
                pmis += pairness[start_of[last]]
                last = start_of[last][1]
                rs *= ranks[last]
                path += (last,)
                tuples[path] = pmis / (len(path) - 1) * rs ** (1 / len(path)) * len(path)

        used = set()
        both = {}
        for k in sorted(tuples, key=tuples.get, reverse=True):
            if used.intersection(set(k)):
                continue
            both[k] = tuples[k]
            for w in k:
                used.add(w)

        return both

max_stars_repo_name: team-asdf/yaboja-crawler

max_issues_repo_name: team-asdf/yaboja-crawler

max_forks_repo_name: team-asdf/yaboja-crawler

reason: import networkx


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Aug 24 01:24:00 2020

@author: jiayingweng
"""

import numpy as np
import scipy.linalg as la
from sklearn.model_selection import train_test_split

__all__ = ['spcov', 'spcovCV']

# covxx = np.cov(X.T)
def spcov(covxx, lam, standard):
    # covxx = np.cov(X.T) * (n-1)/n
    # lam = 0.1
    # standard = True
    if standard:
        covdiag = np.diag(covxx)
        dhat = np.diag(np.sqrt(covdiag))
        dhatinv = np.diag(1/np.sqrt(covdiag))
        S = dhatinv @ covxx @ dhatinv
    else:
        S = covxx

    tmp = np.abs(S) - lam
    tmp = (tmp > 0) * tmp
    Ss = np.sign(S) * tmp
    covss = Ss - np.diag(np.diag(Ss)) + np.diag(np.diag(S))

    if standard:
        covss = dhat @ covss @ dhat

    return covss

def spcovCV(X, standard = False, lambseq = None, K = 10, ntr = None):
    n = X.shape[0]
    covxx = np.cov(X.T) * (n-1)/n
    if ntr is None:
        ntr = np.floor(n*(1-1/np.log(n)))/n
    if standard and (lambseq is None):
        lamseq = np.linspace(0, 0.95, 20)
    elif lambseq is None:
        abscov = np.abs(covxx)
        lammax = np.max(abscov - np.diag(np.diag(abscov)))
        lamseq = np.linspace(0, lammax, 20)

    cvloss = np.zeros((K, 20))
    for k in range(K):
        # print('Fold-', k)
        X_train, X_test = train_test_split(X, test_size=ntr, shuffle = True)
        n_train = X_train.shape[0]
        n_test = X_test.shape[0]
        ss_train = np.cov(X_train.T) * (n_train-1)/n_train
        ss_test = np.cov(X_test.T) * (n_test-1)/n_test
        for i in range(20):
            outcov = spcov(ss_train, lamseq[i], standard)
            cvloss[k, i] = la.norm(outcov-ss_test) ** 2

    l_mean = np.mean(cvloss, axis = 0)
    lambcv = lamseq[np.argmin(l_mean)]
    covss = spcov(covxx, lambcv, standard)
    return covss

max_stars_repo_name: SDR-VariableSelection/ADMMFTIRE

max_issues_repo_name: SDR-VariableSelection/ADMMFTIRE

max_forks_repo_name: SDR-VariableSelection/ADMMFTIRE

reason: import numpy,import scipy


text:
"""Compute dispersion correction using Greenwell & Beran's MP2D executable."""

import pprint
import re
import sys
from decimal import Decimal
from typing import Any, Dict, Optional, Tuple

import numpy as np
import qcelemental as qcel
from qcelemental.models import AtomicResult, Provenance
from qcelemental.util import safe_version, which

from ..exceptions import InputError, ResourceError, UnknownError
from ..util import execute
from . import empirical_dispersion_resources
from .model import ProgramHarness

pp = pprint.PrettyPrinter(width=120, compact=True, indent=1)

class MP2DHarness(ProgramHarness):

    _defaults = {
        "name": "MP2D",
        "scratch": True,
        "thread_safe": True,
        "thread_parallel": False,
        "node_parallel": False,
        "managed_memory": False,
    }
    version_cache: Dict[str, str] = {}

    class Config(ProgramHarness.Config):
        pass

    @staticmethod
    def found(raise_error: bool = False) -> bool:
        return which(
            "mp2d",
            return_bool=True,
            raise_error=raise_error,
            raise_msg="Please install via `conda install mp2d -c psi4`",
        )

    def get_version(self) -> str:
        self.found(raise_error=True)

        which_prog = which("mp2d")
        if which_prog not in self.version_cache:
            # Note: anything below v1.1 will return an input error message here. but that's fine as version compare evals to False.
            command = [which_prog, "--version"]
            import subprocess

            proc = subprocess.run(command, stdout=subprocess.PIPE)
            self.version_cache[which_prog] = safe_version(proc.stdout.decode("utf-8").strip())

        return self.version_cache[which_prog]

    def compute(self, input_model: "AtomicInput", config: "TaskConfig") -> "AtomicResult":
        from ..testing import is_program_new_enough

        self.found(raise_error=True)

        if not is_program_new_enough("mp2d", "1.1"):
            raise ResourceError(f"MP2D version '{self.get_version()}' too old. Please update to at least '1.1'.")

        job_inputs = self.build_input(input_model, config)

        success, dexe = self.execute(job_inputs)

        if success:
            dexe["outfiles"]["stdout"] = dexe["stdout"]
            dexe["outfiles"]["stderr"] = dexe["stderr"]
            output_model = self.parse_output(dexe["outfiles"], input_model)

        else:
            output_model = input_model
            output_model["error"] = {"error_type": "execution_error", "error_message": dexe["stderr"]}

        return output_model

    def execute(
        self, inputs: Dict[str, Any], *, extra_outfiles=None, extra_commands=None, scratch_name=None, timeout=None
    ) -> Tuple[bool, Dict]:

        success, dexe = execute(
            inputs["command"],
            inputs["infiles"],
            inputs["outfiles"],
            scratch_messy=False,
            scratch_directory=inputs["scratch_directory"],
        )
        return success, dexe

    def build_input(
        self, input_model: "AtomicInput", config: "TaskConfig", template: Optional[str] = None
    ) -> Dict[str, Any]:

        # strip engine hint
        mtd = input_model.model.method
        if mtd.startswith("mp2d-"):
            mtd = mtd[5:]

        if input_model.driver.derivative_int() > 1:
            raise InputError(f"Driver {input_model.driver} not implemented for MP2D.")

        # temp until actual options object
        input_model.extras["info"] = empirical_dispersion_resources.from_arrays(
            name_hint=mtd,
            level_hint=input_model.keywords.get("level_hint", None),
            param_tweaks=input_model.keywords.get("params_tweaks", None),
            dashcoeff_supplement=input_model.keywords.get("dashcoeff_supplement", None),
        )

        # Need 'real' field later and that's only guaranteed for molrec
        molrec = qcel.molparse.from_schema(input_model.molecule.dict())
        xyz = qcel.molparse.to_string(molrec, dtype="xyz", units="Angstrom", ghost_format="")
        infiles = {"mp2d_geometry": xyz}
        # jobrec['molecule']['real'] = molrec['real']

        # env = {
        #    'HOME': os.environ.get('HOME'),
        #    'PATH': os.environ.get('PATH'),
        #    #'PATH': os.pathsep.join([os.path.abspath(x) for x in os.environ.get('PSIPATH', '').split(os.pathsep) if x != '']) + \
        #    #        os.pathsep + os.environ.get('PATH'),
        #    #'LD_LIBRARY_PATH': os.environ.get('LD_LIBRARY_PATH'),
        # }

        command = ["mp2d", "mp2d_geometry"]
        command.extend(
            """--TT_a1={a1} --TT_a2={a2} --rcut={rcut} --w={w} --s8={s8}""".format(
                **input_model.extras["info"]["dashparams"]
            ).split()
        )
        if input_model.driver == "gradient":
            command.append("--gradient")

        return {
            "command": command,
            "infiles": infiles,
            "outfiles": ["mp2d_gradient"],
            "scratch_directory": config.scratch_directory,
            "input_result": input_model.copy(deep=True),
        }

    def parse_output(self, outfiles: Dict[str, str], input_model: "AtomicInput") -> "AtomicResult":
        stdout = outfiles.pop("stdout")

        for fl, contents in outfiles.items():
            if contents is not None:
                # LOG text += f'\n  MP2D scratch file {fl} has been read.\n'
                pass

        # parse energy output (could go further and break into UCHF, CKS)
        real = np.array(input_model.molecule.real)
        full_nat = real.shape[0]
        real_nat = np.sum(real)

        for ln in stdout.splitlines():
            if re.match("   MP2D dispersion correction Eh", ln):
                ene = Decimal(ln.split()[4])
            elif re.match("Atomic Coordinates in Angstroms", ln):
                break
        else:
            if not ((real_nat == 1) and (input_model.driver == "gradient")):
                raise UnknownError("Unknown issue occured.")

        # parse gradient output
        if outfiles["mp2d_gradient"] is not None:
            srealgrad = outfiles["mp2d_gradient"]
            realgrad = np.fromstring(srealgrad, count=3 * real_nat, sep=" ").reshape((-1, 3))

        if input_model.driver == "gradient":
            ireal = np.argwhere(real).reshape((-1))
            fullgrad = np.zeros((full_nat, 3))
            try:
                fullgrad[ireal, :] = realgrad
            except NameError as exc:
                raise UnknownError("Unsuccessful gradient collection.") from exc

        qcvkey = input_model.extras["info"]["fctldash"].upper()

        calcinfo = []
        calcinfo.append(qcel.Datum("CURRENT ENERGY", "Eh", ene))
        calcinfo.append(qcel.Datum("DISPERSION CORRECTION ENERGY", "Eh", ene))
        calcinfo.append(qcel.Datum("2-BODY DISPERSION CORRECTION ENERGY", "Eh", ene))
        if qcvkey:
            calcinfo.append(qcel.Datum(f"{qcvkey} DISPERSION CORRECTION ENERGY", "Eh", ene))

        if input_model.driver == "gradient":
            calcinfo.append(qcel.Datum("CURRENT GRADIENT", "Eh/a0", fullgrad))
            calcinfo.append(qcel.Datum("DISPERSION CORRECTION GRADIENT", "Eh/a0", fullgrad))
            calcinfo.append(qcel.Datum("2-BODY DISPERSION CORRECTION GRADIENT", "Eh/a0", fullgrad))
            if qcvkey:
                calcinfo.append(qcel.Datum(f"{qcvkey} DISPERSION CORRECTION GRADIENT", "Eh/a0", fullgrad))

        # LOGtext += qcel.datum.print_variables({info.label: info for info in calcinfo})
        calcinfo = {info.label: info.data for info in calcinfo}
        # calcinfo = qcel.util.unnp(calcinfo, flat=True)

        # got to even out who needs plump/flat/Decimal/float/ndarray/list
        # Decimal --> str preserves precision
        calcinfo = {
            k.upper(): str(v) if isinstance(v, Decimal) else v for k, v in qcel.util.unnp(calcinfo, flat=True).items()
        }

        # jobrec['properties'] = {"return_energy": ene}
        # jobrec["molecule"]["real"] = list(jobrec["molecule"]["real"])

        retres = calcinfo[f"CURRENT {input_model.driver.upper()}"]
        if isinstance(retres, Decimal):
            retres = float(retres)
        elif isinstance(retres, np.ndarray):
            retres = retres.ravel().tolist()

        output_data = {
            "extras": input_model.extras,
            "properties": {},
            "provenance": Provenance(
                creator="MP2D", version=self.get_version(), routine=__name__ + "." + sys._getframe().f_code.co_name
            ),
            "return_result": retres,
            "stdout": stdout,
        }
        output_data["extras"]["local_keywords"] = input_model.extras["info"]
        output_data["extras"]["qcvars"] = calcinfo

        output_data["success"] = True
        return AtomicResult(**{**input_model.dict(), **output_data})

max_stars_repo_name: SinaMostafanejad/QCEngine

max_issues_repo_name: SinaMostafanejad/QCEngine

max_forks_repo_name: SinaMostafanejad/QCEngine


text:
import logging

import numpy

from jyotisha.panchaanga.temporal import body
from jyotisha.panchaanga.temporal.body import Graha, Transit

logging.basicConfig(
  level=logging.DEBUG,
  format="%(levelname)s: %(asctime)s {%(filename)s:%(lineno)d}: %(message)s "
)

def test_graha_get_longitude():
  numpy.testing.assert_approx_equal(Graha.singleton(Graha.SUN).get_longitude(jd=2458434.083333251), 229.12286985575702)

def test_longitude_difference():
  numpy.testing.assert_approx_equal(body.longitude_difference(jd=2458484.545, body1=Graha.singleton(Graha.SUN), body2=Graha.singleton(Graha.MARS)), -79.66, significant=2)
  numpy.testing.assert_approx_equal(body.longitude_difference(jd=2458485.5453, body1=Graha.singleton(Graha.SUN), body2=Graha.singleton(Graha.MARS)), -79.31, significant=2)

def test_graha_get_next_raashi_transit():
  from jyotisha.panchaanga.temporal.zodiac import Ayanamsha
  from jyotisha.panchaanga.temporal.zodiac import AngaType
  transits = Graha.singleton(Graha.JUPITER).get_transits(jd_start=2457755, jd_end=2458120, anga_type=AngaType.RASHI, ayanaamsha_id=Ayanamsha.CHITRA_AT_180)
  from sanskrit_data import collection_helper
  collection_helper.assert_approx_equals(transits, [
           Transit(body=Graha.JUPITER, jd=2458008.4510242934, anga_type=AngaType.RASHI.name, value_1=6, value_2=7)], floating_point_precision=4)
  assert Graha.singleton(Graha.SUN).get_transits(jd_start=2458162.545722, jd_end=2458177.545722, anga_type=AngaType.RASHI, ayanaamsha_id=Ayanamsha.CHITRA_AT_180) == []

def test_get_star_longitude():
  numpy.testing.assert_approx_equal(body.get_star_longitude(star="Spica", jd=2458434.083333251), 204.09485939669307)

max_stars_repo_name: Prabhakaran-cbe/jyotisha

max_issues_repo_name: Prabhakaran-cbe/jyotisha

max_forks_repo_name: Prabhakaran-cbe/jyotisha


text:
import unittest
import warnings
from unittest import TestCase
from contextlib import contextmanager

import numpy as np

import llvmlite.binding as llvm

from numba import types
from numba.core.errors import NumbaInvalidConfigWarning
from numba.core.compiler import compile_isolated
from numba.core.codegen import _parse_refprune_flags
from numba.tests.support import override_config

@contextmanager
def set_refprune_flags(flags):
    with override_config('LLVM_REFPRUNE_FLAGS', flags):
        yield

class TestRefOpPruning(TestCase):

    _numba_parallel_test_ = False

    def check(self, func, *argtys, **prune_types):
        """
        Asserts the the func compiled with argument types "argtys" reports
        refop pruning statistics. The **prune_types** kwargs list each kind
        of pruning and whether the stat should be zero (False) or >0 (True).

        Note: The exact statistic varies across platform.
        """

        with override_config('LLVM_REFPRUNE_PASS', '1'):
            cres = compile_isolated(func, (*argtys,))

        pstats = cres.metadata.get('prune_stats', None)
        self.assertIsNotNone(pstats)

        for k, v in prune_types.items():
            stat = getattr(pstats, k, None)
            self.assertIsNotNone(stat)
            msg = f'failed checking {k}'
            if v:
                self.assertGreater(stat, 0, msg=msg)
            else:
                self.assertEqual(stat, 0, msg=msg)

    def test_basic_block_1(self):
        # some nominally involved control flow and ops, there's only basic_block
        # opportunities present here.
        def func(n):
            a = np.zeros(n)
            acc = 0
            if n > 4:
                b = a[1:]
                acc += b[1]
            else:
                c = a[:-1]
                acc += c[0]
            return acc

        self.check(func, (types.intp), basicblock=True)

    def test_diamond_1(self):
        # most basic?! diamond
        def func(n):
            a = np.ones(n)
            x = 0
            if n > 2:
                x = a.sum()
            return x + 1

        # disable fanout pruning
        with set_refprune_flags('per_bb,diamond'):
            self.check(func, (types.intp), basicblock=True, diamond=True,
                       fanout=False, fanout_raise=False)

    def test_diamond_2(self):
        # more complex diamonds
        def func(n):
            con = []
            for i in range(n):
                con.append(np.arange(i))
            c = 0.0
            for arr in con:
                c += arr.sum() / (1 + arr.size)
            return c

        # disable fanout pruning
        with set_refprune_flags('per_bb,diamond'):
            self.check(func, (types.intp), basicblock=True, diamond=True,
                       fanout=False, fanout_raise=False)

    def test_fanout_1(self):
        # most basic?! fan-out
        def func(n):
            a = np.zeros(n)
            b = np.zeros(n)
            x = (a, b)
            acc = 0.
            for i in x:
                acc += i[0]
            return acc

        self.check(func, (types.intp), basicblock=True, fanout=True)

    def test_fanout_2(self):
        # fanout with raise
        def func(n):
            a = np.zeros(n)
            b = np.zeros(n)
            x = (a, b)
            for i in x:
                if n:
                    raise ValueError
            return x

        with set_refprune_flags('per_bb,fanout_raise'):
            self.check(func, (types.intp), basicblock=True, diamond=False,
                       fanout=False, fanout_raise=True)

class TestRefPruneFlags(TestCase):
    def setUp(self):
        warnings.simplefilter('error', NumbaInvalidConfigWarning)

    def tearDown(self):
        warnings.resetwarnings()

    def test_warn_invalid_flags(self):
        with set_refprune_flags('abc,per_bb,cde'):
            with self.assertWarns(NumbaInvalidConfigWarning) as cm:
                optval = _parse_refprune_flags()
            self.assertEqual(len(cm.warnings), 2)
            self.assertIn('abc', str(cm.warnings[0].message))
            self.assertIn('cde', str(cm.warnings[1].message))
            self.assertEqual(optval, llvm.RefPruneSubpasses.PER_BB)

    def test_valid_flag(self):
        with set_refprune_flags('per_bb, diamond, fanout,fanout_raise'):
            optval = _parse_refprune_flags()
            self.assertEqual(optval, llvm.RefPruneSubpasses.ALL)

    def test_the_all_flag(self):
        with set_refprune_flags('all'):
            optval = _parse_refprune_flags()
            self.assertEqual(optval, llvm.RefPruneSubpasses.ALL)

    def test_some_flags(self):
        with set_refprune_flags('per_bb, fanout'):
            optval = _parse_refprune_flags()
            enumcls = llvm.RefPruneSubpasses
            self.assertEqual(optval, enumcls.PER_BB | enumcls.FANOUT)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: auderson/numba

max_issues_repo_name: auderson/numba

max_forks_repo_name: auderson/numba

reason: import numpy,from numba


text:
import os
import json
import pickle
import sys
import traceback

from sagemaker.tensorflow import TensorFlow

prefix = "/opt/ml/"

input_path = prefix + "input/data"
hyperparams_path = prefix + "input/config/hyperparameters.json"
output_path = os.path.join(prefix, "output")
model_path = os.path.join(prefix, "model")

hvd_instance_type = "ml.m5.4xlarge"
hvd_processes_per_host = 1
hvd_instance_count = 4

distributions = {
    "mpi": {
        "enabled": True,
        "processes_per_host": hvd_processes_per_host,
        "custom_mpi_options": "-verbose --NCCL_DEBUG=INFO -x OMPI_MCA_btl_vader_single_copy_mechanism=none",
    }
}

remote_inputs = {
    "train": input_path + "/train",
    "validation": input_path + "/validate",
    "eval": input_path + "/test",
}

# The function to execute the training.
def train():
    print("Starting the training.")
    try:

        # Read the hyperparameter config json file
        with open(hyperparams_path) as _in_file:
            hyperparams_dict = json.load(_in_file)

        # Usually max_delta_step is not needed, but it might help in logistic regression when class is extremely imbalanced.
        params = {
            # hyperparameters
            "learning_rate": float(hyperparams_dict["learning_rate"]),
            "epochs": int(hyperparams_dict["epochs"]),
            "batch-size": float(hyperparams_dict["batch-size"]),
            "activation": float(hyperparams_dict["activation"]),
            "loss_fn": float(hyperparams_dict["loss_fn"]),
        }

        estimator_hvd = TensorFlow(
            source_dir="source",
            entry_point="train_horovod.py",
            model_dir=model_path,
            base_job_name="sagemaker-horovod",
            hyperparameters=params,
            framework_version="1.14",
            py_version="py3",
            train_instance_count=hvd_instance_count,
            train_instance_type=hvd_instance_type,
            distributions=distributions,
        )

        # train model on whole dataset
        estimator_hvd.fit(remote_inputs)

        # save the model
        with open(os.path.join(model_path, "resnet18-model.pkl"), "wb") as out:
            pickle.dump(estimator_hvd, out, protocol=0)
            print("Training complete.")

    except Exception as e:
        # Write out an error file. This will be returned as the failureReason in the
        # DescribeTrainingJob result.
        trc = traceback.format_exc()
        with open(os.path.join(output_path, "failure"), "w") as s:
            s.write("Exception during training: " + str(e) + "\n" + trc)
        # Printing this causes the exception to be in the training job logs, as well.
        print("Exception during training: " + str(e) + "\n" + trc, file=sys.stderr)
        # A non-zero exit code causes the training job to be marked as Failed.
        sys.exit(255)

if __name__ == "__main__":
    train()

    # A zero exit code causes the job to be marked a Succeeded.
    sys.exit(0)

max_stars_repo_name: muchemwal/dataops-infra

max_issues_repo_name: muchemwal/dataops-infra

max_forks_repo_name: muchemwal/dataops-infra

reason: from sage