text:
"""
Test the CONMIN optimizer component
"""

import unittest
import numpy

# pylint: disable=F0401,E0611
from openmdao.main.api import Assembly, Component, VariableTree, set_as_top, Driver
from openmdao.main.datatypes.api import Float, Array, Str, VarTree
from openmdao.lib.casehandlers.api import ListCaseRecorder
from openmdao.main.interfaces import IHasParameters, implements
from openmdao.main.hasparameters import HasParameters
from openmdao.util.decorators import add_delegate
from openmdao.lib.drivers.conmindriver import CONMINdriver
from openmdao.util.testutil import assert_rel_error

class OptRosenSuzukiComponent(Component):
    """ From the CONMIN User's Manual:
    EXAMPLE 1 - CONSTRAINED ROSEN-SUZUKI FUNCTION. NO GRADIENT INFORMATION.

         MINIMIZE OBJ = X(1)**2 - 5*X(1) + X(2)**2 - 5*X(2) +
                        2*X(3)**2 - 21*X(3) + X(4)**2 + 7*X(4) + 50

         Subject to:

              G(1) = X(1)**2 + X(1) + X(2)**2 - X(2) +
                     X(3)**2 + X(3) + X(4)**2 - X(4) - 8   .LE.0

              G(2) = X(1)**2 - X(1) + 2*X(2)**2 + X(3)**2 +
                     2*X(4)**2 - X(4) - 10                  .LE.0

              G(3) = 2*X(1)**2 + 2*X(1) + X(2)**2 - X(2) +
                     X(3)**2 - X(4) - 5                     .LE.0

    This problem is solved beginning with an initial X-vector of
         X = (1.0, 1.0, 1.0, 1.0)
    The optimum design is known to be
         OBJ = 6.000
    and the corresponding X-vector is
         X = (0.0, 1.0, 2.0, -1.0)
    """

    x = Array(iotype='in', low=-10, high=99)
    g = Array([1., 1., 1.], iotype='out')
    result = Float(iotype='out')
    obj_string = Str(iotype='out')
    opt_objective = Float(iotype='out')

    # pylint: disable=C0103
    def __init__(self):
        super(OptRosenSuzukiComponent, self).__init__()
        self.x = numpy.array([1., 1., 1., 1.], dtype=float)
        self.result = 0.

        self.opt_objective = 6.*10.0
        self.opt_design_vars = [0., 1., 2., -1.]

    def execute(self):
        """calculate the new objective value"""
        x = self.x

        self.result = (x[0]**2 - 5.*x[0] + x[1]**2 - 5.*x[1] +
                       2.*x[2]**2 - 21.*x[2] + x[3]**2 + 7.*x[3] + 50)

        self.obj_string = "Bad"
        #print "rosen", self.x

        self.g[0] = (x[0]**2 + x[0] + x[1]**2 - x[1] +
                     x[2]**2 + x[2] + x[3]**2 - x[3] - 8)
        self.g[1] = (x[0]**2 - x[0] + 2*x[1]**2 + x[2]**2 +
                     2*x[3]**2 - x[3] - 10)
        self.g[2] = (2*x[0]**2 + 2*x[0] + x[1]**2 - x[1] +
                     x[2]**2 - x[3] - 5)
        #print self.x, self.g

class RosenSuzuki2D(Component):
    """ RosenSuzuki with 2D input. """

    x = Array(iotype='in', low=-10, high=99)
    result = Float(iotype='out')
    opt_objective = Float(iotype='out')

    # pylint: disable=C0103
    def __init__(self):
        super(RosenSuzuki2D, self).__init__()
        self.x = numpy.array([[1., 1.], [1., 1.]], dtype=float)
        self.result = 0.

        self.opt_objective = 6.*10.0
        self.opt_design_vars = [0., 1., 2., -1.]

    def execute(self):
        """calculate the new objective value"""
        self.result = (self.x[0][0]**2 - 5.*self.x[0][0] +
                       self.x[0][1]**2 - 5.*self.x[0][1] +
                       2.*self.x[1][0]**2 - 21.*self.x[1][0] +
                       self.x[1][1]**2 + 7.*self.x[1][1] + 50)

class RosenSuzukiMixed(Component):
    """ RosenSuzuki with mixed scalar and 1D inputs. """

    x0 = Float(iotype='in', low=-10, high=99)
    x12 = Array(iotype='in', low=-10, high=99)
    x3 = Float(iotype='in', low=-10, high=99)
    result = Float(iotype='out')
    opt_objective = Float(iotype='out')

    # pylint: disable=C0103
    def __init__(self):
        super(RosenSuzukiMixed, self).__init__()
        self.x0 = 1.
        self.x12 = numpy.array([1., 1.], dtype=float)
        self.x3 = 1.
        self.result = 0.

        self.opt_objective = 6.*10.0
        self.opt_design_vars = [0., 1., 2., -1.]

    def execute(self):
        """calculate the new objective value"""
        self.result = (self.x0**2 - 5.*self.x0 +
                       self.x12[0]**2 - 5.*self.x12[0] +
                       2.*self.x12[1]**2 - 21.*self.x12[1] +
                       self.x3**2 + 7.*self.x3 + 50)

class CONMINdriverTestCase(unittest.TestCase):
    """test CONMIN optimizer component"""

    def setUp(self):
        self.top = set_as_top(Assembly())
        self.top.add('driver', CONMINdriver())
        self.top.add('comp', OptRosenSuzukiComponent())
        self.top.driver.workflow.add('comp')
        self.top.driver.iprint = 0
        self.top.driver.itmax = 30

    def test_opt1(self):
        # Run with scalar parameters, scalar constraints, and OpenMDAO gradient.
        self.top.driver.add_objective('10*comp.result')
        # pylint: disable=C0301
        map(self.top.driver.add_parameter,
            ['comp.x[0]', 'comp.x[1]', 'comp.x[2]', 'comp.x[3]'])

        map(self.top.driver.add_constraint, [
            'comp.x[0]**2+comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2+comp.x[2]+comp.x[3]**2-comp.x[3] < 8',
            'comp.x[0]**2-comp.x[0]+2*comp.x[1]**2+comp.x[2]**2+2*comp.x[3]**2-comp.x[3] < 10',
            '2*comp.x[0]**2+2*comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2-comp.x[3] < 5'])
        self.top.recorders = [ListCaseRecorder()]
        self.top.driver.iprint = 0
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0], 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[2], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[3], 0.05)

        cases = self.top.recorders[0].get_iterator()
        end_case = cases[-1]

        self.assertEqual(self.top.comp.x[1],
                         end_case.get_input('comp.x[1]'))
        self.assertEqual(10*self.top.comp.result,
                         end_case.get_output('_pseudo_0.out0'))

    def test_opt1_a(self):
        # Run with scalar parameters, 1D constraint, and OpenMDAO gradient.
        self.top.driver.add_objective('10*comp.result')
        # pylint: disable=C0301
        map(self.top.driver.add_parameter,
            ['comp.x[0]', 'comp.x[1]', 'comp.x[2]', 'comp.x[3]'])

        self.top.driver.add_constraint('comp.g <= 0')
        self.top.driver.iprint = 0
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[2], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[3], 0.05)

    def test_opt1_with_CONMIN_gradient(self):
        # Note: all other tests use OpenMDAO gradient
        self.top.driver.add_objective('10*comp.result')
        self.top.driver.add_parameter('comp.x[0]', fd_step=.00001)
        self.top.driver.add_parameter('comp.x[1]', fd_step=.00001)
        self.top.driver.add_parameter('comp.x[2]', fd_step=.00001)
        self.top.driver.add_parameter('comp.x[3]', fd_step=.00001)

        # pylint: disable=C0301
        map(self.top.driver.add_constraint, [
            'comp.x[0]**2+comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2+comp.x[2]+comp.x[3]**2-comp.x[3] < 8',
            'comp.x[0]**2-comp.x[0]+2*comp.x[1]**2+comp.x[2]**2+2*comp.x[3]**2-comp.x[3] < 10',
            '2*comp.x[0]**2+2*comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2-comp.x[3] < 5'])

        self.top.driver.conmin_diff = True
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[2], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[3], 0.05)

    def test_opt1_with_CONMIN_gradient_a(self):
        # Scalar parameters, array constraint, CONMIN gradient.
        # Note: all other tests use OpenMDAO gradient
        self.top.driver.add_objective('10*comp.result')
        self.top.driver.add_parameter('comp.x[0]', fd_step=.00001)
        self.top.driver.add_parameter('comp.x[1]', fd_step=.00001)
        self.top.driver.add_parameter('comp.x[2]', fd_step=.00001)
        self.top.driver.add_parameter('comp.x[3]', fd_step=.00001)

        self.top.driver.add_constraint('comp.g <= 0')

        self.top.driver.conmin_diff = True
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0], 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[2], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[3], 0.05)

    def test_opt1_flippedconstraints(self):
        self.top.driver.add_objective('10*comp.result')
        map(self.top.driver.add_parameter,
            ['comp.x[0]', 'comp.x[1]', 'comp.x[2]', 'comp.x[3]'])

        # pylint: disable=C0301
        map(self.top.driver.add_constraint, [
            '8 > comp.x[0]**2+comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2+comp.x[2]+comp.x[3]**2-comp.x[3]',
            '10 > comp.x[0]**2-comp.x[0]+2*comp.x[1]**2+comp.x[2]**2+2*comp.x[3]**2-comp.x[3]',
            '5 > 2*comp.x[0]**2+2*comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2-comp.x[3]'])
        self.top.run()
        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0], 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[2], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[3], 0.05)

    def test_gradient_step_size_large(self):
        # Test that a larger value of fd step-size is less acurate

        self.top.driver.add_objective('10*comp.result')
        map(self.top.driver.add_parameter, ['comp.x[0]', 'comp.x[1]',
                                            'comp.x[2]', 'comp.x[3]'])

        # pylint: disable=C0301
        map(self.top.driver.add_constraint, [
            'comp.x[0]**2+comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2+comp.x[2]+comp.x[3]**2-comp.x[3] < 8.',
            'comp.x[0]**2-comp.x[0]+2*comp.x[1]**2+comp.x[2]**2+2*comp.x[3]**2-comp.x[3] < 10.',
            '2*comp.x[0]**2+2*comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2-comp.x[3] < 5.'])

        self.top.driver.conmin_diff = True
        self.top.driver.fdch = 1.0e-6
        self.top.driver.fdchm = 1.0e-6
        self.top.run()
        baseerror = abs(self.top.comp.opt_objective - self.top.driver.eval_objective())

        self.top.driver.fdch = .3
        self.top.driver.fdchm = .3
        self.top.comp.x = numpy.array([1., 1., 1., 1.], dtype=float)
        self.top.run()
        newerror = abs(self.top.comp.opt_objective - self.top.driver.eval_objective())

        # pylint: disable=E1101
        if baseerror > newerror:
            self.fail("Coarsening CONMIN gradient step size did not make the objective worse.")

    def test_linear_constraint_specification(self):
        # Note, just testing problem specification and setup

        self.top.driver.add_objective('comp.result')
        map(self.top.driver.add_parameter,
            ['comp.x[0]', 'comp.x[1]', 'comp.x[2]', 'comp.x[3]'])

        self.top.driver.add_constraint('comp.x[1] + 3.0*comp.x[2] > 3.0', linear=True)
        self.top.driver.add_constraint('comp.x[2] + comp.x[3] > 13.0', linear=True)
        self.top.driver.add_constraint('comp.x[1] - 0.73*comp.x[3]*comp.x[2] > -12.0', linear=False)
        self.top.driver.itmax = 1

        self.top.run()
        self.assertEqual(self.top.driver._cons_is_linear[0], 1, 1e-6)
        self.assertEqual(self.top.driver._cons_is_linear[1], 1, 1e-6)
        self.assertEqual(self.top.driver._cons_is_linear[2], 0, 1e-6)

        lcons = self.top.driver.get_constraints(linear=True)
        self.assertTrue(len(lcons) == 2)
        self.assertTrue('comp.x[2]+comp.x[3]>13.0' in lcons)
        self.assertTrue('comp.x[1]-0.73*comp.x[3]*comp.x[2]>-12.0' not in lcons)

        lcons = self.top.driver.get_constraints(linear=False)
        self.assertTrue(len(lcons) == 1)
        self.assertTrue('comp.x[2]+comp.x[3]>13.0' not in lcons)
        self.assertTrue('comp.x[1]-0.73*comp.x[3]*comp.x[2]>-12.0' in lcons)

    def test_max_iteration(self):

        self.top.driver.add_objective('comp.result')
        map(self.top.driver.add_parameter, ['comp.x[0]', 'comp.x[1]',
                                            'comp.x[2]', 'comp.x[3]'])
        self.top.driver.nscal = -1

        self.top.driver.itmax = 2

        # pylint: disable=C0301
        self.top.run()

        # pylint: disable=E1101
        self.assertEqual(self.top.driver.iter_count, 2)

    def test_remove(self):
        self.top.driver.add_objective('comp.result')
        map(self.top.driver.add_parameter,
            ['comp.x[0]', 'comp.x[1]', 'comp.x[2]', 'comp.x[3]'])

        # pylint: disable=C0301
        map(self.top.driver.add_constraint, [
            'comp.x[0]**2+comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2+comp.x[2]+comp.x[3]**2-comp.x[3] < 8',
            'comp.x[0]**2-comp.x[0]+2*comp.x[1]**2+comp.x[2]**2+2*comp.x[3]**2-comp.x[3] < 10',
            '2*comp.x[0]**2+2*comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2-comp.x[3] < 5'])

        self.top.remove('comp')
        self.assertEqual(self.top.driver.list_param_targets(), [])
        self.assertEqual(self.top.driver.list_constraints(), [])
        self.assertEqual(self.top.driver.get_objectives(), {})

    def test_initial_run(self):
        # Test the fix that put run_iteration at the top
        #   of the start_iteration method
        class MyComp(Component):

            x = Float(0.0, iotype='in', low=-10, high=10)
            xx = Float(0.0, iotype='in', low=-10, high=10)
            f_x = Float(iotype='out')
            y = Float(iotype='out')

            def execute(self):
                if self.xx != 1.0:
                    self.raise_exception("Lazy", RuntimeError)
                self.f_x = 2.0*self.x
                self.y = self.x

        @add_delegate(HasParameters)
        class SpecialDriver(Driver):

            implements(IHasParameters)

            def execute(self):
                self.set_parameters([1.0])

        top = set_as_top(Assembly())
        top.add('comp', MyComp())
        top.add('driver', CONMINdriver())
        top.add('subdriver', SpecialDriver())
        top.driver.workflow.add('subdriver')
        top.subdriver.workflow.add('comp')

        top.subdriver.add_parameter('comp.xx')
        top.driver.add_parameter('comp.x')
        top.driver.add_constraint('comp.y > 1.0')
        top.driver.add_objective('comp.f_x')

        top.run()

class TestContainer(VariableTree):

    dummy1 = Float(desc='default value of 0.0') #this value is being grabbed by the optimizer
    dummy2 = Float(11.0)

class TestComponent(Component):

    dummy_data = VarTree(TestContainer(), iotype='in')
    x = Float(iotype='out')

    def execute(self):
        self.x = (self.dummy_data.dummy1-3)**2 - self.dummy_data.dummy2

class TestAssembly(Assembly):

    def configure(self):
        self.add('dummy_top', TestContainer())
        self.add('comp', TestComponent())
        self.add('driver', CONMINdriver())

        self.driver.workflow.add(['comp'])
        #self.driver.iprint = 4 #debug verbosity
        self.driver.add_objective('comp.x')
        self.driver.add_parameter('comp.dummy_data.dummy1', low=-10.0, high=10.0)

class CONMINdriverTestCase2(unittest.TestCase):

    def test_vartree_opt(self):
        blah = set_as_top(TestAssembly())
        blah.run()
        self.assertAlmostEqual(blah.comp.dummy_data.dummy1, 3.0, 1) #3.0 should be minimum

class TestCase1D(unittest.TestCase):
    """Test using 1D array connections and 1D array constraint."""

    def setUp(self):
        self.top = set_as_top(Assembly())
        self.top.add('comp', OptRosenSuzukiComponent())
        driver = self.top.add('driver', CONMINdriver())
        driver.workflow.add('comp')
        driver.iprint = 0
        driver.itmax = 30

        driver.add_objective('10*comp.result')
        driver.add_parameter('comp.x')

    def test_conmin_gradient_a(self):
        # Run with 1D parameter, 1D constraint, and CONMIN gradient.

        self.top.driver.add_constraint('comp.g <= 0')
        self.top.driver.conmin_diff = True
        self.top.driver.fdch = .000001
        self.top.driver.fdchm = .000001
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0], 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[2], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[3], 0.05)

    def test_conmin_gradient_s(self):
        # Run with 1D parameter, scalar constraints, and CONMIN gradient.
        # pylint: disable=C0301
        map(self.top.driver.add_constraint, [
            'comp.x[0]**2+comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2+comp.x[2]+comp.x[3]**2-comp.x[3] < 8',
            'comp.x[0]**2-comp.x[0]+2*comp.x[1]**2+comp.x[2]**2+2*comp.x[3]**2-comp.x[3] < 10',
            '2*comp.x[0]**2+2*comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2-comp.x[3] < 5'])

        self.top.driver.conmin_diff = True
        self.top.driver.fdch = .000001
        self.top.driver.fdchm = .000001
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.driver.eval_objective(),
                         self.top.comp.opt_objective,
                         0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0], 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[2], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[3], 0.05)

    def test_openmdao_gradient_a(self):
        # Run with 1D parameter, 1D constraint, and OpenMDAO gradient.
        self.top.driver.add_constraint('comp.g <= 0')
        self.top.driver.conmin_diff = False
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0], 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[2], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[3], 0.05)

    def test_openmdao_gradient_s(self):
        # Run with 1D parameter, scalar constraints, and OpenMDAO gradient.
        # pylint: disable=C0301
        map(self.top.driver.add_constraint, [
            'comp.x[0]**2+comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2+comp.x[2]+comp.x[3]**2-comp.x[3] < 8',
            'comp.x[0]**2-comp.x[0]+2*comp.x[1]**2+comp.x[2]**2+2*comp.x[3]**2-comp.x[3] < 10',
            '2*comp.x[0]**2+2*comp.x[0]+comp.x[1]**2-comp.x[1]+comp.x[2]**2-comp.x[3] < 5'])

        self.top.driver.conmin_diff = False
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0], 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[2], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[3], 0.05)

class TestCase2D(unittest.TestCase):
    """Test using 2D array connections."""

    def setUp(self):
        self.top = set_as_top(Assembly())
        self.top.add('comp', RosenSuzuki2D())
        driver = self.top.add('driver', CONMINdriver())
        driver.workflow.add('comp')
        driver.iprint = 0
        driver.itmax = 30

        driver.add_objective('10*comp.result')
        driver.add_parameter('comp.x')

        # pylint: disable=C0301
        map(driver.add_constraint, [
            'comp.x[0][0]**2+comp.x[0][0]+comp.x[0][1]**2-comp.x[0][1]+comp.x[1][0]**2+comp.x[1][0]+comp.x[1][1]**2-comp.x[1][1] < 8',
            'comp.x[0][0]**2-comp.x[0][0]+2*comp.x[0][1]**2+comp.x[1][0]**2+2*comp.x[1][1]**2-comp.x[1][1] < 10',
            '2*comp.x[0][0]**2+2*comp.x[0][0]+comp.x[0][1]**2-comp.x[0][1]+comp.x[1][0]**2-comp.x[1][1] < 5'])

    def test_conmin_gradient(self):
        # Run with 2D parameter and CONMIN gradient.
        self.top.driver.conmin_diff = True
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0][0], 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[0][1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[1][0], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[1][1], 0.05)

    def test_openmdao_gradient(self):
        # Run with 2D parameter and OpenMDAO gradient.
        self.top.driver.conmin_diff = False
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x[0][0], 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x[0][1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x[1][0], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x[1][1], 0.05)

class TestCaseMixed(unittest.TestCase):
    """Test using mixed scalar and 1D connections."""

    def setUp(self):
        self.top = set_as_top(Assembly())
        self.top.add('comp', RosenSuzukiMixed())
        driver = self.top.add('driver', CONMINdriver())
        driver.workflow.add('comp')
        driver.iprint = 0
        driver.itmax = 30

        driver.add_objective('10*comp.result')
        map(driver.add_parameter, ['comp.x0', 'comp.x12', 'comp.x3'])

        # pylint: disable=C0301
        map(driver.add_constraint, [
            'comp.x0**2+comp.x0+comp.x12[0]**2-comp.x12[0]+comp.x12[1]**2+comp.x12[1]+comp.x3**2-comp.x3 < 8',
            'comp.x0**2-comp.x0+2*comp.x12[0]**2+comp.x12[1]**2+2*comp.x3**2-comp.x3 < 10',
            '2*comp.x0**2+2*comp.x0+comp.x12[0]**2-comp.x12[0]+comp.x12[1]**2-comp.x3 < 5'])

    def test_conmin_gradient(self):
        # Run with mixed parameters and CONMIN gradient.
        self.top.driver.conmin_diff = True
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x0, 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x12[0], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x12[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x3, 0.05)

    def test_openmdao_gradient(self):
        # Run with mixed parameters and OpenMDAO gradient.
        self.top.driver.conmin_diff = False
        self.top.run()

        # pylint: disable=E1101
        assert_rel_error(self, self.top.comp.opt_objective,
                         self.top.driver.eval_objective(), 0.01)
        assert_rel_error(self, 1 + self.top.comp.opt_design_vars[0],
                         1 + self.top.comp.x0, 0.05)
        assert_rel_error(self, self.top.comp.opt_design_vars[1],
                         self.top.comp.x12[0], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[2],
                         self.top.comp.x12[1], 0.06)
        assert_rel_error(self, self.top.comp.opt_design_vars[3],
                         self.top.comp.x3, 0.05)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: mjfwest/OpenMDAO-Framework

max_issues_repo_name: jcchin/OpenMDAO-Framework

max_forks_repo_name: jcchin/OpenMDAO-Framework


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
import copy
import numpy as np
from collections import OrderedDict
from encoder import *
from layer import *

class BMGFModel(nn.Module):
    def __init__(self, **kw):
        super(BMGFModel, self).__init__()
        max_arg = kw.get("max_arg", 512)
        encoder = kw.get("encoder", "roberta")
        hidden_dim = kw.get("hidden_dim", 128)
        num_perspectives = kw.get("num_perspectives", 16)
        dropout = kw.get("dropout", 0.2)
        activation = kw.get("activation", "relu")
        num_rels = kw.get("num_rels", 4)
        num_filters = kw.get("num_filters", 64)
        act_layer = map_activation_str_to_layer(activation)

        self.drop = Dropout(dropout)
        if encoder == "lstm":
            self.encoder = LSTMEncoder(**kw)
        elif encoder == "bert":
            self.encoder = BERTEncoder(num_segments=2, max_len=max_arg*2, **kw)
        elif encoder == "roberta":
            self.encoder = ROBERTAEncoder(num_segments=2, max_len=max_arg*2, **kw)
        else:
            raise NotImplementedError("Error: encoder=%s is not supported now." % (encoder))

        self.bimpm = BiMpmMatching(
            hidden_dim=self.encoder.get_output_dim(),
            num_perspectives=num_perspectives)
        output_dim = self.encoder.get_output_dim() + self.bimpm.get_output_dim()

        self.gated_attn_layer = GatedMultiHeadAttn(
            query_dim=output_dim,
            key_dim=output_dim,
            value_dim=output_dim,
            hidden_dim=hidden_dim,
            num_head=num_perspectives,
            dropatt=dropout,
            act_func="softmax",
            add_zero_attn=False,
            pre_lnorm=False,
            post_lnorm=False)

        self.conv_layer = CnnHighway(
            input_dim=self.gated_attn_layer.get_output_dim(),
            output_dim=hidden_dim,
            filters=[(1, num_filters), (2, num_filters)], # the shortest length is 2
            num_highway=1,
            activation=activation,
            layer_norm=False)

        self.fc_layer = FullyConnectedLayer(2 * hidden_dim, hidden_dim, num_rels)

    def set_finetune(self, finetune):
        for param in self.parameters():
            param.requires_grad = True
        self.encoder.set_finetune(finetune)

    def forward(self, arg1, arg2, arg1_mask=None, arg2_mask=None, encode_pair=True):
        if encode_pair:
            arg1_feats, arg2_feats, arg1_mask, arg2_mask = self.encoder.forward_pair(arg1, arg2, arg1_mask, arg2_mask)
        else:
            arg1_feats, arg1_mask = self.encoder.forward_single(arg1, arg1_mask)
            arg2_feats, arg2_mask = self.encoder.forward_single(arg2, arg2_mask)
        arg1_feats, arg2_feats = self.drop(arg1_feats), self.drop(arg2_feats)

        arg1_matched_feats, arg2_matched_feats = self.bimpm(
            arg1_feats, arg1_mask, arg2_feats, arg2_mask)
        arg1_matched_feats = torch.cat(arg1_matched_feats, dim=2)
        arg2_matched_feats = torch.cat(arg2_matched_feats, dim=2)
        arg1_matched_feats, arg2_matched_feats = self.drop(arg1_matched_feats), self.drop(arg2_matched_feats)

        arg1_self_attned_feats = torch.cat([arg1_feats, arg1_matched_feats], dim=2)
        arg2_self_attned_feats = torch.cat([arg2_feats, arg2_matched_feats], dim=2)
        arg1_self_attned_feats = self.gated_attn_layer(
            arg1_self_attned_feats, arg1_self_attned_feats, arg1_self_attned_feats, attn_mask=arg1_mask)
        arg2_self_attned_feats = self.gated_attn_layer(
            arg2_self_attned_feats, arg2_self_attned_feats, arg2_self_attned_feats, attn_mask=arg2_mask)
        arg1_self_attned_feats, arg2_self_attned_feats = self.drop(arg1_self_attned_feats), self.drop(arg2_self_attned_feats)

        arg1_conv = self.conv_layer(arg1_self_attned_feats, arg1_mask)
        arg2_conv = self.conv_layer(arg2_self_attned_feats, arg2_mask)
        arg1_conv, arg2_conv = self.drop(arg1_conv), self.drop(arg2_conv)

        output = self.fc_layer(torch.cat([arg1_conv, arg2_conv], dim=1))

        return output # unnormalized results

ext: py

max_stars_repo_name: HKUST-KnowComp/BMGF-RoBERTa

max_issues_repo_name: HKUST-KnowComp/BMGF-RoBERTa

max_forks_repo_name: HKUST-KnowComp/BMGF-RoBERTa

reason: import numpy


text:
# -*- coding: utf-8 -*-

# Autor: Luise Köhler
# Datum: Fri Sep 24 14:41:58 2021
# Python 3.8.8
# Ubuntu 20.04.1
"""
Replication project for Wojatzki and Zesch (2016). Run from command line to
process and classify data.
First, the data will be split into a training and a test set (10%), then
using 10-fold cros-validation, a SVM-classifier will be trained for
classification of explicit stance and another SVM to classify debate stance.
Using a decision tree classifier the debate stance labels are also classified
based on the explicit stance labels. Outputs are micro F1 scores of the
classifiers.
"""

import argparse
import logging
import os
from typing import List, Optional, Union

import numpy as np
from sklearn.metrics import f1_score
from sklearn.model_selection import cross_validate, train_test_split
from sklearn.multioutput import MultiOutputClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier

from classify import (
    compute_micro_f1,
    get_best_classifier,
    majority_class_baseline,
    transform_categorical_data,
)
from feature_extractor import FeatureExtractor
from preprocessing import prepare_data

logging.basicConfig(
    filename="stance.log",
    level=logging.DEBUG,
    format="%(asctime)s - %(levelname)s - %(name)s: %(message)s",
)
logger = logging.getLogger(__name__)

def main(arguments: Optional[List[str]] = None) -> None:
    args = parse_arguments(arguments)
    data = prepare_data(args.data_path)
    label_columns = [
        column
        for column in data.columns
        if column not in {"id", "text", "id  text", "[debate stance:polarity]"}
    ]
    logger.info(f"labels: {label_columns}")
    train_set, test_set = train_test_split(data, test_size=0.1, random_state=0)

    extractor = FeatureExtractor()
    extractor.collect_features(list(train_set["text"]))

    X_train = train_set["text"].apply(extractor.get_features_for_instance)
    X_test = test_set["text"].apply(extractor.get_features_for_instance)

    # explicit stance label sets
    y_train = np.array(train_set[label_columns])
    y_test = np.array(test_set[label_columns])

    # debate stance label sets
    y_debate_train = train_set["[debate stance:polarity]"]
    y_debate_test = test_set["[debate stance:polarity]"]

    # Explicit stance classification
    # set up majority class baseline
    majority_baseline = majority_class_baseline(X_train, y_train)
    baseline_label = majority_baseline.predict(X_test)
    logger.info("baseline")
    print(
        f"F1 for majority class baseline of explicit stance: \n \
{round(compute_micro_f1(y_test, baseline_label), 2)}"
    )
    # perform 10-fold cross-validation on the training set with a svm
    svm = SVC(kernel="linear", C=1, random_state=11)
    classifier = MultiOutputClassifier(svm, n_jobs=-1)

    scores = cross_validate(
        classifier,
        X_train,
        y_train,
        cv=10,
        return_estimator=True,
    )
    logger.info(f"Cross-validation accuracy per classifier: \n {scores['test_score']}")

    # use best classifier to predict test set
    estimator = get_best_classifier(scores)
    y_pred = estimator.predict(X_test)
    print("Micro F1 score for classification of explicit stance targets:")
    print(round(compute_micro_f1(y_test, y_pred), 2))

    # Debate stance classification
    logger.info("Debate stance classification")
    debate_majority_baseline = majority_class_baseline(
        X_train,
        y_debate_train,
    ).predict(X_test)
    print(
        f"F1 score for baseline of debate stance: \n\
{round(f1_score(y_debate_test, debate_majority_baseline,average='micro'), 2)}"
    )
    debate_scores = cross_validate(
        svm,
        X_train,
        y_debate_train,
        cv=10,
        return_estimator=True,
    )
    logger.info(
        f"Cross-validation accuracy per classifier: \n {debate_scores['test_score']}"
    )
    debate_estimator = get_best_classifier(debate_scores)
    y_pred_debate = debate_estimator.predict(X_test)
    print(
        f"F1 score for classification of debate stance: \n \
{round(f1_score(y_debate_test, y_pred_debate, average='micro'), 2)}"
    )

    # Decision tree classification
    # prepare data: use one-hot-encoding to transform categorical data
    y_train_num = transform_categorical_data(y_train, label_columns)
    y_pred_num = transform_categorical_data(y_pred, label_columns)
    y_test_num = transform_categorical_data(y_test, label_columns)

    dt = DecisionTreeClassifier(
        criterion="entropy",
        max_depth=10,
        min_samples_leaf=4,
    )
    dt.fit(y_train_num, y_debate_train)
    dt_prediction = dt.predict(y_pred_num)
    print(
        f"F1 score of decision tree classification: \n \
{round(f1_score(np.array(y_debate_test), dt_prediction, average='micro'), 2)}"
    )

    # "orcale" decision tree
    oracle_prediction = dt.predict(y_test_num)
    print(
        f"F1 score of 'oracle' decision tree: \n \
{round(f1_score(y_debate_test, oracle_prediction, average='micro'), 2)}"
    )

def parse_arguments(arguments: Optional[List[str]] = None) -> argparse.Namespace:
    """Construct argument parser"""
    parser = argparse.ArgumentParser(
        add_help=False,
        description="Process and classify data for explicit and debate stance \
using a SVM in 10-fold cross validation and a decision tree classifier.",
    )
    parser.add_argument(
        "--help",
        "-h",
        action="help",
        default=argparse.SUPPRESS,
        help="Show this help message an exit",
    )
    parser.add_argument(
        "data_path",
        help="File to process",
        type=valid_path,
    )

    return parser.parse_args(arguments)

def valid_path(input_string: str) -> Union[bool, str]:
    """
    Checks if a file or directory exists.

    Parameters
    ----------
    input_string : str
        File or folder to check.

    Returns
    -------
    Union[bool, str]
        False if file or folder is not found, else the path of the file or
        directory is returned.

    """
    if os.path.isfile(input_string) or os.path.isdir(input_string):
        return input_string
    return False

if __name__ == "__main__":
    main()


text:
#!/usr/bin/env python
"""
Some sanity checks of the rendering.
"""

from matplotlib import pyplot as plt
import math
import random
import numpy as np
import os
import sys
sys.path.insert(1, os.path.realpath(__file__ + '../lib/'))
import utils
import visualization
import ntpath
import bounding_box
import common

if __name__ == '__main__':

    if len(sys.argv) < 2:
        print('Usage!')

    config_file = sys.argv[1]
    assert os.path.exists(config_file)

    print('[Data] reading ' + config_file)
    config = utils.read_json(config_file)
    dataset = ntpath.basename(config_file)[:-5]

    inputs = utils.read_hdf5(common.filename(config, 'input_file', '_f.h5', dataset))
    print('[Data] read ' + common.filename(config, 'input_file', '_f.h5', dataset))
    #inputs_sdf = utils.read_hdf5(common.filename(config, 'input_sdf_file', '_f.h5', dataset))
    #print('[Data] read ' + common.filename(config, 'input_sdf_file', '_f.h5', dataset))

    #inputs_combined_gt = utils.read_hdf5(common.filename(config, 'input_combined_gt_file', '_f.h5', dataset))
    #print('[Data] read ' + common.filename(config, 'input_combined_gt_file', '_f.h5', dataset))
    #inputs_sdf_combined_gt = utils.read_hdf5(common.filename(config, 'input_sdf_combined_gt_file', '_f.h5', dataset))
    #print('[Data] read ' + common.filename(config, 'input_sdf_combined_gt_file', '_f.h5', dataset))

    #inputs_gt = utils.read_hdf5(common.filename(config, 'input_gt_file', '_f.h5', dataset))
    #print('[Data] read ' + common.filename(config, 'input_gt_file', '_f.h5', dataset))
    #inputs_sdf_gt = utils.read_hdf5(common.filename(config, 'input_sdf_gt_file', '_f.h5', dataset))
    #print('[Data] read ' + common.filename(config, 'input_sdf_gt_file', '_f.h5', dataset))
    #full_space_gt = utils.read_hdf5(common.filename(config, 'full_space_gt_file', '_f.h5', dataset))
    #print('[Data] read ' + common.filename(config, 'full_space_gt_file', '_f.h5', dataset))
    #part_space_gt = utils.read_hdf5(common.filename(config, 'part_space_gt_file', '_f.h5', dataset))
    #print('[Data] read ' + common.filename(config, 'part_space_gt_file', '_f.h5', dataset))

    full_space = utils.read_hdf5(common.filename(config, 'full_space_file', '_f.h5', dataset))
    print('[Data] read ' + common.filename(config, 'full_space_file', '_f.h5', dataset))
    part_space = utils.read_hdf5(common.filename(config, 'part_space_file', '_f.h5', dataset))
    print('[Data] read ' + common.filename(config, 'part_space_file', '_f.h5', dataset))
    bounding_boxes = bounding_box.read_bounding_boxes(common.filename(config, 'bounding_box_file', '.txt', dataset))
    print('[Data] read ' + common.filename(config, 'bounding_box_file', '.txt', dataset))
    #depths = utils.read_hdf5(common.filename(config, 'depth_file'))
    #print('[Data] read ' + common.filename(config, 'depth_file'))
    #cylindral = utils.read_hdf5(common.filename(config, 'cylindral_file'))
    #print('[Data] read ' + common.filename(config, 'cylindral_file'))

    print(inputs.shape)
    print(inputs_sdf.shape)
    print(full_space.shape)
    print(part_space.shape)

    for i in range(min(25, inputs.shape[0])):
        n = i#random.randint(0, inputs.shape[0] - 1)

        print('[Data] visualizing %s %d/%d' % (dataset, (n + 1), inputs.shape[0]))

        #n_voxels = np.sum(inputs[n])
        #n_space = np.sum(space[n])

        #center = bounding_boxes[n].translation
        #distance = math.sqrt(center[0] * center[0] + center[2] * center[2])

        #if n_voxels < config['min_input_voxels'] or n_space < config['min_space_voxels'] or distance > config['max_bounding_box_distance']:
        #    print('   filtered (%d/%d %d/%d %f/%f' % (
        #        n_voxels, config['min_input_voxels'],
        #        n_space, config['min_space_voxels'],
        #        distance, config['max_bounding_box_distance']
        #    ))
        #else:
        #    print('   not filtered (%d/%d %d/%d %f/%f' % (
        #        n_voxels, config['min_input_voxels'],
        #        n_space, config['min_space_voxels'],
        #        distance, config['max_bounding_box_distance']
        #    ))

        slices = []
        for j in range(2, inputs[n][0].shape[0] - 2, 1):
            slices.append(j)

        visualization.plot_specific_slices([inputs[n][0], full_space[n][0], part_space[n][0]], slices, dataset + '_' + str(n) + '.png')
        #visualization.plot_specific_slices([inputs_combined_gt[n][0], inputs_gt[n][0], full_space_gt[n][0], part_space_gt[n][0], inputs_gt[n][1], full_space_gt[n][1], part_space_gt[n][1], inputs_gt[n][2], full_space_gt[n][2], part_space_gt[n][2]], slices, dataset + '_' + str(n) + '_gt.png')
        #visualization.plot_specific_slices([inputs_sdf_combined_gt[n][0], inputs_sdf[n][0]], slices, dataset + '_' + str(n) + '_sdf.png', 0, np.min(inputs_sdf[n]),np.max(inputs_sdf[n]))
        #visualization.plot_volumes(space[n][0], inputs[n][0], '', 10, 1)

        #visualization.plot_volume_mayavi(inputs[n][0])
        #visualization.plot_volume_mayavi(full_space[n][0])
        #visualization.plot_volume_mayavi(part_space[n][0])

        #plt.imshow(depths[n, :, :, 0])
        #plt.imshow(cylindral[n])
        #plt.show()

max_stars_repo_name: davidstutz/daml-shape-completion

max_issues_repo_name: davidstutz/daml-shape-completion

max_forks_repo_name: davidstutz/daml-shape-completion


text:
# -*- coding: utf-8 -*-
""" The AdBandits bandit algorithm, mixing Thompson Sampling and BayesUCB.

- Reference: [AdBandit: A New Algorithm For Multi-Armed Bandits, F.S.Truzzi, V.F.da Silva, A.H.R.Costa, F.G.Cozman](http://sites.poli.usp.br/p/fabio.cozman/Publications/Article/truzzi-silva-costa-cozman-eniac2013.pdf)
- Code inspired from: https://github.com/flaviotruzzi/AdBandits/

.. warning:: This policy is very not famous, but for stochastic bandits it works usually VERY WELL! It is not anytime thought.
"""
from __future__ import division, print_function  # Python 2 compatibility

__author__ = "Flavio Truzzi and Lilian Besson"
__version__ = "0.9"

from random import random, choice
import numpy as np

try:
    from .Posterior import Beta
    from .BasePolicy import BasePolicy
    from .with_proba import with_proba
except ImportError:
    from Posterior import Beta
    from BasePolicy import BasePolicy
    from with_proba import with_proba

# --- Data

#: Default value for the parameter :math:`\alpha` for the :class:`AdBandits` class.
ALPHA = 1

# --- Class

class AdBandits(BasePolicy):
    """ The AdBandits bandit algorithm, mixing Thompson Sampling and BayesUCB.

    - Reference: [AdBandit: A New Algorithm For Multi-Armed Bandits, F.S.Truzzi, V.F.da Silva, A.H.R.Costa, F.G.Cozman](http://sites.poli.usp.br/p/fabio.cozman/Publications/Article/truzzi-silva-costa-cozman-eniac2013.pdf)
    - Code inspired from: https://github.com/flaviotruzzi/AdBandits/

    .. warning:: This policy is very not famous, but for stochastic bandits it works usually VERY WELL! It is not anytime thought.
    """

    def __init__(self, nbArms, horizon=1000, alpha=ALPHA, posterior=Beta, lower=0., amplitude=1.):
        """ New policy."""
        super(AdBandits, self).__init__(nbArms, lower=lower, amplitude=amplitude)
        self.alpha = alpha  #: Parameter alpha
        self.horizon = int(horizon)  #: Parameter :math:`T` = known horizon of the experiment. Default value is 1000.
        self.posterior = [None] * self.nbArms  #: Posterior for each arm. List instead of dict, quicker access
        for arm in range(self.nbArms):
            self.posterior[arm] = posterior()

    def __str__(self):
        # OK, they all have knowledge of T, but it's good to display it to, remember it
        return r"AdBandits($T={}$, $\alpha={:.3g}$)".format(self.horizon, self.alpha)

    def startGame(self):
        """ Reset each posterior."""
        super(AdBandits, self).startGame()
        for arm in range(self.nbArms):
            self.posterior[arm].reset()

    def getReward(self, arm, reward):
        """ Store the reward, and update the posterior for that arm."""
        super(AdBandits, self).getReward(arm, reward)
        reward = (reward - self.lower) / self.amplitude
        self.posterior[arm].update(reward)

    # This decorator @property makes this method an attribute, cf. https://docs.python.org/3/library/functions.html#property
    @property
    def epsilon(self):
        r""" Time variating parameter :math:`\varepsilon(t)`."""
        # Crop it to [0, 1]
        return max(0, min(1, float(self.t / (self.horizon * self.alpha))))

    def choice(self):
        r""" With probability :math:`1 - \varepsilon(t)`, use a Thompson Sampling step, otherwise use a UCB-Bayes step, to choose one arm."""
        # Thompson Exploration
        if with_proba(1 - self.epsilon):  # with proba 1-epsilon
            upperbounds = [self.posterior[i].sample() for i in range(self.nbArms)]
            maxIndex = max(upperbounds)
            bestArms = [arm for (arm, index) in enumerate(upperbounds) if index == maxIndex]
            arm = choice(bestArms)
        # UCB-Bayes
        else:
            expectations = (1.0 + self.rewards) / (2.0 + self.pulls)
            upperbounds = [self.posterior[arm].quantile(1. - 1. / self.t) for arm in range(self.nbArms)]
            regret = np.max(upperbounds) - expectations
            admissible = np.nonzero(regret == np.min(regret))[0]
            arm = choice(admissible)
        return arm

    def choiceWithRank(self, rank=1):
        r""" With probability :math:`1 - \varepsilon(t)`, use a Thompson Sampling step, otherwise use a UCB-Bayes step, to choose one arm of a certain rank."""
        if rank == 1:
            return self.choice()
        else:
            assert rank >= 1, "Error: for AdBandits = {}, in choiceWithRank(rank={}) rank has to be >= 1.".format(self, rank)
            # Thompson Exploration
            if with_proba(1 - self.epsilon):  # with proba 1-epsilon
                indexes = [self.posterior[i].sample() for i in range(self.nbArms)]
            # UCB-Bayes
            else:
                expectations = (1.0 + self.rewards) / (2.0 + self.pulls)
                upperbounds = [self.posterior[arm].quantile(1. - 1. / self.t) for arm in range(self.nbArms)]
                indexes = expectations - np.max(upperbounds)
            # We computed the indexes, OK let's use them
            sortedRewards = np.sort(indexes)  # XXX What happens here if two arms has the same index, being the max?
            chosenIndex = sortedRewards[-rank]
            # Uniform choice among the rank-th best arms
            return choice(np.nonzero(indexes == chosenIndex)[0])


text:
import os
import numpy as np
from collections import Counter
from collections import deque

gram_num = 4
min_count = 60
dim=256
data_type = '6_layer_sigmoid/0.0001_256'
embedding_file = '/nfs/YueLao/grtzsohalf/yeeee/English/embedding/'+data_type+'/train_embeddings_17'
embedding_with_word_file = '/nfs/YueLao/grtzsohalf/yeeee/English/embedding/'+data_type+'/train_embeddings_17_with_words'

# word_dir = '/nfs/YueLao/grtzsohalf/yeeee/English/all_words'
# word_files = os.listdir(word_dir)

# word_freq = Counter()
# for word_file in word_files:
    # with open(os.path.join(word_dir, word_file), 'r') as f_w:
        # for line in f_w:
            # word = line[:-1]
            # word_freq[word] += 1

# embs = {}
# counts = {}
# total_count = 0
# with open(embedding_file, 'r') as f_emb:
    # for word_file in word_files:
        # print (word_file)
        # words = []
        # with open(os.path.join(word_dir, word_file), 'r') as f_word:
            # for word in f_word:
                # words.append(word[:-1])
        # que = deque()
        # for word in words:
            # que.append(word)
            # if len(que) < 2*gram_num+1:
                # continue
            # continue_bool = False
            # for qq in que:
                # if word_freq[qq] < min_count:
                    # continue_bool = True
                    # break
            # if continue_bool == True:
                # que.popleft()
                # continue
            # emb = f_emb.readline()
            # emb = np.array(list(map(float, emb[:-1].split())))
            # ww = que[gram_num]
            # if not ww in embs:
                # embs[ww] = emb
                # counts[ww] = 1
            # else:
                # embs[ww] += emb
                # counts[ww] += 1
            # que.popleft()
            # total_count += 1
# print ('# of feats after: '+str(total_count))
# print ('# of different words after: '+str(len(embs)))

embs = {}
counts = Counter()
total_count = 0
with open(embedding_file, 'r') as f_emb:
    for line in f_emb:
        line = line[:-1].split()
        word = line[0]
        emb = np.array(list(map(float, line[1:])))
        if not word in embs:
            embs[word] = emb
            counts[word] += 1
        else:
            embs[word] += emb
            counts[word] += 1
        total_count += 1
print ('Total #: '+str(total_count))

with open(embedding_with_word_file, 'w') as f_out:
    f_out.write(str(len(embs))+' '+str(dim)+'\n')
    for word in embs:
        emb = embs[word] / counts[word]
        f_out.write(word+' '+' '.join(emb.astype(str))+'\n')


text:
import numpy as np
import os
import torch

from plasticorigins.detection.detect import detect
from plasticorigins.tracking.utils import get_detections_for_video, write_tracking_results_to_file
from plasticorigins.tools.video_readers import IterableFrameReader
from plasticorigins.tools.misc import load_model
from plasticorigins.tracking.trackers import get_tracker
from plasticorigins.tracking.track_video import track_video, Display

def main(args, display):

    if args.device is None:
        device = 'cuda' if torch.cuda.is_available() else 'cpu'
    else:
        device = args.device
    device = torch.device(device)

    engine = get_tracker('EKF')

    print('---Loading model...')
    model = load_model(arch=args.arch, model_weights=args.model_weights, device=device)
    print('Model loaded.')

    detector = lambda frame: detect(frame, threshold=args.detection_threshold, model=model)

    transition_variance = np.load(os.path.join(args.noise_covariances_path, 'transition_variance.npy'))
    observation_variance = np.load(os.path.join(args.noise_covariances_path, 'observation_variance.npy'))

    video_filenames = [video_filename for video_filename in os.listdir(args.data_dir) if video_filename.endswith('.mp4')]

    for video_filename in video_filenames:
        print(f'---Processing {video_filename}')
        reader = IterableFrameReader(video_filename=os.path.join(args.data_dir, video_filename),
                                     skip_frames=args.skip_frames,
                                     output_shape=args.output_shape,
                                     progress_bar=True,
                                     preload=args.preload_frames)

        input_shape = reader.input_shape
        output_shape = reader.output_shape
        ratio_y = input_shape[0] / (output_shape[0] // args.downsampling_factor)
        ratio_x = input_shape[1] / (output_shape[1] // args.downsampling_factor)

        print('Detecting...')
        detections = get_detections_for_video(reader, detector, batch_size=args.detection_batch_size, device=device)

        print('Tracking...')
        results = track_video(reader, iter(detections), args, engine, transition_variance, observation_variance, display)

        output_filename = os.path.join(args.output_dir, video_filename.split('.')[0] +'.txt')
        write_tracking_results_to_file(results, ratio_x=ratio_x, ratio_y=ratio_y, output_filename=output_filename)

if __name__ == '__main__':

    import argparse
    parser = argparse.ArgumentParser(description='Tracking')
    parser.add_argument('--data_dir', type=str)
    parser.add_argument('--detection_threshold', type=float, default=0.3)
    parser.add_argument('--confidence_threshold', type=float, default=0.2)
    parser.add_argument('--model_weights', type=str, default=None)
    parser.add_argument('--output_dir', type=str)
    parser.add_argument('--downsampling_factor', type=int, default=1)
    parser.add_argument('--noise_covariances_path',type=str)
    parser.add_argument('--skip_frames',type=int,default=0)
    parser.add_argument('--output_shape',type=str,default='960,544')
    parser.add_argument('--arch', type=str, default='dla_34')
    parser.add_argument('--display', type=int, default=0)
    parser.add_argument('--device', type=str, default='cuda')
    parser.add_argument('--detection_batch_size',type=int,default=1)
    parser.add_argument('--preload_frames', action='store_true', default=False)
    args = parser.parse_args()

    display = None
    if args.display == 0:
        display = Display(on=False, interactive=True)
    elif args.display == 1:
        display = Display(on=True, interactive=True)
    elif args.display == 2:
        display = Display(on=True, interactive=False)

    args.output_shape = tuple(int(s) for s in args.output_shape.split(','))

    main(args, display)

max_stars_repo_name: surfriderfoundationeurope/surfnet

max_issues_repo_name: surfriderfoundationeurope/surfnet

max_forks_repo_name: surfriderfoundationeurope/surfnet


text:
import csv
import pickle
import glob, re
import numpy as np
import pandas as pd

tags = ['num', 'orange hair', 'white hair', 'aqua hair', 'gray hair', 'green hair', 'red hair', 'purple hair',
            'pink hair', 'blue hair', 'black hair', 'brown hair', 'blonde hair',
            'gray eyes', 'black eyes', 'orange eyes', 'pink eyes', 'yellow eyes',
            'aqua eyes', 'purple eyes', 'green eyes', 'brown eyes', 'red eyes', 'blue eyes']

imgs = glob.glob('../../dataset/data/*.png')
imgs_nums = set(re.findall(r'\d+', img)[-1] for img in imgs)

with open('features.pickle', 'rb') as handle:
    d = pickle.load(handle)

with open('features.csv', 'a') as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=tags)
    writer.writeheader()

    for k, v in d.items():
        if not k in imgs_nums:
            continue

        label = {k2: 0 for k2 in tags}
        label['num'] = k
        for name_feature, zero_or_one in v.items():
            # zero_or_one is always 1
            label[name_feature] = 1
        writer.writerow(label)


text:
"""Stanford 40 Actions dataset."""
from __future__ import absolute_import
from __future__ import division
import os
import logging
import numpy as np
try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET
import mxnet as mx
import pickle as pkl
from ..base import VisionDataset
from ..transforms import bbox

class Stanford40Action(VisionDataset):
    """Stanford 40 Actions dataset.

    Parameters
    ----------
    root : str, default '~/data/Stanford40'
        Path to folder storing the dataset.
    split : str, default 'train'
        Candidates can be: 'train', 'test'.
    transform : callable, defaut None
        A function that takes data and label and transforms them. Refer to
        :doc:`./transforms` for examples.

        A transform function for object detection should take label into consideration,
        because any geometric modification will require label to be modified.
    index_map : dict, default None
        In default, the 11 classes are mapped into indices from 0 to 10. We can
        customize it by providing a str to int dict specifying how to map class
        names to indicies. Use by advanced users only, when you want to swap the orders
        of class labels.
    preload_label : bool, default True
        If True, then parse and load all labels into memory during
        initialization. It often accelerate speed but require more memory
        usage. Typical preloaded labels took tens of MB. You only need to disable it
        when your dataset is extreamly large.
    """
    CLASSES = ("applauding", "blowing_bubbles", "brushing_teeth", "cleaning_the_floor", "climbing", "cooking",
               "cutting_trees", "cutting_vegetables", "drinking", "feeding_a_horse", "fishing", "fixing_a_bike",
               "fixing_a_car", "gardening", "holding_an_umbrella", "jumping", "looking_through_a_microscope",
               "looking_through_a_telescope", "playing_guitar", "playing_violin", "pouring_liquid", "pushing_a_cart",
               "reading", "phoning", "riding_a_bike", "riding_a_horse", "rowing_a_boat", "running",
               "shooting_an_arrow", "smoking", "taking_photos", "texting_message", "throwing_frisby",
               "using_a_computer", "walking_the_dog", "washing_dishes", "watching_TV", "waving_hands",
               "writing_on_a_board", "writing_on_a_book")

    def __init__(self, root=os.path.join('~', 'data', 'Stanford40'),
                 split='train', index_map=None, preload_label=True,
                 augment_box=False, load_box=False):
        super(Stanford40Action, self).__init__(root)
        self._im_shapes = {}
        self._root = os.path.expanduser(root)
        self._load_box = load_box
        self._augment_box = augment_box
        self._split = split
        self._items = self._load_items(split)
        self._anno_path = os.path.join(self._root, 'XMLAnnotations', '{}.xml')
        self._image_path = os.path.join(self._root, 'JPEGImages', '{}.jpg')
        self._box_path = os.path.join(self._root, 'Boxes', '{}.pkl')
        self.index_map = index_map or dict(zip(self.classes, range(self.num_class)))
        self._label_cache = self._preload_labels() if preload_label else None

    def __str__(self):
        return self.__class__.__name__ + '(' + self._split + ')'

    @property
    def classes(self):
        """Category names."""
        return type(self).CLASSES

    def img_path(self, idx):
        img_id = self._items[idx]
        return self._image_path.format(img_id)

    def save_boxes(self, idx, boxes):
        img_id = self._items[idx]
        box_path = self._box_path.format(img_id)
        with open(box_path, 'wb') as f:
            pkl.dump(boxes, f)

    def __len__(self):
        return len(self._items)

    def __getitem__(self, idx):
        img_id = self._items[idx]
        img_path = self._image_path.format(img_id)
        label = self._label_cache[idx] if self._label_cache else self._load_label(idx)
        img = mx.image.imread(img_path, 1)
        if self._augment_box:
            h, w, _ = img.shape
            label = bbox.augment(label, img_w=w, img_h=h, output_num=16)
        if self._load_box:
            box_path = self._box_path.format(img_id)
            with open(box_path, 'rb') as f:
                box = pkl.load(f)
            return img, label, box
        return img, label

    def _load_items(self, split):
        """Load individual image indices from split."""
        ids = []
        set_file = os.path.join(self._root, 'ImageSplits', split + '.txt')
        with open(set_file, 'r') as f:
            # remove file extensions
            ids += [line.strip().rsplit(sep='.', maxsplit=1)[0] for line in f.readlines()]
        return ids

    def _load_label(self, idx):
        """Parse xml file and return labels."""
        img_id = self._items[idx]
        anno_path = self._anno_path.format(img_id)
        root = ET.parse(anno_path).getroot()
        size = root.find('size')
        width = float(size.find('width').text)
        height = float(size.find('height').text)
        if idx not in self._im_shapes:
            # store the shapes for later usage
            self._im_shapes[idx] = (width, height)
        label = []
        for obj in root.iter('object'):
            cls_name = obj.find('name').text.strip().lower()
            if cls_name != 'person':
                continue

            xml_box = obj.find('bndbox')
            xmin = (float(xml_box.find('xmin').text) - 1)
            ymin = (float(xml_box.find('ymin').text) - 1)
            xmax = (float(xml_box.find('xmax').text) - 1)
            ymax = (float(xml_box.find('ymax').text) - 1)
            try:
                self._validate_label(xmin, ymin, xmax, ymax, width, height)
            except AssertionError as e:
                raise RuntimeError("Invalid label at {}, {}".format(anno_path, e))

            act_cls_name = obj.find('action').text
            cls_id = self.index_map[act_cls_name]
            cls_array = [0] * len(self.classes)
            cls_array[cls_id] = 1
            anno = [xmin, ymin, xmax, ymax, cls_id]
            anno.extend(cls_array)
            label.append(anno)
        return np.array(label)

    def _validate_label(self, xmin, ymin, xmax, ymax, width, height):
        """Validate labels."""
        assert 0 <= xmin < width, "xmin must in [0, {}), given {}".format(width, xmin)
        assert 0 <= ymin < height, "ymin must in [0, {}), given {}".format(height, ymin)
        assert xmin < xmax <= width, "xmax must in (xmin, {}], given {}".format(width, xmax)
        assert ymin < ymax <= height, "ymax must in (ymin, {}], given {}".format(height, ymax)

    def _preload_labels(self):
        """Preload all labels into memory."""
        logging.debug("Preloading %s labels into memory...", str(self))
        return [self._load_label(idx) for idx in range(len(self))]


text:
from yolov3_tf2.utils import draw_outputs
from yolov3_tf2.dataset import transform_images
from yolov3_tf2.models import (
    YoloV3, YoloV3Tiny
)
from absl.flags import FLAGS
from absl import app, flags, logging
import time
import os
import glob

import cv2
import matplotlib.pyplot as plt
import dlib

import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import ZeroPadding2D, Convolution2D, MaxPooling2D
from tensorflow.keras.layers import Dense, Dropout, Softmax, Flatten, Activation, BatchNormalization
from tensorflow.keras.preprocessing.image import load_img, img_to_array
from tensorflow.keras.applications.imagenet_utils import preprocess_input
import tensorflow.keras.backend as K

from scipy.spatial.distance import cosine

import random
import websockets
import asyncio
import json
import base64

def get_match(known_embeddings, candidate_embedding, thresh=0.3):

    minim = thresh
    person = "unknown"

    for key in known_embeddings:
        score = cosine(known_embeddings[key], candidate_embedding)
        if minim > score:
            minim = score
            person = key

    return person, minim

def recognize(img, outputs, class_names, vgg_face):
    person_rep = dict()
    person_names = ["angelamerkel", "jinping", "trump"]
    for person in person_names:
        embed = np.loadtxt(person+".txt")
        person_rep[person] = embed

    boxes, objectness, classes, nums = outputs
    boxes, objectness, classes, nums = boxes[0], objectness[0], classes[0], nums[0]
    wh = np.flip(img.shape[0:2])
    for i in range(nums):
        x1y1 = tuple((np.array(boxes[i][0:2]) * wh).astype(np.int32))
        x2y2 = tuple((np.array(boxes[i][2:4]) * wh).astype(np.int32))
        if class_names[int(classes[i])] == "face":

            img_crop = img[x1y1[1]:x2y2[1], x1y1[0]:x2y2[0]]

            crop_img = img_to_array(img_crop)
            crop_img = np.expand_dims(crop_img, axis=0)
            crop_img = preprocess_input(crop_img)

            img_encode = vgg_face(transform_images(crop_img, 224))
            embed = K.eval(img_encode)
            name, score = get_match(person_rep, embed, 0.3)

            img = cv2.rectangle(img, x1y1, x2y2, (205, 0, 0), 2)
            img = cv2.putText(img, '{} {:.4f}'.format(
                name, score),
                x1y1, cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (255, 0, 255), 2)
        else:
            img = cv2.rectangle(img, x1y1, x2y2, (255, 0, 0), 2)
            img = cv2.putText(img, '{} {:.4f}'.format(
                class_names[int(classes[i])], objectness[i]),
                x1y1, cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 2)
    return img

flags.DEFINE_string('classes', './data/face.names', 'path to classes file')
flags.DEFINE_string('weights', './checkpoints/yolov3_train_10.tf',
                    'path to weights file')
flags.DEFINE_boolean('tiny', False, 'yolov3 or yolov3-tiny')
flags.DEFINE_integer('size', 416, 'resize images to')
flags.DEFINE_string('video', './DonaldTrump.mp4',
                    'path to video file or number for webcam)')
flags.DEFINE_string('output', None, 'path to output video')
flags.DEFINE_string('output_format', 'XVID',
                    'codec used in VideoWriter when saving video to file')
flags.DEFINE_integer('num_classes', 80, 'number of classes in the model')

class VideoCamera(object):
    def __init__(self, stream):
        self.video = stream

    def __del__(self):
        self.video

    def get_frame(self):
        while True:
            image = self.video
            ret, jpeg = cv2.imencode('.jpg', image)
            base6 = base64.b64encode(jpeg.tobytes())
            yield base6.decode('utf-8')

def gen(camera):

    while True:
        image = next(camera.get_frame())

        yield(image)

def recog(_argv):

    physical_devices = tf.config.experimental.list_physical_devices('GPU')
    if len(physical_devices) > 0:
        tf.config.experimental.set_memory_growth(physical_devices[0], True)

    if FLAGS.tiny:
        yolo = YoloV3Tiny(classes=FLAGS.num_classes)
    else:
        yolo = YoloV3(classes=FLAGS.num_classes)

    yolo.load_weights(FLAGS.weights)
    logging.info('weights loaded')
    vgg_face = tf.keras.models.load_model("vgg_model.h5")
    logging.info('vgg_model loaded')
    class_names = [c.strip() for c in open(FLAGS.classes).readlines()]
    logging.info('classes loaded')

    times = []

    try:
        vid = cv2.VideoCapture(int(FLAGS.video))
    except:
        vid = cv2.VideoCapture(FLAGS.video)

    out = None

    if FLAGS.output:
        # by default VideoCapture returns float instead of int
        width = int(vid.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))
        fps = int(vid.get(cv2.CAP_PROP_FPS))
        codec = cv2.VideoWriter_fourcc(*FLAGS.output_format)
        out = cv2.VideoWriter(FLAGS.output, codec, fps, (width, height))

    while True:
        _, img = vid.read()

        if img is None:
            logging.warning("Empty Frame")
            time.sleep(0.1)
            continue

        img_in = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        img_in = tf.expand_dims(img_in, 0)
        img_in = transform_images(img_in, FLAGS.size)

        t1 = time.time()
        boxes, scores, classes, nums = yolo.predict(img_in)
        t2 = time.time()
        times.append(t2-t1)
        times = times[-20:]

        img = recognize(img, (boxes, scores, classes, nums),
                        class_names, vgg_face)
        img = cv2.putText(img, "Time: {:.2f}ms".format(sum(times)/len(times)*1000), (0, 30),
                          cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 2)
        if FLAGS.output:
            out.write(img)
        yield img
        if cv2.waitKey(1) == ord('q'):
            break

    cv2.destroyAllWindows()

async def sender(websocket, path):
    camera = VideoCamera(recog)
    i = 0
    while True:
        i = i+1
        data = next(gen(camera))
        js = {'data': data, "infos": [
            {
                "key": "1",
                "name": "John Brown",
                "age": 32,
                "address": "New York No. 1 Lake Park",
            },
            {
                "key": "2",
                "name": "Jim Green",
                "age": 42,
                "address": "London No. 1 Lake Park",
            },
            {
                "key": "3",
                "name": "Joe Black",
                "age": 32,
                "address": "Sidney No. 1 Lake Park",
            },
            {
                "key": "4",
                "name": "Jim Red",
                "age": 32,
                "address": "London No. 2 Lake Park",
            },
        ], }

        res_bytes = json.dumps(js).encode('utf-8')
        base6 = base64.b64encode(res_bytes)
        message = base6.decode('utf-8')

        await websocket.send(message)

def main(_argv):
    start_server = websockets.serve(sender, "127.0.0.1", 5678)
    asyncio.get_event_loop().run_until_complete(start_server)
    asyncio.get_event_loop().run_forever()

if __name__ == '__main__':
    try:

        app.run(main)
    except SystemExit:
        pass

reason: import numpy,from scipy


text:
# coding: utf-8

from numpy import zeros

#$ header mxm_omp(int, int, int, double[:,:], double[:,:], double[:,:])
def mxm_omp(n,m,p,a,b,c):
    a = 0.
    b = 0.
    c = 0.

    #$ omp parallel
    #$ omp do schedule(runtime)
    for i in range(0, n):
        for j in range(0, m):
            a[i,j] = i-j
    #$ omp end do nowait

    #$ omp do schedule(runtime)
    for i in range(0, m):
        for j in range(0, p):
            b[i,j] = i+j
    #$ omp end do nowait

    #$ omp do schedule(runtime)
    for i in range(0, n):
        for j in range(0, p):
            for k in range(0, p):
                c[i,j] = c[i,j] + a[i,k]*b[k,j]
    #$ omp end do
    #$ omp end parallel

n = 800
m = 1600
p = 800

a = zeros((n,m))
b = zeros((m,p))
c = zeros((n,p))

mxm_omp(n,m,p,a,b,c)

max_stars_repo_name: dina-fouad/pyccel

max_issues_repo_name: dina-fouad/pyccel

max_forks_repo_name: dina-fouad/pyccel

reason: from numpy


text:
# Copyright (C) 2020  GreenWaves Technologies, SAS

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import logging
import os
import struct
from io import BufferedWriter, BufferedReader

import numpy as np

LOG = logging.getLogger("nntool." + __name__)

TYPES = {
    8: np.int8,
    16: np.int16,
    32: np.int32
}

CTYPES = {
    np.int8: "signed char",
    np.int16: "short int",
    np.int32: "int"
}

class Header():
    __FMT = '5i4i'

    def __init__(self, step, tensor_type, size, elem_size, num_dims, *dims):
        self.step = step
        self.tensor_type = tensor_type
        self.size = size
        self.elem_size = elem_size
        self.dims = [dims[i] for i in range(num_dims)]

    @classmethod
    def read(cls, fp: BufferedReader):
        buf = fp.read(struct.calcsize(cls.__FMT))
        if buf:
            return cls(*struct.unpack(cls.__FMT, buf))
        return None

    def write(self, fp: BufferedWriter):
        assert len(self.dims) <= 4, "up to 4 dimensions are supported"
        dims = self.dims.copy() + [0] * (4 - len(self.dims))
        buf = struct.pack(self.__FMT, self.step, self.tensor_type, self.size, self.elem_size, len(self.dims), *dims)
        fp.write(buf)

def write_dims(prefix, fp, dims, elem_size):
    assert len(dims) == 3
    dim_names = ["CHANNELS", "HEIGHT", "WIDTH"]
    for i, v in enumerate(dims):
        fp.write('{}_{}?={}\n'.format(prefix, dim_names[i], v))
    fp.write('{}_BITS?={}\n'.format(prefix, elem_size))
    fp.write('{}_TYPE?={}\n'.format(prefix, CTYPES[TYPES[elem_size]]))

def write_make_include_file(filename, header, output_shape, output_elem_size):
    with open(filename, 'w') as fp:
        write_dims("INPUT", fp, header.dims, header.elem_size)
        write_dims("OUTPUT", fp, output_shape, output_elem_size)

def get_bits(tensor):
    bit_size = -1
    for k, v in TYPES.items():
        if tensor.dtype == v:
            bit_size = k
            break
    if bit_size < 0:
        raise ValueError("only int8, int16 and int32 are supported")
    return bit_size

def write_gap_tensor(filename, tensor, step=0, output_tensor=None, make_file=None):
    bit_size = get_bits(tensor)
    with open(filename, 'wb') as fp:
        header = Header(step, 0, tensor.size * bit_size // 8, bit_size, len(tensor.shape), *tensor.shape)
        header.write(fp)
        tensor.tofile(fp)
    if output_tensor is not None and make_file is not None:
        output_bit_size = get_bits(output_tensor)
        write_make_include_file(make_file, header, output_tensor.shape, output_bit_size)

def read_gap_tensors(filename):
    tensors = {}
    size = os.path.getsize(filename)
    max_step = -1
    with open(filename, 'rb') as fp:
        while fp.tell() < size:
            header = Header.read(fp)
            if not header:
                break
            if header.elem_size not in TYPES:
                LOG.error("Bit width %s not found in types list. File may be corrupted.",
                          header.elem_size)
                raise ValueError()
            tensor = np.fromfile(fp, dtype=TYPES[header.elem_size], count=header.size)
            step_rec = tensors.get(header.step)
            if not step_rec:
                step_rec = [None]*3
                tensors[header.step] = step_rec
            elem = 0 if header.tensor_type < 2 else header.tensor_type - 1
            step_rec[elem] = tensor.reshape(header.dims)
            if header.step > max_step:
                max_step = header.step
    return [tensors.get(i) for i in range(max_step + 1)]


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Oct 29 14:45:09 2020

@author: pielsticker
"""
import numpy as np
import os

os.chdir(r"C:\Users\pielsticker\Lukas\MPI-CEC\Projects\deepxps")
from xpsdeeplearning.network.data_handling import DataHandler

#%%
np.random.seed(502)
input_filepath = r"C:\Users\pielsticker\Simulations\20210914_CoFe_individual_without_auger_peaks_35eV_window\20210914_CoFe_individual_without_auger_peaks_35eV_window_peaks_only.h5"

datahandler = DataHandler(intensity_only=False)
train_test_split = 0.2
train_val_split = 0.2
no_of_examples = 200000

(
    X_train,
    X_val,
    X_test,
    y_train,
    y_val,
    y_test,
    aug_values_train,
    aug_values_val,
    aug_values_test,
) = datahandler.load_data_preprocess(
    input_filepath=input_filepath,
    no_of_examples=no_of_examples,
    train_test_split=train_test_split,
    train_val_split=train_val_split,
)
print("Input shape: " + str(datahandler.input_shape))
print("Labels: " + str(datahandler.labels))
print("No. of classes: " + str(datahandler.num_classes))

datahandler.plot_random(
    no_of_spectra=15, dataset="train", with_prediction=False
)

max_stars_repo_name: lukaspie/xpsdeeplearning

max_issues_repo_name: lukaspie/xpsdeeplearning

max_forks_repo_name: lukaspie/xpsdeeplearning


text:
"""Simple convolutional neural network classififer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf

from common import model
from common import ops

class AlexNetModel(model.AbstractModel):

  def get_params(self):
    return {
      "drop_rate": 0.5
    }

  def get_features(self, params):
    image_size = params.image_size
    return {
      "image": tf.placeholder(
        dtype=tf.float32,
        shape=[None, image_size, image_size, 3],
        name="image")
    }

  def model(self, features, labels, mode, params):
    """CNN classifier model."""
    images = features["image"]
    labels = labels["label"]

    training = mode == tf.estimator.ModeKeys.TRAIN
    drop_rate = params.drop_rate if training else 0.0

    features = ops.conv_layers(
      images,
      filters=[64, 128, 256],
      kernels=[3, 3, 3],
      pool_sizes=[2, 2, 2])

    features = tf.keras.layers.Flatten().apply(features)

    logits = ops.dense_layers(
      features, [512, params.num_classes],
      drop_rates=drop_rate,
      linear_top_layer=True)

    predictions = tf.argmax(logits, axis=-1)

    loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)

    tf.summary.image("images", images)

    eval_metrics = {
      "accuracy": tf.metrics.accuracy(labels, predictions),
      "top_1_error": tf.metrics.mean(ops.top_k_error(labels, logits, 1)),
    }

    return {"predictions": predictions}, loss, eval_metrics

model.ModelFactory.register("alexnet", AlexNetModel)

max_stars_repo_name: vahidk/TensorflowFramework

max_issues_repo_name: vahidk/TensorflowFramework

max_forks_repo_name: vahidk/TensorflowFramework


text:
# Copyright 2020 Pure Storage Inc.
# SPDX-License-Identifier: Apache-2.0
# ==============================================================================

"""
Test Fleet format reader/writer interface
"""

from fleetfmt import FileWriter, FileReader

import numpy as np
import pytest

def test_basic(tmp_path):
    """ Test that we can read the entirety of a simple file"""
    # Write file
    pfile = tmp_path / "test.flt"
    with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
        writer.append(100, [1, 2, 3])

    # Read and validate
    with pfile.open('rb') as fhandle, FileReader(fhandle) as reader:
        assert set(reader.keys()) == {100}
        rec = reader.read(100)
        assert len(rec) == 3
        assert rec == [1, 2, 3]

def test_read_twice(tmp_path):
    # Write file
    pfile = tmp_path / "test.flt"
    with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
        writer.append(100, [1, 2, 3])

    with pfile.open('rb') as fhandle:
        with FileReader(fhandle) as reader:
            assert reader.keys() == {100}

        with FileReader(fhandle) as reader:
            assert reader.keys() == {100}

def test_read_complex(tmp_path):
    """ Test that we can read a set of keys """
    # Write file
    pfile = tmp_path / "test.flt"
    with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
        writer.append(100, [1, 2, 3])
        writer.append(101, [4, 5, 6])
        writer.append("alpha", ["7", "8", "9"])

    # Read and validate
    with pfile.open('rb') as fhandle, FileReader(fhandle) as reader:
        assert set(reader.keys()) == {100, 101, "alpha"}
        rec = reader.read([100, 101, "alpha"])
        assert np.array(rec).shape == (3,3)
        assert rec[0] == [1, 2, 3]
        assert rec[1] == [4, 5, 6]
        assert rec[2] == ["7", "8", "9"]

def test_append_to_new(tmp_path):
    """Test that we can append to an empty file"""
    # Use append to write initial records into a new file (opened in 'wb' mode)
    pfile = tmp_path / "test.flt"
    with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
        a = writer.append(100, [0])
        b = writer.append(101, [1])
    # Read and validate
    with pfile.open('rb') as fhandle, FileReader(fhandle) as reader:
        assert set(reader.keys()) == {100, 101}
        assert reader.read(101) == [1]

def test_append_to_existing(tmp_path):
    """ Test that we can add new records to existing file """
    # Write file
    pfile = tmp_path / "test.flt"
    with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
        a = writer.append(100, [0])
        b = writer.append(101, [1])
        c = writer.append(102, [2])
    # Insert new record to existing file
    with pfile.open('rb+') as fhandle, FileWriter(fhandle) as writer:
        d = writer.append(103, [3])
    # Read and validate
    with pfile.open('rb') as fhandle, FileReader(fhandle) as reader:
        assert set(reader.keys()) == {100, 101, 102, 103}
        assert reader.read(103) == [3]

def test_invalid_read_inputs(tmp_path):
    """ Test that exception is raised for invalid reader.read() inputs """
    # Raise exception for malformed read requests
    with pytest.raises(Exception):
        assert reader.read(1.234)

max_stars_repo_name: guillermo-jimenez/fleet-format

max_issues_repo_name: guillermo-jimenez/fleet-format

max_forks_repo_name: guillermo-jimenez/fleet-format


text:
#!/usr/bin/env python

"""
  Author: Adam White, Mohammad M. Ajallooeian, Sina Ghiassian
  Purpose: Code for the Gambler's problem environment from the Sutton and Barto
  Reinforcement Learning: An Introduction Chapter 4.
  For use in the Reinforcement Learning course, Fall 2017, University of Alberta
"""

from utils import rand_norm, rand_in_range, rand_un
import numpy as np
import player_format, card_format
import random

gCurrentState = None
phases = ["cleanup", "action", "buy" ]
player = None
totalDeck = None
supplyCards = None
supplyAmounts = None
totalDeckLists = list()

def env_init():
    global gStartPosition, p1, supplyCards, supplyAmounts, totalDeckLists
    p1 = player_format.playerStats(1,1,0,3) # Easy 3 vp for starting
    totalDeckLists = list()
    supplyCards = card_format.kingdomCards()
    supplyAmounts = card_format.kingdomCardValues(supplyCards)
    cards = card_format.startingCards()
    hand, deck, discard, play = firstHand(cards)
    totalDeck = card_format.allDeckCards(hand, deck, discard, play)
    playAreas = [hand, deck, discard, play]
    possibleBuys = list()
    gStartPosition = [totalDeck, playAreas, possibleBuys]
    return

def env_start():
    """ returns numpy array """
    global gCurrentState, gStartPosition, p1
    gCurrentState = gStartPosition
    totalDeck = gCurrentState[0]
    hand, deck, discard, play = gCurrentState[1]
    possibleBuys = gCurrentState[2]
    hand, deck, discard, play, totalDeck, p1 = actionPhase(hand, deck, discard, play, totalDeck, p1)
    hand, deck, discard, play, totalDeck, p1, buys = buyPhase(hand, deck, discard, play, totalDeck, p1)
    gCurrentState = [totalDeck, [hand, deck, discard, play], buys]
    return gCurrentState

def env_step(action):
    """
    Arguments
    ---------
    action : int
        the action taken by the agent in the current state

    Returns
    -------
    result : dict
        dictionary with keys {reward, state, isTerminal} containing the results
        of the action taken
    """
    global gCurrentState, gTerminalPosition
    global p1
    global totalDeckLists
    reward = -1.0
    isTerminal = False

    totalDeck = gCurrentState[0]
    hand, deck, discard, play = gCurrentState[1]
    possibleBuys = gCurrentState[2]

    for buy in action:
        if buy == "province":
            reward = 1
            isTerminal = True
        if buy != "none":
            supplyAmounts[buy] -= 1
            discard = card_format.newCard(discard, buy)

    totalDeck = card_format.allDeckCards(hand, deck, discard, play)
    hand, deck, discard, play, totalDeck, p1 = cleanupPhase(hand, deck, discard, play, totalDeck, p1)
    hand, deck, discard, play, totalDeck, p1 = actionPhase(hand, deck, discard, play, totalDeck, p1)
    hand, deck, discard, play, totalDeck, p1, buys = buyPhase(hand, deck, discard, play, totalDeck, p1)
    # for phase in phases:
        # if phase == "action": hand, deck, discard, play, totalDeck, p1 = actionPhase(hand, deck, discard, play, totalDeck, p1)
        # elif phase == "buy": buys = buyPhase(hand, deck, discard, play, totalDeck, p1)
        # elif phase == "cleanup": hand, deck, discard, play, totalDeck, p1 = cleanupPhase(hand, deck, discard, play, totalDeck, p1)
        # else: print("OH NO")

    gCurrentState = [totalDeck, [hand, deck, discard, play], buys]
    totalDeckLists.append(totalDeck)
    result = {"reward": reward, "state": gCurrentState, "isTerminal": isTerminal}
    return result

def env_cleanup():
    #
    global gStartPosition, p1, supplyCards, supplyAmounts, totalDeckLists
    p1 = None
    totalDeck = None
    supplyCards = None
    supplyAmounts = None
    totalDeckLists = list()
    p1 = player_format.playerStats(1,1,0,3) # Easy 3 vp for starting
    totalDeckLists = list()
    supplyCards = card_format.kingdomCards()
    supplyAmounts = card_format.kingdomCardValues(supplyCards)
    cards = card_format.startingCards()
    hand, deck, discard, play = firstHand(cards)
    totalDeck = card_format.allDeckCards(hand, deck, discard, play)
    playAreas = [hand, deck, discard, play]
    possibleBuys = list()
    gStartPosition = [totalDeck, playAreas, possibleBuys]
    return

def env_message(in_message): # returns string, in_message: string
    """
    Arguments
    ---------
    inMessage : string
        the message being passed

    Returns
    -------
    string : the response to the message
    """
    return ""

def firstHand(cards):
    ''' Create the first hand to be played; cards are the starting cards
        Returns the hand, deck, discard, and play piles (or locations)
    '''
    random.shuffle(cards)
    hand = list()
    deck = cards
    for _ in range(5): hand.append(deck.pop(-1))
    return (hand, deck, list(), list())

def actionPhase(hand, deck, discard, play, totalDeck, p1):
    ''' No actions right now '''
    return hand, deck, discard, play, totalDeck, p1

def buyPhase(hand, deck, discard, play, totalDeck, p1):
    ''' The buy phase for a regular dominion turn'''
    global supplyCards, supplyAmounts
    # Little dance of destroying our hand and creating a new one
    # to play all the treasures
    tmpHand = list()
    while len(hand) > 0:
        card = hand.pop(-1)
        if card.isTreasure():
            p1.changeCoins(card.getCoins())
            play.append(card)
        else: tmpHand.append(card)
    hand = tmpHand
    possibleBuys = list()
    for card in supplyCards:
        if card.getCost() <= p1.getCoins() and supplyAmounts[card.getName()] > 0:
            possibleBuys.append(card.getName())
    possibleBuys.append("none") # gotta add that it may be better to buy nothing
    return hand, deck, discard, play, totalDeck, p1, possibleBuys

def cleanupPhase(hand, deck, discard, play, totalDeck, p1):
    ''' the cleanup phase for a regular dominion turn '''
    global totalDeckLists
    while len(hand) > 0: discard.append(hand.pop(-1))
    while len(play) > 0: discard.append(play.pop(-1))

    while len(hand) < 5:
        if len(deck) < 1:
            random.shuffle(discard)
            deck = discard
            discard = list()
        if len(deck) < 1:
            print("cleanupPhase deck length: {}".format(len(deck)))
            print("discard length: {}".format(len(discard)))
            print("hand length: {}".format(len(hand)))
            print("play length: {}".format(len(play)))
            print("totalDeck: {}".format(totalDeck))
            print(len(totalDeckLists))
            for x in range(len(totalDeckLists)):
                print(totalDeckLists[x])
                input("> ")
        hand.append(deck.pop(-1))
    player_format.resetStats(p1)
    return hand, deck, discard, play, totalDeck, p1


text:
#!/usr/bin/python

"""
    Example to run pypot with an image from NGC 6951
"""

##== Properties for the galaxy
Distdef = 35.0        # Galaxy distance in Mpc
PAdef=138.7          # Position Angle
Incldef=41.5          # Inclination

"""
Import of the required modules
"""
import pypot
from pypot.pypot_main import *
import pylab as p
import numpy as num
import scipy
from scipy.ndimage.interpolation import rotate

dir_input = "/soft/pytorque/examples/data_6951/"

import pytorque
from pytorque.pytorque import discmodel

gal = discmodel(folder="/soft/python/pypot/examples/data_6951/", massimage="r6951nicmos_f160w.fits", gasimage="co21-un-2sigma-m0.fits", Vcfile="rot-co21un-01.tex", verbose=True, plot=True, distance=35.0, PA=138.7, inclination=41.5, cen=(178,198), cengas=(148,123), box=(90,90), boxgas=(90,90), pixelsize=0.025, pixelsizegas=0.1, azoom=5, Nbins=90)
gal.run_torque()

max_stars_repo_name: emsellem/pytorque

max_issues_repo_name: emsellem/pytorque

max_forks_repo_name: emsellem/pytorque

reason: import numpy,import scipy,from scipy


text:
import asyncio
import requests
from tqdm import tqdm
import csv
import numpy as np
import pandas as pd
import os

async def check_exsit(id, pid):
    df = pd.read_csv("./term.csv", delimiter='|')
    df = df.drop(['alias', 'name', 'categories', 'definition', 'source'], 1)
    r_df = df.loc[df['pid'] == pid]
    r_df = r_df.loc[df['id'] == id]
    if len(r_df.category_ids.values):
        print(f'Exsit id -> {id}')
        return r_df.category_ids.values[0]
    return None

async def tree(category):
    headers = {
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,pt;q=0.6",
        "Connection": "keep-alive",
        "Cookie": "JSESSIONID=168B9F4A47D7596ACFAAC82A85074AF1; zcyuid=955",
        "Host": "zcy.ckcest.cn",
        "Referer": "http://zcy.ckcest.cn/tcm/dic/sd",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.3",
        "X-Requested-With": "XMLHttpRequest"
    }
    response = requests.get(f'http://zcy.ckcest.cn/tcm/dic/treeExpand?category={category}', headers=headers)
    if response.status_code == 200:
        content = response.json()
        return [{'id': item['attr']['id'], 'name': item['data']} for item in content['results']]

async def category(name):
    headers = {
        "Accept": "*/*",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,pt;q=0.6",
        "Connection": "keep-alive",
        "Cookie": "JSESSIONID=CE70C134158DA4724549465B8750017E",
        "Host": "zcy.ckcest.cn",
        "Referer": "http://zcy.ckcest.cn/tcm/dic/home",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
        "X-Requested-With": "XMLHttpRequest"
    }
    response = requests.get(f'http://zcy.ckcest.cn/tcm/dic/NodeClass?name={name}', headers=headers)
    if response.status_code == 200:
        return response.text

async def explain(id, alias):
    headers = {
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,pt;q=0.6",
        "Connection": "keep-alive",
        "Cookie": "JSESSIONID=168B9F4A47D7596ACFAAC82A85074AF1; zcyuid=955",
        "Host": "zcy.ckcest.cn",
        "Referer": "http://zcy.ckcest.cn/tcm/dic/sd",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.3",
        "X-Requested-With": "XMLHttpRequest"
    }
    response = requests.get(f'http://zcy.ckcest.cn/tcm/dic/detailedInfoAjax?indexId={id}', headers=headers)
    if response.status_code == 200:
        content = response.json()
        return {
            'id': id,
            'alias': alias,
            'name': content['word']['name'],
            'category_ids': content['word']['category'],
            'categories': await category(content['word']['name']),
            'definition': content['word']['def'],
            'source': content['word']['source']
        }

async def term(id=0, name='', pid=0):
    print(f'Processing: {id} - {name} - {pid}')
    try:
        exsit_state = await check_exsit(id, pid)
        if exsit_state:
            for item in await tree(exsit_state):
                await term(item['id'], item['name'], id)
        else:
            result = await explain(id, name)

            with open('./term.csv', 'a+') as f:
                f.write(
                    f"{pid}|{result['id']}|{result['alias']}|{result['name']}|{result['category_ids']}|{result['categories']}|{result['definition']}|{result['source']}\n")

            print(
                f"{pid}|{result['id']}|{result['alias']}|{result['name']}|{result['category_ids']}|{result['categories']}|{result['definition']}|{result['source']}")

            for item in await tree(result['category_ids']):
                await term(item['id'], item['name'], id)
    except Exception as e:
        with open('./error.log', 'a+') as f:
            f.write(f"id={id} name={name} parent_id={pid}\n")
            f.write(f'{str(e)}\n\n')

        with open('./tmp.log', 'a+') as f:
            f.write(f"{id},{name},{pid}\n")

async def remove_duplicate():
    df = pd.read_csv("./term.csv", delimiter='|')
    df.sort_values("id", inplace=True)
    df.drop_duplicates(subset="id", keep=False, inplace=True)
    df.sort_values("pid", inplace=True)
    df.to_csv('./term-remove-duplicate.csv', encoding='utf-8', index=False, sep='|')
    # df.to_csv('./term-for-neo4j.csv', encoding='utf-8', index=False, quoting=1)

if __name__ == '__main__':
    if os.path.exists(f'./error.log'):
        os.remove(f'./error.log')
    if os.path.exists(f'./tmp.log'):
        os.remove(f'./tmp.log')

    # asyncio.run(remove_duplicate())

    # if not os.path.exists(f'./failed.csv'):
    #     with open('./failed.csv', 'w') as f:
    #         f.write('id,name,parent_id\n')
    # else:
    #     with open('./failed.csv') as f:
    #         for line in csv.DictReader(f, delimiter=','):
    #             asyncio.run(term(id=int(line['id']), name=line['name'], pid=int(line['parent_id'])))
    #     os.remove(f'./failed.csv')

    with open('./categories.csv') as f:
        for line in csv.DictReader(f, delimiter=','):
            if line['enable'] == '1':
                asyncio.run(term(id=int(line['id']), name=line['name']))