text:
from biosimulators_utils.config import get_config
from biosimulators_utils.log.data_model import (
    Status, CombineArchiveLog, SedDocumentLog, TaskLog, ReportLog)
from biosimulators_utils.log.utils import init_sed_document_log
from biosimulators_utils.plot.data_model import PlotFormat
from biosimulators_utils.report.data_model import VariableResults, DataSetResults, ReportResults, ReportFormat
from biosimulators_utils.report.io import ReportReader
from biosimulators_utils.report.warnings import RepeatDataSetLabelsWarning
from biosimulators_utils.sedml import data_model
from biosimulators_utils.sedml import exec
from biosimulators_utils.sedml import io
from biosimulators_utils.sedml.exceptions import SedmlExecutionError
from biosimulators_utils.sedml.warnings import NoTasksWarning, NoOutputsWarning, InconsistentVariableShapesWarning
from lxml import etree
from unittest import mock
import numpy
import numpy.testing
import os
import shutil
import tempfile
import unittest

class ExecTaskCase(unittest.TestCase):
    def setUp(self):
        self.tmp_dir = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.tmp_dir)

    def test_successful(self):
        doc = data_model.SedDocument()

        doc.models.append(data_model.Model(
            id='model1',
            source='model1.xml',
            language='urn:sedml:language:sbml',
        ))
        doc.models.append(data_model.Model(
            id='model2',
            source='https://models.edu/model1.xml',
            language='urn:sedml:language:cellml',
        ))

        doc.simulations.append(data_model.SteadyStateSimulation(
            id='ss_sim',
        ))
        doc.simulations.append(data_model.UniformTimeCourseSimulation(
            id='time_course_sim',
            initial_time=10.,
            output_start_time=20.,
            output_end_time=30.,
            number_of_points=5,
        ))

        doc.tasks.append(data_model.Task(
            id='task_1_ss',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))
        doc.tasks.append(data_model.Task(
            id='task_2_time_course',
            model=doc.models[1],
            simulation=doc.simulations[1],
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_1',
            variables=[
                data_model.Variable(
                    id='data_gen_1_var_1',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_1']/@concentration",
                    task=doc.tasks[0],
                ),
            ],
            math='data_gen_1_var_1',
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_2',
            variables=[
                data_model.Variable(
                    id='data_gen_2_var_2',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_2']/@concentration",
                    task=doc.tasks[0],
                ),
            ],
            math='data_gen_2_var_2',
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_3',
            variables=[
                data_model.Variable(
                    id='data_gen_3_var_3',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_3']/@concentration",
                    task=doc.tasks[1],
                ),
            ],
            math='data_gen_3_var_3',
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_4',
            variables=[
                data_model.Variable(
                    id='data_gen_4_var_4',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_4']/@concentration",
                    task=doc.tasks[1],
                ),
            ],
            math='data_gen_4_var_4',
        ))

        doc.outputs.append(data_model.Report(
            id='report_1',
            data_sets=[
                data_model.DataSet(
                    id='dataset_1',
                    label='dataset_1',
                    data_generator=doc.data_generators[0],
                ),
                data_model.DataSet(
                    id='dataset_2',
                    label='dataset_2',
                    data_generator=doc.data_generators[2],
                ),
            ],
        ))

        doc.outputs.append(data_model.Report(
            id='report_2',
            data_sets=[
                data_model.DataSet(
                    id='dataset_3',
                    label='dataset_3',
                    data_generator=doc.data_generators[1],
                ),
                data_model.DataSet(
                    id='dataset_4',
                    label='dataset_4',
                    data_generator=doc.data_generators[3],
                ),
            ],
        ))

        doc.outputs.append(data_model.Report(
            id='report_3',
            data_sets=[
                data_model.DataSet(
                    id='dataset_5',
                    label='dataset_5',
                    data_generator=doc.data_generators[0],
                ),
            ],
        ))

        doc.outputs.append(data_model.Report(
            id='report_4',
            data_sets=[
                data_model.DataSet(
                    id='dataset_6',
                    label='dataset_6',
                    data_generator=doc.data_generators[3],
                ),
                data_model.DataSet(
                    id='dataset_7',
                    label='dataset_7',
                    data_generator=doc.data_generators[3],
                ),
            ],
        ))

        filename = os.path.join(self.tmp_dir, 'test.sedml')
        io.SedmlSimulationWriter().run(doc, filename)

        def execute_task(task, variables, log):
            results = VariableResults()
            if task.id == 'task_1_ss':
                results[doc.data_generators[0].variables[0].id] = numpy.array((1., 2.))
                results[doc.data_generators[1].variables[0].id] = numpy.array((3., 4.))
            else:
                results[doc.data_generators[2].variables[0].id] = numpy.array((5., 6.))
                results[doc.data_generators[3].variables[0].id] = numpy.array((7., 8.))
            return results, log

        working_dir = os.path.dirname(filename)
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir = os.path.join(self.tmp_dir, 'results')
        with mock.patch('requests.get', return_value=mock.Mock(raise_for_status=lambda: None, content=b'')):
            output_results, _ = exec.exec_sed_doc(execute_task, filename, working_dir,
                                                  out_dir, report_formats=[ReportFormat.csv], plot_formats=[])

        expected_output_results = ReportResults({
            doc.outputs[0].id: DataSetResults({
                'dataset_1': numpy.array((1., 2.)),
                'dataset_2': numpy.array((5., 6.)),
            }),
            doc.outputs[1].id: DataSetResults({
                'dataset_3': numpy.array((3., 4.)),
                'dataset_4': numpy.array((7., 8.)),
            }),
            doc.outputs[2].id: DataSetResults({
                'dataset_5': numpy.array((1., 2.)),
            }),
            doc.outputs[3].id: DataSetResults({
                'dataset_6': numpy.array((7., 8.)),
                'dataset_7': numpy.array((7., 8.)),
            }),
        })
        self.assertEqual(sorted(output_results.keys()), sorted(expected_output_results.keys()))
        for report_id, data_set_results in output_results.items():
            self.assertEqual(sorted(output_results[report_id].keys()), sorted(expected_output_results[report_id].keys()))
            for data_set_id in data_set_results.keys():
                numpy.testing.assert_allclose(
                    output_results[report_id][data_set_id],
                    expected_output_results[report_id][data_set_id])

        data_set_results = ReportReader().run(doc.outputs[0], out_dir, doc.outputs[0].id, format=ReportFormat.csv)
        for data_set in doc.outputs[0].data_sets:
            numpy.testing.assert_allclose(
                output_results[doc.outputs[0].id][data_set.id],
                data_set_results[data_set.id])

        data_set_results = ReportReader().run(doc.outputs[1], out_dir, doc.outputs[1].id, format=ReportFormat.csv)
        for data_set in doc.outputs[1].data_sets:
            numpy.testing.assert_allclose(
                output_results[doc.outputs[1].id][data_set.id],
                data_set_results[data_set.id])

        # save in HDF5 format
        doc.models[1].source = doc.models[0].source
        io.SedmlSimulationWriter().run(doc, filename)
        shutil.rmtree(out_dir)
        exec.exec_sed_doc(execute_task, filename, os.path.dirname(filename), out_dir, report_formats=[ReportFormat.h5], plot_formats=[])

        data_set_results = ReportReader().run(doc.outputs[0], out_dir, doc.outputs[0].id, format=ReportFormat.h5)
        for data_set in doc.outputs[0].data_sets:
            numpy.testing.assert_allclose(
                output_results[doc.outputs[0].id][data_set.id],
                data_set_results[data_set.id])

        data_set_results = ReportReader().run(doc.outputs[1], out_dir, doc.outputs[1].id, format=ReportFormat.h5)
        for data_set in doc.outputs[1].data_sets:
            numpy.testing.assert_allclose(
                output_results[doc.outputs[1].id][data_set.id],
                data_set_results[data_set.id])

        # track execution status
        shutil.rmtree(out_dir)
        log = SedDocumentLog(
            tasks={
                'task_1_ss': TaskLog(id='task_1_ss', status=Status.QUEUED),
                'task_2_time_course': TaskLog(id='task_2_time_course', status=Status.QUEUED),
            },
            outputs={
                'report_1': ReportLog(id='report_1', status=Status.QUEUED, data_sets={
                    'dataset_1': Status.QUEUED,
                    'dataset_2': Status.QUEUED,
                }),
                'report_2': ReportLog(id='report_2', status=Status.QUEUED, data_sets={
                    'dataset_3': Status.QUEUED,
                    'dataset_4': Status.QUEUED,
                }),
                'report_3': ReportLog(id='report_3', status=Status.QUEUED, data_sets={
                    'dataset_5': Status.QUEUED,
                }),
                'report_4': ReportLog(id='report_4', status=Status.QUEUED, data_sets={
                    'dataset_6': Status.QUEUED,
                    'dataset_7': Status.QUEUED,
                })
            },
        )
        log.parent = CombineArchiveLog(out_dir=out_dir)
        log.tasks['task_1_ss'].parent = log
        log.tasks['task_2_time_course'].parent = log
        log.outputs['report_1'].parent = log
        log.outputs['report_2'].parent = log
        log.outputs['report_3'].parent = log
        log.outputs['report_4'].parent = log
        exec.exec_sed_doc(execute_task, filename, os.path.dirname(filename), out_dir, report_formats=[ReportFormat.h5], plot_formats=[],
                          log=log)

        expected_log = {
            'location': None,
            'status': None,
            'exception': None,
            'skipReason': None,
            'output': None,
            'duration': None,
            'tasks': [
                {
                    'id': 'task_1_ss',
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.tasks['task_1_ss'].output,
                    'duration': log.tasks['task_1_ss'].duration,
                    'algorithm': None,
                    'simulatorDetails': None,
                },
                {
                    'id': 'task_2_time_course',
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.tasks['task_2_time_course'].output,
                    'duration': log.tasks['task_2_time_course'].duration,
                    'algorithm': None,
                    'simulatorDetails': None,
                },
            ],
            'outputs': [
                {
                    'id': 'report_1',
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.outputs['report_1'].output,
                    'duration': log.outputs['report_1'].duration,
                    'dataSets': [
                        {'id': 'dataset_1', 'status': 'SUCCEEDED'},
                        {'id': 'dataset_2', 'status': 'SUCCEEDED'},
                    ],
                },
                {
                    'id': 'report_2',
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.outputs['report_2'].output,
                    'duration': log.outputs['report_2'].duration,
                    'dataSets': [
                        {'id': 'dataset_3', 'status': 'SUCCEEDED'},
                        {'id': 'dataset_4', 'status': 'SUCCEEDED'},
                    ],
                },
                {
                    'id': 'report_3',
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.outputs['report_3'].output,
                    'duration': log.outputs['report_3'].duration,
                    'dataSets': [
                        {'id': 'dataset_5', 'status': 'SUCCEEDED'},
                    ],
                },
                {
                    'id': 'report_4',
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.outputs['report_4'].output,
                    'duration': log.outputs['report_4'].duration,
                    'dataSets': [
                        {'id': 'dataset_6', 'status': 'SUCCEEDED'},
                        {'id': 'dataset_7', 'status': 'SUCCEEDED'},
                    ],
                },
            ],
        }
        actual = log.to_json()
        actual['tasks'].sort(key=lambda task: task['id'])
        actual['outputs'].sort(key=lambda output: output['id'])
        for output in actual['outputs']:
            output['dataSets'].sort(key=lambda dat_set: dat_set['id'])
        self.assertEqual(actual, expected_log)
        self.assertTrue(os.path.isfile(os.path.join(out_dir, get_config().LOG_PATH)))

    def test_with_model_changes(self):
        doc = data_model.SedDocument()

        model1 = data_model.Model(
            id='model1',
            source='model1.xml',
            language='urn:sedml:language:sbml',
        )
        doc.models.append(model1)

        model2 = data_model.Model(
            id='model2',
            source='model1.xml',
            language='urn:sedml:language:sbml',
            changes=[
                data_model.ModelAttributeChange(
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Z']/@id",
                    new_value="Z2",
                )
            ]
        )
        doc.models.append(model2)

        model3 = data_model.Model(
            id='model3',
            source='#model2',
            language='urn:sedml:language:sbml',
            changes=[
                data_model.ModelAttributeChange(
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Z2']/@initialConcentration",
                    new_value="4.0",
                ),
                data_model.ComputeModelChange(
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Z2']/@initialConcentration",
                    parameters=[
                        data_model.Parameter(
                            id='p',
                            value=3.1,
                        ),
                    ],
                    variables=[
                        data_model.Variable(
                            id='var_Z2',
                            target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Z2']/@initialConcentration",
                        ),
                    ],
                    math='p * var_Z2',
                ),
            ],
        )
        model3.changes[1].variables[0].model = model3
        doc.models.append(model3)

        model1.changes.append(
            data_model.ModelAttributeChange(
                target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='X']/@initialConcentration",
                new_value="2.5",
            )
        ),
        model1.changes.append(
            data_model.ComputeModelChange(
                target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Y']/@initialConcentration",
                parameters=[
                    data_model.Parameter(id='a', value=0.2),
                    data_model.Parameter(id='b', value=2.0),
                ],
                variables=[
                    data_model.Variable(
                        id='y',
                        model=model1,
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='X']/@initialConcentration",
                    ),
                    data_model.Variable(
                        id='z',
                        model=model3,
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Z2']/@initialConcentration",
                    ),
                ],
                math='a * y + b * z',
            ),
        )

        doc.simulations.append(data_model.SteadyStateSimulation(
            id='sim1',
        ))

        doc.tasks.append(data_model.Task(
            id='task1',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_1',
            variables=[
                data_model.Variable(
                    id='data_gen_1_var_1',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='X']/@initialConcentration",
                    task=doc.tasks[0],
                ),
            ],
            math='data_gen_1_var_1',
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_2',
            variables=[
                data_model.Variable(
                    id='data_gen_2_var_2',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Y']/@initialConcentration",
                    task=doc.tasks[0],
                ),
            ],
            math='data_gen_2_var_2',
        ))

        doc.outputs.append(data_model.Report(
            id='report_1',
            data_sets=[
                data_model.DataSet(
                    id='dataset_1',
                    label='dataset_1',
                    data_generator=doc.data_generators[0],
                ),
                data_model.DataSet(
                    id='dataset_2',
                    label='dataset_2',
                    data_generator=doc.data_generators[1],
                ),
            ],
        ))

        filename = os.path.join(self.tmp_dir, 'test.sedml')
        working_dir = os.path.dirname(filename)
        io.SedmlSimulationWriter().run(doc, filename)

        shutil.copyfile(
            os.path.join(os.path.dirname(__file__), '..', 'fixtures', 'sbml-three-species.xml'),
            os.path.join(working_dir, 'model1.xml'))

        def execute_task(task, variables, log):
            et = etree.parse(task.model.source)

            results = VariableResults()
            for variable in variables:
                obj_xpath, _, attr = variable.target.rpartition('/@')
                obj = et.xpath(obj_xpath, namespaces={'sbml': 'http://www.sbml.org/sbml/level3/version2'})[0]
                results[variable.id] = numpy.array((float(obj.get(attr)),))

            return results, log

        out_dir = os.path.join(self.tmp_dir, 'results')

        report_results, _ = exec.exec_sed_doc(execute_task, filename, working_dir, out_dir, apply_xml_model_changes=False)
        numpy.testing.assert_equal(report_results[doc.outputs[0].id][doc.outputs[0].data_sets[0].id], numpy.array((1., )))
        numpy.testing.assert_equal(report_results[doc.outputs[0].id][doc.outputs[0].data_sets[1].id], numpy.array((2., )))

        report_results, _ = exec.exec_sed_doc(execute_task, filename, working_dir, out_dir, apply_xml_model_changes=True)
        numpy.testing.assert_equal(report_results[doc.outputs[0].id][doc.outputs[0].data_sets[0].id], numpy.array((2.5, )))
        expected_value = 0.2 * 2.5 + 2.0 * 3.1 * 4.0
        numpy.testing.assert_equal(report_results[doc.outputs[0].id][doc.outputs[0].data_sets[1].id], numpy.array((expected_value)))

    def test_warnings(self):
        # no tasks
        doc = data_model.SedDocument()

        filename = os.path.join(self.tmp_dir, 'test.sedml')
        io.SedmlSimulationWriter().run(doc, filename)

        def execute_task(task, variables, log):
            return VariableResults(), log

        out_dir = os.path.join(self.tmp_dir, 'results')
        with self.assertWarns(NoTasksWarning):
            exec.exec_sed_doc(execute_task, filename, os.path.dirname(filename), out_dir)

        # no outputs
        doc = data_model.SedDocument()

        doc.models.append(data_model.Model(
            id='model1',
            source='model1.xml',
            language='urn:sedml:language:sbml',
            changes=[
                data_model.ModelAttributeChange(
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='X']/@initialConcentration",
                    new_value="2.0",
                ),
            ],
        ))

        doc.simulations.append(data_model.SteadyStateSimulation(
            id='sim1',
        ))

        doc.tasks.append(data_model.Task(
            id='task1',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))

        doc.tasks.append(data_model.Task(
            id='task2',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_1',
            variables=[
                data_model.Variable(
                    id='data_gen_1_var_1',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='X']/@initialConcentration",
                    task=doc.tasks[0],
                ),
            ],
            math='data_gen_1_var_1',
        ))

        doc.outputs.append(data_model.Report(id='report_1', data_sets=[
            data_model.DataSet(id='data_set_1', label='data_set_1', data_generator=doc.data_generators[0]),
        ]))

        filename = os.path.join(self.tmp_dir, 'test.sedml')
        io.SedmlSimulationWriter().run(doc, filename)

        def execute_task(task, variables, log):
            if task.id == 'task1':
                return VariableResults({'data_gen_1_var_1': numpy.array(1.)}), log
            else:
                return VariableResults(), log

        working_dir = os.path.dirname(filename)
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir = os.path.join(self.tmp_dir, 'results')
        with self.assertWarns(NoOutputsWarning):
            exec.exec_sed_doc(execute_task, filename, working_dir, out_dir)

    def test_errors(self):
        # error: variable not recorded
        doc = data_model.SedDocument()

        doc.models.append(data_model.Model(
            id='model1',
            source='model1.xml',
            language='urn:sedml:language:sbml',
        ))

        doc.simulations.append(data_model.SteadyStateSimulation(
            id='sim1',
        ))

        doc.tasks.append(data_model.Task(
            id='task1',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_1',
            variables=[
                data_model.Variable(
                    id='data_gen_1_var_1',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_1']/@concentration",
                    task=doc.tasks[0],
                ),
            ],
            math='data_gen_1_var_1',
        ))

        doc.outputs.append(data_model.Report(
            id='report_1',
            data_sets=[
                data_model.DataSet(
                    id='dataset_1',
                    label='dataset_1',
                    data_generator=doc.data_generators[0],
                ),
            ],
        ))

        filename = os.path.join(self.tmp_dir, 'test.sedml')
        io.SedmlSimulationWriter().run(doc, filename)

        def execute_task(task, variables, log):
            return VariableResults(), log

        working_dir = os.path.dirname(filename)
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir = os.path.join(self.tmp_dir, 'results')
        with self.assertRaisesRegex(SedmlExecutionError, 'did not generate the following expected variables'):
            exec.exec_sed_doc(execute_task, filename, working_dir, out_dir)

        # error: unsupported type of task
        doc = data_model.SedDocument()
        doc.tasks.append(mock.Mock(
            id='task_1_ss',
        ))
        out_dir = os.path.join(self.tmp_dir, 'results')
        with self.assertRaisesRegex(SedmlExecutionError, 'not supported'):
            exec.exec_sed_doc(execute_task, doc, '.', out_dir)

        # error: unsupported data generators
        doc = data_model.SedDocument()

        doc.models.append(data_model.Model(
            id='model1',
            source='model1.xml',
            language='urn:sedml:language:sbml',
        ))

        doc.simulations.append(data_model.SteadyStateSimulation(
            id='sim1',
        ))

        doc.tasks.append(data_model.Task(
            id='task1',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_1',
            variables=[
                data_model.Variable(
                    id='data_gen_1_var_1',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_1']/@concentration",
                    task=doc.tasks[0],
                ),
                data_model.Variable(
                    id='data_gen_1_var_2',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_1']/@concentration",
                    task=doc.tasks[0],
                ),
            ],
            math='data_gen_1_var_1 * data_gen_1_var_2',
        ))

        doc.outputs.append(data_model.Report(
            id='report_1',
            data_sets=[
                data_model.DataSet(
                    id='dataset_1',
                    label='dataset_1',
                    data_generator=doc.data_generators[0],
                ),
            ],
        ))

        def execute_task(task, variables, log):
            results = VariableResults()
            results[doc.data_generators[0].variables[0].id] = numpy.array((1.,))
            results[doc.data_generators[0].variables[1].id] = numpy.array((1.,))
            return results, log

        working_dir = self.tmp_dir
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir = os.path.join(self.tmp_dir, 'results')
        exec.exec_sed_doc(execute_task, doc, working_dir, out_dir)

        # error: inconsistent math
        doc.data_generators = [
            data_model.DataGenerator(
                id='data_gen_1',
                variables=[
                    data_model.Variable(
                        id='data_gen_1_var_1',
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_1']/@concentration",
                        task=doc.tasks[0],
                    ),
                ],
                math='xx',
            ),
        ]
        doc.outputs = [
            data_model.Report(
                id='report_1',
                data_sets=[
                    data_model.DataSet(
                        id='dataset_1',
                        label='dataset_1',
                        data_generator=doc.data_generators[0],
                    ),
                ]
            )
        ]

        def execute_task(task, variables, log):
            results = VariableResults()
            results[doc.data_generators[0].variables[0].id] = numpy.array((1.,))
            return results, log

        working_dir = self.tmp_dir
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir = os.path.join(self.tmp_dir, 'results')
        with self.assertRaisesRegex(SedmlExecutionError, 'could not be evaluated'):
            exec.exec_sed_doc(execute_task, doc, working_dir, out_dir)

        # error: variables have inconsistent shapes
        doc.data_generators = [
            data_model.DataGenerator(
                id='data_gen_1',
                variables=[
                    data_model.Variable(
                        id='data_gen_1_var_1',
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_1']/@concentration",
                        task=doc.tasks[0],
                    ),
                    data_model.Variable(
                        id='data_gen_1_var_2',
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_2']/@concentration",
                        task=doc.tasks[0],
                    ),
                ],
                math='data_gen_1_var_1 * data_gen_1_var_2',
            ),
        ]

        doc.outputs = [
            data_model.Report(
                id='report_1',
                data_sets=[
                    data_model.DataSet(
                        id='dataset_1',
                        label='dataset_1',
                        data_generator=doc.data_generators[0],
                    ),
                ],
            ),
        ]

        def execute_task(task, variables, log):
            results = VariableResults()
            results[doc.data_generators[0].variables[0].id] = numpy.array((1.,))
            results[doc.data_generators[0].variables[1].id] = numpy.array((1., 2.))
            return results, log

        working_dir = self.tmp_dir
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir = os.path.join(self.tmp_dir, 'results')
        with self.assertWarnsRegex(InconsistentVariableShapesWarning, 'do not have consistent shapes'):
            exec.exec_sed_doc(execute_task, doc, working_dir, out_dir)

        # error: data generators have inconsistent shapes
        doc.data_generators = [
            data_model.DataGenerator(
                id='data_gen_1',
                variables=[
                    data_model.Variable(
                        id='data_gen_1_var_1',
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_1']/@concentration",
                        task=doc.tasks[0],
                    ),
                ],
                math='data_gen_1_var_1',
            ),
            data_model.DataGenerator(
                id='data_gen_2',
                variables=[
                    data_model.Variable(
                        id='data_gen_2_var_2',
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_2']/@concentration",
                        task=doc.tasks[0],
                    ),
                ],
                math='data_gen_2_var_2',
            ),
            data_model.DataGenerator(
                id='data_gen_3',
                variables=[
                    data_model.Variable(
                        id='data_gen_3_var_3',
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_3']/@concentration",
                        task=doc.tasks[0],
                    ),
                ],
                math='data_gen_3_var_3',
            ),
        ]

        doc.outputs = [
            data_model.Report(
                id='report_1',
                data_sets=[
                    data_model.DataSet(
                        id='dataset_1',
                        label='dataset_1',
                        data_generator=doc.data_generators[0],
                    ),
                    data_model.DataSet(
                        id='dataset_2',
                        label='dataset_2',
                        data_generator=doc.data_generators[1],
                    ),
                ],
            ),
        ]

        def execute_task(task, variables, log):
            results = VariableResults()
            results[doc.data_generators[0].variables[0].id] = numpy.array((1.,))
            results[doc.data_generators[1].variables[0].id] = numpy.array((1., 2.))
            results[doc.data_generators[2].variables[0].id] = numpy.array(((1., 2., 3.), (4., 5., 6.), (7., 8., 9.)))
            return results, log

        working_dir = self.tmp_dir
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir = os.path.join(self.tmp_dir, 'results')
        with self.assertWarnsRegex(UserWarning, 'do not have consistent shapes'):
            report_results, _ = exec.exec_sed_doc(execute_task, doc, working_dir, out_dir)
        numpy.testing.assert_equal(report_results[doc.outputs[0].id][doc.outputs[0].data_sets[0].id], numpy.array((1.)))
        numpy.testing.assert_equal(report_results[doc.outputs[0].id][doc.outputs[0].data_sets[1].id], numpy.array((1., 2.)))

        doc.outputs[0].data_sets.append(
            data_model.DataSet(
                id='dataset_3',
                label='dataset_3',
                data_generator=doc.data_generators[2],
            ),
        )

        working_dir = self.tmp_dir
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir = os.path.join(self.tmp_dir, 'results2')
        with self.assertRaisesRegex(SedmlExecutionError, 'Multidimensional reports cannot be exported to CSV'):
            with self.assertWarnsRegex(UserWarning, 'do not have consistent shapes'):
                exec.exec_sed_doc(execute_task, doc, working_dir, out_dir)

        with self.assertWarnsRegex(UserWarning, 'do not have consistent shapes'):
            report_results, _ = exec.exec_sed_doc(execute_task, doc, working_dir, out_dir, report_formats=[ReportFormat.h5])
        numpy.testing.assert_equal(report_results[doc.outputs[0].id][doc.outputs[0].data_sets[0].id],
                                   numpy.array((1.,)))
        numpy.testing.assert_equal(report_results[doc.outputs[0].id][doc.outputs[0].data_sets[1].id],
                                   numpy.array((1., 2.)))
        numpy.testing.assert_equal(report_results[doc.outputs[0].id][doc.outputs[0].data_sets[2].id],
                                   numpy.array(((1., 2., 3.), (4., 5., 6.), (7., 8., 9.))))

        # warning: data set labels are not unique
        doc.data_generators=[
            data_model.DataGenerator(
                id='data_gen_1',
                variables=[
                    data_model.Variable(
                        id='data_gen_1_var_1',
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_1']/@concentration",
                        task=doc.tasks[0],
                    ),
                ],
                math='data_gen_1_var_1',
            ),
            data_model.DataGenerator(
                id='data_gen_2',
                variables=[
                    data_model.Variable(
                        id='data_gen_2_var_2',
                        target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var_1']/@concentration",
                        task=doc.tasks[0],
                    ),
                ],
                math='data_gen_2_var_2',
            ),
        ]

        doc.outputs=[
            data_model.Report(
                id='report_1',
                data_sets=[
                    data_model.DataSet(
                        id='dataset_1',
                        label='dataset_label',
                        data_generator=doc.data_generators[0],
                    ),
                    data_model.DataSet(
                        id='dataset_2',
                        label='dataset_label',
                        data_generator=doc.data_generators[1],
                    ),
                ],
            ),
        ]

        def execute_task(task, variables, log):
            results=VariableResults()
            results[doc.data_generators[0].variables[0].id]=numpy.array((1., 2.))
            results[doc.data_generators[1].variables[0].id]=numpy.array((2., 3.))
            return results, log

        working_dir=self.tmp_dir
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir=os.path.join(self.tmp_dir, 'results')
        with self.assertWarnsRegex(RepeatDataSetLabelsWarning, 'should have unique labels'):
            exec.exec_sed_doc(execute_task, doc, working_dir, out_dir)

        # error: unsupported outputs
        doc.outputs=[
            mock.Mock(id='unsupported')
        ]

        working_dir=self.tmp_dir
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        log=SedDocumentLog(tasks={}, outputs={})
        for task in doc.tasks:
            log.tasks[task.id]=TaskLog(parent=log)
        for output in doc.outputs:
            log.outputs[output.id]=ReportLog(parent=log)
        with self.assertRaisesRegex(SedmlExecutionError, 'are not supported'):
            exec.exec_sed_doc(execute_task, doc, working_dir, out_dir, log=log)

    def test_2d_plot(self):
        doc=data_model.SedDocument()

        doc.models.append(data_model.Model(
            id='model',
            source='model1.xml',
            language='urn:sedml:language:sbml',
        ))

        doc.simulations.append(data_model.UniformTimeCourseSimulation(
            id='sim',
            initial_time=0.,
            output_start_time=10.,
            output_end_time=10.,
            number_of_points=10,
        ))

        doc.tasks.append(data_model.Task(
            id='task1',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))

        doc.tasks.append(data_model.Task(
            id='task2',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_time',
            variables=[
                data_model.Variable(
                    id='time',
                    symbol=data_model.Symbol.time,
                    task=doc.tasks[0],
                ),
            ],
            math='time',
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_var',
            variables=[
                data_model.Variable(
                    id='var',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var']/@concentration",
                    task=doc.tasks[1],
                ),
            ],
            math='var',
        ))

        doc.outputs.append(data_model.Plot2D(
            id='plot_2d_1',
            curves=[
                data_model.Curve(
                    id='curve1',
                    x_data_generator=doc.data_generators[0],
                    y_data_generator=doc.data_generators[0],
                    x_scale=data_model.AxisScale.linear,
                    y_scale=data_model.AxisScale.linear,
                ),
                data_model.Curve(
                    id='curve2',
                    x_data_generator=doc.data_generators[1],
                    y_data_generator=doc.data_generators[1],
                    x_scale=data_model.AxisScale.linear,
                    y_scale=data_model.AxisScale.linear,
                ),
            ],
        ))

        doc.outputs.append(data_model.Plot2D(
            id='plot_2d_2',
            curves=[
                data_model.Curve(
                    id='curve3',
                    x_data_generator=doc.data_generators[1],
                    y_data_generator=doc.data_generators[1],
                    x_scale=data_model.AxisScale.linear,
                    y_scale=data_model.AxisScale.linear,
                ),
            ],
        ))

        filename=os.path.join(self.tmp_dir, 'test.sedml')
        io.SedmlSimulationWriter().run(doc, filename)

        def execute_task(task, variables, log=None):
            results=VariableResults()
            results[doc.data_generators[0].variables[0].id]=numpy.linspace(0., 10., 10 + 1)
            results[doc.data_generators[1].variables[0].id]=2 * results[doc.data_generators[0].variables[0].id]
            return results, log

        working_dir=os.path.dirname(filename)
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir=os.path.join(self.tmp_dir, 'results')
        _, log=exec.exec_sed_doc(execute_task, filename, working_dir,
                                   out_dir, plot_formats=[PlotFormat.pdf])

        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_2d_1.pdf')))
        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_2d_2.pdf')))

        self.assertEqual(
            log.to_json()['outputs'],
            [
                {
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.to_json()['outputs'][0]['output'],
                    'duration': log.to_json()['outputs'][0]['duration'],
                    'id': 'plot_2d_1',
                    'curves': [
                        {'id': 'curve1', 'status': 'SUCCEEDED'},
                        {'id': 'curve2', 'status': 'SUCCEEDED'},
                    ],
                },
                {
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.to_json()['outputs'][1]['output'],
                    'duration': log.to_json()['outputs'][1]['duration'],
                    'id': 'plot_2d_2',
                    'curves': [
                        {'id': 'curve3', 'status': 'SUCCEEDED'},
                    ],
                },
            ]
        )

        os.remove(os.path.join(out_dir, 'plot_2d_1.pdf'))
        os.remove(os.path.join(out_dir, 'plot_2d_2.pdf'))

        # error with a curve
        doc.data_generators[0].math='time * var'
        io.SedmlSimulationWriter().run(doc, filename)
        log=init_sed_document_log(doc)
        with self.assertRaisesRegex(SedmlExecutionError, "name 'var' is not defined"):
            exec.exec_sed_doc(execute_task, filename, working_dir,
                              out_dir, log=log, plot_formats=[PlotFormat.pdf])

        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_2d_1.pdf')))
        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_2d_2.pdf')))

        self.assertEqual(
            log.to_json()['outputs'],
            [
                {
                    'status': 'FAILED',
                    'exception': log.to_json()['outputs'][0]['exception'],
                    'skipReason': None,
                    'output': log.to_json()['outputs'][0]['output'],
                    'duration': log.to_json()['outputs'][0]['duration'],
                    'id': 'plot_2d_1',
                    'curves': [
                        {'id': 'curve1', 'status': 'FAILED'},
                        {'id': 'curve2', 'status': 'SUCCEEDED'},
                    ],
                },
                {
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.to_json()['outputs'][1]['output'],
                    'duration': log.to_json()['outputs'][1]['duration'],
                    'id': 'plot_2d_2',
                    'curves': [
                        {'id': 'curve3', 'status': 'SUCCEEDED'},
                    ],
                },
            ]
        )

        # error with a task
        def execute_task(task, variables, log=None):
            results=VariableResults()
            results[doc.data_generators[0].variables[0].id]=None
            results[doc.data_generators[1].variables[0].id]=2 * numpy.linspace(0., 10., 10 + 1)
            return results, log

        doc.data_generators[0].math='time'
        io.SedmlSimulationWriter().run(doc, filename)
        log=init_sed_document_log(doc)
        with self.assertRaisesRegex(SedmlExecutionError, "Some generators could not be produced:"):
            exec.exec_sed_doc(execute_task, filename, working_dir,
                              out_dir, log=log, plot_formats=[PlotFormat.pdf])

        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_2d_1.pdf')))
        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_2d_2.pdf')))

        self.assertEqual(
            log.to_json()['outputs'],
            [
                {
                    'status': 'FAILED',
                    'exception': log.to_json()['outputs'][0]['exception'],
                    'skipReason': None,
                    'output': log.to_json()['outputs'][0]['output'],
                    'duration': log.to_json()['outputs'][0]['duration'],
                    'id': 'plot_2d_1',
                    'curves': [
                        {'id': 'curve1', 'status': 'FAILED'},
                        {'id': 'curve2', 'status': 'SUCCEEDED'},
                    ],
                },
                {
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.to_json()['outputs'][1]['output'],
                    'duration': log.to_json()['outputs'][1]['duration'],
                    'id': 'plot_2d_2',
                    'curves': [
                        {'id': 'curve3', 'status': 'SUCCEEDED'},
                    ],
                },
            ]
        )

    def test_3d_plot(self):
        doc=data_model.SedDocument()

        doc.models.append(data_model.Model(
            id='model',
            source='model1.xml',
            language='urn:sedml:language:sbml',
        ))

        doc.simulations.append(data_model.UniformTimeCourseSimulation(
            id='sim',
            initial_time=0.,
            output_start_time=10.,
            output_end_time=10.,
            number_of_points=10,
        ))

        doc.tasks.append(data_model.Task(
            id='task1',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))

        doc.tasks.append(data_model.Task(
            id='task2',
            model=doc.models[0],
            simulation=doc.simulations[0],
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_time',
            variables=[
                data_model.Variable(
                    id='time',
                    symbol=data_model.Symbol.time,
                    task=doc.tasks[0],
                ),
            ],
            math='time',
        ))

        doc.data_generators.append(data_model.DataGenerator(
            id='data_gen_var',
            variables=[
                data_model.Variable(
                    id='var',
                    target="/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:speces[@id='var']/@concentration",
                    task=doc.tasks[1],
                ),
            ],
            math='var',
        ))

        doc.outputs.append(data_model.Plot3D(
            id='plot_3d_1',
            surfaces=[
                data_model.Surface(
                    id='surface1',
                    x_data_generator=doc.data_generators[0],
                    y_data_generator=doc.data_generators[0],
                    z_data_generator=doc.data_generators[0],
                    x_scale=data_model.AxisScale.linear,
                    y_scale=data_model.AxisScale.linear,
                    z_scale=data_model.AxisScale.linear,
                ),
                data_model.Surface(
                    id='surface2',
                    x_data_generator=doc.data_generators[1],
                    y_data_generator=doc.data_generators[1],
                    z_data_generator=doc.data_generators[1],
                    x_scale=data_model.AxisScale.linear,
                    y_scale=data_model.AxisScale.linear,
                    z_scale=data_model.AxisScale.linear,
                ),
            ],
        ))

        doc.outputs.append(data_model.Plot3D(
            id='plot_3d_2',
            surfaces=[
                data_model.Surface(
                    id='surface3',
                    x_data_generator=doc.data_generators[1],
                    y_data_generator=doc.data_generators[1],
                    z_data_generator=doc.data_generators[1],
                    x_scale=data_model.AxisScale.linear,
                    y_scale=data_model.AxisScale.linear,
                    z_scale=data_model.AxisScale.linear,
                ),
            ],
        ))

        filename=os.path.join(self.tmp_dir, 'test.sedml')
        io.SedmlSimulationWriter().run(doc, filename)

        def execute_task(task, variables, log=None):
            results=VariableResults()
            x=numpy.arange(-5, 5, 0.25)
            x, _=numpy.meshgrid(x, x)
            results[doc.data_generators[0].variables[0].id]=x
            results[doc.data_generators[1].variables[0].id]=x
            return results, log

        working_dir=os.path.dirname(filename)
        with open(os.path.join(working_dir, doc.models[0].source), 'w'):
            pass

        out_dir=os.path.join(self.tmp_dir, 'results')
        _, log=exec.exec_sed_doc(execute_task, filename, working_dir,
                                   out_dir, plot_formats=[PlotFormat.pdf])

        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_3d_1.pdf')))
        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_3d_2.pdf')))

        self.assertEqual(
            log.to_json()['outputs'],
            [
                {
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.to_json()['outputs'][0]['output'],
                    'duration': log.to_json()['outputs'][0]['duration'],
                    'id': 'plot_3d_1',
                    'surfaces': [
                        {'id': 'surface1', 'status': 'SUCCEEDED'},
                        {'id': 'surface2', 'status': 'SUCCEEDED'},
                    ],
                },
                {
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.to_json()['outputs'][1]['output'],
                    'duration': log.to_json()['outputs'][1]['duration'],
                    'id': 'plot_3d_2',
                    'surfaces': [
                        {'id': 'surface3', 'status': 'SUCCEEDED'},
                    ],
                },
            ]
        )

        os.remove(os.path.join(out_dir, 'plot_3d_1.pdf'))
        os.remove(os.path.join(out_dir, 'plot_3d_2.pdf'))

        # error with a surface
        doc.data_generators[0].math='time * var'
        io.SedmlSimulationWriter().run(doc, filename)
        log=init_sed_document_log(doc)
        with self.assertRaisesRegex(SedmlExecutionError, "name 'var' is not defined"):
            exec.exec_sed_doc(execute_task, filename, working_dir,
                              out_dir, log=log, plot_formats=[PlotFormat.pdf])

        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_3d_1.pdf')))
        self.assertTrue(os.path.isfile(os.path.join(out_dir, 'plot_3d_2.pdf')))

        self.assertEqual(
            log.to_json()['outputs'],
            [
                {
                    'status': 'FAILED',
                    'exception': log.to_json()['outputs'][0]['exception'],
                    'skipReason': None,
                    'output': log.to_json()['outputs'][0]['output'],
                    'duration': log.to_json()['outputs'][0]['duration'],
                    'id': 'plot_3d_1',
                    'surfaces': [
                        {'id': 'surface1', 'status': 'FAILED'},
                        {'id': 'surface2', 'status': 'SUCCEEDED'},
                    ],
                },
                {
                    'status': 'SUCCEEDED',
                    'exception': None,
                    'skipReason': None,
                    'output': log.to_json()['outputs'][1]['output'],
                    'duration': log.to_json()['outputs'][1]['duration'],
                    'id': 'plot_3d_2',
                    'surfaces': [
                        {'id': 'surface3', 'status': 'SUCCEEDED'},
                    ],
                },
            ]
        )


text:
import numpy as np
import matplotlib.pyplot as plt
import main.utils as utils
import tools.visualize as vis
import main.MCN as MCN
import main.SMCN as SMCN

# ---------------------------- 说明 ----------------------------------
# generate some result( especially simularity matrix )
# (*) means you should specify according to your case
# omni-SCUT descriptor file *.npy are needed
# ---------------------------- 说明 ----------------------------------

D1 = np.load('./netvlad/day_desc.npy')    #(*)
D2 = np.load('./netvlad/dawn_desc.npy')   #(*)

D1 = D1 / np.linalg.norm(D1, axis=0)
D2 = D2 / np.linalg.norm(D2, axis=0)

num = D1.shape[0]
omni_D1 = D1.reshape((num // 4, -1))
omni_D2 = D2.reshape((num // 4, -1))

GT = utils.getGroundTruthMatrix(num // 4, 1)

sig_D1 = D1[::4, :]
sig_D2 = D2[::4, :]

del D1, D2

id_dict = {}

# single view
num, old_dims = sig_D1.shape
new_dims = 8192     # acturally 8192 * 2
s = 0.25
P = np.random.rand(old_dims, new_dims)
P /= np.linalg.norm(P, axis=1, keepdims=True)
sig_D1_slsbh = utils.getLSBH(sig_D1, P, 0.25)
sig_D2_slsbh = utils.getLSBH(sig_D2, P, 0.25)
del P
nOnes = 2 * s * new_dims
S_slbsh = np.matmul(sig_D1_slsbh, sig_D2_slsbh.T, dtype=np.float) / nOnes
id_dict['sig_f_id'] = np.argmax(S_slbsh, axis=0)
del S_slbsh, sig_D1_slsbh, sig_D2_slsbh

# onmi full
old_dims = omni_D1.shape[1]
new_dims = 8192
s = 0.25
P = np.random.rand(old_dims, new_dims)
P /= np.linalg.norm(P, axis=1, keepdims=True)
omni_D1_slsbh = utils.getLSBH(omni_D1, P, s)
omni_D2_slsbh = utils.getLSBH(omni_D2, P, s)
del P
nOnes = 2 * s * new_dims
S = np.matmul(omni_D1_slsbh, omni_D2_slsbh.T, dtype=np.float) / nOnes
id_dict['onmi_f_id'] = np.argmax(S, axis=0)
del S, omni_D1_slsbh, omni_D2_slsbh

# 1% tiny single
_, old_dims = sig_D1.shape
new_dims = int(8192 * 0.01)     # acturally 8192 * 2
s = 0.25
P = np.random.rand(old_dims, new_dims)
P /= np.linalg.norm(P, axis=1, keepdims=True)
sig_D1_slsbh = utils.getLSBH(sig_D1, P, 0.25)
sig_D2_slsbh = utils.getLSBH(sig_D2, P, 0.25)
del P
nOnes = 2 * s * new_dims
S_slbsh = np.matmul(sig_D1_slsbh, sig_D2_slsbh.T, dtype=np.float) / nOnes
id_dict['sig_t_id'] = np.argmax(S_slbsh, axis=0)
del S_slbsh, sig_D1_slsbh, sig_D2_slsbh

# 1% tiny omni
old_dims = omni_D1.shape[1]
new_dims = int(8192 * 0.01)
s = 0.25
P = np.random.rand(old_dims, new_dims)
P /= np.linalg.norm(P, axis=1, keepdims=True)
tiny_omni_D1_slsbh = utils.getLSBH(omni_D1, P, s)
tiny_omni_D2_slsbh = utils.getLSBH(omni_D2, P, s)
del P
nOnes = 2 * s * new_dims
S_pw = np.matmul(tiny_omni_D1_slsbh, tiny_omni_D2_slsbh.T, dtype=np.float) / nOnes
S_in = np.matmul(tiny_omni_D1_slsbh, tiny_omni_D2_slsbh.T, dtype=np.float) / nOnes
id_dict['onmi_t_id'] = np.argmax(S_pw, axis=0)

# 1% tiny omni MCN
params = MCN.MCNParams(probAddCon=0.05,     #(*)
                       nCellPerCol=32,
                       nConPerCol=200,
                       minColActivity=0.65,
                       nColPerPattern=50,
                       kActiveCol=100)

S = MCN.runMCN_SDR(params, tiny_omni_D1_slsbh, tiny_omni_D2_slsbh, GT)
id_dict['onmi_t_mcn_id'] = np.argmax(S, axis=0)

# 1% tiny omni SMCN
k1 = 1
k2 = 3
cannum = 2
can = SMCN.getCandidate(S_in, S_pw, *(cannum, k1, k2))
S_omni_smcncan = np.zeros_like(S_pw)
for i in range(can.shape[1]):
    tmp = np.where(can[:, i] >= 0)[0]
    if len(tmp):
        S_omni_smcncan[can[tmp, i], i] = S_pw[can[tmp, i], i]

id_dict['onmi_t_smcn_id'] = np.argmax(S_omni_smcncan, axis=0)

# 1% tiny omni SMCNTF
_, p = SMCN.refreshLoss(S_in, S_pw, can, 1)
id_dict['onmi_t_smcntf_id'] = p

ext: py

reason: import numpy


text:
# -*- coding: utf-8 -*-
#
# Creating Sequence to Sequence Models
#-------------------------------------
#  Here we show how to implement sequence to sequence models.
#  Specifically, we will build an English to German translation model.
#

import os
import re
import string
import requests
import io
import numpy as np
import collections
import random
import pickle
import string
import matplotlib.pyplot as plt
import tensorflow as tf
from zipfile import ZipFile
from collections import Counter
from tensorflow.models.rnn.translate import data_utils
from tensorflow.models.rnn.translate import seq2seq_model
from tensorflow.python.framework import ops
ops.reset_default_graph()

# Start a session
sess = tf.Session()

# Model Parameters
learning_rate = 0.1
lr_decay_rate = 0.99
lr_decay_every = 100
max_gradient = 5.0
batch_size = 50
num_layers = 3
rnn_size = 500
layer_size = 512
generations = 10000
vocab_size = 10000
save_every = 1000
eval_every = 500
output_every = 50
punct = string.punctuation

# Data Parameters
data_dir = 'temp'
data_file = 'eng_ger.txt'
model_path = 'seq2seq_model'
full_model_dir = os.path.join(data_dir, model_path)

# Test Translation from English (lowercase, no punct)
test_english = ['hello where is my computer',
                'the quick brown fox jumped over the lazy dog',
                'is it going to rain tomorrow']

# Make Model Directory
if not os.path.exists(full_model_dir):
    os.makedirs(full_model_dir)

# Make data directory
if not os.path.exists(data_dir):
    os.makedirs(data_dir)

print('Loading English-German Data')
# Check for data, if it doesn't exist, download it and save it
if not os.path.isfile(os.path.join(data_dir, data_file)):
    print('Data not found, downloading Eng-Ger sentences from www.manythings.org')
    sentence_url = 'http://www.manythings.org/anki/deu-eng.zip'
    r = requests.get(sentence_url)
    z = ZipFile(io.BytesIO(r.content))
    file = z.read('deu.txt')
    # Format Data
    eng_ger_data = file.decode()
    eng_ger_data = eng_ger_data.encode('ascii',errors='ignore')
    eng_ger_data = eng_ger_data.decode().split('\n')
    # Write to file
    with open(os.path.join(data_dir, data_file), 'w') as out_conn:
        for sentence in eng_ger_data:
            out_conn.write(sentence + '\n')
else:
    eng_ger_data = []
    with open(os.path.join(data_dir, data_file), 'r') as in_conn:
        for row in in_conn:
            eng_ger_data.append(row[:-1])

# Remove punctuation
eng_ger_data = [''.join(char for char in sent if char not in punct) for sent in eng_ger_data]
# Split each sentence by tabs
eng_ger_data = [x.split('\t') for x in eng_ger_data if len(x)>=1]
[english_sentence, german_sentence] = [list(x) for x in zip(*eng_ger_data)]
english_sentence = [x.lower().split() for x in english_sentence]
german_sentence = [x.lower().split() for x in german_sentence]

print('Processing the vocabularies.')
# Process the English Vocabulary
all_english_words = [word for sentence in english_sentence for word in sentence]
all_english_counts = Counter(all_english_words)
eng_word_keys = [x[0] for x in all_english_counts.most_common(vocab_size-1)] #-1 because 0=unknown is also in there
eng_vocab2ix = dict(zip(eng_word_keys, range(1,vocab_size)))
eng_ix2vocab = {val:key for key, val in eng_vocab2ix.items()}
english_processed = []
for sent in english_sentence:
    temp_sentence = []
    for word in sent:
        try:
            temp_sentence.append(eng_vocab2ix[word])
        except:
            temp_sentence.append(0)
    english_processed.append(temp_sentence)

# Process the German Vocabulary
all_german_words = [word for sentence in german_sentence for word in sentence]
all_german_counts = Counter(all_german_words)
ger_word_keys = [x[0] for x in all_german_counts.most_common(vocab_size-1)]
ger_vocab2ix = dict(zip(ger_word_keys, range(1,vocab_size)))
ger_ix2vocab = {val:key for key, val in ger_vocab2ix.items()}
german_processed = []
for sent in german_sentence:
    temp_sentence = []
    for word in sent:
        try:
            temp_sentence.append(ger_vocab2ix[word])
        except:
            temp_sentence.append(0)
    german_processed.append(temp_sentence)

# Process the test english sentences, use '0' if word not in our vocab
test_data = []
for sentence in test_english:
    temp_sentence = []
    for word in sentence.split(' '):
        try:
            temp_sentence.append(eng_vocab2ix[word])
        except:
            # Use '0' if the word isn't in our vocabulary
            temp_sentence.append(0)
    test_data.append(temp_sentence)

# Define Buckets for sequence lengths
# We will split data into the corresponding buckets:
# (x1, y1), (x2, y2), ...
# Where all entries in bucket 1: len(x)<x1 and len(y)<y1 and so on.
x_maxs = [5, 7, 11, 50]
y_maxs = [10, 12, 17, 60]
buckets = [x for x in zip(x_maxs, y_maxs)]
bucketed_data = [[] for _ in range(len(x_maxs))]
for eng, ger in zip(english_processed, german_processed):
    for ix, (x_max, y_max) in enumerate(zip(x_maxs, y_maxs)):
        if (len(eng) <= x_max) and (len(ger) <= y_max):
            bucketed_data[ix].append([eng, ger])
            break

# Print summaries of buckets
train_bucket_sizes = [len(bucketed_data[b]) for b in range(len(buckets))]
train_total_size = float(sum(train_bucket_sizes))
for ix, bucket in enumerate(bucketed_data):
    print('Data pts in bucket {}: {}'.format(ix, len(bucket)))

# Create sequence to sequence model
def translation_model(sess, input_vocab_size, output_vocab_size,
                      buckets, rnn_size, num_layers, max_gradient,
                      learning_rate, lr_decay_rate, forward_only):
    model = seq2seq_model.Seq2SeqModel(
          input_vocab_size,
          output_vocab_size,
          buckets,
          rnn_size,
          num_layers,
          max_gradient,
          batch_size,
          learning_rate,
          lr_decay_rate,
          forward_only=forward_only,
          dtype=tf.float32)
    return(model)

print('Creating Translation Model')
input_vocab_size = vocab_size
output_vocab_size = vocab_size
with tf.variable_scope('translate_model') as scope:
    translate_model = translation_model(sess, vocab_size, vocab_size,
                                        buckets, rnn_size, num_layers,
                                        max_gradient, learning_rate,
                                        lr_decay_rate, False)
    #Reuse the variables for the test model
    scope.reuse_variables()
    test_model = translation_model(sess, vocab_size, vocab_size,
                                   buckets, rnn_size, num_layers,
                                   max_gradient, learning_rate,
                                   lr_decay_rate, True)
    test_model.batch_size = 1

# Initialize all model variables
init = tf.global_variables_initializer()
sess.run(init)

# Start training
train_loss = []
for i in range(generations):
    rand_bucket_ix = np.random.choice(len(bucketed_data))

    model_outputs = translate_model.get_batch(bucketed_data, rand_bucket_ix)
    encoder_inputs, decoder_inputs, target_weights = model_outputs

    # Get the (gradient norm, loss, and outputs)
    _, step_loss, _ = translate_model.step(sess, encoder_inputs, decoder_inputs,
                                           target_weights, rand_bucket_ix, False)

    # Output status
    if (i+1) % output_every == 0:
        train_loss.append(step_loss)
        print('Gen #{} out of {}. Loss: {:.4}'.format(i+1, generations, step_loss))

    # Check if we should decay the learning rate
    if (i+1) % lr_decay_every == 0:
        sess.run(translate_model.learning_rate_decay_op)

    # Save model
    if (i+1) % save_every == 0:
        print('Saving model to {}.'.format(full_model_dir))
        model_save_path = os.path.join(full_model_dir, "eng_ger_translation.ckpt")
        translate_model.saver.save(sess, model_save_path, global_step=i)

    # Eval on test set
    if (i+1) % eval_every == 0:
        for ix, sentence in enumerate(test_data):
            # Find which bucket sentence goes in
            bucket_id = next(index for index, val in enumerate(x_maxs) if val>=len(sentence))
            # Get RNN model outputs
            encoder_inputs, decoder_inputs, target_weights = test_model.get_batch(
                {bucket_id: [(sentence, [])]}, bucket_id)
            # Get logits
            _, test_loss, output_logits = test_model.step(sess, encoder_inputs, decoder_inputs,
                                                           target_weights, bucket_id, True)
            ix_output = [int(np.argmax(logit, axis=1)) for logit in output_logits]
            # If there is a 0 symbol in outputs end the output there.
            ix_output = ix_output[0:[ix for ix, x in enumerate(ix_output+[0]) if x==0][0]]
            # Get german words from indices
            test_german = [ger_ix2vocab[x] for x in ix_output]
            print('English: {}'.format(test_english[ix]))
            print('German: {}'.format(test_german))

# Plot train loss
loss_generations = [i for i in range(generations) if i%output_every==0]
plt.plot(loss_generations, train_loss, 'k-')
plt.title('Sequence to Sequence Loss')
plt.xlabel('Generation')
plt.ylabel('Loss')
plt.show()

max_stars_repo_path: Chapter 09/seq2seq_translation.py

max_issues_repo_path: Chapter 09/seq2seq_translation.py

max_forks_repo_path: Chapter 09/seq2seq_translation.py


text:
import numpy as np
import pandas as pd
import time
import copy
import torch
import torch.nn as nn
import torch.nn.functional as F
from numpy import linalg as LA
import scipy
from scipy import io
import sklearn
from sklearn import preprocessing
import pdb
import random
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from rotateImage import Rotate

class cifarLoader(object):
    def __init__(self, labeldSamples, unlabeldSamples, testingBatchSize = 1000, nEachClassSamples = None):
        # np.random.seed(0)
        # torch.manual_seed(0)
        # random.seed(0)
        self.batchSize  = testingBatchSize
        self.loadData(labeldSamples, unlabeldSamples, nEachClassSamples)

    def loadData(self, labeldSamples, unlabeldSamples, nEachClassSamples):

#         pdb.set_trace()
        # labeled data
        labeledData  = torch.Tensor(labeldSamples.data.shape[0],1,32,32)
        labeledLabel = torch.LongTensor(labeldSamples.data.shape[0])

        for idx, example in enumerate(labeldSamples):
#             pdb.set_trace()
            labeledData[idx]  = example[0][2:3] #torch.unsqueeze(torch.mean(example[0],0), 0) # example[0][2:3]
            labeledLabel[idx] = example[1]

        self.labeledData  = labeledData
        self.labeledLabel = labeledLabel

        self.classes = torch.unique(self.labeledLabel).tolist()
        self.nOutput = torch.unique(self.labeledLabel).shape[0]
        self.nLabeledData = self.labeledData.shape[0]

        if nEachClassSamples is not None:
            selectedLabeledData  = torch.Tensor(self.nOutput*nEachClassSamples,1,32,32)
            selectedLabeledLabel = torch.LongTensor(self.nOutput*nEachClassSamples)

            idx = 0
            selectedDataIdx = []
            for iClass in self.classes:
                # print(iClass)
                dataCount = 0

                for iData in range(0,self.nLabeledData):
                    # print(iData)
                    if labeledLabel[iData] == iClass:
                        selectedLabeledData[idx]  = self.labeledData[iData]
                        selectedLabeledLabel[idx] = self.labeledLabel[iData]
                        # print(labeledLabel[iData])
                        idx += 1
                        dataCount += 1

                        selectedDataIdx.append(iData)
                        iData += 1

                    if dataCount == nEachClassSamples:
                        break

            remainderData     = deleteRowTensor(self.labeledData, selectedDataIdx, 2)
            remainderLabel    = deleteRowTensor(self.labeledLabel, selectedDataIdx, 2)

            self.nLabeledData = selectedLabeledData.shape[0]

            # shuffle
            indices = torch.randperm(self.nLabeledData)
            self.labeledData  = selectedLabeledData[indices]
            self.labeledLabel = selectedLabeledLabel[indices]

        # unlabeled data
        unlabeledData  = torch.Tensor(unlabeldSamples.data.shape[0],1,32,32)
        unlabeledLabel = torch.LongTensor(unlabeldSamples.data.shape[0])

        for idx, example in enumerate(unlabeldSamples):
            unlabeledData[idx]  = example[0][2:3] #torch.unsqueeze(torch.mean(example[0],0), 0) # example[0][2:3]
            unlabeledLabel[idx] = example[1]

        if nEachClassSamples is not None:
            unlabeledData   = torch.cat((unlabeledData,remainderData),0)
            unlabeledLabel  = torch.cat((unlabeledLabel,remainderLabel),0)

        self.unlabeledData  = unlabeledData
        self.unlabeledLabel = unlabeledLabel

        self.nUnlabeledData = self.unlabeledData.shape[0]
        self.nBatch         = int(self.nUnlabeledData/self.batchSize)
        self.taskIndicator  = (torch.zeros(self.nBatch).long()).tolist()

        print('Number of output: ', self.nOutput)
        print('Number of labeled data: ', self.nLabeledData)
        print('Number of unlabeled data: ', self.nUnlabeledData)
        print('Number of unlabeled data batch: ', self.nBatch)

    def createTask(self, nTask = 2, taskList = [], taskType = 1):
        # random seed control
        # random.seed(0)
        if (taskType == 2 or taskType == 3) and (len(taskList) != nTask):
            raise NameError('list of rotaion angle should be the same with the number of task')

        self.taskIndicator        = []

        # clone labeled data
        transformedLabeledData    = self.labeledData.clone()
        transformedLabeledLabel   = self.labeledLabel.clone()
        finalLabeledData          = {}
        finalLabeledLabel         = {}

        # clone unlabeled data
        transformedUnlabeledData  = self.unlabeledData.clone()
        transformedUnlabeledLabel = self.unlabeledLabel.clone()
        finalUnlabeledData        = {}
        finalUnlabeledLabel       = {}

        # testing data
        unlabeledDataTest        = {}
        unlabeledLabelTest       = {}

        # number of data and batch for each task
        self.nTask                  = nTask
        self.nLabeledDataPerTask    = int(self.nLabeledData/nTask)
        self.nBatchPerTask          = int(self.nBatch/nTask)
        self.nBatch                 = self.nBatchPerTask*nTask
        self.nUnlabeledDataPerTask  = self.nBatchPerTask*self.batchSize

        nUnlabeledDataTest = 0

        for iTask in range(0,nTask):
            # load data
            # iTask = iTask + 1

            # load labeled data
            taskLabeledData    = transformedLabeledData[(iTask)*self.nLabeledDataPerTask:(iTask+1)*self.nLabeledDataPerTask]
            taskLabeledLabel   = transformedLabeledLabel[(iTask)*self.nLabeledDataPerTask:(iTask+1)*self.nLabeledDataPerTask]

            # load unlabeled data
            taskUnlabeledData  = transformedUnlabeledData[(iTask)*self.nUnlabeledDataPerTask:(iTask+1)*self.nUnlabeledDataPerTask]
            taskUnlabeledLabel = transformedUnlabeledLabel[(iTask)*self.nUnlabeledDataPerTask:(iTask+1)*self.nUnlabeledDataPerTask]

            if self.nUnlabeledDataPerTask != taskUnlabeledData.shape[0] or self.nLabeledDataPerTask != taskLabeledData.shape[0]:
                raise NameError('Mismatch data size')

            # permute
            if taskType == 1:
                # reshape image
                taskLabeledData   = taskLabeledData.view(taskLabeledData.size(0),-1)
                taskUnlabeledData = taskUnlabeledData.view(taskUnlabeledData.size(0),-1)

                # create permutation
                col_idxs = list(range(taskLabeledData.shape[1]))
                random.shuffle(col_idxs)

                # permute labeled data
                taskLabeledData   = taskLabeledData[:, torch.tensor(col_idxs)]

                # permute unlabeled data
                taskUnlabeledData = taskUnlabeledData[:, torch.tensor(col_idxs)]

                # reshape back image
                taskLabeledData   = taskLabeledData.reshape(taskLabeledData.size(0),1,32,32)
                taskUnlabeledData = taskUnlabeledData.reshape(taskUnlabeledData.size(0),1,32,32)

            elif taskType == 2:
                # rotate labeled data
                for idx, _ in enumerate(taskLabeledData):
                    taskLabeledData[idx] = Rotate(random.randint(taskList[iTask][0], taskList[iTask][1]))(taskLabeledData[idx])

                # rotate unlabeled data
                for idx, _ in enumerate(taskUnlabeledData):
                    taskUnlabeledData[idx] = Rotate(random.randint(taskList[iTask][0], taskList[iTask][1]))(taskUnlabeledData[idx])

            elif taskType == 3:
                # split MNIST, into 5 tasks
                self.nOutputPerTask = 2

                taskLabeledDataSplit    = torch.Tensor().float()
                taskLabeledLabelSplit   = torch.Tensor().long()

                taskUnlabeledDataSplit  = torch.Tensor().float()
                taskUnlabeledLabelSplit = torch.Tensor().long()

                for iClass in taskList[iTask]:
                    # split labeled data
                    taskLabeledDataSplit  = torch.cat((taskLabeledDataSplit,transformedLabeledData[transformedLabeledLabel==iClass]),0)
                    taskLabeledLabelSplit = torch.cat((taskLabeledLabelSplit,transformedLabeledLabel[transformedLabeledLabel==iClass]),0)

                    # split unlabeled data
                    taskUnlabeledDataSplit  = torch.cat((taskUnlabeledDataSplit,transformedUnlabeledData[transformedUnlabeledLabel==iClass]),0)
                    taskUnlabeledLabelSplit = torch.cat((taskUnlabeledLabelSplit,transformedUnlabeledLabel[transformedUnlabeledLabel==iClass]),0)

                # shuffle labeled data
                taskLabeledData  = taskLabeledDataSplit
                taskLabeledLabel = taskLabeledLabelSplit

                row_idxs = list(range(taskLabeledData.shape[0]))
                random.shuffle(row_idxs)
                taskLabeledData  = taskLabeledData[torch.tensor(row_idxs), :]
                taskLabeledLabel = taskLabeledLabel[torch.tensor(row_idxs)]

                # shuffle unlabeled data
                taskUnlabeledData  = taskUnlabeledDataSplit
                taskUnlabeledLabel = taskUnlabeledLabelSplit

                row_idxs = list(range(taskUnlabeledData.shape[0]))
                random.shuffle(row_idxs)
                taskUnlabeledData  = taskUnlabeledData[torch.tensor(row_idxs), :]
                taskUnlabeledLabel = taskUnlabeledLabel[torch.tensor(row_idxs)]

            # store labeled data and labels
            finalLabeledData[iTask]  = taskLabeledData
            finalLabeledLabel[iTask] = taskLabeledLabel

            # store unlabeled data and labels
            finalUnlabeledData[iTask]  = taskUnlabeledData[self.batchSize:]
            finalUnlabeledLabel[iTask] = taskUnlabeledLabel[self.batchSize:]
            self.taskIndicator = self.taskIndicator + (iTask*torch.ones(int(finalUnlabeledData[iTask].shape[0]/self.batchSize)).long()).tolist()

            # store unlabeled data for testing
            unlabeledDataTest[iTask]  = taskUnlabeledData[:self.batchSize]
            unlabeledLabelTest[iTask] = taskUnlabeledLabel[:self.batchSize]
            nUnlabeledDataTest += unlabeledDataTest[iTask].shape[0]

        # labeled data
        self.labeledData    = finalLabeledData
        self.labeledLabel   = finalLabeledLabel

        # unlabeled data
        self.unlabeledData  = finalUnlabeledData
        self.unlabeledLabel = finalUnlabeledLabel

        # testing data
        self.unlabeledDataTest  = unlabeledDataTest
        self.unlabeledLabelTest = unlabeledLabelTest

        # update size
        self.nBatchPerTask          = int(finalUnlabeledData[iTask].shape[0]/self.batchSize)
        self.nBatch                 = self.nBatchPerTask*nTask
        self.nUnlabeledDataPerTask  = int(finalUnlabeledData[iTask].shape[0]/self.batchSize)*self.batchSize
        self.nUnlabeledDataTest     = nUnlabeledDataTest

        print('Number of task: ', nTask)
        print('Number of labeled data per task: ', self.nLabeledDataPerTask)
        print('Number of unlabeled data per task: ', self.nUnlabeledDataPerTask)
        print('Number of unlabeled data batch per task: ', self.nBatchPerTask)
        print('Number of unlabeled data test: ', self.nUnlabeledDataTest)

    def createDrift(self, nDrift = 2, taskList = [], taskType = 1):
        # random seed control
        # random.seed(0)
        if (taskType == 2) and (len(taskList) != nDrift):
            raise NameError('list of rotaion angle should be the same with the number of task')

        # clone unlabeled data
        transformedUnlabeledData  = self.unlabeledData.clone()
        transformedUnlabeledLabel = self.unlabeledLabel.clone()
        finalUnlabeledData        = torch.Tensor().float()
        finalUnlabeledLabel       = torch.Tensor().long()

        # number of data and batch for each task
        self.nDrift                  = nDrift
        self.nBatchPerTask          = int(self.nBatch/nDrift)
        self.nBatch                 = self.nBatchPerTask*nDrift
        self.nUnlabeledDataPerTask  = self.nBatchPerTask*self.batchSize

        for iTask in range(0,nDrift):
            # load data
            # iTask = iTask + 1
            self.taskIndicator = self.taskIndicator + (iTask*torch.ones(self.nBatchPerTask).long()).tolist()

            # load unlabeled data
            taskUnlabeledData  = transformedUnlabeledData[(iTask)*self.nUnlabeledDataPerTask:(iTask+1)*self.nUnlabeledDataPerTask]
            taskUnlabeledLabel = transformedUnlabeledLabel[(iTask)*self.nUnlabeledDataPerTask:(iTask+1)*self.nUnlabeledDataPerTask]

            if self.nUnlabeledDataPerTask != taskUnlabeledData.shape[0]:
                raise NameError('Mismatch data size')

            # permute
            if taskType == 1:
                # reshape image
                taskUnlabeledData = taskUnlabeledData.view(taskUnlabeledData.size(0),-1)

                # create permutation
                col_idxs = list(range(taskLabeledData.shape[1]))
                random.shuffle(col_idxs)

                # permute unlabeled data
                taskUnlabeledData = taskUnlabeledData[:, torch.tensor(col_idxs)]

                # reshape back image
                taskUnlabeledData = taskUnlabeledData.reshape(taskUnlabeledData.size(0),1,32,32)

            elif taskType == 2:
                # rotate unlabeled data
                for idx, _ in enumerate(taskUnlabeledData):
                    taskUnlabeledData[idx] = Rotate(random.randint(taskList[iTask][0], taskList[iTask][1]))(taskUnlabeledData[idx])

            # store unlabeled data and labels
            finalUnlabeledData  = torch.cat((finalUnlabeledData,taskUnlabeledData),0)
            finalUnlabeledLabel = torch.cat((finalUnlabeledLabel,taskUnlabeledLabel),0)

        # unlabeled data
        self.unlabeledData  = finalUnlabeledData
        self.unlabeledLabel = finalUnlabeledLabel

reason: import numpy,from numpy,import scipy,from scipy


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 25 13:13:23 2021

@author: simen
"""

# -*- coding: utf-8 -*-
"""
Created on Tue Jan  5 14:47:31 2021

@author: simen
"""

# -*- coding: utf-8 -*-
"""
Created on Tue Jan  5 14:46:09 2021

@author: simen
"""

# -*- coding: utf-8 -*-
"""
Created on Wed Nov 18 15:38:53 2020

@author: simen
"""

# -*- coding: utf-8 -*-
"""
Created on Wed Sep 30 11:15:37 2020
@author: simen
base code taken from https://github.com/bgrimstad/TTK28-Courseware/blob/master/model/flow_model.ipynb
"""
import matplotlib.pyplot as plt
import pandas as pd
import torch
from torch.utils.data import DataLoader
from math import sqrt
import sys
import xlsxwriter
import numpy as np
import GPUtil as GPU
import math
from threading import Thread
import time
import torch_optimizer as experimental
# class Monitor(Thread):
#     def __init__(self, delay):
#         super(Monitor, self).__init__()
#         self.stopped = False
#         self.delay = delay # Time between calls to GPUtil
#         self.start()

#     def run(self):
#         while not self.stopped:
#             GPU.showUtilization()
#             time.sleep(self.delay)

#     def stop(self):
#         self.stopped = True
class Net(torch.nn.Module):
    """
    PyTorch offers several ways to construct neural networks.
    Here we choose to implement the network as a Module class.
    This gives us full control over the construction and clarifies our intentions.
    """

    def __init__(self, layers):
        """
        Constructor of neural network
        :param layers: list of layer widths. Note that len(layers) = network depth + 1 since we incl. the input layer.
        """
        super().__init__()

        self.device = 'cuda' #if torch.cuda.is_available() else 'cpu'

        assert len(layers) >= 2, "At least two layers are required (incl. input and output layer)"
        self.layers = layers

        # Fully connected linear layers
        linear_layers = []

        for i in range(len(self.layers) - 1):
            n_in = self.layers[i]
            n_out = self.layers[i+1]
            layer = torch.nn.Linear(n_in, n_out)

            # Initialize weights and biases
            a = 1 if i == 0 else 2
            layer.weight.data = torch.randn((n_out, n_in)) * sqrt(a / n_in)
            layer.bias.data = torch.zeros(n_out)

            # Add to list
            linear_layers.append(layer)

        # Modules/layers must be registered to enable saving of model
        self.linear_layers = torch.nn.ModuleList(linear_layers)

        # Non-linearity (e.g. ReLU, ELU, or SELU)
        self.act = torch.nn.ReLU(inplace=False)

    def forward(self, input):
        """
        Forward pass to evaluate network for input values
        :param input: tensor assumed to be of size (batch_size, n_inputs)
        :return: output tensor
        """
        x = input
        for l in self.linear_layers[:-1]:
            x = l(x)
            x = self.act(x)

        output_layer = self.linear_layers[-1]
        return output_layer(x)

    def get_num_parameters(self):
        return sum(p.numel() for p in self.parameters())

    def save(self, path: str):
        """
        Save model state
        :param path: Path to save model state
        :return: None
        """
        torch.save({
            'model_state_dict': self.state_dict(),
        }, path)

    def load(self, path: str):
        """
        Load model state from file
        :param path: Path to saved model state
        :return: None
        """
        checkpoint = torch.load(path, map_location=torch.device("cuda" if torch.cuda.is_available() else "cpu"))
        self.load_state_dict(checkpoint['model_state_dict'])

def train(
        net: torch.nn.Module,
        train_loader: DataLoader,
        val_loader: DataLoader,
        n_epochs: int,
        lr: float,
        l2_reg: float, MSElist, gpu
) -> torch.nn.Module:
    """
    Train model using mini-batch SGD
    After each epoch, we evaluate the model on validation data
    :param net: initialized neural network
    :param train_loader: DataLoader containing training set
    :param n_epochs: number of epochs to train
    :param lr: learning rate (default: 0.001)
    :param l2_reg: L2 regularization factor (default: 0)
    :return: torch.nn.Module: trained model.
    """

    # Define loss and optimizer
    criterion = torch.nn.MSELoss(reduction='sum')
    optimizer = torch.optim.Adagrad (net.parameters(), lr=lr, lr_decay=0.001*lr*(1/n_epochs))
    #optimizer = experimental.AdaBound(net.parameters(), lr=lr)

    # Train Network
    for epoch in range(n_epochs):
        for inputs, labels in train_loader:
            # Zero the parameter gradients (from last iteration)
            optimizer.zero_grad()

            # Forward propagation
            outputs = net(inputs)

            # Compute cost function
            batch_mse = criterion(outputs, labels)

            reg_loss = 0
            for param in net.parameters():
                reg_loss += param.pow(2).sum()

            cost = batch_mse + l2_reg * reg_loss

            # Backward propagation to compute gradient
            cost.backward()

            # Update parameters using gradient
            optimizer.step()
            #print("GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))

        # Evaluate model on validation data
        mse_val = 0
        for inputs, labels in val_loader:
            mse_val += torch.sum(torch.pow(labels - net(inputs), 2)).item()
        mse_val /= len(val_loader.dataset)
        MSElist.append(mse_val)
        #print(f'Epoch: {epoch + 1}: Val MSE: {mse_val}')

    return net

def main(hidden_layers,epochs,learning_rate,result_filename,console_log_filename,time_spent_filename,output,inputset):

    old=sys.stdout
    sys.stdout=open(console_log_filename,'w')
    # monitor = Monitor(5)
    GPUs = GPU.getGPUs()
    gpu = GPUs[0]
    t0=time.time()

    random_seed =13371337  # This seed is also used in the pandas sample() method below
    torch.manual_seed(random_seed)
    df = pd.read_csv(r"C:\Users\simen\OneDrive\Skrivebord\MasterOppgave\Engine\SeptDec2020-ver3.csv", dtype=np.float64)
    for col in df:
        df[col] = pd.to_numeric(df[col], errors='coerce')

    print('Sizze of dataset', df.shape)
    # Test set (this is the period for which we must estimate QTOT)
    test_set = df.iloc[838860:1048570]

    # Make a copy of the dataset and remove the test data
    train_val_set = df.copy().drop(test_set.index)

    # Sample validation data without replacement (10%)
    val_set = train_val_set.sample(frac=0.1, replace=False, random_state=random_seed)

    # The remaining data is used for training (90%)
    train_set = train_val_set.copy().drop(val_set.index)

    # Check that the numbers add up
    n_points = len(train_set) + len(val_set) + len(test_set)
    print(f'{len(df)} = {len(train_set)} + {len(val_set)} + {len(test_set)} = {n_points}')
    INPUT_COLS = inputset	#'Africa.571_TT_114','Africa.871_XI_10151',	'Africa.871_XI_10207','Africa.871_XI_10259','Africa.871_XI_10302','Africa.871_XI_10303','Africa.871_XI_10304','Africa.871_XI_10306','Africa.871_XI_10312','Africa.871_XI_10315','Africa.871_XI_10363','Africa.871_XI_10409']
    OUTPUT_COLS = output

    # Get input and output tensors and convert them to torch tensors
    x_train = torch.from_numpy(train_set[INPUT_COLS].values).to(torch.float)
    y_train = torch.from_numpy(train_set[OUTPUT_COLS].values).to(torch.float)

    x_val = torch.from_numpy(val_set[INPUT_COLS].values).to(torch.float)
    y_val = torch.from_numpy(val_set[OUTPUT_COLS].values).to(torch.float)

    # Create dataset loaders
    # Here we specify the batch size and if the data should be shuffled
    train_dataset = torch.utils.data.TensorDataset(x_train, y_train)
    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=8192, shuffle=True)

    val_dataset = torch.utils.data.TensorDataset(x_val, y_val)
    val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=len(val_set), shuffle=False)

    layers_constructor = [len(INPUT_COLS)]+hidden_layers+[len(OUTPUT_COLS)]
    layers = layers_constructor
    net = Net(layers)

    print(f'Layers: {layers}')
    print(f'Number of model parameters: {net.get_num_parameters()}')

    n_epochs = epochs #orig 12000
    lr =  learning_rate
    l2_reg =  0.05  # 10

    MSElist=[]

    net = train(net, train_loader, val_loader, n_epochs, lr, l2_reg,MSElist,gpu)

    # with open('MSEovertime.txt','w') as f:
    #     sys.stdout=f
    #     epoke=1
    #     for element in MSElist:
    #         print('Epoch:',epoke, '     Val MSE: ' ,element, '\n \n')
    #         epoke+=1

    workbook = xlsxwriter.Workbook(result_filename)
    worksheet = workbook.add_worksheet()
    row=0
    for element in MSElist:
        worksheet.write(row,0,element)
        row+=1
    workbook.close()
    # monitor.stop()
    t1=time.time()-t0
    sys.stdout=open('timespent.txt','w')
    print('Time for entire script to run:', t1)
    sys.stdout=old
    # Get input and output as torch tensors
    #x_test = torch.from_numpy(test_set[INPUT_COLS].values).to(torch.float)

    #y_test = torch.from_numpy(test_set[OUTPUT_COLS].values).to(torch.float)

    # Make prediction
    #pred_test = net(x_test)

    # Compute MSE, MAE and MAPE on test data
    #print('Error on test data')

    #mse_test = torch.mean(torch.pow(pred_test - y_test, 2))
    #print(f'MSE: {mse_test.item()}')

    #mae_test = torch.mean(torch.abs(pred_test - y_test))
    #print(f'MAE: {mae_test.item()}')

    #mape_test = 100*torch.mean(torch.abs(torch.div(pred_test - y_test, y_test)))
    #print(f'MAPE: {mape_test.item()} %')

def multi_run(mode):
    #constants
    epochs=12000
    learning_rate=0.7
    #rectangular mode
    if (mode==1):
       modus="Rectangular"
       hidden_layers=[]
       max_depth=20
       max_width=100
       depthlist=range(2,max_depth)
       widthlist=np.arange(10,max_width+10,10).tolist()
       for depth in depthlist:
           for width in widthlist:
               hidden_layers=[width]*depth
               result_filename="results_using_"+modus+"neuralnet_with_depth_"+str(depth)+"_and width_"+str(width)+".xlsx"
               console_log_filename="console_log_using_"+modus+"neuralnet_with_depth_"+str(depth)+"_and width_"+str(width)+".txt"
               time_spent_filename="time_spent_using_"+modus+"neuralnet_with_depth_"+str(depth)+"_and width_"+str(width)+".txt"
               main(hidden_layers,epochs,learning_rate,result_filename,console_log_filename,time_spent_filename)

    if (mode==2):
        modus="Cone"
        hidden_layers=[]
        max_depth=100
        max_width=500
        depthlist=range(2,max_depth)
        widthlist=np.arange(10,max_width+10,10).tolist()
        ratio_between_width_of_first_and_last_layer=30
        for depth in depthlist:
            for width in widthlist:
                first_layer=width
                layer_width_change=math.floor((first_layer/ratio_between_width_of_first_and_last_layer)/depth)
                hidden_layers=[None]*depth

                for i in range(len(hidden_layers)):
                    if (i==0):
                        hidden_layers[i]=first_layer
                    else:
                        hidden_layers[i]=hidden_layers[i-1]-layer_width_change
                result_filename="results_using_"+modus+"_neuralnet_with_depth_"+str(depth)+"start_width="+str(hidden_layers[0])+"_last_layer_"+str(hidden_layers[-1])+".xlsx"
                console_log_filename="console_log_using_"+modus+"_neuralnet_with_depth_"+str(depth)+"start_width="+str(hidden_layers[0])+"_last_layer_"+str(hidden_layers[-1])+".txt"
                time_spent_filename="time_spent_using_"+modus+"_neuralnet_with_depth_"+str(depth)+"start_width="+str(hidden_layers[0])+"_last_layer_"+str(hidden_layers[-1])+".txt"
                main(hidden_layers,epochs,learning_rate,result_filename,console_log_filename,time_spent_filename)

def GrowingApproach():
    output=['Africa.871_XI_10207']
    inputs=[['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200','Africa.601_TI_10179','Africa.601_TI_10179','Africa.601_TI_10178'],['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200','Africa.601_TI_10179','Africa.601_TI_10179','Africa.601_TI_10178','Africa.601_PI_10177','Africa.601_PT_10163'],['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200','Africa.601_TI_10179','Africa.601_TI_10179','Africa.601_TI_10178','Africa.601_PI_10177','Africa.601_PT_10163','Africa.601_PI_10170','601_TI_10172'],['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200','Africa.601_TI_10179','Africa.601_TI_10179','Africa.601_TI_10178','Africa.601_PI_10177','Africa.601_PT_10163','Africa.601_PI_10170','601_TI_10172','Africa.601_TT_10189','Africa.601_TT_10188','Africa.601_TT_10187','Africa.601_TT_10186','Africa.601_TT_10185','Africa.601_TT_10184','Africa.601_TT_10183','Africa.601_TT_10182','Africa.601_TT_10181']]

    #inputs=[['Africa.601_XI_10114'],['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200'],['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200','Africa.601_TI_10179','Africa.601_TI_10179','Africa.601_TI_10178'],['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200','Africa.601_TI_10179','Africa.601_TI_10179','Africa.601_TI_10178','Africa.601_PI_10177','Africa.601_PT_10163'],['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200','Africa.601_TI_10179','Africa.601_TI_10179','Africa.601_TI_10178','Africa.601_PI_10177','Africa.601_PT_10163','Africa.601_PI_10170','601_TI_10172'],['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200','Africa.601_TI_10179','Africa.601_TI_10179','Africa.601_TI_10178','Africa.601_PI_10177','Africa.601_PT_10163','Africa.601_PI_10170','601_TI_10172','Africa.601_TT_10189','Africa.601_TT_10188','Africa.601_TT_10187','Africa.601_TT_10186','Africa.601_TT_10185','Africa.601_TT_10184','Africa.601_TT_10183','Africa.601_TT_10182','Africa.601_TT_10181']]
    teller=0
    for inputset in inputs:
        if inputset != inputs[-1]:
            continue
        epochs=6000
        learning_rate=0.9
        hidden_layers=[500,10,5]
        result_filename="10CONEresults_using_input_set_"+str(teller)+".xlsx"
        console_log_filename="10CONEconsole_log_using_input_set"+str(teller)+".txt"
        time_spent_filename="10CONEtime_spent_using_input_set"+str(teller)+".txt"
        main(hidden_layers,epochs,learning_rate,result_filename,console_log_filename,time_spent_filename,output,inputset)
        teller+=1

def NoGrow():
    output=['Africa.871_XI_10207']
    inputset=['Africa.601_XI_10114','Africa.601_UA_10176','Africa.601_TI_10199','Africa.601_TI_10200','Africa.601_TI_10179','Africa.601_TI_10179','Africa.601_TI_10178','Africa.601_PI_10177','Africa.601_PT_10163','Africa.601_PI_10170','601_TI_10172','Africa.601_TT_10189','Africa.601_TT_10188','Africa.601_TT_10187','Africa.601_TT_10186','Africa.601_TT_10185','Africa.601_TT_10184','Africa.601_TT_10183','Africa.601_TT_10182','Africa.601_TT_10181']
    epochs=6000
    learning_rate=0.7
    teller=5
    hidden_layers=[5000,100,5]
    result_filename="11CONEresults_using_input_set_ADABOUND"+str(teller)+".xlsx"
    console_log_filename="11CONEconsole_log_using_input_set"+str(teller)+".txt"
    time_spent_filename="11CONEtime_spent_using_input_set"+str(teller)+".txt"
    main(hidden_layers,epochs,learning_rate,result_filename,console_log_filename,time_spent_filename,output,inputset)

max_stars_repo_name: SimenKH/DataDrivenModelling

max_issues_repo_name: SimenKH/DataDrivenModelling

max_forks_repo_name: SimenKH/DataDrivenModelling


text:
from typing import Dict, List, Union

import matplotlib.pylab as plt
import numpy as np
import pytest
import tqdm
from numpy import float64, ndarray

from modes.mode_solver_full import mode_solver_full

def sweep_wavelength(
    wavelengths: ndarray, plot: bool = True, overwrite: bool = False, **wg_kwargs
) -> Dict[str, Union[List[ndarray], List[List[float64]], List[List[float]]]]:
    """

    Solve for the effective indices of a fixed structure at
    different wavelengths.

    Args:
        wavelengths (list): list of wavelengths to sweep
        plot (bool): `True` generates plots
        overwrite: when True forces to resimulate the structure

    Returns:
        wg_kwargs: arguments for the waveguide
        results: dict of results
        resuls['n_effs']: A list of the effective indices found for each wavelength.
        resuls['fractions_te']:
    """
    n_effs = []
    fractions_te = []

    for w in tqdm.tqdm(wavelengths, ncols=70):
        ms = mode_solver_full(wavelength=w, overwrite=overwrite, **wg_kwargs)
        n_effs.append(np.real(ms.n_effs))
        fractions_te.append(ms.fraction_te)

    suffix = "_".join([f"{int(wavelengths[i]*1e3)}" for i in [0, -1]])
    suffix += f"_{len(wavelengths)}"

    filename = ms._modes_directory / f"{ms.name}_{suffix}.dat"
    filename_neffs = filename.parent / f"{filename.stem}_neffs.dat"
    filename_fraction_te = filename.parent / f"{filename.stem}_fraction_te.dat"

    ms._write_n_effs_to_file(n_effs, filename_neffs, wavelengths)

    with open(filename_fraction_te, "w") as fs:
        header = "fraction te"
        fs.write("# param sweep," + header + "\n")
        for param, fte in zip(wavelengths, fractions_te):
            txt = "%.6f," % param
            txt += ",".join("%.2f" % f for f in fte)
            fs.write(txt + "\n")

    if plot:
        title = "$n_{eff}$ vs Wavelength"
        ms._plot_n_effs(
            filename_neffs,
            filename_fraction_te,
            "Wavelength",
            "n_{eff}",
            title,
        )
        plt.ylabel("$n_{eff}$")

    results = dict(n_effs=n_effs, fractions_te=fractions_te)
    return results

@pytest.mark.parametrize("overwrite", [True, False])
def test_sweep(overwrite: bool) -> None:
    wavelengths = np.arange(1.30, 1.60, 0.1)
    r = sweep_wavelength(wavelengths=wavelengths, overwrite=overwrite)
    print(r["n_effs"][0])
    assert np.isclose(r["n_effs"][0], np.array([2.7357584, 2.22395364])).all()
    assert r

if __name__ == "__main__":
    test_sweep(overwrite=False)
    plt.show()

max_stars_repo_name: joamatab/modesolverpy

max_issues_repo_name: joamatab/modesolverpy

max_forks_repo_name: joamatab/modesolverpy

reason: import numpy,from numpy