text:
# -*- coding: utf-8 -*-
"""Tools for the evaluation of reconstruction methods.
"""
import sys
from warnings import warn
from itertools import product
from math import ceil
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
from dival.util.odl_utility import CallbackStoreAfter
from dival.util.odl_utility import CallbackStore
# to be replaced by odl.solvers.util.callback.CallbackStore when
# https://github.com/odlgroup/odl/pull/1539 is included in ODL release
from dival.util.plot import plot_image, plot_images
from dival.util.std_out_err_redirect_tqdm import std_out_err_redirect_tqdm
from dival.measure import Measure
from dival.data import DataPairs
from dival import IterativeReconstructor, LearnedReconstructor

class TaskTable:
    """Task table containing reconstruction tasks to evaluate.

    Attributes
    ----------
    name : str
        Name of the task table.
    tasks : list of dict
        Tasks that shall be run. The fields of each dict are set from the
        parameters to :meth:`append` (or :meth:`append_all_combinations`). Cf.
        documentation of :meth:`append` for details.
    results : :class:`ResultTable` or `None`
        Results from the latest call to :meth:`run`.
    """
    def __init__(self, name=''):
        self.name = name
        self.tasks = []
        self.results = None

    def run(self, save_reconstructions=True, reuse_iterates=True,
            show_progress='text'):
        """Run all tasks and return the results.

        The returned :class:`ResultTable` object is also stored as
        :attr:`results`.

        Parameters
        ----------
        save_reconstructions : bool, optional
            Whether the reconstructions should be saved in the results.
            The default is ``True``.

            If measures shall be applied after this method returns, it must be
            ``True``.

            If ``False``, no iterates (intermediate reconstructions) will be
            saved, even if ``task['options']['save_iterates']==True``.

        reuse_iterates : bool, optional
            Whether to reuse iterates from other sub-tasks if possible.
            The default is ``True``.

            If there are sub-tasks whose hyper parameter choices differ only
            in the number of iterations of an :class:`IterativeReconstructor`,
            only the sub-task with the maximum number of iterations is run and
            the results for the other ones determined by storing iterates if
            this option is ``True``.

            Note 1: If enabled, the callbacks assigned to the reconstructor
            will be run only for the above specified sub-tasks with the maximum
            number of iterations.

            Note 2: If the reconstructor is non-deterministic, this option can
            affect the results as the same realization is used for multiple
            sub-tasks.

        show_progress : str, optional
            Whether and how to show progress. Options are:

                ``'text'`` (default)
                    print a line before running each task
                ``'tqdm'``
                    show a progress bar with ``tqdm``
                `None`
                    do not show progress

        Returns
        -------
        results : :class:`ResultTable`
            The results.
        """
        row_list = []
        with std_out_err_redirect_tqdm(None if show_progress == 'tqdm' else
                                       sys.stdout) as orig_stdout:
            for i, task in enumerate(tqdm(self.tasks,
                                          desc='task',
                                          file=orig_stdout,
                                          disable=(show_progress != 'tqdm'))):
                if show_progress == 'text':
                    print('running task {i}/{num_tasks} ...'.format(
                        i=i, num_tasks=len(self.tasks)))
                test_data = task['test_data']
                reconstructor = task['reconstructor']
                if test_data.ground_truth is None and task['measures']:
                    raise ValueError('missing ground truth, cannot apply '
                                     'measures')
                measures = [(measure if isinstance(measure, Measure) else
                             Measure.get_by_short_name(measure))
                            for measure in task['measures']]
                options = task['options']
                skip_training = options.get('skip_training', False)
                save_best_reconstructor = options.get(
                    'save_best_reconstructor')
                save_iterates = (save_reconstructions and
                                 options.get('save_iterates'))

                hp_choices = task.get('hyper_param_choices')
                if hp_choices:
                    # run all hyper param choices as sub-tasks
                    retrain_param_keys = [k for k, v in
                                          reconstructor.HYPER_PARAMS.items()
                                          if v.get('retrain', False)]
                    orig_hyper_params = reconstructor.hyper_params.copy()

                    def _warn_if_invalid_keys(keys):
                        for k in keys:
                            if k not in reconstructor.HYPER_PARAMS.keys():
                                warn("choice for unknown hyper parameter '{}' "
                                     "for reconstructor of type '{}' will be "
                                     'ignored'.format(k, type(reconstructor)))

                    if isinstance(hp_choices, dict):
                        _warn_if_invalid_keys(hp_choices.keys())
                        keys_retrain_first = sorted(
                            hp_choices.keys(),
                            key=lambda k: k not in retrain_param_keys)
#                        if isinstance(reconstructor, IterativeReconstructor):
                            # 'iterations' treated specially to re-use iterates
#                            keys_retrain_first.remove('iterations')
#                            hp_choices_iterations = hp_choices.get(
#                                'iterations',
#                                [orig_hyper_params['iterations']])
                        param_values = [
                            hp_choices.get(k, [orig_hyper_params[k]]) for k in
                            keys_retrain_first]
                        hp_choice_list = [
                            dict(zip(keys_retrain_first, v)) for
                            v in product(*param_values)]
                    else:
                        hp_choice_list = hp_choices
                        for hp_choice in hp_choice_list:
                            _warn_if_invalid_keys(hp_choice.keys())
#                        if isinstance(reconstructor, IterativeReconstructor):
#                             no special support for re-using iterates
#                            hp_choices_iterations = []
                    if (isinstance(reconstructor, IterativeReconstructor) and
                            reuse_iterates):
                        reuse_iterates_from = []
                        for j, hp_choice_j in enumerate(hp_choice_list):
                            iter_j = hp_choice_j.get(
                                'iterations', orig_hyper_params['iterations'])
                            (k_max, iter_max) = (-1, iter_j)
                            for k, hp_choice_k in enumerate(hp_choice_list):
                                iter_k = hp_choice_k.get(
                                    'iterations',
                                    orig_hyper_params['iterations'])
                                if iter_k > iter_max:
                                    hp_choice_j_rem = hp_choice_j.copy()
                                    hp_choice_j_rem.pop('iterations')
                                    hp_choice_k_rem = hp_choice_k.copy()
                                    hp_choice_k_rem.pop('iterations')
                                    if hp_choice_j_rem == hp_choice_k_rem:
                                        (k_max, iter_max) = (k, iter_k)
                            reuse_iterates_from.append(k_max)
                    if save_best_reconstructor:
                        if len(measures) == 0 and len(hp_choice_list) > 1:
                            warn("No measures are chosen to be evaluated, so "
                                 "no best reconstructor can be selected. Will "
                                 "not save like requested by "
                                 "'save_best_reconstructor' option.")
                            save_best_reconstructor = None
                        else:
                            best_loss = np.inf
                    row_sub_list = [None] * len(hp_choice_list)
                    # run sub-tasks
                    for j, hp_choice in enumerate(
                            tqdm(hp_choice_list, desc='sub-task',
                                 file=orig_stdout,
                                 disable=(show_progress != 'tqdm'),
                                 leave=False)):
                        if show_progress == 'text':
                            print('sub-task {j}/{n} ...'
                                  .format(j=j, n=len(hp_choice_list)))
                        train = (isinstance(reconstructor,
                                            LearnedReconstructor) and (
                            j == 0 or any(
                                (hp_choice.get(k, orig_hyper_params[k]) !=
                                 reconstructor.hyper_params[k]
                                 for k in retrain_param_keys))))
                        reconstructor.hyper_params = orig_hyper_params.copy()
                        reconstructor.hyper_params.update(hp_choice)
#                        if (isinstance(reconstructor, IterativeReconstructor)
#                                and hp_choices_iterations):
#                            reconstructor.hyper_params['iterations'] = max(
#                                hp_choices_iterations)  # only largest number
                        if train and not skip_training:
                            reconstructor.train(task['dataset'])
                        run_sub_task = not (isinstance(reconstructor,
                                                       IterativeReconstructor)
                                            and reuse_iterates
                                            and reuse_iterates_from[j] != -1)
                        if run_sub_task:
                            return_rows_iterates = None
                            if (isinstance(reconstructor,
                                           IterativeReconstructor) and
                                    reuse_iterates):
                                # determine the iteration numbers needed for
                                # other sub-tasks
                                return_iterates_for = [
                                    k for k, from_k in
                                    enumerate(reuse_iterates_from)
                                    if from_k == j]  # sub-task indices
                                return_rows_iterates = [
                                    hp_choice_list[k].get(
                                        'iterations',
                                        orig_hyper_params['iterations'])
                                    for k in return_iterates_for]  # iterations
                            row = self._run_task(
                                reconstructor=reconstructor,
                                test_data=test_data,
                                measures=measures,
                                hp_choice=hp_choice,
                                return_rows_iterates=return_rows_iterates,
                                options=options,
                                save_reconstructions=save_reconstructions,
                                save_iterates=save_iterates,
                                )
                            if return_rows_iterates is not None:
                                (row, rows_iterates) = row
                                # assign rows for other sub-tasks
                                for r_i, k in enumerate(return_iterates_for):
                                    rows_iterates[r_i]['task_ind'] = i
                                    rows_iterates[r_i]['sub_task_ind'] = k
                                    row_sub_list[k] = rows_iterates[r_i]
                            # assign row for current sub-task
                            row['task_ind'] = i
                            row['sub_task_ind'] = j
                            row_sub_list[j] = row
                            if save_best_reconstructor:
                                def save_if_best_reconstructor(
                                        measure_values, iterations=None):
                                    measure = save_best_reconstructor.get(
                                        'measure', measures[0])
                                    if isinstance(measure, str):
                                        measure = Measure.get_by_short_name(
                                            measure)
                                    loss_sign = (
                                        1 if measure.measure_type == 'distance'
                                        else -1)
                                    cur_loss = (
                                        loss_sign * np.mean(measure_values[
                                            measure.short_name]))
                                    if cur_loss < best_loss:
                                        if iterations is not None:
                                            reconstructor.hyper_params[
                                                'iterations'] = iterations
                                        reconstructor.save_params(
                                            save_best_reconstructor['path'])
                                        return cur_loss
                                    return best_loss
                                best_loss = save_if_best_reconstructor(
                                    row['measure_values'])
                                if return_rows_iterates is not None:
                                    for row_iterates, iterations in zip(
                                            rows_iterates, return_rows_iterates):
                                        best_loss = save_if_best_reconstructor(
                                            row_iterates['measure_values'],
                                            iterations=iterations)
                    reconstructor.hyper_params = orig_hyper_params.copy()
                    row_list += row_sub_list
                else:
                    # run task (with hyper params as they are)
                    if (isinstance(reconstructor, LearnedReconstructor) and
                            not skip_training):
                        reconstructor.train(task['dataset'])

                    row = self._run_task(
                        reconstructor=reconstructor,
                        test_data=test_data,
                        measures=measures,
                        hp_choice=None,
                        return_rows_iterates=None,
                        options=options,
                        save_reconstructions=save_reconstructions,
                        save_iterates=save_iterates,
                        )
                    row['task_ind'] = i
                    row['sub_task_ind'] = 0
                    row_list.append(row)
                    if save_best_reconstructor:
                        reconstructor.save_params(
                            save_best_reconstructor['path'])

        self.results = ResultTable(row_list)
        return self.results

    def _run_task(self, reconstructor, test_data, measures, options, hp_choice,
                  return_rows_iterates, save_reconstructions, save_iterates):
        # Parameters
        # ----------
        # return_rows_iterates : list of int or `None`
        #     If specified, also return rows for the specified iterates.
        #     Must be `None` if reconstructor is no `IterativeReconstructor`.
        #
        # Returns
        # -------
        # row [, rows_iterates] : dict or (dict, list of dict)
        #     The resulting row, and if `return_rows_iterates` is specified,
        #     as second output a list of rows for the iterates.
        reconstructions = []
        if isinstance(reconstructor, IterativeReconstructor):
            if save_iterates:
                iterates = []
            if options.get('save_iterates_measure_values'):
                iterates_measure_values = {m.short_name: [] for m in measures}
            save_iterates_step = options.get('save_iterates_step', 1)
            if return_rows_iterates is not None:
                iterates_for_rows = []

        for observation, ground_truth in zip(test_data.observations,
                                             test_data.ground_truth):
            if isinstance(reconstructor, IterativeReconstructor):
                callbacks = []
                if return_rows_iterates is not None:
                    iters_for_rows = []
                    store_after_iters = return_rows_iterates.copy()
                    if 0 in store_after_iters:
                        warn('reporting a dummy zero reconstruction as '
                             'zero-th iterate of `IterativeReconstructor`')
                        # there is no general way to obtain the 0-th iterate,
                        # the first callback call is after first iteration
                        iters_for_rows.append(reconstructor.reco_space.zero())
                        store_after_iters.remove(0)
                    iterates_for_rows.append(iters_for_rows)
                    callback_store_after = CallbackStoreAfter(
                        iters_for_rows,
                        store_after_iters=store_after_iters)
                    callbacks.append(callback_store_after)
                if save_iterates:
                    iters = []
                    iterates.append(iters)
                    callback_store = CallbackStore(
                        iters, step=save_iterates_step)
                    callbacks.append(callback_store)
                if options.get('save_iterates_measure_values'):
                    for measure in measures:
                        iters_mvs = []
                        iterates_measure_values[
                            measure.short_name].append(iters_mvs)
                        callback_store = CallbackStore(
                            iters_mvs, step=save_iterates_step)
                        callbacks.append(
                            callback_store *
                            measure.as_operator_for_fixed_ground_truth(
                                ground_truth))
                callback = None
                if len(callbacks) > 0:
                    callback = callbacks[-1]
                    for c in callbacks[-2::-1]:
                        callback &= c
                reconstruction = reconstructor.reconstruct(
                    observation, callback=callback)
            else:
                reconstruction = reconstructor.reconstruct(observation)

            reconstructions.append(reconstruction)

        measure_values = {}
        for measure in measures:
            measure_values[measure.short_name] = [
                measure.apply(r, g) for r, g in zip(
                    reconstructions, test_data.ground_truth)]
        misc = {}
        if isinstance(reconstructor, IterativeReconstructor):
            if save_iterates:
                misc['iterates'] = iterates
            if options.get('save_iterates_measure_values'):
                misc['iterates_measure_values'] = iterates_measure_values
        if hp_choice:
            misc['hp_choice'] = hp_choice

        row = {'reconstructions': reconstructions,
               'reconstructor': reconstructor,
               'test_data': test_data,
               'measure_values': measure_values,
               'misc': misc}
        if save_reconstructions:
            row['reconstructions'] = reconstructions
        if return_rows_iterates is not None:
            # create rows for iterates given by return_rows_iterates
            rows_iterates = []
            # convert iterates_for_rows[reconstructions_idx][rows_iterates_idx]
            # to
            # reconstructions_iterates[rows_iterates_idx][reconstructions_idx]
            reconstructions_iterates = [
                list(it) for it in zip(*iterates_for_rows)]
            for iterations, recos_iterates in zip(
                    return_rows_iterates, reconstructions_iterates):
                measure_values_iterates = {}
                for measure in measures:
                    measure_values_iterates[measure.short_name] = [
                        measure.apply(r, g) for r, g in zip(
                            recos_iterates, test_data.ground_truth)]
                misc_iterates = {}
                # number of iterates to keep
                n_iterates = ceil(iterations / save_iterates_step)
                if save_iterates:
                    misc_iterates['iterates'] = iterates[:n_iterates]
                if options.get('save_iterates_measure_values'):
                    misc_iterates['iterates_measure_values'] = {
                        short_name: values[:n_iterates] for short_name, values
                        in iterates_measure_values.items()}
                if hp_choice:
                    misc_iterates['hp_choice'] = hp_choice.copy()
                    # specify 'iterations' hyper param, which was emulated by
                    # using CallbackStoreAfter while running for more iters
                    misc_iterates['hp_choice']['iterations'] = iterations
                row_iterates = {'reconstructions': recos_iterates,
                                'reconstructor': reconstructor,
                                'test_data': test_data,
                                'measure_values': measure_values_iterates,
                                'misc': misc_iterates}
                rows_iterates.append(row_iterates)
        return row if return_rows_iterates is None else (row, rows_iterates)

    def append(self, reconstructor, test_data, measures=None, dataset=None,
               hyper_param_choices=None, options=None):
        """Append a task.

        Parameters
        ----------
        reconstructor : :class:`.Reconstructor`
            The reconstructor.
        test_data : :class:`.DataPairs`
            The test data.
        measures : sequence of (:class:`.Measure` or str), optional
            Measures that will be applied. Either :class:`.Measure` objects or
            their short names can be passed.
        dataset : :class:`.Dataset`, optional
            The dataset that will be passed to
            :meth:`reconstructor.train <LearnedReconstructor.train>` if it is a
            :class:`.LearnedReconstructor`.
        hyper_param_choices : dict of list or list of dict, optional
            Choices of hyper parameter combinations to try as sub-tasks.

                * If a dict of lists is specified, all combinations of the
                  list elements (cartesian product space) are tried.
                * If a list of dicts is specified, each dict is taken as a
                  parameter combination to try.

            The current parameter values are read from
            :attr:`Reconstructor.hyper_params` in the beginning and used as
            default values for all parameters not specified in the passed
            dicts. Afterwards, the original values are restored.
        options : dict
            Options that will be used. Options are:

            ``'skip_training'`` : bool, optional
                Whether to skip training. Can be used for manual training
                of reconstructors (or loading of a stored state).
                Default: ``False``.
            ``'save_best_reconstructor'`` : dict, optional
                If specified, save the best reconstructor from the sub-tasks
                (cf. `hyper_param_choices`) by calling
                :meth:`Reconstructor.save_params`.
                For ``hyper_param_choices=None``, the reconstructor from the
                single sub-task is saved.
                This option requires `measures` to be non-empty if there are
                multiple sub-tasks.
                The fields are:

                    ``'path'`` : str
                        The path to save the best reconstructor at (argument to
                        :meth:`save_params`). Note that this path is used
                        during execution of the task to store the best
                        reconstructor params so far, so the file(s) are
                        most likely updated multiple times.
                    ``'measure'`` : :class:`.Measure` or str, optional
                        The measure used to define the "best" reconstructor (in
                        terms of mean performance).
                        Must be one of the `measures`. By default
                        ``measures[0]`` is used.
                        This field is ignored if there is only one sub-task.

            ``'save_iterates'`` : bool, optional
                Whether to save the intermediate reconstructions of iterative
                reconstructors. Default: ``False``.
                Will be ignored if ``save_reconstructions=False`` is passed to
                `run`.
                If ``reuse_iterates=True`` is passed to `run` and there are
                sub-tasks for which iterates are reused, these iterates are the
                same objects for all of those sub-tasks (i.e. no copies).
            ``'save_iterates_measure_values'`` : bool, optional
                Whether to compute and save the measure values for each
                intermediate reconstruction of iterative reconstructors
                (the default is ``False``).
            ``'save_iterates_step'`` : int, optional
                Step size for ``'save_iterates'`` and
                ``'save_iterates_measure_values'`` (the default is 1).
        """
        if measures is None:
            measures = []
        if options is None:
            options = {}
        if (isinstance(reconstructor, LearnedReconstructor) and
                not options.get('skip_training', False) and dataset is None):
            raise ValueError('in order to use a learned reconstructor you '
                             'must specify a `dataset` for training (or set '
                             '``skip_training: True`` in `options` and train '
                             'manually)')
        self.tasks.append({'reconstructor': reconstructor,
                           'test_data': test_data,
                           'measures': measures,
                           'dataset': dataset,
                           'hyper_param_choices': hyper_param_choices,
                           'options': options})

    def append_all_combinations(self, reconstructors, test_data, measures=None,
                                datasets=None, hyper_param_choices=None,
                                options=None):
        """Append tasks of all combinations of test data, reconstructors and
        optionally datasets.
        The order is taken from the lists, with test data changing slowest
        and reconstructor changing fastest.

        Parameters
        ----------
        reconstructors : list of `Reconstructor`
            Reconstructor list.
        test_data : list of `DataPairs`
            Test data list.
        measures : sequence of (`Measure` or str)
            Measures that will be applied. The same measures are used for all
            combinations of test data and reconstructors. Either `Measure`
            objects or their short names can be passed.
        datasets : list of `Dataset`, optional
            Dataset list. Required if `reconstructors` contains at least one
            `LearnedReconstructor`.
        hyper_param_choices : list of (dict of list or list of dict), optional
            Choices of hyper parameter combinations for each reconstructor,
            which are tried as sub-tasks.
            The i-th element of this list is used for the i-th reconstructor.
            See `append` for documentation of how the choices are passed.
        options : dict
            Options that will be used. The same options are used for all
            combinations of test data and reconstructors. See `append` for
            documentation of the options.
        """
        if datasets is None:
            datasets = [None]
        if hyper_param_choices is None:
            hyper_param_choices = [None] * len(reconstructors)
        for test_data_ in test_data:
            for dataset in datasets:
                for reconstructor, hp_choices in zip(reconstructors,
                                                     hyper_param_choices):
                    self.append(reconstructor=reconstructor,
                                test_data=test_data_,
                                measures=measures,
                                dataset=dataset,
                                hyper_param_choices=hp_choices,
                                options=options)

    def __repr__(self):
        return "TaskTable(name='{name}', tasks={tasks})".format(
            name=self.name,
            tasks=self.tasks)

class ResultTable:
    """The results of a :class:`.TaskTable`.

    Cf. :attr:`TaskTable.results`.

    Attributes
    ----------
    results : :class:`pandas.DataFrame`
        The results.
        The index is given by ``'task_ind'`` and ``'sub_task_ind'``, and the
        columns are ``'reconstructions'``, ``'reconstructor'``,
        ``'test_data'``, ``'measure_values'`` and ``'misc'``.
    """
    def __init__(self, row_list):
        """
        Usually, objects of this type are constructed by
        :meth:`TaskTable.run`, which sets :attr:`TaskTable.results`, rather
        than by manually calling this constructor.

        Parameters
        ----------
        row_list : list of dict
            Result rows.
            Used to build :attr:`results` of type :class:`pandas.DataFrame`.
        """
        self.results = pd.DataFrame(row_list).set_index(['task_ind',
                                                         'sub_task_ind'])

    def apply_measures(self, measures, task_ind=None):
        """Apply (additional) measures to reconstructions.

        This is not possible if the reconstructions were not saved, in which
        case a :class:`ValueError` is raised.

        Parameters
        ----------
        measures : list of :class:`.Measure`
            Measures to apply.
        task_ind : int or sequence of ints, optional
            Indexes of tasks to which the measures shall be applied.
            If `None`, this is interpreted as "all results".

        Raises
        ------
        ValueError
            If reconstructions are missing or `task_ind` is not valid.
        """
        if task_ind is None:
            indexes = self.results.index.levels[0]
        elif np.isscalar(task_ind):
            indexes = [task_ind]
        elif isinstance(task_ind, list):
            indexes = task_ind
        else:
            raise ValueError('`task_ind` must be a scalar, a list of ints or '
                             '`None`')
        for i in indexes:
            rows = self.results.loc[i]
            for j in range(len(rows)):
                row = rows.loc[j]
                if row['reconstructions'] is None:
                    raise ValueError('reconstructions missing in task {}{}'
                                     .format(i, '.{}'.format(j) if
                                                len(rows) > 1 else ''))
                for measure in measures:
                    if isinstance(measure, str):
                        measure = Measure.get_by_short_name(measure)
                    row['measure_values'][measure.short_name] = [
                        measure.apply(r, g) for r, g in zip(
                            row['reconstructions'],
                            row['test_data'].ground_truth)]

    def plot_reconstruction(self, task_ind, sub_task_ind=0, test_ind=-1,
                            plot_ground_truth=True, **kwargs):
        """Plot the reconstruction at the specified index.
        Supports only 1d and 2d reconstructions.

        Parameters
        ----------
        task_ind : int
            Index of the task.
        sub_task_ind : int, optional
            Index of the sub-task (default ``0``).
        test_ind : sequence of int or int, optional
            Index in test data. If ``-1``, plot all reconstructions (the
            default).
        plot_ground_truth : bool, optional
            Whether to show the ground truth next to the reconstruction.
            The default is ``True``.
        kwargs : dict
            Keyword arguments that are passed to
            :func:`~dival.util.plot.plot_image` if the reconstruction is 2d.

        Returns
        -------
        ax_list : list of :class:`np.ndarray` of :class:`matplotlib.axes.Axes`
            The axes in which the reconstructions and eventually the ground
            truth were plotted.
        """
        row = self.results.loc[task_ind, sub_task_ind]
        test_data = row.at['test_data']
        reconstructor = row.at['reconstructor']
        ax_list = []
        if isinstance(test_ind, int):
            if test_ind == -1:
                test_ind = range(len(test_data))
            else:
                test_ind = [test_ind]
        for i in test_ind:
            title = 'reconstruction for task {}{}, test_data[{}]'.format(
                task_ind, '.{}'.format(sub_task_ind) if
                          len(self.results.loc[task_ind]) > 1 else '', i)
            reconstruction = row.at['reconstructions'][i]
            ground_truth = test_data.ground_truth[i]
            if reconstruction is None:
                raise ValueError('reconstruction is `None`')
            if reconstruction.asarray().ndim > 2:
                print('only 1d and 2d reconstructions can be plotted')
                return
            if reconstruction.asarray().ndim == 1:
                x = reconstruction.space.points()
                _, ax = plt.subplots()
                ax.plot(x, reconstruction, label=reconstructor.name)
                if plot_ground_truth:
                    ax.plot(x, ground_truth, label='ground truth')
                ax.legend()
                ax.set_title(title)
                ax = np.array(ax)
            elif reconstruction.asarray().ndim == 2:
                if plot_ground_truth:
                    _, ax = plot_images([reconstruction, ground_truth],
                                        **kwargs)
                    ax[1].set_title('ground truth')
                else:
                    _, ax = plot_image(reconstruction, **kwargs)
                ax[0].set_title(reconstructor.name)
                ax[0].figure.suptitle(title)
            ax_list.append(ax)
        return ax_list

    def plot_all_reconstructions(self, **kwargs):
        """Plot all reconstructions.

        Parameters
        ----------
        kwargs : dict
            Keyword arguments that are forwarded to
            :meth:`plot_reconstruction`.

        Returns
        -------
        ax : :class:`np.ndarray` of :class:`matplotlib.axes.Axes`
            The axes the reconstructions were plotted in.
        """
        ax = []
        for i, j in self.results.index:
            ax_ = self.plot_reconstruction(task_ind=i, sub_task_ind=j,
                                           **kwargs)
            ax.append(ax_)
        return np.vstack(ax)

    def plot_convergence(self, task_ind, sub_task_ind=0, measures=None,
                         fig_size=None, gridspec_kw=None):
        """
        Plot measure values for saved iterates.

        This shows the convergence behavior with respect to the measures.

        Parameters
        ----------
        task_ind : int
            Index of the task.
        sub_task_ind : int, optional
            Index of the sub-task (default ``0``).
        measures : [list of ] :class:`.Measure`, optional
            Measures to apply. Each measure is plotted in a subplot.
            If `None` is passed, all measures in ``result['measure_values']``
            are used.

        Returns
        -------
        ax : :class:`np.ndarray` of :class:`matplotlib.axes.Axes`
            The axes the measure values were plotted in.
        """
        row = self.results.loc[task_ind, sub_task_ind]
        iterates_measure_values = row['misc'].get('iterates_measure_values')
        if not iterates_measure_values:
            iterates = row['misc'].get('iterates')
            if not iterates:
                raise ValueError(
                    "no 'iterates_measure_values' or 'iterates' in results "
                    "of task {}{}".format(
                        task_ind, '.{}'.format(sub_task_ind) if
                        len(self.results.loc[task_ind]) > 1 else ''))
        if measures is None:
            measures = row['measure_values'].keys()
        elif isinstance(measures, Measure):
            measures = [measures]
        fig, ax = plt.subplots(len(measures), 1, gridspec_kw=gridspec_kw)
        if not isinstance(ax, np.ndarray):
            ax = np.array([ax])
        if fig_size is not None:
            fig.set_size_inches(fig_size)
        fig.suptitle('convergence of {}'.format(row['reconstructor'].name))
        for measure, ax_ in zip(measures, ax.flat):
            if isinstance(measure, str):
                measure = Measure.get_by_short_name(measure)
            if iterates_measure_values:
                errors = np.mean([iters_mvs[measure.short_name] for iters_mvs
                                  in iterates_measure_values], axis=0)
            else:
                ground_truth = row['test_data'].ground_truth
                errors = np.mean([[measure.apply(x, g) for x in iters] for
                                  iters, g in zip(iterates, ground_truth)],
                                 axis=0)
            ax_.plot(errors)
            ax_.set_title(measure.short_name)
        return ax

    def plot_performance(self, measure, reconstructors=None, test_data=None,
                         weighted_average=False, **kwargs):
        """
        Plot average measure values for different reconstructors.
        The values have to be computed previously, e.g. by
        :meth:`apply_measures`.

        The average is computed over all rows of :attr:`results` with the
        specified `test_data` that store the requested `measure` value.

        Note that for tasks with multiple sub-tasks, all of them are used when
        computing the average (i.e., the measure values for all hyper parameter
        choices are averaged).

        Parameters
        ----------
        measure : :class:`.Measure` or str
            The measure to plot (or its :attr:`~.Measure.short_name`).
        reconstructors : sequence of :class:`.Reconstructor`, optional
            The reconstructors to compare. If `None` (default), all
            reconstructors that are found in the results are compared.
        test_data : [sequence of ] :class:`.DataPairs`, optional
            Test data to take into account for computing the mean value.
            By default, all test data is used.
        weighted_average : bool, optional
            Whether to weight the rows according to the number of pairs in
            their test data.
            Default: ``False``, i.e. all rows are weighted equally.
            If ``True``, all test data pairs are weighted equally.

        Returns
        -------
        ax : `matplotlib.axes.Axes`
            The axes the performance was plotted in.
        """
        if not isinstance(measure, Measure):
            measure = Measure.get_by_short_name(measure)
        if reconstructors is None:
            reconstructors = self.results['reconstructor'].unique()
        if isinstance(test_data, DataPairs):
            test_data = [test_data]
        mask = [measure.short_name in row['measure_values'].keys() and
                row['reconstructor'] in reconstructors and
                (test_data is None or row['test_data'] in test_data)
                for _, row in self.results.iterrows()]
        rows = self.results[mask]
        v = []
        for reconstructor in reconstructors:
            r_rows = rows[rows['reconstructor'] == reconstructor]
            values = [mvs[measure.short_name] for mvs in
                      r_rows['measure_values']]
            weights = None
            if weighted_average:
                weights = [len(test_data.observations) for test_data in
                           r_rows['test_data']]
            v.append(np.average(values, weights=weights))
        fig, ax = plt.subplots(**kwargs)
        ax.bar(range(len(v)), v)
        ax.set_xticks(range(len(v)))
        ax.set_xticklabels([r.name for r in reconstructors], rotation=30)
        ax.set_title('{measure_name}'.format(measure_name=measure.name))
        return ax

    def to_string(self, max_colwidth=70, formatters=None, hide_columns=None,
                  show_columns=None, **kwargs):
        """Convert to string. Used by :meth:`__str__`.

        Parameters
        ----------
        max_colwidth : int, optional
            Maximum width of the columns, c.f. the option
            ``'display.max_colwidth'`` of pandas.
        formatters : dict of functions, optional
            Custom formatter functions for the columns, passed to
            :meth:`results.to_string <pandas.DataFrame.to_string>`.
        hide_columns : list of str, optional
            Columns to hide. Default: ``['reconstructions', 'misc']``.
        show_columns : list of str, optional
            Columns to show. Overrides `hide_columns`.
        kwargs : dict
            Keyword arguments passed to
            :meth:`results.to_string <pandas.DataFrame.to_string>`.

        Returns
        -------
        string : str
            The string.
        """

        def measure_values_formatter(measure_values):
            means = ['{}: {:.4g}'.format(k, np.mean(v)) for k, v in
                     measure_values.items()]
            return 'mean: {{{}}}'.format(', '.join(means))

        def name_or_repr_formatter(x):
            return x.name or x.__repr__()

        formatters_ = {}
        formatters_['measure_values'] = measure_values_formatter
        formatters_['test_data'] = name_or_repr_formatter
        formatters_['reconstructor'] = name_or_repr_formatter
        if formatters is not None:
            formatters_.update(formatters)
        if hide_columns is None:
            hide_columns = ['reconstructions', 'misc']
        if show_columns is None:
            show_columns = []
        columns = [c for c in self.results.columns if c not in hide_columns or
                   c in show_columns]
        with pd.option_context('display.max_colwidth', max_colwidth):
            return "ResultTable(results=\n{}\n)".format(
                self.results.to_string(formatters=formatters_, columns=columns,
                                       **kwargs))

    def print_summary(self):
        """Prints a summary of the results.
        """
        print('ResultTable with {:d} tasks.'.format(
            len(self.results.index.levels[0])))
        if len(self.results.index.levels[1]) > 1:
            print('Total count of sub-tasks: {}'.format(len(self.results)))
        test_data_list = pd.unique(self.results['test_data'])
        if len(test_data_list) == 1:
            print('Test data: {}'.format(test_data_list[0]))

    def __repr__(self):
        return "ResultTable(results=\n{results})".format(results=self.results)

    def __str__(self):
        return self.to_string()

max_stars_repo_name: MBaltz/dival

max_issues_repo_name: MBaltz/dival

max_forks_repo_name: MBaltz/dival


text:
#-*- coding:utf-8 -*-
from scipy.io import savemat
import numpy as np
import struct

def loadImageSet(filename):

    binfile = open(filename, 'rb') # 读取二进制文件
    buffers = binfile.read()

    head = struct.unpack_from('>IIII', buffers, 0) # 取前4个整数，返回一个元组

    offset = struct.calcsize('>IIII')  # 定位到data开始的位置
    imgNum = head[1]
    width = head[2]
    height = head[3]

    bits = imgNum * width * height  # data一共有60000*28*28个像素值
    bitsString = '>' + str(bits) + 'B'  # fmt格式：'>47040000B'

    imgs = struct.unpack_from(bitsString, buffers, offset) # 取data数据，返回一个元组

    binfile.close()
    imgs = np.reshape(imgs, [imgNum, width * height]) # reshape为[60000,784]型数组

    return imgs,head

def loadLabelSet(filename):

    binfile = open(filename, 'rb') # 读二进制文件
    buffers = binfile.read()

    head = struct.unpack_from('>II', buffers, 0) # 取label文件前2个整形数

    labelNum = head[1]
    offset = struct.calcsize('>II')  # 定位到label数据开始的位置

    numString = '>' + str(labelNum) + "B" # fmt格式：'>60000B'
    labels = struct.unpack_from(numString, buffers, offset) # 取label数据

    binfile.close()
    labels = np.reshape(labels, (labelNum,1)) # 转型为列表(一维数组)

    return labels,head

if __name__ == "__main__":
    '''
    file1= 'train-images.idx3-ubyte'
    file2= 'train-labels.idx1-ubyte'
    imgs,data_head = loadImageSet(file1)
    labels,labels_head = loadLabelSet(file2)
    savemat('train',{'X':imgs,'y':labels})
    '''
    file1= 't10k-images.idx3-ubyte'
    file2= 't10k-labels.idx1-ubyte'
    imgs,data_head = loadImageSet(file1)
    labels,labels_head = loadLabelSet(file2)
    savemat('test',{'X':imgs,'y':labels})

ext: py

reason: import numpy,from scipy


text:
# coding: utf-8
import argparse
import time
import math
import os, sys
import json
import random

from pathlib import Path

import numpy as np

import torch
import torch.nn as nn
import torch.optim as optim
import sentencepiece as spm

import json_log_plots

from mem_transformer import MemTransformerLM
from utils.exp_utils import create_exp_dir
from utils.data_parallel import BalancedDataParallel

parser = argparse.ArgumentParser(description='PyTorch Transformer Language Model')
parser.add_argument('--data', type=str,
                    help='location of the data corpus')
parser.add_argument('--work_dir', type=str,
                    help='experiment work dir')
parser.add_argument('--sp_model', type=str,
                    help='sentencepiece model')
parser.add_argument('--n_layer', type=int, default=12,
                    help='number of total layers')
parser.add_argument('--n_head', type=int, default=10,
                    help='number of heads')
parser.add_argument('--d_head', type=int, default=50,
                    help='head dimension')
parser.add_argument('--d_embed', type=int, default=-1,
                    help='embedding dimension')
parser.add_argument('--d_model', type=int, default=500,
                    help='model dimension')
parser.add_argument('--d_inner', type=int, default=1000,
                    help='inner dimension in FF')
parser.add_argument('--dropout', type=float, default=0.0,
                    help='global dropout rate')
parser.add_argument('--dropatt', type=float, default=0.0,
                    help='attention probability dropout rate')
parser.add_argument('--init', default='normal', type=str,
                    help='parameter initializer to use.')
parser.add_argument('--emb_init', default='normal', type=str,
                    help='parameter initializer to use.')
parser.add_argument('--init_range', type=float, default=0.1,
                    help='parameters initialized by U(-init_range, init_range)')
parser.add_argument('--emb_init_range', type=float, default=0.01,
                    help='parameters initialized by U(-init_range, init_range)')
parser.add_argument('--init_std', type=float, default=0.02,
                    help='parameters initialized by N(0, init_std)')
parser.add_argument('--proj_init_std', type=float, default=0.01,
                    help='parameters initialized by N(0, init_std)')
parser.add_argument('--optim', default='adam', type=str,
                    choices=['adam', 'sgd', 'adagrad'],
                    help='optimizer to use.')
parser.add_argument('--lr', type=float, default=0.00025,
                    help='initial learning rate (0.00025|5 for adam|sgd)')
parser.add_argument('--mom', type=float, default=0.0,
                    help='momentum for sgd')
parser.add_argument('--scheduler', default='cosine', type=str,
                    choices=['cosine', 'inv_sqrt', 'dev_perf', 'constant'],
                    help='lr scheduler to use.')
parser.add_argument('--warmup_step', type=int, default=0,
                    help='upper warm up step limit')
parser.add_argument('--decay_rate', type=float, default=0.5,
                    help='decay factor when ReduceLROnPlateau is used')
parser.add_argument('--lr_min', type=float, default=0.0,
                    help='minimum learning rate during annealing')
parser.add_argument('--clip', type=float, default=0.25,
                    help='gradient clipping')
parser.add_argument('--clip_nonemb', action='store_true',
                    help='only clip the gradient of non-embedding params')
parser.add_argument('--num_epochs', type=int, default=10,
                    help='number of epochs to train, default: 10')
parser.add_argument('--batch_size', type=int, default=60,
                    help='batch size')
parser.add_argument('--tgt_len', type=int, default=70,
                    help='number of tokens to predict')
parser.add_argument('--eval_tgt_len', type=int, default=50,
                    help='number of tokens to predict for evaluation')
parser.add_argument('--ext_len', type=int, default=0,
                    help='length of the extended context')
parser.add_argument('--mem_len', type=int, default=0,
                    help='length of the retained previous heads')
parser.add_argument('--not_tied', action='store_true',
                    help='do not tie the word embedding and softmax weights')
parser.add_argument('--seed', type=int, default=1111,
                    help='random seed')
parser.add_argument('--cuda', action='store_true',
                    help='use CUDA')
parser.add_argument('--adaptive', action='store_true',
                    help='use adaptive softmax')
parser.add_argument('--div_val', type=int, default=1,
                    help='divident value for adapative input and softmax')
parser.add_argument('--pre_lnorm', action='store_true',
                    help='apply LayerNorm to the input instead of the output')
parser.add_argument('--log-interval', type=int, default=200,
                    help='report interval')
parser.add_argument('--eval-interval', type=int, default=4000,
                    help='evaluation interval')
parser.add_argument('--debug', action='store_true',
                    help='run in debug mode (do not create exp dir)')
parser.add_argument('--same_length', action='store_true',
                    help='use the same attn length for all tokens')
parser.add_argument('--attn_type', type=int, default=0,
                    help='attention type. 0 for ours, 1 for Shaw et al,'
                    '2 for Vaswani et al, 3 for Al Rfou et al.')
parser.add_argument('--clamp_len', type=int, default=-1,
                    help='use the same pos embeddings after clamp_len')
parser.add_argument('--eta_min', type=float, default=0.0,
                    help='min learning rate for cosine scheduler')
parser.add_argument('--gpu0_bsz', type=int, default=-1,
                    help='batch size on gpu 0')
parser.add_argument('--sample_softmax', type=int, default=-1,
                    help='number of samples in sampled softmax')
parser.add_argument('--patience', type=int, default=0,
                    help='patience')
parser.add_argument('--finetune_v2', action='store_true',
                    help='finetune v2')
parser.add_argument('--finetune_v3', action='store_true',
                    help='finetune v3')
args = parser.parse_args()
args.tied = not args.not_tied

if args.d_embed < 0:
    args.d_embed = args.d_model

assert args.ext_len >= 0, 'extended context length must be non-negative'

logging = create_exp_dir(args.work_dir,
    scripts_to_save=['train.py', 'mem_transformer.py'], debug=args.debug)

# Set the random seed manually for reproducibility.
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
    if not args.cuda:
        logging('WARNING: You have a CUDA device, so you should probably run with --cuda')
    else:
        torch.cuda.manual_seed_all(args.seed)

device = torch.device('cuda' if args.cuda else 'cpu')

###############################################################################
# Load data
###############################################################################

class SPVocab(object):

    def __init__(self, sp_model):
        self.sp_model = sp_model

    def get_symbols(self, indices):
        # return self.sp_model.DecodePieces(map(lambda id: self.sp_model.IdToPiece(id), indices.tolist()))
        return [ self.sp_model.IdToPiece(id) for id in indices.tolist() ]

    def len(self):
        return len(self.sp_model)

class SPCorpus(object):
    def __init__(self, dataset_path, sp_model_path, batch_size, tgt_len, device):

        self.batch_size = batch_size
        self.tgt_len    = tgt_len
        self.device     = device

        logging('Loading sentencepiece model from %s' % sp_model_path)
        self.sp_model = spm.SentencePieceProcessor()
        self.sp_model.load(sp_model_path)
        logging('Vocabulary size: %d' % len(self.sp_model))

        UNK = '<unk>'
        END_OF_LINE = '<endofline>'
        END_OF_TEXT = '<endoftext>'

        self.id_unk = self.sp_model.PieceToId(UNK)
        self.id_eot = self.sp_model.PieceToId(END_OF_TEXT)
        self.id_eol = self.sp_model.PieceToId(END_OF_LINE)

        logging ("sp ids: %s->%d %s->%d %s->%d" % (UNK, self.id_unk, END_OF_TEXT, self.id_eot, END_OF_LINE, self.id_eol))

        self.vocab = SPVocab(self.sp_model)

        logging(f'Loading dataset from {dataset_path}')
        valid_dataset = np.load(os.path.join(dataset_path, 'valid.npy'))
        train_dataset = np.load(os.path.join(dataset_path, 'train.npy'))
        test_dataset  = np.load(os.path.join(dataset_path, 'test.npy'))

        logging(f'Train      dataset has {len(train_dataset):,} tokens')
        logging(f'Validation dataset has {len(valid_dataset):,} tokens')
        logging(f'test       dataset has {len(test_dataset):,} tokens')

        self.valid = valid_dataset.astype(np.int64)
        self.train = train_dataset.astype(np.int64)
        self.test  = test_dataset.astype (np.int64)

    def get_vocab_size(self):
        return len(self.sp_model)

    def _get_data(self, split):
        if split == 'train':
            return self.train
        elif split == 'valid':
            return self.valid
        return self.test

    def get_num_batches(self, split):

        data = self._get_data(split)

        # number of tgt_len samples we can generate from data
        n_samples = (len(data)-1) // self.tgt_len
        logging ("number of %s samples: %d" % (split, n_samples))

        # Work out how cleanly we can divide the dataset into batch_size parts.
        n_batches = n_samples // self.batch_size
        logging ("number of %s batches: %d" % (split, n_batches))

        return n_batches

    def get_batch(self, split, i_batch):

        data = self._get_data(split)

        n_samples = (len(data)-1) // self.tgt_len

        n_batches = n_samples // self.batch_size

        i_batch = i_batch % n_batches

        # start indices (in tokens) for each batch
        batch_indices = [n_batches * self.tgt_len * bi + i_batch * self.tgt_len for bi in range(self.batch_size)]

        # print (batch_indices)

        batch_x = torch.LongTensor( [ [ data[batch_indices[bi] + ci    ] for bi in range(self.batch_size) ] for ci in range(self.tgt_len) ] ).to(self.device)
        batch_y = torch.LongTensor( [ [ data[batch_indices[bi] + ci + 1] for bi in range(self.batch_size) ] for ci in range(self.tgt_len) ] ).to(self.device)
        return batch_x, batch_y

corpus = SPCorpus(args.data, args.sp_model, args.batch_size, args.tgt_len, device)
ntokens = corpus.get_vocab_size()

n_batches_per_epoch = corpus.get_num_batches('train')

max_step = args.num_epochs * n_batches_per_epoch

# corpus.get_batch('train', 0)
# corpus.get_batch('train', 1)

args.n_token = ntokens

# eval_batch_size = 10
# tr_iter = corpus.get_iterator('train', args.batch_size, args.tgt_len,
#     device=device, ext_len=args.ext_len)
# va_iter = corpus.get_iterator('valid', eval_batch_size, args.eval_tgt_len,
#     device=device, ext_len=args.ext_len)
# te_iter = corpus.get_iterator('test', eval_batch_size, args.eval_tgt_len,
#     device=device, ext_len=args.ext_len)

# adaptive softmax / embedding
cutoffs, tie_projs = [], [False]
if args.adaptive:
    # cutoffs = [20000, 40000, 200000]
    cutoffs = [3500, 7500, 37500] # FIXME: make relative to ntokens
    tie_projs += [True] * len(cutoffs)

###############################################################################
# Build the model
###############################################################################
def init_weight(weight):
    if args.init == 'uniform':
        nn.init.uniform_(weight, -args.init_range, args.init_range)
    elif args.init == 'normal':
        nn.init.normal_(weight, 0.0, args.init_std)

def init_bias(bias):
    nn.init.constant_(bias, 0.0)

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Linear') != -1:
        if hasattr(m, 'weight') and m.weight is not None:
            init_weight(m.weight)
        if hasattr(m, 'bias') and m.bias is not None:
            init_bias(m.bias)
    elif classname.find('AdaptiveEmbedding') != -1:
        if hasattr(m, 'emb_projs'):
            for i in range(len(m.emb_projs)):
                if m.emb_projs[i] is not None:
                    nn.init.normal_(m.emb_projs[i], 0.0, args.proj_init_std)
    elif classname.find('Embedding') != -1:
        if hasattr(m, 'weight'):
            init_weight(m.weight)
    elif classname.find('ProjectedAdaptiveLogSoftmax') != -1:
        if hasattr(m, 'cluster_weight') and m.cluster_weight is not None:
            init_weight(m.cluster_weight)
        if hasattr(m, 'cluster_bias') and m.cluster_bias is not None:
            init_bias(m.cluster_bias)
        if hasattr(m, 'out_projs'):
            for i in range(len(m.out_projs)):
                if m.out_projs[i] is not None:
                    nn.init.normal_(m.out_projs[i], 0.0, args.proj_init_std)
    elif classname.find('LayerNorm') != -1:
        if hasattr(m, 'weight'):
            nn.init.normal_(m.weight, 1.0, args.init_std)
        if hasattr(m, 'bias') and m.bias is not None:
            init_bias(m.bias)
    elif classname.find('TransformerLM') != -1:
        if hasattr(m, 'r_emb'):
            init_weight(m.r_emb)
        if hasattr(m, 'r_w_bias'):
            init_weight(m.r_w_bias)
        if hasattr(m, 'r_r_bias'):
            init_weight(m.r_r_bias)
        if hasattr(m, 'r_bias'):
            init_bias(m.r_bias)

def update_dropout(m):
    classname = m.__class__.__name__
    if classname.find('Dropout') != -1:
        if hasattr(m, 'p'):
            m.p = args.dropout

def update_dropatt(m):
    if hasattr(m, 'dropatt'):
        m.dropatt.p = args.dropatt

model_pt_path = os.path.join(args.work_dir, 'cur_model.pt')

if os.path.exists(model_pt_path):

    logging ("restoring model from %s ..." % model_pt_path)

    with open(model_pt_path, 'rb') as f:
        model = torch.load(f)
    model.apply(update_dropout)
    model.apply(update_dropatt)

else:
    logging ("creating MemTransformerLM from scratch ...")

    model = MemTransformerLM(ntokens, args.n_layer, args.n_head, args.d_model,
        args.d_head, args.d_inner, args.dropout, args.dropatt,
        tie_weight=args.tied, d_embed=args.d_embed, div_val=args.div_val,
        tie_projs=tie_projs, pre_lnorm=args.pre_lnorm, tgt_len=args.tgt_len,
        ext_len=args.ext_len, mem_len=args.mem_len, cutoffs=cutoffs,
        same_length=args.same_length, attn_type=args.attn_type,
        clamp_len=args.clamp_len, sample_softmax=args.sample_softmax)

    model_args = {
        'ntokens'       : ntokens,
        'n_layer'       : args.n_layer,
        'n_head'        : args.n_head,
        'd_model'       : args.d_model,
        'd_head'        : args.d_head,
        'd_inner'       : args.d_inner,
        'dropout'       : args.dropout,
        'dropatt'       : args.dropatt,
        'tie_weight'    : args.tied,
        'd_embed'       : args.d_embed,
        'div_val'       : args.div_val,
        'tie_projs'     : tie_projs,
        'pre_lnorm'     : args.pre_lnorm,
        'tgt_len'       : args.tgt_len,
        'ext_len'       : args.ext_len,
        'mem_len'       : args.mem_len,
        'cutoffs'       : cutoffs,
        'same_length'   : args.same_length,
        'attn_type'     : args.attn_type,
        'clamp_len'     : args.clamp_len,
        'sample_softmax': args.sample_softmax,
        }

    logging ("model args: %s" % repr(model_args))

    args_path = os.path.join(args.work_dir, 'params.json')
    with open(args_path, 'w') as argsf:
        argsf.write(json.dumps(model_args))
    logging ("%s written." % args_path)

    model.apply(weights_init)
    model.word_emb.apply(weights_init) # ensure embedding init is not overridden by out_layer in case of weight sharing

args.n_all_param = sum([p.nelement() for p in model.parameters()])
args.n_nonemb_param = sum([p.nelement() for p in model.layers.parameters()])

para_model = model.to(device)

#### optimizer
if args.optim.lower() == 'sgd':
    if args.sample_softmax > 0:
        dense_params, sparse_params = [], []
        for param in model.parameters():
            if param.size() == model.word_emb.weight.size():
                sparse_params.append(param)
            else:
                dense_params.append(param)
        optimizer_sparse = optim.SGD(sparse_params, lr=args.lr * 2)
        optimizer = optim.SGD(dense_params, lr=args.lr, momentum=args.mom)
    else:
        optimizer = optim.SGD(model.parameters(), lr=args.lr,
            momentum=args.mom)
elif args.optim.lower() == 'adam':
    if args.sample_softmax > 0:
        dense_params, sparse_params = [], []
        for param in model.parameters():
            if param.size() == model.word_emb.weight.size():
                sparse_params.append(param)
            else:
                dense_params.append(param)
        optimizer_sparse = optim.SparseAdam(sparse_params, lr=args.lr)
        optimizer = optim.Adam(dense_params, lr=args.lr)
    else:
        optimizer = optim.Adam(model.parameters(), lr=args.lr)
elif args.optim.lower() == 'adagrad':
    optimizer = optim.Adagrad(model.parameters(), lr=args.lr)

#### scheduler
if args.scheduler == 'cosine':
    # here we do not set eta_min to lr_min to be backward compatible
    # because in previous versions eta_min is default to 0
    # rather than the default value of lr_min 1e-6
    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer,
        max_step, eta_min=args.eta_min) # should use eta_min arg
    if args.sample_softmax > 0:
        scheduler_sparse = optim.lr_scheduler.CosineAnnealingLR(optimizer_sparse,
            max_step, eta_min=args.eta_min) # should use eta_min arg
elif args.scheduler == 'inv_sqrt':
    # originally used for Transformer (in Attention is all you need)
    def lr_lambda(step):
        # return a multiplier instead of a learning rate
        if step == 0 and args.warmup_step == 0:
            return 1.
        else:
            return 1. / (step ** 0.5) if step > args.warmup_step \
                   else step / (args.warmup_step ** 1.5)
    scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_lambda)
elif args.scheduler == 'dev_perf':
    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
        factor=args.decay_rate, patience=args.patience, min_lr=args.lr_min)
    if args.sample_softmax > 0:
        scheduler_sparse = optim.lr_scheduler.ReduceLROnPlateau(optimizer_sparse,
            factor=args.decay_rate, patience=args.patience, min_lr=args.lr_min)
elif args.scheduler == 'constant':
    pass

optimizer_pt_path = os.path.join(args.work_dir, 'cur_optimizer.pt')

if os.path.exists(optimizer_pt_path):

    logging ("loading optimizer from %s ..." % optimizer_pt_path)

    with open(optimizer_pt_path, 'rb') as f:
        opt_state_dict = torch.load(f)
        optimizer.load_state_dict(opt_state_dict)

n_steps_txt_path = os.path.join(args.work_dir, 'n_steps.txt')
n_restart_step = 0
if os.path.exists(n_steps_txt_path):
    with open(n_steps_txt_path, 'r') as f:
        n_restart_step = int(f.read().strip())

logging ("restarting from step #%d" % n_restart_step)

logging('=' * 100)
for k in sorted(args.__dict__):
    logging('    - {} : {}'.format(k, args.__dict__[k]))
logging('=' * 100)
logging('#params = {}'.format(args.n_all_param))
logging('#non emb params = {}'.format(args.n_nonemb_param))

###############################################################################
# Training code
###############################################################################

def evaluate(split):
    # Turn on evaluation mode which disables dropout.
    model.eval()

    # If the model does not use memory at all, make the ext_len longer.
    # Otherwise, make the mem_len longer and keep the ext_len the same.
    if args.mem_len == 0:
        model.reset_length(args.eval_tgt_len,
            args.ext_len+args.tgt_len-args.eval_tgt_len, args.mem_len)
    else:
        model.reset_length(args.eval_tgt_len,
            args.ext_len, args.mem_len+args.tgt_len-args.eval_tgt_len)

    max_eval_step = corpus.get_num_batches(split)

    # Evaluation
    total_loss = 0.0
    with torch.no_grad():
        mems = tuple()

        for eval_step in range(max_eval_step):
        # for i, (data, target, seq_len) in enumerate(eval_iter):

            data, target = corpus.get_batch(split, eval_step)

            ret = model(data, target, *mems)
            loss, mems = ret[0], ret[1:]
            loss = loss.mean()
            total_loss += loss.float().item()

            if (eval_step > 0) and (eval_step % 10 == 0):
                print ("| eval batch %6d/%6d [%7.2f%%] | avg loss: %7.3f" % (eval_step, max_eval_step, eval_step * 100.0 / max_eval_step, total_loss / eval_step))

    # Switch back to the training mode
    model.reset_length(args.tgt_len, args.ext_len, args.mem_len)
    model.train()

    return total_loss / max_eval_step

def save_model(model_name):

    logging ("saving %s model..." % model_name)
    with open(os.path.join(args.work_dir, '%s_model.pt' % model_name), 'wb') as f:
        torch.save(model, f)
    with open(os.path.join(args.work_dir, '%s_optimizer.pt' % model_name), 'wb') as f:
        torch.save(optimizer.state_dict(), f)
    with open(os.path.join(args.work_dir, '%s_state_dict.pt' % model_name), 'wb') as f:
        torch.save (model.state_dict(), f)
    logging ("saving %s model... done." % model_name)

def train():
    # Turn on training mode which enables dropout.
    global train_loss, best_val_loss, eval_start_time, log_start_time, corpus, n_batches_per_epoch, max_step
    model.train()
    mems = tuple()
    sample_from_batch = 0

    for train_step in range(n_restart_step, max_step):

        data, target = corpus.get_batch('train', train_step)

        model.zero_grad()

        # logging ("=====>", data, target, seq_len)
        # logging ("%%%%%% data.shape=", data.shape, "target.shape=", target.shape)

        # logging ( list(zip(data[:, 3].tolist(), corpus.vocab.get_symbols(data[:, 3])  )) )
        # logging ( "====> data   :", data[:, 3] )
        # logging ( "====> target :", target[:, 3] )

        # print ( "====> data   [batch #%02d]: %s" % (sample_from_batch, ' '.join(corpus.vocab.get_symbols(data[:, sample_from_batch]))[:120] ))
        # print ( "====> data   [batch #%02d]: %s" % (sample_from_batch, ' '.join(corpus.vocab.get_symbols(data[:, sample_from_batch])) ))
        # print ( "====> target [batch #%02d]: %s" % (sample_from_batch, ' '.join(corpus.vocab.get_symbols(target[:, sample_from_batch]))[:120] ))

        ret = para_model(data, target, *mems)
        loss, mems = ret[0], ret[1:]
        loss = loss.float().mean().type_as(loss)
        loss.backward()
        train_loss += loss.float().item()

        torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)

        optimizer.step()
        if args.sample_softmax > 0:
            optimizer_sparse.step()

        # step-wise learning rate annealing
        if args.scheduler in ['cosine', 'constant', 'dev_perf']:
            # linear warmup stage
            if train_step < args.warmup_step:
                curr_lr = args.lr * train_step / args.warmup_step
                optimizer.param_groups[0]['lr'] = curr_lr
                if args.sample_softmax > 0:
                    optimizer_sparse.param_groups[0]['lr'] = curr_lr * 2
            else:
                if args.scheduler == 'cosine':
                    scheduler.step(train_step)
                    if args.sample_softmax > 0:
                        scheduler_sparse.step(train_step)
        elif args.scheduler == 'inv_sqrt':
            scheduler.step(train_step)

        if (train_step>n_restart_step) and (train_step % args.log_interval == 0):
            cur_loss = train_loss / args.log_interval
            elapsed = time.time() - log_start_time

            epoch = train_step / n_batches_per_epoch + 1

            log_str = '| epoch %2d/%2d batch %6d/%6d [%7.2f%%] | lr %.3g | ms/batch %5.2f | loss %7.3f' % (
                epoch, args.num_epochs, train_step % n_batches_per_epoch, n_batches_per_epoch, train_step * 100.0 / max_step,
                optimizer.param_groups[0]['lr'],
                elapsed * 1000 / args.log_interval, cur_loss)

            log_str += ' | ppl {:9.3f}'.format(math.exp(cur_loss))
            logging(log_str)

            json_log_plots.write_event(Path(args.work_dir), step=train_step, loss=cur_loss, lr=optimizer.param_groups[0]['lr']*100000.0)

            train_loss = 0
            log_start_time = time.time()

            sample_from_batch = random.randint(0, args.batch_size-1)

        if (train_step>n_restart_step) and (train_step % args.eval_interval == 0):
            save_model('cur')
            with open(n_steps_txt_path, 'w') as f:
                f.write("%s\n" % train_step)

            logging ("evaluating model...")
            val_loss = evaluate('valid')

            json_log_plots.write_event(Path(args.work_dir), step=train_step, val_loss=val_loss)

            # Save the model if the validation loss is the best we've seen so far.
            if not best_val_loss or val_loss < best_val_loss:
                logging ("best valid loss so far.")
                save_model('valid')
                best_val_loss = val_loss

            # # dev-performance based learning rate annealing
            # if args.scheduler == 'dev_perf':
            #     scheduler.step(val_loss)
            #     if args.sample_softmax > 0:
            #         scheduler_sparse.step(val_loss)

            # eval_start_time = time.time()

# Loop over epochs.
train_loss = 0
best_val_loss = None

log_start_time = time.time()
eval_start_time = time.time()

logging ("training starts...")

# At any point you can hit Ctrl + C to break out of training early.
try:
    train()
    logging('Training finished.')
except KeyboardInterrupt:
    logging('-' * 100)
    logging('Exiting from training early')

# FIXME

# logging('Load the best saved model.')
# with open(os.path.join(args.work_dir, 'model.pt'), 'rb') as f:
#     model = torch.load(f)
# para_model = model.to(device)
#
# logging('Run on test data.')
# test_loss = evaluate(te_iter)
# logging('=' * 100)
# logging('| End of training | test loss {:5.2f} | test ppl {:9.3f}'.format(
#     test_loss, math.exp(test_loss)))
# logging('=' * 100)

max_stars_repo_name: gooofy/transformer-xl

max_issues_repo_name: gooofy/transformer-xl

max_forks_repo_name: gooofy/transformer-xl

reason: import numpy


text:
from pathlib import Path

import numpy as np
from PIL import Image
import torch
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms

class PennFudanDataset(Dataset):
    """Torch dataset for PennFudan Pedestrians dataset.
    Reference: https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html

    Args:
        root (str): Path to folder containing dataset
        transforms (): Transforms
    """

    def __init__(self, root, transforms):
        self.root = Path(root)
        self.transforms = transforms

        self.images_folder = self.root / "PNGImages"
        self.masks_folder = self.root / "PedMasks"

        # load all image files, sorting them to
        # ensure that they are aligned
        self.img_paths = list(sorted(self.images_folder.glob("*.png")))
        self.mask_paths = list(sorted(self.masks_folder.glob("*.png")))

    def __getitem__(self, idx):
        img_path = self.img_paths[idx]
        mask_path = self.mask_paths[idx]

        img = Image.open(img_path).convert("RGB")
        mask = Image.open(mask_path)
        mask = np.array(mask)

        obj_ids = np.unique(mask)[1:]
        # split one mask with all instances into multiple binary masks
        masks = mask == obj_ids[:, None, None]

        num_objects = len(obj_ids)

        boxes = []
        for m in masks:
            pos = np.where(m)
            xmin = np.min(pos[1])
            xmax = np.max(pos[1])
            ymin = np.min(pos[0])
            ymax = np.max(pos[0])
            boxes.append([xmin, ymin, xmax, ymax])
        boxes = torch.as_tensor(boxes)

        # all labels are 1
        labels = torch.ones((num_objects,), dtype=torch.int64)

        target = {}
        target["image_id"] = 0
        target["boxes"] = boxes
        target["labels"] = labels

        if self.transforms is not None:
            img = self.transforms(img)

        return img, target

    def __len__(self):
        return len(self.img_paths)

def get_transform(train):
    tfms = []
    tfms.append(transforms.ToTensor())
    tfms.append(transforms.Resize((448, 448)))

    if train:
        tfms.append(transforms.RandomHorizontalFlip(0.5))
    return transforms.Compose(tfms)

def get_dataloader(train=True):
    dataset = PennFudanDataset(
        root="../../data/PennFudanPed", transforms=get_transform(train=train)
    )

    data_loader = DataLoader(dataset, batch_size=2, shuffle=True, collate_fn=collate_fn)
    return data_loader

def collate_fn(batch):
    """Taken from https://github.com/pytorch/vision/blob/27745e5be086ecdf2a8e609d65c13969cb5201f7/references/detection/utils.py#L203"""
    return tuple(zip(*batch))

max_stars_repo_name: santhisenan/yolo

max_issues_repo_name: santhisenan/yolo

max_forks_repo_name: santhisenan/yolo


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

class MAE(nn.Module):
    def __init__(self):
        super(MAE, self).__init__()

    def forward(self, outputs, target, *args):
        val_pixels = (target > 0).float().cuda()
        err = torch.abs(target * val_pixels - outputs * val_pixels)
        loss = torch.sum(err.view(err.size(0), 1, -1), -1, keepdim=True)
        cnt = torch.sum(val_pixels.view(val_pixels.size(0), 1, -1), -1, keepdim=True)
        return torch.mean(loss / cnt)

class RMSE(nn.Module):
    def __init__(self):
        super(RMSE, self).__init__()

    def forward(self, outputs, target, *args):
        val_pixels = (target > 0).float().cuda()
        err = (target * val_pixels - outputs * val_pixels) ** 2
        loss = torch.sum(err.view(err.size(0), 1, -1), -1, keepdim=True)
        cnt = torch.sum(val_pixels.view(val_pixels.size(0), 1, -1), -1, keepdim=True)
        return torch.mean(torch.sqrt(loss / cnt))

class Deltas(nn.Module):
    def __init__(self):
        super(Deltas, self).__init__()

    def forward(self, outputs, target, *args):
        val_pixels = (target > 0).float().cuda()
        rel = torch.max((target * val_pixels) / (outputs * val_pixels + 1e-3),
                        (outputs * val_pixels) / (target * val_pixels))

        cnt = torch.sum(val_pixels.view(val_pixels.size(0), 1, -1), -1, keepdim=True)

        def del_i(i):
            r = (rel < 1.01 ** i).float()
            delta = torch.sum(r.view(r.size(0), 1, -1), -1, keepdim=True) / cnt
            return torch.mean(delta)

        return del_i(1), del_i(2), del_i(3)

class Huber(nn.Module):
    def __init__(self):
        super(Huber, self).__init__()

    def forward(self, outputs, target, delta=5):
        l1_loss = F.l1_loss(outputs, target, reduce=False)
        mse_loss = F.mse_loss(outputs, target, reduce=False)

        mask = (l1_loss < delta).float()

        loss = (0.5 * mse_loss) * mask + delta * (l1_loss - 0.5 * delta) * (1 - mask)

        return torch.mean(loss)

class EPE_metric(nn.Module):
    def __init__(self):
        super(EPE_metric, self).__init__()

    def forward(self, outputs, target):
        mask = (target > 0)
        outputs, target = outputs[mask], target[mask]
        err = torch.abs(target - outputs)
        loss = torch.mean(err)
        return loss

class D1_metric(nn.Module):
    def __init__(self):
        super(D1_metric, self).__init__()

    def forward(self, outputs, target):
        mask = (target > 0)
        outputs, target = outputs[mask], target[mask]
        E = torch.abs(outputs - target)
        # err_mask = (E > 3) & (E / target.abs() > 0.05)
        err_mask = (E > 3)
        return torch.mean(err_mask.float())

class Thres_metric(nn.Module):
    def __init__(self):
        super(Thres_metric, self).__init__()

    def forward(self, outputs, target):
        mask_tar = (target > 0)
        mask_out = (outputs > 0)
        mask = mask_tar * mask_out
        # mask = (target > 0)
        thres = 3
        assert isinstance(thres, (int, float))
        outputs, target = outputs[mask], target[mask]
        E = torch.abs(target - outputs)
        err_mask = (E > thres)
        return torch.mean(err_mask.float())

class Deltas_Paint(nn.Module):
    def __init__(self):
        super(Deltas_Paint, self).__init__()

    def forward(self, outputs, target, *args):
        val_pixels = (target > 0).float().cuda()
        rel = torch.max((target * val_pixels) / (outputs * val_pixels + 1e-3),
                        (outputs * val_pixels) / (target * val_pixels))

        cnt = torch.sum(val_pixels.view(val_pixels.size(0), 1, -1), -1, keepdim=True)

        def del_i(i):
            r = (rel < 1.25 ** i).float()
            delta = torch.sum(r.view(r.size(0), 1, -1), -1, keepdim=True) / cnt
            return torch.mean(delta)

        def del_j(i):
            r = (rel < i).float()
            delta = torch.sum(r.view(r.size(0), 1, -1), -1, keepdim=True) / cnt
            return torch.mean(delta)

        return del_j(1.05), del_j(1.10), del_i(1), del_i(2), del_i(3), cnt

class SSIM_Metric(nn.Module):
    def __init__(self):
        super(SSIM_Metric, self).__init__()

    def forward(self, x, y):
        C1 = 0.01 ** 2
        C2 = 0.03 ** 2

        mu_x = F.avg_pool2d(x, 3, 1, 1)
        mu_y = F.avg_pool2d(y, 3, 1, 1)

        sigma_x = F.avg_pool2d(x ** 2, 3, 1, 1) - mu_x ** 2
        sigma_y = F.avg_pool2d(y ** 2, 3, 1, 1) - mu_y ** 2
        sigma_xy = F.avg_pool2d(x * y, 3, 1, 1) - mu_x * mu_y

        SSIM_n = (2 * mu_x * mu_y + C1) * (2 * sigma_xy + C2)
        SSIM_d = (mu_x ** 2 + mu_y ** 2 + C1) * (sigma_x + sigma_y + C2)

        SSIM = SSIM_n / SSIM_d

        # return torch.clamp((1 - SSIM) / 2, 0, 1)
        return SSIM.mean(), torch.tensor([torch.numel(x)])

class MAE_Paint(nn.Module):
    def __init__(self):
        super(MAE_Paint, self).__init__()

    def forward(self, outputs, target, *args):
        val_pixels = (target > 0).float().cuda()
        err = torch.abs(target * val_pixels - outputs * val_pixels)
        loss = torch.sum(err.view(err.size(0), 1, -1), -1, keepdim=True)
        cnt = torch.sum(val_pixels.view(val_pixels.size(0), 1, -1), -1, keepdim=True)
        return torch.mean(loss / torch.numel(outputs)), torch.tensor([torch.numel(outputs)])

class RMSE_Paint(nn.Module):
    def __init__(self):
        super(RMSE_Paint, self).__init__()

    def forward(self, outputs, target, *args):
        val_pixels = (target > 0).float().cuda()
        err = (target * val_pixels - outputs * val_pixels) ** 2
        loss = torch.sum(err.view(err.size(0), 1, -1), -1, keepdim=True)
        cnt = torch.sum(val_pixels.view(val_pixels.size(0), 1, -1), -1, keepdim=True)
        return (loss / torch.numel(outputs)), torch.tensor([torch.numel(outputs)])

max_stars_repo_name: caozidong/Depth-Completion

max_issues_repo_name: caozidong/Depth-Completion

max_forks_repo_name: caozidong/Depth-Completion


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Jan 14 23:32:57 2021

Author: Josef Perktold
License: BSD-3

"""

# import numpy as np
from numpy.testing import assert_allclose
import pytest

import statsmodels.sandbox.distributions.copula as cop

ev_list = [
    [cop.transform_bilogistic, 0.5, 0.9, (0.25, 0.05), 0.5],
    [cop.transform_tawn, 0.5, 0.9, (0.5, 0.5, 0.5), 0.4724570876035117],
    # note evd has asymmetry reversed, interchange variables
    [cop.transform_tawn2, 0.9, 0.5, (0.25, 0.05), 0.464357480263932],
    [cop.transform_tawn2, 0.9, 0.5, (0.5, 0.25), 0.4916117128670654],
    [cop.transform_tawn2, 0.5, 0.9, (0.5, 0.25), 0.48340673415789],
    # note evd has parameter for hr 1/lmbda (inverse of our parameter)
    [cop.transform_hr, 0.5, 0.9, (2,), 0.4551235014298542],
    [cop.transform_joe, 0.5, 0.9, (0.5, 0.75, 1/0.25), 0.4543698299835434],
    [cop.transform_joe, 0.9, 0.5, (0.5, 0.75, 1/0.25), 0.4539773435983587],

    # tev is against R `copula` package
    # > cop = tevCopula(0.8, df = 4)
    # > pCopula(c(0.5, 0.75), cop)
    # [1] 0.456807960674953
    # > pCopula(c(0.5, 0.9), cop)
    # [1] 0.4911039761533587
    [cop.transform_tev, 0.5, 0.75, (0.8, 4), 0.456807960674953],
    [cop.transform_tev, 0.5, 0.9, (0.8, 4), 0.4911039761533587],
    ]

cop_list = [
    [cop.TransfFrank, 0.5, 0.9, (2,), 0.4710805107852225, 0.9257812360337806],
    [cop.TransfGumbel, 0.5, 0.9, (2,), 0.4960348880595387, 0.3973548776136501],
    [cop.TransfClayton, 0.5, 0.9, (2,), 0.485954322440435, 0.8921974147432954],
    [cop.TransfIndep, 0.5, 0.5, (), 0.25, 1],
    ]

@pytest.mark.parametrize("case", ev_list)
def test_ev_copula(case):
    # check ev copulas, cdf and transform against R `evt` package
    ev_tr, v1, v2, args, res1 = case
    res = cop.copula_bv_ev(v1, v2, ev_tr, args=args)
    assert_allclose(res, res1, rtol=1e-13)

@pytest.mark.parametrize("case", cop_list)
def test_copulas(case):
    # check ev copulas, cdf and transform against R `copula` package
    cop_tr, v1, v2, args, cdf2, pdf2 = case
    ca = cop.CopulaArchimedean(cop_tr())
    cdf1 = ca.cdf([v1, v2], args=args)
    pdf1 = ca.pdf([v1, v2], args=args)
    assert_allclose(cdf1, cdf2, rtol=1e-13)
    assert_allclose(pdf1, pdf2, rtol=1e-13)

max_stars_repo_name: NolanMP/statsmodels

max_issues_repo_name: NolanMP/statsmodels

max_forks_repo_name: NolanMP/statsmodels

reason: import numpy,from numpy,import statsmodels


text:
# Bar plots are easy to make too, but do require some adjusted thinking
import matplotlib.pyplot as plt
import numpy as np

# get some random data
data = np.random.randint(20,size=15)
figure = plt.figure()
myplot = figure.add_subplot(1, 1, 1)

# First you will need to get an index.
index = [i for i in range(len(data))]
myplot.bar(index, data)
plt.show()

max_stars_repo_name: vierth/humanitiesTutorial

max_issues_repo_name: vierth/humanitiesTutorial

max_forks_repo_name: vierth/humanitiesTutorial