text:
import curses
import numpy as np

class Colors(object):
    def __init__(self):
        self.GREEN    = '\033[0;32m'
        self.HEADER = '\033[95m'
        self.OKBLUE = '\033[94m'
        self.OKGREEN = '\033[92m'
        self.WARNING = '\033[93m'
        self.RED = '\033[91m'
        self.ENDC = '\033[0m'
        #curses.setupterm()
        #if curses.tigetnum("colors") != 256:
        #    self.disable()

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.RED = ''
        self.ENDC = ''

def output_pass_fail(gnd):
    c = colors()
    return 'Pass: ' + c.GREEN   + str(np.sum(gnd==1)) + c.ENDC + \
          ' Fail: ' + c.WARNING + str(np.sum(gnd==0)) + c.ENDC

max_stars_repo_name: brucedispassion/qikify

max_issues_repo_name: brucedispassion/qikify

max_forks_repo_name: brucedispassion/qikify


text:
def df(model, model_ID, feature_headers):
    """
    Plot the feature importance for a given model

    Arguments:
    ----------
        model: the model object from which the feature importance will be derived
        model_ID: the type/ID of the model
            - valid arguments: 'RandomForest', 'DecisionTree'
        feature_headers: list of feature headers for labeling

    Returns:
    --------
        df_feat_import: Pandas DataFrame with the feature names, importance, and standard deviation, in the case of ensemble models.
    """
    import pandas as pd
    import numpy as np

    df_feat_import= pd.DataFrame(np.array(feature_headers).reshape(-1,1), columns = ['feature'])
    df_feat_import['importance'] = model.feature_importances_.reshape(-1,1)

    if model_ID in ['RandomForest']: #add standard dev of feat importances
        df_feat_import['stddev'] = np.std([tree.feature_importances_ for tree in model.estimators_],axis=0)
    else:
        df_feat_import['stddev']=0

    df_feat_import = df_feat_import.sort_values('importance').reset_index(drop=True)

    return df_feat_import

ext: py

reason: import numpy


text:
""" Plot wrangled B-factors

    This script plots experimental, MD and ENM wrangeled B-factor data.

    This script requires that packgaes within the `comp-biophys` environment
    be installed within the Pyhton environment you are running this script in.
    See `ccenv.yml` for more deatils.

    This file can also be imported as a module and contains the following
    functions:
        * main - the main function of the script
        * import_summary_data - imports and wragles .mdout summary data

"""
import click
import logging
from pathlib import Path
from dotenv import find_dotenv, load_dotenv

import pandas as pd
import numpy as np
from os.path import join as join_paths, basename as get_basename
import glob

import matplotlib as mpl
import matplotlib.pyplot as plt

import sys
sys.path.append("./src")
import src.utilities as utils

@click.command()
@click.argument('input_dir', type=click.Path(exists=True))
@click.argument('output_dir', type=click.Path())
def main_commandline(input_dir, output_dir):
    """ The main command line function.
        Uses `click` decorators.

        Parameters
        ----------
        input_dir : str
            The location of the input directory
        output_dir : str
            The location of the output directory

        Returns
        -------
        None

    """
    logger = logging.getLogger(__name__)
    logger.info('plot eigenvalues')

    main(input_dir, output_dir)

    return None

###################################################################################

def main(input_dir, output_dir):
    """ The main fucniton of this script.

        Parameters
        ----------
        input_dir : str
            The location of the input directory
        output_dir : str
            The location of the output directory

        Returns
        -------
        None

    """
    config = utils.read_config()
    mpl.rcParams.update(mpl.rcParamsDefault)
    # plt.style.use(config['viz']['default'])
    color_cycler = plt.rcParams['axes.prop_cycle'].by_key()['color']

    MD_EIGVALS_PATH   = "data/05-analysis/eigvals.dat"
    ENM_FREQ_PATH   = "data/external/mode.frequencies"
    ENM_EIGVALS_PATH  = "data/external/eigenvalues"

    eigvals_md = pd.read_fwf(MD_EIGVALS_PATH, infer_nrows=7000, header=None)
    eigvals_md.columns = ['mode_number', 'eigenvalue']
    # eigvals_md['eigenvalue'] = np.sqrt(eigvals_md['eigenvalue'].to_numpy())
    eigvals_md['mode_number'] = eigvals_md['mode_number'] + 6
    eigvals_md.set_index('mode_number', drop=True, inplace=True)

    # eigvals_enm = pd.read_csv(ENM_EIGVALS_PATH, index_col='mode_number')

    freq_enm = pd.read_csv(ENM_FREQ_PATH, comment='#', header=None)
    freq_enm.columns = ['eigenvalue']
    freq_enm['mode_number'] = np.arange(1,freq_enm.shape[0]+1)
    freq_enm.set_index('mode_number', drop=True, inplace=True)
    freq_enm = freq_enm[6:]

    plot_modes = 1197
    plot_modes += 6

    # PLOT
    _, axs = plt.subplots(2,2, figsize=(6,6), constrained_layout=True)
    ax = axs[0][0]
    ax.scatter(eigvals_md[:106].index, eigvals_md[:106]['eigenvalue'],
        label='MD', color=color_cycler[0], s=5)
    ax.scatter(freq_enm[:106].index, freq_enm[:106]['eigenvalue'],
        label='ENM', color=color_cycler[1], s=5)
    ax.set_ylabel("$\lambda$ [$cm^{-1}$]")
    ax.set_xlabel("Mode number")
    ax.set_title("First real 100 modes")
    ax.legend(frameon=False)

    eigvals_ratio = eigvals_md['eigenvalue'] / freq_enm['eigenvalue']
    ax = axs[0][1]
    ax.scatter(eigvals_md[:plot_modes].index, eigvals_md[:plot_modes]['eigenvalue'],
        label='MD', color=color_cycler[0], s=5)
    ax.scatter(freq_enm[:plot_modes].index, freq_enm[:plot_modes]['eigenvalue'],
        label='ENM', color=color_cycler[1], s=5)
    # ax.set_ylabel("$\lambda$ [$cm^{-1}$]")
    ax.set_xlabel("Mode number")
    ax.set_title("All modes")
    # ax.legend(frameon=False)

    no_bins = 25
    no_modes = len(eigvals_md.index)
    hist_kwargs = dict( histtype='step',
                        alpha=1,
                        # color = colourWheel[j%len(colourWheel)],
                        # linestyle = lineStyles_hist[j%len(lineStyles_hist)],
                        # density=True,
                        # label = column_name,
                        bins=no_bins)

    ax = axs[1][0]
    ax.hist(eigvals_md['eigenvalue'].to_numpy(), **hist_kwargs,
        color=color_cycler[0], label="MD")
    ax.set_ylabel("# modes per bin")
    ax.set_xlabel("$\lambda$ [$cm^{-1}$]")
    ax.set_title("# bins = {} | # modes = {}".format(no_bins, no_modes))
    ax.legend(frameon=False)

    ax = axs[1][1]
    ax.hist(freq_enm['eigenvalue'][:no_modes+6].to_numpy(), **hist_kwargs,
        color=color_cycler[1], label="ENM")
    # ax.set_ylabel("# modes per bin")
    ax.set_xlabel("$\lambda$ [$cm^{-1}$]")
    ax.legend(frameon=False)

    plt.show()

    plt.savefig(join_paths(output_dir, "eigvals.pdf"), bbox_inches='tight')
    plt.savefig(join_paths(output_dir, "eigvals.png"), bbox_inches='tight')

    # fig, axs = plt.subplots(2,2, figsize=(6,6), constrained_layout=True)

    # ax = axs[0][0]
    # ax.scatter(eigvals_md[:plot_modes].index, eigvals_md[:plot_modes]['eigenvalue'],
    #     label='MD', color=color_cycler[0], s=5)
    # ax.scatter(eigvals_enm[:plot_modes].index, eigvals_enm[:plot_modes]['eigenvalue'],
    #     label='ENM', color=color_cycler[1], s=5)
    # ax.set_ylabel("$\lambda$ [$cm^{-1}$]")
    # ax.set_xlabel("Mode number")
    # ax.set_title("Eigenvalues")
    # ax.legend(frameon=False)

    # eigvals_ratio = eigvals_md['eigenvalue'] / eigvals_enm['eigenvalue']
    # ax = axs[0][1]
    # ax.scatter(eigvals_ratio[:plot_modes].index, eigvals_ratio[:plot_modes],
    #     color=color_cycler[2], s=5)
    # ax.set_ylabel("$\lambda_{MD}/\lambda_{ENM}$")
    # ax.set_xlabel("Mode number")
    # ax.set_title("MD/ENM ratio")

    # no_bins = 35
    # hist_kwargs = dict( histtype='step',
    #                     alpha=1,
    #                     # color = colourWheel[j%len(colourWheel)],
    #                     # linestyle = lineStyles_hist[j%len(lineStyles_hist)],
    #                     # density=True,
    #                     # label = column_name,
    #                     bins=no_bins)

    # ax = axs[1][0]
    # ax.hist(eigvals_md['eigenvalue'][:950].to_numpy(), **hist_kwargs,
    #     color=color_cycler[0])
    # ax.set_ylabel("# modes per bin")
    # ax.set_xlabel("$\lambda$ [$cm^{-1}$]")
    # ax.set_title("MD DOS | # bins = {}".format(no_bins))

    # ax = axs[1][1]
    # ax.hist(eigvals_enm['eigenvalue'][:950].to_numpy(), **hist_kwargs,
    #     color=color_cycler[1])
    # ax.set_ylabel("# modes per bin")
    # ax.set_xlabel("$\lambda$ [?]")
    # ax.set_title("ENM DOS | # bins = {}".format(no_bins))
    # plt.show()

    # plt.savefig(join_paths(output_dir, "eigvals.eigvals.pdf"), bbox_inches='tight')
    # plt.savefig(join_paths(output_dir, "eigvals.eigvals.png"), bbox_inches='tight')

    return None

if __name__ == '__main__':
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    # not used in this stub but often useful for finding various files
    project_dir = Path(__file__).resolve().parents[2]

    # find .env automagically by walking up directories until it's found, then
    # load up the .env entries as environment variables
    load_dotenv(find_dotenv())

    main_commandline()

max_stars_repo_name: igordub/all-atom-md-cap

max_stars_repo_licenses: MIT Unlicense

max_issues_repo_name: igordub/all-atom-md-cap

max_issues_repo_licenses: MIT Unlicense

max_forks_repo_name: igordub/all-atom-md-cap

max_forks_repo_licenses: MIT Unlicense


text:
import numpy as np
from scipy import stats

from river.base import DriftDetector

class KSWIN(DriftDetector):
    """ Kolmogorov-Smirnov Windowing method for concept drift detection.

    Parameters
    ----------
    alpha
        Probability for the test statistic of the Kolmogorov-Smirnov-Test. The alpha parameter is
        very sensitive, therefore should be set below 0.01.
    window_size
        Size of the sliding window.
    stat_size
        Size of the statistic window.
    window
        Already collected data to avoid cold start.

    Notes
    -----
    KSWIN (Kolmogorov-Smirnov Windowing) is a concept change detection method based
    on the Kolmogorov-Smirnov (KS) statistical test. KS-test is a statistical test with
    no assumption of underlying data distribution. KSWIN can monitor data or performance
    distributions. Note that the detector accepts one dimensional input as array.

    KSWIN maintains a sliding window $\Psi$ of fixed size $n$ (window_size). The
    last $r$ (stat_size) samples of $\Psi$ are assumed to represent the last
    concept considered as $R$. From the first $n-r$ samples of $\Psi$,
    $r$ samples are uniformly drawn, representing an approximated last concept $W$.

    The KS-test is performed on the windows $R$ and $W$ of the same size. KS
    -test compares the distance of the empirical cumulative data distribution $dist(R,W)$.

    A concept drift is detected by KSWIN if:

    $$
    dist(R,W) > \sqrt{-\frac{ln\alpha}{r}}
    $$

    The difference in empirical data distributions between the windows $R$ and $W$ is too large
    since $R$ and $W$ come from the same distribution.

    Examples
    --------
    >>> import numpy as np
    >>> from river.drift import KSWIN
    >>> np.random.seed(12345)

    >>> kswin = KSWIN()

    >>> # Simulate a data stream composed by two data distributions
    >>> data_stream = np.concatenate((np.random.randint(2, size=1000),
    ...                               np.random.randint(4, high=8, size=1000)))

    >>> # Update drift detector and verify if change is detected
    >>> for i, val in enumerate(data_stream):
    ...     in_drift, in_warning = kswin.update(val)
    ...     if in_drift:
    ...         print(f"Change detected at index {i}, input value: {val}")
    Change detected at index 1014, input value: 5
    Change detected at index 1828, input value: 6

    References
    ----------
    [^1]: Christoph Raab, Moritz Heusinger, Frank-Michael Schleif, Reactive Soft Prototype Computing for Concept Drift Streams, Neurocomputing, 2020,

    """

    def __init__(self, alpha=0.005, window_size=100, stat_size=30, window=None):
        super().__init__()
        self.window_size = window_size
        self.stat_size = stat_size
        self.alpha = alpha
        self.p_value = 0
        self.n = 0
        if self.alpha < 0 or self.alpha > 1:
            raise ValueError("Alpha must be between 0 and 1")

        if self.window_size < 0:
            raise ValueError("window_size must be greater than 0")

        if self.window_size < self.stat_size:
            raise ValueError("stat_size must be smaller than window_size")

        if not isinstance(window, np.ndarray) or window is None:
            self.window = np.array([])
        else:
            self.window = window

    def update(self, value) -> tuple:
        """Update the change detector with a single data point.

        Adds an element on top of the sliding window and removes the oldest one from the window.
        Afterwards, the KS-test is performed.

        Parameters
        ----------
        value
            New data sample the sliding window should add.

        Returns
        -------
        A tuple (drift, warning) where its elements indicate if a drift or a warning is detected.

        """
        self.n += 1
        currentLength = self.window.shape[0]
        if currentLength >= self.window_size:
            self.window = np.delete(self.window, 0)
            rnd_window = np.random.choice(self.window[:-self.stat_size], self.stat_size)

            (st, self.p_value) = stats.ks_2samp(rnd_window,
                                                self.window[-self.stat_size:], mode="exact")

            if self.p_value <= self.alpha and st > 0.1:
                self._in_concept_change = True
                self.window = self.window[-self.stat_size:]
            else:
                self._in_concept_change = False
        else:  # Not enough samples in sliding window for a valid test
            self._in_concept_change = False

        self.window = np.concatenate([self.window, [value]])

        return self._in_concept_change, self._in_warning_zone

    def reset(self):
        """Reset the change detector."""
        self.p_value = 0
        self.window = np.array([])
        self._in_concept_change = False

max_stars_repo_name: brcharron/creme

max_issues_repo_name: brcharron/creme

max_forks_repo_name: brcharron/creme

reason: import numpy,from scipy


text:
"""
Trains an HGAN, and visulizes results

Author(s): Wei Chen (wchen459@umd.edu)
"""

import argparse
import os.path
import json
import numpy as np
from importlib import import_module

from shape_plot import plot_samples, plot_grid
from metrics.diversity import ci_rdiv
from metrics.mmd import ci_mmd
from metrics.consistency import ci_cons
from metrics.precision import ci_prc
from utils import ElapsedTimer

if __name__ == "__main__":

    # Arguments
    parser = argparse.ArgumentParser(description='Train')
    parser.add_argument('mode', type=str, default='startover', help='startover, continue, or evaluate')
    parser.add_argument('data', type=str, default='AH', help='AH or SE')
    parser.add_argument('--sample_size', type=int, default=10000, help='sample size')
    parser.add_argument('--save_interval', type=int, default=500, help='save interval')
    args = parser.parse_args()
    assert args.mode in ['startover', 'continue', 'evaluate']
    assert args.data in ['AH', 'SE', 'SC']

    print('##################################################################')
    print('Data: {}'.format(args.data))
    model_name = 'hgan'
    print('Model: {}'.format(model_name))
    print('Sample size: {}'.format(args.sample_size))

    # Set hyper-parameters
    if args.data == 'AH':
        data_fname = 'results/AH/AH.npy'
        latent_dim = [3, 3]
        noise_dim = [10, 0]
        n_points = [64, 64]
        bezier_degree = [31, None]
        bounds = (0.0, 1.0)
        train_steps = 100000
        batch_size = 32
    elif args.data == 'SE':
        data_fname = 'results/SE/SE.npy'
        latent_dim = [2, 2]
        noise_dim = [10, 0]
        n_points = [64, 64]
        bezier_degree = [31, None]
        bounds = (0.0, 1.0)
        train_steps = 100000
        batch_size = 32
    elif args.data == 'SC':
        data_fname = 'results/SC/SC.npy'
        latent_dim = [2, 2]
        noise_dim = [10, 0]
        n_points = [64, 64]
        bezier_degree = [31, None]
        bounds = (0.0, 1.0)
        train_steps = 100000
        batch_size = 32

    # Read dataset
    X = np.load(data_fname)
    ind = np.random.choice(X.shape[0], size=args.sample_size, replace=False)
    X = X[ind]

    # Split training and test data
    test_split = 0.8
    N = X.shape[0]
    split = int(N*test_split)
    X_train = X[:split]
    X_test = X[split:]

    X0_test = X_test[:,:n_points[0]]
    X1_test = X_test[:,n_points[0]:]
    X_test_list = [X0_test, X1_test]

    results_dir = 'results/{}/{}/{}'.format(args.data, model_name, args.sample_size)
    if not os.path.exists(results_dir):
        os.makedirs(results_dir)

    # Train
    h = import_module('{}.{}'.format(args.data, model_name))
    model = h.Model(latent_dim, noise_dim, n_points, bezier_degree)
    if args.mode == 'startover':
        timer = ElapsedTimer()
        model.train(X_train, batch_size=batch_size, train_steps=train_steps, save_interval=args.save_interval, save_dir=results_dir)
        elapsed_time = timer.elapsed_time()
        runtime_mesg = 'Wall clock time for training: %s' % elapsed_time
        print(runtime_mesg)
        runtime_file = open('{}/runtime.txt'.format(results_dir), 'w')
        runtime_file.write('%s\n' % runtime_mesg)
        runtime_file.close()
    else:
        model.restore(save_dir=results_dir)

    print('Plotting training samples ...')
    samples_list = []
    n_points_c = np.cumsum([0,] + n_points)
    for i in range(len(n_points)):
        samples_list.append(np.squeeze(X)[:25, n_points_c[i]:n_points_c[i+1]])
    plot_samples(None, samples_list, scatter=False, alpha=.7, c='k', fname='%s/samples' % args.data)

    print('Plotting synthesized x0 ...')
    plot_grid(5, gen_func=model.synthesize_x0, d=latent_dim[0], scale=.95,
              scatter=False, alpha=.7, c='k', fname='{}/x0'.format(results_dir))

    print('Plotting synthesized x1 ...')
    plot_grid(5, gen_func=model.synthesize_x1, d=latent_dim[1], scale=.95,
              scatter=False, alpha=.7, c='k', fname='{}/x1'.format(results_dir))

    print('Plotting synthesized assemblies ...')
    assemblies = model.synthesize_assemblies(25)
    assemblies_list = []
    for i in range(len(n_points)):
        assemblies_list.append(assemblies[:25, n_points_c[i]:n_points_c[i+1]])
    plot_samples(None, assemblies_list, scatter=False, alpha=.7, c='k', fname='{}/assemblies'.format(results_dir))

    n_runs = 10

    feasibility_func = import_module('{}.feasibility'.format(args.data)).check_feasibility
    prc_mean, prc_err = ci_prc(n_runs, model.synthesize_assemblies, feasibility_func, n_points)
    print('Precision for assembly: %.3f +/- %.3f' % (prc_mean, prc_err))
    mmd_mean, mmd_err = ci_mmd(n_runs, model.synthesize_assemblies, X_test)
    rdiv_mean, rdiv_err = ci_rdiv(n_runs, X_test, model.synthesize_assemblies)
    if args.data == 'SC':
        basis = 'polar'
    else:
        basis = 'cartesian'
    cons0_mean, cons0_err = ci_cons(n_runs, model.synthesize_x0, latent_dim[0], bounds, basis='cartesian')
    cons1_mean, cons1_err = ci_cons(n_runs, model.synthesize_x1, latent_dim[1], bounds, basis=basis)

    res = {'CSS': [prc_mean, prc_err],
           'MMD': [mmd_mean, mmd_err],
           'R-Div': [rdiv_mean, rdiv_err],
           'LSC_A': [cons0_mean, cons0_err],
           'LSC_B': [cons1_mean, cons1_err]}
    json.dump(res, open('{}/results.json'.format(results_dir), 'w'))

    results_mesg_0 = 'Precision for assembly: %.3f +/- %.3f' % (prc_mean, prc_err)
    results_mesg_1 = 'Maximum mean discrepancy for assembly: %.4f +/- %.4f' % (mmd_mean, mmd_err)
    results_mesg_2 = 'Relative diversity for assembly: %.3f +/- %.3f' % (rdiv_mean, rdiv_err)
    results_mesg_3 = 'Consistency for latent space 0: %.3f +/- %.3f' % (cons0_mean, cons0_err)
    results_mesg_4 = 'Consistency for latent space 1: %.3f +/- %.3f' % (cons1_mean, cons1_err)

    results_file = open('{}/results.txt'.format(results_dir), 'w')

    print(results_mesg_0)
    results_file.write('%s\n' % results_mesg_0)
    print(results_mesg_1)
    results_file.write('%s\n' % results_mesg_1)
    print(results_mesg_2)
    results_file.write('%s\n' % results_mesg_2)
    print(results_mesg_3)
    results_file.write('%s\n' % results_mesg_3)
    print(results_mesg_4)
    results_file.write('%s\n' % results_mesg_4)

    results_file.close()


text:
# !/usr/bin/env python
# encoding: utf-8
"""
*The modal form class*

:Author:
    David Young

:Date Created:
    July 1, 2014

.. todo::

    @review: when complete pull all general functions and classes into dryxPython
"""
################# GLOBAL IMPORTS ####################
import sys
import os
import datetime
import numpy as np
from docopt import docopt
from dryxPython import logs as dl
from dryxPython import commonutils as dcu
from fundamentals import tools, times
from ..__init__ import *
from . import modal

###################################################################
# CLASSES                                                         #
###################################################################

class modalForm():

    """
    *The worker class for the modalForm module*

    **Key Arguments:**
        - ``log`` -- logger
        - ``title`` -- title
        - ``postToScriptUrl`` -- postToScriptUrl
        - ``reloadToUrl`` -- reloadToUrl

    .. todo::

        - @review: when complete, clean modalForm class
        - @review: when complete add logging
        - @review: when complete, decide whether to abstract class to another module
    """
    # Initialisation

    def __init__(
        self,
        log,
        title,
        postToScriptUrl,
        reloadToUrl,
    ):
        self.log = log
        self.title = title
        self.postToScriptUrl = postToScriptUrl
        self.reloadToUrl = reloadToUrl
        # xt-self-arg-tmpx

        # Variable Data Atrributes
        self.formContent = u""
        self.randNum = int(np.random.rand() * 10000)
        self.hiddenParameterList = []

        # 3. @flagged: what variable attrributes need overriden in any baseclass(es) used
        # Override Variable Data Atrributes

        # Initial Actions

        return None

    def close(self):
        del self
        return None

    # 4. @flagged: what actions does each object have to be able to perform? Add them here
    # Method Attributes
    def get(self):
        """
        *get the modalForm object*

        **Return:**
            - ``modalForm``

        .. todo::

            - @review: when complete, clean get method
            - @review: when complete add logging
        """
        self.log.debug('starting the ``get`` method')

        randNum = self.randNum
        modalTrigger = "modalTrigger%(randNum)s" % locals()
        formId = "form%(randNum)s" % locals()

        self.set_hidden_parameters()

        modalForm = form(
            content=self.formContent,  # list of control groups
            # [ "inline" | "horizontal" | "search" | "navbar-form" | "navbar-search" ]
            formType='horizontal',
            navBarPull=False,  # [ false | right | left ],
            postToScript=self.postToScriptUrl,
            htmlId=formId,
            postInBackground=True,
            redirectUrl=self.reloadToUrl,
            span=12,
            offset=False
        )

        modalForm = grid_row(
            responsive=True,
            columns=modalForm,
            htmlId=False,
            htmlClass=False,
            onPhone=True,
            onTablet=True,
            onDesktop=True
        )

        modalForm = modal(
            modalHeaderContent=self.title,
            modalBodyContent=modalForm,
            modalFooterContent=self.get_form_action_buttons(formId),
            htmlId=modalTrigger,
            centerContent=True
        )

        modalTrigger = "#%(modalTrigger)s" % locals()

        self.log.debug('completed the ``get`` method')
        return modalForm, modalTrigger

    def add_form_object(
            self,
            formObject,
            label="",
            hidden=False):
        """
        *add a form objec to the modal form*

        **Key Arguments:**
            # -

        **Return:**
            - None

        .. todo::

            - @review: when complete, clean addFormObject method
            - @review: when complete add logging
        """
        self.log.debug('starting the ``addFormObject`` method')

        randNum = self.randNum
        randId = "randId%(randNum)s" % locals()
        formObject = formObject.replace(
            "class=", """id="%(randId)s" class=""" % locals())

        thisControlRow = controlRow(
            inputList=[formObject, ]
        )
        thisContentLabel = horizontalFormControlLabel(
            labelText=label,
            forId=randId,
            location="left"
        )
        thisContentCG = horizontalFormControlGroup(
            content=thisContentLabel + thisControlRow,
            validationLevel=False
        )
        self.formContent = self.formContent + thisContentCG

        self.log.debug('completed the ``addFormObject`` method')
        return None

    # use the tab-trigger below for new method
    def get_form_action_buttons(
            self,
            formId):
        """
        *get form action buttons*

        **Key Arguments:**
            # -

        **Return:**
            - None

        .. todo::

            - @review: when complete, clean get_form_action_buttons method
            - @review: when complete add logging
        """
        self.log.debug('starting the ``get_form_action_buttons`` method')

        cancel = button(
            buttonText='cancel',
            # [ default | primary | info | success | warning | danger | inverse | link ]
            buttonStyle='danger',
            buttonSize='small',  # [ large | default | small | mini ]
            htmlId=False,
            href=False,
            pull=False,  # right, left, center
            submit=False,
            block=False,
            disable=False,
            postInBackground=False,
            dataToggle=False,  # [ modal ]
            popover=False,
            close=True
        )

        submit = button(
            buttonText='submit',
            # [ default | primary | info | success | warning | danger | inverse | link ]
            buttonStyle='info',
            buttonSize='small',  # [ large | default | small | mini ]
            htmlId=False,
            href=False,
            pull=False,  # right, left, center
            submit=True,
            block=False,
            disable=False,
            postInBackground=False,
            dataToggle=False,  # [ modal ]
            popover=False,
            formId=formId
        )

        actionButtons = buttonGroup(
            buttonList=[cancel, submit],
            format='default'  # [ default | toolbar | vertical ]
        )

        self.log.debug('completed the ``get_form_action_buttons`` method')
        return actionButtons

    # use the tab-trigger below for new method
    def add_hidden_parameter_value(
            self,
            key,
            value):
        """
        *add hidden parameter value to the form (to be submitted with the form but does not need user input)*

        **Key Arguments:**
            # -

        **Return:**
            - None

        .. todo::

            - @review: when complete, clean add_hidden_parameter_value method
            - @review: when complete add logging
        """
        self.log.debug('starting the ``add_hidden_parameter_value`` method')

        if isinstance(value, str):
            ttype = "text"
        if isinstance(value, unicode):
            ttype = "text"
        elif isinstance(value, datetime.date):
            ttype = "datetime"
        else:
            ttype = "number"

        thisInput = formInput(
            # [ text | password | datetime | datetime-local | date | month | time | week | number | email | url | search | tel | color ]
            ttype=ttype,
            htmlId=key,
            hidden=True,
            defaultValue=value
        )
        self.hiddenParameterList.append(thisInput)

        self.log.debug('completed the ``add_hidden_parameter_value`` method')
        return None

    # use the tab-trigger below for new method
    def set_hidden_parameters(
            self):
        """
        *get hidden parameters*

        **Key Arguments:**
            # -

        **Return:**
            - None

        .. todo::

            - @review: when complete, clean set_hidden_parameters method
            - @review: when complete add logging
        """
        self.log.debug('starting the ``set_hidden_parameters`` method')

        thisControlRow = controlRow(
            inputList=self.hiddenParameterList
        )
        thisContentCG = horizontalFormControlGroup(
            content=thisControlRow,
            validationLevel=False,
            hidden=True
        )

        self.formContent = self.formContent + thisContentCG

        self.log.debug('completed the ``set_hidden_parameters`` method')
        return

    # use the tab-trigger below for new method
    # xt-class-method

# xt-class-tmpx

###################################################################
# PUBLIC FUNCTIONS                                                #
###################################################################
# xt-worker-def

# use the tab-trigger below for new function
# xt-def-with-logger

###################################################################
# PRIVATE (HELPER) FUNCTIONS                                      #
###################################################################

############################################
# CODE TO BE DEPECIATED                    #
############################################

if __name__ == '__main__':
    main()

###################################################################
# TEMPLATE FUNCTIONS                                              #
###################################################################

max_stars_repo_name: thespacedoctor/dryxPython

max_issues_repo_name: thespacedoctor/dryxPython

max_forks_repo_name: thespacedoctor/dryxPython


text:
import distutils.dir_util
import importlib
import inspect
import json
import os
import warnings

import numpy as np
from pybtex.database import parse_file

import skactiveml

for module in skactiveml.__all__:
    importlib.import_module('skactiveml.' + module)

warnings.filterwarnings("ignore")

def generate_api_reference_rst(gen_path):
    """Creates the api_reference.rst file in the specified gen_path.
    Parameters
    ----------
    gen_path : string
        The path of the main directory in which all generated files should be
        created.
    """
    path = os.path.join(os.path.basename(gen_path), 'api_reference.rst')
    gen_path = os.path.join(os.path.basename(gen_path), 'api')
    os.makedirs(os.path.abspath(gen_path), exist_ok=True)
    with open(path, 'w') as file:
        file.write('.. _api_reference:\n')
        file.write('\n')
        file.write('=============\n')
        file.write('API Reference\n')
        file.write('=============\n')
        file.write('\n')
        file.write('This is an overview of the API.\n')
        file.write('\n')
        file.write('.. module:: skactiveml\n')
        file.write('\n')
        for item in skactiveml.__all__:
            if inspect.ismodule(getattr(skactiveml, item)):
                file.write(automodule(getattr(skactiveml, item)))

def automodule(module, level=0):
    rst_str = ''
    modules = []
    classes = []
    functions = []
    constants = []

    for item in module.__all__:
        try:
            importlib.import_module(module.__name__ + "." + item)
        except ModuleNotFoundError:
            pass
        if inspect.ismodule(getattr(module, item)):
            modules.append(item)
        if inspect.isclass(getattr(module, item)):
            classes.append(item)
        if inspect.isfunction(getattr(module, item)):
            functions.append(item)
        if isinstance(getattr(module, item), object) and item.isupper():
            constants.append(item)

    title = f':mod:`{module.__name__}`:'
    rst_str += title + '\n'
    rst_str += ''.ljust(len(title), '=') + '\n\n'

    rst_str += f'.. automodule:: {module.__name__}\n'
    rst_str += f'    :no-members:\n'
    rst_str += f'    :no-inherited-members:\n\n'

    rst_str += f'.. currentmodule:: {module.__name__.split(".")[0]}\n\n'
    if classes:
        rst_str += f'Classes\n'
        rst_str += f'-------\n\n'

        rst_str += f'.. autosummary::\n'
        rst_str += f'   :nosignatures:\n'
        rst_str += f'   :toctree: api\n'
        rst_str += f'   :template: class.rst\n\n'
        for item in classes:
            name = module.__name__
            if 'skactiveml.' in name:
                name = name.replace('skactiveml.', '')
            if name:
                name += '.'
            name += item
            rst_str += f'   {name}' + '\n'
        rst_str += '\n'

    if functions:
        rst_str += f'Functions\n'
        rst_str += f'---------\n\n'

        rst_str += f'.. autosummary::\n'
        rst_str += f'   :nosignatures:\n'
        rst_str += f'   :toctree: api\n'
        rst_str += f'   :template: function.rst\n\n'
        for item in functions:
            name = module.__name__
            if 'skactiveml.' in name:
                name = name.replace('skactiveml.', '')
            if name:
                name += '.'
            name += item
            rst_str += f'   {name}' + '\n'
        rst_str += '\n'

    for item in modules:
        rst_str += automodule(getattr(module, item), level=level + 1)

    return rst_str

def generate_strategy_overview_rst(gen_path, examples_data={}):
    """Creates the strategy_overview.rst file in the specified path.

    Parameters
    ----------
    gen_path : string
        The path of the main directory in which all generated files should be
        created.
    examples_data : dict
        The additional data, needed to create the strategy overview. Shape:
        {'strategy_1' : [
            ['method_1', ['reference_1', ...], {'tab_1': 'category', ...}],
            ['method_2', ...],
            ...],
        'strategy_2' : [...],
        ...
        }
    """
    # Load bibtex database.
    bib_data = parse_file('refs.bib')

    # create directory if it does not exist.
    os.makedirs(os.path.join(gen_path, 'strategy_overview'), exist_ok=True)

    # Generate file
    with open(
            os.path.join(gen_path, 'strategy_overview.rst'),
            'w') as file:
        file.write('=================\n')
        file.write('Strategy Overview\n')
        file.write('=================\n')
        file.write('\n')
        file.write('In the following you\'ll find summaries of all implemented'
                   ' "Query Strategies", based on the categorization of '
                   'different papers.\n')
        file.write('\n')
        file.write('.. toctree::\n')
        file.write('   :maxdepth: 1\n')
        file.write('\n')
        for tab in examples_data.keys():
            file.write(f'   strategy_overview/strategy_overview-{tab}\n')

    # Iterate over the tabs.
    for tab, cats in examples_data.items():
        author = bib_data.entries[tab].persons["author"][0].last_names[0]
        path = os.path.join(gen_path,
                            'strategy_overview',
                            f'strategy_overview-{tab}.rst')
        with open(path, 'w') as file:
            title = f'Strategy Overview By {author}\n'
            file.write(title)
            file.write(''.ljust(len(title) + 1, '=') + '\n')
            file.write('\n')
            file.write(f'This is an overview of all implemented AL strategies. '
                       f'The strategies are categorized according to '
                       f':footcite:t:`{tab}`.\n')
            file.write('\n')
            file.write('Pool Strategies\n')
            file.write('---------------\n')
            file.write('\n')

            # Iterate over the categories in the current tab.
            for cat in sorted(cats):
                if cat != 'Others':
                    file.write(table_from_array(cats[cat], title=cat,
                                                section_level='~',
                                                header_rows=1, indent=0))
            if 'Others' in cats.keys():
                # 'Others' is the fallback, if no category is specified
                # in the json file
                file.write(table_from_array(cats['Others'], title='Others',
                                            section_level='~',
                                            header_rows=1, indent=0))
            file.write('References\n')
            file.write('----------\n')
            file.write('.. footbibliography::')
            # TODO stream
            # file.write('\n')
            # file.write('Stream Strategies:\n')
            # file.write('------------------\n')
            # file.write('\n')
            file.write('\n')

def table_from_array(a, title='', caption='', widths=None, header_rows=0,
                     section_level='-', indent=0):
    """Generates a rst-table and returns it as a string.
    Parameters
    ----------
    a : array-like, shape=(columns, rows)
        Contains the data for the table.
    title : string, optional (default='')
        The title of the table.
    caption : str, optional (default='')
        The caption of the table.
    section_level : str, optional (default='-')
        The rst section level of the title.
    widths : string, optional (default=None)
        A list of relative column widths separated with comma or space
        or 'auto'.
    header_rows : int, optional (default=0)
        The number of rows to use in the table header.
    indent : int, optional (default=0)
        Number of spaces as indent in each line
    Returns
    -------
    string : reStructuredText list-table as String.
    """
    a = np.asarray(a)
    indents = ''.ljust(indent, ' ')
    table = title + '\n'
    table += ''.ljust(len(title), section_level) + '\n'
    table += f'{indents}.. list-table:: {caption}\n' \
             f'{indents}   :header-rows: {header_rows}\n'
    if widths is None:
        table += '\n'
    elif widths == 'auto':
        table += f'{indents}   :widths: auto\n\n'
    else:
        table += f'{indents}   :widths: {widths}\n\n'
    for row in a:
        table += f'{indents}   *'
        for column in row:
            table += ' - ' + str(column) + f'\n{indents}    '
        table = table[0: -4 - indent]
    return table + '\n'

def generate_examples(gen_path, package, json_path):
    """
    Creates all example scripts for the specified package and returns the data
    needed to create the strategy overview.
    Parameters
    ----------
    gen_path : string
        The path of the main directory in which all generated files should be
        created.
    package : module
        The '__init__' module of the package for which the examples should be
        created.
    json_path : string
        The path of the directory where to find the json example files for the
        specified package.
    Returns
    -------
    dict : Holds the data needed to create the strategy overview.
    """
    dir_path_package = os.path.join(gen_path, 'examples',
                                    package.__name__.split('.')[1], '')

    rel_api_path = os.path.join(
        os.path.basename(gen_path), 'api'
    ).replace('\\', '/')

    # create directory if it does not exist.
    os.makedirs(dir_path_package, exist_ok=True)

    # create README.rst files needed for 'sphinx-gallery'
    with open(os.path.join(gen_path, 'examples', 'README.rst'), 'w') as file:
        file.write('Examples\n')
        file.write('========')
    with open(os.path.join(dir_path_package, 'README.rst'), 'w') as file:
        title = f'{package.__name__.split(".")[1]} based'.title()
        file.write(title + '\n')
        file.write(''.ljust(len(title), '-'))

    head_line = ['Method', 'Base Class', 'Reference']
    examples_data = {}  # (Tab, Category, Collum, Row)
    table = np.ndarray(shape=(0, 4))
    # iterate over jason example files
    for filename in os.listdir(json_path):
        if not filename.endswith('.json'):
            continue
        with open(os.path.join(json_path, filename)) as file:
            # iterate over the examples in the json file
            for data in json.load(file):
                # Collect the data needed to generate the strategy overview.
                qs_name = data["class"]
                method = data['method']
                package_name = package.__name__.replace('skactiveml.', '')
                methods_text = \
                    f':doc:`{method} </generated/sphinx_gallery_examples/' \
                    f'{package_name}/plot_{qs_name}_' \
                    f'{method.replace(" ", "_")}>`'
                strategy_text = f':doc:`{qs_name} </{rel_api_path}/' \
                                f'{package.__name__}.{qs_name}>`'
                ref_text = ''
                for ref in data['refs']:
                    ref_text += f':footcite:t:`{ref}`, '
                ref_text = ref_text[0:-2]
                category = data['categories'] if 'categories' in data.keys() \
                    else {}
                table = np.append(
                    table,
                    [[methods_text, strategy_text, ref_text, category]],
                    axis=0
                )

                # create the example python script
                plot_filename = \
                    'plot_' + data["class"] + "_" + method.replace(' ', '_')
                generate_example_script(
                    filename=plot_filename + '.py',
                    dir_path=dir_path_package,
                    data=data,
                    package=package,
                    template_path=os.path.join(json_path, 'template.py')
                )

    # Sort the table alphabetically.
    table = sorted(table, key=lambda row: row[1])

    # Collect the different tabs and categories.
    for i, row in enumerate(table):
        for tab in row[3].keys():
            if tab not in examples_data.keys():
                examples_data[tab] = {}
            if row[3][tab] == '':
                table[i][3][tab] = 'Others'
            if row[3][tab] not in examples_data[tab].keys():
                examples_data[tab][row[3][tab]] = np.array([head_line])

    # Build the dict that holds the data.
    for row in table:
        for tab in examples_data:
            if tab in row[3].keys():
                cat = row[3][tab] if tab in row[3].keys() else 'Others'
            examples_data[tab][cat] = \
                np.append(examples_data[tab][cat], [row[:3]], axis=0)

    return examples_data

def generate_example_script(filename, dir_path, data, package, template_path):
    """
    Generates a python example file needed, for the 'sphinx-gallery' extension.
    Parameters
    ----------
    filename : string
        The name of the python example file
    dir_path : string
        The directory path in which to save the python example file.
    data : dict
        The data from the json example file for the example.
    package : module
        The '__init__' module of the package for which the examples should be
        created.
    template_path : path-like
        The path to the template file.
    """
    # create directory if it does not exist.
    os.makedirs(dir_path, exist_ok=True)

    # Validation of 'data'.
    if data['class'] not in package.__all__:
        raise ValueError(f'"{data["class"]}" is not in "{package}.__all__".')
    data["qs"] = getattr(package, data["class"])

    first_title = True
    # Create the file.
    with open(os.path.join(dir_path, filename), 'w') as file:
        code_blocks = []
        # Iterate over the 'blocks' and generate the corresponding strings
        # expected from sphinx-gallery.
        for block in data["sequence"]:
            if block.startswith("title"):
                block_str = format_title(data[block], first_title)
                first_title = False
            elif block.startswith("subtitle"):
                block_str = format_title(data[block], False)
            elif block.startswith("text"):
                block_str = format_text(data[block])
            elif block.startswith("code"):
                code_blocks.append(data[block])
                block_str = format_code(data[block])
            elif block.startswith("plot"):
                block_str = format_plot(data, template_path)
            elif block.startswith("refs"):
                block_str = format_refs(data[block])

            # Write the corresponding string to the python script.
            file.write(block_str)

        file.write("\n")

def format_title(title, first_title):
    """
    Generates the string for a title of the example page, formatted for a
    'sphinx-gallery' example script.
    Parameters
    ----------
    title : string
        The title string.
    first_title : boolean
        Indicates whether the title is the first title of the example script.
    Returns
    -------
    string : The formatted string for the example script.
    """
    if first_title:
        block_str = '"""\n' \
                    '' + title + '\n' \
                                 ''.ljust(len(title) + 1, '=') + '\n' \
                                                                 '"""\n'
    else:
        block_str = '# %%\n' \
                    '# .. rubric:: ' + title + ':\n'

    return block_str + '\n'

def format_subtitle(title):
    """
    Generates the string for a subtitle of the example page, formatted for a
    'sphinx-gallery' example script.
    Parameters
    ----------
    title : string
        The subtitle string.
    Returns
    -------
    string : The formatted string for the example script.
    """
    block_str = '# %%\n' \
                '# ' + title + '\n' \
                               '# '.ljust(len(title) + 1, '-') + '\n\n'
    return block_str

def format_text(text):
    """
    Generates the string for a paragraph of the example page, formatted for a
    'sphinx-gallery' example script.
    Parameters
    ----------
    text : string
        The paragraph text.
    Returns
    -------
    string : The formatted string for the example script.
    """
    block_str = '# %%\n'
    for line in text.split('\n'):
        block_str += '# ' + line + '\n'
    return block_str + '\n'

def format_code(code):
    """
    Generates the string for a code block of the example page, formatted for a
    'sphinx-gallery' example script.
    Parameters
    ----------
    code : string
        The python source code to be formatted.
    Returns
    -------
    string : The formatted string for the example script.
    """
    block_str = ''
    for line in code:
        block_str += line + '\n'
    return block_str + '\n'

def format_plot(data, template_path):
    """
    Generates the string for the plotting section of the example page,
    formatted for a 'sphinx-gallery' example script.
    Parameters
    ----------
    data : dict
        The data from the jason example file for the example.
    template_path : path-like
        The path to the template file.
    Returns
    -------
    string : The formatted string for the example script.
    """
    block_str = ''
    with open(template_path, "r") as template:
        for line in template:
            if '#_' in line:
                if '#_import' in line:
                    line = f'from skactiveml.pool import {data["qs"].__name__}\n'
                    if 'clf' not in data.keys() and \
                            'clf' not in data['init_params'].keys():
                        line += 'from skactiveml.classifier import ParzenWindowClassifier\n'
                elif 'init_clf' in line:
                    # Decide which classifier to use, if clf is None.
                    if 'clf' not in data.keys():
                        if 'clf' not in data['init_params'].keys():
                            clf = 'ParzenWindowClassifier(classes=[0, 1], random_state=random_state)'
                        else:
                            clf = data['init_params']['clf']
                    else:
                        clf = data['clf']
                    line = ('clf = ' + clf + '\n')
                elif 'init_qs' in line:
                    if 'clf' not in data['init_params'].keys() and 'clf' in \
                            inspect.signature(data["qs"].__init__).parameters:
                        data['init_params']['clf'] = 'clf'
                    # Set the random state if it is not set in the json file.
                    if 'random_state' not in data[
                        'init_params'].keys() and 'random_state' in \
                            inspect.signature(data["qs"].__init__).parameters:
                        data['init_params']['random_state'] = 'random_state'
                    # Initialise the query strategy.
                    line = f'qs = {data["qs"].__name__}' \
                           f'({dict_to_str(data["init_params"])})\n'
                elif '#_query_params' in line:
                    start = line.find('#_query_params') - 1
                    line = line[:start] + '{' + \
                           dict_to_str(data['query_params'], allocator=': ',
                                       key_as_string=True) + '}\n'
                elif '"#_n_cycles"' in line:
                    n_cycles = os.getenv('N_CYCLES', default='2')
                    line = line.replace('"#_n_cycles"', n_cycles)
                elif '"#_n_samples"' in line:
                    n_samples = os.getenv('N_SAMPLES', default='10')
                    line = line.replace('"#_n_samples"', n_samples)
                elif '"#_res"' in line:
                    res = os.getenv('RES', default='3')
                    line = line.replace('"#_res"', res)
                elif '"#_candidates"' in line:
                    if 'use_candidate_plotting' not in data.keys():
                        cand = 'None'
                    else:
                        cand = "np.arange(len(X))"
                    line = line.replace('"#_candidates"', cand)
                elif '#_bp' in line:
                    try:
                        s = line.find('#_')
                        bp = line[s + 2:].split()[0]
                        prefix = line[:s]
                        line = ''
                        if type(data[bp]) != list:
                            data[bp] = [data[bp]]
                        for l in data[bp]:
                            line += prefix + l + '\n'
                    except KeyError:
                        pass

            block_str += line

    return block_str + '\n'

def format_refs(refs: list):
    """
    Generates the string for a references of the example page, formatted for a
    'sphinx-gallery' example script.
    Parameters
    ----------
    refs : list of strings
        A list of references to bibliographic entries in the 'refs.bib' file.
    Returns
    -------
    string : The formatted string for the example script.
    """
    if not refs:
        return ''
    block_str = '# %%\n' \
                '# .. rubric:: References:\n' \
                '# \n' \
                '# The implementation of this strategy is based on '
    if len(refs) > 1:
        for i in range(len(refs) - 1):
            block_str += f':footcite:t:`{refs[i]}`, '
        block_str = block_str[0:-2]
        block_str += f' and :footcite:t:`{refs[-1]}'
    else:
        block_str += f':footcite:t:`{refs[0]}'
    block_str += '`.\n' \
                 '#\n'

    block_str += '# .. footbibliography::\n'

    return block_str + '\n'

def dict_to_str(d, idx=None, allocator='=', key_as_string=False):
    """Converts a dictionary into a string.
    Parameters
    ----------
    d : dict
        The dictionary to be converted.
        Shape: {key1:value1,...} or {key1:[value1, value2,...],...} or a
        combination of both.
    idx: : dict, optional
        If a key has multiple values, idx[key] chooses the used value for the
        specific key. If idx is not given, the first value in the list is
        always used. It is not necessary to specify all keys from d.
        shape: {key1:int1,...}
    allocator: string, optional (Default='=')
        The allocator is used to separate the key and the value.
    Returns
    -------
    String : dict_ as String. Shape: 'key1=value[idx[key1]], key2...' or
             'key1=value1, key2=value2...' or a combination of both.
    """
    dd_str = ""
    for key, value in d.items():
        if not isinstance(value, list):
            value = [value]
        if idx is not None and key in idx.keys():
            value = value[idx[key]]
        else:
            value = value[0]
        key = str(key)
        if key_as_string:
            if not ((key.startswith('"') and key.endswith('"')) or \
                    (key.startswith('\'') and key.endswith('\''))):
                key = '"' + key + '"'
        dd_str += str(key) + allocator + value + ", "
    return dd_str[0:-2]

def generate_tutorials(src_path, dst_path):
    """Includes the tutorials folder from the git root, such that tutorials are
    included in the documentation. Effectively this function copies all
    contents from src_path to dst_path.
    Parameters
    ----------
    src_path: string
        The path where the notebooks are found.
    dst_path: string
        The path where the notebooks are saved, such that tutorials.rst can
        find them.
    """
    distutils.dir_util.copy_tree(src=src_path, dst=dst_path)

max_stars_repo_name: LukasLuehrs/scikit-activeml

max_issues_repo_name: LukasLuehrs/scikit-activeml

max_forks_repo_name: LukasLuehrs/scikit-activeml


text:
#!/usr/bin/env python

# Copyright 2019 Pascal Audet & Helen Janiszewski
#
# This file is part of OBStools.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Import modules and functions
import numpy as np
import pickle
import stdb
from obstools.atacr import StaNoise, Power, Cross, Rotation
from obstools.atacr import utils, plotting
from pathlib import Path

from argparse import ArgumentParser
from os.path import exists as exist
from obspy import UTCDateTime
from numpy import nan

def get_cleanspec_arguments(argv=None):
    """
    Get Options from :class:`~optparse.OptionParser` objects.

    Calling options for the script `obs_clean_spectra.py` that accompany this
    package.

    """

    parser = ArgumentParser(
        usage="%(prog)s [options] <indb>",
        description="Script used "
        "to extract daily spectra calculated from " +
        "`obs_daily_spectra.py` and flag days for outlier " +
        "PSDs and calculate spectral averages of the " +
        "corresponding Fourier transforms over the entire " +
        "time period specified. The stations are processed " +
        "one by one and the data are stored to disk.")
    parser.add_argument(
        "indb",
        help="Station Database to process from.",
        type=str)

    # General Settings
    parser.add_argument(
        "--keys",
        action="store",
        type=str,
        dest="stkeys",
        default="",
        help="Specify a comma separated list of station " +
        "keys for which to perform the analysis. These must " +
        "be contained within the station database. Partial " +
        "keys will be used to match against those in the " +
        "dictionary. For instance, providing IU will match " +
        "with all stations in the IU network. " +
        "[Default processes all stations in the database]")
    parser.add_argument(
        "-O", "--overwrite",
        action="store_true",
        dest="ovr",
        default=False,
        help="Force the overwriting of pre-existing data. " +
        "[Default False]")

    # Event Selection Criteria
    DaysGroup = parser.add_argument_group(
        title="Time Search Settings",
        description="Time settings associated with " +
        "searching for day-long seismograms")
    DaysGroup.add_argument(
        "--start",
        action="store",
        type=str,
        dest="startT",
        default="",
        help="Specify a UTCDateTime compatible string " +
        "representing the start day for the data search. " +
        "This will override any station start times. " +
        "[Default start date of each station in database]")
    DaysGroup.add_argument(
        "--end",
        action="store",
        type=str,
        dest="endT",
        default="",
        help="Specify a UTCDateTime compatible string " +
        "representing the start time for the data search. " +
        "This will override any station end times. " +
        "[Default end date of each station in database]")

    # Constants Settings
    ConstGroup = parser.add_argument_group(
        title='Parameter Settings',
        description="Miscellaneous default values " +
        "and settings")
    ConstGroup.add_argument(
        "--freq-band",
        action="store",
        type=str,
        dest="pd",
        default=None,
        help="Specify comma-separated frequency limits " +
        "(float, in Hz) over which to calculate spectral " +
        "features used in flagging the days/windows. " +
        "[Default 0.004,2.0]")
    ConstGroup.add_argument(
        "--tolerance",
        action="store",
        type=float,
        dest="tol",
        default=1.5,
        help="Specify parameter for tolerance threshold. " +
        "If spectrum > std*tol, window is flagged as bad. " +
        "[Default 1.5]")
    ConstGroup.add_argument(
        "--alpha",
        action="store",
        type=float,
        dest="alpha",
        default=0.05,
        help="Confidence level for f-test, for iterative " +
        "flagging of windows. [Default 0.05, or 95 percent confidence]")

    # Constants Settings
    FigureGroup = parser.add_argument_group(
        title='Figure Settings',
        description="Flags for plotting figures")
    FigureGroup.add_argument(
        "--figQC",
        action="store_true",
        dest="fig_QC",
        default=False,
        help="Plot Quality-Control figure. " +
        "[Default does not plot figure]")
    FigureGroup.add_argument(
        "--debug",
        action="store_true",
        dest="debug",
        default=False,
        help="Plot intermediate steps for debugging. " +
        "[Default does not plot figure]")
    FigureGroup.add_argument(
        "--figAverage",
        action="store_true",
        dest="fig_average",
        default=False,
        help="Plot daily average figure. " +
        "[Default does not plot figure]")
    FigureGroup.add_argument(
        "--figCoh",
        action="store_true",
        dest="fig_coh_ph",
        default=False,
        help="Plot Coherence and Phase figure. " +
        "[Default does not plot figure]")
    FigureGroup.add_argument(
        "--figCross",
        action="store_true",
        dest="fig_av_cross",
        default=False,
        help="Plot cross-spectra figure. " +
        "[Default does not plot figure]")
    FigureGroup.add_argument(
        "--save-fig",
        action="store_true",
        dest="saveplot",
        default=False,
        help="Set this option if you wish to save the figure(s). [Default " +
        "does not save figure]")
    FigureGroup.add_argument(
        "--format",
        action="store",
        type=str,
        dest="form",
        default="png",
        help="Specify format of figure. Can be any one of the valid" +
        "matplotlib formats: 'png', 'jpg', 'eps', 'pdf'. [Default 'png']")

    args = parser.parse_args(argv)

    # Check inputs
    if not exist(args.indb):
        parser.error("Input file " + args.indb + " does not exist")

    # create station key list
    if len(args.stkeys) > 0:
        args.stkeys = args.stkeys.split(',')

    # construct start time
    if len(args.startT) > 0:
        try:
            args.startT = UTCDateTime(args.startT)
        except Exception:
            parser.error(
                "Error: Cannot construct UTCDateTime from start time: " +
                args.startT)
    else:
        args.startT = None

    # construct end time
    if len(args.endT) > 0:
        try:
            args.endT = UTCDateTime(args.endT)
        except Exception:
            parser.error(
                "Error: Cannot construct UTCDateTime from end time: " +
                args.endT)
    else:
        args.endT = None

    if args.pd is None:
        args.pd = [0.004, 2.0]
    else:
        args.pd = [float(val) for val in args.pd.split(',')]
        args.pd = sorted(args.pd)
        if (len(args.pd)) != 2:
            raise(Exception(
                "Error: --freq-band should contain 2 " +
                "comma-separated floats"))

    return args

def main(args=None):

    if args is None:
        # Run Input Parser
        args = get_cleanspec_arguments()

    # Load Database
    # stdb>0.1.3
    try:
        db, stkeys = stdb.io.load_db(fname=args.indb, keys=args.stkeys)

    # stdb=0.1.3
    except Exception:
        db = stdb.io.load_db(fname=args.indb)

        # Construct station key loop
        allkeys = db.keys()
        sorted(allkeys)

        # Extract key subset
        if len(args.stkeys) > 0:
            stkeys = []
            for skey in args.stkeys:
                stkeys.extend([s for s in allkeys if skey in s])
        else:
            stkeys = db.keys()
            sorted(stkeys)

    # Loop over station keys
    for stkey in list(stkeys):

        # Extract station information from dictionary
        sta = db[stkey]

        # Path where spectra are located
        specpath = Path('SPECTRA') / stkey
        if not specpath.is_dir():
            raise(Exception(
                "Path to " + str(specpath) +
                " doesn`t exist - aborting"))

        # Path where average spectra will be saved
        avstpath = Path('AVG_STA') / stkey
        if not avstpath.is_dir():
            print("Path to "+str(avstpath)+" doesn`t exist - creating it")
            avstpath.mkdir(parents=True)

        # Path where plots will be saved
        if args.saveplot:
            plotpath = avstpath / 'PLOTS'
            if not plotpath.is_dir():
                plotpath.mkdir(parents=True)
        else:
            plotpath = False

        # Get catalogue search start time
        if args.startT is None:
            tstart = sta.startdate
        else:
            tstart = args.startT

        # Get catalogue search end time
        if args.endT is None:
            tend = sta.enddate
        else:
            tend = args.endT

        if tstart > sta.enddate or tend < sta.startdate:
            continue

        # Temporary print locations
        tlocs = sta.location
        if len(tlocs) == 0:
            tlocs = ['']
        for il in range(0, len(tlocs)):
            if len(tlocs[il]) == 0:
                tlocs[il] = "--"
        sta.location = tlocs

        # Update Display
        print("\n|===============================================|")
        print("|===============================================|")
        print("|                   {0:>8s}                    |".format(
            sta.station))
        print("|===============================================|")
        print("|===============================================|")
        print("|  Station: {0:>2s}.{1:5s}                            |".format(
            sta.network, sta.station))
        print("|      Channel: {0:2s}; Locations: {1:15s}  |".format(
            sta.channel, ",".join(tlocs)))
        print("|      Lon: {0:7.2f}; Lat: {1:6.2f}                |".format(
            sta.longitude, sta.latitude))
        print("|      Start time: {0:19s}          |".format(
            sta.startdate.strftime("%Y-%m-%d %H:%M:%S")))
        print("|      End time:   {0:19s}          |".format(
            sta.enddate.strftime("%Y-%m-%d %H:%M:%S")))
        print("|-----------------------------------------------|")

        # Filename for output average spectra
        dstart = str(tstart.year).zfill(4)+'.'+str(tstart.julday).zfill(3)+'-'
        dend = str(tend.year).zfill(4)+'.'+str(tend.julday).zfill(3)+'.'
        fileavst = avstpath / (dstart+dend+'avg_sta.pkl')

        if fileavst.exists():
            if not args.ovr:
                print("*   -> file "+str(fileavst)+" exists - continuing")
                continue

        # Containers for power and cross spectra
        coh_all = []
        ph_all = []
        coh_12_all = []
        coh_1Z_all = []
        coh_1P_all = []
        coh_2Z_all = []
        coh_2P_all = []
        coh_ZP_all = []
        ph_12_all = []
        ph_1Z_all = []
        ph_1P_all = []
        ph_2Z_all = []
        ph_2P_all = []
        ph_ZP_all = []
        ad_12_all = []
        ad_1Z_all = []
        ad_1P_all = []
        ad_2Z_all = []
        ad_2P_all = []
        ad_ZP_all = []
        nwins = []

        t1 = tstart

        # Initialize StaNoise object
        stanoise = StaNoise()

        # Loop through each day withing time range
        while t1 < tend:

            year = str(t1.year).zfill(4)
            jday = str(t1.julday).zfill(3)

            tstamp = year+'.'+jday+'.'
            filespec = specpath / (tstamp + 'spectra.pkl')

            # Load file if it exists
            if filespec.exists():
                print("\n"+"*"*60)
                print('* Calculating noise spectra for key ' +
                      stkey+' and day '+year+'.'+jday)
                print("*   -> file "+str(filespec)+" found - loading")
                file = open(filespec, 'rb')
                daynoise = pickle.load(file)
                file.close()
                stanoise += daynoise
            else:
                t1 += 3600.*24.
                continue

            coh_all.append(daynoise.rotation.coh)
            ph_all.append(daynoise.rotation.ph)

            # Coherence
            coh_12_all.append(
                utils.smooth(
                    utils.coherence(
                        daynoise.cross.c12,
                        daynoise.power.c11,
                        daynoise.power.c22), 50))
            coh_1Z_all.append(
                utils.smooth(
                    utils.coherence(
                        daynoise.cross.c1Z,
                        daynoise.power.c11,
                        daynoise.power.cZZ), 50))
            coh_1P_all.append(
                utils.smooth(
                    utils.coherence(
                        daynoise.cross.c1P,
                        daynoise.power.c11,
                        daynoise.power.cPP), 50))
            coh_2Z_all.append(
                utils.smooth(
                    utils.coherence(
                        daynoise.cross.c2Z,
                        daynoise.power.c22,
                        daynoise.power.cZZ), 50))
            coh_2P_all.append(
                utils.smooth(
                    utils.coherence(
                        daynoise.cross.c2P,
                        daynoise.power.c22,
                        daynoise.power.cPP), 50))
            coh_ZP_all.append(
                utils.smooth(
                    utils.coherence(
                        daynoise.cross.cZP,
                        daynoise.power.cZZ,
                        daynoise.power.cPP), 50))

            # Phase
            try:
                ph_12_all.append(
                    180./np.pi*utils.phase(daynoise.cross.c12))
            except Exception:
                ph_12_all.append(None)
            try:
                ph_1Z_all.append(
                    180./np.pi*utils.phase(daynoise.cross.c1Z))
            except Exception:
                ph_1Z_all.append(None)
            try:
                ph_1P_all.append(
                    180./np.pi*utils.phase(daynoise.cross.c1P))
            except Exception:
                ph_1P_all.append(None)
            try:
                ph_2Z_all.append(
                    180./np.pi*utils.phase(daynoise.cross.c2Z))
            except Exception:
                ph_2Z_all.append(None)
            try:
                ph_2P_all.append(
                    180./np.pi*utils.phase(daynoise.cross.c2P))
            except Exception:
                ph_2P_all.append(None)
            try:
                ph_ZP_all.append(
                    180./np.pi*utils.phase(daynoise.cross.cZP))
            except Exception:
                ph_ZP_all.append(None)

            # Admittance
            ad_12_all.append(utils.smooth(utils.admittance(
                daynoise.cross.c12, daynoise.power.c11), 50))
            ad_1Z_all.append(utils.smooth(utils.admittance(
                daynoise.cross.c1Z, daynoise.power.c11), 50))
            ad_1P_all.append(utils.smooth(utils.admittance(
                daynoise.cross.c1P, daynoise.power.c11), 50))
            ad_2Z_all.append(utils.smooth(utils.admittance(
                daynoise.cross.c2Z, daynoise.power.c22), 50))
            ad_2P_all.append(utils.smooth(utils.admittance(
                daynoise.cross.c2P, daynoise.power.c22), 50))
            ad_ZP_all.append(utils.smooth(utils.admittance(
                daynoise.cross.cZP, daynoise.power.cZZ), 50))

            t1 += 3600.*24.

        # Convert to numpy arrays
        coh_all = np.array(coh_all)
        ph_all = np.array(ph_all)
        coh_12_all = np.array(coh_12_all)
        coh_1Z_all = np.array(coh_1Z_all)
        coh_1P_all = np.array(coh_1P_all)
        coh_2Z_all = np.array(coh_2Z_all)
        coh_2P_all = np.array(coh_2P_all)
        coh_ZP_all = np.array(coh_ZP_all)
        ph_12_all = np.array(ph_12_all)
        ph_1Z_all = np.array(ph_1Z_all)
        ph_1P_all = np.array(ph_1P_all)
        ph_2Z_all = np.array(ph_2Z_all)
        ph_2P_all = np.array(ph_2P_all)
        ph_ZP_all = np.array(ph_ZP_all)
        ad_12_all = np.array(ad_12_all)
        ad_1Z_all = np.array(ad_1Z_all)
        ad_1P_all = np.array(ad_1P_all)
        ad_2Z_all = np.array(ad_2Z_all)
        ad_2P_all = np.array(ad_2P_all)
        ad_ZP_all = np.array(ad_ZP_all)

        # Store transfer functions as objects for plotting
        coh = Cross(coh_12_all, coh_1Z_all, coh_1P_all,
                    coh_2Z_all, coh_2P_all, coh_ZP_all)
        ph = Cross(ph_12_all, ph_1Z_all, ph_1P_all,
                   ph_2Z_all, ph_2P_all, ph_ZP_all)
        ad = Cross(ad_12_all, ad_1Z_all, ad_1P_all,
                   ad_2Z_all, ad_2P_all, ad_ZP_all)

        # Quality control to identify outliers
        stanoise.QC_sta_spectra(pd=args.pd, tol=args.tol, alpha=args.alpha,
                                fig_QC=args.fig_QC, debug=args.debug,
                                save=plotpath, form=args.form)

        # Average spectra for good days
        stanoise.average_sta_spectra(
            fig_average=args.fig_average,
            save=plotpath, form=args.form)

        if args.fig_av_cross:
            fname = stkey + '.' + 'av_coherence'
            plot = plotting.fig_av_cross(
                stanoise.f, coh, stanoise.gooddays,
                'Coherence', stanoise.ncomp, key=stkey, lw=0.5)
            # if plotpath.is_dir():
            if plotpath:
                plot.savefig(
                    str(plotpath / (fname + '.' + args.form)),
                    dpi=300, bbox_inches='tight', format=args.form)
            else:
                plot.show()

            fname = stkey + '.' + 'av_admittance'
            plot = plotting.fig_av_cross(
                stanoise.f, ad, stanoise.gooddays,
                'Admittance', stanoise.ncomp, key=stkey, lw=0.5)

            if plotpath:
                plot.savefig(
                    str(plotpath / (fname + '.' + args.form)),
                    dpi=300, bbox_inches='tight', format=args.form)
            else:
                plot.show()

            fname = stkey + '.' + 'av_phase'
            plot = plotting.fig_av_cross(
                stanoise.f, ph, stanoise.gooddays,
                'Phase', stanoise.ncomp, key=stkey, marker=',', lw=0)

            if plotpath:
                plot.savefig(
                    str(plotpath / (fname + '.' + args.form)),
                    dpi=300, bbox_inches='tight', format=args.form)
            else:
                plot.show()

        if args.fig_coh_ph and stanoise.direc is not None:
            fname = stkey + '.' + 'coh_ph'
            plot = plotting.fig_coh_ph(coh_all, ph_all, stanoise.direc)
            if plotpath:
                plot.savefig(
                    str(plotpath / (fname + '.' + args.form)),
                    dpi=300, bbox_inches='tight', format=args.form)
            else:
                plot.show()

        # Save to file
        stanoise.save(fileavst)

if __name__ == "__main__":

    # Run main program
    main()

max_stars_repo_name: paudetseis/OBStools

max_issues_repo_name: paudetseis/OBStools

max_forks_repo_name: paudetseis/OBStools

reason: import numpy,from numpy


text:
import numpy
import math
import scipy

print(math.pi, 'from the math module')
print(numpy.pi, 'from the numpy package')
print(scipy.pi, 'from the scipy package')

reason: import numpy,import scipy


text:
import unittest
import numpy as np
# import os
from statistical_clear_sky.algorithm.initialization.linearization_helper\
 import LinearizationHelper

class TestLinealizationHelper(unittest.TestCase):
    '''
    Unit test for obtaining initial data of Right Vectors component r0,
    which is used as a denomoniator of non-linear equation in order to make
    it linear.
    It convers the first part of the constructor of main.IterativeClearSky
    in the original code.
    '''

    def setUp(self):
        pass

    def test_obtain_component_r0(self):

        power_signals_d = np.array([[3.65099996e-01, 0.00000000e+00,
                                     0.00000000e+00, 2.59570003e+00],
                                    [6.21100008e-01, 0.00000000e+00,
                                     0.00000000e+00, 2.67740011e+00],
                                    [8.12500000e-01, 0.00000000e+00,
                                     0.00000000e+00, 2.72729993e+00],
                                    [9.00399983e-01, 0.00000000e+00,
                                     0.00000000e+00, 2.77419996e+00]])
        rank_k = 4

        expected_result = np.array([1.36527916, 2.70624333, 4.04720749,
                                    5.38817165])

        linearization_helper = LinearizationHelper(solver_type='ECOS')
        left_low_rank_matrix_u, singular_values_sigma, right_low_rank_matrix_v \
            = np.linalg.svd(power_signals_d)
        initial_r_cs_value = np.diag(singular_values_sigma[:rank_k]).dot(
                    right_low_rank_matrix_v[:rank_k, :])
        actual_result = linearization_helper.obtain_component_r0(
            initial_r_cs_value)

        np.testing.assert_almost_equal(actual_result, expected_result,
                                       decimal=2)

max_stars_repo_name: elsirdavid/StatisticalClearSky

max_issues_repo_name: elsirdavid/StatisticalClearSky

max_forks_repo_name: elsirdavid/StatisticalClearSky


text:
"""
Example motion correction script.

https://github.com/ClandininLab/visanalysis
mhturner@stanford.edu
"""
import sys
import time
import nibabel as nib
import numpy as np
from visanalysis.util import registration

t0 = time.time()

# first arg: path to image series base, without .suffix
#   e.g. /oak/stanford/groups/trc/data/Max/ImagingData/Bruker/20210611/TSeries-20210611-001
file_base_path = sys.argv[1]
print('Registering brain file from {}'.format(file_base_path))

# Load metadata from bruker .xml file
metadata = registration.get_bruker_metadata(file_base_path + '.xml')
print('Loaded metadata from {}'.format(file_base_path + '.xml'))

# Load brain images
ch1 = registration.get_ants_brain(file_base_path + '_channel_1.nii', metadata, channel=0)
print('Loaded {}, shape={}'.format(file_base_path + '_channel_1.nii', ch1.shape))
ch2 = registration.get_ants_brain(file_base_path + '_channel_2.nii', metadata, channel=0)
print('Loaded {}, shape={}'.format(file_base_path + '_channel_2.nii', ch2.shape))

# Register both channels to channel 1
merged = registration.register_two_channels_to_red(ch1, ch2, spatial_dims=len(ch1.shape) - 1)

# Save registered, merged .nii
nifti1_limit = (2**16 / 2)
if np.any(np.array(merged.shape) >= nifti1_limit): # Need to save as nifti2
    nib.save(nib.Nifti2Image(merged, np.eye(4)), file_base_path + '_reg.nii')
else: # Nifti1 is OK
    nib.save(nib.Nifti1Image(merged, np.eye(4)), file_base_path + '_reg.nii')
print('Saved registered brain to {}. Total time = {:.1f}'.format(file_base_path + '_reg.nii', time.time()-t0))

max_stars_repo_name: ClandininLab/multistim

max_issues_repo_name: ClandininLab/multistim

max_forks_repo_name: ClandininLab/visanalysis


text:
# Copyright 2020 Hsueh-Ti Derek Liu
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import bpy
import numpy as np

# Inputs:
#   pathRadius: radius of the circle camera path
#   pathHeight: height of the circle camera path
#   lookAtPos: look at position
#   focalLength: focal length of the camera
#   duration: duration of the video (turing 360 degree)
#   startAngle: staring angle of the camera (angle from x-axis)

def setCameraPath(
	pathRadius, pathHeight, lookAtPos, focalLength,\
	duration = 100, startAngle = 0):

	# path circle position
	circPos = tuple(np.add(lookAtPos, (0,0,pathHeight)))

	# create circle path
	bpy.ops.curve.primitive_bezier_circle_add(enter_editmode=False, location=circPos)
	OBJ_circle = bpy.context.object
	OBJ_circle.scale = (pathRadius,pathRadius,pathRadius)

	# create look at position
	bpy.ops.object.empty_add(type='PLAIN_AXES', location=lookAtPos)
	OBJ_empty = bpy.context.object

	# compute initial camera position
	camPos = (pathRadius*np.cos(startAngle/180.0*np.pi), pathRadius*np.sin(startAngle/180.0*np.pi), 0)
	camPos = tuple(np.add(camPos, circPos))

	# initialize camera
	bpy.ops.object.camera_add(location = camPos)
	OBJ_camera = bpy.context.object
	OBJ_camera.data.lens = focalLength

	# set camera to follow a path
	OBJ_camera.select_set(True)
	OBJ_circle.select_set(True)
	bpy.context.view_layer.objects.active = OBJ_circle
	bpy.ops.object.parent_set(type='FOLLOW')
	bpy.ops.object.select_all(action='DESELECT')
	OBJ_circle.data.path_duration = duration

	# set camera lookAt
	lookAt = OBJ_camera.constraints.new(type='TRACK_TO')
	lookAt.target = OBJ_empty
	lookAt.track_axis = 'TRACK_NEGATIVE_Z'
	lookAt.up_axis = 'UP_Y'

	return OBJ_camera

max_stars_repo_name: HTDerekLiu/BlenderToolbox