text:
"""Tests for the `marmot.Environment` class."""

__author__ = "Jake Nunemaker"
__copyright__ = "Copyright 2019, Jake Nunemaker"
__email__ = "jake.d.nunemaker@gmail.com"
__status__ = "Development"

import numpy as np
import pytest

from marmot import Agent, Object, Environment, lt, true
from _simpy.core import EmptySchedule
from marmot.agent import WindowNotFound
from marmot.object import AlreadyRegistered
from marmot.environment import (
    StateExhausted,
    ActionMissingKeys,
    RegistrationFailed,
    RegistrationConflict,
)

def test_registration(env):

    obj = Object("Test Object")
    agent = Agent("Test Agent")

    env.register(obj)
    assert len(env._objects) == 1
    assert obj.env == env

    env.register(agent)
    assert len(env._agents) == 1
    assert agent.env == env

def test_failed_registration(env):

    with pytest.raises(RegistrationFailed):
        env.register("Invalid")

def test_registration_conflict(env):

    obj1 = Object("Test Object")
    obj2 = Object("Test Object")
    agent1 = Agent("Test Agent")
    agent2 = Agent("Test Agent")

    env.register(obj1)
    env.register(obj2)
    assert len(env._objects) == 2

    env.register(agent1)
    with pytest.raises(RegistrationConflict):
        env.register(agent2)

def test_deregistration_with_name(env):

    agent = Agent("Test Agent")

    env.register(agent)
    assert len(env._agents) == 1
    assert agent.env == env

    env.deregister("Test Agent")
    assert len(env._agents) == 0
    assert agent.env == None

def test_deregistration_with_instance(env):

    agent = Agent("Test Agent")

    env.register(agent)
    assert len(env._agents) == 1
    assert agent.env == env

    env.deregister(agent)
    assert len(env._agents) == 0
    assert agent.env == None

def test_unknown_deregistration(env):

    agent = Agent("Test Agent")

    env.register(agent)
    assert len(env._agents) == 1

    env.deregister("Unknown Agent")
    assert len(env._agents) == 1

def test_already_registered():

    env1 = Environment("Test Environment 1")
    env2 = Environment("Test Environment 2")
    agent = Agent("Test Agent")

    env1.register(agent)

    with pytest.raises(AlreadyRegistered):
        env2.register(agent)

    env1.deregister(agent)
    env2.register(agent)
    assert len(env2._agents) == 1

def test_scheduled_agents(env, ExampleAgent):

    agent1 = ExampleAgent("Agent 1")
    env.register(agent1)
    agent1.pause(10)

    assert len(env.scheduled_agents) == 1

    agent2 = ExampleAgent("Agent 2")
    env.register(agent2)
    agent2.pause(10)

    assert len(env.scheduled_agents) == 2
    assert agent1 in env.scheduled_agents
    assert agent2 in env.scheduled_agents

def test_scheduled_agents_completeness(env, ExampleAgent):

    agent1 = ExampleAgent("Agent 1")
    env.register(agent1)
    agent1.pause_then_perform(50, 20)

    agent2 = ExampleAgent("Agent 2")
    env.register(agent2)
    agent2.pause(100)

    agent3 = ExampleAgent("Agent 3")
    env.register(agent3)
    agent3.pause_then_perform(40, 60)

    while True:

        try:
            assert None not in env.scheduled_agents
            env.step()

        except EmptySchedule:
            break

def test_bad_action_log(env):

    with pytest.raises(ActionMissingKeys):
        env._submit_log({"action": "TestAction"}, level="ACTION")

def test_env_state_shape(min_env, env):

    assert min_env.state.shape == (0,)

    assert env.state.shape == (24,)
    assert "temp" in env.state.dtype.names
    assert "workday" in env.state.dtype.names

def test_find_valid_constraints(env):

    valid1 = env._find_valid_constraints(temp=lt(10))
    assert len(valid1) == 1

    valid2 = env._find_valid_constraints(temp=10)
    assert len(valid2) == 0

    valid3 = env._find_valid_constraints(temp=lt(10), workday=true())
    assert len(valid3) == 2

    valid4 = env._find_valid_constraints(test=lt(10))
    assert len(valid4) == 0

def test_apply_constraints(env):

    # No constraints
    expected = np.repeat(True, 24)
    output = env._apply_constraints(env.state, constraints={})

    assert output.shape == expected.shape
    assert all(output == expected)

    # Temperature less than 100
    expected = np.concatenate(
        [np.array([True] * 12), np.array([False] * 6), np.array([True] * 6)]
    )

    output = env._apply_constraints(env.state, constraints={"temp": lt(100)})
    assert all(output == expected)

    # Working hours 7am - 8pm
    expected = np.concatenate(
        [np.array([False] * 6), np.array([True] * 14), np.array([False] * 4)]
    )

    output = env._apply_constraints(env.state, constraints={"workday": true()})
    assert all(output == expected)

    # Temperature less than 100, working hours 7am - 8pm
    expected = np.concatenate(
        [
            np.array([False] * 6),
            np.array([True] * 6),
            np.array([False] * 6),
            np.array([True] * 2),
            np.array([False] * 4),
        ]
    )

    output = env._apply_constraints(
        env.state, constraints={"temp": lt(100), "workday": true()}
    )
    assert all(output == expected)

def test_count_delay(env):
    arr1 = np.array([False, False, False, True, True])
    assert env._count_delays(arr1, 1) == [3, 1]
    assert env._count_delays(arr1, 2) == [3, 2]
    assert env._count_delays(arr1, 3) == None

    arr2 = np.array([True, True, False, False, True])
    assert env._count_delays(arr2, 1) == [1]
    assert env._count_delays(arr2, 2) == [2]
    assert env._count_delays(arr2, 3) == [2, 2, 1]
    assert env._count_delays(arr2, 4) == None

    arr3 = np.array([True, False, False, False, True, True, False, True])
    assert env._count_delays(arr3, 1) == [1]
    assert env._count_delays(arr3, 2) == [1, 3, 1]
    assert env._count_delays(arr3, 3) == [1, 3, 2]
    assert env._count_delays(arr3, 4) == [1, 3, 2, 1, 1]
    assert env._count_delays(arr3, 5) == None

def test_find_first_window(env):

    arr1 = np.array([False, False, False, True, True, True, False, False])
    assert env._find_first_window(arr1, 1) == 3
    assert env._find_first_window(arr1, 2) == 3
    assert env._find_first_window(arr1, 3) == 3
    assert env._find_first_window(arr1, 4) == None

    arr2 = np.array([True, True, False, True, True, True])
    assert env._find_first_window(arr2, 1) == 0
    assert env._find_first_window(arr2, 2) == 0
    assert env._find_first_window(arr2, 3) == 3
    assert env._find_first_window(arr2, 4) == None

def test_calculate_operational_delays(env, min_env):

    # No state, no constraints
    assert min_env.calculate_operational_delays(4, constraints={}) == [4]

    # No state, with constraints
    assert min_env.calculate_operational_delays(4, constraints={"temp": lt(100)}) == [4]

    # With state, no constraints
    assert env.calculate_operational_delays(4, constraints={}) == [4]

    # With state, with constraints
    assert env.calculate_operational_delays(4, constraints={"temp": lt(100)}) == [4]
    assert env.calculate_operational_delays(
        4, constraints={"temp": lt(100), "workday": true()}
    ) == [6, 4]
    assert env.calculate_operational_delays(
        8, constraints={"temp": lt(100), "workday": true()}
    ) == [6, 6, 6, 2]
    with pytest.raises(StateExhausted):
        env.calculate_operational_delays(19, constraints={"temp": lt(100)})

def test_find_operational_window(env, min_env):

    # No state, no constraints
    assert min_env.find_operational_window(4, constraints={}) == 0

    # No state, with constraints
    assert min_env.find_operational_window(4, constraints={"temp": lt(100)}) == 0

    # With state, no constraints
    assert env.find_operational_window(4, constraints={}) == 0

    # With state, with constraints
    assert env.find_operational_window(4, constraints={"temp": lt(100)}) == 0
    assert (
        env.find_operational_window(4, constraints={"temp": lt(100), "workday": true()})
        == 6
    )
    with pytest.raises(WindowNotFound):
        env.find_operational_window(8, constraints={"temp": lt(100), "workday": true()})

max_stars_repo_name: JakeNunemaker/marmot

max_issues_repo_name: JakeNunemaker/marmot

max_forks_repo_name: JakeNunemaker/marmot


text:
__author__ = "alvaro barbeira"

import logging
import os

import shutil
import traceback
from collections import namedtuple
from subprocess import call

import numpy
import pandas

from ...Exceptions import ReportableException
from ... import Utilities
from ...file_formats import BIMBAM
from ...individual_data.Utilities import _StudyBasedContext
from ...miscellaneous import matrices
from ...individual_data import Utilities as StudyUtilities

########################################################################################################################

class ResultStatsTF:
    GENE=0
    GENE_NAME=1
    N_SNPS_IN_MODEL=2
    STATUS = 3

    K_GENE="gene"
    K_GENE_NAME="gene_name"
    K_N_SNPS_IN_MODEL="n_snps_in_model"
    K_STATUS="status"

    order = [(GENE,K_GENE), (GENE_NAME, K_GENE_NAME), (N_SNPS_IN_MODEL, K_N_SNPS_IN_MODEL), (STATUS, K_STATUS)]

HyperParameters = namedtuple("HyperParameters",
    ["gene",
     "h_0_500", "h_0_025", "h_0_975",
     "pve_0_500", "pve_0_025", "pve_0_975",
     "rho_0_500", "rho_0_025", "rho_0_975",
     "pge_0_500", "pge_0_025", "pge_0_975",
     "pi_0_500", "pi_0_025", "pi_0_975",
     "n_gamma_0_500", "n_gamma_0_025", "n_gamma_0_975"
])

class _Context(_StudyBasedContext):
    def get_gemma_path(self): raise RuntimeError("Not implemented")
    def get_intermediate_path(self): raise RuntimeError("Not implemented")

########################################################################################################################

def run_gemma(context, gene):
    annotation = context.get_gene_annotation(gene)
    gene_name = annotation.gene_name
    weights, covariance_data, hyperparameters, stats = pandas.DataFrame(), matrices.matrix_data(gene, [], numpy.matrix([])), _hyper(gene), _stats(gene, gene_name)
    #weights, stats, covariance_data = _run_gemma(context, gene, annotation)
    try:
        #pass
        weights, covariance_data, hyperparameters, stats = _run_gemma(context, gene, annotation)
    except ReportableException as ex:
        status = Utilities.ERROR_REGEXP.sub('_', ex.msg)
        stats = _stats(gene, annotation.gene_name, status=status)
        logging.info("Reportable exception running gemma: %s", ex.msg)
    except Exception as ex:
        msg = '{0}'.format(type(ex))#.replace('"', "'")
        status = '"{0}"'.format(msg)
        stats = _stats(gene, annotation.gene_name, status=status)
        logging.info("Exception running gemma:\n%s", traceback.format_exc())
    finally:
        folder = _intermediate_folder(context, gene)
        if os.path.exists(folder):
            shutil.rmtree(folder)

    return weights, covariance_data, hyperparameters, stats

########################################################################################################################

def _intermediate_folder(context, gene):
    return os.path.join(context.get_intermediate_path(), gene)

def _study_prefix(context, gene):
    _i = _intermediate_folder(context, gene)
    return os.path.join(_i, "study")

def _output_folder(context, gene):
    _i = _intermediate_folder(context, gene)
    return os.path.join(_i, "results")

def _result(context, gene):
    return "results"

def _output_prefix(context, gene):
    return os.path.join(_output_folder(context, gene), _result(context, gene))

def _output_params(context, gene):
    return _output_prefix(context, gene) + ".param.txt"

def _output_hyperparams(context, gene):
    return _output_prefix(context, gene) + ".hyp.txt"

########################################################################################################################

def _stats(gene, gene_name, n_snps_in_model=numpy.nan, status=None):
    return (gene, gene_name, n_snps_in_model, status,)

def _hyper(gene):
    t = tuple([gene] + [None]*(len(HyperParameters._fields)-1))
    return HyperParameters._make(t)

def _run_gemma(context, gene, annotation):
    os.makedirs(_intermediate_folder(context, gene))
    study = StudyUtilities._get_study_for_gene(context, gene)
    study_prefix = _study_prefix(context, gene)
    BIMBAM.save_study(study, study_prefix)

    _run_gemma_exe(context, gene)

    weights = _get_weights(context, gene, study)
    covariance = numpy.cov(study.genotype.variants)
    covariance_data = matrices.matrix_data(gene, weights.rsid, covariance)
    hyper = _get_hyper(context, gene)
    stats = _stats(gene, annotation.gene_name, weights.shape[0])
    return weights, covariance_data, hyper, stats

def _get_weights(context, gene, study):
    params_path = _output_params(context, gene)
    if not os.path.exists(params_path):
        raise ReportableException("missing_gemma_parameters")
    params = pandas.read_table(params_path)
    params["weight"] = params.alpha + params.beta*params.gamma
    params = params[["rs", "weight"]].rename(columns={"rs":"rsid"}).assign(gene=gene)
    m = study.get_variants_metadata(params.rsid)
    m = m[["id", "allele_0", "allele_1"]].rename(columns={"id": "rsid", "allele_0":"ref_allele", "allele_1":"eff_allele"})
    w = m.merge(params, on="rsid")[["rsid", "gene", "weight", "ref_allele", "eff_allele"]]
    return w

def _get_hyper(context, gene):
    hyp_path = _output_hyperparams(context, gene)
    if not os.path.exists(hyp_path):
        raise ReportableException("missing_gemma_hyperparameters")
    #Take the second half of the values, assuming they are closer to Montecarlo Chain convergence
    hyp = pandas.read_table(hyp_path, sep="\s+")
    f_ = int(numpy.ceil(hyp.shape[0]/2))
    hyp = hyp.iloc[f_:,]

    r_ = (gene,)
    columns=["h", "pve", "rho", "pge", "pi", "n_gamma"]
    for k in columns:
        r_ += tuple(numpy.percentile(hyp[k], [50.0, 2.5, 97.5]))
    r_ = HyperParameters._make(r_)
    return r_

def _run_gemma_exe(context, gene):
    _intermediate = _intermediate_folder(context, gene)
    _script = os.path.join(_intermediate, "run.sh")
    _command = gemma_command(context, gene)
    with open(_script, "w") as f: f.write(_command)
    logging.log(6,"command:\n%s\n", _command)
    _o = os.path.join(_intermediate, "gemma_o.txt")
    _e = os.path.join(_intermediate, "gemma_e.txt")
    with open(_o, "w") as o:
        with open(_e, "w") as e:
            logging.log(8, "Running gemma for %s", gene)
            call(["bash", _script], stdout=o, stderr=e)

def gemma_command(context, gene):
    output_folder = _output_folder(context, gene)
    results = _result(context, gene)
    study = _study_prefix(context, gene)
    gemma = context.get_gemma_path()
    command = \
"""
#!/usr/bin/env bash
GEMMA_PATH={}
STUDY_PREFIX="{}"
OUTPUT_DIR="{}"
OUTPUT="{}"

#GEMMA has "output" folder as default
[ -d "$OUTPUT_DIR" ] || mkdir -p "$OUTPUT_DIR"

$GEMMA_PATH \\
-g "$STUDY_PREFIX.geno.txt.gz" \\
-p "$STUDY_PREFIX.pheno.txt" \\
-a "$STUDY_PREFIX.snp.txt" \\
-bslmm 1 \\
-outdir $OUTPUT_DIR \\
-o "$OUTPUT"
""".format(gemma, study, output_folder, results)
    return command

def dataframe_from_stats(stats):
    r = Utilities.to_dataframe(stats, [x[1] for x in ResultStatsTF.order])
    return r

def dataframe_from_covariance_data(data):
    d = matrices.matrices_data_to_dataframe(data, ["GENE", "RSID1", "RSID2", "VALUE"])
    return d

def dataframe_from_hyperparameters(data):
    d = Utilities.to_dataframe(data, list(HyperParameters._fields))
    return d

ext: py

max_stars_repo_name: jhchung/summary-gwas-imputation

max_issues_repo_name: RajLabMSSM/summary-gwas-imputation

max_forks_repo_name: RajLabMSSM/summary-gwas-imputation

reason: import numpy


text:
#!/usr/bin/env /usr/bin/python3
import numpy as np
from matplotlib import pyplot as plt
import argparse
from pathlib import Path
from scipy import optimize
from scipy.special import erf

################################################################################
#===============================================================================
# plot_spherical_output.py
#===============================================================================
################################################################################

#3.80573442,0.0796802,0.95420221,3.66025486,0.01169476

################################################################################

def smooth_runaway (x):
	return np.where(x>0, np.exp(-1/x), 0)

def smooth_step (x):
	return smooth_runaway(x)/(smooth_runaway(x)+smooth_runaway(1-x))

################################################################################

def F (x,x0, alpha, m, b, c):
	return smooth_step((x0-x)/alpha) * G(x, m, b, c)

def G (x, m, b, c):
	return -m*x+b + np.sqrt( (-m*x+b)**2 + c**2 )

def F_flat (x,x0, alpha, m, b, c):
	return ((np.tanh((x0-x)/alpha) + 1)/2)**2 * G(x, m, b, c)

def F_shift (x, x0):
	return F_flat(x+x0,3.80573442,0.0796802,0.95420221,3.66025486,0.01169476)

################################################################################

#def CDF (x,mu,sigma):
#	return 0.5*(1+erf((x-mu)/sigma/np.sqrt(2)))

#def F (x, m, b, mu, sigma):
#	(-m*x-b)*(1-CDF(x,mu,sigma))

################################################################################

n_params = np.array([24, 32, 48, 64, 128])
markers = np.array(['o', 'v', 's', 'P', 'X'])
colors = np.array(['purple', 'blue', 'green',
					'orange', 'red'])
fit_function = F
p0 = [3.8, 0.1, 1., 3.6, 0.1]
#fit_function = F_shift
#p0 = [0.04]

def plot_output (result_file = Path('../sphere_output/results.csv'),
				 output_file = Path('../plots/sphere_output_plot.svg'),
				 fine_scale = False, draw_box = False, return_ax = False):
	fig, ax = plt.subplots(1)
	output_file.parent.mkdir(parents = True, exist_ok = True)
	energy_data = np.loadtxt(result_file, comments = '#', delimiter = ',')
	n_param = energy_data[:,0]
	p0_param = energy_data[:,1]
	median = energy_data[:,2]
	mean = energy_data[:,3]
	error = energy_data[:,4]
	x_values = np.linspace(3.,4.,11)
	ax.plot(x_values, np.zeros_like(x_values),
			color = 'darkgrey', linestyle = '-', marker = '')
	y_values = np.linspace(0.,0.5,11)
	ax.plot(np.ones_like(y_values)*3.813, y_values,
			color = 'black', linestyle = '--', marker = '')
#	p0_values = p0_fits = np.array([3.7451,3.7711,3.7898,3.8016,3.8098])
#	for index, current_n in enumerate(n_params):
#		ax.plot(np.ones_like(y_values)*p0_values[index], y_values,
#				color = colors[index], linestyle = '--', marker = '')
	x = np.linspace(3.58,3.92,200)
	for index, current_n in enumerate(n_params):
		mask = (n_param == current_n)
		params, covar = optimize.curve_fit( fit_function,
										p0_param[mask],
										mean[mask],
										p0 = p0,
										sigma = error[mask],
										bounds = (0,24) )
	#	print(f'{params[0]:1.4f} +/-{covar[0,0]:1.4f}')
		y = fit_function(x,*params)
		ax.plot(x,y, color = colors[index], linestyle = '-')
		ax.plot(p0_param[mask],
				 mean[mask],
				 color = colors[index], linestyle = '',
				 marker = markers[index], markersize = 3,
				 label = 'n = {0:d}'.format(current_n))
#		ax.errorbar(p0_param[mask],
#					 median[mask],
#					 error[mask],
#					 color = colors[index], linestyle = '',
#					 marker = markers[index],
#					 label = 'n = {0:d}'.format(current_n))
	ax.plot(x, F_shift(x,0),
			 color = 'black', linestyle = '-', marker = '',
			 label = r'n $\rightarrow \infty$ (flat)')
	ax.grid(True)
	if(not fine_scale):
		ax.set_ylim((-0.01,0.445))
		ax.set_xlim((3.585,3.915))
		if(draw_box):
			xx = np.linspace(3.715, 3.825, 200)
			yy = np.linspace(-0.005, 0.14, 200)
			ax.plot(xx, np.ones_like(xx)*yy[0],
					 color = 'gray', linestyle = 'dotted', marker = '')
			ax.plot(xx, np.ones_like(xx)*yy[-1],
					 color = 'gray', linestyle = 'dotted', marker = '')
			ax.plot(np.ones_like(yy)*xx[0], yy,
					 color = 'gray', linestyle = 'dotted', marker = '')
			ax.plot(np.ones_like(yy)*xx[-1], yy,
					 color = 'gray', linestyle = 'dotted', marker = '')
	else:
		ax.set_ylim((-0.005,0.14))
		ax.set_xlim((3.715,3.825))
#	ax.set_xticks((p0_param[n_param == 24])[::5])
#	ax.set_title('Energy Barrier versus Shape Index')
	ax.set_xlabel(r'$p_0$')
	ax.set_ylabel(r'$\Delta \varepsilon$')
	ax.legend(loc = 'best', fancybox = True, framealpha = 1.)
	plt.savefig(output_file)
	plt.rc('pgf', texsystem='pdflatex')
	plt.savefig(output_file.with_suffix('.pgf'))
	if return_ax:
		return ax
#	plt.show()

################################################################################

if __name__ == '__main__':
	# Use argparse to get arguements from commandline call
	parser = argparse.ArgumentParser(
							description = 'Plot results of simulation.')
	parser.add_argument('-o', '--outfile',
						required = False,
						default = '../plots/sphere_output_plot.svg',
						type = str,
						help = 'file to put plot in')
	parser.add_argument('-b', '--box', dest='draw_box',
						action='store_const',
						const=True, default=False,
						help = 'Put a dotted box around finer scale.')
	parser.add_argument('-f', '--fine', dest='fine_scale',
						action='store_const',
						const=True, default=False,
						help = 'Plot finer scale.')
	parser.add_argument('result_file',
						nargs = '?',
						default = '../sphere_output/results.csv',
						type = str,
						help = 'file with simulation result data')
	args = parser.parse_args()
	plot_output(result_file = Path(args.result_file),
				output_file = Path(args.outfile),
				fine_scale = args.fine_scale,
				draw_box = args.draw_box )

################################################################################
# EOF

reason: import numpy,from scipy


text:
from __future__ import print_function
import matplotlib.pyplot as plt
import matplotlib.colors as plt_colors
import numpy as np
import scipy.io
import scipy.stats
from rnmu.pme.line import Line
import rnmu.pme.stats as stats

def plot_soft_line(ax, line, sigma, box, n_levels=64, color='r', alpha=0.8):
    x_min, x_max = box
    xi, yi = np.mgrid[slice(x_min[0], x_max[0], .001),
                      slice(x_min[1], x_max[1], .001)]
    pos = np.vstack((xi.flatten(), yi.flatten())).T
    dists = line.distances(pos)
    alphas = scipy.stats.norm.pdf(dists, loc=0, scale=sigma)
    alphas /= alphas.max()
    alphas = alphas.reshape(xi.shape)

    levels = np.linspace(1e-2, 1, num=n_levels, endpoint=True)
    c = plt_colors.ColorConverter().to_rgba(color)
    colors = np.tile(c, (n_levels, 1))
    colors[:, 3] = levels

    ax.contourf(xi, yi, alphas, levels=levels, colors=colors, antialiased=True)
    # line.plot(cax=ax, color='r', linewidth=2)

def plot_orthogonal_projection(data, mss, sigma, cutoff, axes=None):
    line = Line(data[mss, :])

    dists = line.distances(data) / sigma
    membership = np.exp(-(dists ** 2))
    idx = membership > np.exp(-(cutoff ** 2))
    membership = membership[idx]

    pfa = stats.concentration_pfa(membership, len(mss))

    if axes is not None:
        x_lim = (data[:, 0].min() - 0.1, data[:, 0].max() + 0.1)
        y_lim = (data[:, 1].min() - 0.1, data[:, 1].max() + 0.1)
        bbox = np.vstack((x_lim, y_lim)).T

        ax = axes[0]
        ax.set_xlim(x_lim)
        ax.set_ylim(y_lim)
        ax.scatter(data[:, 0], data[:, 1], c='w', s=10)
        ax.scatter(data[mss, 0], data[mss, 1], c='r', s=10)
        plot_soft_line(ax, line, sigma, bbox, n_levels=10, color='r', alpha=0.8)
        ax.set_aspect('equal', adjustable='box')

        membership.sort()
        membership = np.insert(membership, 0, 0)

        n = len(membership)
        acc = np.arange(n, dtype=np.float) / n

        below = membership > acc

        idx = np.where(below)[0]
        starts = np.setdiff1d(idx - 1, idx)
        ends = np.setdiff1d(idx + 1, idx)

        def get_crossing(i):
            if i < 0 or i + 2 >= n:
                return None
            sl = slice(i, i + 2)
            x1 = membership[sl]
            y2 = acc[sl]

            pts2 = np.vstack((x1, y2)).T
            crossing = np.cross([-1, 1, 0], Line(pts2).eq)
            crossing /= crossing[2]
            return crossing

        membership_all_runs = []
        acc_all_runs = []
        for s, e in zip(starts, ends):
            run = slice(s + 1, e)
            membership_run = membership[run]
            acc_run = acc[run]

            crossing = get_crossing(s)
            if crossing is not None:
                membership_run = np.insert(membership_run, 0, crossing[0])
                acc_run = np.insert(acc_run, 0, crossing[1])

            crossing = get_crossing(e - 1)
            if crossing is not None:
                membership_run = np.append(membership_run, crossing[0])
                acc_run = np.append(acc_run, crossing[1])

            membership_all_runs = np.hstack(
                (membership_all_runs, membership_run))
            acc_all_runs = np.hstack((acc_all_runs, acc_run))

        ax = axes[1]
        ax.set_xlim(0, 1)
        ax.set_ylim(0, 1)

        ax.fill_between(membership_all_runs, membership_all_runs, acc_all_runs,
                         color='g', alpha=0.3)

        arg_Dmin = np.argmax(membership - acc)
        pt = membership[arg_Dmin]
        ax.plot([pt, pt], [pt, acc[arg_Dmin]], 'r-', linewidth=2)

        ax.plot(membership, acc, 'k-', linewidth=2)
        ax.plot([0, 1], [0, 1], 'g-', linewidth=2)

        ax.set_aspect('equal', adjustable='box')

        ax.set_title('p-value: {:.3f}'.format(pfa), fontdict={'fontsize': 20})

def main(sigma=0.02, cutoff=3, transversal=True):
    mat = scipy.io.loadmat('../data/JLinkageExamples.mat')

    # data = mat['Stairs4_S00075_O60'].T
    # data = mat['Star5_S00075_O50'].T
    data = np.random.rand(5000, 2)
    n = len(data)
    print(n)

    random_sample = np.random.randint(n, size=2)
    # random_sample = [1, 30]
    print(random_sample)

    fig, axes = plt.subplots(nrows=1, ncols=2)
    plot_orthogonal_projection(data, random_sample, sigma, cutoff, axes)

    # i = 0
    # for j in range(0, 50):
    #     if i == j:
    #         continue
    #     random_sample = [i, j]
    #     plt.figure()
    #     plot_orthogonal_projection(data, random_sample, sigma, [121, 122])

def paper_figure(sigma=0.015, cutoff=3, transversal=True):
    mat = scipy.io.loadmat('../data/JLinkageExamples.mat')

    data = mat['Stairs4_S00075_O60'].T
    n = len(data)
    print(n)

    fig, axes = plt.subplots(nrows=2, ncols=4)

    plot_orthogonal_projection(data, [3, 10], sigma, cutoff, axes[:, 0])
    plot_orthogonal_projection(data, [60, 92], sigma, cutoff, axes[:, 1])
    plot_orthogonal_projection(data, [4, 180], sigma, cutoff, axes[:, 2])
    plot_orthogonal_projection(data, [250, 260], sigma, cutoff, axes[:, 3])

    fig.tight_layout(pad=0, h_pad=-4, w_pad=-3)

def half_layout(sigma=0.02, cutoff=3, add_line=False, transversal=True):
    data = np.random.rand(1000, 2)
    data[:, 0] /= 2
    data = np.append(data, np.array([[1, 0]]), axis=0)

    if add_line:
        width = 1. / 50
        line = np.random.rand(50, 2)
        line[:, 1] *= width
        line = np.append(line, np.array([[0, width/2], [1, width/2]]), axis=0)
        theta = np.radians(30)
        rot = np.array([[np.cos(theta), np.sin(theta)],
                        [-np.sin(theta), np.cos(theta)]])
        line = line.dot(rot)
        line[:, 1] += 0.2

        data = np.append(data, line, axis=0)
    else:
        data = np.append(data, np.array([[.5, 0], [.5, 1]]), axis=0)

    fig, axes = plt.subplots(nrows=1, ncols=2)

    sample = [-1, -2]
    plot_orthogonal_projection(data, sample, sigma, cutoff, axes)

    # i = 0
    # for j in range(0, 50):
    #     if i == j:
    #         continue
    #     random_sample = [i, j]
    #     plt.figure()
    #     plot_orthogonal_projection(data, random_sample, sigma, [121, 122])

if __name__ == '__main__':
    # main()
    # half_layout()
    # half_layout(add_line=True)
    paper_figure()
    plt.show()

reason: import numpy,import scipy


text:
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
sys.path.append('../')

from auto_scan_test import AutoScanTest, IgnoreReasons
from program_config import TensorConfig, ProgramConfig, OpConfig, CxxConfig, TargetType, PrecisionType, DataLayoutType, Place
import unittest

import hypothesis
from hypothesis import given, settings, seed, example, assume
import hypothesis.strategies as st
import argparse
from functools import partial
import random
import numpy as np

class TestBoxClipOp(AutoScanTest):
    def __init__(self, *args, **kwargs):
        AutoScanTest.__init__(self, *args, **kwargs)
        self.enable_testing_on_place(
            TargetType.Host,
            PrecisionType.FP32,
            DataLayoutType.NCHW,
            thread=[1, 4])

    def is_program_valid(self,
                         program_config: ProgramConfig,
                         predictor_config: CxxConfig) -> bool:
        return True

    def sample_program_configs(self, draw):
        lod_data = [[1, 2, 3]]
        dim0 = draw(st.integers(min_value=4, max_value=10))
        dim1 = draw(st.integers(min_value=4, max_value=10))
        in_shape = [dim0, dim1, 4]
        iminfo_shape = [dim0, 3]

        def generate_input(*args, **kwargs):
            return np.random.random(in_shape).astype(np.float32)

        def generate_iminfo(*args, **kwargs):
            return np.random.random(iminfo_shape).astype(np.float32)

        box_clip_op = OpConfig(
            type="box_clip",
            inputs={"Input": ["input_data"],
                    "ImInfo": ["iminfo_data"]},
            outputs={"Output": ["output_data"]},
            attrs={})

        program_config = ProgramConfig(
            ops=[box_clip_op],
            weights={},
            inputs={
                "input_data": TensorConfig(
                    data_gen=partial(generate_input), lod=lod_data),
                "iminfo_data": TensorConfig(
                    data_gen=partial(generate_iminfo), lod=lod_data),
            },
            outputs=["output_data"])
        return program_config

    def sample_predictor_configs(self):
        return self.get_predictor_configs(), ["box_clip"], (1e-5, 1e-5)

    def add_ignore_pass_case(self):
        pass

    def test(self, *args, **kwargs):
        self.run_and_statis(quant=False, max_examples=25)

if __name__ == "__main__":
    unittest.main(argv=[''])

max_forks_repo_name: yingshengBD/Paddle-Lite


text:
import urllib.request, json
import numpy as np
import datetime

def same_day(day1, day2):
    if day1.year == day2.year and day1.month == day2.month and day1.day == day2.day:
        return True
    else:
        return False

def is_in_lecture(current, start, end):
    num = len(start)

    for i in range(num):
        if start[i] <= current <= end[i]:
            return True

    return False

def is_between_lecture(current, start, end):
    num = len(start)

    for i in range(num - 1):
        if end[i] < current < start[i+1]:
            return True

    return False

def is_after_last_lecture(current, start, end):
    num = len(end)
    if current > end[num - 1]:
        return True
    else:
        return False

def is_before_first_lecture(current, start, end, minutes=10, seconds=10):
    if current + datetime.timedelta(minutes=minutes, seconds=seconds) > start[0] and current <= start[0]:
        return True
    else:
        return False

def readurl(current_time=datetime.datetime.fromisoformat('2020-07-13 08:50:00+02:00')):
    url = "https://sciot-calendar.azurewebsites.net/events"

    # current_time = datetime.datetime.now()
    # current_time = datetime.datetime.fromisoformat('2020-07-13 08:50:00+02:00')
    start = []
    end = []
    with urllib.request.urlopen(url) as dat:
        data = json.loads(dat.read().decode())

        for i in range(len(data)):
            temp_start = datetime.datetime.fromisoformat(data[i]['start']['dateTime'])
            if same_day(temp_start, current_time):
                start.append(temp_start)

            temp_end = datetime.datetime.fromisoformat(data[i]['end']['dateTime'])
            if same_day(temp_end, current_time):
                end.append(temp_end)

    if len(start) == 0 or len(end) == 0:
        print('no event today')
        # inLecture, betweenLectures, afterLastLecture, shortBeforeFirstLecture
        return False, False, False, False
    elif is_in_lecture(current_time, start, end):
        print('in lecture')
        # inLecture, betweenLectures, afterLastLecture, shortBeforeFirstLecture
        return True, False, False, False
    elif is_after_last_lecture(current_time, start, end):
        print('after last lecture')
        # inLecture, betweenLectures, afterLastLecture, shortBeforeFirstLecture
        return False, False, True, False
    elif is_between_lecture(current_time, start, end):
        print('lecture pause')
        # inLecture, betweenLectures, afterLastLecture, shortBeforeFirstLecture
        return False, True, False, False
    elif is_before_first_lecture(current_time, start, end, minutes=30):
        print('first lecture about to start')
        # inLecture, betweenLectures, afterLastLecture, shortBeforeFirstLecture
        return False, False, False, True
    else:
        print('too early/ some thing went wrong')
        # inLecture, betweenLectures, afterLastLecture, shortBeforeFirstLecture
        return False, False, False, False

#curr_time = datetime.datetime.fromisoformat('2020-07-13 08:05:00+02:00')
#while True:
    #readurl(curr_time)
    #curr_time += datetime.timedelta(minutes=10)

max_stars_repo_name: Jinaz/scaiot-project

max_issues_repo_name: Jinaz/scaiot-project

max_forks_repo_name: Jinaz/scaiot-project


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: maximov
"""

import torch
import numpy as np
import util_func
import util_data
from torchvision import transforms, utils
from os.path import join
from os import listdir
import arch.arch_unet_flex as arch_gen
import argparse

def inference(generator, out_dir, data_loader, device_comp, num_classes = 1200):
    total_imgs = 0
    for batch in data_loader:
        # prepare data
        im_faces, im_lndm, im_msk, im_ind = [item[0].float().to(device_comp) for item in batch]

        output_id = (int(im_ind[0].cpu())+1)%num_classes #chose next id

        labels_one_hot = np.zeros((1, num_classes))
        labels_one_hot[0, output_id] = 1
        labels_one_hot = torch.tensor(labels_one_hot).float().to(device_comp)

        # inference
        with torch.no_grad():
            input_gen = torch.cat((im_lndm, im_faces * (1 - im_msk)), 1)
            im_gen = generator(input_gen, onehot=labels_one_hot)
            im_gen = torch.clamp(im_gen*im_msk+im_faces * (1 - im_msk), 0, 1) #final image with BG

        # output image
        img_out = transforms.ToPILImage()(im_gen[0].cpu()).convert("RGB")
        img_out.save(join(out_dir, str(total_imgs).zfill(6) + '.jpg'))
        total_imgs+=1

    print("Done.")

def run_inference(data_path='../dataset/celeba/', num_folders = -1, model_path = '../modelG', output_path = '../output'):
    ##### PREPARING DATA
    if num_folders==-1:
        num_folders = len(listdir(join(data_path,'lndm')))

    dataset_test = util_data.ImageDataset(root_dir=data_path, label_num=num_folders, transform_fnc=transforms.Compose([transforms.ToTensor()]), flag_sample=1, flag_augment = False)
    data_loader = torch.utils.data.DataLoader(dataset=dataset_test, batch_size=1, shuffle=False)

    ##### PREPARING MODELS
    device_comp = util_func.set_comp_device(True)
    model = arch_gen.Generator()
    model.load_state_dict(torch.load(model_path + '.pth'))
    model.to(device_comp)
    print('Model is ready')

    inference(model, output_path, data_loader, device_comp=device_comp)

# Instantiate the parser
parser = argparse.ArgumentParser()
parser.add_argument('--data', type=str, help='path to input data directory', default='../dataset/celeba/')
parser.add_argument('--ids', type=int, help='how many folder/ids to process', default=-1)
parser.add_argument('--model', type=str, help='path to a pre-trained model and its name', default='../modelG')
parser.add_argument('--out', type=str, help='path to output data directory', default='../output')

args = parser.parse_args()

run_inference(data_path=args.data, num_folders=args.ids,model_path=args.model, output_path=args.out)

max_stars_repo_name: staskh/ciagan

max_issues_repo_name: staskh/ciagan

max_forks_repo_name: staskh/ciagan


text:
# https://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_moons
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC

import gif

h = .02  # step size in the mesh

X, y = make_moons(noise=0.3, random_state=0)
X = StandardScaler().fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4, random_state=42)

x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5

xx, yy = np.meshgrid(
    np.arange(x_min, x_max, h),
    np.arange(y_min, y_max, h)
)

model = KNeighborsClassifier(3)
model = SVC(gamma=2, C=1, probability=True)

model.fit(X_train, y_train)
score = model.score(X_test, y_test)
Z = model.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]
Z = Z.reshape(xx.shape)

cm = plt.cm.viridis
first, last = cm.colors[0], cm.colors[-1]
cm_bright = ListedColormap([first, last])
fig, ax = plt.subplots()
ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,
           edgecolors='k')
ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,
           edgecolors='k', alpha=0.6)
ax.set_xlim(xx.min(), xx.max())
ax.set_ylim(yy.min(), yy.max())
ax.set_xticks(())
ax.set_yticks(())
ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'),
        size=15, horizontalalignment='right');

# 2

@gif.frame
def plot(gamma, C):
    model = SVC(gamma=gamma, C=C, probability=True)
    model.fit(X_train, y_train)
    score = model.score(X_test, y_test)
    Z = model.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]
    Z = Z.reshape(xx.shape)
    cm = plt.cm.viridis
    first, last = cm.colors[0], cm.colors[-1]
    cm_bright = ListedColormap([first, last])
    fig, ax = plt.subplots(figsize=(8, 8))
    ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,
               edgecolors='k')
    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,
               edgecolors='k', alpha=0.6)
    ax.set_xlim(xx.min(), xx.max())
    ax.set_ylim(yy.min(), yy.max())
    ax.set_xticks(())
    ax.set_yticks(())
    ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'),
            size=15, horizontalalignment='right')
    fig.suptitle(f"SVC({gamma=}, {C=})")

from itertools import product

gammas = [1/100, 1/10, 1, 10, 100]
Cs = [1/100, 1/10, 1, 10, 100]

gif.options.matplotlib['dpi'] = 200

plot(1, 1)

frames = []
for gamma, C in product(gammas, Cs):
    frame = plot(gamma, C)
    frames.append(frame)

gif.save(frames, "output/mesh.gif", duration=2, unit='s')

# knn

@gif.frame
def plot(model):
    model.fit(X_train, y_train)
    score = model.score(X_test, y_test)
    Z = model.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]
    Z = Z.reshape(xx.shape)
    cm = plt.cm.viridis
    cm_bright = ListedColormap([cm.colors[0], cm.colors[-1]])
    fig, ax = plt.subplots(figsize=(8, 8))
    ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,
               edgecolors='k')
    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,
               edgecolors='k', alpha=0.6)
    ax.set_xlim(xx.min(), xx.max())
    ax.set_ylim(yy.min(), yy.max())
    ax.set_xticks(())
    ax.set_yticks(())
    ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'),
            size=15, horizontalalignment='right')
    fig.suptitle(f"KNeighborsClassifier({model.n_neighbors})")

gif.options.matplotlib['dpi'] = 200

frames = []
for n in range(1, 20):
    frame = plot(KNeighborsClassifier(n))
    frames.append(frame)

gif.save(frames, "output/mesh.gif", duration=1, unit='s')

max_stars_repo_name: maxhumber/ADW

max_issues_repo_name: maxhumber/ADW

max_forks_repo_name: maxhumber/ADW


text:
# -*- coding: utf-8 -*-
import logging
l = logging.getLogger(__name__)

from datetime import datetime, timedelta

from django_lean.experiments.models import (DailyEngagementReport,
                                            DailyConversionReport,
                                            DailyConversionReportGoalData,
                                            Experiment, Participant,
                                            GoalRecord, GoalType)
from django_lean.experiments.significance import chi_square_p_value

def calculate_participant_conversion(participant, goal_type, report_date):
    """
    Determines whether a specific participant achieved a specific goal_type
    between the participant's enrollment date and the report date.
    If goal_type is None, then it determines the result for _any_ goal_type.
    """
    if goal_type == None:
        count = GoalRecord.objects.filter(
            created__gte=participant.enrollment_date,
            created__lt=(report_date + timedelta(days=1)),
            anonymous_visitor=participant.anonymous_visitor).count()
    else:
        count = GoalRecord.objects.filter(
            goal_type=goal_type,
            created__gte=participant.enrollment_date,
            created__lt=(report_date + timedelta(days=1)),
            anonymous_visitor=participant.anonymous_visitor).count()

    return count and 1 or 0

def calculate_goal_type_conversion(goal_type,
                                   participants,
                                   report_date,
                                   participant_conversion_calculator=calculate_participant_conversion):
    """
    Calculates the number of conversions for a specific goal type among the group of
    participants between each participant's enrollment date and the given report date.
    """
    count = 0
    for participant in participants:
        count += participant_conversion_calculator(participant, goal_type, report_date)
    return count

def find_experiment_group_participants(group, experiment, report_date):
    """
    Returns a collection of participants belonging to the specified group in the
    given experiment. It only includes participants that were enrolled in the
    given report date.
    """
    return Participant.objects.filter(group=group,
                                      enrollment_date__lte=report_date,
                                      experiment=experiment,
                                      anonymous_visitor__isnull=False)

def __rate(a, b):
    if not b or a == None:
        return None
    return 100. * a / b

def __improvement(a, b):
    if not b or not a:
        return None
    return (a - b) * 100. / b

def get_conversion_data(experiment, date):
    """
    Returns (if report exists):
    {
      "date",
      "test_group_size",
      "control_group_size",
      "goal_types": {
        <goal_type_name>: {
          "test_count",
          "control_count",
          "test_rate",
          "control_rate",
          "improvement",
          "confidence"
        }, ...
      },
      "totals": {
        "test_count",
        "control_count",
        "test_rate",
        "control_rate",
        "improvement",
        "confidence"
      }
    }

    Otherwise, returns 'None'

    <goal_type_name> will map to None if a report was generated for a given day, but no goal type report was generated for <goal_type_name>
    """
    report_set = DailyConversionReport.objects.filter(experiment=experiment, date=date)
    if report_set.count() != 1:
        l.warning("No conversion report for date %s and experiment %s" %
               (date, experiment.name))
        return None

    report = report_set[0]
    test_rate = __rate(report.overall_test_conversion, report.test_group_size)
    control_rate = __rate(report.overall_control_conversion, report.control_group_size)
    improvement = __improvement(test_rate, control_rate)

    all_goal_types = GoalType.objects.all()

    goal_types_data = {}
    for goal_type in all_goal_types:
        goal_type_data_set = report.goal_data.filter(goal_type=goal_type)
        if goal_type_data_set.count() != 1:
            goal_data = None
        else:
            goal_type_data = goal_type_data_set[0]
            goal_test_rate = __rate(goal_type_data.test_conversion, report.test_group_size)
            goal_control_rate = __rate(goal_type_data.control_conversion, report.control_group_size)
            goal_improvement = __improvement(goal_test_rate, goal_control_rate)
            goal_data = {
                "test_count": goal_type_data.test_conversion,
                "control_count": goal_type_data.control_conversion,
                "test_rate": goal_test_rate,
                "control_rate": goal_control_rate,
                "improvement": goal_improvement,
                "confidence": goal_type_data.confidence
                }
        goal_types_data[goal_type.name] = goal_data
    data = {
        "date": report.date,
        "test_group_size": report.test_group_size,
        "control_group_size": report.control_group_size,
        "goal_types": goal_types_data,
        "totals": {
            "test_count": report.overall_test_conversion,
            "control_count": report.overall_control_conversion,
            "test_rate": test_rate,
            "control_rate": control_rate,
            "improvement": improvement,
            "confidence": report.confidence
            }
        }
    return data

class BaseReportGenerator(object):
    def __init__(self, report_model_class):
        self.report_model_class = report_model_class

    def generate_all_daily_reports(self):
        """ Generates all missing reports up until yesterday """
        experiments = Experiment.objects.filter(start_date__isnull=False)
        yesterday = (datetime.today() - timedelta(days=1)).date()
        for experiment in experiments:
            start_date = experiment.start_date
            current_date = start_date
            end_date = experiment.end_date or yesterday
            end_date = min(end_date, yesterday)

            # get or create the report for all the days of the experiment
            while current_date <= end_date:
                if (self.report_model_class.objects.filter(
                        experiment=experiment, date=current_date).count() == 0):
                    daily_report = self.generate_daily_report_for_experiment(
                        experiment=experiment, report_date=current_date)
                current_date = current_date + timedelta(days=1)

class ConversionReportGenerator(BaseReportGenerator):
    def __init__(self, goal_type_conversion_calculator=calculate_goal_type_conversion,
                 participant_finder=find_experiment_group_participants):
        BaseReportGenerator.__init__(self, DailyConversionReport)
        self.goal_type_conversion_calculator = goal_type_conversion_calculator
        self.participant_finder = participant_finder

    def __confidence(self, a_count, a_conversion, b_count, b_conversion):
        contingency_table = [[a_count - a_conversion, a_conversion],
                             [b_count - b_conversion, b_conversion]]

        chi_square, p_value = chi_square_p_value(contingency_table)
        if p_value:
            return (1 - p_value) * 100
        else:
            return None

    def generate_daily_report_for_experiment(self, experiment, report_date):
        """ Generates a single conversion report """
        control_participants = self.participant_finder(Participant.CONTROL_GROUP,
                                                       experiment, report_date)
        test_participants = self.participant_finder(Participant.TEST_GROUP,
                                                    experiment, report_date)
        control_participant_count = control_participants.count()
        test_participant_count = test_participants.count()

        total_control_conversion = self.goal_type_conversion_calculator(
            None, control_participants, report_date)
        total_test_conversion = self.goal_type_conversion_calculator(
            None, test_participants, report_date)

        confidence = self.__confidence(test_participant_count, total_test_conversion,
                                       control_participant_count, total_control_conversion)

        report = DailyConversionReport.objects.create(
            experiment=experiment,
            date=report_date,
            test_group_size=test_participant_count,
            control_group_size=control_participant_count,
            overall_test_conversion=total_test_conversion,
            overall_control_conversion=total_control_conversion,
            confidence=confidence)

        for goal_type in GoalType.objects.all():
            control_count = self.goal_type_conversion_calculator(goal_type,
                                                                 control_participants,
                                                                 report_date)
            test_count = self.goal_type_conversion_calculator(goal_type,
                                                              test_participants,
                                                              report_date)
            confidence = self.__confidence(test_participant_count, test_count,
                                           control_participant_count, control_count)
            DailyConversionReportGoalData.objects.create(
                report=report, goal_type=goal_type,
                test_conversion=test_count,
                control_conversion=control_count,
                confidence=confidence)

class EngagementReportGenerator(BaseReportGenerator):
    def __init__(self, engagement_score_calculator):
        BaseReportGenerator.__init__(self, DailyEngagementReport)
        self.engagement_score_calculator = engagement_score_calculator

    def __generate_scores(self, experiment, group, report_date):
        """
        Returns an array of all scores for participants in the given group in the
        given experiment, as of the specified report date.
        """
        participants = Participant.objects.filter(
                            experiment=experiment,
                            group=group,
                            enrollment_date__lte=report_date).exclude(user=None)
        scores = []
        for participant in participants:
            scores.append(self.engagement_score_calculator.
                          calculate_user_engagement_score(participant.user,
                                                          participant.enrollment_date,
                                                          report_date))
        return scores

    def generate_daily_report_for_experiment(self, experiment, report_date):
        """ Generates a single engagement report """
        try:
            from numpy import mean, isnan
            from scipy.stats import ttest_ind
        except ImportError:
            from django_lean.experiments.stats import mean, isnan, ttest_ind
        test_group_scores = self.__generate_scores(
            experiment, Participant.TEST_GROUP, report_date)
        control_group_scores = self.__generate_scores(
            experiment, Participant.CONTROL_GROUP, report_date)

        test_group_mean = None
        control_group_mean = None
        confidence = None

        if len(test_group_scores):
            test_group_mean = mean(test_group_scores)
        if len(control_group_scores):
            control_group_mean = mean(control_group_scores)
        if len(test_group_scores) and len(control_group_scores):
            t_value, p_value = ttest_ind(test_group_scores, control_group_scores)
            if isnan(p_value):
                confidence = None
            else:
                confidence = (1 - p_value) * 100

        DailyEngagementReport.objects.create(
            experiment=experiment,
            date=report_date,
            test_score=test_group_mean,
            control_score=control_group_mean,
            test_group_size=len(test_group_scores),
            control_group_size=len(control_group_scores),
            confidence=confidence)

max_stars_repo_name: MontmereLimited/django-lean

max_issues_repo_name: MontmereLimited/django-lean

max_forks_repo_name: MontmereLimited/django-lean

reason: from numpy,from scipy


text:
import argparse
import collections
import datetime
import json
import os
import logging
import itertools

import _jsonnet
import attr
import numpy as np
import torch
import wandb

from tensor2struct.utils import registry, random_state, vocab
from tensor2struct.utils import saver as saver_mod
from tensor2struct.commands import train, meta_train
from tensor2struct.training import maml

@attr.s
class MetaTrainConfig(meta_train.MetaTrainConfig):
    use_bert_training = attr.ib(kw_only=True)

class MetaTrainer(meta_train.MetaTrainer):
    def load_train_config(self):
        self.train_config = registry.instantiate(
            MetaTrainConfig, self.config["meta_train"]
        )

        if self.train_config.num_batch_accumulated > 1:
            self.logger.warn("Batch accumulation is used only at MAML-step level")

        if self.train_config.use_bert_training:
            if self.train_config.clip_grad is None:
                self.logger.info("Gradient clipping is recommended for BERT training")

    def load_optimizer(self, config):
        with self.init_random:
            # 0. create inner_optimizer
            # inner_parameters = list(self.model.get_trainable_parameters())
            inner_parameters = list(self.model.get_non_bert_parameters())
            inner_optimizer = registry.construct(
                "optimizer", self.train_config.inner_opt, params=inner_parameters
            )
            self.logger.info(f"{len(inner_parameters)} parameters for inner update")

            # 1. MAML trainer, might add new parameters to the optimizer, e.g., step size
            maml_trainer = maml.MAML(
                model=self.model, inner_opt=inner_optimizer, device=self.device,
            )
            maml_trainer.to(self.device)

            opt_params = maml_trainer.get_inner_opt_params()
            self.logger.info(f"{len(opt_params)} opt meta parameters")

            # 2. Outer optimizer
            # if config["optimizer"].get("name", None) in ["bertAdamw", "torchAdamw"]:
            if self.train_config.use_bert_training:
                bert_params = self.model.get_bert_parameters()
                non_bert_params = self.model.get_non_bert_parameters()
                assert len(non_bert_params) + len(bert_params) == len(
                    list(self.model.parameters())
                )
                assert len(bert_params) > 0
                self.logger.info(
                    f"{len(bert_params)} BERT parameters and {len(non_bert_params)} non-BERT parameters"
                )

                optimizer = registry.construct(
                    "optimizer",
                    config["optimizer"],
                    non_bert_params=non_bert_params,
                    bert_params=bert_params,
                )
                lr_scheduler = registry.construct(
                    "lr_scheduler",
                    config.get("lr_scheduler", {"name": "noop"}),
                    param_groups=[
                        optimizer.non_bert_param_group,
                        optimizer.bert_param_group,
                    ],
                )
            else:
                optimizer = registry.construct(
                    "optimizer",
                    config["optimizer"],
                    params=self.model.get_trainable_parameters(),
                )
                lr_scheduler = registry.construct(
                    "lr_scheduler",
                    config.get("lr_scheduler", {"name": "noop"}),
                    param_groups=optimizer.param_groups,
                )

            lr_scheduler = registry.construct(
                "lr_scheduler",
                config.get("lr_scheduler", {"name": "noop"}),
                param_groups=optimizer.param_groups,
            )
            return inner_optimizer, maml_trainer, optimizer, lr_scheduler

    def step(
        self,
        config,
        train_data_scheduler,
        maml_trainer,
        optimizer,
        lr_scheduler,
        last_step,
    ):
        with self.model_random:
            for _i in range(self.train_config.num_batch_accumulated):
                task = train_data_scheduler.get_batch(last_step)
                inner_batch, outer_batches = task
                ret_dic = maml_trainer.meta_train(
                    self.model, inner_batch, outer_batches
                )
                loss = ret_dic["loss"]

            # clip bert grad
            if self.train_config.clip_grad and self.train_config.use_bert_training:
                for param_group in optimizer.param_groups:
                    torch.nn.utils.clip_grad_norm_(
                        param_group["params"], self.train_config.clip_grad,
                    )

            optimizer.step()
            optimizer.zero_grad()

            # log lr for each step
            outer_lr = lr_scheduler.update_lr(last_step)
            if outer_lr is None:
                outer_lr = [param["lr"] for param in optimizer.param_groups]
            inner_lr = [param["lr"] for param in maml_trainer.inner_opt.param_groups]

        # Report metrics and lr
        if last_step % self.train_config.report_every_n == 0:
            self.logger.info("Step {}: loss={:.4f}".format(last_step, loss))
            self.logger.info(f"Step {last_step}, lr={inner_lr, outer_lr}")
            wandb.log({"train_loss": loss}, step=last_step)
            for idx, lr in enumerate(inner_lr):
                wandb.log({f"inner_lr_{idx}": lr}, step=last_step)
            for idx, lr in enumerate(outer_lr):
                wandb.log({f"outer_lr_{idx}": lr}, step=last_step)

def main(args):
    # setup logger etc
    config, logger = train.setup(args)

    # Construct trainer and do training
    trainer = MetaTrainer(logger, config)
    trainer.train(config, modeldir=args.logdir)

if __name__ == "__main__":
    args = train.add_parser()
    main(args)


text:
#!/usr/bin/env python3

import binning_utils as bu
from collections import defaultdict
from copy import deepcopy
from jediApplicationArgs import jediAppName, nOuterIter
import numpy as np
import os
import var_utils as vu

#======================================================
# outer iteration settings for Variational applications
#======================================================

outerIter = str(nOuterIter)
anIter = str(nOuterIter)
appName = jediAppName

#=============================================================
# groups of diagnostics for which statistics can be calculated
#=============================================================

## model-space diagnostics
modelDiags = ['mmgfsan']

## difference diagnostics
diffDiagnostics_ = defaultdict(list)
diffDiagnostics_['variational'] += ['omb']
diffDiagnostics_['hofx'] += ['omf']

## relative difference diagnostics
relativeDiagnostics_ = defaultdict(list)
relativeDiagnostics_['variational'] += ['rltv_omb']
relativeDiagnostics_['hofx'] += ['rltv_omf']

## absolute diagnostics
absoluteDiagnostics_ = defaultdict(list)
absoluteDiagnostics_['variational'] += ['obs', 'bak']
absoluteDiagnostics_['hofx'] += ['obs', 'h(x)']

## cloudy radiance diagnostics
cloudyRadDiagnostics = ['SCI']

## STD diagnostics
sigmaDiagnostics_ = defaultdict(list)
sigmaDiagnostics_['variational'] = ['omb','sigmaob', 'sigmab', 'CRyb']
sigmaDiagnostics_['hofx'] = ['omf','sigmaof', 'sigmaf', 'CRyf']

# variational analysis diagnostics
if nOuterIter > 0:
    diffDiagnostics_['variational'] += ['oma']
    relativeDiagnostics_['variational'] += ['rltv_oma']
    absoluteDiagnostics_['variational'] += ['ana']
    sigmaDiagnostics_['variational'] += ['oma','sigmaoa','sigmaa', 'CRya']

diffDiagnostics = diffDiagnostics_[jediAppName]
relativeDiagnostics = relativeDiagnostics_[jediAppName]
absoluteDiagnostics = absoluteDiagnostics_[jediAppName]
sigmaDiagnostics = sigmaDiagnostics_[jediAppName]

defaultDiagnostics = diffDiagnostics

## diagnostics for which QC is irrelevant
nonQCedDiagnostics = ['obs']

#==========================
# names and values of bins
#==========================

## heterogeneous/named bins
# note: the order of subplots will follow what is specified here

# latbandsMethod bins, north to south
allNamedLatBands = {}
allNamedLatBands['values']    = ['NPol','NXTro','Tro','SXTro','SPol']
allNamedLatBands['minBounds'] = [60.0, 30.0, -30.0, -90.0, -90.0]
allNamedLatBands['maxBounds'] = [90.0, 90.0,  30.0, -30.0, -60.0]

namedLatBands = {}
namedLatBands['values'] = ['NXTro','Tro','SXTro']
namedLatBands['minBounds'] = []
namedLatBands['maxBounds'] = []

for latBand in namedLatBands['values']:
    iband = allNamedLatBands['values'].index(latBand)
    namedLatBands['minBounds'].append(allNamedLatBands['minBounds'][iband])
    namedLatBands['maxBounds'].append(allNamedLatBands['maxBounds'][iband])

# cloudbandsMethod bins
clrskyThresh = 0.05
cldskyThresh = 1.0 - clrskyThresh

namedCldFracBands = {}
namedCldFracBands['values'] = [bu.clrskyMethod, bu.mixskyMethod, bu.cldskyMethod, bu.allskyMethod]
namedCldFracBands['minBounds'] = [0.0, clrskyThresh, cldskyThresh, 0.0]
namedCldFracBands['maxBounds'] = [clrskyThresh, cldskyThresh, 1.0, 1.0]

# surfbandsMethod bins
seasurfThresh = 0.05
landsurfThresh = 1.0 - seasurfThresh

namedLandFracBands = {}
namedLandFracBands['values'] = [bu.seasurfMethod, bu.mixsurfMethod, bu.landsurfMethod, bu.allsurfMethod]
namedLandFracBands['minBounds'] = [0.0, seasurfThresh, landsurfThresh, 0.0]
namedLandFracBands['maxBounds'] = [seasurfThresh, landsurfThresh, 1.0, 1.0]

# geoirlatlonboxMethod bins
geoirLonBands = deepcopy(bu.geoirlatlonBoxParams)
geoirLatBands = deepcopy(bu.geoirlatlonBoxParams)

# store minBounds/maxBounds
for centerLon in geoirLonBands['centerLon']:
  geoirLonBands['minBounds'] += \
    [centerLon - bu.geoirMaxZenith]
  geoirLonBands['maxBounds'] += \
    [centerLon + bu.geoirMaxZenith]
  geoirLatBands['minBounds'] += \
    [-bu.geoirMaxZenith]
  geoirLatBands['maxBounds'] += \
    [bu.geoirMaxZenith]

# ensure positive-definite longitude
for bound in ['minBounds', 'maxBounds']:
  for ii, lon in enumerate(geoirLonBands[bound]):
    while geoirLonBands[bound][ii] > 360.:
      geoirLonBands[bound][ii] -= 360.
    while geoirLonBands[bound][ii] < 0.:
      geoirLonBands[bound][ii] += 360.

## homogeneous bins
# note: the ordering described here does not make any difference
#       figure axes will be monotonically increasing except for pressure

# binLims is used to auto-generate a large number of binning
# configurations in the binVarConfigs dictionary

binLims = {}

binLims[vu.obsVarPrs] = {}
binLims[vu.obsVarPrs]['start']  =    0.0
binLims[vu.obsVarPrs]['finish'] = 1000.0
binLims[vu.obsVarPrs]['step']   =  100.0
binLims[vu.obsVarPrs]['format'] = '{:.0f}'

binLims[vu.obsVarAlt] = {}
binLims[vu.obsVarAlt]['start']  = 1000.0
binLims[vu.obsVarAlt]['finish'] = 50000.0
binLims[vu.obsVarAlt]['step']   = 2000.0
binLims[vu.obsVarAlt]['format'] = '{:.0f}'

binLims[vu.obsVarLat] = {}
binLims[vu.obsVarLat]['start']  = -90.0
binLims[vu.obsVarLat]['finish'] =  90.0
binLims[vu.obsVarLat]['step']   =  10.0
binLims[vu.obsVarLat]['format'] = '{:.0f}'

binLims[vu.obsVarLT] = {}
binLims[vu.obsVarLT]['start']  = bu.LH0
binLims[vu.obsVarLT]['finish'] = bu.LH1
binLims[vu.obsVarLT]['step']   = bu.LHDT
binLims[vu.obsVarLT]['format'] = '{:.0f}'

binLims[vu.obsVarSenZen] = {}
binLims[vu.obsVarSenZen]['start']  = 0.0
binLims[vu.obsVarSenZen]['finish'] = 70.0
binLims[vu.obsVarSenZen]['step']   = 5.0
binLims[vu.obsVarSenZen]['format'] = '{:.0f}'

binLims[vu.obsVarGlint] = {}
binLims[vu.obsVarGlint]['start']  = 0.0
binLims[vu.obsVarGlint]['finish'] = bu.maxGlint
binLims[vu.obsVarGlint]['step']   = 10.0
binLims[vu.obsVarGlint]['format'] = '{:.0f}'

binLims[vu.obsVarLandFrac] = {}
binLims[vu.obsVarLandFrac]['start']  = 0.0
binLims[vu.obsVarLandFrac]['finish'] = 1.0
binLims[vu.obsVarLandFrac]['step']   = seasurfThresh / 2.0
binLims[vu.obsVarLandFrac]['format'] = '{:.3f}'

binLims[vu.obsVarCldFrac] = {}
binLims[vu.obsVarCldFrac]['start']  = 0.0
binLims[vu.obsVarCldFrac]['finish'] = 1.0
binLims[vu.obsVarCldFrac]['step']   = clrskyThresh / 2.0
binLims[vu.obsVarCldFrac]['format'] = '{:.3f}'

binLims[vu.obsVarSCI] = {}
binLims[vu.obsVarSCI]['start']  = 0.0
binLims[vu.obsVarSCI]['finish'] = 60.0
binLims[vu.obsVarSCI]['step']   = 1.0
binLims[vu.obsVarSCI]['format'] = '{:.0f}'

binLims[vu.obsVarACI] = {}
binLims[vu.obsVarACI]['start']  = -20.0
binLims[vu.obsVarACI]['finish'] = 20.0
binLims[vu.obsVarACI]['step']   = 2.0
binLims[vu.obsVarACI]['format'] = '{:.0f}'

binLims[vu.obsVarNormErr] = {}
binLims[vu.obsVarNormErr]['start']  = -7.0
binLims[vu.obsVarNormErr]['finish'] =  7.0
binLims[vu.obsVarNormErr]['step']   =  0.25
binLims[vu.obsVarNormErr]['format'] = '{:.2f}'

binLims[vu.modVarLat] = {}
binLims[vu.modVarLat]['start']  = -90.0
binLims[vu.modVarLat]['finish'] =  90.0
binLims[vu.modVarLat]['step']   =  5.0
binLims[vu.modVarLat]['format'] = '{:.0f}'

#binLims[vu.modVarAlt] = {}
#binLims[vu.modVarAlt]['start']  = 0.0
#binLims[vu.modVarAlt]['finish'] = 50000.0
#binLims[vu.modVarAlt]['step']   = 5000.0
#binLims[vu.modVarAlt]['format'] = '{:.0f}'

for binType, param in binLims.items():
    binBounds = list(np.arange(
        param['start']-0.5*np.abs(param['step']),
        param['finish']+1.5*param['step'],
        param['step']))
    binLims[binType]['minBounds'] = []
    binLims[binType]['maxBounds'] = []
    binLims[binType]['values'] = []
    for ibin in list(range(len(binBounds)-1)):
        binLims[binType]['minBounds'].append(binBounds[ibin])
        binLims[binType]['maxBounds'].append(binBounds[ibin+1])

        binVal = 0.5 * (binBounds[ibin+1] + binBounds[ibin])
        binLims[binType]['values'].append(param['format'].format(binVal))

#@EffectiveQC* values:
# pass    = 0;   // we like that one!
# passive = 1;   // H(x) is computed (for monitoring, BC...) but obs not assimilated
# missing = 10;  // missing values prevent use of observation
# preQC   = 11;  // observation rejected by pre-processing
# bounds  = 12;  // observation value out of bounds
# domain  = 13;  // observation not within domain of use
# exclude = 14;  // observation excluded
# Hfailed = 15;  // H(x) computation failed
# thinned = 16;  // observation removed due to thinning
# diffref = 17;  // metadata too far from reference
# clw     = 18;  // observation removed due to cloud field
# fguess  = 19;  // observation too far from guess
# seaice  = 20;  // observation based sea ice detection, also flags land points
# track   = 21;  // observation removed as inconsistent with the rest of track
# buddy   = 22;  // observation rejected by the buddy check
# derivative = 23;  // observation removed due to metadata derivative value
# profile = 24;  // observation rejected by at least one profile QC check
# onedvar = 25;  // observation failed to converge in 1dvar check
# bayesianQC = 26;  // observation failed due to Bayesian background check
# modelobthresh = 27;  // observation failed modelob threshold check
# Static list above copied on 22 Apr 2021
# see ufo/src/ufo/filters/QCflags.h for up-to-date list

goodFlags = [0, 1]
goodFlagNames = ['pass', 'passive']

badFlags = [10, 11, 12, 13,
            14, 15, 16, 17,
            18, 19, 20, 21,
            22, 23, 24, 25, 26, 27,
]
badFlagNames = ['missing', 'preQC',      'bounds',  'domain',
                'exclude', 'Hfailed',    'thinned', 'diffref',
                'clw',     'fguess',     'seaice', 'track',
                'buddy',   'derivative', 'profile', 'onedvar', 'bayesianQC', 'modelobthresh',
]

#=================================================
# binning configurations used for all bin methods
#=================================================

# Each binVarConfig member has the following properties
# key: binVar string describes the variable to bin over
# binMethod: used to distinguish between multiple methods with the same binVar
#  (e.g., bu.identityBinMethod, bu.badQC, bu.latbandsMethod, etc.)
#     filters: list of filters that will exclude locations for each method
#     for each filter:
#         where: logical function that determines locations that are excluded
#         variable: string or class that is used to initialize the IdObsFunction or ObsFunction class;
#                   this may refer to a predefined key of vu.ObsGroups and vu.ObsVars
#         bounds: numerical value(s) used in the where test (scalar or Iterable same length as values). At least one filter must have len(bounds)==len(values).
#         except_diags (optional): list of diagnostics for which a particular filter does not apply
#     values (string): list of values associated with each bin
#
#TODO: classify each binmethod as 1D, 2D, etc. to indicate which types of figures apply to it

nullBinMethod = { 'filters': [], 'values': [] }
nullBinVarConfig = { bu.identityBinMethod:nullBinMethod }

# filter locations that do not have goodFlags
AnyBadQC = {
  'where': bu.notEqualAnyBound,
  'variable': vu.selfQCValue,
  'bounds': [goodFlags],
  'except_diags': nonQCedDiagnostics,
}

binVarConfigs = {
    vu.obsVarQC: {
        bu.goodQCMethod: {
            'filters': [
                {'where': bu.notEqualBound,
                 'variable': vu.selfQCValue,
                 'bounds': goodFlags,
                 'except_diags': nonQCedDiagnostics}
            ],
            'values': goodFlagNames,
        },
        bu.badQCMethod: {
            'filters': [
                {'where': bu.notEqualBound,
                 'variable': vu.selfQCValue,
                 'bounds': badFlags,
                 'except_diags': nonQCedDiagnostics},
                {'where': bu.equalBound,
                 'variable': vu.selfQCValue,
                 'bounds': badFlags,
                 'except_diags': nonQCedDiagnostics,
                 'mask_value': 0.0},
            ],
            'values': badFlagNames,
        },
        bu.allQCMethod: {
            'filters': [
                {'where': bu.notEqualBound,
                 'variable': vu.selfQCValue,
                 'bounds': goodFlags+badFlags,
                 'except_diags': nonQCedDiagnostics},
                {'where': bu.equalBound,
                 'variable': vu.selfQCValue,
                 'bounds': goodFlags+badFlags,
                 'except_diags': nonQCedDiagnostics,
                 'mask_value': 0.0},
            ],
            'values': goodFlagNames+badFlagNames,
        },
    },
    vu.obsVarPrs: {
        bu.PjetMethod: {
            'filters': [
# eliminate locations outside bu.P_jet_min to bu.P_jet_max
                {'where': bu.lessBound,
                 'variable': vu.prsMeta,
                 'bounds': bu.P_jet_min},
                {'where': bu.greatEqualBound,
                 'variable': vu.prsMeta,
                 'bounds': bu.P_jet_max},
                AnyBadQC,
            ],
            'values': bu.P_jet_val,
        },
    },
    vu.obsVarAlt: {
        bu.altjetMethod: {
            'filters': [
# eliminate locations outside bu.alt_jet_min to bu.alt_jet_max
                {'where': bu.lessBound,
                 'variable': vu.altMeta,
                 'bounds': bu.alt_jet_min},
                {'where': bu.greatEqualBound,
                 'variable': vu.altMeta,
                 'bounds': bu.alt_jet_max},
                AnyBadQC,
            ],
            'values': bu.alt_jet_val,
        },
    },
    vu.obsVarLat: {
        bu.latbandsMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': vu.latMeta,
                 'bounds': namedLatBands['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': vu.latMeta,
                 'bounds': namedLatBands['maxBounds']},
                AnyBadQC,
            ],
            'values': namedLatBands['values'],
        },
        bu.PjetMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': vu.latMeta,
                 'bounds': binLims[vu.obsVarLat]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': vu.latMeta,
                 'bounds': binLims[vu.obsVarLat]['maxBounds']},
                {'where': bu.lessBound,
                 'variable': vu.prsMeta,
                 'bounds': bu.P_jet_min},
                {'where': bu.greatEqualBound,
                 'variable': vu.prsMeta,
                 'bounds': bu.P_jet_max},
                AnyBadQC,
            ],
            'values': binLims[vu.obsVarLat]['values'],
        },
        bu.altjetMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': vu.latMeta,
                 'bounds': binLims[vu.obsVarLat]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': vu.latMeta,
                 'bounds': binLims[vu.obsVarLat]['maxBounds']},
                {'where': bu.lessBound,
                 'variable': vu.altMeta,
                 'bounds': bu.alt_jet_min},
                {'where': bu.greatEqualBound,
                 'variable': vu.altMeta,
                 'bounds': bu.alt_jet_max},
                AnyBadQC,
            ],
            'values': binLims[vu.obsVarLat]['values'],
        },
    },
    vu.obsVarLandFrac: {
        bu.surfbandsMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': vu.landfracGeo,
                 'bounds': namedLandFracBands['minBounds']},
                {'where': bu.greatBound,
                 'variable': vu.landfracGeo,
                 'bounds': namedLandFracBands['maxBounds']},
                AnyBadQC,
            ],
            'values': namedLandFracBands['values'],
        },
    },
    vu.obsVarCldFrac: {
        bu.cloudbandsMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': vu.cldfracMeta,
                 'bounds': namedCldFracBands['minBounds']},
                {'where': bu.greatBound,
                 'variable': vu.cldfracMeta,
                 'bounds': namedCldFracBands['maxBounds']},
                AnyBadQC,
            ],
            'values': namedCldFracBands['values'],
        },
    },
    vu.obsVarSCI: {
        bu.OkamotoMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': bu.SCIOkamoto,
                 'bounds': binLims[vu.obsVarSCI]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': bu.SCIOkamoto,
                 'bounds': binLims[vu.obsVarSCI]['maxBounds']},
            ],
            'values': binLims[vu.obsVarSCI]['values'],
        },
        bu.ScaleOkamotoMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': bu.ScaledSCIOkamoto,
                 'bounds': binLims[vu.obsVarSCI]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': bu.ScaledSCIOkamoto,
                 'bounds': binLims[vu.obsVarSCI]['maxBounds']},
            ],
            'values': binLims[vu.obsVarSCI]['values'],
        },
        bu.ModHarnischMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': bu.SCIModHarnisch,
                 'bounds': binLims[vu.obsVarSCI]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': bu.SCIModHarnisch,
                 'bounds': binLims[vu.obsVarSCI]['maxBounds']},
            ],
            'values': binLims[vu.obsVarSCI]['values'],
        },
        bu.ScaleModHarnischMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': bu.ScaledSCIModHarnisch,
                 'bounds': binLims[vu.obsVarSCI]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': bu.ScaledSCIModHarnisch,
                 'bounds': binLims[vu.obsVarSCI]['maxBounds']},
            ],
            'values': binLims[vu.obsVarSCI]['values'],
        },
    },
    vu.obsVarNormErr: {
        bu.OkamotoMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': bu.OkamotoNormalizedError,
                 'bounds': binLims[vu.obsVarNormErr]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': bu.OkamotoNormalizedError,
                 'bounds': binLims[vu.obsVarNormErr]['maxBounds']},
            ],
            'values': binLims[vu.obsVarNormErr]['values'],
        },
        bu.ScaleOkamotoMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': bu.ScaledOkamotoNormalizedError,
                 'bounds': binLims[vu.obsVarNormErr]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': bu.ScaledOkamotoNormalizedError,
                 'bounds': binLims[vu.obsVarNormErr]['maxBounds']},
            ],
            'values': binLims[vu.obsVarNormErr]['values'],
        },
        bu.ModHarnischMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': bu.ModHarnischNormalizedError,
                 'bounds': binLims[vu.obsVarNormErr]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': bu.ModHarnischNormalizedError,
                 'bounds': binLims[vu.obsVarNormErr]['maxBounds']},
            ],
            'values': binLims[vu.obsVarNormErr]['values'],
        },
        bu.ScaleModHarnischMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': bu.ScaledModHarnischNormalizedError,
                 'bounds': binLims[vu.obsVarNormErr]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': bu.ScaledModHarnischNormalizedError,
                 'bounds': binLims[vu.obsVarNormErr]['maxBounds']},
            ],
            'values': binLims[vu.obsVarNormErr]['values'],
        },
    },
    vu.obsRegionBinVar: {
        'AFRICA': nullBinMethod,
        'ATLANTIC': nullBinMethod,
        'AUSTRALIA': nullBinMethod,
        'CONUS': {
            'filters': [
                {'where': bu.lessBound,
                 'variable': vu.lonMeta,
                 'bounds': 234.0},
                {'where': bu.greatBound,
                 'variable': vu.lonMeta,
                 'bounds': 294.0},
                {'where': bu.lessBound,
                 'variable': vu.latMeta,
                 'bounds': 25.0},
                {'where': bu.greatBound,
                 'variable': vu.latMeta,
                 'bounds': 50.0},
                AnyBadQC,
            ],
            'values': ['CONUS'],
        },
        'EUROPE': nullBinMethod,
        'E_EUROPE': nullBinMethod,
        'NAMERICA': nullBinMethod,
        'PACIFIC': nullBinMethod,
        'SAMERICA': nullBinMethod,
        'SE_ASIA': nullBinMethod,
        'S_ASIA': nullBinMethod,
        bu.geoirlatlonboxMethod: {
            'filters': [
                {'where': bu.lessBound,
                 'variable': vu.lonMeta,
                 'bounds': geoirLonBands['minBounds']},
                {'where': bu.greatBound,
                 'variable': vu.lonMeta,
                 'bounds': geoirLonBands['maxBounds']},
                {'where': bu.lessBound,
                 'variable': vu.latMeta,
                 'bounds': geoirLatBands['minBounds']},
                {'where': bu.greatBound,
                 'variable': vu.latMeta,
                 'bounds': geoirLatBands['maxBounds']},
            ],
            'values': geoirLonBands['values'],
        },
#TODO: use shapefiles/polygons to describe geographic regions instead of lat/lon boxes, e.g.,
#        'CONUS_POLYGON': {
#            'filters': [
#                {'where': bu.notInBound,
#                 'variable': bu.Regions,
#                 'variable': [vu.lonMeta, vu.latMeta],
#                 'bounds': ['CONUS']},
#            ],
#            'values': ['CONUS'],
#        },
#        'EUROPE_POLYGON': {
#            'filters': [
#                {'where': bu.notInBound,
#                 'variable': bu.Regions,
#                 'variable': [vu.lonMeta, vu.latMeta],
#                 'bounds': ['EUROPE']},
#            ],
#            'values': ['EUROPE'],
#        },
    },
#TODO: enable binning in MPAS Model space
#    vu.modVarPrs: {
#        bu.identityBinMethod: {
#            'filters': [
#                {'where': bu.lessBound,
#                 'variable': vu.modVarPrs,
#                 'bounds': binLims[vu.modVarPrs]['minBounds']},
#                {'where': bu.greatEqualBound,
#                 'variable': vu.modVarPrs,
#                 'bounds': binLims[vu.modVarPrs]['maxBounds']},
#            ],
#            'values': binLims[vu.modVarPrs]['values'],
#        },
#    },
#    vu.modVarAlt: {
#        bu.identityBinMethod: {
#            'filters': [
#                {'where': bu.lessBound,
#                 'variable': vu.modVarAlt,
#                 'bounds': binLims[vu.modVarAlt]['minBounds']},
#                {'where': bu.greatEqualBound,
#                 'variable': vu.modVarAlt,
#                 'bounds': binLims[vu.modVarAlt]['maxBounds']},
#            ],
#            'values': binLims[vu.modVarAlt]['values'],
#        },
#    },
#    vu.modVarLat: {
#        bu.identityBinMethod: {
#            'filters': [
#                {'where': bu.lessBound,
#                 'variable': vu.modVarLat,
#                 'bounds': binLims[vu.modVarLat]['minBounds']},
#                {'where': bu.greatEqualBound,
#                 'variable': vu.modVarLat,
#                 'bounds': binLims[vu.modVarLat]['maxBounds']},
#            ],
#            'values: binLims[vu.modVarLat]['values'],
#        },
#        bu.latbandsMethod: {
#            'filters': [
#                {'where': bu.lessBound,
#                 'variable': vu.modVarLat,
#                 'bounds': namedLatBands['minBounds']},
#                {'where': bu.greatEqualBound,
#                 'variable': vu.modVarLat,
#                 'bounds': namedLatBands['maxBounds']},
#            ],
#            'values': namedLatBands['values'],
#        },
#    },
#    vu.modelRegionBinVar: {
#        'AFRICA': nullBinMethod,
#        'ATLANTIC': nullBinMethod,
#        'AUSTRALIA': nullBinMethod,
#        'CONUS': {
#            'filters': [
#                {'where': bu.lessBound,
#                 'variable': vu.modVarLon,
#                 'bounds': 234.0},
#                {'where': bu.greatBound,
#                 'variable': vu.modVarLon,
#                 'bounds': 294.0},
#                {'where': bu.lessBound,
#                 'variable': vu.modVarLat,
#                 'bounds': 25.0},
#                {'where': bu.greatBound,
#                 'variable': vu.modVarLat,
#                 'bounds': 50.0},
#            ],
#            'values': ['CONUS'],
#        },
#        'EUROPE': nullBinMethod,
#        'E_EUROPE': nullBinMethod,
#        'NAMERICA': nullBinMethod,
#        'PACIFIC': nullBinMethod,
#        'SAMERICA': nullBinMethod,
#        'SE_ASIA': nullBinMethod,
#        'S_ASIA': nullBinMethod,
#        bu.geoirlatlonboxMethod: {
#            'filters': [
#                {'where': bu.lessBound,
#                 'variable': vu.modelVarLon,
#                 'bounds': geoirLonBands['minBounds']},
#                {'where': bu.greatBound,
#                 'variable': vu.modelVarLon,
#                 'bounds': geoirLonBands['maxBounds']},
#                {'where': bu.lessBound,
#                 'variable': vu.modelVarLat,
#                 'bounds': geoirLatBands['minBounds']},
#                {'where': bu.greatBound,
#                 'variable': vu.modelVarLat,
#                 'bounds': geoirLatBands['maxBounds']},
#            ],
#            'values': geoirLonBands['values'],
#        },
#    },
}

#=============================
# Parameterized binVarConfigs
#=============================

# Add bu.identityBinMethod for identity ranged binning variables
identityRangeBinVars = {
    vu.obsVarAlt: [vu.altMeta, []],
    vu.obsVarACI: [bu.AsymmetricCloudImpact, ['obs','bak','ana','SCI']],
    vu.obsVarCldFrac: [vu.cldfracMeta, ['obs','bak','ana','SCI']],
    vu.obsVarGlint: [bu.GlintAngle, ['obs','bak','ana','SCI']],
    vu.obsVarLandFrac: [vu.landfracGeo, ['obs','bak','ana','SCI']],
    vu.obsVarLat: [vu.latMeta, ['obs','bak','ana','SCI']],
    vu.obsVarLT: [bu.LocalHour, ['obs','bak','ana','SCI']],
    vu.obsVarNormErr: [bu.NormalizedError, []],
    vu.obsVarPrs: [vu.prsMeta, []],
    vu.obsVarSenZen: [vu.senzenMeta, ['obs','bak','ana','SCI']],
}
for binVar, rangeVar in identityRangeBinVars.items():
    if binVar not in binVarConfigs: binVarConfigs[binVar] = {}
    binVarConfigs[binVar][bu.identityBinMethod] = {
        'filters': [
            {'where': bu.lessBound,
             'variable': rangeVar[0],
             'bounds': binLims[binVar]['minBounds']},
            {'where': bu.greatEqualBound,
             'variable': rangeVar[0],
             'bounds': binLims[binVar]['maxBounds']},
            AnyBadQC,
        ],
        'values': binLims[binVar]['values'],
        'override_exclusiveDiags': rangeVar[1],
    }

# Add named latitude-band-specific bins for applicable ranged variables
latBinVars = {
    vu.obsVarAlt: [vu.altMeta, []],
    vu.obsVarPrs: [vu.prsMeta, []],
}
for binVar, rangeVar in latBinVars.items():
    if binVar not in binVarConfigs: binVarConfigs[binVar] = {}
    for iband, latBand in enumerate(namedLatBands['values']):
        binVarConfigs[binVar][latBand] = {
            'filters': [
                {'where': bu.lessBound,
                 'variable': rangeVar[0],
                 'bounds': binLims[binVar]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': rangeVar[0],
                 'bounds': binLims[binVar]['maxBounds']},
                {'where': bu.lessBound,
                 'variable': vu.latMeta,
                 'bounds': namedLatBands['minBounds'][iband]},
                {'where': bu.greatEqualBound,
                 'variable': vu.latMeta,
                 'bounds': namedLatBands['maxBounds'][iband]},
                AnyBadQC,
            ],
            'values': binLims[binVar]['values'],
            'override_exclusiveDiags': rangeVar[1],
        }

# Add named cloud fraction-band-specific bins for applicable ranged variables
cldfracBinVars = {
#    vu.obsVarACI: [bu.AsymmetricCloudImpact],
#    vu.obsVarGlint: [bu.GlintAngle],
#    vu.obsVarLandFrac: [vu.landfracGeo],
    vu.obsVarLat: [vu.latMeta],
#    vu.obsVarLT: [bu.LocalHour],
#    vu.obsVarSenZen: [vu.senzenMeta],
}
for binVar, rangeVar in cldfracBinVars.items():
    if binVar not in binVarConfigs: binVarConfigs[binVar] = {}
    for iband, cldBand in enumerate(namedCldFracBands['values']):
        binVarConfigs[binVar][cldBand] = {
            'filters': [
                {'where': bu.lessBound,
                 'variable': rangeVar[0],
                 'bounds': binLims[binVar]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': rangeVar[0],
                 'bounds': binLims[binVar]['maxBounds']},
                {'where': bu.lessBound,
                 'variable': vu.cldfracMeta,
                 'bounds': namedCldFracBands['minBounds'][iband]},
                {'where': bu.greatBound,
                 'variable': vu.cldfracMeta,
                 'bounds': namedCldFracBands['maxBounds'][iband]},
                AnyBadQC,
            ],
            'values': binLims[binVar]['values'],
            'override_exclusiveDiags': ['obs','bak','ana','SCI'],
    }

# Add named land fraction-band-specific bins for applicable ranged variables
landfracBinVars = {
    vu.obsVarSCI: [bu.SCIOkamoto],
}
for binVar, rangeVar in landfracBinVars.items():
    if binVar not in binVarConfigs: binVarConfigs[binVar] = {}
    for iband, surfBand in enumerate(namedLandFracBands['values']):
        binVarConfigs[binVar][surfBand] = {
            'filters': [
                {'where': bu.lessBound,
                 'variable': rangeVar[0],
                 'bounds': binLims[binVar]['minBounds']},
                {'where': bu.greatEqualBound,
                 'variable': rangeVar[0],
                 'bounds': binLims[binVar]['maxBounds']},
                {'where': bu.lessBound,
                 'variable': vu.landfracGeo,
                 'bounds': namedLandFracBands['minBounds'][iband]},
                {'where': bu.greatBound,
                 'variable': vu.landfracGeo,
                 'bounds': namedLandFracBands['maxBounds'][iband]},
                AnyBadQC,
            ],
            'values': binLims[binVar]['values'],
            'override_exclusiveDiags': ['obs','bak','ana','SCI'],
    }

max_stars_repo_name: JCSDA/mpas-jedi

max_issues_repo_name: JCSDA/mpas-jedi

max_forks_repo_name: JCSDA/mpas-jedi