text:
import numpy as np

from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer

tk = Tokenizer(num_words=None, char_level=True, oov_token='UNK')
alphabet = "abcdefghijklmnopqrstuvwxyz0123456789,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{} "
char_dict = {}
for i, char in enumerate(alphabet):
    char_dict[char] = i + 1
tk.word_index = char_dict.copy()
tk.word_index[tk.oov_token] = max(char_dict.values()) + 1
vocab_size = len(tk.word_index)

def texts_to_data(texts, max_length):
    global tk
    seq = tk.texts_to_sequences(texts)
    data = pad_sequences(seq, maxlen=max_length, padding='post')
    return np.array(data, dtype='float32')

ext: py

reason: import numpy


text:
# Copyright (c) Facebook, Inc. and its affiliates.
import json
import pandas as pd

import numpy as np
import torch
from mmf.utils.file_io import PathManager
from mmf.utils.general import get_absolute_path

class AnnotationDatabase(torch.utils.data.Dataset):
    """
    Dataset for Annotations used in MMF

    TODO: Update on docs sprint
    """

    def __init__(self, config, path, *args, **kwargs):
        super().__init__()
        self.metadata = {}
        self.config = config
        self.start_idx = 0
        path = get_absolute_path(path)
        self.load_annotation_db(path)

    def load_annotation_db(self, path):
        if path.find("visdial") != -1 or path.find("visual_dialog") != -1:
            self._load_visual_dialog(path)
        elif path.endswith(".npy"):
            self._load_npy(path)
        elif path.endswith(".jsonl"):
            self._load_jsonl(path)
        elif path.endswith(".json"):
            self._load_json(path)
        elif path.endswith(".csv"):
            self._load_csv(path)
        else:
            print(path)
            raise ValueError("Unknown file format for annotation db")

    def _load_csv(self, path):
        db=pd.read_csv(path)
        db=db.to_dict('index')
        self.data = db

        self.start_idx =0

    def _load_jsonl(self, path):
        with PathManager.open(path, "r") as f:
            db = f.readlines()
            for idx, line in enumerate(db):
                db[idx] = json.loads(line.strip("\n"))
            self.data = db
            self.start_idx = 0

    def _load_npy(self, path):
        with PathManager.open(path, "rb") as f:
            self.db = np.load(f, allow_pickle=True)

        self.start_idx = 0

        if type(self.db) == dict:
            self.metadata = self.db.get("metadata", {})
            self.data = self.db.get("data", [])
        else:
            # TODO: Deprecate support for this
            self.metadata = {"version": 1}
            self.data = self.db
            # Handle old imdb support
            if "image_id" not in self.data[0]:
                self.start_idx = 1

        if len(self.data) == 0:
            self.data = self.db

    def _load_json(self, path):
        with PathManager.open(path, "r") as f:
            data = json.load(f)
        self.metadata = data.get("metadata", {})
        self.data = data.get("data", [])

        if len(self.data) == 0:
            raise RuntimeError("Dataset is empty")

    def _load_visual_dialog(self, path):
        from mmf.datasets.builders.visual_dialog.database import VisualDialogDatabase

        self.data = VisualDialogDatabase(path)
        self.metadata = self.data.metadata
        self.start_idx = 0

    def __len__(self):
        return len(self.data) - self.start_idx

    def __getitem__(self, idx):

        data = self.data[idx + self.start_idx]

        # Hacks for older IMDBs
        if "answers" not in data:
            if "all_answers" in data and "valid_answers" not in data:
                data["answers"] = data["all_answers"]
            if "valid_answers" in data:
                data["answers"] = data["valid_answers"]

        # TODO: Clean up VizWiz IMDB from copy tokens
        if "answers" in data and data["answers"][-1] == "<copy>":
            data["answers"] = data["answers"][:-1]

        return data

    def get_version(self):
        return self.metadata.get("version", None)

max_stars_repo_name: alisonreboud/mmf

max_issues_repo_name: alisonreboud/mmf

max_forks_repo_name: alisonreboud/mmf


text:
import pandas as pd
import numpy as np
import pprint
import datetime
def main():

    sample_startdate=datetime.datetime(1989, 7, 1)
    sample_enddate=datetime.datetime(2006, 2, 1)

    raw(sample_startdate,sample_enddate)
    #linearity()
    #symmetry()
def linearity():
    df = pd.read_csv("../output/monthly_treatment_counts.csv")
    df = df[df.date!="0"]
    cols = ['d_m025', 'd_0', 'd_025']
    df['d_025'] = (df["d_050"] + df["d_075"]+df['d_025']).astype(bool).astype(int)
    df['d_m025'] = (df["d_m050"] + df["d_m075"]+df['d_m025']).astype(bool).astype(int)
    new_df = df.copy()
    mappings = {
        'd_m025':'-0.25',
        'd_0':'0',
        'd_025':'0.25',
    }
    for i in new_df.index:
        bucket = set()
        for col in cols:
            if new_df.loc[i,col]==1:
                bucket.add(mappings[col])
        bucket_str = ",".join(sorted(list(bucket)))
        new_df.loc[i, 'perc_group'] = bucket_str
    targets = pd.read_csv("../output/fed_targets_with_alternatives.csv")[[
        'date', 'decision']]
    targets.loc[168, 'decision'] = -.5
    decisions = set(targets['decision'])
    group_cols = []
    for decision in decisions:
        targets[str(decision)] = targets['decision'] == decision
        #targets[str(decision)] = targets['decision'].apply(lambda x:1 if round_quarter(x)==decision else 0)
        group_cols.append(decision)
    new_df = pd.merge(left=new_df,right=targets,on="date")
    new_df.to_csv("../output/grouped_percent_treatments_linearity.csv")
    new_df['count'] = 1
    #print(new_df.columns)
    group_cols = ['date','perc_group','count']+[str(x) for x in sorted(group_cols)]
    lat_df = new_df.groupby("perc_group")[group_cols].sum()

    #print(lat_df)
    lat_df.to_latex("../output/linearity_percentage_group_counts_no_rounding.tex")

def symmetry():
    df = pd.read_csv("../output/monthly_treatment_counts.csv")
    df = df[df.date!="0"]
    #print(df)
    cols = ['d_m050','d_m025','d_0','d_025','d_050']
    df['d_050'] = (df["d_050"]+df["d_075"]).astype(bool).astype(int)
    df['d_m050'] = (df["d_m050"]+df["d_m075"]).astype(bool).astype(int)

    mappings = {
        'd_m050':50,
        'd_m025':25,
        'd_0':0,
        'd_025':25,
        'd_050':50
    }
    new_df = df.copy()
    for i in new_df.index:
        bucket = set()
        for col in cols:
            if new_df.loc[i,col]==1:
                bucket.add(str(mappings[col]))
        bucket_str = ",".join(sorted(list(bucket)))
        new_df.loc[i,'perc_group'] = bucket_str
        new_df.loc[:,'perc_group'] = new_df['perc_group'].\
            apply(lambda x:"0,25" if x=="25,50" else x)
    new_df['count'] = 1
    targets = pd.read_csv("../output/fed_targets_with_alternatives.csv")[[
        'date','target_before','target_after']]
    targets['decision'] = abs(targets['target_after']-targets['target_before'])
    targets.loc[168,'decision'] = .5
    #print(new_df[new_df.perc_group=="0,25,50"])
    decisions = set(targets['decision'])
    for decision in decisions:
        targets[str(decision)] = targets['decision']
    #    targets[str(decision)] = targets['decision'].apply(lambda x:1 if round_quarter(x)==decision else 0)
    columns = [str(x) for x in decisions]
    columns = columns+['date','decision']
    new_df = pd.merge(left=new_df,right=targets,on="date")
    #print(new_df.columns)
    new_df.to_csv("../output/grouped_percent_treatments_symmetry.csv")
    group_cols = ['date','perc_group','count','0.0','0.25','0.5']
    lat_df = new_df[group_cols].groupby("perc_group").sum()

    #print(lat_df)
    lat_df.to_latex("../output/symmetry_percentage_group_counts.tex")
def raw(sample_startdate,sample_enddate):
    df = pd.read_csv("../output/monthly_treatment_counts.csv")
    df = df[df.date!="0"]
    cols = ['d_m075','d_m050','d_m025','d_0','d_025','d_050','d_075']
    #df['d_025'] = (df["d_050"] + df["d_075"]+df['d_025']).astype(bool).astype(int)
    #df['d_m025'] = (df["d_m050"] + df["d_m075"]+df['d_m025']).astype(bool).astype(int)
    new_df = df.copy()
    mappings = {
        'd_m075': '-0.75',
        'd_m050': '-0.5',
        'd_m025': '-0.25',
        'd_0': '0',
        'd_025': '0.25',
        'd_050': '0.50',
        'd_075': '-0.75',
    }
    for i in new_df.index:
        bucket = set()
        for col in cols:
            if new_df.loc[i,col]==1:
                bucket.add(mappings[col])
        bucket_str = ",".join(sorted(list(bucket)))
        new_df.loc[i, 'perc_group'] = bucket_str
    targets = pd.read_csv("../output/fed_targets_with_alternatives.csv")[[
        'date', 'decision']]

    targets['newdate']=pd.to_datetime(targets['date'])
    targets=targets[(targets['newdate']>=sample_startdate) & (targets['newdate']<=sample_enddate)]
    targets=targets[targets['newdate']!=datetime.datetime(2003, 9, 15)]
    decisions = set(targets['decision'])
    group_cols = []
    for decision in decisions:
        targets[str(decision)] = targets['decision']==decision
        group_cols.append(decision)
    new_df = pd.merge(left=new_df,right=targets,on="date")
    new_df.to_csv("../output/grouped_percent_treatments_no_round.csv")
    new_df['count'] = 1
    columns = ['date','perc_group','count']+[str(x) for x in sorted(group_cols)]
    lat_df = new_df.groupby("perc_group")[columns].sum()
    lat_df = lat_df.astype(int).reset_index().rename({"perc_group":"Menu"})
    #print(lat_df)
    lat_df.rename(columns={'perc_group':'Policy Options'},inplace=True)
    lat_df.rename(columns={'count':'Count'},inplace=True)
    lat_df.to_latex("../output/overleaf_files/percentage_group_counts.tex",index=False)

def round_quarter(val):
    mappings = {
        -.125:-.25,
        .125:.25,
        -.3125:-.25,
        .3125:.25,
        .0625:0,
        -.0625:0,
        .75:.5,
        -.75:-.5
    }
    if val in mappings:
        return mappings[val]
    else:
        return val
if __name__ == "__main__":
    main()


text:
#!/usr/bin/env python

# Programmer(s): Sopan Patil.
# This file is part of the 'hydroutils' package.

import numpy
from scipy import integrate
import warnings

warnings.filterwarnings("ignore")

######################################################################

class OdeSolver(object):

    """ The 'OdeSolver' class contains the method for solving a system of
    Ordinary Differential Equations.
    """

    @staticmethod
    def solve_rk45(userfunction, initstate, userpara, tlength):

        """ This method performs the integration of the user provided function
        over the specified simulation time period.
        The ODE solver used is Runge-Kutta 4-5th order with Dormand-Prince corrector.

        Args:
            (1) userfunction: Function that provides the right hand side equations of the ODE system
            of the user's model.

            (2) initstate: Initial values of the state variables in the ODE system.

            (3) userpara: Parameter set of the user's model.

            (4) tlength: time length of the model simulation period.

        """

        solver = integrate.ode(userfunction)
        solver.set_integrator('dopri5', atol=1e-6, rtol=1e-3)
        solver.set_initial_value(initstate, 0.0)
        solver.set_f_params(userpara)

        while solver.successful() and solver.t < tlength:
            solver.integrate(solver.t+1)

# ---------------------------------------------------------------------------------

    @staticmethod
    def solve_rk4(f, x0, para, tlength):

        """ This method performs the integration of the user provided function
        over the specified simulation time period.
        The ODE solver used in Runge-Kutta 4th order.

        Args:
            (1) f: Function that provides the right hand side equations of the ODE system

            (2) x0: Initial values of the state variables in the ODE system

            (3) para: Parameter set of the user's model

            (4) tlength: time length of the model simulation period.

        """
        t = numpy.arange(tlength)
        n = len(t)
        x = numpy.array([x0] * n)

        for i in range(n - 1):
            h = t[i+1] - t[i]
            k1 = h * f(t[i], x[i], para)
            k2 = h * f(t[i] + 0.5 * h, x[i] + 0.5 * k1, para)
            k3 = h * f(t[i] + 0.5 * h, x[i] + 0.5 * k2, para)
            k4 = h * f(t[i+1], x[i] + k3, para)
            x[i+1] = x[i] + (k1 + 2.0 * (k2 + k3) + k4) / 6.0

##################################################################################

max_stars_repo_name: sopanpatil/hydroutils

max_issues_repo_name: sopanpatil/hydroutils

max_forks_repo_name: sopanpatil/hydroutils

reason: import numpy,from scipy


text:
import numpy as np

def constant_function(x, constant=0):
    """
    Returns a constant value `a` such that f(x) = a.

    Parameters
    ============
    x: the input value.
    constant: the constant value.
    """
    return constant

def dirac_delta_function(x, steps=[(1, 1)]):
    """
    Returns a Dirac delta function such that

    f(x) = y_0 if x = x_0,
           y_1 if x = x_1,
           ...
           else 0

    Parameters
    ============
    x: the input value.
    steps: a list of deltas.
    """
    for x_n, y_n in steps:
        if x == x_n:
            return y_n
    else:
        return 0

def random_stochastic_function(x, delta):
    """
    Creates a random stochastic function that adds a value between
    [-delta, delta]

    Parameters
    ============
    x: the input value.
    delta: defines the range
    """
    return (np.random.random_sample() * 2 * delta) - delta

def random_gaussian_function(x, sigma):
    """
    Samples from a Gaussian distribution.

    Parameters
    ============
    x: the input value.
    delta: defines the variance of the Gaussian curve.
    """
    return np.random.normal(0, sigma)

max_stars_repo_name: patriacaelum/HydraDX-simulations

max_issues_repo_name: patriacaelum/HydraDX-simulations

max_forks_repo_name: patriacaelum/HydraDX-simulations


text:
"""Usage: first download the data (about 12.5GB).
pip install gdown
gdown https://drive.google.com/uc?id=16xlZWvR9Ml0TM0VEgrBGnFbcpzA4uLOY
tar -xvf bair_dataset_numpy.tar.gz

In each numpy file there are the following keys:
data = np.load("train00.npz")
images = data["images"] # T x K x H x 64 x 64 x 3, dtype=np.uint8
actions = data["actions"] # T x H x 4
endeff = data["endeff"] # T x H x 3

where:
T = # of trajectories,
K = camera ID (0 is top, 1 is front),
H = horizon = 15
"""
import numpy as np
import itertools
import torch

from railrl.data_management.images import normalize_image, unnormalize_image
from railrl.torch.core import np_to_pytorch_batch
import railrl.torch.pytorch_util as ptu

from torch.utils import data
from torchvision.transforms import ColorJitter, RandomResizedCrop
from PIL import Image

import torchvision.transforms.functional as F

from railrl.torch.data import BatchLoader, InfiniteBatchLoader
from railrl.data_management.external.bair_dataset.config import BAIR_DATASET_LOCATION

class BAIRDataset(data.Dataset):
    train_data = {}
    test_data = {}

    def __init__(self, is_train, camera=1, n_train_files=10, info=None):
        self.is_train = is_train
        self.N = 1024 if is_train else 256
        self.traj_length = 15
        self.camera = camera

        if is_train:
            self.n_files = n_train_files
            for i in range(self.n_files):
                suffix = 'train{:0>2d}.npz'.format(i)
                filename = BAIR_DATASET_LOCATION + suffix
                data = np.load(filename)
                images = data['images']
                self.train_data[i] = images
        else:
            self.n_files = 1
            suffix = "test.npz"
            filename = BAIR_DATASET_LOCATION + suffix
            data = np.load(filename)
            images = data['images']
            self.test_data[0] = images

        # print("loaded BAIR dataset:", self.images.shape)
        # self.jitter = ColorJitter((0.7,1.3), (0.95,1.05), (0.95,1.05), (-0.05,0.05))
        # self.crop = RandomResizedCrop((48, 48), (0.9, 0.9), (1, 1))
        # RandomResizedCrop((int(sqrt(self.imlength)), int(sqrt(self.imlength))), (0.9, 0.9), (1, 1))

    def __len__(self):
        return self.N * self.traj_length * self.n_files

    def __getitem__(self, idx):
        i_file = idx // (self.N * self.traj_length)
        data = self.train_data if self.is_train else self.test_data
        images = data[i_file]

        i = idx % (self.N * self.traj_length)
        traj_i = i // self.traj_length
        trans_i = i % self.traj_length

        # idx_out = trans_i + traj_i * self.traj_length + i_file * self.N * self.traj_length
        # print(idx, idx_out, i_file, traj_i, trans_i)

        # x = Image.fromarray(self.data['observations'][traj_i, trans_i].reshape(48, 48, 3), mode='RGB')
        # c = Image.fromarray(self.data['env'][traj_i].reshape(48, 48, 3), mode='RGB')

        # # upsampling gives bad images so random resizing params set to 1 for now
        # # crop = self.crop.get_params(c, (0.9, 0.9), (1, 1))
        # crop = self.crop.get_params(c, (1, 1), (1, 1))
        # jitter = self.jitter.get_params((0.7,1.3), (0.95,1.05), (0.95,1.05), (-0.05,0.05))
        # # jitter = self.jitter.get_params((0.5,1.5), (0.9,1.1), (0.9,1.1), (-0.1,0.1))
        # # jitter = self.jitter.get_params(0.5, 0.1, 0.1, 0.1)

        # x = jitter(F.resized_crop(x, crop[0], crop[1], crop[2], crop[3], (48, 48), Image.BICUBIC))
        # c = jitter(F.resized_crop(c, crop[0], crop[1], crop[2], crop[3], (48, 48), Image.BICUBIC))
        # x_t = normalize_image(np.array(x).flatten()).squeeze()
        # env = normalize_image(np.array(c).flatten()).squeeze()

        data_dict = {
            'x_t': images[traj_i, self.camera, trans_i, 8:56, 8:56, :].transpose().flatten() / 255.0,
            'env': images[traj_i, self.camera, 0, 8:56, 8:56, :].transpose().flatten() / 255.0,
        }
        return data_dict

def generate_dataset(variant):
    train_dataset = BAIRDataset(is_train=True)
    test_dataset = BAIRDataset(is_train=False)

    train_batch_loader_kwargs = variant.get(
        'train_batch_loader_kwargs',
        dict(batch_size=32, num_workers=0, )
    )
    test_batch_loader_kwargs = variant.get(
        'test_batch_loader_kwargs',
        dict(batch_size=32, num_workers=0, )
    )

    train_data_loader = data.DataLoader(train_dataset,
        shuffle=True, drop_last=True, **train_batch_loader_kwargs)
    test_data_loader = data.DataLoader(test_dataset,
        shuffle=True, drop_last=True, **test_batch_loader_kwargs)

    train_dataset = InfiniteBatchLoader(train_data_loader)
    test_dataset = InfiniteBatchLoader(test_data_loader)
    info = {}

    return train_dataset, test_dataset, info


text:
import numpy as np
from scipy import stats
import pandas as pd

__all__ = ['unique_rows',
            'argrank',
            'mnmx',
            'mnmxi',
            'argsort_rows',
            'untangle',
            'first_nonzero',
            'complete_index']

def complete_index(df, index_cols, fill_value):
    """Complete the combinations of values in index_cols of df
    so there is one row in df per combination of values across
    index_cols. Missing combination values are filled with fill_value.

    Parameters
    ----------
    df : pd.DataFrame
        A pandas.DataFrame, no special indexing required, though index
        will be replaced in the copy.
    index_cols : list
        Columns in df that form a unique index, for expanding all
        combinations.
    fill_value : any value
        Filled for all columns when
        new rows are added with reindex

    Returns
    -------
    out : pd.DataFrame
        A possibly longer df with a complete combination
        of unique values across index columns, now as the index"""
    new_index = pd.MultiIndex.from_product(iterables=[df[col].unique() for col in index_cols], names=index_cols)
    out = df.reindex(new_index, fill_value=fill_value)
    return out

def unique_rows(a, return_index=False, return_inverse=False, return_counts=False):
    """Performs np.unique on whole rows of matrix a using a "view".
    See http://stackoverflow.com/a/16971324/74616"""
    try:
        dummy, uniqi, inv_uniqi, counts = np.unique(a.view(a.dtype.descr * a.shape[1]), return_index = True, return_inverse = True, return_counts = True)
        out = [a[uniqi,:]]
        if return_index:
            out.append(uniqi)
        if return_inverse:
            out.append(inv_uniqi)
        if return_counts:
            out.append(counts)
    except ValueError:
        """This doesn't work with all types of data, so fall back on slow-simple algo"""
        s = set()
        for i in range(a.shape[0]):
            s.add(tuple(a[i,:].tolist()))
        out = [np.array([row for row in s])]
    if len(out) == 1:
        return out[0]
    else:
        return tuple(out)

def untangle(y_orig, y_tangled, rtol=0.001):
    """Assuming two vectors contain the same elements (to a
    relative tolerance), but in different order, find the
    indices into y_tangled that "untangle" it so that it matches
    the original data.

    Parameters
    ----------
    y_orig : np.ndarray
    y_tangled : np.ndarray

    Returns
    -------
    untanglei : np.ndarray
        Indices into y_tangled such that: y_orig == y_tangled[untanglei]"""

    yo = np.array(y_orig)
    y = np.array(y_tangled)
    assert len(yo) == len(y)

    yo_sorti = np.argsort(yo)
    y_sorti = np.argsort(y)

    yo_sorted = yo[yo_sorti]
    y_sorted = y[y_sorti]
    assert np.allclose(yo_sorted, y_sorted, rtol=rtol)

    yo_unsorti = np.argsort(yo_sorti)

    untanglei = y_sorti[yo_unsorti]
    newy = y[untanglei]
    # assert newy == yo
    return untanglei

def argrank(vec, method='average'):
    """Return the rank (0 based) of the elements in vec"""
    return stats.rankdata(vec, method=method)
    '''sorti = np.argsort(vec)
    ranks = np.empty(len(vec), int)
    try:
        ranks[sorti] = np.arange(len(vec))
    except IndexError:
        ranks[sorti.values] = np.arange(len(vec))
    return ranks'''
def mnmx(arr):
    """Shortcut to return both the min and the max of arr"""
    return (np.min(arr), np.max(arr))
def mnmxi(arr):
    """Shortcut to return both the argmax and argmin of arr"""
    return (np.argmin(arr), np.argmax(arr))
def argsort_rows(a):
    """Performs argsort on whole rows of matrix a.
    See unique_rows and http://stackoverflow.com/a/16971324/74616"""
    sorti = np.argsort(a.view(a.dtype.descr * a.shape[1]), axis=0).flatten()
    return sorti

def first_nonzero(mask, axis, invalid_val=-1):
    """SO: https://stackoverflow.com/questions/47269390/numpy-how-to-find-first-non-zero-value-in-every-column-of-a-numpy-array"""
    return np.where(mask.any(axis=axis), mask.argmax(axis=axis), invalid_val)

max_stars_repo_name: agartland/utils

max_issues_repo_name: agartland/utils

max_forks_repo_name: agartland/utils


text:
import theano, theano.tensor as T
theano.config.optimizer='fast_compile'
theano.config.exception_verbosity='high'

from sklearn.utils import shuffle
import numpy as np
import sys, datetime, pickle, os, re

from Recurrent_Unit import GRU
from Optimization import Adam, rmsprop
from util import init_weight
class RNN_Model:
    def __init__(self, I=None, H=None, rnn_unit=GRU, opt=Adam, activation=T.nnet.elu):
        self.D = I
        self.hidden_layer_sizes = H
        self.rnn_unit = rnn_unit
        self.activation = activation
        self.opt = opt
        self.__setstate__()

    def fit(self, X, Y, epochs=50, mu=0.9, reg=0., batch_sz=50, lr=0.002):

        ### Initialize X and Y, calculating other variables
        thX = T.imatrix('X')
        thY = T.imatrix('Y')
        thStartPoints = T.ivector('start_points')

        Z = thX
        for ru in self.hidden_layers:
            Z = ru.output(Z, thStartPoints)
        py_x = T.nnet.softmax(Z.dot(self.Wo) + self.bo)
        prediction = T.argmax(py_x, axis=1)
        ###

        ### Training variable and function
        cost = T.mean(T.nnet.categorical_crossentropy(py_x, thY))
        updates = rmsprop(cost, params=self.params, lr=lr)

        self.train_op = theano.function(
            inputs=[thX, thY, thStartPoints],
            outputs=[cost, prediction, py_x],
            updates=updates,
        )
        ###

        ### iterating over input values
        n_batches = len(X) // batch_sz
        for i in range(epochs):
            # t0 = datetime.datetime.now()
            X, Y = shuffle(X, Y)
            tn_correct = 0
            tn_total = 0
            cost = 0
            for j in range(n_batches):
                n_correct = 0
                n_total = 0
                sequenceLengths = []
                input_sequence, output_sequence = [], []
                for k in range(j * batch_sz, (j + 1) * batch_sz):
                    # don't always add the end token
                    input_sequence += X[k]
                    output_sequence += Y[k]
                    sequenceLengths.append(len(X[k]))

                startPoints = np.zeros(sum(sequenceLengths), dtype=np.int32)
                last = 0
                for length in sequenceLengths:
                    startPoints[last] = 1
                    last += length
                # try:
                #     input_sequence = np.array(input_sequence, dtype=np.int32)
                #     output_sequence = np.array(output_sequence, dtype=np.int32)
                # except ValueError:
                #     exit()
                c, p, res = self.train_op(input_sequence, output_sequence, startPoints)

                cost += c

                for pj, yj in zip(p, output_sequence):
                    if pj == np.argmax(yj):
                        n_correct += 1
                tn_correct += n_correct
                tn_total += len(output_sequence)
                print("batch: %d/%d" % (j, n_batches), "cost:", c, "accuracy:",(float(n_correct) / len(output_sequence)))
            print("\nepoch: %d/%d cost: %f accuracy: %f\n"%(i,epochs,cost,float(tn_correct/tn_total)))

    def predict(self, X):
        thX = T.imatrix('X')
        thStartPoints = T.ivector('start_points')

        Z = thX
        for ru in self.hidden_layers:
            Z = ru.output(Z, thStartPoints)
        py_x = T.nnet.softmax(Z.dot(self.Wo) + self.bo)
        prediction = T.argmax(py_x, axis=1)

        self.predict_op = theano.function(
            inputs=[thX, thStartPoints],
            outputs=[prediction, py_x],
            allow_input_downcast=True
        )

        ### iterating over input values
        Y, PY_X =[], []
        for x in X:
            startPoints = np.zeros(len(x), dtype=np.int32)
            startPoints[0] = 1
            p, py_x = self.predict_op(x, startPoints)
            Y.append(p)
            PY_X.append(py_x)
        return Y, PY_X
        ###

    def __setstate__(self, state=None):
        if state:
            ru_params, Wo, bo = state
        else:
            Wo = init_weight(self.hidden_layer_sizes[-1], self.D)
            bo = np.zeros(self.D)
            ru_params = [None for i in self.hidden_layer_sizes]

        Mi = self.D
        self.hidden_layers = []
        for i in range(len(self.hidden_layer_sizes)):
            Mo = self.hidden_layer_sizes[i]
            ru = self.rnn_unit(Mi, Mo, self.activation, state=ru_params[i])
            self.hidden_layers.append(ru)
            Mi = Mo

        ### seting tensors
        self.Wo = theano.shared(Wo)
        self.bo = theano.shared(bo)
        self.params = [self.Wo, self.bo]
        for ru in self.hidden_layers:
            self.params += ru.params
        ###

    def __getstate__(self):
        ru_params = []
        for h in self.hidden_layers:
            ru_params.append(h.__getstate__())
        return tuple([ru_params, self.Wo.get_value(), self.bo.get_value()])

    # def save(self, filename):
    #     state = self.__getstate__()
    #     return state
        # pickle.dump(state, open(filename,'wb'))

    # def load(self, filename):
    #     state = pickle.load(open(filename, 'rb'))
    #     self.__setstate__(state)
    #     return

class CharPredictNNModel:
    def __init__(self, hidden_lay_sz=(128,)):
        self.chars = ' !?`-,.:;"\'?<>{}[]+-()&%$@^#*0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
        self.i2c_map = {i + 1: self.chars[i] for i in range(len(self.chars))}
        self.c2i_map = {v: k for k, v in self.i2c_map.items()}
        self.i2c_map[0], self.c2i_map[''] = '', 0
        self.D = len(self.i2c_map)

        self.map_vect = {}
        for i in self.c2i_map:
            self.map_vect[i] = np.zeros(self.D)
            self.map_vect[i][self.c2i_map[i]] = 1.0
        # self.seq_length = seq_len

        self.opt = Adam
        self.activation = T.nnet.relu
        self.hidden_lay_sz = hidden_lay_sz

    def compile(self):
        self.model = RNN_Model(I=self.D, H=self.hidden_lay_sz, rnn_unit=GRU, opt=Adam, activation=T.nnet.relu)
        # if model_file:
        #     self.load(model_file)
        # elif os.path.isfile('model_file.save'):
        #     self.load('model_file.save')
        # else:
        #     self.model.__setstate__()

    def sample_formation(self, text):
        X , Y = [], []
        text = re.sub(r'[ \t\n]{2,}',' ', text)
        tokens = set(text.split(' '))
        for word in tokens:
            if len(word) > 0:
                X.append([self.map_vect[c] if c in self.map_vect else self.map_vect[''] for c in word])
                Y.append([self.map_vect[c] if c in self.map_vect else self.map_vect[''] for c in word[1:] + ' '])
        return X, Y

    def train(self, fname):
        text = open(fname, 'r').read()
        X, Y = self.sample_formation(text)
        self.model.fit(X, Y)
        self.save('model_file.save')

    def save(self, filename):
        params = (self.hidden_lay_sz, self.model.__getstate__())
        pickle.dump(params, open(filename, 'wb'))

    def load(self, filename):
        self.hidden_lay_sz, state = pickle.load(open(filename, 'rb'))
        self.compile()
        self.model.__setstate__(state)

    def test(self, fname):
        text = open(fname, 'r').read()
        X, Y = self.sample_formation(text)
        X, Y = shuffle(X, Y)
        Y_, _ = self.model.predict(X)
        correct = 0
        for i in range(len(Y_)):
            if np.argmax(Y[i][-1]) == Y_[i][-1]:
                correct += 1
        return correct, len(X)

    def pridect(self, text):
        x = [self.map_vect[j] if j in self.map_vect else self.map_vect[''] for j in text]
        p, y = self.model.predict(np.array([x]))
        prob = dict((i, y[0][-1][i]) for i in range(len(y[0][-1])))
        prob = sorted(prob.items(), key=lambda kv: kv[1], reverse=True)
        return {self.i2c_map[k]:v for k,v in prob[:2]}

if __name__ == '__main__':
    fname = 'pg.txt'
    cmodel = CharPredictNNModel(hidden_lay_sz=(128,))
    cmodel.compile()
    cmodel.train(fname)

    # testing
    correct, total = cmodel.test(fname)
    print("accuracy :", float(correct)/total)

reason: import numpy,import theano


text:
import os

import numpy as np
import torch
import torch.nn as nn

from tqdm import tqdm

from data.faceattribute_dataset import FaceAttributesDataset
from models.DecisionDensenetModel import DecisionDensenetModel

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class Args:

    checkpoints_dir = "/path/to/checkopints/dir"
    data_dir = "/path/to/data/dir"

    ##################
    # FOR CELEBAMASK-HQ
    decision_model_name = 'decision_model_celebamhq'
    image_path_train = os.path.join(data_dir, "train", "images")
    image_path_val = os.path.join(data_dir, "test", "images")
    attributes_path = os.path.join(data_dir, "CelebAMask-HQ-attribute-anno.txt")
    load_size = (256, 256)

    ## FOR CELEBA
    #decision_model_name = 'decision_model_celeba'
    #image_path_train = os.path.join(data_dir, "img_squared128_celeba_train")
    #image_path_val = os.path.join(data_dir, "img_squared128_celeba_test")
    #attributes_path = os.path.join(data_dir, "list_attr_celeba.txt")
    #load_size = (128, 128)
    ###################

    train_attributes_idx = [20, 31, 39] # Male, Smile, Young
    batch_size = 32
    optimizer = 'adam'
    lr = 0.0001
    step_size = 10
    gamma_scheduler = 0.5

    num_epochs = 5

opt=Args()

# Load data
data_train = FaceAttributesDataset(image_path=opt.image_path_train, attributes_path=opt.attributes_path, load_size=opt.load_size)
data_val = FaceAttributesDataset(image_path=opt.image_path_val, attributes_path=opt.attributes_path, load_size=opt.load_size)

dataloader_train = torch.utils.data.DataLoader(data_train, batch_size=opt.batch_size, shuffle=True, num_workers=4)
dataloader_val = torch.utils.data.DataLoader(data_val, batch_size=opt.batch_size, shuffle=False, num_workers=4)

def train_one_epoch():

    print("Number of batches:", len(dataloader_train))
    total_loss = 0
    stat_loss = 0

    total_acc = np.zeros(len(opt.train_attributes_idx))
    stat_acc = np.zeros(len(opt.train_attributes_idx))

    model.train()

    for batch_idx, batch_data in enumerate(tqdm(dataloader_train)):
        batch_data['image'] = batch_data['image'].to(device)
        batch_data['attributes'] = batch_data['attributes'].to(device)

        # Forward pass
        optimizer.zero_grad()
        inputs = batch_data['image']

        pred = model(inputs)
        pred_labels = torch.where(pred > 0.5, 1.0, 0.0)
        real_labels = torch.index_select(batch_data['attributes'], 1, torch.tensor(opt.train_attributes_idx).to(device))

        # Compute loss and gradients
        loss = criterion(pred, real_labels)
        acc = compute_accuracy(pred_labels, real_labels)

        stat_loss += loss.item()
        total_loss += loss.item()
        stat_acc += acc
        total_acc += acc

        loss.backward()
        optimizer.step()

        batch_interval = 50
        if (batch_idx+1) % batch_interval == 0:
            log_string(' ---- batch: %03d ----' % (batch_idx+1))
            log_string('mean loss on the last 50 batches: %f'%(stat_loss/batch_interval))
            log_string('mean accuracy on the last 50 batches: '+ str(stat_acc/batch_interval))
            stat_loss = 0
            stat_acc = 0

    total_mean_loss = total_loss / len(dataloader_train)
    total_mean_acc = total_acc / len(dataloader_train)
    log_string('mean loss over training set: %f' % (total_mean_loss))
    log_string('mean accuracy over training set: ' + str(total_mean_acc))

    return total_mean_loss

def evaluate_one_epoch():

    model.eval()

    total_loss = 0
    stat_loss = 0
    total_acc = np.zeros(len(opt.train_attributes_idx))
    stat_acc = np.zeros(len(opt.train_attributes_idx))

    print("Number of batches:", len(dataloader_val))

    for batch_idx, batch_data in enumerate(tqdm(dataloader_val)):
        batch_data['image'] = batch_data['image'].to(device)
        batch_data['attributes'] = batch_data['attributes'].to(device)

        # Forward pass

        inputs = batch_data['image']
        with torch.no_grad():
            pred = model(inputs)
            pred_labels = torch.where(pred > 0.5 , 1.0,0.0)

            real_labels = torch.index_select(batch_data['attributes'],1,torch.tensor(opt.train_attributes_idx).to(device))

        # Compute loss and metrics
        loss = criterion(pred,real_labels)
        acc = compute_accuracy(pred_labels,real_labels)

        stat_loss += loss.item()
        total_loss += loss.item()
        stat_acc += acc
        total_acc += acc

        batch_interval = 50
        if (batch_idx+1) % batch_interval == 0:
            log_string(' ---- batch: %03d ----' % (batch_idx+1))
            log_string('mean loss on the last 50 batches: %f'%(stat_loss/batch_interval))
            log_string('mean accuracy on the last 50 batches: ' + str(stat_acc/batch_interval))
            stat_loss = 0
            stat_acc = 0

    total_mean_loss = total_loss/len(dataloader_val)
    total_mean_acc = total_acc/len(dataloader_val)

    log_string('mean loss over validation set: %f'%(total_mean_loss))
    log_string('mean accuracy over validation set: '+str(total_mean_acc))

    return total_mean_loss

model = DecisionDensenetModel(num_classes=len(opt.train_attributes_idx), pretrained=False)
model.to(device)

if opt.optimizer == 'adam':
    optimizer = torch.optim.Adam(model.parameters(),lr=opt.lr)
else:
    optimizer = torch.optim.SGD(model.parameters(),lr=opt.lr)

criterion = nn.BCELoss(reduction='mean')

def compute_accuracy(pred, target):
    same_ids = (pred == target).float().cpu()
    return torch.mean(same_ids,axis=0).numpy()

scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=opt.step_size, gamma=opt.gamma_scheduler, verbose=True)

LOG_DIR = os.path.join(opt.checkpoints_dir, opt.decision_model_name)
print(LOG_DIR)

if not os.path.exists(LOG_DIR):
    os.mkdir(LOG_DIR)

LOG_FOUT = open(os.path.join(LOG_DIR, 'log_train.txt'), 'a')
LOG_FOUT.write(str(opt)+'\n')

def log_string(out_str):
    LOG_FOUT.write(out_str+'\n')
    LOG_FOUT.flush()
    print(out_str)

lowest_loss = 100000
log_string("Starting training from the beginning.")

for epoch in range(opt.num_epochs):

    # Train one epoch
    log_string(' **** EPOCH: %03d ****' % (epoch+1))
    train_one_epoch()

    # Evaluate one epoch
    log_string(' **** EVALUATION AFTER EPOCH %03d ****' % (epoch+1))
    total_mean_loss = evaluate_one_epoch()
    if total_mean_loss < lowest_loss:
        lowest_loss = total_mean_loss
        save_dict = {'epoch': epoch+1, 'optimizer_state_dict': optimizer.state_dict(), 'loss': total_mean_loss, 'model_state_dict': model.state_dict()}
        torch.save(save_dict, os.path.join(LOG_DIR, 'checkpoint.tar'))

    scheduler.step()

max_stars_repo_name: valeoai/STEEX

max_issues_repo_name: valeoai/STEEX

max_forks_repo_name: valeoai/STEEX


text:
# -*- coding: utf-8 -*-

"""
IBEIS CORE
Defines the core dependency cache supported by the image analysis api

Extracts annotation chips from imaages and applies optional image
normalizations.

TODO:
     * interactive callback functions
     * detection interface
     * identification interface

NOTES:
    HOW TO DESIGN INTERACTIVE PLOTS:
        decorate as interactive
        depc.get_property(recompute=True)
        instead of calling preproc as a generator and then adding,
        calls preproc and passes in a callback function.
        preproc spawns interaction and must call callback function when finished.
        callback function adds the rowids to the table.

Needed Tables:
    Chip
    NormChip
    Feats
    Keypoints
    Descriptors
    ProbChip

    IdentifyQuery
    NeighborIndex
    QualityClassifier
    ViewpointClassifier

CommandLine:
    python -m ibeis.control.IBEISControl --test-show_depc_annot_graph --show

Setup:
    >>> from ibeis.core_annots import *  # NOQA
    >>> import ibeis
    >>> import plottool as pt
    >>> ibs = ibeis.opendb('testdb1')
    >>> depc = ibs.depc_annot
    >>> aid_list = ibs.get_valid_aids()[0:2]
"""
from __future__ import absolute_import, division, print_function, unicode_literals
from six.moves import zip
import dtool
import utool as ut
import vtool as vt
import numpy as np
import cv2
from ibeis.control.controller_inject import register_preprocs, register_subprops
from ibeis.algo.hots.chip_match import ChipMatch
from ibeis.algo.hots import neighbor_index
(print, rrr, profile) = ut.inject2(__name__, '[core_annots]')

derived_attribute = register_preprocs['annot']
register_subprop = register_subprops['annot']
# dtool.Config.register_func = derived_attribute

def testdata_core(defaultdb='testdb1', size=2):
    import ibeis
    # import plottool as pt
    ibs = ibeis.opendb(defaultdb=defaultdb)
    depc = ibs.depc_annot
    aid_list = ut.get_argval(('--aids', '--aid'), type_=list,
                             default=ibs.get_valid_aids()[0:size])
    return ibs, depc, aid_list

class ChipConfig(dtool.Config):
    _param_info_list = [
        #ut.ParamInfo('dim_size', 128, 'sz', hideif=None),
        #ut.ParamInfo('dim_size', 960, 'sz', hideif=None),
        ut.ParamInfo('dim_size', 700, 'sz', hideif=None, type_=eval),  # TODO: allow types to vary
        ut.ParamInfo(
            'resize_dim', 'width', '',
            #'resize_dim', 'area', '',
            valid_values=['area', 'width', 'height', 'diag', 'maxwh', 'wh'],
            hideif=lambda cfg: cfg['dim_size'] is None),
        ut.ParamInfo('dim_tol', 0, 'tol', hideif=0),
        ut.ParamInfo('preserve_aspect', True, hideif=True),
        ut.ParamInfo('histeq', False, hideif=False),
        ut.ParamInfo('adapteq', False, hideif=False),
        ut.ParamInfo('histeq_thresh', False, hideif=False),
        ut.ParamInfo('pad', 0, hideif=0),
        ut.ParamInfo('ext', '.png', hideif='.png'),
    ]

ChipImgType = dtool.ExternType(vt.imread, vt.imwrite, extkey='ext')

@derived_attribute(
    tablename='chips', parents=['annotations'],
    colnames=['img', 'width', 'height', 'M'],
    coltypes=[ChipImgType, int, int, np.ndarray],
    configclass=ChipConfig,
    #depprops=['image_uuid', 'verts', 'theta'],
    fname='chipcache4',
    rm_extern_on_delete=True,
    chunksize=256,
)
def compute_chip(depc, aid_list, config=None):
    r"""
    Extracts the annotation chip from the bounding box

    Args:
        depc (ibeis.depends_cache.DependencyCache):
        aid_list (list):  list of annotation rowids
        config (dict): (default = None)

    Yields:
        (uri, int, int): tup

    CommandLine:
        ibeis --tf compute_chip --show
        ibeis --tf compute_chip --show --pad=64 --dim_size=256 --db PZ_MTEST
        ibeis --tf compute_chip --show --pad=64 --dim_size=None --db PZ_MTEST
        ibeis --tf compute_chip --show --db humpbacks
        ibeis --tf compute_chip:1 --show

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> import ibeis
        >>> defaultdb = 'testdb1'
        >>> ibs = ibeis.opendb(defaultdb=defaultdb)
        >>> depc = ibs.depc_annot
        >>> config = ChipConfig.from_argv_dict(dim_size=None)
        >>> aid_list = ibs.get_valid_aids()[0:8]
        >>> chips = depc.get_property('chips', aid_list, 'img', config={'dim_size': 256})
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> #interact_obj = pt.interact_multi_image.MultiImageInteraction(chips, nPerPage=4)
        >>> import ibeis.viz.interact.interact_chip
        >>> interact_obj = ibeis.viz.interact.interact_chip.interact_multichips(ibs, aid_list, config2_=config)
        >>> interact_obj.start()
        >>> pt.show_if_requested()

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> import ibeis
        >>> defaultdb = 'testdb1'
        >>> ibs = ibeis.opendb(defaultdb=defaultdb)
        >>> depc = ibs.depc_annot
        >>> config = ChipConfig(**{'dim_size': (256, 256), 'resize_dim': 'wh'})
        >>> #dlg = config.make_qt_dialog()
        >>> #config = dlg.widget.config
        >>> aid_list = ibs.get_valid_aids()[0:8]
        >>> chips = depc.get_property('chips', aid_list, 'img', config=config)
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> pt.imshow(vt.stack_image_recurse(chips))
        >>> pt.show_if_requested()

    """
    print('Preprocess Chips')
    print('config = %r' % (config,))

    ibs = depc.controller
    chip_dpath = ibs.get_chipdir() + '2'

    ut.ensuredir(chip_dpath)

    #ext = config['ext']
    pad = config['pad']
    dim_size = config['dim_size']
    dim_tol = config['dim_tol']
    resize_dim = config['resize_dim']

    #cfghashid = config.get_hashid()
    #avuuid_list = ibs.get_annot_visual_uuids(aid_list)

    # TODO: just hash everything together
    #_fmt = 'chip_aid_{aid}_avuuid_{avuuid}_{cfghashid}{ext}'
    #cfname_list = [_fmt.format(aid=aid, avuuid=avuuid, ext=ext, cfghashid=cfghashid)
    #               for aid, avuuid in zip(aid_list, avuuid_list)]
    #cfpath_list = [ut.unixjoin(chip_dpath, chip_fname)
    #               for chip_fname in cfname_list]

    #gfpath_list = ibs.get_annot_image_paths(aid_list)
    gid_list    = ibs.get_annot_gids(aid_list)
    # TODO: use verts instead
    bbox_list   = ibs.get_annot_bboxes(aid_list)
    theta_list  = ibs.get_annot_thetas(aid_list)
    bbox_size_list = ut.take_column(bbox_list, [2, 3])

    # Checks
    invalid_flags = [w == 0 or h == 0 for (w, h) in bbox_size_list]
    invalid_aids = ut.compress(aid_list, invalid_flags)
    assert len(invalid_aids) == 0, 'invalid aids=%r' % (invalid_aids,)

    if resize_dim == 'wh':
        assert isinstance(dim_size, tuple), (
            'must specify both width and height in dim_size when resize_dim=wh')
        # Aspect ratio is not preserved. Use exact specifications.
        newsize_list = [dim_size for _ in range(len(bbox_size_list))]
    else:
        scale_func_dict = {
            'width': vt.get_scaled_size_with_width,
            'area': vt.get_scaled_size_with_area,  # actually root area
        }
        scale_func = scale_func_dict[resize_dim]

        if dim_size is None:
            newsize_list = bbox_size_list
        else:
            if resize_dim == 'area':
                dim_size = dim_size ** 2
                dim_tol = dim_tol ** 2
            newsize_list = [scale_func(dim_size, w, h, dim_tol) for (w, h) in bbox_size_list]

    if pad > 0:
        halfoffset_ms = (pad, pad)
        extras_list = [vt.get_extramargin_measures(bbox, new_size, halfoffset_ms)
                       for bbox, new_size in zip(bbox_list, newsize_list)]

        # Overwrite bbox and new size with margined versions
        bbox_list = ut.take_column(extras_list, 0)
        newsize_list = ut.take_column(extras_list, 1)

    # Build transformation from image to chip
    M_list = [vt.get_image_to_chip_transform(bbox, new_size, theta) for
              bbox, theta, new_size in zip(bbox_list, theta_list, newsize_list)]

    #arg_iter = zip(cfpath_list, gid_list, newsize_list, M_list)
    arg_iter = zip(gid_list, newsize_list, M_list)
    arg_list = list(arg_iter)

    filterfn_list = []
    from vtool import image_filters
    if config['histeq']:
        filterfn_list.append(image_filters.histeq_fn)
    if config['adapteq']:
        filterfn_list.append(image_filters.adapteq_fn)

    warpkw = dict(flags=cv2.INTER_LANCZOS4, borderMode=cv2.BORDER_CONSTANT)

    last_gid = None
    for tup in ut.ProgIter(arg_list, lbl='computing chips', backspace=True):
        # FIXME: THE GPATH SHOULD BE PASSED HERE WITH AN ORIENTATION FLAG
        #cfpath, gid, new_size, M = tup
        gid, new_size, M = tup
        # Read parent image # TODO: buffer this
        if gid != last_gid:  # We assume the gids are nicely ordered, no need to load the image more than once, if so
            imgBGR = ibs.get_image_imgdata(gid)
            last_gid = gid
        # Warp chip
        chipBGR = cv2.warpAffine(imgBGR, M[0:2], tuple(new_size), **warpkw)
        for filtfn in filterfn_list:
            chipBGR = filtfn(chipBGR)
        width, height = vt.get_size(chipBGR)
        yield (chipBGR, width, height, M)
        # Write chip to disk
        #vt.imwrite(cfpath, chipBGR)
        #yield (cfpath, width, height, M)

@register_subprop('chips', 'dlen_sqrd')
def compute_dlen_sqrd(depc, aid_list, config=None):
    size_list = np.array(depc.get('chips', aid_list, ('width', 'height'), config))
    dlen_sqrt_list = (size_list ** 2).sum(axis=1).tolist()
    return dlen_sqrt_list

class AnnotMaskConfig(dtool.Config):
    _param_info_list = [
        ut.ParamInfo('manual', True)
    ]
    _sub_config_list = [
        ChipConfig
    ]

@derived_attribute(
    tablename='annotmask', parents=['annotations'],
    colnames=['img', 'width', 'height'],
    coltypes=[('extern', vt.imread), int, int],
    configclass=AnnotMaskConfig,
    fname='../maskcache2',
    # isinteractive=True,
)
def compute_annotmask(depc, aid_list, config=None):
    r"""
    Interaction dispatcher for annotation masks.

    Args:
        depc (ibeis.depends_cache.DependencyCache):
        aid_list (list):  list of annotation rowids
        config (AnnotMaskConfig): (default = None)

    Yields:
        (uri, int, int): tup

    CommandLine:
        python -m ibeis.core_annots --exec-compute_annotmask --show
        python -m ibeis.core_annots --exec-compute_annotmask --show --edit

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> ibs, depc, aid_list = testdata_core()
        >>> config = AnnotMaskConfig(dim_size=None)
        >>> chip_config = config.chip_cfg
        >>> edit = ut.get_argflag('--edit')
        >>> mask = depc.get_property('annotmask', aid_list, 'img', config, recompute=edit)[0]
        >>> chip = depc.get_property('chips', aid_list, 'img', config=chip_config)[0]
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> resized = vt.resize_mask(mask, chip)
        >>> blended = vt.blend_images_multiply(chip, resized)
        >>> pt.imshow(blended, title='mask')
        >>> pt.show_if_requested()
    """
    from plottool import interact_impaint
    # TODO: Ensure interactive required cache words
    # Keep manual things above the cache dir
    mask_dpath = ut.unixjoin(depc.cache_dpath, '../ManualChipMask')
    ut.ensuredir(mask_dpath)

    ibs = depc.controller
    chip_config = config.chip_cfg
    chip_imgs = depc.get('chips', aid_list, 'img', config=chip_config)

    cfghashid = config.get_hashid()
    avuuid_list = ibs.get_annot_visual_uuids(aid_list)

    # TODO: just hash everything together
    ext = '.png'
    _fmt = 'mask_aid_{aid}_avuuid_{avuuid}_{cfghashid}{ext}'
    fname_list = [_fmt.format(aid=aid, avuuid=avuuid, ext=ext, cfghashid=cfghashid)
                   for aid, avuuid in zip(aid_list, avuuid_list)]

    for img, fname, aid in zip(chip_imgs, fname_list, aid_list):
        mask_fpath = ut.unixjoin(mask_dpath, fname)
        if ut.checkpath(mask_fpath):
            # Allow for editing on recompute
            init_mask = vt.imread(mask_fpath)
        else:
            init_mask = None
        mask = interact_impaint.impaint_mask2(img, init_mask=init_mask)
        vt.imwrite(mask_fpath, mask)
        print('imwrite')
        w, h = vt.get_size(mask)

        yield mask_fpath, w, h
        # Remove the old chips
        #ibs.delete_annot_chips([aid])
        #ibs.delete_annot_chip_thumbs([aid])

class ProbchipConfig(dtool.Config):
    # TODO: incorporate into base
    _named_defaults = {
        'rf': {
            'fw_detector': 'rf',
            'smooth_thresh': None,
            'smooth_ksize': None,
        }
    }
    _param_info_list = [
        #ut.ParamInfo('preserve_aspect', True, hideif=True),
        ut.ParamInfo('fw_detector', 'cnn', 'detector='),
        ut.ParamInfo('fw_dim_size', 256, 'sz'),
        ut.ParamInfo('smooth_thresh', 20, 'thresh='),
        ut.ParamInfo('smooth_ksize', 20, 'ksz=', hideif=lambda cfg: cfg['smooth_thresh'] is None),
        #ut.ParamInfo('ext', '.png'),
    ]
    #_sub_config_list = [
    #    ChipConfig
    #]

ProbchipImgType = dtool.ExternType(ut.partial(vt.imread, grayscale=True),
                                   vt.imwrite, extern_ext='.png')

@derived_attribute(
    tablename='probchip', parents=['annotations'],
    colnames=['img'],
    coltypes=[ProbchipImgType],
    configclass=ProbchipConfig,
    fname='chipcache4',
    # isinteractive=True,
)
def compute_probchip(depc, aid_list, config=None):
    """ Computes probability chips using pyrf

    CommandLine:
        python -m ibeis.core_annots --test-compute_probchip --nocnn --show --db PZ_MTEST
        python -m ibeis.core_annots --test-compute_probchip --show --fw_detector=cnn
        python -m ibeis.core_annots --test-compute_probchip --show --fw_detector=rf --smooth_thresh=None

    Example1:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> import ibeis
        >>> ibs, depc, aid_list = testdata_core()
        >>> aid_list = ibs.get_valid_aids(species='zebra_plains')[0:10]
        >>> config = ProbchipConfig.from_argv_dict(fw_detector='rf', smooth_thresh=None)
        >>> #probchip_fpath_list_ = ut.take_column(list(compute_probchip(depc, aid_list, config)), 0)
        >>> probchip_list_ = ut.take_column(list(compute_probchip(depc, aid_list, config)), 0)
        >>> #result = ut.list_str(probchip_fpath_list_)
        >>> #print(result)
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> #xlabel_list = list(map(str, [vt.image.open_image_size(p) for p in probchip_fpath_list_]))
        >>> #iteract_obj = pt.interact_multi_image.MultiImageInteraction(probchip_fpath_list_, nPerPage=4, xlabel_list=xlabel_list)
        >>> xlabel_list = [str(vt.get_size(img)) for img in probchip_list_]
        >>> iteract_obj = pt.interact_multi_image.MultiImageInteraction(probchip_list_, nPerPage=4, xlabel_list=xlabel_list)
        >>> iteract_obj.start()
        >>> ut.show_if_requested()
    """
    print('[core] COMPUTING FEATWEIGHTS')
    print('config = %r' % (config,))
    import vtool as vt

    ibs = depc.controller

    # Use the labeled species for the fw_detector
    species_list = ibs.get_annot_species_texts(aid_list)

    fw_detector = config['fw_detector']
    dim_size = config['fw_dim_size']
    smooth_thresh = config['smooth_thresh']
    smooth_ksize = config['smooth_ksize']

    if fw_detector == 'rf':
        pad = 64
    else:
        pad = 0

    probchip_dir = ibs.get_probchip_dir() + '2'
    cfghashid = config.get_hashid()

    # TODO: just hash everything together
    ut.ensuredir(probchip_dir)
    _fmt = 'probchip_avuuid_{avuuid}_' + cfghashid + '.png'
    annot_visual_uuid_list  = ibs.get_annot_visual_uuids(aid_list)
    probchip_fpath_list = [ut.unixjoin(probchip_dir, _fmt.format(avuuid=avuuid))
                           for avuuid in annot_visual_uuid_list]

    chip_config = ChipConfig(pad=pad, dim_size=dim_size)
    mchip_path_list = depc.get('chips', aid_list, 'img', config=chip_config, read_extern=False)

    aid_list = np.array(aid_list)
    species_list = np.array(species_list)
    species_rowid = np.array(ibs.get_species_rowids_from_text(species_list))

    # Group by species
    unique_species_rowids, groupxs = vt.group_indices(species_rowid)
    grouped_aids    = vt.apply_grouping(aid_list, groupxs)
    grouped_species = vt.apply_grouping(species_list, groupxs)
    grouped_mpaths = ut.apply_grouping(mchip_path_list, groupxs)
    grouped_ppaths = ut.apply_grouping(probchip_fpath_list, groupxs)
    unique_species = ut.get_list_column(grouped_species, 0)

    if ut.VERBOSE:
        print('[preproc_probchip] +--------------------')
    print(('[preproc_probchip.compute_and_write_probchip] '
          'Preparing to compute %d probchips of %d species')
          % (len(aid_list), len(unique_species)))
    print(config)

    #grouped_probchip_fpath_list = []
    grouped_probchips = []
    _iter = zip(grouped_aids, unique_species, grouped_ppaths, grouped_mpaths)
    _iter = ut.ProgIter(_iter, nTotal=len(grouped_aids),
                        lbl='probchip for species', enabled=ut.VERBOSE, backspace=True)

    if fw_detector == 'rf':
        for aids, species, probchip_fpaths, inputchip_fpaths in _iter:
            if len(aids) == 0:
                continue
            gen = rf_probchips(ibs, aids, species, probchip_fpaths, inputchip_fpaths, pad,
                               smooth_thresh, smooth_ksize)
            #grouped_probchip_fpath_list.append(probchip_fpaths)
            grouped_probchips.append(list(gen))
    elif fw_detector == 'cnn':
        for aids, species, probchip_fpaths, inputchip_fpaths in _iter:
            if len(aids) == 0:
                continue
            gen = cnn_probchips(ibs, species, probchip_fpath_list, inputchip_fpaths,
                                smooth_thresh, smooth_ksize)
            #grouped_probchip_fpath_list.append(probchip_fpaths)
            grouped_probchips.append(list(gen))
    else:
        raise NotImplementedError('unknown fw_detector=%r' % (fw_detector,))

    if ut.VERBOSE:
        print('[preproc_probchip] Done computing probability images')
        print('[preproc_probchip] L_______________________')

    #probchip_fpath_list = vt.invert_apply_grouping2(
    #    grouped_probchip_fpath_list, groupxs, dtype=object)
    #for fpath in probchip_fpath_list:
    #    yield (fpath,)
    probchip_result_list = vt.invert_apply_grouping2(
        grouped_probchips, groupxs, dtype=object)
    for probchip in probchip_result_list:
        yield (probchip,)

def cnn_probchips(ibs, species, probchip_fpath_list, inputchip_fpaths, smooth_thresh, smooth_ksize):
    # dont use extrmargin here (for now)
    mask_gen = ibs.generate_species_background_mask(inputchip_fpaths, species)
    _iter = zip(probchip_fpath_list, mask_gen)
    for chunk in ut.ichunks(_iter, 256):
        _progiter = ut.ProgIter(chunk, lbl='compute probchip chunk', adjust=True, time_thresh=30.0, backspace=True)
        for probchip_fpath, probchip in _progiter:
            if smooth_thresh is not None and smooth_ksize is not None:
                probchip = postprocess_mask(probchip, smooth_thresh, smooth_ksize)
            yield probchip
            #vt.imwrite(probchip_fpath, probchip)

def rf_probchips(ibs, aids, species, probchip_fpaths, inputchip_fpaths, pad,
                 smooth_thresh, smooth_ksize):
    from ibeis.algo.detect import randomforest
    extramargin_probchip_fpaths = [ut.augpath(path, '_margin')
                                   for path in probchip_fpaths]
    rfconfig = {'scale_list': [1.0], 'mode': 1,
                'output_gpath_list': extramargin_probchip_fpaths}
    probchip_generator = randomforest.detect_gpath_list_with_species(
        ibs, inputchip_fpaths, species, **rfconfig)
    # Evalutate genrator until completion
    ut.evaluate_generator(probchip_generator)
    extramargin_mask_gen = (vt.imread(fpath, grayscale=True)
                            for fpath in extramargin_probchip_fpaths)
    # Crop the extra margin off of the new probchips
    _iter = zip(probchip_fpaths, extramargin_mask_gen)
    for (probchip_fpath, extramargin_probchip) in _iter:
        half_w, half_h = (pad, pad)
        probchip = extramargin_probchip[half_h:-half_h, half_w:-half_w]
        if smooth_thresh is not None and smooth_ksize is not None:
            probchip = postprocess_mask(probchip, smooth_thresh, smooth_ksize)
        yield probchip
        #vt.imwrite(probchip_fpath, probchip)

def postprocess_mask(mask, thresh=20, kernel_size=20):
    r"""
    Args:
        mask (ndarray):

    Returns:
        ndarray: mask2

    CommandLine:
        python -m ibeis.core_annots --exec-postprocess_mask --cnn --show --aid=1 --db PZ_MTEST
        python -m ibeis --tf postprocess_mask --cnn --show --db PZ_MTEST --adapteq=True

    SeeAlso:
        python -m ibeis_cnn --tf generate_species_background_mask --show --db PZ_Master1 --aid 9970

    Ignore:
        input_tuple = aid_list
        tablename = 'probchip'
        config = full_config
        rowid_kw = dict(config=config)

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> import plottool as pt
        >>> ibs, depc, aid_list = testdata_core()
        >>> config = ChipConfig.from_argv_dict()
        >>> probchip_config = ProbchipConfig(smooth_thresh=None)
        >>> chip = ibs.depc_annot.get('chips', aid_list, 'img', config)[0]
        >>> mask = ibs.depc_annot.get('probchip', aid_list, 'img', probchip_config)[0]
        >>> mask2 = postprocess_mask(mask)
        >>> ut.quit_if_noshow()
        >>> fnum = 1
        >>> pt.imshow(chip, pnum=(1, 3, 1), fnum=fnum, xlabel=str(chip.shape))
        >>> pt.imshow(mask, pnum=(1, 3, 2), fnum=fnum, title='before', xlabel=str(mask.shape))
        >>> pt.imshow(mask2, pnum=(1, 3, 3), fnum=fnum, title='after', xlabel=str(mask2.shape))
        >>> ut.show_if_requested()
    """
    import cv2
    thresh = 20
    kernel_size = 20
    mask2 = mask.copy()
    # light threshold
    mask2[mask2 < thresh] = 0
    # open and close
    kernel = np.ones((kernel_size, kernel_size), np.uint8)
    mask2 = cv2.morphologyEx(mask2, cv2.MORPH_CLOSE, kernel)
    mask2 = cv2.morphologyEx(mask2, cv2.MORPH_OPEN, kernel)
    mask2 = cv2.morphologyEx(mask2, cv2.MORPH_CLOSE, kernel)
    return mask2

class HOGConfig(dtool.Config):
    _param_info_list = [
        ut.ParamInfo('orientations', 8),
        ut.ParamInfo('pixels_per_cell', (16, 16)),
        ut.ParamInfo('cells_per_block', (1, 1)),
    ]

def make_hog_block_image(hog, config=None):
    """
    References:
        https://github.com/scikit-image/scikit-image/blob/master/skimage/feature/_hog.py
    """

    from skimage import draw

    if config is None:
        config = HOGConfig()

    cx, cy = config['pixels_per_cell']

    normalised_blocks = hog
    (n_blocksy, n_blocksx, by, bx, orientations) = normalised_blocks.shape

    n_cellsx = (n_blocksx - 1) + bx
    n_cellsy = (n_blocksy - 1) + by

    # Undo the normalization step
    orientation_histogram = np.zeros((n_cellsy, n_cellsx, orientations))

    for x in range(n_blocksx):
        for y in range(n_blocksy):
            norm_block = normalised_blocks[y, x, :]
            # hack, this only works right for block sizes of 1
            orientation_histogram[y:y + by, x:x + bx, :] = norm_block

    sx = n_cellsx * cx
    sy = n_cellsy * cy

    radius = min(cx, cy) // 2 - 1
    orientations_arr = np.arange(orientations)
    dx_arr = radius * np.cos(orientations_arr / orientations * np.pi)
    dy_arr = radius * np.sin(orientations_arr / orientations * np.pi)
    hog_image = np.zeros((sy, sx), dtype=float)
    for x in range(n_cellsx):
        for y in range(n_cellsy):
            for o, dx, dy in zip(orientations_arr, dx_arr, dy_arr):
                centre = tuple([y * cy + cy // 2, x * cx + cx // 2])
                rr, cc = draw.line(int(centre[0] - dx),
                                   int(centre[1] + dy),
                                   int(centre[0] + dx),
                                   int(centre[1] - dy))
                hog_image[rr, cc] += orientation_histogram[y, x, o]
    return hog_image

@derived_attribute(
    tablename='hog', parents=['chips'],
    colnames=['hog'],
    coltypes=[np.ndarray],
    configclass=HOGConfig,
    fname='hogcache', chunksize=32,
)
def compute_hog(depc, cid_list, config=None):
    """
    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> ibs, depc, aid_list = testdata_core()
        >>> chip_config = {}
        >>> config = HOGConfig()
        >>> cid_list = depc.get_rowids('chips', aid_list, config=chip_config)
        >>> hoggen = compute_hog(depc, cid_list, config)
        >>> hog = list(hoggen)[0]
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> hog_image = make_hog_block_image(hog, config)
        >>> ut.show_if_requested()
    """
    import skimage.feature
    orientations     = config['orientations']

    ut.assert_all_not_None(cid_list, 'cid_list')
    chip_fpath_list = depc.get_native('chips', cid_list, 'img', read_extern=False)

    for chip_fpath in chip_fpath_list:
        chip = vt.imread(chip_fpath, grayscale=True) / 255.0
        hog = skimage.feature.hog(chip, feature_vector=False,
                                  orientations=orientations,
                                  pixels_per_cell=(16, 16), cells_per_block=(1, 1))
        yield (hog,)

class FeatConfig(dtool.Config):
    r"""
    Example:
        >>> from ibeis.core_annots import *  # NOQA
        >>> feat_cfg = FeatConfig()
        >>> result = str(feat_cfg)
        >>> print(result)
        <FeatConfig(hesaff+sift)>
    """
    # TODO: FIXME
    #_parents = [ChipConfig]

    def get_param_info_list(self):
        import pyhesaff
        default_keys = list(pyhesaff.get_hesaff_default_params().keys())
        default_items = list(pyhesaff.get_hesaff_default_params().items())
        param_info_list = [
            ut.ParamInfo('feat_type', 'hesaff+sift', ''),
            ut.ParamInfo('maskmethod', None, hideif=None)
        ]
        param_info_dict = {
            name: ut.ParamInfo(name, default, hideif=default)
            for name, default in default_items
        }
        #param_info_dict['scale_max'].default = -1
        #param_info_dict['scale_max'].default = 50
        param_info_list += ut.dict_take(param_info_dict, default_keys)
        return param_info_list

    def get_hesaff_params(self):
        # Get subset of these params that correspond to hesaff
        import pyhesaff
        default_keys = list(pyhesaff.get_hesaff_default_params().keys())
        hesaff_param_dict = ut.dict_subset(self, default_keys)
        return hesaff_param_dict

@derived_attribute(
    tablename='feat', parents=['chips'],
    colnames=['num_feats', 'kpts', 'vecs'],
    coltypes=[int, np.ndarray, np.ndarray],
    configclass=FeatConfig,
    fname='featcache', chunksize=1024,
)
def compute_feats(depc, cid_list, config=None):
    r"""
    Computes features and yields results asynchronously: TODO: Remove IBEIS from
    this equation. Move the firewall towards the controller

    Args:
        depc (dtool.DependencyCache):
        cid_list (list):
        config (None):

    Returns:
        generator : generates param tups

    SeeAlso:
        ~/code/ibeis_cnn/ibeis_cnn/_plugin.py

    CommandLine:
        python -m ibeis.core_annots --test-compute_feats:0 --show
        python -m ibeis.core_annots --test-compute_feats:1

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> ibs, depc, aid_list = testdata_core()
        >>> chip_config = {}
        >>> config = FeatConfig()
        >>> cid_list = depc.get_rowids('chips', aid_list, config=chip_config)
        >>> featgen = compute_feats(depc, cid_list, config)
        >>> feat_list = list(featgen)
        >>> assert len(feat_list) == len(aid_list)
        >>> (nFeat, kpts, vecs) = feat_list[0]
        >>> assert nFeat == len(kpts) and nFeat == len(vecs)
        >>> assert kpts.shape[1] == 6
        >>> assert vecs.shape[1] == 128
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> chip = depc.get_native('chips', cid_list[0:1], 'img')[0]
        >>> pt.interact_keypoints.KeypointInteraction(chip, kpts, vecs, autostart=True)
        >>> ut.show_if_requested()

    Example:
        >>> # TIMING
        >>> from ibeis.core_annots import *  # NOQA
        >>> ibs, depc, aid_list = testdata_core('PZ_MTEST', 100)
        >>> config = {'dim_size': 450}
        >>> cid_list = depc.get_rowids('chips', aid_list, config=config)
        >>> config = FeatConfig()
        >>> featgen = compute_feats(depc, cid_list, config)
        >>> feat_list = list(featgen)
        >>> idx = 5
        >>> (nFeat, kpts, vecs) = feat_list[idx]
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> chip = depc.get_native('chips', cid_list[idx:idx + 1], 'img')[0]
        >>> pt.interact_keypoints.KeypointInteraction(chip, kpts, vecs, autostart=True)
        >>> ut.show_if_requested()

        >>> #num_feats = depc.get('feat', aid_list, 'num_feats', config=config, recompute=True)

        ibs.delete_annot_feats(aid_list)
        ibs.get_annot_feat_rowids(aid_list)
    """
    nInput = len(cid_list)
    hesaff_params  = config.get_hesaff_params()
    feat_type      = config['feat_type']
    maskmethod     = config['maskmethod']

    ut.assert_all_not_None(cid_list, 'cid_list')
    chip_fpath_list = depc.get_native('chips', cid_list, 'img', read_extern=False)

    if maskmethod is not None:
        assert False
        #aid_list = ibs.get_chip_aids(cid_list)
        #probchip_fpath_list = ibs.get_annot_probchip_fpath(aid_list)
    else:
        probchip_fpath_list = (None for _ in range(nInput))

    if ut.NOT_QUIET:
        print('[preproc_feat] config = %s' % config)
        if ut.VERYVERBOSE:
            print('full_params = ' + ut.dict_str())

    ibs = depc.controller
    if feat_type == 'hesaff+sift':
        # Multiprocessing parallelization
        dictargs_iter = (hesaff_params for _ in range(nInput))
        arg_iter = zip(chip_fpath_list, probchip_fpath_list, dictargs_iter)
        # eager evaluation.
        # TODO: Check if there is any benefit to just passing in the iterator.
        arg_list = list(arg_iter)
        # TODO: futures_generate
        featgen = ut.generate(gen_feat_worker, arg_list, nTasks=nInput,
                              freq=10, ordered=True,
                              force_serial=ibs.force_serial)
    elif feat_type == 'hesaff+siam128':
        from ibeis_cnn import _plugin
        assert maskmethod is None, 'not implemented'
        assert False, 'not implemented'
        featgen = _plugin.generate_siam_l2_128_feats(ibs, cid_list, config=config)
    else:
        raise AssertionError('unknown feat_type=%r' % (feat_type,))

    for nFeat, kpts, vecs in featgen:
        yield (nFeat, kpts, vecs,)

def gen_feat_worker(tup):
    r"""
    Function to be parallelized by multiprocessing / joblib / whatever.
    Must take in one argument to be used by multiprocessing.map_async

    Args:
        tup (tuple):

    Returns:
        tuple: (None, kpts, vecs)

    CommandLine:
        python -m ibeis.core_annots --exec-gen_feat_worker --show
        python -m ibeis.core_annots --exec-gen_feat_worker --show --aid 1988 --db GZ_Master1 --affine-invariance=False --scale_max=30
        python -m ibeis.core_annots --exec-gen_feat_worker --show --aid 1988 --db GZ_Master1 --affine-invariance=False --maskmethod=None  --scale_max=30

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> ibs, depc, aid_list = testdata_core()
        >>> aid = aid_list[0]
        >>> config = {}
        >>> feat_config = FeatConfig.from_argv_dict()
        >>> chip_fpath = ibs.depc_annot.get('chips', aid_list[0], 'img', config=config, read_extern=False)
        >>> maskmethod = ut.get_argval('--maskmethod', type_=str, default='cnn')
        >>> probchip_fpath = ibs.depc_annot.get('probchip', aid_list[0], 'img', config=config, read_extern=False) if feat_config['maskmethod'] == 'cnn' else None
        >>> hesaff_params = feat_config.asdict()
        >>> # Exec function source
        >>> tup = (chip_fpath, probchip_fpath, hesaff_params)
        >>> masked_chip, num_kpts, kpts, vecs = ut.exec_func_src(
        >>>     gen_feat_worker, key_list=['masked_chip', 'num_kpts', 'kpts', 'vecs'],
        >>>     sentinal='num_kpts = kpts.shape[0]')
        >>> result = ('(num_kpts, kpts, vecs) = %s' % (ut.repr2((num_kpts, kpts, vecs)),))
        >>> print(result)
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> from plottool.interactions import ExpandableInteraction
        >>> interact = ExpandableInteraction()
        >>> interact.append_plot(pt.interact_keypoints.KeypointInteraction(masked_chip, kpts, vecs))
        >>> interact.append_plot(lambda **kwargs: pt.plot_score_histograms([vt.get_scales(kpts)], **kwargs))
        >>> interact.start()
        >>> ut.show_if_requested()
    """
    import pyhesaff
    #import numpy as np
    #import vtool as vt
    chip_fpath, probchip_fpath, hesaff_params = tup
    chip = vt.imread(chip_fpath)
    if probchip_fpath is not None:
        probchip = vt.imread(probchip_fpath, grayscale=True)
        probchip = vt.resize_mask(probchip, chip)
        #vt.blend_images_multiply(chip, probchip)
        masked_chip = (chip * (probchip[:, :, None].astype(np.float32) / 255)).astype(np.uint8)
    else:
        masked_chip = chip
    kpts, vecs = pyhesaff.detect_feats_in_image(masked_chip, **hesaff_params)
    num_kpts = kpts.shape[0]
    return (num_kpts, kpts, vecs)

class FeatWeightConfig(dtool.Config):
    _param_info_list = [
        ut.ParamInfo('featweight_enabled', True, 'enabled='),
    ]
    # FIXME: incorporate config dependencies in dtool
    #_parents = [FeatConfig, ProbchipConfig]

@derived_attribute(
    tablename='featweight', parents=['feat', 'probchip'],
    colnames=['fwg'],
    coltypes=[np.ndarray],
    configclass=FeatWeightConfig,
    fname='featcache', chunksize=512,
)
def compute_fgweights(depc, fid_list, pcid_list, config=None):
    """
    Args:
        depc (dtool.DependencyCache): depc
        fid_list (list):
        config (None): (default = None)

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> ibs, depc, aid_list = testdata_core()
        >>> full_config = {}
        >>> config = FeatConfig()
        >>> fid_list = depc.get_rowids('feat', aid_list, config=full_config)
        >>> pcid_list = depc.get_rowids('probchip', aid_list, config=full_config)
        >>> prop_list = list(compute_fgweights(depc, fid_list, pcid_list))
        >>> featweight_list = ut.take_column(prop_list, 0)
        >>> result = np.array_str(featweight_list[0][0:3], precision=3)
        >>> print(result)
    """
    ibs = depc.controller
    nTasks = len(fid_list)
    print('[compute_fgweights] Computing %d fgweights' % (nTasks,))
    #aid_list = depc.get_ancestor_rowids('feat', fid_list, 'annotations')
    #probchip_fpath_list = depc.get(aid_list, 'img', config={}, read_extern=False)
    probchip_list = depc.get_native('probchip', pcid_list, 'img')
    cid_list = depc.get_ancestor_rowids('feat', fid_list, 'chips')
    chipsize_list = depc.get_native('chips', cid_list, ('width', 'height'))
    kpts_list = depc.get_native('feat', fid_list, 'kpts')
    # Force grayscale reading of chips
    arg_iter = zip(kpts_list, probchip_list, chipsize_list)
    # ibs = depc.controller
    # featweight_gen = ut.generate(gen_featweight_worker, arg_iter,
    #                               nTasks=nTasks, ordered=True, freq=10,
    #                               force_serial=ibs.force_serial)
    featweight_gen = ut.generate(gen_featweight_worker, arg_iter,
                                 nTasks=nTasks, ordered=True, freq=10,
                                 force_serial=ibs.force_serial
                                 )
    featweight_list = list(featweight_gen)
    print('[compute_fgweights] Done computing %d fgweights' % (nTasks,))
    for fw in featweight_list:
        yield (fw,)

def gen_featweight_worker(tup):
    """
    Function to be parallelized by multiprocessing / joblib / whatever.
    Must take in one argument to be used by multiprocessing.map_async

    Args:
        tup (aid, tuple(kpts(ndarray), probchip_fpath )): keypoints and
            probability chip file path aid, kpts, probchip_fpath

    CommandLine:
        python -m ibeis.core_annots --test-gen_featweight_worker --show
        python -m ibeis.core_annots --test-gen_featweight_worker --show --dpath figures --save ~/latex/crall-candidacy-2015/figures/gen_featweight.jpg
        python -m ibeis.core_annots --test-gen_featweight_worker --show --db PZ_MTEST --qaid_list=1,2,3,4,5,6,7,8,9

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> #test_featweight_worker()
        >>> ibs, depc, aid_list = testdata_core()
        >>> aid_list = aid_list[0:1]
        >>> config = {'dim_size': 450, 'resize_dim': 'area', 'smooth_thresh': 0, 'smooth_ksize': 0}
        >>> probchip = depc.get('probchip', aid_list, 'img', config=config)[0]
        >>> chipsize = depc.get('chips', aid_list, ('width', 'height'), config=config)[0]
        >>> kpts = depc.get('feat', aid_list, 'kpts', config=config)[0]
        >>> tup = (kpts, probchip, chipsize)
        >>> weights = gen_featweight_worker(tup)
        >>> assert np.all(weights <= 1.0), 'weights cannot be greater than 1'
        >>> chip = depc.get('chips', aid_list, 'img', config=config)[0]
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> fnum = 1
        >>> pnum_ = pt.make_pnum_nextgen(1, 3)
        >>> pt.figure(fnum=fnum, doclf=True)
        >>> pt.imshow(chip, pnum=pnum_(0), fnum=fnum)
        >>> pt.imshow(probchip, pnum=pnum_(2), fnum=fnum)
        >>> pt.imshow(chip, pnum=pnum_(1), fnum=fnum)
        >>> color_list = pt.draw_kpts2(kpts, weights=weights, ell_alpha=.3)
        >>> cb = pt.colorbar(weights, color_list)
        >>> cb.set_label('featweights')
        >>> pt.show_if_requested()
    """
    (kpts, probchip, chipsize) = tup
    if probchip is None:
        # hack for undetected chips. SETS ALL FEATWEIGHTS TO .25 = 1/4
        assert False, 'should not be in this state'
        weights = np.full(len(kpts), .25, dtype=np.float32)
    else:
        sfx, sfy = (probchip.shape[1] / chipsize[0], probchip.shape[0] / chipsize[1])
        kpts_ = vt.offset_kpts(kpts, (0, 0), (sfx, sfy))
        #vtpatch.get_warped_patches()
        if False:
            # VERY SLOW
            patch_list1  = [vt.get_warped_patch(probchip, kp)[0].astype(np.float32) / 255.0 for kp in kpts_]
            weight_list = [vt.gaussian_average_patch(patch1) for patch1 in patch_list1]
            #weight_list = [patch.sum() / (patch.size) for patch in patch_list]
        else:
            # New way
            weight_list = vt.patch_gaussian_weighted_average_intensities(probchip, kpts_)
        weights = np.array(weight_list, dtype=np.float32)
    return weights

class VsOneRequest(dtool.base.VsOneSimilarityRequest):
    _tablename = 'vsone'

    def postprocess_execute(request, parent_rowids, result_list):
        import ibeis
        depc = request.depc
        ibs = depc.controller
        qaid_list, daid_list = list(zip(*parent_rowids))
        unique_qaids, groupxs = ut.group_indices(qaid_list)
        grouped_daids = ut.apply_grouping(daid_list, groupxs)

        unique_qnids = ibs.get_annot_nids(unique_qaids)
        single_cm_list = ut.take_column(result_list, 1)
        grouped_cms = ut.apply_grouping(single_cm_list, groupxs)

        _iter = zip(unique_qaids, unique_qnids, grouped_daids, grouped_cms)
        cm_list = []
        for qaid, qnid, daids, cms in _iter:
            # Hacked in version of creating an annot match object
            chip_match = ibeis.ChipMatch.combine_cms(cms)
            chip_match.score_maxcsum(request)
            cm_list.append(chip_match)

        #import utool
        #utool.embed()
        #cm = cm_list[0]
        #cm.print_inspect_str(request)
        #cm.assert_self(request, assert_feats=False)
        return cm_list

class VsOneConfig(dtool.Config):
    """
    Example:
        >>> from ibeis.core_annots import *  # NOQA
        >>> cfg = VsOneConfig()
        >>> result = str(cfg)
        >>> print(result)
    """
    _param_info_list = [
        #ut.ParamInfo('sver_xy_thresh', .01),
        ut.ParamInfo('sver_xy_thresh', .001),
        ut.ParamInfo('ratio_thresh', .625),
        ut.ParamInfo('refine_method', 'homog'),
        ut.ParamInfo('symmetric', False),
        ut.ParamInfo('K', 1),
        ut.ParamInfo('Knorm', 1),
        ut.ParamInfo('version', 0),
        ut.ParamInfo('augment_queryside_hack', False),
    ]
    _sub_config_list = [
        FeatConfig,
        ChipConfig,  # TODO: infer chip config from feat config
        FeatWeightConfig,
    ]

def cut_test(ibs, parent_rowids_T, score_list2):

    unique_aids = ut.unique(ut.flatten(parent_rowids_T))
    #for view in set(ibs.get_annot_yaw_texts(unique_aids)):
    #    aid2_idx = ut.make_index_lookup(unique_aids)
    #    #idx2_aid = ut.invert_dict(aid2_idx)
    #    idx_pairs = np.array(ut.unflat_take(aid2_idx, zip(*parent_rowids_T)))
    #    num = len(aid2_idx)
    #    flat_idx = np.ravel_multi_index(idx_pairs.T, (num, num))
    #    score_list2 = np.array(score_list2)
    #    cost_matrix = np.zeros(num * num)
    #    cost_matrix[flat_idx] = score_list2
    #    cost_matrix = cost_matrix.reshape((num, num))
    #    thresh = np.median(cost_matrix)
    #    thresh = 20
    #    labels = vt.unsupervised_multicut_labeling(cost_matrix, thresh)
    #    grouping = ut.group_items(unique_aids, labels)

    if True:
        #vp2_name2_aids = ibs.group_annots_by_multi_prop(unique_aids, [ibs.get_annot_yaw_texts, ibs.get_annot_name_texts])
        aid2_idx = ut.make_index_lookup(unique_aids)
        num = len(aid2_idx)
        idx_pairs = np.array(ut.unflat_take(aid2_idx, zip(*parent_rowids_T)))
        flat_idx = np.ravel_multi_index(idx_pairs.T, (num, num))
        score_list2 = np.array(score_list2)
        cost_matrix = np.zeros(num * num)
        cost_matrix[flat_idx] = score_list2
        cost_matrix = cost_matrix.reshape((num, num))

        vp2_aids = ibs.group_annots_by_multi_prop(unique_aids, [ibs.get_annot_yaw_texts])

        for view, aids in vp2_aids.items():
            print('---')
            print('view = %r' % (view,))
            print('len(aids) = %r' % (len(aids),))
            idxs = ut.take(aid2_idx, aids)
            if len(idxs) == 1:
                continue
            real_group = ibs.group_annots_by_name(aids)[0]
            sub_cost_matrix = cost_matrix[idxs].T[idxs].T
            #ibs = ut.search_stack_for_localvar('ibs')
            for thresh in [5, 7, 10, 15, 25, 50]:
                labels = vt.unsupervised_multicut_labeling(sub_cost_matrix, thresh)
                grouping = ut.group_items(aids, labels)
                diff = ut.compare_groupings(real_group, grouping.values())
                print('thresh = %r, diff=%r' % (thresh, diff))
                #print('--')

        if False:
            # synthetic data
            size = 100
            thresh = 50
            np.random.randint(0, 1)
            np.zeros((size, size))
            #np.random.rand(size, size)
            size = 40
            for size in range(2, 100):
                aids = np.arange(size)
                encounter_lbls = np.random.randint(0, size, size)
                grid1 = np.tile(encounter_lbls, (size, 1))
                is_match = grid1.T == grid1
                good_pos = np.where(is_match)
                bad_pos = np.where(~is_match)
                sub_cost_matrix = np.empty((size, size))
                sub_cost_matrix[good_pos] = np.random.randn(len(good_pos[0])) + 20
                sub_cost_matrix[bad_pos] = np.random.randn(len(bad_pos[0])) - 20
                sub_cost_matrix[np.diag_indices_from(sub_cost_matrix)] = np.inf
                labels = vt.unsupervised_multicut_labeling(sub_cost_matrix, 0)
                diff = ut.compare_groupings(
                    list(ut.group_items(aids, encounter_lbls).values()),
                    list(ut.group_items(aids, labels).values()))
                print('diff = %r' % (diff,))

@derived_attribute(
    tablename='vsone', parents=['annotations', 'annotations'],
    colnames=['score', 'match'], coltypes=[float, ChipMatch],
    requestclass=VsOneRequest,
    configclass=VsOneConfig,
    chunksize=128,
    #chunksize=16,
    fname='vsone',
)
def compute_one_vs_one(depc, qaids, daids, config):
    r"""
    CommandLine:
        python -m ibeis.core_annots --test-compute_one_vs_one --show
        python -m ibeis.control.IBEISControl --test-show_depc_annot_graph --show
        python -m ibeis.control.IBEISControl --test-show_depc_annot_table_input --show --tablename=vsone

    Ignore:
        >>> from ibeis.core_annots import *  # NOQA
        >>> import ibeis
        >>> ibs, aid_list = ibeis.testdata_aids('PZ_Master1', 'default:')
        >>> occurid2_aids = ibs.temp_group_annot_occurrences(aid_list)
        >>> aids_list = [np.unique(aids) for aids in occurid2_aids.values()]
        >>> aids_list = [aids for aids in aids_list if len(aids) > 1 and len(aids) < 100]

        aids = ut.sortedby([a.tolist() for a in aids_list], ut.lmap(len, aids_list))[-1]

        depc = ibs.depc_annot

        progiter = ut.ProgIter(aids_list, freq=1)
        for aids in progiter:
            request = depc.new_request('vsone', aids, aids, {'dim_size': 450})
            qaids, daids = request.parent_rowids_T
            config = request.config
            parent_rowids_T = request.parent_rowids_T
            rawres_list2 = request.execute(postprocess=False)
            #score_list2 = ut.take_column(rawres_list2, 0)
            ut.list_T = ut.list_transpose
            #test_cut(ibs, parent_rowids_T, score_list2)
            # x = 44
            #test_cut(ibs, ut.list_T(ut.list_T(parent_rowids_T)[0:x]), score_list2[0:x])

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> #ibs, depc, aid_list = testdata_core(size=5)
        >>> import ibeis
        >>> #ibs, aid_list = ibeis.testdata_aids('wd_peter2', 'timectrl:pername=2,view=left,view_ext=0,exclude_reference=True')
        >>> ibs, aid_list = ibeis.testdata_aids('testdb2', 'default:')
        >>> _, aids = ut.items_sorted_by_value(ut.group_items(aid_list, ibs.get_annot_occurrence_text(aid_list)), key=len)[-1]
        >>> aid_list = aids[0:4]
        >>> depc = ibs.depc_annot
        >>> request = depc.new_request('vsone', aid_list, aid_list, {'resize_dim': 'width', 'dim_size': 450})
        >>> config = request.config
        >>> parent_rowids_T = request.parent_rowids_T
        >>> qaids, daids = request.parent_rowids_T
        >>> # Compute using request
        >>> print('...Test vsone cache')
        >>> rawres_list2 = request.execute(postprocess=False)
        >>> score_list2 = ut.take_column(rawres_list2, 0)
        >>> res_list2 = request.execute()
        >>> print(res_list2)
        >>> # Compute using function
        >>> #print('...Test vsone function')
        >>> #rawres_list1 = list(compute_one_vs_one(depc, qaids, daids, config))
        >>> #score_list1 = ut.take_column(rawres_list1, 0)
        >>> #print(score_list1)
        >>> #assert np.all(score_list1 == score_list2)
        >>> ut.quit_if_noshow()
        >>> ut.ensure_pylab_qt4()
        >>> match = res_list2[0]
        >>> match.print_inspect_str(request)
        >>> #match.show_analysis(qreq_=request)
        >>> #match.ishow_analysis(qreq_=request)
        >>> #match.ishow_single_annotmatch(qreq_=request)
        >>> match.show_single_annotmatch(qreq_=request, vert=False)
        >>> ut.show_if_requested()

    Example:
        >>> # Example of a one-vs-one query
        >>> import ibeis
        >>> ibs = ibeis.opendb('testdb1')
        >>> config = {'codename': 'vsone'}
        >>> qreq_ = ibs.new_query_request([1], [2], cfgdict=config)
        >>> cm_list = qreq_.execute()
        >>> match = cm_list[0]
        >>> match.print_inspect_str(qreq_)
        >>> match.show_single_annotmatch(qreq_=qreq_, vert=False)
        >>> import utool as ut
        >>> ut.show_if_requested()

    Example:
        >>> # Example of a one-vs-many query
        >>> import ibeis
        >>> ibs = ibeis.opendb('testdb1')
        >>> config = {'codename': 'vsmany'}
        >>> qreq_ = ibs.new_query_request([1], ibs.get_valid_aids(), cfgdict=config)
        >>> cm_list = qreq_.execute()
        >>> match = cm_list[0]
        >>> match.print_inspect_str(qreq_)
        >>> match.show_single_annotmatch(qreq_=qreq_, vert=False)
        >>> import utool as ut
        >>> ut.show_if_requested()
    """
    import ibeis
    ibs = depc.controller
    qconfig2_ = config
    dconfig2_ = config
    unique_qaids = np.unique(qaids)
    unique_daids = np.unique(daids)

    # TODO: Ensure entire pipeline can use new dependencies
    # DEPC Precompute
    ibs.depc.d.get_feat_rowids(unique_qaids, config=qconfig2_)
    ibs.depc.d.get_feat_rowids(unique_daids, config=dconfig2_)
    if True:
        annot1_list = [ibs.get_annot_lazy_dict2(qaid, config=qconfig2_)
                       for qaid in unique_qaids]
        annot2_list = [ibs.get_annot_lazy_dict2(daid, config=dconfig2_)
                       for daid in unique_daids]
    else:
        #config.chip_cfgstr = config.chip_cfg.get_cfgstr()
        #config.chip_cfg_dict = config.chip_cfg.asdict()
        annot1_list = [ibs.get_annot_lazy_dict(qaid, config2_=qconfig2_)
                       for qaid in unique_qaids]
        annot2_list = [ibs.get_annot_lazy_dict(daid, config2_=dconfig2_)
                       for daid in unique_daids]

    # precache flann structures
    # TODO: Make depcache node
    flann_params = {'algorithm': 'kdtree', 'trees': 8}
    for annot1 in annot1_list:
        if 'flann' not in annot1:
            annot1['flann'] = lambda: vt.flann_cache(
                annot1['vecs'], flann_params=flann_params, quiet=True,
                verbose=False)

    qaid_to_annot = dict(zip(unique_qaids, annot1_list))
    daid_to_annot = dict(zip(unique_daids, annot2_list))

    #all_aids = np.unique(ut.flatten([qaids, daids]))
    verbose = False
    #yeild_ = []
    #print("START VSONE")
    for qaid, daid  in ut.ProgIter(zip(qaids, daids), nTotal=len(qaids),
                                   lbl='compute vsone', backspace=True, freq=1):
        annot1 = qaid_to_annot[qaid]
        annot2 = daid_to_annot[daid]
        metadata = {
            'annot1': annot1,
            'annot2': annot2,
        }
        vt_match = vt.vsone_matching(metadata, cfgdict=config, verbose=verbose)
        matchtup = vt_match.matches['TOP+SV']
        H = vt_match.metadata['H_TOP']
        score = matchtup.fs.sum()
        fm = matchtup.fm
        fs = matchtup.fs

        match = ibeis.ChipMatch(
            qaid=qaid,
            daid_list=[daid],
            fm_list=[fm],
            fsv_list=[vt.atleast_nd(fs, 2)],
            H_list=[H],
            fsv_col_lbls=['L2_SIFT'])
        match._update_daid_index()
        match.evaluate_dnids(ibs=ibs)
        match._update_daid_index()
        match.set_cannonical_name_score([score], [score])

        #import utool
        #utool.embed()
        if False:
            ut.ensure_pylab_qt4()
            ibs, depc, aid_list = testdata_core(size=3)
            request = depc.new_request('vsone', aid_list, aid_list, {'dim_size': 450})
            match.ishow_analysis(request)
        #match = SingleMatch_IBEIS(qaid, daid, score, fm)
        #yeild_.append((score, match))
        yield (score, match)

class IndexerConfig(dtool.Config):
    """
    Example:
        >>> from ibeis.core_annots import *  # NOQA
        >>> cfg = VsOneConfig()
        >>> result = str(cfg)
        >>> print(result)
    """
    _param_info_list = [
        ut.ParamInfo('algorithm', 'kdtree', 'alg'),
        ut.ParamInfo('random_seed', 42, 'seed'),
        ut.ParamInfo('trees', 4, hideif=lambda cfg: cfg['algorithm'] != 'kdtree'),
        ut.ParamInfo('version', 1),
    ]
    _sub_config_list = [
        #FeatConfig,
        #ChipConfig,  # TODO: infer chip config from feat config
        #FeatWeightConfig,
    ]

    def get_flann_params(cfg):
        default_params = vt.get_flann_params(cfg['algorithm'])
        flann_params = ut.update_existing(default_params, cfg.asdict())
        return flann_params

testmode = ut.get_argflag('--testmode')

#if 1 or testmode:
@derived_attribute(
    #tablename='neighbor_index', parents=['annotations*'],
    #tablename='neighbor_index', parents=['annotations'],
    #tablename='neighbor_index', parents=['feat*'],
    tablename='neighbor_index', parents=['featweight*'],
    # tablename='neighbor_index', parents=['feat*'],
    #tablename='neighbor_index', parents=['feat'],
    colnames=['indexer'], coltypes=[neighbor_index.NeighborIndex2],
    configclass=IndexerConfig,
    chunksize=1, fname='indexer',
)
def compute_neighbor_index(depc, fids_list, config):
    r"""
    Args:
        depc (dtool.DependencyCache):
        fids_list (list):
        config (dtool.Config):

    CommandLine:
        python -m ibeis.core_annots --exec-compute_neighbor_index --show
        python -m ibeis.control.IBEISControl --test-show_depc_annot_table_input --show --tablename=neighbor_index

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.core_annots import *  # NOQA
        >>> import ibeis
        >>> ibs, aid_list = ibeis.testdata_aids('testdb1')
        >>> depc = ibs.depc_annot
        >>> fid_list = depc.get_rowids('feat', aid_list)
        >>> aids_list = tuple([aid_list])
        >>> fids_list = tuple([fid_list])
        >>> # Compute directly from function
        >>> config = ibs.depc_annot['neighbor_index'].configclass()
        >>> result1 = list(compute_neighbor_index(depc, fids_list, config))
        >>> nnindexer1 = result1[0][0]
        >>> # Compute using depcache
        >>> result2 = ibs.depc_annot.get('neighbor_index', [aids_list], 'indexer', config, recompute=False, _debug=True)
        >>> #result3 = ibs.depc_annot.get('neighbor_index', [tuple(fids_list)], 'indexer', config, recompute=False)
        >>> print(result2)
        >>> print(result3)
        >>> assert result2[0] is not result3[0]
        >>> assert nnindexer1.knn(ibs.get_annot_vecs(1), 1) is not None
        >>> assert result3[0].knn(ibs.get_annot_vecs(1), 1) is not None
    """
    print('[IBEIS] COMPUTE_NEIGHBOR_INDEX:')
    # TODO: allow augment
    assert len(fids_list) == 1, 'only working with one indexer at a time'
    fid_list = fids_list[0]
    aid_list = depc.get_root_rowids('feat', fid_list)
    flann_params = config.get_flann_params()
    cfgstr = config.get_cfgstr()
    verbose = True
    nnindexer = neighbor_index.NeighborIndex2(flann_params, cfgstr)
    # Initialize neighbor with unindexed data
    support = nnindexer.get_support(depc, aid_list, config)
    nnindexer.init_support(aid_list, *support, verbose=verbose)
    nnindexer.config = config
    nnindexer.reindex()
    yield (nnindexer,)

#class FeatNeighborConfig(dtool.Config)

if testmode:
    # NOT YET READY
    @derived_attribute(
        tablename='feat_neighbs', parents=['featweight', 'neighbor_index'],
        colnames=['qfx2_idx', 'qfx2_dist'], coltypes=[np.ndarray, np.ndarray],
        #configclass=IndexerConfig,
        chunksize=1, fname='neighbors',
    )
    def compute_feature_neighbors(depc, fid_list, indexer_rowid_list, config):
        """
        Args:
            depc (dtool.DependencyCache):
            aids_list (list):
            config (dtool.Config):

        CommandLine:
            python -m ibeis.core_annots --exec-compute_feature_neighbors --show
            python -m ibeis.control.IBEISControl --test-show_depc_annot_table_input --show --tablename=feat_neighbs

        Example:
            >>> # DISABLE_DOCTEST
            >>> from ibeis.core_annots import *  # NOQA
            >>> #ibs, depc, aid_list = testdata_core(size=5)
            >>> import ibeis
            >>> ibs, qaid_list = ibeis.testdata_aids('seaturtles')
            >>> daid_list = qaid_list
            >>> depc = ibs.depc_annot
            >>> index_config = ibs.depc_annot['neighbor_index'].configclass()
            >>> fid_list = depc.get_rowids('feat', qaid_list)
            >>> indexer_rowid_list = ibs.depc_annot.get_rowids('neighbor_index', [daid_list], index_config)
            >>> config = ibs.depc_annot['feat_neighbs'].configclass()
            >>> compute_feature_neighbors(depc, fid_list, indexer_rowid_list, config)
        """
        print('[IBEIS] NEAREST NEIGHBORS')
        #assert False
        # do computation
        #num_neighbors = (config['K'] + config['Knorm'])
        ibs = depc.controller
        num_neighbors = 1

        #b = np.broadcast([1, 2, 3], [1])
        #list(b)
        #[(1, 1), (2, 1), (3, 1)]

        # FIXME: not sure how depc should handle this case
        # Maybe it groups by indexer_rowid_list and then goes from there.
        indexer = depc.get_native('neighbor_index', indexer_rowid_list, 'indexer')[0]
        qvecs_list = depc.get_native('feat', fid_list, 'vecs', eager=False, nInput=len(fid_list))
        #qvecs_list = depc.get('feat', qaid_list, 'vecs', config, eager=False, nInput=len(qaid_list))
        qaid_list = depc.get_ancestor_rowids('feat', fid_list)

        ax2_encid = np.array(ibs.get_annot_encounter_text(indexer.ax2_aid))

        for qaid, qfx2_vec in zip(qaid_list, qvecs_list):
            qencid = ibs.get_annot_encounter_text([qaid])[0]
            invalid_axs = np.where(ax2_encid == qencid)[0]
            #indexer.ax2_aid[invalid_axs]
            nnindxer = indexer
            qfx2_idx, qfx2_dist, iter_count = nnindxer.conditional_knn(qfx2_vec,
                                                                       num_neighbors,
                                                                       invalid_axs)
            yield qfx2_idx, qfx2_dist

    # NOT YET READY
    @derived_attribute(
        tablename='sver', parents=['feat_neighbs'],
        colnames=['chipmatch'], coltypes=[ChipMatch],
        #configclass=IndexerConfig,
        chunksize=1, fname='vsmany',
    )
    def compute_sver(depc, fid_list, config):
        pass

    @derived_attribute(
        tablename='vsmany', parents=['sver'],
        colnames=['chipmatch'], coltypes=[ChipMatch],
        #configclass=IndexerConfig,
        chunksize=1, fname='vsmany',
    )
    def compute_vsmany(depc, fid_list, config):
        pass

class LabelerConfig(dtool.Config):
    _param_info_list = [
        ut.ParamInfo('labeler_sensitivity', 0.2),
    ]
    _sub_config_list = [
        ChipConfig
    ]

@derived_attribute(
    tablename='labeler', parents=['annotations'],
    colnames=['score', 'species', 'viewpoint', 'quality', 'orientation', 'probs'],
    coltypes=[float, str, str, str, float, dict],
    configclass=LabelerConfig,
    fname='chipcache4',
    chunksize=128,
)
def compute_labels_annotations(depc, aid_list, config=None):
    r"""
    Extracts the detections for a given input image

    Args:
        depc (ibeis.depends_cache.DependencyCache):
        gid_list (list):  list of image rowids
        config (dict): (default = None)

    Yields:
        (float, str): tup

    CommandLine:
        ibeis compute_labels_annotations

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.core_images import *  # NOQA
        >>> import ibeis
        >>> defaultdb = 'PZ_MTEST'
        >>> ibs = ibeis.opendb(defaultdb=defaultdb)
        >>> depc = ibs.depc_annot
        >>> aid_list = ibs.get_valid_aids()[0:8]
        >>> # depc.delete_property('labeler', aid_list)
        >>> results = depc.get_property('labeler', aid_list, None)
        >>> print(results)
    """
    from ibeis.algo.detect.labeler.labeler import label_chip_list
    print('[ibs] Process Annotation Labels')
    print('config = %r' % (config,))
    # Get controller
    ibs = depc.controller
    depc = ibs.depc_annot
    config = {
        'dim_size' : (128, 128),
        'resize_dim' : 'wh',
    }
    chip_list = depc.get_property('chips', aid_list, 'img', config=config)
    result_list = label_chip_list(chip_list)
    # yield detections
    for result in result_list:
        yield result

if __name__ == '__main__':
    r"""
    CommandLine:
        python -m ibeis.core_annots
        python -m ibeis.core_annots --allexamples
        utprof.py -m ibeis.core_annots --allexamples
    """
    import multiprocessing
    multiprocessing.freeze_support()  # for win32
    import utool as ut  # NOQA
    ut.doctest_funcs()

max_stars_repo_name: holmbergius/ibeisold

max_issues_repo_name: holmbergius/ibeisold

max_forks_repo_name: holmbergius/ibeisold