text:
import numpy as np
import torch
from torch import nn
from torch.utils.data import ConcatDataset
import tqdm
import copy
import utils
from models.cl.continual_learner import ContinualLearner

def train(model, train_loader, iters, loss_cbs=list(), eval_cbs=list(), save_every=None, m_dir="./store/models",
          args=None):
    '''Train a model with a "train_a_batch" method for [iters] iterations on data from [train_loader].

    [model]             model to optimize
    [train_loader]      <dataloader> for training [model] on
    [iters]             <int> (max) number of iterations (i.e., batches) to train for
    [loss_cbs]          <list> of callback-<functions> to keep track of training progress
    [eval_cbs]          <list> of callback-<functions> to evaluate model on separate data-set'''

    device = model._device()

    # Should convolutional layers be frozen?
    freeze_convE = (utils.checkattr(args, "freeze_convE") and hasattr(args, "depth") and args.depth>0)

    # Create progress-bar (with manual control)
    bar = tqdm.tqdm(total=iters)

    iteration = epoch = 0
    while iteration < iters:
        epoch += 1

        # Loop over all batches of an epoch
        for batch_idx, (data, y) in enumerate(train_loader):
            iteration += 1

            # Perform training-step on this batch
            data, y = data.to(device), y.to(device)
            loss_dict = model.train_a_batch(data, y=y, freeze_convE=freeze_convE)

            # Fire training-callbacks (for visualization of training-progress)
            for loss_cb in loss_cbs:
                if loss_cb is not None:
                    loss_cb(bar, iteration, loss_dict, epoch=epoch)

            # Fire evaluation-callbacks (to be executed every [eval_log] iterations, as specified within the functions)
            for eval_cb in eval_cbs:
                if eval_cb is not None:
                    eval_cb(model, iteration, epoch=epoch)

            # Break if max-number of iterations is reached
            if iteration == iters:
                bar.close()
                break

            # Save checkpoint?
            if (save_every is not None) and (iteration % save_every) == 0:
                utils.save_checkpoint(model, model_dir=m_dir)

def train_cl(model, train_datasets, replay_mode="none", scenario="task", rnt=None, classes_per_task=None,
             iters=2000, batch_size=32, batch_size_replay=None, loss_cbs=list(), eval_cbs=list(), sample_cbs=list(),
             generator=None, gen_iters=0, gen_loss_cbs=list(), feedback=False, reinit=False, args=None, only_last=False,
             sample_method='random', curated_multiplier=4, variety_weight=0.5, mir_coef=0.1):
    '''Train a model (with a "train_a_batch" method) on multiple tasks, with replay-strategy specified by [replay_mode].

    [model]             <nn.Module> main model to optimize across all tasks
    [train_datasets]    <list> with for each task the training <DataSet>
    [replay_mode]       <str>, choice from "generative", "current", "offline" and "none"
    [scenario]          <str>, choice from "task", "domain", "class" and "all"
    [classes_per_task]  <int>, # classes per task; only 1st task has [classes_per_task]*[first_task_class_boost] classes
    [rnt]               <float>, indicating relative importance of new task (if None, relative to # old tasks)
    [iters]             <int>, # optimization-steps (=batches) per task; 1st task has [first_task_iter_boost] steps more
    [batch_size_replay] <int>, number of samples to replay per batch
    [generator]         None or <nn.Module>, if a seperate generative model should be trained (for [gen_iters] per task)
    [feedback]          <bool>, if True and [replay_mode]="generative", the main model is used for generating replay
    [only_last]         <bool>, only train on final task / episode
    [*_cbs]             <list> of call-back functions to evaluate training-progress
    [sample_method]     <str> indicating the sample method, choices: 'random', 'uniform', 'curated', 'softmax', 'interfered', 'misclassified'
    [curated_multiplier]<int> choose curated samples out of size curated_multiplier * mutiply batch_size_replay
    [variety_weight]    <float> weight of variety loss as compared with regular loss
    [mir_coef]          <float> weight of previous model's cross entropy score for generated sample (encourages samples which the prior model is confident about)

    '''

    print("\nCurated multiplier: ", curated_multiplier)
    print("\nVariety weight: ", variety_weight)
    print("\nMIR coefficient: ", mir_coef)
    # Should convolutional layers be frozen?
    freeze_convE = (utils.checkattr(args, "freeze_convE") and hasattr(args, "depth") and args.depth>0)

    # Use cuda?
    device = model._device()
    cuda = model._is_on_cuda()

    # Set default-values if not specified
    batch_size_replay = batch_size if batch_size_replay is None else batch_size_replay

    # Initiate indicators for replay (no replay for 1st task)
    Generative = Current = Offline_TaskIL = False
    previous_model = None

    # Register starting param-values (needed for "intelligent synapses").
    if isinstance(model, ContinualLearner) and model.si_c>0:
        for n, p in model.named_parameters():
            if p.requires_grad:
                n = n.replace('.', '__')
                model.register_buffer('{}_SI_prev_task'.format(n), p.detach().clone())

    # Loop over all tasks.
    for task, train_dataset in enumerate(train_datasets, 1):

        # If offline replay-setting, create large database of all tasks so far
        if replay_mode=="offline" and (not scenario=="task"):
            train_dataset = ConcatDataset(train_datasets[:task])
        # -but if "offline"+"task": all tasks so far should be visited separately (i.e., separate data-loader per task)
        if replay_mode=="offline" and scenario=="task":
            Offline_TaskIL = True
            data_loader = [None]*task

        # Initialize # iters left on data-loader(s)
        iters_left = 1 if (not Offline_TaskIL) else [1]*task

        # Prepare <dicts> to store running importance estimates and parameter-values before update
        if isinstance(model, ContinualLearner) and model.si_c>0:
            W = {}
            p_old = {}
            for n, p in model.named_parameters():
                if p.requires_grad:
                    n = n.replace('.', '__')
                    W[n] = p.data.clone().zero_()
                    p_old[n] = p.data.clone()

        # Find [active_classes] (=classes in current task)
        active_classes = None  #-> for "domain"- or "all"-scenarios, always all classes are active
        if scenario=="task":
            # -for "task"-scenario, create <list> with for all tasks so far a <list> with the active classes
            active_classes = [list(range(classes_per_task*i, classes_per_task*(i+1))) for i in range(task)]
        elif scenario=="class":
            # -for "class"-scenario, create one <list> with active classes of all tasks so far
            active_classes = list(range(classes_per_task*task))

        # Reinitialize the model's parameters (if requested)
        if reinit:
            from define_models import init_params
            init_params(model, args)
            if generator is not None:
                init_params(generator, args)

        # Define a tqdm progress bar(s)
        iters_main = iters
        progress = tqdm.tqdm(range(1, iters_main+1))
        if generator is not None:
            iters_gen = gen_iters
            progress_gen = tqdm.tqdm(range(1, iters_gen+1))

        # Loop over all iterations
        iters_to_use = (iters_main if (generator is None) else max(iters_main, iters_gen))
        # -if only the final task should be trained on:
        if only_last and not task==len(train_datasets):
            iters_to_use = 0

        # This helps w/ speeding up curated_classVariety
        mask = None
        if (sample_method=="curated_classVariety" and (task-1)>0):
            sampleAmt = batch_size_replay * curated_multiplier
            classNum = classes_per_task*(task-1)
            indexList = [[idx for idx in range(sampleAmt) if (idx%classNum) == (rowIdx%classNum)] for rowIdx in range(sampleAmt)]
            mask = []
            for rowIdxList in indexList:
                curRow = [0] * sampleAmt
                for idx in rowIdxList:
                    curRow[idx] = 1
                mask.append(curRow)
            mask = torch.tensor(mask, dtype=torch.float).to(device)

        for batch_index in range(1, iters_to_use+1):

            # Update # iters left on current data-loader(s) and, if needed, create new one(s)
            if not Offline_TaskIL:
                iters_left -= 1
                if iters_left==0:
                    data_loader = iter(utils.get_data_loader(train_dataset, batch_size, cuda=cuda, drop_last=True))
                    iters_left = len(data_loader)
            else:
                # -with "offline replay" in Task-IL scenario, there is a separate data-loader for each task
                batch_size_to_use = int(np.ceil(batch_size/task))
                for task_id in range(task):
                    iters_left[task_id] -= 1
                    if iters_left[task_id]==0:
                        data_loader[task_id] = iter(utils.get_data_loader(
                            train_datasets[task_id], batch_size_to_use, cuda=cuda, drop_last=True
                        ))
                        iters_left[task_id] = len(data_loader[task_id])

            #-----------------Collect data------------------#

            #####-----CURRENT BATCH-----#####
            if not Offline_TaskIL:
                x, y = next(data_loader)                                    #--> sample training data of current task
                y = y-classes_per_task*(task-1) if scenario=="task" else y  #--> ITL: adjust y-targets to 'active range'
                x, y = x.to(device), y.to(device)                           #--> transfer them to correct device
                #y = y.expand(1) if len(y.size())==1 else y                 #--> hack for if batch-size is 1
            else:
                x = y = task_used = None  #--> all tasks are "treated as replay"
                # -sample training data for all tasks so far, move to correct device and store in lists
                x_, y_ = list(), list()
                for task_id in range(task):
                    x_temp, y_temp = next(data_loader[task_id])
                    x_.append(x_temp.to(device))
                    y_temp = y_temp - (classes_per_task * task_id) #--> adjust y-targets to 'active range'
                    if batch_size_to_use == 1:
                        y_temp = torch.tensor([y_temp])            #--> correct dimensions if batch-size is 1
                    y_.append(y_temp.to(device))

            #####-----REPLAYED BATCH-----#####
            if not Offline_TaskIL and not Generative and not Current:
                x_ = y_ = scores_ = task_used = None   #-> if no replay

            #--------------------------------------------INPUTS----------------------------------------------------#

            ##-->> Current Replay <<--##
            if Current:
                x_ = x[:batch_size_replay]  #--> use current task inputs
                task_used = None

            ##-->> Generative Replay <<--##
            if Generative:
                #---> Only with generative replay, the resulting [x_] will be at the "hidden"-level
                conditional_gen = True if (
                    (previous_generator.per_class and previous_generator.prior=="GMM") or
                    utils.checkattr(previous_generator, 'dg_gates')
                ) else False

                # Sample [x_]
                if conditional_gen and scenario=="task":
                    # -if a conditional generator is used with task-IL scenario, generate data per previous task
                    x_ = list()
                    task_used = list()
                    for task_id in range(task-1):
                        allowed_classes = list(range(classes_per_task*task_id, classes_per_task*(task_id+1)))
                        batch_size_replay_to_use = int(np.ceil(batch_size_replay / (task-1)))
                        x_temp_ = previous_generator.sample(batch_size_replay_to_use, allowed_classes=allowed_classes,
                                                            only_x=False)
                        x_.append(x_temp_[0])
                        task_used.append(x_temp_[2])
                else:
                    # -which classes are allowed to be generated? (relevant if conditional generator / decoder-gates)
                    allowed_classes = None if scenario=="domain" else list(range(classes_per_task*(task-1)))
                    # -which tasks/domains are allowed to be generated? (only relevant if "Domain-IL" with task-gates)
                    allowed_domains = list(range(task-1))
                    # -generate inputs representative of previous tasks

                    # --- SAMPLE METHOD CHOICES: softmax, random, uniform, curated ---
                    # --- Softmax sampling: use previous model to score images from this new task, generate those classes
                    if sample_method == 'softmax':
                        with torch.no_grad():
                            curTaskID = task - 2
                            newScores_og = previous_model.classify(previous_model.input_to_hidden(x),
                                                                   not_hidden=False if Generative else True)
                            newScores = newScores_og[:, :(classes_per_task * (curTaskID + 1))]
                            softmax = torch.nn.Softmax(dim=1)
                            scores_old = nn.Softmax(dim=1)(newScores)
                            avgError = torch.mean(scores_old, dim=0)
                            sampleProbs = torch.zeros(newScores_og.shape[1])
                            sampleProbs[:(classes_per_task * (curTaskID + 1))] = avgError[
                                                                                :(classes_per_task * (curTaskID + 1))]
                            x_, y_used, task_used = previous_generator.sample(
                                batch_size_replay, allowed_classes=allowed_classes, allowed_domains=allowed_domains,
                                only_x=False, class_probs=sampleProbs,uniform_sampling=False)

                    # --- Uniformly random sampling (baseline) ---
                    elif sample_method == 'random':
                        x_, y_used, task_used = previous_generator.sample(
                            batch_size_replay, allowed_classes=allowed_classes, allowed_domains=allowed_domains,
                            only_x=False, class_probs=None, uniform_sampling=False)

                    # --- Uniform sampling: balanced numbers of samples from each class ---
                    elif sample_method == 'uniform':
                        x_, y_used, task_used = previous_generator.sample(
                            batch_size_replay, allowed_classes=allowed_classes, allowed_domains=allowed_domains,
                            only_x=False, class_probs=None, uniform_sampling=True)
                    # --- Uniform sample curation: pick the best samples to show (by some metric), balance uniformly ---
                    else:

                        if (sample_method == "curated_variety" or sample_method == "interfered"):
                            # Generate x times as many samples as we need to then pick the best of
                            x_, y_used, task_used, varietyVector = previous_generator.sample(
                                batch_size_replay * curated_multiplier, allowed_classes=allowed_classes, allowed_domains=allowed_domains,
                                only_x=False, class_probs=None, uniform_sampling=False, varietyVector=True)

                        # CURATED USING CLASS VARIETY (i.e., generating batch_size_reply*curated_multipler / len(allowed_classes) samples
                        # per class, where each sample is the "most different" sample based off our variety calculation
                        elif(sample_method == "curated_classVariety"):
                            x_, y_used, task_used, varietyVector = previous_generator.sample(
                                batch_size_replay * curated_multiplier, allowed_classes=allowed_classes, allowed_domains=allowed_domains,
                                only_x=False, class_probs=None, uniform_sampling=True, varietyVector=True, classVariety=True, classVarietyMask=mask)

                        elif(sample_method == "curated_softmax"):

                            with torch.no_grad():
                                curTaskID = task - 2
                                newScores_og = previous_model.classify(previous_model.input_to_hidden(x),
                                                                       not_hidden=False if Generative else True)
                                newScores = newScores_og[:, :(classes_per_task * (curTaskID + 1))]
                                softmax = torch.nn.Softmax(dim=1)
                                scores_old = nn.Softmax(dim=1)(newScores)
                                avgError = torch.mean(scores_old, dim=0)
                                sampleProbs = torch.zeros(newScores_og.shape[1])
                                sampleProbs[:(classes_per_task * (curTaskID + 1))] = avgError[
                                                                                :(classes_per_task * (curTaskID + 1))]

                            # Generate x times as many samples as we need to then pick the best of
                            x_, y_used, task_used = previous_generator.sample(
                                batch_size_replay * curated_multiplier, allowed_classes=allowed_classes, allowed_domains=allowed_domains,
                                only_x=False, class_probs=sampleProbs, uniform_sampling=False)

                        else:
                            # Generate x times as many samples as we need to then pick the best of
                            x_, y_used, task_used = previous_generator.sample(
                                batch_size_replay * curated_multiplier, allowed_classes=allowed_classes, allowed_domains=allowed_domains,
                                only_x=False, class_probs=None, uniform_sampling=False)

                        # --- Measure the performance of each of these samples on the current model ---
                        # Use the previous model to score the generated images (code taken from Trevor's softmax above)
                        with torch.no_grad():
                            curTaskID = task - 2
                            newScores_og = model.classify(x_, not_hidden=False if Generative else True).to(device)
                            newScores = newScores_og[:, :(classes_per_task * (curTaskID + 1))].to(device) # Logits that don't sum to 1
                            scores_old = nn.Softmax(dim=1)(newScores).to(device) # Makes the scores sum to 1 (probabilities)
                            cross_entropy = nn.CrossEntropyLoss(reduction='none').to(device)
                            y_used = torch.tensor(y_used, dtype=torch.long).to(device)
                            loss_old = cross_entropy(scores_old, y_used).to(device)

                        # --- Copy the model and perform an update on just the new incoming data (no replayed data) ---
                        # This will lead to catastrophic forgetting, as it has no replays to prevent this from happening
                        model_tmp = copy.deepcopy(model)
                        # NOTE: Can train multiple batches if needed, but it would be on the same data, so any changes will just be exacerbated
                        _ = model_tmp.train_a_batch(x, y=y, x_=None, y_=None, scores_=None,
                                                        tasks_=task_used, active_classes=active_classes, task=task, rnt=(
                                                            1. if task==1 else 1./task
                                                        ) if rnt is None else rnt, freeze_convE=freeze_convE,
                                                        replay_not_hidden=False if Generative else True)

                        # --- Measure the performance of each of the generated samples on this updated model ---
                        # This can tell us how much the model 'forgets' each of these samples, we will replay the worst ones
                        with torch.no_grad():
                            curTaskID = task - 2
                            newScores_og = model_tmp.classify(x_, not_hidden=False if Generative else True).to(device)
                            newScores = newScores_og[:, :(classes_per_task * (curTaskID + 2))].to(device) # Logits that don't sum to 1
                            scores_new = nn.Softmax(dim=1)(newScores).to(device) # Makes the scores sum to 1 (probabilities)

                            # --- Measure the difference in cross entropy loss for predictions before and after ---
                            if sample_method == 'curated' or sample_method == "curated_softmax":
                                cross_entropy = nn.CrossEntropyLoss(reduction='none') # Per-example cross entropy (not avg)
                                loss_new = cross_entropy(scores_new, y_used)

                                # Amount that the loss changes between the model updating
                                diff = loss_new - loss_old
                                metric = diff

                            # TREVOR'S NEW METHOD - This tries to take into account the variety of the samples
                            elif sample_method == "curated_variety" or sample_method == "curated_classVariety":
                                cross_entropy = nn.CrossEntropyLoss(reduction='none').to(device) # Per-example cross entropy (not avg)
                                loss_new = cross_entropy(scores_new, y_used).to(device)

                                # Amount that the loss changes between the model updating
                                diff = loss_new - loss_old

                                # Softmaxing diff and the variety vector (to get probabilities)
                                variety_weight = torch.tensor((variety_weight)).to(device)
                                diff_softmax = nn.Softmax(dim=0)(diff).to(device)
                                variety_softmax = nn.Softmax(dim=0)(varietyVector).to(device)
                                metric = ((1-variety_weight) * diff_softmax) + (variety_weight * variety_softmax).to(device)

                            # Multiply the misclassification error (cross entropy) by the amount that this changes between the model updating
                            # metric = loss_new * diff

                            # --- Measure KL Divergence between predictions before and predictions afterwards ---
                            # Maximally Interfered Retrieval uses a linear combination of KL, entropy, and 'variance'
                            # This ensures the samples are not too close together, but we do not currently measure that
                            elif sample_method == 'interfered':
                                # First, pad with zeros so predictions match (previous model predicts zero score for all new classes)
                                padded_scores = torch.zeros_like(scores_new)
                                padded_scores[:, :scores_old.size(1)] = scores_old
                                kl_div = nn.KLDivLoss(reduction='none')(padded_scores, scores_new)
                                kl_div = torch.mean(kl_div, dim=1)
                                variety_softmax = nn.Softmax(dim=0)(varietyVector).to(device)

                                # Calculate MIR loss and balance with variety (instead of explicitly searching, maximize both variety and MIR loss)
                                mir_loss = kl_div - mir_coef * loss_old
                                metric = (1-variety_weight) * (mir_loss) + variety_weight * variety_softmax

                            # --- New idea: use the examples which the new model misclassifies the most as one of the new classes
                            # This the opposite approach to softmax, where softmax takes the current model and calculates
                            # Which classes does it confuse the new data for the most, this trains on the new data and then
                            # Tries to find generated examples which it confuses for the new data classes the most
                            elif sample_method == 'misclassified' or sample_method == 'uniform_large' or sample_method == 'random_large':
                                metric = scores_new[:, -1] + scores_new[:, -1]

                            # --- Sort based on some metric, then divide up by classes (afterwards) ---
                            _, indices = torch.sort(metric, descending=True) # Descending order, pick first 100

                            # Shuffle indices around to test choosing from this larger pool of generated samples randomly
                            if sample_method == 'uniform_large' or sample_method == 'random_large':
                                indices2 = indices.cpu().numpy()
                                np.random.shuffle(indices2)
                                indices = torch.from_numpy(indices2).to(device)

                            if sample_method != 'random_large' and sample_method != 'curated_softmax':
                                # --- Calculate how many examples for each class should be generated to divide up uniformly ---
                                # Uniform dist will be [0, 1, 2, 3, 0, 1, 2] for allowed classes=4 and batch_size_replay=7
                                uniform_dist = torch.arange(batch_size_replay) % len(allowed_classes)
                                counts_each_class = torch.unique(uniform_dist, return_counts=True)[1]

                                # --- Optional: Calculate unbalanced indices to replay, results in poor performance ---
                                # If we added a variation term to ensure samples are different from each other, this could
                                # be a simpler way to do things, but variance would be pretty complicated to calculate
                                #indices_to_replay = indices[:batch_size_replay]

                                # --- Select the top k_i indices for each class i, where k_i is the number of examples for that class ---
                                # Top x most affected of the generated samples for each class (ensures it is balanced, slightly more computation than unbalanced)
                                indices_to_replay = torch.cat(( [ indices[y_used[indices]==i][:counts_each_class[i]] for i in range(len(allowed_classes)) ] ))
                                x_ = x_[indices_to_replay]
                            else:
                                # Uniformly randomly choose from the 400 samples generated
                                x_ = x_[indices]

            #--------------------------------------------OUTPUTS----------------------------------------------------#

            if Generative or Current:
                # Get target scores & possibly labels (i.e., [scores_] / [y_]) -- use previous model, with no_grad()
                if scenario in ("domain", "class") and previous_model.mask_dict is None:
                    # -if replay does not need to be evaluated for each task (ie, not Task-IL and no task-specific mask)
                    with torch.no_grad():
                        all_scores_ = previous_model.classify(x_, not_hidden=False if Generative else True)
                    scores_ = all_scores_[:, :(classes_per_task*(task-1))] if (
                            scenario=="class"
                    ) else all_scores_ # -> when scenario=="class", zero probs will be added in [loss_fn_kd]-function
                    # -also get the 'hard target'
                    _, y_ = torch.max(scores_, dim=1)
                else:
                    # -[x_] needs to be evaluated according to each previous task, so make list with entry per task
                    scores_ = list()
                    y_ = list()
                    # -if no task-mask and no conditional generator, all scores can be calculated in one go
                    if previous_model.mask_dict is None and not type(x_)==list:
                        with torch.no_grad():
                            all_scores_ = previous_model.classify(x_, not_hidden=False if Generative else True)
                    for task_id in range(task-1):
                        # -if there is a task-mask (i.e., XdG is used), obtain predicted scores for each task separately
                        if previous_model.mask_dict is not None:
                            previous_model.apply_XdGmask(task=task_id+1)
                        if previous_model.mask_dict is not None or type(x_)==list:
                            with torch.no_grad():
                                all_scores_ = previous_model.classify(x_[task_id] if type(x_)==list else x_,
                                                                      not_hidden=False if Generative else True)
                        if scenario=="domain":
                            # NOTE: if scenario=domain with task-mask, it's of course actually the Task-IL scenario!
                            #       this can be used as trick to run the Task-IL scenario with singlehead output layer
                            temp_scores_ = all_scores_
                        else:
                            temp_scores_ = all_scores_[:, (classes_per_task*task_id):(classes_per_task*(task_id+1))]
                        scores_.append(temp_scores_)
                        # - also get hard target
                        _, temp_y_ = torch.max(temp_scores_, dim=1)
                        y_.append(temp_y_)
            # -only keep predicted y_/scores_ if required (as otherwise unnecessary computations will be done)
            y_ = y_ if (model.replay_targets=="hard") else None
            scores_ = scores_ if (model.replay_targets=="soft") else None

            #-----------------Train model(s)------------------#

            #---> Train MAIN MODEL
            if batch_index <= iters_main:

                # Train the main model with this batch
                loss_dict = model.train_a_batch(x, y=y, x_=x_, y_=y_, scores_=scores_,
                                                tasks_=task_used, active_classes=active_classes, task=task, rnt=(
                                                    1. if task==1 else 1./task
                                                ) if rnt is None else rnt, freeze_convE=freeze_convE,
                                                replay_not_hidden=False if Generative else True)

                # UNIFORM SAMPLE CURATION: loss_dict has a "predL_r" key that contains the individual prediction
                # losses

                # Update running parameter importance estimates in W
                if isinstance(model, ContinualLearner) and model.si_c>0:
                    for n, p in model.convE.named_parameters():
                        if p.requires_grad:
                            n = "convE."+n
                            n = n.replace('.', '__')
                            if p.grad is not None:
                                W[n].add_(-p.grad*(p.detach()-p_old[n]))
                            p_old[n] = p.detach().clone()
                    for n, p in model.fcE.named_parameters():
                        if p.requires_grad:
                            n = "fcE."+n
                            n = n.replace('.', '__')
                            if p.grad is not None:
                                W[n].add_(-p.grad * (p.detach() - p_old[n]))
                            p_old[n] = p.detach().clone()
                    for n, p in model.classifier.named_parameters():
                        if p.requires_grad:
                            n = "classifier."+n
                            n = n.replace('.', '__')
                            if p.grad is not None:
                                W[n].add_(-p.grad * (p.detach() - p_old[n]))
                            p_old[n] = p.detach().clone()

                # Fire callbacks (for visualization of training-progress / evaluating performance after each task)
                for loss_cb in loss_cbs:
                    if loss_cb is not None:
                        loss_cb(progress, batch_index, loss_dict, task=task)
                for eval_cb in eval_cbs:
                    if eval_cb is not None:
                        eval_cb(model, batch_index, task=task)
                if model.label=="VAE":
                    for sample_cb in sample_cbs:
                        if sample_cb is not None:
                            sample_cb(model, batch_index, task=task, allowed_classes=None if (
                                    scenario=="domain"
                            ) else list(range(classes_per_task*task)))

            #---> Train GENERATOR
            if generator is not None and batch_index <= iters_gen:

                loss_dict = generator.train_a_batch(x, y=y, x_=x_, y_=y_, scores_=scores_,
                                                    tasks_=task_used, active_classes=active_classes, rnt=(
                                                        1. if task==1 else 1./task
                                                    ) if rnt is None else rnt, task=task,
                                                    freeze_convE=freeze_convE,
                                                    replay_not_hidden=False if Generative else True)

                # Fire callbacks on each iteration
                for loss_cb in gen_loss_cbs:
                    if loss_cb is not None:
                        loss_cb(progress_gen, batch_index, loss_dict, task=task)
                for sample_cb in sample_cbs:
                    if sample_cb is not None:
                        sample_cb(generator, batch_index, task=task, allowed_classes=None if (
                                    scenario=="domain"
                            ) else list(range(classes_per_task*task)))

        # Close progres-bar(s)
        progress.close()
        if generator is not None:
            progress_gen.close()

        ##----------> UPON FINISHING EACH TASK...

        # EWC: estimate Fisher Information matrix (FIM) and update term for quadratic penalty
        if isinstance(model, ContinualLearner) and model.ewc_lambda>0:
            # -find allowed classes
            allowed_classes = list(
                range(classes_per_task*(task-1), classes_per_task*task)
            ) if scenario=="task" else (list(range(classes_per_task*task)) if scenario=="class" else None)
            # -if needed, apply correct task-specific mask
            if model.mask_dict is not None:
                model.apply_XdGmask(task=task)
            # -estimate FI-matrix
            model.estimate_fisher(train_dataset, allowed_classes=allowed_classes)

        # SI: calculate and update the normalized path integral
        if isinstance(model, ContinualLearner) and model.si_c>0:
            model.update_omega(W, model.epsilon)

        # REPLAY: update source for replay
        previous_model = copy.deepcopy(model).eval()
        if replay_mode=="generative":
            Generative = True
            previous_generator = previous_model if feedback else copy.deepcopy(generator).eval()
        elif replay_mode=='current':
            Current = True

max_stars_repo_name: elliottower/brain-inspired-replay

max_issues_repo_name: elliottower/brain-inspired-replay

max_forks_repo_name: elliottower/brain-inspired-replay


text:
""" Testing of the Decision Forest Classifier.
"""

# Author: AI Werkstatt (TM)
# (C) Copyright 2019, AI Werkstatt (TM) www.aiwerkstatt.com. All rights reserved.

import pytest

import numpy as np
import pickle
import graphviz

from sklearn.datasets import load_iris

from sklearn.utils.estimator_checks import check_estimator
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
from sklearn.exceptions import NotFittedError
from sklearn.externals.joblib import parallel_backend

from koho.sklearn import DecisionForestClassifier

precision = 1e-7  # used for floating point "==" test

# iris dataset
@pytest.fixture
def iris():
    return load_iris()

# sklearn compatible
# ==================

# sklearn's check_estimator()
def test_sklearn_check_estimator():
    check_estimator(DecisionForestClassifier)

# sklearn's pipeline
def test_sklearn_pipeline(iris):
    X, y = iris.data, iris.target
    pipe = make_pipeline(DecisionForestClassifier(random_state=0))
    pipe.fit(X, y)
    pipe.predict(X)
    pipe.predict_proba(X)
    score = pipe.score(X, y)
    assert score > 0.9666666666666667 - precision and score < 0.9666666666666667 + precision

# sklearn's grid search
def test_sklearn_grid_search(iris):
    X, y = iris.data, iris.target
    parameters = [{'n_estimators': [10, 20],
                   'bootstrap': [False, True],
                   'max_features': [None, 1],
                   'max_thresholds': [None, 1]
                   }]
    grid_search = GridSearchCV(DecisionForestClassifier(random_state=0), parameters, cv=3)
    grid_search.fit(X, y)
    assert grid_search.best_params_['n_estimators'] == 10
    assert grid_search.best_params_['bootstrap'] == False
    assert grid_search.best_params_['max_features'] is None
    assert grid_search.best_params_['max_thresholds'] == 1
    clf = DecisionForestClassifier(random_state=0)
    clf.set_params(**grid_search.best_params_)
    assert clf.n_estimators == 10
    assert clf.bootstrap == False
    assert clf.class_balance == 'balanced'
    assert clf.max_depth == 3
    assert clf.max_features is None
    assert clf.max_thresholds == 1
    assert clf.random_state == 0
    clf.fit(X, y)
    score = clf.score(X, y)
    assert score > 0.9666666666666667 - precision and score < 0.9666666666666667 + precision

# sklearn's persistence
def test_sklearn_persistence(iris):
    X, y = iris.data, iris.target
    clf = DecisionForestClassifier(random_state=0)
    clf.fit(X, y)
    with open("clf_dfc.pkl", "wb") as f:
        pickle.dump(clf, f)
    with open("clf_dfc.pkl", "rb") as f:
        clf2 = pickle.load(f)
    score = clf2.score(X, y)
    assert score > 0.9666666666666667 - precision and score < 0.9666666666666667 + precision

# sklearn's parallel processing (joblib)
def test_sklearn_parallel_processing(iris):
    X, y = iris.data, iris.target
    clf = DecisionForestClassifier(random_state=0)
    with parallel_backend('loky', n_jobs=-1):
        clf.fit(X, y)
        score = clf.score(X, y)
    assert score > 0.9666666666666667 - precision and score < 0.9666666666666667 + precision

# iris dataset (User's Guide Python)
# ==================================

def test_iris(iris):
    X, y = iris.data, iris.target
    clf = DecisionForestClassifier(random_state=0)
    assert clf.n_estimators == 100
    assert clf.bootstrap == False
    assert clf.oob_score == False
    assert clf.class_balance == 'balanced'
    assert clf.max_depth == 3
    assert clf.max_features == 'auto'
    assert clf.max_thresholds is None
    assert clf.missing_values is None
    assert clf.n_jobs is None
    assert clf.random_state == 0
    # Training
    clf.fit(X, y)
    # Feature Importances
    feature_importances = clf.feature_importances_
    feature_importances_target = [0.09045256, 0.00816573, 0.38807981, 0.5133019]
    for i1, i2 in zip(feature_importances, feature_importances_target):
        assert i1 > i2 - precision and i1 < i2 + precision
    # Visualize Trees
    tree_idx = 0
    dot_data = clf.estimators_[tree_idx].export_graphviz(
        feature_names=iris.feature_names,
        class_names=iris.target_names,
        rotate=True)
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'rankdir=LR ;' '\n' \
        r'0 [label="petal width (cm) <= 0.8\np(class) = [0.33, 0.33, 0.33]\nclass, n = 150", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=3.333333, headlabel="True", labeldistance=2.5, labelangle=-45] ;' '\n' \
        r'0 -> 2 [penwidth=6.666667] ;' '\n' \
        r'1 [label="[1, 0, 0]\nsetosa", fillcolor="#FF0000FF"] ;' '\n' \
        r'2 [label="petal width (cm) <= 1.75\n[0, 0.5, 0.5]", fillcolor="#00FF003F"] ;' '\n' \
        r'2 -> 3 [penwidth=3.600000] ;' '\n' \
        r'2 -> 6 [penwidth=3.066667] ;' '\n' \
        r'3 [label="petal length (cm) <= 4.95\n[0, 0.91, 0.09]", fillcolor="#00FF00BE"] ;' '\n' \
        r'3 -> 4 [penwidth=3.200000] ;' '\n' \
        r'3 -> 5 [penwidth=0.400000] ;' '\n' \
        r'4 [label="[0, 0.98, 0.02]\nversicolor", fillcolor="#00FF00EF"] ;' '\n' \
        r'5 [label="[0, 0.33, 0.67]\nvirginica", fillcolor="#0000FF55"] ;' '\n' \
        r'6 [label="petal width (cm) <= 1.85\n[0, 0.02, 0.98]", fillcolor="#0000FFEE"] ;' '\n' \
        r'6 -> 7 [penwidth=0.800000] ;' '\n' \
        r'6 -> 8 [penwidth=2.266667] ;' '\n' \
        r'7 [label="[0, 0.08, 0.92]\nvirginica", fillcolor="#0000FFC4"] ;' '\n' \
        r'8 [label="[0, 0, 1]\nvirginica", fillcolor="#0000FFFF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target
    # Export textual format
    t = clf.estimators_[tree_idx].export_text()
    t_target = r'0 X[3]<=0.8 [50, 50, 50]; 0->1; 0->2; 1 [50, 0, 0]; 2 X[3]<=1.75 [0, 50, 50]; 2->3; 2->6; 3 X[2]<=4.95 [0, 49, 5]; 3->4; 3->5; 4 [0, 47, 1]; 5 [0, 2, 4]; 6 X[3]<=1.85 [0, 1, 45]; 6->7; 6->8; 7 [0, 1, 11]; 8 [0, 0, 34]; '
    assert t == t_target
    # Persistence
    with open("iris_dfc.pkl", "wb") as f:
        pickle.dump(clf, f)
    with open("iris_dfc.pkl", "rb") as f:
        clf2 = pickle.load(f)
    assert clf2.estimators_[tree_idx].export_text() == clf.estimators_[tree_idx].export_text()
    # Classification
    c = clf2.predict(X)
    assert sum(c) == 147
    cp = clf2.predict_proba(X)
    assert sum(sum(cp)) > 150 - precision and sum(sum(cp)) < 150 + precision
    # Testing
    score = clf2.score(X, y)
    assert score > 0.9666666666666667 - precision and score < 0.9666666666666667 + precision

# simple example
# ==============

classes = ['A', 'B']
features = ['a', 'b', 'c']
X = np.array([
        [0, 0, 0],
        [0, 0, 1],
        [0, 1, 0],
        [0, 1, 1],
        [0, 1, 1],
        [1, 0, 0],
        [1, 0, 0],
        [1, 0, 0],
        [1, 0, 0],
        [1, 1, 1]]).astype(np.double)
y = np.array([0, 0, 1, 1, 1, 1, 1, 1, 1, 1])

X_test = np.array([
        [0, 0, 0],
        [0, 0, 1],
        [0, 1, 0],
        [0, 1, 1],
        [1, 0, 0],
        [1, 0, 1],
        [1, 1, 0],
        [1, 1, 1]]).astype(np.double)
y_test = np.array([0, 0, 1, 1, 1, 1, 1, 1])

def test_simple_example():
    clf = DecisionForestClassifier(
        n_estimators=10,
        bootstrap=False,
        oob_score=False,
        class_balance='balanced',
        max_depth=3,
        max_features=2,
        max_thresholds=None,
        missing_values=None,
        random_state=0)
    assert clf.n_estimators == 10
    assert clf.bootstrap == False
    assert clf.oob_score == False
    assert clf.class_balance == 'balanced'
    assert clf.max_depth == 3
    assert clf.max_features == 2
    assert clf.max_thresholds is None
    assert clf.missing_values is None
    assert clf.random_state == 0
    # Training
    clf.fit(X, y)
    # Feature Importances
    feature_importances = clf.feature_importances_
    feature_importances_target = [0.48484848, 0.47939394, 0.03575758]
    for i1, i2 in zip(feature_importances, feature_importances_target):
        assert i1 > i2 - precision and i1 < i2 + precision
    # Visualize Tree
    tree_idx = 0
    dot_data = clf.estimators_[tree_idx].export_graphviz(
        feature_names=features,
        class_names=classes,
        rotate=True)
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'rankdir=LR ;' '\n' \
        r'0 [label="a <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=-45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="b <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\nA", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target
    # Export textual format
    t = clf.estimators_[tree_idx].export_text()
    t_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert t == t_target
    # Persistence
    with open("simple_example_dfc.pkl", "wb") as f:
        pickle.dump(clf, f)
    with open("simple_example_dfc.pkl", "rb") as f:
        clf2 = pickle.load(f)
    assert clf2.estimators_[tree_idx].export_text() == clf.estimators_[tree_idx].export_text()
    # Classification
    c = clf2.predict(X)
    c_target = [0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
    for i1, i2 in zip(c, c_target):
        assert i1 > i2 - precision and i1 < i2 + precision
    # Testing
    score = clf2.score(X, y)
    assert score > 1.0 - precision and score < 1.0 + precision

# simple multi-output example
# ===========================

# multi-output fed with single-output
# -----------------------------------

def test_simple_multi_output_example_with_single_output():
    classes = [['0', '1', '2', '3', '4', '5', '6', '7']]
    features = ['2^2', '2^1', '2^0']

    X_mo = np.array([[0, 0, 0],
                     [0, 0, 1],
                     [0, 1, 0],
                     [0, 1, 1],
                     [1, 0, 0],
                     [1, 0, 1],
                     [1, 1, 0],
                     [1, 1, 1]]).astype(np.double)
    y_mo = np.array([[0],
                     [1],
                     [2],
                     [3],
                     [4],
                     [5],
                     [6],
                     [7]]).astype(np.long)

    clf = DecisionForestClassifier(
        n_estimators=10,
        bootstrap=False,
        oob_score=False,
        class_balance='balanced',
        max_depth=3,
        max_features=1,
        max_thresholds=1,
        missing_values=None,
        random_state=0)
    # Training
    clf.fit(X_mo, y_mo)
    # Feature Importances
    feature_importances = clf.feature_importances_
    feature_importances_target = [0.32857143, 0.32857143, 0.34285714]
    for i1, i2 in zip(feature_importances, feature_importances_target):
        assert i1 > i2 - precision and i1 < i2 + precision
    # Visualize Tree
    dot_data = clf.estimators_[0].export_graphviz(
        feature_names=features,
        class_names=classes,
        rotate=True)
    #filename = "simple_example_multi_output_with_single_output_dfc_dtc_0"
    #graph = graphviz.Source(dot_data)
    #graph.render(filename, format='pdf')
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'rankdir=LR ;' '\n' \
        r'0 [label="2^2 <= 0.874\np(class) = [0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]\nclass, n = 8", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=5.000000, headlabel="True", labeldistance=2.5, labelangle=-45] ;' '\n' \
        r'0 -> 8 [penwidth=5.000000] ;' '\n' \
        r'1 [label="2^0 <= 0.753\n[0.25, 0.25, 0.25, 0.25, 0, 0, 0, 0]", fillcolor="#FF000024"] ;' '\n' \
        r'1 -> 2 [penwidth=2.500000] ;' '\n' \
        r'1 -> 5 [penwidth=2.500000] ;' '\n' \
        r'2 [label="2^1 <= 0.368\n[0.5, 0, 0.5, 0, 0, 0, 0, 0]", fillcolor="#FF00006D"] ;' '\n' \
        r'2 -> 3 [penwidth=1.250000] ;' '\n' \
        r'2 -> 4 [penwidth=1.250000] ;' '\n' \
        r'3 [label="[1, 0, 0, 0, 0, 0, 0, 0]\n0", fillcolor="#FF0000FF"] ;' '\n' \
        r'4 [label="[0, 0, 1, 0, 0, 0, 0, 0]\n2", fillcolor="#0000FFFF"] ;' '\n' \
        r'5 [label="2^1 <= 0.31\n[0, 0.5, 0, 0.5, 0, 0, 0, 0]", fillcolor="#00FF006D"] ;' '\n' \
        r'5 -> 6 [penwidth=1.250000] ;' '\n' \
        r'5 -> 7 [penwidth=1.250000] ;' '\n' \
        r'6 [label="[0, 1, 0, 0, 0, 0, 0, 0]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'7 [label="[0, 0, 0, 1, 0, 0, 0, 0]\n3", fillcolor="#FFFF00FF"] ;' '\n' \
        r'8 [label="2^1 <= 0.247\n[0, 0, 0, 0, 0.25, 0.25, 0.25, 0.25]", fillcolor="#00FFFF24"] ;' '\n' \
        r'8 -> 9 [penwidth=2.500000] ;' '\n' \
        r'8 -> 12 [penwidth=2.500000] ;' '\n' \
        r'9 [label="2^0 <= 0.316\n[0, 0, 0, 0, 0.5, 0.5, 0, 0]", fillcolor="#00FFFF6D"] ;' '\n' \
        r'9 -> 10 [penwidth=1.250000] ;' '\n' \
        r'9 -> 11 [penwidth=1.250000] ;' '\n' \
        r'10 [label="[0, 0, 0, 0, 1, 0, 0, 0]\n4", fillcolor="#00FFFFFF"] ;' '\n' \
        r'11 [label="[0, 0, 0, 0, 0, 1, 0, 0]\n5", fillcolor="#FF00FFFF"] ;' '\n' \
        r'12 [label="2^0 <= 0.516\n[0, 0, 0, 0, 0, 0, 0.5, 0.5]", fillcolor="#FF80006D"] ;' '\n' \
        r'12 -> 13 [penwidth=1.250000] ;' '\n' \
        r'12 -> 14 [penwidth=1.250000] ;' '\n' \
        r'13 [label="[0, 0, 0, 0, 0, 0, 1, 0]\n6", fillcolor="#FF8000FF"] ;' '\n' \
        r'14 [label="[0, 0, 0, 0, 0, 0, 0, 1]\n7", fillcolor="#00FF80FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target
    # Export textual format
    t = clf.estimators_[0].export_text()
    t_target = r'0 X[0]<=0.874 [1, 1, 1, 1, 1, 1, 1, 1]; 0->1; 0->8; 1 X[2]<=0.753 [1, 1, 1, 1, 0, 0, 0, 0]; 1->2; 1->5; 2 X[1]<=0.368 [1, 0, 1, 0, 0, 0, 0, 0]; 2->3; 2->4; 3 [1, 0, 0, 0, 0, 0, 0, 0]; 4 [0, 0, 1, 0, 0, 0, 0, 0]; 5 X[1]<=0.31 [0, 1, 0, 1, 0, 0, 0, 0]; 5->6; 5->7; 6 [0, 1, 0, 0, 0, 0, 0, 0]; 7 [0, 0, 0, 1, 0, 0, 0, 0]; 8 X[1]<=0.247 [0, 0, 0, 0, 1, 1, 1, 1]; 8->9; 8->12; 9 X[2]<=0.316 [0, 0, 0, 0, 1, 1, 0, 0]; 9->10; 9->11; 10 [0, 0, 0, 0, 1, 0, 0, 0]; 11 [0, 0, 0, 0, 0, 1, 0, 0]; 12 X[2]<=0.516 [0, 0, 0, 0, 0, 0, 1, 1]; 12->13; 12->14; 13 [0, 0, 0, 0, 0, 0, 1, 0]; 14 [0, 0, 0, 0, 0, 0, 0, 1]; '
    assert t == t_target
    # Persistence
    with open("simple_example_multi_output_with_single_output_dtc.pkl", "wb") as f:
        pickle.dump(clf, f)
    with open("simple_example_multi_output_with_single_output_dtc.pkl", "rb") as f:
        clf2 = pickle.load(f)
    assert clf2.estimators_[0].export_text() == clf.estimators_[0].export_text()
    # Classification
    c = clf2.predict(X_mo)
    for i1, i2 in zip(c, y_mo):
        assert i1 > i2 - precision and i1 < i2 + precision
    # Testing
    score = clf2.score(X_mo, y_mo)
    assert score > 1.0 - precision and score < 1.0 + precision

# multi-output
# ------------

def test_simple_multi_output_example():
    classes = [['0', '1', '2', '3', '4', '5', '6', '7'],
               ['0', '4'],
               ['0', '2'],
               ['0', '1']]
    features = ['2^2', '2^1', '2^0']

    X_mo = np.array([[0, 0, 0],
                     [0, 0, 1],
                     [0, 1, 0],
                     [0, 1, 1],
                     [1, 0, 0],
                     [1, 0, 1],
                     [1, 1, 0],
                     [1, 1, 1]]).astype(np.double)
    y_mo = np.array([[0, 0, 0, 0],
                     [1, 0, 0, 1],
                     [2, 0, 1, 0],
                     [3, 0, 1, 1],
                     [4, 1, 0, 0],
                     [5, 1, 0, 1],
                     [6, 1, 1, 0],
                     [7, 1, 1, 1]]).astype(np.long)

    clf = DecisionForestClassifier(
        n_estimators=10,
        bootstrap=False,
        oob_score=False,
        class_balance='balanced',
        max_depth=3,
        max_features=1,
        max_thresholds=1,
        missing_values=None,
        random_state=0)
    # Training
    clf.fit(X_mo, y_mo)
    # Feature Importances
    feature_importances = clf.feature_importances_
    feature_importances_target = [0.33157895, 0.33157895, 0.33684211]
    for i1, i2 in zip(feature_importances, feature_importances_target):
        assert i1 > i2 - precision and i1 < i2 + precision
    # Visualize Tree
    dot_data = clf.estimators_[0].export_graphviz(
        feature_names=features,
        class_names=classes,
        rotate=True)
    #filename = "simple_example_multi_output_dfc_dtc_0"
    #graph = graphviz.Source(dot_data)
    #graph.render(filename, format='pdf')
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'rankdir=LR ;' '\n' \
        r'0 [label="2^2 <= 0.874\np(class) = [0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]\n[0.5, 0.5]\n[0.5, 0.5]\n[0.5, 0.5]\nclass, n = 8", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=5.000000, headlabel="True", labeldistance=2.5, labelangle=-45] ;' '\n' \
        r'0 -> 8 [penwidth=5.000000] ;' '\n' \
        r'1 [label="2^0 <= 0.753\n[0.25, 0.25, 0.25, 0.25, 0, 0, 0, 0]\n[1, 0]\n[0.5, 0.5]\n[0.5, 0.5]\n", fillcolor="#FF000043"] ;' '\n' \
        r'1 -> 2 [penwidth=2.500000] ;' '\n' \
        r'1 -> 5 [penwidth=2.500000] ;' '\n' \
        r'2 [label="2^1 <= 0.368\n[0.5, 0, 0.5, 0, 0, 0, 0, 0]\n[1, 0]\n[0.5, 0.5]\n[1, 0]\n", fillcolor="#FF000093"] ;' '\n' \
        r'2 -> 3 [penwidth=1.250000] ;' '\n' \
        r'2 -> 4 [penwidth=1.250000] ;' '\n' \
        r'3 [label="[1, 0, 0, 0, 0, 0, 0, 0]\n[1, 0]\n[1, 0]\n[1, 0]\n0\n0\n0\n0\n", fillcolor="#FF0000FF"] ;' '\n' \
        r'4 [label="[0, 0, 1, 0, 0, 0, 0, 0]\n[1, 0]\n[0, 1]\n[1, 0]\n2\n0\n2\n0\n", fillcolor="#00FFFFFF"] ;' '\n' \
        r'5 [label="2^1 <= 0.31\n[0, 0.5, 0, 0.5, 0, 0, 0, 0]\n[1, 0]\n[0.5, 0.5]\n[0, 1]\n", fillcolor="#FFFF0093"] ;' '\n' \
        r'5 -> 6 [penwidth=1.250000] ;' '\n' \
        r'5 -> 7 [penwidth=1.250000] ;' '\n' \
        r'6 [label="[0, 1, 0, 0, 0, 0, 0, 0]\n[1, 0]\n[1, 0]\n[0, 1]\n1\n0\n0\n1\n", fillcolor="#FFFF00FF"] ;' '\n' \
        r'7 [label="[0, 0, 0, 1, 0, 0, 0, 0]\n[1, 0]\n[0, 1]\n[0, 1]\n3\n0\n2\n1\n", fillcolor="#00FF80FF"] ;' '\n' \
        r'8 [label="2^1 <= 0.247\n[0, 0, 0, 0, 0.25, 0.25, 0.25, 0.25]\n[0, 1]\n[0.5, 0.5]\n[0.5, 0.5]\n", fillcolor="#FF400043"] ;' '\n' \
        r'8 -> 9 [penwidth=2.500000] ;' '\n' \
        r'8 -> 12 [penwidth=2.500000] ;' '\n' \
        r'9 [label="2^0 <= 0.316\n[0, 0, 0, 0, 0.5, 0.5, 0, 0]\n[0, 1]\n[1, 0]\n[0.5, 0.5]\n", fillcolor="#FF400093"] ;' '\n' \
        r'9 -> 10 [penwidth=1.250000] ;' '\n' \
        r'9 -> 11 [penwidth=1.250000] ;' '\n' \
        r'10 [label="[0, 0, 0, 0, 1, 0, 0, 0]\n[0, 1]\n[1, 0]\n[1, 0]\n4\n4\n0\n0\n", fillcolor="#FF4000FF"] ;' '\n' \
        r'11 [label="[0, 0, 0, 0, 0, 1, 0, 0]\n[0, 1]\n[1, 0]\n[0, 1]\n5\n4\n0\n1\n", fillcolor="#BFFF00FF"] ;' '\n' \
        r'12 [label="2^0 <= 0.516\n[0, 0, 0, 0, 0, 0, 0.5, 0.5]\n[0, 1]\n[0, 1]\n[0.5, 0.5]\n", fillcolor="#00BFFF93"] ;' '\n' \
        r'12 -> 13 [penwidth=1.250000] ;' '\n' \
        r'12 -> 14 [penwidth=1.250000] ;' '\n' \
        r'13 [label="[0, 0, 0, 0, 0, 0, 1, 0]\n[0, 1]\n[0, 1]\n[1, 0]\n6\n4\n2\n0\n", fillcolor="#00BFFFFF"] ;' '\n' \
        r'14 [label="[0, 0, 0, 0, 0, 0, 0, 1]\n[0, 1]\n[0, 1]\n[0, 1]\n7\n4\n2\n1\n", fillcolor="#00FFC0FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target
    # Export textual format
    t = clf.estimators_[0].export_text()
    t_target = r'0 X[0]<=0.874 [1, 1, 1, 1, 1, 1, 1, 1][4, 4][4, 4][4, 4]; 0->1; 0->8; 1 X[2]<=0.753 [1, 1, 1, 1, 0, 0, 0, 0][4, 0][2, 2][2, 2]; 1->2; 1->5; 2 X[1]<=0.368 [1, 0, 1, 0, 0, 0, 0, 0][2, 0][1, 1][2, 0]; 2->3; 2->4; 3 [1, 0, 0, 0, 0, 0, 0, 0][1, 0][1, 0][1, 0]; 4 [0, 0, 1, 0, 0, 0, 0, 0][1, 0][0, 1][1, 0]; 5 X[1]<=0.31 [0, 1, 0, 1, 0, 0, 0, 0][2, 0][1, 1][0, 2]; 5->6; 5->7; 6 [0, 1, 0, 0, 0, 0, 0, 0][1, 0][1, 0][0, 1]; 7 [0, 0, 0, 1, 0, 0, 0, 0][1, 0][0, 1][0, 1]; 8 X[1]<=0.247 [0, 0, 0, 0, 1, 1, 1, 1][0, 4][2, 2][2, 2]; 8->9; 8->12; 9 X[2]<=0.316 [0, 0, 0, 0, 1, 1, 0, 0][0, 2][2, 0][1, 1]; 9->10; 9->11; 10 [0, 0, 0, 0, 1, 0, 0, 0][0, 1][1, 0][1, 0]; 11 [0, 0, 0, 0, 0, 1, 0, 0][0, 1][1, 0][0, 1]; 12 X[2]<=0.516 [0, 0, 0, 0, 0, 0, 1, 1][0, 2][0, 2][1, 1]; 12->13; 12->14; 13 [0, 0, 0, 0, 0, 0, 1, 0][0, 1][0, 1][1, 0]; 14 [0, 0, 0, 0, 0, 0, 0, 1][0, 1][0, 1][0, 1]; '
    assert t == t_target
    # Persistence
    with open("simple_example_multi_output_dtc.pkl", "wb") as f:
        pickle.dump(clf, f)
    with open("simple_example_multi_output_dtc.pkl", "rb") as f:
        clf2 = pickle.load(f)
    assert clf2.estimators_[0].export_text() == clf.estimators_[0].export_text()
    # Classification
    c = clf2.predict(X_mo)
    for i1, i2 in zip(c.ravel(), y_mo.ravel()):
        assert i1 > i2 - precision and i1 < i2 + precision
    # Testing
    score = clf2.score(X_mo, y_mo)
    assert score > 1.0 - precision and score < 1.0 + precision

# DecisionForestClassifier.fit()
# ==============================

def test_fit():

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [2, 8]; 0->1; 0->4; 1 X[1]<=0.5 [2, 3]; 1->2; 1->3; 2 [2, 0]; 3 [0, 3]; 4 [0, 5]; '
    assert data == data_target

# n_estimators
# ------------

def test_fit_nestimators():

    clf = DecisionForestClassifier(n_estimators=-1, bootstrap=False, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'n_estimators' in str(excinfo.value)

    clf = DecisionForestClassifier(n_estimators=0, bootstrap=False, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'n_estimators' in str(excinfo.value)

    clf = DecisionForestClassifier(n_estimators=1, bootstrap=False, class_balance=None, max_depth=1, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [2, 8]; 0->1; 0->2; 1 [2, 3]; 2 [0, 5]; '
    assert data == data_target

    clf = DecisionForestClassifier(n_estimators=10, bootstrap=False, class_balance=None, max_depth=1, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [2, 8]; 0->1; 0->2; 1 [2, 3]; 2 [0, 5]; '
    assert data == data_target

    data = clf.estimators_[9].export_text()
    data_target = r'0 X[1]<=0.5 [2, 8]; 0->1; 0->2; 1 [2, 4]; 2 [0, 4]; '
    assert data == data_target

    with pytest.raises(IndexError):
        data = clf.estimators_[10].export_text()

    # misc

    clf = DecisionForestClassifier(n_estimators=[], bootstrap=False, random_state=0)
    with pytest.raises(TypeError):
        clf.fit(X, y)

# bootstrap
# ---------

def test_fit_bootstrap(iris):

    # simple example

    # data set it too small for bootstrap = True

    # misc

    clf = DecisionForestClassifier(bootstrap=[], random_state=0)
    with pytest.raises(TypeError):
        clf.fit(X, y)

    clf = DecisionForestClassifier(bootstrap=1, random_state=0)
    with pytest.raises(TypeError):
        clf.fit(X, y)

    clf = DecisionForestClassifier(bootstrap=2, random_state=0)
    with pytest.raises(TypeError):
        clf.fit(X, y)

    clf = DecisionForestClassifier(bootstrap=1.0, random_state=0)
    with pytest.raises(TypeError):
        clf.fit(X, y)

    # iris dataset

    X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=.4, random_state=0)

    clf = DecisionForestClassifier(bootstrap=True,
                                   n_estimators=20,
                                   class_balance='balanced',
                                   max_depth=None,
                                   max_features=None,
                                   max_thresholds=None,
                                   random_state=0)

    clf.set_params(n_jobs=-1)
    clf.fit(X_train, y_train)
    score = clf.score(X_test, y_test)
    assert score > .95 - precision and score < .95 + precision

# bootstrap + oob_score
# ---------------------

def test_fit_bootstrap_oob_score(iris):

    # simple example

    clf = DecisionForestClassifier(bootstrap=False, oob_score=True)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'oob_score: only available if bootstrap=True' in str(excinfo.value)

    clf = DecisionForestClassifier(oob_score=[])
    with pytest.raises(TypeError) as excinfo:
        clf.fit(X, y)
    assert 'oob_score: must be boolean.' in str(excinfo.value)

    clf = DecisionForestClassifier(oob_score='abc')
    with pytest.raises(TypeError) as excinfo:
        clf.fit(X, y)
    assert 'oob_score: must be boolean.' in str(excinfo.value)

    # iris dataset

    X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=.4, random_state=0)

    clf = DecisionForestClassifier(bootstrap=True,
                                   oob_score=True,
                                   n_estimators=1,
                                   class_balance='balanced',
                                   max_depth=None,
                                   max_features=None,
                                   max_thresholds=None,
                                   random_state=0)

    clf.set_params(n_jobs=-1)
    clf.fit(X_train, y_train)
    score = clf.score(X_test, y_test)
    assert score > 0.9333333333333333 - precision and score < 0.9333333333333333 + precision
    score = clf.oob_score_
    assert score < precision

    clf = DecisionForestClassifier(bootstrap=True,
                                   oob_score=True,
                                   n_estimators=25,
                                   class_balance='balanced',
                                   max_depth=None,
                                   max_features=None,
                                   max_thresholds=None,
                                   random_state=0)

    clf.set_params(n_jobs=-1)
    clf.fit(X_train, y_train)
    score = clf.score(X_test, y_test)
    assert score > 0.95 - precision and score < 0.95 + precision
    score = clf.oob_score_
    assert score > 0.9555555555555556 - precision and score < 0.9555555555555556 + precision

# max_depth
# ---------

def test_fit_maxdepth():

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, max_depth='abc', random_state=0)
    with pytest.raises(TypeError):
        clf.fit(X, y)

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, max_depth=-999, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_depth' in str(excinfo.value)

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, max_depth=0, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_depth' in str(excinfo.value)

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, max_depth=1, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [2, 8]; 0->1; 0->2; 1 [2, 3]; 2 [0, 5]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, max_depth=2, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [2, 8]; 0->1; 0->4; 1 X[1]<=0.5 [2, 3]; 1->2; 1->3; 2 [2, 0]; 3 [0, 3]; 4 [0, 5]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, max_depth=999, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [2, 8]; 0->1; 0->4; 1 X[1]<=0.5 [2, 3]; 1->2; 1->3; 2 [2, 0]; 3 [0, 3]; 4 [0, 5]; '
    assert data == data_target

# class_balance
# -------------

def test_fit_classbalance():

    clf = DecisionForestClassifier(bootstrap=False, class_balance=0, random_state=0)
    with pytest.raises(TypeError):
        clf.fit(X, y)

    clf = DecisionForestClassifier(bootstrap=False, class_balance='auto', random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'class_balance' in str(excinfo.value)

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

# max_depth + class_balance
# -------------------------

def test_fit_maxdepth_classbalance():

    clf = DecisionForestClassifier(bootstrap=False, max_depth=2, class_balance='balanced', random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

# max_features
# ------------

def test_fit_maxfeatures():

    clf = DecisionForestClassifier(bootstrap=False, max_features=None, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    # integers

    clf = DecisionForestClassifier(bootstrap=False, max_features=-1, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_features' in str(excinfo.value)

    clf = DecisionForestClassifier(bootstrap=False, max_features=0, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_features' in str(excinfo.value)

    clf = DecisionForestClassifier(bootstrap=False, max_features=1, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, max_features=2, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, max_features=4, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_features' in str(excinfo.value)

    # floats

    clf = DecisionForestClassifier(bootstrap=False, max_features=-1.0, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_features' in str(excinfo.value)

    clf = DecisionForestClassifier(bootstrap=False, max_features=0.0, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_features' in str(excinfo.value)

    clf = DecisionForestClassifier(bootstrap=False, max_features=0.1, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, max_features=0.67, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, max_features=1.0, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, max_features=1.1, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_features' in str(excinfo.value)

    # strings

    clf = DecisionForestClassifier(bootstrap=False, max_features='xxx', random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_features' in str(excinfo.value)

    clf = DecisionForestClassifier(bootstrap=False, max_features='auto', random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, max_features='sqrt', random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, max_features='log2', random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    # misc

    clf = DecisionForestClassifier(bootstrap=False, max_features=[], random_state=0)
    with pytest.raises(TypeError) as excinfo:
        clf.fit(X, y)
    assert 'max_features' in str(excinfo.value)

# max_thresholds
# --------------

def test_fit_maxthresholds():

    # None

    clf = DecisionForestClassifier(bootstrap=False, max_thresholds=None, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [5, 5]; 0->1; 0->4; 1 X[1]<=0.5 [5, 1.88]; 1->2; 1->3; 2 [5, 0]; 3 [0, 1.88]; 4 [0, 3.12]; '
    assert data == data_target

    # integers

    clf = DecisionForestClassifier(bootstrap=False, max_thresholds=0, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_thresholds' in str(excinfo.value)

    clf = DecisionForestClassifier(bootstrap=False, max_thresholds=1, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.874 [5, 5]; 0->1; 0->8; 1 X[2]<=0.753 [5, 1.88]; 1->2; 1->5; 2 X[1]<=0.368 [2.5, 0.62]; 2->3; 2->4; 3 [2.5, 0]; 4 [0, 0.62]; 5 X[1]<=0.31 [2.5, 1.25]; 5->6; 5->7; 6 [2.5, 0]; 7 [0, 1.25]; 8 [0, 3.12]; '
    assert data == data_target

    clf = DecisionForestClassifier(bootstrap=False, max_thresholds=99, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X, y)
    assert 'max_thresholds' in str(excinfo.value)

    # misc

    clf = DecisionForestClassifier(bootstrap=False, max_thresholds=[], random_state=0)
    with pytest.raises(TypeError) as excinfo:
        clf.fit(X, y)
    assert 'max_thresholds' in str(excinfo.value)

# max_features and max_thresholds
# -------------------------------

def test_fit_maxfeatures_maxthresholds():

    # decision tree: max_features=None, max_thresholds=None ... covered before
    # random tree: max_features<n_features, max_thresholds=None ... covered before

    # extreme randomized tree: max_features<n_features, max_thresholds=1

    clf = DecisionForestClassifier(bootstrap=False, max_depth=2, max_features=2, max_thresholds=1, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.874 [5, 5]; 0->1; 0->4; 1 X[2]<=0.563 [5, 1.88]; 1->2; 1->3; 2 [2.5, 0.62]; 3 [2.5, 1.25]; 4 [0, 3.12]; '
    assert data == data_target

    # totally randomized tree: max_features=1, max_thresholds=1

    clf = DecisionForestClassifier(bootstrap=False, max_depth=2, max_features=1, max_thresholds=1, random_state=0)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.874 [5, 5]; 0->1; 0->4; 1 X[2]<=0.753 [5, 1.88]; 1->2; 1->3; 2 [2.5, 0.62]; 3 [2.5, 1.25]; 4 [0, 3.12]; '
    assert data == data_target

# missing_values
# --------------

def test_fit_missingvalues():

    # training

    # - no NaN in y ever

    X_train_mv = np.array([
        [np.NaN],
        [np.NaN]
    ]).astype(np.double)
    y_train_mv = np.array([0, np.NaN])

    clf = DecisionForestClassifier(missing_values=None, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X_train_mv, y_train_mv)
    assert 'NaN' in str(excinfo.value)

    clf = DecisionForestClassifier(missing_values='NMAR', random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X_train_mv, y_train_mv)
    assert 'NaN' in str(excinfo.value)

    # - no NaN in X when missing values None

    y_train_mv = np.array([0, 1])

    clf = DecisionForestClassifier(missing_values=None, random_state=0)
    with pytest.raises(ValueError) as excinfo:
        clf.fit(X_train_mv, y_train_mv)
    assert 'NaN' in str(excinfo.value)

    # - only NaN(s)

    y_train_mv = np.array([0, 1])
    clf = DecisionForestClassifier(missing_values='NMAR', random_state=0)
    clf.fit(X_train_mv, y_train_mv)

    data = clf.estimators_[0].export_text()
    data_target = r'0 [1, 1]; '
    assert data == data_target

    # - 1 value : 0, 1 NaN : 1

    X_train_mv = np.array([
        [0],
        [np.NaN]
    ]).astype(np.double)
    y_train_mv = np.array([0, 1])

    clf = DecisionForestClassifier(missing_values='NMAR', random_state=0)
    clf.fit(X_train_mv, y_train_mv)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0] NA [1, 1]; 0->1; 0->2; 1 [0, 1]; 2 [1, 0]; '
    assert data == data_target

    # - 1 value : 0, 1 value and 1 NaN : 1

    X_train_mv = np.array([
        [0],
        [1],
        [np.NaN]
    ]).astype(np.double)
    y_train_mv = np.array([0, 1, 1])

    clf = DecisionForestClassifier(missing_values='NMAR', random_state=0)
    clf.fit(X_train_mv, y_train_mv)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 not NA [1.5, 1.5]; 0->1; 0->2; 1 [1.5, 0]; 2 [0, 1.5]; '
    assert data == data_target

    # testing

    # - simple dataset - no NaN(s) in training, all 1s are NaN(s) in testing

    X_train_mv = np.array([
        [0, 0, 0],
        [0, 0, 1],
        [0, 1, 0],
        [0, 1, 1],
        [1, 0, 0],
        [1, 0, 1],
        [1, 1, 0],
        [1, 1, 1]
    ]).astype(np.double)
    y_train_mv = np.array([0, 1, 2, 3, 4, 5, 6, 7])

    X_test_mv = np.array([
        [0, 0, 0],
        [0, 0, np.NaN],
        [0, np.NaN, 0],
        [0, np.NaN, np.NaN],
        [np.NaN, 0, 0],
        [np.NaN, 0, np.NaN],
        [np.NaN, np.NaN, 0],
        [np.NaN, np.NaN, np.NaN]
    ]).astype(np.double)

    clf = DecisionForestClassifier(missing_values='NMAR', random_state=0)
    clf.fit(X_train_mv, y_train_mv)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [1, 1, 1, 1, 1, 1, 1, 1]; 0->1; 0->8; 1 X[1]<=0.5 [1, 1, 1, 1, 0, 0, 0, 0]; 1->2; 1->5; 2 X[2]<=0.5 [1, 1, 0, 0, 0, 0, 0, 0]; 2->3; 2->4; 3 [1, 0, 0, 0, 0, 0, 0, 0]; 4 [0, 1, 0, 0, 0, 0, 0, 0]; 5 X[2]<=0.5 [0, 0, 1, 1, 0, 0, 0, 0]; 5->6; 5->7; 6 [0, 0, 1, 0, 0, 0, 0, 0]; 7 [0, 0, 0, 1, 0, 0, 0, 0]; 8 X[2]<=0.5 [0, 0, 0, 0, 1, 1, 1, 1]; 8->9; 8->12; 9 X[1]<=0.5 [0, 0, 0, 0, 1, 0, 1, 0]; 9->10; 9->11; 10 [0, 0, 0, 0, 1, 0, 0, 0]; 11 [0, 0, 0, 0, 0, 0, 1, 0]; 12 X[1]<=0.5 [0, 0, 0, 0, 0, 1, 0, 1]; 12->13; 12->14; 13 [0, 0, 0, 0, 0, 1, 0, 0]; 14 [0, 0, 0, 0, 0, 0, 0, 1]; '
    assert data == data_target

    predict_proba = clf.predict_proba(X_test_mv)
    predict_proba_target = [
         [ 1.,     0.,     0.,     0.,     0.,     0.,     0.,     0.   ],
         [ 0.5,    0.5,    0.,     0.,     0.,     0.,     0.,     0.   ],
         [ 0.5,    0.,     0.5,    0.,     0.,     0.,     0.,     0.   ],
         [ 0.25,   0.25,   0.25,   0.25,   0.,     0.,     0.,     0.   ],
         [ 0.5,    0.,     0.,     0.,     0.5,    0.,     0.,     0.   ],
         [ 0.25,   0.25,   0.,     0.,     0.25,   0.25,   0.,     0.   ],
         [ 0.25,   0.,     0.25,   0.,     0.25,   0.,     0.25,   0.   ],
         [ 0.125,  0.125,  0.125,  0.125,  0.125,  0.125,  0.125,  0.125]
    ]

    for a, b in zip(predict_proba, predict_proba_target):
        for ai, bi in zip(a, b):
            assert ai > bi - precision and ai < bi + precision

    # - simple dataset - NaN(s) in training replacing some 1s, all 1s are NaN(s) in testing

    X_train_mv = np.array([
        [0, 0, 0],
        [0, 0, np.NaN],
        [0, np.NaN, 0],
        [0, np.NaN, np.NaN],
        [1, 0, 0],
        [1, 0, 1],
        [1, 1, 0],
        [1, 1, 1]
    ]).astype(np.double)
    y_train_mv = np.array([0, 1, 2, 3, 4, 5, 6, 7])

    X_test_mv = np.array([
        [0, 0, 0],
        [0, 0, np.NaN],
        [0, np.NaN, 0],
        [0, np.NaN, np.NaN],
        [np.NaN, 0, 0],
        [np.NaN, 0, np.NaN],
        [np.NaN, np.NaN, 0],
        [np.NaN, np.NaN, np.NaN]
    ]).astype(np.double)

    clf = DecisionForestClassifier(missing_values='NMAR', random_state=0)
    clf.fit(X_train_mv, y_train_mv)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[0]<=0.5 [1, 1, 1, 1, 1, 1, 1, 1]; 0->1; 0->8; 1 X[1] NA [1, 1, 1, 1, 0, 0, 0, 0]; 1->2; 1->5; 2 X[2] NA [0, 0, 1, 1, 0, 0, 0, 0]; 2->3; 2->4; 3 [0, 0, 0, 1, 0, 0, 0, 0]; 4 [0, 0, 1, 0, 0, 0, 0, 0]; 5 X[2] NA [1, 1, 0, 0, 0, 0, 0, 0]; 5->6; 5->7; 6 [0, 1, 0, 0, 0, 0, 0, 0]; 7 [1, 0, 0, 0, 0, 0, 0, 0]; 8 X[2]<=0.5 [0, 0, 0, 0, 1, 1, 1, 1]; 8->9; 8->12; 9 X[1]<=0.5 [0, 0, 0, 0, 1, 0, 1, 0]; 9->10; 9->11; 10 [0, 0, 0, 0, 1, 0, 0, 0]; 11 [0, 0, 0, 0, 0, 0, 1, 0]; 12 X[1]<=0.5 [0, 0, 0, 0, 0, 1, 0, 1]; 12->13; 12->14; 13 [0, 0, 0, 0, 0, 1, 0, 0]; 14 [0, 0, 0, 0, 0, 0, 0, 1]; '
    assert data == data_target

    predict_proba = clf.predict_proba(X_test_mv)
    predict_proba_target = [
        [0.75,       0.04333333, 0.04333333, 0.,         0.1,        0.02666667,  0.03666667, 0.        ],
        [0.04333333, 0.93333333, 0.,         0.,         0.02333333, 0.,          0.,         0.        ],
        [0.04333333, 0.,         0.94333333, 0.,         0.01333333, 0.,          0.,         0.        ],
        [0.,         0.,         0.,         1.,         0.,         0.,          0.,         0.        ],
        [0.37333333, 0.03333333, 0.02833333, 0.,         0.37333333, 0.08666667,  0.105,      0.        ],
        [0.03333333, 0.65833333, 0.,         0.,         0.1525,     0.12416667,  0.01833333, 0.01333333],
        [0.02833333, 0.,         0.69666667, 0.,         0.13583333, 0.01333333,  0.1175,     0.00833333],
        [0.,         0.,         0.,         0.855 ,     0.03625,    0.03625,     0.03625,    0.03625   ]
    ]
    for a, b in zip(predict_proba, predict_proba_target):
        for ai, bi in zip(a, b):
            assert ai > bi - precision and ai < bi + precision

# random_state
# ------------

def test_fit_randomstate():

    # integers

    clf = DecisionForestClassifier(bootstrap=False, max_features='auto', random_state=-1)
    with pytest.raises(OverflowError) as excinfo:
        clf.fit(X, y)

    clf = DecisionForestClassifier(bootstrap=False, max_depth=2, max_features=1, max_thresholds=1, random_state=999)
    clf.fit(X, y)

    data = clf.estimators_[0].export_text()
    data_target = r'0 X[2]<=0.929 [5, 5]; 0->1; 0->4; 1 X[1]<=0.249 [2.5, 3.12]; 1->2; 1->3; 2 [2.5, 2.5]; 3 [0, 0.62]; 4 X[1]<=0.21 [2.5, 1.88]; 4->5; 4->6; 5 [2.5, 0]; 6 [0, 1.88]; '
    assert data == data_target

    # misc

    clf = DecisionForestClassifier(bootstrap=False, max_features='auto', random_state=[])
    with pytest.raises(TypeError) as excinfo:
        clf.fit(X, y)

# DecisionForestClassifier.predict_proba()
# ========================================

def test_predict_proba():

    clf = DecisionForestClassifier(bootstrap=False, random_state=0)
    clf.fit(X, y)

    predict_proba = clf.predict_proba(X_test)
    predict_proba_target = [
        [1., 0.],
        [1., 0.],
        [0., 1.],
        [0., 1.],
        [0., 1.],
        [0.38, 0.62],
        [0., 1.],
        [0., 1.]
    ]

    for a, b in zip(predict_proba, predict_proba_target):
        for ai, bi in zip(a, b):
            assert ai > bi - precision and ai < bi + precision

    # not fitted

    clf = DecisionForestClassifier(bootstrap=False, random_state=0)
    with pytest.raises(NotFittedError):
        predict_proba = clf.predict_proba(X_test)

# class_balance
# -------------

def test_predict_proba_classbalance():

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)

    predict_proba = clf.predict_proba(X_test)
    predict_proba_target = [
        [1., 0.],
        [1., 0.],
        [0., 1.],
        [0., 1.],
        [0., 1.],
        [0.38, 0.62],
        [0., 1.],
        [0., 1.]
    ]

    for a, b in zip(predict_proba, predict_proba_target):
        for ai, bi in zip(a, b):
            assert ai > bi - precision and ai < bi + precision

# DecisionForestClassifier.predict()
# ==================================

def test_predict():

    clf = DecisionForestClassifier(bootstrap=False, random_state=0)
    clf.fit(X, y)

    predict = clf.predict(X_test)
    predict_target = [0, 0, 1, 1, 1, 1, 1, 1]

    for a, b in zip(predict, predict_target):
        assert a > b - precision and a < b + precision

    # not fitted

    clf = DecisionForestClassifier(bootstrap=False, random_state=0)
    with pytest.raises(NotFittedError):
        predict = clf.predict(X_test)

# class_balance
# -------------

def test_predict_classbalance():

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)

    predict = clf.predict(X_test)
    predict_target = [0, 0, 1, 1, 1, 1, 1, 1]

    for a, b in zip(predict, predict_target):
        assert a > b - precision and a < b + precision

# DecisionForestClassifier.feature_importances_
# =============================================

def test_feature_importances():

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, random_state=0)
    clf.fit(X, y)
    feature_importances = clf.feature_importances_
    feature_importances_target = [0.46291667, 0.46802083, 0.0690625]
    for a, b in zip(feature_importances, feature_importances_target):
        assert a > b - precision

    # not fitted

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, random_state=0)
    with pytest.raises(NotFittedError):
        feature_importances = clf.feature_importances_

# class_balance
# -------------

def test_feature_importances_classbalance():

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)
    feature_importances = clf.feature_importances_
    feature_importances_target = [0.51302453, 0.44893362, 0.03804185]
    for a, b in zip(feature_importances, feature_importances_target):
        assert a > b - precision

# DecisionForestClassifier.estimators_[i].export_graphviz()
# ========================================================

def test_export_graphviz():

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)

    dot_data = clf.estimators_[0].export_graphviz()
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'0 [label="X[0] <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="X[1] <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\n0", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target

# feature_names
# -------------

def test_export_graphviz_featurenames():

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)

    with pytest.raises(TypeError):
        dot_data = clf.estimators_[0].export_graphviz(feature_names=0)

    with pytest.raises(IndexError):
        dot_data = clf.estimators_[0].export_graphviz(feature_names=[ ])

    with pytest.raises(IndexError):
        dot_data = clf.estimators_[0].export_graphviz(feature_names=["f1"])

    dot_data = clf.estimators_[0].export_graphviz(feature_names=["f1", "f2", "f3"])
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'0 [label="f1 <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="f2 <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\n0", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target

    dot_data = clf.estimators_[0].export_graphviz(feature_names=["f1", "f2", "f3", "f4"])
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'0 [label="f1 <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="f2 <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\n0", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target

# class_names
# -----------

def test_export_graphviz_classnames():

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)

    with pytest.raises(TypeError):
        dot_data = clf.estimators_[0].export_graphviz(class_names=0)

    with pytest.raises(IndexError):
        dot_data = clf.estimators_[0].export_graphviz(class_names=[ ])

    with pytest.raises(IndexError):
        dot_data = clf.estimators_[0].export_graphviz(class_names=['A'])

    dot_data = clf.estimators_[0].export_graphviz(class_names=['A', 'B'])
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'0 [label="X[0] <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="X[1] <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\nA", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target

    dot_data = clf.estimators_[0].export_graphviz(class_names=['A', 'B', 'C'])
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'0 [label="X[0] <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="X[1] <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\nA", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target

# rotate
# ------

def test_export_graphviz_rotate():

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)

    dot_data = clf.estimators_[0].export_graphviz(rotate=True)
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'rankdir=LR ;' '\n' \
        r'0 [label="X[0] <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=-45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="X[1] <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\n0", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target

    dot_data = clf.estimators_[0].export_graphviz(rotate=False)
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'0 [label="X[0] <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="X[1] <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\n0", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\n1", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target

# feature_names + class_names
# ---------------------------

def test_export_graphviz_featurenames_classnames():

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)
    dot_data = clf.estimators_[0].export_graphviz(feature_names=["f1", "f2", "f3"],
                                                  class_names=['A', 'B'])
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'0 [label="f1 <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="f2 <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\nA", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target

# feature_names + class_names + rotate
# ------------------------------------

def test_export_graphviz_featurenames_classnames_rotate():

    clf = DecisionForestClassifier(bootstrap=False, class_balance='balanced', random_state=0)
    clf.fit(X, y)
    dot_data = clf.estimators_[0].export_graphviz(feature_names=["f1", "f2", "f3"],
                                                  class_names=['A', 'B'],
                                                  rotate=True)
    dot_data_target = \
        r'digraph Tree {' '\n' \
        r'node [shape=box, style="rounded, filled", color="black", fontname=helvetica, fontsize=14] ;' '\n' \
        r'edge [fontname=helvetica, fontsize=12] ;' '\n' \
        r'rankdir=LR ;' '\n' \
        r'0 [label="f1 <= 0.5\np(class) = [0.5, 0.5]\nclass, n = 10", fillcolor="#FF000000"] ;' '\n' \
        r'0 -> 1 [penwidth=6.875000, headlabel="True", labeldistance=2.5, labelangle=-45] ;' '\n' \
        r'0 -> 4 [penwidth=3.125000] ;' '\n' \
        r'1 [label="f2 <= 0.5\n[0.73, 0.27]", fillcolor="#FF000034"] ;' '\n' \
        r'1 -> 2 [penwidth=5.000000] ;' '\n' \
        r'1 -> 3 [penwidth=1.875000] ;' '\n' \
        r'2 [label="[1, 0]\nA", fillcolor="#FF0000FF"] ;' '\n' \
        r'3 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'4 [label="[0, 1]\nB", fillcolor="#00FF00FF"] ;' '\n' \
        r'}'
    assert dot_data == dot_data_target

# Extreme Data
# ============

# Empty X, y training data
# ------------------------

def test_empty_Xy_train():

    X_train = np.array([]).astype(np.double).reshape(1, -1)
    y_train = np.array([])

    clf = DecisionForestClassifier(bootstrap=False, random_state=0)
    with pytest.raises(ValueError):
        clf.fit(X_train, y_train)

# 1 X, y training data
# --------------------

def test_1_Xy_train():

    X_train = np.array([[0, 0, 0]]).astype(np.double).reshape(1, -1)
    y_train = np.array([0])

    clf = DecisionForestClassifier(bootstrap=False, random_state=0)
    clf.fit(X_train, y_train)

    data = clf.estimators_[0].export_text()
    data_target = r'0 [1]; '
    assert data == data_target

    X_test = np.array([[1, 1, 1]]).astype(np.double).reshape(1, -1)
    predict = clf.predict(X_test)
    predict_target = [0]
    for a, b in zip(predict, predict_target):
        assert a > b - precision and a < b + precision

# All X = 0 training data
# -----------------------

def test_X_0_train():

    X_train = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]]).astype(np.double)
    y_train = np.array([0, 1, 1])

    clf = DecisionForestClassifier(bootstrap=False, class_balance=None, random_state=0)
    clf.fit(X_train, y_train)

    data = clf.estimators_[0].export_text()
    data_target = r'0 [1, 2]; '
    assert data == data_target

    predict = clf.predict(X_train)
    predict_target = [1, 1, 1]
    for a, b in zip(predict, predict_target):
        assert a > b - precision and a < b + precision

# All y = 0 training data
# -----------------------

def test_y_0_train():

    X_train = np.array([[0, 0, 0], [0, 0, 1], [0, 1, 0]]).astype(np.double)
    y_train = np.array([0, 0, 0])

    clf = DecisionForestClassifier(bootstrap=False, random_state=0)
    clf.fit(X_train, y_train)

    data = clf.estimators_[0].export_text()
    data_target = r'0 [3]; '
    assert data == data_target

    predict = clf.predict(X_train)
    predict_target = [0, 0, 0]
    for a, b in zip(predict, predict_target):
        assert a > b - precision and a < b + precision

# Number of classes very large
# ----------------------------
# code coverage for duplication of offset_list in create_rgb_LUT in export_graphviz( )

def test_numberclasses_large():

    n_classes = 97 # max number of colors = 96
    X_train = np.array(range(n_classes)).astype(np.double).reshape(-1,1)
    y_train = np.array(range(n_classes))

    clf = DecisionForestClassifier(bootstrap=False, random_state=0)
    clf.fit(X_train, y_train)
    dot_data = clf.estimators_[0].export_graphviz()
    # no error raised

# Mismatch number of features
# ---------------------------

def test_mismatch_nfeatures():

    X_train = np.array([[0, 0, 0], [0, 0, 1], [0, 1, 0]]).astype(np.double)
    y_train = np.array([0, 1, 2])

    X_test = np.array([[0, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0]]).astype(np.double)
    y_test = np.array([0, 1, 2])

    clf = DecisionForestClassifier(bootstrap=False, random_state=0)
    clf.fit(X_train, y_train)

    with pytest.raises(ValueError) as excinfo:
        predict = clf.predict(X_test)
    assert 'number of features' in str(excinfo.value)

    with pytest.raises(ValueError) as excinfo:
        predict_proba = clf.predict_proba(X_test)
    assert 'number of features' in str(excinfo.value)

ext: py

max_stars_repo_name: AIWerkstatt/koho

max_issues_repo_name: AIWerkstatt/koho

max_forks_repo_name: AIWerkstatt/koho

reason: import numpy


text:
import numpy as np

from neurons.activations import get_activation

class Neuron:
    def __init__(self, number_of_inputs, weights_range, bias, activation):
        self.number_of_inputs = number_of_inputs
        self.weights = np.random.uniform(*weights_range, number_of_inputs)
        self.bias = bias
        self.activation = get_activation(activation)

    def predict(self, features):
        return self.activation(self.raw_output(features))

    def raw_output(self, features):
        features_weights_product = self.bias
        for i in range(self.number_of_inputs):
            features_weights_product += self.weights[i] * features[i]
        return features_weights_product

    @property
    def get_number_of_inputs(self):
        return self.number_of_inputs

    @property
    def get_activation(self):
        return self.activation

max_stars_repo_name: PiotrGrzybowski/NeuralNetworks

max_issues_repo_name: PiotrGrzybowski/NeuralNetworks

max_forks_repo_name: PiotrGrzybowski/NeuralNetworks