text:
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
# Copyright (c) 2020 Ross Wightman
# This file has been modified by Megvii ("Megvii Modifications").
# All Megvii Modifications are Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
import functools
import itertools
import math
import warnings
from typing import List, Sequence, Tuple, Union

import megengine as mge
import megengine.distributed as dist
import megengine.functional as F
import megengine.module as M
import numpy as np
from basecore.config import ConfigDict
from basecore.utils import all_reduce
from megengine.module.batchnorm import _BatchNorm
from megengine.utils.module_stats import register_flops

from basecls.data import DataLoaderType

__all__ = [
    "NORM_TYPES",
    "Preprocess",
    "adjust_block_compatibility",
    "calculate_fan_in_and_fan_out",
    "compute_precise_bn_stats",
    "init_weights",
    "init_vit_weights",
    "trunc_normal_",
    "lecun_normal_",
    "make_divisible",
]

NORM_TYPES = (_BatchNorm, M.GroupNorm, M.InstanceNorm, M.LayerNorm)

class Preprocess(M.Module):
    def __init__(self, mean: Union[float, Sequence[float]], std: Union[float, Sequence[float]]):
        super().__init__()
        self.mean = mge.Tensor(np.array(mean, dtype=np.float32).reshape(1, -1, 1, 1))
        self.std = mge.Tensor(np.array(std, dtype=np.float32).reshape(1, -1, 1, 1))

    def forward(self, inputs: Sequence[np.ndarray]) -> Tuple[mge.Tensor, mge.Tensor]:
        samples, targets = [mge.Tensor(x) for x in inputs]
        samples = (samples - self.mean) / self.std
        return samples, targets

def adjust_block_compatibility(
    ws: Sequence[int], bs: Sequence[float], gs: Sequence[int]
) -> Tuple[List[int], ...]:
    """Adjusts the compatibility of widths, bottlenecks and groups.

    Args:
        ws: widths.
        bs: bottleneck multipliers.
        gs: group widths.

    Returns:
        The adjusted widths, bottlenecks and groups.
    """
    assert len(ws) == len(bs) == len(gs)
    assert all(w > 0 and b > 0 and g > 0 for w, b, g in zip(ws, bs, gs))
    assert all(b < 1 or b % 1 == 0 for b in bs)
    vs = [int(max(1, w * b)) for w, b in zip(ws, bs)]
    gs = [int(min(g, v)) for g, v in zip(gs, vs)]
    ms = [np.lcm(g, int(b)) if b > 1 else g for g, b in zip(gs, bs)]
    vs = [max(m, int(round(v / m) * m)) for v, m in zip(vs, ms)]
    ws = [int(v / b) for v, b in zip(vs, bs)]
    assert all(w * b % g == 0 for w, b, g in zip(ws, bs, gs))
    return ws, bs, gs

def calculate_fan_in_and_fan_out(tensor: mge.Tensor, pytorch_style: bool = False):
    """Fixed :py:func:`megengine.module.init.calculate_fan_in_and_fan_out` for group conv2d.

    Note:
        The group conv2d kernel shape in MegEngine is ``(G, O/G, I/G, K, K)``. This function
        calculates ``fan_out = O/G * K * K`` as default, but PyTorch uses ``fan_out = O * K * K``.

    Args:
        tensor: tensor to be initialized.
        pytorch_style: utilize pytorch style init for group conv. Default: ``False``
    """
    if len(tensor.shape) not in (2, 4, 5):
        raise ValueError(
            "fan_in and fan_out can only be computed for tensor with 2/4/5 " "dimensions"
        )
    if len(tensor.shape) == 5:
        # `GOIKK` to `OIKK`
        tensor = tensor.reshape(-1, *tensor.shape[2:]) if pytorch_style else tensor[0]

    num_input_fmaps = tensor.shape[1]
    num_output_fmaps = tensor.shape[0]
    receptive_field_size = 1
    if len(tensor.shape) > 2:
        receptive_field_size = functools.reduce(lambda x, y: x * y, tensor.shape[2:], 1)
    fan_in = num_input_fmaps * receptive_field_size
    fan_out = num_output_fmaps * receptive_field_size
    return fan_in, fan_out

def compute_precise_bn_stats(cfg: ConfigDict, model: M.Module, dataloader: DataLoaderType):
    """Computes precise BN stats on training data.

    References: https://github.com/facebookresearch/pycls/blob/main/pycls/core/net.py

    Args:
        cfg: config for precising BN.
        model: model for precising BN.
        dataloader: dataloader for precising BN.
    """
    # Prepare the preprocessor
    preprocess = Preprocess(cfg.preprocess.img_mean, cfg.preprocess.img_std)
    # Compute the number of minibatches to use
    num_iter = int(cfg.bn.num_samples_precise / cfg.batch_size / dist.get_world_size())
    num_iter = min(num_iter, len(dataloader))
    # Retrieve the BN layers
    bns = [m for m in model.modules() if isinstance(m, _BatchNorm)]
    # Initialize BN stats storage for computing mean(mean(batch)) and mean(var(batch))
    running_means = [F.zeros_like(bn.running_mean) for bn in bns]
    running_vars = [F.zeros_like(bn.running_var) for bn in bns]
    # Remember momentum values
    momentums = [bn.momentum for bn in bns]
    # Set momentum to 0.0 to compute BN stats that only reflect the current batch
    for bn in bns:
        bn.momentum = 0.0
    # Average the BN stats for each BN layer over the batches
    for data in itertools.islice(dataloader, num_iter):
        samples, _ = preprocess(data)
        model(samples)
        for i, bn in enumerate(bns):
            running_means[i] += bn.running_mean / num_iter
            running_vars[i] += bn.running_var / num_iter
    # Sync BN stats across GPUs (no reduction if 1 GPU used)
    running_means = [all_reduce(x, mode="mean") for x in running_means]
    running_vars = [all_reduce(x, mode="mean") for x in running_vars]
    # Set BN stats and restore original momentum values
    for i, bn in enumerate(bns):
        bn.running_mean = running_means[i]
        bn.running_var = running_vars[i]
        bn.momentum = momentums[i]

def init_weights(m: M.Module, pytorch_style: bool = False, zero_init_final_gamma: bool = False):
    """Performs ResNet-style weight initialization.

    About zero-initialize:
    Zero-initialize the last BN in each residual branch, so that the residual branch starts
    with zeros, and each residual block behaves like an identity.
    This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677.

    References: https://github.com/facebookresearch/pycls/blob/main/pycls/models/blocks.py

    Args:
        m: module to be initialized.
        pytorch_style: utilize pytorch style init for group conv. Default: ``False``
        zero_init_final_gamma: enable zero-initialize or not. Default: ``False``
    """

    if isinstance(m, M.Conv2d):
        _, fan_out = calculate_fan_in_and_fan_out(m.weight, pytorch_style)
        std = math.sqrt(2 / fan_out)
        M.init.normal_(m.weight, 0, std)
        if getattr(m, "bias", None) is not None:
            fan_in, _ = calculate_fan_in_and_fan_out(m.weight, pytorch_style)
            bound = 1 / math.sqrt(fan_in)
            M.init.uniform_(m.bias, -bound, bound)
    elif isinstance(m, NORM_TYPES):
        M.init.fill_(
            m.weight, 0.0 if getattr(m, "final_bn", False) and zero_init_final_gamma else 1.0
        )
        M.init.zeros_(m.bias)
    elif isinstance(m, M.Linear):
        M.init.normal_(m.weight, std=0.01)
        if getattr(m, "bias", None) is not None:
            M.init.zeros_(m.bias)

def init_vit_weights(module: M.Module):
    """Initialization for Vision Transformer (ViT).

    References:
    https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py

    Args:
        m: module to be initialized.
    """
    if isinstance(module, M.Linear):
        if module.name and module.name.startswith("head"):
            M.init.zeros_(module.weight)
            M.init.zeros_(module.bias)
        elif module.name and module.name.startswith("pre_logits"):
            lecun_normal_(module.weight)
            M.init.zeros_(module.bias)
        else:
            trunc_normal_(module.weight, std=0.02)
            if module.bias is not None:
                M.init.zeros_(module.bias)
    elif isinstance(module, M.Conv2d):
        M.init.msra_uniform_(module.weight, a=math.sqrt(5))
        if module.bias is not None:
            fan_in, _ = M.init.calculate_fan_in_and_fan_out(module.weight)
            bound = 1 / math.sqrt(fan_in)
            M.init.uniform_(module.bias, -bound, bound)
    elif isinstance(module, (M.LayerNorm, M.GroupNorm, M.BatchNorm2d)):
        M.init.zeros_(module.bias)
        M.init.ones_(module.weight)

def trunc_normal_(tensor, mean=0.0, std=1.0, a=-2.0, b=2.0):
    # Cut & paste from PyTorch official master until it's in a few official releases - RW
    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
    def norm_cdf(x):
        # Computes standard normal cumulative distribution function
        return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0

    if (mean < a - 2 * std) or (mean > b + 2 * std):
        warnings.warn(
            "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
            "The distribution of values may be incorrect.",
            stacklevel=2,
        )

    # Values are generated by using a truncated uniform distribution and
    # then using the inverse CDF for the normal distribution.
    # Get upper and lower cdf values
    lo = norm_cdf((a - mean) / std)
    up = norm_cdf((b - mean) / std)

    # Uniformly fill tensor with values from [l, u], then translate to
    # [2l-1, 2u-1].
    M.init.uniform_(tensor, 2 * lo - 1, 2 * up - 1)
    # tensor.uniform_(2 * l - 1, 2 * u - 1)

    # Use inverse cdf transform for normal distribution to get truncated
    # standard normal
    tensor._reset(M.Elemwise("erfinv")(tensor))
    # tensor.erfinv_()

    # Transform to proper mean, std
    tensor *= std * math.sqrt(2.0)
    # tensor.mul_(std * math.sqrt(2.))
    tensor += mean
    # tensor.add_(mean)

    # Clamp to ensure it's in the proper range
    tensor._reset(F.clip(tensor, lower=a, upper=b))
    # tensor.clamp_(min=a, max=b)
    return tensor

def lecun_normal_(tensor):
    fan_in, _ = calculate_fan_in_and_fan_out(tensor)
    std = 1 / math.sqrt(fan_in) / 0.87962566103423978
    # constant is stddev of standard normal truncated to (-2, 2)
    trunc_normal_(tensor, std=std)

def make_divisible(value, divisor: int = 8, min_value: int = None, round_limit: float = 0.0):
    min_value = min_value or divisor
    new_value = max(min_value, int(value + divisor / 2) // divisor * divisor)
    if new_value < round_limit * value:
        new_value += divisor
    return new_value

# FIXME: adaptive pool on 1x1 feature map lead to div_zero error
@register_flops(M.AdaptiveAvgPool2d, M.AdaptiveMaxPool2d)
def flops_adaptivePool(module: M.AdaptiveAvgPool2d, inputs, outputs):
    stride_h = (
        np.floor(inputs[0].shape[2] / (inputs[0].shape[2] - 1)) if (inputs[0].shape[2] - 1) else 0
    )
    kernel_h = inputs[0].shape[2] - (inputs[0].shape[2] - 1) * stride_h
    stride_w = (
        np.floor(inputs[0].shape[3] / (inputs[0].shape[3] - 1)) if (inputs[0].shape[3] - 1) else 0
    )
    kernel_w = inputs[0].shape[3] - (inputs[0].shape[3] - 1) * stride_w
    return np.prod(outputs[0].shape) * kernel_h * kernel_w

max_stars_repo_name: megvii-research/basecls

max_issues_repo_name: megvii-research/basecls

max_forks_repo_name: megvii-research/basecls


text:
import sys, time
import numpy as np
import navio.adc
import navio.util

navio.util.check_apm()
adc = navio.adc.ADC()
data = np.zeros(1000)

start_time = time.clock()*1000.0
for i in range(0,1000):
	analog_val = adc.read(4)*0.001
	data[i] = analog_val
	print analog_val
	time.sleep(0.25)
end_time = time.clock()*1000.0
average = np.mean(data)
print end_time - start_time, average

ext: py

max_stars_repo_name: EvanHope/SeniorProject

max_issues_repo_name: EvanHope/SeniorProject

max_forks_repo_name: EvanHope/SeniorProject

reason: import numpy


text:
# import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# read csv into pandas' dataframe
CSV_DATA = pd.read_csv('ori.csv', names=["YEAR", "VALUE"])
CSV_DATA_drop = CSV_DATA.drop([0])
print(CSV_DATA_drop)

# store column items in lists¶
CSV_YEAR = [int(item) for item in CSV_DATA_drop.YEAR.to_list()]
#print(CSV_YEAR)
CSV_VALUE = [float(item) for item in CSV_DATA_drop.VALUE.to_list()]
#print(CSV_VALUE)

# nominalization (let value[0] = 0)
print("CSV_YEAR = ", CSV_YEAR)
print("\n")
NOMINAL_VALUE = [round(item-CSV_VALUE[0], 2) for item in CSV_VALUE]
print("NOMINAL_VALUE = ", NOMINAL_VALUE)

# create new lists for final data sets
Year = []
Month = []
Year_Month = []
Value = []

# fill in missing data
i = 0
j = [i for i in np.arange(1, 469, 1)] # 468 = (2019-1980)*12; why add 1?: np.arange(1, 3, 1) => 1,2
for year in CSV_YEAR:
	if year >= CSV_YEAR[0] and year <= CSV_YEAR[-2]: # year: 1980-2018(here)
		i += 1
		value_init1 = NOMINAL_VALUE[i-1]
		value_init2 = NOMINAL_VALUE[i]
		value_difference = (value_init2 - value_init1)
		unit_difference = round(value_difference/12,3)
		print("Year", str(year), "~", str(year+1), ":")
		print(str(value_init1), "=>", str(value_init2), "(nominalized) ;")
		print("Total difference:", str(value_difference), ";")
		print("Twelve months' average difference:", str(unit_difference), ". \n")
		for month in np.arange(1, 13, 1): # month: 1~12
			Year.append(year) # year: 1980-2018(here)
			Month.append(int(month))
			Year_Month.append(str(year)+"-"+str(month)+"-01")
			if month == 1:
				Value.append(value_init1)
			elif month >= 2 and month <= 12:
				append_value = round(value_init1 + unit_difference*(month-1), 3)
				Value.append(append_value)

#print(Year)
#print(Month)
print(Year_Month)
print(Value)

#print(len(Year))
#print(len(Month))
#print(len(Value))

plt.figure(figsize=(20,10))
plt.plot(j, Value)

# export to a new csv
df = pd.DataFrame.from_dict({'Year':Year, 'Month':Month, 'Year-Month':Year_Month, 'Value': Value})
df.to_csv('tidy_Value.csv', header=True, index=False)


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np
#from pointnet2_cls_msg import PointNet2
#from point_augment import Augmentor
import utils.train_utils
from models.model import ClassificationPointNet
from models.dgcnn import DGCNN_cls
from models.fgsm import fgsm_attack
import os
import tqdm
import logging
from tensorboardX import SummaryWriter
from torch.utils.data import DataLoader
from PointAugment.Common import data_utils as d_utils
import random
from PointAugment.Common import loss_utils
from PointAugment.Common.ModelNetDataLoader import ModelNetDataLoader
from PointAugment.Augment.pointnet import PointNetCls
from PointAugment.Augment.augmentor import Augmentor
import sklearn.metrics as metrics
debug = True

def inplace_relu(m):
    classname = m.__class__.__name__
    if classname.find('ReLU') != -1:
        m.inplace=True

class PointNetTrain():
    def __init__(self, model_config, device):

        self._device          = device
        self._categories     = model_config['NUM_CLASSES']
        self._model           = DGCNN_cls(num_classes = self._categories).to(self._device) #PointNetCls(self._categories).to(self._device) #PointNetClassifier(num_classes = model_config['TRAINING']['NUM_CLASSES']).to(self._device)
        self._epochs          = model_config['TRAINING']['EPOCHS']
        self._criterion_name  = model_config['TRAINING']['CRITERION']
        self._optimizer_name  = model_config['TRAINING']['OPTIMIZER']
        self._lr              = model_config['TRAINING']['LR']
        self._augmentor_name  = model_config['AUGMENTOR']['NAME']
        self._alpha = model_config['AUGMENTOR']['ALPHA']
        self._augmentor = None
        self._path = model_config['TRAINING']['PATH']

        #self.weight = weights.to(self._device).float()

        if(debug == True):
            print("Device: {}".format(self._device))
            print("Epochs: {}".format(self._epochs))
            print("Criterion: {}".format(self._criterion_name))
            print("Optimizer: {}".format(self._optimizer_name))
            print("Learning Rate: {}".format(self._lr))

        if(self._augmentor_name == 'FGSM'):

            self._eps = model_config['AUGMENTOR']['EPS']
            self._augmentor = fgsm_attack
        elif(self._augmentor_name == 'PA'):
            pass

        try:
            if(self._criterion_name == "CrossEntropyLoss"):
                self._criterion = nn.CrossEntropyLoss()
            else:
                self._criterion = nn.CrossEntropyLoss()
        except:
            print("Criterion invalid .... Proceedding with default CrossEntropyLoss")
            self._criterion = nn.CrossEntropyLoss()
        else:
            print("Criterion is {}".format(self._criterion_name))

        try:
            if(self._optimizer_name == "Adam"):
                self._optimizer = optim.Adam(self._model.parameters(), lr = self._lr)
            else:
                self._optimizer = optim.Adam(self._model.parameters(), lr = self._lr)
        except:
            print("Optimizer invalid .... Proceedding with default Adam")
            self._optimizer = optim.Adam(self._model.parameters(), lr = self._lr)
        else:
            print("Optimizer is {}".format(self._optimizer_name))

        self._lr_scheduler_classifier = torch.optim.lr_scheduler.StepLR(self._optimizer, step_size = model_config['TRAINING']['LR_SCHEDULER']['STEP_SIZE'],
                                        gamma = model_config['TRAINING']['LR_SCHEDULER']['GAMMA'])

    def train(self, trainloader, validloader, adv = False, save_flag = True):
        if(adv):
            self.adv_train_model(trainloader, validloader)
        else:
            self.train_model(trainloader, validloader)

    def test(self, validloader, eps_list = [0.3,0.6,0.9,1.2,1.5], adv = True):
        try:
            self._model.load_state_dict(torch.load(self._path))
        except:
            print("Cannot Load the saved Model")

        #self._model.eval()
        if adv == True:
            for eps in eps_list:
                self._eps = eps
                self.adv_valid_model(validloader)
        else:
            self.valid_model(validloader)

    @property
    def epochs(self):
        return self._epochs

    @epochs.setter
    def epochs(self, epochs):
        if epochs > 0:
            self._epochs = epochs
        else:
            print("Enter valid value of greater than zero. ")

    def save_model(self):
        torch.save(self._model.state_dict(), self._path)

    def train_model(self, trainloader, validloader, path = "./saved_model.pth", print_epoch = 5, save_flag = True):

        for epoch in range(self._epochs):

            loss_tot = 0.0
            total_examples, accurate_examples = 0, 0

            for batch_idx, batch_data in enumerate(trainloader):
                self._model.train()
                self._optimizer.zero_grad()
                data   = batch_data[0].type('torch.FloatTensor').to(self._device).transpose(2,1)
                labels = batch_data[1].type('torch.FloatTensor').to(self._device).view(-1)

                outputs, feature_transform = self._model(data)
                loss = self._criterion(outputs, labels.long())
                loss_tot += loss.item()

                pred_label = torch.argmax(outputs, dim=1)
                accurate_examples += np.sum(pred_label.cpu().numpy() == labels.cpu().numpy())
                total_examples += len(pred_label.cpu().numpy())

                loss.backward()
                self._optimizer.step()
                from fgsm import visualize_batch

            print("Loss per Epoch - {}: {}, Train Accuracy : {}".format(epoch, loss_tot, (accurate_examples/ total_examples)))
            self._lr_scheduler_classifier.step()
            if((epoch+1) % print_epoch == 0):
                self.valid_model(validloader, epoch)

    def adv_train_model(self, trainloader, validloader, path = "./saved_model.pth", print_epoch = 5, save_flag = True):
        current_best_accuracy = 0.0
        for epoch in range(self._epochs):
            self._model.train()
            mean_correct, test_pred, test_true = [], [], []
            for batch_idx, batch_data in enumerate(trainloader):

                self._optimizer.zero_grad()

                points = batch_data[0].permute(0,2,1).to(self._device)
                labels = batch_data[1].to(self._device)

                pred_pc, feature_transform, _ = self._model(points)
                loss = self._criterion(pred_pc, labels.long())

                loss.backward()
                self._optimizer.step()

                self._optimizer.zero_grad()
                if(self._augmentor_name == 'FGSM'):
                    aug_data = self._augmentor(self._model, self._criterion, points, labels, self._eps)

                outputs, feature_transform = self._model(aug_data)
                loss = self._criterion(outputs, labels.long())

                loss.backward()
                self._optimizer.step()

                test_true.append(labels.cpu().numpy())
                test_pred.append(torch.argmax(pred_pc, dim = 1).detach().cpu().numpy())
                pred_pc = torch.argmax(pred_pc, dim = 1)
                correct = pred_pc.eq(labels.long().data).cpu().sum()
                mean_correct.append(correct.item() / float(points.size()[0]))

            test_true = np.concatenate(test_true)
            test_pred = np.concatenate(test_pred)
            test_acc = metrics.accuracy_score(test_true, test_pred)

            print("Accuracy: {}".format(test_acc))
            self._lr_scheduler_classifier.step()
            if((epoch+1) % print_epoch == 0):
                val_accuracy = self.valid_model(validloader, epoch)
                if(val_accuracy > current_best_accuracy):
                    current_best_accuracy = val_accuracy
                    self.save_model()
                print("Valid Accuracy {}".format(val_accuracy))

        return

    def valid_model(self, validloader, epoch = 0):
        #self._model.eval()
        mean_correct, test_pred, test_true = [], [], []
        with torch.no_grad():
            for batch_idx, batch_data in enumerate(validloader):
                points   = batch_data[0].permute(0,2,1).to(self._device)
                labels   = batch_data[1].to(self._device)
                pred_pc, _  = self._model(points)

                test_true.append(labels.cpu().numpy())
                test_pred.append(torch.argmax(pred_pc, dim = 1).detach().cpu().numpy())
                pred_pc = torch.argmax(pred_pc, dim = 1)
                correct = pred_pc.eq(labels.long().data).cpu().sum()
                mean_correct.append(correct.item() / float(points.size()[0]))

            test_true = np.concatenate(test_true)
            test_pred = np.concatenate(test_pred)
            valid_acc = metrics.accuracy_score(test_true, test_pred)

        print("Valid Accuracy {}".format(valid_acc))
        return valid_acc

    def adv_valid_model(self, validloader, epoch = 0):
        total_examples, accurate_examples, accurate_aug_examples = 0, 0, 0
        #self._model.eval()
        for batch_idx, batch_data in enumerate(validloader):

            with torch.no_grad():
                labels = batch_data[1].to(self._device)
                data = batch_data[0].permute(0,2,1).to(self._device)
                outputs, feature_transform  = self._model(data)
                pred_label = torch.argmax(outputs, dim=1)

            if(self._augmentor_name == 'FGSM'):
                aug_data = self._augmentor(self._model, self._criterion, data, labels, self._eps)

            outputs, feature_transform = self._model(aug_data)
            pred_label_aug = torch.argmax(outputs, dim=1)

            accurate_aug_examples += np.sum(pred_label_aug.cpu().numpy() == labels.cpu().numpy())
            accurate_examples += np.sum(pred_label.cpu().numpy() == labels.cpu().numpy())
            total_examples += len(pred_label.cpu().numpy())

            #visualize_batch(data, pred_label, labels, categories)

        try:
            accuracy_clean  = accurate_examples/total_examples
            accuracy_adv = accurate_aug_examples/total_examples
        except Exception as e:
            raise e

        print("Valid(Clean) Accuracy {} at epoch {}".format(accuracy_clean, epoch))
        print("Valid(Adv) Accuracy {} at epoch {}".format(accuracy_adv, epoch))

class PointAugmentTrain():
    def __init__(self, model_config, device):

        self._device         = device
        self._categories     = model_config['NUM_CLASSES']
        self._classifier     = PointNetCls(self._categories).to(self._device)
        self._augmentor      = Augmentor().to(self._device)
        self._epochs            = model_config['TRAINING']['EPOCHS']
        self._optimizerA_name   = model_config['TRAINING']['OPTIMIZER']["AUGMENTOR"]
        self._optimizerC_name   = model_config['TRAINING']['OPTIMIZER']["CLASSIFIER"]
        self._lrA               = model_config['TRAINING']['LR']["AUGMENTOR"]
        self._lrC               = model_config['TRAINING']['LR']["CLASSIFIER"]
        self._pathA  = model_config['TRAINING']['PATH_AUG']
        self._pathC  = model_config['TRAINING']['PATH_CLS']

        try:
            if(self._optimizerA_name == "Adam"):
                self._optimizerA = optim.Adam(self._augmentor.parameters(),  lr  = self._lrA)
            if(self._optimizerC_name == "Adam"):
                self._optimizerC = optim.Adam(self._classifier.parameters(), lr = self._lrC)
        except:
            print("Optimizer invalid .... Proceeding with default Adam")
            self._optimizerA = optim.Adam(self._augmentor.parameters(), lr = self._lrA, eps = 1e-08,  betas = (0.9, 0.999), weight_decay = 1e-4)
            self._optimizerC = optim.Adam(self._classifier.parameters(), lr = self._lrC, eps = 1e-08, betas = (0.9, 0.999), weight_decay = 1e-4)
        else:
            print("Optimizer Augmentor  is {}".format(self._optimizerA_name))
            print("Optimizer Classifier is {}".format(self._optimizerC_name))

        self._lr_scheduler_classifier = torch.optim.lr_scheduler.StepLR(self._optimizerC, step_size = model_config['TRAINING']['LR_SCHEDULER']['STEP_SIZE'],
                                        gamma = model_config['TRAINING']['LR_SCHEDULER']['GAMMA'])
        self._classifier.apply(train_utils.weights_init)
        self._augmentor.apply( train_utils.weights_init)

    def train(self, trainloader, validloader, adv = False, save_flag = True):
        if(adv):
            self.adv_train_model(trainloader, validloader)
        else:
            self.train_model(trainloader, validloader)

    def test(self, validloader, eps_list = [0.3,0.6,0.9,1.2,1.5]):
        self._classifier.load_state_dict(torch.load(self._pathC))
        self._classifier.eval()
        for eps in eps_list:
            self._eps = eps
            self.adv_valid_model(validloader)

    @property
    def epochs(self):
        return self._epochs

    @epochs.setter
    def epochs(self, epochs):
        if epochs > 0:
            self._epochs = epochs
        else:
            print("Enter valid value of greater than zero. ")

    def save_model(self, path_aug = "./saved_augmentor.pth", path_class = "./saved_classifier.pth"):
        torch.save(self._augmentor.state_dict() , path_aug)
        torch.save(self._classifier.state_dict(), path_class)

    def train_model(self, trainloader, validloader, path = "./saved_model.pth", print_epoch = 5, save_flag = True):

        ispn = True
        current_best_accuracy = 0.0
        for epoch in range(self._epochs):
            self._augmentor.train()
            self._classifier.train()
            mean_correct, test_pred, test_true = [], [], []
            for idx, batch_data in enumerate(trainloader):
                self._optimizerA.zero_grad()
                self._optimizerC.zero_grad()

                points   = batch_data[0].permute(0,2,1).to(self._device)
                labels   = batch_data[1].to(self._device)

                # Augmentor
                self._optimizerA.zero_grad()
                aug_pc = self._augmentor(points)

                pred_pc, pc_tran, pc_feat = self._classifier(points)
                pred_aug, aug_tran, aug_feat = self._classifier(aug_pc)
                augLoss  = loss_utils.aug_loss(pred_pc, pred_aug, labels, pc_tran, aug_tran, ispn=ispn)

                augLoss.backward(retain_graph=True)
                self._optimizerA.step()

                self._optimizerC.zero_grad()
                clsLoss = loss_utils.cls_loss(pred_pc, pred_aug, labels, pc_tran, aug_tran, pc_feat,
                                              aug_feat, ispn=ispn)
                clsLoss.backward(retain_graph=True)
                self._optimizerC.step()

                test_true.append(labels.cpu().numpy())
                test_pred.append(torch.argmax(pred_pc, dim = 1).detach().cpu().numpy())
                pred_pc = torch.argmax(pred_pc, dim = 1)
                correct = pred_pc.eq(labels.long().data).cpu().sum()
                mean_correct.append(correct.item() / float(points.size()[0]))

            test_true = np.concatenate(test_true)
            test_pred = np.concatenate(test_pred)
            test_acc = metrics.accuracy_score(test_true, test_pred)

            print("Accuracy: {}".format(test_acc))

            self._lr_scheduler_classifier.step()

            if( ((epoch + 1) % print_epoch) == 0):
                val_accuracy = self.valid_model(validloader)
                if(val_accuracy > current_best_accuracy):
                    current_best_accuracy = val_accuracy
                    self.save_model()

                print('Valid accuracy:', val_accuracy)

        return

    def valid_model(self, validloader):
        self._classifier.eval()
        mean_correct , test_pred, test_true = [], [], []
        with torch.no_grad():
            for idx, batch_data in enumerate(validloader):
                points   = batch_data[0].permute(0,2,1).to(self._device)
                labels   = batch_data[1].to(self._device)
                pred_pc, _ , _ = self._classifier(points)

                test_true.append(labels.cpu().numpy())
                test_pred.append(torch.argmax(pred_pc, dim = 1).detach().cpu().numpy())
                pred_pc = torch.argmax(pred_pc, dim = 1)
                correct = pred_pc.eq(labels.long().data).cpu().sum()
                mean_correct.append(correct.item() / float(points.size()[0]))

            test_true = np.concatenate(test_true)
            test_pred = np.concatenate(test_pred)
            valid_acc = metrics.accuracy_score(test_true, test_pred)

        return valid_acc

    def adv_valid_model(self, validloader, epoch = 0):
        total_examples, accurate_examples, accurate_aug_examples = 0, 0, 0
        from fgsm import fgsm_attack
        criterion = nn.CrossEntropyLoss()
        for batch_idx, batch_data in enumerate(validloader):
            #self._model.eval()
            with torch.no_grad():
                labels = batch_data[1].to(self._device)
                points = batch_data[0].permute(0,2,1).to(self._device)
                pred_pc, pc_tran, pc_feat = self._classifier(points)

                pred_label = torch.argmax(pred_pc, dim=1)

            aug_data = fgsm_attack(self._classifier, criterion, points, labels, self._eps)

            outputs, _ , _ = self._classifier(aug_data)
            pred_label_aug = torch.argmax(outputs, dim=1)

            accurate_aug_examples += np.sum(pred_label_aug.cpu().numpy() == labels.cpu().numpy())
            accurate_examples += np.sum(pred_label.cpu().numpy() == labels.cpu().numpy())
            total_examples += len(pred_label.cpu().numpy())

            #visualize_batch(data, pred_label, labels, categories)

        try:
            accuracy_clean  = accurate_examples/total_examples
            accuracy_adv = accurate_aug_examples/total_examples
        except Exception as e:
            raise e

        print("Valid(Clean) Accuracy {} at epoch {}".format(accuracy_clean, epoch))
        print("Valid(Adv) Accuracy {} at epoch {}".format(accuracy_adv, epoch))

class Model:
    def __init__(self, opts):
        self.opts = opts
        self.backup()
        self.set_logger()

    def backup(self):
        if not self.opts.restore:
            source_folder = os.path.join(os.getcwd(),"Augment")
            common_folder = os.path.join(os.getcwd(), "Common")
            os.system("cp %s/config.py '%s/model_cls.py.backup'" % (source_folder,self.opts.log_dir))
            os.system("cp %s/model.py '%s/model.py.backup'" % (source_folder,self.opts.log_dir))
            os.system("cp %s/augmentor.py '%s/augmentor.py.backup'" % (source_folder,self.opts.log_dir))
            os.system("cp %s/%s.py '%s/%s.py.backup'" % (source_folder,self.opts.model_name, self.opts.log_dir, self.opts.model_name))
            os.system("cp %s/loss_utils.py '%s/loss_utils.py.backup'" % (common_folder,self.opts.log_dir))
            os.system("cp %s/ModelNetDataLoader.py '%s/ModelNetDataLoader.py.backup'" % (common_folder,self.opts.log_dir))

    def set_logger(self):
        self.logger = logging.getLogger("CLS")
        self.logger.setLevel(logging.INFO)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        file_handler = logging.FileHandler(os.path.join(self.opts.log_dir, "log_train.txt"))
        file_handler.setLevel(logging.INFO)
        file_handler.setFormatter(formatter)
        self.logger.addHandler(file_handler)

    def train(self):

        self.log_string('PARAMETER ...')
        self.log_string(self.opts)
        with open(os.path.join(self.opts.log_dir, 'args.txt'), 'w') as log:
            for arg in sorted(vars(self.opts)):
                log.write(arg + ': ' + str(getattr(self.opts, arg)) + '\n')  # log of arguments
        writer = SummaryWriter(logdir=self.opts.log_dir)

        '''DATA LOADING'''
        self.log_string('Load dataset ...')

        trainDataLoader = DataLoader(ModelNetDataLoader(self.opts, partition='train'),
                            batch_size=self.opts.batch_size, shuffle=True, drop_last=False)
        testDataLoader = DataLoader(ModelNetDataLoader(self.opts,partition='test'),
                            batch_size=self.opts.batch_size, shuffle=False)

        self.log_string("The number of training data is: %d" % len(trainDataLoader.dataset))
        self.log_string("The number of test data is: %d" % len(testDataLoader.dataset))

        '''MODEL LOADING'''
        num_class = 40
        use_cuda = torch.cuda.is_available()
        device = torch.device("cuda" if use_cuda else "cpu")
        self.dim = 3 if self.opts.use_normal else 0

        classifier = PointNetCls(num_class).cuda()
        augmentor = Augmentor().cuda()
        if torch.cuda.device_count() > 1:
            print("Let's use", torch.cuda.device_count(), "GPUs!")
            classifier = nn.DataParallel(classifier)
            augmentor = nn.DataParallel(augmentor)

        if self.opts.restore:
            self.log_string('Use pretrain Augment...')

            checkpoint = torch.load(self.opts.log_dir)
            start_epoch = checkpoint['epoch']
            classifier.load_state_dict(checkpoint['model_state_dict'])
        else:
            print('No existing Augment, starting training from scratch...')
            start_epoch = 0

        optimizer_c = torch.optim.Adam(
            classifier.parameters(),
            lr=self.opts.learning_rate,
            betas=(0.9, 0.999),
            eps=1e-08,
            weight_decay=self.opts.decay_rate
        )

        optimizer_a = torch.optim.Adam(
            augmentor.parameters(),
            lr=self.opts.learning_rate_a,
            betas=(0.9, 0.999),
            eps=1e-08,
            weight_decay=self.opts.decay_rate
        )
        if self.opts.no_decay:
            scheduler_c = None
        else:
            scheduler_c = torch.optim.lr_scheduler.StepLR(optimizer_c, step_size=20, gamma=self.opts.lr_decay)

        #scheduler_a = torch.optim.lr_scheduler.StepLR(optimizer_a, step_size=20, gamma=self.opts.lr_decay)
        scheduler_a = None

        global_epoch = 0
        best_tst_accuracy = 0.0
        blue = lambda x: '\033[94m' + x + '\033[0m'
        ispn = True if self.opts.model_name=="pointnet" else False
        '''TRANING'''
        self.logger.info('Start training...')
        PointcloudScaleAndTranslate = d_utils.PointcloudScaleAndTranslate()  # initialize augmentation

        for epoch in range(start_epoch, self.opts.epoch):
            self.log_string('Epoch %d (%d/%s):' % (global_epoch + 1, epoch + 1, self.opts.epoch))
            if scheduler_c is not None:
                scheduler_c.step(epoch)
            if scheduler_a is not None:
                scheduler_a.step(epoch)

            for batch_id, data in enumerate(trainDataLoader, 0):
                points, target = data
                target = target[:, 0]
                points, target = points.cuda(), target.cuda().long()

                points = PointcloudScaleAndTranslate(points)
                points = points.transpose(2, 1).contiguous()

                noise = 0.02 * torch.randn(self.opts.batch_size, 1024).cuda()

                classifier = classifier.train()
                augmentor = augmentor.train()
                optimizer_a.zero_grad()
                aug_pc = augmentor(points, noise)

                pred_pc, pc_tran, pc_feat = classifier(points)
                pred_aug, aug_tran, aug_feat = classifier(aug_pc)
                augLoss  = loss_utils.aug_loss(pred_pc, pred_aug, target, pc_tran, aug_tran, ispn=ispn)

                augLoss.backward(retain_graph=True)
                optimizer_a.step()

                optimizer_c.zero_grad()
                clsLoss = loss_utils.cls_loss(pred_pc, pred_aug, target, pc_tran, aug_tran, pc_feat,
                                              aug_feat, ispn=ispn)
                clsLoss.backward(retain_graph=True)
                optimizer_c.step()

            train_acc = self.eval_one_epoch(classifier.eval(), trainDataLoader)
            test_acc = self.eval_one_epoch(classifier.eval(), testDataLoader)

            self.log_string('CLS Loss: %.2f'%clsLoss.data)
            self.log_string('AUG Loss: %.2f'%augLoss.data)

            self.log_string('Train Accuracy: %f' % train_acc)
            self.log_string('Test Accuracy: %f'%test_acc)

            writer.add_scalar("Train_Acc", train_acc, epoch)
            writer.add_scalar("Test_Acc", test_acc, epoch)

            if (test_acc >= best_tst_accuracy) and test_acc >= 0.895:# or (epoch % self.opts.epoch_per_save == 0):
                best_tst_accuracy = test_acc
                self.log_string('Save model...')
                self.save_checkpoint(
                    global_epoch + 1,
                    train_acc,
                    test_acc,
                    classifier,
                    optimizer_c,
                    str(self.opts.log_dir),
                    self.opts.model_name)

            global_epoch += 1
        self.log_string('Best Accuracy: %f' % best_tst_accuracy)
        self.log_string('End of training...')
        self.log_string(self.opts.log_dir)

    def eval_one_epoch(self, model, loader):
        mean_correct = []
        test_pred = []
        test_true = []

        for j, data in enumerate(loader, 0):
            points, target = data
            target = target[:, 0]
            points = points.transpose(2, 1)
            points, target = points.cuda(), target.cuda()
            classifier = model.eval()
            pred, _, _= classifier(points)
            pred_choice = pred.data.max(1)[1]

            test_true.append(target.cpu().numpy())
            test_pred.append(pred_choice.detach().cpu().numpy())

            correct = pred_choice.eq(target.long().data).cpu().sum()
            mean_correct.append(correct.item() / float(points.size()[0]))

        test_true = np.concatenate(test_true)
        test_pred = np.concatenate(test_pred)
        test_acc = metrics.accuracy_score(test_true, test_pred)

        return test_acc

    def save_checkpoint(self, epoch, train_accuracy, test_accuracy, model, optimizer, path, modelnet='checkpoint'):
        savepath = path + '/%s-%f-%04d.pth' % (modelnet, test_accuracy, epoch)
        print(savepath)
        state = {
            'epoch': epoch,
            'train_accuracy': train_accuracy,
            'test_accuracy': test_accuracy,
            #'model_state_dict': model.module.state_dict(),
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
        }
        torch.save(state, savepath)

    def log_string(self, msg):
        print(msg)
        self.logger.info(msg)


text:
import numpy as np
import sys
import os
import matplotlib.gridspec as gs
import matplotlib.pyplot as plt
import pandas as pd
import datetime as dt
from ada.Loggermixin import *

# use self.logger from Loggermixin within classes
# use h_logger outside classes
h_logger=Loggermixin.get_default_logger()

def softmax(x, axis=0, temp=1):
    if axis == 1:
        x = x.T
    probs = np.exp((x - np.max(x))/temp) / np.sum(
        np.exp((x - np.max(x))/temp), axis=0)
    return probs

# Check forecast values
# Run this periodically during testing to ensure forecasts
# are properly calculated. Because of integer conversion in
# Order Books (for calculation speed) error of +/- 31 is accepted.
def check_forecast_consistency(env):
    passed = True
    last_month = env.end_time.month
    for i, month in enumerate(env.shipping_dict.keys()):
        if month >= last_month:
            continue
        last_week = env.month_to_sim_day[month][-7]
        for j, g in enumerate(env.gmids):
            act_demand = env.order_book[np.where(
                (env.order_book[:,env.ob_indices['doc_create_time']]<=env.sim_time) &
                (env.order_book[:,env.ob_indices['doc_num']]>0) &
                (env.order_book[:,env.ob_indices['planned_gi_month']]==month) &
                (env.order_book[:,env.ob_indices['gmid']]==g)
                )[0], env.ob_indices['order_qty']].sum()
            dummy_demand = env.order_book[np.where(
                (env.order_book[:,env.ob_indices['doc_create_time']]<=env.sim_time) &
                (env.order_book[:,env.ob_indices['doc_num']]<=0) &
                (env.order_book[:,env.ob_indices['planned_gi_month']]==month) &
                (env.order_book[:,env.ob_indices['gmid']]==g)
                )[0], env.ob_indices['order_qty']].sum()
            total_demand = act_demand + dummy_demand
            diff = np.abs(total_demand-env.monthly_forecast[i,j])
            current_month = env.sim_day_to_date[env.sim_time][0]
            if env.settings['FORECAST'] == False or env.settings['FORECAST'] == 'False':
                if dummy_demand > 0:
                    h_logger.debug("Run on:\t{}-{}".format(current_month, env.sim_day_to_date[env.sim_time][1]))
                    h_logger.debug("Month: {}\tGMID: {}\tForecast: {}".format(month, g, env.settings['FORECAST']))
                    h_logger.debug("Actual Demand:\t{:.0f}\tDummy Demand:\t{:.0f}\t".format(
                        act_demand, dummy_demand))
                    h_logger.debug("Total Demand:\t{:.0f}\tForecasted Demand:\t{:.0f}\tDiff={:.0f}".format(
                        total_demand, env.monthly_forecast[i,j], total_demand-env.monthly_forecast[i,j]))
                    passed = False
            elif env.settings['FORECAST']:
                if diff > 31 and month >= current_month and env.sim_time < last_week: # Diff could be 31 given rounding errors over 31 days in a month
                    h_logger.debug("Run on:\t{}-{}".format(current_month, env.sim_day_to_date[env.sim_time][1]))
                    h_logger.debug("Month: {}\tGMID: {}\tForecast: {}".format(month, g, env.settings['FORECAST']))
                    h_logger.debug("Actual Demand:\t{:.0f}\tDummy Demand:\t{:.0f}\t".format(
                        act_demand, dummy_demand))
                    h_logger.debug("Total Demand:\t{:.0f}\tForecasted Demand:\t{:.0f}\tDiff={:.0f}".format(
                        total_demand, env.monthly_forecast[i,j], total_demand-env.monthly_forecast[i,j]))
                    passed = False
    return passed

# Visualize forecasts
def plot_net_forecast(env, gmid, month):
    v_adj = 2 # Adjust placement of vertical text
    plt.style.use('ggplot')
    #https://coolors.co/345995-f0c808-ff6700-34d1bf-4cb944
    colors = ['#345995',
              '#ff6700',
              '#4cb944',
              '#34d1bf',
              '#f0c808']
    plt.rcParams.update({'font.size': 8})
    days = env.month_to_sim_day[month]
    df = env.get_orders()
    # TODO: Update to match on forecast flag
    p_demand = df.loc[(df['gmid']==gmid) &
                      (df['doc_num']>0) &
                      (df['doc_create_time']<=env.sim_time) &
                      (df['planned_gi_month']==month)]
    p_fcast = df.loc[(df['gmid']==gmid) &
                     (df['doc_num']<=0) &
                     (df['planned_gi_month']==month)]

    # Visible demand at Day = 1
    act_agg = p_demand.groupby('planned_gi_time')['order_qty'].sum()
    if act_agg.values.size == 0:
        act_agg = pd.Series(np.zeros(len(days)), index=days)
    fcast_agg = p_fcast.groupby('planned_gi_time')['order_qty'].sum()

    gs.GridSpec(4, 4)
    fig = plt.figure(1, figsize=(12,15))
    # Actual Demand Plot
    plt.subplot2grid((4, 4), (0, 0), colspan=2, rowspan=2)
    plt.bar(act_agg.index, act_agg, color=colors[0])
    plt.xlim([min(days) - 1, max(days) + 1])
    plt.xlabel('Day')
    plt.title('Actual Demand Visible at Day {} for Product {}'.format(
        env.sim_time, gmid))
    plt.ylabel('Demand (MT)')
    if env.fixed_planning_horizon + env.sim_time in days:
        plt.axvline(x=env.fixed_planning_horizon + env.sim_time,
                    c='k')
        plt.text(s='Fixed Schedule Horizon',
                 x=env.fixed_planning_horizon + env.sim_time,
                 y=max(act_agg)/v_adj, rotation=270, fontsize=12)
        plt.axvline(x=env.lookahead_planning_horizon + env.sim_time,
                    c='k')
        plt.text(s='Planning Schedule Horizon',
                 x=env.lookahead_planning_horizon + env.sim_time,
                 y=max(act_agg)/v_adj, rotation=270, fontsize=12)

    # Forecasted Demand Plot
    plt.subplot2grid((4, 4), (0, 2), colspan=2, rowspan=2)
    plt.bar(fcast_agg.index, fcast_agg, color=colors[-1])
    plt.xlim([min(days) - 1, max(days) + 1])
    plt.xlabel('Day')
    plt.title('Forecasted Demand from Day {} for Product {}'.format(
        env.sim_time, gmid))
    if env.fixed_planning_horizon + env.sim_time in days:
        plt.axvline(x=env.fixed_planning_horizon + env.sim_time,
                    c='k')
        plt.text(s='Fixed Schedule Horizon',
                 x=env.fixed_planning_horizon + env.sim_time,
                 y=max(fcast_agg)/v_adj, rotation=270, fontsize=12)
        plt.axvline(x=env.lookahead_planning_horizon + env.sim_time,
                    c='k')
        plt.text(s='Planning Schedule Horizon',
                 x=env.lookahead_planning_horizon + env.sim_time,
                 y=max(fcast_agg)/v_adj, rotation=270, fontsize=12)

    # Net forecast
    net_agg = np.zeros(max(days) + 1)
    net_agg[fcast_agg.index] += fcast_agg.values
    bottoms = np.zeros(max(days) + 1)
    bottoms[act_agg.index] += act_agg.values

    plt.subplot2grid((4, 4), (2, 0), colspan=4, rowspan=2)
    plt.bar(act_agg.index, act_agg, color=colors[0])
    plt.bar(days, net_agg[min(days):], color=colors[1],
            bottom=bottoms[min(days):])
    plt.title('Net Forecast Total ({} MT)'.format(
        int(bottoms.sum() + net_agg.sum())))
    plt.xlim([min(days) - 1, max(days) + 1])
    plt.xlabel('Day')
    plt.ylabel('Demand (MT)')
    if env.fixed_planning_horizon + env.sim_time in days:
        plt.axvline(x=env.fixed_planning_horizon + env.sim_time,
                    c='k')
        plt.text(s='Fixed Schedule Horizon',
                 x=env.fixed_planning_horizon + env.sim_time,
                 y=max(net_agg + bottoms)/v_adj, rotation=270, fontsize=12)
        plt.axvline(x=env.lookahead_planning_horizon + env.sim_time,
                    c='k')
        plt.text(s='Planning Schedule Horizon',
                 x=env.lookahead_planning_horizon + env.sim_time,
                 y=max(net_agg + bottoms)/v_adj, rotation=270, fontsize=12)
    plt.legend(prop={'size': 20})
    plt.tight_layout()
    plt.show()

def load_excel_demand_file(env):
    file_name = "{0}.xlsx".format(env.settings["EXCEL_DEMAND_FILE"])
    ob_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),"demand_files",file_name)
    order_book = pd.read_excel(ob_file)
    order_book = order_book.values
    return order_book


text:
from scipy.signal import butter, filtfilt
import numpy as np
import matplotlib.pyplot as plt

import pandas as pd
from pandas import DataFrame, Series
import numpy as np
import neo
import mne
from mne import io, read_proj, read_selection
from mne.datasets import sample
from mne.time_frequency import psd_multitaper
import mne

print(__doc__)

sfreq = 250
tsample = 1 / sfreq
f_low = 50
f_high = 1
order = 2
channel_vector = [1,2, 3, 4, 5,6,7,8,9,10,11,12]
data = []
ch_types = []
ch_names = []
n_ch = 5

df = pd.read_csv("/home/runge/openbci/git/OpenBCI_Python/build/dataset/train/result/raw_reconstructed_signals.csv")
df = df.dropna(axis=0)

# Set parameters
# data_path = sample.data_path()
# raw_fname = data_path + '/MEG/sample/sample_audvis_raw.fif'
# proj_fname = data_path + '/MEG/sample/sample_audvis_eog-proj.fif'

for i in range(0,n_ch):
    data.append(df.ix[:,i])
    ch_types.append('mag')
    ch_names.append("kkkk"+str(i))

info = mne.create_info(ch_names=ch_names, sfreq=sfreq, ch_types=ch_types)
raw = mne.io.RawArray(data, info)
scalings = 'auto'

# Add SSP projection vectors to reduce EOG and ECG artifacts
# projs = read_proj(proj_fname)
# raw.add_proj(projs, remove_existing=True)

raw.plot(n_channels=n_ch, scalings=scalings, title='MEG data visualization over time', show=True, block=True)

# EpochsArray
# event_id = 1
# events = np.array([[200, 0, event_id],
#                    [1200, 0, event_id],
#                    [2000, 0, event_id]])
# epochs_data = np.array([[data[0][:700], data[1][:700]],
#                         [data[0][1000:1700], data[1][1000:1700]],
#                         [data[0][1800:2500], data[1][1800:2500]]])
#
# epochs = mne.EpochsArray(epochs_data, info=info, events=events,
#                          event_id={'arbitrary': 1})
# picks = mne.pick_types(info, meg=True, eeg=False, misc=False)
# epochs.plot(picks=picks, scalings='auto', show=True, block=True)

# ###############################################################################
# # EvokedArray
#
# nave = len(epochs_data)  # Number of averaged epochs
# evoked_data = np.mean(epochs_data, axis=0)
#
# evokeds = mne.EvokedArray(evoked_data, info=info, tmin=-0.2,
#                           comment='Arbitrary', nave=nave)
# evokeds.plot(picks=picks, show=True, units={'mag': '-'},
#              titles={'mag': 'sin and cos averaged'})
#
# ###############################################################################
# # Create epochs by windowing the raw data.
#
# # The events are spaced evenly every 1 second.
# duration = 1.
#
# # create a fixed size events array
# # start=0 and stop=None by default
# events = mne.make_fixed_length_events(raw, event_id, duration=duration)
# print(events)
#
# # for fixed size events no start time before and after event
# tmin = 0.
# tmax = 0.99  # inclusive tmax, 1 second epochs
#
# # create :class:`Epochs <mne.Epochs>` object
# epochs = mne.Epochs(raw, events=events, event_id=event_id, tmin=tmin,
#                     tmax=tmax, baseline=None, verbose=True)
# epochs.plot(scalings='auto', block=True)
#
# ###############################################################################
# # Create overlapping epochs using :func:`mne.make_fixed_length_events` (50 %
# # overlap). This also roughly doubles the amount of events compared to the
# # previous event list.
#
# duration = 0.5
# events = mne.make_fixed_length_events(raw, event_id, duration=duration)
# print(events)
# epochs = mne.Epochs(raw, events=events, tmin=tmin, tmax=tmax, baseline=None,
#                     verbose=True)
# epochs.plot(scalings='auto', block=True)
#
# ###############################################################################
# # Extracting data from NEO file
#
# # The example here uses the ExampleIO object for creating fake data.
# # For actual data and different file formats, consult the NEO documentation.
# reader = neo.io.ExampleIO('fakedata.nof')
# bl = reader.read(cascade=True, lazy=False)[0]
#
# # Get data from first (and only) segment
# seg = bl.segments[0]
# title = seg.file_origin
#
# ch_names = list()
# data = list()
# for asig in seg.analogsignals:
#     # Since the data does not contain channel names, channel indices are used.
#     ch_names.append(str(asig.channel_index))
#     asig = asig.rescale('V').magnitude
#     data.append(asig)
#
# sfreq = int(seg.analogsignals[0].sampling_rate.magnitude)
#
# # By default, the channel types are assumed to be 'misc'.
# info = mne.create_info(ch_names=ch_names, sfreq=sfreq)
#
# raw = mne.io.RawArray(data, info)
# raw.plot(n_channels=4, scalings={'misc': 1}, title='Data from NEO',
#          show=True, block=True, clipping='clamp')

max_stars_repo_name: GPrathap/OpenBCIPython

max_issues_repo_name: GPrathap/OpenBCIPython

max_forks_repo_name: GPrathap/OpenBCIPython

reason: import numpy,from scipy


text:
from dataclasses import dataclass
from typing import Optional, Sequence

import numpy as np
import pandas as pd
import rpy2.robjects.packages as rpackages
from epysurv.simulation.base import BaseSimulation
from epysurv.simulation.utils import add_date_time_index_to_frame, r_list_to_frame
from rpy2 import robjects
from scipy.stats import nbinom, poisson

surveillance = rpackages.importr("surveillance")

@dataclass
class SeasonalNoisePoisson(BaseSimulation):
    r"""Simulation of an endemic time series based on a Poisson distribution.

    The mean of the Poisson distribution is modelled as:

        :math:`\mu(t) = \exp{(A\sin{(frequency \cdot \omega \cdot (t + \phi))}
        + \alpha + \beta \cdot t + K \cdot state)}`

    with :math:`\omega = \pi / 52`, :math:`A` being the amplitude, :math:`\beta` the trend parameter, :math:`t`
    the current week, and :math:`\theta` the seasonal move.

    Parameters
    ----------
    amplitude
        Amplitude of the sine. Determines the range of simulated cases.
    alpha
        Parameter to move simulation along the y-axis (negative values are not allowed) with `alpha` >= `amplitude`.
    frequency
        Factor in oscillation term. Is multiplied with the annual term :math:`\omega` and the current time point.
    seasonal_move
        A term added to each time point :math:`t` to move the curve along the x-axis.
    seed
        Seed for the random number generation.
    trend
        Controls the influence of the current week on :math:`\mu`.

    References
    ----------
        http://surveillance.r-forge.r-project.org/
    """

    alpha: float = 1.0
    amplitude: float = 1.0
    frequency: int = 1
    seasonal_move: int = 0
    seed: Optional[int] = None
    trend: float = 0.0

    def simulate(
        self,
        length: int,
        state_weight: Optional[float] = None,
        state: Optional[Sequence[int]] = None,
    ) -> pd.DataFrame:
        r"""
        Simulate outbreaks.

        Parameters
        ----------
        length
            Number of weeks to model. ``length`` is ignored if ``state`` is given. In this case the length of ``state``
            is used.
        state
            Use a state chain to define the status at this time point (outbreak or not). If not given, a Markov chain is
            generated automatically.
        state_weight
            Additional weight for an outbreak which influences the distribution parameter :math:`\mu`.

        Returns
        -------
            A ``DataFrame`` of an endemic time series where each row contains the case counts of this week.
            It also contains the mean case count value based on the underlying sinus model.
        """
        if self.seed:
            base = robjects.packages.importr("base")
            base.set_seed(self.seed)
        simulated = surveillance.sim_seasonalNoise(
            A=self.amplitude,
            alpha=self.alpha,
            beta=self.trend,
            phi=self.seasonal_move,
            length=length,
            frequency=self.frequency,
            state=robjects.NULL if state is None else robjects.IntVector(state),
            K=robjects.NULL if state_weight is None else state_weight,
        )

        simulated = r_list_to_frame(simulated, ["mu", "seasonalBackground"])
        simulated = simulated.pipe(add_date_time_index_to_frame).rename(
            columns={"mu": "mean", "seasonalBackground": "n_cases"}
        )
        return simulated

@dataclass
class SeasonalNoiseNegativeBinomial(BaseSimulation):
    r"""A time series simulation that generates case counts based on a negative binomial model.

    The model is described by a mean :math:`\mu`, variance :math:`\phi \cdot \mu`, and a linear predictor including
    trend and seasonality determined by Fourier terms. :math:`\mu` of the model depends on the current week and
    is defined as follows:

        :math:`\mu(t) = \exp \left\{ \theta + \beta t + \sum_{j=1}^{m} \left\{ \gamma_{1} \cos (\frac{2\pi j t}{52})
        + \gamma_{2} \sin (\frac{2\pi j t}{52}) \right\} \right\}`

    where :math:`t` is the current week, :math:`m` the seasonality length, :math:`\beta` equals to the trend parameter,
    :math:`\gamma` is a seasonality parameter, and :math:`\theta` is the baseline frequency of the cases.

    The simulation is then run using
    :math:`\mu` and the dispersion parameter :math:`\phi` to specify the
    negative binomial model we draw case counts from.

    Parameters
    ----------
    baseline_frequency
        Baseline frequency of cases.
    dispersion
        Regulates the overdispersion compared to the Poisson distribution (:math:`\phi \cdot \mu`).
    seasonality_cos
        Seasonality parameter to model :math:`\cos` of the Fourier term.
    seasonality_sin
        seasonality parameter to model :math:`\sin` of the Fourier term.
    seasonality_length
        Models the annual-wise seasonality. 0 equals to no seasonality, 1 to annual seasonality, 2 to
        biannual seasonality and so forth.
    seed
        A seed for the random number generation.
    trend
        Controls the influence of the current week on :math:`\mu`.

    References
    ----------
    .. [1] Noufaily, A., Enki, D.G., Farrington, C.P., Garthwaite, P., Andrews, N.J., Charlett, A. (2012): An
        improved algorithm for outbreak detection in multiple surveillance systems. Statistics in Medicine,
        32 (7), 1206-1222.
    """

    baseline_frequency: float = 1.5
    dispersion: float = 1.0
    seasonality_cos: float = 0.2
    seasonality_sin: float = -0.4
    seasonality_length: int = 1
    seed: Optional[int] = None
    trend: float = 0.003

    def _seasonality(self, week: int):
        """A Fourier-based seasonality term to model the season-depended case counts.

        Parameters
        ----------
        week
            The week to model the season-based case count.
        """
        years = np.arange(1, self.seasonality_length + 1)
        return np.sum(
            self.seasonality_cos * np.cos((2 * np.pi * years * week) / 52)
            + self.seasonality_sin * np.sin((2 * np.pi * years * week) / 52)
        )

    def simulate(self, length: int) -> pd.DataFrame:
        r"""Simulate outbreaks.

        Parameter
        ---------
        length
            Number of weeks to model.

        Returns
        -------
            A ``DataFrame`` of an endemic time series where each row contains the case counts ot this week.
        """
        if self.seed:
            np.random.seed(self.seed)
        mu_s = [
            np.exp(
                self.baseline_frequency + self.trend * week + self._seasonality(week)
            )
            for week in range(length)
        ]
        if self.dispersion == 1:
            cases = [poisson.rvs(mu, size=1)[0] for mu in mu_s]
        else:
            cases = []
            for mu in mu_s:
                r = np.float(mu / (self.dispersion - 1))
                p = r / (r + mu)
                cases.append(nbinom.rvs(r, p, size=1)[0])
        return (
            pd.DataFrame({"n_cases": cases})
            .pipe(add_date_time_index_to_frame)
            .assign(timestep=list(range(1, length + 1)))
        )

max_stars_repo_name: aauss/epysurv

max_issues_repo_name: aauss/epysurv

max_forks_repo_name: aauss/epysurv


text:
import numpy as np
import torch
import lmdb
import pickle
import sys
import os.path
from os import makedirs
from baselines.common.trex_utils import preprocess

from .constants import MUJOCO_ENVS

_print = print
def print(*args, **kwargs):
    _print(*args, **kwargs)
    sys.stdout.flush()

def generate_demos(env, env_name, agent, model_dir, checkpoint_range, save_dir='demos', episodes_per_checkpoint=5, map_increment=1e9):
    save_path = save_dir + '/' + env_name + '.lmdb'
    if os.path.exists(save_path):
        print('Demonstrations not collected as %s already exists' % save_path)
        return

    checkpoints = []
    for i in checkpoint_range:
        if i < 10:
            checkpoints.append('0000' + str(i))
        elif i < 100:
            checkpoints.append('000' + str(i))
        elif i < 1000:
            checkpoints.append('00' + str(i))
        elif i < 10000:
            checkpoints.append('0' + str(i))
    print(checkpoints)

    makedirs(save_dir, exist_ok=True)
    map_counter = 1
    keys = []
    with lmdb.open(save_path, map_size=map_counter*map_increment) as lmdb_env:
        for checkpoint in checkpoints:
            model_path = model_dir + "/models/" + env_name + "_25/" + checkpoint
            if env_name == "seaquest":
                model_path = model_dir + "/models/" + env_name + "_5/" + checkpoint

            agent.load(model_path)
            for i in range(episodes_per_checkpoint):
                done = False
                traj = []
                gt_rewards = []
                actions = []
                r = 0

                ob = env.reset()
                steps = 0
                acc_reward = 0
                while True:
                    action = agent.act(ob, r, done)
                    ob, r, done, _ = env.step(action)
                    ob_processed = preprocess(ob, env_name)
                    traj.append(ob_processed)
                    actions.append(action)

                    gt_rewards.append(r[0])
                    acc_reward += r[0]
                    steps += 1
                    if done:
                        print("checkpoint: {}, steps: {}, return: {}".format(checkpoint, steps, acc_reward))
                        break

                traj = (np.concatenate(traj, axis=0)*255).astype(np.uint8)
                actions = np.array(actions)
                gt_rewards = np.array(gt_rewards)
                value = {'states':traj,
                         'actions':actions,
                         'rewards':gt_rewards,
                         'length':steps,
                         'return':acc_reward}
                key = '%s_%s_%d' % (env_name, checkpoint, i)
                lmdb_env, key = lmdb_submit(key, value, lmdb_env, save_path, map_counter, map_increment)
                keys += [key]
        with lmdb_env.begin(write=True) as txn:
            txn.put(b'__keys__', pickle.dumps(keys))
    print('%d total demonstrations gathered' % len(keys))

def create_training_data(env_name, num_trajs, num_snippets, min_snippet_length, max_snippet_length, demo_dir='demos', save_dir='datasets', map_increment=1e9):
    save_path = save_dir + '/' + env_name + ('_%d_%d.lmdb' % (num_snippets, num_trajs))
    if os.path.exists(save_path):
        print('Dataset not created as %s already exists' % save_path)
        return
    demos_path = demo_dir + '/' + env_name + '.lmdb'
    if not os.path.exists(demos_path):
        print('Dataset not created as %s does not exist' % demos_path)
        return

    makedirs(save_dir, exist_ok=True)
    max_traj_length = 0
    keys = []
    with lmdb.open(demos_path, readonly=True, map_size=map_increment) as lmdb_demos_env:
        with lmdb_demos_env.begin() as txn:
            demo_keys = pickle.loads(txn.get(b'__keys__'))
            num_demos = len(demo_keys)
        with lmdb.open(save_path, map_size=map_increment) as lmdb_data_env:
            with lmdb_data_env.begin(write=True) as txn:
                txn.put(b'__demo_path__', pickle.dumps(demos_path))
            #add full trajs
            if num_trajs > 0:
                max_step = 6
                max_trajs = num_demos * (num_demos-1) * (max_step-1) * (max_step-1) / 2
                print('Building %d of approxmately %d possible trajectories...' % (num_trajs, max_trajs))
            for n in range(num_trajs):
                print('%d of %d complete...' % (n, num_trajs), end='\r')
                ti_key, traj_i, tj_key, traj_j, label = get_random_demos(demo_keys, lmdb_demos_env)

                #create random partial trajs by finding random start frame and random skip frame

                step = np.random.randint(2, max_step + 1)
                si = np.random.randint(step)
                sj = np.random.randint(step)

                traj_i = slice(si, traj_i['length'], step)
                traj_j = slice(sj, traj_j['length'], step)
                value = {'trajectories': ((ti_key, traj_i), (tj_key, traj_j)),
                         'label': label}
                key = 't_%d' % n
                lmdb_data_env, key = lmdb_submit(key, value, lmdb_data_env, save_path, 1, map_increment)
                keys += [key]

                #max_traj_length = max(max_traj_length, len(traj_i), len(traj_j))

            #fixed size snippets with progress prior
            if num_snippets > 0:
                print('Building %d snippets...' % num_snippets)
            for n in range(num_snippets):
                print('%d of %d complete...' % (n, num_snippets), end='\r')
                ti_key, traj_i, tj_key, traj_j, label = get_random_demos(demo_keys, lmdb_demos_env)

                #create random snippets
                #find min length of both demos to ensure we can pick a demo no earlier than that chosen in worse preferred demo
                min_length = min(traj_i['length'], traj_j['length'])
                rand_length = np.random.randint(min(min_snippet_length, min_length-1), min(max_snippet_length, min_length))
                ti_start = np.random.randint(traj_i['length'] - rand_length + 1)
                tj_start = np.random.randint(traj_j['length'] - rand_length + 1)
                traj_i = slice(ti_start, ti_start+rand_length)
                traj_j = slice(tj_start, tj_start+rand_length)
                value = {'trajectories': ((ti_key, traj_i), (tj_key, traj_j)),
                         'label': label}
                key = 't_%d' % n
                lmdb_data_env, key = lmdb_submit(key, value, lmdb_data_env, save_path, 1, map_increment)
                keys += [key]

                #max_traj_length = max(max_traj_length, len(traj_i), len(traj_j))

            with lmdb_data_env.begin(write=True) as txn:
                txn.put(b'__keys__', pickle.dumps(keys))

    #print("maximum traj length", max_traj_length)

def get_random_demos(demo_keys, lmdb_demos_env):
    num_demos = len(demo_keys)
    ti_return = 0
    tj_return = 0
    #only add trajectories that are different returns
    while(ti_return == tj_return):
        #pick two random demonstrations
        ti_key = demo_keys[np.random.randint(num_demos)]
        tj_key = demo_keys[np.random.randint(num_demos)]
        with lmdb_demos_env.begin() as txn:
            traj_i = pickle.loads(txn.get(ti_key))
            traj_j = pickle.loads(txn.get(tj_key))
        ti_return = traj_i['return']
        tj_return = traj_j['return']

    if ti_return > tj_return:
        label = 0
    else:
        label = 1

    return ti_key, traj_i, tj_key, traj_j, label

def lmdb_submit(key, value, env, env_path, map_counter, map_increment):
    key = key.encode('ascii')
    value = pickle.dumps(value)
    try:
        with env.begin(write=True) as txn:
            txn.put(key, value)
        return env, key
    except lmdb.MapFullError:
        #print('map full: %d' % map_counter)
        map_counter += 1
        env.close()
        env = lmdb.open(env_path, map_size=map_counter*map_increment)
        return lmdb_submit(key.decode('ascii'), pickle.loads(value), env, env_path, map_counter, map_increment)

max_stars_repo_name: Sean-Hastings/TREX-NoRL

max_issues_repo_name: Sean-Hastings/TREX-NoRL

max_forks_repo_name: Sean-Hastings/TREX-NoRL


text:
import vdomr as vd
import time
import multiprocessing
import sys
from .stdoutsender import StdoutSender
import mtlogging
import numpy as np
import spikeforest_analysis as sa
from spikeforest import EfficientAccessRecordingExtractor
import json
import mlprocessors as mlpr
import traceback
from mountaintools import client as mt
from .tablewidget import TableWidget

class UnitsTableView(vd.Component):
    def __init__(self, context, opts=None, prepare_result=None):
        vd.Component.__init__(self)
        self._sorting_context = context
        self._recording_context = context.recordingContext()
        self._size = (100, 100)
        self._unit_table_widget = UnitTableWidget(
            context=self._sorting_context,
            units_info=prepare_result['units_info']
        )
        self._unit_table_widget.setSize(self._size)

    @staticmethod
    def prepareView(context, opts):
        sorting_context = context
        recording_context = context.recordingContext()
        try:
            recording_context.initialize()
            sorting_context.initialize()
            print('***** Preparing efficient access recording extractor...')
            earx = EfficientAccessRecordingExtractor(recording=recording_context.recordingExtractor())
            print('***** computing units info...')
            info0 = mt.loadObject(path=ComputeUnitsInfo.execute(recording=earx, sorting=sorting_context.sortingExtractor(), json_out=True).outputs['json_out'])
            print('*****')
        except:
            traceback.print_exc()
            raise
        return dict(
            units_info=info0
        )

    def setSize(self, size):
        self._size = size
        if self._unit_table_widget:
            self._unit_table_widget.setSize(size)

    def size(self):
        return self._size

    def tabLabel(self):
        return 'Units table'

    def render(self):
        if self._unit_table_widget:
            return vd.div(
                self._unit_table_widget
            )
        else:
            return vd.div(
                vd.h3('Initializing......'),
                style=dict(overflow='auto')
            )

class UnitTableWidget(vd.Component):
    def __init__(self, *, context, units_info):
        vd.Component.__init__(self)
        self._sorting_context = context
        self._recording_context = context.recordingContext()
        self._size = (100, 100)
        self._units_info = units_info
        self._update_table()
        self._sorting_context.onCurrentUnitIdChanged(self._on_context_selection_changed)
        self._sorting_context.onSelectedUnitIdsChanged(self._on_context_selection_changed)

    def setSize(self, size):
        self._size = size
        self._update_table()

    def _update_table(self):
        self._table_widget = TableWidget(
            columns=[
                dict(label='Unit ID', name='unit_id'),
                dict(label='SNR', name='snr'),
                dict(label='Peak channel', name='peak_channel'),
                dict(label='Num. events', name='num_events'),
                dict(label='Firing rate', name='firing_rate')
            ],
            records=self._units_info,
            height=self._size[1]
        )
        self._table_widget.onSelectionChanged(self._on_widget_selection_changed)
        self._on_context_selection_changed()
        self.refresh()

    def _on_widget_selection_changed(self):
        current_row_index = self._table_widget.currentRowIndex()
        if current_row_index is not None:
            unit_id = self._units_info[current_row_index]['unit_id']
            self._sorting_context.setCurrentUnitId(unit_id)
            if unit_id is not None:
                self._sorting_context.setSelectedUnitIds([unit_id])
            else:
                self._sorting_context.setSelectedUnitIds([])
        else:
            self._sorting_context.setCurrentUnitId(None)

    def _on_context_selection_changed(self):
        unit_id = self._sorting_context.currentUnitId()
        unit_ids = [info['unit_id'] for info in self._units_info]
        if unit_id is not None:
            try:
                index0 = unit_ids.index(unit_id)
            except:
                index0 = None
        else:
            index0 = None
        self._table_widget.setCurrentRowIndex(index0)

    def render(self):
        return self._table_widget

class ComputeUnitsInfo(mlpr.Processor):
    NAME = 'ComputeUnitsInfo'
    VERSION = '0.1.3'
    recording = mlpr.Input()
    sorting = mlpr.Input()
    json_out = mlpr.Output()

    def run(self):
        info0 = sa.compute_units_info(recording=self.recording, sorting=self.sorting)
        with open(self.json_out, 'w') as f:
            json.dump(info0, f)

max_stars_repo_name: mhhennig/spikeforest

max_issues_repo_name: mhhennig/spikeforest

max_forks_repo_name: mhhennig/spikeforest


text:
from .draw import Draw
from .rect import Rect
from .window import Window
from . import colors

import time as tm
import copy

# For the camera only
from pygame.locals import *
import numpy as np
import pygame
import cv2

# For the camera only

class Camera:
    """The camera relies on opencv, pygame and numpy."""

    def __init__(self, draw, position=[0, 0],
                 build_capture=False,
                 build_screen_writer=False,
                 build_capture_writer=False,
                 filename="unnamed.mp4",
                 framerate=15):
        """Create an opencv camera. By default the capture is not active, it must
        be built using the build method, or by setting build to True in the
        init method."""
        self._draw = draw
        self.position = position
        self.filename = filename
        self.fourccs = {'mp4': 'MP4V', 'avi': 'DIVX'}
        self.framerate = framerate
        # Building videos components
        if build_capture:
            self.buildCapture()
        if build_screen_writer:
            self.buildScreenWriter()
        if build_capture_writer:
            self.buildCaptureWriter()

    def buildCapture(self):
        """Build the capture with opencv. Having a separate function for the
        capture allows to have more efficiency for programs that do not need the
        camera."""
        self.capture = cv2.VideoCapture(0)

    def buildCaptureWriter(self, filename=None, framerate=None):
        """Write the videos of the camera, this is just an odd concept, nothing
        really functional."""
        if not filename: filename = self.filename
        _, extension = filename.split('.')
        fourcc = self.fourccs[extension]
        print(fourcc)
        fourcc = cv2.VideoWriter_fourcc(*fourcc)
        if not framerate: framerate = self.framerate
        self.capture_writer = cv2.VideoWriter(filename, fourcc, framerate, frameSize=self._draw.window.size)

    def buildScreenWriter(self, filename=None, framerate=None):
        """Write the videos of the screen."""
        if not filename: filename = self.filename
        _, extension = filename.split('.')
        fourcc = self.fourccs[extension]
        fourcc = cv2.VideoWriter_fourcc(*fourcc)
        if not framerate: framerate = self.framerate
        self.screen_writer = cv2.VideoWriter(filename, fourcc, framerate, frameSize=self._draw.window.size)

    def writeCapture(self):
        """Write the capture."""
        # Note this cannot work if the video capture is not built
        _, frame = self.capture.read()
        self.capture_writer.write(frame)

    def writeScreen(self):
        """Write the screen."""
        output = pygame.surfarray.array3d(self._draw.window.screen)
        output = cv2.transpose(output)
        output = cv2.cvtColor(output, cv2.COLOR_RGB2BGR)
        self.screen_writer.write(output)

    def show(self):
        """Show the video."""
        ret, frame = self.capture.read()
        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        frame = np.rot90(frame)
        frame = pygame.surfarray.make_surface(frame)
        self._draw.image(self._draw.window.screen, frame, self.position)

    __call__ = show

    def endCaptureWriter(self):
        """End the video writer and save its content."""
        del self.capture_writer

    def endScreenWriter(self):
        """End the screen writer and record its content."""
        del self.screen_writer

    def endCapture(self):
        """End the camera by ending the video capture."""
        del self.capture

    def destroy(self):
        """Destroy all cv2 windows."""
        cv2.destroyAllWindows()

    def isCapturing(self):
        """Determine if the capture is set."""
        return "capture" in self.__dict__

    def isWriting(self):
        """Determine if some content is being recorded."""
        return ("screen_writer" in self.__dict__) or ("capture_writer" in self.__dict__)

    def isScreenWriting(self):
        """Determine if the screen is being recorded."""
        return "screen_writer" in self.__dict__

    def isCaptureWriting(self):
        """Determine if the capture is being recorded."""
        return "capture_writer" in self.__dict__

    def setCapturing(self, capturing):
        """Set the capturing to the boolean 'capturing'."""
        if not self.capturing and capturing:
            self.buildCapture()
        if self.capturing and not capturing:
            self.endCapture()

    def setScreenWriting(self, screen_writing):
        """Set the screen writing to the boolean 'screen_writing'."""
        if not self.screen_writing and screen_writing:
            self.buildScreenWriter()
        if self.screen_writing and not screen_writing:
            self.endScreenWriter()

    def setCaptureWriting(self, capture_writing):
        """Set the capture writing to the boolean 'capture_writing'."""
        if not self.capture_writing and capture_writing:
            self.buildCaptureWriter()
        if self.capture_writing and not capture_writing:
            self.endCaptureWriter()

    capturing = property(isCapturing, setCapturing)
    screen_writing = property(isScreenWriting)
    capture_writing = property(isCaptureWriting)
    writing = property(isWriting)

    def switchCapture(self):
        """Switch the capture mode."""
        if self.capturing:
            self.endCapture()
        else:
            self.buildCapture()

    def switchScreenWriting(self):
        """Switch the screen writing mode."""
        if self.screen_writing:
            self.endScreenWriter()
        else:
            self.buildScreenWriter()

    def switchCaptureWriting(self):
        """Switch the capture writing mode."""
        if self.capture_writing:
            self.endCaptureWriter()
        else:
            self.buildCaptureWriter()

    def __del__(self):
        """Delete the camera components if they exist."""
        if self.capturing:
            del self.capture
        if self.screen_writing:
            del self.screen_writer
        if self.capture_writing:
            del self.capture_writer
        self.destroy()

    def write(self):
        """Write the screen or the capture video if they are on."""
        if self.screen_writing:
            self.writeScreen()
        if self.capture_writing:
            self.writeCapture()

    def release(self):
        """Release the screen or the capture video if they are on."""
        if self.screen_writing:
            self.screen_writer.release()
        if self.capture_writing:
            self.capture_writer.release()

class Line:
    """Representation of a line in the console.
    This might be used for visual debugging but also for typing commands."""

    def __init__(self, *content, time=None, color=colors.WHITE, separator=" "):
        """Object of line created using the text and optional time."""
        if time is None: time=tm.time()
        self.content = list(content)
        self.time = time
        self.color = color
        self.separator = separator

    def __str__(self):
        """Return the string representation of a line."""
        return self.separator.join(map(str, self.content))

    def __iter__(self):
        """Iterate a line by giving its content of the instant the line was written."""
        self.iterator = 0
        return self

    def __next__(self):
        """Return first the content, then the instant the line was written."""
        if self.iterator < 2:
            self.iterator += 1
            if self.iterator + 1 == 0:
                return self.content
            elif self.iterator + 1 == 1:
                return self.time

    def getText(self):
        """Return the content of a line."""
        return self.content

    text = property(getText)

    def refresh(self):
        """Refresh the time."""
        self.time = tm.time()

    @property
    def empty(self):
        return self.content[0] == ""  # difficult to do uglier

class Console:
    def __init__(self, draw,
                 lines=[],
                 interline=15,
                 left_padding=10,
                 down_padding=20,
                 conversion=False,
                 size=20,
                 position=[0, 0],
                 font=None,
                 max_lines_shown=10,
                 duration_lines_shown=10,
                 disappearance_lines_shown=10,
                 colors=[colors.WHITE, colors.BLACK]):
        """Create a console."""
        self._draw = draw  # Draw is protected and can only be read
        self.lines = lines  # List of Lines
        self.interline = interline
        self.left_padding = left_padding
        self.down_padding = down_padding
        self.conversion = conversion
        self.size = size
        self.position = position
        self.font = font
        self.max_lines_shown = max_lines_shown
        self.duration_lines_shown = duration_lines_shown
        self.disappearance_lines_shown = disappearance_lines_shown
        self.colors = colors
        self.line_index = 0
        self.line_memory = None

    def eval(self):
        """Execute the last line."""
        content = self.lines[-1].content
        if len(content) >= 1:
            if content[0] == "marc":
                content[0] += " is awesome."
            elif content[0] == "time":
                content[0] = "The time is " + time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime()) + "."
            elif content[0] == "minecraft":
                content[0] += "is the best game ever."
            elif content[0][:6] == "print(":
                self(eval(content[0][6:-1]))
            elif content[0] == "refresh":
                self.refresh()

    def back(self):
        """Set the line to an older one."""
        if self.line_index == 0:
            self.line_memory = copy.deepcopy(self.line)
        if self.line_index < len(self.lines) - 1:
            self.line_index += 1
            self.line = copy.deepcopy(self.lines[-1 - self.line_index])
            self.line.refresh()

    def forward(self):
        """Set the line to a new one."""
        if self.line_index > 1:
            self.line_index -= 1
            self.line = copy.deepcopy(self.lines[-1 - self.line_index])
        elif self.line_index == 1:
            self.line_index -= 1
            self.line = self.line_memory
        self.line.refresh()

    def clear(self):
        """Clear the console by removing all the lines."""
        self.lines = []

    def refresh(self):
        """Refresh the console by showing its previous messages."""
        for line in self.lines:
            line.time = tm.time()

    def __getitem__(self, i):
        """Return the i-th line."""
        return self.lines[i]

    def __setitem__(self, v, i):
        """Set the i-th line."""
        self.lines[i] = v

    def __iadd__(self, *args):
        """Add a line to the console."""
        l = Line(*args, time=tm.time())
        self.lines.append(l)

    append = __iadd__

    def appendLines(self, lines):
        """Append multiple lines."""
        for line in lines:
            self.append(line)

    def __call__(self, *args, **kwargs):
        """Display a message on the context as a console would do."""
        if len(args) > 0:
            l = Line(*args, **kwargs)
            self.lines.append(l)

    def nextArg(self):
        """Create a new argument."""
        self.lines[-1].content.append("")

    def nextLine(self):
        """Create a new line."""
        self("")

    def setLine(self, line):
        self.lines[-1] = line

    def getLine(self):
        return self.lines[-1]

    def delLine(self):
        del self.lines[-1]

    line = property(getLine, setLine, delLine)

    def show(self):
        """Show the console without adding a text."""
        sx, sy = self._draw.window.size
        n = len(self.lines)
        nmax = self.max_lines_shown
        dls = self.disappearance_lines_shown
        for i in range(min(n, nmax)):
            position = (self.left_padding, sy - self.interline * i - self.down_padding)
            to = self.lines[-i - 1].time
            t = (tm.time() - to) / self.duration_lines_shown
            if t < 1:
                c1 = self.lines[-i-1].color + (t ** dls,)
                # c2 = self.colors[1] + (t ** dls,)
                self._draw.print(self.lines[-i - 1], position, self.size, c1, self.font, self.conversion)

class Context(Rect):
    """The context is an object that allows the user to display graphical objects
    on the screen in a virtual mathematical plane."""

    def createFromSizeAndCorners(size, corners, **kwargs):
        c = Context(size=size, **kwargs)
        c.corners = corners
        return c

    def __init__(self, draw=None, console=None, camera=None, **kwargs):
        """Create a context using the optional draw, console and camera, with
        some of the draw's arguments for the window.
        If those are not given, they will be created automatically."""
        if draw is None:
            draw = Draw(**kwargs)
        if console is None:
            console = Console(draw)
        if camera is None:
            camera = Camera(draw)
        self.start_time = tm.time()
        self.draw = draw
        self.console = console
        self.camera = camera
        self.clear = self.draw.clear
        self.flip = self.draw.window.flip
        self.press = self.draw.window.press
        self.build = self.draw.window.build
        self.click = self.draw.window.click
        self.__call__ = self.draw.window.__call__
        self.wait = self.draw.window.wait
        self.control = self.draw.control
        self.checking = self.draw.window.checking
        self.loadImage = self.draw.window.loadImage
        self.print = self.draw.print
        self.update = self.draw.window.update
        self.switch = self.draw.window.switch
        self.count = self.draw.window.count
        self.alert = self.draw.window.alert
        self.scale = self.draw.window.scale

    def getScreen(self):
        return self.draw.window.screen

    def setScreen(self, screen):
        self.draw.window.screen = screen

    screen = property(getScreen, setScreen)

    def getKeys(self):
        """Return the keys of the window."""
        return self.press()

    def getCorners(self):
        """Return the corners of the surface shown."""
        return self.draw.plane.getCorners(self.draw.window)

    def setCorners(self, corners):
        """Set the corners of the plane using the new corners."""
        self.draw.plane.setCorners(corners, self.draw.window)

    def getRect(self):
        """Return the rect of the surface shown."""
        return Plane.getRectFromCorners(self.getCorners())

    def setRect(self, rect):
        """Set the rect of the surface shown."""
        self.draw.plane.setCorners(Plane.getCornersFromRect(rect))

    def getCoordonnates(self):
        """Return the coordonnates of the surface shown."""
        return Plane.getCoordonnatesFromCorners(self.getCorners())

    def setCoordonnates(self, coordonnates):
        """Set the coordonnates of the surface shown."""
        self.draw.plane.setCorners(
            Plane.getCornersFromCoordonnates(coordonnates))

    def getPosition(self):
        """Return the position of the surface shown."""
        return self.draw.plane.position

    def setPosition(self, position):
        """Set the position of the surface shown."""
        self.draw.plane.position = position

    def getSize(self):
        """Return the size of the surface shown."""
        return self.draw.window.size

    def setSize(self, size):
        """Set the size of the surface shown."""
        self.draw.window.size = size

    def getUnits(self):
        """Return the units of the surface shown."""
        return self.draw.plane.units

    def setUnits(self, units):
        """Set the units of the surface shown."""
        self.draw.plane.units = units

    def point(self):
        """Adapt the position of the cursor in plane's coordonnates."""
        x, y = self.draw.window.point()
        x, y = self.draw.plane.getFromScreen([x, y], self.draw.window)
        return (x, y)

    def check(self):
        """Check if the surface is open."""
        self.draw.window.check()
        self.open = self.draw.window.open

    def __call__(self):
        """Calling the surface allow the user to move on screen."""
        while self.open:
            self.check()
            self.draw.plane.control(self.draw.window)
            self.draw.window.clear()
            self.draw.plane.show(self.draw.window)
            self.flip()

    def __contains__(self, position):
        """Determine if a position is in the context."""
        return self.draw.plane.contains(position, self.draw.window)

    def show(self):
        """Show the plane on screen."""
        self.draw.plane.show(self.draw.window)

    def refresh(self):
        """Refresh the context by clearing the screen and showing the plane."""
        self.draw.window.clear()
        self.draw.plane.show(self.draw.window)

    def controlZoom(self):
        """Control the zoom of the surface's plane."""
        self.draw.plane.controlZoom(self.draw.window)

    def controlPosition(self):
        """Control the position of the surface's plane."""
        self.draw.plane.controlPosition(self.draw.window)

    def getFromScreen(self, position):
        """Behave like the get from screen of the plan without having to put
        the window in parameter."""
        return self.draw.plane.getFromScreen(position, self.draw.window)

    def getToScreen(self, position):
        """Behave like the get to screen of the plan without having to put
        the window in parameter."""
        return self.draw.plane.getToScreen(position, self.draw.window)

    def blit(self, surface, position):
        """Blit a given surface to a given position."""
        size = surface.get_size()
        position = self.draw.plane.getToScreen(position, self.window)
        self.draw.window.screen.blit(surface, position)

    def getOpen(self):
        return self.draw.window.open

    def setOpen(self, open):
        self.draw.window.open = open

    def __enter__(self):
        """Opening the context."""
        self.check()
        self.control()
        self.clear()
        self.show()
        print("went in")
        return self

    def __exit__(self):
        """Ending the context."""
        print("going out")
        self.flip()
        if self.open:
            self.__enter__()

    def getWidth(self):
        """Return the width of the window."""
        return self.size[0]

    def setWidth(self, width):
        """Set the width of the window."""
        self.size[0] = width

    def getHeight(self):
        """Return the height of the window."""
        return self.size[1]

    def setHeight(self, height):
        """Set the height of the window."""
        self.size[1] = height

    def __bool__(self):
        """Determine if the context is opened or not."""
        return self.open

    def getText(self):
        """Text is an alias for console and can be read."""
        return self.console

    def setText(self, value):
        """Text is an alias for console and can be written."""
        self.console = value

    def showConsole(self, *args):
        """Show the console, this function is a fix for deprecated programs."""
        self.console.show(*args)

    def getCounter(self):
        """Return the window counter."""
        return self.draw.window.counter

    def setCounter(self, coutner):
        """Set the window counter."""
        self.draw.window.counter = counter

    def getPlane(self):
        """Return the draw's plane."""
        return self.draw.plane

    def setPlane(self, plane):
        """Set the draw's plane."""
        self.plane = plane

    def getWindow(self):
        """Return the draw's window."""
        return self.draw.window

    def setWindow(self, window):
        """Set the draw's window."""
        self.draw.window = window

    def getFullscreen(self):
        """Determine if the context is in fullscreen mode."""
        return self.draw.window.fullscreen

    def setFullscreen(self, fullscreen):
        """Set the fullscreen mode."""
        self.draw.window.fullscreen = fullscreen

    def getResolution(self):
        """Return the resolution of the screen."""
        return self.width / self.height

    @property
    def rate(self):
        return self.counter / (tm.time() - self.start_time)

    window = property(getWindow, setWindow)
    plane = property(getPlane, setPlane)

    corners = property(getCorners, setCorners)
    rect = property(getRect, setRect)
    coordonnates = property(getCoordonnates, setCoordonnates)  # Coordinates
    position = property(getPosition, setPosition)
    size = property(getSize, setSize)
    units = property(getUnits, setUnits)
    keys = property(getKeys)
    open = property(getOpen, setOpen)
    width = property(getWidth, setWidth)
    height = property(getHeight, setHeight)
    text = property(getText, setText)
    counter = property(getCounter, setCounter)
    fullscreen = property(getFullscreen, setFullscreen)
    resolution = property(getResolution)

Surface = Context  # Ugly fix for deprectated programs

if __name__ == "__main__":
    context = Context(name="Context test")
    context.camera.buildCapture()
    context.camera.buildScreenWriter("context test.mp4")
    # context.camera.buildCaptureWriter('zzzz.mp4')
    context.console.duration_lines_shown = 2
    context.console.max_lines_shown = 40
    # context.corners=[0,0,1,1]
    print(context.corners)
    print(bool(context))
    while context:
        context.count()
        print(context.counter)
        context.check()
        context.update()
        context.control()
        context.clear()
        context.show()

        context.camera.show()
        context.camera.write()
        context.console(context.rate)

        # if context.counter > 100:
        #    context.camera.destroy()

        context.flip()
    context.camera.capture.release()
    context.camera.screen_writer.release()
    print('released')

max_stars_repo_name: MarcPartensky/Pygame-Geometry

max_issues_repo_name: MarcPartensky/Pygame-Geometry

max_forks_repo_name: MarcPartensky/Pygame-Geometry