text:
import os

import numpy as np
import torch
import torch.nn as nn
import torchvision.transforms as transforms
from PIL import Image
from torch.autograd import Variable
from torch.optim.lr_scheduler import StepLR
from torch.utils.data import DataLoader, Dataset

import resnet
import myResnet
import myInceptionNet
from Attention import Attention

torch.cuda.empty_cache()
"""Load datasets"""

NUMBER = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
ALPHABET = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u',
            'v', 'w', 'x', 'y', 'z']
# NONE = ['NONE']  # label for empty space
NONE = ['-']  # label for empty space
ALL_CHAR_SET = NUMBER + ALPHABET + NONE
ALL_CHAR_SET_LEN = len(ALL_CHAR_SET)
MAX_CAPTCHA = 7

# print(ALL_CHAR_SET.index('NONE'))

def encode(a):
    onehot = [0] * ALL_CHAR_SET_LEN
    idx = ALL_CHAR_SET.index(a)
    onehot[idx] += 1
    return onehot

# modified dataset class
class Mydataset(Dataset):
    def __init__(self, img_path, label_path, is_train=True, transform=None):
        self.path = img_path
        self.label_path = label_path
        if is_train:
            self.img = os.listdir(self.path)[:10000]
            self.labels = open(self.label_path, 'r').read().split('\n')[:-1][:10000]
        else:
            self.img = os.listdir(self.path)[:1000]
            self.labels = open(self.label_path, 'r').read().split('\n')[:-1][:1000]

        self.transform = transform
        self.max_length = MAX_CAPTCHA

    def __getitem__(self, idx):
        img_path = self.img[idx]
        img = Image.open(f'{self.path}/{self.img[idx]}')
        img = img.convert('L')
        label = self.labels[idx]
        label_oh = []
        # one-hot for each character
        for i in range(self.max_length):
            if i < len(label):
                label_oh += encode(label[i])
            else:
                # label_oh += [0]*ALL_CHAR_SET_LEN
                # label_oh += encode('NONE')
                label_oh += encode('-')

        if self.transform is not None:
            img = self.transform(img)
        return img, np.array(label_oh), label

    def __len__(self):
        return len(self.img)

transform = transforms.Compose([
    transforms.Resize([160, 60]),
    transforms.ToTensor(),
    ##############################################################################
    transforms.Normalize((0.8958,), (0.1360,)),
    ##############################################################################
])

"""Loading DATA"""
# Change to your own data foler path!
gPath = ''

train_ds = Mydataset(gPath + 'Data/train/', gPath + 'Data/train.txt', transform=transform)
test_ds = Mydataset(gPath + 'Data/train/', gPath + 'Data/train.txt', False, transform)
train_dl = DataLoader(train_ds, batch_size=128, num_workers=4)
test_dl = DataLoader(test_ds, batch_size=1, num_workers=4)
# train_dl = DataLoader(train_ds, batch_size=128)
# test_dl = DataLoader(test_ds, batch_size=1)

# mean = 0.
# std = 0.
# for step, i in enumerate(train_dl):
#     img, label_oh, label = i
#     img = Variable(img).cuda()
#     mean += img.mean()
#     std += img.std()
# print(step)
# mean /= step
# std /= step
# print("mean=", mean, " std=", std)

"""To CUDA for local run"""
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
GPUID = '1'  # define GPUID
os.environ["CUDA_VISIBLE_DEVICES"] = str(GPUID)

"""Problem 1: Design LSTM model for catcha image recognition. (10 points)"""
class LSTM(nn.Module):
    def __init__(self, cnn_dim, hidden_size, vocab_size, num_layers=2):
        super(LSTM, self).__init__()

        # define the properties
        self.cnn_dim = cnn_dim
        self.hidden_size = hidden_size
        self.vocab_size = vocab_size
        self.num_layers = num_layers

        # lstm
        self.bilstm = nn.GRU(input_size=self.hidden_size, hidden_size=hidden_size, num_layers=2,
                              bidirectional=True, bias=True, batch_first=True)

        # output fully connected layer
        self.fc_in = nn.Linear(in_features=self.cnn_dim, out_features=self.vocab_size)
        self.fc_out = nn.Linear(in_features=self.hidden_size * 2, out_features=self.vocab_size)

        # embedding layer
        self.embed = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=self.vocab_size)

        # activations
        self.softmax = nn.Softmax(dim=2)

        self.avgpool = nn.AdaptiveAvgPool2d(1)

    def forward(self, features, isTraining=True):

        ##############################################################################
        hidden = self.init_hidden(batch_size, next(self.parameters()).is_cuda)
        outputs, hidden = self.bilstm(features, hidden)
        outputs = self.fc_out(outputs)
        if not isTraining:
            outputs = self.softmax(outputs)
        ##############################################################################
        return outputs

    def init_hidden(self, batch_size, gpu=False):
        h0 = Variable(torch.zeros(self.num_layers * 2, batch_size, self.hidden_size)).cuda()
        return h0

"""Problem 2:

*   1.Connect CNN model to the desinged LSTM model.
*   2.Replace ResNet to your own CNN model from Assignment3.
"""

##############################################################################
"""ResNet"""
# CNN
cnn_model_path = './trained_myResnet_only2.pth'
betternet = myResnet.betternet()
betternet.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
betternet.fc = nn.Linear(in_features=512, out_features=ALL_CHAR_SET_LEN * MAX_CAPTCHA, bias=True)
betternet.load_state_dict(torch.load(cnn_model_path))
betternet.to(device)
betternet.eval()

##############################################################################

# LSTM
cnn_dim = 512  # resnet18-512
hidden_size = 37  # 8->37
vocab_size = 37  # ALL_CHAR_SET_LEN
lstm = LSTM(cnn_dim=cnn_dim, hidden_size=hidden_size, vocab_size=hidden_size).to(device)
lstm.train()
# loss, optimizer
##############################################################################
# params = list(betternet.parameters()) + list(lstm.parameters())
loss_func = nn.MultiLabelSoftMarginLoss()
nn.utils.clip_grad_norm_(lstm.parameters(), 5)
optimizer = torch.optim.Adam(lstm.parameters(), lr=0.001, weight_decay=0.0001)
lr_scheduler = StepLR(optimizer, step_size=500)
##############################################################################

def get_char_count(arg1):
    c0 = ALL_CHAR_SET[np.argmax(arg1.cpu().tolist()[0:ALL_CHAR_SET_LEN])]
    c1 = ALL_CHAR_SET[np.argmax(arg1.cpu().tolist()[ALL_CHAR_SET_LEN:ALL_CHAR_SET_LEN * 2])]
    c2 = ALL_CHAR_SET[np.argmax(arg1.cpu().tolist()[ALL_CHAR_SET_LEN * 2:ALL_CHAR_SET_LEN * 3])]
    c3 = ALL_CHAR_SET[np.argmax(arg1.cpu().tolist()[ALL_CHAR_SET_LEN * 3:ALL_CHAR_SET_LEN * 4])]
    c4 = ALL_CHAR_SET[np.argmax(arg1.cpu().tolist()[ALL_CHAR_SET_LEN * 4:ALL_CHAR_SET_LEN * 5])]
    c5 = ALL_CHAR_SET[np.argmax(arg1.cpu().tolist()[ALL_CHAR_SET_LEN * 5:ALL_CHAR_SET_LEN * 6])]
    c6 = ALL_CHAR_SET[np.argmax(arg1.cpu().tolist()[ALL_CHAR_SET_LEN * 6:ALL_CHAR_SET_LEN * 7])]
    return c0, c1, c2, c3, c4, c5, c6
"""step3: Find hyper-parameters."""

"""TRAINING"""
print_interval = 50
# max_epoch = 1
max_epoch = 50
threshold = 1e-4
isConverged = False
for epoch in range(max_epoch):
    if isConverged:
        break
    for step, i in enumerate(train_dl):
        img, label_oh, label = i
        img = Variable(img).cuda()
        text = Variable(label_oh.float()).cuda()
        label_oh = Variable(label_oh.long()).cuda()
        batch_size, _ = label_oh.shape
        cnn_pred, features = betternet(img)
        ##############################################################################
        optimizer.zero_grad()
        # cnn_pred = cnn_pred.reshape(batch_size, MAX_CAPTCHA, ALL_CHAR_SET_LEN).to(device)

        pred = lstm(text, isTraining=True)
        pred = torch.flatten(pred, 1)  # 128, 259
        # for end-to-end learning
        loss = loss_func(pred, label_oh)  # softmax(128, 259) vs. one-hot(128, 259)

        if loss < threshold:
            isConverged = True

        loss.backward()
        optimizer.step()
        lr_scheduler.step()
        ##############################################################################
        if (step + 1) % print_interval == 0:
            print('epoch:', epoch + 1, 'step:', step + 1, 'loss:', loss.item())
        if (epoch + 1) % 101 == 0:
            print("predict:", nn.Softmax(pred[0, :]))
            print("target:", label_oh[0, :])

# cnn_model_path = './trained_cnn_model_bidirect_end_to_end.pth'
lstm_model_path = './trained_lstm_model_bidirect_end_to_end.pth'
print('Finished Training')
# torch.save(betternet.state_dict(), cnn_model_path)
torch.save(lstm.state_dict(), lstm_model_path)
print('Saved Trained Model')

"""TEST"""

char_correct = 0
word_correct = 0
total = 0

# betternet.eval()
lstm.eval()

with torch.no_grad():
    for step, (img, label_oh, label) in enumerate(test_dl):
        char_count = 0
        img = Variable(img).cuda()
        label_oh = Variable(label_oh.long()).cuda()
        cnn_pred, feature = betternet(img)
        batch_size, _ = label_oh.shape
        cnn_pred = cnn_pred.reshape(batch_size, MAX_CAPTCHA, ALL_CHAR_SET_LEN).to(device)
        pred = lstm(cnn_pred, isTraining=False)
        outputs = torch.flatten(pred, 1)  # 128, 259
        # for end-to-end learning

        label_len = label[0]
        outputs = outputs.squeeze(0)
        label_oh = label_oh.squeeze(0)

        c0, c1, c2, c3, c4, c5, c6 = get_char_count(outputs)
        d0, d1, d2, d3, d4, d5, d6 = get_char_count(label_oh)

        c = '%s%s%s%s%s%s%s' % (c0, c1, c2, c3, c4, c5, c6)
        d = '%s%s%s%s%s%s%s' % (d0, d1, d2, d3, d4, d5, d6)

        char_count += (c0 == d0) + (c1 == d1) + (c2 == d2) + (c3 == d3) + (c4 == d4) + (c5 == d5) + (c6 == d6)
        char_correct += char_count

        print("predict:", c)
        print("label:", d)

        if bool(str(label[0]) in str(c)):
            word_correct += 1

        total += 1

print(100/7*char_correct/total)
print(100*word_correct/total)

"""END TEST"""


text:
"""
Helpers when analyzing backtest data
"""
import logging
from pathlib import Path
from typing import Dict

import numpy as np
import pandas as pd
import pytz

from freqtrade import persistence
from freqtrade.misc import json_load
from freqtrade.persistence import Trade

logger = logging.getLogger(__name__)

# must align with columns in backtest.py
BT_DATA_COLUMNS = ["pair", "profitperc", "open_time", "close_time", "index", "duration",
                   "open_rate", "close_rate", "open_at_end", "sell_reason"]

def load_backtest_data(filename) -> pd.DataFrame:
    """
    Load backtest data file.
    :param filename: pathlib.Path object, or string pointing to the file.
    :return: a dataframe with the analysis results
    """
    if isinstance(filename, str):
        filename = Path(filename)

    if not filename.is_file():
        raise ValueError(f"File {filename} does not exist.")

    with filename.open() as file:
        data = json_load(file)

    df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)

    df['open_time'] = pd.to_datetime(df['open_time'],
                                     unit='s',
                                     utc=True,
                                     infer_datetime_format=True
                                     )
    df['close_time'] = pd.to_datetime(df['close_time'],
                                      unit='s',
                                      utc=True,
                                      infer_datetime_format=True
                                      )
    df['profitabs'] = df['close_rate'] - df['open_rate']
    df = df.sort_values("open_time").reset_index(drop=True)
    return df

def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int) -> pd.DataFrame:
    """
    Find overlapping trades by expanding each trade once per period it was open
    and then counting overlaps
    :param results: Results Dataframe - can be loaded
    :param freq: Frequency used for the backtest
    :param max_open_trades: parameter max_open_trades used during backtest run
    :return: dataframe with open-counts per time-period in freq
    """
    dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, freq=freq))
             for row in results[['open_time', 'close_time']].iterrows()]
    deltas = [len(x) for x in dates]
    dates = pd.Series(pd.concat(dates).values, name='date')
    df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns)

    df2 = pd.concat([dates, df2], axis=1)
    df2 = df2.set_index('date')
    df_final = df2.resample(freq)[['pair']].count()
    return df_final[df_final['pair'] > max_open_trades]

def load_trades_from_db(db_url: str) -> pd.DataFrame:
    """
    Load trades from a DB (using dburl)
    :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite)
    :return: Dataframe containing Trades
    """
    trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS)
    persistence.init(db_url, clean_open_orders=False)

    columns = ["pair", "open_time", "close_time", "profit", "profitperc",
               "open_rate", "close_rate", "amount", "duration", "sell_reason",
               "fee_open", "fee_close", "open_rate_requested", "close_rate_requested",
               "stake_amount", "max_rate", "min_rate", "id", "exchange",
               "stop_loss", "initial_stop_loss", "strategy", "ticker_interval"]

    trades = pd.DataFrame([(t.pair,
                            t.open_date.replace(tzinfo=pytz.UTC),
                            t.close_date.replace(tzinfo=pytz.UTC) if t.close_date else None,
                            t.calc_profit(), t.calc_profit_percent(),
                            t.open_rate, t.close_rate, t.amount,
                            (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)
                                if t.close_date else None),
                            t.sell_reason,
                            t.fee_open, t.fee_close,
                            t.open_rate_requested,
                            t.close_rate_requested,
                            t.stake_amount,
                            t.max_rate,
                            t.min_rate,
                            t.id, t.exchange,
                            t.stop_loss, t.initial_stop_loss,
                            t.strategy, t.ticker_interval
                            )
                           for t in Trade.query.all()],
                          columns=columns)

    return trades

def load_trades(source: str, db_url: str, exportfilename: str) -> pd.DataFrame:
    """
    Based on configuration option "trade_source":
    * loads data from DB (using `db_url`)
    * loads data from backtestfile (using `exportfilename`)
    """
    if source == "DB":
        return load_trades_from_db(db_url)
    elif source == "file":
        return load_backtest_data(Path(exportfilename))

def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame:
    """
    Compare trades and backtested pair DataFrames to get trades performed on backtested period
    :return: the DataFrame of a trades of period
    """
    trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) &
                        (trades['close_time'] <= dataframe.iloc[-1]['date'])]
    return trades

def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "close"):
    """
    Combine multiple dataframes "column"
    :param tickers: Dict of Dataframes, dict key should be pair.
    :param column: Column in the original dataframes to use
    :return: DataFrame with the column renamed to the dict key, and a column
        named mean, containing the mean of all pairs.
    """
    df_comb = pd.concat([tickers[pair].set_index('date').rename(
        {column: pair}, axis=1)[pair] for pair in tickers], axis=1)

    df_comb['mean'] = df_comb.mean(axis=1)

    return df_comb

def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
                      timeframe: str) -> pd.DataFrame:
    """
    Adds a column `col_name` with the cumulative profit for the given trades array.
    :param df: DataFrame with date index
    :param trades: DataFrame containing trades (requires columns close_time and profitperc)
    :param col_name: Column name that will be assigned the results
    :param timeframe: Timeframe used during the operations
    :return: Returns df with one additional column, col_name, containing the cumulative profit.
    """
    from freqtrade.exchange import timeframe_to_minutes
    ticker_minutes = timeframe_to_minutes(timeframe)
    # Resample to ticker_interval to make sure trades match candles
    _trades_sum = trades.resample(f'{ticker_minutes}min', on='close_time')[['profitperc']].sum()
    df.loc[:, col_name] = _trades_sum.cumsum()
    # Set first value to 0
    df.loc[df.iloc[0].name, col_name] = 0
    # FFill to get continuous
    df[col_name] = df[col_name].ffill()
    return df

ext: py

reason: import numpy


text:
"""
To read .nex or .nex5 files, use the following code:
    import nex
    reader = nex.Reader()
    fileData = reader.ReadNexFile('C:\\Data\\file.nex')
    fileData1 = reader.ReadNexFile('C:\\Data\\file.nex5')
    print(fileData['Variables'][0])

If your files are larger than a few MB, use numpy version of the reader:
    import nex
    reader = nex.Reader(useNumpy=True)
    fileData = reader.ReadNexFile('C:\\Data\\LargeFile.nex')

See comments below for the description of the content of fileData.

To write .nex file, use this code:
    timestampFrequency = 50000
    writer = nex.NexWriter(timestampFrequency)
then, add variable data using Add... methods in NexWriter class
(see method doc strings below for more info):
    writer.AddContVarWithSingleFragment('cont1', 0, 10000, [5, 6, 7, 8])
    writer.AddContVarWithSingleFragment('cont2', 0, 10000, [9, 10, 11, 12])
then, use WriteNexFile method:
    writer.WriteNexFile('C:\\Data\\python.nex')

If your files are larger than a few MB, use numpy version of the NexWriter:
    import nex
    import numpy as np
    timestampFrequency = 50000
    writer = nex.NexWriter(timestampFrequency, useNumpy=True)
    writer.AddNeuron('neuron1', np.array([1, 2, 3, 4]))
    writer.AddContVarWithSingleFragment('cont1', 2, 10000, np.array([5, 6, 7, 8]))
    writer.WriteNex5File('C:\\Data\\pythonWithFloatContValues.nex5', saveContValuesAsFloats=1)

fileData:
    fileData["FileHeader"] -- file header:
        fileData["FileHeader"]["Comment"] -- comment
        fileData["FileHeader"]["Beg"] -- file data start in ticks
        fileData["FileHeader"]["End"] -- file data end in ticks
        fileData["FileHeader"]["NexFileVersion"] -- file version
        fileData["FileHeader"]["MagicNumber"] -- magic number
        fileData["FileHeader"]["Frequency"] -- timestamp frequency
        fileData["FileHeader"]["MetaOffset"] -- location of metadata (position in file)
        fileData["FileHeader"]["NumVars"] -- number of variables

    fileData["MetaData"] -- file metadata: various extra information fields that do not fit
        into file or variable headers

    fileData["Variables"] -- a list of data variables
        For each variable var in fileData["Variables"]:
            var["Timestamps"] -- array of timestamps in seconds (neurons, events, waveforms)
            var["Intervals"] -- array of intervals in seconds (interval variables)
            var["WaveformValues"] -- array of waveform values in milliVolts (waveform variables)

            for marker variables:
                var["MarkerFieldNames"] -- array of marker field names
                var["Markers"] -- array of marker values for all fields
                var["Fields"] -- array of marker fields
                for each field:
                    var["Fields"][i]["Name"] -- the name of the marker field
                    var["Fields"][i]["Marker"] -- marker values

            for continuous variables:
                var["FragmentCounts"] -- array of fragment counts (how many samples in each fragment)
                var["FragmentIndexes"] -- array of fragment indexes (index of the first data point in each fragment)
                var["ContinuousValues"] -- array of continuous values in milliVolts
                var["FragmentTimestamps"] -- array of fragment timestamps (timestamp of the first data point in each fragment)
            var["Header"] -- variable header:
                var["Header"]["Type"] -- variable type: 0 - neuron, 1 - event, 2- interval, 3 - waveform, 4 - pop. vector, 5 - continuously recorded, 6 - marker
                var["Header"]["Name"] -- variable name
                var["Header"]["Version"] -- variable version in file
                var["Header"]["DataOffset"] -- where the data array for this variable is located in the file
                var["Header"]["Count"] --  neuron variable: number of timestamps
                             event variable: number of timestamps
                             interval variable: number of intervals
                             waveform variable: number of waveforms
                             continuous variable: number of fragments
                             population vector: number of weights
                var["Header"]["TsDataType"] -- if 0, timestamps are stored as 32-bit integers;
                            if 1, timestamps are stored as 64-bit integers;  supported by NeuroExplorer version 5.100 or greater
                var["Header"]["ContDataType"] -- waveforms and continuous variables only,
                            if 0, waveform and continuous values are stored as 16-bit integers;
                            if 1, waveform and continuous values are stored as 32-bit floating point values in units specified in Units field
                var["Header"]["SamplingRate"] -- waveforms and continuous variables only, waveform or continuous variable sampling frequency in Hertz
                var["Header"]["ADtoMV"] -- waveforms and continuous variables only, coefficient
                            to convert from A/D values stored in file to units.
                            A/D values in fileData are already scaled to units
                            see formula below UnitsOffset below; ignored if ContinuousDataType == 1
                var["Header"]["MVOffset"] -- waveforms and continuous variables only,
                            this offset is used to convert A/D values stored in file to units:
                            value_in_units = raw * ADtoUnitsCoefficient + UnitsOffset; ignored if ContinuousDataType == 1
                             A/D values in fileData are already scaled to units.
                var["Header"]["NPointsWave"] -- waveform variable: number of data points in each wave
                                           continuous variable: overall number of data points in the variable
                var["Header"]["PreThrTime"] -- waveform variables only, pre-threshold time in seconds
                            if waveform timestamp in seconds is t,
                            then the timestamp of the first point of waveform is t - PrethresholdTimeInSeconds
                var["Header"]["MarkerDataType"] -- marker events only,
                            if 0, marker values are stored as strings;
                            if 1, marker values are stored as 32-bit unsigned integers
                var["Header"]["MarkerLength"] -- marker events only, how many characters are in each marker value; ignored if MarkerDataType is 1
                var["Header"]["ContFragIndexType"] -- continuous variables only,
                            if 0, indexes of first data point in fragments are stored as unsigned 32-bit integers;
                            if 1, indexes of first data point in fragments are stored as unsigned 64-bit integers; not supported in NeuroExplorer as of October 2017 (version 5.107)
                var["Header"]["Units"] -- waveforms and continuous variables only, units that should be used for the variable values
                            not supported as of January 2022 (version 5.312)
                var["Header"]["NMarkers"] -- marker events only, how many values are associated with each marker

"""

import sys
import os
import struct
import json
import numbers

class NexFileVarType:
    """
    Constants for .nex and .nex5 variable types
    """
    NEURON = 0
    EVENT = 1
    INTERVAL = 2
    WAVEFORM = 3
    POPULATION_VECTOR = 4
    CONTINUOUS = 5
    MARKER = 6

class DataFormat:
    # constants for reading data
    INT16 = 0
    UINT16 = 1
    INT32 = 2
    UINT32 = 3
    INT64 = 4
    UINT64 = 5
    FLOAT32 = 6
    FLOAT64 = 7

    @staticmethod
    def NumBytesPerItem(dataType):
        numBytes = [2,2,4,4,8,8,4,8]
        return numBytes[dataType]

    @staticmethod
    def StructTypeFromDataType(dataType):
        formats = ['h', 'H', 'i', 'I', 'q', 'Q', 'f', 'd']
        return formats[dataType]

class Reader(object):
    """
    Nex file reader class.
    """
    def __init__(self, useNumpy=True):
        """
        Constructor
        :param useNumpy: option to use numpy to read data arrays.
        """
        self.theFile = None
        self.fileData = None
        self.useNumpy = useNumpy
        self.fromTicksToSeconds = 1

    def ReadNex5File(self, filePath:str) -> dict:
        """
        Reads data from .nex5 file.
        :param filePath: full path of file
        :return: file data
        """
        extension = os.path.splitext(filePath)[1].lower()
        if extension == '.nex':
            return self.ReadNexFile(filePath)
        self.fileData = {}
        self.theFile = open(filePath, 'rb')

        # read file header
        self.fileData['FileHeader'] = self._ReadNex5FileHeader()
        self.fileData['Variables'] = []

        # read variable headers and create variables
        for varNum in range(self.fileData['FileHeader']['NumVars']):
            var = {'Header': self._ReadNex5VarHeader()}
            self.fileData['Variables'].append(var)

        # read variable data
        self._ReadData()

        # read metadata
        metaOffset = self.fileData['FileHeader']['MetaOffset']
        if metaOffset > 0:
            self.theFile.seek(0, os.SEEK_END)
            size = self.theFile.tell()
            if metaOffset < size:
                self.theFile.seek(metaOffset)
                metaString = self.theFile.read(size - metaOffset).decode('utf-8').strip('\x00')
                metaString = metaString.strip()
                try:
                    self.fileData['MetaData'] = json.loads(metaString)
                except Exception as error:
                    print('Invalid file metadata: ' + repr(error))

        self.theFile.close()
        return self.fileData

    def ReadNexFile(self, filePath:str) -> dict:
        """
        Reads data from .nex file.
        :param filePath:
        :return: file data
        """
        extension = os.path.splitext(filePath)[1].lower()
        if extension == '.nex5':
            return self.ReadNex5File(filePath)

        self.fileData = {}
        self.theFile = open(filePath, 'rb')

        self.fileData['FileHeader'] = self._ReadFileHeader()
        self.fileData['Variables'] = []

        for varNum in range(self.fileData['FileHeader']['NumVars']):
            var = {'Header': self._ReadVarHeader()}
            self.fileData['Variables'].append(var)

        self._ReadData()

        self.theFile.close()
        return self.fileData

    def _ReadData(self):
        for var in self.fileData['Variables']:
            self.theFile.seek(var['Header']['DataOffset'])
            varType = var['Header']['Type']
            methods = [self._ReadTimestamps, self._ReadTimestamps, self._ReadIntervals,
                       self._ReadWaveforms, self._ReadPopVectors, self._ReadContinuous, self._ReadMarker ]
            methods[varType](var)

    def _ToString(self, theBytes, discardAfterFirstZero=False):
        """Converts bytes read from a file to a string.
           Some .nex file writers may write garbage after the zero-terminaged string.
           We need to discard this garbage before converting bytes to string.
        """
        if discardAfterFirstZero:
            theBytesBeforeZero = theBytes.split(b'\0', 1)[0]
            try:
                return theBytesBeforeZero.decode('ascii')
            except:
                return theBytesBeforeZero.decode('ascii', errors='replace')

        try:
            str = theBytes.decode('utf-8').strip('\x00')
            return str
        except:
            # try to discard garbage after zero and then decode
            theBytesBeforeZero = theBytes.split(b'\0', 1)[0]
            try:
                return theBytesBeforeZero.decode('utf-8').strip('\x00')
            except:
                return theBytesBeforeZero.decode('utf-8', errors='replace').strip('\x00')

    def _ReadNex5FileHeader(self):
        fileHeaderFormat = '<i i 256s d q i Q q 56s'
        headerBytes = self.theFile.read(struct.calcsize(fileHeaderFormat))
        if len(headerBytes) < struct.calcsize(fileHeaderFormat):
            raise ValueError("unable to read file header")
        fhValues = struct.unpack(fileHeaderFormat, headerBytes)
        keys = ['MagicNumber', 'NexFileVersion', 'Comment', 'Frequency', 'Beg', 'NumVars', 'MetaOffset', 'End', 'Padding']
        fileHeader = dict(zip(keys, fhValues))
        del fileHeader['Padding']

        if fileHeader['MagicNumber'] != 894977358 or fileHeader['Frequency'] <= 0:
            raise ValueError('Invalid .nex5 file')

        fileHeader['Comment'] = self._ToString(fileHeader['Comment'])
        self.tsFreq = fileHeader['Frequency']
        self.fromTicksToSeconds = 1.0 / self.tsFreq
        fileHeader['Beg'] /= self.tsFreq
        fileHeader['End'] /= self.tsFreq
        return fileHeader

    def _ReadFileHeader(self):
        fileHeaderFormat = '<i i 256s d i i i 260s'
        headerBytes = self.theFile.read(struct.calcsize(fileHeaderFormat))
        if len(headerBytes) < struct.calcsize(fileHeaderFormat):
            raise ValueError("unable to read file header")
        fhValues = struct.unpack(fileHeaderFormat, headerBytes)
        keys = ['MagicNumber', 'NexFileVersion', 'Comment', 'Frequency', 'Beg', 'End', 'NumVars', 'Padding']
        fileHeader = dict(zip(keys, fhValues))
        del fileHeader['Padding']

        if fileHeader['MagicNumber'] != 827868494 or fileHeader['Frequency'] <= 0:
            raise ValueError('Invalid .nex file')

        fileHeader['Comment'] = self._ToString(fileHeader['Comment'], True)
        self.tsFreq = fileHeader['Frequency']
        self.fromTicksToSeconds = 1.0 / self.tsFreq
        fileHeader['Beg'] /= self.tsFreq
        fileHeader['End'] /= self.tsFreq
        return fileHeader

    def _ReadVarHeader(self):
        varHeaderFormat = '<i i 64s i i i i i i d d d d i i i d d 52s'
        varHeaderSize = struct.calcsize(varHeaderFormat)
        headerBytes = self.theFile.read(varHeaderSize)
        if len(headerBytes) != varHeaderSize:
            raise ValueError('unable to read variable header')
        vhValues = struct.unpack(varHeaderFormat, headerBytes)
        keys = ['Type', 'Version', 'Name', 'DataOffset', 'Count', 'Wire', 'Unit', 'Gain', 'Filter', 'XPos', 'YPos',
                'SamplingRate', 'ADtoMV', 'NPointsWave', 'NMarkers', 'MarkerLength', 'MVOffset', 'PreThrTime', 'Padding']
        varHeader = dict(zip(keys, vhValues))
        del varHeader['Padding']

        varHeader['Name'] = self._ToString(varHeader['Name'], True)
        # add fields that are in nex5 only
        varHeader['TsDataType'] = 0
        varHeader['ContDataType'] = 0
        varHeader['ContFragIndexType'] = 0
        varHeader['MarkerDataType'] = 0
        return varHeader

    def _ReadNex5VarHeader(self):
        varHeaderFormat = '<i i 64s Q Q i i d 32s d d Q d i i i i 60s'
        varHeaderSize = struct.calcsize(varHeaderFormat)
        headerBytes = self.theFile.read(varHeaderSize)
        if len(headerBytes) != varHeaderSize:
            raise ValueError('unable to read variable header')
        vhValues = struct.unpack(varHeaderFormat, headerBytes)
        keys = ['Type', 'Version', 'Name', 'DataOffset', 'Count', 'TsDataType', 'ContDataType', 'SamplingRate', 'Units',
                'ADtoMV', 'MVOffset', 'NPointsWave', 'PreThrTime', 'MarkerDataType', 'NMarkers', 'MarkerLength',
                'ContFragIndexType', 'Padding']
        varHeader = dict(zip(keys, vhValues))
        del varHeader['Padding']
        if varHeader['Version'] != 500:
            raise ValueError("invalid variable header version")

        varHeader['Name'] = self._ToString(varHeader['Name'])
        varHeader['Units'] = self._ToString(varHeader['Units'])
        if varHeader['ContDataType'] == 1:
            varHeader['ADtoMV'] = 1
            varHeader['MVOffset'] = 0
        return varHeader

    def _ReadTimestamps(self, var):
        tsValueType = DataFormat.INT32
        if var['Header']['TsDataType'] == 1:
                tsValueType = DataFormat.INT64
        var['Timestamps'] = self._ReadAndScaleValues(tsValueType, var['Header']['Count'], self.tsFreq, True)

    def _ReadAndScaleValuesUsingNumpy(self, valueType, count, coeff=1.0, divide=False):
        import numpy as np
        numpyTypes = [np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64, np.float32, np.float64]
        numpyType = numpyTypes[valueType]
        values = np.fromfile(self.theFile, numpyType, count)
        if len(values) != count:
            raise ValueError("unable to read all values")
        if coeff == 1.0:
            return values
        if divide:
            return values / coeff
        else:
            return values * coeff

    def _ReadAndScaleValues(self, valueType, count, coeff=1.0, divide=False):
        if self.useNumpy:
            return self._ReadAndScaleValuesUsingNumpy(valueType, count, coeff, divide)
        nb = DataFormat.NumBytesPerItem(valueType)
        vtype = DataFormat.StructTypeFromDataType(valueType)
        valuesBytes = self.theFile.read(nb*count)
        if len(valuesBytes) != nb*count:
            raise ValueError("unable to read all values")
        x = struct.unpack(vtype*count, valuesBytes)
        valuesAsList = list(x)
        if coeff == 1.0:
            return valuesAsList
        if divide:
            return [x / coeff for x in valuesAsList]
        else:
            return [x * coeff for x in valuesAsList]

    def _ReadIntervals(self, var):
        if var['Header']['Count'] == 0:
            var['Intervals'] = [[], []]
            return
        tsValueType = DataFormat.INT32
        if var['Header']['TsDataType'] == 1:
            tsValueType = DataFormat.INT64
        intStarts = self._ReadAndScaleValues(tsValueType, var['Header']['Count'], self.tsFreq, True)
        intEnds = self._ReadAndScaleValues(tsValueType, var['Header']['Count'], self.tsFreq, True)
        var['Intervals'] = [intStarts, intEnds]

    def _ReadWaveforms(self, var):
        if var['Header']['Count'] == 0:
            var['WaveformValues'] = []
            return
        if var['Header']['NPointsWave'] <= 0:
            raise ValueError('invalid waveform header: NPointsWave is not positive')
        self._ReadTimestamps(var)
        wfValueType = DataFormat.INT16
        coeff = var['Header']['ADtoMV']
        woffset = var['Header']['MVOffset']
        if var['Header']['ContDataType'] == 1:
            wfValueType = DataFormat.FLOAT32
            coeff = 1.0
            woffset = 0.0

        wf = self._ReadAndScaleValues(wfValueType, var['Header']['Count'] * var['Header']['NPointsWave'], coeff)
        if self.useNumpy:
            import numpy as np
            var['WaveformValues'] = wf.reshape( var['Header']['Count'], var['Header']['NPointsWave'])
            if woffset != 0:
                var['WaveformValues'] = var['WaveformValues'] + woffset
        else:
            if woffset != 0:
                var['WaveformValues'] = list(self._Chunks([x + woffset for x in wf], var['Header']['NPointsWave']))
            else:
                var['WaveformValues'] = list(self._Chunks(wf, var['Header']['NPointsWave']))

    def _Chunks(self, theList, n):
        """Yield successive n-sized chunks from l."""
        for i in range(0, len(theList), n):
            yield theList[i:i + n]

    def _ReadPopVectors(self, var):
        var['Weights'] = self._ReadAndScaleValues(DataFormat.FLOAT64, var['Header']['Count'])

    def _ReadContinuous(self, var):
        tsValueType = DataFormat.INT32
        if var['Header']['TsDataType'] == 1:
            tsValueType = DataFormat.INT64
        var['FragmentTimestamps'] = self._ReadAndScaleValues(tsValueType, var['Header']['Count'], self.tsFreq, True)
        indexValueType = DataFormat.UINT32
        if var['Header']['ContFragIndexType'] == 1:
            indexValueType = DataFormat.UINT64
        var['FragmentIndexes'] = self._ReadAndScaleValues(indexValueType, var['Header']['Count'])
        var['FragmentCounts'] = []
        for frag in range(len(var['FragmentIndexes'])):
            if frag < var['Header']['Count'] - 1:
                count = var['FragmentIndexes'][frag+1] - var['FragmentIndexes'][frag]
            else:
                count = var['Header']['NPointsWave'] - var['FragmentIndexes'][frag]
            var['FragmentCounts'].append(count)

        coeff = var['Header']['ADtoMV']
        woffset = var['Header']['MVOffset']

        contValueType = DataFormat.INT16

        if var['Header']['ContDataType'] == 1:
            contValueType = DataFormat.FLOAT32
            coeff = 1.0
            woffset = 0.0
        var['ContinuousValues'] = self._ReadAndScaleValues(contValueType, var['Header']['NPointsWave'], coeff)
        if woffset != 0:
            var['ContinuousValues'] = [x + woffset for x in var['ContinuousValues']]

    def _ReadMarker(self, var):
        self._ReadTimestamps(var)
        var['Fields'] = []
        var['MarkerFieldNames'] = []
        var['Markers'] = []
        isNumeric = True
        for field in range(var['Header']['NMarkers']):
            # marker names and values should be ASCII strings
            field = {'Name': self._ToString(self.theFile.read(64), True).strip()}
            var['MarkerFieldNames'].append(field['Name'])
            if var['Header']['MarkerDataType'] == 0:
                length = var['Header']['MarkerLength']
                field['Markers'] = [self._ToString(self.theFile.read(length), True) for m in range(var['Header']['Count'])]
                if isNumeric:
                    for m in field['Markers']:
                        try:
                            n = int(m)
                        except ValueError:
                            isNumeric = False
                            break

                var['Fields'].append(field)
            else:
                field['Markers'] = self._ReadAndScaleValues(DataFormat.UINT32, var['Header']['Count'])
                var['Fields'].append(field)
        # convert to numbers if all the fields that contain only numbers to have the same values as in nex python interface
        if isNumeric:
            for f in var['Fields']:
                # we can use int (not uint that does not exist in Python anyway) since integers have no limit in Python
                f['Markers'] = [int(m) for m in f['Markers']]
        for f in var['Fields']:
            var['Markers'].append(f['Markers'])

class NexWriter(object):
    """
    Nex file writer class.
    Sample code:

    import nex
    w = nex.NexWriter(100000)
    w.fileData['FileHeader']['Comment'] = 'this is a comment'
    w.AddNeuron('neuron1', [1, 2, 3, 4])
    w.AddContVarWithSingleFragment('cont1', 2, 10000, [5, 6, 7, 8])
    w.WriteNexFile('C:\\Data\\testFileWrittenInPython.nex')
    w.WriteNex5File('C:\\Data\\testFileWrittenInPython.nex5', 1)

    """
    def __init__(self, timestampFrequency, useNumpy=False):
        """
        Constructor
        :param timestampFrequency: timestamp frequency in Hertz. Timestamps are stored as integers representing
                number of ticks, where tick = 1.0/timestampFrequency
        :param useNumpy: option to use numpy
        """
        self.theFile = None
        self.tsFreq = timestampFrequency
        self.useNumpy = useNumpy
        self.fromTicksToSeconds = 1.0/timestampFrequency

        self.varHeaderKeys = ['Type', 'Version', 'Name', 'DataOffset', 'Count', 'Wire', 'Unit', 'Gain', 'Filter', 'XPos', 'YPos',
                'SamplingRate', 'ADtoMV', 'NPointsWave', 'NMarkers', 'MarkerLength', 'MVOffset', 'PreThrTime', 'Padding']

        self.nex5VarHeaderKeys = ['Type', 'Version', 'Name', 'DataOffset', 'Count', 'TsDataType', 'ContDataType',
                                  'SamplingRate', 'Units',
                'ADtoMV', 'MVOffset', 'NPointsWave', 'PreThrTime', 'MarkerDataType', 'NMarkers', 'MarkerLength',
                'ContFragIndexType', 'Padding']

        self.fileHeaderKeys = ['MagicNumber', 'NexFileVersion', 'Comment', 'Frequency', 'Beg', 'End', 'NumVars', 'Padding']
        self.fileHeaderKeysToWrite = ['MagicNumber', 'NexFileVersion', 'Comment', 'Frequency', 'BegTicks', 'EndTicks', 'NumVars', 'Padding']
        self.nex5fileHeaderKeys = ['MagicNumber', 'NexFileVersion', 'Comment', 'Frequency', 'Beg', 'NumVars', 'MetaOffset', 'End',
                'Padding']
        self.nex5fileHeaderKeysToWrite = ['MagicNumber', 'NexFileVersion', 'Comment', 'Frequency', 'BegTicks', 'NumVars',
                                   'MetaOffset', 'EndTicks', 'Padding']
        # we will add nex5 file header keys (add MetaOffset)
        fhValues = [827868494, 106, '', timestampFrequency, 0, 0, 0, 0, '']
        fileHeader = dict(zip(self.nex5fileHeaderKeys, fhValues))
        self.fileData = {'FileHeader': fileHeader, 'Variables': []}

    def AddNeuron(self, name, timestamps, wire=0, unit=0, xpos=0, ypos=0):
        """
        Adds neuron file variable
        :param name: neuron name
        :param timestamps: list of timestamps in seconds or numpy array if numpy option is specified in constructor
        :param wire: wire (electrode) number
        :param unit: unit number
        :param xpos: x position in [0, 100] range (used in 3d displays)
        :param ypos: y position in [0, 100] range (used in 3d displays)
        :return: none
        """
        if self.useNumpy:
            self._VerifyIsNumpyArray('Timestamps', timestamps)
        vhValues = [NexFileVarType.NEURON, 100, name, 0, len(timestamps), wire, unit, 0, 0, xpos, ypos, 0, 0, 0, 0, 0, 0, 0, '']
        var = {'Header': dict(zip(self.varHeaderKeys, vhValues))}
        self._AddNex5VarHeaderFields(var)
        var['Timestamps'] = timestamps
        self.fileData['Variables'].append(var)

    def AddEvent(self, name, timestamps):
        """
        Adds event file variable
        :param name: event name
        :param timestamps: list of timestamps in seconds or numpy array if numpy option is specified in constructor
        :return: none
        """
        if self.useNumpy:
            self._VerifyIsNumpyArray('Timestamps', timestamps)
        vhValues = [NexFileVarType.EVENT, 100, name, 0, len(timestamps), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '']
        var = {'Header': dict(zip(self.varHeaderKeys, vhValues))}
        self._AddNex5VarHeaderFields(var)
        var['Timestamps'] = timestamps
        self.fileData['Variables'].append(var)

    def AddIntervalVariable(self, name, intStarts, intEnds):
        """
        Adds interval variable to file data
        :param name: variable name
        :param intStarts: list interval starts in seconds or numpy array if numpy option is specified in constructor
        :param intEnds: list interval ends in seconds or numpy array if numpy option is specified in constructor
        :return: none
        """
        if self.useNumpy:
            self._VerifyIsNumpyArray('Interval starts', intStarts)
            self._VerifyIsNumpyArray('Interval ends', intEnds)
        vhValues = [NexFileVarType.INTERVAL, 100, name, 0, len(intStarts), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '']
        var = {'Header': dict(zip(self.varHeaderKeys, vhValues))}
        self._AddNex5VarHeaderFields(var)
        var['Intervals'] = [intStarts, intEnds]
        self.fileData['Variables'].append(var)

    def AddContVarWithSingleFragment(self, name, timestampOfFirstDataPoint, SamplingRate, values):
        """
        Adds continuous variable with a single fragment
        :param name: variable name
        :param timestampOfFirstDataPoint: time of first data point in seconds
        :param SamplingRate: sampling rate in Hz
        :param values: list of variable values in mV or numpy array of values if numpy option is specified in constructor
        :return: none
        """
        if self.useNumpy:
            self._VerifyIsNumpyArray('Values', values)
        if SamplingRate <= 0 or SamplingRate > self.tsFreq:
            raise ValueError('invalid sampling rate in continuous')
        vhValues = [NexFileVarType.CONTINUOUS, 100, name, 0, 1, 0, 0, 0, 0, 0, 0, SamplingRate, 1.0,
                    len(values), 0, 0, 0, 0, '']
        var = {'Header': dict(zip(self.varHeaderKeys, vhValues))}
        self._AddNex5VarHeaderFields(var)
        var['Timestamps'] = [timestampOfFirstDataPoint]
        var['FragmentIndexes'] = [0]
        var['FragmentCounts'] = [len(values)]
        if self.useNumpy:
            import numpy as np
            var['Timestamps'] = np.array(var['Timestamps'])
            var['FragmentIndexes'] = np.array(var['FragmentIndexes'])
            var['FragmentCounts'] = np.array(var['FragmentCounts'])
        var['ContinuousValues'] = values
        self.fileData['Variables'].append(var)

    def AddContVarWithMultipleFragments(self, name, timestamps, SamplingRate, fragmentValues):
        """
        Adds continuous variable with multiple fragments
        :param name: variable name
        :param timestamps: list fragment start times in seconds or numpy array of fragment start times
                     if numpy option is specified in constructor
        :param SamplingRate: sampling rate in Hz
        :param fragmentValues: list of lists, each sublist is array data values in mV (or list of numpy arrays)
        :return:
        """
        if self.useNumpy:
            self._VerifyIsNumpyArray('Timestamps', timestamps)
            for fragment in fragmentValues:
                self._VerifyIsNumpyArray('FragmentValues', fragment)
        if SamplingRate <= 0 or SamplingRate > self.tsFreq:
            raise ValueError('invalid sampling rate in continuous')
        if len(timestamps) != len(fragmentValues):
            raise ValueError('should be the same number of timestamps and fragments')
        totalValues = 0
        var = {'FragmentIndexes': [], 'FragmentCounts': [], 'ContinuousValues': []}
        for fragment in fragmentValues:
            var['FragmentIndexes'].append(totalValues)
            var['FragmentCounts'].append(len(fragment))
            if not self.useNumpy:
                var['ContinuousValues'].extend(fragment)
            totalValues += len(fragment)
        if self.useNumpy:
            import numpy as np
            var['ContinuousValues'] = np.array([])
            for fragment in fragmentValues:
                var['ContinuousValues'] = np.append(var['ContinuousValues'], fragment)

        vhValues = [NexFileVarType.CONTINUOUS, 100, name, 0, 1, 0, 0, 0, 0, 0, 0, SamplingRate, 1.0,
                    totalValues, 0, 0, 0, 0, '']
        var['Header'] = dict(zip(self.varHeaderKeys, vhValues))
        self._AddNex5VarHeaderFields(var)
        var['Timestamps'] = timestamps

        if self.useNumpy:
            import numpy as np
            var['FragmentIndexes'] = np.array(var['FragmentIndexes'])
            var['FragmentCounts'] = np.array(var['FragmentCounts'])

        self.fileData['Variables'].append(var)

    def AddMarker(self, name, timestamps, fieldNames, markerFields):
        """
        Adds marker variable.
        :param name: variable name
        :param timestamps: list of timestamps in seconds or numpy array if numpy option is specified in constructor
        :param fieldNames: list of field names
        :param markerFields: a list of lists (one list of numbers or strings per marker field)
        :return:
        """
        if self.useNumpy:
            self._VerifyIsNumpyArray('Timestamps', timestamps)
        if len(fieldNames) != len(markerFields):
            raise ValueError('different number of field names and field values')
        for x in markerFields:
            if len(x) != len(timestamps):
                raise ValueError('different number of timestamps and field values in a single field')
        vhValues = [NexFileVarType.MARKER, 100, name, 0, len(timestamps), 0, 0, 0, 0, 0, 0, 0, 0, 0, len(fieldNames), 6, 0, 0, '']
        var = {'Header': dict(zip(self.varHeaderKeys, vhValues))}
        self._AddNex5VarHeaderFields(var)
        var['Header']['MarkerDataType'] = 1
        var['Timestamps'] = timestamps
        var['MarkerFieldNames'] = fieldNames
        var['Markers'] = markerFields
        self._CalcMarkerLength(var)
        self.fileData['Variables'].append(var)

    def AddWave(self, name, timestamps, SamplingRate, WaveformValues, NPointsWave=0, PrethresholdTimeInSeconds=0, wire=0, unit=0):
        """
        Adds waveform variable.
        :param name: variable name
        :param timestamps: list of timestamps in seconds or numpy array if numpy option is specified in constructor
        :param SamplingRate: sampling rate of w/f values in Hz
        :param WaveformValues: a list of lists, each sublist contains values of a single waveform in mV;
               if numpy option is specified in constructor, numpy matrix
        :param NPointsWave: number of data points in each wave
        :param PrethresholdTimeInSeconds: pre-threshold time in seconds
        :param wire: wire (electrode) number
        :param unit: unit number
        :return:
        """
        if self.useNumpy:
            self._VerifyIsNumpyArray('Timestamps', timestamps)
            self._VerifyIsNumpyArray('WaveformValues', WaveformValues)
        if len(timestamps) != len(WaveformValues):
            raise ValueError('different number of timestamps and number of waveforms')
        if SamplingRate <= 0 or SamplingRate > self.tsFreq:
            raise ValueError('invalid sampling rate in wave')
        var = {}
        if len(WaveformValues) > 0 and len(WaveformValues[0]) > 0:
            NPointsWave = len(WaveformValues[0])
        if NPointsWave == 0:
            raise ValueError('invalid number of data points in wave')
        vhValues = [NexFileVarType.WAVEFORM, 100, name, 0, len(timestamps), wire, unit, 0, 0, 0, 0, SamplingRate, 1.0, NPointsWave, 0, 0, 0, PrethresholdTimeInSeconds, '']
        var['Header'] = dict(zip(self.varHeaderKeys, vhValues))
        self._AddNex5VarHeaderFields(var)
        var['Timestamps'] = timestamps
        var['WaveformValues'] = WaveformValues
        self.fileData['Variables'].append(var)

    def WriteNexFile(self, filePath):
        """
        Writes file data as .nex file.
        :param filePath: full path of file
        :return: none
        """
        self.theFile = open(filePath, 'wb')
        self.fileData['FileHeader']['MagicNumber'] = 827868494
        self.fileData['FileHeader']['NexFileVersion'] = 106
        numberOfVariables = len(self.fileData['Variables'])
        self.fileData['FileHeader']['NumVars'] = numberOfVariables

        maxTs = self._MaximumTimestamp()
        if round(maxTs * self.tsFreq) > pow(2, 31):
            raise ValueError('unable to save as .nex file: max timestamp exceeds 32-bit range; you can save as .nex5 file instead')
        self.fileData['FileHeader']['BegTicks'] = int(round(self.fileData['FileHeader']['Beg'] * self.tsFreq))
        self.fileData['FileHeader']['EndTicks'] = int(round(maxTs * self.tsFreq))

        for v in self.fileData['Variables']:
            v['Header']['TsDataType'] = 0
            v['Header']['Version'] = 102
            v['Header']['ContDataType'] = 0
            v['Header']['MarkerDataType'] = 0
            self._CalculateScaling(v)
            self._CalcMarkerLength(v)

        dataOffset = 544 + numberOfVariables*208
        for v in self.fileData['Variables']:
            v['Header']['Count'] = self._VarCount(v)
            v['Header']['DataOffset'] = dataOffset
            dataOffset += self._VarNumDataBytes(v)

        fileHeaderFormat = '<i <i 256s <d <i <i <i 260s'.split()
        for i in range(len(self.fileHeaderKeysToWrite)):
            self._WriteField(fileHeaderFormat[i], self.fileData['FileHeader'][self.fileHeaderKeysToWrite[i]])

        varHeaderFormat = '<i <i 64s <i <i <i <i <i <i <d <d <d <d <i <i <i <d <d 52s'.split()
        for v in self.fileData['Variables']:
            for i in range(len(self.varHeaderKeys)):
                self._WriteField(varHeaderFormat[i], v['Header'][self.varHeaderKeys[i]])

        for v in self.fileData['Variables']:
            self._VarWriteData(v)

        self.theFile.close()

    def WriteNex5File(self, filePath, saveContValuesAsFloats=0):
        """
        Writes file data as .nex5 file.
        :param filePath: full path of file
        :param saveContValuesAsFloats: if zero, continuous values are saved as 16-bit integers; if 1, saved as floats
        :return:
        """
        self.theFile = open(filePath, 'wb')
        self.fileData['FileHeader']['MagicNumber'] = 894977358
        self.fileData['FileHeader']['NexFileVersion'] = 501
        nvars = len(self.fileData['Variables'])
        self.fileData['FileHeader']['NumVars'] = nvars

        maxTs = self._MaximumTimestamp()
        tsAs64 = 0
        if round(maxTs * self.tsFreq) > pow(2, 31):
            tsAs64 = 1
            self.fileData['FileHeader']['NexFileVersion'] = 502

        for v in self.fileData['Variables']:
            v['Header']['TsDataType'] = tsAs64
            v['Header']['Version'] = 500
            v['Header']['ContDataType'] = saveContValuesAsFloats
            if v['Header']['Type'] == NexFileVarType.MARKER:
                self._CalcMarkerLength(v)
                if v['AllNumbers']:
                    v['Header']['MarkerDataType'] = 1
                else:
                    v['Header']['MarkerDataType'] = 0
            self._CalculateScaling(v)

        self.fileData['FileHeader']['BegTicks'] = int(round(self.fileData['FileHeader']['Beg'] * self.tsFreq))
        self.fileData['FileHeader']['EndTicks'] = int(round(maxTs * self.tsFreq))

        dataOffset = 356 + nvars * 244
        for v in self.fileData['Variables']:
            v['Header']['Count'] = self._VarCount(v)
            v['Header']['DataOffset'] = dataOffset
            dataOffset += self._VarNumDataBytes(v)

        fileHeaderFormat = '<i <i 256s <d <q <i <Q <q 56s'.split()
        for i in range(len(self.nex5fileHeaderKeysToWrite)):
            self._WriteField(fileHeaderFormat[i], self.fileData['FileHeader'][self.nex5fileHeaderKeysToWrite[i]])

        varHeaderFormat = '<i <i 64s <Q <Q <i <i <d 32s <d <d <Q <d <i <i <i <i 60s'.split()
        for v in self.fileData['Variables']:
            for i in range(len(self.nex5VarHeaderKeys)):
                self._WriteField(varHeaderFormat[i], v['Header'][self.nex5VarHeaderKeys[i]])

        for v in self.fileData['Variables']:
            self._VarWriteData(v)

        metaData = {"file": {}, 'variables': []}
        metaData["file"]["writerSoftware"] = {}
        metaData["file"]["writerSoftware"]["name"] = 'nexfile.py'
        metaData["file"]["writerSoftware"]["version"] = 'Sep-11-2019'
        for v in self.fileData['Variables']:
            varMeta = {'name': v['Header']['Name']}
            if v['Header']['Type'] == NexFileVarType.NEURON or v['Header']['Type'] == NexFileVarType.WAVEFORM:
                varMeta['unitNumber'] = v['Header']['Unit']
                varMeta['probe'] = {}
                varMeta['probe']['wireNumber'] = v['Header']['Wire']
                varMeta['probe']['position'] = {}
                varMeta['probe']['position']['x'] = v['Header']['XPos']
                varMeta['probe']['position']['y'] = v['Header']['YPos']
            metaData['variables'].append(varMeta)

        metaString = json.dumps(metaData).encode('utf-8')
        pos = self.theFile.tell()
        self.theFile.write(metaString)
        metaPosInHeader = 284
        self.theFile.seek(metaPosInHeader, 0)
        self.theFile.write(struct.pack('<Q', pos))

        self.theFile.close()

    # the following class methods are internal
    def _VerifyIsNumpyArray(self, name, a):
        import numpy as np
        if not isinstance(a, np.ndarray):
            raise ValueError(name + ' should be a numpy array')

    def _ConvertStringToBytesIfNeeded(self, stringOrBytes):
        if isinstance(stringOrBytes, bytes):
            return stringOrBytes
        else:
            return stringOrBytes.encode('utf-8')

    def _WriteField(self, theFormat, theField):
        if theFormat.endswith('s'):
            theField = self._ConvertStringToBytesIfNeeded(theField)
        self.theFile.write(struct.pack(theFormat, theField))

    def _WriteList(self, theList, valueType):
        vtype = DataFormat.StructTypeFromDataType(valueType)
        self.theFile.write(struct.pack(vtype*len(theList), *theList))

    def _AddNex5VarHeaderFields(self, var):
        """
        Adds .nex5 variable header fields.
        :param var: file data variable
        :return: none
        """
        var['Header']['TsDataType'] = 0
        var['Header']['ContDataType'] = 0
        var['Header']['Units'] = ''
        var['Header']['MarkerDataType'] = 0
        var['Header']['ContFragIndexType'] = 0

    def _BytesInTimestamp(self, var):
        """
        Calculates number of bytes in timestamp.
        :param var: file data variable
        :return: number of bytes in timestamp
        """
        if var['Header']['TsDataType'] == 0:
            return 4
        else:
            return 8

    def _BytesInContValue(self, var):
        """
        :param var:  file data variable
        :return: number of bytes in continuous value
        """
        if var['Header']['ContDataType'] == 0:
            return 2
        else:
            return 4

    def _VarNumDataBytes(self, var):
        """
        :param var: file data variable
        :return: number of bytes in variable data
        """
        varType = var['Header']['Type']
        if varType == NexFileVarType.NEURON or varType == NexFileVarType.EVENT:
            return self._BytesInTimestamp(var) * len(var['Timestamps'])
        elif varType == NexFileVarType.INTERVAL:
            return 2 * self._BytesInTimestamp(var) * len(var['Intervals'][0])
        elif varType == NexFileVarType.WAVEFORM:
            return self._BytesInTimestamp(var) * len(var['Timestamps']) + self._BytesInContValue(var) * len(var['Timestamps']) * var['Header']['NPointsWave']
        elif varType == NexFileVarType.POPULATION_VECTOR:
            # population vectors are not supported in this writer class
            return 0
        elif varType == NexFileVarType.CONTINUOUS:
            return (self._BytesInTimestamp(var) + 4) * len(var['Timestamps']) + self._BytesInContValue(var) * len(var['ContinuousValues'])
        elif varType == NexFileVarType.MARKER:
            if var['Header']['MarkerDataType'] == 0:
                singleMarkerLength = var['Header']['MarkerLength']
            else:
                singleMarkerLength = 4
            nts = len(var['Timestamps'])
            nm = len(var['MarkerFieldNames'])
            return self._BytesInTimestamp(var) * nts + nm * (64 + nts * singleMarkerLength)

    def _VarCount(self, var):
        """
        Calculates count field for file variables
        :param var:
        :return:
        """
        varType = var['Header']['Type']
        if varType == NexFileVarType.NEURON or varType == NexFileVarType.EVENT:
            return len(var['Timestamps'])
        elif varType == NexFileVarType.INTERVAL:
            return len(var['Intervals'][0])
        elif varType == NexFileVarType.WAVEFORM:
            return len(var['Timestamps'])
        elif varType == NexFileVarType.POPULATION_VECTOR:
            return 0
        elif varType == NexFileVarType.CONTINUOUS:
            return len(var['Timestamps'])
        elif varType == NexFileVarType.MARKER:
            return len(var['Timestamps'])

    def _MaxOfNumpyArrayOrZero(self, x):
        import numpy as np
        if len(x) == 0:
            return 0
        else:
            return np.max(x)

    def _VarMaxTimestampNumpy(self, var):
        import numpy
        varType = var['Header']['Type']
        if varType == NexFileVarType.NEURON or varType == NexFileVarType.EVENT or varType == NexFileVarType.MARKER:
            return self._MaxOfNumpyArrayOrZero(var['Timestamps'])
        elif varType == NexFileVarType.INTERVAL:
            return self._MaxOfNumpyArrayOrZero(var['Intervals'][1])
        elif varType == NexFileVarType.WAVEFORM:
            if len(var['Timestamps']) == 0:
                return 0
            return self._MaxOfNumpyArrayOrZero(var['Timestamps']) + (var['Header']['NPointsWave']-1)/var['Header']['SamplingRate']
        elif varType == NexFileVarType.POPULATION_VECTOR:
            return 0
        elif varType == NexFileVarType.CONTINUOUS:
            if len(var['Timestamps']) == 0:
                return 0
            return var['Timestamps'][-1] + (var['FragmentCounts'][-1] - 1) / var['Header']['SamplingRate']

    def _MaxValueOrZero(self, theList):
        if len(theList) > 0:
            return max(theList)
        return 0

    def _VarMaxTimestamp(self, var):
        if self.useNumpy:
            return self._VarMaxTimestampNumpy(var)
        varType = var['Header']['Type']
        if varType == NexFileVarType.NEURON or varType == NexFileVarType.EVENT or varType == NexFileVarType.MARKER:
            return self._MaxValueOrZero(var['Timestamps'])
        elif varType == NexFileVarType.INTERVAL:
            return self._MaxValueOrZero(var['Intervals'][1])
        elif varType == NexFileVarType.WAVEFORM:
            if len(var['Timestamps']) == 0:
                return 0
            return max(var['Timestamps']) + (var['Header']['NPointsWave']-1)/var['Header']['SamplingRate']
        elif varType == NexFileVarType.POPULATION_VECTOR:
            return 0
        elif varType == NexFileVarType.CONTINUOUS:
            if len(var['Timestamps']) == 0:
                return 0
            return var['Timestamps'][-1] + (var['FragmentCounts'][-1] - 1) / var['Header']['SamplingRate']

    def _VarWriteTimestampsNumpy(self, var, timestamps):
        import numpy as np
        if self._BytesInTimestamp(var) == 4:
            np.round(timestamps * self.tsFreq).astype(np.int32).tofile(self.theFile)
        else:
            np.round(timestamps * self.tsFreq).astype(np.int64).tofile(self.theFile)

    def _VarWriteTimestamps(self, var, timestamps):
        if self.useNumpy:
            return self._VarWriteTimestampsNumpy(var, timestamps)
        if self._BytesInTimestamp(var) == 4:
            tsTicks = [int(round(x * self.tsFreq)) for x in timestamps]
            self._WriteList(tsTicks, DataFormat.INT32)
        else:
            if sys.version_info < (3,):
                tsTicks = [long(round(x * self.tsFreq)) for x in timestamps]
            else:
                tsTicks = [int(round(x * self.tsFreq)) for x in timestamps]
            self._WriteList(tsTicks, DataFormat.INT64)

    def _VarWriteWaveformsNumpy(self, var):
        import numpy as np
        if self._BytesInContValue(var) == 2:
            np.round(var['WaveformValues'] / var['Header']['ADtoMV']).astype(np.int16).tofile(self.theFile)
        else:
            var['WaveformValues'].astype(np.float32).tofile(self.theFile)

    def _VarWriteContinuousValuesNumpy(self, var):
        import numpy as np
        if self._BytesInContValue(var) == 2:
            np.round(var['ContinuousValues'] / var['Header']['ADtoMV']).astype(np.int16).tofile(self.theFile)
        else:
            var['ContinuousValues'].astype(np.float32).tofile(self.theFile)

    def _WriteWaveformVarData(self,var):
        self._VarWriteTimestamps(var, var['Timestamps'])
        if self.useNumpy:
            self._VarWriteWaveformsNumpy(var)
            return
        if self._BytesInContValue(var) == 2:
            for w in var['WaveformValues']:
                waveValues = [int(x / var['Header']['ADtoMV']) for x in w]
                self._WriteList(waveValues, DataFormat.INT16)
        else:
            for w in var['WaveformValues']:
                self._WriteList(w, DataFormat.FLOAT32)

    def _WriteContinuousVarData(self,var):
        self._VarWriteTimestamps(var, var['Timestamps'])
        if self.useNumpy:
            import numpy as np
            var['FragmentIndexes'].astype(np.uint32).tofile(self.theFile)
            self._VarWriteContinuousValuesNumpy(var)
        else:
            self._WriteList(var['FragmentIndexes'], DataFormat.UINT32)
            if self._BytesInContValue(var) == 2:
                contValues = [int(x / var['Header']['ADtoMV']) for x in var['ContinuousValues']]
                self._WriteList(contValues, DataFormat.INT16)
            else:
                self._WriteList(var['ContinuousValues'], DataFormat.FLOAT32)

    def _VarWriteData(self, var):
        varType = var['Header']['Type']
        if varType == NexFileVarType.NEURON or varType == NexFileVarType.EVENT:
            self._VarWriteTimestamps(var, var['Timestamps'])
            return
        elif varType == NexFileVarType.INTERVAL:
            for i in range(2):
                self._VarWriteTimestamps(var, var['Intervals'][i])
            return
        elif varType == NexFileVarType.WAVEFORM:
            self._WriteWaveformVarData(var)
            return
        elif varType == NexFileVarType.POPULATION_VECTOR:
            return
        elif varType == NexFileVarType.CONTINUOUS:
            self._WriteContinuousVarData(var)
            return
        elif varType == NexFileVarType.MARKER:
            self._VarWriteTimestamps(var, var['Timestamps'])
            for i, name in enumerate(var['MarkerFieldNames']):
                self._WriteField('64s', name)
                if var['Header']['MarkerDataType'] == 0:
                    for v in var['Markers'][i]:
                        # convert number to string if needed first
                        if isinstance(v, numbers.Number):
                            sv = '{0:05d}'.format(v) + '\x00'
                        else:
                            sv = v
                            while len(sv) < var['Header']['MarkerLength']:
                                sv += '\x00'
                        self.theFile.write(sv.encode('utf-8'))
                else:
                    self._WriteList(var['Markers'][i], DataFormat.UINT32)
            return

    def _CalcMarkerLength(self, var):
        if var['Header']['Type'] != NexFileVarType.MARKER:
            return
        maxStringLength = 0
        allNumbers = True
        for field in var['Markers']:
            for x in field:
                if not isinstance(x, numbers.Number):
                    allNumbers = False
                    if not isinstance(x, str):
                        raise ValueError('marker values should be either numbers or strings')
                    maxStringLength = max(maxStringLength, len(x))
        var['AllNumbers'] = allNumbers
        if allNumbers:
            var['Header']['MarkerLength'] = 6
        else:
            var['Header']['MarkerLength'] = max(6, maxStringLength + 1)

    def _MaximumTimestamp(self):
        maxTs = 0
        for v in self.fileData['Variables']:
            maxTs = max(maxTs, self._VarMaxTimestamp(v))
        return maxTs

    def _SignalAbsMaxNumPy(self, signal):
        import numpy as np
        return np.max(np.abs(signal))

    def _CalculateScaling(self, var):
        varType = var['Header']['Type']
        if varType == NexFileVarType.CONTINUOUS:
            if var['Header']['ContDataType'] == 1:
                var['Header']['ADtoMV'] = 1
                return
            contMax = 0
            if self.useNumpy:
                contMax = self._SignalAbsMaxNumPy(var['ContinuousValues'])
            else:
                contMax = max(contMax, abs(max(var['ContinuousValues'])))
                contMax = max(contMax, abs(min(var['ContinuousValues'])))
            if contMax == 0:
                var['Header']['ADtoMV'] = 1
            else:
                var['Header']['ADtoMV'] = contMax / 32767.0
            return
        if varType == NexFileVarType.WAVEFORM:
            if var['Header']['ContDataType'] == 1:
                var['Header']['ADtoMV'] = 1
                return
            waveMax = 0
            if self.useNumpy:
                waveMax = self._SignalAbsMaxNumPy(var['WaveformValues'])
            else:
                for w in var['WaveformValues']:
                    waveMax = max(waveMax, abs(max(w)))
                    waveMax = max(waveMax, abs(min(w)))
            if waveMax == 0:
                var['Header']['ADtoMV'] = 1
            else:
                var['Header']['ADtoMV'] = waveMax / 32767.0

max_stars_repo_name: NeuroExplorer/nex-Python-package

max_issues_repo_name: NeuroExplorer/nex-Python-package

max_forks_repo_name: NeuroExplorer/nex-Python-package


text:
import vtk
import numpy as np
import time
import LinearSubdivisionFilter as lsf

import argparse
import os

import PNS
import json

import tensorflow as tf

print("Tensorflow version:", tf.__version__)

def normalize_points(poly, radius=1.0):
	polypoints = poly.GetPoints()
	for pid in range(polypoints.GetNumberOfPoints()):
		spoint = polypoints.GetPoint(pid)
		spoint = np.array(spoint)
		norm = np.linalg.norm(spoint)
		spoint = spoint/norm * radius
		polypoints.SetPoint(pid, spoint)
	poly.SetPoints(polypoints)
	return poly

def CreateIcosahedron(radius, sl):
	icosahedronsource = vtk.vtkPlatonicSolidSource()
	icosahedronsource.SetSolidTypeToIcosahedron()
	icosahedronsource.Update()
	icosahedron = icosahedronsource.GetOutput()

	subdivfilter = lsf.LinearSubdivisionFilter()
	subdivfilter.SetInputData(icosahedron)
	subdivfilter.SetNumberOfSubdivisions(sl)
	subdivfilter.Update()

	icosahedron = subdivfilter.GetOutput()
	icosahedron = normalize_points(icosahedron, radius)

	return icosahedron

pns = PNS.PNS()

# lets create dummy {x,y,z} coordinates
rand_points = tf.cast(tf.abs(pns.NormalizeVectors(np.random.normal(size=[500, 10]))), tf.float32)

dimension = tf.shape(rand_points)[-1]

# if(args.continue_fitting):

#   min_dim = dimension

#   for d in range(dimension.numpy(), 2, -1):
#     fit_path = os.path.join(args.out, "sphere_fit_" + str(d) + ".json")

#     if(os.path.exists(fit_path)):
#       min_dim = d

#   fit_path = os.path.join(args.out, "sphere_fit_" + str(min_dim) + ".json")

#   if(os.path.exists(fit_path)):
#     print("Continue fitting at dimension", min_dim)
#     points = pns.GetFkPoints_g(encoded_images, min_dim)
# else:
#   points = encoded_images

points = rand_points
pns.Fit(points)

points = pns.GetFkPoints(3)
# points_ = pns.GetFkPoints_(3)
projected = pns.GetProjectedPoints(points=points)
circle_center_v1, angle_r1, rot_mat = pns.GetSubSphereFit(3)

rand_points_ = pns.GetFk_1Points(points)

print(rand_points, rand_points_)

# build points & polydata from numpy_to_vtk
vtk_points = vtk.vtkPoints()
vertices = vtk.vtkCellArray()

for p in points:
	pid = vtk_points.InsertNextPoint(p)
	vertices.InsertNextCell(1)
	vertices.InsertCellPoint(pid)

poly_rand = vtk.vtkPolyData()
poly_rand.SetPoints(vtk_points)
poly_rand.SetVerts(vertices)

# build points & polydata from numpy_to_vtk
# vtk_points_g = vtk.vtkPoints()
# vertices_g = vtk.vtkCellArray()

# for p in points_:
# 	pid = vtk_points_g.InsertNextPoint(p)
# 	vertices_g.InsertNextCell(1)
# 	vertices_g.InsertCellPoint(pid)

# poly_rand_g = vtk.vtkPolyData()
# poly_rand_g.SetPoints(vtk_points_g)
# poly_rand_g.SetVerts(vertices_g)

projected_points = vtk.vtkPoints()
projected_cells = vtk.vtkCellArray()

for p, prp in zip(points, projected):

	pid0 = projected_points.InsertNextPoint(p)
	pid1 = projected_points.InsertNextPoint(prp)

	vtk_line = vtk.vtkLine()
	vtk_line.GetPointIds().SetId(0, pid0)
	vtk_line.GetPointIds().SetId(1, pid1)
	projected_cells.InsertNextCell(vtk_line)

projected_poly = vtk.vtkPolyData()
projected_poly.SetPoints(projected_points)
projected_poly.SetLines(projected_cells)

circle_points = vtk.vtkPoints()
circle_cells = vtk.vtkCellArray()
circle_radius = np.sin(angle_r1)
lastpid = -1

for t in np.arange(0., 2*np.pi, np.pi/180):

	pcircle = np.array([circle_radius*np.cos(t), circle_radius*np.sin(t), np.cos(angle_r1)])

	pcircle = np.matmul(pcircle, np.transpose(rot_mat))

	pid = circle_points.InsertNextPoint(pcircle)

	if(lastpid != -1):
		vtk_line = vtk.vtkLine()
		vtk_line.GetPointIds().SetId(0, lastpid)
		vtk_line.GetPointIds().SetId(1, pid)
		lastpid = pid

		circle_cells.InsertNextCell(vtk_line);

	else:
		lastpid = pid

poly_circle = vtk.vtkPolyData()
poly_circle.SetPoints(circle_points)
poly_circle.SetLines(circle_cells)

poly_rand_mapper = vtk.vtkPolyDataMapper()
poly_rand_mapper.SetInputData(poly_rand)
poly_rand_actor = vtk.vtkActor()
poly_rand_actor.SetMapper(poly_rand_mapper)
# poly_rand_actor.GetProperty().SetRepresentationToPoints()
poly_rand_actor.GetProperty().SetColor(1.0, 0, 0)
poly_rand_actor.GetProperty().SetPointSize(20)

# poly_rand_g_mapper = vtk.vtkPolyDataMapper()
# poly_rand_g_mapper.SetInputData(poly_rand_g)
# poly_rand_g_actor = vtk.vtkActor()
# poly_rand_g_actor.SetMapper(poly_rand_g_mapper)
# # poly_rand_actor.GetProperty().SetRepresentationToPoints()
# poly_rand_g_actor.GetProperty().SetColor(0.2, 1.0, 0)
# poly_rand_g_actor.GetProperty().SetPointSize(10)

poly_circle_mapper = vtk.vtkPolyDataMapper()
poly_circle_mapper.SetInputData(poly_circle)
poly_circle_actor = vtk.vtkActor()
poly_circle_actor.SetMapper(poly_circle_mapper)
poly_circle_actor.GetProperty().SetColor(1.0, 0.5, 0)
poly_circle_actor.GetProperty().SetLineWidth(20)

circle_center_source = vtk.vtkLineSource()
circle_center_source.SetPoint1([0, 0, 0])
circle_center_source.SetPoint2(circle_center_v1)

circle_center_source_mapper = vtk.vtkPolyDataMapper()
circle_center_source_mapper.SetInputConnection(circle_center_source.GetOutputPort())
circle_center_source_actor = vtk.vtkActor()
circle_center_source_actor.SetMapper(circle_center_source_mapper)
circle_center_source_actor.GetProperty().SetColor(0, 0, 1.0)
circle_center_source_actor.GetProperty().SetLineWidth(5)

north_source = vtk.vtkLineSource()
north_source.SetPoint1([0, 0, 0])
north_source.SetPoint2([0, 0, 1])

north_source_mapper = vtk.vtkPolyDataMapper()
north_source_mapper.SetInputConnection(north_source.GetOutputPort())
north_source_actor = vtk.vtkActor()
north_source_actor.SetMapper(north_source_mapper)
north_source_actor.GetProperty().SetColor(0.0, 1.0, 0)
north_source_actor.GetProperty().SetLineWidth(10)

projected_points_mapper = vtk.vtkPolyDataMapper()
projected_points_mapper.SetInputData(projected_poly)
projected_actor = vtk.vtkActor()
projected_actor.SetMapper(projected_points_mapper)
projected_actor.GetProperty().SetColor(1.0, 1.0, 0)
projected_actor.GetProperty().SetLineWidth(10)

icosahedron = CreateIcosahedron(1, 10)

icosahedronmapper = vtk.vtkPolyDataMapper()
icosahedronmapper.SetInputData(icosahedron)
icosahedronactor = vtk.vtkActor()
icosahedronactor.SetMapper(icosahedronmapper)
icosahedronactor.GetProperty().SetRepresentationToWireframe()

ren1 = vtk.vtkRenderer()
renWin = vtk.vtkRenderWindow()
renWin.AddRenderer(ren1)
iren = vtk.vtkRenderWindowInteractor()
iren.SetRenderWindow(renWin)

ren1.AddActor(icosahedronactor)
ren1.AddActor(poly_rand_actor)
# ren1.AddActor(poly_rand_g_actor)
ren1.AddActor(poly_circle_actor)
ren1.AddActor(circle_center_source_actor)
ren1.AddActor(north_source_actor)
ren1.AddActor(projected_actor)

ren1.SetBackground(0,.5,1)
ren1.ResetCamera()
ren1.GetActiveCamera().Dolly(1.2)
ren1.ResetCameraClippingRange()
renWin.SetSize(1080, 960)
renWin.Render()
iren.Start()

max_stars_repo_name: juanprietob/minimal-geodesic

max_issues_repo_name: juanprietob/minimal-geodesic

max_forks_repo_name: juanprietob/minimal-geodesic

reason: import numpy,from numpy


text:
import typing as t
from rich import print

from .base_info import BaseInfo

class QueryError(RuntimeError):
    """
    Indicate that a query failed.
    """

try:
    import pycuda
    import pycuda.driver as cuda
    import pycuda.autoinit  # noqa F401
    is_CUDA_available = True
except (ModuleNotFoundError, ImportError):
    is_CUDA_available = False

class GpusInfo(BaseInfo):
    """
    Query any info on GPUs available for the users system (currently only supports NVIDIA GPUs)
    """
    def __init__(self):
        super().__init__()
        self.compute_capability_to_architecture = {
            2: 'Fermi',
            3: 'Kepler',
            5: 'Maxwell',
            6: 'Pascal',
            7: 'Volta',
            8: 'Ampere'
        }

    def query_gpus(self, **_) -> t.List[t.Mapping[str, t.Any]]:
        """
        Get information about all GPUs.
        """
        if not is_CUDA_available:
            print('[bold yellow]Unable to import package pycuda. GPU information may be limited.')
            return []

        gpus = []
        for i in range(cuda.Device.count()):
            device = cuda.Device(i)
            gpus.append(self.query_gpu(device))

        return gpus

    def query_gpu(self, device: 'cuda.Device') -> t.Mapping[str, t.Any]:
        """
        Get information about a given GPU.
        """
        attributes = device.get_attributes()
        compute_capability = device.compute_capability()
        multiprocessors = attributes[cuda.device_attribute.MULTIPROCESSOR_COUNT]
        cuda_cores = GpusInfo.calculate_cuda_cores(compute_capability, multiprocessors)
        try:
            try:
                compute_cap_arch = self.compute_capability_to_architecture[compute_capability[0]]
            # compute capability in case its not implemented in SI yet
            except KeyError:
                compute_cap_arch = "Unknown"
            return {
                'architecture': compute_cap_arch,
                'brand': device.name(),
                'compute_capability': str(float('.'.join(str(_) for _ in compute_capability))),
                'memory': GpusInfo.format_bytes(self, device.total_memory()),
                'memory_clock': GpusInfo.hz_to_hreadable_string(attributes[cuda.device_attribute.MEMORY_CLOCK_RATE]),
                'clock': GpusInfo.hz_to_hreadable_string(attributes[cuda.device_attribute.CLOCK_RATE]),
                'multiprocessors': str(multiprocessors),
                'cores': str(cuda_cores),
                'warp_size': str(attributes[cuda.device_attribute.WARP_SIZE])
            }
        except KeyError as err:
            raise QueryError(f'expected value not present among device attributes: {device.get_attributes()}') from err

    @staticmethod
    def calculate_cuda_cores(compute_capability: t.Tuple[int, int], multiprocessors: int) -> t.Optional[int]:
        """
        Calculate number of cuda cores according to Nvidia's specifications.
        """
        if compute_capability[0] == 2:  # Fermi
            if compute_capability[1] == 1:
                return multiprocessors * 48
            return multiprocessors * 32
        if compute_capability[0] == 3:  # Kepler
            return multiprocessors * 192
        if compute_capability[0] == 5:  # Maxwell
            return multiprocessors * 128
        if compute_capability[0] == 6:  # Pascal
            if compute_capability[1] == 0:
                return multiprocessors * 64
            if compute_capability[1] == 1:
                return multiprocessors * 128
        if compute_capability[0] == 7:  # Volta
            return multiprocessors * 64
        if compute_capability[0] == 8:  # Ampere
            return multiprocessors * 64
        return None

    def print_gpus_info(self, gpus_info: list):
        """
        Print info on any GPUs on the users system
        """
        if gpus_info:
            column_names = ['Architecture', 'Brand', 'Compute Capability', 'Memory', 'Memory Clock', 'Clock', 'Multiprocessors', 'Cores', 'Warp Size']
            self.init_table(title='Graphical Processing Unit', column_names=column_names)
            for gpu in gpus_info:
                self.table.add_row(*gpu.values())

            self.print_table()

max_stars_repo_name: mlf-core/system-intelligence

max_issues_repo_name: mlf-core/system-intelligence

max_forks_repo_name: mlf-core/system-intelligence

reason: import pycuda


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# MIT License
#
# Copyright (c) 2018 Virginia Tech Ground Station
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

import numpy
from gnuradio import gr
import json,time,os,pmt,binascii, math, datetime

class es_tag_to_utc(gr.sync_block):
    """
    Event Stream Time Stamp Generator

    uses the event stream 'es::event_time' tag and the 'rx_time' tag to generate an ISO 8601 timestamp.

    samp_rate is needed to convert the proper event time value.
    """
    def __init__(self, samp_rate):
        gr.sync_block.__init__(self,
            name="es_tag_to_utc",
            in_sig=[],
            out_sig=[])

        self.samp_rate = samp_rate
        self.message_port_register_in(pmt.intern('in'))
        self.set_msg_handler(pmt.intern('in'), self.handler)
        self.message_port_register_out(pmt.intern('out'))

    def work(self, input_items, output_items):
        assert(False)

    def handler(self, pdu):
        meta = pmt.to_python(pmt.car(pdu))
        samples = pmt.cdr(pdu)
        x = numpy.array(pmt.c32vector_elements(samples), dtype=numpy.complex64)
        #print meta
        #print meta['rx_time'], meta['es::event_time']
        time_delta = meta['es::event_time'] / self.samp_rate
        rx_int_sec = meta['rx_time'][1][0]
        rx_frac_sec = meta['rx_time'][1][1]
        #print rx_int_sec, rx_frac_sec
        #print time_delta, math.modf(time_delta)

        td = math.modf(time_delta)
        dt = (numpy.int64(rx_int_sec + td[1]) , rx_frac_sec + td[0])
        #print datetime
        ts = numpy.datetime64(datetime.datetime.utcfromtimestamp(dt[0]))
        ts = ts + numpy.timedelta64(int(dt[1]*1e9), 'ns')
        ts_ns = numpy.datetime_as_string(ts) +"Z"

        meta = pmt.to_pmt(meta)
        #meta = pmt.dict_add(meta, pmt.intern("pkt_rx_time"), pmt.make_tuple(dt));
        meta = pmt.dict_add(meta, pmt.intern("datetime"), pmt.string_to_symbol(ts_ns));
        #meta['pkt_rx_time'] = dt
        #meta['datetime'] = ts_ns

        samples_out = pmt.init_c32vector(len(x), map(lambda i: complex(i), x))
        cpdu = pmt.cons(meta,samples_out)
        self.message_port_pub(pmt.intern("out"), cpdu);

        #print meta

max_stars_repo_name: vt-gs/gr-vcc

max_issues_repo_name: vt-gs/gr-vcc

max_forks_repo_name: vt-gs/gr-vcc


text:
"""
Create a plot of where the integration window lays on the trace for the pixel
with the highest charge, its neighbours, and the pixel with the lowest max
charge and its neighbours. Also shows a disgram of which pixels count as a
neighbour, the camera image for the max charge timeslice, the true pe camera
image, and a calibrated camera image
"""

import numpy as np
from matplotlib import pyplot as plt
from traitlets import Dict, List, Int, Bool, Enum

from ctapipe.calib.camera.dl0 import CameraDL0Reducer
from ctapipe.calib.camera.dl1 import CameraDL1Calibrator
from ctapipe.calib.camera.r1 import CameraR1CalibratorFactory
from ctapipe.core import Tool
from ctapipe.image.charge_extractors import ChargeExtractorFactory
from ctapipe.io.eventseeker import EventSeeker
from ctapipe.io.eventsourcefactory import EventSourceFactory
from ctapipe.visualization import CameraDisplay

def plot(event, telid, chan, extractor_name):
    # Extract required images
    dl0 = event.dl0.tel[telid].waveform[chan]

    t_pe = event.mc.tel[telid].photo_electron_image
    dl1 = event.dl1.tel[telid].image[chan]
    max_time = np.unravel_index(np.argmax(dl0), dl0.shape)[1]
    max_charges = np.max(dl0, axis=1)
    max_pix = int(np.argmax(max_charges))
    min_pix = int(np.argmin(max_charges))

    geom = event.inst.subarray.tel[telid].camera
    nei = geom.neighbors

    # Get Neighbours
    max_pixel_nei = nei[max_pix]
    min_pixel_nei = nei[min_pix]

    # Get Windows
    windows = event.dl1.tel[telid].extracted_samples[chan]
    length = np.sum(windows, axis=1)
    start = np.argmax(windows, axis=1)
    end = start + length - 1

    # Draw figures
    ax_max_nei = {}
    ax_min_nei = {}
    fig_waveforms = plt.figure(figsize=(18, 9))
    fig_waveforms.subplots_adjust(hspace=.5)
    fig_camera = plt.figure(figsize=(15, 12))

    ax_max_pix = fig_waveforms.add_subplot(4, 2, 1)
    ax_min_pix = fig_waveforms.add_subplot(4, 2, 2)
    ax_max_nei[0] = fig_waveforms.add_subplot(4, 2, 3)
    ax_min_nei[0] = fig_waveforms.add_subplot(4, 2, 4)
    ax_max_nei[1] = fig_waveforms.add_subplot(4, 2, 5)
    ax_min_nei[1] = fig_waveforms.add_subplot(4, 2, 6)
    ax_max_nei[2] = fig_waveforms.add_subplot(4, 2, 7)
    ax_min_nei[2] = fig_waveforms.add_subplot(4, 2, 8)

    ax_img_nei = fig_camera.add_subplot(2, 2, 1)
    ax_img_max = fig_camera.add_subplot(2, 2, 2)
    ax_img_true = fig_camera.add_subplot(2, 2, 3)
    ax_img_cal = fig_camera.add_subplot(2, 2, 4)

    # Draw max pixel traces
    ax_max_pix.plot(dl0[max_pix])
    ax_max_pix.set_xlabel("Time (ns)")
    ax_max_pix.set_ylabel("DL0 Samples (ADC)")
    ax_max_pix.set_title(
        "(Max) Pixel: {}, True: {}, Measured = {:.3f}"
        .format(max_pix, t_pe[max_pix], dl1[max_pix])
    )
    max_ylim = ax_max_pix.get_ylim()
    ax_max_pix.plot([start[max_pix], start[max_pix]],
                    ax_max_pix.get_ylim(),
                    color='r',
                    alpha=1)
    ax_max_pix.plot([end[max_pix], end[max_pix]],
                    ax_max_pix.get_ylim(),
                    color='r',
                    alpha=1)
    for i, ax in ax_max_nei.items():
        if len(max_pixel_nei) > i:
            pix = max_pixel_nei[i]
            ax.plot(dl0[pix])
            ax.set_xlabel("Time (ns)")
            ax.set_ylabel("DL0 Samples (ADC)")
            ax.set_title(
                "(Max Nei) Pixel: {}, True: {}, Measured = {:.3f}"
                .format(pix, t_pe[pix], dl1[pix])
            )
            ax.set_ylim(max_ylim)
            ax.plot([start[pix], start[pix]],
                    ax.get_ylim(),
                    color='r',
                    alpha=1)
            ax.plot([end[pix], end[pix]], ax.get_ylim(), color='r', alpha=1)

    # Draw min pixel traces
    ax_min_pix.plot(dl0[min_pix])
    ax_min_pix.set_xlabel("Time (ns)")
    ax_min_pix.set_ylabel("DL0 Samples (ADC)")
    ax_min_pix.set_title(
        "(Min) Pixel: {}, True: {}, Measured = {:.3f}"
        .format(min_pix, t_pe[min_pix], dl1[min_pix])
    )
    ax_min_pix.set_ylim(max_ylim)
    ax_min_pix.plot([start[min_pix], start[min_pix]],
                    ax_min_pix.get_ylim(),
                    color='r',
                    alpha=1)
    ax_min_pix.plot([end[min_pix], end[min_pix]],
                    ax_min_pix.get_ylim(),
                    color='r',
                    alpha=1)
    for i, ax in ax_min_nei.items():
        if len(min_pixel_nei) > i:
            pix = min_pixel_nei[i]
            ax.plot(dl0[pix])
            ax.set_xlabel("Time (ns)")
            ax.set_ylabel("DL0 Samples (ADC)")
            ax.set_title(
                "(Min Nei) Pixel: {}, True: {}, Measured = {:.3f}"
                .format(pix, t_pe[pix], dl1[pix])
            )
            ax.set_ylim(max_ylim)
            ax.plot([start[pix], start[pix]],
                    ax.get_ylim(),
                    color='r',
                    alpha=1)
            ax.plot([end[pix], end[pix]], ax.get_ylim(), color='r', alpha=1)

    # Draw cameras
    nei_camera = np.zeros_like(max_charges, dtype=np.int)
    nei_camera[min_pixel_nei] = 2
    nei_camera[min_pix] = 1
    nei_camera[max_pixel_nei] = 3
    nei_camera[max_pix] = 4
    camera = CameraDisplay(geom, ax=ax_img_nei)
    camera.image = nei_camera
    ax_img_nei.set_title("Neighbour Map")
    ax_img_nei.annotate(
        "Pixel: {}".format(max_pix),
        xy=(geom.pix_x.value[max_pix], geom.pix_y.value[max_pix]),
        xycoords='data',
        xytext=(0.05, 0.98),
        textcoords='axes fraction',
        arrowprops=dict(facecolor='red', width=2, alpha=0.4),
        horizontalalignment='left',
        verticalalignment='top'
    )
    ax_img_nei.annotate(
        "Pixel: {}".format(min_pix),
        xy=(geom.pix_x.value[min_pix], geom.pix_y.value[min_pix]),
        xycoords='data',
        xytext=(0.05, 0.94),
        textcoords='axes fraction',
        arrowprops=dict(facecolor='orange', width=2, alpha=0.4),
        horizontalalignment='left',
        verticalalignment='top'
    )
    camera = CameraDisplay(geom, ax=ax_img_max)
    camera.image = dl0[:, max_time]
    camera.add_colorbar(ax=ax_img_max, label="DL0 Samples (ADC)")
    ax_img_max.set_title("Max Timeslice (T = {})".format(max_time))
    ax_img_max.annotate(
        "Pixel: {}".format(max_pix),
        xy=(geom.pix_x.value[max_pix], geom.pix_y.value[max_pix]),
        xycoords='data',
        xytext=(0.05, 0.98),
        textcoords='axes fraction',
        arrowprops=dict(facecolor='red', width=2, alpha=0.4),
        horizontalalignment='left',
        verticalalignment='top'
    )
    ax_img_max.annotate(
        "Pixel: {}".format(min_pix),
        xy=(geom.pix_x.value[min_pix], geom.pix_y.value[min_pix]),
        xycoords='data',
        xytext=(0.05, 0.94),
        textcoords='axes fraction',
        arrowprops=dict(facecolor='orange', width=2, alpha=0.4),
        horizontalalignment='left',
        verticalalignment='top'
    )

    camera = CameraDisplay(geom, ax=ax_img_true)
    camera.image = t_pe
    camera.add_colorbar(ax=ax_img_true, label="True Charge (p.e.)")
    ax_img_true.set_title("True Charge")
    ax_img_true.annotate(
        "Pixel: {}".format(max_pix),
        xy=(geom.pix_x.value[max_pix], geom.pix_y.value[max_pix]),
        xycoords='data',
        xytext=(0.05, 0.98),
        textcoords='axes fraction',
        arrowprops=dict(facecolor='red', width=2, alpha=0.4),
        horizontalalignment='left',
        verticalalignment='top'
    )
    ax_img_true.annotate(
        "Pixel: {}".format(min_pix),
        xy=(geom.pix_x.value[min_pix], geom.pix_y.value[min_pix]),
        xycoords='data',
        xytext=(0.05, 0.94),
        textcoords='axes fraction',
        arrowprops=dict(facecolor='orange', width=2, alpha=0.4),
        horizontalalignment='left',
        verticalalignment='top'
    )

    camera = CameraDisplay(geom, ax=ax_img_cal)
    camera.image = dl1
    camera.add_colorbar(ax=ax_img_cal, label="Calib Charge (Photo-electrons)")
    ax_img_cal.set_title("Charge (integrator={})".format(extractor_name))
    ax_img_cal.annotate(
        "Pixel: {}".format(max_pix),
        xy=(geom.pix_x.value[max_pix], geom.pix_y.value[max_pix]),
        xycoords='data',
        xytext=(0.05, 0.98),
        textcoords='axes fraction',
        arrowprops=dict(facecolor='red', width=2, alpha=0.4),
        horizontalalignment='left',
        verticalalignment='top'
    )
    ax_img_cal.annotate(
        "Pixel: {}".format(min_pix),
        xy=(geom.pix_x.value[min_pix], geom.pix_y.value[min_pix]),
        xycoords='data',
        xytext=(0.05, 0.94),
        textcoords='axes fraction',
        arrowprops=dict(facecolor='orange', width=2, alpha=0.4),
        horizontalalignment='left',
        verticalalignment='top'
    )

    fig_waveforms.suptitle("Integrator = {}".format(extractor_name))
    fig_camera.suptitle("Camera = {}".format(geom.cam_id))

    plt.show()

class DisplayIntegrator(Tool):
    name = "DisplayIntegrator"
    description = "Calibrate dl0 data to dl1, and plot the various camera " \
                  "images that characterise the event and calibration. Also " \
                  "plot some examples of waveforms with the " \
                  "integration window."

    event_index = Int(0, help='Event index to view.').tag(config=True)
    use_event_id = Bool(
        False,
        help='event_index will obtain an event using'
        'event_id instead of '
        'index.'
    ).tag(config=True)
    telescope = Int(
        None,
        allow_none=True,
        help='Telescope to view. Set to None to display the first'
        'telescope with data.'
    ).tag(config=True)
    channel = Enum([0, 1], 0, help='Channel to view').tag(config=True)

    aliases = Dict(
        dict(
            r='EventSourceFactory.product',
            f='EventSourceFactory.input_url',
            max_events='EventSourceFactory.max_events',
            extractor='ChargeExtractorFactory.product',
            window_width='ChargeExtractorFactory.window_width',
            window_shift='ChargeExtractorFactory.window_shift',
            sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG',
            sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG',
            lwt='ChargeExtractorFactory.lwt',
            clip_amplitude='CameraDL1Calibrator.clip_amplitude',
            radius='CameraDL1Calibrator.radius',
            E='DisplayIntegrator.event_index',
            T='DisplayIntegrator.telescope',
            C='DisplayIntegrator.channel',
        )
    )
    flags = Dict(
        dict(
            id=({
                'DisplayDL1Calib': {
                    'use_event_index': True
                }
            }, 'event_index will obtain an event using '
                'event_id instead of index.')
        )
    )
    classes = List([
        EventSourceFactory,
        ChargeExtractorFactory,
        CameraDL1Calibrator,
    ])

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.eventseeker = None
        self.r1 = None
        self.dl0 = None
        self.extractor = None
        self.dl1 = None

    def setup(self):
        self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
        kwargs = dict(config=self.config, tool=self)

        eventsource = EventSourceFactory.produce(**kwargs)
        self.eventseeker = EventSeeker(eventsource, **kwargs)

        self.extractor = ChargeExtractorFactory.produce(**kwargs)

        self.r1 = CameraR1CalibratorFactory.produce(
            eventsource=eventsource, **kwargs
        )

        self.dl0 = CameraDL0Reducer(**kwargs)

        self.dl1 = CameraDL1Calibrator(extractor=self.extractor, **kwargs)

    def start(self):
        event_num = self.event_index
        if self.use_event_id:
            event_num = str(event_num)
        event = self.eventseeker[event_num]

        # Calibrate
        self.r1.calibrate(event)
        self.dl0.reduce(event)
        self.dl1.calibrate(event)

        # Select telescope
        tels = list(event.r0.tels_with_data)
        telid = self.telescope
        if telid is None:
            telid = tels[0]
        if telid not in tels:
            self.log.error(
                "[event] please specify one of the following "
                "telescopes for this event: {}".format(tels)
            )
            exit()

        extractor_name = self.extractor.__class__.__name__

        plot(event, telid, self.channel, extractor_name)

    def finish(self):
        pass

if __name__ == '__main__':
    exe = DisplayIntegrator()
    exe.run()

max_stars_repo_name: mpecimotika/ctapipe

max_issues_repo_name: mpecimotika/ctapipe

max_forks_repo_name: mpecimotika/ctapipe