text:
from __future__ import print_function
import nifty
import numpy
import nifty
import nifty.graph
import nifty.graph.rag
import tempfile
import shutil
import os

nrag = nifty.graph.rag

def ensureDir(f):
    d = os.path.dirname(f)
    if not os.path.exists(d):
        os.makedirs(d)

def genericRagTest(rag, numberOfNodes, shouldEdges, shouldNotEdges):
    assert rag.numberOfNodes == numberOfNodes
    assert rag.numberOfEdges == len(shouldEdges)

    edgeList = []
    for edge in rag.edges():
        edgeList.append(edge)

    assert len(edgeList) == len(shouldEdges)

    for shouldEdge in shouldEdges:

        fRes = rag.findEdge(shouldEdge)
        assert fRes >= 0
        uv = rag.uv(fRes)
        uv = sorted(uv)
        assert uv[0] == shouldEdge[0]
        assert uv[1] == shouldEdge[1]

    for shouldNotEdge in shouldNotEdges:
        fRes = rag.findEdge(shouldNotEdge)
        assert fRes == -1

def testInsert():

    labels = numpy.zeros(shape=[2,2],dtype='uint32')

    labels[0,0] = 0
    labels[1,0] = 1
    labels[0,1] = 0
    labels[1,1] = 2

    g =  nifty.graph.rag.explicitLabelsGridRag2D(labels)
    weights = numpy.ones(g.numberOfEdges)*1
    obj = nifty.graph.multicut.multicutObjective(g, weights)

    greedy=obj.greedyAdditiveFactory().create(obj)
    visitor = obj.multicutVerboseVisitor()
    ret = greedy.optimize()
    #print("greedy",obj.evalNodeLabels(ret))

    assert g.numberOfNodes == 3
    assert g.numberOfEdges == 3

    insertWorked = True
    try:
        g.insertEdge(0,1)
    except:
        insertWorked = False
    assert insertWorked == False

def testExplicitLabelsRag2d():

    labels = [
        [0,1,2],
        [0,0,2],
        [3,3,2],
        [4,4,4]
    ]

    ragA = nifty.graph.rag.gridRag(labels,0)
    ragB = nifty.graph.rag.gridRag(labels)

    assert isinstance(ragA, nifty.graph.rag.ExplicitLabelsGridRag2D)
    assert isinstance(ragB, nifty.graph.rag.ExplicitLabelsGridRag2D)

    shouldEdges = [
        (0,1),
        (0,2),
        (0,3),
        (1,2),
        (2,3),
        (2,4),
        (3,4)
    ]

    shouldNotEdges = [
        (0,4),
        (1,3),
        (1,4)
    ]

    genericRagTest(rag=ragA, numberOfNodes=5,
               shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

    genericRagTest(rag=ragB, numberOfNodes=5,
               shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

def testExplicitLabelsRag2dSerializeDeserialize():

    labels = [
        [0,1,2],
        [0,0,2],
        [3,3,2],
        [4,4,4]
    ]

    ragA = nifty.graph.rag.gridRag(labels,0)
    ragB = nifty.graph.rag.gridRag(labels)

    assert isinstance(ragA, nifty.graph.rag.ExplicitLabelsGridRag2D)
    assert isinstance(ragB, nifty.graph.rag.ExplicitLabelsGridRag2D)

    shouldEdges = [
        (0,1),
        (0,2),
        (0,3),
        (1,2),
        (2,3),
        (2,4),
        (3,4)
    ]

    shouldNotEdges = [
        (0,4),
        (1,3),
        (1,4)
    ]

    genericRagTest(rag=ragA, numberOfNodes=5,
               shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

    genericRagTest(rag=ragB, numberOfNodes=5,
               shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

def testExplicitLabelsRag3d():

    labels = [
        [
            [0,1],
            [0,0]
        ],
        [
            [1,1],
            [2,2]
        ],
        [
            [3,3],
            [3,3]
        ]
    ]

    labels = numpy.array(labels)

    ragA = nifty.graph.rag.gridRag(labels,0)
    ragB = nifty.graph.rag.gridRag(labels)

    assert isinstance(ragA, nifty.graph.rag.ExplicitLabelsGridRag3D)
    assert isinstance(ragB, nifty.graph.rag.ExplicitLabelsGridRag3D)

    shouldEdges = [
        (0,1),
        (0,2),
        (1,2),
        (1,3),
        (2,3)

    ]

    shouldNotEdges = [
       (0,3)
    ]

    genericRagTest(rag=ragA, numberOfNodes=4,
               shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

    genericRagTest(rag=ragB, numberOfNodes=4,
               shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

if nifty.Configuration.WITH_HDF5:

    nhdf5 = nifty.hdf5

    def testHdf5Rag2d():
        tempFolder = tempfile.mkdtemp()
        ensureDir(tempFolder)
        fpath = os.path.join(tempFolder,'_nifty_test_array4_.h5')

        try:

            shape = [3,3]
            chunkShape = [1,1]
            blockShape =  [2,2]

            hidT = nhdf5.createFile(fpath)
            array = nhdf5.Hdf5ArrayUInt32(hidT, "data", shape, chunkShape)

            assert array.shape[0] == shape[0]
            assert array.shape[1] == shape[1]

            labels = numpy.array( [
                [0,  1, 1],
                [2,  2, 2],
                [3,  3, 3]
            ],dtype='uint32')

            assert labels.shape[0] == shape[0]
            assert labels.shape[1] == shape[1]

            array[0:shape[0], 0:shape[1]] = labels

            rag = nrag.gridRagHdf5(array, numberOfLabels=labels.max()+1,
                                   blockShape=blockShape, numberOfThreads=2)

            shouldEdges = [
                (0,1),
                (0,2),
                (1,2),
                (2,3)
            ]

            shouldNotEdges = [
                (0,3),
                (1,3)
            ]

            genericRagTest(rag=rag, numberOfNodes=labels.max()+1,
                           shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

        finally:
            try:
                os.remove(fpath)
                shutil.rmtree(tempFolder)

            except:
                pass

    def testHdf5Rag2dLarge():

        tempFolder = tempfile.mkdtemp()
        ensureDir(tempFolder)
        fpath = os.path.join(tempFolder,'_nifty_testHdf5Rag2dLarge_.h5')

        try:

            shape = [5,6]
            chunkShape = [3,2]
            blockShape =  [2,3]

            hidT = nhdf5.createFile(fpath)
            array = nhdf5.Hdf5ArrayUInt32(hidT, "data", shape, chunkShape)

            assert array.shape[0] == shape[0]
            assert array.shape[1] == shape[1]

            labels = numpy.array( [
                [0, 0, 0, 0, 1, 1],
                [0, 2, 2, 0, 1, 3],
                [0, 3, 3, 3, 3, 3],
                [0, 3, 4, 5, 5, 5],
                [0, 0, 4, 6, 6, 6],
            ],dtype='uint32')

            assert labels.shape[0] == shape[0]
            assert labels.shape[1] == shape[1]

            array[0:shape[0], 0:shape[1]] = labels
            rag = nrag.gridRagHdf5(array, numberOfLabels=labels.max()+1,
                                   blockShape=blockShape, numberOfThreads=1)

            shouldEdges = [
                (0,1),
                (0,2),
                (0,3),
                (0,4),
                (1,3),
                (2,3),
                (3,4),
                (3,5),
                (4,5),
                (4,6),
                (5,6)
            ]

            shouldNotEdges = [
                (0,6),
                (0,5),
                (1,6),
                (1,5)
            ]

            genericRagTest(rag=rag, numberOfNodes=labels.max()+1,
                           shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

        finally:
            try:
                os.remove(fpath)
                shutil.rmtree(tempFolder)

            except:
                pass

    def testHdf5Rag3d():

        tempFolder = tempfile.mkdtemp()
        ensureDir(tempFolder)
        fpath = os.path.join(tempFolder,'_nifty_testHdf5Rag3d_.h5')

        try:

            shape = [3,2,2]
            chunkShape = [1,2,1]
            blockShape =  [1,2,3]

            hidT = nhdf5.createFile(fpath)
            array = nhdf5.Hdf5ArrayUInt32(hidT, "data", shape, chunkShape)

            assert array.shape[0] == shape[0]
            assert array.shape[1] == shape[1]
            assert array.shape[2] == shape[2]

            labels = [
                [
                    [0,1],
                    [0,0]
                ],
                [
                    [1,1],
                    [2,2]
                ],
                [
                    [3,3],
                    [3,3]
                ]
            ]
            labels = numpy.array( labels,dtype='uint32')

            assert labels.shape[0] == shape[0]
            assert labels.shape[1] == shape[1]
            assert labels.shape[2] == shape[2]

            array[0:shape[0], 0:shape[1], 0:shape[2]] = labels
            rag = nrag.gridRagHdf5(array, numberOfLabels=labels.max()+1,
                           blockShape=blockShape, numberOfThreads=-1)

            shouldEdges = [
                (0,1),
                (0,2),
                (1,2),
                (1,3),
                (2,3)
            ]

            shouldNotEdges = [
               (0,3)
            ]

            genericRagTest(rag=rag, numberOfNodes=labels.max()+1,
                           shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

        finally:
            try:
                os.remove(fpath)
                shutil.rmtree(tempFolder)

            except:
                pass

    def testGridRag3DStacked2D():

        tempFolder = tempfile.mkdtemp()
        ensureDir(tempFolder)
        fpath = os.path.join(tempFolder,'_nifty_testHdf5Rag3d_.h5')

        try:

            shape = [3,2,2]
            chunkShape = [1,2,1]
            blockShape =  [1,2,3]

            hidT = nhdf5.createFile(fpath)
            array = nhdf5.Hdf5ArrayUInt32(hidT, "data", shape, chunkShape)

            assert array.shape[0] == shape[0]
            assert array.shape[1] == shape[1]
            assert array.shape[2] == shape[2]

            labels = [
                [
                    [0,1],
                    [0,1]
                ],
                [
                    [2,2],
                    [2,3]
                ],
                [
                    [4,5],
                    [6,6]
                ]
            ]
            labels = numpy.array( labels,dtype='uint32')

            assert labels.shape[0] == shape[0]
            assert labels.shape[1] == shape[1]
            assert labels.shape[2] == shape[2]

            array[0:shape[0], 0:shape[1], 0:shape[2]] = labels
            rag = nrag.gridRagStacked2DHdf5(array, numberOfLabels=labels.max()+1,
                                            numberOfThreads=-1)

            shouldEdges = [
                (0,1),
                (0,2),
                (1,2),
                (1,3),
                (2,3),
                (2,4),
                (2,5),
                (2,6),
                (3,6),
                (4,5),
                (4,6),
                (5,6)
            ]

            shouldNotEdges = [
                (0,3),
                (0,4),
                (0,5),
                (0,6),
                (1,4),
                (1,5),
                (1,6)
            ]

            #("edges in rag",rag.numberOfEdges,len(shouldEdges))

            genericRagTest(rag=rag, numberOfNodes=labels.max()+1,
                           shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

        finally:
            try:
                os.remove(fpath)
                shutil.rmtree(tempFolder)

            except:
                pass

    def testGridRag3DStacked2DLarge():

        tempFolder = tempfile.mkdtemp()
        ensureDir(tempFolder)
        fpath = os.path.join(tempFolder,'_nifty_testHdf5Rag3d_.h5')

        try:

            shape = [3,4,4]
            chunkShape = [1,2,1]

            hidT = nhdf5.createFile(fpath)
            array = nhdf5.Hdf5ArrayUInt32(hidT, "data", shape, chunkShape)

            assert array.shape[0] == shape[0]
            assert array.shape[1] == shape[1]
            assert array.shape[2] == shape[2]

            labels = [
                [
                    [0,0,0,0],
                    [1,1,1,1],
                    [2,2,2,2],
                    [2,2,2,2]
                ],
                [
                    [3,3,3,3],
                    [3,3,3,3],
                    [3,3,3,3],
                    [3,3,3,3]
                ],
                [
                    [4,4,5,5],
                    [4,4,5,5],
                    [4,4,5,5],
                    [4,4,5,5]
                ]
            ]
            labels = numpy.array( labels,dtype='uint32')

            assert labels.shape[0] == shape[0]
            assert labels.shape[1] == shape[1]
            assert labels.shape[2] == shape[2]

            array[0:shape[0], 0:shape[1], 0:shape[2]] = labels
            rag = nrag.gridRagStacked2DHdf5(array, numberOfLabels=labels.max()+1,
                                            numberOfThreads=-1)

            shouldEdges = [
               (0,1),
               (0,3),
               (1,2),
               (1,3),
               (2,3),
               (3,4),
               (3,5),
               (4,5)
            ]

            shouldNotEdges = [
                (0,4),
                (0,5),
                (1,4),
                (1,5),
                (2,4),
                (2,5)
            ]

            genericRagTest(rag=rag, numberOfNodes=labels.max()+1,
                           shouldEdges=shouldEdges, shouldNotEdges=shouldNotEdges)

        finally:
            try:
                os.remove(fpath)
                shutil.rmtree(tempFolder)

            except:
                pass

max_stars_repo_name: k-dominik/nifty

max_issues_repo_name: k-dominik/nifty

max_forks_repo_name: k-dominik/nifty


text:
import torch
import numpy as np

import pfrl

import pytest
import unittest

assertions = unittest.TestCase("__init__")

@pytest.mark.parametrize("in_size", [1, 5])
@pytest.mark.parametrize("out_size", [1, 3])
@pytest.mark.parametrize("hidden_sizes", [(), (1,), (1, 1), (7, 8)])
@pytest.mark.parametrize("normalize_input", [True, False])
@pytest.mark.parametrize("normalize_output", [True, False])
@pytest.mark.parametrize("nonlinearity", ["relu", "elu"])
@pytest.mark.parametrize("last_wscale", [1, 1e-3])
class TestMLPBN:
    @pytest.fixture(autouse=True)
    def setUp(
        self,
        in_size,
        out_size,
        hidden_sizes,
        normalize_input,
        normalize_output,
        nonlinearity,
        last_wscale,
    ):
        self.in_size = in_size
        self.out_size = out_size
        self.hidden_sizes = hidden_sizes
        self.normalize_input = normalize_input
        self.normalize_output = normalize_output
        self.nonlinearity = nonlinearity
        self.last_wscale = last_wscale

    def _test_call(self, gpu):
        nonlinearity = getattr(torch.nn.functional, self.nonlinearity)
        mlp = pfrl.nn.MLPBN(
            in_size=self.in_size,
            out_size=self.out_size,
            hidden_sizes=self.hidden_sizes,
            normalize_input=self.normalize_input,
            normalize_output=self.normalize_output,
            nonlinearity=nonlinearity,
            last_wscale=self.last_wscale,
        )
        batch_size = 7
        x = np.random.rand(batch_size, self.in_size).astype(np.float32)
        x = torch.from_numpy(x)
        if gpu >= 0:
            assertions.assertTrue(torch.cuda.is_available())
            device = torch.device("cuda:{}".format(gpu))
            mlp = mlp.to(device)
            x = x.cuda()
        y = mlp(x)
        assertions.assertEqual(y.shape, (batch_size, self.out_size))
        assertions.assertEqual(y.device, x.device)

    def test_call_cpu(self):
        self._test_call(gpu=-1)

    @pytest.mark.gpu
    def test_call_gpu(self):
        self._test_call(gpu=0)

ext: py

max_stars_repo_name: g-votte/pfrl

max_issues_repo_name: g-votte/pfrl

max_forks_repo_name: g-votte/pfrl

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""

@author: Adam Reinhold Von Fisher - https://www.linkedin.com/in/adamrvfisher/

"""

#Developed in Python 3.5

#This is a trading strategy model
#Does not function properly see DonchianTrendEfficiencyFilterSingleStockSingleFrequency.py

#R Multiple Finder; Trade Tracking
#Import modules
import numpy as np
import random as rand
import pandas as pd
import time as t
from DatabaseGrabber import DatabaseGrabber
from YahooGrabber import YahooGrabber

#Assign ticker
Ticker1 = 'VXX'
#Request data
Asset1 = YahooGrabber(Ticker1)

#Declaration/Assignment
#Empty list
Empty = []
#Empty dataframe
Trades = pd.DataFrame()

#Timing statistics and iteration counter for optimization
#Start = t.time()
#Counter = 0
#start = t.time()

#The next 4 declarations are for use in fixed profit and loss based exits
#Exit stop loss - in percentages --------- however, looking to use ATR based stops
#LongStopLoss = .005
#ShortStopLoss = .005
##Exit profit take -------- However, looking to use other exits, time based, trailing, ATR, etc.
#LongProfitTake = .01
#ShortProfitTake = .01

#Constraints in percentages
Commission = .01
Slippage = .01

#Time series trimmer for in/out sample data
#Asset1a = Asset1[-1250:] #Out
Asset1 = Asset1[:] #In

#Numbered subindex
Asset1['SubIndex'] = range(1,len(Asset1)+1)

#Assign variable windows
donchianwindow = 15
exitwindow = 13
ATRwindow = 20
stopwindow = 13
Counter = 0

#Calculate log Returns
Asset1['LogRet'] = np.log(Asset1['Adj Close']/Asset1['Adj Close'].shift(1))
Asset1['LogRet'] = Asset1['LogRet'].fillna(0)
#Calculate ATR
Asset1['Method1'] = Asset1['High'] - Asset1['Low']
Asset1['Method2'] = abs((Asset1['High'] - Asset1['Close'].shift(1)))
Asset1['Method3'] = abs((Asset1['Low'] - Asset1['Close'].shift(1)))
Asset1['Method1'] = Asset1['Method1'].fillna(0)
Asset1['Method2'] = Asset1['Method2'].fillna(0)
Asset1['Method3'] = Asset1['Method3'].fillna(0)
Asset1['TrueRange'] = Asset1[['Method1','Method2','Method3']].max(axis = 1)
Asset1['ATR'] = Asset1['TrueRange'].rolling(window = ATRwindow,
                                center=False).mean()

#Market top and bottom calculation
Asset1['RollingMax'] = Asset1['High'].rolling(window=donchianwindow, center=False).max()
Asset1['RollingMin'] = Asset1['Low'].rolling(window=donchianwindow, center=False).min()
#Graphical display
#Asset1[['RollingMax','RollingMin','Adj Close']].plot()

#Signal = Price </> min/max
#If price is less than the min go long
#If price is greater than the max go short
Asset1['Signal'] = np.where(Asset1['High'] >= Asset1['RollingMax'].shift(1) , 1, 0)
Asset1['Signal'] = np.where(Asset1['Low'] <= Asset1['RollingMin'].shift(1) , -1, Asset1['Signal'])
#If Rolling Min/Max is still being computed, stay out of market
Asset1['Signal'] = np.where(Asset1['RollingMax'] == np.nan, 0, Asset1['Signal'])

#To help identify "regime changes" i.e. last signal switches from short to long, vice versa
#Asset1['FilledSignal'] = np.where(Asset1['Signal'] == 0, np.nan, Asset1['Signal'] )
#Asset1['FilledSignal'] = Asset1['FilledSignal'].ffill(inplace = False)
#Asset1['FilledSignal'] = Asset1['FilledSignal'].fillna(0)

#Signal sub index numbers for segmenting data for trade analysis
SignalDates = Asset1['SubIndex'].loc[((Asset1['Signal'] != 0))]

#Trade ATR for signal
Asset1['TradeATR'] = np.where(Asset1['Signal'] != 0, Asset1['ATR'].shift(1), np.nan)

#Experimental exits
Asset1['LimitExitPrice'] = np.nan
Asset1['ShortExitPrice'] =  Asset1['High'].rolling(window=stopwindow, center=False).max()
Asset1['LongExitPrice'] =  Asset1['Low'].rolling(window=stopwindow, center=False).min()

#Find the first trade of the signal period, document entry prices
#Declare columns to record entry price and stop for unit one
Asset1['EntryPriceUnitOne'] = np.nan
Asset1['StopPriceUnitOne'] = np.nan

#Check for gaps on first unit entry and later on exits.
#Default stops and entries
#Long entry first unit
Asset1['EntryPriceUnitOne'] = np.where(Asset1['Signal'] == 1,
                              Asset1['RollingMax'].shift(1) + .01, np.nan)
#Long gap entry first unit
Asset1['EntryPriceUnitOne'].loc[(Asset1['Signal'] == 1) & (
            Asset1['Open'] > Asset1['EntryPriceUnitOne'])] = Asset1['Open']
#Short entry first unit
Asset1['EntryPriceUnitOne'] = np.where(Asset1['Signal'] == -1,
              Asset1['RollingMin'].shift(1) - .01, Asset1['EntryPriceUnitOne'])
#Short gap entry first unit
Asset1['EntryPriceUnitOne'].loc[(Asset1['Signal'] == -1) & (
            Asset1['Open'] < Asset1['EntryPriceUnitOne'])] = Asset1['Open']
#Long stop first unit
Asset1['StopPriceUnitOne'] = np.where(Asset1['Signal'] == 1,
                              Asset1['EntryPriceUnitOne'] - (Asset1['TradeATR'] * 2), np.nan)
#Short stop first unit
Asset1['StopPriceUnitOne'] = np.where(Asset1['Signal'] == -1,
              Asset1['EntryPriceUnitOne'] + (Asset1['TradeATR'] * 2), Asset1['StopPriceUnitOne'])

#Experimental exits
Asset1['HybridShortExitPrice'] = np.where(Asset1['ShortExitPrice'] < Asset1['StopPriceUnitOne'],
                          Asset1['ShortExitPrice'], Asset1['StopPriceUnitOne'])
Asset1['HybridLongExitPrice'] = np.where(Asset1['LongExitPrice'] > Asset1['StopPriceUnitOne'],
                          Asset1['LongExitPrice'], Asset1['StopPriceUnitOne'])
Asset1['HybridShortExitPrice'] = Asset1['HybridShortExitPrice'].ffill()
Asset1['HybridLongExitPrice'] = Asset1['HybridLongExitPrice'].ffill()

#This is a profit target for long trades
Asset1['LimitExitPrice'] = np.where(Asset1['Signal'] == 1,
                 Asset1['EntryPriceUnitOne'] + (5 * Asset1['TradeATR']), np.nan)
#This is a profit target for short trades
Asset1['LimitExitPrice'] = np.where(Asset1['Signal'] == -1,
    Asset1['EntryPriceUnitOne'] - (5 * Asset1['TradeATR']), Asset1['LimitExitPrice'])

#Begin loops for individual trade examination
#Iterable
TradeRanger = range(0,len(SignalDates))
#for r in TradeRanger:
TradeSubset = Asset1.loc[(Asset1['SubIndex'] >= SignalDates[0])]
#TradeSubset = Asset1.loc[(Asset1['SubIndex'] >= 59) & (Asset1['SubIndex'] <= 87)]
TradeDirection = TradeSubset['Signal'][0]
TradeSubset['Exit'] = 0
#
#Short exit, 1 = yes, 0 = no
TradeSubset['ShortExit'] = 0
#Long exit, 1 = yes, 0 = no
TradeSubset['LongExit'] = 0
#Did the exit gap overnight? or hit after open
TradeSubset['GapShortExit'] = 0
#Did the exit gap overnight? or hit after open
TradeSubset['GapLongExit'] = 0

#1 = Short exit being hit starting the day of the signal.
if TradeDirection == -1:
    TradeSubset['ShortExit'].loc[(TradeSubset['High'] > TradeSubset['HybridShortExitPrice'])] = 1
if TradeDirection == 1:
#1 = Long exit being hit starting the day of the signal.
    TradeSubset['LongExit'].loc[(TradeSubset['Low'] < TradeSubset['HybridLongExitPrice'])] = 1

#Assess gaps on days where trade closes
TradeSubset['GapShortExit'].loc[(TradeSubset['ShortExit'] == 1) & (
                TradeSubset['Open'] > TradeSubset['HybridShortExitPrice'])] = 1

TradeSubset['GapLongExit'].loc[(TradeSubset['LongExit'] == 1) & (
                 TradeSubset['Open'] < TradeSubset['HybridLongExitPrice'])] = 1

#Types of exit
TradeSubset['Exit'].loc[(TradeSubset['ShortExit'] == 1)] = 1 #1 indicating short exit
TradeSubset['Exit'].loc[(TradeSubset['LongExit'] == 1)] = 2 #1 indicating long exit
TradeSubset['Exit'].loc[(TradeSubset['GapShortExit'] == 1)] = 3 #1 indicating short exit w/ gap
TradeSubset['Exit'].loc[(TradeSubset['GapLongExit'] == 1)] = 4 #1 indicating long exit w/ gap

while sum(abs(TradeSubset['Exit'])) != 0:
#while Counter < 1:
    EntryPriceUnitOne = TradeSubset['EntryPriceUnitOne'][0]
    StopPriceUnitOne = TradeSubset['StopPriceUnitOne'][0]
    #TradeDirection
    TradeDirection = TradeSubset['Signal'][0]

    TradeSubset['Exit'] = 0

    #Short exit, 1 = yes, 0 = no
    TradeSubset['ShortExit'] = 0
    #Long exit, 1 = yes, 0 = no
    TradeSubset['LongExit'] = 0
    #Did the exit gap overnight? or hit after open
    TradeSubset['GapShortExit'] = 0
    #Did the exit gap overnight? or hit after open
    TradeSubset['GapLongExit'] = 0

    #1 = Short exit being hit starting the day of the signal.
    if TradeDirection == -1:
        TradeSubset['ShortExit'].loc[(TradeSubset['High'] > TradeSubset['HybridShortExitPrice'])] = 1
    if TradeDirection == 1:
    #1 = Long exit being hit starting the day of the signal.
        TradeSubset['LongExit'].loc[(TradeSubset['Low'] < TradeSubset['HybridLongExitPrice'])] = 1

    #Assess Gaps on days where trade closes
    TradeSubset['GapShortExit'].loc[(TradeSubset['ShortExit'] == 1) & (
                    TradeSubset['Open'] > TradeSubset['HybridShortExitPrice'])] = 1

    TradeSubset['GapLongExit'].loc[(TradeSubset['LongExit'] == 1) & (
                     TradeSubset['Open'] < TradeSubset['HybridLongExitPrice'])] = 1

    #Types of exit
    TradeSubset['Exit'].loc[(TradeSubset['ShortExit'] == 1)] = 1 #1 indicating short exit
    TradeSubset['Exit'].loc[(TradeSubset['LongExit'] == 1)] = 2 #1 indicating long exit
    TradeSubset['Exit'].loc[(TradeSubset['GapShortExit'] == 1)] = 3 #1 indicating short exit w/ gap
    TradeSubset['Exit'].loc[(TradeSubset['GapLongExit'] == 1)] = 4 #1 indicating long exit w/ gap

    #List comprehension to find exit taken for subset.
    #The next function gives a position on the TradeSubset index
    ExitTaken = TradeSubset['Exit'][next((n for n, x in enumerate(TradeSubset['Exit']) if x), 0)]
    #The length of the trade
    LengthOfTrade = int(next((n for n, x in enumerate(TradeSubset['Exit']) if x), 0))
    #The SubIndex of the exit date is for continuing looking for rentry in new subset
    SubIndexOfEntry = TradeSubset['SubIndex'][0]
    SubIndexOfExit = TradeSubset['SubIndex'][next((n for n, x in enumerate(TradeSubset['Exit']) if x), 0)]

    #Price of gap open
    OpenPriceOnGap = TradeSubset['Open'][LengthOfTrade]
    #Exit taken
    if ExitTaken == 1: # if exiting short trade, exit during market day
        TradeReturn = (EntryPriceUnitOne - StopPriceUnitOne)/EntryPriceUnitOne
    elif ExitTaken == 2: # if exiting long trade, exitduring market day
        TradeReturn = (StopPriceUnitOne - EntryPriceUnitOne)/EntryPriceUnitOne
    elif ExitTaken == 3: # if exiting short trade with gap
        TradeReturn = (EntryPriceUnitOne - OpenPriceOnGap)/EntryPriceUnitOne
    elif ExitTaken == 4: # if exiting long trade with gap
        TradeReturn = (OpenPriceOnGap - EntryPriceUnitOne)/EntryPriceUnitOne
    #Save trade details to list
    Empty.append(ExitTaken)
    Empty.append(LengthOfTrade)
    Empty.append(EntryPriceUnitOne)
    Empty.append(StopPriceUnitOne)
    Empty.append(SubIndexOfEntry)
    Empty.append(SubIndexOfExit)
    Empty.append(TradeDirection)
    Empty.append(OpenPriceOnGap)
    Empty.append(TradeReturn)
    #Empty.append(RMultiple)
    #List to series
    Emptyseries = pd.Series(Empty)
    #Series to dataframe
    Trades[Counter] = Emptyseries.values
    #Clear list
    Empty[:] = []
    #Trimmer trims the Trade out of the TradeSubset, then trims to the next signal
    TradeSubset = TradeSubset[(LengthOfTrade + 1):]
    SignalTrim = next((n for n, x in enumerate(TradeSubset['Signal']) if x), 0)
    TradeSubset = TradeSubset[SignalTrim:]
    #Iteration tracking
    Counter = Counter + 1
    print(Counter)

#The last trade that is still open
if sum(abs(TradeSubset['Signal'])) != 0:
    #Entry price
    EntryPriceUnitOne = TradeSubset['EntryPriceUnitOne'][0]
    #Exit price
    StopPriceUnitOne = TradeSubset['StopPriceUnitOne'][0]
    ExitTaken = 0
    #Trade duration
    LengthOfTrade = len(TradeSubset)
    #Trade direction
    TradeDirection = TradeSubset['Signal'][0]
    #If long trade, calculate return
    if TradeDirection == 1:
        TradeReturn = (TradeSubset['HybridLongExitPrice'][-1] - EntryPriceUnitOne)/EntryPriceUnitOne
    #If short trade, calculate return
    elif TradeDirection == -1:
        TradeReturn = (EntryPriceUnitOne - TradeSubset['HybridLongExitPrice'][-1])/EntryPriceUnitOne
    #Exit date
    SubIndexOfEntry = TradeSubset['SubIndex'][0]
    SubIndexOfExit = np.nan
    OpenPriceOnGap = np.nan
    #Save details to list
    Empty.append(ExitTaken)
    Empty.append(LengthOfTrade)
    Empty.append(EntryPriceUnitOne)
    Empty.append(StopPriceUnitOne)
    Empty.append(SubIndexOfEntry)
    Empty.append(SubIndexOfExit)
    Empty.append(TradeDirection)
    Empty.append(OpenPriceOnGap)
    Empty.append(TradeReturn)
    #List to series
    Emptyseries = pd.Series(Empty)
    #Series to dataframe
    Trades[Counter] = Emptyseries.values
    #Clear list
    Empty[:] = []

#Rename rows
Trades = Trades.rename(index={0: "ExitTaken", 1: "LengthOfTrade", 2: "EntryPriceUnitOne",
                3: "StopPriceUnitOne", 4: "SubIndexOfEntry", 5: "SubIndexOfExit",
                6: "TradeDirection", 7: "OpenPriceOnGap", 8: "TradeReturn"})
#Ones
Asset1['Brackets'] = 1
#Calculate cumulative returns
for d in Trades:
    Asset1['Brackets'].loc[(Asset1['SubIndex'] == Trades[d]['SubIndexOfExit'])] = 1 + Trades[d]['TradeReturn']
#System statistics
NumWinningTrades = len(Asset1['Brackets'][Asset1['Brackets'] > 1])
NumLosingTrades = len(Asset1['Brackets'][Asset1['Brackets'] < 1])
AvgWin = Asset1['Brackets'][Asset1['Brackets'] > 1].mean()
AvgLoss = Asset1['Brackets'][Asset1['Brackets'] < 1].mean()
RewardRisk = AvgWin/AvgLoss
WinRate = NumWinningTrades / (NumWinningTrades + NumLosingTrades)
LossRate = NumLosingTrades / (NumWinningTrades + NumLosingTrades)
Expectancy = (WinRate * RewardRisk) - (LossRate)
#Returns on $1
Asset1['Multiplier'] = Asset1['Brackets'].cumprod()
#Graphical display
Asset1['Multiplier'].plot()
#Display results
print(Expectancy)

max_stars_repo_name: adamrvfisher/TechnicalAnalysisLibrary

max_issues_repo_name: adamrvfisher/TechnicalAnalysisLibrary

max_forks_repo_name: adamrvfisher/TechnicalAnalysisLibrary


text:
import numpy as np
import sys
import time
import torch
from torch.autograd import Variable

import src.modelsPytorch as mopy
from src.globalStuff import printing, threshold_consecutive_epochs, max_consecutive_epochs_no_improvement, reorderData, IND_PAD, timeSince, asMinutes, list_event_names, metrics_to_use, possible_outputs, SAVE_EVERY_TH_EPOCH, SAVE_STATE

def saveIntermediateState(test_output_dir, epoch, trainedModels, model_params, pred_train, pred_val, pred_test, data_train, data_val, data_test):
    import os

    try:
        os.makedirs(os.path.join(test_output_dir, 'epoch_number_%d' %epoch))
    except OSError:
        pass # already exists

    #save pytorch models
    torch.save(trainedModels['encoder'].state_dict(), '%s/epoch_number_%d/%s_optenc%s_lrenc%s_layrenc%s_hiddenenc%s_dropoutenc%s_epochnum%s_embed%s_batch%s' %(test_output_dir, epoch, 'encoder', model_params['encoder_params']['optimizer_name'], model_params['encoder_params']['learning_rate'], model_params['encoder_params']['num_layers'], model_params['encoder_params']['hidden_size'], model_params['encoder_params']['dropout_prob'], epoch, model_params['encoder_params']['embed_size'], model_params['encoder_params']['batch_size']))
    for model_type in trainedModels['output_types']:
        hidden_sizes_str = '_'.join(map(str, model_params['nn_params'][model_type]['hidden_sizes']))
        torch.save(trainedModels[model_type].state_dict(), '%s/epoch_number_%d/%s_opt%s_epochnum%s_lr%s_layr%s_hidden%s_dropout%s' %(test_output_dir, epoch, model_type, model_params['nn_params'][model_type]['optimizer_name'], epoch, model_params['nn_params'][model_type]['learning_rate'], model_params['nn_params'][model_type]['num_layers'], hidden_sizes_str, model_params['nn_params'][model_type]['dropout_prob']))

    #save most recent epoch predictions
    predicted_values = {}
    measured_values = {}
    errors_values = {}
    for model_type in trainedModels['output_types']:
        predicted_values[model_type] = {'Training': pred_train['last_epoch_predictions'][model_type], 'Validation': pred_val['last_epoch_predictions'][model_type]}
        measured_values[model_type] = {'Training': data_train[model_type], 'Validation': data_val[model_type]}
        errors_values[model_type] = {'Training': pred_train['abs_error_per_epoch'][-1][model_type], 'Validation': pred_val['abs_error_per_epoch'][-1][model_type]}
        np.savetxt('%s/epoch_number_%d/last_epoch_prediction_train_%s.csv' %(test_output_dir, epoch, model_type), predicted_values[model_type]['Training'], delimiter=",")
        np.savetxt('%s/epoch_number_%d/last_epoch_prediction_val_%s.csv' %(test_output_dir, epoch, model_type), predicted_values[model_type]['Validation'], delimiter=",")
        if data_test != []:
            predicted_values[model_type]['Testing'] = pred_test['last_epoch_predictions'][model_type]
            measured_values[model_type]['Testing'] = data_test[model_type]
            errors_values[model_type]['Testing'] = pred_test['abs_error_per_epoch'][-1][model_type]
            np.savetxt('%s/epoch_number_%d/last_epoch_prediction_test_%s.csv' %(test_output_dir, epoch, model_type), predicted_values[model_type]['Testing'], delimiter=",")

def padSequence(sequence, max_length):
    sequence += [IND_PAD for i in range(max_length - len(sequence))]
    return sequence

def tensorFromWord(program, max_size_batch, device):
    if not len(program) == max_size_batch:
        program = padSequence(program, max_size_batch)
    return torch.tensor(program, dtype=torch.long).view(-1, 1).to(device)

def tensorFromBatch(program_insts, batch_index, batch_size, device):
    tensors_batch=[]
    len_list = [len(i) for i in program_insts[batch_index:batch_index+batch_size]]
    max_size_batch = np.max(len_list)
    for program_id in range(batch_size):
        tensors_batch.append(tensorFromWord(program_insts[batch_index+program_id], max_size_batch, device))
    tensors_batch = torch.stack(tensors_batch)
    tensors_batch = torch.squeeze(tensors_batch)
    return tensors_batch

def evaluateBatch(device, clocks, models, model_name, input, target_variable, batch_index, batch_size, loss_function, pc, metrics_batch):
    encoder = models['encoder']
    output_model = models[model_name]

    loss = 0
    pred_output = torch.empty([batch_size, clocks['num_mem_clocks'], np.max(clocks['num_core_clocks'])] , dtype=torch.float).to(device)

    with torch.no_grad():
        input_tensor = tensorFromBatch(input, batch_index, batch_size, device).requires_grad_(requires_grad=False).detach()
        if (batch_size == 1):
            input_variable = Variable(input_tensor)
        else:
            input_variable = Variable(input_tensor.transpose(0, 1))

        seq_length = input_variable.size()[0]
        target_length = target_variable.view(-1).size()[0]

        encoder.hidden = encoder.initHidden(batch_size)
        for ei in range(seq_length):
            lstm_output = encoder(input_variable[ei], batch_size)

        lstm_output = lstm_output.view(batch_size, -1)

        for bi in range(batch_size):
            #if using performance_counters
            if pc == True:
                metrics_bi = []
                for metric_id, metric in enumerate(metrics_to_use):
                    metrics_bi.append(metrics_batch[metric][bi])
                metrics_tensor = torch.tensor(metrics_bi, dtype=torch.float).to(device)

            for clock_mem_id, clock_mem in enumerate(clocks['mem_clocks']):
                for clock_core_id, clock_core in enumerate(clocks['core_clocks'][clock_mem_id]):
                    clock_tensor = torch.tensor([clock_mem/clocks['highest_mem_clock'], clock_core/clocks['highest_core_clock']], dtype=torch.float).to(device)
                    newfeatures = torch.cat((lstm_output[bi,:], clock_tensor), 0).to(device) #concatenate V-F level to the feature vector
                    if pc == True:
                        newfeatures = torch.cat((newfeatures, metrics_tensor), 0).to(device)
                    pred_output[bi][clock_mem_id, clock_core_id] = output_model(newfeatures)
                    loss += loss_function(pred_output[bi][clock_mem_id, clock_core_id], target_variable[bi][clock_mem_id,clock_core_id])

    return pred_output, loss.item() / target_length

def processBatch(device, clocks, verbose, models, model_name, batch_index, batch_size, args_batch, X, y, names, pc, metrics_batch):
    loss_function = torch.nn.SmoothL1Loss()
    errors = args_batch[0]
    predicted_values = args_batch[1]
    total_loss = args_batch[2]

    real_target =  torch.tensor(y[batch_index:batch_index+batch_size], dtype=torch.float).to(device).requires_grad_(requires_grad=False).detach()
    target_variable = real_target

    pred_target, loss_batch = evaluateBatch(device, clocks, models, model_name, X, target_variable, batch_index, batch_size, loss_function, pc, metrics_batch)

    total_loss += loss_batch

    for bench_i in range(len(real_target)):
        for clock_mem_id, clock_mem in enumerate(clocks['mem_clocks']):
            for clock_core_id, clock_core in enumerate(clocks['core_clocks'][clock_mem_id]):
                real = real_target[bench_i][clock_mem_id, clock_core_id].item()
                pred = pred_target[bench_i][clock_mem_id, clock_core_id].item()
                predicted_values.append(pred)
                if verbose == True:
                    if names == []:
                            print('\t\tValidation --- (%d, %d) Predicted factor (%s): %2.3f, Real factor: %2.3f (difference %5.2f%%)' %(clock_mem, clock_core, model_name, pred, real, 100.0*np.abs(real-pred)/real))
                    else:
                        maxwidth=len(max(names,key=len))
                        str_aux = '\t{message: >{width}}'.format(message=names[batch_index+bench_i], width=maxwidth+3)
                        str_aux += ': %5.2f%% --- (%d, %d) Predicted factor (%s): %2.3f, Real factor: %2.3f' %(100.0*np.abs(real-pred)/real, clock_mem, clock_core, model_name, pred, real)
                        print(str_aux)
                errors.append(np.absolute(pred-real)/real*100.0)

    output_batch = [errors, predicted_values, total_loss]
    return output_batch

def evaluate_lstm(device, clocks, verbose, models, X, y, model_name, pc, data, names=[]):
    total_programs = len(X)
    errors = []
    predicted_values = []
    total_loss=0.0

    batch_size = 4
    batch_index = 0
    while batch_index + batch_size <= total_programs:
        next_index = batch_index + batch_size
        args_batch = [errors, predicted_values, total_loss]
        metrics_batch={}
        if pc == True:
            for metric_id, metric in enumerate(metrics_to_use):
                metrics_batch[metric] = data[metric][batch_index:batch_index+batch_size]
        output_batch = processBatch(device, clocks, verbose, models, model_name, batch_index, batch_size, args_batch, X, y, names, pc, metrics_batch)
        errors = output_batch[0]
        predicted_values = output_batch[1]
        total_loss = output_batch[2]

        batch_index = next_index

    if batch_index < total_programs:
        batch_size_aux = total_programs - batch_index
        args_batch = [errors, predicted_values, total_loss]
        metrics_batch={}
        if pc == True:
            for metric_id, metric in enumerate(metrics_to_use):
                metrics_batch[metric] = data[metric][batch_index:batch_index+batch_size]
        output_batch = processBatch(device, clocks, verbose, models, model_name, batch_index, batch_size_aux, args_batch, X, y, names, pc, metrics_batch)
        errors = output_batch[0]
        predicted_values = output_batch[1]
        total_loss = output_batch[2]

    return total_loss, np.mean(errors), errors, predicted_values

def trainBatch(device, clocks, model_name, input_tensor, target_tensor, models, output_name, optimizers, criterion, batch_size, pc, metrics_batch):
    encoder = models['encoder']
    optimizer_encoder = optimizers['encoder']

    output_model = models[output_name]
    optimizer_output = optimizers[output_name]

    optimizer_encoder.zero_grad()
    optimizer_output.zero_grad()

    if (batch_size == 1):
        input_variable = Variable(input_tensor)
    else:
        input_variable = Variable(input_tensor.transpose(0, 1))

    target_variable = Variable(target_tensor)

    seq_length = input_variable.size()[0]
    target_length = target_variable.view(-1).size()[0]

    loss = 0

    encoder.hidden = encoder.initHidden(batch_size)
    for ei in range(seq_length):
        features = encoder(input_variable[ei], batch_size)

    features = features.view(batch_size, -1)

    for bi in range(batch_size):
        #if using performance_counters
        if pc == True:
            metrics_bi = []
            for metric_id, metric in enumerate(metrics_to_use):
                metrics_bi.append(metrics_batch[metric][bi])
            metrics_tensor = torch.tensor(metrics_bi, dtype=torch.float).to(device)

        for clock_mem_id, clock_mem in enumerate(clocks['mem_clocks']):
            for clock_core_id, clock_core in enumerate(clocks['core_clocks'][clock_mem_id]):
                clock_tensor = torch.tensor([clock_mem/clocks['highest_mem_clock'], clock_core/clocks['highest_core_clock']], dtype=torch.float).to(device)
                newfeatures = torch.cat((features[bi,:], clock_tensor), 0).to(device) #concatenate V-F level to the feature vector
                if pc == True:
                    newfeatures = torch.cat((newfeatures, metrics_tensor), 0).to(device)
                output = output_model(newfeatures)
                loss += criterion(output[0], target_variable[bi][clock_mem_id,clock_core_id])

    loss.backward()
    optimizer_encoder.step()
    optimizer_output.step()
    return loss.item() / target_length

def evaluate(device, clocks, verbose, no_output, epoch, dataset_name, models, data, pc):
    X = data['inst_seq']

    avg_loss = {}
    mean_abs_error = {}
    errors_per_bench = {}
    predicted_values = {}
    for model_name in models['output_types']:
        y = data[model_name]

        bench_names = data['names']

        aux_avg_loss, aux_mean_abs_error, aux_errors_per_bench, aux_predicted_values = evaluate_lstm(device, clocks, verbose, models, X, y, model_name, pc, data, bench_names)

        printing('\tEpoch %d - %s (%s) Absolute loss: %.4f' %(epoch, dataset_name, model_name, aux_avg_loss), no_output)
        printing('\tEpoch %d - %s (%s) Mean Absolute error: %.4f%%' %(epoch, dataset_name, model_name, aux_mean_abs_error), no_output)

        avg_loss[model_name] = aux_avg_loss
        mean_abs_error[model_name] = aux_mean_abs_error
        errors_per_bench[model_name] = np.array(aux_errors_per_bench)
        predicted_values[model_name] = aux_predicted_values

    return avg_loss, mean_abs_error, errors_per_bench, predicted_values

def trainPytorchModel(device, clocks, verbose, fast, pc, model_params, test_output_dir, original_data_train, original_data_val, original_data_test=[]):
    #unpacking arguments
    model_name = model_params['model_name']
    max_epochs = model_params['max_epochs']

    batch_size = model_params['encoder_params']['batch_size']
    never_stop = model_params['never_stop']
    no_output = model_params['no_output']

    #parameters for printing output
    output_dim = 1
    print_every = 2
    print_every = print_every*batch_size

    plot_losses = []
    print_loss_total = 0  # Reset every print_every
    plot_loss_total = 0  # Reset every plot_every

    counter_no_improvements = 0

    trainTime = time.time()
    torch.cuda.manual_seed(42)

    feature_vector_size = model_params['encoder_params']['hidden_size']

    #Pytorch Models
    encoder_lstm = mopy.EncoderLSTM(model_params['encoder_params']['vocab_size'], model_params['encoder_params']['embed_size'], feature_vector_size, model_params['encoder_params']['num_layers'], model_params['encoder_params']['dropout_prob'], batch_size, device).to(device)
    models = {'encoder': encoder_lstm, 'output_types': []}
    optimizers = {}
    if model_params['encoder_params']['optimizer_name'] == 'SGD':
        optimizers['encoder'] = torch.optim.SGD(encoder_lstm.parameters(), lr=model_params['encoder_params']['learning_rate'])
    elif model_params['encoder_params']['optimizer_name'] == 'Adam':
        optimizers['encoder'] = torch.optim.Adam(encoder_lstm.parameters(), lr=model_params['encoder_params']['learning_rate'])
    else:
        optimizers['encoder'] = torch.optim.Adagrad(encoder_lstm.parameters(), lr=model_params['encoder_params']['learning_rate'])

    for model_type in possible_outputs:
        if model_params['outputs'][model_type] == False:
            if pc == True:
                model_params['nn_params'][model_type]['input_size'] = feature_vector_size+2+len(metrics_to_use)
            else:
                model_params['nn_params'][model_type]['input_size'] = feature_vector_size+2

            #create neural network
            models[model_type] = mopy.multiNN2(model_params['nn_params'][model_type]['input_size'], 1, model_params['nn_params'][model_type]['hidden_sizes'], model_params['nn_params'][model_type]['dropout_prob']).to(device)
            models['output_types'].append(model_type)

            #select correct optimizer
            if model_params['nn_params'][model_type]['optimizer_name'] == 'SGD':
                optimizers[model_type] = torch.optim.SGD(models[model_type].parameters(), lr=model_params['nn_params'][model_type]['learning_rate'])
            elif model_params['nn_params'][model_type]['optimizer_name'] == 'Adam':
                optimizers[model_type] = torch.optim.Adam(models[model_type].parameters(), lr=model_params['nn_params'][model_type]['learning_rate'])
            else:
                optimizers[model_type] = torch.optim.Adagrad(models[model_type].parameters(), lr=model_params['nn_params'][model_type]['learning_rate'])

    num_output_models = len(models) - 1 - 1
    printing("Num output models: %d" %num_output_models, no_output)
    printing(models['output_types'], no_output)
    printing('Training model (vocab_size: %d, max_epochs: %d, batch_size: %d, num_output_models: %d, using_performance_metrics: %r)...' % (model_params['encoder_params']['vocab_size'], max_epochs, batch_size, num_output_models, pc), False)
    printing('\tLSTM Encoder -> eta: %f, drop_prob: %f, opt: %s, num_layers: %d, layer_size: %d, embed_size: %d' %(model_params['encoder_params']['learning_rate'], model_params['encoder_params']['dropout_prob'], model_params['encoder_params']['optimizer_name'], model_params['encoder_params']['num_layers'], model_params['encoder_params']['hidden_size'], model_params['encoder_params']['embed_size']), False)
    for model_type in possible_outputs:
        if model_params['outputs'][model_type] == False:
            printing('\tOutput \'%s\' -> eta: %f, drop_prob: %f, opt: %s, num_layers: %d, hidden_layer_sizes: %s' %(model_type, model_params['nn_params'][model_type]['learning_rate'], model_params['nn_params'][model_type]['dropout_prob'], model_params['nn_params'][model_type]['optimizer_name'], model_params['nn_params'][model_type]['num_layers'], model_params['nn_params'][model_type]['hidden_sizes']), False)

    loss_function = torch.nn.SmoothL1Loss()
    losses = []

    num_ubench_train = len(original_data_train['names'])
    printing("\tNumber programs training: %d" %num_ubench_train, no_output)
    # if fast == 0:
    num_ubench_val = len(original_data_val['names'])
    printing("\tNumber programs validation: %d" %num_ubench_val, no_output)

    num_bench_test = 0
    if original_data_test != []:
        num_bench_test = len(original_data_test['names'])
        printing("\tNumber programs testing: %d" %num_bench_test, no_output)

    num_clocks_mem = clocks['num_mem_clocks']
    num_clocks_core = clocks['num_core_clocks']

    results_train = {'loss_per_epoch': [], 'abs_error_per_epoch': [], 'errors_per_bench_all_epochs': [], 'last_epoch_predictions': []}
    results_val = {'loss_per_epoch': [], 'abs_error_per_epoch': [], 'errors_per_bench_all_epochs': [], 'last_epoch_predictions': []}
    results_test = {'loss_per_epoch': [], 'abs_error_per_epoch': [], 'errors_per_bench_all_epochs': [], 'last_epoch_predictions': []}

    data_train = reorderData(original_data_train, pc)

    data_val = reorderData(original_data_val, pc)
    if original_data_test != []:
        data_test = reorderData(original_data_test, pc)

    previous_mae = {}
    error_too_high = 0
    for epoch in range(max_epochs):
        printing("Starting Epoch %d" %epoch, no_output)
        batch_index = 0
        start = time.time()
        aux_iter=0
        model_type = epoch % num_output_models #to make sure all benchmarks are used to train the different output models

        while batch_index + batch_size <= num_ubench_train:
            next_index = batch_index + batch_size
            input_tensor =  tensorFromBatch(data_train['inst_seq'], batch_index, batch_size, device)

            target_train = data_train[models['output_types'][model_type]]
            target_tensor =  torch.tensor(target_train[batch_index:batch_index+batch_size], dtype=torch.float).to(device)

            metrics_batch={}
            if pc == True:
                for metric_id, metric in enumerate(metrics_to_use):
                    metrics_batch[metric] = data_train[metric][batch_index:batch_index+batch_size]

            loss = trainBatch(device, clocks, model_name, input_tensor, target_tensor, models, models['output_types'][model_type], optimizers, loss_function, batch_size, pc, metrics_batch)

            print_loss_total += loss
            plot_loss_total += loss

            batch_index = next_index

            model_type = (model_type + 1) % num_output_models
            aux_iter += batch_size
            if aux_iter % print_every == 0:
                print_loss_avg = print_loss_total / print_every
                print_loss_total = 0
                if verbose == True:
                    print('\t%s (%d: %.1f%%) %.4f' % (timeSince(start, aux_iter / num_ubench_train), aux_iter, aux_iter / num_ubench_train * 100, print_loss_avg))

        if batch_index < num_ubench_train:
            batch_size_aux = num_ubench_train - batch_index
            input_tensor =  tensorFromBatch(data_train['inst_seq'], batch_index, batch_size_aux, device)

            target_train = data_train[models['output_types'][model_type]]
            target_tensor =  torch.tensor(target_train[batch_index:batch_index+batch_size_aux], dtype=torch.float).to(device)

            metrics_batch={}
            if pc == True:
                for metric_id, metric in enumerate(metrics_to_use):
                    metrics_batch[metric] = data_train[metric][batch_index:batch_index+batch_size_aux]

            loss = trainBatch(device, clocks, model_name, input_tensor, target_tensor, models, models['output_types'][model_type], optimizers, loss_function, batch_size_aux, pc, metrics_batch)

            print_loss_total += loss
            plot_loss_total += loss

            aux_iter += batch_size_aux
            model_type = (model_type + 1) % num_output_models
            print_loss_avg = print_loss_total / print_every
            print_loss_total = 0
            if verbose == True:
                print('\t%s (%d: %.1f%%) %.4f' % (timeSince(start, aux_iter / num_ubench_train), aux_iter, aux_iter / num_ubench_train * 100, print_loss_avg))

        printing('\tEpoch %d Training Duration: %s' %(epoch, asMinutes(time.time()-start)), no_output)

        if fast == 0:
            #evaluate on training set
            temp_avg_loss,  temp_aux_mean_abs_error,  temp_errors_per_bench, results_train['last_epoch_predictions'] = evaluate(device, clocks, verbose, no_output, epoch, 'Training', models, data_train, pc)
            results_train['loss_per_epoch'].append(temp_avg_loss)
            results_train['abs_error_per_epoch'].append(temp_aux_mean_abs_error)
            results_train['errors_per_bench_all_epochs'].append(temp_errors_per_bench)

            if SAVE_STATE == True and (epoch+1) % SAVE_EVERY_TH_EPOCH == 0:
                temp_avg_loss,  temp_aux_mean_abs_error,  temp_errors_per_bench, results_val['last_epoch_predictions'] = evaluate(device, clocks, verbose, no_output, epoch, 'Validation', models, data_val, pc)
                results_val['loss_per_epoch'].append(temp_avg_loss)
                results_val['abs_error_per_epoch'].append(temp_aux_mean_abs_error)
                results_val['errors_per_bench_all_epochs'].append(temp_errors_per_bench)

                if original_data_test != []:
                    #evaluate on testing set
                    temp_avg_loss,  temp_aux_mean_abs_error,  temp_errors_per_bench, results_test['last_epoch_predictions'] = evaluate(device, clocks, verbose, no_output, epoch, 'Testing', models, data_test, pc)
                    results_test['loss_per_epoch'].append(temp_avg_loss)
                    results_test['abs_error_per_epoch'].append(temp_aux_mean_abs_error)
                    results_test['errors_per_bench_all_epochs'].append(temp_errors_per_bench)

                saveIntermediateState(test_output_dir, epoch, models, model_params, results_train, results_val, results_test, data_train, data_val, data_test)

            if epoch % 5 == 0:
                #evaluate on validation set
                temp_avg_loss,  temp_aux_mean_abs_error,  temp_errors_per_bench, results_val['last_epoch_predictions'] = evaluate(device, clocks, verbose, no_output, epoch, 'Validation', models, data_val, pc)
                results_val['loss_per_epoch'].append(temp_avg_loss)
                results_val['abs_error_per_epoch'].append(temp_aux_mean_abs_error)
                results_val['errors_per_bench_all_epochs'].append(temp_errors_per_bench)

                if epoch > 0:
                    for model_type in models['output_types']:
                        if np.absolute(temp_aux_mean_abs_error[model_type] - previous_mae[model_type]) < threshold_consecutive_epochs:
                            counter_no_improvements += 1
                            break
                        else:
                            counter_no_improvements = 0

                previous_mae = temp_aux_mean_abs_error

                if counter_no_improvements == max_consecutive_epochs_no_improvement:
                    printing('\tLimit of consecutive epochs without improvement was achieved', no_output)
                    error_too_high = 1
                else:
                    for model_type in models['output_types']:
                        if epoch > 5 and temp_aux_mean_abs_error[model_type] > 50:
                            printing('\tError is too high (%s) (>50%% after 5 epochs)' %(model_type), no_output)
                            error_too_high = 1

            elif epoch == max_epochs-1:
                temp_avg_loss,  temp_aux_mean_abs_error,  temp_errors_per_bench, results_val['last_epoch_predictions'] = evaluate(device, clocks, verbose, no_output, epoch, 'Validation', models, data_val, pc)
                results_val['loss_per_epoch'].append(temp_avg_loss)
                results_val['abs_error_per_epoch'].append(temp_aux_mean_abs_error)
                results_val['errors_per_bench_all_epochs'].append(temp_errors_per_bench)

            if epoch == max_epochs-1 or (error_too_high == 1 and never_stop == False): #if last epoch
                if original_data_test != []:
                    #evaluate on testing set
                    temp_avg_loss,  temp_aux_mean_abs_error,  temp_errors_per_bench, results_test['last_epoch_predictions'] = evaluate(device, clocks, verbose, no_output, epoch, 'Testing', models, data_test, pc)
                    results_test['loss_per_epoch'].append(temp_avg_loss)
                    results_test['abs_error_per_epoch'].append(temp_aux_mean_abs_error)
                    results_test['errors_per_bench_all_epochs'].append(temp_errors_per_bench)

            printing('\tEpoch %d Full Duration: %s' %(epoch, asMinutes(time.time()-start)), no_output)

        if never_stop == False and error_too_high == 1:
            break

    endTime = time.time()
    printing('Training Time: %s' %asMinutes(endTime-trainTime), False)

    return (endTime-trainTime), models, results_train, results_val, results_test

max_stars_repo_name: hpc-ulisboa/gpuPTXModel

max_issues_repo_name: hpc-ulisboa/gpuPTXModel

max_forks_repo_name: hpc-ulisboa/gpuPTXModel


text:
from sklearn.metrics import mean_squared_error
from scipy.stats.stats import pearsonr
from copy import copy

def ForwardPropFeatureImp(model, X_true, Y_true, mp):
    '''
    Leave one feature out  importance
    '''
    try:
        df_grid = mp.df_grid
    except:
        H_grid = mp.plot_grid()
        df_grid = mp.df_grid

    df_grid = df_grid.sort_values(['y', 'x']).reset_index(drop=True)
    Y = model.predict(X_true)
    mse = mean_squared_error(Y_true, Y)
    N, W, H, C = X_true.shape
    results = []
    for i in tqdm(range(len(df_grid)), ascii= True):
        ts = df_grid.iloc[i]
        y = ts.y
        x = ts.x
        X1 = copy(X_true)
        X1[:, y, x,:] = np.zeros(X1[:, y, x,:].shape)
        Y1 = model.predict(X1)
        mse_mutaion = mean_squared_error(Y_true, Y1)
        res = mse_mutaion - mse  # if res > 0, important, othervise, not important
        results.append(res)

    S = pd.Series(results, name = 'importance')
    df = df_grid.join(S)
    return df

max_stars_repo_name: riversdark/bidd-molmap

reason: from scipy


text:
import unittest

import numpy as np

from topocalc.gradient import gradient_d8
from topocalc.shade import shade

class TestShade(unittest.TestCase):

    # with self.dx and self.dy equal to 1, the cardinal direction
    # slope values will be np.pi/4 as one of the differences
    # will be zero
    dx = 1
    dy = 1

    dem = np.tile(range(10), (10, 1))

    def calc_slope(self, dem):
        """Calculate the slope

        Arguments:
            dem {numpy} -- dem array
        """

        return gradient_d8(dem, self.dx, self.dy, aspect_rad=True)

    def test_shade_cosz_bounds_error(self):

        with self.assertRaises(Exception) as context:
            shade(0, 0, 0, cosz=1.5)

        self.assertTrue('cosz must be > 0 and <= 1' in str(context.exception))

    def test_shade_zenith_bounds_error(self):

        with self.assertRaises(Exception) as context:
            shade(0, 0, 0, zenith=-10)

        self.assertTrue(
            'Zenith must be >= 0 and < 90' in str(context.exception))

    def test_shade_zenith_cosz_not_specified_error(self):

        with self.assertRaises(Exception) as context:
            shade(0, 0, 0)

        self.assertTrue(
            'Must specify either cosz or zenith' in str(context.exception))

    def test_shade_aspect_degrees_error(self):

        with self.assertRaises(Exception) as context:
            shade(0, 100, 0, zenith=10)

        self.assertTrue(
            'Aspect is not in radians from south' in str(context.exception))

    def test_shade_azimuth_value_error(self):

        with self.assertRaises(Exception) as context:
            shade(0, 0, 360, zenith=10)

        self.assertTrue(
            'Azimuth must be between -180 and 180 degrees' in str(context.exception))  # noqa

    def test_shade_west(self):

        # test west slope and aspect
        dem = np.tile(range(10), (10, 1))
        slope, asp = self.calc_slope(dem)

        zenith = 45
        mu = shade(slope, asp, 0, zenith=zenith)
        mu_cos = shade(slope, asp, 0, cosz=np.cos(zenith * np.pi / 180))

        self.assertAlmostEqual(np.mean(mu), 0.43769265754174774, places=7)
        np.testing.assert_allclose(mu, mu_cos)

    def test_shade_north(self):

        # test north slope and aspect
        dem = np.tile(range(10), (10, 1)).transpose()
        slope, asp = self.calc_slope(dem)

        zenith = 45
        mu = shade(slope, asp, 0, zenith=zenith)
        mu_cos = shade(slope, asp, 0, cosz=np.cos(zenith * np.pi / 180))

        self.assertAlmostEqual(np.mean(mu), 0.0, places=7)
        np.testing.assert_allclose(mu, mu_cos)

    def test_shade_east(self):

        # test east slope and aspect
        dem = np.fliplr(np.tile(range(10), (10, 1)))
        slope, asp = self.calc_slope(dem)

        zenith = 45
        mu = shade(slope, asp, 0, zenith=zenith)
        mu_cos = shade(slope, asp, 0, cosz=np.cos(zenith * np.pi / 180))

        self.assertAlmostEqual(np.mean(mu), 0.43769265754174774, places=7)
        np.testing.assert_allclose(mu, mu_cos)

    def test_shade_south(self):

        # test south slope and aspect
        dem = np.flipud(np.tile(range(10), (10, 1)).transpose())
        slope, asp = self.calc_slope(dem)

        zenith = 45
        mu = shade(slope, asp, 0, zenith=zenith)
        mu_cos = shade(slope, asp, 0, cosz=np.cos(zenith * np.pi / 180))

        self.assertAlmostEqual(np.mean(mu), 0.9930530248115434, places=7)
        np.testing.assert_allclose(mu, mu_cos)

max_stars_repo_name: cgallinger/topocalc

max_issues_repo_name: cgallinger/topocalc

max_forks_repo_name: cgallinger/topocalc


text:
import numpy as np
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.modules.normalization import LayerNorm

class MultiheadAttention(nn.Module):

	def __init__(self, embed_dim, num_heads, output_dim, dropout=0., bias=True, add_bias_kv=False, add_zero_attn=False, kdim=None, vdim=None):
		super(MultiheadAttention, self).__init__()
		self.embed_dim = embed_dim
		self.kdim = kdim if kdim is not None else embed_dim
		self.vdim = vdim if vdim is not None else embed_dim
		self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim

		self.num_heads = num_heads
		self.dropout = dropout
		self.head_dim = embed_dim // num_heads
		assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"

		self.in_proj_weight = nn.Parameter(torch.empty(3 * embed_dim, embed_dim))

		if self._qkv_same_embed_dim is False:
			self.q_proj_weight = nn.Parameter(torch.Tensor(embed_dim, embed_dim))
			self.k_proj_weight = nn.Parameter(torch.Tensor(embed_dim, self.kdim))
			self.v_proj_weight = nn.Parameter(torch.Tensor(embed_dim, self.vdim))

		if bias:
			self.in_proj_bias = nn.Parameter(torch.empty(3 * embed_dim))
		else:
			self.register_parameter('in_proj_bias', None)
		self.out_proj = nn.Linear(embed_dim, output_dim, bias=bias)

		if add_bias_kv:
			self.bias_k = nn.Parameter(torch.empty(1, 1, embed_dim))
			self.bias_v = nn.Parameter(torch.empty(1, 1, embed_dim))
		else:
			self.bias_k = self.bias_v = None

		self.add_zero_attn = add_zero_attn

		self._reset_parameters()

	def _reset_parameters(self):
		if self._qkv_same_embed_dim:
			nn.init.xavier_uniform_(self.in_proj_weight)
		else:
			nn.init.xavier_uniform_(self.q_proj_weight)
			nn.init.xavier_uniform_(self.k_proj_weight)
			nn.init.xavier_uniform_(self.v_proj_weight)

		if self.in_proj_bias is not None:
			nn.init.constant_(self.in_proj_bias, 0.)
			nn.init.constant_(self.out_proj.bias, 0.)
		if self.bias_k is not None:
			nn.init.nn.init.xavier_normal_(self.bias_k)
		if self.bias_v is not None:
			nn.init.xavier_normal_(self.bias_v)

	def forward(self, query, key, value, key_padding_mask=None,
				need_weights=True, attn_mask=None):

		if hasattr(self, '_qkv_same_embed_dim') and self._qkv_same_embed_dim is False:
			return F.multi_head_attention_forward(
				query, key, value, self.embed_dim, self.num_heads,
				self.in_proj_weight, self.in_proj_bias,
				self.bias_k, self.bias_v, self.add_zero_attn,
				self.dropout, self.out_proj.weight, self.out_proj.bias,
				training=self.training,
				key_padding_mask=key_padding_mask, need_weights=need_weights,
				attn_mask=attn_mask, use_separate_proj_weight=True,
				q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
				v_proj_weight=self.v_proj_weight)
		else:
			if not hasattr(self, '_qkv_same_embed_dim'):
				warnings.warn('A new version of MultiheadAttention module has been implemented. \
					Please re-train your model with the new module',
							  UserWarning)

			return F.multi_head_attention_forward(
				query, key, value, self.embed_dim, self.num_heads,
				self.in_proj_weight, self.in_proj_bias,
				self.bias_k, self.bias_v, self.add_zero_attn,
				self.dropout, self.out_proj.weight, self.out_proj.bias,
				training=self.training,
				key_padding_mask=key_padding_mask, need_weights=need_weights,
				attn_mask=attn_mask)


text:
import numpy as np
from sklearn import metrics
import pandas as pd
import os

from src.config import config

# client = config.client
# summary_collection = config.summary_collection

def deduplicate_summaries(summaries):
    result = []
    summaries.sort(key=lambda x: x['config']['abac_params']['itemset_limit'], reverse=False)
    summaries.sort(key=lambda x:x['config']['abac_params']['metric']['beta'])
    last_beta = None
    for summary in summaries:
        if summary['config']['abac_params']['metric']['beta'] == last_beta:
            continue
        last_beta = summary['config']['abac_params']['metric']['beta']
        result.append(summary)
    return result

if __name__ == "__main__":
    path = '../resources/csv/'
    files = os.listdir(path)
    for filename in files:
        if not filename.endswith('.csv'):
            continue
        filename = path + filename
        tprs = []
        tprs.append(1.0)
        fprs = []
        fprs.append(1.0)
        with open(filename) as f:
            for line in f:
                fpr = float(line.split()[0])
                tpr = float(line.split()[1])
                tprs.append(tpr)
                fprs.append(fpr)

        tprs.append(0.0)
        fprs.append(0.0)
        fpr = np.array(fprs)
        tpr = np.array(tprs)
        mean_auc = metrics.auc(fpr, tpr, reorder=True)
        print('%s: %f' % (filename, mean_auc))

    # open('../resources/uncoveredUnique.csv')
    # for gen_param_name in summary_collection.distinct('config.abac_params.generation_param_info_id'):
    #     summaries = list(summary_collection.find({'config.abac_params.generation_param_info_id':gen_param_name}))

    #     summaries = deduplicate_summaries(summaries)
    #     for summary in summaries:
    #         tprs.append(summary['cRecall'])
    #         fprs.append(summary['uFPR'])
    #     tprs.append(1.0)
    #     fprs.append(1.0)
    #     fpr = np.array(fprs)
    #     tpr = np.array(tprs)
    #     mean_auc = metrics.auc(fpr, tpr, reorder=True)
    #     print('%s: %f' % (gen_param_name, mean_auc))

max_stars_repo_name: MWSanders/AssociationAbacMiner

max_issues_repo_name: MWSanders/AssociationAbacMiner

max_forks_repo_name: MWSanders/AssociationAbacMiner


text:
from opymize import Variable, LinOp
from opymize.linear.sparse import idxop, einsumop

import numba
import numpy as np
from numpy.linalg import norm

try:
    import opymize.tools.gpu
    from opymize.tools.gpu import prepare_kernels
    from pkg_resources import resource_stream
except:
    # no cuda support
    pass

def indexedmult_prepare_gpu(B, P, x, type_t="double"):
    J = B.shape[0]
    K = x[0]['shape'][1]
    L = B.shape[2]
    M = B.shape[1]
    N = x[0]['shape'][0]
    constvars = {
        'J': J, 'K': K, 'L': L, 'M': M, 'N': N,
        'P': P, 'B': B, 'TYPE_T': type_t
    }
    files = [resource_stream('opymize.linear', 'indexed.cu')]
    templates = [
        ("indexedmult", "PP", (N, J, M), (32, 24, 1)),
        ("indexedmultadj", "PP", (N, 1, 1), (512, 1, 1))
    ]
    return prepare_kernels(files, templates, constvars)

class IndexedMultAdj(LinOp):
    """ for k,l,i do (Ax)[i,P[j,l]] -= \sum_m B[j,m,l] * x[j,i,m] """
    def __init__(self, K, N, P, B, adjoint=None):
        LinOp.__init__(self)
        assert P.shape[0] == B.shape[0]
        assert P.shape[1] == B.shape[2]
        self.x = Variable((B.shape[0],N,B.shape[1]))
        self.y = Variable((N,K))
        self.P = P
        self.B = B
        if adjoint is None:
            self.adjoint = IndexedMult(K, N, B, P, adjoint=self)
        else:
            self.adjoint = adjoint
        self._kernel = None
        self.spmat = self.adjoint.spmat.T

    def prepare_gpu(self, kernels=None, type_t="double"):
        if self._kernel is not None: return
        if kernels is None:
            kernels = indexedmult_prepare_gpu(self.B, self.P, self.y, type_t=type_t)
        self._kernel = kernels['indexedmultadj']
        self.adjoint.prepare_gpu(kernels, type_t=type_t)

    def _call_gpu(self, x, y=None, add=False):
        assert y is not None
        if not add: y.fill(0.0)
        self._kernel(x, y)

class IndexedMult(LinOp):
    """ (Ax)[j,i,m] -= \sum_l B[j,m,l] * x[i,P[j,l]] """
    def __init__(self, K, N, B, P, adjoint=None):
        LinOp.__init__(self)
        assert P.shape[0] == B.shape[0]
        assert P.shape[1] == B.shape[2]
        self.x = Variable((N,K))
        self.y = Variable((B.shape[0],N,B.shape[1]))
        self.P = P
        self.B = B

        self._kernel = None
        spP = [idxop(Pj, K) for Pj in P]
        spP = einsumop("jlk,ik->jil", spP, dims={ 'i': N })
        self.spmat = -einsumop("jml,jil->jim", B, dims={ 'i': N }).dot(spP)

        if adjoint is None:
            self.adjoint = IndexedMultAdj(K, N, P, B, adjoint=self)
        else:
            self.adjoint = adjoint

    def prepare_gpu(self, kernels=None, type_t="double"):
        if self._kernel is not None: return
        if kernels is None:
            kernels = indexedmult_prepare_gpu(self.B, self.P, self.x, type_t=type_t)
        self._kernel = kernels['indexedmult']
        self.adjoint.prepare_gpu(kernels, type_t=type_t)

    def _call_gpu(self, x, y=None, add=False):
        assert y is not None
        if not add: y.fill(0.0)
        self._kernel(x, y)

reason: import numpy,from numpy,import numba


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from model.base_module import BaseModule
from model.gcn_layer import GCNLayer
from model.gat_layer import GatLayer
from model.mlp_layer import MLPLayer

class GraphModel(BaseModule):
    def __init__(self, config):
        super().__init__(config)
        self.layer1 = self.new_layer(config.graph_layer, config.laplacian1)
        self.layer2 = self.new_layer(config.graph_layer, config.laplacian2)
        self.fc = MLPLayer(self.config.encoding_dim*2, self.config.num_classes)
        self.cached_embeds1 = torch.zeros(config.concept_num, config.encoding_dim).to(config.device)
        self.cached_embeds2 = torch.zeros(config.concept_num, config.encoding_dim).to(config.device)

    def new_layer(self, graph_layer, laplacian):
        if graph_layer == 'gcn':
            layer = GCNLayer(self.config, laplacian)
        if graph_layer == 'gat':
            layer = GatLayer(self.config, laplacian)
        assert layer, 'Graph layer is not supported.'
        return layer

    def forward(self, batch):
        i1, i2 = batch['i1'], batch['i2']
        o1 = self.layer1(self.config.embeddings, i1)
        o2 = self.layer2(self.config.embeddings, i2)
        o = torch.cat([o1, o2], -1)
        outs = self.fc(o)
        return outs

    def set_cached_embeds(self, batch):
        i = batch['i']
        o1 = self.layer1(self.config.embeddings, i)
        o2 = self.layer2(self.config.embeddings, i)
        pos = [i]
        self.cached_embeds1.index_put_(pos, o1.detach())
        self.cached_embeds2.index_put_(pos, o2.detach())

    def predict(self, batch):
        i1, i2 = batch['i1'], batch['i2']
        o1 = self.cached_embeds1.index_select(0, i1)
        o2 = self.cached_embeds2.index_select(0, i2)
        o = torch.cat([o1, o2], -1)
        outs = self.fc(o)
        return outs


text:
import pandas as pd
import numpy as np
from sklearn.base import TransformerMixin
import warnings
import scipy.stats as ss

########################################################################################
#
# DiaImputer to combine the other methods in this module
#
########################################################################################

class DiaImputer(TransformerMixin):

    def __init__(self, cutoff=0.02, cont_impute=None, cat_impute=None, missing_value=None):
        self.cutoff = cutoff
        self.cont_impute = cont_impute
        self.missing_value = missing_value
        self.cat_impute = cat_impute

    def fit(self, X, y=None):
        self.new_cols = NewMissingColumn(cutoff=self.cutoff).fit(X)
        imputer_construct_dict = {}
        if self.cont_impute is not None: imputer_construct_dict['cont_impute'] = self.cont_impute
        if self.missing_value is not None: imputer_construct_dict['missing_value'] = self.missing_value
        if self.cat_impute is not None: imputer_construct_dict['cat_impute'] = self.cat_impute
        self.imputer = DataFrameImputer(**imputer_construct_dict).fit(X)
        return self

    def transform(self, X):
        new_X = self.new_cols.transform(X)
        new_X = self.imputer.transform(new_X)
        return new_X

########################################################################################
#
# Create New Variables for columns with missing values above threshold
#
########################################################################################

class NewMissingColumn(TransformerMixin):

    def __init__(self, cutoff=0.02):
        self.cutoff = cutoff

        # Throw errors if the inputted parameters don't meet the necessary criteria
        if (cutoff < 0) | (cutoff >= 1):
            raise ValueError('cutoff ' + str(cutoff) + ' is not in the range [0, 1)')

    def fit(self, X, y=None):
        #Check if there are any NA values
        if X.isnull().values.any():
            self._new_column_id(X)
        else:
            warnings.warn("No NA values in dataframe. Process Finished")
        return self

    def transform(self, X):
        #call new_missing_columns
        new_columns = [j + "_IsMissing" for j in self.new_missing_columns]
        for i in range(len(new_columns)):
            this_column = self.new_missing_columns[i]
            this_new_column = new_columns[i]
            #Create new column by coercing old boolean column into 1/0
            X[this_new_column] = X[this_column].isnull() * 1
        #call ohe if required
        return X

    def _new_column_id(self, X):
        nrow = X.shape[1]
        #Find the missing columns
        missing_columns = X.columns[X.isnull().any()]

        # if above threshold, create new columns
        self.new_missing_columns = []
        for i in missing_columns:
            if (sum(X[i].isnull()) / nrow) > self.cutoff:
                self.new_missing_columns.append(i)
        return self

########################################################################################
#
# Impute values - median for categorical, user choice for continuous
#
########################################################################################
#https://stackoverflow.com/questions/25239958/impute-categorical-missing-values-in-scikit-learn

class DataFrameImputer(TransformerMixin):

    def __init__(self, cont_impute='median', cat_impute='mode', missing_value='missing'):
        """Impute missing values.
        Columns of dtype object are imputed with the most frequent value in column.
        Columns of other types are imputed with user selected (median as default) of column.
        """
        try:
            self.missing_value = str(missing_value)
        except ValueError:
            print("must enter missing_value parameter as a string")

        if cont_impute not in ["mean", "median", "mode", "missing_value"]:
            raise ValueError(str(cont_impute) + " is not a valid value for cont_impute. Must enter one of the following: ('mean', 'median', 'mode')")

        if cat_impute not in ['mode', 'missing_value']:
            raise ValueError(str(cat_impute) + " is not a valid value for cat_impute. Must enter one of the following: ('mode', 'missing_value')")

        self.cont_impute = cont_impute
        self.cat_impute = cat_impute

    def fit(self, X, y=None):
        def set_fill(self, X):
            self.fill = pd.Series([X[c].value_counts().index[0]
                                   if X[c].dtype == np.dtype('O') else self.this_fun(X[c]) for c in X],
                                  index=X.columns)

        def fill_cat_value(self, X):
            self.fill = pd.Series([self.missing_value
                                   if X[c].dtype == np.dtype('O') else self.this_fun(X[c]) for c in X],
                                  index=X.columns)

        #Set the imputer desired for each of the imputer values desired

        if self.cont_impute == 'mode':
            holder_fun = getattr(ss, 'mode')
            self.this_fun = lambda x: holder_fun(x)[0][0] #This is weird, but it works
            if self.cat_impute == 'missing_value':
                fill_cat_value(self, X)
            else:
                set_fill(self, X)
        else:
            self.this_fun = getattr(np, self.cont_impute)
            if self.cont_impute == 'median':
                self.fill = pd.Series([X[c].value_counts().index[0]
                                  if X[c].dtype == np.dtype('O') else X[c].median() for c in X],
                                  index=X.columns)
            else:
                if self.cat_impute == 'missing_value':
                    fill_cat_value(self, X)
                else:
                    set_fill(self, X)
        return self

    def transform(self, X, y=None):
        return X.fillna(self.fill)

max_stars_repo_name: chasedehan/diaml

max_issues_repo_name: chasedehan/diaml

max_forks_repo_name: chasedehan/diaml

reason: import numpy,import scipy


text:
# -*- coding: utf-8 -*-
"""CXR_model1_cnf_1000+600 -> 150 epochs.ipynb

Automatically generated by Colaboratory.

"""

#from google.colab import drive
#drive.mount('/content/drive')

# Commented out IPython magic to ensure Python compatibility.
#from tensorflow.keras.optimizers import Adam
from keras import backend as K
from keras.preprocessing.image import ImageDataGenerator,load_img, img_to_array
from keras.models import Sequential, Model
from keras.layers import Conv2D, MaxPooling2D,GlobalAveragePooling2D
from keras.layers import Activation, Dropout, BatchNormalization, Flatten, Dense, AvgPool2D,MaxPool2D
from keras.models import Sequential, Model
from keras.applications.vgg16 import VGG16, preprocess_input
from tensorflow.keras.optimizers import Adam, SGD, RMSprop

import tensorflow as tf

import os
import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
# %matplotlib inline

DATASET_DIR = "dataset path"

os.listdir(DATASET_DIR)

# Commented out IPython magic to ensure Python compatibility.
import glob
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# %matplotlib inline

normal_images = []
for img_path in glob.glob(DATASET_DIR + '/normal/*'):
    normal_images.append(mpimg.imread(img_path))

fig = plt.figure()
fig.suptitle('normal')
plt.imshow(normal_images[0], cmap='gray')

covid_images = []
for img_path in glob.glob(DATASET_DIR + '/covid/*'):
    covid_images.append(mpimg.imread(img_path))
fig = plt.figure()
fig.suptitle('covid')
plt.imshow(covid_images[0], cmap='gray')

print(len(normal_images))
print(len(covid_images))

IMG_W = 150
IMG_H = 150
CHANNELS = 3

INPUT_SHAPE = (IMG_W, IMG_H, CHANNELS)
NB_CLASSES = 2
EPOCHS = 150
BATCH_SIZE = 48

model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=INPUT_SHAPE))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(64,(3,3)))
model.add(Activation("relu"))
model.add(Conv2D(250,(3,3)))
model.add(Activation("relu"))

model.add(Conv2D(128,(3,3)))
model.add(Activation("relu"))
model.add(AvgPool2D(2,2))
model.add(Conv2D(64,(3,3)))
model.add(Activation("relu"))
model.add(AvgPool2D(2,2))

model.add(Conv2D(256,(2,2)))
model.add(Activation("relu"))
model.add(MaxPool2D(2,2))

model.add(Flatten())
model.add(Dense(32))
model.add(Dropout(0.25))
model.add(Dense(1))
model.add(Activation("sigmoid"))

model.compile(loss='binary_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])

model.summary()

train_datagen = ImageDataGenerator(rescale=1./255,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True,
    validation_split=0.4)

train_generator = train_datagen.flow_from_directory(
    DATASET_DIR,
    target_size=(IMG_H, IMG_W),
    batch_size=BATCH_SIZE,
    class_mode='binary',
    subset='training')

validation_generator = train_datagen.flow_from_directory(
    DATASET_DIR,
    target_size=(IMG_H, IMG_W),
    batch_size=BATCH_SIZE,
    class_mode='binary',
    shuffle= False,
    subset='validation')

history = model.fit_generator(
    train_generator,
    steps_per_epoch = train_generator.samples // BATCH_SIZE,
    validation_data = validation_generator,
    validation_steps = validation_generator.samples // BATCH_SIZE,
    epochs = EPOCHS)

plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

print("training_accuracy", history.history['accuracy'][-1])
print("validation_accuracy", history.history['val_accuracy'][-1])

label = validation_generator.classes

pred= model.predict(validation_generator)
predicted_class_indices=np.argmax(pred,axis=1)
labels = (validation_generator.class_indices)
labels2 = dict((v,k) for k,v in labels.items())
predictions = [labels2[k] for k in predicted_class_indices]
#print(predicted_class_indices)
#print (labels)
#print (predictions)

from sklearn.metrics import confusion_matrix

cm = confusion_matrix(predicted_class_indices,label)
total = sum(sum(cm))
acc = (cm[0, 0] + cm[1, 1]) / total
sensitivity = cm[0, 0] / (cm[0, 0] + cm[0, 1])
specificity = cm[1, 1] / (cm[1, 0] + cm[1, 1])

plt.matshow(cm)
plt.title('Confusion Matrix Plot')
plt.colorbar()
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show();

print(cm)
print("acc: {:.4f}".format(acc))
print("sensitivity: {:.4f}".format(sensitivity))
print("specificity: {:.4f}".format(specificity))

exp_series = pd.Series(label)
pred_series = pd.Series(predicted_class_indices)
pd.crosstab(exp_series, pred_series, rownames=['Actual'], colnames=['Predicted'],margins=True)

plt.matshow(cf)
plt.title('Confusion Matrix Plot')
plt.colorbar()
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show();


text:
import torch
from torch import nn
from torch.nn import functional as F

from pathlib import Path
import pandas as pd
import numpy as np
from collections import defaultdict
import pickle
import json

from sklearn.metrics import confusion_matrix, precision_recall_curve, auc
import src.models.metrics as metrics
from src.models.data_loader import create_data_loader
from src.models.models import HIBERT

from transformers import BertTokenizerFast, AdamW, get_linear_schedule_with_warmup

class Trainer():

    def __init__(self, model_dir, model_config,
                max_len=512*4, epochs=4, out_dir=None, device='cpu',
                loss_fn='cross_entropy_loss', optimizer='adamw', lr=1e-5, scheduler='linear_schedule_with_warmup',
                train_batch_size=32, val_batch_size=32, chunksize=512, sampler = None, print_n = 1):

        self.out_dir = out_dir
        self.model_dir = model_dir
        self.model_config = model_config
        self.epochs = epochs
        self.device = device

        if self.out_dir is None:
            self.out_dir = ''

        # load model and tokenizer
        self.model = HIBERT.from_pretrained(self.model_dir, **self.model_config)
        self.tokenizer = BertTokenizerFast.from_pretrained(self.model_dir)

        # loss function, optimizer and scheduler
        self.loss_fn = loss_fn
        self.optimizer = optimizer
        self.lr = lr
        self.scheduler = scheduler

        # dataloader parameters
        self.max_len = max_len
        self.train_batch_size = train_batch_size
        self.val_batch_size = val_batch_size
        self.chunksize = chunksize
        self.sampler = sampler
        self.shuffle = None
        self.print_n = print_n

        if self.sampler is None:
            self.shuffle = True
        else:
            self.shuffle = False

        # other attributes
        self.history = None
        self.best_f1 = None
        self.best_threshold = None
        self.metrics_names = None

        # compile constructor args as a dict (to save)
        self.params = {
            'model_dir': self.model_dir,
            'model_config': self.model_config,
            'max_len': self.max_len,
            'epochs': self.epochs,
            'out_dir': self.out_dir,
            'device': self.device,
            'loss_fn': self.loss_fn,
            'optimizer': self.optimizer,
            'lr': self.lr,
            'scheduler': self.scheduler,
            'train_batch_size': self.train_batch_size,
            'val_batch_size': self.val_batch_size,
            'chunksize': self.chunksize,
            'sampler': self.sampler
        }

    # loss function
    def get_loss_function(self, loss_fn, device):

        if loss_fn == 'cross_entropy_loss':
            out = nn.CrossEntropyLoss().to(device)

        return out

    def get_scheduler(self, scheduler, optimizer, total_steps):

        if scheduler == 'linear_schedule_with_warmup':
            out = get_linear_schedule_with_warmup(
                optimizer=optimizer,
                num_warmup_steps=0,
                num_training_steps=total_steps
                )

        return out

    def get_optimizer(self, model, optimizer, lr):

        if optimizer == 'adamw':
            out = AdamW(model.parameters(), lr=lr, correct_bias=False)

        return out

    # function for printing metrics
    def print_metrics(self, loss, loss_prefix, metrics_dict, batch_no=None):

        # print batch number if required
        if batch_no is None:
            out_str = ''
        else:
            out_str = 'Batch: ' + str(batch_no) + '     '

        # append loss
        out_str = out_str + loss_prefix + ': ' + '{:.4f}'.format(round(loss, 4))

        # loop through dict to add metrics to string
        for key in metrics_dict.keys():
            out_str = out_str + '     ' + key + ': ' + '{:.4f}'.format(round(metrics_dict[key], 4))

        print(out_str)

    # training for one epoch
    def train_epoch(self, model, data_loader, loss_fn, optimizer, device, scheduler):

        # Set to training mode
        model = model.train()

        losses = []
        pred_prob_list = []
        target_list = []
        batch = 0

        # training loop for one epoch
        for d in data_loader:
            input_ids = d['input_ids'].type(torch.LongTensor).to(device)
            attention_mask = d['attention_mask'].to(device)
            n_chunks = d['n_chunks'].to(device)
            target = d['target'].to(device)

            # pass through model
            outputs = model(
                input_ids = input_ids,
                attention_mask = attention_mask,
                n_chunks = n_chunks
            )

            # make predictions and calculate loss
            pred_prob = F.softmax(outputs, 1)[:, 1]
            loss = loss_fn(outputs, target)

            # collect loss
            losses.append(loss.item())

            # compulsory steps
            optimizer.zero_grad()
            loss.backward()

            # clip gradients
            nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)

            optimizer.step()
            scheduler.step()

            # detach tensors for metrics
            target_np = target.detach().numpy()
            pred_prob_np = pred_prob.detach().numpy()

            # collect targets and predicted prob for epoch level metrics
            target_list.append(target_np)
            pred_prob_list.append(pred_prob_np)

            # batch level metrics
            best_threshold_batch = metrics.calculate_best_threshold(y_true=target_np, probas_pred=pred_prob_np)
            metrics_batch = metrics.calculate_metrics(y_true=target_np, probas_pred=pred_prob_np, best_threshold=best_threshold_batch)

            # print batch level metrics
            if batch % self.print_n == 0:
                self.print_metrics(loss=loss.item(), loss_prefix='Batch Train loss', metrics_dict=metrics_batch, batch_no=batch)

            # update batch counter
            batch = batch + 1

        # Epoch level metrics (also return best threshold)
        best_threshold_epoch = \
            metrics.calculate_best_threshold(
                y_true=np.concatenate(target_list),
                probas_pred=np.concatenate(pred_prob_list)
                )

        metrics_epoch = \
            metrics.calculate_metrics(
                y_true=np.concatenate(target_list),
                probas_pred=np.concatenate(pred_prob_list),
                best_threshold=best_threshold_epoch
            )

        loss_epoch = np.mean(losses)

        return loss_epoch, best_threshold_epoch, metrics_epoch

    # evaluate model on validation set (based on best threshold)
    def eval_model(self, model, data_loader, loss_fn, device, best_threshold):

        # Turn on evaluation mode which disables dropout.
        model = model.eval()

        losses = []
        pred_prob_list = []
        target_list = []

        # evaluate model through the validation set
        # torch.no_grad() for speed and also since backprop is not needed
        with torch.no_grad():

            for d in data_loader:
                input_ids = d['input_ids'].type(torch.LongTensor).to(device)
                attention_mask = d['attention_mask'].to(device)
                n_chunks = d['n_chunks'].to(device)
                target = d['target'].to(device)

                # pass through model
                outputs = model(
                    input_ids = input_ids,
                    attention_mask = attention_mask,
                    n_chunks = n_chunks
                )

                # make predictions and calculate loss
                pred_prob = F.softmax(outputs, 1)[:, 1]
                loss = loss_fn(outputs, target)

                # collect loss
                losses.append(loss.item())

                # detach tensors for metrics
                target_np = target.detach().numpy()
                pred_prob_np = pred_prob.detach().numpy()

                # collect targets and predicted prob for epoch level metrics
                target_list.append(target_np)
                pred_prob_list.append(pred_prob_np)

        # Epoch level metrics (use threshold from training to make predictions)
        metrics_epoch = \
            metrics.calculate_metrics(
                y_true=np.concatenate(target_list),
                probas_pred=np.concatenate(pred_prob_list),
                best_threshold=best_threshold
            )

        loss_epoch = np.mean(losses)

        return loss_epoch, metrics_epoch

    # main training loop
    def train_model(self, epochs, model, train_data_loader, val_data_loader, loss_fn, optimizer, device, scheduler):

        self.history = defaultdict(list)
        best_f1 = 0

        # training loop
        for epoch in range(epochs):

            print(f'Epoch {epoch + 1} / {epochs}')
            print('-'*20)

            train_loss, best_threshold, train_metrics = self.train_epoch(
                model = model,
                data_loader = train_data_loader,
                loss_fn = loss_fn,
                optimizer = optimizer,
                device = device,
                scheduler = scheduler
            )

            print()
            self.print_metrics(loss=train_loss, loss_prefix='Epoch Train loss', metrics_dict=train_metrics)

            val_loss, val_metrics = self.eval_model(
                model = model,
                data_loader = val_data_loader,
                loss_fn = loss_fn,
                device = device,
                best_threshold = best_threshold
            )

            self.print_metrics(loss=val_loss, loss_prefix='Epoch val   loss', metrics_dict=val_metrics)
            print()

            # save history
            train_metrics['loss'] = train_loss
            val_metrics['loss'] = val_loss

            # save metric names for easier plotting
            self.metrics_names = [s.lower() for s in list(train_metrics.keys())]

            for key in train_metrics.keys():
                self.history['train_' + key.lower()].append(train_metrics[key])
                self.history['val_' + key.lower()].append(val_metrics[key])

            self.history['best_threshold'].append(best_threshold)

            # save best model
            if val_metrics['F1'] > best_f1:

                # save model
                model.save_pretrained(Path(self.out_dir, 'trained_model'))
                self.tokenizer.save_pretrained(Path(self.out_dir, 'trained_model'))

                # save threshold (convert to float because json does not allow numpy format)
                best_threshold_dict = {'best_threshold': float(best_threshold)}
                json.dump(best_threshold_dict, open(Path(self.out_dir, 'trained_model', 'best_threshold.json'), 'w'), indent=4)
                self.best_threshold = best_threshold

                # update f1
                best_f1 = val_metrics['F1']
                self.best_f1 = best_f1

        # when training is complete, dump parameters used in Trainer class
        json.dump(self.params, open(Path(self.out_dir, 'trained_model', 'trainer_params.json'), 'w'), indent=4)

    # call this function for training
    def fit(self, df_train, df_val):

        # create data loaders
        train_data_loader = create_data_loader(
            df = df_train,
            tokenizer = self.tokenizer,
            max_len = self.max_len,
            batch_size = self.train_batch_size,
            chunksize = self.chunksize,
            sampler = self.sampler,
            shuffle = self.shuffle,
            drop_last = True)

        val_data_loader = create_data_loader(
            df = df_val,
            tokenizer = self.tokenizer,
            max_len = self.max_len,
            batch_size = self.val_batch_size,
            chunksize = self.chunksize,
            sampler = None,
            shuffle = False,
            drop_last = False)

        # get loss function, optimizer and scheduler
        loss_fn = self.get_loss_function(loss_fn=self.loss_fn, device=self.device)
        optimizer = self.get_optimizer(model=self.model, optimizer=self.optimizer, lr=self.lr)
        scheduler = self.get_scheduler(scheduler=self.scheduler, optimizer=optimizer,
                                        total_steps=(len(train_data_loader) * self.epochs))

        # call main training loop
        self.train_model(
            epochs=self.epochs,
            model=self.model,
            train_data_loader=train_data_loader,
            val_data_loader=val_data_loader,
            loss_fn=loss_fn,
            optimizer=optimizer,
            device=self.device,
            scheduler=scheduler)

class Predictor():

    def __init__(self, model_dir, test_batch_size=32, device='cpu'):

        self.model_dir = model_dir
        self.test_batch_size = test_batch_size
        self.device = device

        # load params from Trainer() used to train model
        self.params = json.load(open(Path(self.model_dir, 'trainer_params.json')))

        # get required params from trainer
        self.model_config = self.params['model_config']
        self.max_len = self.params['max_len']
        self.chunksize = self.params['chunksize']

        # check if there is already a best threshold in the model folder
        try:
            best_threshold_dict = json.load(open(Path(self.model_dir, 'best_threshold.json')))
            self.best_threshold = best_threshold_dict['best_threshold']
        except (OSError, IOError) as e:
            self.best_threshold = None

        # load model and tokenizer
        self.model = HIBERT.from_pretrained(self.model_dir, **self.model_config)
        self.tokenizer = BertTokenizerFast.from_pretrained(self.model_dir)

        # other attributes
        self.is_unseen = None

    # Prediction function
    def pred_model(self, model, data_loader, device, best_threshold):

        # Turn on evaluation mode which disables dropout.
        model = model.eval()

        pred_prob_list = []
        target_list = []
        attn_wts_list = []
        input_ids_list = []
        n_chunks_list = []

        # make predictions on test set
        # torch.no_grad() for speed and also since backprop is not needed
        with torch.no_grad():

            for d in data_loader:
                input_ids = d['input_ids'].type(torch.LongTensor).to(device)
                attention_mask = d['attention_mask'].to(device)
                n_chunks = d['n_chunks'].to(device)
                target = d['target'].to(device)

                # pass through model
                outputs = model(
                    input_ids = input_ids,
                    attention_mask = attention_mask,
                    n_chunks = n_chunks
                )

                # make predictions
                pred_prob = F.softmax(outputs, 1)[:, 1]

                # collect targets and predicted prob for epoch level metrics
                target_list.append(target)
                pred_prob_list.append(pred_prob)
                attn_wts_list.extend(model.attn_weights)
                input_ids_list.append(input_ids)
                n_chunks_list.extend(n_chunks.tolist())

        # concat output outside the loop
        target_array = torch.cat(target_list).tolist()
        pred_prob_array = torch.cat(pred_prob_list).tolist()
        pred_array = (pd.Series(pred_prob_array) > best_threshold).astype(int).tolist()

        # compile attention weights
        attn_wts_compiled = [i.numpy().flatten().tolist() for i in attn_wts_list]

        # compile input_ids
        input_ids_array = torch.cat(input_ids_list)
        input_ids_chunks = input_ids_array.split_with_sizes(n_chunks_list)

        input_ids_compiled = []
        for chunk in input_ids_chunks:
            input_ids_compiled.append(chunk.view(1, -1).numpy().flatten().tolist())

        output = {
            'target': target_array,
            'pred_proba': pred_prob_array,
            'pred': pred_array,
            'attn_wts': attn_wts_compiled,
            'input_ids': input_ids_compiled,
        }

        return output

        # call this function for prediction
    def predict(self, df_test, threshold=0.5):

        df_test = df_test.copy()

        # initialize this variable
        self.is_unseen = False

        # df_test may not have a 'target' column
        # manually create one here
        if 'target' not in df_test.columns:
            df_test['target'] = [np.nan] * df_test.shape[0]
            self.is_unseen = True

        # create test data loader
        test_data_loader = create_data_loader(
            df = df_test,
            tokenizer = self.tokenizer,
            max_len = self.max_len,
            batch_size = self.test_batch_size,
            chunksize = self.chunksize,
            is_unseen = self.is_unseen,
            sampler = None,
            shuffle = False,
            drop_last = False)

        # make predictions
        return self.pred_model(
            model=self.model,
            data_loader=test_data_loader,
            device=self.device,
            best_threshold=threshold
            )