text:
import cv2
import numpy as np

# Read and show the raw images
ImagePath = './data/crop_digits/8.jpg'
rawImage = cv2.imread(ImagePath)
rawImage = cv2.resize(rawImage, (480,270))
rawImage = cv2.rotate(rawImage, cv2.ROTATE_90_CLOCKWISE)
print('Now showing the raw images')
cv2.imshow('rawImage', rawImage)
cv2.waitKey(0)

# pad the image for close operation
pad_img = np.zeros([50, rawImage.shape[1], 3]).astype(np.uint8)
padImage = np.concatenate([pad_img, rawImage, pad_img], axis=0)
pad_img = np.zeros([padImage.shape[0], 50, 3]).astype(np.uint8)
padImage = np.concatenate([pad_img, padImage, pad_img], axis=1)
cv2.imshow('rawImage', rawImage)
cv2.waitKey(0)

# Detect the user interface according HSV
hsvImage = cv2.cvtColor(padImage, cv2.COLOR_BGR2HSV)
low_hsv = np.array([100, 50, 150])
high_hsv = np.array([160, 255, 255])
binImage = cv2.inRange(hsvImage, lowerb=low_hsv, upperb=high_hsv)
cv2.imshow("HSV Binary", binImage)
cv2.waitKey(0)

# MedianBlur filtering
MedianImage = cv2.medianBlur(binImage, 3)
cv2.imshow("MedianBlur Filtering", MedianImage)
cv2.waitKey(0)

# Integrate the target area by using close operation
kernelX = cv2.getStructuringElement(cv2.MORPH_RECT, (30, 30))
CloseImage = cv2.morphologyEx(MedianImage, cv2.MORPH_CLOSE, kernelX)
cv2.imshow("Close Op", CloseImage)
cv2.waitKey(0)

# Eliminate the independent, small areas by using dilation and erosion
kernelX = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 10))
kernelY = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 10))
DEImage = cv2.dilate(CloseImage, kernelX)
DEImage = cv2.erode(DEImage, kernelX)
DEImage = cv2.erode(DEImage, kernelY)
DEImage = cv2.dilate(DEImage, kernelY)
cv2.imshow("dilate and erode", DEImage)
cv2.waitKey(0)

print('Done!')


text:
# -*- coding: UTF-8 -*-

# 11、简述面向对象中__new__和__init__区别,
'''
__init__ : 初始化方法, 创建对象后就默认被调用了, 可以传参数
1、__new__至少要有一个参数cls，代表当前类，此参数在实例化时由Python解释器自动识别

2、__new__必须要有返回值，返回实例化出来的实例，这点在自己实现__new__时要特别注意，
可以return父类（通过super(当前类名, cls)）__new__出来的实例，或者直接是object的__new__出来的实例

3、__init__有一个参数self，就是这个__new__返回的实例，__init__在__new__的基础上可以完成一些其它初始化的动作，__init__不需要返回值

4、如果__new__创建的是当前类的实例，会自动调用__init__函数，通过return语句里面调用的__new__函数的第一个参数是cls来保证是当前类实例，
如果是其他类的类名，；那么实际创建返回的就是其他类的实例，其实就不会调用当前类的__init__函数，也不会调用其他类的__init__函数。

'''
import random

# import numpy as np
import re

class Foo(object):
    def __init__(self, color, money):
        self.color = color
        self.money = money

    def move(self):
        print('running.........')

ferrari = Foo('red', '200w')
lawse = Foo('black', '300w')
print('法拉利的顔色为:{},价格{}:'.format(ferrari.color, ferrari.money))
print('劳斯的顔色为:{},价格{}:'.format(lawse.color, lawse.money))

class Obj(object):
    def __init__(self, name, age):
        print('3__init__:', self)  # 3__init__: <__main__.Obj object at 0x7f2e2e100240>
        self.name = name
        self.age = age

    def __new__(cls, *args, **kwargs):
        print('1_cls:', id(cls))  # 1_cls: 14915272
        if not hasattr(cls, '_inst'):
            cls._inst = super(Obj, cls).__new__(cls)
            print('2__new__:', cls._inst)  # 2__new__: <__main__.Obj object at 0x7f2e2e100240>
        return cls._inst

a = Obj('zhangsan', 19)
# b = Obj('lisi', 20)
print(a.name, a.age)
# print(b.name, b.age)
print('4_Obj:', id(Obj))  # 4_Obj: 14915272
print('*'*60)

# 12、简述with方法打开处理文件帮我我们做了什么？
"""
打开文件在进行读写的时候可能会出现一些异常状况，如果按照常规的f.open

写法，我们需要try,except,finally，做异常判断，并且文件最终不管遇到什么情况，
都要执行finally f.close()关闭文件，with方法帮我们实现了finally中f.close
"""
f = open('./note.txt', 'wb')
try:
    f.write('hello world')
except:
    pass
finally:
    f.close()

# with open('./note2.txt', 'wb') as f2:
#     f2.write('made in China')

# 13、列表[1,2,3,4,5],请使用map()函数输出[1,4,9,16,25]，并使用列表推导式提取出大于10的数，最终输出[16,25]
# map（）函数第一个参数是fun，第二个参数是一般是list，第三个参数可以写list，也可以不写，根据需求
li1 = [1, 2, 3, 4, 5]

def foo(x):
    return x ** 2

li2 = list(map(foo, li1))
print(li2)  # [1, 4, 9, 16, 25]
print([x for x in li2 if x > 10])  # [16, 25]
print('='*60)

# 14、python中生成随机整数、随机小数、0--1之间小数方法
print(random.randint(1, 3))
# print(np.random.randn(5))
print(random.random())  # 0.23450313090215458
print('&'*60)

# 15、避免转义给字符串加哪个字母表示原始字符串？
# r , 表示需要原始字符串，不转义特殊字符

# 16、re
str1 = '
中国
'
res = re.findall(r'
(.*?)
', str1)  # .* 类名可有可元, () 提取
print(res)
print("%"*60)

# 17、python 断言方法
n = 3
assert(n > 1)
print('success')  # 打印
# assert(n < 2)
# print('failure')  # 不打印

# 18、数据表student有id,name,score,city字段，其中name中的名字可有重复，需要消除重复行,请写sql语句
# select distinct name from *;

# 19、10个Linux常用命令
# ls pwd cp mv tar zip unzip lsof grep top tree cat tail herf du cal ssh
# touch mkdir sudo install version pip list cd more echo rm

# 20、python2和python3区别？列举5个,
# name = raw_input('')
# print(name)
# age = input('')
# print(age)
'''
1、Python3 使用 print 必须要以小括号包裹打印内容，比如 print('hi')

Python2 既可以使用带小括号的方式，也可以使用一个空格来分隔打印内容，比如 print 'hi'

2、python2 range(1,10)返回列表，python3中返回迭代器，节约内存

3、python2中使用ascii编码，python中使用utf-8编码

4、python2中unicode表示字符串序列，str表示字节序列

      python3中str表示字符串序列，byte表示字节序列

5、python2中为正常显示中文，引入coding声明，python3中不需要

6、python2中是raw_input()函数，python3中是input()函数

'''

ext: py

reason: import numpy


text:
import numpy as np
from numba import int64, float32, float64, jitclass, njit, types, generated_jit, int32
from numba.extending import overload
from numba.types import Tuple, UniTuple, unicode_type, List
from scipy.sparse import csr_matrix, csc_matrix
from DEC_Pipeline import DEC_Pipeline, DEC_Options
import os

####   Decades Sparse Graph   ####
spec_sparse = [('indptr', int64[:]), ('indices', int64[:]),
               ('node_attr', int64[:]), ('num_nodes', int64)]

@jitclass(spec_sparse)
class mk_DecSparseGraph:
    def __init__(self, graph_indptr, graph_indices, node_attr, num_nodes):
        self.indptr = graph_indptr
        self.indices = graph_indices
        self.node_attr = node_attr
        self.num_nodes = num_nodes

def DecSparseGraphSpec():
    return mk_DecSparseGraph.class_type.instance_type

@njit(DecSparseGraphSpec()(int64[:], int64[:], int64[:], int64), nogil=True, pipeline_class=DEC_Pipeline)
def DecSparseGraph(i, j, k, l):
    return mk_DecSparseGraph(i, j, k, l)

def LoadDecSparseGraph(data_path, vertical=False):
    filename='/'.join([data_path,'edge_list.tsv'])
    with open(filename) as f:
        graph_edge_data = np.fromfile(f, count=-1, sep='\t',dtype=np.int64)
        graph_edge_data = np.reshape(graph_edge_data, (int(len(graph_edge_data) / 2), 2))

    num_nodes = np.max(graph_edge_data) + 1
    graph_edge_data = np.concatenate((graph_edge_data,np.flip(graph_edge_data, axis=1)),axis=0)
    data=np.ones(graph_edge_data.shape[0])
    row=graph_edge_data[:,0]
    column=graph_edge_data[:,1]
    if vertical:
        graph=csc_matrix((data,(row,column)),shape=(num_nodes,num_nodes))
    else:
        graph=csr_matrix((data,(row,column)),shape=(num_nodes,num_nodes))

    filename='/'.join([data_path,'node_attr.tsv'])
    if os.path.isfile(filename):
        with open(filename) as f:
            graph_node_attr = np.fromfile(f, count=-1, sep='\t',dtype=np.int64)
            graph_node_attr = np.reshape(graph_node_attr, (int(len(graph_node_attr) / 2), 2))
            graph_node_attr = graph_node_attr[:,1]
    else:
        graph_node_attr = np.zeros(num_nodes, dtype=np.int64)

    sparse_graph = DecSparseGraph(np.array(graph.indptr, dtype=np.int64), np.array(graph.indices, dtype=np.int64), graph_node_attr,num_nodes)

    return sparse_graph

####   Decades Bipartite Graph   ####
spec_bipartite = [
    ('x_nodes', int64),                     # number of x nodes
    ('y_nodes', int64),                     # number of y nodes
    ('edges', int64),                       # number of edges
    ('node_array', int64[:]),               # vector of node_ids
    ('edge_array', int64[:]),               # vector of edge_ids
    ('edge_data', float64[:]),              # vector of edge_weights
    ('projection_size', int64),             # size of one dim of projection array
    ('first_val', int64),                   # first neighbor node id
    ('graph_edge_data', float64[:, ::1]),   # projected graph edge weights
    ('tri_size', int64)                     # size of projected triangular matrix
]

@jitclass(spec_bipartite)
class mk_DecBipartiteGraph():
    def __init__(self, edges, x_nodes, y_nodes, graph_edge_data):
        self.x_nodes = x_nodes
        self.y_nodes = y_nodes
        self.edges = edges
        self.node_array = np.zeros(self.x_nodes+1, dtype=np.int64)
        self.edge_array = np.zeros(self.edges, dtype=np.int64)
        self.edge_data = np.zeros(self.edges, dtype=np.float64)
        self.projection_size = np.int64(((y_nodes * y_nodes) - y_nodes) / 2)
        i = 0
        node = 0
        while i < self.edges:
            first, second, weight = graph_edge_data[i, :]
            if np.int64(first) != node:
                node += 1
                self.node_array[node] = i
            self.edge_array[i] = np.int64(second)
            self.edge_data[i] = weight
            i += 1
        self.node_array[self.x_nodes] = self.edges
        self.tri_size = np.int64(y_nodes * (y_nodes - 1) / 2)

def DecBipartiteGraphSpec():
     return mk_DecBipartiteGraph.class_type.instance_type

@njit(DecBipartiteGraphSpec()(int64, int64, int64, float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def DecBipartiteGraph(i, j, k, l):
    return mk_DecBipartiteGraph(i, j, k, l)

def LoadDecBipartiteGraph(filename):
    with open(filename) as f:
        _ = f.readline()
        line = f.readline().split()
        edges = np.int64(line[1])
        x_nodes = np.int64(line[2])
        y_nodes = np.int64(line[3])
        edge_data = np.loadtxt(f, dtype=np.float64)
    bipartite_graph = DecBipartiteGraph(edges, x_nodes, y_nodes, edge_data)

    return bipartite_graph

# Decades class implementation for csr, csc, coo
# sparse matrix data structures (see below for matrix operations)
spec_sparse_float64 = [('indptr', int64[:]), ('indices', int64[:]),
                       ('data', float64[:]), ('shape', int64[:]), ('size', int64)]

@jitclass(spec_sparse_float64)
class mk_DecCSR:
    def __init__(self, indptr, indices, data, shape):
        self.indptr = indptr
        self.indices = indices
        #self.shape = np.array([len(indptr)-1, np.max(indptr)], dtype=np.int64)
        self.shape = shape
        self.size = indptr[-1]
        self.data = data

@jitclass(spec_sparse_float64)
class mk_DecCSC:
    def __init__(self, indptr, indices, data, shape):
        self.indptr = indptr
        self.indices = indices
        # np.array([len(indptr), np.max(indptr)+1], dtype=np.int64)
        self.shape = shape
        self.size = indptr[-1]
        self.data = data

spec_sparse_coo_float64 = [('rows', int64[:]), ('cols', int64[:]),
                       ('data', float64[:]), ('shape', int64[:]), ('size', int64)]

@jitclass(spec_sparse_coo_float64)
class mk_DecCOO:
    def __init__(self, rows, cols, data, shape):
        self.rows = rows
        self.cols = cols
        # shape = np.array([np.max(rows)+1, np.max(cols)+1], dtype=np.int64)
        self.shape =shape
        self.size = np.int64(len(rows))
        self.data = data

def DecCSRSpec():
    return mk_DecCSR.class_type.instance_type

def DecCSCSpec():
    return mk_DecCSC.class_type.instance_type

def DecCOOSpec():
    return mk_DecCOO.class_type.instance_type

@njit(DecCSRSpec()(int64[:], int64[:], float64[:], int64[:]), nogil=True, pipeline_class=DEC_Pipeline)
def DecCSR(i, j, k, l):
    return mk_DecCSR(i,j,k, l)

@njit(DecCSCSpec()(int64[:], int64[:], float64[:], int64[:]), nogil=True, pipeline_class=DEC_Pipeline)
def DecCSC(i, j, k, l):
    return mk_DecCSC(i, j, k, l)

@njit(DecCOOSpec()(int64[:], int64[:], float64[:], int64[:]), nogil=True, pipeline_class=DEC_Pipeline)
def DecCOO(i, j, k, l):
    return mk_DecCOO(i, j, k, l)

# eliminate zeros in CSR matrix if any
@njit(DecCSRSpec()(DecCSRSpec()))
def eliminate_zeros(A):
    nrow, ncol = A.shape
    nnz, row_end = 0, 0
    for i in range(nrow):
        jj = row_end
        row_end = A.indptr[i+1]
        while (jj < row_end):
            j, x = A.indices[jj], A.data[jj]
            if x!=0:
                A.indices[nnz], A.data[nnz] = j, x
                nnz += 1
            jj +=1
        A.indptr[i+1] = nnz
    A.indices = A.indices[:nnz]
    A.data = A.data[:nnz]
    return A

# Decades implementation of sparse matrix operations
#
#   matrix transformations:
#        csr to csc
#        csc to csr
#        coo to csr
#        csr to coo
#        tranpose
#
#   matrix-matrix operations:
#        (for all combinations: sparse-sparse, sparse-dense, dense-sparse, dense-dense
#                              and for all three sparse matrix classes: csr, csc, coo)
#         dot product
#         Hadamard product

@njit(DecCSRSpec()(DecCOOSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def coo_to_csr(A):
    n_rows,n_cols= A.shape
    nnz = A.size
    indptr= np.zeros(n_cols+1, dtype=np.int64)
    indices= np.zeros(A.size, dtype=np.int64)
    data = np.zeros(A.size, dtype=np.float64)
    last, cumsum = 0, 0

    for n in range(nnz):
        indptr[A.rows[n]] += 1

    for i in range(n_rows):
        temp = indptr[i]
        indptr[i] = cumsum
        cumsum += temp

    for n in range(nnz):
        row = A.rows[n]
        dest = indptr[row]
        indices[dest] = A.cols[n]
        data[dest] = A.data[n]
        indptr[row] += 1

    for i in range(n_rows+1):
        temp = indptr[i]
        indptr[i] = last
        last = temp

    return DecCSR(indptr, indices, data, A.shape)

@njit(DecCOOSpec()(DecCSRSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def csr_to_coo(A):
    n_rows, n_cols = A.shape
    cols = A.indices
    rows = np.empty(len(cols), dtype=A.indices.dtype)

    for i in range(n_rows):
        for j in range(A.indptr[i],A.indptr[i+1]):
            rows[j] = i

    return DecCOO(rows, cols, A.data, A.shape)

@njit(DecCSCSpec()(DecCSRSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def csr_to_csc(A):
    n_rows,n_cols= A.shape
    indptr= np.zeros(n_cols+1, dtype=np.int64)
    indices= np.zeros(A.size, dtype=np.int64)
    data = np.zeros(A.size, dtype=np.float64)
    last = 0

    # enter number of elements per column:
    for i in range(A.indptr[-1]):
        indptr[A.indices[i]+1] += 1

    # cumsum number of elements per column:
    for i in range(1,len(indptr)):
        indptr[i] += indptr[i-1]

    # fill indices of rows into index array and
    # fill columnwise stacked data into data array
    for row in range(n_rows):
        for j in range(A.indptr[row], A.indptr[row+1]):
            col = A.indices[j]
            dest = indptr[col]
            indices[dest] = row
            data[dest] = A.data[j]
            indptr[col] += 1

    for col in range(n_cols+1):
        temp = indptr[col]
        indptr[col] = last
        last = temp

    return DecCSC(indptr, indices, data, A.shape)

@njit(DecCSRSpec()(DecCSCSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def csc_to_csr(A):
    B = DecCSR(A.indptr, A.indices, A.data, A.shape)
    B = csr_to_csc(B)
    return DecCSR(B.indptr, B.indices, B.data, A.shape)

@njit(DecCSRSpec()(DecCSRSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def transpose_csr(A):
    B = csr_to_csc(A)
    return DecCSR(B.indptr, B.indices, B.data, A.shape[::-1])

@njit(DecCSCSpec()(DecCSCSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def transpose_csc(A):
    B = DecCSR(A.indptr,A.indices,A.data, A.shape[::-1])
    B = csr_to_csc(B)
    return B

@njit(DecCOOSpec()(DecCOOSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def transpose_coo(A):
    return DecCOO(A.cols, A.rows, A.data, A.shape[::-1])

@njit(float64[:,:](float64[:,:],float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def dec_dot(x,y):
    m,d1=x.shape
    d2,n=y.shape
    if d2!=d1:
        print('matrices must have (m x k) and (k x n) dimensions')
        assert(False)
    d = d1
    z= np.zeros((m,n), dtype=np.float64)
    for i in range(m):
        for j in range(n):
            ij = 0
            for k in range(d):
                ij +=x[i,k]*y[k,j]
            z[i,j]= ij
    return z

@njit(float64[:,:](float64[:,:],float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def dec_mult(x,y):
    m,n=x.shape
    m2,n2 =y.shape
    if m2!=m or n2!=n:
        print('matrices must be the same dimensions')
        assert(False)
    z= np.zeros((m,n), dtype=np.float64)
    for i in range(m):
        for j in range(n):
            z[i,j] =x[i,j]*y[i,j]
    return z

@njit(float64[:,:](float64[:,:],float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def dec_div(x,y):
    m,n=x.shape
    m2,n2 =y.shape
    if m2!=m or n2!=n:
        print('matrices must be the same dimensions')
        assert(False)
    z= np.zeros((m,n), dtype=np.float64)
    for i in range(m):
        for j in range(n):
            z[i,j] =x[i,j]/y[i,j]
    return z

@njit(float64[:,:](float64[:,:],float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def dec_add(x,y):
    m,n=x.shape
    m2,n2 =y.shape
    if m2!=m or n2!=n:
        print('matrices must be the same dimensions')
        assert(False)
    z= np.zeros((m,n), dtype=np.float64)
    for i in range(m):
        for j in range(n):
            z[i,j] =x[i,j]+y[i,j]
    return z

@njit(float64[:,:](float64[:,:],float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def dec_subt(x,y):
    m,n=x.shape
    m2,n2 =y.shape
    if m2!=m or n2!=n:
        print('matrices must be the same dimensions')
        assert(False)
    z= np.zeros((m,n), dtype=np.float64)
    for i in range(m):
        for j in range(n):
            z[i,j] =x[i,j]-y[i,j]
    return z

@njit(float64[:,:](float64[:,:],float64), nogil=True, pipeline_class=DEC_Pipeline)
def dec_mul_scalar(x,y):
    m,n=x.shape
    z= np.zeros((m,n), dtype=np.float64)
    for i in range(m):
        for j in range(n):
            z[i,j] = y*x[i,j]
    return z

@njit(DecCSRSpec()(DecCSRSpec(),DecCSRSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def sparse_sparse_dot(A,B):
    if (A.shape[1]!=B.shape[0]):
        print('Matrix dimensions do not match!Make sure you are multiplying (m x k) and (k x n) matrices.')
        print('Extending one of the sparse matrix to match dimensions, beware if this is not intended')
    n_rows, n_cols = A.shape[0], B.shape[1]
    indptr = np.zeros((n_rows+1), dtype=np.int64)
    indptr[0] = 0
    mask = np.ones(n_cols, dtype=np.int64) * -1

    # pass 1:
    nnz = 0
    for i in range(n_rows):
        row_nnz = 0
        for jj in range(A.indptr[i], A.indptr[i+1]):
            j = A.indices[jj]
            for kk in range(B.indptr[j],B.indptr[j+1]):
                k = B.indices[kk]
                if mask[k] != i:
                    mask[k] = i
                    row_nnz += 1

        next_nnz = nnz + row_nnz
        nnz = next_nnz
        indptr[i+1] = nnz

    # pass 2:
    nnz = 0
    indptr[0] = 0;
    indices = np.zeros(indptr[-1], dtype=np.int64)
    data = np.zeros(indptr[-1], dtype=np.float64)

    for i in range(n_rows):
        sums = np.zeros(n_cols, dtype=np.float64)

        for jj in range(A.indptr[i], A.indptr[i+1]):
            j = A.indices[jj]
            v = A.data[jj]

            for kk in range(B.indptr[j],B.indptr[j+1]):
                k = B.indices[kk]
                sums[k] += v*B.data[kk]

        for jj in range(n_rows):
            if (sums[jj] != 0):
                indices[nnz] = jj
                data[nnz] = sums[jj]
                nnz += 1

        indptr[i+1] = nnz

    return DecCSR(indptr, indices, data, np.array([ n_rows, n_cols], dtype=np.int64))

@njit(DecCSRSpec()(DecCSRSpec(), DecCSCSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def dot_sparse_sparse(A, B):
    if (A.shape[1]!=B.shape[0]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x k) and (k x n) matrices.')
        print('Extending one of the sparse matrix to match dimensions, beware if this is not intended')
    return sparse_sparse_dot(A,csc_to_csr(B))

@njit((float64[:,:])(DecCSRSpec(),float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def dot_sparse_dense(A,B):
    if (A.shape[1]!=B.shape[0]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x k) and (k x n) matrices.')
        if (A.shape[1] > B.shape[0]):
            assert(False)
        else:
            print('Extending the sparse matrix (2nd argument) to match dimensions, beware if this is not intended')

    n_rows = A.shape[0]
    n_cols = np.int64(B.shape[1])
    C = np.zeros((n_rows, n_cols), dtype=np.float64)

    for i in range(n_rows):
        for j in range(n_cols):
            tmp = 0.0
            for k in range(A.indptr[i],A.indptr[i+1]):
                tmp += B[A.indices[k],j] * A.data[k]
            C[i,j] = tmp

        #C[i,:] = np.sum(np.multiply(A_row.reshape(-1,1), B[A.indices[A.indptr[i]:A.indptr[i+1]],:]), axis=0)
        #C[i,:] = np.dot(A.data[A.indptr[i]:A.indptr[i+1]],B[A.indices[A.indptr[i]:A.indptr[i+1]],:])
    return C

@njit((float64[:,:])(float64[:,:],DecCSCSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def dot_dense_sparse(B,A):

    if A.shape[0]!=B.shape[1]:
        print('Matrix dimensions do not match! Make sure you are multiplying (m x k) and (k x n) matrices.')
        if (A.shape[0] > B.shape[1]):
            assert(False)
        else:
            print('Extending the sparse matrix (2nd argument) to match dimensions, beware if this is not intended')

    n_cols = A.shape[1]
    n_rows = np.int64(B.shape[0])
    C = np.zeros((n_rows, n_cols), dtype=np.float64)
    for i in range(n_rows):
        for j in range(n_cols):
            tmp = 0.0
            for k in range(A.indptr[j],A.indptr[j+1]):
                tmp += B[i,A.indices[k]] * A.data[k]
            C[i,j] = tmp
            #C[:,i] = np.dot(,A.data[A.indptr[i]:A.indptr[i+1]])

    return C

@njit((float64[:,:],DecCSCSpec(), float64[:,:], int32, int32), nogil=True, pipeline_class=DEC_Pipeline)
def dot_dense_sparse_parallel(B,A,C, tid, num_threads):

    if A.shape[0]!=B.shape[1]:
        print('Matrix dimensions do not match! Make sure you are multiplying (m x k) and (k x n) matrices.')
        if (A.shape[0] > B.shape[1]):
            assert(False)
        else:
            print('Extending the sparse matrix (2nd argument) to match dimensions, beware if this is not intended')

    n_cols = A.shape[1]
    n_rows = np.int64(B.shape[0])

    #not the best way to distribute, the best would distribute remainder evenly rather than rounding up
    if n_cols < num_threads:
        chunk = 1
    elif n_cols%num_threads>0:
        chunk = (n_cols//num_threads) +1
    else:
        chunk = (n_cols//num_threads)

    for i in range(n_rows):
        for j in range((tid)*chunk, min(chunk*(tid+1), n_cols)):
            tmp = 0.0
            for k in range(A.indptr[j],A.indptr[j+1]):
                tmp += B[i,A.indices[k]] * A.data[k]
            C[i,j] = tmp
            #C[:,i] = np.dot(,A.data[A.indptr[i]:A.indptr[i+1]])

@njit(Tuple((int64[:],int64[:],int64[:]))(int64[:],int64[:]), nogil=True, pipeline_class=DEC_Pipeline)
def list_intersection(list1,list2):
    len1 = len(list1)
    len2 = len(list2)

    x = np.zeros(max(len1,len2), dtype=np.int64)
    xidx = np.zeros(max(len1,len2), dtype=np.int64)
    xjdx = np.zeros(max(len1,len2), dtype=np.int64)
    i, j, lenx = 0, 0, 0
    while (i < len1) and (j < len2):
        if list1[i] == list2[j]:
            x[lenx] = list1[i]
            xidx[lenx] = i
            xjdx[lenx] = j
            i += 1
            j += 1
            lenx += 1
        elif list1[i] < list2[j]:
            i += 1
        elif list1[i] > list2[j]:
            j += 1
    return x[:lenx], xidx[:lenx], xjdx[:lenx]

#@njit(Tuple((int64[:], int64[:]))(int64[:],int64[:]), nogil=True, pipeline_class=DEC_Pipeline)
@njit(Tuple((int64[:], int64[:], int64[:,:], int64[:,:], int64[:], int64[:], int64[:]))(int64[:],int64[:]), nogil=True, pipeline_class=DEC_Pipeline)
def list_union(list1,list2):
    len1 = len(list1)
    len2 = len(list2)
    minlen = min(len1,len2)
    intersection = np.zeros(minlen, dtype=np.int64)
    x_index = np.zeros(minlen, dtype=np.int64)

    union = np.zeros(len1+len2, dtype=np.int64)
    diff_list1 = np.zeros((2,len1), dtype=np.int64)
    diff_list2 = np.zeros((2,len2), dtype=np.int64)
    x_index = np.zeros(minlen, dtype=np.int64)
    xidx = np.zeros(minlen, dtype=np.int64)
    xjdx = np.zeros(minlen, dtype=np.int64)

    i, j, lenu, lenx, diff_i, diff_j, diff_ix, diff_jx = 0, 0, 0, 0, 0, 0, 0 ,0
    k=0
    while (i < len1) and (j < len2) and k<1:
        k +=1
        '''
        if list1[i] == list2[j]:
            union[lenu] = list1[i]
            intersection[lenx] = list1[i]
            x_index[lenx] = lenu
            xidx[lenx] = i
            xjdx[lenx] = j
            i += 1
            j += 1
            lenx += 1
            lenu += 1
        '''

        if list1[i] == list2[j]:
            union[lenu] = list1[i]
            diff_list1[0, diff_ix] = lenu
            diff_list1[1, diff_ix] =  i
            i += 1
            lenu += 1
            diff_ix +=1

        elif list1[i] > list2[j]:
            union[lenu] = list2[j]
            diff_list2[0,diff_jx] = lenu
            diff_list2[1,diff_jx] = j
            j += 1
            lenu += 1
            diff_jx += 1

    while i < len1:
            union[lenu] = list1[i]
            diff_list1[0,diff_ix] = lenu
            diff_list1[1,diff_ix] = i
            i += 1
            lenu += 1
            diff_ix +=1

    while j < len2:
            union[lenu] = list2[j]
            diff_list2[0,diff_jx] = lenu
            diff_list2[1,diff_jx] = j
            j += 1
            lenu += 1
            diff_jx += 1

    #return intersection, x_index
    return union[:lenu], intersection[:lenx], diff_list1[:,:diff_ix], diff_list2[:,:diff_jx], x_index[:lenx], xidx[:lenx], xjdx[:lenx]

    #return union, intersection, diff_list1, diff_list2, x_index, xidx, xjdx

    #zzz=np.array([0], dtype=np.int64)
    #zzz2=np.array([[0]], dtype=np.int64)
    #return zzz[:0], zzz[:0], zzz2[:0], zzz2[:0], zzz[:0], zzz[:0], zzz[:0]

@njit(DecCSRSpec()(DecCSRSpec(), DecCSRSpec()), nogil=True,  pipeline_class=DEC_Pipeline)
def ew_add_sparse_sparse(A, B):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)
    n_rows=A.shape[0]
    n_cols = A.shape[1]
    indptr= np.zeros(n_rows+1, dtype=np.int64)
    size = A.size+B.size
    data = np.zeros(size, dtype=np.float64)
    indices = np.zeros(size, dtype=np.int64)
    counter = np.int64(0)

    union , _ , diffA, diffB, x_index, xidx, xjdx = list_union(A.indices[A.indptr[0]: A.indptr[1]], B.indices[B.indptr[0]:B.indptr[1]])
    _ , x_index = list_union(A.indices[A.indptr[0]: A.indptr[1]], B.indices[B.indptr[0]:B.indptr[1]])

    for i in range(n_rows):
        union , _ , diffA, diffB, x_index, xidx, xjdx = list_union(A.indices[A.indptr[i]: A.indptr[i+1]], B.indices[B.indptr[i]:B.indptr[i+1]])
        offset = union.size
        n = x_index.size

        if x_index.size>0:
            data[counter:counter+offset][x_index] = np.array([A.data[A.indptr[i]:A.indptr[i+1]][xidx[k]] + B.data[B.indptr[i]:B.indptr[i+1]][xjdx[k]] for k  in range(n)], dtype=np.float64)

        if diffA.size>0:
            data[counter:counter+offset][diffA[0,:]]=A.data[A.indptr[i]:A.indptr[i+1]][diffA[1,:]]
        if diffB.size>0:
            data[counter:counter+offset][diffB[0,:]]=B.data[B.indptr[i]:B.indptr[i+1]][diffB[1,:]]

        indices[counter:counter+offset] = union
        counter += offset
        indptr[i+1] = counter

    return DecCSR(indptr, indices[:counter], data[:counter], A.shape)

    #return A

@njit(DecCSRSpec()(DecCSRSpec(), DecCSRSpec()), nogil=True,  pipeline_class=DEC_Pipeline)
def ew_subt_sparse_sparse(A, B):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)
    n_rows=A.shape[0]
    n_cols = A.shape[1]
    indptr= np.zeros(n_rows+1, dtype=np.int64)
    size = A.size+B.size
    data = np.zeros(size, dtype=np.float64)
    indices = np.zeros(size, dtype=np.int64)
    counter = np.int64(0)

    for i in range(n_rows):
        union , _ , diffA, diffB, x_index, xidx, xjdx = list_union(A.indices[A.indptr[i]: A.indptr[i+1]], B.indices[B.indptr[i]:B.indptr[i+1]])
        offset = union.size
        n = x_index.size

        if x_index.size>0:
            data[counter:counter+offset][x_index] = np.array([A.data[A.indptr[i]:A.indptr[i+1]][xidx[k]] - B.data[B.indptr[i]:B.indptr[i+1]][xjdx[k]] for k  in range(n)], dtype=np.float64)
        if diffA.size>0:
            data[counter:counter+offset][diffA[0,:]]=A.data[A.indptr[i]:A.indptr[i+1]][diffA[1,:]]
        if diffB.size>0:
            data[counter:counter+offset][diffB[0,:]]= -1 * B.data[B.indptr[i]:B.indptr[i+1]][diffB[1,:]]

        indices[counter:counter+offset] = union
        counter += offset
        indptr[i+1] = counter

    return DecCSR(indptr, indices[:counter], data[:counter], A.shape)

@njit(DecCSRSpec()(DecCSRSpec(), DecCSRSpec(), List(unicode_type)), nogil=True,  pipeline_class=DEC_Pipeline)
def elementwise_sparse_sparse(A, B, o):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)
    n_rows = A.shape[0]
    n_cols = A.shape[1]
    indptr= np.zeros(n_rows+1, dtype=np.int64)
    size = A.size+B.size
    data = np.zeros(size, dtype=np.float64)
    indices = np.zeros(size, dtype=np.int64)
    counter = np.int64(0)

    if o[0] =='*':
        for i in range(n_rows):
            x, xidx, xjdx = list_intersection(A.indices[A.indptr[i]: A.indptr[i+1]], B.indices[B.indptr[i]:B.indptr[i+1]])
            offset = xidx.size
            if offset>0:
                data[counter:counter+offset] = np.array([A.data[A.indptr[i]:A.indptr[i+1]][xidx[k]] * B.data[B.indptr[i]:B.indptr[i+1]][xjdx[k]] for k  in range(offset)], dtype=np.float64)
            indices[counter:counter+offset] = x
            counter += offset
            indptr[i+1] = counter

    else:
        for i in range(n_rows):
            union , _ , diffA, diffB, x_index, xidx, xjdx = list_union(A.indices[A.indptr[i]: A.indptr[i+1]], B.indices[B.indptr[i]:B.indptr[i+1]])
            offset = union.size
            n = x_index.size

            if o[0] =='+':
                if x_index.size>0:
                    data[counter:counter+offset][x_index] = np.array([A.data[A.indptr[i]:A.indptr[i+1]][xidx[k]] + B.data[B.indptr[i]:B.indptr[i+1]][xjdx[k]] for k  in range(n)], dtype=np.float64)
                if diffA.size>0:
                    data[counter:counter+offset][diffA[0,:]]=A.data[A.indptr[i]:A.indptr[i+1]][diffA[1,:]]
                if diffB.size>0:
                    data[counter:counter+offset][diffB[0,:]]=B.data[B.indptr[i]:B.indptr[i+1]][diffB[1,:]]
                #zzz = 0
            elif o[0] == '-':
                if x_index.size>0:
                    data[counter:counter+offset][x_index] = np.array([A.data[A.indptr[i]:A.indptr[i+1]][xidx[k]] - B.data[B.indptr[i]:B.indptr[i+1]][xjdx[k]] for k  in range(n)], dtype=np.float64)
                if diffA.size>0:
                    data[counter:counter+offset][diffA[0,:]]=A.data[A.indptr[i]:A.indptr[i+1]][diffA[1,:]]
                if diffB.size>0:
                    data[counter:counter+offset][diffB[0,:]]= -1*B.data[B.indptr[i]:B.indptr[i+1]][diffB[1,:]]

            else:
                print('this operation is not supported')
                assert(False)

            indices[counter:counter+offset] = union
            counter += offset
            indptr[i+1] = counter

    return DecCSR(indptr, indices[:counter], data[:counter], A.shape)

@njit(DecCSRSpec()(DecCSRSpec(), float64[:,:], List(unicode_type)), nogil=True, pipeline_class=DEC_Pipeline)
def elementwise_sparse_dense(A, B, o):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)

    #if o[0]=='/':
    #    print('Division of zero by zero is assigned to 0, instead of nan!')

    n_rows, n_cols = A.shape
    data = np.zeros(A.size, dtype=np.float64)
    indices = np.zeros(A.size, dtype=np.int64)
    indptr = np.zeros(n_rows+1, dtype=np.int64)
    counter=0
    for i in range(n_rows):
        cols=A.indices[A.indptr[i]: A.indptr[i+1]]
        offset=len(cols)
        if o[0] == '*':
            data[counter:counter+offset] =  np.array([A.data[A.indptr[i]:A.indptr[i+1]][k] * B[i,cols[k]] for k  in range(len(cols))], dtype=np.float32)
        elif o[0] == '/':
            data[counter:counter+offset] =  np.array([A.data[A.indptr[i]:A.indptr[i+1]][k] / B[i,cols[k]] for k  in range(len(cols))], dtype=np.float32)
        else:
            print('This operation is not supported')
            assert(False)
        indices[counter:counter+offset] = cols
        counter += offset
        indptr[i+1] = counter

    return DecCSR(indptr, indices, data, A.shape)

@njit(DecCSRSpec()(DecCSRSpec(), float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def ew_sparse_dense_div(A, B):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)

    n_rows, n_cols = A.shape
    data = np.zeros(A.size, dtype=np.float64)
    indices = np.zeros(A.size, dtype=np.int64)
    indptr = np.zeros(n_rows+1, dtype=np.int64)
    counter=0
    for i in range(n_rows):
        cols=A.indices[A.indptr[i]: A.indptr[i+1]]
        offset=len(cols)
        data[counter:counter+offset] =  np.array([A.data[A.indptr[i]:A.indptr[i+1]][k] / B[i,cols[k]] for k  in range(len(cols))], dtype=np.float32)
        indices[counter:counter+offset] = cols
        counter += offset
        indptr[i+1] = counter

    return DecCSR(indptr, indices, data, A.shape)

@njit((DecCSRSpec(), float64[:,:], DecCSRSpec(), int32, int32), nogil=True, pipeline_class=DEC_Pipeline)
def ew_sparse_dense_div_parallel(A, B, C, tid, num_threads):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)

    n_rows, n_cols = A.shape
    counter=0

    if n_rows< num_threads:
        chunk = 1
    elif n_rows%num_threads > 0:
        chunk = (n_rows//num_threads) +1
    else:
        chunk = (n_rows//num_threads)

    for i in range(tid*chunk, min((tid+1)* chunk, n_rows)):
        cols=A.indices[A.indptr[i]: A.indptr[i+1]]
        offset=len(cols)
        C.data[counter:counter+offset][:] =  np.array([A.data[A.indptr[i]:A.indptr[i+1]][k] / B[i,cols[k]] for k  in range(len(cols))], dtype=np.float32)
        C.indices[counter:counter+offset] = cols
        counter += offset
        C.indptr[i+1] = counter

@njit(DecCSRSpec()(DecCSRSpec(), float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def ew_sparse_dense_mult(A, B):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)

    n_rows, n_cols = A.shape
    data = np.zeros(A.size, dtype=np.float64)
    indices = np.zeros(A.size, dtype=np.int64)
    indptr = np.zeros(n_rows+1, dtype=np.int64)
    counter=0
    for i in range(n_rows):
        cols=A.indices[A.indptr[i]: A.indptr[i+1]]
        offset=len(cols)
        data[counter:counter+offset] =  np.array([A.data[A.indptr[i]:A.indptr[i+1]][k] * B[i,cols[k]] for k  in range(len(cols))], dtype=np.float32)
        indices[counter:counter+offset] = cols
        counter += offset
        indptr[i+1] = counter

    return DecCSR(indptr, indices, data, A.shape)

@njit(DecCSRSpec()(float64[:,:], DecCSRSpec(), List(unicode_type)), nogil=True, pipeline_class=DEC_Pipeline)
def elementwise_dense_sparse(A, B, o):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)
    if o[0] != '*':
        print('This operation is not supported with dense-sparse matrices')
        assert(False)
    return elementwise_sparse_dense(B, A, o)

@njit(DecCSRSpec()(DecCSRSpec(), float64[:,:]), nogil=True, pipeline_class=DEC_Pipeline)
def multiply_sparse_dense(A,B):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)
    n_rows, n_cols = A.shape
    data = np.zeros(A.size, dtype=np.float64)
    indices = np.zeros(A.size, dtype=np.int64)
    indptr = np.zeros(n_rows+1, dtype=np.int64)
    counter = np.int64(0)
    for i in range(n_rows):
        cols=A.indices[A.indptr[i]: A.indptr[i+1]]
        offset=len(cols)
        data[counter:counter+offset] =  np.array([A.data[A.indptr[i]:A.indptr[i+1]][k] * B[i,cols[k]] for k  in range(len(cols))], dtype=np.float32)
        indices[counter:counter+offset] = cols
        counter += offset
        indptr[i+1] = counter

    return DecCSR(indptr, indices, data, A.shape)

@njit(DecCSRSpec()(float64[:,:], DecCSRSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def multiply_dense_sparse(A, B):
    if (A.shape[0]!=B.shape[0]) or (A.shape[1] != B.shape[1]):
        print('Matrix dimensions do not match! Make sure you are multiplying (m x n) and (m x n) matrices.')
        assert(False)
    return multiply_sparse_dense(B, A)

@generated_jit(nopython=True, nogil=True)
def dot(A, B):
    if A is DecCSRSpec():
        if B is DecCSRSpec():
            return lambda A,B: dot_sparse_sparse(A, B)
        elif B is DecCSCSpec():
            return lambda A,B: dot_sparse_sparse(A, csc_to_csr(B))
        elif isinstance(B, np.ndarray):
            return lambda A,B: dot_sparse_dense(A,B)
        else:
            print('input types not supported')
            assert(False)
    elif A is DecCSCSpec():
        if B is DecCSRSpec():
            return lambda A,B: dot_sparse_sparse(csc_to_csr(A), B)
        elif B is DecCSCSpec():
            return lambda A,B: dot_sparse_sparse(csc_to_csr(A), csc_to_csr(B))
        elif isinstance(B, np.ndarray):
            return lambda A,B: dot_sparse_dense(csc_to_csr(A), B)
        else:
            print('input types not supported')
            assert(False)
    elif isinstance(A, np.ndarray):
       	if B is DecCSRSpec():
            return lambda A,B: dot_dense_sparse(A, csr_to_csc(B))
        elif B is DecCSCSpec():
            return lambda A,B: dot_dense_sparse(A, B)
        elif isinstance(B, np.ndarray):
            return lambda A,B: dec_dot(A, B)
        else:
            print('input types not supported')
            assert(False)
    else:
        print('input types not supported')
        assert(False)

# Hadamard product
@generated_jit(nopython=True, nogil=True)
def multiply(A, B):
    if A is DecCSRSpec():
        if B is DecCSRSpec():
            return lambda A,B: multiply_sparse_sparse(A, B)
        elif B is DecCSCSpec():
            return lambda A,B: multiply_sparse_sparse(A, csc_to_csr(B))
        elif isinstance(B, np.ndarray):
            return lambda A,B: multiply_sparse_dense(A,B)
        else:
            print('input types not supported')
            assert(False)
    elif A is DecCSCSpec():
        if B is DecCSRSpec():
            return lambda A,B: multiply_sparse_sparse(csc_to_csr(A), B)
        elif B is DecCSCSpec():
            return lambda A,B: multiply_sparse_sparse(csc_to_csr(A), csc_to_csr(B))
        elif isinstance(B, np.ndarray):
            return lambda A,B: multiply_sparse_dense(csc_to_csr(A), B)
        else:
            print('input types not supported')
            assert(False)
    elif isinstance(A, np.ndarray):
       	if B is DecCSRSpec():
            return lambda A,B: multiply_dense_sparse(A, B)
        elif B is DecCSCSpec():
            return lambda A,B: multiply_dense_sparse(A, csc_to_csr(B))
        elif isinstance(B, np.ndarray):
            return lambda A,B: dec_mult(A, B)
        else:
            print('input types not supported')
            assert(False)
    else:
        print('input types not supported')
        assert(False)

@njit((float64[:], int64[:], int64[:], int64[:],int64[:], int64[:], int64[:]), nogil=True, pipeline_class=DEC_Pipeline)
def _minimum_spanning_tree(data, cols, indptr, i_sort, rows, predecessors, rank):

    # step through the edges from smallest to largest.
    #  V1 and V2 are connected vertices.
    i, n_edges_in_mst = 0, 0
    n_verts = predecessors.shape[0]
    n_data = i_sort.shape[0]

    while i < n_data and n_edges_in_mst < n_verts - 1:
        j =  i_sort[i]
        V1, V2 = rows[j], cols[j]
        # progress upward to the head node of each subtree
        R1 = V1
        while predecessors[R1] != R1:
            R1 = predecessors[R1]
        R2 = V2
        while predecessors[R2] != R2:
            R2 = predecessors[R2]

        # Compress both paths.
        while predecessors[V1] != R1:
            predecessors[V1] = R1
        while predecessors[V2] != R2:
            predecessors[V2] = R2

        # if the subtrees are different, then we connect them and keep the
        # edge.  Otherwise, we remove the edge: it duplicates one already
        # in the spanning tree.
        if R1 != R2:
            n_edges_in_mst += 1

            # Use approximate (because of path-compression) rank to try
            # to keep balanced trees.
            if rank[R1] > rank[R2]:
                predecessors[R2] = R1
            elif rank[R1] < rank[R2]:
                predecessors[R1] = R2
            else:
                predecessors[R2] = R1
                rank[R1] += 1
        else:
            data[j] = 0

        i += 1

    # We may have stopped early if we found a full-sized MST so zero out the rest
    while i < n_data:
        j = i_sort[i]
        data[j] = 0
        i += 1

@njit(DecCSRSpec()(DecCSRSpec()), nogil=True, pipeline_class=DEC_Pipeline)
def dec_minimum_spanning_tree(G):
    #  Work-horse routine for computing minimum spanning tree using
    #  Kruskal's algorithm.  By separating this code here, we get more
    #  efficient indexing.
    n_verts = G.shape[0]
    i_sort = np.argsort(G.data)
    predecessors = np.arange(n_verts)
    rank = np.zeros(predecessors.shape[0], dtype=np.int64)
    rows = np.zeros(G.size, dtype=np.int64)
    for i in range(n_verts):
        for j in range(G.indptr[i], G.indptr[i + 1]):
            rows[j] = i

    _minimum_spanning_tree(G.data, G.indices, G.indptr,
                                   i_sort, rows, predecessors, rank)
    eliminate_zeros(G)
    return G

@njit(int64(int64[:]), nogil=True, pipeline_class=DEC_Pipeline)
def dec_int64_max(f):
    ret = np.int64(-2147483648)
    for i in f:
        if i > ret:
            ret = i
    return ret

####   Triangular Dense Matrix    ####

@njit(int64(int64, int64, int64), nogil=True, pipeline_class=DEC_Pipeline)
def TriDenseGraph_num_ele_i_rows(n, i, tri_size):
    return np.int64(tri_size - ((n-i)*((n-i)-1)/2))

@njit(UniTuple(int64, 2)(int64, int64), nogil=True, pipeline_class=DEC_Pipeline)
def TriDenseGraph_k_to_i_j(k, n):
    i = np.int64(n - 2 - np.floor(np.sqrt(-8 * k + 4 * n * (n - 1) - 7) / 2.0 - 0.5))
    j = np.int64(k + i + 1 - n * (n - 1) / 2 + (n - i) * ((n - i) - 1) / 2)
    return i, j

@njit((), nogil=True, pipeline_class=DEC_Pipeline)
def PyDECADES_barrier():
    assert(False)

@njit(int32(int32[:]), nogil=True, pipeline_class=DEC_Pipeline)
def PyDECADES_fetchAdd(addr):
    tmp = addr[0]
    addr[0] += 1
    return tmp

@njit(int32(int32[:], int32), nogil=True, pipeline_class=DEC_Pipeline)
def PyDECADES_atomicAdd(addr, val):
    tmp = addr[0]
    addr[0] += val
    return tmp

max_stars_repo_name: PrincetonUniversity/DecadesCompiler

max_issues_repo_name: PrincetonUniversity/GraphAttack

max_forks_repo_name: PrincetonUniversity/GraphAttack

reason: import numpy,from scipy,from numba


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

'''
plot pre-, post 2013, and post 2015 longitudinal profiles along Flat Creek Glacier.
'''
in_file = '/Users/mistral/Documents/CUBoulder/Science/Sulzer/data/TerrainAnalysis/slope_profiles.csv'
data = pd.read_csv(in_file, header = 1,
    names = ['d12', 'x12', 'y12', 'z12', 'd14', 'x14','y14', 'z14', 'd16', 'x16', 'y16', 'z16'])

fig = plt.figure(figsize = (11,5))
#plt.style.use('ggplot')
plt.plot(data.d12, data.z12, label = 'Elevation 2012', color = 'purple')
plt.plot(data.d14, data.z14, label = 'Elevation 2014', color = 'darkorange')
#plt.plot(data.d15, data.z15, label = 'Elevation 2015')
plt.plot(data.d16, data.z16, color = 'dimgray', label = 'Elevation 2016')
plt.xlabel('Distance [m]', fontsize = 18)
plt.ylabel('Elevation [m]', fontsize = 18)
plt.title('Profile view of Flat Creek detachments', fontsize = 18)
plt.ylim([1900,2750])
plt.xlim([-20, 1650])
ax = plt.gca()
plt.yticks(np.arange(2000, 2800, step=200))
ax.tick_params('both', labelsize = 16)
plt.legend(fontsize = 16)
plt.grid(True, color='gainsboro', linestyle='-', linewidth=1)
plt.tight_layout()
#plt.show()
plt.savefig('FlatCreek_profile_view.pdf', format = 'pdf')

max_stars_repo_name: mjacqu/FlatCreekProject

max_issues_repo_name: mjacqu/FlatCreekProject

max_forks_repo_name: mjacqu/FlatCreekProject


text:
# Import numpy
import numpy as np

# Assign the filename: file
file = 'digits_header.txt'

# Load the data: data
data = np.loadtxt(file, delimiter="\t", skiprows=1, usecols= [0,3])

# Print data
print(data)


text:
import numpy as np
from PIL import Image

def numpy2pil(array):
    return Image.fromarray(np.transpose(array, (1, 2, 0)))


text:
"""
  Copyright 2020 ETH Zurich, Secure, Reliable, and Intelligent Systems Lab

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
"""
import warnings

import numpy as np
import onnx
from onnx import numpy_helper
from config import config

def onnxshape_to_intlist(onnxshape):
	"""
	ONNX has its own wrapper for shapes. Our optimizer expects a list of ints.

	Arguments
	---------
	onnxshape : TensorShapeProto

	Return
	------
	output : list
	    list of ints corresponding to onnxshape
	"""
	result = list(map(lambda j: 1 if j.dim_value is None else int(j.dim_value), onnxshape.dim))

	# No shape means a single value
	if not result:
		return [1]

	# convert NCHW to NHWC
	if len(result) == 4:
		return [result[0], result[2], result[3], result[1]]

	return result

def nchw_to_nhwc_shape(shape):
	"""
	Reorders dimensions of a 1D array from NCHW to NHWC, since ONNX uses NCHW, ELINA expects NHWC.

	:param index: the array to be converted

	:return: converted array
	"""
	assert len(shape) == 4, "Unexpected shape size"
	return [shape[0], shape[2], shape[3], shape[1]]

def nchw_to_nhwc_index(index: int) -> int:
	"""
	Converts an single index from NCHW to NHWC, since ONNX uses NCHW, ELINA expects NHWC,

	:param index: the index to be converted

	:return: converted index
	"""
	assert 0 <= index <= 3, f"index out of range: {index}"
	if index == 0:  # batch (N)
		return 0
	elif index == 1:  # channel (C)
		return 3
	else:
		return index - 1

def nchw_to_nhwc(array):
	"""
	ONNX uses NCHW. ELINA expects NHWC

	:param array: array to be converted

	:return: converted array
	"""
	if array.ndim == 4:
		return array.transpose(0, 2, 3, 1)

	return array

def reshape_nhwc(shape_in, shape_out):
	#print(shape_in, shape_out)
	ndim_in = len(shape_in)
	ndim_out = len(shape_out)
	total_in = np.prod(shape_in[1:ndim_in])
	total_out = np.prod(shape_out[1:ndim_out])
	assert total_in == total_out, "Reshape doesn't have same number of neurons before and after"
	array = np.asarray(range(total_in)).reshape(shape_in[1:ndim_in])
	if array.ndim == 3:
		array = array.transpose((2, 0, 1))
	array = array.reshape(shape_out[1:ndim_out])
	if array.ndim == 3:
		return array.transpose((1, 2, 0))
	else:
		return array

def prepare_model(model):
	"""
	The constructor has produced a graph_def with the help of the functions graph_util.convert_variables_to_constants and graph_util.remove_training_nodes.
	translate() takes that graph_def, imports it, and translates it into two lists which then can be processed by an Optimzer object.

	Return
	------
	(operation_types, operation_resources) : (list, list)
	    A tuple with two lists, the first one has items of type str and the second one of type dict. In the first list the operation types are stored (like "Add", "MatMul", etc.).
	    In the second list we store the resources (matrices, biases, etc.) for those operations. It is organised as follows: operation_resources[i][domain] has the resources related to
	    operation_types[i] when analyzed with domain (domain is currently either 'deepzono' or 'deeppoly', as of 8/30/18)
	"""
	shape_map = {}
	constants_map = {}
	output_node_map = {}
	input_node_map = {}

	for initial in model.graph.initializer:
		const = nchw_to_nhwc(numpy_helper.to_array(initial)).copy()
		constants_map[initial.name] = const
		shape_map[initial.name] = const.shape

	placeholdernames = []
	#print("graph ", model.graph.node)
	for node_input in model.graph.input:
		placeholdernames.append(node_input.name)
		if node_input.name not in shape_map:
			shape_map[node_input.name] = onnxshape_to_intlist(node_input.type.tensor_type.shape)
			input_node_map[node_input.name] = node_input

	for node in model.graph.node:
		#print(node.op_type)
		output_node_map[node.output[0]] = node
		for node_input in node.input:
			input_node_map[node_input] = node
		if node.op_type == "Flatten":
			#shape_map[node.output[0]] = shape_map[node.input[0]]
			shape_map[node.output[0]] = [1,] + [np.prod(shape_map[node.input[0]][1:]),]
		elif node.op_type == "Constant":
			const = node.attribute
			const = nchw_to_nhwc(numpy_helper.to_array(const[0].t)).copy()
			constants_map[node.output[0]] = const
			shape_map[node.output[0]] = const.shape

		elif node.op_type in ["MatMul", "Gemm"]:
			transA = 0
			transB = 0
			for attribute in node.attribute:
				if 'transA' == attribute.name:
					transA = attribute.i
				elif 'transB' == attribute.name:
					transB = attribute.i
			input_shape_A = ([1] if len(shape_map[node.input[0]])==1 else []) + list(shape_map[node.input[0]])
			input_shape_B =  list(shape_map[node.input[1]]) + ([1] if len(shape_map[node.input[1]])==1 else [])
			M = input_shape_A[transA]
			N = input_shape_B[1 - transB]
			shape_map[node.output[0]] = [M, N]

		elif node.op_type in ["Add", "Sub", "Mul", "Div"]:
			shape_map[node.output[0]] = shape_map[node.input[0]]
			if node.input[0] in constants_map and node.input[1] in constants_map:
				if node.op_type == "Add":
					result = np.add(constants_map[node.input[0]], constants_map[node.input[1]])
				elif node.op_type == "Sub":
					result = np.subtract(constants_map[node.input[0]], constants_map[node.input[1]])
				elif node.op_type == "Mul":
					result = np.multiply(constants_map[node.input[0]], constants_map[node.input[1]])
				elif node.op_type == "Div":
					result = np.divide(constants_map[node.input[0]], constants_map[node.input[1]])
				constants_map[node.output[0]] = result
		elif node.op_type in ["Conv", "MaxPool", "AveragePool"]:
			output_shape = []
			input_shape = shape_map[node.input[0]]

			require_kernel_shape = node.op_type in ["MaxPool", "AveragePool"]
			if not require_kernel_shape:
				filter_shape = shape_map[node.input[1]]
				kernel_shape = filter_shape[1:-1]

			strides = [1, 1]
			padding = [0, 0, 0, 0]
			auto_pad = 'NOTSET'
			dilations = [1, 1]
			group = 1
			ceil_mode = 0
			for attribute in node.attribute:
				if attribute.name == 'strides':
					strides = attribute.ints
				elif attribute.name == 'pads':
					padding = attribute.ints
				elif attribute.name == 'auto_pad':
					auto_pad = attribute.s
				elif attribute.name == 'kernel_shape':
					kernel_shape = attribute.ints
				elif attribute.name == 'dilations':
					dilations = attribute.ints
				elif attribute.name == 'group':
					group = attribute.i
				elif attribute.name == 'ceil_mode':
					ceil_mode = attribute.i

			effective_kernel_shape = [(kernel_shape[i] - 1) * dilations[i] + 1 for i in range(len(kernel_shape))]

			output_shape.append(input_shape[0])

			for i in range(len(kernel_shape)):
				effective_input_size = input_shape[1 + i]
				effective_input_size += padding[i]
				effective_input_size += padding[i + len(kernel_shape)]
				if ceil_mode == 1:
					strided_kernel_positions = int(np.ceil((effective_input_size - effective_kernel_shape[i]) / float(strides[i])))
				else:
					strided_kernel_positions = int(np.floor((effective_input_size - effective_kernel_shape[i]) / strides[i]))
				output_shape.append(1 + strided_kernel_positions)

			if require_kernel_shape:
				output_shape.append(input_shape[3])
			else:
				output_shape.append(filter_shape[0])

			shape_map[node.output[0]] = output_shape
		elif node.op_type in ["Relu", "Sigmoid", "Tanh", "Softmax", "BatchNormalization", "LeakyRelu"]:
			shape_map[node.output[0]] = shape_map[node.input[0]]

		# Gather is for the moment solely for shapes
		elif node.op_type == "Gather":
			axis = 0
			for attribute in node.attribute:
				axis = attribute.i
			if node.input[0] in constants_map and node.input[1] in constants_map:
				data = constants_map[node.input[0]]
				indexes = constants_map[node.input[1]]
				constants_map[node.output[0]] = np.take(data, indexes, axis)

			if node.input[0] in shape_map and node.input[1] in shape_map:
				r = len(shape_map[node.input[0]])
				q = len(shape_map[node.input[1]])
				out_rank = q + r - 1
				if out_rank == 0:
					shape_map[node.output[0]] = shape_map[node.input[1]]
				else:
					output_shape = []
					for i in range(out_rank):
						if i < axis:
							output_shape.append(shape_map[node.input[0]][i]) # i < axis < r
						elif i >= axis and i < axis + q:
							output_shape.append(shape_map[node.input[0]][i-axis]) # i - axis < q
						else:
							output_shape.append(shape_map[node.input[0]][i - q + 1]) # i < out_rank < q + r - 1
					shape_map[node.output[0]] = output_shape
		elif node.op_type == "Shape":
			if node.input[0] in shape_map:
				constants_map[node.output[0]] = shape_map[node.input[0]]
				shape_map[node.output[0]] = [len(shape_map[node.input[0]])]

		#elif node.op_type == "Cast":
			#shape_map[node.output[0]] = shape_map[node.input[0]]
			#print("CASTING ", node.input[0], shape_map[node.input[0]], shape_map[node.output[0]])

		elif node.op_type == "Reshape":
			#print("RESHAPE ", node.input, node.output)
			if node.input[1] in constants_map:
				total = 1
				replace_index = -1
				for index in range(len(constants_map[node.input[1]])):
					if constants_map[node.input[1]][index] == -1:
						replace_index = index
					else:
						total *= constants_map[node.input[1]][index]

				if replace_index != -1:
					constants_map[node.input[1]][replace_index] = np.prod(shape_map[node.input[0]]) / total

				if len(constants_map[node.input[1]]) == 4:
					shape_map[node.output[0]] = [constants_map[node.input[1]][0], constants_map[node.input[1]][2], constants_map[node.input[1]][3], constants_map[node.input[1]][1]]
				else:
					shape_map[node.output[0]] = constants_map[node.input[1]]

		elif node.op_type == "Unsqueeze":
			if node.input[0] in shape_map:
				axis = node.attribute[0].ints
				output_shape = list(shape_map[node.input[0]])
				if node.input[0] in constants_map:
					constants_map[node.output[0]] = constants_map[node.input[0]]
				for i in axis:
					output_shape.insert(i, 1)
					if node.input[0] in constants_map:
						constants_map[node.output[0]] = np.expand_dims(constants_map[node.output[0]], axis=i)
				shape_map[node.output[0]] = output_shape

		elif node.op_type == "Concat":
			all_constant = True
			n_dim = len(shape_map[node.input[0]])
			if n_dim > 2:
				axis = nchw_to_nhwc_index(node.attribute[0].i)
			else:
				axis = node.attribute[0].i
			for node_input in node.input:
				if not node_input in constants_map:
					all_constant = False
					break
			if all_constant:
				constants_map[node.output[0]] = np.concatenate([constants_map[input] for input in node.input], axis=axis)
			all_shape_known = True
			for node_input in node.input:
				if not node_input in shape_map:
					all_shape_known = False
					break
			assert all_shape_known, "Unknown shape for at least one node input!"
			new_axis_size = 0
			for node_input in node.input:
				new_axis_size += shape_map[node_input][axis]
			shape_map[node.output[0]] = [shape_map[node.input[0]][i] if i != axis else new_axis_size for i in range(len(shape_map[node.input[0]]))]
			if not all_constant:
				assert axis == n_dim-1, "ELINA currently only supports concatenation on the channel dimension"

		elif node.op_type == "Tile":
			repeats = nchw_to_nhwc_shape(constants_map[node.input[1]])
			input_shape = list(shape_map[node.input[0]])
			assert len(repeats) == len(input_shape), "Expecting one repeat factor per dimension"
			output_shape = [factor * size for factor, size in zip(repeats, input_shape)]
			shape_map[node.output[0]] = output_shape

			repeat_index = np.where(np.array(repeats) != 1)[0]
			assert len(repeat_index) == 1, "ELINA backend currently only supports repeats for one dimension"
			repeat_index = repeat_index.item()
			assert repeat_index == 1, "ELINA backend currently only supports repeats for the first dimension"
			assert input_shape[0] == 1, "ELINA backend currently only supports repeats for dimensions of size 1"

		elif node.op_type == "Expand":
			if node.input[1] in constants_map:
				if len(constants_map[node.input[1]]) == 4:
					shape_map[node.output[0]] = [constants_map[node.input[1]][0], constants_map[node.input[1]][2], constants_map[node.input[1]][3], constants_map[node.input[1]][1]]
				else:
					shape_map[node.output[0]] = constants_map[node.input[1]]

				result = np.zeros(shape_map[node.output[0]]) + constants_map[node.input[0]]
				constants_map[node.output[0]] = result
		elif node.op_type == "Pad":
			input_shape = np.array(shape_map[node.input[0]])
			for attribute in node.attribute:
				if attribute.name == "pads":
					padding = np.array(attribute.ints)
				if attribute.name == "mode":
					assert attribute.s == bytes(b'constant'), "only zero padding supported"
				if attribute.name == "value":
					assert attribute.f == 0, "only zero padding supported"
			output_shape = np.copy(input_shape)
			input_dim = len(input_shape)
			assert len(padding) == 2* input_dim
			for i in range(2,input_dim): # only pad spatial dimensions
				output_shape[i-1] += padding[i]+padding[i+input_dim]
			shape_map[node.output[0]] = list(output_shape)
		elif node.op_type == "Transpose":
			dim_order = None
			for attribute in node.attribute:
				if 'perm' == attribute.name:
					dim_order = attribute.ints
			assert dim_order is not None
			shape_map[node.output[0]] = np.array(shape_map[node.input[0]])[dim_order].tolist()
		else:
			assert 0, f"Operations of type {node.op_type} are not yet supported."

	#print('const_map')
	#print(constants_map)
	#print('shape_map')
	#print(shape_map)
	return shape_map, constants_map, output_node_map, input_node_map, placeholdernames

class ONNXTranslator:
	"""
	This class is used to turn a ONNX model into two lists that then can be processed by an Optimizer object
	"""
	def __init__(self, model, is_gpupoly):
		"""
		This constructor takes a reference to a ONNX Model and checks model, infers intermediate shapes and sets up maps from name to type and node or constant value
		graph_util.convert_variables_to_constants and graph_util.remove_training_nodes to cleanse the graph of any nodes that are linked to training. This leaves us with
		the nodes you need for inference.
		In the resulting graph there should only be tf.Operations left that have one of the following types [Const, MatMul, Add, BiasAdd, Conv2D, Reshape, MaxPool, AveragePool, Placeholder, Relu, Sigmoid, Tanh, LeakyRelu]
		If the input should be a Keras model we will ignore operations with type Pack, Shape, StridedSlice, and Prod such that the Flatten layer can be used.

		Arguments
		---------
		model : onnx.ModelProto
		"""
		if issubclass(model.__class__, onnx.ModelProto):
			onnx.checker.check_model(model)
			self.model = model
			self.nodes = self.model.graph.node
			self.is_gpupoly = is_gpupoly
			self.shape_map, self.constants_map, self.output_node_map, self.input_node_map, self.placeholdernames = prepare_model(model)
		else:
			assert 0, 'not onnx model'

	def find_input(self):
		inputs_dir = {x.name: x for x in self.model.graph.input}
		all_inputs = [x for y in self.nodes for x in y.input]
		[all_inputs.remove(x) for y in self.nodes for x in y.output if x in all_inputs]
		[all_inputs.remove(x.name) for x in self.model.graph.initializer if x.name in all_inputs]

		assert all_inputs[0] in inputs_dir

		return inputs_dir[all_inputs[0]]

	@staticmethod
	def clean_shape(shape_raw):
		'''
		Onnx translator expects the inputs and outputs of each node to not have 0-sized dimensions.
		These can occur, if other formats are converted to onnx instead of directly exporting an onnx model.
		This function handles such occurances, setting the 0-sized dimension to 1.

		Arguments
		--------
		shape_raw : A shape in form of a list
		'''
		shape_cleaned = [1 if x == 0 else x for x in shape_raw]
		if 0 in shape_raw:
			warnings.warn(f"0-sized dimension encountered: {shape_raw} and changed to: {shape_cleaned}",RuntimeWarning)
		return shape_cleaned

	def translate(self):
		"""
		The constructor has produced a graph_def with the help of the functions graph_util.convert_variables_to_constants and graph_util.remove_training_nodes.
		translate() takes that graph_def, imports it, and translates it into two lists which then can be processed by an Optimzer object.

		Return
		------
		(operation_types, operation_resources) : (list, list)
		    A tuple with two lists, the first one has items of type str and the second one of type dict. In the first list the operation types are stored (like "Add", "MatMul", etc.).
		    In the second list we store the resources (matrices, biases, etc.) for those operations. It is organised as follows: operation_resources[i][domain] has the resources related to
		    operation_types[i] when analyzed with domain (domain is currently either 'deepzono' or 'deeppoly', as of 8/30/18)
		"""
		operation_types     = ["Placeholder"]
		placeholder = self.find_input()
		in_out_placeholder = ([], placeholder.name, self.clean_shape(onnxshape_to_intlist(placeholder.type.tensor_type.shape)))
		orig_input_shape = in_out_placeholder[-1]
		if len(orig_input_shape) == 4:
			orig_input_shape = orig_input_shape[:1] + orig_input_shape[-1:] + orig_input_shape[1:-1]
		orig_input_shape = self.clean_shape(orig_input_shape)
		operation_resources = [{'deepzono':in_out_placeholder, 'deeppoly':in_out_placeholder}]
		reshape_map = {}
		operations_to_be_ignored = ["Pack", "Shape", "StridedSlice", "Prod", "Unsqueeze", "Softmax", "Flatten", "BatchNormalization"]
		padding_merger_dict = {}

		### Check if there Are Add/Sub and Div/Mul layers that can be interpreted as normalization layer
		stop_norm_layers = ["MatMul","Gemm","Conv","MaxPool","Relu","Sigmoid","Tanh","LeakyRelu"]
		stop_norm_layer = len(self.nodes)
		extract_mean = False
		extract_std = False
		for node_idx, node in enumerate(self.nodes):
			if node.op_type in stop_norm_layers or (extract_mean and extract_std):
				stop_norm_layer = node_idx
				break
			if node.op_type in ["Add","Sub"]:
				extract_mean = True
			elif node.op_type in ["Div", "Mult"]:
				extract_std = True
		extract_norm = extract_std and extract_mean
		apply_permute = None

		for node_idx, node in enumerate(self.nodes):
			# print("node ", node.op_type)
			if node.op_type == "Constant":
				continue
			elif node.op_type in operations_to_be_ignored:
				input_name = node.input[0]
				output_name = node.output[0]
				if input_name in reshape_map:
					reshape_map[output_name] = reshape_map[input_name]
				else:
					reshape_map[output_name] = input_name
				continue

			operation_types.append(node.op_type)
			# take means and stds out of the network
			if extract_norm and node_idx <= stop_norm_layer and len(operation_types) == 2 and node.op_type in ["Add", "Sub", "Mul", "Div"] and node.output[0] not in self.constants_map:
				constant = self.add_resources(node)[0].reshape(-1)
				if node.op_type == "Add":
					config.mean = np.multiply(constant, -1)
					print(f"Mean of {config.mean} extracted from network")
				elif node.op_type == "Sub":
					config.mean = constant
					print(f"Mean of {config.mean} extracted from network")
				elif node.op_type == "Mul":
					config.std = np.divide(1, constant)
					print(f"Std of {config.std} extracted from network")
				elif node.op_type == "Div":
					config.std = constant
					print(f"Std of {config.std} extracted from network")

				self.ignore_node(node, operation_types, reshape_map)
				continue

			input_onnx_names = []
			for name in node.input:
				if name in reshape_map:
					name = reshape_map[name]
				kind = self.get_kind(name)
				if kind == 'Constant':
					continue
				input_onnx_names.append(name)
			shape = self.get_shape(node.output[0])
			shape = self.clean_shape(shape)
			in_out_info = (input_onnx_names, node.output[0], shape)

			if node.op_type == "MatMul":
				weight, transform = self.matmul_resources(node)
				if apply_permute is not None:
					input_shape = [x for x in operation_resources if x["deeppoly"][-2] == deeppoly_res[-3][0]][0]["deeppoly"][-1]
					if self.is_gpupoly:
						idx = np.arange(np.prod(input_shape)).reshape(np.array(input_shape)).transpose(np.argsort(apply_permute)).flatten() # gpu
					else:
						idx = np.arange(np.prod(input_shape)).reshape(np.array(input_shape)[np.argsort(apply_permute)]).transpose(np.argsort(apply_permute)).flatten()  # cpu
					weight = weight[:,idx]
				apply_permute = None
				deeppoly_res = (weight,) + in_out_info
				deepzono_res = deeppoly_res
				operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})
			elif node.op_type == "Gemm":
				weight, bias, transform = self.gemm_resources(node)
				if apply_permute is not None and not self.is_gpupoly:
					input_shape = [x for x in operation_resources if x["deeppoly"][-2] == deeppoly_res[-3][0]][0]["deeppoly"][-1]
					idx = np.arange(np.prod(input_shape)).reshape(np.array(input_shape)[np.argsort(apply_permute)]).transpose(np.argsort(apply_permute)).flatten()
					weight = weight[:, idx]
					apply_permute = None
				deeppoly_res = (weight, bias) + in_out_info
				deepzono_res = deeppoly_res
				operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})
			elif node.op_type in ["Add", "Mul"]:
				left_type  = self.get_kind(node.input[0])
				right_type = self.get_kind(node.input[1])
				if left_type == 'Constant' and right_type == 'Constant':
					operation_types.pop()
				elif left_type == 'Constant' or right_type == 'Constant':
					deeppoly_res = self.add_resources(node) + in_out_info
					deepzono_res = deeppoly_res
					operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})
				else:
					if node.op_type != "Add":
						assert 0, "we don't support residual operations other then add"
					operation_types[-1] = "Resadd"
					operation_resources.append({'deepzono':in_out_info, 'deeppoly':in_out_info})

			elif node.op_type == "Sub":
				left_type  = self.get_kind(node.input[0])
				right_type = self.get_kind(node.input[1])
				if left_type == 'Constant' and right_type == 'Constant':
					assert 0, "we don't support the subraction of two constants yet"
				elif left_type == 'Constant' or right_type == 'Constant':
					deeppoly_res = self.sub_resources(node) + in_out_info
					if np.all(deeppoly_res[0]==0):
						self.ignore_node(node, operation_types, reshape_map)
						continue
					deepzono_res = deeppoly_res
					operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})
				else:
					assert 0, "we don't support the ressub yet"
					operation_types[-1] = "Ressub"
					operation_resources.append({'deepzono':in_out_info, 'deeppoly':in_out_info})
			elif node.op_type == "Conv":
				filters, bias, image_shape, strides, pad_top, pad_left, pad_bottom, pad_right, kernel_shape = self.conv_resources(node)
				if node.name in padding_merger_dict:
					image_shape, in_out_info, pad_top, pad_left, pad_bottom, pad_right = self.merge_padding(node, padding_merger_dict, in_out_info, pad_top, pad_left, pad_bottom, pad_right)
				deeppoly_res = (filters, bias, image_shape, strides, pad_top, pad_left, pad_bottom, pad_right) + in_out_info
				deepzono_res = deeppoly_res
				operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})
			elif node.op_type == "Pad":
				image_shape, pad_top, pad_left, pad_bottom, pad_right = self.pad_resources(node)
				if pad_top==0 and pad_left==0 and pad_bottom==0 and pad_right==0:
					self.ignore_node(node, operation_types, reshape_map)
					continue
				deeppoly_res = (image_shape, pad_top, pad_left, pad_bottom, pad_right) + in_out_info
				deepzono_res = deeppoly_res
				consequent_nodes = [node_i for node_i in self.nodes if node.output[0] in node_i.input]
				can_be_merged = all([node_i.op_type in ["Conv"] for node_i in consequent_nodes])
				if can_be_merged:
					padding_merger_dict.update({node_i.name: deeppoly_res for node_i in consequent_nodes})
					self.ignore_node(node, operation_types, reshape_map)
				else:
					operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})
			elif node.op_type == "MaxPool" or node.op_type == "AveragePool":
				image_shape, kernel_shape, strides, padding, dilations, pad_top, pad_left, pad_bottom, pad_right, ceil_mode, storage_order = self.pool_resources(node)
				if node.name in padding_merger_dict:
					image_shape, in_out_info, pad_top, pad_left, pad_bottom, pad_right = self.merge_padding(node, padding_merger_dict, in_out_info, pad_top, pad_left, pad_bottom, pad_right)
				deeppoly_res = (image_shape, tuple(kernel_shape), tuple(strides), pad_top, pad_left, pad_bottom, pad_right) + in_out_info
				# TODO padding is expected to be string in tf. dilations, auto_pad, ceil_mode, storage_order are unused at the moment
				deepzono_res = deeppoly_res
				operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})
			elif node.op_type == "Placeholder":
				assert 0, "Placeholder is not in the ONNX graph"
			elif node.op_type in ["Relu", "Sigmoid", "Tanh", "LeakyRelu"]:
				deeppoly_res = self.nonlinearity_resources(node) + in_out_info
				deepzono_res = deeppoly_res
				operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})

			# Gather is for the moment solely for shapes
			elif node.op_type == "Gather":
				only_shape, image_shape, indexes, axis = self.gather_resources(node)

				if only_shape:
					self.ignore_node(node, operation_types, reshape_map)
				else:
					deeppoly_res = (image_shape, indexes, axis) + in_out_info
					deepzono_res = deeppoly_res
					operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})

			elif node.op_type == "Expand":
				only_shape, image_shape, to_expand = self.expand_resources(node)
				if only_shape:
					operation_types.pop()
				else:
					deeppoly_res = (image_shape, indexes, axis) + in_out_info
					deepzono_res = deeppoly_res
					operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})

			elif node.op_type == "Reshape":
				if node.output[0] in self.input_node_map and self.input_node_map[node.output[0]].op_type in ["MatMul", "Gemm"]:

					self.ignore_node(node, operation_types, reshape_map)

				elif node.output[0] in self.input_node_map and self.input_node_map[node.output[0]].op_type in ["Relu", "Sigmoid", "Tanh", "LeakyRelu"] and self.input_node_map[self.input_node_map[node.output[0]].output[0]].op_type == "Reshape":
					# ignore this reshape even in the shape_map
					self.shape_map[node.output[0]] = self.shape_map[node.input[0]]
					self.shape_map[self.input_node_map[node.output[0]].output[0]] = self.shape_map[node.input[0]]
					self.ignore_node(node, operation_types, reshape_map)
				else:
					shape_in = self.get_shape(node.input[0])
					shape_out = self.get_shape(node.output[0])
					if len(shape_in) == 2 and len(shape_out) == 2:
						self.ignore_node(node, operation_types, reshape_map)
					else:
						indexes = reshape_nhwc(shape_in, shape_out)
						deeppoly_res = (indexes,) + in_out_info
						deepzono_res = deeppoly_res
						operation_resources.append({'deepzono':deepzono_res, 'deeppoly':deeppoly_res})

			elif node.op_type == "Concat":
				if len(in_out_info[0])<2:
					self.ignore_node(node, operation_types, reshape_map)
					continue
				n_dim = len(self.shape_map[node.input[0]])
				if n_dim > 2:
					axis = nchw_to_nhwc_index(node.attribute[0].i)
				else:
					axis = node.attribute[0].i
				assert axis == n_dim - 1, "ELINA backend currently only supports concatenation on the channel dimension"
				channels = []
				for input_node in node.input:
					channels.append(self.get_shape(input_node)[axis])
				# width = shape[1]
				# height = shape[2]
				operation_resources.append({'deeppoly': (channels,) + in_out_info})

			elif node.op_type == "Tile":
				repeats = nchw_to_nhwc_shape(self.constants_map[node.input[1]])
				repeat_factor = repeats[repeats != 1].item()
				operation_resources.append({'deeppoly': (repeat_factor,) + in_out_info})

			elif node.op_type == "Transpose":
				if node_idx <= stop_norm_layer:
					self.ignore_node(node, operation_types, reshape_map)
					continue
				# no_conv = not any(["Conv" in self.nodes[i].op_type for i in range(node_idx,len(self.nodes))])
				if self.nodes[node_idx+1].op_type in ["MatMul","GEMM"] or \
						(self.nodes[node_idx+1].op_type in ["Flatten","Reshape"] and
						 self.nodes[node_idx+2].op_type in ["MatMul","GEMM"]):
					apply_permute = None
					for attribute in node.attribute:
						if 'perm' == attribute.name:
							apply_permute = attribute.ints
					assert apply_permute is not None
					self.ignore_node(node, operation_types, reshape_map)
					continue
				assert 0, "Operations of type " + node.op_type + " are not yet supported."
			else:
				assert 0, "Operations of type " + node.op_type + " are not yet supported."

			assert all([0 not in y[-1] for x in operation_resources for y in x.values()]), "Ensure inputs and outpus include no dimensions of size 0"

		return operation_types, operation_resources, orig_input_shape

	def ignore_node(self, node, operation_types, reshape_map):
		operation_types.pop()
		input_name = node.input[0]
		#print("ignore ", len(node.input), reshape_map)
		output_name = node.output[0]
		if input_name in reshape_map:
			reshape_map[output_name] = reshape_map[input_name]
		else:
			reshape_map[output_name] = input_name

	def merge_padding(self, node, padding_merger_dict, in_out_info, pad_top, pad_left, pad_bottom, pad_right):
		image_shape, m_pad_top, m_pad_left, m_pad_bottom, m_pad_right, input_node, _, _ = padding_merger_dict[node.name]
		in_out_info = (input_node, in_out_info[1], in_out_info[2])
		pad_top += m_pad_top
		pad_left += m_pad_left
		pad_bottom += m_pad_bottom
		pad_right += m_pad_right
		return image_shape, in_out_info, pad_top, pad_left, pad_bottom, pad_right

	def get_kind(self, name):
		if name in self.constants_map:
			kind = 'Constant'
		elif name in self.placeholdernames:
			kind = 'Placeholder'
		else:
			kind = self.output_node_map[name].op_type
		return kind

	def get_shape(self, name):
		if name in self.shape_map:
			return self.shape_map[name]

	def matmul_resources(self, node):
		"""
		checks which one of the direct ancestor tf.Operations is a constant and returns the underlying onnx as a numpy.ndarray inside a tuple. The matrix is manipulated in a way that it can be
		used as the left multiplier in the matrix multiplication.

		Arguments
		---------
		node : ONNX.Node
		    must have op_type "MatMul"

		Return
		------
		output : tuple
		    tuple with the matrix (of type numpy.ndarray) as its only item
		"""
		inputs = node.input
		left   = inputs[0]
		right  = inputs[1]

		if left in self.constants_map:
			matrix = self.constants_map[left]
			matrix, transform = self.reshape_adjust(right, matrix, True)
		else:
			matrix = self.constants_map[right].transpose()
			matrix, transform = self.reshape_adjust(left, matrix)
		return matrix, transform

	def reshape_adjust(self, element, matrix, is_right=False):
		if self.get_kind(element) in ['Reshape', 'Flatten'] and not self.is_gpupoly: #TODO check whether it should be triggered for Flatten layers to
			shape_in = self.get_shape(self.output_node_map[element].input[0])
			shape_out = self.get_shape(self.output_node_map[element].output[0])
			if config.debug:
				print('reshape adjust ', str(shape_in), 'to', str(shape_out))

			indexes = reshape_nhwc(shape_in, shape_out)
			#indexes = indexes[0]
			inverse_perm = np.arange(len(indexes))[np.argsort(indexes)]
			if is_right:
				matrix = matrix[inverse_perm, :]
			else:
				matrix = matrix[:, inverse_perm]
		else:
			inverse_perm = np.arange(matrix.shape[0 if is_right else 1])
		return matrix, inverse_perm

	def gemm_resources(self, node):
		"""
		checks which one of the direct ancestor tf.Operations is a constant and returns the underlying onnx as a numpy.ndarray inside a tuple. The matrix is manipulated in a way that it can be
		used as the left multiplier in the matrix multiplication.

		Arguments
		---------
		node : ONNX.Node
		    must have op_type "Gemm"

		Return
		------
		output : tuple
		    tuple with the matrix and bias (of type numpy.ndarray) and is_left used to calculate the output shape
		"""
		inputs = node.input
		left   = inputs[0]
		right  = inputs[1]
		bias  = self.constants_map[inputs[2]]

		transA = False
		transB = False
		alpha = 1.0
		beta = 1.0
		for att in node.attribute:
			if 'transA' == att.name:
				transA = att.i == 1
			elif 'transB' == att.name:
				transB = att.i == 1
			elif 'alpha' == att.name:
				alpha = att.f
			elif 'beta' == att.name:
				beta = att.f
			else:
				assert 0, "Unkown attribute " + att.name + " for operation type " + node.op_type

		if left in self.constants_map:
			matrix = self.constants_map[left] if not transA else self.constants_map[left].transpose()
			matrix, transform = self.reshape_adjust(right, matrix, True)
		else:
			matrix = self.constants_map[right].transpose() if not transB else self.constants_map[right]
			matrix, transform = self.reshape_adjust(left, matrix)
		return matrix * alpha, bias * beta, transform

	def add_resources(self, node):
		"""
		checks which one of the direct ancestor tf.Operations is a constant and returns the underlying onnx as a numpy.ndarray inside a tuple.

		Arguments
		---------
		node : ONNX.Node
		    must have op_type "Add"

		Return
		------
		output : tuple
		    tuple with the addend (of type numpy.ndarray) as its only item
		"""
		inputs = node.input
		left = inputs[0]
		right = inputs[1]

		if left in self.constants_map:
			addend = self.constants_map[left]
		else:
			addend = self.constants_map[right]
		return addend,

	def sub_resources(self, node):
		"""
		checks which one of the direct ancestors is a constant and returns the underlying onnx as a numpy.ndarray and a bool is_minuend, whether the returned ndarray is the minuend, inside a tuple.

		Arguments
		---------
		node : ONNX.Node
		    must have op_type "Sub"

		Return
		------
		output : tuple
		    tuple with the addend (of type numpy.ndarray) and left_constant
		"""
		inputs = node.input
		left   = inputs[0]
		right  = inputs[1]

		if left in self.constants_map:
			addend = self.constants_map[left]
			is_minuend = True
		else:
			addend = self.constants_map[right]
			is_minuend = False
		return addend, is_minuend

	def conv_resources(self, node):
		"""
		Extracts the filter, the stride of the filter, and the padding from node as well as the shape of the input coming into node

		Arguments
		---------
		node : ONNX.Node
		    must have op_type "Conv"

		Return
		------
		output : tuple
		    has 4 entries (numpy.ndarray, numpy.ndarray, numpy.ndarray, str)
		"""
		inputs  = node.input
		image   = inputs[0]
		filters = self.constants_map[node.input[1]].transpose(1, 2, 3, 0)
		if len(node.input) == 3:
			bias = self.constants_map[node.input[2]]
		else:
			bias = np.zeros(filters.shape[3])
		image_shape = self.get_shape(image)[1:]
		pads = [0, 0, 0, 0]
		for attribute in node.attribute:
			if attribute.name == 'strides':
				strides = attribute.ints
			elif attribute.name == 'pads':
				pads = attribute.ints
			elif attribute.name == 'kernel_shape':
				kernel_shape = attribute.ints

		pad_top = pads[0]
		pad_left = pads[1]
		pad_bottom = pads[2]
		pad_right = pads[3]
		# assert pad_top == pad_bottom, 'different padding for top and bottom is not supported in ERAN'
		# assert pad_left == pad_right, 'different padding for left and right is not supported in ERAN'
		return filters, bias, image_shape, strides, pad_top, pad_left, pad_bottom, pad_right, kernel_shape

	def pad_resources(self, node):
		"""
		Extracts the padding from node as well as the shape of the input coming into node

		Arguments
		---------
		node : ONNX.Node
		    must have op_type "Pad"

		Return
		------
		output : tuple
		    has 4 entries (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray)
		"""
		inputs = node.input
		image = inputs[0]
		image_shape = self.get_shape(image)[1:]

		pads = [0, 0, 0, 0]
		for attribute in node.attribute:
			if attribute.name == 'pads':
				pads = attribute.ints

		pad_top = pads[2]
		pad_left = pads[3]
		pad_bottom = pads[6]
		pad_right = pads[7]
		return image_shape, pad_top, pad_left, pad_bottom, pad_right

	def pool_resources(self, node):
		"""
		Extracts the incoming image size (heigth, width, channels), the size of the maxpool/averagepool window (heigth, width), and the strides of the window (heigth, width)

		Arguments
		---------
		node : ONNX.Node
		    must have op_type "MaxPool" or "AveragePool"

		Return
		------
		output : tuple
		    has 4 entries - (list, numpy.ndarray, numpy.ndarray, numpy.ndarray, int, int, str)
		"""
		image       = node.input[0]

		image_shape = self.get_shape(image)[1:]

		padding = 'NOTSET'
		ceil_mode = 0
		storage_order = 0
		pads = [0, 0, 0, 0]
		dilations = None

		for attribute in node.attribute:
			if attribute.name == 'kernel_shape':
				kernel_shape = attribute.ints
			if attribute.name == 'strides':
				strides = attribute.ints
			elif attribute.name == 'pads':
				pads = attribute.ints
			elif attribute.name == 'dilations':
				dilations = attribute.ints
			elif attribute.name == 'auto_pad':
				padding = attribute.s
			elif attribute.name == 'ceil_mode':
				ceil_mode = attribute.i
			elif attribute.name == 'storage_order':
				storage_order = attribute.i
		pad_top = pads[0]
		pad_left = pads[1]
		pad_bottom = pads[2]
		pad_right = pads[3]
		assert pad_top == pad_bottom, 'different padding for top and bottom is not supported in ERAN'
		assert pad_left == pad_right, 'different padding for left and right is not supported in ERAN'
		return image_shape, kernel_shape, strides, padding, dilations, pad_top, pad_left, pad_bottom, pad_right, ceil_mode, storage_order

	def nonlinearity_resources(self, op):
		"""
		This function only outputs an empty tuple, to make the code look more consistent

		Return
		------
		output : tuple
		    but is empty
		"""
		return ()

	def gather_resources(self, node):
		"""
		Extracts the indexes in the image which have to be gathered.

		Arguments
		---------
		node : ONNX.Node
		    must have op_type "Gather"

		Return
		------
		output : tuple
		    has 4 entries - (list, numpy.ndarray, numpy.ndarray, numpy.ndarray, int, int, str)
		"""
		inputs  = node.input
		image   = inputs[0]
		if node.output[0] in self.constants_map:
			only_shape = True
			image_shape, indexes, axis = None, None, None
		else:
			only_shape = False
			image_shape = self.get_shape(image)[1:]
			indexes = self.constants_map[node.input[1]]
			axis = node.attribute[0].i
		return only_shape, image_shape, indexes, axis

	def expand_resources(self, node):
		if node.output[0] in self.constants_map:
			only_shape = True
			image_shape, to_expand = None, None
		else:
			assert 0, "Implementation for 'Expand' is missing."
		return only_shape, image_shape, to_expand


text:
import cv2
import numpy as np

path = 'Resources/'
img = cv2.imread(path+'base.jpeg')

#print current pixel color BGR values
px = img[200,200]
print(px)

#accessing certain B value
blue = img[200, 200, 0]

#accessing certain G value
green = img[200, 200, 1]

#accessing certain R value
red = img[200, 200, 2]

print(blue)
print(green)
print(red)

#using array item for individual access
print(img.item(200,200,0))
print(img.item(200,200,1))
print(img.item(200,200,2))

#modifying BGR value
img.itemset((200,200,2), 100)
print(img.item(200,200,2))

#accessing image properties
print(img.shape)

#total number of pixels is accessed by:
print(img.size)

#image datatype is obtained:
print(img.dtype)

#image ROI
area_to_be_copied = img[280:340, 330:390]
img[273:333, 100:160] = area_to_be_copied
cv2.imshow("test", img)
cv2.waitKey(0)

#splitting and merging image channels
#B,G,R channels of an image can be split into their individual planes
b, g, r = cv2.split(img)
img = cv2.merge((b,g,r))
#making all red channels to be zero
img[:,:,2] = 0
# remember that cv2.split() is a costly operation so use it if necessary

#padding
from matplotlib import pyplot as plt

BLUE = [255,0,0]

replicate = cv2.copyMakeBorder(img,10,10,10,10,cv2.BORDER_REPLICATE)
reflect = cv2.copyMakeBorder(img,10,10,10,10,cv2.BORDER_REFLECT)
reflect101 = cv2.copyMakeBorder(img,10,10,10,10,cv2.BORDER_REFLECT_101)
wrap = cv2.copyMakeBorder(img,10,10,10,10,cv2.BORDER_WRAP)
constant= cv2.copyMakeBorder(img,10,10,10,10,cv2.BORDER_CONSTANT,value=BLUE)

plt.subplot(231),plt.imshow(img,'gray'),plt.title('ORIGINAL')
plt.subplot(232),plt.imshow(replicate,'gray'),plt.title('REPLICATE')
plt.subplot(233),plt.imshow(reflect,'gray'),plt.title('REFLECT')
plt.subplot(234),plt.imshow(reflect101,'gray'),plt.title('REFLECT_101')
plt.subplot(235),plt.imshow(wrap,'gray'),plt.title('WRAP')
plt.subplot(236),plt.imshow(constant,'gray'),plt.title('CONSTANT')

plt.show()

max_stars_repo_path: Core Operations/Basic_Operations.py

max_issues_repo_path: Core Operations/Basic_Operations.py

max_forks_repo_path: Core Operations/Basic_Operations.py


text:
import csv
import os
from collections import defaultdict

import numpy as np
import spacy

COURTESY_SENTENCES_WORDS = 9

EMOTION_SCORE_THRESHOLD = 50
WASSA_THRESHOLD = 0.5

HASHTAG_EMOTION_CORPUS = 'data/raw/emotion/Jan9-2012-tweets-clean.txt'
CROWDFLOWER_EMOTION_CORPUS = 'data/raw/emotion/text_emotion.csv'
AFFECTIVETEXT_TRIAL_LABELS = 'data/raw/emotion/AffectiveText.Semeval.2007/AffectiveText.trial/affectivetext_trial' \
                             '.emotions.gold'
AFFECTIVETEXT_TRIAL_TEXT = 'data/raw/emotion/AffectiveText.Semeval.2007/AffectiveText.trial/affectivetext_trial.xml'
AFFECTIVETEXT_TEST_LABELS = 'data/raw/emotion/AffectiveText.Semeval.2007/AffectiveText.test/affectivetext_test' \
                            '.emotions.gold'
AFFECTIVETEXT_TEST_TEXT = 'data/raw/emotion/AffectiveText.Semeval.2007/AffectiveText.test/affectivetext_test.xml'
ELECTORAL_DATA_BATCH1 = 'data/raw/emotion/ElectoralTweetsData/Annotated-US2012-Election-Tweets/Questionnaire2/Batch1' \
                        '/AnnotatedTweets.txt'
ELECTORAL_DATA_BATCH2 = 'data/raw/emotion/ElectoralTweetsData/Annotated-US2012-Election-Tweets/Questionnaire2/Batch2' \
                        '/AnnotatedTweets.txt'
WASSA_FOLDER = 'data/raw/emotion/Wassa-2017'
LOVE_LETTERS = 'data/raw/emotion/LoveHateSuicide/love-letters.txt'
SPUDISK_TRAIN = 'data/raw/emotion/spudisc-emotion-classification-master/train.txt'
SPUDISK_TEST = 'data/raw/emotion/spudisc-emotion-classification-master/test.txt'
NRC_LEXICON = 'data/raw/emotion/NRC-Sentiment-Emotion-Lexicons/NRC-Emotion-Lexicon-v0.92/NRC-Emotion-Lexicon' \
              '-Wordlevel-v0.92.txt'
FULL_DATASET = 'data/processed/emotions_initial.csv'

def parse_hashtag_emotion_corpus():
    """Extract tweets and emotions from http://saifmohammad.com/WebDocs/Jan9-2012-tweets-clean.txt.zip.

    Returns:
        ([(str, str)]): Tweets and their emotion labels zipped.
    """
    data = []
    with open(HASHTAG_EMOTION_CORPUS) as corpus:
        for line in corpus:
            _, sentence_label_str = line.split(':', 1)
            preprocessed_sentence_label_str = sentence_label_str.replace('\t', '').replace('\n', '')
            sentence, label = preprocessed_sentence_label_str.rsplit('::', 1)
            data.append((sentence.strip(), label.strip()))
    return data

def parse_crowdflower_emotion_corpus():
    """Extract tweets and emotions from https://www.crowdflower.com/wp-content/uploads/2016/07/text_emotion.csv.

    Returns:
        ([(str, str)]): Tweets and their emotion labels zipped.
    """
    data = []
    with open(CROWDFLOWER_EMOTION_CORPUS) as data_file:
        reader = csv.reader(data_file)
        reader.__next__()
        for line in reader:
            emotion = line[1]
            text = line[3]
            data.append((text, emotion))
    return data

def parse_affective_text_corpuses():
    """ Extract and merge trial and test data downloaded from
        http://web.eecs.umich.edu/~mihalcea/downloads/AffectiveText.Semeval.2007.tar.gz.

    Returns:
        ([(str, str)]): News headlines and their top scored emotion labels zipped.
    """
    data = parse_affective_text_corpus(AFFECTIVETEXT_TRIAL_TEXT, AFFECTIVETEXT_TRIAL_LABELS)
    data.extend(parse_affective_text_corpus(AFFECTIVETEXT_TEST_TEXT, AFFECTIVETEXT_TEST_LABELS))
    return data

def parse_affective_text_corpus(text_filepath, labels_filepath):
    """Extract news headlines and emotions from affectivetext dataset. http://web.eecs.umich.edu/~mihalcea/downloads.html

    Args:
        text_filepath (str): Path to affectivetext xml file with ids and texts.
        labels_filepath (str): Path to affectivetext emotions file.
    Returns:
        ([(str, str)]): News headlines and their top scored emotion labels zipped.
    """
    data = []
    columns = ['anger', 'disgust', 'fear', 'joy', 'sadness', 'surprise']
    with open(text_filepath) as text_file, open(labels_filepath) as labels_file:
        text_file.__next__()
        for text_line, label_line in zip(text_file, labels_file):
            emotion_scores = np.fromstring(label_line, dtype='int', sep=' ')[1:]
            max_score_index = np.argmax(emotion_scores)
            if emotion_scores[max_score_index] <= EMOTION_SCORE_THRESHOLD:
                continue
            label = columns[max_score_index]
            text = text_line.split('>', 1)[1].rsplit('<', 1)[0]
            data.append((text, label))
    return data

def join_electoral_tweets_data():
    """ Extract and join 2 batches of data from electoral tweets:
    http://saifmohammad.com/WebDocs/ElectoralTweetsData.zip

    Returns:
        ([(str, str)]): Tweets and their emotion labels zipped.
    """
    data = parse_electoral_tweets_data(ELECTORAL_DATA_BATCH1)
    data.extend(parse_electoral_tweets_data(ELECTORAL_DATA_BATCH2))
    return data

def parse_electoral_tweets_data(data_path):
    """Extract electoral tweets and emotions from the dataset.

    Args:
        data_path (str): Path to annotated tweets file.
    Returns:
        ([(str, str)]): Tweets and their emotion labels zipped.
    """
    data = []
    with open(data_path) as data_file:
        data_file.__next__()
        previous_row = ''
        for line in data_file:
            line = previous_row + ' ' + line
            row = line.rstrip('\t\n').split('\t')
            if len(row) < 29:
                # Some rows in the data file are improperly newlined
                previous_row = line
                continue
            else:
                previous_row = ''
            tweet = row[13]
            emotion = row[15]
            data.append((tweet, emotion))
    return data

def parse_wassa_data():
    """Extract tweets and their emotions from WASSA-2017 dataset:
    http://saifmohammad.com/WebPages/EmotionIntensity-SharedTask.html
    Filter by WASSA_THRESHOLD intensity.

    Returns:
        ([(str, str)]): Tweets and emotion labels zipped.
    """
    data = []
    for filename in os.listdir(WASSA_FOLDER):
        if not filename.startswith('.'):
            with open(WASSA_FOLDER + '/' + filename) as data_file:
                reader = csv.reader(data_file, delimiter='\t')
                for row in reader:
                    if float(row[3]) > WASSA_THRESHOLD:
                        data.append((row[1], row[2]))
    return data

def parse_love_letters():
    """Extract love sentences from love letter collection from http://saifmohammad.com/WebDocs/LoveHateSuicide.tar.gz

    Returns:
        ([(str, str)]): Sentence, 'love' pairs.
    """
    data = []
    with open(LOVE_LETTERS) as letters_file:
        nlp = spacy.load('en')
        for line in letters_file:
            doc = nlp(line)
            if len(doc) <= COURTESY_SENTENCES_WORDS:
                continue
            for sentence in doc.sents:
                data.append((sentence.text, 'love'))
    return data

def join_spudisc_datasets():
    """Extract and join test/train imdb review parts annotated with emotions from
    https://github.com/NLeSC/spudisc-emotion-classification.

    Returns:
        ([(str, str)]): Review part, emotion.
    """
    data = parse_spudisc_dataset(SPUDISK_TRAIN)
    data.extend(parse_spudisc_dataset(SPUDISK_TEST))
    return data

def parse_spudisc_dataset(data_path):
    """Extract imdb review parts annotated with emotions

    Args:
        data_path (str): Path to annotated review file.
    Returns:
        ([(str, str)]): Review part, emotion.
    """
    data = []
    with open(data_path) as data_file:
        for line in data_file:
            sentence, label = line.rsplit(maxsplit=1)
            if label == 'None':
                continue
            new_label = label.split('_')[0]
            data.append((sentence, new_label))
    return data

def parse_nrc_lexicon():
    """Extract National Resource Council Canada emotion lexicon from http://saifmohammad.com/WebPages/lexicons.html

    Returns:
        {str: [str]} A defaultdict of emotion to list of associated words
    """
    emotion2words = defaultdict(list)
    with open(NRC_LEXICON) as lexicon_file:
        lexicon_file.__next__()
        for line in lexicon_file:
            word, emotion, associated = line.split()
            if associated == '1':
                emotion2words[emotion].append(word)
    return emotion2words

# TODO maybe add StanceDataset

def write_data(writer, data, name):
    """Write a particular dataset into the file

    Args:
        writer (csv.Writer): writer handle of output file
        data ([(str, str)]: text, label tuples
        name (str): dataset identification
    """
    for text, label in data:
        writer.writerow([text, label, name])

if __name__ == '__main__':
    extractors = [parse_hashtag_emotion_corpus, parse_crowdflower_emotion_corpus, parse_affective_text_corpuses,
                  join_electoral_tweets_data, parse_wassa_data, parse_love_letters, join_spudisc_datasets]
    names = ["hashtag_emotion", "crowdflower", "affective_text", "electoral_tweets", "wassa", "love_letters", "spudisc"]
    with open(FULL_DATASET, 'w') as full_data_file:
        writer = csv.writer(full_data_file, quoting=csv.QUOTE_MINIMAL)
        for extractor_function, name in zip(extractors, names):
            data = extractor_function()
            write_data(writer, data, name)
        emotion2words = parse_nrc_lexicon()
        for emotion, word_list in emotion2words.items():
            for word in word_list:
                writer.writerow((word, emotion, "nrc_lexicon"))

max_stars_repo_name: Neronuser/EmoCourseChat

max_issues_repo_name: Neronuser/EmoCourseChat

max_forks_repo_name: Neronuser/EmoCourseChat