text:
"""Module for eigenvalue problems.

We have an abstract base-class representing an eigensolver. This has a number of functions which must be overloaded"""

from abc import ABC,abstractmethod

from petsc4py import PETSc
from slepc4py import SLEPc

import dolfin as dolf

import time

import numpy as np

import os

import gc

class EigenError(Exception):
    """Error class for errors arising in eigenvalue problems"""
    def __init__(self,message):
        self.message=message

class Eigensolver(ABC):
    """Abstract base class for an eigensolver.

    This abstract class implements some of the required grunt work in :py:meth:`_buildEPS`. The methods that need to
    be overloaded are:

    * :py:meth:`build` - builsd the matrices and SLEPc.EPS object needed for the eigensolve.
    * :py:meth:`validateEigenpair` - validates an eigenapair (some eigensolvers will introduce false solutions)
    * :py:meth:`getEigenpair` - returns an eigenpair.
    """

    def __init__(self):
        super().__init__()
        self._defaultLogTime = 10.

        self.built = False
        """Flag to indicate whether the matrices have been built"""

    def _resetTimes(self):
        """Resets internal timers for the stop and logging during eps solves"""
        self._lastLog   = time.time()
        self._startTime = time.time()

    def _buildEPS(self,*ops):
        """
        Performs the grunt work for build a SLEPC eps object with PETSC matrices as the operators.

        This is called by classes inheriting from :py:class:`Eigensolver`.

        :params list(PETSc.Mat) ops: set of operator matrices to pass to the eigensolver
        """
        eps = SLEPc.EPS().create()      #Generate EPS
        st = eps.getST()
        st.setType('sinvert')           #shift invert

        #Set up the linear solver
        ksp = st.getKSP()
        ksp.setType('preonly')
        pc = ksp.getPC()
        pc.setType('lu')
        pc.setFactorSolverType('mumps')

        #Set the operators
        eps.setOperators(*ops)

        eps.setStoppingTest(self._eigensolveStop)

        self.eps = eps

    def _eigensolveStop(self, solver, its, max_it, nconv, nev):
        currTime    = time.time()
        solveTime   = currTime - self._startTime

        if currTime - self._lastLog > self._defaultLogTime:
            self._lastLog = currTime

        if nconv >= nev:
            return SLEPc.EPS.ConvergedReason.CONVERGED_TOL

        if self._maxSolveTime is not None and solveTime > self._maxSolveTime and its > 8:
            return SLEPc.EPS.ConvergedReason.CONVERGED_USER

        if its > max_it:
            return SLEPc.EPS.ConvergedReason.DIVERGED_ITS

        return SLEPc.EPS.ConvergedReason.ITERATING

    @abstractmethod
    def build(self):
        """Builds any matrices required and creates the eigensolver.

        Must be overloaded by any class inheriting this.
        """
        pass

    def solve(self,nEigen, maxSolveTime = 240):
        """Solves for eigenvalues.

        This is performs the grunt work of performing an eigensolve through the SLEPC interface. It uses the :py:attr:`built`
        flag to determing whether or not to call :py:meth:`build` .

        :param int nEigen: number of eigenvalues to look for
        :return: number of eigenvalues found
        :rtype: int
        """
        #We make sure we have built all the required matrices and solvers

        if not self.built:
            self.build()

        #TODO: maybe check if the build flag has raised?
        #Set how many eigenvalues to look for
        self.eps.setDimensions(nEigen,SLEPc.DECIDE)

        self._resetTimes()
        self._maxSolveTime = maxSolveTime

        gc.collect(2) #We should garbage collect before solving #TODO - need to sort out when and how this happens. May do it too oftern
        self.eps.solve()

        self.validatedEigenpairs = []
        self.storedEigenpairs = []

        self.solved = True

        return self.eps.getConverged()

    def nConverged(self):
        """Returns number of eigenvalues converged.

        :return: number of eigenvalues converged
        :rtype: int
        """
        return self.eps.getConverged()

    @abstractmethod
    def getEigenpair(self,i,absSpectra=True):
        """Retrieves eigenpair.

        Must be overloaded by any class inheriting this.

        :param int i: which eigenpair to return.
        :param bool absSpectra: (optional) for use if a shift is involved. If true then the eigenvalue is the absolute position in the spectra as opposed to the position relative to the shift
        """
        pass

#COMPLEX EIGENVALUE PROBLEM#
class ComplexEigensolver(Eigensolver):
    """This is an eigensolver that takes the real representation of a complex eigenvalue and allows it to be solver using
    real PETSc.

    .. todo::

        With no shift then this struggles validating the eigenvalues, it should always be run with a shift

    Because all the matrices are real, it morphs an NxN problem into a 2Nx2N problem. See :ref:`complex-eigensolver-label` for
    details.

    :param dolfin.PETScMatrix AReal: The real part of the A operator
    :param dolfin.PETScMatrix AImag: The imaginary part of the A operator
    :param dolfin.PETScMatrix B: (optional) the B operator of a generalised eigenvalue problem
    :param str mode: (optional) whether to use direct mode (right eigenvectors, default) or adjoint mode (left eigenvectors)
    """
    def __init__(self, AReal, AImag, B=None, mode = None):
        super().__init__()

        #TODO: implement dealing with no B matrix
        if B == None:
            raise EigenError('Not yet implemented ComplexEigensolver without B specified')

        self.AReal  = AReal
        self.AImag  = AImag
        self.B      = B

        self.shift = 0.
        self.solved = False

        self.validatedEigenpairs = []
        self.storedEigenpairs = []

        if mode is None:
            self.mode = 'direct'
        else:
            self.mode = mode

    def build(self):
        """Builds the necessary matrices needed for SLEPc eigensolver."""

        self.solved = False  #we make sure we clear the solved flag
        self._validated = False
        self.validatedEigenpairs = []
        self.storedEigenpairs = []

        A00 = self.AReal.mat().copy()

        if self.AImag != 0:
            A10 = self.AImag.mat().copy()

        B00 = self.B.mat().copy()

        dimBlock = A00.getSize()

        #The case of zero real part of the shift causes problems and a real eigenvalue problem, leads to the case where ghosts can't be differentiated
        if self.shift.imag == 0 and self.AImag == 0:
            #We have a special case where we have a purely real system. The splitting into 2Nx2N makes no sense and so we
            #produce a reduced system
            A00.axpy(-self.shift.real, B00)
            self.AA = A00
            self.BB = B00

        else:
            #We have to produce a split system

            #We check all the matrices we've been given are compatible
            if dimBlock != B00.getSize():
                raise EigenError('Matrices A and B must agree in dimension')
            if self.AImag != 0 and dimBlock != A10.getSize():
                raise EigenError('Real part and complex part of A must agree')

            dummy = PETSc.Mat() #dummy matrix for building the block system

            comm = A00.getComm() #We use the same mpi communicator that dolfin was using

            #We construct the augmented A matrix
            A11 = A00.copy() #get base copy for A11
            A00.axpy(-self.shift.real,B00)
            A11.axpy(-self.shift.real,B00)

            if self.AImag == 0:
                A01 = B00.copy()
                A01.scale(self.shift.imag)

                A10 = B00.copy()
                A10.scale(-self.shift.imag)

            else:
                A01 = B00.copy()
                A01.scale(self.shift.imag)
                A01.axpy(-1., A10)

                A10.axpy(-self.shift.imag, B00)

            #We construct the augmented B matrix
            BB  = PETSc.Mat().createNest([[B00  ,dummy],
                                          [dummy,  B00]] ,comm=comm)

            AA = PETSc.Mat().createNest([[A00, A01],
                                         [A10, A11]],comm=comm)

            #We now convert the matrices to SEQ AIJ form
            AA.convert(mat_type=PETSc.Mat.Type.AIJ,out = AA)
            BB.convert(mat_type=PETSc.Mat.Type.AIJ,out = BB)

            self.AA = AA
            self.BB = BB

            self.indicesReal = PETSc.IS().createBlock(dimBlock[0],0)
            self.indicesImag = PETSc.IS().createBlock(dimBlock[0],1)

        #store dimensions
        self.dimBlock = dimBlock
        self.dim      = self.AA.getSize()

        #We generate the eps obejct
        if self.mode == 'adjoint':
            self.AA.transpose(out=self.AA)
            self.BB.transpose(out=self.BB)
        self._buildEPS(self.AA,self.BB)

        self.built=True

    def applyShift(self,shift):
        """Applies a complex shift.

        This clears the built flag and so the matrices will be rebuilt for the next solve.

        :param complex shift: complex shift to apply
        """
        self.shift = shift
        self.built = False

    def _reconstructEigenvector(self,rx,cx):
        """Reconstructs the original real and imaginary parts of the eigenvector.

        Reconstruction is required because the spltting into 2Nx2N redistributes the eigenvector
        into real and imaginary parts.

        :param PETSC.Vec rx: PETSc vector representing the real part of the eigenvector
        :param PETSC.Vec cx: PETSc vector representing the complex part of the eigenvector
        :return: tuple of PETSC.Vec of the reconstructed real and imaginary parts of the eigenvector
        """
        #We generate the index sets for the block which represents the real component and the block which represents the complex component

        if self.shift.imag != 0:

            r0 = PETSc.Vec().createSeq(self.dimBlock[0]);  rx.getSubVector(self.indicesReal, subvec=r0)
            r1 = PETSc.Vec().createSeq(self.dimBlock[0]);  rx.getSubVector(self.indicesImag, subvec=r1)
            c0 = PETSc.Vec().createSeq(self.dimBlock[0]);  cx.getSubVector(self.indicesReal, subvec=c0)
            c1 = PETSc.Vec().createSeq(self.dimBlock[0]);  cx.getSubVector(self.indicesImag, subvec=c1)

            r0.axpy(-1.,c1)
            r1.axpy( 1.,c0)

            return r0, r1
        else:
            return rx,cx

    def validateEigenpairs(self, store=False, sTol = 1e-3, rTol = 1e3):
        """Validates the converged eigenpairs.

        This is done to eliminate the additional 'ghost' solutions that arise from the 2Nx2N real representation of a
        complex matrix. The way this works is that we know the ghost solutions appear as eigenvalues that are exactly mirrored
        about the sinvert point

        :param bool store: (optional) whether to store the validated eigenpairs for future use
        :param float sTol: (optional) spectral tolerance for checking mirrored eigenvalues. Default is 1e-3
        :param flaot rTol: (optional) relative tolerance for checking which mode is the ghost and which is the real eigenvector. Default is 1e3.

        :return: list of validated eigenpairs
        :rtype: bool

        """
        if not self.solved:
            raise EigenError('Must call solve() before validating eigenpairs')

        #we clear the

        #We identify the pairs
        evList      = []
        procEvs     = []
        evGroups    = []
        validPairs  = []

        nEv = self.eps.getConverged()

        rx = PETSc.Vec().createSeq(self.dim[0])
        cx = PETSc.Vec().createSeq(self.dim[0])

        counter = 0

        if self.shift.imag == 0:

            self.validatedEigenpairs = [i for i in range(nEv)]
            self.storedEigenpairs    = [[self.eps.getEigenvalue(i)] for i in range(nEv)]

            if store:
                for i in range(nEv):
                    self.eps.getEigenpair(i, rx, cx)
                    self.storedEigenpairs[i].extend([rx.copy(), cx.copy()])

        else:
            #The imaginary shift means we get possible ghost solutions
            #We get all the eigenvalues
            for i in range(nEv):
                evList.append([self.eps.getEigenvalue(i), i])

            #We organise them into potential ghost/no ghost pairs
            for i in range(nEv):
                #We locate groups
                if i in procEvs: continue

                evI = evList[i][0]

                newGroup = []

                newGroup.append(evList[i])
                procEvs.append(i)

                for j in range(i+1, nEv):
                    #in the relative spectrum frame these ghost pairs should be complex conjugates of each other
                    if j in procEvs: continue

                    evJ = evList[j][0]

                    if np.abs(evI.real - evJ.real) < sTol and np.abs(evI.imag + evJ.imag) < sTol:
                        #this is a candidate for being part of a ghost/no ghost pair
                        newGroup.append(evList[j])
                        procEvs.append(j)

                evGroups.append(newGroup)

            for evGroup in evGroups:
                #We process the groups looking for which of the pair has the largest norm

                if len(evGroup) > 2:
                    continue
                    #raise Exception('Cannot handle ev groups of more than 2 members')
                elif len(evGroup) == 1:

                    pair = [evGroup[0][0]]

                    if store:

                        #self.eps.getEigenpair(evGroup[0][1], rx, cx)
                        norm, real, imag = self._eigenpairNorm(evGroup[0][1])
                        pair.extend([real, imag])

                    self.validatedEigenpairs.append(evGroup[0][1])
                    self.storedEigenpairs.append(pair)

                else:
                    #We have a two group. We need to check the relative magnitudes
                    norm0, rx0, cx0 = self._eigenpairNorm(evGroup[0][1])
                    norm1, rx1, cx1 = self._eigenpairNorm(evGroup[1][1])

                    if norm0 > norm1:
                        index = evGroup[0][1]
                        ev    = evGroup[0][0]
                        rx    = rx0
                        cx    = cx0

                    else:
                        index   = evGroup[1][1]
                        ev      = evGroup[1][0]
                        rx      = rx1
                        cx      = cx1

                    self.validatedEigenpairs.append(index)

                    if store:
                        self.storedEigenpairs.append([ev, rx, cx])
                    else:
                        self.storedEigenpairs.append([ev])

    def _eigenpairNorm(self,i):
        """Reconstructs an eigenpair and checks what the l2 norm is. This is used when testing for ghost solutions.

        :param int i: which eigenpair to validate
        :return: the calculated l2 norm
        :rtype: float and the combined test vector
        """

        rx = PETSc.Vec().createSeq(self.dim[0])
        cx = PETSc.Vec().createSeq(self.dim[0])

        ev = self.eps.getEigenpair(i,rx,cx)

        real,imag = self._reconstructEigenvector(rx,cx)

        storeReal = real.copy() #We have to keep a backup - using restoreVector seems to free() the memory pointed to by these vectors
        storeImag = imag.copy()

        #We create the vector used to check whether the solution is a ghost or not
        testVec = PETSc.Vec().createSeq(self.dim[0])
        testVec.restoreSubVector(self.indicesReal,real)
        testVec.restoreSubVector(self.indicesImag,imag)

        #testVec.setType(PETSc.Vec.Type.SEQ) #change its type back to a normal seq
        resVec = testVec.duplicate() #allocate the emmory for the test vector

        self.AA.mult(testVec,resVec)

        norm = resVec.norm(PETSc.NormType.N2)

        return norm, storeReal, storeImag

    def solve(self,nEigen, store=True, maxSolveTime = 240):
        super().solve(nEigen, maxSolveTime)
        #We validate
        self.validateEigenpairs(store=store)

        return len(self.validatedEigenpairs)

    def getEigenpair(self,i,absSpectra=True):
        """Gets the i-th eigenpair.

        :param int i: index of the eigenpair to resturn
        :param bool absSpectra: (optional) whether to return the absolute spectra or spectra relative to shift
        """

        [ev, real, imag] = self.storedEigenpairs[i]

        if absSpectra:

            if self.mode == 'direct':
                ev += self.shift
            else:
                ev += self.shift.conjugate()

        return ev, dolf.PETScVector(real), dolf.PETScVector(imag)

class RealEigensolver(ComplexEigensolver):
    """This is an eigensolver that takes a real eigenvalue problem and enables a complex shift to be applied. This is implemented
    by extending the ComplexEigensolver and setting the imaginary part to zero.

    Because all the matrices are real, it morphs an NxN problem into a 2Nx2N problem. See :ref:`real-eigensolver-label` for
    details.

    :param dolfin.PETScMatrix A: The A operator
    :param dolfin.PETScMatrix B: (optional) The B operator of a generalised eigenvalue problem.
    :param str mode: (optional) whether to use direct mode (right eigenvectors) or adjoint mode (left eigenvectors)
    """

    def __init__(self,A,B=None,mode=None):
        super().__init__(AReal = A, AImag = 0, B = B, mode = mode)


text:
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
__author__ = 'Shilin He'

import numpy as np
import math
import utils.evaluation as ev

def weighting(para, raw_data):
	""" weight the matrix using idf

	Args:
	--------
	raw_data: log sequences matrix

	Returns:
	--------
	weigh_data: weighted raw data
	"""
	inst_size, event_num = raw_data.shape
	if para['tf-idf']:
		idf_data = np.zeros((inst_size, event_num))
		for j in range(event_num):
			cnt = np.count_nonzero(raw_data[:, j])
			if cnt != 0:
				for i in range(inst_size):
					idf_data[i, j] = raw_data[i, j]*math.log(inst_size/float(cnt))
		raw_data = idf_data
	raw_data = raw_data.T
	mean_value = raw_data.mean(axis=1)
	weigh_data = np.zeros((event_num, inst_size))
	for i in range(inst_size):
		weigh_data[:, i] = raw_data[:, i] - mean_value
	return weigh_data

def get_threshold(para, weigh_data):
	""" compute the threshold for anomaly detection

	Args:
	--------
	weigh_data: weighted raw data

	Returns:
	--------
	threshold: used as the threshold, which determines the anomalies
	C: the projection matrix
	"""
	U, sigma, VT = np.linalg.svd(weigh_data, full_matrices=False)
	event_num, inst_size = weigh_data.shape

	# calculate the number of principal components
	k=4
	tot = np.dot(sigma,sigma)
	tmp=0
	for i in range(len(sigma)):
		tmp += sigma[i] * sigma[i]
		if (tmp/tot) >= para['fraction']:
			break
	k = i+1
	print('principal components number is %d' % (k))

	for i in range(event_num):
		sigma[i] = sigma[i] * sigma[i]/float(inst_size)

	phi = np.zeros(3)
	for i in range(3):
		for j in range(k,event_num):
			phi[i] += math.pow(sigma[j],i+1)

	h0 = 1-2*phi[0]*phi[2]/(3*phi[1]*phi[1])
	c_alpha = 3.2905
	threshold = phi[0]*math.pow((c_alpha*math.sqrt(2*phi[1]*h0*h0)/phi[0] + 1.0 + phi[1]*h0*(h0-1)/phi[0]/phi[0]), (1.0/h0))
	P = U[:,:k]
	I = np.identity(event_num,int)
	C = I - np.dot(P,P.transpose())
	return threshold, C

def anomaly_detection(weigh_data, label_data, C, threshold):
	""" detect anomalies by projecting into a subspace with C

	Args:
	--------
	weigh_data: weighted raw data
	label_data: the labels list
	threshold: used as the threshold, which determines the anomalies
	C: the projection matrix

	Returns:
	--------

	"""
	print ('there are %d anomalies' % (sum(label_data)))
	event_num, inst_size  = weigh_data.shape
	predict_results = np.zeros((inst_size),int)
	print('the threshold is %f' % (threshold))
	for i in range(inst_size):
		ya = np.dot(C,weigh_data[:,i])
		SPE = np.dot(ya,ya)
		if SPE > threshold:
			predict_results[i] = 1	#1 represent failure
	assert len(label_data) == len(predict_results)
	ev.evaluate(label_data, predict_results)

ext: py

reason: import numpy


text:
import unittest
from .cost_function import QCBMCostFunction
from .ansatz import get_qcbm_ansatz
from zquantum.core.bitstring_distribution import BitstringDistribution
from zquantum.core.utils import create_object
import numpy as np

class TestQCBMCostFunction(unittest.TestCase):

    def test_init(self):
        # Given
        num_qubits = 4
        topology = "all"
        epsilon = 1e-6
        n_layers = 1
        distance_measure = "clipped_log_likelihood"
        ansatz = get_qcbm_ansatz(num_qubits, n_layers, topology)
        target_bitstring_distribution = BitstringDistribution(
            {"0000": 1.0, "0001": 0.0, "0010": 0.0,
             "0011": 1.0, "0100": 0.0, "0101": 1.0,
             "0110": 0.0, "0111": 0.0, "1000": 0.0,
             "1001": 0.0, "1010": 1.0, "1011": 0.0,
             "1100": 1.0, "1101": 0.0, "1110": 0.0,
             "1111": 1.0})
        backend = create_object({'module_name': 'zquantum.core.interfaces.mock_objects', 'function_name': 'MockQuantumSimulator', 'n_samples': 1})

        # When
        cost_function = QCBMCostFunction(ansatz, backend, distance_measure, target_bitstring_distribution, epsilon)

        # Then
        self.assertEqual(cost_function.ansatz, ansatz)
        self.assertEqual(cost_function.backend, backend)
        self.assertEqual(cost_function.distance_measure, distance_measure)
        self.assertEqual(cost_function.target_bitstring_distribution, target_bitstring_distribution)
        self.assertEqual(cost_function.epsilon, epsilon)
        self.assertEqual(cost_function.gradient_type, 'finite_difference')
        self.assertEqual(cost_function.save_evaluation_history, True)
        self.assertEqual(cost_function.evaluations_history, [])

    def test_evaluate(self):
        # Given
        num_qubits = 4
        topology = "all"
        epsilon = 1e-6
        n_layers = 1
        params = np.array([0, 0, 0, 0])
        distance_measure = "clipped_log_likelihood"
        ansatz = get_qcbm_ansatz(num_qubits, n_layers, topology)
        target_bitstring_distribution = BitstringDistribution(
            {"0000": 1.0, "0001": 0.0, "0010": 0.0,
             "0011": 1.0, "0100": 0.0, "0101": 1.0,
             "0110": 0.0, "0111": 0.0, "1000": 0.0,
             "1001": 0.0, "1010": 1.0, "1011": 0.0,
             "1100": 1.0, "1101": 0.0, "1110": 0.0,
             "1111": 1.0})
        backend = create_object({'module_name': 'zquantum.core.interfaces.mock_objects', 'function_name': 'MockQuantumSimulator', 'n_samples': 1})
        cost_function = QCBMCostFunction(ansatz, backend, distance_measure, target_bitstring_distribution, epsilon)

        # When
        value = cost_function.evaluate(params)
        history = cost_function.evaluations_history

        # Then
        self.assertEqual(type(value), float)
        self.assertIn("bitstring_distribution", history[0].keys())
        self.assertEqual(dict, type(history[0]["bitstring_distribution"]))
        self.assertIn("value", history[0].keys())
        self.assertIn("params", history[0].keys())

max_stars_repo_name: jmalliaros/qcbm

max_issues_repo_name: jmalliaros/qcbm

max_forks_repo_name: jmalliaros/qcbm


text:
from keras.models import Model #将各个层组合成全连接神经网络的函数
from keras.layers import Input, Dense #构建各层用到的函数
from keras.utils import np_utils
import matplotlib.pyplot as plt
import os
import pandas as pd
import numpy as np

batch_size = 20000 #每次训练用到的数据规模
num_epochs = 40 #训练次数
hidden_size = 50 #隐藏层节点数
num_train = 200000 #训练样本数
spe_len = 36 #每个样本参数数量
num_classes = 2 #二分类

#读取数据集
train_data = np.array(pd.read_csv('train.csv'))
test_data = np.array(pd.read_csv('test.csv'))
X_train = train_data[:,1:-1]
X_train = X_train.astype('float32') #源数据是int类型，这里需要转换成float
y_train = train_data[:,-1]
#由于是二分类问题，输出层有两个节点，所以需要将label处理成两列。
Y_train = np_utils.to_categorical(y_train, num_classes)
X_test = test_data[:,1:]

'''创建输入层，隐藏层1、2，输出层并建立先后顺序关系。
两个隐藏层节点数和激活函数相同。
注意到两个隐藏层和输出层都有bias，可以使预测模型更精确。
输出层激活函数是softmax。'''
inp = Input(shape=(spe_len,))
hidden_1 = Dense(hidden_size, activation='relu', use_bias = True)(inp)
hidden_2 = Dense(hidden_size, activation='relu', use_bias = True)(hidden_1)
out = Dense(num_classes, activation='softmax', use_bias = True)(hidden_2)

#用Model函数将上述各层连接到一起
model = Model(inputs=inp, outputs=out)

'''定义好模型之后需要通过编译来对学习过程进行配置，
我们可以为模型的编译指定各类参数包括：优化器optimizer，损失函数loss，评估指标metrics'''
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

'''采用反向传播算法训练神经网络。训练完成后，history 会保存模型训练后的相关描述'''
history = model.fit(X_train, Y_train, batch_size=batch_size, epochs=num_epochs, verbose=1, validation_split=0.1)

# 绘制训练和测试精度曲线
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()

# 绘制训练和测试损失函数曲线
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()

#对测试集进行预测
test_predict = model.predict_on_batch(X_test)
test_predict = test_predict[:,0]
print(test_predict)

#将预测输出到submit_data.csv中
submitData = pd.read_csv("submit_data.csv")
submitData['Evaluation'] = test_predict
submitData.to_csv("submit_data.csv", index=False)

max_stars_repo_path: report_01_ Traffic Accidents Claims/code.py

max_stars_repo_name: haohlin/Pattern-Recognition-Homework

max_issues_repo_name: haohlin/Pattern-Recognition-Homework

max_forks_repo_name: haohlin/Pattern-Recognition-Homework


text:
#!/usr/bin/env python3

import re
import statsmodels.api as sm
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import math

def parse_times_file(filename):
    f = open(filename, "r")
    lines = f.readlines()
    f.close()

    # Get the filename of the manifest these results are for
    p = re.compile(".*(/.*\.json).*")
    manifest_filename = p.search(lines[0]).group(1)[1:]
    print("Timing results for " + manifest_filename)

    num_files = 0
    mount_time_pattern = re.compile(".*mount filesystem: ([0-9]+\.[0-9]*)")
    mount_times = []

    ls_time_pattern = re.compile(".*list ([0-9]+) files: ([0-9]+\.[0-9]*)")
    ls_times = []

    cat_time_pattern = re.compile(".*to cat .* of size ([0-9]+) bytes: ([0-9]+\.[0-9]*)")
    cat_times = []

    for line in lines[1:]:
        try:
            time_to_mount = float(mount_time_pattern.search(line).group(1))
            mount_times.append(time_to_mount)
        except Exception:
            pass

        try:
            time_to_ls = float(ls_time_pattern.search(line).group(2))
            ls_times.append(time_to_ls)
            num_files = float(ls_time_pattern.search(line).group(1))
        except Exception:
            pass

        try:
            # convert filesizes bytes -> megabytes
            filesize = float(cat_time_pattern.search(line).group(1)) / (1000000.0)
            # convert seconds to minutes
            time_to_cat = float(cat_time_pattern.search(line).group(2)) / 60.0
            cat_times.append((filesize, time_to_cat))
        except Exception:
            pass

    return num_files, mount_times, ls_times, cat_times, manifest_filename

def plot_file_size_performance_graph(cat_times):
    X = np.asarray([x[0] for x in cat_times])
    Y = np.asarray([ x[1] for x in cat_times])

    mpl_fig = plt.figure()
    ax = mpl_fig.add_subplot(111)
    dot_area = np.asarray([60 for x in cat_times])
    alphas = np.asarray([0.3 for x in cat_times])
    colors = np.asarray(['green' for x in cat_times])
    plt.scatter(X,Y, s=dot_area, marker='o', c=colors, edgecolors='black')
    plt.title('Gen3Fuse cat times by file size')
    plt.grid(True)
    ax.set_xlabel('filesize in MB')
    ax.set_ylabel('minutes to cat')

    plt.show()

def plot_num_files_mount_time_performance_graph(to_plot):
    x_arr = []
    y_arr = []
    for entry in to_plot:
        num_files = entry["num_files"]
        for time in entry["mount_times"]:
            x_arr.append(num_files)
            y_arr.append(time)

    X = np.asarray(x_arr)
    Y = np.asarray(y_arr)

    mpl_fig = plt.figure()
    ax = mpl_fig.add_subplot(111)
    dot_area = np.asarray([100 for x in x_arr])
    alphas = np.asarray([0.3 for x in x_arr])
    colors = np.asarray(['green' for x in x_arr])
    plt.scatter(X,Y, s=dot_area, marker='o', c=colors, edgecolors='black')
    plt.title('Gen3Fuse mount times by the number of files')
    plt.grid(True)
    ax.set_xlabel('number of files in manifest')
    ax.set_ylabel('seconds to mount')

    plt.show()

def plot_num_files_ls_time_performance_graph(to_plot):
    x_arr = []
    y_arr = []
    for entry in to_plot:
        num_files = entry["num_files"]
        for time in entry["ls_times"]:
            x_arr.append(num_files)
            y_arr.append(time)

    X = np.asarray(x_arr)
    Y = np.asarray(y_arr)

    mpl_fig = plt.figure()
    ax = mpl_fig.add_subplot(111)
    dot_area = np.asarray([100 for x in x_arr])
    alphas = np.asarray([0.3 for x in x_arr])
    colors = np.asarray(['green' for x in x_arr])
    plt.scatter(X,Y, s=dot_area, marker='o', c=colors, edgecolors='black')
    plt.title('Gen3Fuse ls times by the number of files')
    plt.grid(True)
    ax.set_xlabel('number of files in manifest')
    ax.set_ylabel('seconds to ls')

    plt.show()

# Cat times by file size
num_files, mount_times, ls_times, cat_times, manifest = parse_times_file("results/performance-file-sizes.txt")
plot_file_size_performance_graph(cat_times)

# ls and mount times by # of files
manifest_sizes = [10, 100, 1000, 2000, 5000, 10000]
to_plot = []
for i in range(len(manifest_sizes)):
    num_files, mount_times, ls_times, cat_times, manifest = parse_times_file("results/performance-{}-files.txt".format(str(manifest_sizes[i])))
    to_plot.append({
        "num_files" : num_files,
        "ls_times" : ls_times,
        "mount_times" : mount_times
    })

plot_num_files_mount_time_performance_graph(to_plot)
plot_num_files_ls_time_performance_graph(to_plot)

reason: import numpy,import statsmodels


text:
import argparse
import os
import numpy as np
from tqdm import tqdm
import torch.nn as nn
from torchviz import make_dot, make_dot_from_trace

from mypath import Path
from dataloaders import make_data_loader

from utils.loss import SegmentationLosses
from utils.calculate_weights import calculate_weigths_labels
from utils.lr_scheduler import LR_Scheduler
from utils.saver import Saver
from utils.summaries import TensorboardSummary
from utils.metrics import Evaluator
from utils.copy_state_dict import copy_state_dict
from utils.eval_utils import AverageMeter
# from utils.encoding import *

from modeling.baseline_model import *
# from modeling.ADD import *
from modeling.ADD import *
from modeling.operations import normalized_shannon_entropy
from modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d
from modeling.sync_batchnorm.replicate import patch_replication_callback

from apex import amp
from ptflops import get_model_complexity_info
from torch.utils.data import TensorDataset, DataLoader

torch.backends.cudnn.benchmark = True

class trainNew(object):
    def __init__(self, args):
        self.args = args

        """ Define Saver """
        self.saver = Saver(args)
        self.saver.save_experiment_config()

        """ Define Tensorboard Summary """
        self.summary = TensorboardSummary(self.saver.experiment_dir)
        self.writer = self.summary.create_summary()

        """ Define Dataloader """
        kwargs = {'num_workers': args.workers, 'pin_memory': True, 'drop_last': True}
        self.train_loader, self.val_loader, _, self.nclass = make_data_loader(args, **kwargs)

        self.criterion = nn.L1Loss()
        if args.network == 'searched-dense':
            cell_path = os.path.join(args.saved_arch_path, 'autodeeplab', 'genotype.npy')
            cell_arch = np.load(cell_path)

            if self.args.C == 2:
                C_index = [5]
                network_arch = [1, 2, 2, 2, 3, 2, 2, 1, 1, 1, 1, 2]
                low_level_layer = 0
            elif self.args.C == 3:
                C_index = [3, 7]
                network_arch = [1, 2, 3, 2, 2, 3, 2, 3, 2, 3, 2, 3]
                low_level_layer = 0
            elif self.args.C == 4:
                C_index = [2, 5, 8]
                network_arch = [1, 2, 3, 3, 2, 3, 3, 3, 3, 3, 2, 2]
                low_level_layer = 0

            model = ADD(network_arch,
                            C_index,
                            cell_arch,
                            self.nclass,
                            args,
                            low_level_layer)

        elif args.network.startswith('autodeeplab'):
            network_arch = [0, 0, 0, 1, 2, 1, 2, 2, 3, 3, 2, 1]
            cell_path = os.path.join(args.saved_arch_path, 'autodeeplab', 'genotype.npy')
            cell_arch = np.load(cell_path)
            low_level_layer = 2
            if self.args.C == 2:
                C_index = [5]
            elif self.args.C == 3:
                C_index = [3, 7]
            elif self.args.C == 4:
                C_index = [2, 5, 8]

            if args.network == 'autodeeplab-dense':
                model = ADD(network_arch,
                            C_index,
                            cell_arch,
                            self.nclass,
                            args,
                            low_level_layer)

            elif args.network == 'autodeeplab-baseline':
                model = Baselin_Model(network_arch,
                                    C_index,
                                    cell_arch,
                                    self.nclass,
                                    args,
                                    low_level_layer)

        self.edm = EDM().cuda()
        optimizer = torch.optim.Adam(self.edm.parameters(), lr=args.lr)
        self.model, self.optimizer = model, optimizer

        if args.cuda:
            self.model = self.model.cuda()

        """ Resuming checkpoint """
        if args.resume is not None:
            if not os.path.isfile(args.resume):
                raise RuntimeError("=> no checkpoint found at '{}'".format(args.resume))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch']

            """ if the weights are wrapped in module object we have to clean it """
            if args.clean_module:
                self.model.load_state_dict(checkpoint['state_dict'])
                state_dict = checkpoint['state_dict']
                new_state_dict = OrderedDict()
                for k, v in state_dict.items():
                    name = k[7:]  # remove 'module.' of dataparallel
                    new_state_dict[name] = v
                copy_state_dict(self.model.state_dict(), new_state_dict)

            else:
                if (torch.cuda.device_count() > 1):
                    copy_state_dict(self.model.module.state_dict(), checkpoint['state_dict'])
                else:
                    copy_state_dict(self.model.state_dict(), checkpoint['state_dict'])

        if os.path.isfile('feature.npy'):
            train_feature = np.load('feature.npy')
            train_entropy = np.load('entropy.npy')
            train_set = TensorDataset(torch.tensor(train_feature), torch.tensor(train_entropy, dtype=torch.float))
            train_set = DataLoader(train_set, batch_size=self.args.train_batch, shuffle=True, pin_memory=True)
            self.train_set = train_set
        else:
            self.make_data(self.args.train_batch)

    def make_data(self, batch_size):
        self.model.eval()
        tbar = tqdm(self.train_loader, desc='\r')
        train_feature = []
        train_entropy = []
        for i, sample in enumerate(tbar):
            image, target = sample['image'], sample['label']
            if self.args.cuda:
                image, target = image.cuda(), target.cuda()

            with torch.no_grad():
                output, feature = self.model.get_feature(image)
                train_entropy.append(normalized_shannon_entropy(output))
                train_feature.append(feature.cpu())

        train_feature = [t.numpy() for t in train_feature]
        np_entropy = np.array(train_entropy)
        np.save('feature', train_feature)
        np.save('entropy', train_entropy)
        train_set = TensorDataset(torch.tensor(train_feature, dtype=torch.float), torch.tensor(train_entropy, dtype=torch.float))
        train_set = DataLoader(train_set, batch_size=batch_size, shuffle=True, pin_memory=True)
        self.train_set = train_set

    def training(self, epoch):
        train_loss = 0.0
        self.edm.train()
        tbar = tqdm(self.train_set)
        for i, (feature,entropy) in enumerate(tbar):
            if self.args.cuda:
                feature, entropy = feature.cuda(), entropy.cuda()
            output = self.edm(feature)
            loss = self.criterion(output, entropy)
            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()
            train_loss += loss.item()
            tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1)))
        self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch)
        print('[Epoch: %d' % (epoch))
        print('Loss: %.3f' % train_loss)

def main():
    parser = argparse.ArgumentParser(description="Train EDM")

    """ model setting """
    parser.add_argument('--network', type=str, default='searched-dense', \
        choices=['searched-dense', 'autodeeplab-baseline', 'autodeeplab-dense'])
    parser.add_argument('--F', type=int, default=20)
    parser.add_argument('--B', type=int, default=5)
    parser.add_argument('--C', type=int, default=2, help='num of classifiers')

    """ dataset config"""
    parser.add_argument('--dataset', type=str, default='cityscapes', choices=['cityscapes', 'cityscapes_edm'], help='dataset name (default: pascal)')
    parser.add_argument('--workers', type=int, default=4, metavar='N', help='dataloader threads')

    """ training config """
    parser.add_argument('--epochs', type=int, default=10, metavar='N')
    parser.add_argument('--start_epoch', type=int, default=0)
    parser.add_argument('--batch-size', type=int, default=1, metavar='N')
    parser.add_argument('--test-batch-size', type=int, default=1, metavar='N')
    parser.add_argument('--train-batch', type=int, default=16, metavar='N')
    parser.add_argument('--dist', action='store_true', default=False)

    """ optimizer params """
    parser.add_argument('--lr', type=float, default=0.001, metavar='LR')
    parser.add_argument('--clean-module', type=int, default=0)
    parser.add_argument('--sync-bn', type=bool, default=False, help='whether to use sync bn (default: auto)')

    """ cuda, seed and logging """
    parser.add_argument('--no-cuda', action='store_true', default=False)
    parser.add_argument('--gpu-ids', type=str, default='0', help='use which gpu to train, must be a comma-separated list of integers only (default=0)')
    parser.add_argument('--seed', type=int, default=1, metavar='S')

    """ checking point """
    parser.add_argument('--resume', type=str, default=None, help='put the path to resuming file if needed')
    parser.add_argument('--saved-arch-path', type=str, default='searched_arch/')
    parser.add_argument('--checkname', type=str, default='edm')

    args = parser.parse_args()
    args.cuda = not args.no_cuda and torch.cuda.is_available()
    if args.cuda:
        try:
            args.gpu_ids = [int(s) for s in args.gpu_ids.split(',')]
        except ValueError:
            raise ValueError('Argument --gpu_ids must be a comma-separated list of integers only')

    if args.checkname is None:
        args.checkname = 'deeplab-'+str(args.network)

    print(args)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    new_trainer = trainNew(args)
    # new_trainer.mac()

    # new_trainer.make_data(args.train_batch)
    print('start training')
    for epoch in range(args.epochs):
        new_trainer.training(epoch)
    new_trainer.saver.save_checkpoint({
                'epoch':args.epochs,
                'state_dict': new_trainer.edm.state_dict(),
                'optimizer': new_trainer.optimizer.state_dict(),
                'best_pred': 1},
                True)
    new_trainer.writer.close()

if __name__ == "__main__":
   main()

max_stars_repo_name: HankKung/Dynamic-AutoDeepLab

max_issues_repo_name: HankKung/Distributed-AutoDeepLab

max_forks_repo_name: HankKung/Distributed-AutoDeepLab


text:
import numpy as np
import tensorflow as tf
from a_nice_mc.objectives.expression import Expression
from a_nice_mc.utils.logger import create_logger

logger = create_logger(__name__)

class LordOfRings(Expression):
    def __init__(self, name='lord_of_rings', display=True):
        super(LordOfRings, self).__init__(name=name, display=display)
        self.z = tf.placeholder(tf.float32, [None, 2], name='z')

    def __call__(self, z):
        with tf.variable_scope(self.name):
            z1 = tf.reshape(tf.slice(z, [0, 0], [-1, 1]), [-1])
            z2 = tf.reshape(tf.slice(z, [0, 1], [-1, 1]), [-1])
            v1 = (tf.sqrt(z1 * z1 + z2 * z2) - 1) / 0.2
            v2 = (tf.sqrt(z1 * z1 + z2 * z2) - 2) / 0.2
            v3 = (tf.sqrt(z1 * z1 + z2 * z2) - 3) / 0.2
            v4 = (tf.sqrt(z1 * z1 + z2 * z2) - 4) / 0.2
            v5 = (tf.sqrt(z1 * z1 + z2 * z2) - 5) / 0.2
            p1, p2, p3, p4, p5 = v1 * v1, v2 * v2, v3 * v3, v4 * v4, v5 * v5
            return tf.minimum(tf.minimum(tf.minimum(tf.minimum(p1, p2), p3), p4), p5)

    @staticmethod
    def mean():
        return np.array([3.6])

    @staticmethod
    def std():
        return np.array([1.24])

    @staticmethod
    def xlim():
        return [-6, 6]

    @staticmethod
    def ylim():
        return [-6, 6]

    @staticmethod
    def statistics(z):
        z_ = np.sqrt(np.sum(np.square(z), axis=-1, keepdims=True))
        return z_

max_issues_repo_name: jiamings/a-nice-mc

max_forks_repo_name: jiamings/a-nice-mc


text:
import cv2
import numpy as np

img = cv2.imread('../images/star.jpg')
imgray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret,thresh = cv2.threshold(imgray,127,255,0)
im,contours,hierarchy = cv2.findContours(thresh, 1, 2)

cnt = contours[0]

# Bounding Rectangle
x,y,w,h = cv2.boundingRect(cnt)
img = cv2.rectangle(img,(x,y),(x+w,y+h),(0,0,255),2)

# Aspect Ratio:
# Ratio of width to hieght of bounding Rectangle
aspect_ratio = float(w)/h
print 'Aspect Ratio: ', aspect_ratio

# Extent
# Ratio of contour area to bounding rectangle area
area = cv2.contourArea(cnt)
bound_rect_area = w * h
extent = float(area)/bound_rect_area
print 'Extent: ', extent

# Solidity
# Ratio of contour area to its convex hull area
hull = cv2.convexHull(cnt)
hull_area = cv2.contourArea(hull)
solidity = float(area)/hull_area
print 'Solidity: ', solidity

# Equivalent Diameter
# Diameter of the circle whose area is the same as the contour area
equ_diameter = np.sqrt(4*area/np.pi)
print 'Equivalent Diameter: ', equ_diameter

# Orientation
# angle at which the object is directed. Also returns Major and Minor Axis length
(x,y), (Ma, ma), angle = cv2.fitEllipse(cnt)
print 'Orientation: '
print '     Angle on X: ', x
print '     Angle on Y: ', y
print '     length of Major Axis: ', Ma
print '     length of minor axis: ', ma

# Mask and Get Pixel Points
# Get all the points comprising of that object
mask = np.zeros(imgray.shape, np.uint8)
cv2.drawContours(mask, [cnt], 0,255,-1)
# Numpy function returns coordinates in (row, column)
pixelpoints = np.transpose(np.nonzero(mask))
# CV gives coordinates in (x,y) format
# Note: row = x, column = y
# pixelpoints = cv2.findNonZero(mask)
print '     Length of mask of object (all points): ', len(pixelpoints)

# Max Value, Min Value, Max Value Location, Min Value Location
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(imgray, mask = mask)
print 'Min Value: ', min_val
print 'Max Value: ', max_val
print 'Min Loc: ', min_loc
print 'Min Loc: ', max_loc

# Mean Color/ Mean Intensity
mean_val = cv2.mean(imgray, mask = mask)
print 'Mean Value: ', mean_val

# Get extreme points
leftmost = tuple(cnt[cnt[:,:,0].argmin()][0])
rightmost = tuple(cnt[cnt[:,:,0].argmax()][0])
topmost = tuple(cnt[cnt[:,:,1].argmin()][0])
bottommost = tuple(cnt[cnt[:,:,1].argmax()][0])
print 'Left most point: ', leftmost
img = cv2.circle(img, leftmost, 5, (255,0,0), -1)
img = cv2.circle(img, rightmost, 5, (255,0,0), -1)
img = cv2.circle(img, topmost, 5, (255,0,0), -1)
img = cv2.circle(img, bottommost, 5, (255,0,0), -1)

# Convexity Defects
# Any deviation of this object from the convex hull can be
# considered as Convexity Defects
# adding parameter returnPoints = False, finds the convexity defects
# returns an array where each row is
# [start pt, end pt, farthest pt, approx dist to farthest pt]
dhull = cv2.convexHull(cnt, returnPoints = False)
defects = cv2.convexityDefects(cnt, dhull)

for i in range(defects.shape[0]):
    s,e,f,d = defects[i,0]
    start = tuple(cnt[s][0])
    end = tuple(cnt[e][0])
    far = tuple(cnt[f][0])
    cv2.line(img, start, end, [0,255,0],2)
    cv2.circle(img, far, 5, [0,0,255], -1)

print 'Dimensions of defects: ', defects.shape

# Point Polygon Test
# Finds the shortest distance between a point in the image and a
# contour. It returns the distance which is negative when the point
# is outside the contour, positive when the point is inside and zero
# if the point is on the contour
# if measureDist = False, it returns -1,0,+1
dist = cv2.pointPolygonTest(cnt, (50,50), measureDist = True)
print 'Point Polygon Test: ', dist

# Match Shapes
# compare two shapes or contours and returns a metric demonstrating
# the similarity. The lower the result, the better match it is
# calculate on hu-moment values
# ret = cv2.matchShapes(cnt1, cnt2, 1, 0, 0)

# Bounding Rectangle
x,y,w,h = cv2.boundingRect(cnt)
img = cv2.rectangle(img,(x,y),(x+w,y+h),(0,0,255),2)

cv2.imshow('star', img)
# cv2.imshow('points', pixelpoints)
cv2.waitKey(0)
cv2.destroyAllWindows()

max_stars_repo_name: vibinash/vision

max_issues_repo_name: vibinash/vision

max_forks_repo_name: vibinash/vision


text:
import os.path as op

from nose.tools import assert_true
from numpy.testing import assert_array_equal

import mne
from mne import Epochs, read_events, pick_channels
from mne.utils import run_tests_if_main
from mne.realtime import MockRtClient, RtEpochs

base_dir = op.join(op.dirname(__file__), '..', '..', 'io', 'tests', 'data')
raw_fname = op.join(base_dir, 'test_raw.fif')
event_name = op.join(base_dir, 'test-eve.fif')

events = read_events(event_name)

def test_mockclient():
    """Test the RtMockClient."""

    raw = mne.io.Raw(raw_fname, preload=True, verbose=False)
    picks = mne.pick_types(raw.info, meg='grad', eeg=False, eog=True,
                           stim=True, exclude=raw.info['bads'])

    event_id, tmin, tmax = 1, -0.2, 0.5

    epochs = Epochs(raw, events[:7], event_id=event_id, tmin=tmin, tmax=tmax,
                    picks=picks, baseline=(None, 0), preload=True)
    data = epochs.get_data()

    rt_client = MockRtClient(raw)
    rt_epochs = RtEpochs(rt_client, event_id, tmin, tmax, picks=picks,
                         isi_max=0.5)

    rt_epochs.start()
    rt_client.send_data(rt_epochs, picks, tmin=0, tmax=10, buffer_size=1000)

    rt_data = rt_epochs.get_data()

    assert_true(rt_data.shape == data.shape)
    assert_array_equal(rt_data, data)

def test_get_event_data():
    """Test emulation of realtime data stream."""

    raw = mne.io.Raw(raw_fname, preload=True, verbose=False)
    picks = mne.pick_types(raw.info, meg='grad', eeg=False, eog=True,
                           stim=True, exclude=raw.info['bads'])

    event_id, tmin, tmax = 2, -0.1, 0.3
    epochs = Epochs(raw, events, event_id=event_id,
                    tmin=tmin, tmax=tmax, picks=picks, baseline=None,
                    preload=True, proj=False)

    data = epochs.get_data()[0, :, :]

    rt_client = MockRtClient(raw)
    rt_data = rt_client.get_event_data(event_id=event_id, tmin=tmin,
                                       tmax=tmax, picks=picks,
                                       stim_channel='STI 014')

    assert_array_equal(rt_data, data)

def test_find_events():
    """Test find_events in rt_epochs."""

    raw = mne.io.Raw(raw_fname, preload=True, verbose=False)
    picks = mne.pick_types(raw.info, meg='grad', eeg=False, eog=True,
                           stim=True, exclude=raw.info['bads'])

    event_id = [0, 5, 6]
    tmin, tmax = -0.2, 0.5

    stim_channel = 'STI 014'
    stim_channel_idx = pick_channels(raw.info['ch_names'],
                                     include=stim_channel)

    # Reset some data for ease of comparison
    raw.first_samp = 0
    raw.info['sfreq'] = 1000
    # Test that we can handle consecutive events with no gap
    raw._data[stim_channel_idx, :] = 0
    raw._data[stim_channel_idx, 500:520] = 5
    raw._data[stim_channel_idx, 520:530] = 6
    raw._data[stim_channel_idx, 530:532] = 5
    raw._data[stim_channel_idx, 540] = 6

    # consecutive=False
    find_events = dict(consecutive=False)

    rt_client = MockRtClient(raw)
    rt_epochs = RtEpochs(rt_client, event_id, tmin, tmax, picks=picks,
                         stim_channel='STI 014', isi_max=0.5,
                         find_events=find_events)
    rt_client.send_data(rt_epochs, picks, tmin=0, tmax=10, buffer_size=1000)
    rt_epochs.start()
    events = [5, 6]
    for ii, ev in enumerate(rt_epochs.iter_evoked()):
        assert_true(ev.comment == str(events[ii]))
    assert_true(ii == 1)

    # consecutive=True
    find_events = dict(consecutive=True)
    rt_client = MockRtClient(raw)
    rt_epochs = RtEpochs(rt_client, event_id, tmin, tmax, picks=picks,
                         stim_channel='STI 014', isi_max=0.5,
                         find_events=find_events)
    rt_client.send_data(rt_epochs, picks, tmin=0, tmax=10, buffer_size=1000)
    rt_epochs.start()
    events = [5, 6, 5, 6]
    for ii, ev in enumerate(rt_epochs.iter_evoked()):
        assert_true(ev.comment == str(events[ii]))
    assert_true(ii == 3)

    # min_duration=0.002
    find_events = dict(consecutive=False, min_duration=0.002)
    rt_client = MockRtClient(raw)
    rt_epochs = RtEpochs(rt_client, event_id, tmin, tmax, picks=picks,
                         stim_channel='STI 014', isi_max=0.5,
                         find_events=find_events)
    rt_client.send_data(rt_epochs, picks, tmin=0, tmax=10, buffer_size=1000)
    rt_epochs.start()
    events = [5]
    for ii, ev in enumerate(rt_epochs.iter_evoked()):
        assert_true(ev.comment == str(events[ii]))
    assert_true(ii == 0)

    # ouput='step', consecutive=True
    find_events = dict(output='step', consecutive=True)
    rt_client = MockRtClient(raw)
    rt_epochs = RtEpochs(rt_client, event_id, tmin, tmax, picks=picks,
                         stim_channel='STI 014', isi_max=0.5,
                         find_events=find_events)
    rt_client.send_data(rt_epochs, picks, tmin=0, tmax=10, buffer_size=1000)
    rt_epochs.start()
    events = [5, 6, 5, 0, 6, 0]
    for ii, ev in enumerate(rt_epochs.iter_evoked()):
        assert_true(ev.comment == str(events[ii]))
    assert_true(ii == 5)

run_tests_if_main()

max_stars_repo_name: TanayGahlot/mne-python

max_issues_repo_name: TanayGahlot/mne-python

max_forks_repo_name: TanayGahlot/mne-python

reason: from numpy


text:
import math
import numpy as np
from skfeature.utility.sparse_learning import tree_lasso_projection, tree_norm

def group_fs(X, y, z1, z2, idx, **kwargs):
    """
    This function implements supervised sparse group feature selection with least square loss, i.e.,
    min_{w} ||Xw-y||_2^2 + z_1||w||_1 + z_2*sum_{i} h_{i}||w_{G_{i}}|| where h_i is the weight for the i-th group

    Input
    -----
    X: {numpy array}, shape (n_samples, n_features)
        input data
    y: {numpy array}, shape (n_samples,)
        input class labels or regression target
    z1: {float}
        regularization parameter of L1 norm for each element
    z2: {float}
        regularization parameter of L2 norm for the non-overlapping group
    idx: {numpy array}, shape (3, n_nodes)
        3*nodes matrix, where nodes denotes the number of groups
        idx[1,:] contains the starting index of a group
        idx[2,: contains the ending index of a group
        idx[3,:] contains the corresponding weight (w_{j})
    kwargs: {dictionary}
        verbose: {boolean}
            True if user want to print out the objective function value in each iteration, false if not

    Output
    ------
    w: {numpy array}, shape (n_features, )
        weight matrix
    obj: {numpy array}, shape (n_iterations, )
        objective function value during iterations
    value_gamma: {numpy array}, shape (n_iterations, )
        suitable step size during iterations

    Reference
    ---------
        Liu, Jun, et al. "Moreau-Yosida Regularization for Grouped Tree Structure Learning." NIPS. 2010.
        Liu, Jun, et al. "SLEP: Sparse Learning with Efficient Projections." http://www.public.asu.edu/~jye02/Software/SLEP, 2009.
    """
    if 'verbose' not in kwargs:
        verbose = False
    else:
        verbose = kwargs['verbose']

    # starting point initialization
    n_samples, n_features = X.shape

    # compute X'y
    Xty = np.dot(np.transpose(X), y)

    # initialize a starting point
    w = np.zeros(n_features)

    # compute Xw = X*w
    Xw = np.dot(X, w)

    # starting the main program, the Armijo Goldstein line search scheme + accelerated gradient descent
    # initialize step size gamma = 1
    gamma = 1

    # assign wp with w, and Xwp with Xw
    Xwp = Xw
    wwp = np.zeros(n_features)
    alphap = 0
    alpha = 1

    # indicates whether the gradient step only changes a little
    flag = False

    max_iter = 1000
    value_gamma = np.zeros(max_iter)
    obj = np.zeros(max_iter)
    for iter_step in range(max_iter):
        # step1: compute search point s based on wp and w (with beta)
        beta = (alphap-1)/alpha
        s = w + beta*wwp

        # step2: line search for gamma and compute the new approximation solution w
        Xs = Xw + beta*(Xw - Xwp)
        # compute X'* Xs
        XtXs = np.dot(np.transpose(X), Xs)
        # obtain the gradient g
        G = XtXs - Xty
        # copy w and Xw to wp and Xwp
        wp = w
        Xwp = Xw

        while True:
            # let s walk in a step in the antigradient of s to get v and then do the L1/L2-norm regularized projection
            v = s - G/gamma
            # tree overlapping group lasso projection
            n_nodes = int(idx.shape[1])
            idx_tmp = np.zeros((3, n_nodes+1))
            idx_tmp[0:2, :] = np.concatenate((np.array([[-1], [-1]]), idx[0:2, :]), axis=1)
            idx_tmp[2, :] = np.concatenate((np.array([z1/gamma]), z2/gamma*idx[2, :]), axis=1)
            w = tree_lasso_projection(v, n_features, idx_tmp, n_nodes+1)
            # the difference between the new approximate solution w and the search point s
            v = w - s
            # compute Xw = X*w
            Xw = np.dot(X, w)
            Xv = Xw - Xs
            r_sum = np.inner(v, v)
            l_sum = np.inner(Xv, Xv)
            # determine weather the gradient step makes little improvement
            if r_sum <= 1e-20:
                flag = True
                break

            # the condition is ||Xv||_2^2 <= gamma * ||v||_2^2
            if l_sum <= r_sum*gamma:
                break
            else:
                gamma = max(2*gamma, l_sum/r_sum)
        value_gamma[iter_step] = gamma

        # step3: update alpha and alphap, and check weather converge
        alphap = alpha
        alpha = (1+math.sqrt(4*alpha*alpha+1))/2

        wwp = w - wp
        Xwy = Xw -y

        # calculate the regularization part
        idx_tmp = np.zeros((3, n_nodes+1))
        idx_tmp[0:2, :] = np.concatenate((np.array([[-1], [-1]]), idx[0:2, :]), axis=1)
        idx_tmp[2, :] = np.concatenate((np.array([z1]), z2*idx[2, :]), axis=1)
        tree_norm_val = tree_norm(w, n_features, idx_tmp, n_nodes+1)

        # function value = loss + regularization
        obj[iter_step] = np.inner(Xwy, Xwy)/2 + tree_norm_val

        if verbose:
            print('obj at iter {0}: {1}'.format(iter_step+1, obj[iter_step]))

        if flag is True:
            break

        # determine weather converge
        if iter_step >= 2 and math.fabs(obj[iter_step] - obj[iter_step-1]) < 1e-3:
            break

    return w, obj, value_gamma


text:
import pyqtgraph as pg
from pyqtgraph.dockarea import *
import numpy as np
import os
import numbers

try:
    from PyQt4.QtGui import QFileDialog
    from PyQt4 import QtCore, QtGui
    from PyQt4.QtGui import QMainWindow
except ImportError:
    from PyQt5.QtWidgets import QFileDialog
    from PyQt5 import QtCore, QtGui
    from PyQt5.QtWidgets import QApplication, QMainWindow

from neutronbraggedge.experiment_handler import *
from ImagingReso import _utilities

from __code.ui_resonance_imaging_experiment_vs_theory import Ui_MainWindow as UiMainWindow

class ImageWindow(QMainWindow):

    pen_color = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'w']
    pen_symbol = ['o', 's', 't', 'd', '+']

    stack = []
    integrated_stack = []
    working_folder = ''
    x_axis = {'file_index': [],
              'tof': [],
              'ev': [],
              'lambda': []}
    x_axis_label = {'file_index': 'file index',
                   'tof': u'TOF (\u00B5s)',
                   'ev': 'eV',
                   'lambda': u'\u03BB (\u212B)',
                   }
    y_axis = {'label': 'Mean Counts', 'data': []}
    elements_to_plot = {}  # ex U, U235...etc to plot
    spectra_file = ''
    b_enable_only_file_index_button = True

    def __init__(self, parent=None, stack=[], working_folder='', o_reso=None):
        QMainWindow.__init__(self, parent=parent)
        self.ui = UiMainWindow()
        self.ui.setupUi(self)
        self.setWindowTitle("Select Rotation Angle for All Images")

        self.stack = np.array(stack)
        self.integrated_stack = self.stack.sum(axis=0)
        self.working_folder = working_folder
        self.o_reso = o_reso

        self.initialize_pyqtgraph()
        self.init_label()
        self.init_list_of_things_to_plot()

        self.update_radio_button_status()
        self.display_image()
        self.update_x_axis()
        self.roi_changed()

    def update_plot(self):
        #        self.update_x_axis()
        self.plot()

    def init_label(self):
        _tof_label = u"TOF (\u00B5s)"
        self.ui.tof_radio_button.setText(_tof_label)
        _lambda_label = u"lambda (\u212B)"
        self.ui.lambda_radio_button.setText(_lambda_label)
        _offset_label = u"\u00B5s"
        self.ui.detector_offset_units.setText(_offset_label)

    def display_image(self):
        self.ui.image_view.setImage(self.integrated_stack)

    def plot(self):
        x_axis_selected = self.get_x_axis_selected()
        x_axis_data = self.x_axis[x_axis_selected]
        y_axis_data = self.y_axis['data']

#         print("for {}".format(x_axis_selected))
#         pprint.pprint(y_axis_data[0:10])
#         pprint.pprint(x_axis_data[0:10])
#         print()

        y_axis_label = self.y_axis['label']

        if x_axis_selected == 'ev':
            y_axis_data = y_axis_data[::-1]
            x_axis_data = x_axis_data[::-1]

        x_axis_data = x_axis_data[0: len(y_axis_data)]

        self.counts_vs_index.clear()
        try:
            self.legend.scene().removeItem(self.legend)
        except:
            pass
        self.legend = self.counts_vs_index.addLegend()
        self.counts_vs_index.plot(
            x_axis_data, y_axis_data, name='Experimental')

        self.counts_vs_index.setLabel('bottom', x_axis_selected)
        self.counts_vs_index.setLabel('left', y_axis_label)

        # plot all elements
        elements_to_plot = self.elements_to_plot
        _index_pen_color = 0
        _index_pen_symbol = 0
        for _label in elements_to_plot.keys():
            _x_axis_data = elements_to_plot[_label]['x_axis']
            _y_axis_data = elements_to_plot[_label]['y_axis']
            self.counts_vs_index.plot(
                _x_axis_data,
                _y_axis_data,
                name=_label,
                pen=self.pen_color[_index_pen_color],
                penSymbol=self.pen_symbol[_index_pen_symbol])
            _index_pen_color += 1
            if _index_pen_color >= len(self.pen_color):
                _index_pen_color = 0
                _index_pen_symbol += 1

            if _index_pen_symbol == len(self.pen_symbol):
                _index_pen_color = 0
                _index_pen_symbol = 0

    def initialize_pyqtgraph(self):
        area = DockArea()
        area.setVisible(True)
        d1 = Dock("Image Integrated Preview", size=(300, 800))
        d2 = Dock("Counts vs Image Index of Selection", size=(300, 800))

        area.addDock(d1, 'right')
        area.addDock(d2, 'left')

        preview_widget = pg.GraphicsLayoutWidget()
        pg.setConfigOptions(antialias=True)

        # image view
        self.ui.image_view = pg.ImageView()
        self.ui.image_view.ui.menuBtn.hide()
        self.ui.image_view.ui.roiBtn.hide()

        # default ROI
        self.ui.roi = pg.ROI([0, 0], [20, 20],
                             pen=(62, 13, 244),
                             scaleSnap=True)  #blue
        self.ui.roi.addScaleHandle([1, 1], [0, 0])
        self.ui.image_view.addItem(self.ui.roi)
        self.ui.roi.sigRegionChanged.connect(self.roi_changed)
        d1.addWidget(self.ui.image_view)

        self.counts_vs_index = pg.PlotWidget(title='')
        self.counts_vs_index.plot()
        d2.addWidget(self.counts_vs_index)

        vertical_layout = QtGui.QVBoxLayout()
        vertical_layout.addWidget(area)

        self.ui.widget.setLayout(vertical_layout)

    def roi_changed(self):
        region = self.ui.roi.getArraySlice(self.integrated_stack,
                                           self.ui.image_view.imageItem)
        x0 = region[0][0].start
        x1 = region[0][0].stop - 1
        y0 = region[0][1].start
        y1 = region[0][1].stop - 1

        mean_selection = [_data[x0:x1, y0:y1].mean() for _data in self.stack]
        self.y_axis['data'] = mean_selection
        self.plot()

    # x_axis
    def get_x_axis_selected(self):
        if self.ui.file_index_ratio_button.isChecked():
            return 'file_index'
        elif self.ui.tof_radio_button.isChecked():
            return 'tof'
        elif self.ui.lambda_radio_button.isChecked():
            return 'lambda'
        else:
            return 'ev'

    def update_radio_button_status(self):

        x_axis_selected = self.get_x_axis_selected()

        # enable or not list of element to display
        if x_axis_selected == 'file_index':
            list_status = False
        else:
            list_status = True
        self.ui.list_to_plot_widget.setEnabled(list_status)
        b_enable_only_file_index_button = False

        spectra_file = self.spectra_file
        if not os.path.exists(spectra_file):
            x_axis_selected = 'file_index'
            b_enable_only_file_index_button = True

        distance_source_detector = self.ui.distance_source_detector_value.text()
        if not distance_source_detector:
            x_axis_selected = 'file_index'
            b_enable_only_file_index_button = True

        elif not isinstance(float(distance_source_detector), numbers.Number):
            x_axis_selected = 'file_index'
            b_enable_only_file_index_button = True

        detector_offset = str(self.ui.detector_offset_value.text())
        if not detector_offset:
            x_axis_selected = 'file_index'
            b_enable_only_file_index_button = True
        elif not isinstance(float(detector_offset), numbers.Number):
            x_axis_selected = 'file_index'
            b_enable_only_file_index_button = True

        self.set_radio_buttons_status(
            b_enable_only_file_index_button=b_enable_only_file_index_button)
        self.b_enable_only_file_index_button = b_enable_only_file_index_button

        self.update_x_axis()

    def update_x_axis(self):

        self.x_axis['file_index'] = np.arange(len(self.stack))
        if not self.b_enable_only_file_index_button:

            # tof
            spectra_file = self.spectra_file
            _tof_handler = TOF(filename=spectra_file)
            self.x_axis['tof'] = _tof_handler.tof_array

            # lambda
            distance_source_detector = self.ui.distance_source_detector_value.text()
            detector_offset = str(self.ui.detector_offset_value.text())
            _exp = Experiment(
                tof=_tof_handler.tof_array,
                distance_source_detector_m=float(distance_source_detector),
                detector_offset_micros=float(detector_offset))
            self.x_axis['lambda'] = _exp.lambda_array * 1e10

            # ev
            _exp = Experiment(tof = _tof_handler.tof_array,
                              distance_source_detector_m = float(distance_source_detector),
                              detector_offset_micros= float(detector_offset))
            _exp_ev = _utilities.convert_x_axis(array=_exp.lambda_array*1e10,
                                               from_units='angstroms',
                                               to_units='ev',
                                               offset_us=float(detector_offset),
                                               source_to_detector_m=float(distance_source_detector))

#             _exp_ev = np.linspace(1, 3000, len(_tof_handler.tof_array))
#             import scipy
#             _exp_ev = scipy.random.ranf(len(_tof_handler.tof_array)) * 3000000
#             _exp_ev.sort()
#             _exp_ev = _exp_ev[::-1]
            self.x_axis['ev'] = _exp_ev

#             with open('/users/j35/Desktop/test_output.txt', 'w') as f:
#                 for _data in _exp_ev:
#                     f.write(str(_data) + '\n')
        else:

            self.x_axis['ev'] = []
            self.x_axis['tof'] = []
            self.x_axis['lambda'] = []

    def set_radio_buttons_status(self, b_enable_only_file_index_button=False):
        self.ui.tof_radio_button.setEnabled(
            not b_enable_only_file_index_button)
        self.ui.lambda_radio_button.setEnabled(
            not b_enable_only_file_index_button)
        self.ui.energy_radio_button.setEnabled(
            not b_enable_only_file_index_button)
        if b_enable_only_file_index_button:
            self.ui.file_index_ratio_button.setChecked(True)

    def radio_button_clicked(self):
        self.update_radio_button_status()
        self.plot()

    def distance_source_detector_validated(self):
        self.update_radio_button_status()
        self.update_x_axis()
        self.plot()

    def detector_offset_validated(self):
        self.update_radio_button_status()
        self.update_x_axis()
        self.plot()

    def time_spectra_file_browse_button_clicked(self):
        spectra_file = QFileDialog.getOpenFileName(
            caption='Select Time Spectra',
            directory=self.working_folder,
            filter='txt (*_Spectra.txt);;All (*.*)')
        if spectra_file:
            self.ui.time_spectra_file.setText(os.path.basename(spectra_file))
            self.spectra_file = spectra_file
            self.update_radio_button_status()
            self.update_x_axis()
            self.plot()

    def init_list_of_things_to_plot(self):
        list_things_to_plot = []
        stack = self.o_reso.stack
        list_layers = stack.keys()
        for _layer in list_layers:
            list_things_to_plot.append(_layer)
            list_elements = stack[_layer]['elements']
            for _element in list_elements:
                list_things_to_plot.append(_layer + ' -> ' + _element)
                list_isotopes = stack[_layer][_element]['isotopes']['list']
                for _isotope in list_isotopes:
                    list_things_to_plot.append(_layer + ' -> ' + _element +
                                               ' -> ' + _isotope)

        self.ui.list_to_plot_widget.addItems(list_things_to_plot)

    def done_button_clicked(self):
        self.close()

    def plot_selection_changed(self, item):
        _elements_to_plot = {}

        # init
        x_axis_ev = []

        x_axis_selected = self.get_x_axis_selected()
        if x_axis_selected == 'file_index':
            self.elements_to_plot = _elements_to_plot
            return

        # retrieve data to display
        for _item in self.ui.list_to_plot_widget.selectedIndexes():
            _row_selected = _item.row()
            _text = self.ui.list_to_plot_widget.item(_row_selected).text()
            _layer_element_isotope = self.__parse_layer_element_isotope(_text)

            _layer = _layer_element_isotope['layer']
            _element = _layer_element_isotope['element']
            _isotope = _layer_element_isotope['isotope']

            if _element == '':
                transmission = self.o_reso.stack_signal[_layer]['transmission']
                x_axis_ev = self.o_reso.stack_signal[_layer]['energy_eV']
            elif _isotope == '':
                transmission = self.o_reso.stack_signal[_layer][_element][
                    'transmission']
                x_axis_ev = self.o_reso.stack_signal[_layer][_element][
                    'energy_eV']
            else:
                transmission = self.o_reso.stack_signal[_layer][_element][
                    _isotope]['transmission']
                x_axis_ev = self.o_reso.stack_signal[_layer][_element][
                    _isotope]['energy_eV']

            _elements_to_plot[_text] = {}
            _elements_to_plot[_text]['y_axis'] = transmission

            x_axis = []
            if x_axis_selected == 'lambda':
                x_axis = _utilities.convert_x_axis(
                    array=x_axis_ev, from_units='ev', to_units='angstroms')
            elif x_axis_selected == 'tof':
                detector_offset = float(self.ui.detector_offset_value.text())
                distance_source_detector = float(
                    self.ui.distance_source_detector_value.text())
                x_axis = _utilities.convert_x_axis(
                    array=x_axis_ev,
                    from_units='ev',
                    to_units='s',
                    offset_us=detector_offset,
                    source_to_detector_m=distance_source_detector)
            else:  # ev
                x_axis = x_axis_ev

            _elements_to_plot[_text]['x_axis'] = x_axis

        self.elements_to_plot = _elements_to_plot
        self.plot()

    def __parse_layer_element_isotope(self, text):
        ''' this will create a dictionary of each data to plot
        '''
        _dict = {'layer': '', 'element': '', 'isotope': ''}

        parse_text = text.split(' -> ')
        _dict['layer'] = parse_text[0]
        if len(parse_text) >= 2:
            _dict['element'] = parse_text[1]
        if len(parse_text) >= 3:
            _dict['isotope'] = parse_text[2]

        return _dict

    def closeEvent(self, event=None):
        pass

reason: import numpy,import scipy


text:
# Original Version: Taehoon Kim (http://carpedm20.github.io)
#   + Source: https://github.com/carpedm20/DCGAN-tensorflow/blob/e30539fb5e20d5a0fed40935853da97e9e55eee8/utils.py
#   + License: MIT
from __future__ import division
import math
import random
import scipy.misc
import numpy as np
import cv2

def get_image(image_path, image_size, is_crop=True):
    return transform(imread(image_path), image_size, is_crop)

def save_images(images, size, image_path):
    return imsave(inverse_transform(images), size, image_path)

def imread(path):
    return scipy.misc.imread(path).astype(np.float)

def merge_images(images, size):
    return inverse_transform(images)

def merge(images, size):
    h, w = images.shape[1], images.shape[2]
    img = np.zeros((int(h * size[0]), int(w * size[1]), 3))
    for idx, image in enumerate(images):
        i = idx % size[1]
        j = idx // size[1]
        img[j*h:j*h+h, i*w:i*w+w, :] = image
    return img

def imsave(images, size, path):
    img = merge(images, size)
    return scipy.misc.imsave(path, (255*img).astype(np.uint8))

# The transform() function of utils.py should resize the images to IMAGE_SIZE*IMAGE_SIZE
# Otherwise we get errors: ValueError: Cannot feed value of shape (64, 218, 178, 3) for Tensor 'D/Placeholder:0', which has shape '(?, 64, 64, 3)'
# Because the images of the dataset are 218*178, and our discriminator accepts only 64*64
# So I copy-paste the code from the source URL given by the teachers, and I modify it a little bit
# https://github.com/carpedm20/DCGAN-tensorflow/blob/e30539fb5e20d5a0fed40935853da97e9e55eee8/utils.py
def center_crop(x, crop_h, crop_w=None):
    # The original images are 218*178. If I crop directly 64*64 at the center, I will probably get only
    # a small part of a face, and the result won't be good. So I crop 128*128, and I resize at 64*64
    if crop_w is None:
        crop_w = crop_h
    h, w = x.shape[:2]
    j = h//2 - crop_h
    i = w//2 - crop_w
    result = cv2.resize(x[j:j+crop_h*2, i:i+crop_w*2], (crop_h, crop_w))
    return result

def transform(image, npx=64, is_crop=True):
    if is_crop:
        cropped_image = center_crop(image, npx)
    else:
        cropped_image = image
    return np.array(cropped_image)/127.5 - 1.

def inverse_transform(images):
    return (images+1.)/2.

max_stars_repo_path: Deep Learning in Computer Vision/Week5/gan-task/utils.py


text:
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import manifold

from scipy import misc
import os

# Look pretty...
# matplotlib.style.use('ggplot')
plt.style.use('ggplot')

#
# TODO: Start by creating a regular old, plain, "vanilla"
# python list. You can call it 'samples'.
#
samples = []

# TODO: Write a for-loop that iterates over the images in the
# Module4/Datasets/ALOI/32/ folder, appending each of them to
# your list. Each .PNG image should first be loaded into a
# temporary NDArray, just as shown in the Feature
# Representation reading.
#
# Optional: Resample the image down by a factor of two if you
# have a slower computer. You can also convert the image from
# 0-255  to  0.0-1.0  if you'd like, but that will have no
# effect on the algorithm's results.
#

#indir = 'Datasets/ALOI/32/'
#for root, dirs, filenames in os.walk(indir):
#    for f in filenames:
#        print(indir +f)
#        log = open(indir + f, 'r')

path = "Datasets/ALOI/32/"
for root, dirs, filenames in os.walk(path):
    for file in filenames:
        img = misc.imread(path+'/{0}'.format(file))
        img = ((img[::2, ::2]/250.0).reshape(-1))
        samples.append(img)

# TODO: Once you're done answering the first three questions,
# right before you converted your list to a dataframe, add in
# additional code which also appends to your list the images
# in the Module4/Datasets/ALOI/32_i directory. Re-run your
# assignment and answer the final question below.
#
#
# TODO: Convert the list to a dataframe
#
df=pd.DataFrame(samples)
#
# TODO: Implement Isomap here. Reduce the dataframe df down
# to three components, using K=6 for your neighborhood size
#
# .. your code here ..

iso = manifold.Isomap(n_neighbors=2, n_components=3)
iso.fit(df)
manifold.Isomap(eigen_solver='auto', max_iter=None, n_components=3, n_neighbors=2,
    neighbors_algorithm='auto', path_method='auto', tol=0)
manifold = iso.transform(df)

#
# TODO: Create a 2D Scatter plot to graph your manifold. You
# can use either 'o' or '.' as your marker. Graph the first two
# isomap components
#
# .. your code here ..
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_title('2D ISOMAP')
ax.set_xlabel('Component: {0}'.format(0))
ax.set_ylabel('Component: {0}'.format(1))
ax.scatter(manifold[:,0],manifold[:,1], marker='o',alpha=0.7)

#
# TODO: Create a 3D Scatter plot to graph your manifold. You
# can use either 'o' or '.' as your marker:
#
fig2 = plt.figure()
ax1 = fig2.add_subplot(111,projection='3d')
ax1.set_title('3D ISOMAP')
ax1.set_xlabel('Component: {0}'.format(0))
ax1.set_ylabel('Component: {0}'.format(1))
ax1.set_zlabel('Component: {0}'.format(2))

ax1.scatter(manifold[:,0],manifold[:,1],manifold[:,2], marker='o',alpha=0.7)
plt.show()

reason: from scipy


text:
# -*- coding: utf-8 -*-

#from keras.models import Sequential
import numpy as np
from keras.models import load_model
from keras.preprocessing import image

# returns a compiled model
# identical to the previous one
classifier1 = load_model('mri_model_weights.h5')

#Image index i
i = 4

#Predicting the Dogs dataset
image_name = 'tumor.'+str(i)+'.jpg'
#file_name = 'D:/datasets/brain_MRI_CNN-master/dataset/test_set/tumor/tumor.'+str(i)+'.jpg'
file_name = 'D:/datasets/brain_MRI_CNN-master/tumor/samples/s.'+str(i)+'.jpg'
test_image = image.load_img(file_name, target_size = (128, 128))
test_image = image.img_to_array(test_image)
test_image = np.expand_dims(test_image, axis = 0)
#result = classifier1.predict(test_image)
result = classifier1.predict_classes(test_image)

if result[0][0] == 1:
     print('Model prediction for image '+image_name+':  TUMOR')
else:
     print('Model prediction for image '+image_name+':   NORMAL')

max_stars_repo_name: aakashvarma/Med-I

max_issues_repo_name: aakashvarma/Med-I

max_forks_repo_name: aakashvarma/Med-I


text:
#!/usr/bin/python

import numpy as np
import random

# Nom du reseau
nom = "Parametrable"

# Initialisation du generateur aleatoire
T = 7 # Nombre de niveaux
gral = 123 # Graine Aleatoire
random.seed(gral)

# Dimensions du reseau
m  = (2**(T+1)) - 1
mr = 1
md = m - mr
n  = ((2**(T+1))-1) + ((2**(T+1))-1) - (T+1) - 1

# Numeros des noeuds initiaux et finaux des arcs
orig = []
dest = []

# Arcs de l'arbre
num = 1
for t in range (0,T):
      ni = (2**t);
      nf = (2**(t+1)) - 1;
      nz = 2 * (nf-ni+1);
      #aorg = [ [ni:nf] ; [ni:nf] ];
      aorg = np.array([range(ni,nf+1),range(ni,nf+1)])
      #orig = [ orig aorg(1:nz)' ];
      orig = np.concatenate((orig,aorg.flatten('F')), axis=0)
      #dest = [ dest [num+1:num+nz] ];
      dest = np.concatenate((dest,np.array(range(num+1,num+nz+1))), axis=0)
      num = num + nz;

# Arcs du coarbre
for t in range(1, T+1):
      ni = (2**t);
      nf = (2**(t+1)) - 1;
      #orig = [ orig [ni:nf-1] ];
      orig = np.concatenate((orig,np.array(range(ni,nf))), axis=0)
      #dest = [ dest [ni+1:nf] ];
      dest = np.concatenate((dest,np.array(range(ni+1,nf+1))), axis=0)

# Coordonnees des noeuds
absn = []
for t in range(0,T+1):
      ni = (2**t);
      nf = (2**(t+1)) - 1;
      na = 2**(T-t+1);
      nb = 2**(T-t);
      #num = na*[0:nf-ni] + nb;
      num = na*np.array(range(0,nf-ni+1)) + nb
      #absn = [ absn num ];
      absn = np.concatenate((absn,num),axis=0)
ordn = []
for t in range(0,T+1):
      ni = (2**t);
      nf = (2**(t+1)) - 1;
      #ordn = [ ordn (T-t+1)*ones(1,nf-ni+1) ];
      ordn = np.concatenate((ordn,(T-t+1)*np.ones(nf-ni+1)),axis=0)

# Resistances des arcs
r = 1000 * np.random.rand(n,1)

# Pressions au pied du reservoir (en m)
pr = np.array([200])

# Flux aux noeuds de demande (en m3/s)
fd = 0.1 * (np.random.rand(md,1)-0.5)

max_stars_repo_name: LouisDumont/Projet-Opti-Controle

max_issues_repo_name: LouisDumont/Projet-Opti-Controle

max_forks_repo_name: LouisDumont/Projet-Opti-Controle


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Aug  6 10:58:17 2018

@author: P. Wolf

title: pix2deg
"""
#%% Imports
import numpy as np

#%% main function
def pix2deg(x_px,y_px,cfg,mode):
    if mode == '2D':
        x_rad = np.arctan((x_px * cfg.screen_width)/(cfg.screen_xres * cfg.screen_dist))
        y_rad = np.arctan((y_px * cfg.screen_height)/(cfg.screen_yres * cfg.screen_dist))

        x_deg = x_rad / np.pi * 180
        y_deg = y_rad / np.pi * 180
    elif mode == '3D':
        x_rad = np.arctan(x_px / cfg.screen_dist)
        y_rad = np.arctan(y_px / cfg.screen_dist)

        x_deg = x_rad / np.pi * 180
        y_deg = y_rad / np.pi * 180
    return x_deg, y_deg


text:
import numpy as np
from precise.skaters.covarianceutil.conventions import X_TYPE, X_DATA_TYPE, is_data
from precise.skaters.covarianceutil.datafunctions import data_population_covariance, data_population_correlation

# State machines that track stats for finite buffers of vectors

def buf_pcov_factory(func, y:X_TYPE=None, s:dict=None, n_buffer:int=100, e=1):
    """
         Factory for building estimators from functions that return dicts with 'loc' and 'pcov' keys

    :param func:  Acts on buffer and returns object with 'loc', 'pcov'
    :param y:
    :param s:
    :param n_buffer:
    :return:
    """
    s = _buf(funcs=[func], func_names=['mav'], func_kwargs=[{}],s=s, x=y, n_buffer=n_buffer, e=e)
    if s.get('mav') is not None:
        x = np.array(s['mav']['loc'])
        x_cov = np.array(s['mav']['pcov'])
    else:
        x = y
        x_cov = np.eye(len(y))
    return x, x_cov, s

def buf_cov(s:dict=None, x:X_TYPE=None, n_buffer:int=100)->dict:
    # Equivalent to np.nanstd(xs[max(0, k - n_buffer + 1):k + 1], axis=0)
    return _buf1(func=data_population_correlation, func_name='corrcoef', s=s, x=x, n_buffer=n_buffer)

def buf_std(s:dict=None, x:X_TYPE=None, n_buffer:int=100)->dict:
    # Equivalent to np.nanstd(xs[max(0, k - n_buffer + 1):k + 1], axis=0)
    return _buf1(func=np.nanstd, func_name='mean', s=s, x=x, n_buffer=n_buffer, axis=0)

def buf_mean(s:dict=None, x:X_TYPE=None, n_buffer:int=100)->dict:
    # Equivalent to np.nanmean(xs[max(0, k - n_buffer + 1):k + 1], axis=0)
    return _buf1(func=np.nanmean, func_name='mean', s=s, x=x, n_buffer=n_buffer, axis=0)

def buf_median(s:dict=None, x:X_TYPE=None, n_buffer:int=100)->dict:
    # Equivalent to np.nanmedian(xs[max(0, k - n_buffer + 1):k + 1], axis=0)
    return _buf1(func=np.nanmedian, func_name='median', s=s, x=x, n_buffer=n_buffer, axis=0)

def _buf1(func, func_name, s:dict=None, x:X_TYPE=None, n_buffer:int=100, e=1, **func_kwarg)->dict:
    return _buf(funcs=[func], func_names=[func_name], func_kwargs=[func_kwarg], s=s, x=x, n_buffer=n_buffer,e=e)

def buf_mean_and_median(s:dict=None, x:X_TYPE=None, n_buffer:int=100, e=1)->dict:
    return _buf(funcs=[np.nanmean, np.nanmedian], func_names=['mean','median'], func_kwargs=[{'axis':0},{'axis':0}], s=s, x=x, n_buffer=n_buffer, e=e)

def buf_mean_and_pcov(s:dict=None, x:X_TYPE=None, n_buffer:int=100, e=1)->dict:
    # Equivalent to np.cov(xs[max(0, k - n_buffer + 1):k + 1], axis=0)
    return _buf(funcs=[np.nanmean, data_population_covariance], func_names=['mean', 'pcov'], func_kwargs=[{'axis':0}, {}], s=s, x=x, e=e, n_buffer=n_buffer)

def _buf(funcs, func_names:[str], func_kwargs:[dict], s:dict=None, x:X_TYPE=None, n_buffer:int=100, e=1)->dict:
    """
    :param funcs:            [func(xs, axis=0) -> 1d array]
    :param func_names:
    :param s:
    :param x:
    :param n_buffer:
    :param e:             If e>1 calculation is performed
    :return:
    """
    if not s:
        s = _buf_init(s=s, n_buffer=n_buffer)
    if is_data(x):
        s = _buf_update(funcs=funcs, func_names=func_names, func_kwargs=func_kwargs, s=s, x=x, n_buffer=n_buffer, e=e)
    return s

def _buf_init(s:dict=None, n_buffer:int=None)->dict:
    if not s:
        s = dict()
    s.update( {'buffer':list(), 'n_buffer':n_buffer} )
    return s

def _buf_update(funcs, func_names, func_kwargs, s:dict, x:X_DATA_TYPE, n_buffer:int=None, e=1)->dict:
    if n_buffer is None:
        n_buffer = s['n_buffer']
    if len(s['buffer']):
        assert len(s['buffer'][0])==len(x), 'dimension mismatch'
    s['buffer'].append(np.atleast_1d(x))
    if len(s['buffer'])>n_buffer:
        _ = s['buffer'].pop(0)
    for func, func_name, func_kwargs in zip(funcs, func_names, func_kwargs):
        if e>0:
            s[func_name] = func(s['buffer'],**func_kwargs)
    return s

if __name__=='__main__':
    import random
    xs = np.random.randn(500,100)
    xs[17,1] = np.nan
    s = {}
    n_buffer = random.choice([1,5,100])
    for k,x in enumerate(xs):
        s = buf_mean_and_median(s=s, x=x, n_buffer=n_buffer)
        np_median = np.nanmedian(xs[max(0,k-n_buffer+1):k+1], axis=0)
        s_median = s['median']
        if not np.allclose( np_median, s_median, atol=1e-8 ):
            print('urgh')
            assert False

max_stars_repo_name: MarcoGorelli/precise

max_issues_repo_name: MarcoGorelli/precise

max_forks_repo_name: MarcoGorelli/precise


text:
#!/usr/bin/python3
# -*- coding: utf-8 -*-

from absl.testing import parameterized, absltest

import tempfile
import numpy as np
from ant.core import TopKSearch
from ant.core import MultiInstances

class MultiInstancesTest(parameterized.TestCase):

    def test(self):
        with tempfile.TemporaryDirectory() as tmp:
            mis = MultiInstances(tmp)
            self.assertEqual(mis.list_instances(), [])
            mis.create_instance("tests", {"dim": 10})
            self.assertEqual(mis.list_instances(), ["tests"])
            instance = mis.get_instance("tests")
            self.assertEqual(type(instance), TopKSearch)
            mis.delete_instance("tests")
            self.assertEqual(mis.list_instances(), [])

if __name__ == '__main__':
    absltest.main()

max_stars_repo_name: LongmaoTeamTf/ant

max_issues_repo_name: LongmaoTeamTf/ant

max_forks_repo_name: LongmaoTeamTf/ant


text:
import torch
from torch import nn
from torch.nn import Parameter
import torch.nn.functional as F
from numpy import pi
import math

epsilon = 1e-16 #regularization value in Keras

def NSS(input,fixation):
    input = input.view(input.size(0),-1)
    # input = torch.div(input,input.max(-1,keepdim=True)[0].expand_as(input)+epsilon)
    fixation = fixation.view(fixation.size(0),-1)
    input = torch.div(input-input.mean(-1,keepdim=True).expand_as(input),input.std(-1,keepdim=True).expand_as(input) + epsilon)
    loss = torch.div(torch.mul(input,fixation).sum(-1), fixation.sum(-1) + epsilon)

    return torch.mean(loss)

def CC(input,fixmap,weight=None):
    if weight is not None:
        input = torch.mul(input,weight)
        fixmap = torch.mul(fixmap,weight)
    input = input.view(input.size(0),-1)

    # input = torch.div(input,input.max(-1,keepdim=True)[0].expand_as(input)+epsilon)
    # fixmap = torch.div(fixmap,fixmap.max(-1,keepdim=True)[0].expand_as(fixmap)+epsilon)

    fixmap = fixmap.view(fixmap.size(0),-1)
    fixmap = torch.div(fixmap,fixmap.sum(-1,keepdim=True).expand_as(fixmap)+epsilon)
    input = torch.div(input,input.sum(-1,keepdim=True).expand_as(input)+epsilon)

    sum_prod = torch.mul(input,fixmap).sum(-1,keepdim=True)
    sum_x = input.sum(-1,keepdim=True)
    sum_y = fixmap.sum(-1,keepdim=True)
    sum_x_square = (input**2).sum(-1,keepdim=True)
    sum_y_square = (fixmap**2).sum(-1,keepdim=True)
    num = sum_prod - torch.mul(sum_x,sum_y)/input.size(-1)
    den = torch.sqrt((sum_x_square-sum_x**2/input.size(-1))*(sum_y_square-sum_y**2/input.size(-1)))
    loss = torch.div(num,den+epsilon)

    return torch.mean(loss)

def KLD(input,fixmap,weight=None):
    input = input.view(input.size(0),-1)
    # input = torch.div(input,input.max(-1,keepdim=True)[0].expand_as(input)+epsilon)
    fixmap = fixmap.view(fixmap.size(0),-1)
    fixmap = torch.div(fixmap,fixmap.sum(-1,keepdim=True).expand_as(fixmap))
    input = torch.div(input,input.sum(-1,keepdim=True).expand_as(input))
    loss = torch.mul(fixmap,torch.log(torch.div(fixmap,input+epsilon) + epsilon))
    if weight is not None:
        weight = weight.view(weight.size(0),-1)
        loss = torch.mul(loss,weight).sum(-1)
    else:
        loss = loss.sum(-1)

    return torch.mean(loss)

def SIM(input,fixmap):
    input = input.view(input.size(0), -1)
    fixmap = fixmap.view(fixmap.size(0),-1)
    fixmap = torch.div(fixmap,fixmap.sum(-1,keepdim=True).expand_as(fixmap))
    input = torch.div(input,input.sum(-1,keepdim=True).expand_as(input))
    loss = torch.min(input,fixmap).sum(-1)
    return torch.mean(loss)

def LL(input,fixmap):
    input = input.view(input.size(0),-1)
    input = F.softmax(input,dim=-1)
    fixmap = fixmap.view(fixmap.size(0),-1)
    loss =  torch.mul(torch.log(input+epsilon),fixmap).sum(-1)

    return -torch.sum(loss)

def cross_entropy(input,target):
    input = input.view(input.size(0), -1)
    input = F.softmax(input,dim=-1)
    target = target.view(target.size(0),-1)
    loss = (-target*torch.log(torch.clamp(input,min=epsilon,max=1))).sum(-1)
    return loss.mean()

class SphereMSE(nn.Module):
    def __init__(self, h, w):
        super(SphereMSE, self).__init__()
        self.h, self.w = h, w
        weight = torch.zeros(1, 1, h, w)
        theta_range = torch.linspace(0, pi, steps=h + 1)
        dtheta = pi / h
        dphi = 2 * pi / w
        for theta_idx in range(h):
            weight[:, :, theta_idx, :] = dphi * (math.sin(theta_range[theta_idx]) + math.sin(theta_range[theta_idx+1]))/2 * dtheta
        self.weight = Parameter(weight, requires_grad=False)

    def forward(self, out, target):
        normalized = out.view(out.size(0),-1).max(-1,keepdim=True)[0].unsqueeze(-1).unsqueeze(-1).expand_as(out)
        out = torch.div(out,normalized+epsilon)

        return torch.sum((out - target) ** 2 * self.weight) / out.size(0)

class SphereCE(nn.Module):
    def __init__(self, h, w):
        super(SphereCE, self).__init__()
        self.h, self.w = h, w
        weight = torch.zeros(1, 1, h, w)
        theta_range = torch.linspace(0, pi, steps=h + 1)
        dtheta = pi / h
        dphi = 2 * pi / w
        for theta_idx in range(h):
            weight[:, :, theta_idx, :] = dphi * (math.sin(theta_range[theta_idx]) + math.sin(theta_range[theta_idx+1]))/2 * dtheta
        self.weight = Parameter(weight, requires_grad=False)

    def forward(self, out, target):
        loss = (-target*torch.log(torch.clamp(out,min=epsilon,max=1))) * self.weight
        return torch.sum(loss) / out.size(0)

def binary_cross_entropy(input,target):
    input = input.view(input.size(0), -1)
    target = target.view(target.size(0),-1)
    loss = target*torch.log(torch.clamp(input,min=epsilon,max=1)) + (1-target)*torch.log(1-torch.clamp(input,min=epsilon,max=1))
    loss = -torch.mean(loss)
    return loss

def difference_loss_bce(input,target,weight=None):
    input = (input[:,0]-input[:,1]).view(input.size(0),-1)
    target = target.view(target.size(0),-1)
    input = 1/(1+torch.exp(-input / (input.std(-1,keepdim=True).expand_as(input)+epsilon)))
    target = 1/(1+torch.exp(-target / (target.std(-1,keepdim=True).expand_as(target)+epsilon)))
    loss = target*torch.log(torch.clamp(input,min=epsilon,max=1)) + (1-target)*torch.log(1-torch.clamp(input,min=epsilon,max=1))
    if weight is not None:
        weight = weight.view(weight.size(0),-1)
        loss = torch.mul(loss,weight)
    loss = -torch.sum(loss)

    return loss

def mse(input,target):
    loss = (input-target)**2
    return loss.sum()

def non_linear_layer(input,no_batch=False):
    if no_batch:
        h, w = input.size()
        input = input.view(-1)
        input = 1/(1+torch.exp(-input / (input.std(-1,keepdim=True).expand_as(input)+epsilon)))
        return input.view(h,w)
    else:
        batch, h, w = input.size()
        input = input.view(batch, -1)
        input = 1/(1+torch.exp(-input / (input.std(-1,keepdim=True).expand_as(input)+epsilon)))
        return input.view(batch,h,w)

def difference_loss_mse(input,target,fixation):
    loss = torch.mul(fixation,(input-target)**2).sum(-1).sum(-1)
    return loss.sum()

class Discriminator(nn.Module):
    def __init__(self,):
        super(Discriminator,self).__init__()
        # self.conv1 = nn.Conv2d(2,32,kernel_size=3,padding=1,stride=1,bias=True)
        # self.conv2 = nn.Conv2d(32,16,kernel_size=1,padding=0,stride=1,bias=True)
        #self.pred = nn.Linear(512,1,bias=True)
        self.conv1 = nn.Conv2d(2,64,kernel_size=3,padding=1,stride=1)
        self.conv2 = nn.Conv2d(64,128,kernel_size=3,padding=1,stride=1)
        self.pred = nn.Linear(128,1,bias=True)

    def forward(self,x,y):
        x = torch.cat((x.unsqueeze(1),y.unsqueeze(1)),dim=1)
        x = F.relu(self.conv1(x))
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x,kernel_size=(16,32))
        # x = F.max_pool2d(x,kernel_size=4)
        x = torch.sigmoid(self.pred(x.view(x.size(0),-1)))
        return x

class Discriminator_warpper(nn.Module):
    def __init__(self,discriminator_1,discriminator_2):
        super(Discriminator_warpper,self).__init__()
        self.discriminator_1 = discriminator_1
        self.discriminator_2 = discriminator_2

    def forward(self,x_diff,y_diff,x_pos,y_pos,x_neg,y_neg):
        pred_diff = self.discriminator_1(x_diff,y_diff)
        pred_pos = self.discriminator_2(x_pos,y_pos)
        pred_neg = self.discriminator_2(x_neg,y_neg)
        return pred_diff, pred_pos, pred_neg

max_stars_repo_name: mjiang/iqva

max_issues_repo_name: mjiang/iqva

max_forks_repo_name: mjiang/iqva