text:
from datetime import date, timedelta
from pandas_datareader import data
from pandas_datareader._utils import RemoteDataError
import numpy as np
from numpy import polyval
from PyQt5.QtWidgets import QApplication, QMainWindow, QSizePolicy, QMessageBox, QAction, QFileDialog
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.figure import Figure

# from scipy import stats
import random, sys
import pandas as pd
# import statsmodels.formula.api as sm
from PyQt5 import uic
## local modules
import modules.tableWidget
import modules.dataLoader as md

MainUI = "UI/testMainUI.ui"
LINE_EDIT_1 = 0
LINE_EDIT_2 = 1

from UI.testMainUI import Ui_StockAnlyst
## 메인 클래스
class Main(QMainWindow, Ui_StockAnlyst):
# class Main(QMainWindow):  # ui파일로 로드 하는 방식
    def __init__(self):
        super().__init__()

        # uic.loadUi(MainUI, self)
        self.setupUi(self)

        self.iniUI()
        self.setWindowTitle('금융상품분석툴')

        menubar = self.menuBar()

        helpMenu = menubar.addMenu('Help')

        aboutButton = QAction('About', self)
        aboutButton.triggered.connect(self.aboutMsg)
        helpMenu.addAction(aboutButton)

        exitButton = QAction('Exit', self)
        exitButton.setShortcut('Ctrl+Q')
        exitButton.setStatusTip('Exit application')
        exitButton.triggered.connect(self.close)
        helpMenu.addAction(exitButton)

        self.statusBar().showMessage('빅데이터 분석 2차 프로젝트 1조 2020.1.3')
        self.setUI()

    def iniUI(self):
        self.m = PlotCanvas(self)
        # self.m = modules.plotCanvas.PlotCanvas(self)
        self.m.move(30, 290)  # 초기 위치 설정

        self.rb_1m.clicked.connect(self.rbtn_setPeriod)
        self.rb_3m.clicked.connect(self.rbtn_setPeriod)
        self.rb_6m.clicked.connect(self.rbtn_setPeriod)
        self.rb_1y.clicked.connect(self.rbtn_setPeriod)
        self.rb_2y.clicked.connect(self.rbtn_setPeriod)

        self.btn_more1.clicked.connect(self.btnMore1)
        self.btn_more2.clicked.connect(self.btnMore2)

        self.btn_resultClear.clicked.connect(self.btnResultClear)
        self.btn_save.clicked.connect(self.btnResultSave)
        # self.stockName1 = ''
        # self.stockName2 = ''

        # self.lbl_prierd.clicked.connect(self.btnMore2)

        # self.show()

    # ===============================================================================

    def setLineEditValue(self, code , opt, cat, name):
        # print('name :', name)
        if cat == 'kospi':
            code = code + '.KS'
        elif cat == 'kosdaq':
            code = code + '.KQ'

        if opt == LINE_EDIT_1:
            self.lineEdit_1.setText(code)
            # self.stockName1 = name
        elif opt == LINE_EDIT_2:
            self.lineEdit_2.setText(code)
            # self.stockName2 = name

    # ===============================================================================

    def setUI(self):

        print(date.today())
        self.dateEdit.setDate(date.today() + timedelta(days=-30))
        # time2 + timedelta(days=-3)
        self.dateEdit_2.setDate(date.today())

        self.btn_start.setEnabled(True)
        self.lineEdit_1.setText('AMD')
        self.lineEdit_2.setText('AAPL')
        self.lineEdit_1.setFocus()

        # connect 방식으로 함수를 연결할때 인자를 사용할수 없기때문에
        # 중간에 람다함수를 사용하여 인자를 전달하도록 작성한다.
        # 이때 clicked 함수에서 리턴되는 값이 2개 이므로 값2개를 받고 그중 button객체만 전달하면 된다.
        # 필요하다면 state변수도 전달 한다.
        # self.btn_start.clicked.connect(lambda state, button = self.btn_start : self.btnTest(state, button))

    def btnTest(self, state, btn):
        print(state, type(state))
        print(btn)
        print('btn test')

    ## UI slots
    ## start 버튼
    def aStart(self):

        # 입력 ui 비활성화
        self.btn_start.setEnabled(False)
        self.lineEdit_1.setEnabled(False)
        self.lineEdit_2.setEnabled(False)

        # print(self.lineEdit_1.text())
        # print(self.lineEdit_2.text())

        # 소문자 대문자로
        input1 = self.lineEdit_1.text()
        input2 = self.lineEdit_2.text()
        input1 = input1.upper()
        input2 = input2.upper()
        self.lineEdit_1.setText(input1.upper())
        self.lineEdit_2.setText(input2.upper())

        # qstart = self.dateEdit.date()
        # start = qstart.toPyDate()
        start = self.dateEdit.date().toPyDate()

        # qend = self.dateEdit_2.date()
        # end = qend.toPyDate()
        end = self.dateEdit_2.date().toPyDate()

        self.lbl_prierd.setText(str((end - start).days) + " day")

        if (input1, input2 != None) and (len(input1) > 0 and len(input2) > 0):
            self.inputData(input1, input2, start, end)
        else:
            print('입력코드값 오류')

        # test = modules.dataLoader.DataLoader()
        #
        # test.get_NameBySymbol(input1)

    # 취소버튼
    def cancel(self):

        # self.stockName1 = ''
        # self.stockName2 = ''

        self.btn_start.setEnabled(True)
        self.lineEdit_1.setEnabled(True)
        self.lineEdit_2.setEnabled(True)
        self.lineEdit_1.setFocus()
        self.m.plotClear()

        self.lbl_name1.setText('')
        self.lbl_name2.setText('')

    def rbtn_setPeriod(self):
        if self.rb_1m.isChecked():
            self.dateEdit.setDate(date.today() + timedelta(days=-30))
            self.dateEdit_2.setDate(date.today())
        elif self.rb_3m.isChecked():
            self.dateEdit.setDate(date.today() + timedelta(days=-30 * 3))
            self.dateEdit_2.setDate(date.today())
        elif self.rb_6m.isChecked():
            self.dateEdit.setDate(date.today() + timedelta(days=-30 * 6))
            self.dateEdit_2.setDate(date.today())
        elif self.rb_1y.isChecked():
            self.dateEdit.setDate(date.today() + timedelta(days=-30 * 12))
            self.dateEdit_2.setDate(date.today())
        elif self.rb_2y.isChecked():
            self.dateEdit.setDate(date.today() + timedelta(days=-30 * 24))
            self.dateEdit_2.setDate(date.today())

    def btnMore1(self):
        self.tw1 = modules.tableWidget.TableWidget(self, LINE_EDIT_1)
        self.tw1.move(self.pos())
        self.tw1.show()

    def btnMore2(self):
        self.tw2 = modules.tableWidget.TableWidget(self, LINE_EDIT_2)
        self.tw2.move(self.pos())
        self.tw2.show()

    def btnResultSave(self):
        temptxt = self.textEdit_info.toPlainText()
        self.save_dialog(temptxt)

    def btnResultClear(self):
        self.textEdit_info.clear()

    def aboutMsg(self):
        QMessageBox.about(self, "about",
                          '''

프로그램명: 금융상품분석툴
제작자: nbvc
e-mail: nbvc1003@gmail.com
내용: 과제 프로젝트 산출물
버전: v 0.1.alpha

        ''')

    def inputData(self, targetStockCode, compStockCode, start, end):
        print('inputData targetStockCode :', targetStockCode, compStockCode)

        # 코드 채크
        if targetStockCode == None:
            targetStockCode = 'AMD'

        if compStockCode == None:
            compStockCode = 'AAPL'

        # 날짜 채크
        if end > date.today():
            end = date.today()
            self.dateEdit_2.setDate(date.today())

        if start > end:
            print("날짜 입력 오류 !!!")
            self.dateEdit.setDate(date.today() + timedelta(days=-30))
            self.dateEdit_2.setDate(date.today())
            return

        self.textEdit_info.append("========================")

        try:
            targetStock_df = data.DataReader(targetStockCode, "yahoo", start, end)  # start ~ end 까지
            compStock_df = data.DataReader(compStockCode, "yahoo", start, end)  # start ~ end 까지

        except RemoteDataError as ede:
            self.textEdit_info.append('애러발생 {}'.format(ede))
            print('애러발생 1{}'.format(ede))
            return
        except Exception as err:
            self.textEdit_info.append('애러발생 {}'.format(err))
            print('애러발생 2{}'.format(err))
            return

        if targetStock_df is None or compStock_df is None:
            self.textEdit_info.append('오류발생 !!!!!')
            return

        tsd = targetStock_df['Close']
        csd = compStock_df['Close']

        # 가져온 데이터의 길이가 다르면
        if tsd.size > csd.size:
            csd = pd.merge(tsd, csd, on='Date', how='outer')
            csd = csd['Close_y']
        elif tsd.size < csd.size:
            tsd = pd.merge(csd, tsd, on='Date', how='outer')
            tsd = tsd['Close_y']

        print("2", tsd.size)
        print("2", csd.size)

        if len(tsd) < 1:
            self.textEdit_info.append(targetStockCode + ' 종목 정보를 가져오지 못했습니다.')
            return
        if len(csd) < 1:
            self.textEdit_info.append(compStockCode + ' 종목 정보를 가져오지 못했습니다.')
            return

        # 데이터 길이가 다를경우 강제로 사이즈 조절
        if len(tsd) > len(csd):
            tsd = tsd[0:len(csd)]

        elif len(tsd) < len(csd):
            csd = csd[0:len(tsd)]

        tsd.fillna(method='bfill')
        csd.fillna(method='bfill')
        tsd.fillna(method='ffill')
        csd.fillna(method='ffill')

        if tsd.isnull().values.any():
            print('tsd isnull')
            tsd = tsd.fillna(tsd.mean())
        if csd.isnull().values.any():
            print('csd isnull')
            csd = csd.fillna(csd.mean())

        corr = tsd.corr(csd)
        targetName = modules.dataLoader.DataLoader.instance().get_NameBySymbol(targetStockCode)
        compName = modules.dataLoader.DataLoader.instance().get_NameBySymbol(compStockCode)
        # codes = '{} / {}'.format(targetStockCode, compStockCode)
        self.textEdit_info.append("종목: {}({})".format(targetName,targetStockCode ))
        self.textEdit_info.append("비교: {}({})".format(compName,compStockCode ))
        # self.textEdit_info.append("심볼: " + codes)
        self.textEdit_info.append('기간:{} ~ {}'.format(start, end))
        self.textEdit_info.append('상관관계 : {}'.format(corr))
        self.m.plot2(tsd, csd)

        title = '{} / {}'.format(targetStockCode, compStockCode)
        self.m.plot1(list(tsd), list(csd), title=title, markup='k.', xlabel=targetStockCode, ylabel=compStockCode,
                     start=start, end=end)

        self.lbl_name1.setText(targetName)
        self.lbl_name2.setText(compName)

    def save_dialog(self, text):
        defualtFileName = './resutText.txt'
        try:
            save_dialog = QFileDialog()
            save_dialog.setAcceptMode(QFileDialog.AcceptSave)
            file_path = save_dialog.getSaveFileName(self,  'Save as... File', defualtFileName,
                                                    filter='All Files(*.*);; Text Files(*.txt)')

            if file_path[0]:
                self.file_path = file_path
                file_open = open(self.file_path[0], 'w')
                self.file_name = (self.file_path[0].split('/'))[-1]
                self.statusBar().showMessage('Saved at: {}'.format(self.file_path[0]))
                self.setWindowTitle("{} - Notepad".format(self.file_name))
                with file_open:
                    file_open.write(text)
                    # self.need_saving(False)

        except FileNotFoundError as why:
            self.error_box(why)
            pass

## PlotCanvas
## matplotlib 와 PyQt5 를 연동하는 클래스
class PlotCanvas(FigureCanvas):
    def __init__(self, parent=None, width=5, height=4, dpi=100):
        fig = Figure(figsize=(width, height), dpi=dpi)
        FigureCanvas.__init__(self, fig)
        self.setParent(parent)
        FigureCanvas.setSizePolicy(self,
                                   QSizePolicy.Expanding,
                                   QSizePolicy.Expanding)
        FigureCanvas.updateGeometry(self)
        self.ax = self.figure.add_subplot(111)

    def plot1(self, *data, markup=None, title='', xlabel='', ylabel='', start=None, end=None):

        if len(data) == 0:
            print('data len is 0 !!!!!!!')
            data = [random.random() for i in range(25)]
        if markup is None:
            markup = 'k.'

        self.ax.plot(data[0], data[1], markup)

        titleplus = title + ' ({} ~ {})'.format(start, end)
        self.ax.set_title(titleplus)
        self.ax.set_xlabel(xlabel)
        self.ax.set_ylabel(ylabel)
        self.draw()

    def plot2(self, xdata, ydata):

        data = np.array([xdata.values, ydata.values])
        # print('data[0]',data[0])

        # data = np.array()
        # print('data',data)
        # data[0] = xdata.values
        # data[1] = ydata.values
        # print(data)
        data = data.T
        m, n = data.shape
        # print(m, n)
        A = np.array([data[:, 0], np.ones(m)]).T
        b = data[:, 1]
        # print("A :", A)
        Q, R = self.qr_householder(A)
        b_hat = Q.T.dot(b)

        R_upper = R[:n, :]
        b_upper = b_hat[:n]

        # print(R_upper, b_upper)

        x = np.linalg.solve(R_upper, b_upper)
        slope, intercept = x

        print(slope, intercept)

        ry = polyval([slope, intercept], xdata)
        self.ax.plot(xdata, ry, 'r-')
        self.draw()

    def plotClear(self):
        self.ax.cla()
        self.draw()

    def qr_householder(self, A):
        m, n = A.shape
        Q = np.eye(m)  # Orthogonal transform so far
        R = A.copy()  # Transformed matrix so far

        for j in range(n):
            # Find H = I - beta*u*u' to put zeros below R[j,j]
            x = R[j:, j]
            normx = np.linalg.norm(x)
            rho = -np.sign(x[0])
            u1 = x[0] - rho * normx
            u = x / u1
            u[0] = 1
            beta = -rho * u1 / normx

            R[j:, :] = R[j:, :] - beta * np.outer(u, u).dot(R[j:, :])
            Q[:, j:] = Q[:, j:] - beta * Q[:, j:].dot(np.outer(u, u))

        return Q, R

app = QApplication([])
ex = Main()
ex.show()
if app.exec_() == 0:
    sys.exit()

reason: import numpy,from numpy,from scipy,import statsmodels


text:
import numpy as np
import pandas
import os

from sklearn import decomposition
#from sklearn import datasets

names = ['ICA1', 'ICA1bis', 'ICA1bis2', 'ICA2', 'ICA2bis','ICA2bis2', 'ICA3', 'ICA3bis', 'ICA3bis2','ICA4', 'ICA4bis', 'ICA4bis2', 'class']
#names = ['ICA1', 'ICA1bis', 'ICA2', 'ICA2bis', 'ICA3', 'ICA3bis','ICA4', 'ICA4bis', 'class']
dataset = pandas.read_csv('.\\csv\\csvMLData.csv', names=names)

array = dataset.values
X = array[:,0:12]
Y = array[:,12]

n_components=4

pca = decomposition.PCA(n_components=n_components)
pca.fit(X)
X = pca.transform(X)

if os.path.exists('.\\csv\\pcaData.csv'):
    os.remove('.\\csv\\pcaData.csv')

with open('.\\csv\\pcaData.csv', 'a+') as csvfile:
    for i in range(0, len(X)):
        info = ''
        for ii in range(0, n_components):
            value = X[i][ii]
            info += "{0:.7}".format(str(value)) + ','
        info+= Y[i] + '\n'
        csvfile.write(info)

ext: py

max_stars_repo_name: N-Lambin/TFE-EEG

max_issues_repo_name: N-Lambin/TFE-EEG

max_forks_repo_name: N-Lambin/TFE-EEG

reason: import numpy


text:
"""

Object shape recognition with L-shape fitting

author: Atsushi Sakai (@Atsushi_twi)

Ref:
- Efficient L-Shape Fitting for Vehicle Detection Using Laser Scanners -
The Robotics Institute Carnegie Mellon University
https://www.ri.cmu.edu/publications/
efficient-l-shape-fitting-for-vehicle-detection-using-laser-scanners/

"""

import matplotlib.pyplot as plt
import numpy as np
import itertools
from enum import Enum
from scipy.spatial.transform import Rotation as Rot

from Mapping.rectangle_fitting.simulator \
    import VehicleSimulator, LidarSimulator

show_animation = True

class LShapeFitting:

    class Criteria(Enum):
        AREA = 1
        CLOSENESS = 2
        VARIANCE = 3

    def __init__(self):
        # Parameters
        self.criteria = self.Criteria.VARIANCE
        self.min_dist_of_closeness_criteria = 0.01  # [m]
        self.d_theta_deg_for_search = 1.0  # [deg]
        self.R0 = 3.0  # [m] range segmentation param
        self.Rd = 0.001  # [m] range segmentation param

    def fitting(self, ox, oy):

        # step1: Adaptive Range Segmentation
        id_sets = self._adoptive_range_segmentation(ox, oy)

        # step2 Rectangle search
        rects = []
        for ids in id_sets:  # for each cluster
            cx = [ox[i] for i in range(len(ox)) if i in ids]
            cy = [oy[i] for i in range(len(oy)) if i in ids]
            rects.append(self._rectangle_search(cx, cy))

        return rects, id_sets

    @staticmethod
    def _calc_area_criterion(c1, c2):
        c1_max = max(c1)
        c2_max = max(c2)
        c1_min = min(c1)
        c2_min = min(c2)

        alpha = -(c1_max - c1_min) * (c2_max - c2_min)

        return alpha

    def _calc_closeness_criterion(self, c1, c2):
        c1_max = max(c1)
        c2_max = max(c2)
        c1_min = min(c1)
        c2_min = min(c2)

        # Vectorization
        D1 = np.minimum(c1_max - c1, c1 - c1_min)
        D2 = np.minimum(c2_max - c2, c2 - c2_min)
        d = np.maximum(np.minimum(D1, D2), self.min_dist_of_closeness_criteria)
        beta = (1.0 / d).sum()

        return beta

    @staticmethod
    def _calc_variance_criterion(c1, c2):
        c1_max = max(c1)
        c2_max = max(c2)
        c1_min = min(c1)
        c2_min = min(c2)

        # Vectorization
        D1 = np.minimum(c1_max - c1, c1 - c1_min)
        D2 = np.minimum(c2_max - c2, c2 - c2_min)
        E1 = D1[D1 < D2]
        E2 = D2[D1 >= D2]
        V1 = - np.var(E1) if len(E1) > 0 else 0.
        V2 = - np.var(E2) if len(E2) > 0 else 0.
        gamma = V1 + V2

        return gamma

    def _rectangle_search(self, x, y):

        X = np.array([x, y]).T

        d_theta = np.deg2rad(self.d_theta_deg_for_search)
        min_cost = (-float('inf'), None)
        for theta in np.arange(0.0, np.pi / 2.0 - d_theta, d_theta):

            rot = Rot.from_euler('z', theta).as_matrix()[0:2, 0:2]
            c = X @ rot
            c1 = c[:, 0]
            c2 = c[:, 1]

            # Select criteria
            cost = 0.0
            if self.criteria == self.Criteria.AREA:
                cost = self._calc_area_criterion(c1, c2)
            elif self.criteria == self.Criteria.CLOSENESS:
                cost = self._calc_closeness_criterion(c1, c2)
            elif self.criteria == self.Criteria.VARIANCE:
                cost = self._calc_variance_criterion(c1, c2)

            if min_cost[0] < cost:
                min_cost = (cost, theta)

        # calc best rectangle
        sin_s = np.sin(min_cost[1])
        cos_s = np.cos(min_cost[1])

        c1_s = X @ np.array([cos_s, sin_s]).T
        c2_s = X @ np.array([-sin_s, cos_s]).T

        rect = RectangleData()
        rect.a[0] = cos_s
        rect.b[0] = sin_s
        rect.c[0] = min(c1_s)
        rect.a[1] = -sin_s
        rect.b[1] = cos_s
        rect.c[1] = min(c2_s)
        rect.a[2] = cos_s
        rect.b[2] = sin_s
        rect.c[2] = max(c1_s)
        rect.a[3] = -sin_s
        rect.b[3] = cos_s
        rect.c[3] = max(c2_s)

        return rect

    def _adoptive_range_segmentation(self, ox, oy):

        # Setup initial cluster
        S = []
        for i, _ in enumerate(ox):
            C = set()
            R = self.R0 + self.Rd * np.linalg.norm([ox[i], oy[i]])
            for j, _ in enumerate(ox):
                d = np.hypot(ox[i] - ox[j], oy[i] - oy[j])
                if d <= R:
                    C.add(j)
            S.append(C)

        # Merge cluster
        while 1:
            no_change = True
            for (c1, c2) in list(itertools.permutations(range(len(S)), 2)):
                if S[c1] & S[c2]:
                    S[c1] = (S[c1] | S.pop(c2))
                    no_change = False
                    break
            if no_change:
                break

        return S

class RectangleData:

    def __init__(self):
        self.a = [None] * 4
        self.b = [None] * 4
        self.c = [None] * 4

        self.rect_c_x = [None] * 5
        self.rect_c_y = [None] * 5

    def plot(self):
        self.calc_rect_contour()
        plt.plot(self.rect_c_x, self.rect_c_y, "-r")

    def calc_rect_contour(self):

        self.rect_c_x[0], self.rect_c_y[0] = self.calc_cross_point(
            self.a[0:2], self.b[0:2], self.c[0:2])
        self.rect_c_x[1], self.rect_c_y[1] = self.calc_cross_point(
            self.a[1:3], self.b[1:3], self.c[1:3])
        self.rect_c_x[2], self.rect_c_y[2] = self.calc_cross_point(
            self.a[2:4], self.b[2:4], self.c[2:4])
        self.rect_c_x[3], self.rect_c_y[3] = self.calc_cross_point(
            [self.a[3], self.a[0]], [self.b[3], self.b[0]], [self.c[3], self.c[0]])
        self.rect_c_x[4], self.rect_c_y[4] = self.rect_c_x[0], self.rect_c_y[0]

    @staticmethod
    def calc_cross_point(a, b, c):
        x = (b[0] * -c[1] - b[1] * -c[0]) / (a[0] * b[1] - a[1] * b[0])
        y = (a[1] * -c[0] - a[0] * -c[1]) / (a[0] * b[1] - a[1] * b[0])
        return x, y

def main():

    # simulation parameters
    sim_time = 30.0  # simulation time
    dt = 0.2  # time tick

    angle_resolution = np.deg2rad(3.0)  # sensor angle resolution

    v1 = VehicleSimulator(-10.0, 0.0, np.deg2rad(90.0),
                          0.0, 50.0 / 3.6, 3.0, 5.0)
    v2 = VehicleSimulator(20.0, 10.0, np.deg2rad(180.0),
                          0.0, 50.0 / 3.6, 4.0, 10.0)

    l_shape_fitting = LShapeFitting()
    lidar_sim = LidarSimulator()

    time = 0.0
    while time <= sim_time:
        time += dt

        v1.update(dt, 0.1, 0.0)
        v2.update(dt, 0.1, -0.05)

        ox, oy = lidar_sim.get_observation_points([v1, v2], angle_resolution)

        rects, id_sets = l_shape_fitting.fitting(ox, oy)

        if show_animation:  # pragma: no cover
            plt.cla()
            # for stopping simulation with the esc key.
            plt.gcf().canvas.mpl_connect(
                'key_release_event',
                lambda event: [exit(0) if event.key == 'escape' else None])
            plt.axis("equal")
            plt.plot(0.0, 0.0, "*r")
            v1.plot()
            v2.plot()

            # Plot range observation
            for ids in id_sets:
                x = [ox[i] for i in range(len(ox)) if i in ids]
                y = [oy[i] for i in range(len(ox)) if i in ids]

                for (ix, iy) in zip(x, y):
                    plt.plot([0.0, ix], [0.0, iy], "-og")

                plt.plot([ox[i] for i in range(len(ox)) if i in ids],
                         [oy[i] for i in range(len(ox)) if i in ids],
                         "o")
            for rect in rects:
                rect.plot()

            plt.pause(0.1)

    print("Done")

if __name__ == '__main__':
    main()

reason: import numpy,from scipy


text:
import numpy as np
from NvTK.Evaluator import *

prediction = np.random.rand((10, 4))

target = np.random.choice(range(4), 10)

target = onehot_encode(target)
assert target.shape == (10, 4)

fpr, tpr, roc_auc = calculate_roc(target, prediction)
visualize_roc_curves(target, prediction)

p, r, ap = calculate_pr(target, prediction)
visualize_pr_curves(target, prediction)

target = np.random.rand((10, 4))
c, p = calculate_correlation(target, prediction, method="pearson")
c, p = calculate_correlation(target, prediction, method="spearman")
c, p = calculate_correlation(target, prediction, method="kendall")

max_stars_repo_name: JiaqiLiZju/NvTK

max_issues_repo_name: JiaqiLiZju/NvTK

max_forks_repo_name: JiaqiLiZju/NvTK


text:
""" Usage:
trained_oie_extractor [--model=MODEL_DIR] --in=INPUT_FILE --out=OUTPUT_FILE [--tokenize] [--conll] [--beam=BEAM]

Options:
    --beam=BEAM  Beam search size [default: 1].

Run a trined OIE model on raw sentences.

MODEL_DIR - Pretrained RNN model folder (containing model.json and pretrained weights).
INPUT FILE - File where each row is a tokenized sentence to be parsed with OIE.
OUTPUT_FILE - File where the OIE tuples will be output.
tokenize - indicates that the input sentences are NOT tokenized.
conll - Print a CoNLL represenation with probabilities

Format of OUTPUT_FILE:
    Sent, prob, pred, arg1, arg2, ...
"""

from rnn.model import load_pretrained_rnn
from docopt import docopt
import logging
import nltk
import re
import numpy as np
from collections import defaultdict

logging.basicConfig(level = logging.INFO)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

class Trained_oie:
    """
    Compose OIE extractions given a pretrained RNN OIE model predicting classes per word
    """
    def __init__(self, model, tokenize):
        """
        model - pretrained supervised model
        tokenize - instance-wide indication whether all of the functions should
                   tokenize their input
        """
        self.model = model
        self.tokenize = tokenize

    def split_words(self, sent):
        """
        Apply tokenization if needed, else just split by space
        sent - string
        """
        return nltk.word_tokenize(sent) if self.tokenize\
            else re.split(r' +', sent) # Allow arbitrary number of spaces

    def get_extractions(self, sent, beam=1):
        """
        Returns a list of OIE extractions for a given sentence
        sent - a list of tokens
        """
        ret = []

        avg_conf = lambda probs: np.average(probs)
        prod_conf = lambda probs: reduce(lambda x, y: x * y, probs) + 0.001

        for ((pred_ind, pred_word), labels) in self.model.predict_sentence_beamsearch(sent, k=beam):
            cur_args = []
            cur_arg = []
            probs = []

            # collect args
            assert len(labels) == len(sent), '#labels should be equal to #tokens in the sentence'
            for i, ((label, prob), word) in enumerate(zip(labels, sent)):
                probs.append(prob)
                # TODO: (1) only focus on argument (2) what if arguments are not in order
                if label.startswith("A"):
                    cur_arg.append((word, i))
                    #probs.append(prob)

                elif cur_arg:
                    cur_args.append(cur_arg)
                    cur_arg = []

            # Create extraction
            if cur_args:
                ret.append(Extraction(sent,
                                      (pred_word, pred_ind),
                                      cur_args,
                                      probs,
                                      calc_prob=avg_conf,
                                  ))
        return ret

    def conll_with_prob(self, sent):
        """
        Returns a conll representation of sentence
        Format:
        word index, word, pred_index, label, probability
        """
        logger.debug("Parsing: {}".format(sent))
        sent = self.split_words(sent)
        ret = ""
        for ((pred_ind, pred_word), labels) in self.model.predict_sentence(sent):
            for (word_ind, ((label, prob), word)) in enumerate(zip(labels, sent)):
                ret+= "\t".join(map(str,
                                         [word_ind, word, pred_ind, label, prob]
                                     )) + '\n'
            ret += '\n'
        return ret

    def parse_sent(self, sent, beam=1):
        """
        Returns a list of extractions for the given sentence
        sent - a tokenized sentence
        tokenize - boolean indicating whether the sentences should be tokenized first
        """
        logger.debug("Parsing: {}".format(sent))
        return self.get_extractions(self.split_words(sent), beam=beam)

    def parse_sents(self, sents):
        """
        Returns a list of extractions per sent in sents.
        sents - list of tokenized sentences
        tokenize - boolean indicating whether the sentences should be tokenized first
        """
        return [self.parse_sent(sent)
                for sent in sents]

class Extraction:
    """
    Store and print an OIE extraction
    """
    def __init__(self, sent, pred, args, probs,
                 calc_prob = lambda probs: 1.0 / (reduce(lambda x, y: x * y, probs) + 0.001)):
        """
        sent - Tokenized sentence - list of strings
        pred - Predicate word
        args - List of arguments (each a tuple <string, position (zero based)>)
        probs - list of float in [0,1] indicating the probability
               of each of the items in the extraction
        calc_prob - function which takes a list of probabilities for each of the
                    items and computes a single probability for the joint occurence of this extraction.
        """
        self.sent = sent
        self.calc_prob = calc_prob
        self.probs = probs
        self.prob = self.calc_prob(self.probs)
        self.pred = pred
        self.args = args
        logger.debug(self)

    def __old_str__(self):
        """
        Format (tab separated):
        Sent, prob, pred, arg1, arg2, ...
        """
        return '\t'.join(map(str,
                             [' '.join(self.sent),
                              self.prob,
                              self.pred,
                              '\t'.join([' '.join(arg)
                                         for arg in self.args])]))
    def __str__(self):
        '''
        store both the word string and the start position in the original sentence.
        '''
        return '\t'.join(map(str,
                             [' '.join(self.sent),
                              self.prob,
                              '{}##{}'.format(*self.pred),
                              '\t'.join([' '.join(map(lambda x: x[0], arg)) + '##' + str(list(map(lambda x: x[1], arg))[0])
                                         for arg in self.args])]))

class Mock_model:
    """
    Load a conll file annotated with labels And probabilities
    and present an external interface of a trained rnn model (through predict_sentence).
    This can be used to alliveate running the trained model.
    """
    def __init__(self, conll_file):
        """
        conll_file - file from which to load the annotations
        """
        self.conll_file = conll_file
        logger.debug("loading file {}".format(self.conll_file))
        self.dic, self.sents = self.load_annots(self.conll_file)
        logger.debug("Done loading file")

    def load_annots(self, conll_file):
        """
        Updates internal state according to file
        for ((pred_ind, pred_word), labels) in self.model.predict_sentence(sent):
                    for (label, prob), word in zip(labels, sent):
        """
        cur_ex = []
        cur_sent = []
        pred_word = ''
        ret = defaultdict(lambda: {})
        sents = []

        # Iterate over lines and populate return dictionary
        for line_ind, line in enumerate(open(conll_file)):
            if not (line_ind % pow(10,5)):
                logger.debug(line_ind)
            line = line.strip()
            if not line:
                if cur_ex:
                    assert(pred_word != '') # sanity check
                    cur_sent = " ".join(cur_sent)

                    # This is because of the dups bug --
                    # doesn't suppose to happen any more
                    ret[cur_sent][pred_word] = (((pred_ind, pred_word), cur_ex),)

                    sents.append(cur_sent)
                    cur_ex = []
                    pred_ind = -1
                    cur_sent = []
            else:
                word_ind, word, pred_ind, label, prob = line.split('\t')
                prob = float(prob)
                word_ind = int(word_ind)
                pred_ind = int(pred_ind)
                cur_sent.append(word)
                if word_ind == pred_ind:
                    pred_word = word
                cur_ex.append((label, prob))
        return (self.flatten_ret_dic(ret, 1),
                list(set(sents)))

    def flatten_ret_dic(self, dic, num_of_dups):
        """
        Given a dictionary of dictionaries, flatten it
        to a dictionary of lists
        """
        ret = defaultdict(lambda: [])
        for sent, preds_dic in dic.iteritems():
            for pred, exs in preds_dic.iteritems():
                ret[sent].extend(exs * num_of_dups)
        return ret

    def predict_sentence(self, sent):
        """
        Return a pre-predicted answer
        """
        return self.dic[" ".join(sent)]

example_sent = "The Economist is an English language weekly magazine format newspaper owned by the Economist Group\
    and edited at offices in London."

if __name__ == "__main__":
    args = docopt(__doc__)
    logger.debug(args)
    model_dir = args["--model"]
    input_fn = args["--in"]
    output_fn = args["--out"]
    tokenize = args["--tokenize"]
    beam = int(args['--beam'])

    if model_dir:
        # If model dir is given, use it to load the model
        model = load_pretrained_rnn(model_dir)
        sents = [line.strip() for line in open(input_fn) if line.strip()]

    else:
        # If no model_dir is given, assume input file already contains annotations and probs
        model = Mock_model(input_fn)
        sents = model.sents

    oie = Trained_oie(model,
                      tokenize = tokenize)

    logger.debug("generating output for {} sentences".format(len(sents)))

    # Iterate over all raw sentences
    if args["--conll"]:
        with open(output_fn, 'w') as fout:
            fout.write('\n\n'.join([oie.conll_with_prob(sent.strip())
                                    for sent in sents]))

    else:
        with open(output_fn, 'w') as fout:
            fout.write('\n'.join([str(ex)
                                  for sent in sents
                                  for ex in oie.parse_sent(sent.strip(), beam=beam)
            ]))

max_stars_repo_name: jzbjyb/supervised-oie

max_issues_repo_name: jzbjyb/supervised-oie

max_forks_repo_name: jzbjyb/supervised-oie


text:
# Copyright (c) OpenMMLab. All rights reserved.
import numpy as np

import mmocr.datasets.pipelines.dbnet_transforms as transforms

def test_imgaug():
    args = [['Fliplr', 0.5],
            dict(cls='Affine', rotate=[-10, 10]), ['Resize', [0.5, 3.0]]]
    imgaug = transforms.ImgAug(args)
    img = np.random.rand(3, 100, 200)
    poly = np.array([[[0, 0, 50, 0, 50, 50, 0, 50]],
                     [[20, 20, 50, 20, 50, 50, 20, 50]]])
    box = np.array([[0, 0, 50, 50], [20, 20, 50, 50]])
    results = dict(img=img, masks=poly, bboxes=box)
    results['mask_fields'] = ['masks']
    results['bbox_fields'] = ['bboxes']
    results = imgaug(results)
    assert np.allclose(results['bboxes'][0],
                       results['masks'].masks[0][0][[0, 1, 4, 5]])
    assert np.allclose(results['bboxes'][1],
                       results['masks'].masks[1][0][[0, 1, 4, 5]])

def test_eastrandomcrop():
    crop = transforms.EastRandomCrop(target_size=(60, 60), max_tries=100)
    img = np.random.rand(3, 100, 200)
    poly = np.array([[[0, 0, 50, 0, 50, 50, 0, 50]],
                     [[20, 20, 50, 20, 50, 50, 20, 50]]])
    box = np.array([[0, 0, 50, 50], [20, 20, 50, 50]])
    results = dict(img=img, gt_masks=poly, bboxes=box)
    results['mask_fields'] = ['gt_masks']
    results['bbox_fields'] = ['bboxes']
    results = crop(results)
    assert np.allclose(results['bboxes'][0],
                       results['gt_masks'].masks[0][0][[0, 2]].flatten())

max_stars_repo_name: yuexy/mmocr

max_issues_repo_name: yuexy/mmocr

max_forks_repo_name: yuexy/mmocr


text:
# -*- encoding:utf-8 -*-
"""封装常用的分析方式及流程模块"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import os

import matplotlib.cbook as cbook
import matplotlib.pyplot as plt
import numpy as np
from scipy import interp
from sklearn import metrics
from sklearn import tree
from sklearn.base import ClusterMixin, clone
from sklearn.metrics import roc_curve, auc

from ..CoreBu.ABuFixes import KFold, learning_curve
from ..UtilBu.ABuDTUtil import warnings_filter
# noinspection PyUnresolvedReferences
from ..CoreBu.ABuFixes import range
from ..UtilBu.ABuFileUtil import file_exist

__author__ = '阿布'
__weixin__ = 'abu_quant'

__all__ = [
           'run_silhouette_cv_estimator',
           'run_prob_cv_estimator',
           'run_cv_estimator',
           'plot_learning_curve',
           'plot_decision_boundary',
           'plot_confusion_matrices',
           'plot_roc_estimator',
           'graphviz_tree',
           'visualize_tree'
           ]

# noinspection PyUnresolvedReferences
def run_silhouette_cv_estimator(estimator, x, n_folds=10):
    """
    只针对kmean的cv验证，使用silhouette_score对聚类后的结果labels_
    进行度量使用silhouette_score，kmean的cv验证只是简单的通过np.random.choice
    进行随机筛选x数据进行聚类的silhouette_score度量，并不涉及训练集测试集
    :param estimator: keman或者支持estimator.labels_, 只通过if not isinstance(estimator, ClusterMixin)进行过滤
    :param x: x特征矩阵
    :param n_folds: int，透传KFold参数，切割训练集测试集参数，默认10
    :return: eg: array([ 0.693 ,  0.652 ,  0.6845,  0.6696,  0.6732,  0.6874,  0.668 ,
                         0.6743,  0.6748,  0.671 ])
    """

    if not isinstance(estimator, ClusterMixin):
        print('estimator must be ClusterMixin')
        return

    silhouette_list = list()
    # eg: n_folds = 10, len(x) = 150 -> 150 * 0.9 = 135
    choice_cnt = int(len(x) * ((n_folds - 1) / n_folds))
    choice_source = np.arange(0, x.shape[0])

    # 所有执行fit的操作使用clone一个新的
    estimator = clone(estimator)
    for _ in np.arange(0, n_folds):
        # 只是简单的通过np.random.choice进行随机筛选x数据
        choice_index = np.random.choice(choice_source, choice_cnt)
        x_choice = x[choice_index]
        estimator.fit(x_choice)
        # 进行聚类的silhouette_score度量
        silhouette_score = metrics.silhouette_score(x_choice, estimator.labels_, metric='euclidean')
        silhouette_list.append(silhouette_score)
    return silhouette_list

def run_prob_cv_estimator(estimator, x, y, n_folds=10):
    """
    通过KFold和参数n_folds拆分训练集和测试集，使用
    np.zeros((len(y), len(np.unique(y))))初始化prob矩阵，
    通过训练estimator.fit(x_train, y_train)后的分类器使用
    predict_proba将y_prob中的对应填数据

    :param estimator: 支持predict_proba的有监督学习, 只通过hasattr(estimator, 'predict_proba')进行过滤
    :param x: 训练集x矩阵，numpy矩阵
    :param y: 训练集y序列，numpy序列
    :param n_folds: int，透传KFold参数，切割训练集测试集参数，默认10
    :return: eg: y_prob
                array([[ 0.8726,  0.1274],
                       [ 0.0925,  0.9075],
                       [ 0.2485,  0.7515],
                       ...,
                       [ 0.3881,  0.6119],
                       [ 0.7472,  0.2528],
                       [ 0.8555,  0.1445]])

    """
    if not hasattr(estimator, 'predict_proba'):
        print('estimator must has predict_proba')
        return

    # 所有执行fit的操作使用clone一个新的
    estimator = clone(estimator)
    kf = KFold(len(y), n_folds=n_folds, shuffle=True)
    y_prob = np.zeros((len(y), len(np.unique(y))))
    """
        根据y序列的数量以及y的label数量构造全是0的矩阵
        eg: y_prob
        array([[ 0.,  0.,  0.],
               [ 0.,  0.,  0.],
               [ 0.,  0.,  0.],
                ..............
               [ 0.,  0.,  0.],
               [ 0.,  0.,  0.],
               [ 0.,  0.,  0.],
    """

    for train_index, test_index in kf:
        x_train, x_test = x[train_index], x[test_index]
        y_train = y[train_index]

        # clf = clone(estimator)
        estimator.fit(x_train, y_train)
        # 使用predict_proba将y_prob中的对应填数据
        y_prob[test_index] = estimator.predict_proba(x_test)

    return y_prob

def run_cv_estimator(estimator, x, y, n_folds=10):
    """
    通过KFold和参数n_folds拆分训练集和测试集，使用
    y.copy()初始化y_pred矩阵，迭代切割好的训练集与测试集，
    不断通过 estimator.predict(x_test)将y_pred中的值逐步替换

    :param estimator: 有监督学习器对象
    :param x: 训练集x矩阵，numpy矩阵
    :param y: 训练集y序列，numpy序列
    :param n_folds: int，透传KFold参数，切割训练集测试集参数，默认10
    :return: y_pred序列
    """
    if not hasattr(estimator, 'predict'):
        print('estimator must has predict')
        return

    # 所有执行fit的操作使用clone一个新的
    estimator = clone(estimator)
    kf = KFold(len(y), n_folds=n_folds, shuffle=True)
    # 首先copy一个一摸一样的y
    y_pred = y.copy()
    """
        eg: y_pred
        array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
               0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
               0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
               1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
               1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
               2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
               2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])
    """

    for train_index, test_index in kf:
        x_train, x_test = x[train_index], x[test_index]
        y_train = y[train_index]
        estimator.fit(x_train, y_train)
        # 通过 estimator.predict(x_test)将y_pred中的值逐步替换
        y_pred[test_index] = estimator.predict(x_test)
    return y_pred

# warnings_filter针对多标签使用OneVsRestClassifier出现的版本警告
@warnings_filter
def plot_learning_curve(estimator, x, y, cv=5, n_jobs=1,
                        train_sizes=np.linspace(.05, 1., 20)):
    """
    绘制学习曲线，train_sizes使用np.linspace(.05, 1., 20)即训练集从5%－100%递进

            np.linspace(.05, 1., 20)
            array([ 0.05,  0.1 ,  0.15,  0.2 ,  0.25,  0.3 ,  0.35,  0.4 ,  0.45,
                    0.5 ,  0.55,  0.6 ,  0.65,  0.7 ,  0.75,  0.8 ,  0.85,  0.9 ,
                    0.95,  1.  ])

    套接sklern中learning_curve函数，传递estimator，cv等参数

    :param estimator: 学习器对象，透传learning_curve
    :param x: 训练集x矩阵，numpy矩阵
    :param y: 训练集y序列，numpy序列
    :param cv: 透传learning_curve，cv参数，默认5，int
    :param n_jobs: 透传learning_curve，并行进程数，默认1，即使用单进程执行
    :param train_sizes: train_sizes使用np.linspace(.05, 1., 20)即训练集从5%－100%递进
    """

    # 套接learning_curve，返回训练集和测试集的score和对应的size
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, x, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
    """
        eg:  train_scores shape = (20, 5)
        array([[ 0.8571,  0.9143,  0.9143,  0.9143,  0.9143],
               [ 0.8169,  0.8732,  0.8732,  0.8732,  0.8732],
               [ 0.8208,  0.8396,  0.8396,  0.8396,  0.8396],
               [ 0.8028,  0.8099,  0.8099,  0.8099,  0.8099],
               [ 0.8146,  0.8202,  0.8146,  0.8146,  0.8146],
               [ 0.8263,  0.8263,  0.8216,  0.8216,  0.8216],
               [ 0.8153,  0.8273,  0.8112,  0.8112,  0.8112],
               [ 0.8063,  0.8169,  0.7993,  0.7993,  0.7993],
               [ 0.8156,  0.8281,  0.8063,  0.8063,  0.8063],
               [ 0.8169,  0.8254,  0.8254,  0.8254,  0.8254],
               [ 0.8184,  0.8235,  0.8261,  0.8312,  0.8312],
               [ 0.815 ,  0.822 ,  0.8197,  0.822 ,  0.822 ],
               [ 0.816 ,  0.8203,  0.8203,  0.8182,  0.8182],
               [ 0.8133,  0.8173,  0.8173,  0.8253,  0.8253],
               [ 0.8109,  0.8127,  0.8146,  0.8202,  0.8221],
               [ 0.8155,  0.819 ,  0.8172,  0.8207,  0.8225],
               [ 0.8149,  0.8248,  0.8231,  0.8248,  0.8198],
               [ 0.8187,  0.8281,  0.825 ,  0.8328,  0.8219],
               [ 0.8254,  0.8299,  0.8284,  0.8343,  0.8166],
               [ 0.8272,  0.8315,  0.8301,  0.8343,  0.8174]])
    """
    train_scores_mean = np.mean(train_scores, axis=1)
    """
        eg: train_scores_mean
            array([ 0.9029,  0.862 ,  0.8358,  0.8085,  0.8157,  0.8235,  0.8153,
            0.8042,  0.8125,  0.8237,  0.8261,  0.8201,  0.8186,  0.8197,
            0.8161,  0.819 ,  0.8215,  0.8253,  0.8269,  0.8281])
    """
    train_scores_std = np.std(train_scores, axis=1)

    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    """
         eg: test_scores_std
            array([ 0.0751,  0.0607,  0.0314,  0.0059,  0.0047,  0.0066,  0.0074,
                    0.0051,  0.0107,  0.0115,  0.0107,  0.012 ,  0.0142,  0.018 ,
                    0.0134,  0.0167,  0.0167,  0.0127,  0.0128,  0.0113])
    """
    # 开始可视化学习曲线
    plt.figure()
    plt.title('learning curve')
    plt.xlabel("train sizes")
    plt.ylabel("scores")
    plt.gca().invert_yaxis()
    plt.grid()
    # 对train_scores的均值和方差区域进行填充
    plt.fill_between(train_sizes, train_scores_mean - train_scores_std, train_scores_mean + train_scores_std,
                     alpha=0.1, color="g")
    # 对test_scores的均值和方差区域进行填充
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std, test_scores_mean + test_scores_std,
                     alpha=0.1, color="r")
    # 把train_scores_mean标注圆圈
    plt.plot(train_sizes, train_scores_mean, 'o-', color="g", label="train scores")
    # 把ttest_scores_mean标注圆圈
    plt.plot(train_sizes, test_scores_mean, 'o-', color="r", label="test scores")
    plt.legend(loc="best")
    plt.draw()
    plt.gca().invert_yaxis()
    plt.show()

def graphviz_tree(estimator, features, x, y):
    """
    绘制决策树或者core基于树的分类回归算法的决策示意图绘制，查看
    学习器本身hasattr(fiter, 'tree_')是否有tree_属性，内部clone(estimator)学习器
    后再进行训练操作，完成训练后使用sklearn中tree.export_graphvizd导出graphviz.dot文件
    需要使用第三方dot工具将graphviz.dot进行转换graphviz.png，即内部实行使用
    运行命令行：
                os.system("dot -T png graphviz.dot -o graphviz.png")
    最后读取决策示意图显示

    :param estimator: 学习器对象，透传learning_curve
    :param x: 训练集x矩阵，numpy矩阵
    :param y: 训练集y序列，numpy序列
    :param features: 训练集x矩阵列特征所队员的名称，可迭代序列对象
    """
    if not hasattr(estimator, 'tree_'):
        logging.info('only tree can graphviz!')
        return

    # 所有执行fit的操作使用clone一个新的
    estimator = clone(estimator)
    estimator.fit(x, y)
    # TODO out_file path放倒cache中
    tree.export_graphviz(estimator.tree_, out_file='graphviz.dot', feature_names=features)
    os.system("dot -T png graphviz.dot -o graphviz.png")

    '''
        !open $path
        要是方便用notebook直接open其实显示效果好，plt，show的大小不好调整
    '''
    graphviz = os.path.join(os.path.abspath('.'), 'graphviz.png')

    # path = graphviz
    # !open $path
    if not file_exist(graphviz):
        logging.info('{} not exist! please install dot util!'.format(graphviz))
        return

    image_file = cbook.get_sample_data(graphviz)
    image = plt.imread(image_file)
    image_file.close()
    plt.imshow(image)
    plt.axis('off')  # clear x- and y-axes
    plt.show()

def visualize_tree(estimator, x, y, boundaries=True):
    """
    需要x矩阵特征列只有两个维度，根据x，y，通过meshgrid构造训练集平面特征
    通过z = estimator.predict(np.c_[xx.ravel(), yy.ravel()])对特征平面
    进行predict生成z轴，可视化meshgrid构造训练集平面特征使用生成的z生成
    pcolormesh进行可视化

    :param estimator: 学习器对象，内部clone(estimator)
    :param x: 训练集x矩阵，numpy矩阵，需要特征列只有两个维度
    :param y: 训练集y序列，numpy序列
    :param boundaries: 是否绘制决策边界

    """
    if x.shape[1] != 2:
        logging.info('be sure x shape[1] == 2!')
        return

    # 所有执行fit的操作使用clone一个新的
    estimator = clone(estimator)
    estimator.fit(x, y)

    xlim = (x[:, 0].min() - 0.1, x[:, 0].max() + 0.1)
    ylim = (x[:, 1].min() - 0.1, x[:, 1].max() + 0.1)
    x_min, x_max = xlim
    y_min, y_max = ylim
    # 通过训练集中x的min和max，y的min，max构成meshgrid
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, 100),
                         np.linspace(y_min, y_max, 100))
    # 摊平xx，yy进行z轴的predict
    z = estimator.predict(np.c_[xx.ravel(), yy.ravel()])
    # z的shape跟随xx
    z = z.reshape(xx.shape)
    plt.figure()
    plt.pcolormesh(xx, yy, z, alpha=0.2, cmap='rainbow')
    plt.clim(y.min(), y.max())

    # 将之前的训练集中的两个特征进行scatter绘制，颜色使用y做区分
    plt.scatter(x[:, 0], x[:, 1], c=y, s=50, cmap='rainbow')
    plt.axis('off')

    plt.xlim(x_min, x_max)
    plt.ylim(y_min, y_max)
    plt.clim(y.min(), y.max())

    def plot_boundaries(i, p_xlim, p_ylim):
        """
        针对有tree_属性的学习器绘制决策边界
        :param i: 内部递归调用使用ree_inner.children_left[i]和tree_inner.children_right[i]
        :param p_xlim: 原始参数使用plt.xlim()
        :param p_ylim: 原始参数使用plt.ylim()
        """
        if i < 0:
            return
        # 拿到tree_使用plot_boundaries继续递归绘制
        tree_inner = estimator.tree_

        if tree_inner.feature[i] == 0:
            # 绘制0的边界
            plt.plot([tree_inner.threshold[i], tree_inner.threshold[i]], p_ylim, '-k')
            # 即x轴固定p_ylim，xlim=[p_xlim[0], tree_inner.threshold[i]], [tree_inner.threshold[i], p_xlim[1]]
            plot_boundaries(tree_inner.children_left[i],
                            [p_xlim[0], tree_inner.threshold[i]], p_ylim)
            plot_boundaries(tree_inner.children_right[i],
                            [tree_inner.threshold[i], p_xlim[1]], p_ylim)
        elif tree_inner.feature[i] == 1:
            # 绘制1的边界
            plt.plot(p_xlim, [tree_inner.threshold[i], tree_inner.threshold[i]], '-k')
            # 即y轴固定p_xlim，ylim=[p_ylim[0], tree_inner.threshold[i]], [tree_inner.threshold[i], p_ylim[1]]
            plot_boundaries(tree_inner.children_left[i], p_xlim,
                            [p_ylim[0], tree_inner.threshold[i]])
            plot_boundaries(tree_inner.children_right[i], p_xlim,
                            [tree_inner.threshold[i], p_ylim[1]])

    if boundaries and hasattr(estimator, 'tree_'):
        # 简单决策树才去画决策边界
        plot_boundaries(0, plt.xlim(), plt.ylim())

def plot_decision_boundary(pred_func, x, y):
    """
    通过x，y以构建meshgrid平面区域，要x矩阵特征列只有两个维度，在区域中使用外部传递的
    pred_func函数进行z轴的predict，通过contourf绘制特征平面区域，最后使用
    plt.scatter(x[:, 0], x[:, 1], c=y, cmap=plt.cm.Spectral)在平面区域上填充原始特征
    点

    :param pred_func: callable函数，eg：pred_func: lambda p_x: fiter.predict(p_x), x, y
    :param x: 训练集x矩阵，numpy矩阵，需要特征列只有两个维度
    :param y: 训练集y序列，numpy序列
    """
    xlim = (x[:, 0].min() - 0.1, x[:, 0].max() + 0.1)
    ylim = (x[:, 1].min() - 0.1, x[:, 1].max() + 0.1)
    x_min, x_max = xlim
    y_min, y_max = ylim
    # 通过训练集中x的min和max，y的min，max构成meshgrid
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, 200),
                         np.linspace(y_min, y_max, 200))
    # 摊平xx，yy进行z轴的predict, pred_func: lambda p_x: fiter.predict(p_x), x, y
    z = pred_func(np.c_[xx.ravel(), yy.ravel()])
    # z的shape跟随xx
    z = z.reshape(xx.shape)
    # 使用contourf绘制xx, yy, z，即特征平面区域以及z的颜色区别
    # noinspection PyUnresolvedReferences
    plt.contourf(xx, yy, z, cmap=plt.cm.Spectral)
    # noinspection PyUnresolvedReferences
    # 在特征区域的基础上将原始，两个维度使用scatter绘制以y为颜色的点
    plt.scatter(x[:, 0], x[:, 1], c=y, cmap=plt.cm.Spectral)
    plt.show()

def plot_roc_estimator(estimator, x, y, pos_label=None):
    """
    固定n_folds=10通过kf = KFold(len(y), n_folds=10, shuffle=True)拆分
    训练测试集，使用estimator.predict_proba对测试集数据进行概率统计，直接使用
    sklearn中的roc_curve分别对多组测试集计算fpr, tpr, thresholds，并计算roc_auc
    最后绘制roc_auc曲线进行可视化操作

    :param estimator: 分类器对象，内部clone(estimator)
    :param x: 训练集x矩阵，numpy矩阵
    :param y: 训练集y序列，numpy序列
    :param pos_label: 对y大于2个label的数据，roc_curve需要指定pos_label，如果不指定，默认使用y的第一个label值
    """

    if not hasattr(estimator, 'predict_proba'):
        # 分类器必须要有predict_proba方法
        logging.info('estimator must has predict_proba!')
        return

    # 所有执行fit的操作使用clone一个新的
    estimator = clone(estimator)
    estimator.fit(x, y)

    # eg: y_unique = [0, 1]
    y_unique = np.unique(y)
    kf = KFold(len(y), n_folds=10, shuffle=True)
    y_prob = np.zeros((len(y), len(y_unique)))
    """
        eg:  y_prob
            array([[ 0.,  0.],
                   [ 0.,  0.],
                   [ 0.,  0.],
                   ...,
                   [ 0.,  0.],
                   [ 0.,  0.],
                   [ 0.,  0.]])
    """
    mean_tpr = 0.0
    # 0-1分布100个
    mean_fpr = np.linspace(0, 1, 100)
    for i, (train_index, test_index) in enumerate(kf):
        x_train, x_test = x[train_index], x[test_index]
        y_train = y[train_index]
        estimator.fit(x_train, y_train)
        y_prob[test_index] = estimator.predict_proba(x_test)
        """
            eg: y_prob[test_index]
            array([[ 0.8358,  0.1642],
                   [ 0.4442,  0.5558],
                   [ 0.1351,  0.8649],
                   [ 0.8567,  0.1433],
                   [ 0.6953,  0.3047],
                   ..................
                   [ 0.1877,  0.8123],
                   [ 0.8465,  0.1535],
                   [ 0.1916,  0.8084],
                   [ 0.8421,  0.1579]])

        """
        if len(y_unique) != 2 and pos_label is None:
            # 对y大于2个label的数据，roc_curve需要指定pos_label，如果不指定，默认使用y的第一个label值
            pos_label = y_unique[0]
            logging.info('y label count > 2 and param pos_label is None, so choice y_unique[0]={} for pos_label!'.
                         format(pos_label))

        fpr, tpr, thresholds = roc_curve(y[test_index], y_prob[test_index, 1], pos_label=pos_label)
        """
            eg:
                fpr
                array([ 0.    ,  0.0169,  0.0169,  0.0339,  0.0339,  0.0508,  0.0508,
                        0.0847,  0.0847,  0.1017,  0.1017,  0.1186,  0.1186,  0.2034,
                        0.2034,  0.2542,  0.2542,  0.5254,  0.5254,  0.5763,  0.5763,  1.    ])
                tpr
                array([ 0.0323,  0.0323,  0.4839,  0.4839,  0.5484,  0.5484,  0.6452,
                        0.6452,  0.7419,  0.7419,  0.7742,  0.7742,  0.8387,  0.8387,
                        0.9032,  0.9032,  0.9355,  0.9355,  0.9677,  0.9677,  1.    ,  1.    ])

                thresholds
                array([ 0.9442,  0.9288,  0.8266,  0.8257,  0.8123,  0.8122,  0.8032,
                        0.7647,  0.7039,  0.5696,  0.5558,  0.4854,  0.4538,  0.2632,
                        0.2153,  0.2012,  0.1902,  0.1616,  0.1605,  0.1579,  0.1561,
                        0.1301])
        """

        # interp线性插值计算
        mean_tpr += interp(mean_fpr, fpr, tpr)
        # 把第一个值固定0，最后会使用mean_tpr[-1] = 1.0把最后一个固定1.0
        mean_tpr[0] = 0.0
        # 直接使用 sklearn中的metrics.auc计算
        roc_auc = auc(fpr, tpr)
        plt.plot(fpr, tpr, lw=1, label='ROC fold %d (area = %0.2f)' % (i, roc_auc))

    mean_tpr /= len(kf)
    # 最后一个固定1.0
    mean_tpr[-1] = 1.0
    mean_auc = auc(mean_fpr, mean_tpr)
    plt.plot(mean_fpr, mean_tpr, 'k--', label='Mean ROC (area = %0.2f)' % mean_auc, lw=2)

    plt.plot([0, 1], [0, 1], '--', color=(0.6, 0.6, 0.6), label='Random')
    plt.xlim([-0.05, 1.05])
    plt.ylim([-0.05, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic')
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.show()

def plot_confusion_matrices(estimator, x, y, n_folds=10):
    """
    套接run_cv_estimator进行通过参数n_folds进行训练集测试集拆封
    使用y_pred和y做为参数，透传给metrics.confusion_matrix函数
    进行混淆矩阵的计算，通过ax.matshow可视化混淆矩阵

    :param estimator: 分类器对象，内部clone(estimator)
    :param x: 训练集x矩阵，numpy矩阵
    :param y: 训练集y序列，numpy序列
    :param n_folds: 透传KFold参数，切割训练集测试集参数
    """
    y_pred = run_cv_estimator(estimator, x, y, n_folds=n_folds)
    """
        eg: y_pred
        array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
           0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
           1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1,
           1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2,
           2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
           2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])
    """
    y_unique = np.unique(y)

    confusion_matrix = metrics.confusion_matrix(y, y_pred)
    """
        eg: confusion_matrix
         [[50  0  0]
         [ 0 47  3]
         [ 0  1 49]]
    """
    logging.info(confusion_matrix)

    fig = plt.figure()
    # 颜色条的颜色数量设置使用len(y_unique) * len(y_unique)，即如果y是3个label->9颜色。2->4
    cmap = plt.get_cmap('jet', len(y_unique) * len(y_unique))
    cmap.set_under('gray')
    ax = fig.add_subplot(111)
    # ax.matshow可视化化混淆矩阵
    cax = ax.matshow(confusion_matrix, cmap=cmap,
                     vmin=confusion_matrix.min(),
                     vmax=confusion_matrix.max())
    plt.title('Confusion matrix for %s' % estimator.__class__.__name__)
    # 辅助颜色边bar显示
    fig.colorbar(cax)
    # noinspection PyTypeChecker
    ax.set_xticklabels('x: '.format(y_unique))
    # noinspection PyTypeChecker
    ax.set_yticklabels('y: '.format(y_unique))
    plt.xlabel('Predicted')
    plt.ylabel('True')
    plt.show()

max_stars_repo_name: luqin/firefly

max_issues_repo_name: momantang/cobrass

max_forks_repo_name: luqin/firefly


text:
import numpy as np
from scipy.stats import levy_stable, norm
from scipy.signal import convolve2d
import matplotlib.pyplot as plt

def moving_average(x, w):
    if len(x.shape) == 1:
        return np.convolve(x, np.ones(w), 'valid') / w
    elif len(x.shape) == 2:
        return convolve2d(x,np.ones([w, len(x[0, :])]))/w

def dist(i, j):
    return np.sqrt(i ** 2 + j ** 2)

def normalize(vec, idx="All"):
    if idx=="All":
        return vec/np.max(np.abs(vec))
    else:
        return vec/vec[idx]

def integer_difference(vec, t, n=1):
    if len(vec) != len(t):
        raise IndexError("lengths of vector and index must match")
    dvec = vec
    for i in range(0, n):
        dvec = np.diff(np.append([dvec[0]], dvec))/abs(t[1]-t[0])
    return dvec

def shift(arr, num, fill_value=0):
    # https://www.delftstack.com/howto/numpy/python-numpy-shift-array/
    result = np.empty_like(arr)
    if num > 0:
        result[:num] = fill_value
        result[num:] = arr[:-num]
    elif num < 0:
        result[num:] = fill_value
        result[:num] = arr[-num:]
    else:
        result[:] = arr
    return result

def autocorr(x, lag):
    lag = int(lag)
    return np.corrcoef(np.array([x[:-lag], x[lag:]]))


text:
"""

Mask R-CNN
Train on my occ5000 dataset

Copyright (c) 2018 Matterport, Inc.
Licensed under the MIT License (see LICENSE for details)
Written by Kidd Liu

"""

import os
import sys
import datetime
import numpy as np
import skimage.draw

# Root directory of the projec
ROOT_DIR = os.getcwd()
if ROOT_DIR.endswith("samples/occ5000"):
    # Go up two levels to the repo root
    ROOT_DIR = os.path.dirname(os.path.dirname(ROOT_DIR))

# Import Mask RCNN
sys.path.append(ROOT_DIR)
from mrcnn.config import Config
from mrcnn import utils
import mrcnn.model as modellib

# Path to trained weights file
COCO_WEIGHTS_PATH = os.path.join(ROOT_DIR, 'mask_rcnn_coco.h5')

# Directory to save logs and model checkpoints, if not provided
# through the command line argument --logs
DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs")

############################################################
#  Configurations
############################################################

class Occ5000Config(Config):
    """
    Configuration for training on the occ5000 dataset.
    Derives from the base Config class and overrides some values.
    """
    # Give the configuration a recognizable name
    NAME = "occ5000"

    # We use a GPU with 12GB memory, which can fit two images.
    # Adjust down if you use a smaller GPU.
    # The default image size is 1024x1024px.
    IMAGES_PER_GPU = 2

    # Number of classes (including background)
    # Background+hair+face+upper closes+left arm
    # +right arm+left hand +right hand+ left leg
    # +right leg+ left feet+ right feet+ accesessory
    NUM_CLASSES = 12 + 1

    # Number of training steps per epoch
    STEPS_PER_EPOCH = 1250

    # Class names and accessory can be ignored
    CLASS_NAMES = ['hair', 'face', 'upper_clothes', 'left_arm',
             'right_arm', 'left_hand', 'right_hand',
             'left_leg', 'right_leg', 'left_feet', 'right_feet',
             'accessory']

    IMAGE_MIN_DIM = 512

    IMAGE_MAX_DIM = 1024

############################################################
#  Dataset
############################################################

class Occ5000Dataset(utils.Dataset):

    def load_occ5000(self, dataset_dir, subset):
        """
        Load a subset of the Balloon dataset.
        dataset_dir: The root directory of the COCO dataset(like /home/kidd/kidd1/occ5000)
        subset: What to load (train_all2500, val_all2500, see in occ5000/list)

        TODO: resize the train val to 4000(3200occ + 800unOcc) and 1000(800occ+200unocc)
              which is 80% of the whole dataset
        """

        # Add classes, can ignore some class here
        classNames = Occ5000Config.CLASS_NAMES
        for className, i in zip(classNames, np.arange(len(Occ5000Config.CLASS_NAMES))):
            self.add_class('occ5000', i+1, Occ5000Config.CLASS_NAMES[i])

        # Train or validation datset
        assert subset in ['train_all2500', 'val_all2500']
        #dateset_dir = os.path.join(dataset_dir, sub)

        # Get image and annotation path in the format 'impath annpath'
        lines = []
        with open(os.path.join(dataset_dir, 'list', subset+'.txt'), 'r') as list_file:
            while True:
                line = list_file.readline()
                if not line:
                    break
                line = line.strip('\n')
                lines.append(line)

        # Seperate image and annotation path
        im_paths = []
        ann_paths = []
        for line in lines:
            im_path = line[0:line.find('.png')+4]
            ann_path = line[line.find('.png')+5:]
            im_paths.append(im_path)
            ann_paths.append(ann_path)

        # Read image and annotation from png and add images
        for im_path, ann_path in zip(im_paths, ann_paths):
            im = skimage.io.imread(dataset_dir + im_path)
            height, width = im.shape[:2]
            _, image_id = os.path.split(im_path)
            path = dataset_dir + im_path
            ann = skimage.io.imread(dataset_dir + ann_path)

            self.add_image('occ5000',
                          image_id = image_id,
                          path = path,
                          width = width, height = height,
                          annotation = ann)

    def load_mask(self, image_id):
        """
        Generate instance masks for an image.
        Returns:
        masks: A bool array of shape [height, width, instance count] with
            one mask per instance.
        class_ids: a 1D array of class IDs of the instance masks.
        """
        # If not a balloon dataset image, delegate to parent class.
        info = self.image_info[image_id]
        if info["source"] != 'occ5000':
            return super(self.__class__, self).load_mask(image_id)

        # Build mask of shape [height, width, instance_count] and list
        # of class IDs that correspond to each channel of the mask.

        masks = []
        classID = []
        ann = info['annotation']
        for i in np.arange(1, len(Occ5000Config.CLASS_NAMES) + 1):
            mask = np.where(ann == i,
                            np.ones(ann.shape, dtype=np.uint8),
                           np.zeros(ann.shape, dtype=np.uint8))
            if np.sum(mask) != 0:
                masks.append(mask)
                classID.append(i)
        masks = np.stack(masks, axis = 2)
        #classID = np.arange(1,len(Occ5000Config.CLASS_NAMES) + 1)

        return masks, np.array(classID).astype(np.int32)

    def image_reference(self, image_id):
        """Return the path of the image."""
        info = self.image_info[image_id]
        if info['source'] == 'occ5000':
            return info['path']
        else:
            super(self.__class__, self).image_reference(image_id)

############################################################
#  Training
############################################################
def train(model):
    """Train the model."""
    # Training dataset.
    dataset_train = Occ5000Dataset()
    dataset_train.load_occ5000(args.dataset, "train_all2500")
    dataset_train.prepare()

    # Validation dataset
    dataset_val = Occ5000Dataset()
    dataset_val.load_occ5000(args.dataset, "val_all2500")
    dataset_val.prepare()

    # *** This training schedule is an example. Update to your needs ***
    # Since we're using a very small dataset, and starting from
    # COCO trained weights, we don't need to train too long. Also,
    # no need to train all layers, just the heads should do it.
    print("Training network heads")
    model.train(dataset_train, dataset_val,
                learning_rate=config.LEARNING_RATE,
                epochs=30,
                layers='heads')

############################################################
#  Evaluation
###########################################################
#def evaluate_occ5000(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None):

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(
    description='Train Mask R-CNN on occ5000.')

    parser.add_argument('command',
                        metavar="<command>",
                        help="'train'")
    parser.add_argument('--dataset', required=False,
                        metavar='/path/to/occ5000',
                        help='Directory of the occ5000 dataset')
    parser.add_argument('--weights', required=True,
                        metavar="/path/to/weights.h5",
                        help="Path to weights .h5 file or 'coco'")
    parser.add_argument('--logs', required=False,
                        default=DEFAULT_LOGS_DIR,
                        metavar="/path/to/logs/",
                        help='Logs and checkpoints directory (default=logs/)')
    parser.add_argument('--image', required=False,
                        metavar="path or URL to image",
                        help='Image to apply the color splash effect on')

    args = parser.parse_args()

    if args.command == "train":
        assert args.dataset, "Argument --dataset is required for training"
    print("Weights: ", args.weights)
    print("Dataset: ", args.dataset)
    print("Logs: ", args.logs)

    # Configurations
    if args.command == "train":
        config = Occ5000Config()
    else:
        class InferenceConfig(BalloonConfig):
            # Set batch size to 1 since we'll be running inference on
            # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
            GPU_COUNT = 1
            IMAGES_PER_GPU = 1
        config = InferenceConfig()
    config.display()

    # Creat model
    if args.command == "train":
        model = modellib.MaskRCNN(mode="training", config=config,
                                  model_dir=args.logs)
    else:
        model = modellib.MaskRCNN(mode="inference", config=config,
                                  model_dir=args.logs)

    # Select weights file to load
    if args.weights.lower() == "coco":
        weights_path = COCO_WEIGHTS_PATH
        # Download weights file
        if not os.path.exists(weights_path):
            utils.download_trained_weights(weights_path)
    elif args.weights.lower() == "last":
        # Find last trained weights
        weights_path = model.find_last()
    elif args.weights.lower() == "imagenet":
        # Start from ImageNet trained weights
        weights_path = model.get_imagenet_weights()
    else:
        weights_path = args.weights

    # Load weights
    print("Loading weights ", weights_path)
    if args.weights.lower() == "coco":
        # Exclude the last layers because they require a matching
        # number of classes
        model.load_weights(weights_path, by_name=True, exclude=[
            "mrcnn_class_logits", "mrcnn_bbox_fc",
            "mrcnn_bbox", "mrcnn_mask"])
    else:
        model.load_weights(weights_path, by_name=True)

    # Train or evaluate
    if args.command == "train":
        train(model)
    else:
        print("'{}' is not recognized. "
              "Use 'train'".format(args.command))


text:
# --------------
import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
# code starts here

# Loading dataset
df =pd.read_csv(path)

print(df.head())
#print(df.columns)

X =df[['ages', 'num_reviews', 'piece_count', 'play_star_rating',
       'review_difficulty', 'star_rating', 'theme_name', 'val_star_rating',
       'country']]
y =df['list_price']

X_train,X_test,y_train,y_test =train_test_split(X, y, test_size =0.3, random_state = 6)
# code ends here

# --------------
import matplotlib.pyplot as plt

# code starts here
cols =X_train.columns

fig,axes =plt.subplots(3,3, sharey =True)

axes =axes.flatten()

df.plot( kind ="scatter",x='ages', y =1, ax =axes[0], figsize =(18,18))
df.plot( kind ="scatter",x='num_reviews', y =1,  ax =axes[1])
df.plot( kind ="scatter",x='piece_count', y =1, ax =axes[2])
df.plot( kind ="scatter",x='play_star_rating', y =1, ax =axes[3])
df.plot( kind ="scatter",x='review_difficulty', y =1 , ax =axes[4])
df.plot( kind ="scatter",x='star_rating', y =1, ax =axes[5])
df.plot( kind ="scatter",x='theme_name', y =1, ax =axes[6])
df.plot( kind ="scatter",x='val_star_rating', y =1, ax =axes[7])
df.plot( kind ="scatter",x='country', y =1, ax =axes[8])
# code ends here

# --------------
# Code starts here

# Calculating correlation between all features
corr =X_train.corr()
print(corr)

# Dropping features with correlation > (+/-)0.75
X_train.drop(['play_star_rating','val_star_rating'], axis=1, inplace=True)

X_test.drop(['play_star_rating','val_star_rating'], axis=1, inplace=True)

# Code ends here

# --------------
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score

# Code starts here

# Model creation
regressor =LinearRegression()

# Fitting the model
regressor.fit(X_train, y_train)

# Predicting on X_test
y_pred =regressor.predict(X_test)

# Calculating mean_sqaured and r-sqaured values
mse =mean_squared_error(y_test, y_pred)
print("Mean sqaured Erro is {}".format(mse))

r2 =r2_score(y_test, y_pred)
print("R-sqaured value is {}".format(r2))

# Code ends here

# --------------
# Code starts here

residual =y_test - y_pred

residual.plot(kind ="hist")

# Code ends here


text:
# coding: utf-8
"""
Data Provider: Build snapshots through a python function
========================================================

This provider builds its data using an external python function.
"""
import os
import sys
import logging
import importlib
import numpy as np
from .sample_cache import SampleCache
from ..space import Sample

class ProviderFunction:
    """Provides Snapshots built through an external python function."""

    logger = logging.getLogger(__name__)

    def __init__(self, plabels, flabels, module, function,
                 psizes=None, fsizes=None,
                 discover_pattern=None, save_dir=None,
                 space_fname='sample-space.json',
                 space_format='json',
                 data_fname='sample-data.json',
                 data_format='json'):
        """Initialize the provider.

        Load a python function to be called for computing new snpahots.

        :param list(str) plabels: input parameter names.
        :param list(str) flabels: output feature names.
        :param str module: python module to load.
        :param str function: function in `module` to execute for generating data.
        :param list(int) psizes: number of components of parameters.
        :param list(int) fsizes: number of components of output features.
        :param str discover_pattern: UNIX-style patterns for directories
          with pairs of sample files to import.
        :param str save_dir: path to a directory for saving known snapshots.
        :param str space_fname: name of space file to write.
        :param str data_fname: name of data file to write.
        :param str space_format: space file format.
        :param str data_format: data file format.
        """
        # import python function
        sys.path.append(os.path.abspath('.'))
        plugin = importlib.import_module(module)
        self._function = getattr(plugin, function)

        # discover existing snapshots
        self._cache = SampleCache(plabels, flabels, psizes, fsizes, save_dir,
                                  space_fname, space_format,
                                  data_fname, data_format)
        if discover_pattern:
            self._cache.discover(discover_pattern)
            self._cache.save()

    @property
    def plabels(self):
        """Names of space parameters."""
        return self._cache.plabels

    @property
    def flabels(self):
        """Names of data features."""
        return self._cache.flabels

    @property
    def psizes(self):
        """Shape of space parameters."""
        return self._cache.psizes

    @property
    def fsizes(self):
        """Shape of data features."""
        return self._cache.fsizes

    @property
    def known_points(self):
        """List of points whose associated data is already known."""
        return self._cache.space

    def require_data(self, points):
        """Return samples for requested points.

        Data for unknown points is generated through a python function.

        :param array_like points: points to build data from,
          (n_points, n_features).
        :return: samples for requested points (carry both space and data)
        :rtype: :class:`Sample`
        """
        # locate results in cache
        idx = self._cache.locate(points)

        # build missing results
        new_points = points[idx >= len(self._cache)]
        if len(new_points) > 0:
            self._cache += self.build_data(new_points)
            self._cache.save()

        return self._cache[idx]

    def build_data(self, points):
        """Compute data for requested points.

        :param array_like points: points to build data from,
          (n_points, n_features).
        :return: samples for requested points (carry both space and data).
        :rtype: :class:`Sample`
        """
        self.logger.debug("Build Snapshot for points {}".format(points))
        points = np.atleast_2d(points)
        result = [self._function(point) for point in points]
        sample = Sample(space=points, data=result, plabels=self.plabels,
                        flabels=self.flabels, psizes=self.psizes, fsizes=self.fsizes)
        return sample

max_stars_repo_name: tupui/batman

max_issues_repo_name: tupui/batman

max_forks_repo_name: tupui/batman


text:
#%%

import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D,Dense,MaxPool2D,Dropout,Flatten
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from shutil import copyfile
import zipfile
import os
import random
physical_devices = tf.config.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)

#%%
path = './Datasets/cats-and-dogs.zip'
zip = zipfile.ZipFile(path,'r')
zip.extractall("./DataSets/cats-dog")
zip.close()
#%%
print(len(os.listdir('./DataSets/cats-dog/PetImages/Cat')))
print(len(os.listdir('./DataSets/cats-dog/PetImages/Dog')))
#%%
try:
    os.mkdir('./DataSets/cats-dog/CatvDog/')
    os.mkdir('./DataSets/cats-dog/CatvDog/training/')
    os.mkdir('./DataSets/cats-dog/CatvDog/testing/')
    os.mkdir('./DataSets/cats-dog/CatvDog/training/cats/')
    os.mkdir('./DataSets/cats-dog/CatvDog/training/dogs/')
    os.mkdir('./DataSets/cats-dog/CatvDog/testing/cats/')
    os.mkdir('./DataSets/cats-dog/CatvDog/testing/dogs/')

except OSError:
    pass
#%%
def split_data(SOURCE, TRAINING, TESTING, SPILT_SIZE):
    filesname = os.listdir(SOURCE)
    files=[]
    for f in filesname:
        if os.path.getsize(SOURCE+f)!=0:
            files.append(f)

    n= len(files)
    shuffle_list = random.sample(files,n)
    up_limit = int(n*SPILT_SIZE)
    training_list = shuffle_list[:up_limit]
    testing_list = shuffle_list[up_limit:]

    for f in training_list:
        copyfile(SOURCE+f,TRAINING+f)
    for f in testing_list:
        copyfile(SOURCE+f,TESTING+f)

cats_source_dir ='./DataSets/cats-dog/PetImages/Cat/'
dog_source_dir = './DataSets/cats-dog/PetImages/Dog/'
Training_cat = './DataSets/cats-dog/CatvDog/training/cats/'
Testing_cat = './DataSets/cats-dog/CatvDog/testing/cats/'
Training_dog = './DataSets/cats-dog/CatvDog/training/dogs/'
Testing_dog = './DataSets/cats-dog/CatvDog/testing/dogs/'
split_size = 0.9
split_data(cats_source_dir, Training_cat,Testing_cat,split_size)
split_data(dog_source_dir, Training_dog, Testing_dog, split_size)
#%%
print(len(os.listdir('./DataSets/cats-dog/CatvDog/training/cats/')))
print(len(os.listdir('./DataSets/cats-dog/CatvDog/training/dogs/')))
print(len(os.listdir('./DataSets/cats-dog/CatvDog/testing/cats/')))
print(len(os.listdir('./DataSets/cats-dog/CatvDog/testing/dogs/')))
#%%
model = Sequential([
    Conv2D(16, 3, activation='relu', input_shape=(150, 150, 3)),
    MaxPool2D(2,2),
    Conv2D(32, 3, activation='relu'),
    MaxPool2D(2,2),
    Conv2D(64, 3, activation='relu'),
    MaxPool2D(2,2),
    Flatten(),
    Dense(128, activation='relu'),
    Dense(1, activation='sigmoid')
])

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])
model.summary()
#%%
train_datagen = ImageDataGenerator(
    rescale=1./255,
    rotation_range=40,
    width_shift_range=0.2,
    height_shift_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True,
    shear_range=0.2,
    fill_mode='nearest'
)

val_datagen = ImageDataGenerator(
    rescale=1./255,
    rotation_range=40,
    width_shift_range=0.2,
    height_shift_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True,
    shear_range=0.2,
    fill_mode='nearest'
)

train_dir = './DataSets/cats-dog/CatvDog/training'
val_dir = './DataSets/cats-dog/CatvDog/testing'

train_gen = train_datagen.flow_from_directory(
    train_dir,
    target_size=(150,150),
    batch_size= 32,
    class_mode ='binary'

)
val_gen = val_datagen.flow_from_directory(
    val_dir,
    target_size=(150, 150),
    batch_size=32,
    class_mode='binary'
)
#%%
class MyCallBack(tf.keras.callbacks.Callback):
    def on_epoch_end(self, epoch, logs={}):
        if logs['val_acc'] > 0.95:
            self.model.stop_training= True

stopcall = MyCallBack()
check = tf.keras.callbacks.ModelCheckpoint('./models/catvdog/', save_best_only=True )
earlstop = tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)

#%%
#model = tf.keras.models.load_model('./Model_saves/Cat-Dog/mymodel.h5')
hist = model.fit_generator(train_gen, epochs = 1, validation_data=val_gen, callbacks=[stopcall,check,earlstop])
model.save('./Model_saves/Cat-Dog/mymodel.h5')
#%%
import matplotlib.image  as mpimg

#-----------------------------------------------------------
# Retrieve a list of list results on training and test data
# sets for each training epoch
#-----------------------------------------------------------
acc=hist.history['acc']
val_acc=hist.history['val_acc']
loss=hist.history['loss']
val_loss=hist.history['val_loss']

epochs=range(len(acc)) # Get number of epochs

#------------------------------------------------
# Plot training and validation accuracy per epoch
#------------------------------------------------
plt.plot(epochs, acc, 'r', "Training Accuracy")
plt.plot(epochs, val_acc, 'b', "Validation Accuracy")
plt.title('Training and validation accuracy')
plt.figure()
plt.show()

#------------------------------------------------
# Plot training and validation loss per epoch
#------------------------------------------------
plt.plot(epochs, loss, 'r', "Training Loss")
plt.plot(epochs, val_loss, 'b', "Validation Loss")

plt.title('Training and validation loss')
plt.show()


text:
import argparse
import glob
import os
import subprocess

import numpy as np
import pandas as pd

from IM_calculation.Advanced_IM import runlibs_2d

model_dir = os.path.dirname(__file__)

def main(comp_000, comp_090, output_dir, OpenSees_path):

    if not os.path.exists(output_dir):
        os.makedirs(args.output_dir)

    script = [
        OpenSees_path,
        os.path.join(model_dir, "Run_script.tcl"),
        comp_000,
        comp_090,
        output_dir,
    ]

    print(" ".join(script))

    subprocess.run(script)

    im_name = "Spear_3D_Model"
    # create CSV for norm
    # replaced by calculate_geom(). keeping these line for references
    # component='norm'
    # component_dir = args.output_dir
    # create_im_csv(args.output_dir, im_name, component, component_dir, remove_gravity = False)

    # create CSV for each component
    for component in ["000", "090"]:
        component_dir = os.path.join(output_dir, component)

        create_im_csv(
            output_dir, im_name, component, component_dir, check_converge=False
        )
    # create a geom csv
    calculate_geom(output_dir, im_name)

    # calculate the disp&drift Norm from records from 000 and 090
    calculate_norm(im_name, output_dir)

    agg_csv(output_dir, im_name)

# calc norm
def read_recorder_all(direction_dir, recorder_name, im_recorder_fname):

    # find corrosponding gravity file
    im_gravity_dir = os.path.join(direction_dir, f"gravity_{recorder_name}")
    im_gravity_recorder = os.path.join(im_gravity_dir, f"gr_{im_recorder_fname}")

    if os.path.exists(im_gravity_recorder):
        gr_value = float(read_out_file(im_gravity_recorder))
    else:
        gr_value = 0

    im_recorder = os.path.join(
        direction_dir, os.path.join(recorder_name, im_recorder_fname)
    )

    with open(im_recorder) as f_im_recorder:
        im_records_list_tmp = [float(line.split()[1]) for line in f_im_recorder]
    im_records_list = []
    for im_record in im_records_list_tmp[:-1]:
        im_record = im_record - gr_value
        im_records_list.append(im_record)

    return im_records_list

def calculate_norm(im_name, output_dir, print_header=True):
    # get 000 and 090 dir
    dir_000 = os.path.join(output_dir, "000")
    dir_090 = os.path.join(output_dir, "090")
    component = "norm"

    im_csv_fname = os.path.join(output_dir, f"{im_name}_{component}.csv")

    value_dict = {}

    # read data from recordings
    for recorder_name in ["disp", "drift", "accl"]:
        # find recorder in 000 first
        # read all 2nd coloumn
        recorder_dir = os.path.join(dir_000, recorder_name)
        im_recorder_glob = os.path.join(recorder_dir, "*.out")
        im_recorders = glob.glob(im_recorder_glob)

        for im_recorder in im_recorders:
            im_recorder_fname = os.path.basename(im_recorder)
            im_recorder_name = os.path.splitext(im_recorder_fname)[0]
            im_record_list = {}

            im_record_list["000"] = read_recorder_all(
                dir_000, recorder_name, im_recorder_fname
            )
            im_record_list["090"] = read_recorder_all(
                dir_090, recorder_name, im_recorder_fname
            )
            im_record_list["norm"] = []
            if len(im_record_list["000"]) != len(im_record_list["090"]):
                raise (
                    ValueError(
                        f"length of 000 and 090 does not match for {im_recorder_fname}"
                    )
                )
            # calulate norm
            for i in range(0, len(im_record_list["000"])):
                norm = np.sqrt(
                    (
                        np.power(im_record_list["000"][i], 2)
                        + np.power(im_record_list["090"][i], 2)
                    )
                )
                im_record_list["norm"].append(norm)

            im_value = max(im_record_list["norm"])
            # value_dict[recorder_name][im_recorder_name] = im_value
            value_dict[im_recorder_name] = im_value

    value_dict = {component: value_dict}
    result_df = pd.DataFrame.from_dict(value_dict, orient="index")

    cols = list(result_df.columns)
    cols.sort()

    # test if file exist, if exist, no header
    if os.path.isfile(im_csv_fname):
        print_header = False

    result_df.to_csv(im_csv_fname, mode="a", header=print_header, columns=cols)

def calculate_geom(output_dir, im_name):
    """
    generates geom by globing results from 000 and 090
    output_dir: folder that contains adv_im_comp.csv. used to grab data from 000, 090.
    im_name: adv_im model name
    """
    df_000 = pd.read_csv(
        os.path.join(output_dir, f"{im_name}_000.csv"), dtype={"component": str}
    )
    df_090 = pd.read_csv(
        os.path.join(output_dir, f"{im_name}_090.csv"), dtype={"component": str}
    )
    #
    ims = df_000.append(df_090)
    ims.set_index("component", inplace=True)

    if "000" in ims.index and "090" in ims.index:
        line = np.sqrt(ims.loc["090"] * ims.loc["000"])
        line.rename("geom", inplace=True)
        im_geom = pd.DataFrame()
        im_geom = im_geom.append(line)
        im_geom.index.name = "component"
    else:
        raise (ValueError("Error: no 000 or 090 found in csv"))
    cols = list(im_geom.columns)
    cols.sort()
    im_csv_fname = os.path.join(output_dir, f"{im_name}_geom.csv")
    im_geom.to_csv(im_csv_fname, columns=cols, index=True, header=True)

def agg_csv(output_dir, im_name, print_header=True):
    """
    aggregate all data into one huge csv that contains multiple rows
    000,090, geom, norm
    """
    # read all df
    # im_csv_glob = os.path.join(output_dir, im_name + "_*" + ".csv")
    df = pd.DataFrame()
    df.index.name = "component"
    for component in ["000", "090", "geom", "norm"]:
        component_csv_fname = os.path.join(output_dir, f"{im_name}_{component}.csv")
        df = df.append(pd.read_csv(component_csv_fname, dtype={"component": str}))
    df.set_index("component", inplace=True)
    # im_csvs = glob.glob(im_csv_glob)
    cols = list(df.columns)
    cols.sort()
    df.to_csv(
        os.path.join(output_dir, im_name + ".csv"), header=print_header, columns=cols
    )
    # write a csv file

def create_im_csv(
    output_dir,
    im_name,
    component,
    component_dir,
    check_converge=True,
    print_header=True,
    remove_gravity=True,
):
    """
    create a csv file for each single component/analysis
    """
    if check_converge:
        success_glob = os.path.join(component_dir, "Analysis_*")
        success_files = glob.glob(success_glob)
        model_converged = False
        for f in success_files:
            with open(f) as fp:
                contents = fp.read()
            model_converged = model_converged or (contents.strip() == "Successful")
    else:
        # read even if not converged
        model_converged = True

    im_csv_fname = os.path.join(output_dir, f"{im_name}_{component}.csv")
    result_df = pd.DataFrame()
    im_recorder_glob = os.path.join(component_dir, "env*/*.out")
    im_recorders = glob.glob(im_recorder_glob)
    value_dict = {}

    for im_recorder in im_recorders:
        folder_name = os.path.dirname(im_recorder)
        recorder_name = os.path.splitext(os.path.basename(im_recorder))[0]
        im_name = os.path.basename(folder_name)
        # get base name
        # im_type = im_name.split("_")[0]
        im_type = "_".join(im_name.split("_")[1:])
        im_gravity_dir = os.path.join(component_dir, "gravity_" + im_type)
        im_gravity_recorder = os.path.join(
            im_gravity_dir, "gr_" + os.path.basename(im_recorder)
        )

        # find corrosponding gravity file
        #        im_value_tmp = read_out_file(im_recorder, model_converged)
        if os.path.exists(im_gravity_recorder) and remove_gravity:
            gr_value = float(read_out_file(im_gravity_recorder, model_converged))
        #            print(f"{im_recorder} gr_value:{gr_value}")
        else:
            #            print(f'cannot find gr files for {im_gravity_recorder}')
            gr_value = 0
        #        print(f"{im_recorder} gr_value:{gr_value} im_value_tmp:{im_value_tmp} gr_value{gr_value}")
        # im_value = float(im_value_tmp) - float(gr_value)
        with open(im_recorder) as f_im_recorder:
            im_records_list_tmp = [float(line.split()[1]) for line in f_im_recorder]
        # read all lines except the last
        im_records_list = []
        for im_record in im_records_list_tmp[:-1]:
            # loop through all records
            # print(im_record)
            # print(f"im_record: {im_record}")
            im_record = im_record - gr_value
            im_records_list.append(abs(im_record))
        im_value = max(im_records_list)

        #        im_value = read_out_file(im_recorder, model_converged)
        value_dict[recorder_name] = im_value
    value_dict = {component: value_dict}
    result_df = pd.DataFrame.from_dict(value_dict, orient="index")
    result_df.index.name = "component"
    cols = list(result_df.columns)
    cols.sort()
    #    if os.path.isfile(im_csv_fname):
    #        print_header = False
    # result_df = result_df.append(value_dict, ignore_index=True)
    # print(result_df)
    result_df.to_csv(im_csv_fname, header=print_header, columns=cols)

def read_out_file(file, success=True):
    if success:
        with open(file) as f:

            lines = f.readlines()

            value = lines[-1].split()[1]

            return value
    else:
        return float("NaN")

if __name__ == "__main__":

    args = runlibs_2d.parse_args()
    main(args.comp_000, args.comp_090, args.output_dir, args.OpenSees_path)