text:
import copy

import numpy as np

from .problem import LpProblem
from .solve import LpSolve
from .solver import LpSolver

class DualSimplex(LpSolver):
    """ 对偶单纯形算法

    待解决应符合标准型，且问题可用对偶单纯形法求解.
    """

    class Problem(LpProblem):
        """
        对偶单纯形算法内部的线性规划问题表示
        """
        def __init__(self, c, a, b):
            super().__init__(c, a, b)
            self.base_idx = np.ones(len(b), 'int') * -1
            self.cc = copy.deepcopy(c)

    def __init__(self, problem: LpProblem):
        super().__init__(problem)
        self.problem = self.Problem(problem.c, problem.a, problem.b)

    def find_idt_base(self):
        """
        尝试找单位阵作初始基

        :return: True if success False else
        """
        base_idx = np.ones(len(self.problem.b), 'int') * -1
        aT = self.problem.a.T
        for i in range(len(self.problem.b)):
            e = np.zeros(len(self.problem.b))
            e[i] = 1
            for j in range(len(aT)):
                if np.all(aT[j] == e):
                    base_idx[i] = j

        self.problem.base_idx = base_idx
        return np.all(base_idx >= 0)

    def big_m(self, **kwargs):
        """
        用大M法得到初始基

        :return: None
        """
        M = ((max(abs(self.problem.c)) + 1) ** 2) * 10 + 10

        for i in range(len(self.problem.base_idx)):
            if self.problem.base_idx[i] < 0:
                self.problem.c = np.insert(self.problem.c, len(self.problem.c), np.array([-M]))

                ap = np.zeros(len(self.problem.b))
                ap[i] = 1
                self.problem.a = np.c_[self.problem.a, ap]

                self.problem.base_idx[i] = len(self.problem.c) - 1

    def two_step(self, **kwargs):
        """
        用两阶段法得到初始基，第一阶段在此计算

        :return: 第一阶段的解
        """
        p = copy.deepcopy(self.problem)
        p.c = np.zeros(len(p.c))
        for i in range(len(self.problem.base_idx)):
            if self.problem.base_idx[i] < 0:
                p.c = np.insert(p.c, len(p.c), np.array([-1]))

                ap = np.zeros(len(p.b))
                ap[i] = 1
                p.a = np.c_[p.a, ap]

                self.problem.base_idx[i] = len(p.c) - 1
        p.base_idx = self.problem.base_idx
        s1 = _dual_simplex_solve(p)

        self.problem.c = copy.deepcopy(self.problem.cc)
        self.problem.base_idx = p.base_idx
        self.problem.b = p.b
        self.problem.a = (p.a.T[0: len(self.problem.c)]).T
        return s1

    def solve(self, **kwargs) -> LpSolve:
        """
        单纯形算法入口

        :param kwargs:
            base_idx=[...] (default []): 指定初始基，缺省则由算法自行确定
            two_step=True  (default False): 使用两阶段法
            big_m=True     (default True):  使用大 M 法
        :return: 问题的解
        """
        base_idx = kwargs.get("base_idx", [])
        if base_idx:    # 用户指定了基
            self.problem.base_idx = base_idx
        else:
            if not self.find_idt_base():    # 没有找到单位阵作初始基，用人工变量法（大M法 / 两阶段法）
                if kwargs.get("two_step", False):
                    s1 = self.two_step(**kwargs)
                    if not s1.success:  # 第一阶段确定无可行解
                        return s1
                else:
                    self.big_m(**kwargs)
        self.problem.base = np.identity(len(self.problem.b))
        self.problem.base_inv = np.identity(len(self.problem.b))

        pb = copy.deepcopy(self.problem)
        return _dual_simplex_solve(pb)

def _dual_simplex_solve(p: DualSimplex.Problem) -> LpSolve:
    """ _dual_simplex_solve 对给定了初始基的标准型使用对偶单纯形法进行求解

    :return: 问题的解
    """
    base_idx = p.base_idx
    n_base_idx = _get_n_base_idx(p, base_idx)

    base_inv = np.linalg.inv(p.a.T[base_idx].T)

    # 初始的检验数计算
    sigma = p.c - np.dot(np.dot(p.c[base_idx], base_inv), p.a)

    if np.any(sigma > 0):
        return LpSolve(False, "对偶问题非可行解，无法使用对偶单纯形算法。", None, None)

    x = np.dot(base_inv, p.b)

    while np.any(x < 0):
        # print(x)
        # 确定换出变量
        lb = np.argmin(x)
        leaving_idx = base_idx[lb]

        if np.all(p.a[lb] >= 0):
            return LpSolve(False, "对偶问题无界解，原问题无可行解", None, None)

        # 确定换出变量
        theta_nb = _get_theta_nb(sigma, p, n_base_idx, lb, base_inv)

        eb = np.argmin(theta_nb)
        entering_idx = n_base_idx[eb]

        # 基变换
        base_idx[lb] = entering_idx
        n_base_idx[eb] = leaving_idx

        base_inv = np.linalg.inv(p.a.T[base_idx].T)
        x = np.dot(base_inv, p.b)

        # Next Sigma_N
        sigma = p.c - np.dot(np.dot(p.c[base_idx], base_inv), p.a)

    # 迭代结束，计算解的值
    # base_inv = np.linalg.inv(p.a.T[base_idx].T)
    xb = np.dot(base_inv, p.b)
    x = np.zeros(len(p.c))
    x[base_idx] = xb

    # 分析解的情况
    return _analyse_solve(p, x, sigma, n_base_idx)

def _get_theta_nb(sigma, p, n_base_idx, lb, base_inv):
    """
    非基变量检验数计算

    :return: 非基变量的检验数
    """
    theta = np.ones(len(p.c[n_base_idx])) * np.inf
    n = np.dot(base_inv, p.a.T[n_base_idx].T)   # 基变换后的非基矩阵
    a_l = n[lb]
    for i in range(len(theta)):
        a_li = a_l[i]
        if a_li < 0:
            theta[i] = sigma[n_base_idx][i] / a_li
    return theta

def _get_n_base_idx(p: DualSimplex.Problem, base_idx):
    """
    初始化非基

    :param p: SimplexVectorized.Problem
    :param base_idx: 已确定的基indices
    :return: 非基indices
    """
    n_base_idx = list(range(len(p.c)))
    for i in base_idx:
        try:
            n_base_idx.pop(n_base_idx.index(i))
        except ValueError:
            pass
    return np.array(n_base_idx, 'int')

def _analyse_solve(p: DualSimplex.Problem, x, last_sigma, last_n_base_idx) -> LpSolve:
    """
    单纯形求解完成后，分析解的情况

    :return: 单纯形求解结果(LpSolve实例)
    """
    x_real = x[0: len(p.cc)]
    x_personal = x[len(p.cc):]    # 人工变量

    if np.any(x_personal != 0):
        return LpSolve(False, "无可行解", None, None)

    z = np.dot(x_real, p.cc)

    for i in last_sigma[last_n_base_idx]:
        if abs(i) < 1e-8:  # 非基变量检验数为0
            return LpSolve(True, "无穷多最优解", x_real, z)

    return LpSolve(True, "唯一最优解", x_real, z)

max_stars_repo_name: Allvey/SimplexLinprog

max_issues_repo_name: Allvey/SimplexLinprog

max_forks_repo_name: Allvey/SimplexLinprog


text:
import numpy as np
import pandas as pd
from sklearn.metrics import confusion_matrix
import sys
import time

from iforest import IsolationTreeEnsemble, find_TPR_threshold

def score(X, y, n_trees, desired_TPR, datafile,
          reqd_fit_time,
          reqd_score_time,
          reqd_FPR,
          reqd_n_nodes):
    it = IsolationTreeEnsemble(sample_size=256, n_trees=n_trees)

    fit_start = time.time()
    it.fit(X, improved=improved)
    fit_stop = time.time()
    fit_time = fit_stop - fit_start
    print(f"INFO {datafile} fit time {fit_time:3.2f}s")

    n_nodes = sum([t.n_nodes for t in it.trees])
    print(f"INFO {datafile} {n_nodes} total nodes in {n_trees} trees")

    score_start = time.time()
    scores = it.anomaly_score(X)
    score_stop = time.time()
    score_time = score_stop - score_start
    print(f"INFO {datafile} score time {score_time:3.2f}s")

    threshold, FPR = find_TPR_threshold(y, scores, desired_TPR)

    y_pred = it.predict_from_anomaly_scores(scores, threshold=threshold)
    confusion = confusion_matrix(y, y_pred)
    TN, FP, FN, TP = confusion.flat
    TPR = TP / (TP + FN)
    FPR = FP / (FP + TN)

    errors = 0
    if fit_time > reqd_fit_time * 2:
        print(f"FAIL {datafile} fit time {fit_time:.1f} > {reqd_fit_time}")
        errors += 1

    if score_time > reqd_score_time * 2:
        print(f"FAIL {datafile} score time {score_time:.1f} > {reqd_score_time}")
        errors += 1

    if TPR < desired_TPR*.9: # TPR must be within 10% (or above)
        print(f"FAIL {datafile} TPR {TPR:.2f} < {desired_TPR} +- 10%")
        errors += 1

    if FPR > reqd_FPR*1.3: # TPR must be within 30%
        print(f"FAIL {datafile} FPR {FPR:.4f} > {reqd_FPR} +- 30%")
        errors += 1

    if n_nodes > reqd_n_nodes*1.15:
        print(f"FAIL {datafile} n_nodes {n_nodes} > {reqd_n_nodes} +- 20%")
        errors += 1

    if errors==0:
        print(f"SUCCESS {datafile} {n_trees} trees at desired TPR {desired_TPR*100.0:.1f}% getting FPR {FPR:.4f}%")
    else:
        print(f"ERRORS {datafile} {errors} errors {n_trees} trees at desired TPR  {desired_TPR*100.0:.1f}% getting FPR {FPR:.4f}%")

def score_cc():
    df = pd.read_csv("creditcard.csv")
    N = 15_000
    df = df.sample(N)  # grab random subset (too slow otherwise)
    if noise: add_noise(df)
    X, y = df.drop('Class', axis=1), df['Class']

    score(X, y, n_trees=200, desired_TPR=.8,
          datafile='creditcard.csv',
          reqd_fit_time=.32 if noise and improved else 0.25,
          reqd_score_time=13,
          reqd_FPR=.08,
          reqd_n_nodes=15800 if noise and improved else 18900)

def score_http():
    df = pd.read_csv("http.csv")
    N = 16_000
    df = df.sample(N)  # grab random subset (too slow otherwise)
    if noise: add_noise(df)
    X, y = df.drop('attack', axis=1), df['attack']

    score(X, y, n_trees=300, desired_TPR=.99,
          datafile='http.csv',
          reqd_fit_time=.37 if noise and improved else 0.2,
          reqd_score_time=21 if noise and improved else 13,
          reqd_FPR=.17 if noise and improved else 0.006,
          reqd_n_nodes=26300 if noise and improved else 22700)

def score_cancer():
    df = pd.read_csv("cancer.csv")
    N = len(df)
    df = df.sample(N)  # grab random subset (too slow otherwise)
    if noise: add_noise(df)
    X, y = df.drop('diagnosis', axis=1), df['diagnosis']

    score(X, y, n_trees=1000, desired_TPR=.75,
          datafile='cancer.csv',
          reqd_fit_time=1.5,
          reqd_score_time=2.4,
          reqd_FPR=.4,
          reqd_n_nodes=109000 if noise and improved else 130_000)

def add_noise(df):
    n_noise = 5
    for i in range(n_noise):
        df[f'noise_{i}'] = np.random.normal(0,100,len(df))

if __name__ == '__main__':
    noise = False
    improved = False
    if '-noise' in sys.argv:
        noise = True
    if '-improved' in sys.argv:
        improved = True

    print(f"Running noise={noise} improved={improved}")
    score_cc()
    print()
    score_http()
    print()
    score_cancer()

ext: py

reason: import numpy


text:
'''
COMP9418 Assignment 2
This file is the example code to show how the assignment will be tested.

Name:     zID:

Name:     zID:
'''

# Make division default to floating-point, saving confusion
from __future__ import division
from __future__ import print_function

# Allowed libraries
import numpy as np
import pandas as pd

from collections import OrderedDict as odict

from tabulate import tabulate

data = pd.read_csv("data.csv")

#calculate the reliability of
#the sensor i.e the number of times the sensor is predicting the correct values

cols_sensor = ['reliable_sensor1',	'reliable_sensor2',	'reliable_sensor3',	'reliable_sensor4',	'unreliable_sensor1',	'unreliable_sensor2',	'unreliable_sensor3',	'unreliable_sensor4']

groundTruth = ['r1',	'r2',	'r3',	'r4',	'r5',	'r6',	'r7',	'r8',	'r9',	'r10',	'r11',	'r12',	'r13',	'r14',	'r15',	'r16',	'r17',	'r18',	'r19',	'r20',	'r21',	'r22',	'r23',	'r24',	'r25',	'r26',	'r27',	'r28',	'r29',	'r30',	'r31',	'r32',	'r33',	'r34',	'r35',	'c1',	'c2',	'c3',	'c4',	'o1',	'outside']

reliable_sensors = {
    'reliable_sensor1' : 'r16',
    'reliable_sensor2' : 'r5',
    'reliable_sensor3' : 'r25',
    'reliable_sensor4' : 'r31',
    }

un_reliable_sensor = {
    'unreliable_sensor1' : 'o1',
    'unreliable_sensor2' : 'c3',
    'unreliable_sensor3' : 'r1',
    'unreliable_sensor4' : 'r24',
    }

door_sensor = {
    'door_sensor1' : ['r8','r9'],
    'door_sensor2' : ['c1','c2'],
    'door_sensor3' : ['r26','r27'],
    'door_sensor4' : ['c4','r35'],
    }

#replace the motion and non_motion values with 1 and 0 respectivily
data = data.replace("motion",1)
data = data.replace("no motion",0)

#check the data after replacing the motion and no motion values with 0 and 1
#print(data.head())

#data to be used for calculating the reliability of
#all the sensors against the ground truth

datat_reliability_cols = list(cols_sensor+groundTruth)

data_reliability = np.array(data[datat_reliability_cols])

#
#print(data_reliability[:5])

#use index func of list to find the index of the col you want in numpy

#creating the sensors list to use later
sensors = list(reliable_sensors.keys()) + list(un_reliable_sensor.keys())

#creating combined locations and sensors dict for latter use
combined_sensors_and_locations_dict = {**reliable_sensors,**un_reliable_sensor}

master_reliability_table = odict()

#print("sensors",sensors)
#pick the first column from cols_sensor
#and then find  that column in numpy and compare its value with the number of people in the room
for i in sensors:
    sensor_index = cols_sensor.index(i)

    #print(len(data_reliability[:,sensor_index]))

    sensor_col = data_reliability[:,sensor_index]

    # print("sensor_col")

    # print(pd.Series(sensor_col).head(25))

    #debugging
    # print(sensor_col[sensor_index])
    #print("i",i)
    #get the room name from the locations and sensors index
    room = combined_sensors_and_locations_dict[i]
    #print(room)

    #get the column of the ground truth for the sensor
    groundTruth_index = datat_reliability_cols.index(room)
    groundTruth_col = data_reliability[:,groundTruth_index]

    # print("groundTruth")

    # print(pd.Series(groundTruth_col).head(25))

    count_T = 0
    for k in range(0,len(sensor_col)):

        #if the k value of  both columns is equal to 1 then count it
        if sensor_col[k] == 1 and groundTruth_col[k] > 0:
            # print("k",k)
            # print('sensor_col',sensor_col[k])
            # print('ground_truth',groundTruth_col[k])
            count_T = count_T + 1

    #calculating the true prob
    total_num_values = len(sensor_col)

    # print(i)
    # print("count_T",count_T)
    # print("total",total_num_values)

    prob_T = count_T / total_num_values
    prob_F = 1 - prob_T

    #creating the probability table with dom and table
    sensor_reliability_prob_table = odict()

    sensor_reliability_prob_table['dom'] = tuple((i,))

    prob_table = odict()

    prob_table[tuple((True,))] = prob_T

    prob_table[tuple((False,))] = prob_F

    sensor_reliability_prob_table['table'] = prob_table

    master_reliability_table[i] = sensor_reliability_prob_table
#if the sensors  reading is motion and number of people is greater then zero. this is the number of
#correct predictions the sensor is making

#count the number of the total number of predictions which is equal to len of index of of the  sensor column

#divide the number of true predictions with total count. we call this the reliability of the sensor and store

#it to with some value and if it is greater than a certain threshold then we will replace it in our next state

def printFactor(f):
    """
    argument
    `f`, a factor to print on screen
    """
    # Create a empty list that we will fill in with the probability table entries
    table = list()

    # Iterate over all keys and probability values in the table
    for key, item in f['table'].items():
        # Convert the tuple to a list to be able to manipulate it
        k = list(key)
        # Append the probability value to the list with key values
        k.append(item)
        # Append an entire row to the table
        table.append(k)
    # dom is used as table header. We need it converted to list
    dom = list(f['dom'])
    # Append a 'Pr' to indicate the probabity column
    dom.append('Pr')
    print(tabulate(table,headers=dom,tablefmt='orgtbl'))

# printFactor(master_reliability_table['unreliable_sensor1'])

reliability_df = pd.DataFrame()

for i in master_reliability_table.keys():
    #print(master_reliability_table[i]['table'][(True,)])

    #create a series for a sensor
    col_values = []

    col_values.append(master_reliability_table[i]['table'][(True,)])

    col_values.append(master_reliability_table[i]['table'][(False,)])

    #print(pd.Series(col_values))

    #create the col using series

    reliability_df[i] = col_values

reliability_df.to_csv("reliability_sensors.csv")

#print(reliability_df.head())


text:
import matplotlib as mpl
# PyQt5 package required for mac users
# mpl.use('Qt5Agg')
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.transforms import Bbox
from matplotlib.gridspec import GridSpec
from matplotlib.widgets import Button
from matplotlib.ticker import EngFormatter, AutoMinorLocator, MaxNLocator
from mpl_toolkits.axes_grid1.inset_locator import InsetPosition
import stmpy
from .gui_components import *
from .misc import grab_bg, blit_bg
import os.path as _ospath

def view3ds(data, topos=[], ch_names=[], didv=None, fit_didv=None, extent=[],
            cmap_topo=stmpy.cm.blue1, cmap_didv=stmpy.cm.bone, cbar_topo='salmon', cbar_didv='salmon',
             use_blit=False):
    '''
    Interactive visulization tools of any dI/dV map.
    Requires `%matplotlib qt` to open a new window. Run `%matplotlib inline` to switch back to inline jupyter notebook.
    Inputs:
          data - Spy object returned from stmpy.load().
          topos - list of channels (2d arrays of the same shape) to be shown in the topo window. If not given, line-subtracted topography (data.Z) will be shown.
          ch_names - list of channel names (str) to be shown in the topo window. If not given, ['Topo',] will be used.
          didv - A 3d array, each slice of which has a same shape as a topo channel, to be shown in the map window. If not given, data.LIY will be shown.
          fit_didv - An additional 3d array, having a same shape as didv, whose same location spectrum to be overlayed in the spectrum window.
          extent - list of four floats, same the extent argument in plt.imshow, defining the extent of topo and map windows. If not given, scan fields info will be used.
          cmap_topo, cmap_didv - Colormap instances for topo and map window, respectively.
          cbar_topo, cbar_didv - colors to be assigned to the color limit indicators in the colorbars
          use_blit - boolean, use blitting or not. Defaut is False, try switching it to True if interactive tools are sluggish.
    Example in jupyter notebook:
          ...
          %matplotlib qt # switch to qt backend
          from stmpy.mapviewer.view3ds import view3ds
          view3ds(data, topos=[data.Z,], ch_names=['Topo',], didv=data.liy, fit_didv=data.fit)
          %matplotlib inline # switch back to inline
          ...
    '''
    # ----- Default rc Parameters -----
    mpl.rcParams['axes.labelsize'] = 10
    mpl.rcParams['axes.titlesize'] = 12
    mpl.rcParams['xtick.labelsize'] = 8
    mpl.rcParams['ytick.labelsize'] = 8
    mpl.rcParams['font.size'] = 8
    mpl.rcParams['axes.linewidth'] = 0.5
    mpl.rcParams['xtick.direction'] = 'out'
    mpl.rcParams['ytick.direction'] = 'out'
    mpl.rcParams['xtick.major.width'] = 0.5
    mpl.rcParams['ytick.major.width'] = 0.5
    mpl.rcParams['xtick.minor.width'] = 0.3
    mpl.rcParams['ytick.minor.width'] = 0.3
    mpl.rcParams['xtick.major.size'] = 2
    mpl.rcParams['ytick.major.size'] = 2
    mpl.rcParams['xtick.minor.size'] = 1
    mpl.rcParams['ytick.minor.size'] = 1

    # ----- Figure Settings -----
    ### Create axes ###
    fig_main = plt.figure(num=1, figsize=(8, 6), dpi=120, tight_layout=False)
    fig_main.canvas.manager.set_window_title('3DS Viewer')
    gs = GridSpec(3, 6, left=0.07, right=0.98, bottom=0.08, top=0.98, hspace=0.1, wspace=0.1, height_ratios=[2.5, 0.5, 3], width_ratios=[5, 0.2, 0.5, 5, 0.2, 0.5])
    axctrl = fig_main.add_subplot(gs[0, 0])
    axspec = fig_main.add_subplot(gs[0, 1:])
    axtopo = fig_main.add_subplot(gs[2, 0])
    axtcbar = fig_main.add_subplot(gs[2, 1])
    axmap = fig_main.add_subplot(gs[2, 3])
    axmcbar = fig_main.add_subplot(gs[2, 4])
    ### Set axis labels ###
    axspec.set_xlabel('Bias [V]')
    axspec.set_ylabel('dI/dV [A]')
    axtopo.set_xlabel('x [m]')
    axtopo.set_ylabel('y [m]')
    axmap.set_xlabel('x [m]')
    fig_main.align_labels()
    # axmap.set_ylabel('y [m]')
    ### Set ticks and tick labels ###
    axes = [axspec, axtopo, axmap]
    for ax in axes:
        ax.xaxis.set_major_formatter(EngFormatter(sep=''))
        ax.yaxis.set_major_formatter(EngFormatter(sep=''))
    axspec.xaxis.set_minor_locator(AutoMinorLocator(n=5))
    axspec.yaxis.set_minor_locator(AutoMinorLocator(n=5))
    axtopo.xaxis.set_major_locator(MaxNLocator(nbins=4))
    axtopo.yaxis.set_major_locator(MaxNLocator(nbins=4))
    axmap.xaxis.set_major_locator(MaxNLocator(nbins=4))
    axmap.yaxis.set_major_locator(MaxNLocator(nbins=4))

    # ----- Load Data -----
    if not topos:
        topo = stmpy.tools.lineSubtract(data.Z, n=1)
        topos = [topo,]
    if didv is None:
        didv = data.LIY
    # ----- Spectrum Plot -----
    axspec.plot(data.en, didv[:, 0, 0], 'b', lw=1,) # will be updated by cursor, HAS to be here to pass data.en to SpecLinkedCursors object
    axspec.grid(which='major', alpha=0.3, color='k', linewidth=0.5)
    axspec.grid(which='minor', alpha=0.3, color='k', linewidth=0.3)
    # ----- 2D Images -----
    if not extent:
        fov = [float(val) for val in data.header['Scan>Scanfield'].split(';')]
        extent = [0, fov[2], 0, fov[3]]
    sm2 = ScrollableMap(topos, axtopo, ch_names=ch_names, cmap=cmap_topo, origin='lower', aspect=1, extent=extent)
    rs2 = RangeSlider(axtcbar, '', topos[0].min(), topos[0].max(),
     orientation='vertical', cmap=cmap_topo, color=cbar_topo, lw=0.8, alpha=0.7)
    # ----- 3D Slice -----
    sm3 = ScrollableMap(didv, axmap, cmap=cmap_didv, origin='lower', aspect=1, extent=extent)
    rs3 = RangeSlider(axmcbar, '', didv.min(), didv.max(), valinit=(didv[0].min(), didv[0].max()),
     orientation='vertical', cmap=cmap_didv, color=cbar_didv, lw=0.8, alpha=0.7)
    lla = LimLinkedAxes(axtopo, axmap)
    slc = SpecLinkedCursors(axtopo, axmap, axspec, rs2, rs3, fit_didv=fit_didv, use_blit=use_blit)
    rs2.on_changed(slc.topo_clim_update)
    rs3.on_changed(slc.map_clim_update)
    # ----- Control Aera -----
    axctrl.axis('off')
    axctrl.set_navigate(False)
    ### Cursor button ###
    axcur = plt.axes([0.1, 0, 1, 1])
    ipcur = InsetPosition(axctrl, [0.1, 0.1, 0.3, 0.2])
    axcur.set_axes_locator(ipcur)
    bvis = Button(axcur, 'Cursor', color='gold', hovercolor='gold')
    bvis.label.set_fontsize(12)
    cursorswitch = LinesVisible(bvis, [slc.vline1, slc.hline1, slc.vline2, slc.hline2])
    bvis.on_clicked(cursorswitch.vis)
    ### Fit button ###
    axfit = plt.axes([0.2, 0, 1, 1])
    ipfit = InsetPosition(axctrl, [0.1, 0.4, 0.3, 0.2])
    axfit.set_axes_locator(ipfit)
    fvis = Button(axfit, 'Fit', color='tomato', hovercolor='tomato')
    fvis.label.set_fontsize(12)
    if fit_didv is None:
        fvis.color = '0.85'
        fvis.hovercolor = '0.85'
    else:
        fitswitch = LinesVisible(fvis, [slc.spec_fit,])
        fvis.on_clicked(fitswitch.vis)
    ### Help button ###
    axhelp = plt.axes([0.3, 0, 1, 1])
    iphelp = InsetPosition(axctrl, [0.1, 0.7, 0.3, 0.2])
    axhelp.set_axes_locator(iphelp)
    helpbtn = Button(axhelp, 'Help', color='0.9', hovercolor='0.9')
    helpbtn.label.set_fontsize(12)
    helpfile_path = _ospath.join(_ospath.dirname(__file__), 'helptext.txt')
    with open(helpfile_path) as helpfile:
        helptext = helpfile.read()
    helpwin = HelpWindow(helpbtn, helptext)
    helpbtn.on_clicked(helpwin.helpwindow)
    ### Buttons not in use ###
    # axpts = plt.axes([0.1, 0.1, 1, 1])
    # ippts = InsetPosition(axctrl, [0.5, 0.1, 0.3, 0.2])
    # axpts.set_axes_locator(ippts)
    # scatter = Button(axpts, 'Scatter', color='skyblue', hovercolor='skyblue')
    # scatter.label.set_fontsize(12)

    # axlcut = plt.axes([0.2, 0.1, 1, 1])
    # iplcut = InsetPosition(axctrl, [0.5, 0.4, 0.3, 0.2])
    # axlcut.set_axes_locator(iplcut)
    # linecut = Button(axlcut, 'Linecut', color='skyblue', hovercolor='skyblue')
    # linecut.label.set_fontsize(12)

    # axrcut = plt.axes([0.3, 0.1, 1, 1])
    # iprcut = InsetPosition(axctrl, [0.5, 0.7, 0.3, 0.2])
    # axrcut.set_axes_locator(iprcut)
    # radcut = Button(axrcut, 'Radialcut', color='skyblue', hovercolor='skyblue')
    # radcut.label.set_fontsize(12)

    plt.show(block=True)

max_stars_repo_name: harrispirie/stmpy

max_issues_repo_name: harrispirie/stmpy

max_forks_repo_name: harrispirie/stmpy


text:
import numpy as np
import logging
import threading
from sklearn.metrics import confusion_matrix, classification_report
from imblearn.over_sampling import SMOTE, RandomOverSampler
from imblearn.under_sampling import NearMiss

class Metrics(object):
    def __init__(self, predicted_values, true_values):
        self.predicted_values = predicted_values
        self.true_values = true_values
        self.logger = logging.getLogger(__name__)

    def build_conf_matrix(self):
        """Build Confusion matrix given True values and Predicted Values.

        Returns: confusion matrix with specified labels.

        """
        pred_vals = []
        for prediction in self.predicted_values:
            pred_vals.append(np.argmax(prediction))

        print confusion_matrix(self.true_values, pred_vals, labels=[1, 2, 3, 4])

        self.logger.info("Confusion Matrix : {}".format(confusion_matrix(self.true_values, pred_vals,
                                                                         labels=[1, 2, 3, 4])))

    def get_classification_report(self):

        pred_vals = []
        for prediction in self.predicted_values:
            pred_vals.append(np.argmax(prediction))

        report = classification_report(self.true_values, pred_vals, labels=[1, 2, 3, 4])
        print report
        self.logger.info("Classification report : {}".format(report))

        return report

class ResampleDataset(object):

    def __init__(self, sampling_mode='random_over_sample'):
        self.sampling_mode = sampling_mode
        self.logger = logging.getLogger(__name__)

    def balance_classes(self, heart_signal, labels,):

        # heart_list = [item for sublist in heart_signal for item in sublist]
        heart_list = []
        labels_list = []
        for ind, sublist in enumerate(heart_signal):
            if labels[ind] < 0:
                labels[ind] = 4
            new_label_list = [int(x) for x in str(labels[ind])]*len(sublist)
            labels_list.append(new_label_list)
            for item in sublist:
                heart_list.append(item)

        flat_labels_list = [item for sublist in labels_list for item in sublist]

        self.logger.debug("Flat heart = {}, Flat labels = {}".format(len(heart_list), len(flat_labels_list)))
        assert len(heart_list) == len(flat_labels_list)

        heart_list = np.reshape(heart_list, (-1,1))

        heart_signal_res, labels_res = [], []
        if self.sampling_mode == 'under_sample':
            heart_signal_res, labels_res = self.undersample_data(X = heart_list, y = flat_labels_list)

        if self.sampling_mode == 'over_sample':
            heart_signal_res, labels_res = self.smote_oversample(X = heart_list, y = flat_labels_list)

        if self.sampling_mode == 'random_over_sample':
            heart_signal_res, labels_res = self.random_oversample(X = heart_list, y = flat_labels_list)

        return heart_signal_res, labels_res

    def undersample_data(self, X, y):
        under_sampler = NearMiss('majority', n_jobs=2)
        heart_signal_res, labels_res = under_sampler.fit_sample(X, y)

        heart_signal_res = np.reshape(heart_signal_res, (heart_signal_res.shape[0],))
        return heart_signal_res, labels_res

    def smote_oversample(self, X, y):
        sm = SMOTE(n_jobs=2)
        heart_signal_res, labels_res = sm.fit_sample(X, y)

        heart_signal_res = np.reshape(heart_signal_res, (heart_signal_res.shape[0],))
        return heart_signal_res, labels_res

    def random_oversample(self, X, y):
        ros = RandomOverSampler(random_state=42, ratio='all')
        heart_signal_res, labels_res = ros.fit_sample(X, y)

        heart_signal_res = np.reshape(heart_signal_res, (heart_signal_res.shape[0],))
        return heart_signal_res, labels_res

class ThreadSafe(object):
    """Make a generator thread safe.

    Takes an iterator/generator and makes it thread-safe by
    serializing call to the `next` method of given iterator/generator.

    """
    def __init__(self, it):
        self.it = it
        self.lock = threading.Lock()

    def __iter__(self):
        return self

    def next(self):  # Py2
        with self.lock:
            return self.it.next()

def threadsafe_generator(f):
    """A decorator that takes a generator function and makes it thread-safe.

    Args:
        f (generator): A generator function

    Returns:

    """
    def g(*a, **kw):
        return ThreadSafe(f(*a, **kw))
    return g

max_stars_repo_name: shashankpr/DeepSleep

max_issues_repo_name: shashankpr/DeepSleep

max_forks_repo_name: shashankpr/DeepSleep


text:
# Add global arguments
import random
import os
import logging
import numpy as np
from profiler.globalvar import *
from profiler.learner import StructureLearner
from profiler.data.embedding import EmbeddingEngine
from profiler.data.transformer import TransformEngine
from profiler.data.dataset import Dataset
from profiler.utility import GlobalTimer
import matplotlib
# matplotlib.use("Agg")

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

logging.basicConfig(
    format="%(asctime)s - [%(levelname)5s] - %(message)s", datefmt='%H:%M:%S')
root_logger = logging.getLogger()
gensim_logger = logging.getLogger('gensim')
root_logger.setLevel(logging.INFO)
gensim_logger.setLevel(logging.WARNING)

# Arguments for Profiler
arguments = [
    (('-u', '--db_user'),
        {'metavar': 'DB_USER',
         'dest': 'db_user',
         'default': 'profileruser',
         'type': str,
         'help': 'User for DB used to persist state.'}),
    (('-p', '--db-pwd', '--pass'),
        {'metavar': 'DB_PWD',
         'dest': 'db_pwd',
         'default': 'abcd1234',
         'type': str,
         'help': 'Password for DB used to persist state.'}),
    (('-h', '--db-host'),
        {'metavar': 'DB_HOST',
         'dest': 'db_host',
         'default': 'localhost',
         'type': str,
         'help': 'Host for DB used to persist state.'}),
    (('-d', '--db_name'),
        {'metavar': 'DB_NAME',
         'dest': 'db_name',
         'default': 'profiler',
         'type': str,
         'help': 'Name of DB used to persist state.'}),
    (('-w', '--workers'),
     {'metavar': 'WORKERS',
      'dest': 'workers',
      'default': 1,
      'type': int,
      'help': 'How many workers to use for parallel execution. If <= 1, then no pool workers are used.'}),
    (('-p', '--null_policy'),
        {'metavar': 'DB_NAME',
         'dest': 'null_policy',
         'default': NULL_NEQ,
         'type': str,
         'help': 'Policy to handle null. [neq, eq, skip]'}),
    (('-s', '--seed'),
        {'metavar': 'SEED',
         'dest': 'seed',
         'default': 10,
         'type': int,
         'help': 'random seed'}),
    (('-t', '--tol'),
        {'metavar': 'TOL',
         'dest': 'tol',
         'default': 1e-6,
         'type': float,
         'help': 'tolerance for differences'}),
    (('-e', '--eps'),
        {'metavar': 'EPS',
         'dest': 'eps',
         'default': 0.01,
         'type': float,
         'help': "error bound for inverse_covariance estimation"}),
    (('-n', '--null'),
        {'metavar': 'NULL',
         'dest': 'null',
         'default': "_empty_",
         'type': str,
         'help': 'null values'}),
]

# Flags for Profiler
flags = [
    (tuple(['--verbose']),
        {'default': False,
         'dest': 'verbose',
         'action': 'store_true',
         'help': 'use DEBUG logging level if verbose enabled'}),
    (tuple(['--usedb']),
        {'default': False,
         'dest': 'usedb',
         'action': 'store_true',
         'help': 'use database engine'}),
    (tuple(['--embedtxt']),
        {'default': False,
         'dest': 'embedtxt',
         'action': 'store_true',
         'help': 'use language embedding for textual data'}),
    (tuple(['--inequality']),
        {'default': False,
         'dest': 'inequality',
         'action': 'store_true',
         'help': 'enable inequality operators'}),
    (tuple(['--continuous']),
     {'default': False,
      'dest': 'continuous',
      'action': 'store_true',
      'help': "use [0,1] instead of {0,1} for operator [EQ, NEQ, GT, LT] evaluation"
      }
     )
]

class Profiler(object):
    """
    Main Entry Point for Profiler
    """

    def __init__(self, **kwargs):
        """
        Constructor for Holoclean
        :param kwargs: arguments for HoloClean
        """

        # Initialize default execution arguments
        arg_defaults = {}
        for arg, opts in arguments:
            if 'directory' in arg[0]:
                arg_defaults['directory'] = opts['default']
            else:
                arg_defaults[opts['dest']] = opts['default']

        # Initialize default execution flags
        for arg, opts in flags:
            arg_defaults[opts['dest']] = opts['default']

        # check env vars
        for arg, opts in arguments:
            # if env var is set use that
            if opts["metavar"] and opts["metavar"] in os.environ.keys():
                logging.debug(
                    "Overriding {} with env varible {} set to {}".format(
                        opts['dest'],
                        opts["metavar"],
                        os.environ[opts["metavar"]])
                )
                arg_defaults[opts['dest']] = os.environ[opts["metavar"]]

        # Override defaults with manual flags
        for key in kwargs:
            arg_defaults[key] = kwargs[key]

        # Initialize additional arguments
        for (arg, default) in arg_defaults.items():
            setattr(self, arg, kwargs.get(arg, default))

        # Init empty session collection
        self.session = Session(arg_defaults)

class Session(object):

    def __init__(self, env, name="session"):
        """
        Constructor for Profiler session
        :param env: Profiler environment
        :param name: Name for the Profiler session
        """
        # use DEBUG logging level if verbose enabled
        if env['verbose']:
            root_logger.setLevel(logging.DEBUG)
            gensim_logger.setLevel(logging.DEBUG)

        logging.debug('initiating session with parameters: %s', env)

        # Initialize random seeds.
        random.seed(env['seed'])
        np.random.seed(seed=env['seed'])

        # Initialize members
        self.embed = None
        self.training_data = None
        self.name = name
        self.env = env
        self.timer = GlobalTimer()
        self.ds = Dataset(name, env)
        self.trans_engine = TransformEngine(env, self.ds)
        self.struct_engine = StructureLearner(self, env, self.ds)
        #self.eval_engine = EvalEngine(env, self.ds)

    def load_data(self, name="", src=FILE, fpath='', df=None, check_param=False, **kwargs):
        """
        load_data takes the filepath to a CSV file to load as the initial dataset.
        :param name: (str) name to initialize dataset with.
        :param src: (str) input source ["file", "df", "db"]
        :param fpath: (str) filepath to CSV file.
        :param kwargs: 'na_values', 'header', 'dropcol', 'encoding', 'nan' (representation for null values)
        """
        self.timer.time_start('Load Data')
        self.ds.load_data(name, src, fpath, df, check_param, **kwargs)
        self.timer.time_end('Load Data')

    def change_operators(self, names, ops):
        self.ds.change_operators(names, ops)

    def change_dtypes(self, names, types, regexs=None):
        if regexs is None:
            regexs = [None] * len(names)
        self.ds.change_dtypes(names, types, regexs)

    def load_embedding(self, **kwargs):
        self.timer.time_start('Load Embedding')
        if not self.embed:
            self.embed = EmbeddingEngine(self.env, self.ds)
        self.embed.train(**kwargs)
        self.timer.time_end('Load Embedding')

    def load_training_data(self, multiplier=None, sample_frac=1, difference=True):
        self.timer.time_start('Create Training Data')
        self.trans_engine.create_training_data(multiplier=multiplier, sample_frac=sample_frac,
                                               embed=self.embed, difference=difference)
        self.timer.time_end('Create Training Data')

    def learn_structure(self, **kwargs):
        self.timer.time_start('Learn Structure')
        results = self.struct_engine.learn(**kwargs)
        self.timer.time_end('Learn Structure')
        return results

    def get_dependencies(self, heatmap=None, score="training_data_vio_ratio", write_to='FDs'):
        self.timer.time_start('Get Dependencies')
        results = self.struct_engine.get_dependencies(
            heatmap=heatmap, score=score, write_to=write_to)
        self.timer.time_end('Get Dependencies')
        print(results)
        return results

    def get_corelations(self, heatmap=None, write_to=None):
        self.timer.time_start('Get Corelations')
        results = self.struct_engine.get_corelations(
            heatmap=heatmap, write_to=write_to)
        self.timer.time_end('Get Corelations')
        print(results)
        return results

    def visualize_covariance(self, filename='Covariance Matrix', write_pairs_file=None):
        self.struct_engine.visualize_covariance(
            filename=filename, write_pairs_file=write_pairs_file)

    def visualize_inverse_covariance(self, filename='Inverse Covariance Matrix'):
        self.struct_engine.visualize_inverse_covariance(filename=filename)

    def visualize_autoregression(self, filename='Autoregression Matrix'):
        self.struct_engine.visualize_autoregression(filename=filename)

max_stars_repo_name: rekords-uw/Profiler-public

max_issues_repo_name: rekords-uw/Profiler-public

max_forks_repo_name: rekords-uw/Profiler-public


text:
# https://github.com/methylDragon/opencv-python-reference/blob/master/04%20OpenCV%20Machine%20Learning%20and%20AI%20Detectors.md
# Source: https://docs.opencv.org/3.4.4/d5/d26/tutorial_py_knn_understanding.html

# help(cv.FUNCTION_YOU_NEED_HELP_WITH)

import numpy as np
import cv2 as cv
from matplotlib import pyplot as plt

X = np.random.randint(25, 50, (25, 2))
Y = np.random.randint(60, 85, (25, 2))
Z = np.vstack((X, Y))

# convert to np.float32
Z = np.float32(Z)

# define criteria and apply kmeans()
criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 10, 1.0)
ret, label, center = cv.kmeans(Z, 2, None, criteria, 10, cv.KMEANS_RANDOM_CENTERS)

# Now separate the data, Note the flatten()
A = Z[label.ravel() == 0]
B = Z[label.ravel() == 1]

# Plot the data
plt.scatter(A[:, 0], A[:, 1])
plt.scatter(B[:, 0], B[:, 1], c="r")
plt.scatter(center[:, 0], center[:, 1], s=80, c="y", marker="s")
plt.xlabel("Height"), plt.ylabel("Weight")

plt.show()

max_stars_repo_name: vladiant/OpenCVMLsamples

max_issues_repo_name: vladiant/OpenCVMLsamples

max_forks_repo_name: vladiant/OpenCVMLsamples


text:
# Copyright (c) OpenMMLab. All rights reserved.
import rflib
import numpy as np
from concurrent import futures as futures
from os import path as osp
from scipy import io as sio

def random_sampling(points, num_points, replace=None, return_choices=False):
    """Random sampling.

    Sampling point cloud to a certain number of points.

    Args:
        points (ndarray): Point cloud.
        num_points (int): The number of samples.
        replace (bool): Whether the sample is with or without replacement.
        return_choices (bool): Whether to return choices.

    Returns:
        points (ndarray): Point cloud after sampling.
    """

    if replace is None:
        replace = (points.shape[0] < num_points)
    choices = np.random.choice(points.shape[0], num_points, replace=replace)
    if return_choices:
        return points[choices], choices
    else:
        return points[choices]

class SUNRGBDInstance(object):

    def __init__(self, line):
        data = line.split(' ')
        data[1:] = [float(x) for x in data[1:]]
        self.classname = data[0]
        self.xmin = data[1]
        self.ymin = data[2]
        self.xmax = data[1] + data[3]
        self.ymax = data[2] + data[4]
        self.box2d = np.array([self.xmin, self.ymin, self.xmax, self.ymax])
        self.centroid = np.array([data[5], data[6], data[7]])
        self.w = data[8]
        self.l = data[9]  # noqa: E741
        self.h = data[10]
        self.orientation = np.zeros((3, ))
        self.orientation[0] = data[11]
        self.orientation[1] = data[12]
        self.heading_angle = -1 * np.arctan2(self.orientation[1],
                                             self.orientation[0])
        self.box3d = np.concatenate([
            self.centroid,
            np.array([self.l * 2, self.w * 2, self.h * 2, self.heading_angle])
        ])

class SUNRGBDData(object):
    """SUNRGBD data.

    Generate scannet infos for sunrgbd_converter.

    Args:
        root_path (str): Root path of the raw data.
        split (str): Set split type of the data. Default: 'train'.
        use_v1 (bool): Whether to use v1. Default: False.
    """

    def __init__(self, root_path, split='train', use_v1=False):
        self.root_dir = root_path
        self.split = split
        self.split_dir = osp.join(root_path, 'sunrgbd_trainval')
        self.classes = [
            'bed', 'table', 'sofa', 'chair', 'toilet', 'desk', 'dresser',
            'night_stand', 'bookshelf', 'bathtub'
        ]
        self.cat2label = {cat: self.classes.index(cat) for cat in self.classes}
        self.label2cat = {
            label: self.classes[label]
            for label in range(len(self.classes))
        }
        assert split in ['train', 'val', 'test']
        split_file = osp.join(self.split_dir, f'{split}_data_idx.txt')
        rflib.check_file_exist(split_file)
        self.sample_id_list = map(int, rflib.list_from_file(split_file))
        self.image_dir = osp.join(self.split_dir, 'image')
        self.calib_dir = osp.join(self.split_dir, 'calib')
        self.depth_dir = osp.join(self.split_dir, 'depth')
        if use_v1:
            self.label_dir = osp.join(self.split_dir, 'label_v1')
        else:
            self.label_dir = osp.join(self.split_dir, 'label')

    def __len__(self):
        return len(self.sample_id_list)

    def get_image(self, idx):
        img_filename = osp.join(self.image_dir, f'{idx:06d}.jpg')
        return rflib.imread(img_filename)

    def get_image_shape(self, idx):
        image = self.get_image(idx)
        return np.array(image.shape[:2], dtype=np.int32)

    def get_depth(self, idx):
        depth_filename = osp.join(self.depth_dir, f'{idx:06d}.mat')
        depth = sio.loadmat(depth_filename)['instance']
        return depth

    def get_calibration(self, idx):
        calib_filepath = osp.join(self.calib_dir, f'{idx:06d}.txt')
        lines = [line.rstrip() for line in open(calib_filepath)]
        Rt = np.array([float(x) for x in lines[0].split(' ')])
        Rt = np.reshape(Rt, (3, 3), order='F').astype(np.float32)
        K = np.array([float(x) for x in lines[1].split(' ')])
        K = np.reshape(K, (3, 3), order='F').astype(np.float32)
        return K, Rt

    def get_label_objects(self, idx):
        label_filename = osp.join(self.label_dir, f'{idx:06d}.txt')
        lines = [line.rstrip() for line in open(label_filename)]
        objects = [SUNRGBDInstance(line) for line in lines]
        return objects

    def get_infos(self, num_workers=4, has_label=True, sample_id_list=None):
        """Get data infos.

        This method gets information from the raw data.

        Args:
            num_workers (int): Number of threads to be used. Default: 4.
            has_label (bool): Whether the data has label. Default: True.
            sample_id_list (list[int]): Index list of the sample.
                Default: None.

        Returns:
            infos (list[dict]): Information of the raw data.
        """

        def process_single_scene(sample_idx):
            print(f'{self.split} sample_idx: {sample_idx}')
            # convert depth to points
            SAMPLE_NUM = 50000
            # TODO: Check whether can move the point
            #  sampling process during training.
            pc_upright_depth = self.get_depth(sample_idx)
            pc_upright_depth_subsampled = random_sampling(
                pc_upright_depth, SAMPLE_NUM)

            info = dict()
            pc_info = {'num_features': 6, 'lidar_idx': sample_idx}
            info['point_cloud'] = pc_info

            rflib.mkdir_or_exist(osp.join(self.root_dir, 'points'))
            pc_upright_depth_subsampled.tofile(
                osp.join(self.root_dir, 'points', f'{sample_idx:06d}.bin'))

            info['pts_path'] = osp.join('points', f'{sample_idx:06d}.bin')
            img_path = osp.join('image', f'{sample_idx:06d}.jpg')
            image_info = {
                'image_idx': sample_idx,
                'image_shape': self.get_image_shape(sample_idx),
                'image_path': img_path
            }
            info['image'] = image_info

            K, Rt = self.get_calibration(sample_idx)
            calib_info = {'K': K, 'Rt': Rt}
            info['calib'] = calib_info

            if has_label:
                obj_list = self.get_label_objects(sample_idx)
                annotations = {}
                annotations['gt_num'] = len([
                    obj.classname for obj in obj_list
                    if obj.classname in self.cat2label.keys()
                ])
                if annotations['gt_num'] != 0:
                    annotations['name'] = np.array([
                        obj.classname for obj in obj_list
                        if obj.classname in self.cat2label.keys()
                    ])
                    annotations['bbox'] = np.concatenate([
                        obj.box2d.reshape(1, 4) for obj in obj_list
                        if obj.classname in self.cat2label.keys()
                    ],
                                                         axis=0)
                    annotations['location'] = np.concatenate([
                        obj.centroid.reshape(1, 3) for obj in obj_list
                        if obj.classname in self.cat2label.keys()
                    ],
                                                             axis=0)
                    annotations['dimensions'] = 2 * np.array([
                        [obj.l, obj.w, obj.h] for obj in obj_list
                        if obj.classname in self.cat2label.keys()
                    ])  # lwh (depth) format
                    annotations['rotation_y'] = np.array([
                        obj.heading_angle for obj in obj_list
                        if obj.classname in self.cat2label.keys()
                    ])
                    annotations['index'] = np.arange(
                        len(obj_list), dtype=np.int32)
                    annotations['class'] = np.array([
                        self.cat2label[obj.classname] for obj in obj_list
                        if obj.classname in self.cat2label.keys()
                    ])
                    annotations['gt_boxes_upright_depth'] = np.stack(
                        [
                            obj.box3d for obj in obj_list
                            if obj.classname in self.cat2label.keys()
                        ],
                        axis=0)  # (K,8)
                info['annos'] = annotations
            return info

        sample_id_list = sample_id_list if \
            sample_id_list is not None else self.sample_id_list
        with futures.ThreadPoolExecutor(num_workers) as executor:
            infos = executor.map(process_single_scene, sample_id_list)
        return list(infos)

reason: import numpy,from scipy


text:
import paho.mqtt.client as mqtt
import os
import serial
import time
import random
from time import strftime
from datetime import datetime
import requests
import json
import schedule

import numpy as np
import tensorflow as tf
import random
import time

model2 = tf.keras.models.load_model('./my_model')

def on_message(client, obj, msg):
    print(msg.topic + " " + str(msg.qos) + " " + str(msg.payload))

def on_publish(client, obj, mid):
    print("mid: " + str(mid))

# gettiing dict with temperature, date and icon for forecast
def day_forecast():
    temp_day = []
    for i in forecast_response['list']:
        foo = '12:00:00'
        if foo in i['dt_txt']:
            dictor = {
                'date': i['dt'],
                'temp': i['main']['temp'],
                'icon': i['weather'][0]['icon'],
                'date_txt': i['dt_txt']
            }
            temp_day.append(dictor)

    # This for loop is selecting all DT from respoonse and making list of it
    temport = []
    for d in temp_day:
        temport.append(d['date'])

    # This loop converting timestamp DT format to week days names and making list of it
    dates_formated = []
    for value in temport:
        dates_formated.append(
            datetime.utcfromtimestamp(value).strftime('%A'))

    return [temp_day, dates_formated]

def night_forecast():
    temp_night = []
    for i in forecast_response['list']:
        foo = '03:00:00'
        if foo in i['dt_txt']:
            dictor = {
                'date': i['dt_txt'],
                'temp': i['main']['temp'],
            }
            temp_night.append(dictor)
    return temp_night

def send_mail(city, temperature, humidity, pressure, wind, description):
    import smtplib
    from email.mime.multipart import MIMEMultipart
    from email.mime.text import MIMEText
    mail= MIMEMultipart()

    sender_email = "sender@gmail.com" # replace with sender mail
    rec_email = "reciver@gmail.com" # replace with reciver mail
    password = "Passwd" # replace with sender mail password

    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.starttls()
    server.login(sender_email, password)

    mail['From']='Weather Notification System'
    mail['To'] = rec_email
    mail['Subject']='Weather App – Alert'

    city = city
    temperature = str(temperature)+ " C"
    humidity = str(humidity) + " %"
    pressure = str(pressure) + " hPa"
    wind = str(wind) + " m/s"
    description = description

    body=" City: "+str(city)+"\n Temperature: "+str(temperature)+"\n Humidity: "+str(humidity)+"\n Pressure: "+str(pressure)+"\n Wind: "+str(wind)+"\n Description: "+ str(description)

    mail.attach(MIMEText(body,'plain'))
    msg=mail.as_string()
    server.sendmail(sender_email, rec_email, msg)
    print('Mail Sent')

email = "Email Will Send Your Mail."

def email12():
    global email
    email = "Email Send At 12PM. Please Check Your Mail."

def email06():
    global email
    email = "Email Send At 06PM. Please Check Your Mail."

schedule.every().day.at("00:00").do(lambda: send_mail(city_float, temp_float, hum_float, pre_float, wind_float, des_float))
schedule.every().day.at("18:00").do(lambda: send_mail(city_float, temp_float, hum_float, pre_float, wind_float, des_float))

schedule.every().day.at("00:00").do(email12)
schedule.every().day.at("18:00").do(email06)

def generate_sensor_data():
    global temp, hum, pre

    temp = random.randint(20, 30)
    hum = random.randint(60, 90)
    pre = random.randint(1000, 1120)

def predict(temp_float, hum_float, pre_float):
    input = np.array([[temp_float, hum_float, pre_float]])
    pred = model2.predict_classes(input)
    suggestion = 0

    if pred == [1]:
        suggestion = "Most Probably Today Will Rain. So, Don't Miss Your Jacket."
    if pred == [2]:
        suggestion = "Most Probably Today Will Snow."
    else:
        suggestion = "I Cannot Predict Whether Rain or Snow."
    return suggestion

def check_temp(temp_float, temp):
    instuction = 0
    if temp_float > temp:
        instuction = "Outside Temperature Higher Than Inside."
    else:
        instuction = "Inside Temperature Higher Than Outside."
    return instuction

try:
    mqttc = mqtt.Client()
    mqttc.on_message = on_message
    mqttc.on_publish = on_publish

    # Connect
    mqttc.username_pw_set("mqtt_username", "mqtt_passwd")  # Replace with mqtt username and passwd
    mqttc.connect('AWS_E2C_IP_address', 1883, 60) # Replace your AWS E2C IP_address

    # Continue the network loop, exit when an error occurs
    while True :
        global temp_float, hum_float, pre_float, wind_float, city_float, des_float

        generate_sensor_data()

        API_KEY = '30ad27b312182fa9f7569003a337536b'

        # Replace your city name
        city = 'Dambulla'
        # getting api
        url = f'http://api.openweathermap.org/data/2.5/weather?q={city}&units=metric&appid={API_KEY}'

        response = requests.get(url).json()

        # If name of city is wrong spell or unknown
        if response.get('cod') != 200:
            message = response.get('message', '')

        weather = {
            'city': city,
            'temperature': response['main']['temp'],
            'humidity': response['main']['humidity'],
            'pressure': response['main']['pressure'],
            'wind': response['wind']['speed'],
            'description': response['weather'][0]['description'],
            'icon': response['weather'][0]['icon'],
        }

        temp_float = weather.get('temperature')
        hum_float = weather.get('humidity')
        pre_float = weather.get('pressure')
        wind_float = weather.get('wind')
        city_float = weather.get('city')
        des_float = weather.get('description')
        temp_int = round(temp_float)

        # This api is showing forecast for five days with days/nights
        url_forecast = f'http://api.openweathermap.org/data/2.5/forecast?q={city}&units=metric&appid={API_KEY}'

        forecast_response = requests.get(url_forecast).json()

        day = day_forecast()
        night = night_forecast()

        prediction = predict(temp_float, hum_float, pre_float)
        instuction = check_temp(temp_float, temp)

        # print(prediction)

        sensor = {
            "temp": temp,
            "hum": hum,
            "pre": pre
        }

        api = {
            "temperature": temp_int,
            "humidity": weather.get('humidity'),
            "pressure": weather.get('pressure'),
            "wind": weather.get('wind'),
            "city" :weather.get('city'),
            "description": weather.get('description'),
            "icon": weather.get('icon'),
            "prediction": prediction,
            "instuction": instuction,
            "email": email
        }

        forecast = {
            "day": day,
            "night": night
        }

        mqttc.publish("sensor", (json.dumps(sensor)))
        mqttc.publish("api", (json.dumps(api)))
        mqttc.publish("forecast", (json.dumps(forecast)))
        print('published')
        schedule.run_pending()
        time.sleep(1)

except:
    exit

max_stars_repo_name: LMJayasundara/IoT

max_issues_repo_name: LMJayasundara/IoT

max_forks_repo_name: LMJayasundara/IoT


text:
# Re-compute sfrs using new method from Av, useful for galaxies with lower limits of SFR_CORR
from astropy.io import ascii
import pandas as pd
import numpy as np
import initialize_mosdef_dirs as imd
from cosmology_calcs import flux_to_luminosity
import matplotlib.pyplot as plt

def convert_ha_to_sfr():
    '''Performs the whole process of adding the new sfr column to mosdef_all_cats_v2'''
    # Read in the dataframe and isolate the relevant columns
    ar_df, halpha_fluxes, avs_stellar, redshifts = read_ha_and_av()

    # Convert ha to luminsoty
    halpha_lums = flux_to_luminosity(halpha_fluxes, redshifts)

    #Add in a factor to go from AV_Stellar to AV_HII
    avs_balmer = correct_av_for_dust(avs_stellar)

    # Shift the AV from 5500 angstroms to 6565
    avs_balmer_6565 = apply_dust_law(avs_balmer, target_wave=6565)

    # Get dust-corrected halpha
    intrinsic_halpha_lums = correct_ha_lum_for_dust(halpha_lums, avs_balmer_6565)

    # Derive SFR from Kennicutt 1998
    halpha_sfrs = ha_lum_to_sfr(intrinsic_halpha_lums, imf='Hao_Chabrier')

    # Append the new sfrs to the catalog
    # Save the catalog
    ar_df['halpha_sfrs'] = halpha_sfrs
    ar_df.to_csv(imd.loc_axis_ratio_cat, index=False)

def add_use_sfr():
    """Adds the use_sfr column to the catalog, which will use the sfr_corr for most galaxies but the halpha_sfr for those without hbeta"""
    ar_df = ascii.read(imd.loc_axis_ratio_cat).to_pandas()

    # Set the new column to a all -999s
    ar_df['use_sfr'] = np.ones(len(ar_df)) * -999

    # Find where the Hbeta readings are good, use sfr_corr for that
    halpha_good = ar_df['ha_detflag_sfr'] == 0
    hbeta_good = ar_df['hb_detflag_sfr'] == 0
    use_corr = np.logical_and(halpha_good, hbeta_good)
    ar_df.loc[use_corr, 'use_sfr'] = ar_df[use_corr]['sfr_corr']

    print(len(ar_df[use_corr]))

    # Find where Halpha is good but Hbeta is not, and use the halpha sfrs for that
    use_hasfr = np.logical_and(halpha_good, ~hbeta_good)
    ar_df.loc[use_hasfr, 'use_sfr'] = ar_df[use_hasfr]['halpha_sfrs']

    print(len(ar_df[use_hasfr]))

    # ar_df = ar_df.rename(columns={"sfr": "sfr2", "err_sfr": "err_sfr2"})
    # ar_df = ar_df.rename(columns={"use_sfr": "sfr"})

    # ar_df.to_csv(imd.loc_axis_ratio_cat, index=False)

def read_ha_and_av():
    """Reads in the catalog"""
    ar_df = ascii.read(imd.loc_axis_ratio_cat).to_pandas()
    halpha_fluxes = ar_df['ha_flux']
    avs_stellar = ar_df['AV']
    redshifts = ar_df['z']
    return ar_df, halpha_fluxes, avs_stellar, redshifts

def filter_ar_df(ar_df):
    '''Cut the catalog down to only those galaxies that we are using'''
    ar_df = ar_df[ar_df['agn_flag'] == 0]
    ar_df = ar_df[ar_df['z_qual_flag'] == 7]
    ar_df = ar_df[ar_df['ha_detflag_sfr'] == 0]
    ar_df = ar_df[ar_df['hb_detflag_sfr'] == 0]

    return ar_df

def correct_av_for_dust(avs_stellar):
    """Find the formula in Reddy et al 2015"""
    avs_balmer = avs_stellar
    return avs_balmer

def apply_dust_law(avs_balmer, target_wave=6565, R_V=4.05):
    """Apply a Calzetti dust law

    Parameters:
    avs_balmer (pd.DataFrame) = AV of the star forming regions, in the V band at 5500 angstroms
    target_wave (float) = wavelenght to correct the extinction to (angstrom)
    R_V (float): R_V value fromt he Calzetti law

    Returns:
    avs_balmer_target (pd.DataFrame): Extinction now shifted to target wavelength
    """
    target_wave_um = target_wave / 10**4
    # Compute k(lambda) using Calzetti
    if target_wave < 6360:
        k_lambda = 2.659*(-2.156 + 1.509/target_wave_um - 0.198/(target_wave_um**2) + 0.011/(target_wave_um**3))+R_V
    elif target_wave >= 6360:
        k_lambda = 2.659*(-1.857 + 1.040/(target_wave_um)) + R_V

    avs_balmer_target = (k_lambda * avs_balmer)/R_V

    return avs_balmer_target

def correct_ha_lum_for_dust(obs_luminosity, attenuation):
    """Corrects observed luminsoity for attenuation

    Parameters:
    obs_luminosity (float): Observed luminosity, erg/s
    attenuation (float): Amount of aattenuation

    Returns:
    int_luminosity (float): Intrinsic luminosity
    """
    int_luminosity = obs_luminosity / (10**(-0.4 * attenuation))
    return int_luminosity

def ha_lum_to_sfr(intrinsic_halpha_lums, imf='Chabrier'):
    """Converts halpha luminosity to SFR, values taken from Brown et al 2017, SFR Indicators

    Parameters:
    intrinsic_halpha_lums (pd.DataFrame): Dust corrected halpha luminosities
    imf (str): Imf name used

    Returns:
    sfr (pd.DataFrame): Star formation rate (m_sun/yr)
    """
    if imf == 'Chabrier':
        sfr = 1.2 * 10**(-41) * intrinsic_halpha_lums
    if imf == 'Hao_Chabrier':  ## Hao et al 2013
        sfr = 10**(-41.056) * intrinsic_halpha_lums
    if imf == 'Salpeter':
        sfr = 7.9 * 10**(-42) * intrinsic_halpha_lums
    return sfr

def plot_sfrs():
    """Plots the old vs new sfrs"""
    ar_df, halpha_fluxes, avs_stellar, redshifts = read_ha_and_av()

    hb_flag_filt = ar_df['hb_detflag_sfr'] == 1

    fig, ax = plt.subplots(figsize=(8,8))

    high_sfrs_bad =  ar_df[hb_flag_filt]['sfr'] > 100

    ax.plot(ar_df[~hb_flag_filt]['sfr2'], ar_df[~hb_flag_filt]['halpha_sfrs'], color='black', marker='o', ls='None', label = 'H$_\\beta$ > 3 sigma')
    ax.plot(ar_df[hb_flag_filt]['sfr2'], ar_df[hb_flag_filt]['halpha_sfrs'], color='orange', marker='o', ls='None', label = 'H$_\\beta$ < 3 sigma')
    ax.plot(ar_df[hb_flag_filt][high_sfrs_bad]['sfr2'], ar_df[hb_flag_filt][high_sfrs_bad]['halpha_sfrs'], color='mediumseagreen', marker='o', ls='None', label = 'H$_\\beta$ < 3 sigma')
    ax.plot((0.001, 10**5), (0.001, 10**5), ls='--', color='blue')

    ax.set_xscale('log')
    ax.set_yscale('log')
    ax.set_xlabel('SFR2', fontsize=14)
    ax.set_xlim(0.1, 10000)
    ax.set_ylim(0.1, 10000)
    ax.set_ylabel('SFR from H$_\\alpha$', fontsize=14)
    ax.legend(fontsize=14)
    ax.tick_params(labelsize=12)
    fig.savefig(imd.axis_output_dir + '/sfr_diagnostics/sfr_comparison.pdf')

    # Figure similar to the one in Sedona's paper
    def sedona_plot(ar_df=ar_df):
        fig, ax = plt.subplots(figsize=(9,8))

        ar_df = filter_ar_df(ar_df)
        ar_df = ar_df[ar_df['z']>=1.7]
        ar_df = ar_df[ar_df['z']<=2.4]
        sfr_type = 'sfr_corr' #sfr_corr or sfr as strings

        high_sfrs_bad =  ar_df[hb_flag_filt][sfr_type] > 100

        filter_999s = np.logical_and(ar_df[~hb_flag_filt]['halpha_sfrs'] > 0, ar_df[~hb_flag_filt][sfr_type] > 0)

        sfr_ratio = ar_df[~hb_flag_filt][filter_999s]['halpha_sfrs'] / ar_df[~hb_flag_filt][filter_999s][sfr_type]

        color_map = plt.cm.get_cmap('RdYlBu')
        reversed_color_map = color_map.reversed()
        sc = ax.scatter(ar_df[~hb_flag_filt][filter_999s]['log_mass'], np.log10(sfr_ratio) , c=ar_df[~hb_flag_filt][filter_999s]['AV'], cmap = reversed_color_map, edgecolors='black', vmin=0, vmax=1.4)
        cbar = plt.colorbar(sc)
        ax.plot((8, 12), (0, 0), ls='--', color='black')

        ax.set_xlabel('Stellar Mass', fontsize=14)
        ax.set_xlim(8.7, 11.6)
        ax.set_ylim(-2.2, 1.3)

        ax.set_ylabel('log10(SFR H$_\\alpha$ / SFR Balmer)', fontsize=14)
        cbar.set_label('FAST AV', fontsize=14)
        ax.tick_params(labelsize=12)
        fig.savefig(imd.axis_output_dir + '/sfr_diagnostics/sfr_comparison_vsmass.pdf')
        return
    sedona_plot()

    fig, ax = plt.subplots(figsize=(8,8))

    bins = np.arange(0, 500, 25)

    ax.hist(ar_df[~hb_flag_filt]['sfr2'], bins=bins, color='black', label = 'H$_\\beta$ > 3 sigma')
    ax.hist(ar_df[hb_flag_filt]['sfr2'], bins=bins, color='orange', alpha=0.5, label = 'H$_\\beta$ < 3 sigma')
    ax.hist(ar_df[hb_flag_filt][high_sfrs_bad]['sfr2'], bins=bins, color='mediumseagreen', alpha=1, label = 'H$_\\beta$ < 3 sigma')

    ax.set_xlabel('SFR2', fontsize=14)
    ax.set_ylabel('Count', fontsize=14)
    ax.legend(fontsize=14)
    ax.tick_params(labelsize=12)
    fig.savefig(imd.axis_output_dir + '/sfr_diagnostics/sfr2_hist.pdf')

    ### DIagnostic plots for why sfr is so highsfr

    fig, ax = plt.subplots(figsize=(8,8))

    remove_999s_good = np.logical_and(ar_df[~hb_flag_filt]['err_hb_flux']>-999, ar_df[~hb_flag_filt]['hb_flux']>-999)
    remove_999s_bad = np.logical_and(ar_df[hb_flag_filt]['err_hb_flux']>-999, ar_df[hb_flag_filt]['hb_flux']>-999)

    ax.plot(ar_df[~hb_flag_filt][remove_999s_good]['err_hb_flux'], ar_df[~hb_flag_filt][remove_999s_good]['hb_flux'], color='black', marker='o', ls='None', label = 'H$_\\beta$ > 3 sigma')
    ax.plot(ar_df[hb_flag_filt][remove_999s_bad]['err_hb_flux'], ar_df[hb_flag_filt][remove_999s_bad]['hb_flux'], color='orange', marker='o', ls='None', label = 'H$_\\beta$ < 3 sigma')
    ax.plot(ar_df[hb_flag_filt][high_sfrs_bad]['err_hb_flux'], ar_df[hb_flag_filt][high_sfrs_bad]['hb_flux'], color='mediumseagreen', marker='o', ls='None', label = 'H$_\\beta$ < 3 sigma')
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    ax.plot((0.001*10**-18, 10**5), (0.003*10**-18, 3*10**5), ls='--', color='blue', label='Flux = 3*error')
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)

    ax.set_xlabel('Hbeta flux error', fontsize=14)
    ax.set_ylabel('Hbeta flux', fontsize=14)
    ax.legend(fontsize=14)
    ax.tick_params(labelsize=12)
    fig.savefig(imd.axis_output_dir + '/sfr_diagnostics/sfr_hbflux_hbflxuerr.pdf')

    fig, ax = plt.subplots(figsize=(8,8))

    remove_999s_good = np.logical_and(ar_df[~hb_flag_filt]['err_ha_flux']>-999, ar_df[~hb_flag_filt]['ha_flux']>-999)
    remove_999s_bad = np.logical_and(ar_df[hb_flag_filt]['err_ha_flux']>-999, ar_df[hb_flag_filt]['ha_flux']>-999)

    ax.plot(ar_df[~hb_flag_filt][remove_999s_good]['err_ha_flux'], ar_df[~hb_flag_filt][remove_999s_good]['ha_flux'], color='black', marker='o', ls='None', label = 'H$_\\beta$ > 3 sigma')
    ax.plot(ar_df[hb_flag_filt][remove_999s_bad]['err_ha_flux'], ar_df[hb_flag_filt][remove_999s_bad]['ha_flux'], color='orange', marker='o', ls='None', label = 'H$_\\beta$ < 3 sigma')
    ax.plot(ar_df[hb_flag_filt][high_sfrs_bad]['err_ha_flux'], ar_df[hb_flag_filt][high_sfrs_bad]['ha_flux'], color='mediumseagreen', marker='o', ls='None', label = 'H$_\\beta$ < 3 sigma')

    ax.set_xlabel('Halpha flux error', fontsize=14)
    ax.set_ylabel('Halpha flux', fontsize=14)
    ax.legend(fontsize=14)
    ax.tick_params(labelsize=12)
    ax.set_xlim(-0.1*10**(-17), 4*10**(-17))
    fig.savefig(imd.axis_output_dir + '/sfr_diagnostics/sfr_haflux_haflxuerr.pdf')

    fig, ax = plt.subplots(figsize=(8,8))

    remove_999s_good = np.logical_and(ar_df[~hb_flag_filt]['ha_flux']>-999, ar_df[~hb_flag_filt]['hb_flux']>-999)
    remove_999s_bad = np.logical_and(ar_df[hb_flag_filt]['ha_flux']>-999, ar_df[hb_flag_filt]['hb_flux']>-999)

    ax.plot(ar_df[~hb_flag_filt][remove_999s_good]['ha_flux'], ar_df[~hb_flag_filt][remove_999s_good]['hb_flux'], color='black', marker='o', ls='None', label = 'H$_\\beta$ > 3 sigma')
    ax.plot(ar_df[hb_flag_filt][remove_999s_bad]['ha_flux'], ar_df[hb_flag_filt][remove_999s_bad]['hb_flux'], color='orange', marker='o', ls='None', label = 'H$_\\beta$ < 3 sigma')
    ax.plot(ar_df[hb_flag_filt][high_sfrs_bad]['ha_flux'], ar_df[hb_flag_filt][high_sfrs_bad]['hb_flux'], color='mediumseagreen', marker='o', ls='None', label = 'H$_\\beta$ < 3 sigma')

    # 2.86 theory line
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    ax.plot((0, 2.86*10**-(15)), (0, 1*10**(-15)), ls='--', color='blue', label='Balmer dec = 2.86')
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)

    ax.set_xlabel('Halpha flux', fontsize=14)
    ax.set_ylabel('Hbeta flux', fontsize=14)
    ax.legend(fontsize=14)
    ax.tick_params(labelsize=12)
    fig.savefig(imd.axis_output_dir + '/sfr_diagnostics/sfr_haflux_hbflux.pdf')

# convert_ha_to_sfr()
# add_use_sfr()
plot_sfrs()

max_stars_repo_name: brianlorenz/code

max_issues_repo_name: brianlorenz/code

max_forks_repo_name: brianlorenz/code

reason: import numpy,from astropy


text:
import tensorflow as tf

import numpy as np
import json
import sys
import os
import argparse

sys.path.append(os.path.abspath('../lib/'))
from dataloader.dataloader import dataloader
from model.cnnmodel import CNN
from module.clusters import create_msg
from evaluation.evaluation import ClusterEvaluation
from kit.messager import messager

def train_CNN(train_data_file,test_data_file,wordvec_file,load_model_name,save_model_name,
    loss_type,max_len=120, pos_emb_dim=5,batch_size=100,batch_num=1000,epoch_num=1,val_size=1000):

    # preparing saving files
    save_path = os.path.join('model_file',save_model_name).replace('\\','/')
    if not os.path.exists(save_path):
        os.makedirs(save_path)

    # train data loading
    print('-----Data Loading-----')
    dataloader_train = dataloader(train_data_file, wordvec_file, max_len=max_len)
    dataloader_test = dataloader(test_data_file, wordvec_file, max_len=max_len)
    word_emb_dim = dataloader_train._word_emb_dim_()
    word_vec_mat = dataloader_train._word_vec_mat_()
    print('word_emb_dim is {}'.format(word_emb_dim))

    # compile model
    print('-----Model Intializing-----')
    config = tf.ConfigProto()
    config.gpu_options.per_process_gpu_memory_fraction = 0.5
    config.gpu_options.allow_growth = True
    sess = tf.Session(config = config)
    cnn = CNN(session=sess,word_vec_mat=word_vec_mat,max_len=max_len, pos_emb_dim=pos_emb_dim,dropout=0.2)
    cnn.set_ph(batch_size=batch_size)
    cnn.set_train_op(loss_type=loss_type,p_mult=0.02)
    cnn.init_model()

    print('-----Testing Data Preparing-----')

    # preparing testing samples
    val_testset_input, val_testset_label = dataloader_test.next_batch_cnn(val_size)
    val_trainset_input, val_trainset_label = dataloader_train.next_batch_cnn(val_size)

    # intializing parameters
    batch_num_list = [batch_num]

    for epoch in range(epoch_num):
        # preparing message lists
        msger = messager(save_path=save_path,types=['batch_num','train_acc','train_l','test_acc','test_l'], json_name='CNNmsg'+str(epoch)+'.json')

        print('------epoch {}------'.format(epoch))
        print('max batch num to train is {}'.format(batch_num_list[epoch]))
        for i in range(batch_num_list[epoch]):
            # training
            cnn.train(dataloader_train)

            # testing and saving
            if i % 10 == 0:
                print('temp_batch_num: ', i,' total_batch_num: ', batch_num_list[epoch])
            if i % 100 == 0:
                print('model_name',save_model_name)
                print('trainset:')
                val_trainset_info = cnn.validation(val_trainset_input, val_trainset_label)
                print('testset:')
                val_testset_info = cnn.validation(val_testset_input, val_testset_label)
                msger.record_message((i,)+val_trainset_info+val_testset_info)
                msger.save_json()
                cnn.save_model(save_path=save_path,global_step=i)
                print('model and messages saved.')

        # Clustering
        print('Data to cluster loading...')
        msger = messager(save_path=save_path,types=['method','F1','precision','recall','msg'], json_name='cluster_msg'+str(epoch)+'.json')
        data_to_cluster, gt = dataloader_test._data_()
        for i,item in enumerate(gt):
            gt[i]=dataloader_test.relid_dict[item]

        print('-----CNN Clustering-----')
        cluster_result = cnn.pred_X(data_to_cluster)
        cluster_result = np.squeeze(cluster_result).tolist()
        cluster_msg = create_msg(cluster_result)

        print('Evaluating...')
        cluster_eval = ClusterEvaluation(gt,cluster_result).printEvaluation()
        msger.record_message(['CNN',cluster_eval['F1'],cluster_eval['precision'],
            cluster_eval['recall'],cluster_msg])
        msger.save_json()
        print(cluster_eval)
        print('clustering messages saved.')

        print("-----End-----")
        print("The model name is:",save_model_name)
        print("loss type is:",loss_type)

if __name__=='__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--gpu",type=str,default='0')
    parser.add_argument("--train_data_file",type=str,default='../data-bin/fewrel_ori/fewrel80_test_train.json')
    parser.add_argument("--test_data_file",type=str,default='../data-bin/fewrel_ori/fewrel80_test_test.json')
    parser.add_argument("--wordvec_file",type=str,default='../data-bin/wordvec/word_vec.json')
    parser.add_argument("--load_model_name",type=str,default=None)
    parser.add_argument("--save_model_name",type=str,default="CNN")
    parser.add_argument("--loss_type",type=str,default='none')
    parser.add_argument("--batch_num",type=str,default=2000)
    parser.add_argument("--val_size",type=str,default=1000)
    args = parser.parse_args()

    os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu

    train_CNN(
        train_data_file=args.train_data_file,
        test_data_file=args.test_data_file,
        wordvec_file=args.wordvec_file,
        load_model_name=args.load_model_name,
        save_model_name=args.save_model_name,
        loss_type=args.loss_type,
        batch_num=args.batch_num,
        val_size=args.val_size)


text:
from __future__ import division, print_function, absolute_import
import os
os.chdir('/home/orram/Documents/GitHub/imagewalker')
import numpy as np
import cv2
import misc
from RL_networks import Stand_alone_net
import pickle
import importlib
import matplotlib.pyplot as plt

# PyTorch libraries and modules
import torchvision
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from torch.autograd import Variable
from torch.nn import Linear, ReLU, CrossEntropyLoss, Sequential, Conv2d, MaxPool2d, Module, Softmax, BatchNorm2d, Dropout
from torch.optim import Adam, SGD
import torch.nn as nn
from mnist import MNIST

mnist = MNIST('/home/orram/Documents/datasets/MNIST/')
images, labels = mnist.load_training()

# Network Parameters
size=None
padding_size=(128,128)
# num_input = padding_size[0]*padding_size[1] # MNIST data input (img shape: 28*28)
num_classes = None
# dropout = 0.25 # Dropout, probability to drop a unit

import SYCLOP_env as syc

#Define function for low resolution lens on syclop
def bad_res101(img,res):
    sh=np.shape(img)
    dwnsmp=cv2.resize(img,res, interpolation = cv2.INTER_CUBIC)
    upsmp = cv2.resize(dwnsmp,sh, interpolation = cv2.INTER_CUBIC)
    return upsmp
#%%
################ Show an example of a sample from the dataset #################

plt.figure()
plt.imshow(misc.build_mnist_padded(1./256*np.reshape(images[0],[1,28,28])))
plt.title('The padded img to feed to the Syclop')

img=misc.build_mnist_padded(1./256*np.reshape(images[45],[1,28,28]))

scene = syc.Scene(image_matrix=img)
sensor = syc.Sensor(winx=56,winy=56,centralwinx=28,centralwiny=28)
agent = syc.Agent(max_q = [scene.maxx-sensor.hp.winx,scene.maxy-sensor.hp.winy])

#Create a random walk coordinates over the image
starting_point = np.array([agent.max_q[0]//2,agent.max_q[1]//2])
steps  = []
for j in range(5):
    steps.append(starting_point*1)
    starting_point += np.random.randint(-5,5,2)

q_sequence = np.array(steps).astype(int)
print('The random ealk coordinate:',q_sequence)

#Run Syclop
sensor.hp.resolution_fun = lambda x: bad_res101(x,(28,28))
imim=[]
dimim=[]
agent.set_manual_trajectory(manual_q_sequence=q_sequence)
for t in range(10):
    agent.manual_act()
    sensor.update(scene, agent)
    imim.append(sensor.central_frame_view)
    dimim.append(sensor.central_dvs_view)

#Plot Syclop outputs
for i in range(5):
    plt.figure()
    plt.imshow(imim[i])

#%%
############### Create a Dataset from the Syclops visual inputs ###############
#We are starting with a simple time series where the syclop starts from the
#same starting point, at the middle of the img on the x axis
#and the middle - 10 pixles on the y axis - (middle_point, middle_point - 10)
#Each time step the syclop will move one pixle up on the y axis,
#to a final point at (middle_point, middle_point + 10) and make a
#circular movement in the x direction.

def create_dataset(images, labels, res, sample = 5, mixed_state = True):
    '''
    Creates a torch dataloader object of syclop outputs
    from a list of images and labels.

    Parameters
    ----------
    images : List object holding the images to proces
    labels : List object holding the labels
    res : resolution dawnsampling factor - to be used in cv.resize(orig_img, res)
    sample: the number of samples to have in syclop
    mixed_state : if False, use the same trajectory on every image.

    Returns
    -------
    train_dataloader, test_dataloader - torch DataLoader class objects

    '''
    count = 0
    ts_images = []
    dvs_images = []
    count = 0
    #create subplot to hold examples from the dataset
    fig, ax = plt.subplots(2,5)
    i = 0 #indexises for the subplot for image and for syclop vision
    for img in images:
        orig_img = np.reshape(img,[28,28])
        #Set the padded image
        img=misc.build_mnist_padded(1./256*np.reshape(img,[1,28,28]))

        if count < 5:
            ax[0,i].imshow(orig_img)
            plt.title(labels[count])
        #Set the sensor and the agent
        scene = syc.Scene(image_matrix=img)
        sensor = syc.Sensor(winx=56,winy=56,centralwinx=28,centralwiny=28)
        agent = syc.Agent(max_q = [scene.maxx-sensor.hp.winx,scene.maxy-sensor.hp.winy])
        #Setting the coordinates to visit
        starting_point = np.array([agent.max_q[0]//2,agent.max_q[1]//2])

        steps  = []
        for j in range(5):
            steps.append(starting_point*1)
            starting_point += np.random.randint(-5,5,2)

        if mixed_state:
            q_sequence = np.array(steps).astype(int)
        else:
            if count == 0:
                q_sequence = np.array(steps).astype(int)
        #Setting the resolution function - starting with the regular resolution
        sensor.hp.resolution_fun = lambda x: bad_res101(x,(res,res))
        #Create empty lists to store the syclops outputs
        imim=[]
        dimim=[]
        agent.set_manual_trajectory(manual_q_sequence=q_sequence)
        #Run Syclop for 20 time steps
        for t in range(5):
            agent.manual_act()
            sensor.update(scene, agent)
            imim.append(sensor.central_frame_view)
            dimim.append(sensor.central_dvs_view)
        #Create a unified matrix from the list
        if count < 5:
            ax[1,i].imshow(imim[0])
            plt.title(labels[count])
            i+=1

        imim = np.array(imim)
        dimim = np.array(dimim)
        #Add current proccessed image to lists
        ts_images.append(imim)
        dvs_images.append(dimim)
        count += 1

    ts_train = ts_images[:55_000]
    train_labels = labels[:55_000]
    ts_val = ts_images[55_000:]
    val_labels = labels[55_000:]

    dvs_train = dvs_images[:55_000]
    dvs_val = dvs_images[55_000:]

    class mnist_dataset(Dataset):
        def __init__(self, data, labels, transform = None):

            self.data = data
            self.labels = labels

            self.transform = transform
        def __len__(self):
            return len(self.data)

        def __getitem__(self, idx):
            '''
            args idx (int) :  index

            returns: tuple(data, label)
            '''
            data = self.data[idx]
            label = self.labels[idx]

            if self.transform:
                data = self.transform(data)
                return data, label
            else:
                return data, label

        def dataset(self):
            return self.data
        def labels(self):
            return self.labels

    train_dataset = mnist_dataset(dvs_train, train_labels)
    test_dataset = mnist_dataset(dvs_val, val_labels)
    batch = 64
    train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size = batch, shuffle = True)
    test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size = batch, shuffle = True)

    return train_dataloader, test_dataloader

class CNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(5,16,3,stride=1, padding=1)
        self.bn1 = nn.BatchNorm2d(16)
        self.conv2 = nn.Conv2d(16,32,3, stride=1, padding=2)
        self.bn2 = nn.BatchNorm2d(32)
        self.conv3 = nn.Conv2d(32,16,3,stride=1, padding=1)
        self.bn3 = nn.BatchNorm2d(16)

        self.pool = nn.MaxPool2d(2)

        #After the layers and pooling the first two we should get
        # 16,3,3
        #Flatting it we get:
        # 144

        self.fc1 = nn.Linear(8*8*16,64)
        self.fc2 = nn.Linear(64,10)

        self.relu = nn.ReLU()

    def forward(self, img):

        img = self.pool(self.relu(self.bn1(self.conv1(img.double()))))
        img = self.pool(self.relu(self.bn2(self.conv2(img))))
        img = self.relu(self.bn3(self.conv3(img)))
        #print(img.shape)
        img = img.view(img.shape[0],8*8*16)
        img = self.relu(self.fc1(img))
        img = self.fc2(img)

        return img

class CNN_one_layer(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1,16,3,stride=1, padding=1)
        self.bn1 = nn.BatchNorm2d(16)
        self.conv2 = nn.Conv2d(16,32,3, stride=1, padding=2)
        self.bn2 = nn.BatchNorm2d(32)
        self.conv3 = nn.Conv2d(32,16,3,stride=1, padding=1)
        self.bn3 = nn.BatchNorm2d(16)

        self.pool = nn.MaxPool2d(2)

        #After the layers and pooling the first two we should get
        # 16,3,3
        #Flatting it we get:
        # 144

        self.fc1 = nn.Linear(8*8*16,64)
        self.fc2 = nn.Linear(64,10)

        self.relu = nn.ReLU()

    def forward(self, img):
        img = img[:,0,:,:]
        img = img.unsqueeze(1)
        img = self.pool(self.relu(self.bn1(self.conv1(img.double()))))
        img = self.pool(self.relu(self.bn2(self.conv2(img))))
        img = self.relu(self.bn3(self.conv3(img)))
        #print(img.shape)
        img = img.view(img.shape[0],8*8*16)
        img = self.relu(self.fc1(img))
        img = self.fc2(img)

        return img

class RNN_Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1,16,3,stride=1, padding=1)
        self.bn1 = nn.BatchNorm2d(16)
        self.conv2 = nn.Conv2d(16,32,3, stride=1, padding=2)
        self.bn2 = nn.BatchNorm2d(32)
        self.conv3 = nn.Conv2d(32,16,3,stride=1, padding=1)
        self.bn3 = nn.BatchNorm2d(16)

        self.pool = nn.MaxPool2d(2)

        #After the layers and pooling the first two we should get
        # 16,3,3
        #Flatting it we get:
        # 144
        self.gru = nn.GRU(8*8*16,100)
        self.fc1 = nn.Linear(100,10)
        #self.fc2 = nn.Linear(6,10)

        self.relu = nn.ReLU()

    def forward(self, data):
        hn = torch.zeros([1,data.shape[0],100]).double()
        #print(data.shape)
        for i in range(data.shape[1]):
            img = data[:,i,:,:,:]
            #print(img.shape)
            #plt.figure()
            #plt.imshow(img[0][0])
            img = self.pool(self.relu(self.bn1(self.conv1(img.double()))))
            img = self.pool(self.relu(self.bn2(self.conv2(img))))
            img = self.relu(self.bn3(self.conv3(img)))
            img = img.view(img.shape[0],8*8*16)
            out, hn = self.gru(img.unsqueeze(0),hn)
            #print(out.shape)
        output = self.fc1(out[0, :, :])

        return output

class RNN_Net2(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1,4,3,stride=1, padding=1)
        self.bn1 = nn.BatchNorm2d(16)
        self.conv2 = nn.Conv2d(16,32,3, stride=1, padding=2)
        self.bn2 = nn.BatchNorm2d(32)
        self.conv3 = nn.Conv2d(32,16,3,stride=1, padding=1)
        self.bn3 = nn.BatchNorm2d(16)

        self.pool = nn.MaxPool2d(2)

        #After the layers and pooling the first two we should get
        # 16,3,3
        #Flatting it we get:
        # 144
        self.gru = nn.GRU(8*8*16,100, batch_first=True)
        self.fc1 = nn.Linear(100,10)
        #self.fc2 = nn.Linear(6,10)

        self.relu = nn.ReLU()

    def forward(self, data):
        batch_size, timesteps, C, H, W = data.size()
        img = data.view(batch_size * timesteps, C, H, W)
        img = self.pool(self.relu(self.bn1(self.conv1(img.double()))))
        img = self.pool(self.relu(self.bn2(self.conv2(img))))
        img = self.relu(self.bn3(self.conv3(img)))
        print(img.shape)
        img = img.view(img.shape[0],img.shape[1],8*8*16)
        out, hn = self.gru(img)
        output = self.fc1(hn.squeeze(0))

        return output

def train(train_dataloader, test_dataloader, net, epochs = 10):

    lr = 3e-3
    #net = CNN().double()
    optimizer = Adam(net.parameters(), lr = lr)
    loss_func = nn.CrossEntropyLoss()
    if torch.cuda.is_available():
        net = net.cuda()

    train_loss = []
    test_loss = []
    test_accur = []
    for epoch in range(epochs):

        batch_loss = []
        for batch_idx, (data,targets) in enumerate(train_dataloader):
            if torch.cuda.is_available():
                data = data.to('cuda', non_blocking=True)
                targets = targets.to('cuda', non_blocking = True)
            #print(batch_idx, data.shape, targets.shape)
            if net.__class__.__name__ == 'RNN_Net':
                data = data.unsqueeze(2)
            optimizer.zero_grad()
            output = net(data.double())
            loss = loss_func(output, targets)
            loss.backward()
            optimizer.step()
            batch_loss.append(loss.item())

        train_loss.append(np.mean(batch_loss))

        if epoch%1 == 0:
            correct = 0
            test_batch_loss = []
            test_accuracy = []
            for batch_idx, (test_data,test_targets) in enumerate(test_dataloader):
                if torch.cuda.is_available():
                    test_data = test_data.to('cuda', non_blocking=True)
                    test_targets = test_targets.to('cuda', non_blocking = True)
                #print(batch_idx, data.shape, targets.shape)
                if net.__class__.__name__ == 'RNN_Net':
                    test_data = test_data.unsqueeze(2)
                test_output = net(test_data)
                loss = loss_func(test_output, test_targets)
                test_batch_loss.append(loss.item())
                test_pred = test_output.data.max(1, keepdim = True)[1]
                correct = test_pred.eq(test_targets.data.view_as(test_pred)).sum()
                test_accuracy.append(100.*correct.to('cpu')/len(test_targets))

            print('Net',net.__class__.__name__,'Epoch : ',epoch+1, '\t', 'loss :', loss.to('cpu').item(), 'accuracy :',np.mean(test_accuracy) )
            test_loss.append(np.mean(test_batch_loss))
            test_accur.append(np.mean(test_accuracy))

    return train_loss, test_loss, test_accur
#%%
################## Mooving on to lower resolutions ############################

res = 6
epochs = 10
#Load the training data
train_dataloader, test_dataloader = create_dataset(images, labels,res = res)
#%%
print('Run the CNN over one image only')
train_loss, test_loss, test_accur =  train(train_dataloader, test_dataloader,net = CNN_one_layer().double(), epochs = epochs)

print('Run the CNN over the stacked images')
train_loss, test_loss, test_accur =  train(train_dataloader, test_dataloader,net = CNN().double(), epochs = epochs)

print('Run the CNN+RNN over the stacked images')
train_loss, test_loss, test_accur =  train(train_dataloader, test_dataloader,net = RNN_Net().double(), epochs = epochs)
#%%
#Check DVS output
print('Run the CNN over one image only')
train_loss, test_loss, test_accur =  train(train_dataloader, test_dataloader,net = CNN_one_layer().double(), epochs = 10)
print('Run the CNN over the stacked images')
train_loss, test_loss, test_accur =  train(train_dataloader, test_dataloader,net = CNN().double(), epochs = 10)
print('Run the CNN+RNN over the stacked images')
train_loss, test_loss, test_accur =  train(train_dataloader, test_dataloader,net = RNN_Net().double(), epochs = 10)

#%%
res_list = [10,8,6,5,4,3,2]
cnn_one_image = []
cnn_lasagna = []
cnn_rnn = []
cnn_one_image_train_loss = []
cnn_lasagna_train_loss = []
cnn_rnn_train_loss = []
for res in res_list:
    #Load the training data
    train_dataloader, test_dataloader = create_dataset(images, labels,res = res)
    #Run the CNN over one image only
    cnn1_train_loss, cnn1_test_loss, cnn1_test_accur =  train(train_dataloader, test_dataloader,net = CNN_one_layer().double(), epochs = 4)
    cnn_one_image.append(cnn1_test_accur[-1])
    cnn_one_image_train_loss.append(cnn1_train_loss[-1])
    #Run the CNN over the stacked images
    cnn_train_loss, cnn_test_loss, cnn_test_accur =  train(train_dataloader, test_dataloader,net = CNN().double(), epochs = 4)
    cnn_lasagna.append(cnn_test_accur[-1])
    cnn_lasagna_train_loss.append(cnn_train_loss[-1])
    #Run the CNN+RNN over the stacked images
    rnn_train_loss, rnn_test_loss, rnn_test_accur =  train(train_dataloader, test_dataloader,net = RNN_Net().double(), epochs = 4)
    cnn_rnn.append(rnn_test_accur[-1])
    cnn_rnn_train_loss.append(rnn_train_loss[-1])

plt.figure()
plt.plot(28/np.array(res_list),cnn_one_image,'o', label = 'CNN 1 img')
plt.plot(28/np.array(res_list),cnn_lasagna,'o', label = 'CNN lasagna')
plt.plot(28/np.array(res_list),cnn_rnn,'o', label = 'CNN RNN')
plt.title('Accuracy')
plt.xlabel('Degragation Factor (28/dwnsmple_size)')
plt.ylabel('Accuracy %')
plt.legend()
plt.savefig('Accuracy FIG Random Walk Changing Trajectories')

plt.figure()
plt.plot(28/np.array(res_list),cnn_one_image_train_loss,'o', label = 'CNN 1 img')
plt.plot(28/np.array(res_list),cnn_lasagna_train_loss, 'o',label = 'CNN lasagna')
plt.plot(28/np.array(res_list),cnn_rnn_train_loss, 'o',label = 'CNN RNN')
plt.title('Train Loss')
plt.xlabel('Degragation Factor (28/dwnsmple_size)')
plt.ylabel('Loss')
plt.legend()
plt.savefig('Train Loss FIG Random Walk Changing Trajectories')

max_stars_repo_name: PopGalacticHistory/imagewalker

max_issues_repo_name: PopGalacticHistory/imagewalker

max_forks_repo_name: PopGalacticHistory/imagewalker


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Classify all the test dataset through our CNN to get accuracy.
"""
import numpy as np
import operator
import random
import glob
import argparse
import os.path
from data import DataSet
from processor import process_image
from tensorflow.keras.models import load_model

import tensorflow as tf
import tensorflow.keras.backend as KTF

config = tf.ConfigProto()
config.gpu_options.allow_growth=True   #dynamic alloc GPU resource
config.gpu_options.per_process_gpu_memory_fraction = 0.3  #GPU memory threshold 0.3
session = tf.Session(config=config)

# set session
KTF.set_session(session)

def validate_cnn_model(model_file):
    data = DataSet()
    model = load_model(model_file)

    # Get all our test images.
    images = glob.glob(os.path.join('data', 'test_full', '**', '*.jpg'))

    # Count the correct predict
    result_count = 0

    for image in images:
        print('-'*80)
        # Get a random row.
        #sample = random.randint(0, len(images) - 1)
        #image = images[sample]

        # Get groundtruth class string
        class_str = image.split(os.path.sep)[-2]

        # Turn the image into an array.
        print(image)
        image_arr = process_image(image, (224, 224, 3))
        image_arr = np.expand_dims(image_arr, axis=0)

        # Predict.
        predictions = model.predict(image_arr)

        # Show how much we think it's each one.
        label_predictions = {}
        for i, label in enumerate(data.classes):
            label_predictions[label] = predictions[0][i]

        sorted_lps = sorted(label_predictions.items(), key=operator.itemgetter(1), reverse=True)

        # Get top-1 predict class as result
        predict_class_str = sorted_lps[0][0]
        if predict_class_str == class_str:
            result_count = result_count + 1

        for i, class_prediction in enumerate(sorted_lps):
            # Just get the top five.
            if i > 4:
                break
            print("%s: %.2f" % (class_prediction[0], class_prediction[1]))
            i += 1

    print("\nval_acc: %f" % (result_count/float(len(images))))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--model_file', help='model file to predict', type=str)

    args = parser.parse_args()
    if not args.model_file:
        raise ValueError('model file is not specified')

    validate_cnn_model(args.model_file)

if __name__ == '__main__':
    main()