text:
# formulate loss function based on supplied ground truth and outputs from network

import importlib
import tensorflow as tf
import numpy as np
import os
from pinv import pinv
from model import SfMNet, lambSH_layer, pred_illuDecomp_layer, sup_illuDecomp_layer, reproj_layer

def loss_formulate(albedos, nm_pred, am_sup, nm_gt, inputs, dms, cams, scale_xs, scale_ys, masks, pair_label,
                   preTrain_flag, am_smt_w_var, reproj_w_var, reg_loss_flag=True):
    # define gamma nonlinear mapping factor
    gamma = tf.constant(2.2)

    albedos = tf.nn.sigmoid(albedos) * masks + tf.constant(1e-4)

    ### pre-process nm_pred such that in range (-1,1)
    nm_pred_norm = tf.sqrt(tf.reduce_sum(nm_pred ** 2, axis=-1, keepdims=True) + tf.constant(1.))
    nm_pred_xy = nm_pred / nm_pred_norm
    nm_pred_z = tf.constant(1.) / nm_pred_norm
    nm_pred_xyz = tf.concat([nm_pred_xy, nm_pred_z], axis=-1) * masks

    # selete normal map used in rendering - gt or pred
    normals = nm_gt if preTrain_flag else nm_pred_xyz

    # reconstruct SH lightings from predicted statistical SH lighting model
    lighting_model = '../hdr_illu_pca'
    lighting_vectors = tf.constant(np.load(os.path.join(lighting_model, 'pcaVector.npy')), dtype=tf.float32)
    lighting_means = tf.constant(np.load(os.path.join(lighting_model, 'mean.npy')), dtype=tf.float32)
    lightings_var = tf.constant(np.load(os.path.join(lighting_model, 'pcaVariance.npy')), dtype=tf.float32)

    if preTrain_flag:
        lightings = sup_illuDecomp_layer.illuDecomp(inputs, albedos, nm_gt, gamma)
    else:
        lightings = pred_illuDecomp_layer.illuDecomp(inputs, albedos, nm_pred_xyz, gamma, masks)

    lightings_pca = tf.matmul((lightings - lighting_means), pinv(lighting_vectors))

    # recompute lightings from lightins_pca which could add weak constraint on lighting reconstruction
    lightings = tf.matmul(lightings_pca, lighting_vectors) + lighting_means

    # reshape 27-D lightings to 9*3 lightings
    lightings = tf.reshape(lightings, [tf.shape(lightings)[0], 9, 3])

    ### lighting prior loss
    var = tf.reduce_mean(lightings_pca ** 2, axis=0)

    illu_prior_loss = tf.losses.absolute_difference(var, lightings_var)

    illu_prior_loss = tf.log(illu_prior_loss + 1.)

    ### stereo supervision based on albedos reprojection consistancy
    reproj_tb = tf.to_float(tf.equal(pair_label, tf.transpose(pair_label)))
    reproj_tb = tf.cast(tf.matrix_set_diag(reproj_tb, tf.zeros([tf.shape(inputs)[0]])), tf.bool)
    reproj_list = tf.where(reproj_tb)
    img1_inds = tf.expand_dims(reproj_list[:, 0], axis=-1)
    img2_inds = tf.expand_dims(reproj_list[:, 1], axis=-1)
    albedo1 = tf.gather_nd(albedos, img1_inds)
    dms1 = tf.gather_nd(dms, img1_inds)
    cams1 = tf.gather_nd(cams, img1_inds)
    albedo2 = tf.gather_nd(albedos, img2_inds)
    cams2 = tf.gather_nd(cams, img2_inds)
    scale_xs1 = tf.gather_nd(scale_xs, img1_inds)
    scale_xs2 = tf.gather_nd(scale_xs, img2_inds)
    scale_ys1 = tf.gather_nd(scale_ys, img1_inds)
    scale_ys2 = tf.gather_nd(scale_ys, img2_inds)

    input1 = tf.gather_nd(inputs, img1_inds)

    # mask_indices contains indices for image index inside batch and spatial locations, and ignores the rgb channel
	# index
    reproj_albedo1, reproj_mask = reproj_layer.map_reproj(dms1, albedo2, cams1, cams2, scale_xs1, scale_xs2, scale_ys1,
                                                          scale_ys2)

    reproj_albedo1 = reproj_albedo1 + tf.constant(1e-4)  # numerical stable constant

    ### scale intensities for each image
    num_imgs = tf.shape(reproj_mask)[0]
    im_ = tf.constant(0)
    output = tf.TensorArray(dtype=tf.float32, size=num_imgs)

    def body(im_, output):
        reproj_mask_ = reproj_mask[im_]
        albedo1_ = tf.boolean_mask(albedo1[im_], reproj_mask_)
        reproj_albedo1_ = tf.boolean_mask(reproj_albedo1[im_], reproj_mask_)

        k = tf.reduce_sum(albedo1_ * reproj_albedo1_, keepdims=True) / (
                    tf.reduce_sum(reproj_albedo1_ ** 2, keepdims=True) + tf.constant(1e-4))

        output = output.write(im_, k)
        im_ += tf.constant(1)

        return im_, output

    def condition(im_, output):
        return tf.less(im_, num_imgs)

    _, output = tf.while_loop(condition, body, loop_vars=[im_, output])

    ks = tf.expand_dims(output.stack(), axis=-1)

    albedo1_pixels = tf.boolean_mask(albedo1, reproj_mask)
    reproj_albedo1_pixels = tf.boolean_mask(reproj_albedo1 * ks, reproj_mask)
    reproj_err = tf.losses.mean_squared_error(cvtLab(albedo1_pixels), cvtLab(reproj_albedo1_pixels))

    ### formulate loss based on paired batches ###
    # self-supervision based on intensity reconstruction
    shadings, renderings_mask = lambSH_layer.lambSH_layer(tf.ones_like(albedos), normals, lightings, 1.)

    # compare rendering intensity by Lab
    inputs_pixels = cvtLab(tf.boolean_mask(inputs, renderings_mask))
    renderings = cvtLab(tf.boolean_mask(tf.pow(albedos * shadings, 1. / gamma), renderings_mask))
    render_err = tf.losses.mean_squared_error(inputs_pixels, renderings)

    ### compute rendering loss from cross-projected alebdo map
    cross_shadings = tf.gather_nd(shadings, img1_inds)
    inputs_pixels = cvtLab(tf.boolean_mask(input1, reproj_mask))
    cross_renderings = cvtLab(
        tf.boolean_mask(tf.pow(tf.nn.relu(cross_shadings * reproj_albedo1 * ks), 1. / gamma), reproj_mask))
    cross_render_err = tf.losses.mean_squared_error(inputs_pixels, cross_renderings)

    ### measure smoothness of albedo map
    Gx = tf.constant(1 / 2) * tf.expand_dims(tf.expand_dims(tf.constant([[-1, 1]], dtype=tf.float32), axis=-1), axis=-1)
    Gy = tf.constant(1 / 2) * tf.expand_dims(tf.expand_dims(tf.constant([[-1], [1]], dtype=tf.float32), axis=-1),
                                             axis=-1)
    Gx_3 = tf.tile(Gx, multiples=(1, 1, 3, 1))
    Gy_3 = tf.tile(Gy, multiples=(1, 1, 3, 1))
    albedo_lab = tf.reshape(cvtLab(tf.reshape(albedos, [-1, 3])), [-1, 200, 200, 3])

    aGx = tf.nn.conv2d(albedos, Gx_3, padding='SAME', strides=(1, 1, 1, 1))
    aGy = tf.nn.conv2d(albedos, Gy_3, padding='SAME', strides=(1, 1, 1, 1))
    aGxy = tf.concat([aGx, aGy], axis=-1)

    # compute pixel-wise smoothness weights by angle distance between neighbour pixels' chromaticities
    inputs_pad = tf.pad(inputs, paddings=tf.constant([[0, 0], [0, 1], [0, 1], [0, 0]]))
    chroma_pad = tf.nn.l2_normalize(inputs_pad, axis=-1)

    chroma = chroma_pad[:, :-1, :-1, :]
    chroma_X = chroma_pad[:, :-1, 1:, :]
    chroma_Y = chroma_pad[:, 1:, :-1, :]
    chroma_Gx = tf.reduce_sum(chroma * chroma_X, axis=-1, keepdims=True) ** tf.constant(2.) - tf.constant(1.)
    chroma_Gy = tf.reduce_sum(chroma * chroma_Y, axis=-1, keepdims=True) ** tf.constant(2.) - tf.constant(1.)
    chroma_Gx = tf.exp(chroma_Gx / tf.constant(0.0001))
    chroma_Gy = tf.exp(chroma_Gy / tf.constant(0.0001))
    chroma_Gxy = tf.concat([chroma_Gx, chroma_Gy], axis=-1)

    int_pad = tf.reduce_sum(inputs_pad ** tf.constant(2.), axis=-1, keepdims=True)
    int = int_pad[:, :-1, :-1, :]
    int_X = int_pad[:, :-1, 1:, :]
    int_Y = int_pad[:, 1:, :-1, :]

    int_Gx = tf.where(condition=int < int_X, x=int, y=int_X)
    int_Gy = tf.where(condition=int < int_Y, x=int, y=int_Y)
    int_Gx = tf.constant(1.) + tf.exp(- int_Gx / tf.constant(.8))
    int_Gy = tf.constant(1.) + tf.exp(- int_Gy / tf.constant(.8))
    int_Gxy = tf.concat([int_Gx, int_Gy], axis=-1)

    Gxy_weights = int_Gxy * chroma_Gxy
    albedo_smt_error = tf.reduce_mean(tf.abs(aGxy) * Gxy_weights)

    ### albedo map pseudo-supervision loss
    if preTrain_flag:
        am_loss = tf.constant(0.)
    else:
        amSup_mask = tf.not_equal(tf.reduce_sum(nm_gt, axis=-1), 0)
        am_sup_pixel = cvtLab(tf.boolean_mask(am_sup, amSup_mask))
        albedos_pixel = cvtLab(tf.boolean_mask(albedos, amSup_mask))
        am_loss = tf.losses.mean_squared_error(am_sup_pixel, albedos_pixel)

    ### regualarisation loss
    reg_loss = sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))

    ### compute nm_pred error
    nmSup_mask = tf.not_equal(tf.reduce_sum(nm_gt, axis=-1), 0)
    nm_gt_pixel = tf.boolean_mask(nm_gt, nmSup_mask)
    nm_pred_pixel = tf.boolean_mask(nm_pred_xyz, nmSup_mask)
    nm_prod = tf.reduce_sum(nm_pred_pixel * nm_gt_pixel, axis=-1, keepdims=True)
    nm_cosValue = tf.constant(0.9999)
    nm_prod = tf.clip_by_value(nm_prod, -nm_cosValue, nm_cosValue)
    nm_angle = tf.acos(nm_prod) + tf.constant(1e-4)
    nm_loss = tf.reduce_mean(nm_angle ** 2)

    ### compute gradient loss
    nm_pred_Gx = conv2d_nosum(nm_pred_xyz, Gx)
    nm_pred_Gy = conv2d_nosum(nm_pred_xyz, Gy)
    nm_pred_Gxy = tf.concat([nm_pred_Gx, nm_pred_Gy], axis=-1)
    normals_Gx = conv2d_nosum(nm_gt, Gx)
    normals_Gy = conv2d_nosum(nm_gt, Gy)
    normals_Gxy = tf.concat([normals_Gx, normals_Gy], axis=-1)

    nm_pred_smt_error = tf.losses.mean_squared_error(nm_pred_Gxy, normals_Gxy)

    ### total loss
    render_err *= tf.constant(.1)
    reproj_err *= tf.constant(.05) * reproj_w_var
    cross_render_err *= tf.constant(.1)
    am_loss *= tf.constant(.1)
    illu_prior_loss *= tf.constant(.01)
    albedo_smt_error *= tf.constant(50.) * am_smt_w_var
    nm_pred_smt_error *= tf.constant(1.)
    nm_loss *= tf.constant(1.)

    if reg_loss_flag == True:
        loss = render_err + reproj_err + cross_render_err + reg_loss + illu_prior_loss + albedo_smt_error + \
			   nm_pred_smt_error + nm_loss + am_loss
    else:
        loss = render_err + reproj_err + cross_render_err + illu_prior_loss + albedo_smt_error + nm_pred_smt_error + \
			   nm_loss + am_loss

    return lightings, albedos, nm_pred_xyz, loss, render_err, reproj_err, cross_render_err, reg_loss, \
		   illu_prior_loss, albedo_smt_error, nm_pred_smt_error, nm_loss, am_loss

# input RGB is 2d tensor with shape (n_pix, 3)
def cvtLab(RGB):
    # threshold definition
    T = tf.constant(0.008856)

    # matrix for converting RGB to LUV color space
    cvt_XYZ = tf.constant(
        [[0.412453, 0.35758, 0.180423], [0.212671, 0.71516, 0.072169], [0.019334, 0.119193, 0.950227]])

    # convert RGB to XYZ
    XYZ = tf.matmul(RGB, tf.transpose(cvt_XYZ))

    # normalise for D65 white point
    XYZ /= tf.constant([[0.950456, 1., 1.088754]]) * 100

    mask = tf.to_float(tf.greater(XYZ, T))

    fXYZ = XYZ ** (1 / 3) * mask + (1. - mask) * (tf.constant(7.787) * XYZ + tf.constant(0.137931))

    M_cvtLab = tf.constant([[0., 116., 0.], [500., -500., 0.], [0., 200., -200.]])

    Lab = tf.matmul(fXYZ, tf.transpose(M_cvtLab)) + tf.constant([[-16., 0., 0.]])
    mask = tf.to_float(tf.equal(Lab, tf.constant(0.)))

    Lab += mask * tf.constant(1e-4)

    return Lab

# compute regular 2d convolution on 3d data
def conv2d_nosum(input, kernel):
    input_x = input[:, :, :, 0:1]
    input_y = input[:, :, :, 1:2]
    input_z = input[:, :, :, 2:3]

    output_x = tf.nn.conv2d(input_x, kernel, strides=(1, 1, 1, 1), padding='SAME')
    output_y = tf.nn.conv2d(input_y, kernel, strides=(1, 1, 1, 1), padding='SAME')
    output_z = tf.nn.conv2d(input_z, kernel, strides=(1, 1, 1, 1), padding='SAME')

    return tf.concat([output_x, output_y, output_z], axis=-1)

# compute regular 2d convolution on 3d data
def conv2d_nosum_2ch(input, kernel):
    input_x = input[:, :, :, 0:1]
    input_y = input[:, :, :, 1:2]

    output_x = tf.nn.conv2d(input_x, kernel, strides=(1, 1, 1, 1), padding='SAME')
    output_y = tf.nn.conv2d(input_y, kernel, strides=(1, 1, 1, 1), padding='SAME')

    return tf.concat([output_x, output_y], axis=-1)

max_stars_repo_name: hannesdelbeke/InverseRenderNet

max_issues_repo_name: hannesdelbeke/InverseRenderNet

max_forks_repo_name: hannesdelbeke/InverseRenderNet


text:
# --------------
# Import packages
import numpy as np
import pandas as pd
from scipy.stats import mode

bank = pd.read_csv(path)

# code starts here

categorical_var = bank.select_dtypes(include = 'object')
categorical_var

numerical_var = bank.select_dtypes(include = 'number')
numerical_var

# code ends here

# --------------
# code starts here

# load the dataset and drop the Loan_ID
banks= bank.drop(columns='Loan_ID')

# check  all the missing values filled.

print(banks.isnull().sum())

# apply mode

bank_mode = banks.mode().iloc[0]

# Fill the missing values with

banks.fillna(bank_mode, inplace=True)

# check again all the missing values filled.

print(banks.isnull().sum())
#code ends here

# --------------
# Code starts here
import pandas as pd
avg_loan_amount = pd.pivot_table(banks,index=["Gender","Married","Self_Employed"],values=["LoanAmount"],aggfunc=np.mean)
print(avg_loan_amount)

# code ends here

# --------------
# code starts here
loan_approved_se = banks[(banks['Self_Employed']== 'Yes') & (banks['Loan_Status']=='Y')].count()

loan_approved_nse = banks[(banks['Self_Employed']== 'No') & (banks['Loan_Status']=='Y')].count()

percentage_se = (loan_approved_se.iloc[0] * 100 / 614)
percentage_nse = (loan_approved_nse.iloc[0] * 100 / 614)

print(percentage_se)
print(percentage_nse)

# code ends here

# --------------
# code starts here

loan_term = banks['Loan_Amount_Term'].apply(lambda x: int(x)/12)

big_loan_term=len(loan_term[loan_term>=25])

print(big_loan_term)

# code ends here

# --------------
# code starts here

loan_groupby = banks.groupby('Loan_Status')[["ApplicantIncome", "Credit_History"]]
mean_values = loan_groupby.mean()
print(mean_values)

# code ends here

ext: py

reason: import numpy,from scipy


text:
from __future__ import print_function, division
import numpy as np
import os
import sys
import torch
from torch.autograd import Variable
import argparse
from utils import *

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument('--score_file',  default='./val_submissions/',required=True, help='path to val submission file')
  parser.add_argument('--val_idx',  default='./data_utils/val_idx_0505.csv', help='path to val idx file')
  args = parser.parse_args()

  f_score = open(args.score_file, 'r')
  i = 0
  scores = []
  for line in f_score:
    if i == 0:
      i += 1
      continue
    line_list = line.strip('\r\n').split(',')
    img_name, score_1, score_2, score_3 = line_list
    scores.append([float(score_1), float(score_2), float(score_3)])
    i += 1
  f_score.close()

  scores = np.array(scores).astype('float32')

  f_val = open(args.val_idx, 'r')
  targets = []
  for line in f_val:
    line_list = [item.strip() for item in line.strip('\r\n').split(',')]
    targets.append(int(line_list[2]))
  f_val.close()

  targets = torch.LongTensor(targets)
  targets_onehot = Variable(convert_to_one_hot(targets, 3))

  scores = Variable(torch.FloatTensor(scores))

  log_loss = KaggleLogLoss()
  log_loss_res = log_loss(scores, targets_onehot)
  print('{}, log loss: {}'.format(args.score_file, log_loss_res.data[0]))

reason: import numpy


text:
import os
os.sys.path.append("../../perception")

from Perception import Perception
import cv2
import numpy as np

class AdvancedLaneDetection(Perception):
    def __init__(self, points_src, points_dst, yellow_threshold, white_threshold, min_pix=100):
        self.points_src = points_src
        self.points_dst = points_dst
        self.yellow_threshold = yellow_threshold
        self.white_threshold = white_threshold
        self.min_pix = min_pix

    def Perceive(self, data):
        assert data.shape == (480, 640, 3), "AdvancedLaneDetector must receive a (480, 640, 3) image"

        # 1. Warp the image to a bird's eye view
        M, M_inv = self.getTransform()
        warped_image = cv2.warpPerspective(data, M, (data.shape[0], data.shape[1]), flags=cv2.INTER_AREA)

        # 2. Change the color space of the image from RGB to HSV
        hsv_image = self.toHSV(warped_image)

        # 3. Segment the lanes by their color (yellow and white)
        yellow_thresh, white_threshold = self.segmentLanes(hsv_image)
        binary = cv2.bitwise_or(yellow_thresh, white_threshold)

        # 4. Find left and right lanes
        w_height = 30
        w_width = 40

        yellow_lane_dict = self.findLane(yellow_thresh, w_height, w_width, M_inv, name="Yellow", debug=True)
        white_lane_dict = self.findLane(white_threshold, w_height, w_width, M_inv, name="White", debug=True)
        #white_lane_dict = findLane(white_lane, w_height, w_width, min_pix, debug)

        #yellow_hist = np.sum(yellow_thresh, axis=0)

        return binary, yellow_lane_dict, white_lane_dict

    def segmentLanes(self, image):
        """
            Function that applies the segmentation thresholds to the HSV image.
        """
        yellow = cv2.inRange(image, self.yellow_threshold[0], self.yellow_threshold[1])
        white = cv2.inRange(image, self.white_threshold[0], self.white_threshold[1])

        return yellow, white

    def getTransform(self):
        """
            Function that compute the transformation between two sets of points.
        """

        src = np.float32(self.points_src)
        dst = np.float32(self.points_dst)

        M = cv2.getPerspectiveTransform(src, dst)
        M_inv = cv2.getPerspectiveTransform(dst, src)

        return M, M_inv

    def toHSV(self, image):
        """
            Function to transform the color space to HSV
        """
        return cv2.cvtColor(image, cv2.COLOR_BGR2HSV)

    def findLane(self, binary_image, w_height, w_width, M_inv, name, debug=False):

        dict_find_lanes = {}

        binary_hist = np.sum(binary_image, axis=0)
        nonzero = binary_image.nonzero()
        nonzero_y = np.array(nonzero[0])
        nonzero_x = np.array(nonzero[1])

        w_x = int(np.argmax(binary_hist))
        w_y = int(binary_image.shape[0] - w_height/2)

        #find first window
        dict_first_window = self.findFirstWindow(w_x, w_y, w_height, w_width, nonzero_x, nonzero_y, debug)

        if dict_first_window["found"]:
            dict_remaining_windows = self.findRemainingWindows(dict_first_window["w_center"], w_height, w_width, nonzero_x, nonzero_y, name, debug)

            dict_find_lanes["windows"] = [dict_first_window["w_center"]] + dict_remaining_windows["chosen_windows"]

            if len(dict_find_lanes["windows"]) > 3:
                dict_find_lanes["found"] = True
                dict_find_lanes["inside_points"] = dict_remaining_windows["inside_points"]

                all_idexes = np.concatenate(dict_find_lanes["inside_points"])
                all_points_x = nonzero_x[all_idexes]
                all_points_y = nonzero_y[all_idexes]
                fit = np.polyfit(all_points_y, all_points_x, 3)
                dict_find_lanes["polynimial_fit"] = fit

                if debug:
                    initial_window_img = self.debugInitialWindow(binary_image, dict_first_window["w_centroids"], w_width, w_height)
                    remaining_windows_img = self.debugRemainingWindows(binary_image, dict_remaining_windows, w_height, w_width)

                    warped_zeros = np.zeros((640, 480,3), np.uint8)
                    curve_points_x = (fit[0]*all_points_y**3 + fit[1]*all_points_y**2 + fit[2]*all_points_y + fit[3]).astype(np.int32)
                    pts = np.flipud(np.transpose(np.vstack([curve_points_x, all_points_y])))
                    color = (255,0,0) if name == "White" else (0,0,255)

                    out_img = cv2.polylines(warped_zeros, np.int_([pts]), isClosed=False, color=color, thickness = 20)
                    out_img = cv2.warpPerspective(out_img, M_inv, (out_img.shape[0], out_img.shape[1]), flags=cv2.INTER_AREA)

                    dict_find_lanes["debug_images"] = [initial_window_img, remaining_windows_img, out_img]
            else:
                print("Too few boxes found for " + name + " lane")
                dict_find_lanes["found"] = False
        else:
            print(name + " lane didn't found")
            dict_find_lanes["found"] = False

        return dict_find_lanes

    def findRemainingWindows(self, w_center, w_height, w_width, nonzero_x, nonzero_y, name, debug=False):
        offsets = [(-w_width, 0), (-w_width, -w_height), (0, -w_height), (w_width, w_height), (w_width, 0)] #left, upper-left, above, uper right, right

        dict_remaining_w = {}

        if debug:
            dict_remaining_w["search_windows"] = []
        dict_remaining_w["chosen_windows"] = []
        dict_remaining_w["inside_points"] = []

        current_x = w_center[0]
        current_y = w_center[1]

        done = False
        count = 0
        while not done and (count < 30):
            best_offset_index = -1
            best_offset_p = -1
            best_inside_indexes = None

            for i, off in enumerate(offsets):
                w_x = current_x + off[0]
                w_y = current_y + off[1]

                w_x_high = int(w_x - w_width/2)
                w_y_high = int(w_y - w_height/2)

                w_x_low = int(w_x + w_width/2)
                w_y_low = int(w_y + w_height/2)

                if w_x_high < 0 or w_x_low > 480 or w_y_high < 0 or w_y_low > 640:
                    pixels_inside = 0
                    inside_window = None
                else:
                    inside_window = ( (nonzero_y >= w_y_high) & (nonzero_y < w_y_low) &
                (nonzero_x >= w_x_high) &  (nonzero_x <= w_x_low)).nonzero()[0]
                    pixels_inside = len(inside_window)

                    if debug:
                        dict_remaining_w["search_windows"].append((w_x,w_y))

                if pixels_inside > best_offset_p:
                    best_offset_index = i
                    best_offset_p = pixels_inside
                    best_inside_indexes = inside_window

            if best_offset_p <= 0:
                if name == "Yellow":
                    #Gaps in yellow line can be bigger than the size of the box so try one last time
                    w_x = current_x + 0
                    w_y = current_y + -w_height*1.5

                    w_x_high = int(w_x - w_width/2)
                    w_y_high = int(w_y - w_height/2)

                    w_x_low = int(w_x + w_width/2)
                    w_y_low = int(w_y + w_height/2)

                    if w_x_high < 0 or w_x_low > 480 or w_y_high < 0 or w_y_low > 640:
                        pixels_inside = 0
                        inside_window = None
                    else:
                        inside_window = ( (nonzero_y >= w_y_high) & (nonzero_y < w_y_low) &
                    (nonzero_x >= w_x_high) &  (nonzero_x <= w_x_low)).nonzero()[0]
                        pixels_inside = len(inside_window)

                        if debug:
                            dict_remaining_w["search_windows"].append((w_x,w_y))

                    if pixels_inside > best_offset_p:
                        best_offset_index = i
                        best_offset_p = pixels_inside
                        best_inside_indexes = inside_window

                    if best_offset_p <= 0:
                        done = True
                    else:
                        current_x = int(np.mean(nonzero_x[best_inside_indexes]))
                        current_y = int(np.mean(nonzero_y[best_inside_indexes]))
                        dict_remaining_w["inside_points"].append(best_inside_indexes)
                        if len(dict_remaining_w["chosen_windows"]) > 0 and current_x == dict_remaining_w["chosen_windows"][-1][0] and current_y == dict_remaining_w["chosen_windows"][-1][1]:
                            done = True
                        dict_remaining_w["chosen_windows"].append((current_x,current_y))
                else:
                    done = True
            else:
                current_x = int(np.mean(nonzero_x[best_inside_indexes]))
                current_y = int(np.mean(nonzero_y[best_inside_indexes]))
                dict_remaining_w["inside_points"].append(best_inside_indexes)
                if len(dict_remaining_w["chosen_windows"]) > 0 and current_x == dict_remaining_w["chosen_windows"][-1][0] and current_y == dict_remaining_w["chosen_windows"][-1][1]:
                    done = True
                #print(current_x, current_y)
                dict_remaining_w["chosen_windows"].append((current_x,current_y))

            count += 1
        #print(done, count)
        return dict_remaining_w

    def findFirstWindow(self, w_x, w_y, w_height, w_width, nonzero_x, nonzero_y, debug=False):

        dict_first_window = {}
        if debug:
            dict_first_window["w_centroids"] = []

        done = False
        dict_first_window["found"] = False

        while not done:
            if debug:
                dict_first_window["w_centroids"].append([w_x,w_y])

            w_x_high = int(w_x - w_width/2)
            w_y_high = int(w_y - w_height/2)

            w_x_low = int(w_x + w_width/2)
            w_y_low = int(w_y + w_height/2)

            if w_x_high < 0 or w_x_low > 480 or w_y_high < 0 or w_y_low > 640:
                done = True
                continue

            inside_window = ( (nonzero_y >= w_y_high) & (nonzero_y < w_y_low) &
                (nonzero_x >= w_x_high) &  (nonzero_x <= w_x_low)).nonzero()[0]

            if len(inside_window) > self.min_pix:
                w_x = int(np.mean(nonzero_x[inside_window]))
                w_y = int(np.mean(nonzero_y[inside_window]))

                if debug:
                    dict_first_window["w_centroids"].append([w_x, w_y])

                done = True
                dict_first_window["found"] = True
                dict_first_window["w_center"] = (w_x, w_y)
            else:
                w_y = int(w_y - w_height)

        return dict_first_window

    def debugRemainingWindows(self, img, dict_remaining_windows, w_height, w_width):
        remaining_windows_img = np.dstack((img, img, img))

        for window in dict_remaining_windows["search_windows"]:
            w_x_high = int(window[0] - w_width/2)
            w_y_high = int(window[1] - w_height/2)

            w_x_low = int(window[0] + w_width/2)
            w_y_low = int(window[1] + w_height/2)

            remaining_windows_img = cv2.rectangle(remaining_windows_img, (w_x_high, w_y_high), (w_x_low, w_y_low), (255,0,0), 5)

        for window in dict_remaining_windows["chosen_windows"]:
            w_x_high = int(window[0] - w_width/2)
            w_y_high = int(window[1] - w_height/2)

            w_x_low = int(window[0] + w_width/2)
            w_y_low = int(window[1] + w_height/2)

            remaining_windows_img = cv2.rectangle(remaining_windows_img, (w_x_high, w_y_high), (w_x_low, w_y_low), (255,255,0), 5)

        return remaining_windows_img

    def debugInitialWindow(self, image, initial_centroids, w_width, w_height):
        initial_window_img = np.dstack((image, image, image))

        for window in initial_centroids[:-1]:
            w_x_high = int(window[0] - w_width/2)
            w_y_high = int(window[1] - w_height/2)

            w_x_low = int(window[0] + w_width/2)
            w_y_low = int(window[1] + w_height/2)

            initial_window_img = cv2.rectangle(initial_window_img, (w_x_high, w_y_high), (w_x_low, w_y_low), (255,0,0), 5)

        window = initial_centroids[-1]
        w_x_high = int(window[0] - w_width/2)
        w_y_high = int(window[1] - w_height/2)

        w_x_low = int(window[0] + w_width/2)
        w_y_low = int(window[1] + w_height/2)

        initial_window_img = cv2.rectangle(initial_window_img, (w_x_high, w_y_high), (w_x_low, w_y_low), (255,255,0), 5)

        return initial_window_img

max_stars_repo_name: goncamateus/patocin

max_issues_repo_name: goncamateus/patocin

max_forks_repo_name: goncamateus/patocin


text:
from dataclasses import dataclass
import json
import logging
import subprocess
import random
import re
from typing import List, Optional, Tuple
import numpy as np
import pandas as pd
import io
import torch
from torch.utils.dlpack import to_dlpack
from torch.utils.dlpack import from_dlpack
import os
import nn_pruning
import aiohttp
import concurrent
from requests import post as POST
from scipy.special import softmax

# from experiment_impact_tracker.compute_tracker import ImpactTracker

from transformers import (
    T5ForConditionalGeneration,
    AutoModelForQuestionAnswering,
    DistilBertForQuestionAnswering,
    ViTForImageClassification,
    GPT2LMHeadModel,
    AutoModelForCausalLM,
)

import triton_python_backend_utils as pb_utils
import json
import time

from hfutils.logger import Logger
import gc
from hfutils.pipe.t5 import (
    T5_ENCODER_INPUTS,
    T5_ENCODER_OUTPUTS,
    T5_DECODER_INPUTS,
    T5_DECODER_OUTPUTS,
    T5PyTorchPipe,
)
from hfutils.pipe.bert import (
    BERT_INPUTS,
    BERT_OUTPUTS,
    BertPyTorchPipeForQuestionAnswering,
)
from hfutils.pipe.vit import (
    VIT_INPUTS,
    VIT_OUTPUTS,
    ViTPyTorchPipeForImageClassification,
)
from hfutils.pipe.distilbert import (
    DISTILBERT_INPUTS,
    DISTILBERT_OUTPUTS,
    DistilBertPyTorchPipeForQuestionAnswering,
)
from hfutils.pipe.gpt import GPT_INPUTS, GPT_OUTPUTS, GPTLMHeadModelPipe
from hfutils.calibration import temperature_scale
from hfutils.constants import np_to_torch_dtype
import dill

m = torch.nn.Softmax(dim=1)

T5_TASK_LABELS = [1176, 6136, 59]  # HACK with GLUE labels

from multiprocessing import shared_memory

@dataclass
class ModelConfig:
    # name:   Optional[str]   # model full name
    path: Optional[str]  # model checkpoint path
    type: Optional[str]  # model type, e.g., t5 or bert
    stages: Optional[int]  # number of parallel stages
    ppos: Optional[int]  # current stage
    epos: Optional[int]  # current ensemble
    ens: Optional[int]  # number of total ensembles
    alpha: Optional[float]  # ensemble exp smooth weight
    temp: Optional[float]  # temperature scaling
    th: Optional[float]  # confidence threshold

class TritonPythonModel:
    """Your Python model must use the same class name. Every Python model
    that is created must have "TritonPythonModel" as the class name.
    """

    def initialize(self, args):
        """`initialize` is called only once when the model is being loaded.
        Implementing `initialize` function is optional. This function allows
        the model to intialize any state associated with this model.
        Parameters
        ----------
        args : dict
          Both keys and values are strings. The dictionary keys and values are:
          * model_config: A JSON string containing the model configuration
          * model_instance_kind: A string containing model instance kind
          * model_instance_device_id: A string containing model instance device ID
          * model_repository: Model repository path
          * model_version: Model version
          * model_name: Model name
        """

        self.logger = Logger(__file__, logging.DEBUG, 50000000, 5)
        self.config = ModelConfig("", "", 0, 0, 0, 0, 0, 0, 0)

        self.model_config = json.loads(args["model_config"])
        self.device_id = int(args["model_instance_device_id"])
        self.device = torch.device("cuda:" + args["model_instance_device_id"])

        self._get_gpu_uuid()

        self.model_name = args["model_name"]  # HACK model_name = <TYPE>_e<epos>p<ppos>
        self.config.type, deploy = tuple(self.model_name.split("_"))
        groups = re.findall(r"e(\d+)p(\d+)", deploy)[0]
        # self.config.name = model_name[:model_name.rfind("_")] # HACK model_name always end with partx, indicating parallel stages
        self.config.ppos = int(groups[1])
        self.config.epos = int(groups[0])
        # self.config.type = model_name[:model_name.find("-")] # HACK model_name always start with type

        meta_path = os.path.join(args["model_repository"], os.path.pardir, "meta.json")
        print(meta_path, flush=True)
        self._read_cfg(meta_path)

        self._load_model()

        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=3,)

        # self.tracker = ImpactTracker(os.path.join("impact_tracker", self.model_name))
        # self.tracker.launch_impact_monitor()

    def _get_gpu_uuid(self):
        command = "nvidia-smi --query-gpu=index,uuid,gpu_bus_id --format=csv"

        result = subprocess.run(command.split(), stdout=subprocess.PIPE)
        # print(result.stdout)
        df = pd.read_csv(io.StringIO(result.stdout.decode("utf-8")), index_col="index")
        df = df.sort_index()
        df.iloc[:, 0] = df.iloc[:, 0].str.strip()
        self.gpu_uuid = df.iloc[self.device_id][" uuid"]
        # print(df)

    def _load_model(self):
        if "t5" == self.config.type:
            model = T5ForConditionalGeneration.from_pretrained(self.config.path)
            self.model = T5PyTorchPipe(model)

        if "bert" == self.config.type:
            model = AutoModelForQuestionAnswering.from_pretrained(self.config.path)
            self.model = BertPyTorchPipeForQuestionAnswering(model)

        if "distilbert" == self.config.type:
            model = DistilBertForQuestionAnswering.from_pretrained(self.config.path)
            self.model = DistilBertPyTorchPipeForQuestionAnswering(model)

        if "vit" == self.config.type:
            model = ViTForImageClassification.from_pretrained(self.config.path)
            self.model = ViTPyTorchPipeForImageClassification(model)

        if "gpt" == self.config.type:
            model = AutoModelForCausalLM.from_pretrained(self.config.path)
            self.model = GPTLMHeadModelPipe(model)

        self.model.eval()
        self.model.partition_by_parameter(self.config.ppos, self.config.stages)
        # self.model.partition_by_parameter(self.config.ppos, 4) # TEST MULTIPLEX
        self.model.convert(self.device)

        del model
        gc.collect()
        torch.cuda.empty_cache()

    def _read_cfg(self, path):
        ensemble_name = "_".join([self.config.type, "ensemble"])

        with open(path, "r") as fp:
            config = json.load(fp)

        self.config.alpha = config[ensemble_name]["alpha"]
        self.config.ens = len(config[ensemble_name]["ensembles"])

        model_name = config[ensemble_name]["ensembles"][self.config.epos]
        self.config.stages = config[model_name]["parallel_stages"]
        self.config.path = os.path.join(config["base_dir"], config[model_name]["path"])
        self.config.temp = config[model_name]["temperature"]
        self.config.th = config[model_name]["threshold"]
        # self.config.epos = config[model_name]["ensemble_pos"]

        util_params = config[model_name]["util_params"]
        self.util_func = np.poly1d(util_params)

        self.is_last_stage = self.config.ppos == self.config.stages - 1
        # self.is_last_stage = False # TEST MULTIPLEX
        with open(
            f"/home/xly/model-inference/inference_dump/{model_name}_calibrator", "rb"
        ) as f:
            self.calibrator = dill.load(f)
        # HACK bert tiny model
        if "bert" == self.config.type and "distilbert" in self.config.path:
            self.config.type = "distilbert"

        self.logger.info("%s", self.config)

    @torch.no_grad()
    async def model_inference(self, args):
        start_time = time.perf_counter()
        uuid = random.randint(1e9, 2e9)
        self.logger.info(
            "%s inference[%s] start %s", self.model_name, uuid, time.time_ns(),
        )
        outputs = self.model(args)
        self.logger.info(
            "%s inference[%s] end %s", self.model_name, uuid, time.time_ns(),
        )
        if self.is_last_stage:
            # print(outputs.shape, flush=True)
            outputs = outputs.squeeze(1) / self.config.temp
            if "t5" == self.config.type:
                outputs = outputs[:, T5_TASK_LABELS]
            if "gpt" == self.config.type:
                outputs = outputs[:, -1, :50257]
            outputs = outputs.detach().cpu().numpy()
            # outputs = temperature_scale(outputs, self.config.temp)

        end_time = time.perf_counter()
        self.logger.info(
            "%s inference %s (ms)", self.model_name, (end_time - start_time) * 1000,
        )

        return outputs

    async def execute(self, requests):
        responses = []

        exec_start_time = time.perf_counter()
        for request in requests:

            # request_id = int(request.request_id())
            # correlation_id = int(request.correlation_id())
            # if self.is_last_stage:
            batch_mask = self.parse_input(request, "batch_mask").detach().cpu().numpy()
            hist_outputs = self.parse_input(request, "logits").detach().cpu().numpy()

            local_mask = batch_mask[self.config.epos]
            self.logger.debug("local_mask %s", local_mask)

            output_tensors = []
            # request_start_time = time.perf_counter()
            if np.any(local_mask):
                args = self.parse_request(request, local_mask)
                outputs = await self.model_inference(
                    args
                )  # MOVE TO CPU, SAVE GPU MEMORY

                # local_mask = local_mask.to(outputs.device)
                # ensemble_outputs = ensemble_outputs.to(outputs.device)
                start_time = time.perf_counter()
                if self.is_last_stage:

                    # self.logger.trace("%s outputs %s", self.model_name, outputs)

                    outputs = self.model_ensemble(
                        None
                        if self.config.epos == 0
                        or np.all(hist_outputs.astype(int) == 0)
                        else hist_outputs,
                        outputs,
                        local_mask,
                    )

                    # self.logger.trace(
                    #     "%s ensemble outputs %s", self.model_name, outputs
                    # )

                    local_mask, max_prob = self.offload_mask(outputs, local_mask)
                    self.logger.debug(
                        "%s local_mask updated %s", self.model_name, local_mask
                    )

                    if np.any(local_mask):
                        batch_mask = self.update_batch_mask(
                            max_prob, batch_mask, local_mask
                        )
                        # batch_mask[self.config.epos] &= ~local_mask
                        # batch_mask[self.config.epos + 1] |= local_mask
                        self.logger.debug(
                            "%s batch_mask updated %s", self.model_name, batch_mask
                        )

                    # inference_response = pb_utils.InferenceResponse(
                    #     output_tensors=[
                    #         self.parse_output(outputs, "outputs"),
                    #         self.parse_output(batch_mask, "batch_mask"),
                    #     ]
                    # )
                    # responses.append(inference_response)
                # else:
                end_time = time.perf_counter()
                self.logger.info(
                    "%s postprocessing time elapsed (%s, %s) %s (ms)",
                    self.model_name,
                    start_time,
                    end_time,
                    (end_time - start_time) * 1000,
                )

                if not isinstance(outputs, Tuple):
                    outputs = (outputs,)

                for output in outputs:
                    self.logger.debug(
                        "%s output %s", self.model_name, output.shape,
                    )

                output_tensors = self.parse_response(outputs)
                if not self.is_last_stage:
                    output_tensors += [
                        self.parse_output(hist_outputs, "logits"),
                        self.parse_output(batch_mask, "batch_mask"),
                    ]
                else:
                    output_tensors += [
                        self.parse_output(batch_mask, "batch_mask"),
                    ]
                # # TEST MULTIPLEX
                # inference_response = pb_utils.InferenceResponse(
                #     output_tensors=[
                #         self.parse_output(hist_outputs, "logits"),
                #         self.parse_output(batch_mask, "batch_mask"),
                #     ]
                # )
            else:
                output_tensors = [
                    self.parse_output(hist_outputs, "logits"),
                    self.parse_output(batch_mask, "batch_mask"),
                ]

                if "gpt" == self.config.type and not self.is_last_stage:
                    output_tensors += [
                        self.parse_output(np.zeros((1, 512, 4096)), "hidden_states"),
                    ]

            # request_end_time = time.perf_counter()
            # self.executor.submit(
            #     POST,
            #     url=f"http://127.0.0.1:10000/meter/{self.gpu_uuid}",
            #     json={
            #         "request_id": request_id,
            #         "correlation_id": correlation_id,
            #         "epos": self.config.epos,
            #         "ppos": self.config.ppos,
            #         "type": self.config.type,
            #         "start": request_start_time,
            #         "end": request_end_time,
            #         "util": self.util_func(np.sum(local_mask))
            #         if np.any(local_mask)
            #         else 0,
            #     },
            #     timeout=1,
            # )

            # output_tensors.append(self.parse_output(timestamp, "timestamp"))
            inference_response = pb_utils.InferenceResponse(
                output_tensors=output_tensors
            )
            responses.append(inference_response)

        exec_end_time = time.perf_counter()
        self.logger.info(
            "%s requests (%s, %s) %s (ms)",
            self.model_name,
            exec_end_time,
            exec_start_time,
            (exec_end_time - exec_start_time) * 1000,
        )
        return responses

    def parse_response(self, outputs):
        layer = self.model.layers[self.model.exec_map[-1] - 1]
        layer_name = type(layer).__name__
        output_tensors = []
        if "t5" == self.config.type:
            input_names = (
                T5_DECODER_OUTPUTS[layer_name]
                if layer.is_decoder
                else T5_ENCODER_OUTPUTS[layer_name]
            )
        if "bert" == self.config.type:
            input_names = BERT_OUTPUTS[layer_name]
        if "distilbert" == self.config.type:
            input_names = DISTILBERT_OUTPUTS[layer_name]
        if "vit" == self.config.type:
            input_names = VIT_OUTPUTS[layer_name]
        if "gpt" == self.config.type:
            input_names = GPT_OUTPUTS[layer_name]

        for i, name in enumerate(input_names):
            if "gpt" == self.config.type and name == "attention_mask":
                continue
            tensor = self.parse_output(outputs[i], name)
            output_tensors.append(tensor)

        return output_tensors

    def parse_request(self, request, local_mask):
        layer = self.model.layers[self.model.exec_map[0]]
        layer_name = type(layer).__name__
        args = []
        if "t5" == self.config.type:
            input_names = (
                T5_DECODER_INPUTS[layer_name]
                if layer.is_decoder
                else T5_ENCODER_INPUTS[layer_name]
            )
        if "bert" == self.config.type:
            input_names = BERT_INPUTS[layer_name]
        if "distilbert" == self.config.type:
            input_names = DISTILBERT_INPUTS[layer_name]
        if "vit" == self.config.type:
            input_names = VIT_INPUTS[layer_name]
        if "gpt" == self.config.type:
            input_names = GPT_INPUTS[layer_name]

        for name in input_names:
            arg = self.parse_input(request, name)
            if self.config.epos == 0 and self.config.ppos == 0:
                arg = arg[local_mask]
            args.append(arg)
        return tuple(args)

    def parse_input(self, request, field):
        # self.logger.debug("parse_input: request %s, field %s", request, field)
        input = pb_utils.get_input_tensor_by_name(request, field)
        if input is None:
            return
        # input = from_dlpack(input.to_dlpack())
        input = input.as_numpy()
        input = torch.as_tensor(
            input, dtype=np_to_torch_dtype(input.dtype), device=self.device
        )
        return input

    def parse_output(self, output, field):
        # return pb_utils.Tensor.from_dlpack(field, to_dlpack(output))
        output_config = pb_utils.get_output_config_by_name(self.model_config, field)
        output_dtype = pb_utils.triton_string_to_numpy(output_config["data_type"])
        if isinstance(output, torch.Tensor):
            output = output.detach().cpu().numpy()
        output_pb = pb_utils.Tensor(field, output.astype(output_dtype))
        return output_pb

    def update_batch_mask(self, max_prob, mask, local_mask):
        start_time = time.perf_counter()
        num_next_models = self.config.ens - self.config.epos - 1
        base_step = self.config.th / num_next_models
        for skip in range(num_next_models):
            skip_th_lower = base_step * (num_next_models - 1 - skip)
            skip_th_upper = base_step * (num_next_models - skip)
            skip_mask = (
                (max_prob >= skip_th_lower) & (max_prob < skip_th_upper) & local_mask
            )
            self.logger.debug(
                "skip_th_lower %s, skip_th_upper %s, skip_mask %s",
                skip_th_lower,
                skip_th_upper,
                skip_mask,
            )
            mask[skip + 1 + self.config.epos] |= skip_mask

        mask[self.config.epos] &= ~local_mask
        end_time = time.perf_counter()
        self.logger.info(
            "%s update_batch_mask time elapsed (%s,%s) %s (ms)",
            start_time,
            end_time,
            self.model_name,
            (end_time - start_time) * 1000,
        )
        return mask

    def offload_mask(self, logits, mask):
        start_time = time.perf_counter()
        probabilities = np.power(softmax(logits, axis=1), 2)
        max_prob = self.calibrator.calibrate(probabilities)
        prob_mask = max_prob < self.config.th

        # prob_mask = np.all(probabilities < self.config.th, axis=1)
        # max_prob = np.max(probabilities, axis=1)
        if "bert" in self.config.type:
            prob_mask = prob_mask.squeeze(1)
            max_prob = max_prob.squeeze(1)
        # max_prob = np.max(probabilities, axis=1)
        # # probabilities = torch.float_power(m(logits), 2)
        # # max_prob, _ = torch.max(probabilities, dim=1)
        # if "bert" in self.config.type:
        #     # max_prob, _ = torch.min(max_prob.squeeze(1), dim=1)
        #     max_prob = np.min(max_prob.squeeze(1), axis=1)
        # prob_mask = max_prob < self.config.th
        self.logger.debug(
            "(offload_mask) prob_mask %s %s", prob_mask, mask,
        )
        combined_mask = mask & prob_mask
        # combined_mask[mask] &= prob_mask[mask]
        self.logger.debug("max_prob %s, combined_mask %s", max_prob, combined_mask)
        end_time = time.perf_counter()
        self.logger.info(
            "%s offload_mask time elapsed (%s, %s) %s (ms)",
            self.model_name,
            start_time,
            end_time,
            (end_time - start_time) * 1000,
        )
        return combined_mask, max_prob

    def model_ensemble(self, hist_outputs, outputs, mask):
        start_time = time.perf_counter()
        if hist_outputs is not None:
            outputs[mask] = (
                hist_outputs[mask] * (1 - self.config.alpha)
                + outputs[mask] * self.config.alpha
            )
        end_time = time.perf_counter()
        self.logger.info(
            "%s model_ensemble time elapsed (%s, %s) %s (ms)",
            self.model_name,
            start_time,
            end_time,
            (end_time - start_time) * 1000,
        )
        return outputs

    def finalize(self):
        # self.tracker.stop()
        print("Cleaning up...")

max_stars_repo_name: drunkcoding/model-inference

max_issues_repo_name: drunkcoding/model-inference

max_forks_repo_name: drunkcoding/model-inference


text:
'''
Ejemplo de entrenamiento de un percepton para reconocer MNIST
v1.0 14-Dic-2018
Juan D. Tardos
'''

import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.optimizers import RMSprop, Adam, SGD
from keras.callbacks import EarlyStopping
from keras import regularizers
import time
#from sklearn.metrics import confusion_matrix
import numpy as np
#import matplotlib.pyplot as plt
import P5_utils

verbose = True

print('Loading MNIST dataset...')
# Problem dimensions
img_rows, img_cols = 28, 28
num_pixels = img_rows * img_cols
num_classes = 10
# The data, split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, num_pixels)
x_test = x_test.reshape(10000, num_pixels)
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test  = keras.utils.to_categorical(y_test,  num_classes)
# Random permutation of training data
np.random.seed(0)
p = np.arange(x_train.shape[0])
np.random.shuffle(p)
x_train = x_train[p]
y_train = y_train[p]

# Función para parar cuando ya no mejora el error de validacion
earlystop=EarlyStopping(monitor='val_loss', patience=5,
                        verbose=1, mode='auto')

# Perceptron de un solo nivel
model = Sequential()

#Capa oculta
# model.add(Dense(512, activation='relu',
#                 kernel_regularizer=regularizers.l2(0.01),
#                 input_shape=(num_pixels,)))

model.add(Dense(512, activation='relu',
                input_shape=(num_pixels,)))

#evitar sobreajuste
model.add(Dropout(0.20))

#Capa de salida
model.add(Dense(num_classes, activation='softmax'))

model.compile(loss='categorical_crossentropy',
              optimizer='adam',                    #algoritmo de entrenamiento
              metrics=['accuracy'])

model.summary()

print('Training the NN....')
t0 = time.clock()
history = model.fit(x_train, y_train,
                    batch_size=128,
                    epochs=40, #cuantas veces queremos usarlo
                    #validation_split=0.1,
                    callbacks=[earlystop],
                    verbose=verbose)

train_time = time.clock() - t0
print('%s %.3f%s' %  ('Training time: ', train_time, 's') )
P5_utils.plot_history(history)

# Evaluar la red
train_score = model.evaluate(x_train, y_train, verbose=0)
test_score = model.evaluate(x_test, y_test, verbose=0)
print('%s %2.2f%s' % ('Accuracy train: ', 100*train_score[1], '%' ))
print('%s %2.2f%s' % ('Accuracy test:  ', 100*test_score[1], '%'))

y_pred = model.predict(x_test)
P5_utils.plot_mnist_confusion_matrix(y_test, y_pred)

max_stars_repo_name: javierherrer/inteligencia-artificial

max_issues_repo_name: javierherrer/inteligencia-artificial

max_forks_repo_name: javierherrer/inteligencia-artificial


text:
import errno
import os
import sys
import gdown
import numpy as np
import requests
import torch
from hsh.library.hash import Hasher
from PIL import Image
from torchvision import transforms
from tqdm import tqdm

from . import data_loader, u2net
from ...backgroundremover import utilities

def load_model(model_name: str = "u2net"):
    hasher = Hasher()

    model = {
        'u2netp': (u2net.U2NETP,
                   'e4f636406ca4e2af789941e7f139ee2e',
                   '1rbSTGKAE-MTxBYHd-51l2hMOQPT_7EPy',
                   'U2NET_PATH'),
        'u2net': (u2net.U2NET,
                  '09fb4e49b7f785c9f855baf94916840a',
                  '1ao1ovG1Qtx4b7EoskHXmi2E9rp5CHLcZ',
                  'U2NET_PATH'),
        'u2net_human_seg': (u2net.U2NET,
                            '347c3d51b01528e5c6c071e3cff1cb55',
                            '1-Yg0cxgrNhHP-016FPdp902BR-kSsA4P',
                            'U2NET_PATH')
    }[model_name]

    if model_name == "u2netp":
        net = u2net.U2NETP(3, 1)
        path = os.environ.get(
            "U2NETP_PATH",
            os.path.expanduser(os.path.join("~", ".u2net", model_name + ".pth")),
        )
        if (
            not os.path.exists(path)
            or hasher.md5(path) != "e4f636406ca4e2af789941e7f139ee2e"
        ):
            utilities.download_file_from_google_drive(
                model, path,
            )

    elif model_name == "u2net":
        net = u2net.U2NET(3, 1)
        path = os.environ.get(
            "U2NET_PATH",
            os.path.expanduser(os.path.join("~", ".u2net", model_name + ".pth")),
        )
        if (
            not os.path.exists(path)
            or hasher.md5(path) != "347c3d51b01528e5c6c071e3cff1cb55"
        ):
            utilities.download_file_from_google_drive(
                model, path,
            )

    elif model_name == "u2net_human_seg":
        net = u2net.U2NET(3, 1)
        path = os.environ.get(
            "U2NET_PATH",
            os.path.expanduser(os.path.join("~", ".u2net", model_name + ".pth")),
        )
        if (
            not os.path.exists(path)
            or hasher.md5(path) != "09fb4e49b7f785c9f855baf94916840a"
        ):
            utilities.download_file_from_google_drive(
                model, path,
            )
    else:
        print("Choose between u2net, u2net_human_seg or u2netp", file=sys.stderr)

    try:
        if torch.cuda.is_available():
            net.load_state_dict(torch.load(path))
            net.to(torch.device("cuda"))
        else:
            net.load_state_dict(
                torch.load(
                    path,
                    map_location="cpu",
                )
            )
    except FileNotFoundError:
        raise FileNotFoundError(
            errno.ENOENT, os.strerror(errno.ENOENT), model_name + ".pth"
        )

    net.eval()

    return net

def norm_pred(d):
    ma = torch.max(d)
    mi = torch.min(d)
    dn = (d - mi) / (ma - mi)

    return dn

def preprocess(image):
    label_3 = np.zeros(image.shape)
    label = np.zeros(label_3.shape[0:2])

    if 3 == len(label_3.shape):
        label = label_3[:, :, 0]
    elif 2 == len(label_3.shape):
        label = label_3

    if 3 == len(image.shape) and 2 == len(label.shape):
        label = label[:, :, np.newaxis]
    elif 2 == len(image.shape) and 2 == len(label.shape):
        image = image[:, :, np.newaxis]
        label = label[:, :, np.newaxis]

    transform = transforms.Compose(
        [data_loader.RescaleT(320), data_loader.ToTensorLab(flag=0)]
    )
    sample = transform({"imidx": np.array([0]), "image": image, "label": label})

    return sample

def predict(net, item):
    sample = preprocess(item)

    with torch.no_grad():

        if torch.cuda.is_available():
            inputs_test = torch.cuda.FloatTensor(
                sample["image"].unsqueeze(0).cuda().float()
            )
        else:
            inputs_test = torch.FloatTensor(sample["image"].unsqueeze(0).float())

        d1, d2, d3, d4, d5, d6, d7 = net(inputs_test)

        pred = d1[:, 0, :, :]
        predict = norm_pred(pred)

        predict = predict.squeeze()
        predict_np = predict.cpu().detach().numpy()
        img = Image.fromarray(predict_np * 255).convert("RGB")

        del d1, d2, d3, d4, d5, d6, d7, pred, predict, predict_np, inputs_test, sample

        return img

max_stars_repo_name: basharbme/backgroundremover

max_issues_repo_name: basharbme/backgroundremover

max_forks_repo_name: basharbme/backgroundremover


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr  2 18:14:37 2019

@author: kazuki.onodera
"""

import numpy as np
import pandas as pd
import os, gc
from glob import glob
from tqdm import tqdm

import sys
sys.path.append(f'/home/{os.environ.get("USER")}/PythonLibrary')
import lgbextension as ex
import lightgbm as lgb
from multiprocessing import cpu_count
from time import sleep
#sleep(60 * 5)

from sklearn.metrics import roc_auc_score

import utils
utils.start(__file__)
#==============================================================================

SEED = np.random.randint(9999)
print('SEED:', SEED)

NFOLD = 5

param = {
         'objective': 'binary',
         'metric': 'None',

         'learning_rate': 0.01,
         'max_depth': -1,
         'num_leaves': 2**6 -1,
#         'num_leaves': 2**4 -1,
         'max_bin': 255,

         'min_child_weight': 10,
         'min_data_in_leaf': 150,
         'reg_lambda': 0.5,  # L2 regularization term on weights.
         'reg_alpha': 0.5,  # L1 regularization term on weights.

         'colsample_bytree': 0.5,
         'subsample': 0.7,
#         'nthread': 32,
         'nthread': cpu_count(),
         'bagging_freq': 5,
         'verbose':-1,
         }

NROUND = 99999
ESR = 100
VERBOSE_EVAL = 50

USE_PREF = [
        'f001',
        'f003',
#        'f004',
#        'f005',
#        'f006',
#        'f007',
#        'f008',
#        'f009',
#        'f010',
#        'f011',
#        'f012',
        ]

COL = pd.read_csv('LOG/imp_801_cv.py.csv').feature.tolist()

# =============================================================================
# load
# =============================================================================

files_tr = sorted(glob('../data/train_f*.pkl'))

# USE_PREF
li = []
for i in files_tr:
    for j in USE_PREF:
        if j in i:
            li.append(i)
            break
files_tr = li

[print(i,f) for i,f in enumerate(files_tr)]

X_train = pd.concat([
                pd.read_pickle(f) for f in tqdm(files_tr, mininterval=30)
               ], axis=1)

y_train = utils.load_target()['target']

if X_train.columns.duplicated().sum()>0:
    raise Exception(f'duplicated!: { X_train.columns[X_train.columns.duplicated()] }')
print('no dup :) ')
print(f'X_train.shape {X_train.shape}')

gc.collect()

# =============================================================================
# cv
# =============================================================================

model_all = []
nround_mean = 0
loss_list = []

for i in range(100, 900, 100):
    col = COL[:i]
    dtrain = lgb.Dataset(X_train[col], y_train.values,
                     free_raw_data=False)
    gc.collect()

    param['seed'] = np.random.randint(9999)

    ret, models = lgb.cv(param, dtrain, NROUND,
                         nfold=NFOLD,
                         stratified=True, shuffle=True,
                         feval=ex.eval_auc,
                         early_stopping_rounds=ESR,
                         verbose_eval=VERBOSE_EVAL,
                         seed=SEED+i)

    p_train = ex.eval_oob(X_train[col], y_train.values, models, SEED+i,
                         stratified=True, shuffle=True)

    model_all += models
    nround_mean += len(ret['auc-mean'])
    loss_list.append( ret['auc-mean'][-1] )

    utils.send_line(f'oof AUC({i}): {round(roc_auc_score(y_train, p_train), 5)}')

#==============================================================================
utils.end(__file__)
#utils.stop_instance()


text:
# "numericalPhi.py"
# 2017-02-07
# David Merrell
#
# A class that bridges the gap between Z3 and
# numpy. It encodes a formula in linear real arithmetic
# as a set of linear constraints
# Ax <= b.
# For now it just handles atoms containing linear
# real inequalities.
#
# Can be used to query whether an array of points
# satisfy the formula.
#
# Assumes variables in the Z3 formula are represented
# by Z3_OP_UNINTERPRETED

from z3 import *
import numpy as np
import z3extra as ze

class numericalPhi:

    def __init__(self, phi, valid_vars=None):
        """
        Build an instance from a Z3 formula.
        Ignore predicates containing variables
        that are not valid.
        """

        print("PHI:\n", phi)
        phi = simplify(phi)
        print("SIMPLIFIED PHI:\n", phi)
        phi = ze.close(phi)
        print("CLOSED PHI:\n", phi)

        if valid_vars is None:
            self.valid_vars = list(ze.get_vars(phi))
        else:
            self.valid_vars = valid_vars

        # We want to encode the formula as a set
        # of linear constraints of the form
        # Ax <= b.
        # We also want a vector d which maps
        # inequalities to disjuncts.
        self.constraint_matrix, self.constraint_b, self.disjuncts \
                = self.build_constraint_matrix(phi)

    def build_constraint_matrix(self, phi):
        """
        Build arrays A, b, and d, where
        Ax <= b
        encodes the formula, and d keeps track
        of which disjunct a given constraint
        belongs to.
        """
        A = []
        b = []
        d = []

        disjuncts = ze.toDNF(phi)
        print("DNF PHI:\n", disjuncts)

        dj_num = 0
        for dj in disjuncts:
            print("DISJUNCT:\n", dj)

            if is_and(dj):
                for cnj in dj.children():

                    print("CONJUNCT:\n", cnj)
                    row, bconst = self.extract_ineq(cnj)
                    if len(row) > 0 :
                        A.append(row)
                        b.append(bconst)
                        d.append(dj_num)

            elif is_le(dj) or is_ge(dj) or is_not(dj):
                row, bconst = self.extract_ineq(dj)
                if len(row) > 0 :
                    A.append(row)
                    b.append(bconst)
                    d.append(dj_num)

            dj_num += 1

        A = np.array(A)
        b = np.array(b)
        d = np.array(d)

        assert A.shape[0] == len(b) and len(d) == len(b)

        return A, b, d

    def extract_ineq(self, ineq):

        row = []
        vnames = self.get_ordered_varnames()

        # We only handle linear inequalities.
        if not( is_le(ineq) or is_ge(ineq) or is_not(ineq) ):
            print("{} is not an inequality.".format(ineq))
            return [], 0

        # Handle the case of a negation.
        sgn = 1
        if is_not(ineq):
            assert len(ineq.children()) == 1,\
                    "{} MULTIPLE CHILDREN?".format(ineq)
            ineq = ineq.children()[0]
            sgn = -1

        # Extract coefficients from the atom.
        coeff_d = self.get_atom_coeffs(ineq)

        # Add coefficients to the row vector, in order.
        for vname in vnames:
            if vname in list(coeff_d.keys()):
                row.append(coeff_d[vname])
            else:
                row.append(0.0)

        if len(coeff_d) > 0 and 'const' in list(coeff_d.keys()):
            return row, coeff_d['const']
        else:
            return [], 0

    def get_ordered_vars(self):
        """
        Get the variables in sorted (alphabetical) order.
        This is the canonical ordering used throughout
        this class.
        """
        varnames = np.array([str(v) for v in self.valid_vars])
        sorted_inds = np.argsort(varnames)
        sorted_vars = list(varnames[sorted_inds])

        return sorted_vars

    def get_ordered_varnames(self):
        """
        Get variable names in sorted order.
        """
        return [str(v) for v in self.get_ordered_vars()]

    def get_atom_coeffs(self, atom):
        """
        Given an atom, extract the coefficients of
        its variables in a dictionary indexed by
        variable names.
        """

        atom = simplify(atom)

        # Assume that we're given an atomic *inequality*.
        # We want to have "<="; if we have a ">=", we
        # will multiply everything by -1.
        sgn = 1
        if is_le(atom):
            sgn = 1
        elif is_ge(atom):
            sgn = -1
        else:
            print("ATOM MUST BE A <= or >=")
            return {}

        # Return a dictionary whose keys are gaussian variable names
        # and values are numbers --- the variables' coefficients.
        coeff_d = {}

        varnames = self.get_ordered_varnames()

        lhs_coeffs = self.get_lin_coeffs(atom.children()[0])
        rhs_coeffs = self.get_lin_coeffs(atom.children()[1])
        key_union = set(lhs_coeffs.keys()).union(list(rhs_coeffs.keys()))

        for varname in key_union:
            coeff_d[varname] = 0
            if varname in varnames:
                if varname in list(lhs_coeffs.keys()):
                    coeff_d[varname] += sgn*lhs_coeffs[varname]
                if varname in list(rhs_coeffs.keys()):
                    coeff_d[varname] += -1*sgn*rhs_coeffs[varname]
            elif varname=='const':
                if varname in list(lhs_coeffs.keys()):
                    coeff_d[varname] += -1*sgn*lhs_coeffs[varname]
                if varname in list(rhs_coeffs.keys()):
                    coeff_d[varname] += sgn*rhs_coeffs[varname]
            else:
                print("{}\nNOT A USABLE ATOM.".format(atom))
                return {}

        return coeff_d

    def get_lin_coeffs(self,lincom):

        result_dict = {}

        # Some code that gets reused a lot:
        def extract_from_product(prod):
            var_inds = []
            coeff = 1
            for i, factor in enumerate(prod.children()):
                if is_rational_value(factor) or is_algebraic_value(factor):
                    coeff = coeff * factor
                else:
                    var_inds.append(i)
            assert len(var_inds) <= 1, "{} must have only one variable.".format(prod)
            value = simplify(coeff)
            value = 1.0*value.numerator_as_long()/value.denominator_as_long()
            return value, prod.children()[var_inds[0]]

        result_dict['const'] = 0

        #print "LINEAR COMBINATION", lincom

        if is_rational_value(lincom):
            value = simplify(lincom)
            value = 1.0*value.numerator_as_long()/value.denominator_as_long()
            result_dict['const'] += value
        elif is_mul(lincom):
            coeff, var = extract_from_product(lincom)
            result_dict[str(var)] = coeff
        elif is_add(lincom):
            for child in lincom.children():
                ch = simplify(child)
                if is_mul(child):
                    coeff, var = extract_from_product(child)
                    result_dict[str(var)] = coeff
                elif is_rational_value(child) or is_algebraic_value(child):
                    result_dict['const'] += child
                else:
                    result_dict[str(child)] = 1
        elif lincom.decl().kind() == Z3_OP_UNINTERPRETED:
            result_dict[str(lincom)] = 1

        return result_dict

if __name__=="__main__":

    x = Real('x')
    y = Real('y')

    phi = Or(And(2*x + 3*y < 100,x > 0, y > 0), And(-7*x + -4*y < 123, x < -2, y < -3))

    nphi = numericalPhi(phi)

    print("MATRIX:\n", nphi.constraint_matrix)
    print("RHS:\n", nphi.constraint_b)
    print("DISJUNCTS:\n", nphi.disjuncts)

max_stars_repo_name: sedrews/fairsquare

max_issues_repo_name: sedrews/fairsquare

max_forks_repo_name: sedrews/fairsquare


text:
import os
import itertools
from collections import Counter

import h5py
from scipy.spatial import KDTree
import scipy.sparse as sp
from scipy.sparse import csgraph
import numpy as np
import zmesh

from . import skel, u

def mesh_seg(seg, voxel_offset, voxel_res, segid=None):

    if segid is not None:
        seg = seg == segid

    mesher = zmesh.Mesher((1, 1, 1))
    mesher.mesh(np.swapaxes(seg, 0, 2))
    mesh_id = mesher.ids()[0]
    mesh = mesher.get_mesh(mesh_id, normals=False,
                           simplification_factor=0,
                           max_simplification_error=0)
    mesh.vertices = (mesh.vertices + voxel_offset) * voxel_res
    mesher.erase(mesh_id)
    mesher.clear()
    return {"vertices": mesh.vertices, "faces": mesh.faces.reshape(-1, 3),
            "num_vertices": len(mesh.vertices)}

def spine_head_volume_near_pts(skeleton, mesh, pts, bbox_margin,
                               skel_kdtree=None, path_inds=None,
                               spine_labels=None, cv_skel=True,
                               verbose=True, root=None,
                               simplest_cut=False):
    """Computes the spine head volume near a list of points (often synapses)"""
    if path_inds is None:
        path_inds, _ = skel.compute_path_info(skeleton, root=root,
                                              cv_skel=cv_skel)

    if spine_labels is None:
        path_radii = [skeleton.radii[inds] for inds in path_inds]
        spine_labels = skel.label_skeleton(skeleton, path_inds=path_inds,
                                           path_radii=path_radii)

    if skel_kdtree is None:
        skel_kdtree = KDTree(skeleton.vertices)

    volumes = list()
    errors = list()
    for (i, pt) in enumerate(pts):
        if verbose:
            print(f"\r#{i+1} of {len(pts)}", end="")
        try:
            # Extract spine and mesh
            spine_skel_inds, root = skel.extract_spine_near_pts(
                                    skeleton, spine_labels,
                                    [pt], path_inds=path_inds,
                                    return_all_paths=True)[0]
            spine_skel = skel.skel_by_inds(skeleton, spine_skel_inds)
            spine_root = skel.translate_skel_ids(spine_skel, [root])[0]
            spine_mesh = isolate_spine_mesh(mesh, skeleton, spine_skel_inds,
                                            kdtree=skel_kdtree,
                                            bbox_margin=bbox_margin)

            # Extract head
            spine_head, _ = isolate_spine_head(spine_skel, pt,
                                               spine_mesh=spine_mesh,
                                               spine_root=spine_root,
                                               cv_skel=False,
                                               simplest_cut=simplest_cut)
            sealed = simple_mesh_seal(spine_head)
            # I'm pretty sure that this helps to avoid overflow
            centered = center_mesh(sealed)
            scaled = scale_mesh(centered, [4, 4, 40], [3.58, 3.58, 40])
            volumes.append(mesh_volume(scaled))

        except Exception as e:
            if verbose:
                print("")
            print(f"EXCEPTION with pt {pt}: {e}")
            print(f"Recording volume of -1 for this pt")
            volumes.append(-1)
            errors.append((i, pt))

    if verbose:
        print("")
    return volumes, errors

def spine_and_head_near_pt(skeleton, mesh, pt, bbox_margin,
                           skel_kdtree=None, path_inds=None,
                           spine_labels=None, cv_skel=True,
                           root=None, simplest_cut=False):
    """Separates a spine and spine head (along with meshes)"""
    if path_inds is None:
        path_inds, _ = skel.compute_path_info(skeleton, root=root,
                                              cv_skel=cv_skel)

    if spine_labels is None:
        path_radii = [skeleton.radii[inds] for inds in path_inds]
        spine_labels = skel.label_skeleton(skeleton, path_inds=path_inds,
                                           path_radii=path_radii)

    if skel_kdtree is None:
        skel_kdtree = KDTree(skeleton.vertices)

    # Extract spine and mesh
    spine_skel_inds, root = skel.extract_spine_near_pts(
                            skeleton, spine_labels,
                            [pt], path_inds=path_inds,
                            return_all_paths=True,
                            kdt=skel_kdtree)[0]
    spine_skel = skel.skel_by_inds(skeleton, spine_skel_inds)
    spine_root = skel.translate_skel_ids(spine_skel, [root])[0]
    spine_mesh = isolate_spine_mesh(mesh, skeleton, spine_skel_inds,
                                    kdtree=skel_kdtree,
                                    bbox_margin=bbox_margin)

    # Extract head
    head_mesh, head_inds = isolate_spine_head(spine_skel, pt,
                                              spine_mesh=spine_mesh,
                                              spine_root=spine_root,
                                              cv_skel=False,
                                              simplest_cut=simplest_cut)

    head_skel = skel.skel_by_inds(spine_skel, head_inds)

    return spine_skel, spine_mesh, head_skel, head_mesh

def isolate_spine_head(spine_skel, synapse_pt, spine_mesh=None,
                       skel_kdtree=None, spine_root=None, cv_skel=False,
                       simplest_cut=False):
    """
    Spine head extraction using a spine mesh, a spine skeleton, and a synapse
    point on the spine.
    """
    if skel_kdtree is None:
        skel_kdtree = KDTree(spine_skel.vertices)

    syn_skel_pt = skel_kdtree.query(synapse_pt)[1]

    if spine_root is None:
        spine_root, path_inds = skel.find_furthest_pt(spine_skel, syn_skel_pt)
    else:
        path_inds = skel.paths_containing_pair(
                        spine_skel, spine_root, syn_skel_pt,
                        max_len=True, cv_skel=cv_skel)

    # Ensuring that the root node is at the beginning of the path
    # for the cut fn
    if np.nonzero(path_inds == spine_root)[0] > len(path_inds) // 2:
        path_inds = path_inds[::-1]
    cut_ind, local_cut_ind = skel.medium_radius_cut_pt(
                                 spine_skel, path_inds,
                                 simplest_vers=simplest_cut)
    remaining_pts = path_inds[local_cut_ind:]

    # Restricting mesh filtering to a single path to avoid "split path" problem
    # -> this wasn't worth it, it fixed the split spine cases, but made others worse
    #    by artificially merging them
    #path_skel = skel.skel_by_inds(spine_skel, path_inds)
    #path_kdtree = KDTree(path_skel.vertices)
    #path_rem_pts = skel.translate_skel_ids(path_skel, remaining_pts)

    if spine_mesh is None:
        return remaining_pts
    else:
        return filter_mesh_by_node_prox(
                   spine_mesh, spine_skel, remaining_pts,
                   kdtree=skel_kdtree), remaining_pts

def isolate_spine_mesh(mesh, skel, spine_nodes, kdtree=None, bbox_margin=None):
    """
    Spine mesh extraction using a cell's skeleton and the nodes of that
    skeleton that describe the spine.
    """
    return filter_mesh_by_node_prox(mesh, skel, spine_nodes,
                                    kdtree=kdtree, bbox_margin=bbox_margin)

def filter_mesh_by_node_prox(mesh, skel, nodes, kdtree=None, bbox_margin=None):
    """
    Finds the subset of a mesh whose closest points on the skeletons are listed
    by the node indices within nodes
    """
    if kdtree is None:
        kdtree = KDTree(skel.vertices)

    if bbox_margin is not None:
        skel_coords = skel.vertices[nodes]
        bbox_min, bbox_max = u.make_bbox(skel_coords, bbox_margin)
        mesh = mesh_within_bbox(mesh, bbox_min, bbox_max)

    node_set = set(nodes)
    new_inds = np.array([i for (i, v) in enumerate(mesh["vertices"])
                         if kdtree.query(v)[1] in node_set])
    return mesh_by_inds(mesh, new_inds)

def mesh_within_bbox(mesh, bbox_min, bbox_max):
    valid_inds = inds_within_bbox(mesh["vertices"], bbox_min, bbox_max)
    return mesh_by_inds(mesh, valid_inds)

def mesh_by_inds(mesh, inds):
    assert len(inds) > 0, "empty inds"
    new_verts = mesh["vertices"][inds]

    ind_map = np.empty((max(inds)+1,), dtype=inds.dtype)
    ind_map[inds] = np.arange(len(inds))
    explicit_faces = mesh["faces"].reshape((-1, 3))
    face_inds = np.all(np.isin(explicit_faces, inds), axis=1)
    new_faces = ind_map[explicit_faces[face_inds]].ravel()

    return dict(vertices=new_verts, faces=new_faces,
                num_vertices=len(new_verts))

def inds_within_bbox(vertices, bbox_min, bbox_max):
    overmin = np.all(vertices >= bbox_min, axis=1)
    undermax = np.all(vertices <= bbox_max, axis=1)

    return np.nonzero(overmin & undermax)[0]

def simple_mesh_seal(mesh):
    """Seals a mesh by adding one vertex at the centroid of each boundary"""
    components = find_mesh_boundary_components(mesh)
    for comp in components:
        new_pt = np.mean(mesh["vertices"][comp], axis=0)
        mesh = attach_new_pt(mesh, new_pt, boundary=comp)

    return mesh

def find_mesh_boundary_vertices(mesh):
    boundary_edges = find_mesh_boundary_edges(mesh)
    return list(set(itertools.chain.from_iterable(boundary_edges)))

def find_mesh_boundary_components(mesh):
    boundary_edges = find_mesh_boundary_edges(mesh)

    return edge_components(boundary_edges)

def edge_components(edges):
    rows, cols = zip(*edges)

    ids = list(set(rows + cols))
    idmap = {v: i for (i, v) in enumerate(ids)}
    rows = [idmap[v] for v in rows]
    cols = [idmap[v] for v in cols]

    vals = np.ones((len(rows),), dtype=np.uint8)
    num_ids = max(max(rows), max(cols)) + 1
    g = sp.coo_matrix((vals, (rows, cols)), shape=(num_ids, num_ids)).tocsr()

    num_comps, labels = csgraph.connected_components(
                            g, directed=False, return_labels=True)
    graph_comps = [np.nonzero(labels == i)[0] for i in range(num_comps)]

    return [[ids[v] for v in comp] for comp in graph_comps]

def find_mesh_boundary_edges(mesh):
    faces = mesh["faces"].reshape((-1, 3))
    edges = [tuple(sorted(edge)) for edge in
             np.vstack((faces[:, :2],
                        faces[:, 1:],
                        faces[:, [2, 0]]))]

    counter = Counter(edges)
    return [edge for (edge, count) in counter.items() if count == 1]

def attach_new_pt(mesh, new_pt, boundary=None):
    new_verts = np.vstack((mesh["vertices"], new_pt))
    new_i = len(mesh["vertices"])

    boundary_edges = find_mesh_boundary_edges(mesh)
    if boundary is not None:
        bset = set(boundary)
        boundary_edges = [edge for edge in boundary_edges
                          if (edge[0] in bset) and (edge[1] in bset)]

    new_faces = np.array(assemble_consistent_faces(boundary_edges, new_i))
    new_faces = flip_if_mostly_inwards(new_faces, new_verts, new_pt)
    all_new_faces = np.hstack((mesh["faces"], new_faces.ravel()))

    return dict(vertices=new_verts, faces=all_new_faces,
                num_vertices=len(new_verts))

def assemble_consistent_faces(edges, new_i):
    vertex_order = dfs_order_vertices(edges)
    return [[vertex_order[i-1], vertex_order[i], new_i]
            for i in range(len(vertex_order))]

def dfs_order_vertices(edges):
    rows, cols = zip(*edges)

    ids = list(set(rows + cols))
    idmap = {v: i for (i, v) in enumerate(ids)}
    rows = [idmap[v] for v in rows]
    cols = [idmap[v] for v in cols]

    vals = np.ones((len(rows),), dtype=np.uint8)
    num_ids = max(max(rows), max(cols)) + 1
    g = sp.coo_matrix((vals, (rows, cols)), shape=(num_ids, num_ids)).tocsr()

    return [ids[i] for i in csgraph.depth_first_order(g, 0, directed=False)[0]]

def flip_if_mostly_inwards(faces, vertices, new_pt):
    bad_faces = find_bad_face_normals(faces, vertices, new_pt)
    if sum(bad_faces) > len(faces) // 2:
        faces = faces[:, [1, 0, 2]]
    return faces

def fix_bad_face_normals(faces, vertices, new_pt):
    bad_faces = find_bad_face_normals(faces, vertices, new_pt)
    faces = faces.copy()
    faces[bad_faces] = faces[bad_faces][:, [1, 0, 2]]

    return faces

def find_bad_face_normals(faces, vertices, new_pt):
    centroid = np.mean(vertices, axis=0)
    normals = compute_normals(faces, vertices)

    return (normals @ (new_pt - centroid)) < 0

def compute_normals(faces, vertices):
    v0 = vertices[faces[:, 0]]
    v1 = vertices[faces[:, 1]]
    v2 = vertices[faces[:, 2]]

    return np.cross(v1-v0, v2-v0)

def mesh_volume(mesh, scale=1000):

    vs = mesh["vertices"] / scale
    fs = mesh["faces"].reshape((-1, 3))

    v0 = vs[fs[:, 0]]
    v1 = vs[fs[:, 1]]
    v2 = vs[fs[:, 2]]

    return np.abs(np.sum(np.cross(v0, v1) * v2 / 6.))

def center_mesh(mesh):
    centroid = np.mean(mesh["vertices"], axis=0)
    return dict(vertices=mesh["vertices"]-centroid,
                faces=mesh["faces"], num_vertices=mesh["num_vertices"])

def scale_mesh(mesh, old_res, new_res):
    verts = (mesh["vertices"] / old_res) * new_res
    return dict(vertices=verts, faces=mesh["faces"],
                num_vertices=mesh["num_vertices"])

def write_mesh(mesh, filename):
    if os.path.isfile(filename):
        os.remove(filename)

    with h5py.File(filename) as f:
        f.create_dataset("vertices", data=mesh["vertices"])
        f.create_dataset("faces", data=mesh["faces"])
        f.create_dataset("num_vertices", data=mesh["num_vertices"])

def read_mesh(filename):
    assert os.path.isfile(filename)

    with h5py.File(filename) as f:
        vertices = f["vertices"][()]
        faces = f["faces"][()]
        num_vertices = f["num_vertices"][()]

    return dict(vertices=vertices, faces=faces, num_vertices=num_vertices)

reason: import numpy,import scipy,from scipy


text:
# Linear Regression : It is only for study
# Copyright : yoonkt200@gmail.com
# Apache License 2.0
# =============================================================================

import numpy as np

class OLS():
    def __init__(self, x_data, y_data):
        self._x_data = x_data
        self._y_data = y_data

    def fit(self):
        # exactly same as MLE (when y has normal distribution)
        self._coefficients = np.linalg.inv(self._x_data.transpose().dot(self._x_data)).dot(self._x_data.transpose()).dot(self._y_data)
        y_hat = np.sum(self._x_data * self._coefficients, axis=1)
        mse = np.mean(np.square(self._y_data - y_hat))
        mse_of_average_data = np.mean(np.square(self._y_data - np.mean(self._y_data)))
        self._r2_score = 1 - (mse / mse_of_average_data)

    def print_results(self):
        print(self._coefficients)
        print(self._r2_score)

if __name__ == "__main__":
    # intercept, friends, work hour, Acquired doctoral degree
    input = [[1, 49, 4, 0], [1, 41, 9, 0], [1, 40, 8, 0], [1, 25, 6, 0], [1, 21, 1, 0], [1, 21, 0, 0], [1, 19, 3, 0],
             [1, 19, 0, 0], [1, 18, 9, 0], [1, 18, 8, 0], [1, 16, 4, 0], [1, 15, 3, 0], [1, 15, 0, 0], [1, 15, 2, 0],
             [1, 15, 7, 0], [1, 14, 0, 0], [1, 14, 1, 0], [1, 13, 1, 0], [1, 13, 7, 0], [1, 13, 4, 0], [1, 13, 2, 0],
             [1, 12, 5, 0], [1, 12, 0, 0], [1, 11, 9, 0], [1, 10, 9, 0], [1, 10, 1, 0], [1, 10, 1, 0], [1, 10, 7, 0],
             [1, 10, 9, 0], [1, 10, 1, 0], [1, 10, 6, 0], [1, 10, 6, 0], [1, 10, 8, 0], [1, 10, 10, 0], [1, 10, 6, 0],
             [1, 10, 0, 0], [1, 10, 5, 0], [1, 10, 3, 0], [1, 10, 4, 0], [1, 9, 9, 0], [1, 9, 9, 0], [1, 9, 0, 0],
             [1, 9, 0, 0], [1, 9, 6, 0], [1, 9, 10, 0], [1, 9, 8, 0], [1, 9, 5, 0], [1, 9, 2, 0], [1, 9, 9, 0],
             [1, 9, 10, 0], [1, 9, 7, 0], [1, 9, 2, 0], [1, 9, 0, 0], [1, 9, 4, 0], [1, 9, 6, 0], [1, 9, 4, 0],
             [1, 9, 7, 0], [1, 8, 3, 0], [1, 8, 2, 0], [1, 8, 4, 0], [1, 8, 9, 0], [1, 8, 2, 0], [1, 8, 3, 0],
             [1, 8, 5, 0], [1, 8, 8, 0], [1, 8, 0, 0], [1, 8, 9, 0], [1, 8, 10, 0], [1, 8, 5, 0], [1, 8, 5, 0],
             [1, 7, 5, 0], [1, 7, 5, 0], [1, 7, 0, 0], [1, 7, 2, 0], [1, 7, 8, 0], [1, 7, 10, 0], [1, 7, 5, 0],
             [1, 7, 3, 0], [1, 7, 3, 0], [1, 7, 6, 0], [1, 7, 7, 0], [1, 7, 7, 0], [1, 7, 9, 0], [1, 7, 3, 0],
             [1, 7, 8, 0], [1, 6, 4, 0], [1, 6, 6, 0], [1, 6, 4, 0], [1, 6, 9, 0], [1, 6, 0, 0], [1, 6, 1, 0],
             [1, 6, 4, 0], [1, 6, 1, 0], [1, 6, 0, 0], [1, 6, 7, 0], [1, 6, 0, 0], [1, 6, 8, 0], [1, 6, 4, 0],
             [1, 6, 2, 1], [1, 6, 1, 1], [1, 6, 3, 1], [1, 6, 6, 1], [1, 6, 4, 1], [1, 6, 4, 1], [1, 6, 1, 1],
             [1, 6, 3, 1], [1, 6, 4, 1], [1, 5, 1, 1], [1, 5, 9, 1], [1, 5, 4, 1], [1, 5, 6, 1], [1, 5, 4, 1],
             [1, 5, 4, 1], [1, 5, 10, 1], [1, 5, 5, 1], [1, 5, 2, 1], [1, 5, 4, 1], [1, 5, 4, 1], [1, 5, 9, 1],
             [1, 5, 3, 1], [1, 5, 10, 1], [1, 5, 2, 1], [1, 5, 2, 1], [1, 5, 9, 1], [1, 4, 8, 1], [1, 4, 6, 1],
             [1, 4, 0, 1], [1, 4, 10, 1], [1, 4, 5, 1], [1, 4, 10, 1], [1, 4, 9, 1], [1, 4, 1, 1], [1, 4, 4, 1],
             [1, 4, 4, 1], [1, 4, 0, 1], [1, 4, 3, 1], [1, 4, 1, 1], [1, 4, 3, 1], [1, 4, 2, 1], [1, 4, 4, 1],
             [1, 4, 4, 1], [1, 4, 8, 1], [1, 4, 2, 1], [1, 4, 4, 1], [1, 3, 2, 1], [1, 3, 6, 1], [1, 3, 4, 1],
             [1, 3, 7, 1], [1, 3, 4, 1], [1, 3, 1, 1], [1, 3, 10, 1], [1, 3, 3, 1], [1, 3, 4, 1], [1, 3, 7, 1],
             [1, 3, 5, 1], [1, 3, 6, 1], [1, 3, 1, 1], [1, 3, 6, 1], [1, 3, 10, 1], [1, 3, 2, 1], [1, 3, 4, 1],
             [1, 3, 2, 1], [1, 3, 1, 1], [1, 3, 5, 1], [1, 2, 4, 1], [1, 2, 2, 1], [1, 2, 8, 1], [1, 2, 3, 1],
             [1, 2, 1, 1], [1, 2, 9, 1], [1, 2, 10, 1], [1, 2, 9, 1], [1, 2, 4, 1], [1, 2, 5, 1], [1, 2, 0, 1],
             [1, 2, 9, 1], [1, 2, 9, 1], [1, 2, 0, 1], [1, 2, 1, 1], [1, 2, 1, 1], [1, 2, 4, 1], [1, 1, 0, 1],
             [1, 1, 2, 1], [1, 1, 2, 1], [1, 1, 5, 1], [1, 1, 3, 1], [1, 1, 10, 1], [1, 1, 6, 1], [1, 1, 0, 1],
             [1, 1, 8, 1], [1, 1, 6, 1], [1, 1, 4, 1], [1, 1, 9, 1], [1, 1, 9, 1], [1, 1, 4, 1], [1, 1, 2, 1],
             [1, 1, 9, 1], [1, 1, 0, 1], [1, 1, 8, 1], [1, 1, 6, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 5, 1]]
    x_data = np.array(input)

    # output hour
    output = [68.77, 51.25, 52.08, 38.36, 44.54, 57.13, 51.4, 41.42, 31.22, 34.76, 54.01, 38.79, 47.59, 49.1, 27.66,
              41.03, 36.73, 48.65, 28.12, 46.62, 35.57, 32.98, 35, 26.07, 23.77, 39.73, 40.57, 31.65, 31.21, 36.32,
              20.45, 21.93, 26.02, 27.34, 23.49, 46.94, 30.5, 33.8, 24.23, 21.4, 27.94, 32.24, 40.57, 25.07, 19.42,
              22.39, 18.42, 46.96, 23.72, 26.41, 26.97, 36.76, 40.32, 35.02, 29.47, 30.2, 31, 38.11, 38.18, 36.31,
              21.03, 30.86, 36.07, 28.66, 29.08, 37.28, 15.28, 24.17, 22.31, 30.17, 25.53, 19.85, 35.37, 44.6, 17.23,
              13.47, 26.33, 35.02, 32.09, 24.81, 19.33, 28.77, 24.26, 31.98, 25.73, 24.86, 16.28, 34.51, 15.23, 39.72,
              40.8, 26.06, 35.76, 34.76, 16.13, 44.04, 18.03, 19.65, 32.62, 35.59, 39.43, 14.18, 35.24, 40.13, 41.82,
              35.45, 36.07, 43.67, 24.61, 20.9, 21.9, 18.79, 27.61, 27.21, 26.61, 29.77, 20.59, 27.53, 13.82, 33.2, 25,
              33.1, 36.65, 18.63, 14.87, 22.2, 36.81, 25.53, 24.62, 26.25, 18.21, 28.08, 19.42, 29.79, 32.8, 35.99,
              28.32, 27.79, 35.88, 29.06, 36.28, 14.1, 36.63, 37.49, 26.9, 18.58, 38.48, 24.48, 18.95, 33.55, 14.24,
              29.04, 32.51, 25.63, 22.22, 19, 32.73, 15.16, 13.9, 27.2, 32.01, 29.27, 33, 13.74, 20.42, 27.32, 18.23,
              35.35, 28.48, 9.08, 24.62, 20.12, 35.26, 19.92, 31.02, 16.49, 12.16, 30.7, 31.22, 34.65, 13.13, 27.51,
              33.2, 31.57, 14.1, 33.42, 17.44, 10.12, 24.42, 9.82, 23.39, 30.93, 15.03, 21.67, 31.09, 33.29, 22.61,
              26.89, 23.48, 8.38, 27.81, 32.35, 23.84]
    y_data = np.array(output)

    # fit OLS
    ols = OLS(x_data=x_data, y_data=y_data)
    ols.fit()
    ols.print_results()


text:
import pandas as pd
import os
import numpy as np
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler

org_path = r'D:\dataset\2021智慧農業數位分身創新應用競賽\org'
all_path = r'D:\dataset\2021智慧農業數位分身創新應用競賽\org\train-test.csv'
save_path = r'D:\dataset\2021智慧農業數位分身創新應用競賽\dataset_lstm'

if not os.path.isdir(save_path):
    os.mkdir(save_path)

train_df = pd.read_csv(os.path.join(org_path, 'train_data.csv'), index_col=0)
col_name = train_df.columns.to_list()

# 在測試資料新增label欄位並填入NAN
test_df = pd.read_csv(os.path.join(org_path, 'test_data.csv'), index_col=0)
col_num = train_df.shape[1]
data_num = test_df.shape[1]
nan_array = np.full([test_df.shape[0], col_num - data_num], np.nan)
test_df = test_df.reindex(columns=col_name)
test_df.iloc[:, data_num:] = nan_array
# concat 訓練資料與測試資料
all_df = pd.concat([train_df, test_df], axis=0, ignore_index=True)
# 對時間排列
all_df['d.log_time'] = pd.to_datetime(all_df['d.log_time'])
all_df.sort_values('d.log_time', inplace=True)
all_df.to_csv(all_path, index=None)

# 正規化
data = all_df.iloc[:, 1:data_num]
x_scaler = StandardScaler().fit(data)
data = x_scaler.transform(data)
all_df.iloc[:, 1:data_num] = data

# 前10個時間點(包含當下)當做訓練資料，若遇到nan則跳過
n = 10
train_data = []
train_label = []
test_data = []
for i in tqdm(range(len(all_df) - n)):
    if np.isnan(all_df.iloc[i + n - 1]['actuator01']):  # 測試集
        data = all_df.iloc[i:i + n, 1:data_num].values  # 第一行是時間
        test_data.append(data)
    else:  # 訓練驗證集
        data = all_df.iloc[i:i + n, 1:data_num].values  # 第一行是時間
        label = all_df.iloc[i + n - 1, data_num:].values
        train_data.append(data)
        train_label.append(label)

train_data = np.array(train_data)
train_label = np.array(train_label)
test_data = np.array(test_data)
print('train_data:', train_data)
print('train_label:', train_label.shape)
print('test_data:', test_data.shape)

# 存檔.npy
np.save(os.path.join(save_path, 'train_val_data.npy'), train_data)
np.save(os.path.join(save_path, 'train_val_label.npy'), train_label)
np.save(os.path.join(save_path, 'test_data.npy'), test_data)