text:
import cv2
import random
import colorsys
import numpy as np
import tensorflow as tf
from core.config import cfg
import tensorflow.keras.backend as K
import os

def load_freeze_layer(model='yolov4', num_detection_layers=1):
    if num_detection_layers == 3:  # for yolo
        if model == 'yolov3':
            freeze_layouts = ['conv2d_58', 'conv2d_66', 'conv2d_74']
        else:
            freeze_layouts = ['conv2d_93', 'conv2d_101', 'conv2d_109']
    elif num_detection_layers == 2: # yolo-tiny
        if model == 'yolov3':
            freeze_layouts = ['conv2d_9', 'conv2d_12']
        else:
            freeze_layouts = ['conv2d_17', 'conv2d_20']
    elif num_detection_layers == 1: # yolo-custom-tiny
        # custom yolov4
        freeze_layouts = ['conv2d_17']
    return freeze_layouts

def load_weights(model, weights_file, model_name='yolov4', num_detection_layer=1):
    if num_detection_layer == 3:
        if model_name == 'yolov3':
            layer_size = 75
            output_pos = [58, 66, 74]
        else:
            layer_size = 110
            output_pos = [93, 101, 109]
    elif num_detection_layer == 2:   # yolo-tiny ver
        if model_name == 'yolov3':
            layer_size = 13
            output_pos = [9, 12]
        else:
            layer_size = 21
            output_pos = [17, 20]
    elif num_detection_layer == 1:   # custom model (detection 층 1개)
        if model_name == 'yolov3':  # 얘 안 씀
            layer_size = 13
            output_pos = [12]
        else:
            layer_size = 18
            output_pos = [17]

    wf = open(weights_file, 'rb')
    major, minor, revision, seen, _ = np.fromfile(wf, dtype=np.int32, count=5)

    j = 0
    for i in range(layer_size):
        conv_layer_name = 'conv2d_%d' %i if i > 0 else 'conv2d'
        bn_layer_name = 'batch_normalization_%d' %j if j > 0 else 'batch_normalization'

        conv_layer = model.get_layer(conv_layer_name)
        filters = conv_layer.filters
        k_size = conv_layer.kernel_size[0]
        in_dim = conv_layer.input_shape[-1]

        if i not in output_pos:
            # darknet weights: [beta, gamma, mean, variance]
            bn_weights = np.fromfile(wf, dtype=np.float32, count=4 * filters)
            # tf weights: [gamma, beta, mean, variance]
            bn_weights = bn_weights.reshape((4, filters))[[1, 0, 2, 3]]
            bn_layer = model.get_layer(bn_layer_name)
            j += 1
        else:
            conv_bias = np.fromfile(wf, dtype=np.float32, count=filters)

        # darknet shape (out_dim, in_dim, height, width)
        conv_shape = (filters, in_dim, k_size, k_size)
        conv_weights = np.fromfile(wf, dtype=np.float32, count=np.product(conv_shape))
        # tf shape (height, width, in_dim, out_dim)
        conv_weights = conv_weights.reshape(conv_shape).transpose([2, 3, 1, 0])

        if i not in output_pos:
            conv_layer.set_weights([conv_weights])
            bn_layer.set_weights(bn_weights)
        else:
            conv_layer.set_weights([conv_weights, conv_bias])

    # assert len(wf.read()) == 0, 'failed to read all data'
    wf.close()

def read_class_names(class_file_name):
    names = {}
    with open(class_file_name, 'r') as data:
        for ID, name in enumerate(data):
            names[ID] = name.strip('\n')
    return names

def load_config(FLAGS):
    if cfg.YOLO.NUM_YOLOLAYERS == 3:    # 3 for yolov4
        STRIDES = np.array(cfg.YOLO.STRIDES)
        if FLAGS.model == 'yolov4':
            ANCHORS = get_anchors(cfg.YOLO.ANCHORS)
        elif FLAGS.model == 'yolov3':
            ANCHORS = get_anchors(cfg.YOLO.ANCHORS_V3)

        XYSCALE = cfg.YOLO.XYSCALE if FLAGS.model == 'yolov4' else [1, 1, 1]
    elif cfg.YOLO.NUM_YOLOLAYERS == 2:    # 2 for tiny-yolov4
        STRIDES = np.array(cfg.YOLO.STRIDES_TINY)
        ANCHORS = get_anchors(cfg.YOLO.ANCHORS_TINY)
        XYSCALE = cfg.YOLO.XYSCALE_TINY if FLAGS.model == 'yolov4' else [1, 1]
    else:
        STRIDES = np.array(cfg.YOLO.STRIDES_CUSTOM)
        ANCHORS = get_anchors(cfg.YOLO.ANCHORS_CUSTOM)
        XYSCALE = cfg.YOLO.XYSCALE_CUSTOM if FLAGS.model == 'yolov4' else [1]
    NUM_CLASS = len(read_class_names(cfg.YOLO.CLASSES))

    return STRIDES, ANCHORS, NUM_CLASS, XYSCALE

def get_anchors(anchors_path):  # 디폴트 ver=2  (custom model)
    anchors = np.array(anchors_path)
    if cfg.YOLO.NUM_YOLOLAYERS == 3:
        return anchors.reshape(3, 3, 2)
    elif cfg.YOLO.NUM_YOLOLAYERS == 2:
        return anchors.reshape(2, 3, 2)
    else:
        return anchors.reshape(1, 3, 2)

def image_preprocess_convertedw(image, target_size, gt_boxes=None):
    ih, iw = target_size
    h, w, ch = image.shape
    if ch == 1:
        image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)

    scale = min(iw / w, ih / h)
    nw, nh = int(scale * w), int(scale * h)
    image_resized = cv2.resize(image, (nw, nh))

    image_paded = np.full(shape=[ih, iw, 3], fill_value=128.0)
    dw, dh = (iw - nw) // 2, (ih - nh) // 2

    image_paded[dh:nh + dh, dw:nw + dw, :] = image_resized
    image_paded = image_paded / 255.

    if gt_boxes is None:
        return image_paded
    else:
        gt_boxes[:, [0,]] = gt_boxes[:, [0,]] * scale + dw
        gt_boxes[:, [2,]] = gt_boxes[:, [2,]] * scale
        gt_boxes[:, [1,]] = gt_boxes[:, [1,]] * scale + dh
        gt_boxes[:, [3,]] = gt_boxes[:, [3,]] * scale
        # gt_boxes[:, [0, 2]] = gt_boxes[:, [0, 2]] * scale + dw
        # gt_boxes[:, [1, 3]] = gt_boxes[:, [1, 3]] * scale + dh
        return image_paded, gt_boxes

def image_preprocess(image, target_size, gt_boxes=None):

    ih, iw    = target_size
    h,  w, _  = image.shape

    scale = min(iw/w, ih/h)
    nw, nh  = int(scale * w), int(scale * h)
    image_resized = cv2.resize(image, (nw, nh))

    image_paded = np.full(shape=[ih, iw, 3], fill_value=128.0)
    dw, dh = (iw - nw) // 2, (ih-nh) // 2
    image_paded[dh:nh+dh, dw:nw+dw, :] = image_resized

    image_paded = image_paded / 255.

    if gt_boxes is None:
        return image_paded

    else:
        gt_boxes[:, [0, 2]] = gt_boxes[:, [0, 2]] * scale + dw
        gt_boxes[:, [1, 3]] = gt_boxes[:, [1, 3]] * scale + dh
        return image_paded, gt_boxes

def load_multiple_img(image_path, input_size, is_multiple):
    extensions = ['bmp', 'jpg', 'jpeg', 'png']
    isfolder = os.path.basename(image_path)
    if is_multiple:
        image_path = [os.path.join(image_path, path) for path in os.listdir(image_path) if any(path.endswith(ext) for ext in extensions)]
        images_data = []
        for idx, path in enumerate(image_path):
            original_image = cv2.imread(path)
            original_image = cv2.cvtColor(original_image, cv2.COLOR_BGR2RGB)
            original_image = np.array(original_image)
            ih, iw, channels = original_image.shape

            if channels == 1:
                original_image = cv2.cvtColor(original_image, cv2.COLOR_GRAY2BGR)

            image_data = image_preprocess(np.copy(original_image), [input_size, input_size])
            images_data.append(image_data)
        images_data = np.asarray(images_data).astype(np.float32)
    else:
        original_image = cv2.imread(image_path)

        image_data = image_preprocess(np.copy(original_image), [input_size, input_size])
        images_data = [image_data]
        images_data = np.asarray(images_data).astype(np.float32)
    return image_path, images_data

# from PIL import Image
# def load_multiple_img_converted(image_path, input_size, is_multiple):
#     extensions = ['bmp', 'jpg', 'jpeg', 'png']
#     isfolder = os.path.basename(image_path)
#     if is_multiple:
#         image_path = [os.path.join(image_path, path) for path in os.listdir(image_path) if any(path.endswith(ext) for ext in extensions)]
#         images_data = []
#         for idx, path in enumerate(image_path):
#             original_image = Image.open(path)
#             original_image = np.array(original_image)
#             _, _, ch = original_image.shape
#             if ch == 1:
#                 cv2.cvtColor(original_image, cv2.COLOR_GRAY2BGR)
#             # original_image = cv2.cvtColor(original_image, cv2.COLOR_BGR2RGB)
#             image_data = image_preprocess(np.copy(original_image), [input_size, input_size])
#             images_data.append(image_data)
#         images_data = np.asarray(images_data).astype(np.float32)
#     else:
#         original_image = cv2.imread(image_path)
#         image_data = image_preprocess(np.copy(original_image), [input_size, input_size])
#         images_data = [image_data]
#         images_data = np.asarray(images_data).astype(np.float32)
#     return image_path, images_data

# def compute_map(pred, target_bboxes, num_classes):
#     """
#         >> gt_bboxes preprocess된 거에서 conf=1인 것들 숫자 세서 그게 진짜 gt개수와 같은지 확인
#
#         0. 배치 단위로 batch x ~ 함, class별로 나눔
#         1. 배치 단위로 batch x bboxes num x (5+num_classes) (xywh, conf, classprob) gt,pred 가져와 iou계산
#         2. iou 0.5 미만인 경우 일괄 fp처리
#         3. iou 0.5 이상인 경우 highest iou=>tp
#         4. 3 을 gt마다 매칭되는 pred들로 묶음(iou 0.5이상) 가장 confidence높은 애가 tp, 나머지 fp처리. tp,fp 확정되면 process되었음
#         5. 각 pred box마다 recall, precision 구함(해당되는 gt없는 경우는? 모두 fp이므로 tp=0으로 계산)
#         5. recall을 0.1단위로 나누어 0~1 사이 11개 구간에서 해당 구간(0.1) 내에서 가장 높은 precision을 구함
#         6. 위에서 계산한 11개의 max precision들의 평균 구함. 걔가 ap
#         7. 각 클래스 별로 구한 ap들의 평균을 구함
#     """
#     aps = np.zeros(num_classes, dtype=np.float32)
#
#     # for class_idx in range(num_classes):
#     #     class_true_bboxes = target_bboxes[]
#     pred_infos = {0: {"detection_id": [0],
#                       "confidence": [0], "iou": [0], "processed": [False],
#                       "acc_tp": [0], "acc_fp": [0]}}  # 0 for image_id
#     # pred
#     batch_, _, _, _, channels_ = tf.keras.backend.int_shape(pred)
#     pred = tf.reshape(pred, (batch_, -1, channels_))
#
#     # gt
#     _, gt_bboxes, _ = target_bboxes.shape
#
#     true_bboxes = []
#     for batch_idx in range(batch_):
#         gt_count = 0
#         for bbox_idx in range(gt_bboxes):
#             if np.sum(target_bboxes[batch_idx, bbox_idx, :]) > 0:
#                 gt_count += 1
#         true_bbox = target_bboxes[batch_idx, :gt_count, :]
#         true_bboxes.append(true_bbox)
#
#     _, max_bboxes_, _ = tf.keras.backend.int_shape(pred)
#     for batch_idx in range(batch_):
#
#         gt_count = len(true_bboxes[batch_idx])
#         for bbox_idx in range(max_bboxes_):
#             if pred[batch_idx, bbox_idx, 4] > 0:
#                 image_id = batch_idx
#                 pred_infos[image_id]['detection_id'].append(bbox_idx)
#                 pred_infos[image_id]['confidence'].append(pred[batch_idx, bbox_idx, 4])
#
#                 iou = bbox_iou(pred[batch_idx, bbox_idx:bbox_idx + 1, :4], true_bboxes[batch_idx][:])
#                 iou = np.array(iou)
#                 iou_idx = np.argmax(iou)
#
#                 iou = iou[iou_idx]
#                 pred_infos[image_id]['iou'].append(iou)
#                 if iou >= 0.5:
#                     try:
#                         pred_infos[image_id]['acc_tp'].append(pred_infos[image_id]['acc_tp'][-1] + 1)
#                     except Exception:
#                         pred_infos[image_id]['acc_tp'][-1] += 1
#                 else:
#                     try:
#                         pred_infos[image_id]['acc_fp'].append(pred_infos[image_id]['acc_fp'][-1] + 1)
#                     except Exception:
#                         pred_infos[image_id]['acc_fp'][-1] += 1
#
#     rec_list = []
#     prec_list = []
#     # recall, precision 으로 ap 계산
#     for idx in range(len(pred_infos)):
#         rec = pred_infos['recall'][idx + 1] - pred_infos['recall'][idx]
#         prec = pred_infos['precision'][idx + 1] - pred_infos['precision'][idx]
#         rec_list.append(rec)
#         prec_list.append(prec)
#
#     rec, prec = zip(*sorted(zip(rec_list, prec_list)))
#     # get highest pred from rec range
#     rec_range = np.arange(0, 1, 0.1)
#     # print("rec_range: {}".format(rec_range))
#
#
# def calc_map(pred, target_bboxes, target_infos, num_classes):
#     print("### clac_map inside")
#     print("target_bboxes:{}".format(tf.keras.backend.int_shape(target_bboxes)))
#     print("target_infos:{}".format(tf.keras.backend.int_shape(target_infos)))
#     print("num_classes:{}".format(tf.keras.backend.int_shape(num_classes)))
#     """
#         >> gt_bboxes preprocess된 거에서 conf=1인 것들 숫자 세서 그게 진짜 gt개수와 같은지 확인
#
#         0. 배치 단위로 batch x ~ 함, class별로 나눔
#         1. 배치 단위로 batch x bboxes num x (5+num_classes) (xywh, conf, classprob) gt,pred 가져와 iou계산
#         2. iou 0.5 미만인 경우 일괄 fp처리
#         3. iou 0.5 이상인 경우 highest iou=>tp
#         4. 3 을 gt마다 매칭되는 pred들로 묶음(iou 0.5이상) 가장 confidence높은 애가 tp, 나머지 fp처리. tp,fp 확정되면 process되었음
#         5. 각 pred box마다 recall, precision 구함(해당되는 gt없는 경우는? 모두 fp이므로 tp=0으로 계산)
#         5. recall을 0.1단위로 나누어 0~1 사이 11개 구간에서 해당 구간(0.1) 내에서 가장 높은 precision을 구함
#         6. 위에서 계산한 11개의 max precision들의 평균 구함. 걔가 ap
#         7. 각 클래스 별로 구한 ap들의 평균을 구함
#     """
#     aps = np.zeros(num_classes, dtype=np.float32)
#
#     pred_infos = {"image_id":[], "detection":[],
#                   "confidence":[], "iou":[], "processed":[],
#                   "acc_tp":[], "acc_fp":[]}
#     # pred
#     batch_, bboxes_, ch_ = tf.keras.backend.int_shape(pred)
#
#     # gt
#     _, gt_bboxes, _ = target_bboxes.shape
#
#     true_bboxes = []
#     for batch_idx in range(batch_):
#         gt_count = 0
#         for bbox_idx in range(gt_bboxes):
#             if np.sum(target_bboxes[batch_idx, bbox_idx, :]) > 0:
#                 gt_count +=1
#         true_bbox = target_bboxes[batch_idx,:gt_count,:]
#         true_bboxes.append(true_bbox)
#
#     for batch_idx in range(batch_):
#
#         gt_count = len(true_bboxes[batch_idx])
#         for bbox_idx in range(bboxes_):
#             if pred[batch_idx, bbox_idx, 4] > 0:
#
#                 pred_infos['image_id'].append(batch_idx)
#                 pred_infos['detection'].append(bbox_idx)
#                 pred_infos['confidence'].append(pred[batch_idx, bbox_idx, 4])
#
#                 # print("true_bboxes[{}][{}]: {}".format(batch_idx, bbox_idx, true_bboxes[batch_idx][bbox_idx]))
#                 iou = bbox_iou(pred[batch_idx, bbox_idx, :4], true_bboxes[batch_idx][bbox_idx])
#                 iou = np.array(iou)
#                 print("iou: {}".format(iou))
#
#                 iou_idx = np.argmax(iou)
#
#                 iou = iou[iou_idx]
#                 if iou >= 0.5:
#                     pred_infos['iou'].append(iou)
#                     pred_infos['acc_tp'] += 1
#                 else:
#                     pred_infos['iou'].append(fp)
#                     pred_infos['acc_fp'] += 1
#
#     print("len(pred_infos):{}".format(len(pred_infos)))
#
#     rec_list = []
#     prec_list = []
#     # recall, precision 으로 ap 계산
#     for idx in range(len(pred_infos)):
#         rec = pred_infos['recall'][idx+1] - pred_infos['recall'][idx]
#         prec = pred_infos['precision'][idx+1] - pred_infos['precision'][idx]
#         rec_list.append(red)
#         prec_list.append(pred)
#
#     # get highest pred from rec range
#     rec_range = np.arange(0,1,0.1)
#     print("rec_range: {}".format(rec_range))
#
#     # return np.mean(aps)
#
# def group_by_key(gt, key="image_id"):
#     for idx in range(len(gt)):
#         print("gt[idx]:{}".format(gt[idx]))
#         image_gts = {key:idx, boxes:gt[idx]}
#
# # gt == annotation
# def recall_precision(preds, gt_bboxes):
#     image_gts = group_by_key(gt, "image_id")
#
#     image_gt_boxes = {
#         img_id: np.array([[float(z) for z in b["bbox"]] for b in boxes]) for img_id, boxes in image_gts.items()
#     }
#     image_gt_checked = {img_id: np.zeros(len(boxes)) for img_id, boxes in image_gts.items()}
#
#     predictions = sorted(predictions, key=lambda x: x["score"], reverse=True)
#
#     # go down dets and mark TPs and FPs
#     num_predictions = len(predictions)
#     tp = np.zeros(num_predictions)
#     fp = np.zeros(num_predictions)
#
#     for prediction_index, prediction in enumerate(predictions):
#         box = prediction["bbox"]
#
#         max_overlap = -np.inf
#         jmax = -1
#
#         try:
#             gt_boxes = image_gt_boxes[prediction["image_id"]]  # gt_boxes per image
#             gt_checked = image_gt_checked[prediction["image_id"]]  # gt flags per image
#         except KeyError:
#             gt_boxes = []
#             gt_checked = None
#
#         if len(gt_boxes) > 0:
#             overlaps = get_overlaps(gt_boxes, box)
#
#             max_overlap = np.max(overlaps)
#             jmax = np.argmax(overlaps)
#
#         if max_overlap >= iou_threshold:
#             if gt_checked[jmax] == 0:
#                 tp[prediction_index] = 1.0
#                 gt_checked[jmax] = 1
#             else:
#                 fp[prediction_index] = 1.0
#         else:
#             fp[prediction_index] = 1.0
#
#     # compute precision recall
#     fp = np.cumsum(fp, axis=0)
#     tp = np.cumsum(tp, axis=0)
#
#     recalls = tp / float(num_gts)
#
#     # avoid divide by zero in case the first detection matches a difficult ground truth
#     precisions = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
#
#     ap = get_ap(recalls, precisions)
#
#     return recalls, precisions, ap
#
# def get_envelope(precisions: np.ndarray) -> np.array:
#     for i in range(precisions.size - 1, 0, -1):
#         precisions[i - 1] = np.maximum(precisions[i - 1], precisions[i])
#     return precisions
#
# def get_ap(recalls: np.ndarray, precisions: np.ndarray) -> float:
#     # correct AP calculation
#     # first append sentinel values at the end
#     recalls = np.concatenate(([0.0], recalls, [1.0]))
#     precisions = np.concatenate(([0.0], precisions, [0.0]))
#
#     precisions = get_envelope(precisions)
#
#     # to calculate area under PR curve, look for points where X axis (recall) changes value
#     i = np.where(recalls[1:] != recalls[:-1])[0]
#
#     # and sum (\Delta recall) * prec
#     ap = np.sum((recalls[i + 1] - recalls[i]) * precisions[i + 1])
#     return ap

def draw_bbox_trainw(original_image, image_size, pred_coord , pred_classes, pred_scores, thres_score=0.6):
    hsv_tuples = [(1.0 * x / cfg.YOLO.NUM_CLASSES, 1., 1.) for x in range(cfg.YOLO.NUM_CLASSES)]
    colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
    colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors))

    original_image = np.reshape(original_image, (image_size, image_size, 3))

    for idx in range(len(pred_coord)):
        pred_class = int(pred_classes[idx][0])
        pred_score = pred_scores[idx][0]

        coord = pred_coord[idx]
        x_left = int((float(coord[0])-float(coord[2])/2))
        x_right = int((float(coord[0]) + float(coord[2])/2))
        y_top = int((float(coord[1] - float(coord[3])/2)))
        y_bottom = int((float(coord[1] + float(coord[3]) / 2)))

        color = colors[pred_class]
        cv2.rectangle(original_image, (x_left,y_top),(x_right,y_bottom), thickness=3, color=color)
        cv2.putText(original_image, text="{} score:{:.02f}".format(pred_class, pred_score), org=(x_left+40,y_bottom-40), thickness=2, color=color, fontFace=cv2.FONT_HERSHEY_SIMPLEX ,fontScale=0.7)

    return original_image

def draw_bbox_convertedw(original_image, image_size, pred_coord , pred_classes, pred_scores):
    hsv_tuples = [(1.0 * x / cfg.YOLO.NUM_CLASSES, 1., 1.) for x in range(cfg.YOLO.NUM_CLASSES)]
    colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
    colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors))

    original_image = np.reshape(original_image, (image_size, image_size, 3))

    for idx in range(len(pred_coord)):
        pred_class = int(pred_classes[idx][0])
        pred_score = pred_scores[idx][0]
        coord = pred_coord[idx]

        y_top = int(float(coord[0]) * image_size)
        x_left = int(float(coord[1]) * image_size)
        y_bottom = int(float(coord[2]) * image_size)
        x_right = int(float(coord[3]) * image_size)

        color = colors[pred_class]
        cv2.rectangle(original_image, (x_left, y_top), (x_right, y_bottom), thickness=3, color=color)
        cv2.putText(original_image, text="{} score:{:.02f}".format(pred_class, pred_score),
                    org=(x_left + 30, y_bottom - 30), thickness=2, color=color, fontFace=cv2.FONT_HERSHEY_SIMPLEX,
                    fontScale=0.5)
    return original_image

def draw_bbox(image, bboxes, classes=read_class_names(cfg.YOLO.CLASSES), show_label=True):
    num_classes = len(classes)
    image_h, image_w, _ = image.shape
    hsv_tuples = [(1.0 * x / num_classes, 1., 1.) for x in range(num_classes)]
    colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
    colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors))

    random.seed(0)
    random.shuffle(colors)
    random.seed(None)

    out_boxes, out_scores, out_classes, num_boxes = bboxes

    for i in range(num_boxes[0]):
        if int(out_classes[0][i]) < 0 or int(out_classes[0][i]) > num_classes: continue
        coor = out_boxes[0][i]
        coor[0] = int(coor[0] * image_h)
        coor[2] = int(coor[2] * image_h)
        coor[1] = int(coor[1] * image_w)
        coor[3] = int(coor[3] * image_w)

        fontScale = 0.5
        score = out_scores[0][i]
        class_ind = int(out_classes[0][i])
        bbox_color = colors[class_ind]
        bbox_thick = int(0.6 * (image_h + image_w) / 600)
        c1, c2 = (coor[1], coor[0]), (coor[3], coor[2])
        cv2.rectangle(image, c1, c2, bbox_color, bbox_thick)

        if show_label:
            bbox_mess = '%s: %.2f' % (classes[class_ind], score)
            t_size = cv2.getTextSize(bbox_mess, 0, fontScale, thickness=bbox_thick // 2)[0]
            c3 = (c1[0] + t_size[0], c1[1] - t_size[1] - 3)
            cv2.rectangle(image, c1, (np.float32(c3[0]), np.float32(c3[1])), bbox_color, -1) #filled
            cv2.putText(image, bbox_mess, (c1[0], np.float32(c1[1] - 2)), cv2.FONT_HERSHEY_SIMPLEX,
                        fontScale, (0, 0, 0), bbox_thick // 2, lineType=cv2.LINE_AA)
    return image

def bbox_iou(bboxes1, bboxes2):
    """
    @param bboxes1: (a, b, ..., 4)
    @param bboxes2: (A, B, ..., 4)
        x:X is 1:n or n:n or n:1
    @return (max(a,A), max(b,B), ...)
    ex) (4,):(3,4) -> (3,)
        (2,1,4):(2,3,4) -> (2,3)
    """
    bboxes1_area = bboxes1[..., 2] * bboxes1[..., 3]
    bboxes2_area = bboxes2[..., 2] * bboxes2[..., 3]

    bboxes1_coor = tf.concat(
        [
            bboxes1[..., :2] - bboxes1[..., 2:] * 0.5,
            bboxes1[..., :2] + bboxes1[..., 2:] * 0.5,
        ],
        axis=-1,
    )
    bboxes2_coor = tf.concat(
        [
            bboxes2[..., :2] - bboxes2[..., 2:] * 0.5,
            bboxes2[..., :2] + bboxes2[..., 2:] * 0.5,
        ],
        axis=-1,
    )

    left_up = tf.maximum(bboxes1_coor[..., :2], bboxes2_coor[..., :2])
    right_down = tf.minimum(bboxes1_coor[..., 2:], bboxes2_coor[..., 2:])

    inter_section = tf.maximum(right_down - left_up, 0.0)
    inter_area = inter_section[..., 0] * inter_section[..., 1]

    union_area = bboxes1_area + bboxes2_area - inter_area

    iou = tf.math.divide_no_nan(inter_area, union_area)

    return iou

def bbox_giou(bboxes1, bboxes2):
    """
    Generalized IoU
    @param bboxes1: (a, b, ..., 4)
    @param bboxes2: (A, B, ..., 4)
        x:X is 1:n or n:n or n:1
    @return (max(a,A), max(b,B), ...)
    ex) (4,):(3,4) -> (3,)
        (2,1,4):(2,3,4) -> (2,3)
    """
    bboxes1_area = bboxes1[..., 2] * bboxes1[..., 3]
    bboxes2_area = bboxes2[..., 2] * bboxes2[..., 3]

    bboxes1_coor = tf.concat(
        [
            bboxes1[..., :2] - bboxes1[..., 2:] * 0.5,
            bboxes1[..., :2] + bboxes1[..., 2:] * 0.5,
        ],
        axis=-1,
    )
    bboxes2_coor = tf.concat(
        [
            bboxes2[..., :2] - bboxes2[..., 2:] * 0.5,
            bboxes2[..., :2] + bboxes2[..., 2:] * 0.5,
        ],
        axis=-1,
    )

    left_up = tf.maximum(bboxes1_coor[..., :2], bboxes2_coor[..., :2])
    right_down = tf.minimum(bboxes1_coor[..., 2:], bboxes2_coor[..., 2:])

    inter_section = tf.maximum(right_down - left_up, 0.0)
    inter_area = inter_section[..., 0] * inter_section[..., 1]

    union_area = bboxes1_area + bboxes2_area - inter_area

    iou = tf.math.divide_no_nan(inter_area, union_area)

    enclose_left_up = tf.minimum(bboxes1_coor[..., :2], bboxes2_coor[..., :2])
    enclose_right_down = tf.maximum(
        bboxes1_coor[..., 2:], bboxes2_coor[..., 2:]
    )

    enclose_section = enclose_right_down - enclose_left_up
    enclose_area = enclose_section[..., 0] * enclose_section[..., 1]

    giou = iou - tf.math.divide_no_nan(enclose_area - union_area, enclose_area)

    return giou

def bbox_ciou(bboxes1, bboxes2):
    """
    Complete IoU
    @param bboxes1: (a, b, ..., 4)
    @param bboxes2: (A, B, ..., 4)
        x:X is 1:n or n:n or n:1
    @return (max(a,A), max(b,B), ...)
    ex) (4,):(3,4) -> (3,)
        (2,1,4):(2,3,4) -> (2,3)
    """
    bboxes1_area = bboxes1[..., 2] * bboxes1[..., 3]
    bboxes2_area = bboxes2[..., 2] * bboxes2[..., 3]

    bboxes1_coor = tf.concat(
        [
            bboxes1[..., :2] - bboxes1[..., 2:] * 0.5,
            bboxes1[..., :2] + bboxes1[..., 2:] * 0.5,
        ],
        axis=-1,
    )
    bboxes2_coor = tf.concat(
        [
            bboxes2[..., :2] - bboxes2[..., 2:] * 0.5,
            bboxes2[..., :2] + bboxes2[..., 2:] * 0.5,
        ],
        axis=-1,
    )

    left_up = tf.maximum(bboxes1_coor[..., :2], bboxes2_coor[..., :2])
    right_down = tf.minimum(bboxes1_coor[..., 2:], bboxes2_coor[..., 2:])

    inter_section = tf.maximum(right_down - left_up, 0.0)
    inter_area = inter_section[..., 0] * inter_section[..., 1]

    union_area = bboxes1_area + bboxes2_area - inter_area

    iou = tf.math.divide_no_nan(inter_area, union_area)

    enclose_left_up = tf.minimum(bboxes1_coor[..., :2], bboxes2_coor[..., :2])
    enclose_right_down = tf.maximum(
        bboxes1_coor[..., 2:], bboxes2_coor[..., 2:]
    )

    enclose_section = enclose_right_down - enclose_left_up

    c_2 = enclose_section[..., 0] ** 2 + enclose_section[..., 1] ** 2

    center_diagonal = bboxes2[..., :2] - bboxes1[..., :2]

    rho_2 = center_diagonal[..., 0] ** 2 + center_diagonal[..., 1] ** 2

    diou = iou - tf.math.divide_no_nan(rho_2, c_2)

    v = (
        (
            tf.math.atan(
                tf.math.divide_no_nan(bboxes1[..., 2], bboxes1[..., 3])
            )
            - tf.math.atan(
                tf.math.divide_no_nan(bboxes2[..., 2], bboxes2[..., 3])
            )
        )
        * 2
        / np.pi
    ) ** 2

    alpha = tf.math.divide_no_nan(v, 1 - iou + v)

    ciou = diou - alpha * v

    return ciou

def bbox_confidence(cls_ind, cls_bboxes, iou):
    first_bbox_classprob_ = pred_classprob[...,0, :]
    second_bbox_classprob_ = pred_classprob[...,1, :]
    third_bbox_classprob_ = pred_classprob[...,2, :]

    first_bbox_class_ = tf.expand_dims(tf.argmax(first_bbox_classprob_, axis=-1), axis=-1)
    second_bbox_class_ = tf.expand_dims(tf.argmax(second_bbox_classprob_, axis=-1), axis=-1)
    third_bbox_class_ = tf.expand_dims(tf.argmax(third_bbox_classprob_, axis=-1), axis=-1)

    pred_class = tf.cast(tf.concat([first_bbox_class_, second_bbox_class_, third_bbox_class_], axis=-1), dtype=tf.float32)

    if istrain == True:
        iou_score = bbox_iou([true_xy, true_wh, true_classprob], [pred_xy, pred_wh, pred_classprob])
        return tf.multiply(pred_objness, pred_class) * iou_score, iou_score
    return tf.multiply(pred_objness, pred_class)

def yolo_eval(yolo_outputs,
              anchors,
              num_classes,
              image_shape,
              max_boxes=20,
              score_threshold=.6,
              iou_threshold=.5):
    """Evaluate YOLO model on given input and return filtered boxes."""
    num_layers = len(yolo_outputs)
    anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [1,2,3]] # default setting
    input_shape = K.shape(yolo_outputs[0])[1:3] * 32
    boxes = []
    box_scores = []
    for l in range(num_layers):
        _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
            anchors[anchor_mask[l]], num_classes, input_shape, image_shape)
        boxes.append(_boxes)
        box_scores.append(_box_scores)
    boxes = K.concatenate(boxes, axis=0)
    box_scores = K.concatenate(box_scores, axis=0)

    mask = box_scores >= score_threshold
    max_boxes_tensor = K.constant(max_boxes, dtype='int32')
    boxes_ = []
    scores_ = []
    classes_ = []
    for c in range(num_classes):
        # TODO: use keras backend instead of tf.
        class_boxes = tf.boolean_mask(boxes, mask[:, c])
        class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
        nms_index = tf.image.non_max_suppression(
            class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold)
        class_boxes = K.gather(class_boxes, nms_index)
        class_box_scores = K.gather(class_box_scores, nms_index)
        classes = K.ones_like(class_box_scores, 'int32') * c
        boxes_.append(class_boxes)
        scores_.append(class_box_scores)
        classes_.append(classes)
    boxes_ = K.concatenate(boxes_, axis=0)
    scores_ = K.concatenate(scores_, axis=0)
    classes_ = K.concatenate(classes_, axis=0)

    return boxes_, scores_, classes_

def yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape):
    '''Get corrected boxes'''
    box_yx = box_xy[..., ::-1]
    box_hw = box_wh[..., ::-1]
    input_shape = K.cast(input_shape, K.dtype(box_yx))
    image_shape = K.cast(image_shape, K.dtype(box_yx))
    new_shape = K.round(image_shape * K.min(input_shape/image_shape))
    offset = (input_shape-new_shape)/2./input_shape
    scale = input_shape/new_shape
    box_yx = (box_yx - offset) * scale
    box_hw *= scale

    box_mins = box_yx - (box_hw / 2.)
    box_maxes = box_yx + (box_hw / 2.)
    boxes =  K.concatenate([
        box_mins[..., 0:1],  # y_min
        box_mins[..., 1:2],  # x_min
        box_maxes[..., 0:1],  # y_max
        box_maxes[..., 1:2]  # x_max
    ])

    # Scale boxes back to original image shape.
    boxes *= K.concatenate([image_shape, image_shape])
    return boxes

def yolo_boxes_and_scores(box_xy, box_wh, box_confidence, box_class_probs,input_shape, image_shape,num_classes):
    '''Process Conv layer output'''
    boxes = yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape)
    boxes = K.reshape(boxes, [-1, 4])
    box_scores = box_confidence * box_class_probs
    box_scores = K.reshape(box_scores, [-1, num_classes])
    return boxes, box_scores

def yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape):
    '''Get corrected boxes'''
    box_yx = box_xy[..., ::-1]
    box_hw = box_wh[..., ::-1]
    input_shape = K.cast(input_shape, K.dtype(box_yx))
    image_shape = K.cast(image_shape, K.dtype(box_yx))
    new_shape = K.round(image_shape * K.min(input_shape/image_shape))
    # print("input_shape:{} image_shape:{} new_shape:{}".format(tf.keras.backend.int_shape(input_shape),
    #                                                           tf.keras.backend.int_shape(image_shape),
    #                                                           tf.keras.backend.int_shape(new_shape)))
    offset = (input_shape-new_shape)/2./input_shape
    scale = input_shape/new_shape
    box_yx = (box_yx - offset) * scale
    box_hw *= scale

    box_mins = box_yx - (box_hw / 2.)
    box_maxes = box_yx + (box_hw / 2.)
    boxes =  K.concatenate([
        box_mins[..., 0:1],  # y_min
        box_mins[..., 1:2],  # x_min
        box_maxes[..., 0:1],  # y_max
        box_maxes[..., 1:2]  # x_max
    ])

    # Scale boxes back to original image shape.
    boxes *= K.concatenate([image_shape, image_shape])
    return boxes

def freeze_all(model, frozen=True):
    model.trainable = not frozen
    if isinstance(model, tf.keras.Model):
        for l in model.layers:
            freeze_all(l, frozen)

def unfreeze_all(model, frozen=False):
    model.trainable = not frozen
    if isinstance(model, tf.keras.Model):
        for l in model.layers:
            unfreeze_all(l, frozen)


text:
import sys
import json
import numpy as np
from tqdm import tqdm

def calc_f1(precision, recall):
    return 2.0 * precision * recall / (precision + recall)

def method1_calc_statistic(scores):
    precisions = [
        precision_good / float(precision_good + precision_bad)
        for [precision_good, precision_bad, _, _] in scores
    ]
    recalls = [
        recall_good / float(recall_good + recall_bad)
        for [_, _, recall_good, recall_bad] in scores
    ]
    f1s = [calc_f1(precision, recall) for precision, recall in zip(precisions, recalls)]
    score1, score2 = f1s
    statistic = f1s[0] - f1s[1]
    return score1, score2, statistic

# Input format: python significance_test.py predictions1.json predictions2.json sample_size
# This tests the null hypothesis that module_accuracy(predictions1) == module_accuracy(predictions2) against
# the alternative hypothesis that module_accuracy(predictions1) > module_accuracy(predictions2)
# sample_size is an integer specifying the number of permutations that will be sampled to conduct this test

f1 = open(sys.argv[1])
f2 = open(sys.argv[2])
sample_size = int(sys.argv[3])
avg_method = int(sys.argv[4])
lines1 = f1.readlines()
lines2 = f2.readlines()
predictions1 = [json.loads(line) for line in lines1]
predictions2 = [json.loads(line) for line in lines2]
assert len(predictions1) == len(predictions2)
predictions = [predictions1, predictions2]

if avg_method == 1:
    # For each model, we store all of the precision good/bad counts and the recall good/bad counts
    scores = [[0, 0, 0, 0] for _ in range(2)]
else:
    scores = [[], []]
# i indexes which model (there are 2 models we are comparing)
for i in range(2):
    # Iterating through each instance
    for j, pred in enumerate(predictions[i]):
        if avg_method == 1:
            precision_good, precision_bad, recall_good, recall_bad = pred[
                "box_f1_overall_score"
            ]
            scores[i][0] += precision_good
            scores[i][1] += precision_bad
            scores[i][2] += recall_good
            scores[i][3] += recall_bad
        elif avg_method == 2:
            scores[i].append(pred["box_f1_overall_score"])
        elif avg_method == 3:
            # Iterating through each module for which there is at least 1 gold box
            # pair = (a,b), where a is the index of a module in the program, and
            # b indexes image 0 or image 1
            # Previously used for mAP:
            for pair in pred["box_f1"]["non_empty_modules"]:
                scores[i].append(pred["box_f1"][str(pair[0])][pair[1]])
        assert predictions[i][j]["image_id"] == predictions[1 - i][j]["image_id"]
        assert predictions[i][j]["utterance"] == predictions[1 - i][j]["utterance"]
if avg_method == 1:
    score1, score2, original_statistic = method1_calc_statistic(scores)
    print("Score 1: " + str(score1))
    print("Score 2: " + str(score2))
elif avg_method in {2, 3}:
    assert len(scores[0]) == len(scores[1])
    original_statistic = np.mean(scores[0]) - np.mean(scores[1])
    print("Score 1: " + str(np.mean(scores[0])))
    print("Score 2: " + str(np.mean(scores[1])))
print("Original statistic: " + str(original_statistic))

num_exceeding = 0
# Sampling permutations, a permutation consists of a 0/1 for each module in the scores
# list
num_examples = len(scores[0])
if avg_method == 1:
    num_examples = len(predictions[0])
signs = np.random.binomial(1, 0.5, size=(sample_size, num_examples))
for trial in tqdm(range(signs.shape[0])):
    if avg_method == 1:
        scores = [[0, 0, 0, 0] for _ in range(2)]
    else:
        scores = [[], []]
    for i in range(2):
        # count indexes which of the 0/1 values in the permutation should be used for a given
        # module
        count = 0
        # Previously used for mAP:
        for j, pred in enumerate(predictions[i]):
            if avg_method == 1:
                if signs[trial][count] >= 0.5:
                    for k in range(len(pred["box_f1_overall_score"])):
                        scores[i][k] += pred["box_f1_overall_score"][k]
                else:
                    for k in range(len(pred["box_f1_overall_score"])):
                        scores[1 - i][k] += pred["box_f1_overall_score"][k]
                count += 1
            elif avg_method == 2:
                if signs[trial][count] >= 0.5:
                    scores[i].append(pred["box_f1_overall_score"])
                else:
                    scores[1 - i].append(pred["box_f1_overall_score"])
                count += 1
            elif avg_method == 3:
                for pair in pred["box_f1"]["non_empty_modules"]:
                    if signs[trial][count] >= 0.5:
                        scores[i].append(pred["box_f1"][str(pair[0])][pair[1]])
                    else:
                        scores[1 - i].append(pred["box_f1"][str(pair[0])][pair[1]])
                    count += 1
    if avg_method == 1:
        _, _, statistic = method1_calc_statistic(scores)
    elif avg_method in {2, 3}:
        assert len(scores[0]) == len(scores[1])
        statistic = np.mean(scores[0]) - np.mean(scores[1])
    if abs(statistic) >= original_statistic:
        num_exceeding += 1

print("p-value: " + str((1.0 + float(num_exceeding)) / (1.0 + signs.shape[0])))

ext: py

max_stars_repo_name: allenai/faithful-nmn

max_issues_repo_name: arjunakula/faithful-nmn

max_forks_repo_name: arjunakula/faithful-nmn

reason: import numpy


text:
from __future__ import print_function, division

import numpy as np
from astropy.table import Table

CART_COL_NAMES = ['X', 'Y', 'Z', 'U', 'V', 'W',
                  'dX', 'dY', 'dZ', 'dU', 'dV', 'dW',
                  'c_XY', 'c_XZ', 'c_XU', 'c_XV', 'c_XW',
                  'c_YZ', 'c_YU', 'c_YV', 'c_YW',
                  'c_ZU', 'c_ZV', 'c_ZW',
                  'c_UV', 'c_UW',
                  'c_VW']

def buildMeanAndCovMatFromRow(row):
    """
    Build a covariance matrix from a row

    Paramters
    ---------
    row : astropy Table row
        Entries: {X, Y, Z, U, V, W, dX, dY, ..., cXY, cXZ, ...}

    Return
    ------
    cov_mat : [6,6] numpy array
        Diagonal elements are dX^2, dY^2, ...
        Off-diagonal elements are cXY*dX*dY, cXZ*dX*dZ, ...
    """
    dim = 6
    # CART_COL_NAMES = ['X', 'Y', 'Z', 'U', 'V', 'W',
    #                   'dX', 'dY',   'dZ',   'dU',   'dV',   'dW',
    #                         'c_XY', 'c_XZ', 'c_XU', 'c_XV', 'c_XW',
    #                                 'c_YZ', 'c_YU', 'c_YV', 'c_YW',
    #                                         'c_ZU', 'c_ZV', 'c_ZW',
    #                                                 'c_UV', 'c_UW',
    #                                                         'c_VW']
    mean = np.zeros(dim)
    for i, col_name in enumerate(CART_COL_NAMES[:6]):
        mean[i] = row[col_name]

    std_vec = np.zeros(dim)
    for i, col_name in enumerate(CART_COL_NAMES[6:12]):
        std_vec[i] = row[col_name]
    corr_tri = np.zeros((dim,dim))
    # Insert upper triangle (top right) correlations
    for i, col_name in enumerate(CART_COL_NAMES[12:]):
        corr_tri[np.triu_indices(dim,1)[0][i],np.triu_indices(dim,1)[1][i]]\
            =row[col_name]
    # Build correlation matrix
    corr_mat = np.eye(6) + corr_tri + corr_tri.T
    # Multiply through by standard deviations
    cov_mat = corr_mat * std_vec * std_vec.reshape(6,1)
    return mean, cov_mat

def insertCartesainData(gt_row, xyzuvw_mean, xyzuvw_cov):
    dim = 6
    # CART_COL_NAMES = ['X', 'Y', 'Z', 'U', 'V', 'W',
    #                   'dX', 'dY', 'dZ', 'dU', 'dV', 'dW',
    #                   'c_XY', 'c_XZ', 'c_XU', 'c_XV', 'c_XW',
    #                           'c_YZ', 'c_YU', 'c_YV', 'c_YW',
    #                                   'c_ZU', 'c_ZV', 'c_ZW',
    #                                           'c_UV', 'c_UW',
    #                                                   'c_VW']

    # fill in cartesian mean
    try:
        for col_ix, col_name in enumerate(CART_COL_NAMES[:6]):
            gt_row[col_name] = xyzuvw_mean[col_ix]
    except IndexError:
        import pdb; pdb.set_trace()

    # fill in standard deviations
    xyzuvw_stds = np.sqrt(xyzuvw_cov[np.diag_indices(dim)])
    for col_ix, col_name in enumerate(CART_COL_NAMES[6:12]):
        gt_row[col_name] = xyzuvw_stds[col_ix]

    correl_matrix = xyzuvw_cov / xyzuvw_stds / xyzuvw_stds.reshape(6, 1)
    # fill in correlations
    for col_ix, col_name in enumerate(CART_COL_NAMES[12:]):
        gt_row[col_name] = correl_matrix[
            np.triu_indices(dim, k=1)[0][col_ix],
            np.triu_indices(dim, k=1)[1][col_ix]
        ]

def eraseCartesianData(gt_row):
    """
    Replace all cartesian data with blanks (for erasing binary companions that
    have been incoroporated into host star's kinematics)
    """
    dim = 6

    for col_name in CART_COL_NAMES:
        gt_row[col_name] = np.nan

if __name__ == '__main__':
    gagne_filename = '../data/gagne_bonafide_full_kinematics_with_lit' \
                     '_and_best_radial_velocity.fits'
    gt = Table.read(gagne_filename)
    new_gagne_filename = '../data/gagne_bonafide_full_kinematics_with_lit' \
                     '_and_best_radial_velocity_comb_binars.fits'

    # companion_ixs = np.where(gt['Companion'] == 'True')
    # host_ixs = (companion_ixs[0]-1,)

    row_ix = 0
    while row_ix < len(gt['Companion']) - 1:
        # find next instance of host
        system_ixs = []
        while (row_ix+1 < len(gt['Companion']) and
               gt[row_ix+1]['Companion'] == 'False'):
            row_ix += 1
        # Check if we've reached end of table
        if row_ix == len(gt['Companion'])-1:
            break
        system_ixs.append(row_ix)

        # find subsequent associated companions
        row_ix += 1
        while row_ix < len(gt['Companion']) and gt[row_ix]['Companion'] == 'True':
            system_ixs.append(row_ix)
            row_ix += 1

        print(system_ixs)

        masses = []
        means = []
        cov_mats = []
        for row_ix in system_ixs:
            # Check 'U' (for e.g.) has been calculated
            # (requires both RV and parallax so good guide for 6D completeness)
            if np.isfinite(gt[row_ix]['U']):
                masses.append(gt[row_ix]['approx_mass'])
                mean, cov_mat = buildMeanAndCovMatFromRow(gt[row_ix])
                means.append(mean)
                cov_mats.append(cov_mat)

        masses = np.array(masses)
        means = np.array(means)
        cov_mats = np.array(cov_mats)

        nstars = len(means)
        if nstars != 0:
            # establish average of available masses
            av_present_mass = masses[np.where(np.isfinite(masses))].mean()
            # replace everything with 1 if there are no valid masses
            if np.isnan(av_present_mass):
                masses[:] = 1.0
            # replace any missing masses with average of available masses
            else:
                masses[np.where(np.isnan(masses))] = av_present_mass

            assert np.all(np.isfinite(masses))
            total_mass = np.sum(masses)

            aver_mean = (means.T * masses).sum(axis=-1) / total_mass
            aver_cov_mat = (cov_mats.T * masses).sum(axis=-1) / total_mass

            insertCartesainData(gt[system_ixs[0]], aver_mean, aver_cov_mat)

            for system_ix in system_ixs[1:]:
                eraseCartesianData(gt[system_ix])

    for pos_dim in CART_COL_NAMES[:3]:
        gt[pos_dim].unit = 'pc'
    for vel_dim in CART_COL_NAMES[3:6]:
        gt[vel_dim].unit = 'km/s'
    for pos_std in CART_COL_NAMES[6:9]:
        gt[pos_std].unit = 'pc'
    for vel_std in CART_COL_NAMES[9:12]:
        gt[vel_std].unit = 'km/s'
    gt['approx_mass'].unit = 'M_solar'

    gt.write(new_gagne_filename)

max_stars_repo_name: tcrundall/chronostar

max_issues_repo_name: tcrundall/chronostar

max_forks_repo_name: tcrundall/chronostar

reason: import numpy,from astropy


text:
import numpy as np
import plotly.colors as pc

PALETTE = [
    "#66C2A5", "#FC8D62", "#8DA0CB", "#E78AC3", "#A6D854", "#FFD92F",
    "#B3B3B3", "#E5C494", "#9C9DBB", "#E6946B", "#DA8DC4", "#AFCC63",
    "#F2D834", "#E8C785", "#BAB5AE", "#D19C75", "#AC9AAD", "#CD90C5",
    "#B8C173", "#E5D839", "#ECCA77", "#C1B7AA", "#BBA37E", "#BC979D",
    "#C093C6", "#C1B683", "#D8D83E", "#F0CD68", "#C8BAA5", "#A6AB88",
    "#CC958F", "#B396C7", "#CBAB93", "#CCD844", "#F3D05A", "#CFBCA1",
    "#90B291", "#DC9280", "#A699C8", "#D4A0A3", "#BFD849", "#F7D34B",
    "#D6BF9C", "#7BBA9B", "#EC8F71", "#999CC9", "#DD95B3", "#B2D84E",
    "#FBD63D", "#DDC198"
]

def get_col_i(pal, i):
    return pal[i % len(pal)]

def palette_to_rgb(
        palette=None, maxval=None, black="#000000", white="#FFFFFF"):
    # Colors when labels are provided
    if palette is None:
        palette = PALETTE
    if maxval is not None:
        while len(palette) - 1 < maxval:
            palette += palette
    # add 0 for black and -1 for white
    palette = [black] + palette + [white]
    palr = np.array([int(i[1:3], 16) for i in palette])  # hex string to int
    palb = np.array([int(i[3:5], 16) for i in palette])
    palg = np.array([int(i[5:], 16) for i in palette])
    return palr, palb, palg

def interpolate_grayimage(im, colorscale='magma'):
    """
    Maps a 2D image to a 3D image using the given colormap.
    We use this manually since plotly is very slow when rendering
    large 2D images using imshow as in the case of CODEX data.
    """
    assert im.ndim == 2
    colorscale = pc.get_colorscale(colorscale)
    colors = pc.colorscale_to_colors(colorscale)
    colors = pc.validate_colors(colors, colortype="tuple")

    # First we bin each value in im so that we know which colors
    # to use for interpolation
    unit = (im - im.min()) / (im.max() - im.min())
    binned = (unit * (len(colors) - 1)) - 0.5
    binned = np.round(binned).astype(int).clip(min=0)
    # empty 3d im
    im3d = np.zeros((*im.shape, 3))
    for i in range(len(colors) - 1):  # Small for loop so it's ok
        r1, g1, b1 = colors[i]
        r2, g2, b2 = colors[i + 1]
        idx = np.where(binned == i)
        if idx[0].size == 0:
            continue
        minv, maxv = im[idx].min(), im[idx].max()
        if minv == maxv:
            intermeds = 0.5
        else:
            intermeds = (im[idx] - minv) / (maxv - minv)
        im3d[idx] = np.stack([
            (r2 - r1) * intermeds + r1,
            (g2 - g1) * intermeds + g1,
            (b2 - b1) * intermeds + b1
        ], axis=-1)
    return im3d

max_stars_repo_name: euxhenh/cellar

max_issues_repo_name: euxhenh/cellar

max_forks_repo_name: euxhenh/cellar


text:
import PyQuantum.DarkState.operators as op
from PyQuantum.DarkState.config import *
from PyQuantum.Common.Matrix import *
from scipy import sparse
import copy
from scipy.linalg.interpolative import *
import math
import sys
from scipy.sparse import vstack, hstack
from scipy.sparse.linalg import *

# ---------------------------

class bits:
    def __init__(self, size, n_levels):
        self.n = size
        self.n_levels = n_levels
        self.data = [0] * self.n

    def inc(self):
        inced = False

        for i in range(self.n-1, -1, -1):
            if self.data[i] < self.n_levels-1:
                self.data[i] += 1
                inced = True
                break
            else:
                self.data[i] = 0

        return inced

    def print(self):
        print(self.data)

def to_n(n, n_levels):
    if n == 0:
        return [[0]]

    bitsize = math.ceil(math.log(n+1)/math.log(n_levels))

    d = bits(size=bitsize, n_levels=n_levels)

    arr = []

    cnt = 1

    while True:
        arr.append(copy.copy(d.data))

        if cnt > n or not d.inc():
            break

        cnt += 1

    return arr

def get_system1(n_levels, n_atoms):
    Sigma_ij = None

    DIM = math.pow(n_levels, n_atoms)
    DIM = int(DIM)

    for i in range(0, n_levels):
        for j in range(0, i):
            s_ij = None

            for i_atom in range(1, n_atoms+1):
                # -----------------------------------------------------------------------------------------
                s = op.Sigma2(i_=i, j_=j, num=i_atom,
                              n_atoms=n_atoms, n_levels=n_levels)

                if s_ij is None:
                    s_ij = s
                else:
                    s_ij += s
                # -----------------------------------------------------------------------------------------

            Sigma_ij = s_ij

            m_, n_ = np.shape(Sigma_ij)

            print('m =', m_, ', n =', n_)
            mat = Sigma_ij.todense()

            rg = np.linalg.matrix_rank(mat)

            print('rg = ', rg, ', n = ', n_, ', dim = ', n_ - rg, sep='')
            # for i in mat:
            # print(i)
            return n_ - rg
    return 1

def get_system2(n_levels, n_atoms):
    Sigma_ij = None
    Sigma_ij_t = None

    DIM = math.pow(n_levels, n_atoms)
    DIM = int(DIM)

    # ab = to_n(DIM-1, n_levels)

    for i in range(0, n_levels):
        for j in range(0, i):
            # print('σ(', i, ',', j, '):', sep='')

            s_ij = None
            s_ij_t = None

            for i_atom in range(1, n_atoms+1):
                # -----------------------------------------------------------------------------------------
                s = op.Sigma2(i_=i, j_=j, num=i_atom,
                              n_atoms=n_atoms, n_levels=n_levels)
                # print(np.shape(s))
                # st = np.transform(s)
                # st = op.Sigma2T(i_=i, j_=j, num=i_atom,
                # n_atoms=n_atoms, n_levels=n_levels)

                if s_ij is None:
                    s_ij = s
                    # s_ij_t = st
                else:
                    s_ij += s

                    # s_ij_t += st
                # -----------------------------------------------------------------------------------------

            # if Sigma_ij is None:
            Sigma_ij = s_ij
            # Sigma_ij_t = s_ij_t
            # else:
            # Sigma_ij = vstack((Sigma_ij, s_ij))
            # Sigma_ij_t = vstack((Sigma_ij_t, s_ij_t))

            # Sigma_ij = vstack((Sigma_ij, Sigma_ij_t))

            m_, n_ = np.shape(Sigma_ij)
            print('m =', m_, ', n =', n_)
            # print('size:', sys.getsizeof(min(m_, n_)-1)
            # u, s, vt = svds(Sigma_ij, k=min(m_, n_-1))

            mat = Sigma_ij.todense()
            # print(mat)
            # for i in mat:
            # print(','.join([str(elem) for elem in i]))
            # print()

            rg = np.linalg.matrix_rank(mat)
            # print('rank_dense=', rg_ok)

            print('rg = ', rg, ', n = ', n_, ', dim = ', n_ - rg, sep='')

    return 1
    # return n_ - rg

max_stars_repo_name: alexfmsu/pyquantum

max_issues_repo_name: alexfmsu/pyquantum

max_forks_repo_name: alexfmsu/pyquantum

reason: from scipy


text:
# coding: utf8

import numpy as np

from clinica.pipelines.machine_learning import algorithm, base, input, validation

class VoxelBasedKFoldDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        output_dir,
        fwhm=0,
        modulated="on",
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        precomputed_kernel=None,
        mask_zeros=True,
        n_threads=15,
        n_folds=10,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
        splits_indices=None,
    ):

        super(VoxelBasedKFoldDualSVM, self).__init__(
            input.CAPSVoxelBasedInput,
            validation.KFoldCV,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

class VoxelBasedRepKFoldDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        output_dir,
        fwhm=0,
        modulated="on",
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        precomputed_kernel=None,
        mask_zeros=True,
        n_threads=15,
        n_iterations=100,
        n_folds=10,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
        splits_indices=None,
    ):

        super(VoxelBasedRepKFoldDualSVM, self).__init__(
            input.CAPSVoxelBasedInput,
            validation.RepeatedKFoldCV,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

class VoxelBasedRepHoldOutDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        output_dir,
        fwhm=0,
        modulated="on",
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        precomputed_kernel=None,
        mask_zeros=True,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
        splits_indices=None,
    ):

        super().__init__(
            input.CAPSVoxelBasedInput,
            validation.RepeatedHoldOut,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

class VertexBasedRepHoldOutDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        output_dir,
        image_type="PET",
        acq_label=None,
        suvr_reference_region=None,
        fwhm=20,
        precomputed_kernel=None,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-10, 2, 1000),
        splits_indices=None,
    ):

        super(VertexBasedRepHoldOutDualSVM, self).__init__(
            input.CAPSVertexBasedInput,
            validation.RepeatedHoldOut,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

class RegionBasedRepHoldOutDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        atlas,
        output_dir,
        use_pvc_data=False,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
        splits_indices=None,
    ):

        super(RegionBasedRepHoldOutDualSVM, self).__init__(
            input.CAPSRegionBasedInput,
            validation.RepeatedHoldOut,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

class RegionBasedRepHoldOutLogisticRegression(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        atlas,
        output_dir,
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
        splits_indices=None,
    ):

        super(RegionBasedRepHoldOutLogisticRegression, self).__init__(
            input.CAPSRegionBasedInput,
            validation.RepeatedHoldOut,
            algorithm.LogisticReg,
            locals(),
            output_dir,
        )

class RegionBasedRepHoldOutRandomForest(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        atlas,
        output_dir,
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        grid_search_folds=10,
        balanced=True,
        n_estimators_range=(100, 200, 400),
        max_depth_range=[None],
        min_samples_split_range=[2],
        max_features_range=("auto", 0.25, 0.5),
        splits_indices=None,
    ):

        super(RegionBasedRepHoldOutRandomForest, self).__init__(
            input.CAPSRegionBasedInput,
            validation.RepeatedHoldOut,
            algorithm.RandomForest,
            locals(),
            output_dir,
        )

class RegionBasedLearningCurveRepHoldOutDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        atlas,
        output_dir,
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        precomputed_kernel=None,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        n_learning_points=10,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
    ):

        super(RegionBasedLearningCurveRepHoldOutDualSVM, self).__init__(
            input.CAPSRegionBasedInput,
            validation.LearningCurveRepeatedHoldOut,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

class VoxelBasedLearningCurveRepHoldOutDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        output_dir,
        fwhm=0,
        modulated="on",
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        precomputed_kernel=None,
        mask_zeros=True,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        n_learning_points=10,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
    ):

        super(VoxelBasedLearningCurveRepHoldOutDualSVM, self).__init__(
            input.CAPSVoxelBasedInput,
            validation.LearningCurveRepeatedHoldOut,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

class RegionBasedRepKFoldDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        atlas,
        output_dir,
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        n_folds=10,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
        splits_indices=None,
    ):

        super(RegionBasedRepKFoldDualSVM, self).__init__(
            input.CAPSRegionBasedInput,
            validation.RepeatedKFoldCV,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

class CAPSTsvRepHoldOutDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        atlas,
        dataset,
        output_dir,
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
        splits_indices=None,
    ):

        super(CAPSTsvRepHoldOutDualSVM, self).__init__(
            input.CAPSTSVBasedInput,
            validation.RepeatedHoldOut,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

class CAPSTsvRepHoldOutRandomForest(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        atlas,
        dataset,
        output_dir,
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        n_threads=15,
        n_iterations=100,
        test_size=0.3,
        grid_search_folds=10,
        balanced=True,
        n_estimators_range=(100, 200, 400),
        max_depth_range=[None],
        min_samples_split_range=[2],
        max_features_range=("auto", 0.25, 0.5),
        splits_indices=None,
    ):

        super(CAPSTsvRepHoldOutRandomForest, self).__init__(
            input.CAPSTSVBasedInput,
            validation.RepeatedHoldOut,
            algorithm.RandomForest,
            locals(),
            output_dir,
        )

# SVM reg

class VoxelBasedREGRepKFoldDualSVM(base.MLWorkflow):
    def __init__(
        self,
        caps_directory,
        subjects_visits_tsv,
        diagnoses_tsv,
        group_label,
        image_type,
        output_dir,
        fwhm=0,
        modulated="on",
        acq_label=None,
        suvr_reference_region=None,
        use_pvc_data=False,
        precomputed_kernel=None,
        mask_zeros=True,
        n_threads=15,
        n_iterations=100,
        n_folds=10,
        test_size=0.1,
        grid_search_folds=10,
        balanced=True,
        c_range=np.logspace(-6, 2, 17),
        splits_indices=None,
    ):

        super(VoxelBasedREGRepKFoldDualSVM, self).__init__(
            input.CAPSTSVBasedInput,
            validation.RepeatedKFoldCV,
            algorithm.DualSVMAlgorithm,
            locals(),
            output_dir,
        )

# TSV

class TsvRepHoldOutRandomForest(base.MLWorkflow):
    def __init__(
        self,
        data_tsv,
        columns,
        output_dir,
        n_threads=20,
        n_iterations=250,
        test_size=0.2,
        grid_search_folds=10,
        balanced=True,
        n_estimators_range=(100, 200, 400),
        max_depth_range=[None],
        min_samples_split_range=[2],
        max_features_range=("auto", 0.25, 0.5),
        splits_indices=None,
        inner_cv=False,
    ):

        super(TsvRepHoldOutRandomForest, self).__init__(
            input.TsvInput,
            validation.RepeatedHoldOut,
            algorithm.RandomForest,
            locals(),
            output_dir,
        )


text:
import pandas

from IMLearn.utils import split_train_test
from IMLearn.learners.regressors import LinearRegression

from typing import NoReturn
import numpy as np
import pandas as pd
import plotly.graph_objects as go
import plotly.express as px
import plotly.io as pio
import datetime as dt

# from sklearn.metrics import r2_score

pio.templates.default = "simple_white"

def load_data(filename: str):
    """
    Load house prices dataset and preprocess data.
    Parameters
    ----------
    filename: str
        Path to house prices dataset

    Returns
    -------
    Design matrix and response vector (prices) - either as a single
    DataFrame or a Tuple[DataFrame, Series]
    """
    full_data = pd.read_csv(filename).drop_duplicates()
    # Drop unused values here---
    full_data = full_data.drop(["id", "date"], axis=1)
    full_data = full_data.dropna()

    # remove outlier that are likely to be errors.
    full_data = full_data.drop(
        full_data[
            (full_data["price"] <= 0) |
            (full_data["sqft_living"] <= 0) |
            (full_data["sqft_lot"] <= 0) |
            (full_data["sqft_above"] <= 0) |
            (full_data["yr_built"] <= 0) |
            (full_data["sqft_lot15"] <= 0) |

            (full_data["bathrooms"] < 0) |
            (full_data["bedrooms"] < 0) |
            (full_data["floors"] < 0) |
            (full_data["sqft_basement"] < 0) |
            (full_data["yr_renovated"] < 0) |

            (~full_data["waterfront"].isin([0, 1])) |
            (~full_data["view"].isin(range(5))) |
            (~full_data["condition"].isin(range(1, 6))) |
            (~full_data["grade"].isin(range(1, 15))) |

            (full_data["bedrooms"] > 30)
            ].index
    )
    # full_data = full_data.drop(
    #     full_data[(full_data["bedrooms"] == 0) & (full_data["bathrooms"] == 0)].index
    # )

    # create new features from existing features:
    full_data["zipcode"] = full_data["zipcode"].astype(int)
    full_data = pd.get_dummies(full_data,
                               prefix="zipcode_",
                               columns=["zipcode"])

    full_data["yr_last_worked_on"] = full_data["yr_renovated"].astype(int)
    full_data["yr_last_worked_on"].mask(
        full_data["yr_renovated"] <= full_data["yr_built"],
        full_data["yr_built"],
        axis=0,
        inplace=True
    )
    # drop data used only for new features:
    full_data = full_data.drop(["yr_built", "yr_renovated"], axis=1)
    return full_data.drop("price", axis=1), full_data["price"]

def feature_evaluation(X: pd.DataFrame, y: pd.Series,
                       output_path: str = ".") -> NoReturn:
    """
    Create scatter plot between each feature and the response.
        - Plot title specifies feature name
        - Plot title specifies Pearson Correlation between feature and response
        - Plot saved under given folder with file name including feature name
    Parameters
    ----------
    X : DataFrame of shape (n_samples, n_features)
        Design matrix of regression problem

    y : array-like of shape (n_samples, )
        Response vector to evaluate against

    output_path: str (default ".")
        Path to folder in which plots are saved
    """
    std_y = np.std(y)
    for name, values in X.items():
        array = values.to_numpy()
        p_cor = np.cov(array, y)[0, 1] / (np.std(array) * std_y)
        go.Figure(
            data=go.Scatter(
                x=array,
                y=y,
                mode="markers",
            )
        ).update_layout(
            title=f"feature {name} correlation with response. pearson = {p_cor}",
            xaxis_title=f"{name}",
            yaxis_title=f"response"
        ).write_image(f"{output_path}/{name}.png")

if __name__ == '__main__':
    np.random.seed(0)
    # Question 1 - Load and preprocessing of housing prices dataset
    features, responses = load_data("../datasets/house_prices.csv")

    # Question 2 - Feature evaluation with respect to response
    feature_evaluation(features, responses)

    # Question 3 - Split samples into training- and testing sets.
    train_X, train_y, test_X, test_y = split_train_test(features, responses)

    # Question 4 - Fit model over increasing percentages of the overall training data
    # For every percentage p in 10%, 11%, ..., 100%, repeat the following 10 times:
    #   1) Sample p% of the overall training data
    #   2) Fit linear model (including intercept) over sampled set
    #   3) Test fitted model over test set
    #   4) Store average and variance of loss over test set
    # Then plot average loss as function of training size with error ribbon of size (mean-2*std, mean+2*std)
    test_X, test_y = test_X.to_numpy(), test_y.to_numpy()
    model = LinearRegression()
    p_arr = np.arange(10, 101)
    mean_loss_array = np.zeros(91)
    std_loss_array = np.zeros(91)
    for i in range(10, 101):
        losses = []
        for _ in range(10):
            ipX_train, ipy_train, ipX_test, ipy_test = \
                split_train_test(train_X, train_y, i / 100)
            ipX_train, ipy_train = ipX_train.to_numpy(), ipy_train.to_numpy()
            losses.append(model.fit(ipX_train, ipy_train).loss(test_X, test_y))

        mean_loss_array[i - 10], std_loss_array[i - 10] = \
            np.mean(losses, axis=0), np.std(losses, axis=0)

    fig = go.Figure(
        data=[
            go.Scatter(
                x=p_arr,
                y=mean_loss_array,
                name="Mean loss",
                mode="markers+lines",
                line=dict(dash="dash"),
                marker=dict(color="green", opacity=.7)
            ),
            go.Scatter(
                x=p_arr,
                y=mean_loss_array - 2 * std_loss_array,
                name="confidence",
                fill=None, mode="lines",
                line=dict(color="lightgrey"),
                showlegend=False),
            go.Scatter(
                x=p_arr,
                y=mean_loss_array + 2 * std_loss_array,
                name="confidence",
                fill='tonexty',
                mode="lines",
                line=dict(color="lightgrey"),
                showlegend=False)],
        layout=go.Layout(
            title="Mean loss as function of p% of the training set",
            xaxis=dict(title="p% of training set"),
            yaxis=dict(title="Mean loss")
        )
    )
    fig.show()


text:
# -*- encoding: utf-8 -*-
"""
@Author     :   zYx.Tom
@Contact    :   526614962@qq.com
@site       :   https://github.com/zhuyuanxiang/tensorflow_cookbook
---------------------------
@Software   :   PyCharm
@Project    :   TensorFlow_Machine_Learning_Cookbook
@File       :   C0107_activation_functions.py
@Version    :   v0.1
@Time       :   2019-10-29 10:26
@License    :   (C)Copyright 2018-2019, zYx.Tom
@Reference  :   《TensorFlow机器学习实战指南，Nick McClure》, Sec0107，P12
@Desc       :   TensorFlow 基础, Implementing Activation Functions
"""
# common imports
import os
import sys

import matplotlib.pyplot as plt
import numpy as np  # pip install numpy<1.17，小于1.17就不会报错
import sklearn
import tensorflow as tf
import winsound
from tensorflow.python.framework import ops

from tools import show_values

# 设置数据显示的精确度为小数点后3位
np.set_printoptions(precision = 8, suppress = True, threshold = np.inf, linewidth = 200)

# 利用随机种子，保证随机数据的稳定性，使得每次随机测试的结果一样
np.random.seed(42)

# 初始化默认的计算图
ops.reset_default_graph()
# Python ≥3.5 is required
assert sys.version_info >= (3, 5)
# Scikit-Learn ≥0.20 is required
assert sklearn.__version__ >= "0.20"
# 屏蔽警告：Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
# Open graph session
sess = tf.Session()

test_data = [-10., -3., -1., 0., 1., 3., 10.]

# 部分线性的非线性函数
# 1. 整流线性单元（Rectifier Linear Unit，ReLU），非线性函数，max(0,x)：连续但不平滑
show_values(tf.nn.relu(test_data), "tf.nn.relu({})".format(test_data))
# 2. ReLUMax6, min(max(0,x),6)：计算运行速度快，解决梯度消失
show_values(tf.nn.relu6(test_data), "tf.nn.relu6({})".format(test_data))
# 6. softplus函数，ReLU函数的平滑版，log(exp(x)+1)
show_values(tf.nn.softplus(test_data), "tf.nn.softplus({})".format(test_data))
# 7. ELU激励函数（Exponential Linear Unit，ELU），
# 与softplus函数相似，只是输入无限小时，趋近于-1，而softplus函数趋近于0.
show_values(tf.nn.elu(test_data), "tf.nn.elu({})".format(test_data))

# 都是类似于Logistic函数
# 3. sigmoid函数，Logistic函数，1/(1+exp(-x))：最常用的连续的、平滑的激励函数，也叫逻辑函数
# sigmoid函数的取值范围为-1到1
show_values(tf.nn.sigmoid(test_data), "tf.nn.sigmoid({})".format(test_data))
# 4. 双曲正切函数（Hyper Tangent，tanh），((exp(x)-exp(-x))/(exp(x)+exp(-x))
# 双曲正切函数的的取值范围为0到1
show_values(tf.nn.tanh(test_data), "tf.nn.tanh({})".format(test_data))
# 5. softsign函数，x/(abs(x)+1)：符号函数的连续估计
show_values(tf.nn.softsign(test_data), "tf.nn.softsign({})".format(test_data))

# X range
x_vals = np.linspace(start = -10., stop = 10., num = 100)
y_relu = sess.run(tf.nn.relu(x_vals))
y_relu6 = sess.run(tf.nn.relu6(x_vals))
y_sigmoid = sess.run(tf.nn.sigmoid(x_vals))
y_tanh = sess.run(tf.nn.tanh(x_vals))
y_softsign = sess.run(tf.nn.softsign(x_vals))
y_softplus = sess.run(tf.nn.softplus(x_vals))
y_elu = sess.run(tf.nn.elu(x_vals))

# Plot the different functions
plt.figure()
plt.plot(x_vals, y_relu, 'k-', label = 'ReLU', linewidth = 4)
plt.plot(x_vals, y_elu, 'b--', label = 'ExpLU', linewidth = 3)
plt.plot(x_vals, y_relu6, 'g-.', label = 'ReLU6', linewidth = 2)
plt.plot(x_vals, y_softplus, 'r:', label = 'Softplus', linewidth = 1)
plt.ylim([-1.5, 7])
plt.legend(loc = 'upper left')
plt.title("图1-3：ReLU、ReLU6、softplus 和 ELU 激励函数")

plt.figure()
plt.plot(x_vals, y_sigmoid, 'r--', label = 'Sigmoid', linewidth = 2)
plt.plot(x_vals, y_tanh, 'b:', label = 'Tanh', linewidth = 2)
plt.plot(x_vals, y_softsign, 'g-.', label = 'Softsign', linewidth = 2)
plt.ylim([-2, 2])
plt.legend(loc = 'upper left')
plt.title("图1-3：sigmoid、softsign 和 tanh 激励函数")

# -----------------------------------------------------------------
# 运行结束的提醒
winsound.Beep(600, 500)
if len(plt.get_fignums()) != 0:
    plt.show()
pass