text:
#!/usr/bin/python

#
#DFZ 11/15/2016: it's hard to control the chunk size read from the
# stream() interface, see run_mri_stream.output for a concrete idea.
#

#the following import block is for testing only
import dipy.core.gradients as dpg
import os.path as op
from dipy.segment.mask import median_otsu
import nibabel as nib
from dipy.denoise import nlmeans
from dipy.denoise.noise_estimate import estimate_sigma
import time
import sys
import numpy as np
from __builtin__ import float

tm_start = time.time()
sys.stderr.write("=====> DFZ DEBUG 1/25/2017: " + time.ctime() + " OMG I start again! \n")

#SUB_ID = 101107
SUB_ID = 100307
DATA_LOC = "/home/dongfang/download/mri_data/" + str(SUB_ID) +"/"

end_of_interaction = 0
while (end_of_interaction != 1):
  header = sys.stdin.readline().rstrip()
  if(header != "0"):
    #We receive a message from the SciDB instance:
    num_lines = int(header)  #how many lines did we get?
    sys.stderr.write("=====> DFZ 1/25/2017: num_lines = "+str(num_lines)+"\n")
    #n_vol = num_lines / 145 / 174 / 145
    #sys.stderr.write("=====> DFZ 1/25/2017: n_vol = "+str(n_vol)+"(should equal to 288/4 )\n")

    #Collect all lines into a list:
    input_lines = []
    for i in range(0, num_lines):
      line = sys.stdin.readline().rstrip()
      try:
        f = float(line)
      except:
        f = 0.0
      input_lines.append(f)

#################################################
############## MRI Logic ########################
#################################################

    #construct the values into a numpy array for MRI
    nparray = np.asarray(input_lines, dtype=np.float32)
#      sys.stderr.write("=====> DFZ DEBUG: convertion completed.\n")
    sys.stderr.write("=====> DFZ DEBUG 2/16/2017: nparray.shape = " + str(nparray.size) + "; len(input_lines) = " + str(len(input_lines)) +"\n")
    data = np.reshape(nparray, (29, 29, 29, 288))#TODO: make a smaller xyz-dimension chunk
    sys.stderr.write("=====> DFZ DEBUG: data loading completed.\n")

    #masking
#    mask = nib.load(DATA_LOC + 'mask.nii.gz').get_data()
#    sys.stderr.write("=====> DFZ DEBUG: mask loaded. mask.shape = " + str(mask.shape) + "\n")

    #denosing
    #sigma = estimate_sigma(data)
    #sys.stderr.write("=====> DFZ DEBUG: sigma calculated.\n")
    #denoised_data = nlmeans.nlmeans(data, sigma=sigma, mask=mask)
    #sys.stderr.write("=====> DFZ DEBUG: denoising completed.\n")

    #model fitting
    #TODO: we need to (1) merge smaller chunks into the entire denoised subject; (2) load the denoised data across the T-dimension
    #Now, let's suppose data[0:29,0:87,0:29,0:144] is a denoised slice
    import dipy.reconst.dti as dti
    gtab = dpg.gradient_table(DATA_LOC + 'bvals', DATA_LOC + 'bvecs', b0_threshold=10)

    ten_model = dti.TensorModel(gtab)
    #ten_fit = ten_model.fit(data[0:29,0:87,0:29,0:144], mask=mask[0:29,0:87,0:29])
    ten_fit = ten_model.fit(data[0:29,0:29,0:29,:])
    sys.stderr.write("=====> DFZ DEBUG: model fitting completed.\n")

    # if you need interative results:
    print(2)
    print("Total lines: " + str(num_lines))
    print("I'm tired ----> First line: " + str(input_lines[0]))
    sys.stdout.flush()

    #This will appear in the scidb-sterr.log file:
    sys.stderr.write(time.ctime() + "I finished a chunk with "+ str(num_lines) + " lines of text!\n")
  else:
    #If we receive "0", it means the SciDB instance has no more
    #Data to give us. Here we have the option of also responding with "0"
    #Or sending some other message (i.e. a global sum):
    end_of_interaction = 1
    print("1")
#    print("KTHXBYE")
    print("KTHXBYE: subject " + str(SUB_ID) + " done in " + str(time.time() - tm_start) + " seconds.")
    sys.stdout.flush()
    sys.stderr.write("I got the end-of-data message. Exiting.\n")
ok = 0
# So I cannot 'return' or 'print' even after 'return'; the following statements would cause errors
exit(0)
# print "Start at " + str(time.ctime())


text:
# Random Forest Regression-even this is a non-linear regression model
# Ensemble algorithms are more stable becasue any changes in your dataset could really impact one tree but for them to impact a forest of trees is much harder so therefore ensemble is much more powerful in that way

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('Position_Salaries.csv')
X = dataset.iloc[:, 1:2].values
y = dataset.iloc[:, 2].values

# Splitting the dataset into the Training set and Test set
"""from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)"""

# Feature Scaling
"""from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)
sc_y = StandardScaler()
y_train = sc_y.fit_transform(y_train)"""

# Fitting Random Forest Regression to the dataset
from sklearn.ensemble import RandomForestRegressor  #import the class and create an object of that class
regressor = RandomForestRegressor(n_estimators = 10, random_state = 0)  #N-estimators-its the no of tress you want to have in your forest
regressor.fit(X, y)  #We are builiding a random forest with 10 trees,  fit(matrix of features,the dependent vector)

# Predicting a new result
y_pred = regressor.predict([[6.5]]) # So here if we give 6.6/6.75/6.9 it will return the same value which is 21K dollars(they are in the same range 6.5-7 so they all will return the same predicted value)
y_pred # here we got the predicted value as 167K dollars,so almost all the algorithms worked fine and have predicted in the range 150-170K dollars

# Visualising the Random Forest Regression results (higher resolution)
X_grid = np.arange(min(X), max(X), 0.01)
X_grid = X_grid.reshape((len(X_grid), 1))
plt.scatter(X, y, color = 'red')
plt.plot(X_grid, regressor.predict(X_grid), color = 'blue')
plt.title('Truth or Bluff (Random Forest Regression)')
plt.xlabel('Position level')
plt.ylabel('Salary')
plt.show()

ext: py

reason: import numpy


text:
from scipy import spatial
import numpy as np

pi = np.pi
bins = np.linspace(-pi, pi, num=9)
print(bins)

K = 10
points = np.zeros((36000, 128))
res = np.random.randint(0, points.shape[0], K)
print(points[res].shape)

points_1 = np.array([[1,2],
                     [3,4],
                     [1,6]])
points_2 = np.array([[2,2], [5,5], [3,1]])
res = spatial.distance.cdist(points_1, points_2, metric='euclidean')

print(f"euclid: \n{res.shape}\n{res}")

am = np.argmin(res, axis=1)
print(f"argmin = {am}")

ind = [0, 1, 2, 3, 4, 5, 5, 1, 0, 0]
res = np.array(ind)[np.where(np.array(ind) == 0)]
print(res)

mean = np.mean(points_1, axis=0)
print(f"mean: {mean}")

labels = 0 + np.zeros((500, 1))
labels = np.concatenate((labels, 1 + np.zeros((500, 1))))
print(labels)
print(np.sum(labels[0:500]))
print(np.sum(labels[500:1000]))
nn_indices = np.array([334, 319,  51, 694,  49, 311,  95,  921,  60, 731])
print("labels of test images:")
print(labels[nn_indices])

print("-j-------------------------")
labels = 0 + np.zeros((4, 1))
labels = np.concatenate((labels, 1 + np.zeros((4, 1))))
print(labels)
print(np.sum(labels[0:4]))
print(np.sum(labels[4:8]))
nn_indices = np.array([4, 7,  1, 6,  4, 3,  5,  5,  0, 7])
print("labels of test images:")
print(labels[nn_indices])

reason: import numpy,from scipy


text:
#!/usr/bin/python2
"""
This program detects neon yellow.
"""

print('Importing libraries...')
import time
import numpy as np
import freenect
import cv2
import matplotlib.pyplot as plt

if __name__ == '__main__':

    while True:

        print('Analyzing frame...')
        rgb, _ = freenect.sync_get_video()
        rgb = rgb[::8,::8,:]
        bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
        hsv = cv2.cvtColor(bgr, cv2.COLOR_BGR2HSV)
        #plt.imshow(hsv); plt.show()
        color_min_hsv = np.array([25, 128, 128])
        color_max_hsv = np.array([35, 255, 255])
        mask = cv2.inRange(hsv, color_min_hsv, color_max_hsv)
        res = cv2.blur(mask, (10,10))
        _, thresh = cv2.threshold(res, 30, 255, cv2.THRESH_BINARY)
        hist = np.sum(thresh, axis=0)
        location = np.argmin(np.abs(np.cumsum(hist)-np.sum(hist)/2))
        presence = np.sum(hist) > 100 # whether there's enough yellow in the image
        print(np.sum(hist))

        print('Ignoring depth!')
        """
        print('Obtaining depth image...')
        def get_depth_frame():
            frame, _ = freenect.sync_get_depth()
            #frame = frame.astype(np.uint8)
            return frame

        # detect obstacles
        depth = get_depth_frame()
        print('Calculating midpoint...')
        mindepths = np.min(depth, axis=0)
        safety_dist = 170
        safe = True
        for i in range(len(mindepths)):
            if mindepths[i] < 500:
                if 320-safety_dist < i < 320 + safety_dist:
                    safe = False
        """
        safe = True

        if presence:
            if safe:
                print('GO to {}'.format(location-320))
            else:
                print('OBSTACLE!')
        else:
            print('N/A')

        if presence:
            cv2.rectangle(bgr, (location,0), (location+1,45), (0,255,0), 2)

        cv2.imshow('image', bgr)
        cv2.imshow('thresh', thresh)

        # exit on 'q'
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    cv2.destroyAllWindows()

max_stars_repo_name: qaute/zeitgeist

max_issues_repo_name: qaute/zeitgeist

max_forks_repo_name: qaute/zeitgeist


text:
# Licensed under a 3-clause BSD style license - see LICENSE.rst

"""
<Put Your Tool Name Here>
-------------------------

:author: <your name> (<your email>)
"""

# Make the URL of the server, timeout and other items configurable
# See <http://docs.astropy.org/en/latest/config/index.html#developer-usage>
# for docs and examples on how to do this
# Below is a common use case
from astropy import config as _config

class Conf(_config.ConfigNamespace):
    """
    Configuration parameters for `astroquery.template_module`.
    """
    server = _config.ConfigItem(
        ['http://dummy_server_mirror_1',
         'http://dummy_server_mirror_2',
         'http://dummy_server_mirror_n'],
        'Name of the template_module server to use.')

    timeout = _config.ConfigItem(
        30,
        'Time limit for connecting to template_module server.')

conf = Conf()

# Now import your public class
# Should probably have the same name as your module
from .core import Template, TemplateClass

__all__ = ['Template', 'TemplateClass',
           'Conf', 'conf',
           ]

max_stars_repo_name: hamogu/astroquery

max_issues_repo_name: astrocatalogs/astroquery

max_forks_repo_name: hamogu/astroquery

reason: from astropy


text:
# Measure Watts-Strogatz network

import os
import sys
import numpy as np
import networkx as nx
from matplotlib import pyplot as plt

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
import votermodel as voter

# Parameters
# ====================
nnodes = 200
edges_per_node_BA = 2
edges_per_node_WS = 4
rewire_probs = np.array([0, 0.001, 0.01, 0.1, 1])
sims_per_prob =  10
nsteps = 1000

# Measure
meanstates = np.zeros((len(rewire_probs), nsteps), dtype=float)
interfacedens = np.zeros((len(rewire_probs), nsteps), dtype=float)

for j_prob, rewire_prob in enumerate(rewire_probs):
    meanstates_sum = np.zeros(nsteps, dtype=float)
    interfacedens_sum = np.zeros(nsteps, dtype=float)
    for j_sim in range(sims_per_prob):
        print("Prob: {0} ({1}/{2}), Sim: {3}/{4}".format(
                rewire_prob, j_prob + 1, len(rewire_probs), j_sim + 1,
                sims_per_prob))
        # Initialize instance
        net_WS = nx.watts_strogatz_graph(nnodes, edges_per_node_WS, rewire_prob)
        VM_WS = voter.VoterModel(net_WS)

        for j_step in range(nsteps):
            VM_WS.update(1)
            meanstates_sum[j_step] += np.abs(VM_WS.meanstate() - 0.5)/0.5
            interfacedens_sum[j_step] += VM_WS.interface_dens()

    # Calculate and store mean
    meanstates[j_prob] = meanstates_sum/sims_per_prob
    interfacedens[j_prob] = interfacedens_sum/sims_per_prob

# Save data
data_meanstate = np.hstack((rewire_probs[np.newaxis].T, meanstates))
np.savetxt("WS_meanstates.dat", data_meanstate)

data_interfacedens = np.hstack((rewire_probs[np.newaxis].T, interfacedens))
np.savetxt("WS_interfacedens.dat", data_interfacedens)

# Plot results
size = 3
fig, ax = plt.subplots(figsize=(1.62*size, size))
#ax.set_ylabel(r"$(\langle s \rangle - 0.5)/0.5$")
#
#for j_prob, rewire_prob in enumerate(rewire_probs):
#    ax.plot(meanstates_WS[j_prob], label=r"$p={0}$".format(rewire_prob))

ax.set_xlabel(r"Time")
ax.set_ylabel(r"Interface density")

for j_edges, rewire_prob in enumerate(rewire_probs):
    ax.plot(interfacedens[j_edges], label=r"$m={0}$".format(rewire_prob))

ax.legend()
fig.tight_layout()

fig.savefig("WS_p.png", dpi=200)
#plt.show()

max_stars_repo_name: ruhugu/votermodel

max_issues_repo_name: ruhugu/votermodel

max_forks_repo_name: ruhugu/votermodel

reason: import numpy,import networkx


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) IBM Corporation 2018
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""model.py: contains base abstract model for all models"""
__author__ = "Tomasz Kornuta"

import torch
from torch import nn
from abc import abstractmethod

import numpy as np

import logging
logger = logging.getLogger('Model')

from utils.app_state import AppState

class Model(nn.Module):
    """
    Class representing base class of all models.

    Provides basic plotting functionality.

    """

    def __init__(self, params):
        """
        Initializes application state and sets plot if visualization flag is
        turned on.

        :param params: Parameters read from configuration file.

        """
        # Call base class inits here.
        super(Model, self).__init__()

        # Initialize app state.
        self.app_state = AppState()

        # Store pointer to params.
        self.params = params

        # Window in which the data will be ploted.
        self.plotWindow = None

        # Initialization of best loss - as INF.
        self.best_loss = np.inf

        # Flag indicating whether intermediate checkpoints should be saved or
        # not (DEFAULT: False).
        if "save_intermediate" not in params:
            params.add_default_params({"save_intermediate": False})
        self.save_intermediate = params["save_intermediate"]

        # "Default" model name.
        self.name = 'Model'

    def add_statistics(self, stat_col):
        """
        Add statistics to collector.

        EMPTY - To be redefined in inheriting classes.

        :param stat_col: Statistics collector.

        """
        pass

    def collect_statistics(self, stat_col, data_tuple, logits):
        """
        Base statistics collection.

        EMPTY - To be redefined in inheriting classes.

        :param stat_col: Statistics collector.
        :param data_tuple: Data tuple containing inputs and targets.
        :param logits: Logits being output of the model.

        """
        pass

    @abstractmethod
    def plot(self, data_tuple, predictions, sample_number=0):
        """
        Plots inputs, targets and predictions, along with model-dependent
        variables.

        Abstract - to be defined in derived classes.

        :param data_tuple: Data tuple containing input and target batches.
        :param predictions: Prediction.
        :param sample_number: Number of sample in batch (DEFAULT: 0)

        """

    def save(self, model_dir, stat_col):
        """
        Generic method saving the model parameters to file. It can be
        overloaded if one needs more control.

        :param model_dir: Directory where the model will be saved.
        :param stat_col: Statistics collector that contain current loss and episode number (and other statistics).
        :return: True if this is the best model that is found till now (considering loss).

        """
        # Get two elementary statistics.
        loss = stat_col['loss']
        episode = stat_col['episode']

        # Checkpoint to be saved.
        chkpt = {
            'name': self.name,
            'state_dict': self.state_dict(),
            'stats': stat_col.statistics
        }

        # for key, value in stat_col.statistics.items():
        #    logger.warning("{}: {}".format(key, value))

        # Save the intermediate checkpoint.
        if self.save_intermediate:
            filename = model_dir + 'model_episode_{:05d}.pt'.format(episode)
            torch.save(chkpt, filename)
            logger.info(
                "Model and statistics exported to checkpoint {}".format(
                    filename))

        # Save the best model.
        if (loss < self.best_loss):
            self.best_loss = loss
            filename = model_dir + 'model_best.pt'
            torch.save(chkpt, filename)
            logger.info(
                "Model and statistics exported to checkpoint {}".format(
                    filename))
            return True
        # Else: that was not the best model.
        return False

    def load(self, checkpoint_file):
        """
        Loads model from the checkpoint file.

        :param checkpoint_file: File containing dictionary with model state and statistics.

        """
        # Load checkpoint
        # This is to be able to load CUDA-trained model on CPU
        chkpt = torch.load(
            checkpoint_file, map_location=lambda storage, loc: storage)

        # Load model.
        self.load_state_dict(chkpt['state_dict'])

        # Print statistics.
        logger.info(
            "Imported {} parameters from checkpoint (episode {}, loss {})".format(
                chkpt['name'],
                chkpt['stats']['episode'],
                chkpt['stats']['loss']))

    def summarize(self):
        """Summarizes model by showing trainable/non-trainable parameters and weights.
            Uses recursive_summarize to interate through nested structure of the mode.

            :param: Model object for which the summary will be created.

        """
        #add name of the current module
        summary_str = '\n' + '='*80 + '\n'
        summary_str += 'Model name (Type) \n'
        summary_str += '  + Submodule name (Type) \n'
        summary_str += '      Matrices: [(name, dims), ...]\n'
        summary_str += '      Trainable Params: #\n'
        summary_str += '      Non-trainable Params: #\n'
        summary_str += '='*80 + '\n'
        summary_str += self.recursive_summarize(self, 0, self.name)
        # Sum the model parameters.
        num_total_params = sum([np.prod(p.size()) for p in self.parameters()])
        mod_trainable_params = filter(lambda p: p.requires_grad, self.parameters())
        num_trainable_params = sum([np.prod(p.size()) for p in mod_trainable_params])
        summary_str += '\nTotal Trainable Params: {}\n'.format(num_trainable_params)
        summary_str += 'Total Non-trainable Params: {}\n'.format(num_total_params-num_trainable_params)
        summary_str += '='*80 + '\n'
        return summary_str

    def recursive_summarize(self, module_, indent_, module_name_):
        """
            Function that recursively inspects the (sub)modules and records their statistics (like names, types, parameters, their numbers etc.)

            :param module_: Module to be inspected.
            :param indent_: Current indentation level.
            :param module_name_: Name of the module that will be displayed before its type.

        """
        # Recursivelly inspect the children.
        child_lines = []
        for key, module in module_._modules.items():
            child_lines.append(self.recursive_summarize(module, indent_+1, key))

        # "Leaf information".
        mod_str = ''
        if indent_ > 0:
            mod_str += '  ' + '| '*(indent_-1) + '+ '
        mod_str += module_name_ + " ("+ module_._get_name() + ')'
        #mod_str += '-'*(80 - len(mod_str))
        mod_str += '\n'
        mod_str += ''.join(child_lines)
        # Get leaf weights and number of params - only for leafs!
        if not child_lines:
            # Collect names and dimensions of all (named) params.
            mod_weights = [(n,tuple(p.size())) for n,p in module_.named_parameters()]
            mod_str += '  ' + '| '* (indent_) + '  Matrices: {}\n'.format(mod_weights)
            # Sum the parameters.
            num_total_params = sum([np.prod(p.size()) for p in module_.parameters()])
            mod_trainable_params = filter(lambda p: p.requires_grad, module_.parameters())
            num_trainable_params = sum([np.prod(p.size()) for p in mod_trainable_params])
            mod_str += '  ' + '| '* (indent_) + '  Trainable Params: {}\n'.format(num_trainable_params)
            mod_str += '  ' + '| '* (indent_) + '  Non-trainable Params: {}\n'.format(num_total_params-num_trainable_params)
            mod_str += '  ' + '| '* (indent_) + '\n'

        return mod_str

max_stars_repo_name: aasseman/mi-prometheus

max_issues_repo_name: aasseman/mi-prometheus

max_forks_repo_name: aasseman/mi-prometheus


text:
"""
MIT License

Copyright (c) 2018 Mukesh Mithrakumar

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import keras
from utils import anchors as utils_anchors
import numpy as np
import tensorflow as tf

def filter_detections(
        boxes,
        classification,
        other=[],
        class_specific_filter=True,
        nms=True,
        score_threshold=0.05,
        max_detections=300,
        nms_threshold=0.5
):
    """ Filter detections using the boxes and classification values.
    Args
        boxes                 : Tensor of shape (num_boxes, 4) containing the boxes in (x1, y1, x2, y2) format.
        classification        : Tensor of shape (num_boxes, num_classes) containing the classification scores.
        other                 : List of tensors of shape (num_boxes, ...) to filter along with the boxes and
                                classification scores.
        class_specific_filter : Whether to perform filtering per class, or take the best scoring class and filter those.
        nms                   : Flag to enable/disable non maximum suppression.
        score_threshold       : Threshold used to prefilter the boxes with.
        max_detections        : Maximum number of detections to keep.
        nms_threshold         : Threshold for the IoU value to determine when a box should be suppressed.
    Returns
        A list of [boxes, scores, labels, other[0], other[1], ...].
        boxes is shaped (max_detections, 4) and contains the (x1, y1, x2, y2) of the non-suppressed boxes.
        scores is shaped (max_detections,) and contains the scores of the predicted class.
        labels is shaped (max_detections,) and contains the predicted label.
        other[i] is shaped (max_detections, ...) and contains the filtered other[i] data.
        In case there are less than max_detections detections, the tensors are padded with -1's.
    """

    def _filter_detections(scores, labels):
        # threshold based on score
        indices = tf.where(keras.backend.greater(scores, score_threshold))

        if nms:
            filtered_boxes = tf.gather_nd(boxes, indices)
            filtered_scores = keras.backend.gather(scores, indices)[:, 0]

            # perform NMS
            nms_indices = tf.image.non_max_suppression(filtered_boxes, filtered_scores, max_output_size=max_detections,
                                                       iou_threshold=nms_threshold)

            # filter indices based on NMS
            indices = keras.backend.gather(indices, nms_indices)

        # add indices to list of all indices
        labels = tf.gather_nd(labels, indices)
        indices = keras.backend.stack([indices[:, 0], labels], axis=1)

        return indices

    if class_specific_filter:
        all_indices = []
        # perform per class filtering
        for c in range(int(classification.shape[1])):
            scores = classification[:, c]
            labels = c * keras.backend.ones((keras.backend.shape(scores)[0],), dtype='int64')
            all_indices.append(_filter_detections(scores, labels))

        # concatenate indices to single tensor
        indices = keras.backend.concatenate(all_indices, axis=0)
    else:
        scores = keras.backend.max(classification, axis=1)
        labels = keras.backend.argmax(classification, axis=1)
        indices = _filter_detections(scores, labels)

    # select top k
    scores = tf.gather_nd(classification, indices)
    labels = indices[:, 1]
    scores, top_indices = tf.nn.top_k(scores, k=keras.backend.minimum(max_detections, keras.backend.shape(scores)[0]))

    # filter input using the final set of indices
    indices = keras.backend.gather(indices[:, 0], top_indices)
    boxes = keras.backend.gather(boxes, indices)
    labels = keras.backend.gather(labels, top_indices)
    other_ = [keras.backend.gather(o, indices) for o in other]

    # zero pad the outputs
    pad_size = keras.backend.maximum(0, max_detections - keras.backend.shape(scores)[0])
    boxes = tf.pad(boxes, [[0, pad_size], [0, 0]], constant_values=-1)
    scores = tf.pad(scores, [[0, pad_size]], constant_values=-1)
    labels = tf.pad(labels, [[0, pad_size]], constant_values=-1)
    labels = keras.backend.cast(labels, 'int32')
    other_ = [tf.pad(o, [[0, pad_size]] + [[0, 0] for _ in range(1, len(o.shape))], constant_values=-1) for o in
              other_]

    # set shapes, since we know what they are
    boxes.set_shape([max_detections, 4])
    scores.set_shape([max_detections])
    labels.set_shape([max_detections])
    for o, s in zip(other_, [list(keras.backend.int_shape(o)) for o in other]):
        o.set_shape([max_detections] + s[1:])

    return [boxes, scores, labels] + other_

class FilterDetections(keras.layers.Layer):
    """ Keras layer for filtering detections using score threshold and NMS.
    """

    def __init__(
            self,
            nms=True,
            class_specific_filter=True,
            nms_threshold=0.5,
            score_threshold=0.05,
            max_detections=300,
            parallel_iterations=32,
            **kwargs
    ):
        """ Filters detections using score threshold, NMS and selecting the top-k detections.
        Args
            nms                   : Flag to enable/disable NMS.
            class_specific_filter : Whether to perform filtering per class, or take the best scoring class and filter
                                    those.
            nms_threshold         : Threshold for the IoU value to determine when a box should be suppressed.
            score_threshold       : Threshold used to prefilter the boxes with.
            max_detections        : Maximum number of detections to keep.
            parallel_iterations   : Number of batch items to process in parallel.
        """
        self.nms = nms
        self.class_specific_filter = class_specific_filter
        self.nms_threshold = nms_threshold
        self.score_threshold = score_threshold
        self.max_detections = max_detections
        self.parallel_iterations = parallel_iterations
        super(FilterDetections, self).__init__(**kwargs)

    def call(self, inputs, **kwargs):
        """ Constructs the NMS graph.
        Args
            inputs : List of [boxes, classification, other[0], other[1], ...] tensors.
        """
        boxes = inputs[0]
        classification = inputs[1]
        other = inputs[2:]

        # wrap nms with our parameters
        def _filter_detections(args):
            boxes = args[0]
            classification = args[1]
            other = args[2]

            return filter_detections(
                boxes,
                classification,
                other,
                nms=self.nms,
                class_specific_filter=self.class_specific_filter,
                score_threshold=self.score_threshold,
                max_detections=self.max_detections,
                nms_threshold=self.nms_threshold,
            )

        # call filter_detections on each batch
        outputs = tf.map_fn(
            _filter_detections,
            elems=[boxes, classification, other],
            dtype=[keras.backend.floatx(), keras.backend.floatx(), 'int32'] + [o.dtype for o in other],
            parallel_iterations=self.parallel_iterations
        )

        return outputs

    def compute_output_shape(self, input_shape):
        """ Computes the output shapes given the input shapes.
        Args
            input_shape : List of input shapes [boxes, classification, other[0], other[1], ...].
        Returns
            List of tuples representing the output shapes:
            [filtered_boxes.shape, filtered_scores.shape, filtered_labels.shape, filtered_other[0].shape,
            filtered_other[1].shape, ...]
        """
        return [(input_shape[0][0], self.max_detections, 4),
                (input_shape[1][0], self.max_detections),
                (input_shape[1][0], self.max_detections),
                ] +[tuple([input_shape[i][0], self.max_detections] +
                          list(input_shape[i][2:])) for i in range(2, len(input_shape))]

    def compute_mask(self, inputs, mask=None):
        """ This is required in Keras when there is more than 1 output.
        """
        return (len(inputs) + 1) * [None]

    def get_config(self):
        """ Gets the configuration of this layer.
        Returns
            Dictionary containing the parameters of this layer.
        """
        config = super(FilterDetections, self).get_config()
        config.update({
            'nms': self.nms,
            'class_specific_filter': self.class_specific_filter,
            'nms_threshold': self.nms_threshold,
            'score_threshold': self.score_threshold,
            'max_detections': self.max_detections,
            'parallel_iterations': self.parallel_iterations,
        })

        return config

def shift(shape, stride, anchors):
    """ Produce shifted anchors based on shape of the map and stride size.
    Args
        shape  : Shape to shift the anchors over.
        stride : Stride to shift the anchors with over the shape.
        anchors: The anchors to apply at each location.
    """
    shift_x = (keras.backend.arange(0, shape[1], dtype=keras.backend.floatx())
               + keras.backend.constant(0.5, dtype=keras.backend.floatx())) * stride
    shift_y = (keras.backend.arange(0, shape[0], dtype=keras.backend.floatx())
               + keras.backend.constant(0.5, dtype=keras.backend.floatx())) * stride

    shift_x, shift_y = tf.meshgrid(shift_x, shift_y)
    shift_x = keras.backend.reshape(shift_x, [-1])
    shift_y = keras.backend.reshape(shift_y, [-1])

    shifts = keras.backend.stack([
        shift_x,
        shift_y,
        shift_x,
        shift_y
    ], axis=0)

    shifts = keras.backend.transpose(shifts)
    number_of_anchors = keras.backend.shape(anchors)[0]

    k = keras.backend.shape(shifts)[0]  # number of base points = feat_h * feat_w

    shifted_anchors = keras.backend.reshape(anchors, [1, number_of_anchors, 4]) + keras.backend.cast(
        keras.backend.reshape(shifts, [k, 1, 4]), keras.backend.floatx())
    shifted_anchors = keras.backend.reshape(shifted_anchors, [k * number_of_anchors, 4])

    return shifted_anchors

def resize_images(images, size, method='bilinear', align_corners=False):
    """ See https://www.tensorflow.org/versions/master/api_docs/python/tf/image/resize_images .
    Args
        method: The method used for interpolation. One of ('bilinear', 'nearest', 'bicubic', 'area').
    """
    methods = {
        'bilinear': tf.image.ResizeMethod.BILINEAR,
        'nearest': tf.image.ResizeMethod.NEAREST_NEIGHBOR,
        'bicubic': tf.image.ResizeMethod.BICUBIC,
        'area': tf.image.ResizeMethod.AREA,
    }
    return tf.image.resize_images(images, size, methods[method], align_corners)

def bbox_transform_inv(boxes, deltas, mean=None, std=None):
    """ Applies deltas (usually regression results) to boxes (usually anchors).
    Before applying the deltas to the boxes, the normalization that was previously applied (in the generator) has to
    be removed.
    The mean and std are the mean and std as applied in the generator. They are unnormalized in this function and then
    applied to the boxes.
    Args
        boxes : np.array of shape (B, N, 4), where B is the batch size, N the number of boxes and 4 values for
        (x1, y1, x2, y2).
        deltas: np.array of same shape as boxes. These deltas (d_x1, d_y1, d_x2, d_y2) are a factor of the width/height.
        mean  : The mean value used when computing deltas (defaults to [0, 0, 0, 0]).
        std   : The standard deviation used when computing deltas (defaults to [0.2, 0.2, 0.2, 0.2]).
    Returns
        A np.array of the same shape as boxes, but with deltas applied to each box.
        The mean and std are used during training to normalize the regression values (networks love normalization).
    """
    if mean is None:
        mean = [0, 0, 0, 0]
    if std is None:
        std = [0.2, 0.2, 0.2, 0.2]

    width = boxes[:, :, 2] - boxes[:, :, 0]
    height = boxes[:, :, 3] - boxes[:, :, 1]

    x1 = boxes[:, :, 0] + (deltas[:, :, 0] * std[0] + mean[0]) * width
    y1 = boxes[:, :, 1] + (deltas[:, :, 1] * std[1] + mean[1]) * height
    x2 = boxes[:, :, 2] + (deltas[:, :, 2] * std[2] + mean[2]) * width
    y2 = boxes[:, :, 3] + (deltas[:, :, 3] * std[3] + mean[3]) * height

    pred_boxes = keras.backend.stack([x1, y1, x2, y2], axis=2)

    return pred_boxes

class Anchors(keras.layers.Layer):
    """ Keras layer for generating achors for a given shape.
    """

    def __init__(self, size, stride, ratios=None, scales=None, *args, **kwargs):
        """ Initializer for an Anchors layer.

        Args
            size: The base size of the anchors to generate.
            stride: The stride of the anchors to generate.
            ratios: The ratios of the anchors to generate (defaults to [0.5, 1, 2]).
            scales: The scales of the anchors to generate (defaults to [2^0, 2^(1/3), 2^(2/3)]).
        """
        self.size = size
        self.stride = stride
        self.ratios = ratios
        self.scales = scales

        if ratios is None:
            self.ratios = np.array([0.5, 1, 2], keras.backend.floatx()),
        elif isinstance(ratios, list):
            self.ratios = np.array(ratios)
        if scales is None:
            self.scales = np.array([2 ** 0, 2 ** (1.0 / 3.0), 2 ** (2.0 / 3.0)], keras.backend.floatx()),
        elif isinstance(scales, list):
            self.scales = np.array(scales)

        self.num_anchors = len(ratios) * len(scales)
        self.anchors = keras.backend.variable(utils_anchors.generate_anchors(
            base_size=size,
            ratios=ratios,
            scales=scales,
        ))

        super(Anchors, self).__init__(*args, **kwargs)

    def call(self, inputs, **kwargs):
        features = inputs
        features_shape = keras.backend.shape(features)[:3]

        # generate proposals from bbox deltas and shifted anchors
        anchors = shift(features_shape[1:3], self.stride, self.anchors)
        anchors = keras.backend.tile(keras.backend.expand_dims(anchors, axis=0), (features_shape[0], 1, 1))

        return anchors

    def compute_output_shape(self, input_shape):
        if None not in input_shape[1:]:
            total = np.prod(input_shape[1:3]) * self.num_anchors
            return (input_shape[0], total, 4)
        else:
            return (input_shape[0], None, 4)

    def get_config(self):
        config = super(Anchors, self).get_config()
        config.update({
            'size': self.size,
            'stride': self.stride,
            'ratios': self.ratios.tolist(),
            'scales': self.scales.tolist(),
        })

        return config

class UpsampleLike(keras.layers.Layer):
    """ Keras layer for upsampling a Tensor to be the same shape as another Tensor.
    """

    def call(self, inputs, **kwargs):
        source, target = inputs
        target_shape = keras.backend.shape(target)
        return resize_images(source, (target_shape[1], target_shape[2]), method='nearest')

    def compute_output_shape(self, input_shape):
        return (input_shape[0][0],) + input_shape[1][1:3] + (input_shape[0][-1],)

class RegressBoxes(keras.layers.Layer):
    """ Keras layer for applying regression values to boxes.
    """

    def __init__(self, mean=None, std=None, *args, **kwargs):
        """ Initializer for the RegressBoxes layer.

        Args
            mean: The mean value of the regression values which was used for normalization.
            std: The standard value of the regression values which was used for normalization.
        """
        if mean is None:
            mean = np.array([0, 0, 0, 0])
        if std is None:
            std = np.array([0.2, 0.2, 0.2, 0.2])

        if isinstance(mean, (list, tuple)):
            mean = np.array(mean)
        elif not isinstance(mean, np.ndarray):
            raise ValueError('Expected mean to be a np.ndarray, list or tuple. Received: {}'.format(type(mean)))

        if isinstance(std, (list, tuple)):
            std = np.array(std)
        elif not isinstance(std, np.ndarray):
            raise ValueError('Expected std to be a np.ndarray, list or tuple. Received: {}'.format(type(std)))

        self.mean = mean
        self.std = std
        super(RegressBoxes, self).__init__(*args, **kwargs)

    def call(self, inputs, **kwargs):
        anchors, regression = inputs
        return bbox_transform_inv(anchors, regression, mean=self.mean, std=self.std)

    def compute_output_shape(self, input_shape):
        return input_shape[0]

    def get_config(self):
        config = super(RegressBoxes, self).get_config()
        config.update({
            'mean': self.mean.tolist(),
            'std': self.std.tolist(),
        })

        return config

class ClipBoxes(keras.layers.Layer):
    """ Keras layer to clip box values to lie inside a given shape.
    """

    def call(self, inputs, **kwargs):
        image, boxes = inputs
        shape = keras.backend.cast(keras.backend.shape(image), keras.backend.floatx())

        x1 = tf.clip_by_value(boxes[:, :, 0], 0, shape[2])
        y1 = tf.clip_by_value(boxes[:, :, 1], 0, shape[1])
        x2 = tf.clip_by_value(boxes[:, :, 2], 0, shape[2])
        y2 = tf.clip_by_value(boxes[:, :, 3], 0, shape[1])

        return keras.backend.stack([x1, y1, x2, y2], axis=2)

    def compute_output_shape(self, input_shape):
        return input_shape[1]


text:
# © 2020 Nokia
#
# Licensed under the BSD 3 Clause license
#
# SPDX-License-Identifier: BSD-3-Clause
# ============================================

from pathlib import Path
import dill
import numpy as np
import functools
import logging

from tqdm import tqdm

from codesearch.retrieval import RetrievalModel
from codesearch.utils import Saveable
from codesearch.snippet_embedding import SnippetTitleEmbedder

logger = logging.getLogger(__name__)

EPS = 10e-8

class RetrievalEmbedder(object):
    """ Embeds queries and snippets in the same vector space."""

    def embed_queries(self, queries):
        pass

    def embed_snippets(self, snippets):
        pass

def nan_logger(embedder):
    def nan_logger_fn(*args, **kwargs):
        embs = embedder(*args, **kwargs)
        if np.isnan(embs).any():
            print("Embedder produced NaN values")
        if len(np.where(~embs.any(axis=1))[0]) > 0:
            print("Contains all zero rows")
        return embs
    return nan_logger_fn

class EmbeddingRetrievalModel(RetrievalModel):

    def __init__(self, retrieval_embedder):
        super().__init__()
        self._snippetids = []
        self._snippetid2idx = {} # snippet id to embedding index
        self._unused_array_rows = 0
        self._embs = None
        self._embedder = retrieval_embedder

    @functools.lru_cache(maxsize=5000)
    def similarities(self, query):
        query = self._embedder.embed_queries([query])[0]
        query = query/np.linalg.norm(query)
        sims = np.dot(self._embs, query)
        return sims

    @functools.lru_cache(maxsize=5000)
    def similarities_batch(self, queries):
        # N x dim
        queries = self._embedder.embed_queries(queries)
        queries = queries/np.linalg.norm(queries)
        # N x M
        sims = np.dot(queries, self._embs.T)
        return sims

    def get_snippetid(self, idx):
        return self._snippetids[idx]

    def add_snippets(self, snippets):
        super().add_snippets(snippets)
        embs = self._embedder.embed_snippets(snippets)
        norms = np.linalg.norm(embs, keepdims=True, axis=1) + EPS
        embs /= norms
        idxs = []
        num_new = 0
        for s in snippets:
            id_ = s["id"]
            if id_ not in self._snippetid2idx:
                num_new += 1
                self._snippetid2idx[id_] = len(self._snippetids)
                self._snippetids.append(id_)
            idxs.append(self._snippetid2idx[id_])
        self._grow_embeddings(num_new, embs.shape[1])
        self._update_embeddings(idxs, embs)

    def _grow_embeddings(self, N, dim):
        new_embs = np.zeros(shape=(N,dim), dtype=np.float32)
        if self._embs is None:
            self._embs = new_embs
        else:
            self._embs = np.concatenate([self._embs, new_embs], axis=0)

    def _update_embeddings(self, idxs, embeddings):
        self._embs[idxs, :] = embeddings

    def save(self, path):
        super().save(path)
        path = Path(path)
        np.save(path/'embs.npy', self._embs)
        self._embedder.save(path/"retrieval_embedder")
        with open(path/"index", "wb") as f:
            dill.dump((self._snippetids, self._snippetid2idx, self._unused_array_rows), f)

    @classmethod
    def load(cls, path):
        path = Path(path)
        embs = np.load(path/'embs.npy')
        embedder = Saveable.load(path/"retrieval_embedder")
        with open(path/"index", "rb") as f:
            snippetids, snippetid2idx, unused_array_rows = dill.load(f)

        obj = cls(embedder)
        obj._embs = embs
        obj._snippetids = snippetids
        obj._snippetid2dix = snippetid2idx
        obj._unused_array_rows = unused_array_rows

        return obj

max_stars_repo_name: chinchila/codesearch


text:
'''
-----------------------------------------------
File Name: data_seg$
Description:
Author: Jing$
Date: 6/29/2021$
-----------------------------------------------
'''
from __future__ import division

import warnings

warnings.filterwarnings('ignore')  # ignore warnings
import os

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"  # only show error
import numpy as np
import pandas as pd
import cv2 as cv
from PIL import Image
import skimage.io as io
from skimage import img_as_ubyte
import os
from medpy.metric.binary import dc, hd, assd
from keras import backend as K
from keras.optimizers import Adam
#from tensorflow.keras.optimizers import Adam # only for doubleunet
from keras.callbacks import CSVLogger, ModelCheckpoint, EarlyStopping
import segmentation_models as sm
from model_seg import *
from doubleu_net import *

def load_data(img_path_aug, img_path_ori, gt_path_aug, gt_path_ori, csv_aug, csv_ori, H, W):
    df_ori = pd.read_csv(csv_ori)
    df_aug = pd.read_csv(csv_aug)
    filename_list_ori = df_ori['filename'].values
    filename_list_aug = df_aug['filename'].values
    pixel_size_ori = df_ori['pixel size(mm)'].values
    hcpx_ori = df_ori['head circumference(px)'].values
    img_ori = []
    label_ori = []
    img_aug = []
    label_aug = []
    pixel_ori = []
    label_hc = []

    for (i, f) in enumerate(filename_list_ori):
        img = Image.open(img_path_ori + f).convert('RGB')  # 3 channels
        img = img.resize((H,W))
        img = np.array(img)
        img_norm = (img - np.mean(img)) / np.std(img)  # normalize
        img_ori.append(img_norm)
        pixel_ori.append(pixel_size_ori[i])
        label_hc.append(hcpx_ori[i])

        gt = Image.open(gt_path_ori + f).convert('L')
        gt = gt.resize((H,W))
        gt = np.array(gt)
        gt[gt > 0.5] = 1  # normalize
        gt[gt <= 0.5] = 0
        gt = gt[:, :, np.newaxis]
        label_ori.append(gt)

    for (i, f) in enumerate(filename_list_aug):
        img = Image.open(img_path_aug + f).convert('RGB')
        img = img.resize((H,W))
        img = np.array(img)
        img_norm = (img - np.mean(img)) / np.std(img)  # normalize
        # img = img_norm[:, :, np.newaxis]
        img_aug.append(img_norm)

        gt = Image.open(gt_path_aug + f).convert('L')
        gt = gt.resize((H,W))
        gt = np.array(gt)
        gt[gt > 0.5] = 1  # normalize
        gt[gt <= 0.5] = 0
        gt = gt[:, :, np.newaxis]
        label_aug.append(gt)

    print("load data successfully!")
    return np.asarray(img_aug, dtype=np.float64), np.asarray(label_aug), np.asarray(label_hc), \
           np.asarray(img_ori, dtype=np.float64), np.asarray(label_ori), np.asarray(pixel_ori)

def save_data(save_path, segment_results, label, shape=(800, 540)):
    image_resize = []
    label_resize = []
    for i, item in enumerate(segment_results):
        img = item[:, :, 0]
        if np.isnan(np.sum(img)):
            img = img[~np.isnan(img)]  # just remove nan elements from vector

        img[img > 0.5] = 1
        img[img <= 0.5] = 0
        img_resize = cv.resize(img, shape, interpolation=cv.INTER_AREA)
        image_resize.append(img_resize)
        io.imsave(os.path.join(save_path, "%d_predict.png" % i), img_as_ubyte(img_resize))
    for i, item in enumerate(label):
        gt_resize = cv.resize(item, shape, interpolation=cv.INTER_AREA)
        label_resize.append(gt_resize)

    print("save data successfully!")
    return np.asarray(image_resize), np.asarray(label_resize)

def Dice(y_true, y_pred):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    return (2. * intersection) / (K.sum(y_true_f) + K.sum(y_pred_f))

def Dice_score(gt, seg):
    dice = []
    for item in range(len(gt)):
        dice.append(dc(gt[item], seg[item]))
    print("The mean and std dice score is: ", '%.2f' % np.mean(dice), '%.2f' % np.std(dice))
    return np.mean(dice), np.std(dice)

def HausdorffDistance_score(gt, seg, pixelsize):
    hausdorff = []
    for item in range(len(gt)):
        if np.sum(seg[item]) > 0:  # If the structure is predicted on at least one pixel
            hausdorff.append(hd(seg[item], gt[item], voxelspacing=[pixelsize[item], pixelsize[item]]))
    print("The mean and std Hausdorff Distance is: ", '%.2f' % np.mean(hausdorff), '%.2f' % np.std(hausdorff))
    return np.mean(hausdorff), np.std(hausdorff)

def ASSD_score(gt, seg, pixelsize):
    ASSD = []
    for item in range(len(gt)):
        if np.sum(seg[item]) > 0:
            ASSD.append(assd(seg[item], gt[item], voxelspacing=[pixelsize[item], pixelsize[item]]))
    print("The mean and std ASSD is: ", '%.2f' % np.mean(ASSD), '%.2f' % np.std(ASSD))
    return np.mean(ASSD), np.std(ASSD)

def EllipseCircumference(a,b):
    # Ramanujan approximation II
    #  HC = pi*(a+b)*(1+3h/(10+sqrt(4-3*h))),h = (a-b)**2/(a+b)**2
    h = ((a / 2 - b / 2) ** 2) / ((a / 2 + b / 2) ** 2)
    perimeter_ellipse = np.pi * (a / 2 + b / 2) * (1 + 3 * h / (10 + np.sqrt(4 - 3 * h)))
    return perimeter_ellipse

def HC_calculate(pred):
    '''
    3 ways to calculate HC:
    1. the number of contour points;
    2. the length of contour;
    3. the length of fitted ellipse
    '''
    num_points_pp = []
    len_contour_pp = []
    len_ellipse_pp = []
    num_points = []
    len_contour = []
    len_ellipse = []

    for item in range(len(pred)):
        image = np.uint8(pred[item])
        image[image > 0.5] = 255
        image[image <= 0.5] = 0
        contour = cv.Canny(image, 80, 160)
        contours, hierarchy = cv.findContours(contour, mode=cv.RETR_EXTERNAL, method=cv.CHAIN_APPROX_NONE)

        #print("performing with post processing")
        max_contour = []
        for i in range(len(contours)):
            if len(contours[i]) > len(max_contour):
                max_contour = contours[i]
        if len(max_contour) != 0:
            perimeter_contour = cv.arcLength(max_contour, True)  # para2:indicating whether the curve is closed or not
            # fitting ellipse, return center points, axis
            (cx, cy), (a, b), angle = cv.fitEllipse(max_contour)
            perimeter_ellipse = EllipseCircumference(a,b)
        else:
            perimeter_contour = 0
            perimeter_ellipse = 0

        num_points_pp.append(len(max_contour))
        len_contour_pp.append(perimeter_contour)
        len_ellipse_pp.append(perimeter_ellipse)

        #print("performing without post processing")
        if len(contours) !=0:
            num_points_unit=0
            len_contour_unit=0
            len_ellipse_unit=0
            for i in range(len(contours)):
                num_points_unit +=len(contours[i])
                len_contour_unit +=cv.arcLength(contours[i], True)
                if len(contours[i])>5:#There should be at least 5 points to fit the ellipse in function 'cv::fitEllipse'
                    (cx, cy), (a, b), angle = cv.fitEllipse(contours[i])
                    len_ellipse_unit +=EllipseCircumference(a,b)
        else:
            num_points_unit = 0
            len_contour_unit = 0
            len_ellipse_unit = 0

        num_points.append(num_points_unit)
        len_contour.append(len_contour_unit)
        len_ellipse.append(len_ellipse_unit)

    return np.asarray(num_points), np.asarray(len_contour), np.asarray(len_ellipse),\
           np.asarray(num_points_pp), np.asarray(len_contour_pp), np.asarray(len_ellipse_pp)

def predictions(x_text, y_test, label_hc_px, pixelsize, model, save_path):
    score = model.evaluate(x_text, y_test, verbose=2)
    print("Loss,iou sore:", '%.2f' % score[0], '%.2f' % score[1])

    results = model.predict(x_text)  # return probability
    pred, y_test = save_data(save_path, results, y_test)
    # segmentation analysis
    mean_dice, std_dice = Dice_score(y_test, pred)
    mean_hd, std_hd = HausdorffDistance_score(y_test, pred, pixelsize)
    mean_assd, std_assd = ASSD_score(y_test, pred, pixelsize)
    # HC analysis
    HC_pred_points, HC_pred_contour, HC_pred_ellipse,\
    HC_pred_points_pp, HC_pred_contour_pp, HC_pred_ellipse_pp= HC_calculate(pred)
    print("predicted value in mm:", HC_pred_ellipse_pp * pixelsize)
    print("predicted value in mm wo pp:", HC_pred_ellipse * pixelsize)

    absDiff_points = np.abs((HC_pred_points - label_hc_px) * pixelsize)
    absDiff_contour = np.abs((HC_pred_contour - label_hc_px) * pixelsize)
    absDiff_ellipse = np.abs((HC_pred_ellipse - label_hc_px) * pixelsize)

    absDiff_points_pp = np.abs((HC_pred_points_pp - label_hc_px) * pixelsize)
    absDiff_contour_pp = np.abs((HC_pred_contour_pp - label_hc_px) * pixelsize)
    absDiff_ellipse_pp = np.abs((HC_pred_ellipse_pp - label_hc_px) * pixelsize)

    mean_mae_points = round(np.mean(absDiff_points), 2)  # compute mae in mm
    mean_mae_contour = round(np.mean(absDiff_contour), 2)  # compute mae in mm
    mean_mae_ellipse = round(np.mean(absDiff_ellipse), 2)  # compute mae in mm

    mean_mae_points_pp = round(np.mean(absDiff_points_pp), 2)  # compute mae in mm
    mean_mae_contour_pp = round(np.mean(absDiff_contour_pp), 2)  # compute mae in mm
    mean_mae_ellipse_pp = round(np.mean(absDiff_ellipse_pp), 2)  # compute mae in mm

    std_mae_points = round(np.std(absDiff_points), 2)
    std_mae_contour = round(np.std(absDiff_contour), 2)
    std_mae_ellipse = round(np.std(absDiff_ellipse), 2)

    std_mae_points_pp = round(np.std(absDiff_points_pp), 2)
    std_mae_contour_pp = round(np.std(absDiff_contour_pp), 2)
    std_mae_ellipse_pp = round(np.std(absDiff_ellipse_pp), 2)

    mean_mae_px_points = round(np.mean(absDiff_points / pixelsize), 2)  # compute mae in pixel
    mean_mae_px_contour = round(np.mean(absDiff_contour / pixelsize), 2)  # compute mae in pixel
    mean_mae_px_ellipse = round(np.mean(absDiff_ellipse / pixelsize), 2)  # compute mae in pixel

    mean_mae_px_points_pp = round(np.mean(absDiff_points_pp / pixelsize), 2)  # compute mae in pixel
    mean_mae_px_contour_pp = round(np.mean(absDiff_contour_pp / pixelsize), 2)  # compute mae in pixel
    mean_mae_px_ellipse_pp = round(np.mean(absDiff_ellipse_pp / pixelsize), 2)  # compute mae in pixel

    std_mae_px_points = round(np.std(absDiff_points / pixelsize), 2)
    std_mae_px_contour = round(np.std(absDiff_contour / pixelsize), 2)
    std_mae_px_ellipse = round(np.std(absDiff_ellipse / pixelsize), 2)

    std_mae_px_points_pp = round(np.std(absDiff_points_pp / pixelsize), 2)
    std_mae_px_contour_pp = round(np.std(absDiff_contour_pp / pixelsize), 2)
    std_mae_px_ellipse_pp = round(np.std(absDiff_ellipse_pp / pixelsize), 2)

    mean_pmae_points = np.mean(np.abs((label_hc_px - HC_pred_points) / label_hc_px)) * 100  # compute percentage mae
    mean_pmae_contour = np.mean(np.abs((label_hc_px - HC_pred_contour) / label_hc_px)) * 100  # compute percentage mae
    mean_pmae_ellipse = np.mean(np.abs((label_hc_px - HC_pred_ellipse) / label_hc_px)) * 100  # compute percentage mae

    mean_pmae_points_pp = np.mean(np.abs((label_hc_px - HC_pred_points_pp) / label_hc_px)) * 100  # compute percentage mae
    mean_pmae_contour_pp = np.mean(np.abs((label_hc_px - HC_pred_contour_pp) / label_hc_px)) * 100  # compute percentage mae
    mean_pmae_ellipse_pp = np.mean(np.abs((label_hc_px - HC_pred_ellipse_pp) / label_hc_px)) * 100  # compute percentage mae

    std_pmae_points = np.std(np.abs((label_hc_px - HC_pred_points) / label_hc_px)) * 100
    std_pmae_contour = np.std(np.abs((label_hc_px - HC_pred_contour) / label_hc_px)) * 100
    std_pmae_ellipse = np.std(np.abs((label_hc_px - HC_pred_ellipse) / label_hc_px)) * 100

    std_pmae_points_pp = np.std(np.abs((label_hc_px - HC_pred_points_pp) / label_hc_px)) * 100
    std_pmae_contour_pp = np.std(np.abs((label_hc_px - HC_pred_contour_pp) / label_hc_px)) * 100
    std_pmae_ellipse_pp = np.std(np.abs((label_hc_px - HC_pred_ellipse_pp) / label_hc_px)) * 100

    print('\t  HC_mae in mm(points) w/o pp:', round(mean_mae_points, 2), 'mm (+-)', round(std_mae_points, 2), 'mm')
    print('\t  HC_mae in mm(contour)w/o pp:', round(mean_mae_contour, 2), 'mm (+-)', round(std_mae_contour, 2), 'mm')
    print('\t  HC_mae in mm(ellipse)w/o pp:', round(mean_mae_ellipse, 2), 'mm (+-)', round(std_mae_ellipse, 2), 'mm')
    print('\t  HC_mae in mm(points)  w pp:', round(mean_mae_points_pp, 2), 'mm (+-)', round(std_mae_points_pp, 2), 'mm')
    print('\t  HC_mae in mm(contour) w pp:', round(mean_mae_contour_pp, 2), 'mm (+-)', round(std_mae_contour_pp, 2), 'mm')
    print('\t  HC_mae in mm(ellipse) w pp:', round(mean_mae_ellipse_pp, 2), 'mm (+-)', round(std_mae_ellipse_pp, 2), 'mm')

    print('\t  HC_mae in px(points) w/o pp:', round(mean_mae_px_points, 2), 'pixels (+-)', round(std_mae_px_points, 2))
    print('\t  HC_mae in px(contour)w/o pp:', round(mean_mae_px_contour, 2), 'pixels (+-)', round(std_mae_px_contour, 2))
    print('\t  HC_mae in px(ellipse)w/o pp:', round(mean_mae_px_ellipse, 2), 'pixels (+-)', round(std_mae_px_ellipse, 2))
    print('\t  HC_mae in px(points)  w pp:', round(mean_mae_px_points_pp, 2), 'pixels (+-)', round(std_mae_px_points_pp, 2))
    print('\t  HC_mae in px(contour) w pp:', round(mean_mae_px_contour_pp, 2), 'pixels (+-)', round(std_mae_px_contour_pp, 2))
    print('\t  HC_mae in px(ellipse) w pp:', round(mean_mae_px_ellipse_pp, 2), 'pixels (+-)', round(std_mae_px_ellipse_pp, 2))

    print('\t  pmae(points) w/o pp:', round(mean_pmae_points, 2), '% (+-)', round(std_pmae_points, 2))
    print('\t  pmae(contour)w/o pp:', round(mean_pmae_contour, 2), '% (+-)', round(std_pmae_contour, 2))
    print('\t  pmae(ellipse)w/o pp:', round(mean_pmae_ellipse, 2), '% (+-)', round(std_pmae_ellipse, 2))
    print('\t  pmae(points)  w pp:', round(mean_pmae_points_pp, 2), '% (+-)', round(std_pmae_points_pp, 2))
    print('\t  pmae(contour) w pp:', round(mean_pmae_contour_pp, 2), '% (+-)', round(std_pmae_contour_pp, 2))
    print('\t  pmae(ellipse) w pp:', round(mean_pmae_ellipse_pp, 2), '% (+-)', round(std_pmae_ellipse_pp, 2))

    return mean_dice, std_dice, mean_hd, std_hd, mean_assd, std_assd, \
           mean_mae_points, mean_mae_contour, mean_mae_ellipse, \
           std_mae_points, std_mae_contour, std_mae_ellipse, \
           mean_mae_px_points, mean_mae_px_contour, mean_mae_px_ellipse, \
           std_mae_px_points, std_mae_px_contour, std_mae_px_ellipse, \
           mean_pmae_points, mean_pmae_contour, mean_pmae_ellipse, \
           std_pmae_points, std_pmae_contour, std_pmae_ellipse,\
           mean_mae_points_pp, mean_mae_contour_pp, mean_mae_ellipse_pp, \
           std_mae_points_pp, std_mae_contour_pp, std_mae_ellipse_pp, \
           mean_mae_px_points_pp, mean_mae_px_contour_pp, mean_mae_px_ellipse_pp, \
           std_mae_px_points_pp, std_mae_px_contour_pp, std_mae_px_ellipse_pp, \
           mean_pmae_points_pp, mean_pmae_contour_pp, mean_pmae_ellipse_pp, \
           std_pmae_points_pp, std_pmae_contour_pp, std_pmae_ellipse_pp

def fold_cross_valid(root,x_aug, y_aug, x_ori, y_ori, label_hc, ps_ori, inputshape2D, loss, save_path,
                     nb_epoch=50, batch_size=8, learning_rate=1e-3, best_filename='best.h5'):
    test_dice = []
    test_hd = []
    test_assd = []

    test_mae_HC_px_point = []
    test_mae_HC_px_contour = []
    test_mae_HC_px_ellipse = []
    test_mae_HC_px_point_pp = []
    test_mae_HC_px_contour_pp = []
    test_mae_HC_px_ellipse_pp = []

    test_mae_HC_mm_point = []
    test_mae_HC_mm_contour = []
    test_mae_HC_mm_ellipse = []
    test_mae_HC_mm_point_pp = []
    test_mae_HC_mm_contour_pp = []
    test_mae_HC_mm_ellipse_pp = []

    test_pmae_HC_point = []
    test_pmae_HC_contour = []
    test_pmae_HC_ellipse = []
    test_pmae_HC_point_pp = []
    test_pmae_HC_contour_pp = []
    test_pmae_HC_ellipse_pp = []

    early_stopping = EarlyStopping(monitor='val_loss', patience=90, verbose=1)
    model_checkpoint = ModelCheckpoint(best_filename, verbose=0, save_best_only=True)
    log_path = [
        'logs/log-1.csv',
        'logs/log-2.csv',
        'logs/log-3.csv',
        'logs/log-4.csv',
        'logs/log-5.csv'
    ]

    for i in range(0, 5):
        idx_train = np.load('cv_array/train' + str(i) + '.npy', allow_pickle=True)
        idx_test = np.load('cv_array/test' + str(i) + '.npy', allow_pickle=True)
        idx_valid = np.load('cv_array/valid' + str(i) + '.npy', allow_pickle=True)

        # idx_train2 = [i + 999 for i in idx_train]
        # x_train = np.concatenate((x_ori[idx_train], x_aug[idx_train]), axis=0)  # first 600 data augmentation
        # y_train = np.concatenate((y_ori[idx_train], y_aug[idx_train]), axis=0)
        #
        # x_train = np.concatenate((x_train, x_aug[idx_train2]), axis=0)  # second 600 data augmentation
        # y_train = np.concatenate((y_train, y_aug[idx_train2]), axis=0)

        x_train = x_ori[idx_train]
        y_train = y_ori[idx_train]

        x_valid = x_ori[idx_valid]
        y_valid = y_ori[idx_valid]

        x_test = x_ori[idx_test]
        y_test = y_ori[idx_test]
        ps_test = ps_ori[idx_test]
        hc_test = label_hc[idx_test]
        metric = sm.metrics.iou_score
        #model = unet(inputshape2D)
        model = sm.Unet(backbone,input_shape=inputshape2D,encoder_weights = None)
        #model = smx.Xnet(backbone,input_shape=inputshape2D)
        #model = sm.FPN(backbone,input_shape=inputshape2D) #
        #model = sm.Linknet(backbone,input_shape=inputshape2D)
        #model = sm.PSPNet(backbone,input_shape=inputshape2D) # require 480,480!!!
        #model = doubleUNet(inputshape2D) # require two keras frames!!!
        #model = sm.PSPNet(backbone, encoder_weights = 'imagenet', classes = 1,
        #encoder_freeze=False, activation='sigmoid', downsample_factor=16, input_shape=(480,480,3),
        #psp_conv_filters=1024, psp_pooling_type='avg')
        #model.summary()
        model.compile(loss=loss, optimizer=Adam(lr=learning_rate), metrics=[metric])

        model.fit(x_train, y_train, validation_data=(x_valid, y_valid),
                  callbacks=[early_stopping, model_checkpoint, CSVLogger(log_path[i], append=True)],
                  batch_size=batch_size, epochs=nb_epoch, shuffle=True, verbose=1)

        model.load_weights(best_filename)

        # -------------- Test Predictions ----------------------------
        mean_dice, std_dice, mean_hd, std_hd, mean_assd, std_assd, \
        mean_mae_points, mean_mae_contour, mean_mae_ellipse, \
        std_mae_points, std_mae_contour, std_mae_ellipse, \
        mean_mae_px_points, mean_mae_px_contour, mean_mae_px_ellipse, \
        std_mae_px_points, std_mae_px_contour, std_mae_px_ellipse, \
        mean_pmae_points, mean_pmae_contour, mean_pmae_ellipse, \
        std_pmae_points, std_pmae_contour, std_pmae_ellipse, \
        mean_mae_points_pp, mean_mae_contour_pp, mean_mae_ellipse_pp, \
        std_mae_points_pp, std_mae_contour_pp, std_mae_ellipse_pp, \
        mean_mae_px_points_pp, mean_mae_px_contour_pp, mean_mae_px_ellipse_pp, \
        std_mae_px_points_pp, std_mae_px_contour_pp, std_mae_px_ellipse_pp, \
        mean_pmae_points_pp, mean_pmae_contour_pp, mean_pmae_ellipse_pp, \
        std_pmae_points_pp, std_pmae_contour_pp, std_pmae_ellipse_pp = \
        predictions(x_test, y_test, hc_test, ps_test, model, save_path[i])

        test_dice.append([mean_dice, std_dice])
        test_hd.append([mean_hd, std_hd])
        test_assd.append([mean_assd, std_assd])

        test_mae_HC_px_point.append([mean_mae_px_points, std_mae_px_points])
        test_mae_HC_px_contour.append([mean_mae_px_contour, std_mae_px_contour])
        test_mae_HC_px_ellipse.append([mean_mae_px_ellipse, std_mae_px_ellipse])
        test_mae_HC_px_point_pp.append([mean_mae_px_points_pp, std_mae_px_points_pp])
        test_mae_HC_px_contour_pp.append([mean_mae_px_contour_pp, std_mae_px_contour_pp])
        test_mae_HC_px_ellipse_pp.append([mean_mae_px_ellipse_pp, std_mae_px_ellipse_pp])

        test_mae_HC_mm_point.append([mean_mae_points, std_mae_points])
        test_mae_HC_mm_contour.append([mean_mae_contour, std_mae_contour])
        test_mae_HC_mm_ellipse.append([mean_mae_ellipse, std_mae_ellipse])
        test_mae_HC_mm_point_pp.append([mean_mae_points_pp, std_mae_points_pp])
        test_mae_HC_mm_contour_pp.append([mean_mae_contour_pp, std_mae_contour_pp])
        test_mae_HC_mm_ellipse_pp.append([mean_mae_ellipse_pp, std_mae_ellipse_pp])

        test_pmae_HC_point.append([mean_pmae_points, std_pmae_points])
        test_pmae_HC_contour.append([mean_pmae_contour, std_pmae_contour])
        test_pmae_HC_ellipse.append([mean_pmae_ellipse, std_pmae_ellipse])
        test_pmae_HC_point_pp.append([mean_pmae_points_pp, std_pmae_points_pp])
        test_pmae_HC_contour_pp.append([mean_pmae_contour_pp, std_pmae_contour_pp])
        test_pmae_HC_ellipse_pp.append([mean_pmae_ellipse_pp, std_pmae_ellipse_pp])

    # end of for loop

    CV_mean_dice, CV_std_dice = np.mean(test_dice, axis=0)
    CV_mean_hd, CV_std_hd = np.mean(test_hd, axis=0)
    CV_mean_assd, CV_std_assd = np.mean(test_assd, axis=0)

    CV_mean_mae_HC_px_point, CV_std_mae_HC_px_point = np.mean(test_mae_HC_px_point, axis=0)
    CV_mean_mae_HC_px_contour, CV_std_mae_HC_px_contour = np.mean(test_mae_HC_px_contour, axis=0)
    CV_mean_mae_HC_px_ellipse, CV_std_mae_HC_px_ellipse = np.mean(test_mae_HC_px_ellipse, axis=0)
    CV_mean_mae_HC_px_point_pp, CV_std_mae_HC_px_point_pp = np.mean(test_mae_HC_px_point_pp, axis=0)
    CV_mean_mae_HC_px_contour_pp, CV_std_mae_HC_px_contour_pp = np.mean(test_mae_HC_px_contour_pp, axis=0)
    CV_mean_mae_HC_px_ellipse_pp, CV_std_mae_HC_px_ellipse_pp = np.mean(test_mae_HC_px_ellipse_pp, axis=0)

    CV_mean_mae_HC_mm_point, CV_std_mae_HC_mm_point = np.mean(test_mae_HC_mm_point, axis=0)
    CV_mean_mae_HC_mm_contour, CV_std_mae_HC_mm_contour = np.mean(test_mae_HC_mm_contour, axis=0)
    CV_mean_mae_HC_mm_ellipse, CV_std_mae_HC_mm_ellipse = np.mean(test_mae_HC_mm_ellipse, axis=0)
    CV_mean_mae_HC_mm_point_pp, CV_std_mae_HC_mm_point_pp = np.mean(test_mae_HC_mm_point_pp, axis=0)
    CV_mean_mae_HC_mm_contour_pp, CV_std_mae_HC_mm_contour_pp = np.mean(test_mae_HC_mm_contour_pp, axis=0)
    CV_mean_mae_HC_mm_ellipse_pp, CV_std_mae_HC_mm_ellipse_pp = np.mean(test_mae_HC_mm_ellipse_pp, axis=0)

    CV_pmae_mean_HC_point, CV_pmae_std_HC_point = np.mean(test_pmae_HC_point, axis=0)
    CV_pmae_mean_HC_contour, CV_pmae_std_HC_contour = np.mean(test_pmae_HC_contour, axis=0)
    CV_pmae_mean_HC_ellipse, CV_pmae_std_HC_ellipse = np.mean(test_pmae_HC_ellipse, axis=0)
    CV_pmae_mean_HC_point_pp, CV_pmae_std_HC_point_pp = np.mean(test_pmae_HC_point_pp, axis=0)
    CV_pmae_mean_HC_contour_pp, CV_pmae_std_HC_contour_pp = np.mean(test_pmae_HC_contour_pp, axis=0)
    CV_pmae_mean_HC_ellipse_pp, CV_pmae_std_HC_ellipse_pp = np.mean(test_pmae_HC_ellipse_pp, axis=0)

    print('-' * 60)
    print('5CV Mean dice score :', round(CV_mean_dice, 3), '(+-)', round(CV_std_dice, 3))
    print('5CV Mean hd score :', round(CV_mean_hd, 3), 'mm (+-)', round(CV_std_hd, 3), 'mm')
    print('5CV Mean assd :', round(CV_mean_assd, 3), 'mm (+-)', round(CV_std_assd, 3), 'mm')
    print('-' * 60)
    print('5CV mae HC(px) in points  w/o pp:', round(CV_mean_mae_HC_px_point, 3), 'px (+-)', round(CV_std_mae_HC_px_point, 3), 'px')
    print('5CV mae HC(px) in contour w/o pp:', round(CV_mean_mae_HC_px_contour, 3), 'px (+-)', round(CV_std_mae_HC_px_contour, 3), 'px')
    print('5CV mae HC(px) in ellipse w/o pp:', round(CV_mean_mae_HC_px_ellipse, 3), 'px (+-)', round(CV_std_mae_HC_px_ellipse, 3), 'px')
    print('5CV mae HC(px) in points w pp:', round(CV_mean_mae_HC_px_point_pp, 3), 'px (+-)',
          round(CV_std_mae_HC_px_point_pp, 3), 'px')
    print('5CV mae HC(px) in contour w pp:', round(CV_mean_mae_HC_px_contour_pp, 3), 'px (+-)',
          round(CV_std_mae_HC_px_contour_pp, 3), 'px')
    print('5CV mae HC(px) in ellipse w pp:', round(CV_mean_mae_HC_px_ellipse_pp, 3), 'px (+-)',
          round(CV_std_mae_HC_px_ellipse_pp, 3), 'px')

    print('5CV mae HC(mm) in points  w/o pp:', round(CV_mean_mae_HC_mm_point, 3), 'mm (+-)', round(CV_std_mae_HC_mm_point, 3), 'mm')
    print('5CV mae HC(mm) in contour w/o pp:', round(CV_mean_mae_HC_mm_contour, 3), 'mm (+-)', round(CV_std_mae_HC_mm_contour, 3), 'mm')
    print('5CV mae HC(mm) in ellipse w/o pp:', round(CV_mean_mae_HC_mm_ellipse, 3), 'mm (+-)', round(CV_std_mae_HC_mm_ellipse, 3), 'mm')
    print('5CV mae HC(mm) in points w pp:', round(CV_mean_mae_HC_mm_point_pp, 3), 'mm (+-)',
          round(CV_std_mae_HC_mm_point_pp, 3), 'mm')
    print('5CV mae HC(mm) in contour w pp:', round(CV_mean_mae_HC_mm_contour_pp, 3), 'mm (+-)',
          round(CV_std_mae_HC_mm_contour_pp, 3), 'mm')
    print('5CV mae HC(mm) in ellipse w pp:', round(CV_mean_mae_HC_mm_ellipse_pp, 3), 'mm (+-)',
          round(CV_std_mae_HC_mm_ellipse_pp, 3), 'mm')

    print('5CV pmae HC in points  w/o pp:', round(CV_pmae_mean_HC_point, 3), '% (+-)', round(CV_pmae_std_HC_point, 3))
    print('5CV pmae HC in contour w/o pp:', round(CV_pmae_mean_HC_contour, 3), '% (+-)', round(CV_pmae_std_HC_contour, 3))
    print('5CV pmae HC in ellipse w/o pp:', round(CV_pmae_mean_HC_ellipse, 3), '% (+-)', round(CV_pmae_std_HC_ellipse, 3))
    print('5CV pmae HC in points  w pp:', round(CV_pmae_mean_HC_point_pp, 3), '% (+-)', round(CV_pmae_std_HC_point_pp, 3))
    print('5CV pmae HC in contour w pp:', round(CV_pmae_mean_HC_contour_pp, 3), '% (+-)', round(CV_pmae_std_HC_contour_pp, 3))
    print('5CV pmae HC in ellipse w pp:', round(CV_pmae_mean_HC_ellipse_pp, 3), '% (+-)', round(CV_pmae_std_HC_ellipse_pp, 3))

def Dice_loss(y_true, y_pred):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    return 1 - (2. * intersection) / (K.sum(y_true_f) + K.sum(y_pred_f))

def Kappa_loss(y_true, y_pred, N=224 * 224):
    Gi = K.flatten(y_true)
    Pi = K.flatten(y_pred)
    numerator = 2 * K.sum(Pi * Gi) - K.sum(Pi) * K.sum(Gi) / N
    denominator = K.sum(Pi * Pi) + K.sum(Gi * Gi) - 2 * K.sum(Pi * Gi) / N
    Kappa = 1 - numerator / denominator
    return Kappa


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 27 12:55:40 2017

@author: elena
"""

import collections
# import token
import os
import pickle
import random
# import utils_nlp
import time

import numpy as np
import sklearn.preprocessing

from examples.Cliner.CliNER.code import helper_dataset

def lists_to_dataset_structure(sentences_tokens, sentence_tags,
        total_token_counter, token_count, label_count,
        character_count):
    labels = []
    tokens = []
    new_label_sequence = []
    new_token_sequence = []

    features = ""
    feature_file_name = ""
    feature_vector_size = 0

    for idx, sentence in enumerate(sentences_tokens):
        for token_idx, token_i in enumerate(sentence):
            new_token_sequence.append(token_i)
            new_label_sequence.append(sentence_tags[idx][token_idx])

            token_count[token_i] += 1
            label_count[sentence_tags[idx][token_idx]] += 1

            if token_idx == len(sentence) - 1:
                labels.append(new_label_sequence)
                tokens.append(new_token_sequence)
                new_token_sequence = []
                new_label_sequence = []
            # FEATURES ARE NOT SUPPORTED:  Can be done if we are getting a \
            # third list that looks like [[f1,f2,f3],[f1,f2,f3]... for each \
            # token]
            token_features = []
            features_as_array = np.array(token_features,
                dtype=np.dtype('int32'))
            features_as_array = features_as_array.reshape(
                (features_as_array.shape[0], 1))
            features_as_array = np.transpose(features_as_array)

            features = ""
            feature_file_name = ""
            feature_vector_size = 0

            total_token_counter += 1
            for character in token_i:
                character_count[character] += 1

    return labels, tokens, token_count, label_count, character_count, \
           features, feature_file_name, feature_vector_size

class Dataset():
    """A class for handling data sets."""

    def __init__(self, name='', verbose=False, debug=False):
        self.name = name
        self.verbose = verbose
        self.debug = debug

    def _parse_dataset(self, sentences_list, tags_list, Not_here=False):

        token_count = collections.defaultdict(lambda: 0)
        label_count = collections.defaultdict(lambda: 0)
        character_count = collections.defaultdict(lambda: 0)

        total_token_counter = 0

        tokens = []
        labels = []
        features = []

        if not Not_here:
            labels, tokens, token_count, label_count, character_count, \
            features, feature_file_name, feature_vector_size = \
                lists_to_dataset_structure(
                    sentences_list, tags_list, total_token_counter,
                    token_count,
                    label_count, character_count)

        return labels, tokens, token_count, label_count, character_count, \
               features, feature_file_name, feature_vector_size

    def _convert_to_indices(self, dataset_types):
        # Frank and Jennies Function

        tokens = self.tokens
        labels = self.labels
        token_to_index = self.token_to_index
        character_to_index = self.character_to_index
        label_to_index = self.label_to_index
        index_to_label = self.index_to_label

        # Map tokens and labels to their indices
        token_indices = {}
        label_indices = {}
        characters = {}
        token_lengths = {}
        character_indices = {}
        character_indices_padded = {}
        for dataset_type in dataset_types:
            print(dataset_type)
            token_indices[dataset_type] = []
            characters[dataset_type] = []
            character_indices[dataset_type] = []
            token_lengths[dataset_type] = []
            character_indices_padded[dataset_type] = []

            for token_sequence in tokens[dataset_type]:
                token_indices[dataset_type].append([
                    token_to_index.get(token, self.UNK_TOKEN_INDEX)
                    for token in token_sequence
                ])
                characters[dataset_type].append(
                    [list(token) for token in token_sequence])
                character_indices[dataset_type].append([[
                    character_to_index.get(
                        character,
                        random.randint(1, max(self.index_to_character.keys())))
                    for character in token
                ] for token in token_sequence])
                token_lengths[dataset_type].append(
                    [len(token) for token in token_sequence])
                longest_token_length_in_sequence = \
                    max(token_lengths[dataset_type][-1])
                character_indices_padded[dataset_type].append([
                    helper_dataset.pad_list(temp_token_indices,
                        longest_token_length_in_sequence,
                        self.PADDING_CHARACTER_INDEX) for
                    temp_token_indices in character_indices[dataset_type][-1]
                ])

            label_indices[dataset_type] = []
            for label_sequence in labels[dataset_type]:
                label_indices[dataset_type].append(
                    [label_to_index[label] for label in label_sequence])

        label_binarizer = sklearn.preprocessing.LabelBinarizer()
        label_binarizer.fit(range(max(index_to_label.keys()) + 1))
        label_vector_indices = {}
        for dataset_type in dataset_types:
            label_vector_indices[dataset_type] = []
            for label_indices_sequence in label_indices[dataset_type]:
                label_vector_indices[dataset_type].append(
                    label_binarizer.transform(label_indices_sequence))

        return token_indices, label_indices, character_indices_padded, \
               character_indices, token_lengths, characters, \
               label_vector_indices

    def update_dataset(self, dataset_types, Datasets_tokens, Datasets_labels):
        """
        dataset_filepaths : dictionary with keys 'train', 'valid', 'test', \
        'deploy'
        Overwrites the data of type specified in dataset_types using the \
        existing token_to_index, character_to_index, and label_to_index
        mappings.
        """

        # def _parse_dataset(self, dataset_filepath, dataset_type, \
        # sentences_list=[],tags_list=[], Not_here=False):
        for dataset_type in dataset_types:
            print(dataset_type)
            self.labels[dataset_type], self.tokens[dataset_type], _, _, \
            _, _, _, _ = self._parse_dataset(
                Datasets_tokens[
                    dataset_type],
                Datasets_labels[
                    dataset_type])

        token_indices, label_indices, character_indices_padded, \
        character_indices, token_lengths, characters, label_vector_indices \
            = self._convert_to_indices(
            dataset_types)

        self.token_indices.update(token_indices)
        self.label_indices.update(label_indices)
        self.character_indices_padded.update(character_indices_padded)
        self.character_indices.update(character_indices)
        self.token_lengths.update(token_lengths)
        self.characters.update(characters)
        self.label_vector_indices.update(label_vector_indices)

    def load_dataset(self,
            avaliable_datasets_sent,
            avaliable_datasets_labels,
            parameters,
            token_to_vector=None,
            pretrained_dataset=None):
        """
        dataset_filepaths : dictionary with keys 'train', 'valid', 'test',
        'deploy'
        """
        start_time = time.time()
        print('Load dataset... \n')
        if parameters['token_pretrained_embedding_filepath'] != '':
            if not token_to_vector:
                token_to_vector = \
                    helper_dataset.load_pretrained_token_embeddings(
                    parameters)
        else:
            token_to_vector = {}

        all_tokens_in_pretraining_dataset = []
        all_characters_in_pretraining_dataset = []

        if parameters['use_pretrained_model']:

            if not pretrained_dataset:
                temp_pretrained_dataset_adress = \
                    parameters['model_folder'] + os.sep + "dataset.pickle"
                pretraining_dataset = \
                    pickle.load(open(temp_pretrained_dataset_adress, "rb"))
                print("Pre-loading Pre-trained dataset objects")
            else:
                pretraining_dataset = pretrained_dataset
                print("Pretrained dataset was pre-loaded")

            all_tokens_in_pretraining_dataset = \
                pretraining_dataset.index_to_token.values()
            all_characters_in_pretraining_dataset = \
                pretraining_dataset.index_to_character.values()

        remap_to_unk_count_threshold = 1
        # pylint: disable=attribute-defined-outside-init
        self.UNK_TOKEN_INDEX = 0
        self.PADDING_CHARACTER_INDEX = 0
        self.tokens_mapped_to_unk = []
        self.UNK = 'UNK'
        self.unique_labels = []
        labels = {}
        tokens = {}
        label_count = {}
        token_count = {}
        character_count = {}

        features = {}
        features_file_names = {}
        feature_vector_size = {}
        # deploy

        for dataset_type in ['train', 'valid', 'test', 'deploy']:
            Not_here = False

            if dataset_type not in avaliable_datasets_sent:
                Not_here = True
            if not Not_here:
                labels[dataset_type], tokens[dataset_type], \
                token_count[dataset_type], label_count[dataset_type], \
                character_count[dataset_type], features[dataset_type], \
                features_file_names[dataset_type], \
                feature_vector_size[dataset_type] \
                    = self._parse_dataset(
                    sentences_list=avaliable_datasets_sent
                    [dataset_type],
                    tags_list=avaliable_datasets_labels[
                        dataset_type])

            if Not_here:
                labels[dataset_type], tokens[dataset_type], \
                token_count[dataset_type], label_count[dataset_type], \
                character_count[dataset_type], features[dataset_type], \
                features_file_names[dataset_type], \
                feature_vector_size[dataset_type] \
                    = self._parse_dataset(sentences_list=[], tags_list=[])

        token_count['all'] = {}
        for token in list(token_count['train'].keys()) + \
                     list(token_count['valid'].keys()) + list(
            token_count['test'].keys()) + \
                     list(token_count['deploy'].keys()):
            token_count['all'][token] = token_count['train'][token] + \
                                        token_count['valid'][token] + \
                                        token_count['test'][token] \
                                        + token_count['deploy'][token]

        if parameters['load_all_pretrained_token_embeddings']:
            for token in token_to_vector:
                if token not in token_count['all']:
                    token_count['all'][token] = -1
                    token_count['train'][token] = -1
            for token in all_tokens_in_pretraining_dataset:
                if token not in token_count['all']:
                    token_count['all'][token] = -1
                    token_count['train'][token] = -1

        character_count['all'] = {}
        for character in list(character_count['train'].keys()) + \
                         list(character_count['valid'].keys()) + \
                         list(character_count['test'].keys()) + \
                         list(character_count['deploy'].keys()):
            character_count['all'][character] = \
                character_count['train'][character] + character_count['valid'][
                    character] + character_count['test'][character] + \
                character_count['deploy'][character]

        for character in all_characters_in_pretraining_dataset:
            if character not in character_count['all']:
                character_count['all'][character] = -1
                character_count['train'][character] = -1

        label_count['all'] = {}
        for character in list(label_count['train'].keys()) + \
                         list(label_count['valid'].keys()) + list(
            label_count['test'].keys()) + \
                         list(label_count['deploy'].keys()):
            label_count['all'][character] = \
                label_count['train'][character] + \
                label_count['valid'][character] + \
                label_count['test'][character] + label_count['deploy'][
                    character]

        token_count['all'] = helper_dataset.order_dictionary(token_count['all'],
            'value_key',
            reverse=True)
        label_count['all'] = helper_dataset.order_dictionary(label_count['all'],
            'key',
            reverse=False)
        character_count['all'] = helper_dataset.order_dictionary(
            character_count['all'],
            'value',
            reverse=True)
        if self.verbose:
            print('character_count[\'all\']: {0}'.format(
                character_count['all']))

        token_to_index = {}
        token_to_index[self.UNK] = self.UNK_TOKEN_INDEX
        iteration_number = 0
        number_of_unknown_tokens = 0
        if self.verbose:
            print("parameters['remap_unknown_tokens_to_unk']: {0}".format(
                parameters['remap_unknown_tokens_to_unk']))
        if self.verbose:
            print("len(token_count['train'].keys()): {0}".format(
                len(token_count['train'].keys())))
        for token, count in token_count['all'].items():
            if iteration_number == self.UNK_TOKEN_INDEX:
                iteration_number += 1

            # pylint: disable=too-many-function-args
            if parameters['remap_unknown_tokens_to_unk'] == 1 and \
                    (token_count['train'][token] == 0 or
                     parameters['load_only_pretrained_token_embeddings']) and \
                    not helper_dataset.is_token_in_pretrained_embeddings(token,
                        token_to_vector,
                        parameters) and \
                    token not in all_tokens_in_pretraining_dataset:
                token_to_index[token] = self.UNK_TOKEN_INDEX
                number_of_unknown_tokens += 1
                self.tokens_mapped_to_unk.append(token)
            else:
                token_to_index[token] = iteration_number
                iteration_number += 1

        infrequent_token_indices = []
        for token, count in token_count['train'].items():
            if 0 < count <= remap_to_unk_count_threshold:
                infrequent_token_indices.append(token_to_index[token])

        labels_without_bio = set()
        for label in label_count['all'].keys():
            new_label = helper_dataset.remove_bio_from_label_name(label)
            labels_without_bio.add(new_label)
        for label in labels_without_bio:
            if label == 'O':
                continue
            if parameters['tagging_format'] == 'bioes':
                prefixes = ['B-', 'I-', 'E-', 'S-']
            else:
                prefixes = ['B-', 'I-']
            for prefix in prefixes:
                l = prefix + label
                if l not in label_count['all']:
                    label_count['all'][l] = 0
        label_count['all'] = helper_dataset.order_dictionary(label_count['all'],
            'key',
            reverse=False)

        if parameters['use_pretrained_model']:

            print("USE_PRETRAINED_MODEL ACTIVE")
            # pylint: disable=attribute-defined-outside-init
            self.unique_labels = sorted(list(
                pretraining_dataset.label_to_index.keys()))
            # Make sure labels are compatible with the pretraining dataset.
            for label in label_count['all']:
                if label not in pretraining_dataset.label_to_index:
                    raise AssertionError(
                        "The label {0} does not exist in"
                        "the pretraining dataset. ".format(label) +
                        "Please ensure that only the"
                        "following labels exist in the dataset: {0}".format(
                            ', '.join(self.unique_labels)))
        else:
            label_to_index = {}
            iteration_number = 0
            for label, count in label_count['all'].items():
                label_to_index[label] = iteration_number
                iteration_number += 1
                self.unique_labels.append(label)

        character_to_index = {}
        iteration_number = 0
        for character, count in character_count['all'].items():
            if iteration_number == self.PADDING_CHARACTER_INDEX:
                iteration_number += 1
            character_to_index[character] = iteration_number
            iteration_number += 1

        token_to_index = helper_dataset.order_dictionary(token_to_index,
            'value',
            reverse=False)
        if self.verbose:
            print('token_to_index: {0}'.format(token_to_index))
        index_to_token = helper_dataset.reverse_dictionary(token_to_index)
        if parameters['remap_unknown_tokens_to_unk'] == 1:
            index_to_token[self.UNK_TOKEN_INDEX] = self.UNK
        if self.verbose:
            print('index_to_token: {0}'.format(index_to_token))

        label_to_index = helper_dataset.order_dictionary(label_to_index,
            'value',
            reverse=False)
        index_to_label = helper_dataset.reverse_dictionary(label_to_index)

        character_to_index = helper_dataset.order_dictionary(character_to_index,
            'value',
            reverse=False)
        index_to_character = helper_dataset.reverse_dictionary(
            character_to_index)
        # pylint: disable=attribute-defined-outside-init
        self.token_to_index = token_to_index
        self.index_to_token = index_to_token
        self.index_to_character = index_to_character
        self.character_to_index = character_to_index
        self.index_to_label = index_to_label
        self.label_to_index = label_to_index
        self.tokens = tokens
        self.labels = labels

        dataset_types = ['train', 'test', 'valid', 'deploy']
        token_indices, label_indices, character_indices_padded, \
        character_indices, token_lengths, characters, label_vector_indices \
            = self._convert_to_indices(
            dataset_types)

        self.token_indices = token_indices
        self.label_indices = label_indices
        self.character_indices_padded = character_indices_padded
        self.character_indices = character_indices
        self.token_lengths = token_lengths
        self.characters = characters
        self.label_vector_indices = label_vector_indices

        self.number_of_classes = max(self.index_to_label.keys()) + 1
        self.vocabulary_size = max(self.index_to_token.keys()) + 1
        self.alphabet_size = max(self.index_to_character.keys()) + 1

        # unique_labels_of_interest is used to compute F1-scores.
        self.unique_labels_of_interest = list(self.unique_labels)
        self.unique_labels_of_interest.remove('O')

        self.unique_label_indices_of_interest = []
        for lab in self.unique_labels_of_interest:
            self.unique_label_indices_of_interest.append(label_to_index[lab])

        self.infrequent_token_indices = infrequent_token_indices

        elapsed_time = time.time() - start_time
        print('done ({0:.2f} seconds)'.format(elapsed_time))

        self.feature_vector_size = 0

        return token_to_vector

max_stars_repo_name: DianLiI/forte-wrappers

max_issues_repo_name: DianLiI/forte-wrappers

max_forks_repo_name: DianLiI/forte-wrappers