text:
import os.path
import os

from data.base_dataset import BaseDataset, get_params, get_transform
from data.image_folder import make_dataset
from PIL import Image
import skimage
import numpy as np
import cv2
import argparse
from skimage.color import xyz2rgb
from skimage.color import xyz2lab
from skimage.color import lab2rgb
from skimage.color import xyz2rgb
from PIL.PngImagePlugin import PngImageFile, PngInfo
import random
import torchvision.transforms as transforms
import torch
import copy

#MIDAS
import midas.utils
from midas.models.midas_net import MidasNet
from midas.models.transforms import Resize, NormalizeImage, PrepareForNet
from torchvision.transforms import Compose
from depthmerge.options.test_options import TestOptions
from depthmerge.models.pix2pix4depth_model import Pix2Pix4DepthModel

import matplotlib.pyplot as plt
def showImage(img,title=None):
    plt.imshow(img, cmap= 'inferno')
    plt.colorbar()
    if title is not None:
        plt.title(title)
    plt.show()

class SNETDataset(BaseDataset):

    def __init__(self, opt):

        BaseDataset.__init__(self, opt)

        # 10000 is the max dataset size
        if opt.phase == 'test':
            self.dir_ourdataset = os.path.join(opt.dataroot,'our_dataset2_test')
            self.images_dir_ourdataset = sorted(make_dataset(self.dir_ourdataset, 100000 ))

            self.dir_multidataset = os.path.join(opt.dataroot,'multi_dataset_test')
            self.images_dir_multidataset = sorted(make_dataset(self.dir_multidataset, 100000 ))

            self.images_dir_portraitdataset = []
        else:
            self.dir_ourdataset = os.path.join(opt.dataroot,'our_dataset2')
            self.images_dir_ourdataset = sorted(make_dataset(self.dir_ourdataset, 100000 ))

            self.dir_multidataset = os.path.join(opt.dataroot,'multi_dataset')
            self.images_dir_multidataset = sorted(make_dataset(self.dir_multidataset, 100000 ))

            self.dir_portraitdataset = os.path.join(opt.dataroot, 'portrait_dataset_extra')
            self.images_dir_portraitdataset = sorted(make_dataset(self.dir_portraitdataset, 100000))

        self.images_dir_all = self.images_dir_ourdataset + self.images_dir_multidataset + self.images_dir_portraitdataset
        # self.images_dir_all = self.images_dir_multidataset

        self.data_size = opt.load_size
        self.data_root = opt.dataroot

        for i in range(len(self.images_dir_all)):
            self.__getitem__(i)

        exit()

    def __getitem__(self, index):
        image_path = self.images_dir_all[index]
        print("process and save",image_path)

        if 'our_dataset' in image_path:
            image_pair = Image.open(image_path)
            hyper_des = int(PngImageFile(image_path).text['des'])

            A,B = self.divide_imagepair(image_pair)
            ambient = skimage.img_as_float(A)
            flash = skimage.img_as_float(B)

            if hyper_des == 21:
                flash = self.changeTemp(flash, 48, hyper_des)
                ambient = self.changeTemp(ambient, 48, hyper_des)
            flashPhoto = flash + ambient
            flashPhoto[flashPhoto>1]=1
            flashPhoto[flashPhoto<0]=0

            flashPhoto = self.xyztorgb(flashPhoto, hyper_des)
            ambient = self.xyztorgb(ambient, hyper_des)

            flashphoto_depth = self.getDepth(flashPhoto,image_path,'flash')
            ambient_depth = self.getDepth(ambient,image_path,'ambient')

        elif 'multi_dataset' in image_path:
            image_pair = Image.open(image_path)

            A, B = self.divide_imagepair(image_pair)
            ambient = skimage.img_as_float(A)
            flash = skimage.img_as_float(B)

            flashPhoto = flash + ambient
            flashPhoto[flashPhoto>1]=1
            flashPhoto[flashPhoto<0]=0

            ambient = Image.fromarray((ambient*255).astype('uint8'))
            flashPhoto = Image.fromarray((flashPhoto*255).astype('uint8'))

            flashphoto_depth = self.getDepth(flashPhoto, image_path, 'flash')
            ambient_depth = self.getDepth(ambient, image_path, 'ambient')

        elif 'portrait_dataset_extra' in image_path:
            image_pair = Image.open(image_path)

            A, B = self.divide_imagepair(image_pair)
            ambient = skimage.img_as_float(A)
            flash = skimage.img_as_float(B)

            ambient = self.lin(ambient)
            flash = self.lin(flash)

            flashPhoto = flash + ambient
            flashPhoto[flashPhoto>1]=1
            flashPhoto[flashPhoto<0]=0

            ambient = Image.fromarray((ambient * 255).astype('uint8'))
            flashPhoto = Image.fromarray((flashPhoto * 255).astype('uint8'))

            flashphoto_depth = self.getDepth(flashPhoto, image_path, 'flash')
            ambient_depth = self.getDepth(ambient, image_path, 'ambient')

        torch.cuda.empty_cache()

        ambient_orgsize = skimage.img_as_float(ambient)
        flashPhoto_orgsize = skimage.img_as_float(flashPhoto)

        ambient = ambient.resize((self.data_size, self.data_size))
        flashPhoto = flashPhoto.resize((self.data_size, self.data_size))
        ambient_depth = ambient_depth.resize((self.data_size, self.data_size))
        flashphoto_depth = flashphoto_depth.resize((self.data_size, self.data_size))

        transform_params = get_params(self.opt, ambient.size)
        rgb_transform = get_transform(self.opt, transform_params, grayscale=False)
        depth_transform = get_transform(self.opt, transform_params, grayscale=True)

        image_path_beheaded = image_path.replace('.png', '')
        image_path_beheaded = image_path_beheaded.replace('.jpg', '')
        image_path_beheaded = image_path_beheaded.replace('.jpeg', '')

        ambient_path = image_path_beheaded[:len(self.data_root)] + '/postprocessed' + image_path_beheaded[
                                                                            len(self.data_root):] + '_ambient' + '.png'
        flash_path = image_path_beheaded[:len(self.data_root)] + '/postprocessed' + image_path_beheaded[
                                                                              len(
                                                                                self.data_root):] + '_flash' + '.png'
        dir = '/'
        dir = dir.join(flash_path.split('/')[0:-1])
        if not os.path.exists(dir):
            os.makedirs(dir)

        ambient.save(ambient_path)
        flashPhoto.save(flash_path)

        # ambient = rgb_transform(ambient)
        # flashPhoto = rgb_transform(flashPhoto)
        #
        # ambient_depth = depth_transform(ambient_depth)
        # flashphoto_depth = depth_transform(flashphoto_depth)
        #
        # return {'A': flashPhoto, 'B': ambient,'A_org':flashPhoto_orgsize,'B_org':ambient_orgsize, 'depth_A': flashphoto_depth, 'depth_B': ambient_depth, 'A_paths': image_path, 'B_paths': image_path}

    def __len__(self):
        """Return the total number of images in the dataset."""
        return len(self.images_dir_all)

    def getDepth(self,image,image_path,id):
        image_path_beheaded = image_path.replace('.png','')
        image_path_beheaded = image_path_beheaded.replace('.jpg','')
        image_path_beheaded = image_path_beheaded.replace('.jpeg','')

        depth_path = image_path_beheaded[:len(self.data_root)]+'/depth'+image_path_beheaded[len(self.data_root):] + '_' + id +'.png'
        if os.path.exists(depth_path):
            depth = Image.open(depth_path)
        else:
            depth = self.estimateDepth(np.asarray(image)/255)
            depth_dir = depth_path.replace(depth_path.split('/')[-1],'')
            if not os.path.exists(depth_dir):
                os.makedirs(depth_dir)
            depth = (depth*255).astype('uint8')
            cv2.imwrite(depth_path, depth)
            depth = Image.fromarray(depth)
            # depth.save(depth_path)
            print('Depth file cached |',depth_path)

        return depth

    def divide_imagepair(self,image_pair):
        w, h = image_pair.size
        w2 = int(w / 2)
        A = image_pair.crop((0, 0, w2, h))
        B = image_pair.crop((w2, 0, w, h))
        return A,B

    def gama_corect(self, rgb):
        srgb = np.zeros_like(rgb)
        mask1 = (rgb > 0) * (rgb < 0.0031308)
        mask2 = (1 - mask1).astype(bool)
        srgb[mask1] = 12.92 * rgb[mask1]
        srgb[mask2] = 1.055 * np.power(rgb[mask2], 0.41666) - 0.055
        srgb[srgb < 0] = 0
        return srgb

    def doubleestimate(self, img, size1, size2):
        estimate1 = self.singleestimate(img, size1)
        estimate1 = cv2.resize(estimate1, (1024, 1024), interpolation=cv2.INTER_CUBIC)

        estimate2 = self.singleestimate(img, size2)
        estimate2 = cv2.resize(estimate2, (1024, 1024), interpolation=cv2.INTER_CUBIC)

        self.mergenet.set_input(estimate1, estimate2)
        self.mergenet.test()
        torch.cuda.empty_cache()
        visuals = self.mergenet.get_current_visuals()
        prediction_mapped = visuals['fake_B']
        prediction_mapped = (prediction_mapped + 1) / 2
        prediction_mapped = (prediction_mapped - torch.min(prediction_mapped)) / (
                torch.max(prediction_mapped) - torch.min(prediction_mapped))
        prediction_mapped = prediction_mapped.squeeze().cpu().numpy()

        prediction_end_res = cv2.resize(prediction_mapped, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_CUBIC)

        return prediction_end_res

    def singleestimate(self, img, msize):
        return self.estimateMidas(img, msize)

    def estimateMidas(self, img, msize):
        transform = Compose(
            [
                Resize(
                    msize,
                    msize,
                    resize_target=None,
                    keep_aspect_ratio=True,
                    ensure_multiple_of=32,
                    resize_method="upper_bound",
                    image_interpolation_method=cv2.INTER_CUBIC,
                ),
                NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
                PrepareForNet(),
            ]
        )

        img_input = transform({"image": img})["image"]
        # compute
        with torch.no_grad():
            sample = torch.from_numpy(img_input).to(self.device).unsqueeze(0)
            prediction = self.midasmodel.forward(sample)
            torch.cuda.empty_cache()

        prediction = prediction.squeeze().cpu().numpy()
        prediction = cv2.resize(prediction, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_CUBIC)

        depth_min = prediction.min()
        depth_max = prediction.max()

        if depth_max - depth_min > np.finfo("float").eps:
            prediction = (prediction - depth_min) / (depth_max - depth_min)
        else:
            prediction = 0

        return prediction

    def estimateDepth(self, rgb_mix):
        rgb_mix = self.gama_corect(rgb_mix)
        depth_temp = self.doubleestimate(rgb_mix, 384, 768)
        return depth_temp

    def xyztorgb(self, image, des):
        illum = self.chromaticityAdaptation(des)
        mat = [[3.2404542, -0.9692660, 0.0556434], [-1.5371385, 1.8760108, -0.2040259],
               [-0.4985314, 0.0415560, 1.0572252]]
        image = np.matmul(image, illum)
        image = np.matmul(image, mat)
        image = np.where(image < 0, 0, image)
        image = np.where(image > 1, 1, image)
        out = (image * 255).astype('uint8')
        out = Image.fromarray(out)
        return out

    def chromaticityAdaptation(self, calibrationIlluminant):
        if (calibrationIlluminant == 17):
            illum = [[0.8652435, 0.0000000, 0.0000000],
                     [0.0000000, 1.0000000, 0.0000000],
                     [0.0000000, 0.0000000, 3.0598005]]
        elif (calibrationIlluminant == 19):
            illum = [[0.9691356, 0.0000000, 0.0000000],
                     [0.0000000, 1.0000000, 0.0000000],
                     [0.0000000, 0.0000000, 0.9209267]]
        elif (calibrationIlluminant == 20):
            illum = [[0.9933634, 0.0000000, 0.0000000],
                     [0.0000000, 1.0000000, 0.0000000],
                     [0.0000000, 0.0000000, 1.1815972]]
        elif (calibrationIlluminant == 21):
            illum = [[1, 0, 0],
                     [0, 1, 0],
                     [0, 0, 1]]
        elif (calibrationIlluminant == 23):
            illum = [[1.0077340, 0.0000000, 0.0000000],
                     [0.0000000, 1.0000000, 0.0000000],
                     [0.0000000, 0.0000000, 0.8955170]]
        return illum

    def getRatio(self, t, low, high):
        dist = t - low
        range = (high - low) / 100
        return dist / range

    def changeTemp(self, image, tempChange, des):
        if (des == 17):
            t1 = 5500
            if tempChange == 44:
                tempChange = -400
            elif tempChange == 40:
                tempChange = - 450
            elif tempChange == 52:
                tempChange = 234
            elif tempChange == 54:
                tempChange = 468
            t = t1 + tempChange
            if t <= 10000 and t > 6500:
                r = self.getRatio(t, 6500, 10000)
                xD = 0.3
                yD = 0.3
                xS = 0.3118
                yS = 0.3224
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y

            elif 6500 >= t and t > 5500:
                r = self.getRatio(t, 5500, 6500)
                xD = 0.3118
                yD = 0.3224
                xS = 0.3580
                yS = 0.3239
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y

            elif 5000 <= t and t < 5500:
                r = self.getRatio(t, 5500, 5000)
                xD = 0.3752
                yD = 0.3238
                xS = 0.3580
                yS = 0.3239
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y

            elif (t >= 4500 and t < 5000):
                r = self.getRatio(t, 5000, 4500)
                xD = 0.4231
                yD = 0.3304
                xS = 0.3752
                yS = 0.3238
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y

            elif t < 4500 and t >= 4000:
                r = self.getRatio(t, 4500, 4000)
                xD = 0.4949
                yD = 0.3564
                xS = 0.4231
                yS = 0.3304
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y

            elif t >= 0 and t < 4000:
                r = self.getRatio(t, 4000, 0)
                xD = 0.5041
                yD = 0.3334
                xS = 0.4949
                yS = 0.3564
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y

            chromaticity_x = 0.3580
            chromaticity_y = 0.3239
        elif des == 21:
            t1 = 4500
            if tempChange == 48:
                tempChange = 700
            if tempChange == 44:
                tempChange = 400
            elif tempChange == 40:
                tempChange = 250
            elif tempChange == 52:
                tempChange = 800
            elif tempChange == 54:
                tempChange = 1000

            t = tempChange + t1
            if t >= 4500 and t <= 7500:
                r = self.getRatio(t, 4500, 7500)
                xD = 0.17
                yD = 0.17
                xS = 0.4231
                yS = 0.3304
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y

            elif (t < 4500 and t >= 4000):
                r = self.getRatio(t, 4500, 4000)
                xD = 0.4949
                yD = 0.3564
                xS = 0.4231
                yS = 0.3304
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y

            elif (t >= 3500 and t < 4000):
                r = self.getRatio(t, 4000, 3500)
                xD = 0.5141
                yD = 0.3434
                xS = 0.4949
                yS = 0.3564
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y
            elif (t > 0 and t < 3500):
                r = self.getRatio(t, 3500, 0)
                xD = 0.5189
                yD = 0.3063
                xS = 0.5141
                yS = 0.3434
                r_x = (xD - xS) / 100
                xD = xS + r * r_x
                r_y = (yD - yS) / 100
                yD = yS + r * r_y

            chromaticity_x = 0.4231

            chromaticity_y = 0.3304

        offset_x = xD / chromaticity_x
        offset_y = yD / chromaticity_y

        out = image
        h, w, c = image.shape
        img0 = image[:, :, 0]
        img1 = image[:, :, 1]
        img2 = image[:, :, 2]
        sumImage = img0 + img1 + img2
        x_pix = np.zeros((h, w))
        y_pix = np.zeros((h, w))

        nonZeroSum = np.where(sumImage != 0)
        x_pix[nonZeroSum] = img0[nonZeroSum] / sumImage[nonZeroSum]
        y_pix[nonZeroSum] = img1[nonZeroSum] / sumImage[nonZeroSum]

        x_pix = x_pix * offset_x
        y_pix = y_pix * offset_y

        out0 = np.zeros((h, w))
        out2 = np.zeros((h, w))

        nonZeroY = np.where(y_pix != 0)
        ones = np.ones((h, w))
        out0[nonZeroY] = x_pix[nonZeroY] * img1[nonZeroY] / y_pix[nonZeroY]
        out2[nonZeroY] = (ones[nonZeroY] - x_pix[nonZeroY] - y_pix[nonZeroY]) * img1[nonZeroY] / y_pix[nonZeroY]
        out[:, :, 0] = out0
        out[:, :, 2] = out2

        return out

    def lin(self, srgb):
        srgb = srgb.astype(np.float)
        rgb = np.zeros_like(srgb).astype(np.float)
        srgb = srgb
        mask1 = srgb <= 0.04045
        mask2 = (1 - mask1).astype(bool)
        rgb[mask1] = srgb[mask1] / 12.92
        rgb[mask2] = ((srgb[mask2] + 0.055) / 1.055) ** 2.4
        rgb = rgb
        return rgb

max_stars_repo_name: sepideh-srj/flashNoflash

max_issues_repo_name: sepideh-srj/flashNoflash

max_forks_repo_name: sepideh-srj/flashNoflash


text:
import os
import time
import numpy as np
import asap3
from asap3.mpi import world
from asap3.io.trajectory import _GetAtoms
try:
    from ase.io.bundletrajectory import BundleTrajectory as _BundleTrajectory
    from ase.io.bundletrajectory import PickleBundleBackend
except ImportError:
    class _BundleTrajectory:
        "This version of ASE does not support BundleTrajectory."
        def __init__(self, *args, **kwargs):
            raise NotImplementedError(self.__doc__)
    class PickleBundleBackend:
        "This version of ASE does not support BundleTrajectory."
        def __init__(self, *args, **kwargs):
            raise NotImplementedError(self.__doc__)

class BundleTrajectory(_BundleTrajectory, _GetAtoms):
    """Reads and writes atoms into a .bundle directory.

    The BundleTrajectory is an alternative way of storing
    trajectories, intended for large-scale molecular dynamics
    simulations, where a single flat file becomes unwieldy.  Instead,
    the data is stored in directory, a 'bundle' (the name bundle is
    inspired from bundles in Mac OS, which are really just directories
    the user is supposed to think of as a single file-like unit).

    Parameters:

    filename:
        The name of the directory.  Preferably ending in .bundle.

    mode (optional):
        The file opening mode.  'r' means open for reading, 'w' for
        writing and 'a' for appending.  Default: 'r'.  If opening in
        write mode, and the filename already exists, the old file is
        renamed to .bak (any old .bak file is deleted), except if the
        existing file is empty.

    atoms (optional):
        The atoms that will be written.  Can only be specified in
        write or append mode.  If not specified, the atoms must be
        given as an argument to the .write() method instead.

    backup (optional):
        Use backup=False to disable renaming of an existing file.

    split (optional):
        If set to True or False, determines whether a split file
        format is used instead of the normal one.  In the split
        format, each processor in a parallel simulation writes its own
        files inside the BundleTrajectory, instead of leaving all I/O
        to the master.  In not specified, a split format is used if
        more than one million atoms.  Ignored in serial simulations.

    iolimit (optional):
        Limits the number of MPI tasks performing I/O simultaneously,
        to prevent overloading the NFS server.  Only enforced if the
        number of tasks is somewhat larger than the limit.

    """
    def __init__(self, filename, mode='r', atoms=None,
                 backup=True, split=None, iolimit=10):
        if split is None:
            # Decide if subtype should be split based on number of atoms
            split = (atoms is not None) and atoms.get_number_of_atoms() > 1000000
        # Never use subtype split for serial simulations
        if not getattr(atoms, "parallel", False):
            split = False
        # Use the collector object to join all data on master if subtype is normal
        # and the simulation is parallel.
        if not split and atoms is not None and getattr(atoms, "parallel", False):
            atoms = asap3.Collector(atoms)
        if split:
            self.subtype = 'split'
        else:
            self.subtype = 'normal'
        # self.iolimit may be needed if reading a split bundle.
        if world.size < 1.5 * iolimit:
            self.iolimit = None
        else:
            self.iolimit = iolimit

        _BundleTrajectory.__init__(self, filename, mode, atoms,
                                   backup=backup)
        if self.subtype == 'split':
            self.set_extra_data('ID')  # So the atoms can be sorted when read.

    def _set_defaults(self):
        subtype = self.subtype   # Preserve it
        _BundleTrajectory._set_defaults(self)
        self.subtype = subtype
        self.datatypes['forces'] = False

    def _set_backend(self, backend=None):
        """Set the backed doing the actual I/O."""
        if backend is not None:
            self.backend_name = backend
        if self.backend_name == 'pickle':
            if self.subtype == 'normal':
                # Use the standard ASE backend
                self.backend = PickleBundleBackend(self.master)
            elif self.subtype == 'split':
                self.backend = PickleSplitBundleBackend(self.master,
                                                        self.iolimit)
        else:
            raise NotImplementedError(
                "This version of ASE cannot use BundleTrajectory with backend '%s'"
                % self.backend_name)

    def write(self, atoms=None):
        if self.subtype == 'normal' and atoms is not None and getattr(atoms, "parallel", False):
            atoms = asap3.Collector(atoms)
        _BundleTrajectory.write(self, atoms)

    def _make_bundledir(self, filename):
        """Make the main bundle directory.

        Since all MPI tasks might write to it, all tasks must wait for
        the directory to appear.

        For performance reasons, the first frame directory is created immediately.
        """
        assert not os.path.isdir(filename)
        world.barrier()
        if self.master:
            self.log("Making directory "+filename)
            os.mkdir(filename)
            framedir = os.path.join(filename, "F0")
            self.log("Making directory "+ framedir)
            os.mkdir(framedir)
        else:
            i = 0
            while not os.path.isdir(filename):
                time.sleep(1)
                i += 1
            if i > 10:
                self.log("Waiting %d seconds for %s to appear!"
                         % (i, filename))

    def _make_framedir(self, frame):
        """Make subdirectory for the frame.

        For a split bundle, all MPI tasks write to the frame
        directory.  The slaves must therefore wait until it becomes
        available.  To minimize the waiting time, frames are
        pre-created.
        """
        if self.subtype == 'split':
            numdirs = 10
        else:
            numdirs = 1
        if self.master:
            for i in range(frame, frame+numdirs):
                framedir = os.path.join(self.filename, "F"+str(i))
                if not os.path.exists(framedir):
                    self.log("Making directory " + framedir)
                    os.mkdir(framedir)
        framedir = os.path.join(self.filename, "F"+str(frame))
        # Wait for the directory to appear
        i = 0
        while not os.path.isdir(framedir):
            time.sleep(1)
            i += 1
        if i > 10:
            self.log("Waiting %d seconds for %s to appear!"
                     % (i, framedir))
        return framedir

    def close(self):
        """Clean up when closing."""
        if self.state == 'write' and self.master:
            i = self.nframes
            while True:
                fname = os.path.join(self.filename, "F" + str(i))
                if not os.path.exists(fname):
                    break
                self.log("Closing, removing empty directory "+fname)
                os.rmdir(fname)
                i += 1
        _BundleTrajectory.close(self)

    def __del__(self):
        self.close()

class PickleSplitBundleBackend(PickleBundleBackend):
    """A special backend for writing split bundles (ASAP only)."""
    def __init__(self, master, iolimit):
        # Store if this backend will actually write anything
        self.writesmall = master
        self.writelarge = True
        self.writenonarray = master
        self.iolimit = iolimit
        if iolimit:
            self.iostart = np.round(
                np.linspace(0, world.size, iolimit+1)).astype(int)
            self.iotag = 413151
        self.lastwritedir = None

    def write_small(self, framedir, smalldata):
        "Write small data to be written jointly."
        smalldata['fragments'] = world.size
        PickleBundleBackend.write_small(self, framedir, smalldata)

    def write(self, framedir, name, data):
        "Write data to separate file."
        if hasattr(data, "shape"):
            # We need to store which kind of data was written in this frame
            # so NFS synchronization is possible when closing file.
            if framedir != self.lastwritedir:
                self.lastwritedir = framedir
                self.writenames = []
            self.writenames.append(name)
            # As expected, we are writing a NumPy array
            self.iosync_start()
            name = "%s_%d" % (name, world.rank)
            PickleBundleBackend.write(self, framedir, name, data)
            self.iosync_end()
        elif self.writenonarray:
            # If the data is not a NumPy array, only the master writes.
            PickleBundleBackend.write(self, framedir, name, data)

    def read(self, framedir, name):
        "Read data from separate file."
        self.iosync_start()
        x = PickleBundleBackend.read(self, framedir, name)
        self.iosync_end()
        return x

    def iosync_start(self):
        "Prevents too many simultaneous IO tasks from trashing server."
        if self.iolimit and world.rank not in self.iostart:
            # I must wait.
            token = np.zeros(1, int)
            world.receive(token, world.rank-1, self.iotag)

    def iosync_end(self):
        if self.iolimit and world.rank+1 not in self.iostart:
            # Another task is waiting for me.
            token = np.zeros(1, int)
            world.send(token, world.rank+1, self.iotag)

    def close(self, log=None):
        """Make sure that all data is available on disk for all MPI tasks."""
        if self.lastwritedir:
            for name in self.writenames:
                for part in range(world.size):
                    fname = os.path.join(self.lastwritedir, "%s_%d.pickle" % (name, part))
                    if not os.path.exists(fname):
                        if log:
                            log.write("Task %i is waiting for '%s' to appear.\n" %
                                      (world.rank, fname))
                        for i in range(20):
                            time.sleep(5)
                            if os.path.exists(fname):
                                break
                        if not os.path.exists(fname) and log:
                            log.write("WARNING: Task %i gave up waiting for '%s'.\n" %
                                      (world.rank, fname))

ext: py

reason: import numpy


text:
import utils
import librosa as lr
import pandas as pd
import numpy as np
import json
import os

"""

"""

# Path for source activations
SOURCEID_PATH = os.environ["MEDLEYDB_PATH"]+\
		"Annotations/Instrument_Activations/SOURCEID/"
AUDIO_PATH = os.environ["MEDLEYDB_PATH"]+"Audio/"
LABEL_PATH = os.environ["CODE_PATH"]+"Labels/"

# Same from MedleyDB API
VOCALS = ["male singer", "female singer", "male speaker", "female speaker",
          "male rapper", "female rapper", "beatboxing", "vocalists"]

# Process files to create label and save on a given path (adjusted for VGGish size of frame)
def save_labels(files):

    if not os.path.exists(LABEL_PATH):
        os.makedirs(LABEL_PATH)

    # Process musics
    for music in files:
        source_activation = pd.read_csv(SOURCEID_PATH + music +\
                                        "_SOURCEID.lab", index_col=None)

        y, sr = lr.load(AUDIO_PATH + music + "/" + music + "_MIX.wav", sr=None)
        duration = lr.get_duration(y, sr)

        # Get music duration in miliseconds (1000 transform into miliseconds)
        label_vector = np.zeros(int(duration*1000))

        for idx, source in source_activation.iterrows():

            if source.instrument_label in VOCALS:
                start, end = source.start_time, source.end_time
                label_vector[int(start*1000):int(end*1000)] = 1

        df = pd.DataFrame(label_vector.astype('int').T,columns=None,index=None)

        # Save vocal labels and copy the audio file to the right place
        df.to_csv(LABEL_PATH + music+"_vocal.csv", index=False, header=False)
        #os.system('cp ' + AUDIO_PATH + music + '/' + music + '_MIX.wav' \
        #          + ' ' + dir_path)

        print ('Generated vocals file for', music)

    print (" >> Vocal labels completed.")

if __name__ == "__main__":

    # Create a list of all musics
    music_files = []

    with open('vocal_pieces.json') as json_file:
        data = json.load(json_file)

        for music in data.keys():
            music_files.append(music)

    # Saving vocal labels
    save_labels(music_files)

max_stars_repo_name: shayenne/vocaldetection

max_issues_repo_name: shayenne/vocaldetection

max_forks_repo_name: shayenne/vocaldetection


text:
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import numpy as np
import tensorflow as tf
import tensorflow.keras.preprocessing.image as tf_image

class Plotter:
   FILE_GRID = "grid.svg"
   FILE_AUG_IMG = "aug_img.svg"
   FILE_AUG_MASK = "aug_mask.svg"

   def __init__(self, AE, dataset, sess):
      self.AE = AE
      self.dataset = dataset
      self.sess = sess

   # Plot Nx4 grid of examples from datagen
   def plot_grid(self, N, datagen):
      (sample_batch, mask_batch) = next(datagen)
      fig, ax = plt.subplots(N, 4)

      assert N <= len(sample_batch)

      # Combine prediction hard mask and ground truth
      def combine_img(pred, target):
         A = np.full((256, 256, 3), 0)   # Black output

         C1 = [255, 255, 255]   # 1 in target, 1 in pred
         C2 = [0, 192, 222]     # 1 in target, 0 in pred
         C3 = [255, 0, 0]       # 0 in target, 1 in pred

         for y in range(len(pred)):
            for x in range(len(pred)):
               if pred[y][x] == target[y][x] and pred[y][x] == 1:
                  A[y][x] = C1
               elif pred[y][x] != target[y][x] and pred[y][x] == 0:
                  A[y][x] = C2
               elif pred[y][x] != target[y][x] and pred[y][x] == 1:
                  A[y][x] = C3
         return A

      pred = self.sess.run(self.AE(sample_batch[:N], training=False))
      for i in range(N):
         img = sample_batch[i]
         target = mask_batch[i]

         # Input image
         ax[i,0].imshow(img)
         ax[i,0].set_xticks([])
         ax[i,0].set_yticks([])

         # Target mask
         ax[i,1].imshow(target, cmap="gray")
         ax[i,1].set_xticks([])
         ax[i,1].set_yticks([])

         # Predicted soft mask
         ax[i,2].imshow(pred[i], cmap="gray")
         ax[i,2].set_xticks([])
         ax[i,2].set_yticks([])

         # Prediction converted into hard mask
         hardmask = self.AE.to_hardmask(pred[i])
         combine = combine_img(hardmask, target)
         ax[i,3].imshow(combine)
         ax[i,3].set_xticks([])
         ax[i,3].set_yticks([])

      ax[0,0].set_title("Image")
      ax[0,1].set_title("Ground Truth")
      ax[0,2].set_title("Soft mask\nPrediction")
      ax[0,3].set_title("Hard mask\nPrediction")
      plt.show()
      #plt.savefig(self.FILE_GRID, bbox_inches="tight")

   # Plot NxN grid of different augmentations of the same image
   def plot_augmentation(self, N, img, mask):
      import random

      def generate_plot(N, tensor, seeds, filename, datagen, cmap=None):
         fig, ax = plt.subplots(N, N)
         for i in range(N):
            for j in range(N):
               aug = datagen.random_transform(tensor, seed=seeds[i*N + j])

               ax[i,j].set_xticks([])
               ax[i,j].set_yticks([])
               ax[i,j].imshow(aug, cmap=cmap)

         plt.savefig(filename, bbox_inches="tight")

      seeds = random.sample(range(1000), N**2)

      datagen = tf_image.ImageDataGenerator(**self.dataset.get_datagen_args())
      generate_plot(N, img, seeds, self.FILE_AUG_IMG, datagen)
      generate_plot(N, mask, seeds, self.FILE_AUG_MASK, datagen, "gray")

   # Plot trajectory of loss on training and test set over N replications
   # Possible metrics: loss, precision, recall, F1-score
   # To get test metrics, add "val_" as prefix to metric
   def plot_training_process(self, data_dir, filename, metric):
      import os
      import numpy as np

      # Do some optional preprocessing of datapoints (in case of recall, precision, F1)
      def get_datapoint(rep, i, metric, test=False):
         if metric == "loss":
            return data[rep]["val_"+metric][i] if test else data[rep][metric][i]
         else:  # Recall, Precision or F1-score
            TP = data[rep]["val_binary_TP"][i] if test else data[rep]["binary_TP"][i]
            FP = data[rep]["val_binary_FP"][i] if test else data[rep]["binary_FP"][i]
            FN = data[rep]["val_binary_FN"][i] if test else data[rep]["binary_FN"][i]

            if metric == "precision":
               return TP / (TP + FP)
            elif metric == "recall":
               return TP / (TP + FN)
            elif metric == "F1-score":
               return TP / (TP + 0.5 * (FP + FN))

      # Plot line with confidence interval
      def plot(X, data, label, color):
         Y1 = [i[0] for i in data]  # AVG
         CI = [i[1] for i in data]  # CI

         Y2 = [x + y for x, y in zip(Y1, CI)]  # Upper CI
         Y3 = [x - y for x, y in zip(Y1, CI)]  # Lower CI

         plt.plot(X, Y1, label=label, color=color)
         plt.fill_between(X, y1=Y2, y2=Y3, color=color, alpha=0.3)

      rep_dirs = [next(os.walk(data_dir))[1]][0]
      N_REP = len(rep_dirs)

      # Load history data
      data = [np.load(os.path.join(data_dir, rep_dirs[i], filename), allow_pickle=True).item() \
              for i in range(N_REP)]

      X = range(len(data[0]["loss"]))
      Y1, Y2 = [], []

      # Compute AVG and CI for training and test sets
      for i in X:
         SUM_TRAIN, SUM_TEST = 0., 0.

         for rep in range(N_REP):
            SUM_TRAIN += get_datapoint(rep, i, metric)
            SUM_TEST  += get_datapoint(rep, i, metric, True)

         AVG_TRAIN = round(SUM_TRAIN / N_REP, 4)
         AVG_TEST  = round(SUM_TEST / N_REP, 4)

         DIST_TRAIN, DIST_TEST = 0., 0.
         for rep in range(N_REP):
            DIST_TRAIN += (get_datapoint(rep, i, metric) - AVG_TRAIN)**2
            DIST_TEST += (get_datapoint(rep, i, metric, True) - AVG_TEST)**2

         CI_TRAIN = np.sqrt(DIST_TRAIN / N_REP) / np.sqrt(N_REP)
         CI_TEST = np.sqrt(DIST_TEST / N_REP) / np.sqrt(N_REP)

         Y1.append((AVG_TRAIN, CI_TRAIN))
         Y2.append((AVG_TEST, CI_TEST))

      # Plot the training and test performance
      plot(X, Y1, "Train set", "navy")
      plot(X, Y2, "Test set", "darkred")

      plt.xlabel("Epoch")
      plt.ylabel("MSE loss")#metric.capitalize())
      plt.legend()
      plt.grid(b=True)
      plt.xlim(0, max(X))
      plt.title(f"MSE loss throughout the training process (3 replications)")
      plt.savefig(os.path.join(data_dir, metric+".svg"), bbox_inches="tight")

max_stars_repo_name: JerrySchonenberg/CloudGAN

max_issues_repo_name: JerrySchonenberg/CloudGAN

max_forks_repo_name: JerrySchonenberg/CloudGAN


text:
"""
Skyline functions

These are shared functions that are required in multiple modules.
"""
import logging
from os.path import dirname, join, abspath, isfile
from os import path
from time import time
import socket
import datetime
import errno

import traceback
import json
import requests
try:
    import urlparse
except ImportError:
    import urllib.parse
try:
    import urllib2
except ImportError:
    import urllib.request
    import urllib.error

import settings

try:
    # @modified 20190518 - Branch #3002: docker
    # from settings import GRAPHITE_HOST
    from settings import CARBON_HOST
except:
    # @modified 20190518 - Branch #3002: docker
    # GRAPHITE_HOST = ''
    CARBON_HOST = ''
try:
    from settings import CARBON_PORT
except:
    CARBON_PORT = ''

# @added 20191007 - Feature #3250: Allow Skyline to send metrics to another Carbon host
try:
    from settings import SKYLINE_METRICS_CARBON_HOST
    skyline_metrics_carbon_host = SKYLINE_METRICS_CARBON_HOST
except:
    skyline_metrics_carbon_host = CARBON_HOST
try:
    from settings import SKYLINE_METRICS_CARBON_PORT
    skyline_metrics_carbon_port = SKYLINE_METRICS_CARBON_PORT
except:
    skyline_metrics_carbon_port = CARBON_PORT

config = {'user': settings.PANORAMA_DBUSER,
          'password': settings.PANORAMA_DBUSERPASS,
          'host': settings.PANORAMA_DBHOST,
          'port': settings.PANORAMA_DBPORT,
          'database': settings.PANORAMA_DATABASE,
          'raise_on_warnings': True}

def send_graphite_metric(current_skyline_app, metric, value):
    """
    Sends the skyline_app metrics to the `GRAPHITE_HOST` if a graphite
    host is defined.

    :param current_skyline_app: the skyline app using this function
    :param metric: the metric namespace
    :param value: the metric value (as a str not an int)
    :type current_skyline_app: str
    :type metric: str
    :type value: str
    :return: ``True`` or ``False``
    :rtype: boolean

    """

    # @added 20190805 - Task #2828: Skyline - Python 3.7
    try:
        python_version
    except:
        from sys import version_info
        python_version = int(version_info[0])

    # @added 20190518 - Branch #3002: docker
    # If the CARBON_HOST is set to the default do not send_graphite_metric
    # @modified 20191007 - Feature #3250: Allow Skyline to send metrics to another Carbon host
    # if CARBON_HOST == 'YOUR_GRAPHITE_HOST.example.com':
    if skyline_metrics_carbon_host == 'YOUR_GRAPHITE_HOST.example.com':
        current_skyline_app_logger = str(current_skyline_app) + 'Log'
        current_logger = logging.getLogger(current_skyline_app_logger)
        current_logger.info('CARBON_HOST is not configured in settings.py no CARBON_HOST to send metrics to')
        return False

    # @modified 20190518 - Branch #3002: docker
    # Use the CARBON_HOST rather than GRAPHITE_HOST to allow for the 2 to be
    # on different hosts
    # if GRAPHITE_HOST != '':
    # @modified 20191007 - Feature #3250: Allow Skyline to send metrics to another Carbon host
    # if CARBON_HOST != '':
    if skyline_metrics_carbon_host != '':
        sock = socket.socket()
        sock.settimeout(10)

        # Handle connection error to Graphite #116 @etsy
        # Fixed as per https://github.com/etsy/skyline/pull/116 and
        # mlowicki:etsy_handle_connection_error_to_graphite
        # Handle connection error to Graphite #7 @ earthgecko
        # merged 1 commit into earthgecko:master from
        # mlowicki:handle_connection_error_to_graphite on 16 Mar 2015
        try:
            # @modified 20190518 - Branch #3002: docker
            # sock.connect((GRAPHITE_HOST, CARBON_PORT))
            # @modified 20191007 - Feature #3250: Allow Skyline to send metrics to another Carbon host
            # sock.connect((CARBON_HOST, CARBON_PORT))
            sock.connect((skyline_metrics_carbon_host, skyline_metrics_carbon_port))
            sock.settimeout(None)
        except socket.error:
            sock.settimeout(None)
            # @modified 20190518 - Branch #3002: docker
            # endpoint = '%s:%d' % (GRAPHITE_HOST, CARBON_PORT)
            # @modified 20191007 - Feature #3250: Allow Skyline to send metrics to another Carbon host
            # endpoint = '%s:%d' % (CARBON_HOST, CARBON_PORT)
            endpoint = '%s:%d' % (skyline_metrics_carbon_host, skyline_metrics_carbon_port)
            current_skyline_app_logger = str(current_skyline_app) + 'Log'
            current_logger = logging.getLogger(current_skyline_app_logger)
            current_logger.error(
                'error :: cannot connect to Graphite at %s' % endpoint)
            return False

        # For the same reason as above
        # sock.sendall('%s %s %i\n' % (name, value, time()))
        try:
            # @modified 20190805 - Task #2828: Skyline - Python 3.7
            if python_version == 2:
                sock.sendall('%s %s %i\n' % (metric, value, time()))
            if python_version == 3:
                message = '%s %s %i\n' % (metric, value, time())
                sock.sendall(message.encode())
            sock.close()
            return True
        except:
            # @modified 20190518 - Branch #3002: docker
            # endpoint = '%s:%d' % (GRAPHITE_HOST, CARBON_PORT)
            # @modified 20191007 - Feature #3250: Allow Skyline to send metrics to another Carbon host
            # endpoint = '%s:%d' % (CARBON_HOST, CARBON_PORT)
            endpoint = '%s:%d' % (skyline_metrics_carbon_host, skyline_metrics_carbon_port)
            current_skyline_app_logger = str(current_skyline_app) + 'Log'
            current_logger = logging.getLogger(current_skyline_app_logger)
            current_logger.error(
                'error :: could not send data to Graphite at %s' % endpoint)
            return False

    return False

def mkdir_p(path):
    """
    Create nested directories.

    :param path: directory path to create
    :type path: str
    :return: returns True

    """
    try:
        os.getpid()
    except:
        import os
    try:
        os.makedirs(path, mode=0o755)
        return True
    # Python >2.5
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def load_metric_vars(current_skyline_app, metric_vars_file):
    """
    Import the metric variables for a check from a metric check variables file

    :param current_skyline_app: the skyline app using this function
    :param metric_vars_file: the path and filename to the metric variables files
    :type current_skyline_app: str
    :type metric_vars_file: str
    :return: the metric_vars module object or ``False``
    :rtype: object or boolean

    """
    try:
        os.getpid()
    except:
        import os

    try:
        imp
    except:
        import imp

    metric_vars = False
    metric_vars_got = False
    if os.path.isfile(metric_vars_file):
        current_skyline_app_logger = str(current_skyline_app) + 'Log'
        current_logger = logging.getLogger(current_skyline_app_logger)
        current_logger.info(
            'loading metric variables from import - metric_check_file - %s' % (
                str(metric_vars_file)))
        # Bug #1460: panorama check file fails
        # global metric_vars
        # current_logger.info('set global metric_vars')
        with open(metric_vars_file) as f:
            try:
                metric_vars = imp.load_source('metric_vars', '', f)
                metric_vars_got = True
            except:
                current_logger.info(traceback.format_exc())
                msg = 'failed to import metric variables - metric_check_file'
                current_logger.error(
                    'error :: %s - %s' % (msg, str(metric_vars_file)))
                metric_vars = False

        if settings.ENABLE_DEBUG and metric_vars_got:
            current_logger.info(
                'metric_vars determined - metric variable - metric - %s' % str(metric_vars.metric))
    else:
        current_logger.error('error :: metric_vars_file not found - %s' % (str(metric_vars_file)))

    return metric_vars

def write_data_to_file(current_skyline_app, write_to_file, mode, data):
    """
    Write date to a file

    :param current_skyline_app: the skyline app using this function
    :param file: the path and filename to write the data into
    :param mode: ``w`` to overwrite, ``a`` to append
    :param data: the data to write to the file
    :type current_skyline_app: str
    :type file: str
    :type mode: str
    :type data: str
    :return: ``True`` or ``False``
    :rtype: boolean

    """
    try:
        os.getpid()
    except:
        import os

    try:
        python_version
    except:
        from sys import version_info
        python_version = int(version_info[0])

    file_dir = os.path.dirname(write_to_file)
    if not os.path.exists(file_dir):
        try:
            os.makedirs(file_dir, mode=0o755)
        # Python >2.5
        except OSError as exc:
            if exc.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise

    if not os.path.exists(file_dir):
        current_skyline_app_logger = str(current_skyline_app) + 'Log'
        current_logger = logging.getLogger(current_skyline_app_logger)
        current_logger.error(
            'error :: could not create directory - %s' % (str(file_dir)))

    try:
        with open(write_to_file, mode) as fh:
            fh.write(data)
        if python_version == 2:
            os.chmod(write_to_file, 0o644)
        if python_version == 3:
            os.chmod(write_to_file, mode=0o644)

        return True
    except:
        return False

    return False

def fail_check(current_skyline_app, failed_check_dir, check_file_to_fail):
    """
    Move a failed check file.

    :param current_skyline_app: the skyline app using this function
    :param failed_check_dir: the directory where failed checks are moved to
    :param check_file_to_fail: failed check file to move
    :type current_skyline_app: str
    :type failed_check_dir: str
    :type check_file_to_fail: str
    :return: ``True``, ``False``
    :rtype: boolean

    """
    try:
        os.getpid()
    except:
        import os

    try:
        shutil
    except:
        import shutil

    try:
        python_version
    except:
        from sys import version_info
        python_version = int(version_info[0])

    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)

    if not os.path.exists(failed_check_dir):
        try:
            mkdir_p(failed_check_dir)
            current_logger.info(
                'created failed_check_dir - %s' % str(failed_check_dir))
        except:
            current_logger.info(traceback.format_exc())
            current_logger.error(
                'error :: failed to create failed_check_dir - %s' %
                str(failed_check_dir))
            return False

    check_file_name = os.path.basename(str(check_file_to_fail))
    failed_check_file = '%s/%s' % (failed_check_dir, check_file_name)

    try:
        shutil.move(check_file_to_fail, failed_check_file)
        if python_version == 2:
            os.chmod(failed_check_file, 0o644)
        if python_version == 3:
            os.chmod(failed_check_file, mode=0o644)

        current_logger.info('moved check file to - %s' % failed_check_file)
        return True
    except OSError:
        current_logger.info(traceback.format_exc())
        msg = 'failed to move check file to -%s' % failed_check_file
        current_logger.error('error :: %s' % msg)
        pass

    return False

def get_graphite_metric(
    current_skyline_app, metric, from_timestamp, until_timestamp, data_type,
        output_object):
    """
    Fetch data from graphite and return it as object or save it as file

    :param current_skyline_app: the skyline app using this function
    :param metric: metric name
    :param from_timestamp: unix timestamp
    :param until_timestamp: unix timestamp
    :param data_type: image or json
    :param output_object: object or path and filename to save data as, if set to
                          object, the object is returned
    :type current_skyline_app: str
    :type metric: str
    :type from_timestamp: str
    :type until_timestamp: str
    :type data_type: str
    :type output_object: str
    :return: timeseries string, ``True``, ``False``
    :rtype: str or boolean

    """
    try:
        os.getpid()
    except:
        import os

    try:
        python_version
    except:
        from sys import version_info
        python_version = int(version_info[0])

    try:
        quote(current_skyline_app, safe='')
    except:
        from requests.utils import quote

    try:
        time.time()
    except:
        import time

    try:
        re
    except:
        import re

    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)

#    if settings.ENABLE_DEBUG:
    current_logger.info('graphite_metric - %s' % (metric))

    # @added 20160803 - Unescaped Graphite target - https://github.com/earthgecko/skyline/issues/20
    #                   bug1546: Unescaped Graphite target
    new_metric_namespace = metric.replace(':', '\:')
    metric_namespace = new_metric_namespace.replace('(', '\(')
    metric = metric_namespace.replace(')', '\)')

    # Graphite timeouts
    connect_timeout = int(settings.GRAPHITE_CONNECT_TIMEOUT)
    if settings.ENABLE_DEBUG:
        current_logger.info('connect_timeout - %s' % str(connect_timeout))

    current_logger.info('connect_timeout - %s' % str(connect_timeout))

    read_timeout = int(settings.GRAPHITE_READ_TIMEOUT)
    if settings.ENABLE_DEBUG:
        current_logger.info('read_timeout - %s' % str(read_timeout))

    current_logger.info('read_timeout - %s' % str(read_timeout))

    graphite_from = datetime.datetime.fromtimestamp(int(from_timestamp)).strftime('%H:%M_%Y%m%d')
    current_logger.info('graphite_from - %s' % str(graphite_from))

    graphite_until = datetime.datetime.fromtimestamp(int(until_timestamp)).strftime('%H:%M_%Y%m%d')
    current_logger.info('graphite_until - %s' % str(graphite_until))

    output_format = data_type

    # @modified 20170603 - Feature #2034: analyse_derivatives
    # Added deriative functions to convert the values of metrics strictly
    # increasing monotonically to their deriative products in Graphite now
    # graphs
    known_derivative_metric = False
    REDIS_CONN = None
    try:
        # @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow
        if settings.REDIS_PASSWORD:
            REDIS_CONN = StrictRedis(password=settings.REDIS_PASSWORD, unix_socket_path=settings.REDIS_SOCKET_PATH)
        else:
            REDIS_CONN = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH)
    except:
        from redis import StrictRedis
    if not REDIS_CONN:
        try:
            # @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow
            if settings.REDIS_PASSWORD:
                REDIS_CONN = StrictRedis(password=settings.REDIS_PASSWORD, unix_socket_path=settings.REDIS_SOCKET_PATH)
            else:
                REDIS_CONN = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH)
        except:
            current_logger.error('error :: alert_smtp - redis connection failed')
    try:
        derivative_metrics = list(REDIS_CONN.smembers('derivative_metrics'))
    except:
        derivative_metrics = []
    redis_metric_name = '%s%s' % (settings.FULL_NAMESPACE, str(metric))

    # @added 20180423 - Feature #2034: analyse_derivatives
    #                   Branch #2270: luminosity
    metric_found_in_redis = False

    if redis_metric_name in derivative_metrics:
        known_derivative_metric = True
        metric_found_in_redis = True
    if known_derivative_metric:
        try:
            non_derivative_metrics = list(REDIS_CONN.smembers('non_derivative_metrics'))
        except:
            non_derivative_metrics = []
        if redis_metric_name in non_derivative_metrics:
            known_derivative_metric = False
            metric_found_in_redis = True

    # @added 20180423 - Feature #2034: analyse_derivatives
    #                   Branch #2270: luminosity
    if not metric_found_in_redis and settings.OTHER_SKYLINE_REDIS_INSTANCES:
        # @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow
        # for redis_ip, redis_port in settings.OTHER_SKYLINE_REDIS_INSTANCES:
        for redis_ip, redis_port, redis_password in settings.OTHER_SKYLINE_REDIS_INSTANCES:
            if not metric_found_in_redis:
                try:
                    if redis_password:
                        other_redis_conn = StrictRedis(host=str(redis_ip), port=int(redis_port), password=str(redis_password))
                    else:
                        other_redis_conn = StrictRedis(host=str(redis_ip), port=int(redis_port))
                    other_derivative_metrics = list(other_redis_conn.smembers('derivative_metrics'))
                except:
                    current_logger.error(traceback.format_exc())
                    current_logger.error('error :: failed to connect to Redis at %s on port %s' % (str(redis_ip), str(redis_port)))
                if redis_metric_name in other_derivative_metrics:
                    known_derivative_metric = True
                    metric_found_in_redis = True
                    current_logger.info('%s found in derivative_metrics in Redis at %s on port %s' % (redis_metric_name, str(redis_ip), str(redis_port)))

    target_metric = metric
    if known_derivative_metric:
        target_metric = 'nonNegativeDerivative(%s)' % str(metric)

    # graphite URL
    graphite_port = '80'
    if settings.GRAPHITE_PORT != '':
        graphite_port = str(settings.GRAPHITE_PORT)
    # @added 20191002 - Branch #3002: docker
    # Unset the graphite_port if normal https
    if settings.GRAPHITE_PORT == '443' and settings.GRAPHITE_PROTOCOL == 'https':
        graphite_port = ''

    # @modified 20190520 - Branch #3002: docker
    # image_url = settings.GRAPHITE_PROTOCOL + '://' + settings.GRAPHITE_HOST + ':' + graphite_port + '/render?from=' + graphite_from + '&until=' + graphite_until + '&target=' + target_metric
    # image_url = settings.GRAPHITE_PROTOCOL + '://' + settings.GRAPHITE_HOST + ':' + graphite_port + '/api/datasources/proxy/1/render/?from=' + graphite_from + '&until=' + graphite_until + '&target=' + target_metric
    image_url = settings.GRAPHITE_PROTOCOL + '://' + settings.GRAPHITE_HOST + ':' + graphite_port + '/' + settings.GRAPHITE_RENDER_URI + '?from=' + graphite_from + '&until=' + graphite_until + '&target=' + target_metric
    url = image_url + '&format=' + output_format

    if settings.ENABLE_DEBUG:
        current_logger.info('graphite url - %s' % (url))

    current_logger.info('graphite url - %s' % (url))

    get_image = False
    if data_type == 'image' and output_object != 'object':
        if not os.path.isfile(output_object):
            get_image = True
        else:
            if settings.ENABLE_DEBUG:
                current_logger.info('graph file exists - %s' % str(output_object))
            return True

    if get_image:
        current_logger.info(
            'retrieving png - surfacing %s graph from graphite from %s to %s' % (
                metric, str(graphite_from), str(graphite_until)))

        graphite_image_file = output_object
        if 'width' not in image_url:
            image_url += '&width=586'
        if 'height' not in image_url:
            image_url += '&height=308'
        # @added 20170106 - Feature #1842: Ionosphere - Graphite now graphs
        # settings, color and title
        if str(current_skyline_app) == 'webapp':
            get_ionosphere_graphs = False
            if 'graphite_now' in output_object:
                get_ionosphere_graphs = True
            # @added 20170107 - Feature #1852: Ionosphere - features_profile matched graphite graphs
            # Added graphite_matched_images matched.fp_id
            if 'matched.fp_id' in output_object:
                get_ionosphere_graphs = True
            # @added 20170308 - Feature #1960: ionosphere_layers
            # Added graphite_layers_matched_images matched.layer
            if 'matched.layers.fp_id' in output_object:
                get_ionosphere_graphs = True
            if get_ionosphere_graphs:
                int_hours = int((int(until_timestamp) - int(from_timestamp)) / 60 / 60)
                if 'graphite_now' in output_object:
                    no_extension = os.path.splitext(output_object)[0]
                    _hours = os.path.splitext(no_extension)[1]
                    hours = _hours.replace('.', '')
                    int_hours = hours.replace('h', '')
                str_value = str(int_hours)
                period = 'hours'
                if int(int_hours) > 24:
                    str_value = str(int(int_hours) / 24)
                    period = 'days'
                if 'graphite_now' in output_object:
                    unencoded_graph_title = 'Graphite NOW at %s %s' % (str_value, period)
                # @added 20170308 - Feature #1960: ionosphere_layers
                matched_graph = False
                if 'matched.fp_id' in output_object:
                    matched_graph = True
                if 'matched.layers.fp_id' in output_object:
                    matched_graph = True
                # @modified 20170308 - Feature #1960: ionosphere_layers
                # if 'matched.fp_id' in output_object:
                if matched_graph:
                    human_date = time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(int(until_timestamp)))
                    if 'matched.fp_id' in output_object:
                        tail_piece = re.sub('.*\.fp_id-', '', output_object)
                        matched_fp_id = re.sub('\..*', '', tail_piece)
                        unencoded_graph_title = 'fp_id %s matched - %s at %s hours' % (
                            str(matched_fp_id), str(human_date),
                            str(int_hours))
                    if 'matched.layers.fp_id' in output_object:
                        # layers_id
                        tail_piece = re.sub('.*\.layers_id-', '', output_object)
                        matched_layers_id = re.sub('\..*', '', tail_piece)
                        unencoded_graph_title = 'layers_id %s matched - %s at %s hours' % (
                            str(matched_layers_id), str(human_date),
                            str(int_hours))
                graph_title_string = quote(unencoded_graph_title, safe='')
                graph_title = '&title=%s' % graph_title_string
                add_parameters = '%s&colorList=blue%s' % (settings.GRAPHITE_GRAPH_SETTINGS, graph_title)
                if 'matched.layers.fp_id' in output_object:
                    add_parameters = '%s&colorList=green%s' % (settings.GRAPHITE_GRAPH_SETTINGS, graph_title)
                image_url += add_parameters
                current_logger.info('Ionosphere graphite NOW url - %s' % (image_url))

        if settings.ENABLE_DEBUG:
            current_logger.info('graphite image url - %s' % (str(image_url)))
        image_url_timeout = int(connect_timeout)

        image_data = None

        try:
            # @modified 20170913 - Task #2160: Test skyline with bandit
            # Added nosec to exclude from bandit tests
            image_data = urllib2.urlopen(image_url, timeout=image_url_timeout).read()  # nosec
            current_logger.info('url OK - %s' % (image_url))
        except urllib2.URLError:
            image_data = None
            current_logger.error('error :: url bad - %s' % (image_url))

        if image_data is not None:
            with open(graphite_image_file, 'w') as f:
                f.write(image_data)
            current_logger.info('retrieved - %s' % (graphite_image_file))
            if python_version == 2:
                os.chmod(graphite_image_file, 0o644)
            if python_version == 3:
                os.chmod(graphite_image_file, mode=0o644)
        else:
            current_logger.error(
                'error :: failed to retrieved - %s' % (graphite_image_file))

        if not os.path.isfile(graphite_image_file):
            msg = 'retrieve failed to surface %s graph from Graphite' % (metric)
            current_logger.error('error :: %s' % msg)
        # @added 20170107 - Feature #1842: Ionosphere - Graphite now graphs
        # In order to determine whether a Graphite image was retrieved or not
        # this needs to return here.  This should not be backwards incompatible
        # as it has never been used to determine the outcome before it appears,
        # which as they say is my bad.
            return False
        else:
            return True

    if data_type == 'json':

        if output_object != 'object':
            if os.path.isfile(output_object):
                return True

        msg = 'surfacing timeseries data for %s from graphite from %s to %s' % (
            metric, graphite_from, graphite_until)
        current_logger.info('%s' % msg)
        if requests.__version__ >= '2.4.0':
            use_timeout = (int(connect_timeout), int(read_timeout))
        else:
            use_timeout = int(connect_timeout)
        if settings.ENABLE_DEBUG:
            current_logger.info('use_timeout - %s' % (str(use_timeout)))

        graphite_json_fetched = False
        try:
            r = requests.get(url, timeout=use_timeout)
            graphite_json_fetched = True
        except:
            datapoints = [[None, str(graphite_until)]]
            current_logger.error('error :: data retrieval from Graphite failed')

        if graphite_json_fetched:
            try:
                js = r.json()
                datapoints = js[0]['datapoints']
                if settings.ENABLE_DEBUG:
                    current_logger.info('data retrieved OK')
            except:
                datapoints = [[None, str(graphite_until)]]
                current_logger.error('error :: failed to parse data points from retrieved json')

        converted = []
        for datapoint in datapoints:
            try:
                new_datapoint = [float(datapoint[1]), float(datapoint[0])]
                converted.append(new_datapoint)
            # @modified 20170913 - Task #2160: Test skyline with bandit
            # Added nosec to exclude from bandit tests
            except:  # nosec
                continue

        if output_object != 'object':
            with open(output_object, 'w') as f:
                f.write(json.dumps(converted))
            if python_version == 2:
                os.chmod(output_object, 0o644)
            if python_version == 3:
                os.chmod(output_object, mode=0o644)
            if settings.ENABLE_DEBUG:
                current_logger.info('json file - %s' % output_object)
        else:
            timeseries_json = json.dumps(converted)
            return timeseries_json

        if not os.path.isfile(output_object):
            current_logger.error(
                'error :: failed to surface %s json from graphite' % (metric))
            return False
        else:
            return True

    return True

# @added 20170206 - Bug #1904: Handle non filesystem friendly metric names in check files
def filesafe_metricname(metricname):
    '''
    Returns a file system safe name for a metric name in terms of creating
    check files, etc
    '''
    keepchars = ('.', '_', '-')
    try:
        sane_metricname = ''.join(c for c in str(metricname) if c.isalnum() or c in keepchars).rstrip()
        return str(sane_metricname)
    except:
        return False

# @added 20160922 - Branch #922: Ionosphere
# Added the send_anomalous_metric_to function for Analyzer and Mirage
# @modified 20161228 Feature #1828: ionosphere - mirage Redis data features
# Added full_duration which needs to be recorded to allow Mirage metrics
# to be profiled on Redis timeseries data at FULL_DURATION
# e.g. mirage.redis.24h.json
# @modified 20170127 - Feature #1886: Ionosphere learn - child like parent with evolutionary maturity
# Added parent_id, always zero from Analyzer and Mirage

def send_anomalous_metric_to(
    current_skyline_app, send_to_app, timeseries_dir, metric_timestamp,
        base_name, datapoint, from_timestamp, triggered_algorithms, timeseries,
        full_duration, parent_id):
    """
    Assign a metric and timeseries to Crucible or Ionosphere.
    """
    try:
        os.getpid()
    except:
        import os

    try:
        python_version
    except:
        from sys import version_info
        python_version = int(version_info[0])

    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)

    # @added 20170116 - Feature #1854: Ionosphere learn
    added_by_context = str(current_skyline_app)

    # @added 20170117 - Feature #1854: Ionosphere learn
    # Added the ionosphere_learn_to_ionosphere to fix ionosphere_learn not being
    # logged.
    if str(send_to_app) == 'ionosphere_learn_to_ionosphere':
        added_by_context = 'ionosphere_learn'
        new_send_to_app = 'ionosphere'
        send_to_app = new_send_to_app

    # @added 20170206 - Bug #1904: Handle non filesystem friendly metric names in check files
    sane_metricname = filesafe_metricname(str(base_name))

    if str(send_to_app) == 'crucible':
        anomaly_dir = '%s/%s/%s' % (
            settings.CRUCIBLE_DATA_FOLDER, str(timeseries_dir), str(metric_timestamp))
        check_file = '%s/%s.%s.txt' % (
            settings.CRUCIBLE_CHECK_PATH, str(metric_timestamp), str(sane_metricname))
        check_dir = '%s' % (settings.CRUCIBLE_CHECK_PATH)
    if str(send_to_app) == 'ionosphere':
        # @modified 20161121 - Branch #922: ionosphere
        # Use the timestamp as the parent dir for training_data, it is easier
        # to manage and clean on timestamps.  Walking through 1000s of Graphite
        # style dirs with dotted metric namespaces for timestamps.
        # anomaly_dir = settings.IONOSPHERE_DATA_FOLDER + '/' + timeseries_dir + '/' + metric_timestamp
        anomaly_dir = '%s/%s/%s' % (
            settings.IONOSPHERE_DATA_FOLDER, str(metric_timestamp),
            str(timeseries_dir))
        check_file = '%s/%s.%s.txt' % (
            settings.IONOSPHERE_CHECK_PATH, str(metric_timestamp),
            str(sane_metricname))
        check_dir = '%s' % (settings.IONOSPHERE_CHECK_PATH)

        # @added 20170116 - Feature #1854: Ionosphere learn
        # So as to not have to backport a context to all the instances of
        # send_anomalous_metric_to identify what this was added_by, here the
        # learn directory path string overrides the dirs if it is a learn
        # related check
        # @modified 20170117 - Feature #1854: Ionosphere learn
        # The ionosphere_learn_to_ionosphere to fix ionosphere_learn not being
        # logged.
        # if str(settings.IONOSPHERE_LEARN_FOLDER) in anomaly_dir:
        #     added_by_context = 'ionosphere_learn'
        if added_by_context == 'ionosphere_learn':
            current_logger.info('send_anomalous_metric_to :: this is an ionosphere_learn check')

    if not os.path.exists(check_dir):
        os.makedirs(check_dir, mode=0o755)

    if not os.path.exists(anomaly_dir):
        os.makedirs(anomaly_dir, mode=0o755)

    # Note:
    # The values are enclosed is single quoted intentionally
    # as the imp.load_source used in crucible results in a
    # shift in the decimal position when double quoted, e.g.
    # value = "5622.0" gets imported as
    # 2016-03-02 12:53:26 :: 28569 :: metric variable - value - 562.2
    # single quoting results in the desired,
    # 2016-03-02 13:16:17 :: 1515 :: metric variable - value - 5622.0
    now_timestamp = int(time())
    # @modified 20161228 Feature #1828: ionosphere - mirage Redis data features
    # Added full_duration
    # @modified 20170116 - Feature #1854: Ionosphere learn
    # Changed added_by parameter from current_skyline_app to added_by_context
    # @modified 20170127 - Feature #1886: Ionosphere learn - child like parent with evolutionary maturity
    # Added ionosphere_parent_id, always zero from Analyzer and Mirage
    anomaly_data = 'metric = \'%s\'\n' \
                   'value = \'%s\'\n' \
                   'from_timestamp = \'%s\'\n' \
                   'metric_timestamp = \'%s\'\n' \
                   'algorithms = %s\n' \
                   'triggered_algorithms = %s\n' \
                   'anomaly_dir = \'%s\'\n' \
                   'graphite_metric = True\n' \
                   'run_crucible_tests = False\n' \
                   'added_by = \'%s\'\n' \
                   'added_at = \'%s\'\n' \
                   'full_duration = \'%s\'\n' \
                   'ionosphere_parent_id = \'%s\'\n' \
        % (str(base_name), str(datapoint), str(from_timestamp),
            str(metric_timestamp), str(settings.ALGORITHMS),
            str(triggered_algorithms), anomaly_dir, added_by_context,
            str(now_timestamp), str(int(full_duration)), str(parent_id))

    # @modified 20170116 - Feature #1854: Ionosphere learn
    # In the Ionosphere context there is no requirement to create a timeseries
    # json file or anomaly_file, just the check
    if added_by_context != 'ionosphere_learn':

        # Create an anomaly file with details about the anomaly
        anomaly_file = '%s/%s.txt' % (anomaly_dir, str(base_name))
        try:
            write_data_to_file(str(current_skyline_app), anomaly_file, 'w', anomaly_data)
            current_logger.info('added %s anomaly file :: %s' % (str(send_to_app), anomaly_file))
        except:
            current_logger.info(traceback.format_exc())
            current_logger.error(
                'error :: failed to add %s anomaly file :: %s' %
                (str(send_to_app), anomaly_file))

        # Create timeseries json file with the timeseries
        json_file = '%s/%s.json' % (anomaly_dir, str(base_name))
        timeseries_json = str(timeseries).replace('[', '(').replace(']', ')')
        try:
            write_data_to_file(str(current_skyline_app), json_file, 'w', timeseries_json)
            current_logger.info('added %s timeseries file :: %s' % (str(send_to_app), json_file))
        except:
            current_logger.error(
                'error :: failed to add %s timeseries file :: %s' %
                (str(send_to_app), json_file))
            current_logger.info(traceback.format_exc())

    # Create a check file
    try:
        write_data_to_file(str(current_skyline_app), check_file, 'w', anomaly_data)
        current_logger.info(
            'added %s check :: %s,%s' % (
                str(send_to_app), str(base_name), str(metric_timestamp)))
    except:
        current_logger.info(traceback.format_exc())
        current_logger.error('error :: failed to add %s check file :: %s' % (str(send_to_app), check_file))

def RepresentsInt(s):
    '''
    As per http://stackoverflow.com/a/1267145 and @Aivar I must agree with
    @Triptycha > "This 5 line function is not a complex mechanism."
    '''
    try:
        int(s)
        return True
    except ValueError:
        return False

################################################################################

def mysql_select(current_skyline_app, select):
    """
    Select data from mysql database

    :param current_skyline_app: the Skyline app that is calling the function
    :param select: the select string
    :type current_skyline_app: str
    :type select: str
    :return: tuple
    :rtype: tuple, boolean

    - **Example usage**::

        from skyline_functions import mysql_select
        query = 'select id, metric from anomalies'
        results = mysql_select(query)

    - **Example of the 0 indexed results tuple, which can hold multiple results**::

        >> print('results: %s' % str(results))
        results: [(1, u'test1'), (2, u'test2')]

        >> print('results[0]: %s' % str(results[0]))
        results[0]: (1, u'test1')

    .. note::
        - If the MySQL query fails a boolean will be returned not a tuple
            * ``False``
            * ``None``

    """
    ENABLE_DEBUG = False
    try:
        if settings.ENABLE_PANORAMA_DEBUG:
            ENABLE_DEBUG = True
    except:
        nothing_to_do = True
    try:
        if settings.ENABLE_WEBAPP_DEBUG:
            ENABLE_DEBUG = True
    except:
        nothing_to_do = True
    try:
        if settings.ENABLE_IONOSPHERE_DEBUG:
            ENABLE_DEBUG = True
    except:
        nothing_to_do = True

    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)
    if ENABLE_DEBUG:
        current_logger.info('debug :: entering mysql_select')

    try:
        mysql.connector
    except:
        import mysql.connector
    try:
        conversion
    except:
        from mysql.connector import conversion
    try:
        re
    except:
        import re

    try:
        cnx = mysql.connector.connect(**config)
        if ENABLE_DEBUG:
            current_logger.info('debug :: connected to mysql')
    except mysql.connector.Error as err:
        current_logger.error('error :: mysql error - %s' % str(err))
        current_logger.error('error :: failed to connect to mysql')
        return False

    if cnx:
        try:
            if ENABLE_DEBUG:
                current_logger.info('debug :: %s' % (str(select)))

            # NOTE: when a LIKE SELECT is made the query string is not run
            # through the conversion.MySQLConverter().escape method as it
            # it would not work and kept on returning mysql error - 1064 with
            # multiple single quotes e.g. use near '\'carbon%\'' at line 1
            # Various patterns were attempted to no avail, it seems to be
            # related to % character. pseudo basic HTTP auth has been added to
            # the webapp just in someone does not secure it properly, a little
            # defence in depth, so added WEBAPP_ALLOWED_IPS too.
            pattern_match = None
            try:
                pattern_match = re.search(' LIKE ', str(select))
            except:
                current_logger.error('error :: pattern_match - %s' % traceback.format_exc())
            if not pattern_match:
                try:
                    pattern_match = re.search(' like ', str(select))
                except:
                    current_logger.error('error :: pattern_match - %s' % traceback.format_exc())

            # TBD - not sure how to get it escaping safely
            pattern_match = True
            if pattern_match:
                query = str(select)
                if ENABLE_DEBUG:
                    current_logger.info('debug :: unescaped query - %s' % (str(query)))
                cursor = cnx.cursor()
                cursor.execute(query)
            else:
                query = conversion.MySQLConverter().escape(select)
                if ENABLE_DEBUG:
                    current_logger.info('debug :: escaped query - %s' % (str(query)))
                cursor = cnx.cursor()
                cursor.execute(query.format(query))

            # cursor = cnx.cursor()
            # cursor.execute(query)
            result = cursor.fetchall()
            cursor.close()
            cnx.close()
            return result
        except mysql.connector.Error as err:
            current_logger.error('error :: mysql error - %s' % str(err))
            current_logger.error(
                'error :: failed to query database - %s' % (str(select)))
            try:
                cnx.close()
                return False
            except:
                return False
    else:
        if ENABLE_DEBUG:
            current_logger.error('error - failed to connect to mysql')

    # Close the test mysql connection
    try:
        cnx.close()
        return False
    except:
        return False

    return False

# @added 20170602 - Feature #2034: analyse_derivatives
def nonNegativeDerivative(timeseries):
    """
    This function is used to convert an integral or incrementing count to a
    derivative by calculating the delta between subsequent datapoints.  The
    function ignores datapoints that trend down and is useful for metrics that
    increase over time and then reset.
    This based on part of the Graphite render function nonNegativeDerivative at:
    https://github.com/graphite-project/graphite-web/blob/1e5cf9f659f5d4cc0fa53127f756a1916e62eb47/webapp/graphite/render/functions.py#L1627
    """

    derivative_timeseries = []
    prev = None

    for timestamp, datapoint in timeseries:
        if None in (prev, datapoint):
            # derivative_timeseries.append((timestamp, None))
            prev = datapoint
            continue

        diff = datapoint - prev
        if diff >= 0:
            derivative_timeseries.append((timestamp, diff))
        # else:
        #    derivative_timeseries.append((timestamp, None))

        prev = datapoint

    return derivative_timeseries

def strictly_increasing_monotonicity(timeseries):
    """
    This function is used to determine whether timeseries is strictly increasing
    monotonically, it will only return True if the values are strictly
    increasing, an incrementing count.
    """
    import numpy as np

    test_ts = []
    for timestamp, datapoint in timeseries:
        # This only identifies and handles positive, strictly increasing
        # monotonic timeseries
        if datapoint < 0.0:
            return False
        test_ts.append(datapoint)

    # Exclude timeseries that are all the same value, these are not increasing
    if len(set(test_ts)) == 1:
        return False

    # Exclude timeseries that sum to 0, these are not increasing
    ts_sum = sum(test_ts[1:])
    if ts_sum == 0:
        return False

    diff_ts = np.asarray(test_ts)
    return np.all(np.diff(diff_ts) >= 0)

def in_list(metric_name, check_list):
    """
    Check if the metric is in list.

    # @added 20170602 - Feature #2034: analyse_derivatives
    #                   Feature #1978: worker - DO_NOT_SKIP_LIST
    This is a part copy of the SKIP_LIST allows for a string match or a match on
    dotted elements within the metric namespace used in Horizon/worker

    """

    metric_namespace_elements = metric_name.split('.')
    metric_in_list = False
    for in_list in check_list:
        if in_list in metric_name:
            metric_in_list = True
            break
        in_list_namespace_elements = in_list.split('.')
        elements_matched = set(metric_namespace_elements) & set(in_list_namespace_elements)
        if len(elements_matched) == len(in_list_namespace_elements):
            metric_in_list = True
            break

    if metric_in_list:
        return True

    return False

# @added 20170825 - Task #2132: Optimise Ionosphere DB usage
# Get the metric db object data to memcache
def get_memcache_metric_object(current_skyline_app, base_name):
    """
    Return the metrics_db_object from memcache if it exists.

    """
    try:
        pymemcache_Client
    except:
        from pymemcache.client.base import Client as pymemcache_Client

    try:
        literal_eval
    except:
        from ast import literal_eval

    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)

    if settings.MEMCACHE_ENABLED:
        memcache_client = pymemcache_Client((settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT), connect_timeout=0.1, timeout=0.2)
    else:
        return False

    memcache_metrics_db_object = None
    metrics_db_object_key = 'metrics_db_object.%s' % str(base_name)
    memcache_result = None
    if settings.MEMCACHE_ENABLED:
        try:
            memcache_result = memcache_client.get(metrics_db_object_key)
        except:
            current_logger.error('error :: failed to get %s from memcache' % metrics_db_object_key)
        try:
            memcache_client.close()
        # @modified 20170913 - Task #2160: Test skyline with bandit
        # Added nosec to exclude from bandit tests
        except:  # nosec
            pass

    memcache_metric_dict = None
    if memcache_result:
        try:
            memcache_metrics_db_object = literal_eval(memcache_result)
            memcache_metric_dict = {}
            # for k, v in memcache_result_list:
            for k, v in memcache_metrics_db_object.iteritems():
                key_name = str(k)
                key_value = str(v)
                memcache_metric_dict[key_name] = key_value
        except:
            current_logger.error('error :: failed to process data from memcache key %s' % metrics_db_object_key)
            memcache_metric_dict = False
        try:
            memcache_client.close()
        # @modified 20170913 - Task #2160: Test skyline with bandit
        # Added nosec to exclude from bandit tests
        except:  # nosec
            pass

    if memcache_metric_dict:
        metrics_id = int(memcache_metric_dict['id'])
        metric_ionosphere_enabled = int(memcache_metric_dict['ionosphere_enabled'])
        metrics_db_object = memcache_metric_dict
        current_logger.info('get_memcache_metric_object :: returned memcache data for key - %s' % metrics_db_object_key)
        return metrics_db_object

    return False

# @added 20170826 - Task #2132: Optimise Ionosphere DB usage
# Get the fp_ids list object data to memcache
def get_memcache_fp_ids_object(current_skyline_app, base_name):
    """
    Return the fp_ids list from memcache if it exists.

    .. warning: this returns a list and to mimic the MySQL results rows, when
        this is used the list item must to turned into a dict to match the
        MySQL/SQLAlchemy format.

    """
    try:
        pymemcache_Client
    except:
        from pymemcache.client.base import Client as pymemcache_Client

    try:
        literal_eval
    except:
        from ast import literal_eval

    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)

    if settings.MEMCACHE_ENABLED:
        memcache_client = pymemcache_Client((settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT), connect_timeout=0.1, timeout=0.2)
    else:
        return False

    fp_ids_list_object_key = 'fp_ids_list_object.%s' % (str(base_name))

    memcache_result = None
    if settings.MEMCACHE_ENABLED:
        try:
            memcache_result = memcache_client.get(fp_ids_list_object_key)
        except:
            current_logger.error('error :: failed to get %s from memcache' % fp_ids_list_object_key)
        try:
            memcache_client.close()
        # @modified 20170913 - Task #2160: Test skyline with bandit
        # Added nosec to exclude from bandit tests
        except:  # nosec
            pass

    result = None
    if memcache_result:
        try:
            result = literal_eval(memcache_result)
        except:
            current_logger.error('error :: failed to process data from memcache key %s' % fp_ids_list_object_key)
            result = False
        try:
            memcache_client.close()
        # @modified 20170913 - Task #2160: Test skyline with bandit
        # Added nosec to exclude from bandit tests
        except:  # nosec
            pass

    return result

# @added 20171216 - Task #2236: Change Boundary to only send to Panorama on alert
def move_file(current_skyline_app, dest_dir, file_to_move):
    """
    Move a file.

    :param current_skyline_app: the skyline app using this function
    :param dest_dir: the directory the file is to be moved to
    :param file_to_move: path and filename of the file to move
    :type current_skyline_app: str
    :type dest_dir: str
    :type file_to_move: str
    :return: ``True``, ``False``
    :rtype: boolean

    """
    try:
        os.getpid()
    except:
        import os

    try:
        shutil
    except:
        import shutil

    try:
        python_version
    except:
        from sys import version_info
        python_version = int(version_info[0])

    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)

    if not os.path.exists(dest_dir):
        try:
            mkdir_p(dest_dir)
            current_logger.info(
                'created dest_dir - %s' % str(dest_dir))
        except:
            current_logger.info(traceback.format_exc())
            current_logger.error(
                'error :: failed to create dest_dir - %s' %
                str(dest_dir))
            return False

    move_file_name = os.path.basename(str(file_to_move))
    moved_file = '%s/%s' % (dest_dir, move_file_name)

    try:
        shutil.move(file_to_move, moved_file)
        if python_version == 2:
            os.chmod(moved_file, 0o644)
        if python_version == 3:
            os.chmod(moved_file, mode=0o644)

        current_logger.info('moved file to - %s' % moved_file)
        return True
    except OSError:
        current_logger.info(traceback.format_exc())
        msg = 'failed to move file to - %s' % moved_file
        current_logger.error('error :: %s' % msg)
        pass

    return False

# @added 20180107 - Branch #2270: luminosity
def is_derivative_metric(current_skyline_app, base_name):
    """
    Determine if a metric is a known derivative metric.

    :param current_skyline_app: the Skyline app that is calling the function
    :type current_skyline_app: str
    :param base_name: The metric base_name
    :type base_name: str
    :return: boolean
    :rtype: boolean

    """

    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)

    # Feature #2034: analyse_derivatives
    known_derivative_metric = False
    REDIS_CONN = None
    try:
        # @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow
        if settings.REDIS_PASSWORD:
            REDIS_CONN = StrictRedis(password=settings.REDIS_PASSWORD, unix_socket_path=settings.REDIS_SOCKET_PATH)
        else:
            REDIS_CONN = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH)
    except:
        from redis import StrictRedis
    if not REDIS_CONN:
        try:
            # @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow
            if settings.REDIS_PASSWORD:
                REDIS_CONN = StrictRedis(password=settings.REDIS_PASSWORD, unix_socket_path=settings.REDIS_SOCKET_PATH)
            else:
                REDIS_CONN = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH)
        except:
            current_logger.error('error :: known_derivative_metric - Redis connection failed')
    try:
        derivative_metrics = list(REDIS_CONN.smembers('derivative_metrics'))
    except:
        derivative_metrics = []
    try:
        non_derivative_metrics = list(REDIS_CONN.smembers('non_derivative_metrics'))
    except:
        non_derivative_metrics = []
    try:
        non_derivative_monotonic_metrics = settings.NON_DERIVATIVE_MONOTONIC_METRICS
    except:
        non_derivative_monotonic_metrics = []

    redis_metric_name = '%s%s' % (settings.FULL_NAMESPACE, str(base_name))
    if redis_metric_name in derivative_metrics:
        known_derivative_metric = True

    # First check if it has its own Redis z.derivative_metric key
    # that has not expired
    derivative_metric_key = 'z.derivative_metric.%s' % str(base_name)
    if not known_derivative_metric:
        last_derivative_metric_key = False
        try:
            last_derivative_metric_key = REDIS_CONN.get(derivative_metric_key)
        except Exception as e:
            current_logger.error('error :: could not query Redis for last_derivative_metric_key: %s' % e)

        if last_derivative_metric_key:
            # Until the z.derivative_metric key expires, it is classed
            # as such
            known_derivative_metric = True

    skip_derivative = in_list(base_name, non_derivative_monotonic_metrics)
    if skip_derivative:
        known_derivative_metric = False

    if known_derivative_metric:
        if redis_metric_name in non_derivative_metrics:
            known_derivative_metric = False

    if known_derivative_metric:
        return True

    return False

# @added 20180804 - Feature #2488: Allow user to specifically set metric as a derivative metric in training_data
def set_metric_as_derivative(current_skyline_app, base_name):
    """
    Add the metric to the derivative_metrics Redis set and create a
    z_derivative_metrics Redis key.

    :param current_skyline_app: the Skyline app that is calling the function
    :type current_skyline_app: str
    :param base_name: the metric base_name
    :type base_name: str
    :return: boolean
    :rtype: boolean

    """

    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)

    current_logger.info('set_metric_as_derivative :: %s' % str(base_name))
    return_boolean = True
    REDIS_CONN = None

    try:
        if settings.REDIS_PASSWORD:
            REDIS_CONN = StrictRedis(password=settings.REDIS_PASSWORD, unix_socket_path=settings.REDIS_SOCKET_PATH)
        else:
            REDIS_CONN = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH)
    except:
        from redis import StrictRedis
    if not REDIS_CONN:
        try:
            if settings.REDIS_PASSWORD:
                REDIS_CONN = StrictRedis(password=settings.REDIS_PASSWORD, unix_socket_path=settings.REDIS_SOCKET_PATH)
            else:
                REDIS_CONN = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH)
        except:
            current_logger.error('error :: set_metric_as_derivative - Redis connection failed')
    if not REDIS_CONN:
        return return_boolean

    metric_name = '%s%s' % (str(settings.FULL_NAMESPACE), str(base_name))
    try:
        REDIS_CONN.sadd('derivative_metrics', metric_name)
        current_logger.info('set_metric_as_derivative :: %s added to derivative_metrics Redis set' % str(metric_name))
    except:
        current_logger.info(traceback.format_exc())
        current_logger.error('error :: failed to add metric to Redis derivative_metrics set')
        return_boolean = False

    derivative_metric_key = 'z.derivative_metric.%s' % str(base_name)
    try:
        last_expire_set = int(time())
        REDIS_CONN.setex(
            derivative_metric_key, settings.FULL_DURATION, last_expire_set)
    except Exception as e:
        current_logger.error('error :: could not set Redis derivative_metric key: %s' % e)
        return_boolean = False

    # @added 20190108 - Bug #2796: set_metric_as_derivative remove from non_derivative_metrics
    # Remove the metric from the non_derivative_metrics Redis set.  This set is
    # normally managed by Analyzer and although the metric is added to the
    # derivative_metrics set above, it only gets removed from the
    # non_derivative_metrics set after the Analyzer Redis key
    # analyzer.derivative_metrics_expiry expires, which can be up to 300 seconds
    # when the derivative_metrics and non_derivative_metrics Redis sets are
    # recreated
    current_logger.info('set_metric_as_derivative :: removing %s from non_derivative_metrics' % str(base_name))
    try:
        non_derivative_metrics = list(REDIS_CONN.smembers('non_derivative_metrics'))
    except:
        non_derivative_metrics = []
    new_non_derivative_metrics = []
    len_non_derivative_metrics = len(non_derivative_metrics)
    current_logger.info('set_metric_as_derivative :: %s metrics in non_derivative_metrics before the removal' % str(len_non_derivative_metrics))
    if non_derivative_metrics:
        for i_metric_name in non_derivative_metrics:
            if i_metric_name != metric_name:
                new_non_derivative_metrics.append(i_metric_name)
    if new_non_derivative_metrics:
        all_redis_ops_ok = True
        for i_metric_name in new_non_derivative_metrics:
            try:
                REDIS_CONN.sadd('new_non_derivative_metrics', i_metric_name)
            except:
                all_redis_ops_ok = False
                current_logger.error('error :: failed to add metric %s to Redis new_non_derivative_metrics set' % str(i_metric_name))
        if all_redis_ops_ok:
            try:
                REDIS_CONN.rename('non_derivative_metrics', 'old_non_derivative_metrics')
            except Exception as e:
                current_logger.error('error :: could not rename Redis set non_derivative_metrics to old_non_derivative_metrics: %s' % str(e))
                return_boolean = False
            try:
                REDIS_CONN.rename('new_non_derivative_metrics', 'non_derivative_metrics')
                current_logger.info('set_metric_as_derivative :: removed %s from non_derivative_metrics' % str(base_name))
            except Exception as e:
                current_logger.error('error :: could not rename Redis set new_non_derivative_metrics to non_derivative_metrics: %s' % str(e))
                return_boolean = False
            try:
                REDIS_CONN.delete('old_non_derivative_metrics')
            except Exception as e:
                current_logger.error('error :: could not delta Redis set old_non_derivative_metrics: %s' % str(e))
    try:
        non_derivative_metrics = list(REDIS_CONN.smembers('non_derivative_metrics'))
    except:
        non_derivative_metrics = []
    len_non_derivative_metrics = len(non_derivative_metrics)
    current_logger.info('set_metric_as_derivative :: %s metrics in non_derivative_metrics after the removal of %s' % (str(len_non_derivative_metrics), str(metric_name)))

    return return_boolean

# @added 20190920 - Feature #3230: users DB table
#                   Ideas #2476: Label and relate anomalies
#                   Feature #2516: Add label to features profile
def get_user_details(current_skyline_app, desired_value, key, value):
    """
    Determines the user details for a user given the desired_value, key and
    value.  If you want the username of the user with id 1 then:
    get_user_details('username', 'id', 1)  # Will return 'Skyline'
    get_user_details('id', 'username', 'Skyline')  # Will return '1'

    :param current_skyline_app: the app calling the function so the function
        knows which log to write too.
    :param desired_value: the id or username
    :param key: the field for the value
    :param value: the value of the item you want to query in the key field
    :type current_skyline_app: str
    :type desired_value: str
    :type key: str
    :type value: str or int
    :return: tuple
    :rtype:  (boolean, str)

    """
    current_skyline_app_logger = str(current_skyline_app) + 'Log'
    current_logger = logging.getLogger(current_skyline_app_logger)

    if key == 'id':
        select_field = 'user'
        select_where = 'id'

    if key == 'username':
        select_field = 'id'
        select_where = 'user'

    if desired_value != select_field:
        fail_msg = 'error :: %s :: get_user_details :: desired_value %s is invalid in get_user_details(%s, %s %s)' % (
            str(current_skyline_app), str(desired_value), str(desired_value),
            str(key), str(value))
        current_logger.error('%s' % fail_msg)
        if current_skyline_app == 'webapp':
            # Raise to webbapp
            raise
        else:
            return False, None

    try:
        query = 'select %s from users WHERE %s = \'%s\'' % (select_field, select_where, str(value))  # nosec
        results = mysql_select(str(current_skyline_app), query)
        result = results[0][0]
    except:
        trace = traceback.format_exc()
        current_logger.error('%s' % trace)
        fail_msg = 'error :: %s :: get_user_details :: could not get user %s from database for get_user_details(%s, %s %s)' % (
            str(current_skyline_app), select_field, str(desired_value),
            str(key), str(value))
        current_logger.error('%s' % fail_msg)
        if current_skyline_app == 'webapp':
            # Raise to webbapp
            raise
        else:
            return False, None
        fail_msg = 'error :: %s :: get_user_details :: could not get user %s from database for get_user_details(%s, %s %s)' % (
            str(current_skyline_app), select_field, str(desired_value),
            str(key), str(value))
        current_logger.info('%s :: get_user_details :: determined %s of %s for users.%s = %s' % (
            select_field, str(result), select_where), str(value))
    if select_field == 'id':
        return True, int(result)
    return True, str(result)

max_stars_repo_name: wfloutier/skyline

max_issues_repo_name: wfloutier/skyline

max_forks_repo_name: wfloutier/skyline