text:
# The MIT License (MIT)
# Copyright (c) 2018 Massachusetts Institute of Technology
#
# Author: Guillaume Rongier
# This software has been created in projects supported by the US National
# Science Foundation and NASA (PI: Pankratius)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import numpy as np

from osgeo import gdal, osr
from numba import jit

################################################################################
# Array coordinates
################################################################################

def transform_to_pixel_coordinates(x, y,
                                   x_min, x_max,
                                   y_min, y_max,
                                   array_width, array_height):
    '''
    Transform some geographic coordinates to pixel coordinates in an array

    @param x: Coordinate along the x axis to transform
    @param y: Coordinate along the y axis to transform
    @param x_min: Minimal coordinate of the array along the x axis (along the cell border)
    @param x_max: Maximal coordinate of the array along the x axis (along the cell border)
    @param y_min: Minimal coordinate of the array along the y axis (along the cell border)
    @param y_max: Maximal coordinate of the array along the y axis (along the cell border)
    @param array_width: Width of the array (i.e., along the x axis)
    @param array_height: Height of the array (i.e., along the y axis)

    @return The pixel coordinates
    '''
    u = (x - x_min)*(array_width/(x_max - x_min))
    v = (y_max - y)*(array_height/(y_max - y_min))

    return np.int_(u), np.int_(v)

def transform_to_geographic_coordinates(u, v,
                                        x_min, x_max,
                                        y_min, y_max,
                                        array_width, array_height):
    '''
    Transform some pixel coordinates in an array to geographic coordinates

    @param u: Pixel coordinate along the x axis to transform
    @param v: Pixel coordinate along the y axis to transform
    @param x_min: Minimal coordinate of the array along the x axis (along the cell border)
    @param x_max: Maximal coordinate of the array along the x axis (along the cell border)
    @param y_min: Minimal coordinate of the array along the y axis (along the cell border)
    @param y_max: Maximal coordinate of the array along the y axis (along the cell border)
    @param array_width: Width of the array (i.e., along the x axis)
    @param array_height: Height of the array (i.e., along the y axis)

    @return The geographic coordinates at the center of the pixel
    '''
    pixel_x_size = (x_max - x_min)/array_width
    pixel_y_size = (y_max - y_min)/array_height
    x = x_min + u*pixel_x_size + 0.5*pixel_x_size
    y = y_max - v*pixel_y_size - 0.5*pixel_y_size

    return x, y

def compute_x_and_y_coordinates_maps(x_min, x_max,
                                     y_min, y_max,
                                     array_width, array_height):
    '''
    Compute an array of x and y coordinates based on an extent and array shape

    @param x_min: Minimal coordinate along the x axis (along the cell border)
    @param x_max: Maximal coordinate along the x axis (along the cell border)
    @param y_min: Minimal coordinate along the y axis (along the cell border)
    @param y_max: Maximal coordinate along the y axis (along the cell border)
    @param array_width: Width of the array (i.e., along the x axis)
    @param array_height: Height of the array (i.e., along the y axis)

    @return The coordinates' arrays
    '''
    pixel_x_size = (x_max - x_min)/array_width
    pixel_y_size = (y_max - y_min)/array_height
    x_array = np.linspace(x_min + 0.5*pixel_x_size,
                          x_max - 0.5*pixel_x_size,
                          array_width)
    y_array = np.linspace(y_max - 0.5*pixel_y_size,
                          y_min + 0.5*pixel_y_size,
                          array_height)

    return np.meshgrid(x_array, y_array)

def extract_subgeoarray(georaster_array,
                        georaster_extent,
                        x_min, x_max,
                        y_min, y_max,
                        center_extent = False):
    '''
    Extract a sub-array given some geographical coordinates. The new extent's
    coordinates don't have to be along a pixel border

    @param georaster_array: A 2D NumPy array
    @param georaster_extent: The array current extent (along the cells' borders)
    @param x_min: New minimal coordinate along the x axis (along the cell border)
    @param x_max: New maximal coordinate along the x axis (along the cell border)
    @param y_min: New minimal coordinate along the y axis (along the cell border)
    @param y_max: New maximal coordinate along the y axis (along the cell border)
    @param center_extent: Whether the new extent should be along the cells'
                          borders or centers

    @return The sub-array and its extent
    '''
    assert len(georaster_array.shape) >= 2, "The array must be two-dimensional"
    assert len(georaster_extent) == 4, "The extent must have contain 4 coordinates"

    i_min, j_min = transform_to_pixel_coordinates(x_min, y_min,
                                                  georaster_extent[0], georaster_extent[1],
                                                  georaster_extent[2], georaster_extent[3],
                                                  georaster_array.shape[-1], georaster_array.shape[-2])
    i_max, j_max = transform_to_pixel_coordinates(x_max, y_max,
                                                  georaster_extent[0], georaster_extent[1],
                                                  georaster_extent[2], georaster_extent[3],
                                                  georaster_array.shape[-1], georaster_array.shape[-2])
    new_georaster_array = georaster_array[..., j_max:j_min, i_min:i_max]

    new_x_min, new_y_min = transform_to_geographic_coordinates(i_min, j_min,
                                                               georaster_extent[0], georaster_extent[1],
                                                               georaster_extent[2], georaster_extent[3],
                                                               georaster_array.shape[-1], georaster_array.shape[-2])
    new_x_max, new_y_max = transform_to_geographic_coordinates(i_max, j_max,
                                                               georaster_extent[0], georaster_extent[1],
                                                               georaster_extent[2], georaster_extent[3],
                                                               georaster_array.shape[-1], georaster_array.shape[-2])

    if center_extent == False:
        pixel_x_size = (georaster_extent[1] - georaster_extent[0])/georaster_array.shape[-1]
        pixel_y_size = (georaster_extent[3] - georaster_extent[2])/georaster_array.shape[-2]
        new_x_min -= 0.5*pixel_x_size
        new_y_min += 0.5*pixel_y_size
        new_x_max -= 0.5*pixel_x_size
        new_y_max += 0.5*pixel_y_size

    new_georaster_extent = (new_x_min, new_x_max, new_y_min, new_y_max)

    return new_georaster_array, new_georaster_extent

@jit(nopython = True)
def get_valid_subarray_indexes(array,
                               subarray_shape,
                               steps = (1, 1),
                               pads = (0, 0),
                               is_shape_centered = False):
    '''
    Get the indexes of all the possible sub-arrays that do not contain any NaN

    @param array: A ND NumPy array. The N - 2 first axes represent the variables,
                  the last two axes represent which the indexes are gotten
    @param subarray_shape: The 2D shape of the sub-arrays
    @param steps: The step between each sub-array for the last two axes, to avoid
                  sampling all the possible sub-arrays
    @param pads: Extra row(s) and column(s) to get around the subarrays, without
                 influencing the subarray indexes
    @param is_shape_centered: True if the sub-arrays are defined from the
                              central cell, false if they are defined from the
                              top-left cell

    @return The 2D indexes of the sub-arrays in a list
    '''
    assert len(array.shape) >= 2, 'Array must be at least 2D'
    assert len(subarray_shape) == 2, 'Sub-array shape must be 2D'

    left_shape = (0, 0)
    right_shape = (subarray_shape[-2],
                   subarray_shape[-1])
    if is_shape_centered == True:
        left_shape = (int(subarray_shape[-2]/2),
                      int(subarray_shape[-1]/2))
        right_shape = (int(subarray_shape[-2]/2) + 1,
                       int(subarray_shape[-1]/2) + 1)

    subarray_indexes = []
    for j in range(left_shape[-2] + pads[-2],
                   array.shape[-2] - right_shape[-2] - pads[-2],
                   steps[-2]):
        for i in range(left_shape[-1] + pads[-1],
                       array.shape[-1] - right_shape[-1] - pads[-1],
                       steps[-1]):
            subarray = array[...,
                             j - left_shape[-2]:j + right_shape[-2],
                             i - left_shape[-1]:i + right_shape[-1]]
            if np.isnan(subarray).any() == False:
                subarray_indexes.append((j, i))

    return subarray_indexes

@jit(nopython = True)
def extract_subarrays(array,
                      sample_array_shape,
                      subarray_indexes,
                      pads = (0, 0),
                      is_shape_centered = False):
    '''
    Extract all the possible sub-arrays from their indexes

    @param array: A ND NumPy array. The N - 2 first axes represent the variables,
                  the last two axes represent which the indexes are gotten
    @param sample_array_shape: The (N + 1)D shape of the array that will contain
                               the sub-arrays
    @param subarray_indexes: A list of 2D indexes that locate the sub-arrays
    @param pads: Extra row(s) and column(s) to get around the subarrays, without
                 influencing the subarray indexes
    @param is_shape_centered: True if the sub-arrays are defined from the
                              central cell, false if they are defined from the
                              top-left cell

    @return The sub-arrays in a (N + 1)D NumPy array
    '''
    assert len(array.shape) >= 2, 'Array must be at least 2D'
    assert len(sample_array_shape) == len(array.shape) + 1, 'Sample array shape must be one dimension higher than array'

    left_shape = (pads[-2], pads[-1])
    right_shape = (sample_array_shape[-2] - pads[-2],
                   sample_array_shape[-1] - pads[-1])
    if is_shape_centered == True:
        left_shape = (int(sample_array_shape[-2]/2),
                      int(sample_array_shape[-1]/2))
        right_shape = (int(sample_array_shape[-2]/2) + 1,
                       int(sample_array_shape[-1]/2) + 1)

    sample_array = np.empty(sample_array_shape)
    for i in range(sample_array_shape[0]):
        sample_array[i] = array[...,
                                subarray_indexes[i][0] - left_shape[0]:subarray_indexes[i][0] + right_shape[0],
                                subarray_indexes[i][1] - left_shape[1]:subarray_indexes[i][1] + right_shape[1]]

    return sample_array

def sample_array(array,
                 subarray_shape,
                 steps = (1, 1),
                 pads = (0, 0),
                 is_shape_centered = False,
                 return_subarray_indexes = False):
    '''
    Sample an array of all the possible sub-arrays that do not contain any NaN

    @param array: A ND NumPy array. The N - 2 first axes represent the variables,
                  the last two axes represent which the indexes are gotten
    @param subarray_shape: The 2D shape of the sub-arrays
    @param steps: The step between each sub-array for the last two axes, to avoid
                  sampling all the possible sub-arrays
    @param pads: Extra row(s) and column(s) to get around the subarrays, without
                 influencing the subarray indexes
    @param is_shape_centered: True if the sub-arrays are defined from the
                              central cell, false if they are defined from the
                              top-left cell

    @return The sub-arrays in a (N + 1)D NumPy array
    '''
    assert len(array.shape) >= 2, 'Array must be at least 2D'
    assert len(subarray_shape) == 2, 'Sub-array shape must be 2D'
    assert len(steps) == 2, 'Steps must be 2D'
    assert (is_shape_centered == False or
            (is_shape_centered == True
             and (subarray_shape[0] - 1)%2 == 0
             and (subarray_shape[1] - 1)%2 == 0)), 'When centered, the subarray shape must be odd'

    subarray_indexes = get_valid_subarray_indexes(array,
                                                  subarray_shape,
                                                  steps = steps,
                                                  pads = pads,
                                                  is_shape_centered = is_shape_centered)
    sample_array_shape = tuple([len(subarray_indexes)]
                               + list(array.shape[:-2])
                               + [subarray_shape[-2] + 2*pads[-2], subarray_shape[-1] + 2*pads[-1]])
    samples_array = extract_subarrays(array,
                                      sample_array_shape,
                                      subarray_indexes,
                                      pads = pads,
                                      is_shape_centered = is_shape_centered)

    if return_subarray_indexes == False:
        return samples_array
    else:
        return samples_array, subarray_indexes

################################################################################
# Projection
################################################################################

def reproject_point(lon,
                    lat,
                    old_projection_EPSG = None,
                    old_projection_wkt = None,
                    old_projection_utm = None,
                    new_projection_EPSG = None,
                    new_projection_wkt = None,
                    new_projection_utm = None):
    '''
    Reproject a single point

    @param lon: Longitude of the point
    @param lat: Latitude of the point
    @param old_projection_EPSG: EPSG code of the old projection
    @param old_projection_wkt: WKT code of the old projection (can be used instead
                               of the old_projection_EPSG)
    @param old_projection_utm: Tuple with the UTM zone code and if it's northern or not
    @param new_projection_EPSG: EPSG code of the new projection
    @param new_projection_wkt: WKT code of the new projection (can be used instead
                               of the new_projection_EPSG)
    @param new_projection_utm: Tuple with the UTM zone code and if it's northern or not

    @return The coordinates' arrays
    '''
    assert (old_projection_EPSG is not None
            or old_projection_wkt is not None
            or old_projection_utm is not None), 'No old projection provided'
    assert (new_projection_EPSG is not None
            or new_projection_wkt is not None
            or new_projection_utm is not None), 'No new projection provided'

    old_spatial_reference = osr.SpatialReference()
    if old_projection_EPSG is not None:
        old_spatial_reference.ImportFromEPSG(old_projection_EPSG)
    elif old_projection_wkt is not None:
        old_spatial_reference.ImportFromWkt(old_projection_wkt)
    elif new_projection_utm is not None:
        assert len(new_projection_utm) == 2, 'UTM projection requires a zone code and an hemisphere code'
        old_spatial_reference.SetUTM(old_projection_utm[0], old_projection_utm[1])

    new_spatial_reference = osr.SpatialReference()
    if new_projection_EPSG is not None:
        new_spatial_reference.ImportFromEPSG(new_projection_EPSG)
    elif new_projection_wkt is not None:
        new_spatial_reference.ImportFromWkt(new_projection_wkt)
    elif new_projection_utm is not None:
        assert len(new_projection_utm) == 2, 'UTM projection requires a zone code and an hemisphere code'
        new_spatial_reference.SetUTM(new_projection_utm[0], new_projection_utm[1])

    transform = osr.CoordinateTransformation(old_spatial_reference,
                                             new_spatial_reference)

    return transform.TransformPoints([(lon, lat)])[0]

def find_utm_area(longitude, latitude):
    '''
    Find the UTM code and hemisphere from the longitude and latitude of a point

    @param longitude: A float for the longitude
    @param latitude: A float for the latitude

    @return A tuple with the code of the UTM zone and the hemisphere
            (1: northern hemisphere; 0: southern hemisphere)
    '''
    utm_zone = int(1 + (longitude + 180.)/6.)
    is_northern = 1
    if latitude < 0.:
        is_northern = 0

    return (utm_zone, is_northern)

def reproject_georaster(georaster,
                        new_cell_sizes,
                        new_projection_EPSG = None,
                        new_projection_wkt = None,
                        new_projection_utm = None,
                        new_extent = None,
                        interpolation_method = gdal.GRA_Cubic,
                        file_type = 'MEM',
                        file_path = '',
                        data_type = gdal.GDT_Float64,
                        no_data_value = -99999.,
                        scale = 1.,
                        offset = 0.,
                        options = []):
    '''
    Change the projection of a GDAL georaster

    @param georaster: The GDAL georaster
    @param new_cell_sizes: Sizes (x, y) for cells of the georaster in the new projection
    @param new_projection_EPSG: EPSG code of the new projection
    @param new_projection_wkt: WKT code of the new projection (can be used instead
                               of the new_projection_EPSG)
    @param new_projection_utm: Tuple with the UTM zone code and if it's northern or not
    @param new_extent: Tuple with the minimal x, maximal x, minimal y, maximal y
                       for the new georaster
    @param interpolation_method: Interpolation method used during the projection
    @param file_type: Type to save the file (default is memory)
    @param file_path: Where to store the new georasterEPSG_code (default is memory)
    @param data_type: Data type of the georaster
    @param no_data_value: No data value for the georaster
    @param scale: Scaling factor for the georaster
    @param offset: Offset factor for the georaster
    @param options: List of options for compression

    @return The GDAL georaster
    '''
    old_spatial_reference = osr.SpatialReference()
    old_spatial_reference.ImportFromWkt(georaster.GetProjectionRef())
    old_geotransform = georaster.GetGeoTransform()
    old_georaster_x_size = georaster.RasterXSize
    old_georaster_y_size = georaster.RasterYSize

    new_spatial_reference = osr.SpatialReference()
    if new_projection_EPSG is not None:
        new_spatial_reference.ImportFromEPSG(new_projection_EPSG)
    elif new_projection_wkt is not None:
        new_spatial_reference.ImportFromWkt(new_projection_wkt)
    elif new_projection_utm is not None:
        assert len(new_projection_utm) == 2, 'UTM projection requires a zone code and an hemisphere code'
        new_spatial_reference.SetUTM(new_projection_utm[0], new_projection_utm[1])
    else:
        if new_extent is None:
            print('No new spatial reference provided, will use the one from the georaster')
        new_spatial_reference.ImportFromWkt(georaster.GetProjectionRef())

    new_x_size = None
    new_y_size = None
    new_geotransform = None
    if new_extent is None:
        transform = osr.CoordinateTransformation(old_spatial_reference,
                                                 new_spatial_reference)
        (ulx, uly, ulz) = transform.TransformPoint(old_geotransform[0],
                                                   old_geotransform[3])
        (lrx, lry, lrz) = transform.TransformPoint(old_geotransform[0] + old_geotransform[1]*old_georaster_x_size,
                                                   old_geotransform[3] + old_geotransform[5]*old_georaster_y_size)
        new_geotransform = (ulx,
                            new_cell_sizes[0],
                            old_geotransform[2],
                            uly,
                            old_geotransform[4],
                            -new_cell_sizes[1])
        new_x_size = int(abs(lrx - ulx)/new_cell_sizes[0])
        new_y_size = int(abs(uly - lry)/new_cell_sizes[1])
    else:
        new_geotransform = (new_extent[0],
                            new_cell_sizes[0],
                            0.,
                            new_extent[3],
                            0.,
                            -new_cell_sizes[1])
        new_x_size = int(round(abs(new_extent[1] - new_extent[0])/new_cell_sizes[0]))
        new_y_size = int(round(abs(new_extent[3] - new_extent[2])/new_cell_sizes[1]))

    number_of_bands = georaster.RasterCount

    driver = gdal.GetDriverByName(file_type)
    new_georaster = driver.Create(file_path,
                                  new_x_size,
                                  new_y_size,
                                  number_of_bands,
                                  data_type,
                                  options = options)
    new_georaster.SetGeoTransform(new_geotransform)
    new_georaster.SetProjection(new_spatial_reference.ExportToWkt())
    for i_band in range(1, number_of_bands + 1):
        new_georaster_band = new_georaster.GetRasterBand(i_band)
        new_georaster_band.SetNoDataValue(no_data_value)
        new_georaster_band.SetScale(scale)
        new_georaster_band.SetOffset(offset)
        # Fill the georaster band, otherwise no data values are not set in the new georaster
        new_georaster_band.Fill(no_data_value)

    res = gdal.ReprojectImage(georaster,
                              new_georaster,
                              old_spatial_reference.ExportToWkt(),
                              new_spatial_reference.ExportToWkt(),
                              interpolation_method)

    return new_georaster

def georaster_vertical_datum_shift(georaster,
                                   old_datum_proj4 = '+proj=longlat +datum=WGS84 +no_defs +geoidgrids=egm96_15.gtx',
                                   new_datum_proj4 = '+proj=longlat +datum=WGS84 +no_defs',
                                   file_type = 'MEM',
                                   file_path = '',
                                   data_type = gdal.GDT_Float64,
                                   no_data_value = -99999.,
                                   scale = 1.,
                                   offset = 0.):
    '''
    Change the vertical datum of a GDAL georaster, from a geoid to an ellipsoid
    or the other way around

    @param georaster: The GDAL georaster
    @param old_datum_proj4: Proj4 code for the old datum, with the elevation
                            being defined relatide to an ellipsoid or a geoid.
                            Default value is for the EGM96 geoid, whose grid
                            needs to be downloaded beforehand
    @param new_datum_proj4: Proj4 code for the new datum, with the elevation
                            being defined relatide to an ellipsoid or a geoid.
                            Default value is for the WGS84 ellipsoid.
    @param file_type: Type to save the file (default is memory)
    @param file_path: Where to store the new georasterEPSG_code (default is memory)
    @param data_type: Data type of the georaster
    @param no_data_value: No data value for the georaster
    @param scale: Scaling factor for the georaster
    @param offset: Offset factor for the georaster

    @return The GDAL georaster
    '''
    geotransform = georaster.GetGeoTransform()
    georaster_x_size = georaster.RasterXSize
    georaster_y_size = georaster.RasterYSize

    number_of_bands = georaster.RasterCount

    driver = gdal.GetDriverByName(file_type)
    new_georaster = driver.Create(file_path,
                                  georaster_x_size,
                                  georaster_y_size,
                                  number_of_bands,
                                  data_type)
    new_georaster.SetGeoTransform(geotransform)
    new_georaster.SetProjection(new_datum_proj4)
    for i_band in range(1, number_of_bands + 1):
        new_georaster_band = new_georaster.GetRasterBand(i_band)
        new_georaster_band.SetNoDataValue(no_data_value)
        new_georaster_band.SetScale(scale)
        new_georaster_band.SetOffset(offset)
        # Fill the georaster band, otherwise no data values are not set in the new georaster
        new_georaster_band.Fill(no_data_value)

    res = gdal.Warp(new_georaster,
                    georaster,
                    srcSRS = old_datum_proj4,
                    dstSRS = new_datum_proj4)

    return new_georaster

max_stars_repo_name: MITeaps/pyinsar

max_issues_repo_name: MITeaps/pyinsar

max_forks_repo_name: MITeaps/pyinsar

reason: import numpy,from numba


text:
import numpy as np
from PIL import Image, ImageDraw
from util import BBox, V2
from math import floor, ceil

def calculate_composite_bbox_and_dimensions(tiles):
    """
    calculate the geographic bounding box and the required
    composite image dimensions for the given tiles.
    """
    bbox = None
    for i, tile in enumerate(tiles):
        if i == 0:
            bbox = BBox(tile['bbox'].min, tile['bbox'].max)
        else:
            bbox.expand(tile['bbox'].min)
            bbox.expand(tile['bbox'].max)

    # Use the first tile bounding box and dimensions - the assumption here is
    # that all the tile are of the same size, but this is not checked
    # The fucntion should really assert this assumption.
    tile_image_width = tiles[0]['img'].size[0]
    tile_image_height = tiles[0]['img'].size[1]
    width = round(bbox.width / tiles[0]['bbox'].width) * tile_image_width
    height = round(bbox.height / tiles[0]['bbox'].height) * tile_image_height
    return (bbox, (width, height))

def create_composite_image(tiles, bbox, dims):
    """
    With the given tileset, create a new image with the given
    bounding box and dimensions. Returns a PIL image.
    """
    (width, height) = dims

    # Default is black with 0 opacity
    composite_img = Image.new('RGBA', (width, height), (0, 0, 0, 0))

    for i, tile in enumerate(tiles):
        offset_x = round(
            (tile['bbox'].min.x - bbox.min.x) / bbox.width * width)
        offset_y = round(
            -(tile['bbox'].max.y - bbox.max.y) / bbox.height * height)
        composite_img.paste(tile['img'], (offset_x, offset_y))
    return composite_img

def geo_json_bbox(coordinates):
    """
    Calculate the bounding box from the GeoJSON coordinates.
    """
    bbox = None
    for i, coordinate in enumerate(coordinates):
        vec = V2(coordinate[0], coordinate[1])
        if i == 0:
            bbox = BBox(vec, vec)
        else:
            bbox.expand(vec)
    return bbox

def calculate_crop(image, uncropped_bbox, boundary_bbox):
    """
    Crop an image with a given geographic bounding to the new bounding box.
    As we round up the pixel size of the new image, the geographic bounding
    box of new image will be potentially slightly larger then the required one,
    so also calculate the correct bounding box for the result.
    """
    pixel_min_x = floor(
        (boundary_bbox.min.x - uncropped_bbox.min.x) /
        uncropped_bbox.width * image.size[0])
    pixel_min_y = floor(
        (uncropped_bbox.max.y - boundary_bbox.max.y) /
        uncropped_bbox.height * image.size[1])
    pixel_max_x = ceil(
        (boundary_bbox.max.x - uncropped_bbox.min.x) /
        uncropped_bbox.width * image.size[0])
    pixel_max_y = ceil(
        (uncropped_bbox.max.y - boundary_bbox.min.y) /
        uncropped_bbox.height * image.size[1])
    cropped_pixels = (pixel_min_x, pixel_min_y, pixel_max_x, pixel_max_y)

    cropped_bbox = BBox(
        V2(
            uncropped_bbox.min.x + pixel_min_x /
            image.size[0] * uncropped_bbox.width,
            uncropped_bbox.max.y - pixel_max_y /
            image.size[1] * uncropped_bbox.height),
        V2(
            uncropped_bbox.min.x + pixel_max_x /
            image.size[0] * uncropped_bbox.width,
            uncropped_bbox.max.y - pixel_min_y /
            image.size[1] * uncropped_bbox.height))
    return (cropped_pixels, cropped_bbox)

def crop_to_boundary(boundary_coordinates, composite_img, composite_bbox):
    """
    Crop an image to the given GeoJSON boundary.
    """
    boundary_bbox = geo_json_bbox(boundary_coordinates)
    (cropped_pixels, cropped_bbox) = calculate_crop(
        composite_img, composite_bbox, boundary_bbox)
    return (composite_img.crop(cropped_pixels), cropped_bbox)

def create_boundary_pixels(coordinates, bbox, image_dims):
    """
    For the given GeoJSON coordinates, map to a list
    of pixel positions.
    """
    (image_width, image_height) = image_dims

    def coordinate_to_pixels(coordinate):
        return (
            round(
                (coordinate[0] - bbox.min.x) / bbox.width * image_width),
            round(
                (bbox.max.y - coordinate[1]) / bbox.height * image_height))

    pixels = map(coordinate_to_pixels, coordinates)
    return list(pixels)

def mask_pixels_outside_boundary(cropped_to_boundary_image,
                                 boundary_coordinates,
                                 cropped_bbox):
    """
    Mask the image to only include pixels inside the GeoJSON boundary.
    """
    polygon_pixels = create_boundary_pixels(
        boundary_coordinates,
        cropped_bbox,
        cropped_to_boundary_image.size)
    mask_image = Image.new('L', cropped_to_boundary_image.size, 0)
    ImageDraw.Draw(mask_image).polygon(polygon_pixels, outline=0, fill=1)
    mask_raw = np.array(mask_image)

    cropped_to_bbox_raw = np.asarray(cropped_to_boundary_image)
    result_raw = np.empty(cropped_to_bbox_raw.shape, dtype='uint8')
    # Mask all channels to reduce file size.
    # How to do RGB in one step?
    result_raw[:, :, 0] = mask_raw * cropped_to_bbox_raw[:, :, 0]
    result_raw[:, :, 1] = mask_raw * cropped_to_bbox_raw[:, :, 1]
    result_raw[:, :, 2] = mask_raw * cropped_to_bbox_raw[:, :, 2]
    result_raw[:, :, 3] = mask_raw * 255
    result_image = Image.fromarray(result_raw, 'RGBA')
    return result_image

ext: py

max_stars_repo_name: bjnortier/tile-crop-py

max_issues_repo_name: bjnortier/tile-crop-py

max_forks_repo_name: bjnortier/tile-crop-py

reason: import numpy


text:
import cv2
import shutil
import os.path as osp
import os
import numpy as np
from tqdm import tqdm
from xml.etree import cElementTree as ET
from glob import glob
from sklearn.model_selection import train_test_split

import sys
sys.path.append('/mnt/data/rz/programe/KREP')

from tools.filter_char import find_unchinese
from util.order_points import sort_box
from util.make_dirs import make_dirs

# Truncates numbers to N decimals
def truncate(n, decimals=0):
	multiplier = 10 ** decimals
	return int(n * multiplier) / multiplier

# extract voc format data
def extract_voc(xmlp, split_flag, default_class_name=None):
	if not os.path.exists(xmlp):
		print('{} file is not exists.'.format(xmlp))
		return
	with open(xmlp) as f:
		tree = ET.parse(f)
		root = tree.getroot()

	filename = root.find('filename').text  # get image file name
	folder = root.find('folder').text
	imgH = root.find('imagesize').find('nrows').text
	imgW = root.find('imagesize').find('ncols').text
	imgH, imgW = int(imgH), int(imgW)
	lines = []
	objects = root.findall('object')
	if len(objects) > 0:
		for obj in objects:
			name = obj.find('name').text  # class name
			if default_class_name is not None:
				name = default_class_name
			# find polygon
			polys = obj.find('polygon').findall('pt')
			points = []
			for pt in polys:
				x, y = pt.find('x').text, pt.find('y').text
				points.extend(map(int, [float(x), float(y)]))
			# find attributes
			att = obj.find('attributes').text

			if att is None:
				att = '###'
			line = split_flag.join(map(str, points)) + split_flag + name + split_flag + att
			lines.append(line)
		return {'filename': filename,
				'items': lines,
				'imgH': imgH,
				'imgW': imgW,
				'folder': folder,
				'attributes': att}
	return {'filename': filename,
			'items': '',
			'imgH': imgH,
			'imgW': imgW,
			'folder': folder,
			'attributes': '###'}

def extract_cvat_xml(xmlp):
	error_image_names = [] # store empty image name
	if not os.path.exists(xmlp):
		print('{} file is not exists.'.format(xmlp))
		return
	with open(xmlp) as f:
		tree = ET.parse(f)
		root = tree.getroot()

	image_obj = root.findall('image')
	results = []
	for imobj in image_obj:
		location = ''
		filename = imobj.get('name')
		filename = filename.split('/')[-1]
		points = imobj.findall('points')
		if len(points) == 0:
			error_image_names.append(filename)
			continue
		for pt in points:
			loc = pt.get('points')
			location += loc + ','
		if location[-1] == ',':
			location = location[:-1]
		location = location.split(',')
		item = {'location': location,
				'filename': filename}
		results.append(item)
	return results, error_image_names

# convert to dota format
def voc_to_dota(xml_root, txt_store_root, split_flag):
	xml_paths = glob(os.path.join(xml_root, '*.xml'))
	for xmlp in tqdm(xml_paths):
		content = extract_voc(xmlp, split_flag)
		if content is None:
			print('xml path:{} is error'.format(xmlp))
			continue
		name = content['filename']
		lines = content['items']
		# dota line format: x1,y1,x2,y2,x3,y3,x4,y4,category,difficult
		dota_lines = ['imagesource:GF-2', 'gsd:null']
		# dota_lines = []
		difficult = '0'  # 0 or 1; 1->difficult, 0->not difficult
		for line in lines:
			line = line.strip().split('\t')[:2]
			line = split_flag.join(line)
			line = line + split_flag + difficult
			dota_lines.append(line)
		txt_name = name.split('.')[0] + '.txt'
		txt_store_path = os.path.join(txt_store_root, txt_name)
		with open(txt_store_path, 'w') as f:
			f.write('\n'.join(dota_lines))

# convert to icdar format
def voc_to_icdar(xml_root, txt_store_root, split_flag, filter_classes=[]):
	error_names = []
	xml_paths = glob(os.path.join(xml_root, '*.xml'))
	for xmlp in tqdm(xml_paths):
		content = extract_voc(xmlp, split_flag)
		if content is None:
			print('xml path:{} is error'.format(xmlp))
			continue
		name = content['filename']
		lines = content['items']
		icdar_lines = []
		for line in lines:
			loc, cls, ct = line.split('\t')
			if cls in filter_classes:  # 过滤掉在filter classes中出现的类
				continue
			txt = ct.replace('content=', '')
			if len(txt) == 0:
				txt = '###'
			if txt[0] == "'":
				txt = txt[1:]
			line = loc + ',' + txt + ',' + cls
			icdar_lines.append(line)

		# store
		if len(icdar_lines) == 0:
			error_names.append(name)
			continue
		txt_name = name.split('.')[0] + '.txt'
		txt_store_path = os.path.join(txt_store_root, txt_name)
		with open(txt_store_path, 'w') as f:
			f.write('\n'.join(icdar_lines))
	return error_names

def voc_to_yolov5(xml_root, txt_store_root, class_idx, split_flag=',', filter_classes=[]):
	error_names = []
	if not osp.exists(txt_store_root):
		os.makedirs(txt_store_root)
	xml_paths = glob(osp.join(xml_root, '*.xml'))
	for xmlp in tqdm(xml_paths):
		content = extract_voc(xmlp, split_flag=split_flag)
		img_name = content['filename']
		lines = content['items']
		imgH = content['imgH']
		imgW = content['imgW']
		new_lines = []
		for line in lines:
			line = line.split(split_flag)
			loc, cls = line[:8], line[8]
			if cls in filter_classes:  # 过滤掉在filter classes中出现的类
				continue
			# if cls == 'icon':
			#     cls = 'element'
			loc = list(map(int, loc))
			xmin, ymin, xmax, ymax = loc[0], loc[1], loc[4], loc[5]
			dw, dh = 1. / imgW, 1 / imgH
			x = (xmin + xmax)/2
			y = (ymin + ymax)/2
			w = xmax - xmin
			h = ymax - ymin
			x = x * dw
			w = w * dw
			y = y * dh
			h = h * dh
			idx = class_idx[cls]
			# idx = '0'
			new_line = ' '.join([idx,
								 str(truncate(x, 7)),
								 str(truncate(y, 7)),
								 str(truncate(w, 7)),
								 str(truncate(h, 7))])
			new_lines.append(new_line)
		txt_name = img_name.split('.')[0] + '.txt'
		txt_store_path = osp.join(txt_store_root, txt_name)
		if len(new_lines) > 0:
			with open(txt_store_path, 'w') as f:
				f.write('\n'.join(new_lines))
		else:
			error_names.append(img_name)
	return error_names

# 将voc格式的xml文件转成单个字段识别的格式
# suffix: 在做line的filename那一栏时，是否需要添加前缀
# 登记证里目前没有加入全中文的训练，所以需要过滤掉一些样本
def voc_to_rec(xmlp, split_flag, suffix=''):
	content = extract_voc(xmlp, split_flag)
	filename = content['filename']
	att = content['attributes']
	if len(att) > 0:
		txt = att.split('=')[-1]
		if txt[0] == "'":
			txt = txt[1:]
		filtered_txt = find_unchinese(txt)
		if len(filtered_txt) < 4:
			return None, filename
		elif '/' in txt or '.' in txt:
			return None, filename
		if suffix != '':
			line = os.path.join(suffix, filename) + '\t' + txt
		else:
			line = filename + '\t' + txt
		return line, filename
	else:
		return None, filename

# cvat icdar format to common icdar format
def load_cvat_icdar(txtp, content='content'):
	results = []
	with open(txtp) as f:
		for line in f:
			line = line.strip().split()
			line = list(map(float, line))
			line = [round(n) for n in line]
			x1, y1, x2, y2 = line
			line = [x1, y1, x2, y1, x2, y2, x1, y2]
			line = [str(x) for x in line]
			line.append('content')
			line = ','.join(line)
			results.append(line)
	return results

# icdar format to labelme format: x1,y1,x2,y2,x3,y3,x4,y4 \t content ---> labelme3.0 xml format
def icdar_to_labelme():
	pass
# =======================================================

# 将voc格式的xml文件，转成icdar的格式去做文字检测:x1,y1,x2,y2,x3,y3,x4,y4,cls
def get_icdar_label():
	root = '/mnt/data/rz/data/idCard/v3'
	image_root = os.path.join(root, 'images')
	txt_store_root = os.path.join(root, 'personLabelTxt')
	error_store_root = os.path.join(root, 'error')
	if not osp.exists(txt_store_root):
		os.makedirs(txt_store_root)
	if not osp.exists(error_store_root):
		os.makedirs(error_store_root)
	# txt_paths = glob(osp.join(root, 'xml/*.txt'))
	# class_names = ['number', 'owner', 'frame_number', 'registration_number',
	#                'manner', 'property', 'cover_code', 'cover_number', 'idCard']
	# class_idx = {cls: str(idx) for idx, cls in enumerate(class_names)}

	# for d in ['1', '2', '3', '4', '5']:
	xml_root = os.path.join(root, 'xml')
	error_names = voc_to_icdar(xml_root, txt_store_root, split_flag=',',
							   filter_classes=['other'])
	for ename in error_names:
		epath = os.path.join(image_root, ename)
		# epath = epath.split('.')[0] + '.jpg'
		shutil.move(epath, error_store_root)
	# if not osp.exists(txt_store_root):
	#     os.makedirs(txt_store_root)
	# for txtp in txt_paths:
	#     txtn = osp.basename(txtp)
	#     txt_store_path = osp.join(txt_store_root, txtn)
	#     result = load_cvat_icdar(txtp)
	#     with open(txt_store_path, 'w') as f:
	#         f.write('\n'.join(result))

def get_dota_label():
	root = '/mnt/data/rz/data/idCard/v3'
	txt_store_root = '/mnt/data/rz/data/idCard/exp/20220104/train'
	txt_store_root = os.path.join(txt_store_root, 'labelTxt')
	if not osp.exists(txt_store_root):
		os.makedirs(txt_store_root)
	xml_root = os.path.join(root, 'xml')
	voc_to_dota(xml_root, txt_store_root, split_flag=' ')

# 将voc格式的xml文件，转成yolov5的格式去做文字检测:x1,y1,x2,y2,x3,y3,x4,y4,cls
def get_yolov5_label():
	root = '/mnt/data/rz/data/idCard/v4/cornered/person/split'
	txt_store_root = '/mnt/data/rz/data/idCard/exp/20220106/labels'
	img_root = '/mnt/data/rz/data/idCard/v4/cornered/person/split/0'
	error_store_root = '/mnt/data/rz/data/idCard/v4/cornered/person/split/error'

	if not os.path.exists(txt_store_root):
		os.makedirs(txt_store_root)
	# class_names = ['number', 'owner', 'frame_number', 'registration_number',
	#                'manner', 'property', 'cover_code', 'cover_number', 'number_code',
	#                'oil_type', 'vehicle_brand', 'domestic_import', 'factory', 'idCard']
	# class_names = ['element', 'txt', 'button', 'inputBox', 'checkBox', 'comboBox',
	# 			   'minimize', 'maximize', 'close', 'slidingBoxes', 'radioButton']
	# class_names = ['rectangle']
	class_names = ('name', 'sex', 'birthday', 'address', 'number', 'authority', 'validity', 'nation')
	class_idx = {cls: str(idx) for idx, cls in enumerate(class_names)}
	# for d in map(str, range(0, 1)):
	# xml_root = os.path.join(root, d)
	xml_root= os.path.join(root, 'xml/0')
	error_img_name = voc_to_yolov5(xml_root, txt_store_root, class_idx, split_flag=',',
								   filter_classes=['other'])

	make_dirs(error_store_root)
	for errorn in error_img_name:
		error_imgp = os.path.join(img_root, errorn)
		shutil.move(error_imgp, error_store_root)

# 将voc格式的xml文件，转成txt格式的识别格式去做文字识别:filename '\t' image_content(exp:1234)
def get_rec_label():
	root = '/mnt/data/rz/data/register/clean/rec/v6/registration_number'
	error_path = os.path.join(root, 'error')
	img_root = os.path.join(root, 'images')
	xml_root = os.path.join(root, 'xml')
	xml_paths = glob(os.path.join(xml_root, '*.xml'))
	# xml_paths = []
	# for dr in range(8, 11):
	#     path = os.path.join(xml_root, str(dr), '*.xml')
	#     xml_paths.extend(glob(path))

	if not os.path.exists(error_path):
		os.makedirs(error_path)

	train_xml_paths = xml_paths
	train_xml_paths, valid_xml_paths = train_test_split(
		xml_paths, test_size=0.2, random_state=47)
	train_xml_paths = xml_paths
	def store(paths, label_path, suffix, img_root, error_path, img_store_root):
		results = []
		for xmlp in tqdm(paths):
			line, filename = voc_to_rec(xmlp, split_flag=',', suffix=suffix)
			img_path = os.path.join(img_root, filename)
			if line is not None:
				results.append(line)
				if os.path.exists(img_path):
					shutil.copy(img_path, img_store_root)
			else:
				if os.path.exists(img_path):
					shutil.move(img_path, error_path)
		with open(label_path, 'w') as f:
			f.write('\n'.join(results))
	train_dir_name = 'truth_images_train'
	train_img_store_root = os.path.join(root, train_dir_name)
	valid_dir_name = 'truth_images_test'
	valid_img_store_root = os.path.join(root, valid_dir_name)
	train_label_path = os.path.join(root, 'train_rec_gt_label.txt')
	valid_label_path = os.path.join(root, 'test_rec_gt_label.txt')
	if not os.path.exists(train_img_store_root):
		os.makedirs(train_img_store_root)
	if not os.path.exists(valid_img_store_root):
		os.makedirs(valid_img_store_root)
	store(train_xml_paths, train_label_path, train_dir_name,
		  img_root, error_path, img_store_root=train_img_store_root)
	store(valid_xml_paths, valid_label_path, valid_dir_name,
		  img_root, error_path, img_store_root=valid_img_store_root)

def icdar_to_yolov5():
	txt_root = '/mnt/data/rz/data/idCard/v3/personLabelTxt/'
	img_root = '/mnt/data/rz/data/idCard/v3/images'
	txt_store_root = '/mnt/data/rz/data/idCard/v3/yolov5LabelTxt'
	if not osp.exists(txt_store_root):
		os.makedirs(txt_store_root)
	txt_paths = glob(osp.join(txt_root, '*.txt'))
	split_flag = '\t'
	filter_class = ['Advertisement', 'Card']
	# class_names = ['number', 'owner', 'frame_number', 'registration_number',
	# 				'manner', 'property', 'cover_code', 'cover_number']
	# class_names = ['List Item', 'Toolbar', 'Video', 'Background Image', 'Radio Button', 'On/Off Switch', 'Button Bar', 'Multi-Tab', 'Icon', 'Input', 'Modal', 'Checkbox',
	#     'Number Stepper', 'Pager Indicator', 'Date Picker', 'Text Button', 'Slider', 'Image', 'Drawer', 'Bottom Navigation', 'Text', 'Web View', 'Map View']
	class_names = ('name', 'sex', 'birthday', 'address', 'number', 'authority', 'validity', 'nation')
	# class_names_map = {'出生':'birthday', '公民身份号码':'number', '失效日期':'validity', '姓名':'name', '性别':'sex', '住址':'address', '签发日期':'validity', '签发机关':'authority', '民族':'nation'}
	class_idx = {cls: str(idx) for idx, cls in enumerate(class_names)}
	for txtp in tqdm(txt_paths):
		suffix = osp.basename(txtp).split('.')[0]
		imgn = suffix + '.jpg'
		imgp = osp.join(img_root, imgn)
		imgH, imgW = cv2.imread(imgp).shape[:2]
		new_lines = []
		with open(txtp) as f:
			for line in f:
				line = line.strip().split(split_flag)
				if len(line) == 2:
					loc, cls = line[0].split(','), line[-1]
				else:
					loc, cls = line[0], line[-1]
				# cls = class_names_map[cls]
				cls = class_idx[cls]
				loc = loc.split(',')
				if cls in filter_class:
					continue
				loc = list(map(float, loc))
				loc = list(map(int, loc))
				if len(loc) == 4:
					loc = [loc[0], loc[1], loc[2], loc[1], loc[2], loc[3], loc[0], loc[3]]
				loc = sort_box(loc) # sort the box points orders
				xmin, ymin, xmax, ymax = loc[0], loc[1], loc[4], loc[5]
				dw, dh = 1. / imgW, 1 / imgH
				x = (xmin + xmax)/2
				y = (ymin + ymax)/2
				w = xmax - xmin
				h = ymax - ymin
				x = x * dw
				w = w * dw
				y = y * dh
				h = h * dh
				idx = class_idx[cls]
				new_line = ' '.join([idx,
										str(truncate(x, 7)),
										str(truncate(y, 7)),
										str(truncate(w, 7)),
										str(truncate(h, 7))])
				new_lines.append(new_line)
		txt_store_path = osp.join(txt_store_root, suffix + '.txt')
		with open(txt_store_path, 'w') as f:
			f.write('\n'.join(new_lines))

# 获取图片前景的边角(身份证、银行卡之类的)
def get_corner_box():
	xmlp = '/mnt/data/rz/data/idCard/v3/annotations.xml'
	image_root = '/mnt/data/rz/data/idCard/v3/images'
	image_store_root = '/mnt/data/rz/data/idCard/v3/corner/images'
	txt_store_root = '/mnt/data/rz/data/idCard/v3/corner/labelTxt'
	if not os.path.exists(image_store_root):
		os.makedirs(image_store_root)
	if not os.path.exists(txt_store_root):
		os.makedirs(txt_store_root)

	def general_box(loc, shape, box_side_ratio=0.1):
		H, W = shape
		sideH, sideW = int(H*box_side_ratio), int(W*box_side_ratio)
		minX, minY, maxX, maxY = 0, 0, W, H

		boxes = []
		for lc in loc:
			cx, cy = lc
			half_side_x, half_side_y = sideW / 2, sideH / 2
			leftx, lefty = cx - half_side_x, cy - half_side_y
			rightx, righty = cx + half_side_x, cy + half_side_y
			if leftx < minX:
				leftx = minX
			if lefty < minY:
				lefty = minY
			if rightx > maxX:
				rightx = maxX
			if righty > maxY:
				righty = maxY

			# convert to yolo5 format
			dw, dh = 1. / W, 1 / H
			x = (leftx + rightx)/2
			y = (lefty + righty)/2
			w = rightx - leftx
			h = righty - lefty
			x = x * dw
			w = w * dw
			y = y * dh
			h = h * dh
			idx = '0'
			new_line = ' '.join([idx,
									str(truncate(x, 7)),
									str(truncate(y, 7)),
									str(truncate(w, 7)),
									str(truncate(h, 7))])
			boxes.append(new_line)
		return boxes

	result, error_image_file = extract_cvat_xml(xmlp)
	for res in tqdm(result):
		imgn = res['filename']
		imgp = os.path.join(image_root, imgn)
		txtn = imgn.split('.')[0] + '.txt'
		txtp = os.path.join(txt_store_root, txtn)
		img = cv2.imread(imgp)
		h, w = img.shape[:2]
		loc = res['location']
		loc = list(map(float, loc))
		loc = np.array(loc, dtype='int')
		loc = loc.reshape((-1, 2))
		loc = loc.tolist()
		boxes = general_box(loc, (h, w))
		shutil.copy(imgp, image_store_root)
		with open(txtp, 'w') as f:
			f.write('\n'.join(boxes))

if __name__ == "__main__":
	get_yolov5_label()

max_stars_repo_name: HkkSimple/KREP

max_issues_repo_name: HkkSimple/KREP

max_forks_repo_name: HkkSimple/KREP


text:
"""Kevin Fung - Github Alias: kkf18

Stage 3: nested class for exploratory data analysis.
This module contains the features to apply machine learning models, validate, and evaluate model performances.
Several helper features also provided for external EDA.

The current validation and evaluation methods focus on Scale Detection Data:
    Validation: Nested K-Fold Cross Validation
    Evaluation: Comparing to MIKON data, both qualitatively and quantitively.

Todo:
    * None.

"""

from mlflowrate.backend.base import Base

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import metrics

from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

from sklearn.metrics import make_scorer
from sklearn.metrics import mean_squared_error

from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_validate

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import ElasticNet
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.ensemble import RandomForestRegressor
from sklearn.cross_decomposition import PLSRegression

class Explore(Base):
    """Nested class to enable users to explore machine learning approaches to the dataset.

    The ML features have been designed for Scale Detection data.
    Class features are to be used in conjunction with Scikit Learn Methods.

    Attributes:
        results (dict): Dictionary of Result objects.

    """

    def __init__(self, datasets):
        super().__init__(datasets=datasets)
        self.results = {}  # dictionary of results objects

    def status(self):
        """Display the datasets created and the results objects."""
        print("\nDatasets Available: ")
        print("~~~~~~~~~~~~~~~~~~~")
        for dname, dobj in self.datasets.items():
            if dobj.is_pandas:
                print("{0}  |  Samples {1}  |  Num Features: {2}  |  Dates: {3}  to  {4}".format(dname,
                                                                                dobj.X.shape[0],
                                                                                dobj.X.shape[1],
                                                                                dobj.date[
                                                                                "datetime"].head(
                                                                                1).dt.strftime(
                                                                                '%Y/%m/%d').values[
                                                                                0],
                                                                                dobj.date[
                                                                                "datetime"].tail(
                                                                                1).dt.strftime(
                                                                                '%Y/%m/%d').values[
                                                                                0]))
            else:
                print("{0}  |  Samples {1}  |  Num Features: {2} ".format(dname,
                                                                          dobj.y.count(),
                                                                          len(dobj.X.columns())))
        print("\nResults Collected: ")
        print("~~~~~~~~~~~~~~~~~")
        for res, resobj in self.results.items():
            print("{0}  |  R2 {1}  |  RMSE {2} ".format(res, resobj.r2, resobj.rmse))

    def benchmark(self, date, ytrue, ypred):
        """Store a benchmark results object.

        Args:
            date (Array-like):  Array-like object which contains the corresponding dates to the results.
            ytrue (Array-like): Array-like object which contains the list of true values.
            ypred (Array-like): Array-like object which contains the list of predicted values.

        """
        self.results["benchmark"] = Result(date=date, ytrue=ytrue, ypred=ypred)

    def input_results(self, result_name, date=None, ypred=None, ytrue=None, r2=0.0, rmse=0.0):
        """Import results into the results dictionary.

        Args:
            result_name (str): Name of the results object.
            date (Array-like):  Array-like object which contains the corresponding dates to the results.
            ytrue (Array-like): Array-like object which contains the list of true values.
            ypred (Array-like): Array-like object which contains the list of predicted values.
            r2 (double): r2 score.
            rmse (double): root mean squared error.

        """
        self.results[result_name] = Result(date, ypred, ytrue, r2, rmse)

    def validate(self, model, traindataset, param_grid, outer=10, inner=5, verb=False):
        """Nested K-fold cross validation method based on grid searches.

        Method wraps around Scikit Learn objects.

        Args:
            model (Obj): Scikit Learn model.
            traindataset (Obj): dset object for training.
            param_grid (dict): Scikit Learn parameter tuning dictionary.
            outer (int): Number of outer layer folds.
            inner (int): Number of inner layer folds.
            verb (bool): Display scores.

        Returns:
            Averaged R2 and RMSE scores from nested cross validation.

        """
        X = traindataset.X
        y = np.squeeze(traindataset.y)

        outer_cv = KFold(n_splits=outer, shuffle=True, random_state=42)
        inner_cv = KFold(n_splits=inner, shuffle=True, random_state=42)
        scoring = {'r2': 'r2',
                   'rmse': make_scorer(mean_squared_error)}

        # Pass the gridSearch estimator to cross_val_score
        clf = GridSearchCV(estimator=model, param_grid=param_grid, cv=inner_cv, scoring=scoring, refit='r2')
        scores = cross_validate(clf, X=X, y=y, scoring=scoring, cv=outer_cv)
        avgr2 = scores['test_r2'].mean()
        avgrmse = np.sqrt(scores['test_rmse']).mean()

        if verb:
            print("R-Squared: ", avgr2)
            print("RMSE: ", avgrmse)

        return avgr2, avgrmse

    def evaluate(self, model, traindataset, testdataset, param_grid, splits=10, testunc=0.15):
        """Final test: Train model on entire dataset with best hyperparameters and compare to MIKON.

        Method compares predictions to MIKON:
        Plot against MIKON: To see if predictions produces correct trends.
        RMSE from MIKON uncertainty range: Quantitive performance metric.
        Percentage of predictions within MIKON uncertainty range: Quantitive performance metric.

        Args:
            model (Obj): Scikit Learn model.
            traindataset (Obj): dset object for training.
            testdataset (Obj): dset object for testing.
            param_grid (dict): Scikit Learn parameter tuning dictionary.
            splits (int): Number of folds for hyper parameter tuning.
            testunc (double): Relative uncertainty of MIKON model.

        Returns:
            model_params which are the best hyperparameters for the model tuned on all the data.

        """
        traindate = traindataset.date
        trainX = traindataset.X
        trainy = np.squeeze(traindataset.y)

        testdate = testdataset.date
        testX = testdataset.X
        testy = np.squeeze(testdataset.y)

        kcv = KFold(n_splits=splits, shuffle=True, random_state=42)
        clf = GridSearchCV(model, param_grid, cv=kcv)
        trainpred = clf.fit(trainX, trainy).predict(trainX)
        model_params = clf.best_params_
        testpred = clf.predict(testX)

        trainpred = np.squeeze(trainpred)
        testpred = np.squeeze(testpred)

        fig, axs = plt.subplots(2, 1, figsize=(12, 8))
        self.trainplot(traindate, trainpred, trainy, axs=axs[0])
        self.evalplot(traindate, testdate, trainy, testy, testpred, testunc=testunc, axs=axs[1])
        display(fig)
        return model_params

    def trainplot(self, traindate, trainpred, trainy, axs=None):
        """Display the training plot of the model.

        Args:
            traindate (Array-like): Array-like object containing dates corresponding to results.
            trainpred (Array-like): Array-like object containing training predictions.
            trainy (Array-like): Array-like object containing the true training labels.
            axs (Obj): Matplotlib axes object.

        """
        trainsse = ((trainpred - trainy) ** 2).sum()
        trainrmse = np.sqrt(trainsse / trainy.shape[0])

        if axs is None:
            fig, axs = plt.subplots(1, 1, figsize=(12, 4))

        axs.plot(traindate, trainy, "*--", label="Truth")
        axs.plot(traindate, trainpred, "+--", label="Prediction")
        axs.grid(True)
        axs.set_title("Train Accuracy, RMSE: {0:.2f}".format(trainrmse))
        axs.set_xlabel("Date", fontsize=12)
        axs.set_ylabel("Liquid Rate", fontsize=12)
        axs.legend(loc="best")

        if axs is None:
            display(fig)

    def evalplot(self, traindate, testdate, trainy, testy, testpred, testunc=0.15, axs=None):
        """Display the evaluation MIKON comparison plot.

        Args:
            traindate (Array-like): Array-like object containing dates corresponding to results.
            testdate (Array-like): Array-like object containing dates corresponding to test results.
            trainy (Array-like): Array-like object containing the true training labels.
            testy (Array-like): Array-like object containing the true test labels.
            testpred (Array-like): Array-like object containing test predictions.
            testunc (double): Relative uncertainty of the MIKON model.
            axs (Obj): Matplotlib axes object.

        """
        trainy = np.squeeze(trainy)
        testy = np.squeeze(testy)
        testpred = np.squeeze(testpred)
        unc_err, rmse_unc, inrange = self._unc_errors(testy, testpred, testunc)

        if axs is None:
            fig, axs = plt.subplots(1, 1, figsize=(12, 4))
            axs.fill_between(testdate.set_index(['datetime']).index, testy + unc_err, testy - unc_err, alpha=0.2,
                             color='gray')
            axs.plot(traindate, trainy, "*", label="Train Truth")
            axs.plot(testdate, testy, "--", label="Test Truth")
            axs.plot(testdate, testpred, "--", label="Prediction")
            axs.grid(True)
            axs.set_title(
                "Test Accuracy, RMSE: {0:.2f}, Predictions Within Test Range (%): {1:.2f}".format(rmse_unc, inrange))
            axs.set_xlabel("Date", fontsize=12)
            axs.set_ylabel("Liquid Rate", fontsize=12)
            axs.legend(loc="best")

            fig.tight_layout()
            display(fig)
        else:
            axs.fill_between(testdate.set_index(['datetime']).index, testy + unc_err, testy - unc_err, alpha=0.2,
                             color='gray')
            axs.plot(traindate, trainy, "*", label="Train Truth")
            axs.plot(testdate, testy, "--", label="Test Truth")
            axs.plot(testdate, testpred, "--", label="Prediction")
            axs.grid(True)
            axs.set_title(
                "Test Accuracy, RMSE: {0:.2f}, Predictions Within Test Range (%): {1:.2f}".format(rmse_unc, inrange))
            axs.set_xlabel("Date", fontsize=12)
            axs.set_ylabel("Liquid Rate", fontsize=12)
            axs.legend(loc="best")

    def _unc_errors(self, testy, testpred, testunc):
        """Private method to calculate uncertainty boundary RMSE and percentages.

        Args:
            testy (Array-like): Array-like object containing the true test labels.
            testpred (Array-like): Array-like object containing test predictions.
            testunc (double): Relative uncertainty of the MIKON model.

        Returns:
            Uncertainty errors and performance metrics.

        """
        unc_err = testy * testunc
        # Calculate RMSE away from the truths uncertainty range:
        upp = testy + unc_err
        low = testy - unc_err
        within = ((testpred >= low) & (testpred <= upp))

        err = np.stack((within, upp, low, testpred), axis=1)
        err = pd.DataFrame(err, columns=['mask', 'upp', 'low', 'testpred'])

        def f(row):
            if row['mask']:
                val = 0.0
            else:
                if row['testpred'] > row['upp']:
                    val = (row['testpred'] - row['upp']) ** 2
                elif row['testpred'] < row['low']:
                    val = (row['testpred'] - row['low']) ** 2
                else:
                    val = None
            return val

        err['dif'] = err.apply(f, axis=1)
        rmse_unc = (np.sqrt(err['dif'].sum() / err.shape[0]))
        inliers = err['mask'].sum()
        inrange = inliers / err.shape[0] * 100

        return unc_err, rmse_unc, inrange

    def fitpredict(self, model, traindataset, testdataset, param_grid=None, splits=10):
        """Wrapper method to fit a model to a train dataset and predict on a test dataset.

        Args:
            model (Obj): Scikit Learn model.
            traindataset (Obj): dset object for training.
            testdataset (Obj): dset object for testing.
            param_grid (dict): Scikit Learn parameter tuning dictionary.
            splits (int): Number of folds for hyper parameter tuning.

        Returns:
            Predictions of test dataset from model.

        """
        assert len(traindataset.X.columns) == len(
            testdataset.X.columns), "Train feature set is not the same length as test feature set"

        trainX = traindataset.X
        trainy = np.squeeze(traindataset.y)
        testX = testdataset.X

        if param_grid:
            kcv = KFold(n_splits=splits, shuffle=True, random_state=42)
            clf = GridSearchCV(model, param_grid, cv=kcv)
            fittedmodel = clf.fit(trainX, trainy)
        else:
            fittedmodel = model.fit(trainX, trainy)

        return fittedmodel.predict(testX)

    def fitpredict_naivemodels(self, traindataset, testdataset, errmetric="r2", eval_all=False):
        """Wrapper method to fit a list of common models to a train dataset and predict on a test dataset.

        Method will only return the best model results if eval_all is False, i.e. it will not
        return the performances of the other models.

        Models are:
            Linear Regression
            ElasticNet
            Lasso
            Ridge
            PLS Regression
            Random Forests Regression

        Hyperparameter tuning ranges are preselected in this method.

        Args:
            traindataset (Obj): dset object for training.
            testdataset (Obj): dset object for testing.
            errmetric (str): "r2" or "rmse" for k-fold nested validation metric.
            eval_all (bool): Evaluate all models on test dataset.

        Returns:
            evaluation metrics and results in a dictionary.

        """
        assert (errmetric == "r2") or (errmetric == "rmse"), "error metric can only be r2 or rmse"
        r2_comp = {}
        rmse_comp = {}
        model_list = {}
        eval_pred = {}
        names = ["LinearRegression", "ElasticNet", "Lasso", "Ridge", "PLSRegression", "RandomForests"]
        models = [Pipeline(steps=[('ss', StandardScaler()), ('lr', LinearRegression())]),
                  Pipeline(steps=[('ss', StandardScaler()), ('lr', ElasticNet())]),
                  Pipeline(steps=[('ss', StandardScaler()), ('lr', Lasso())]),
                  Pipeline(steps=[('ss', StandardScaler()), ('lr', Ridge())]),
                  Pipeline(steps=[('ss', StandardScaler()), ('lr', PLSRegression())]),
                  Pipeline(steps=[('ss', StandardScaler()), ('rf', RandomForestRegressor())])
                  ]
        param_grids = [{'lr__fit_intercept': [True, False], 'lr__normalize': [True, False]},
                       {'lr__alpha': [0.1, 1.0, 10, 100, 1000, 1e4],
                        'lr__l1_ratio': [0.2, 0.4, 0.6, 0.8, 1.0]},
                       {'lr__alpha': [0.1, 1.0, 10, 100, 1000, 1e4]},
                       {'lr__alpha': [0.1, 1.0, 10, 100, 1000, 1e4]},
                       {'lr__n_components': [2, 3, 4]},
                       {'rf__bootstrap': [True],
                        'rf__max_depth': [70, 80, 90],
                        'rf__max_features': [2, 3],
                        'rf__n_estimators': [50, 100, 150]}
                       ]

        key2grid = {key: idx for idx, key in enumerate(names)}

        print("Training")
        for name, model, param_grid in zip(names, models, param_grids):
            print("Training with Model: ", name)
            r2_comp[name], rmse_comp[name] = self.validate(model, traindataset, param_grid)
            model_list[name] = model
            if eval_all:
                eval_pred[name] = self.fitpredict(model, traindataset, testdataset, param_grid)

        if errmetric == "r2":
            best_model = max(r2_comp, key=r2_comp.get)
            # model = model_list[best_model]
        else:
            best_model = max(rmse_comp, key=rmse_comp.get)
            # model = model_list[best_model]

        print("Best naive model from cross validation found to be: ", best_model)
        if not eval_all:
            param_grid = param_grids[key2grid[best_model]]
            eval_pred[best_model] = self.fitpredict(model_list[best_model], traindataset, testdataset, param_grid,
                                                    splits=5)

        return eval_pred, r2_comp, rmse_comp

    def plotmodels(self, traindataset, testdataset, modelpreds, testunc=0.15):
        """Evaluation plot (MIKON Comparison) for a list of different model results on the same graph.

        Args:
            traindataset (Obj): dset object for training.
            testdataset (Obj): dset object for testing.
            modelpreds (dict): Dictionary of different model predictive results: {"model name" : Array-like results}
            testunc (double): Relative uncertainty of the MIKON model.

        """
        rmse_unc = 0.0
        inrange = 0.0
        unc_err = 0.0

        traindate = traindataset.date
        testdate = testdataset.date
        testy = np.squeeze(testdataset.y)
        trainy = np.squeeze(traindataset.y)

        fig, axs = plt.subplots(1, 1, figsize=(12, 4))
        axs.plot(testdate, testy, "--", label="Test Truth")
        axs.plot(traindate, trainy, "*", label="Train Truth")

        for model, pred in modelpreds.items():
            pred = np.squeeze(pred)
            unc_err, rmse_unc, inrange = self._unc_errors(testy, pred, testunc)
            axs.plot(testdate, pred, "--",
                     label="{0}: RMSE {1:.2f}, {2:.2f}% within uncertainty".format(model, rmse_unc, inrange))

        axs.grid(True)
        axs.set_title("Model Comparison".format(rmse_unc, inrange))
        axs.set_xlabel("Date", fontsize=12)
        axs.set_ylabel("Liquid Rate", fontsize=12)
        axs.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=8, borderaxespad=0.)

        axs.fill_between(testdate.set_index(['datetime']).index, testy + unc_err, testy - unc_err, alpha=0.2,
                         color='gray')

        display(fig)
        plt.close(fig)

    def get_datasets(self):
        """Return datasets dictionary for modelling outside the WorkFlow framework."""
        return self.datasets

    def get_results(self):
        """Return results dictionary containing the different results objects."""
        return self.results

class Result:
    """Data storing class for evaluation results.

    If the truth values are provided, the R2 and rmse scores are calculated.

    Attributes:
        date (Array-like): Corresponding dates.
        ypred (Array-like): Model predictions.
        ytrue (Array-like): Truth values.
        r2 (double): R2 score.
        rmse (double): Root mean squared error.
        n (int): Number of results.

    """

    def __init__(self, date=None, ypred=None, ytrue=None, r2=0.0, rmse=0.0):
        self.date = date
        self.ypred = ypred
        self.ytrue = ytrue
        self.r2 = r2
        self.rmse = rmse
        if ytrue is not None:
            assert len(ypred) == len(ytrue), "predictions and truths arent the same size!"
            assert ypred is not None, "ypred not given"
            self.n = len(ytrue)
            self.r2 = metrics.r2_score(self.ytrue, self.ypred)
            self.rmse = self.calc_rmse(self.ytrue, self.ypred)

    def calc_rmse(self, ytrue, ypred):
        """Calculate the root mean squared error.

        Args:
            ypred (Array-like): Model predictions.
            ytrue (Array-like): Truth values.

        """
        residual = ypred - ytrue
        sse = sum(residual ** 2)
        mse = sse / self.n
        return np.sqrt(mse)

    def plot_residual(self):
        """Plot the error residual graph."""
        assert self.ytrue is not None, "No truth values provided!"
        residual = self.ypred - self.ytrue
        fig, axs = plt.subplots(1, 2, figsize=(10, 4))
        axs[0].plot(self.ypred, residual, ".")
        axs[0].hlines(0, 0, 2200)
        axs[0].grid(True)
        axs[1].plot(self.date, self.ypred, ".--", label="Prediction")
        axs[1].plot(self.date, self.ytrue, ".--", label="True")
        axs[1].legend(loc='best')
        axs[1].grid(True)
        display(fig)
        plt.close(fig)

max_stars_repo_name: kev-fung/mlflowrate

max_issues_repo_name: kev-fung/mlflowrate

max_forks_repo_name: kev-fung/mlflowrate


text:
import numpy as np
import matplotlib.pyplot as plt

class EXP3:
    def __init__(self, Delta:float, lr: float):
        #self.true_means = avg  # true means of the arms

        self.true_means = [0.5]*10
        self.true_means[8] = 0.5 + Delta
        self.true_means[9] = 0.5 - Delta
        self.true_means = np.asarray(self.true_means)

        self.num_arms = 10  # num arms (k)
        self.best_arm = 9 #int(np.argmax(self.true_means))  # True best arm
        self.lr = lr

        self.restart()

    def restart(self):
        # Reset counters
        self.time = 0
        self.L = np.array([0.0] * self.num_arms)                 # S_t,j = initialize to zero
        self.P = None                                            # P_t,j = initialized uniformly at t=0 by update_exp3()
        self.arm_ix = None

        self.regret = []

        self.true_means = [0.5]*10
        self.true_means[8] = 0.5 + Delta
        self.true_means[9] = 0.5 - Delta
        self.true_means = np.asarray(self.true_means)

        self.best_arm = 8

    def get_best_arm(self):
        # For each time index, sample the best arm based off P_(t-1),j
        all_ix = np.arange(self.num_arms)
        return np.random.choice(a=all_ix, size=1, replace=False, p=self.P)

    def update_exp3(self):
        # calculate and update P_t,j
        exp_wt = np.exp(-self.L * self.lr)
        self.P = exp_wt / sum(exp_wt)

    def update_stats(self, rew_vec):
        # update regret
        genie_rew = rew_vec[self.best_arm]
        player_rew = rew_vec[self.arm_ix]
        self.regret.append((genie_rew - player_rew))

        # update L
        self.L[self.arm_ix] += ((1 - rew_vec[self.arm_ix]) / (self.P[self.arm_ix] + 0.5*self.lr))

        self.time += 1

    def get_reward(self):
        #return self.true_means + np.random.normal(0, 0.01, np.shape(self.true_means))
        #rewards = []
        #for j in range(self.num_arms):
        #    rewards.append(np.random.binomial(n=1, p=self.true_means[j]))

        rewards = np.random.binomial(n=1, p=self.true_means)

        return rewards

    def iterate(self):
        self.time += 1

        if self.time > 5e4:
            self.true_means[9] = 0.5 + 4*Delta
            self.best_arm = 9

        self.update_exp3()
        self.arm_ix = self.get_best_arm()
        rew_vec = self.get_reward()
        #self.per_arm_total_rew  += rew_vec
        self.update_stats(rew_vec=rew_vec)

def run(Delta, iterations, num_repeat, eta):
    regret = np.zeros((num_repeat, iterations))
    exp3 = EXP3(Delta=Delta, lr=eta)

    for j in range(num_repeat):
        for t in range(iterations):
            exp3.iterate()

        # calculate cumulative regret
        regret[j, :] = np.cumsum(np.asarray(exp3.regret))
        print(j)
        mean_runs = np.mean(regret[0:j+1,:], axis=0)
        std_runs = np.std(regret[0:j+1,:], axis=0)
        print("Mean total regret at the end:", mean_runs[-1])
        print("Std:", std_runs[-1])
        exp3.restart()
        #print(j)

    return regret

if __name__ == '__main__':
    #mu = np.asarray([0.8, 0.7, 0.5])
    num_arms = 10
    num_iter, num_inst = int(1e5), 10
    Delta = 0.1

    #eta = np.sqrt(np.log(num_arms) / (num_iter * num_arms))
    eta = 0.01

    reg = run(Delta=Delta,
              iterations=num_iter,
              num_repeat=num_inst,
              eta=eta)

    mean_runs = np.mean(reg, axis=0)
    std_runs = np.std(reg, axis=0)

    print("Mean total regret at the end:", mean_runs[-1])
    print("Std:", std_runs[-1])

    UB = mean_runs + 1 * std_runs
    LB = mean_runs - 1 * std_runs

    x = np.arange(len(mean_runs))
    plt.plot(x, mean_runs, color='b')
    plt.fill_between(x, LB, UB, alpha=0.3, linewidth=0.5, color='b')

    plt.xlabel('Time', fontsize=10)
    plt.ylabel('Cumulative Regret with EXP3-IX', fontsize=10)
    #plt.xscale('log')
    plt.grid(True, which='both', linestyle='--')
    plt.show()

    #print("Mean total regret at the end:", mean_runs[-1])
    #print("Std:", std_runs[-1])

max_stars_repo_name: anishacharya/Bandits

max_issues_repo_name: anishacharya/Bandits

max_forks_repo_name: anishacharya/Bandits


text:
import multiprocessing as mp
import pdb
from functools import partial
from itertools import product

# Levenshtein.ratio(str1, str2)
import Levenshtein
# import numpy as np
# import pandas as pd
from pandas import DataFrame

# from operator import itemgetter
# from time import time

def fuzzymerge(left, right, uselower=True, threshold=0.9, multi=0,
               how='outer', on=None, left_on=None, right_on=None,
               left_index=False, right_index=False):

    if left_index:
        left_on = left_index

    if right_index:
        right_on = right_index

    if on:
        left_on = on
        right_on = on

    left = left.reset_index(drop=True)
    right = right.reset_index(drop=True)
    df1 = left.reset_index()[['index', left_on]].dropna(subset=[left_on])
    df2 = right.reset_index()[['index', right_on]].dropna(subset=[right_on])
    # pdb.set_trace()

    if uselower:
        df1[left_on] = df1[left_on].str.lower()
        df2[right_on] = df2[right_on].str.lower()

    list1 = df1[['index', left_on]].to_numpy()
    list2 = df2[['index', right_on]].to_numpy()

    if multi < 0:
        multi = mp.cpu_count() + multi

    # TODO: longer list on right-hand side to benefit from multiprocessing

    matched = _match_by_left(list1, list2, threshold, multi)
    matched = matched.astype({'leftind': 'Int64', 'rightind': 'Int64'})

    mergeleft = (left
                 .merge(matched, how=how, left_index=True, right_on='leftind'))

    mergeright = (matched
                  .merge(right, how=how, left_on='rightind', right_index=True))

    # pdb.set_trace()
    if how == 'left' or how == 'inner':
        merge_on, delete_on = 'leftind', 'rightind'
    elif how == 'right' or how == 'outer':
        merge_on, delete_on = 'rightind', 'leftind'

    if how == 'left':
        merged = (mergeleft
                  .drop(columns=[delete_on])
                  .merge(mergeright, how=how, on=merge_on)
                  .drop_duplicates(subset=[left_on, right_on])
                  .drop(columns=['leftind', 'rightind'])
                  .dropna(subset=[left_on])
                  .reset_index(drop=True))
    elif how == 'right':
        merged = (mergeleft
                  .drop(columns=[delete_on])
                  .merge(mergeright, how=how, on=merge_on)
                  .drop_duplicates(subset=[left_on, right_on])
                  .drop(columns=['leftind', 'rightind'])
                  .dropna(subset=[right_on])
                  .reset_index(drop=True))
    elif how == 'inner':
        merged = (mergeleft
                  .drop(columns=[delete_on])
                  .merge(mergeright, how=how, on=merge_on)
                  .drop_duplicates(subset=[left_on, right_on])
                  .drop(columns=['leftind', 'rightind'])
                  .dropna(subset=[left_on, right_on])
                  .reset_index(drop=True))

    elif how == 'outer':
        merged = (mergeleft
                  .drop(columns=[delete_on])
                  .merge(mergeright, how=how, on=merge_on)
                  .drop_duplicates(subset=[left_on, right_on])
                  .drop(columns=['leftind', 'rightind'])
                  .dropna(subset=[left_on, right_on], how='all')
                  .reset_index(drop=True))

    return merged

def _levenshtein(str_a, ind_str_b):
    """
    str_a: str
    ind_str_b: (ind, str_b)
    """
    # ind, str_b = ind_str_b
    return (ind_str_b[0], Levenshtein.ratio(str_a, ind_str_b[1]))

def _levenshtein_mp(list2, threshold, item1):
    matchedindices = []
    ind, x = item1
    ind_distances = [_levenshtein(x, y) for y in list2]
    # pdb.set_trace()
    rightindices = [i[0] for i in ind_distances if i[1] >= threshold]
    matchedindices += list(product([ind], rightindices))
    return matchedindices

def _match_by_left(list1, list2, threshold, multi):
    "fuzzy matching by left"
    matchedindices = []

    if multi:
        singlefunc = partial(_levenshtein_mp, list2, threshold)
        with mp.Pool(multi) as p:
            results = p.map(singlefunc, list1)

        matchedindices = [i for s in results for i in s]

    else:
        for ind, leftentry in list1:
            ind_distances = [_levenshtein(leftentry, y) for y in list2]
            # pdb.set_trace()
            rightindices = [i[0] for i in ind_distances if i[1] >= threshold]
            matchedindices += list(product([ind], rightindices))
    return DataFrame(set(matchedindices), columns=['leftind', 'rightind'])

max_stars_repo_name: mark-fangzhou-xie/levenpandas

max_issues_repo_name: mark-fangzhou-xie/levenpandas

max_forks_repo_name: mark-fangzhou-xie/levenpandas


text:
#!/usr/bin/python
#encoding:utf-8

import tensorflow.contrib.slim as slim
import tensorflow.contrib.keras as kr
import tensorflow as tf
import tkinter as tk
import numpy as np
import warnings
import codecs
import jieba
import time
import re
import sys
import os
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from gensim.models import word2vec
from collections import  Counter
from datetime import timedelta
from sklearn import metrics
from tc_data import *

class tc_cnn(object):

    def __init__(self,tensorboard_dir,trained_modelfile,embedding_size=200,vocab_size=77216,pre_trianing=None,seq_length=100,num_classes=8,num_filters=128,
                 filter_sizes=[2,3,4],hidden_dim=128,keep_prob=0.5,lr=1e-3,lr_decay=0.9,clip=5.0,l2_reg_lambda=0.001,
                 num_epochs=10,batch_size=640,print_per_batch=100,save_per_batch=10):
        self.tensorboard_dir=tensorboard_dir
        self.trained_modelfile=trained_modelfile
        self.config={}
        self.config['embedding_size']=embedding_size##
        self.config['vocab_size']=vocab_size##
        self.config['pre_trianing']=pre_trianing##
        self.config['seq_length']=seq_length##
        self.config['num_classes']=num_classes
        self.config['num_filters']=num_filters
        self.config['filter_sizes']=filter_sizes
        self.config['hidden_dim']=hidden_dim
        self.config['keep_prob']=keep_prob
        self.config['lr']=lr
        self.config['lr_decay']=lr_decay
        self.config['clip']=clip
        self.config['l2_reg_lambda']=l2_reg_lambda
        self.config['num_epochs']=num_epochs
        self.config['batch_size']=batch_size##
        self.config['print_per_batch']=print_per_batch##
        self.config['save_per_batch']=print_per_batch##

    def init_param(self):
        input_x=self.__set_input_x()
        input_y=self.__set_input_y()
        keep_prob=self.__set_keep_prob()
        global_step = self.__set_global_step()
        l2_loss=self.__set_l2loss()
        self.__set_embedding()
        self.__set_cov_max_pool()
        self.__set_dropout()
        self.__set_output()
        self.__set_loss()
        self.__set_accuracy()
        self.__set_optimizer()

#
#instance_param
#input_x
#
    def __set_input_x(self):
        self.input_x=tf.placeholder(tf.int32,shape=[None,self.config['seq_length']],name='input_x')
        return self.input_x

#
#instance_param
#input_y
#
    def __set_input_y(self):
        self.input_y=tf.placeholder(tf.float32,shape=[None,self.config['num_classes']],name='input_y')
        return self.input_y

#
#instance_param
#keep_prob
#
    def __set_keep_prob(self):
        self.keep_prob=tf.placeholder(tf.float32,name='dropout')
        return self.keep_prob

#
#instance_param
#global_step
#
    def __set_global_step(self):
        self.global_step = tf.Variable(0, trainable=False, name='global_step')
        return self.global_step

#
#instance_param
#l2_loss
#
    def __set_l2loss(self):
        self.l2_loss = tf.constant(0.0)
        return self.l2_loss
#
#instance_param
#embedding
#
    def __set_embedding_r(self):
        self.embedding_r = tf.get_variable("embeddings_r", shape=[self.config['vocab_size'], self.config['embedding_size']])
        return self.embedding_r

    def __set_embedding_expand_r(self,embedding_r):
        embedding_inputs_r = tf.nn.embedding_lookup(embedding_r, self.input_x)
        self.embedding_expand_r = tf.expand_dims(embedding_inputs_r, -1)
        return self.embedding_expand_r

    def __set_embedding_s(self):
        self.embedding_s = tf.get_variable("embeddings_s", shape=[self.config['vocab_size'], self.config['embedding_size']],initializer=tf.constant_initializer(self.config['pre_trianing']))
        return self.embedding_s

    def __set_embedding_expand_s(self,embedding_s):
        embedding_inputs_s = tf.nn.embedding_lookup(embedding_s, self.input_x)
        self.embedding_expand_s = tf.expand_dims(embedding_inputs_s, -1)
        return self.embedding_expand_s

    def __set_embedding_2(self):
        self.embedding_2 = tf.concat([self.embedding_expand_r, self.embedding_expand_s], 3,name='embedding_2')
        return self.embedding_2

    def __set_embedding(self):
        with tf.device('/cpu:0'), tf.name_scope('embedding'):
            embedding_r = self.__set_embedding_r()
            embedding_expand_r=self.__set_embedding_expand_r(embedding_r)
            embedding_s = self.__set_embedding_s()
            embedding_expand_s=self.__set_embedding_expand_s(embedding_s)
            embedding_2=self.__set_embedding_2()
            embedding_2_cbam = self.__set_cbam_()
#
#instance_param
#embedding
#cbam
#
    def __set_combined_static_and_dynamic_shape(self,tensor):
        static_tensor_shape = tensor.shape.as_list()
        dynamic_tensor_shape = tf.shape(tensor)
        combined_shape = []
        for index, dim in enumerate(static_tensor_shape):
            if dim is not None:
                combined_shape.append(dim)
            else:
                combined_shape.append(dynamic_tensor_shape[index])
        return combined_shape

    def __set_channel_weight_reshape(self,feature_map,feature_map_shape):
        channel_avg_weights = tf.nn.avg_pool(
            value=feature_map,
            ksize=[1, feature_map_shape[1], feature_map_shape[2], 1],
            strides=[1, 1, 1, 1],
            padding='VALID'
        )
        channel_max_weights = tf.nn.max_pool(
            value=feature_map,
            ksize=[1, feature_map_shape[1], feature_map_shape[2], 1],
            strides=[1, 1, 1, 1],
            padding='VALID'
        )
        channel_avg_reshape = tf.reshape(channel_avg_weights,[feature_map_shape[0], 1, feature_map_shape[3]])
        channel_max_reshape = tf.reshape(channel_max_weights,[feature_map_shape[0], 1, feature_map_shape[3]])
        channel_w_reshape = tf.concat([channel_avg_reshape, channel_max_reshape], axis=1)
        return channel_w_reshape

    def __set_channel_attention(self,feature_map,feature_map_shape,channel_w_reshape,inner_units_ratio):
        fc_1 = tf.layers.dense(
            inputs=channel_w_reshape,
            units=feature_map_shape[3] * inner_units_ratio,
            name="fc_1",
            activation=tf.nn.relu
        )
        fc_2 = tf.layers.dense(
            inputs=fc_1,
            units=feature_map_shape[3],
            name="fc_2",
            activation=tf.nn.sigmoid
        )
        channel_attention = tf.reduce_sum(fc_2, axis=1, name="channel_attention_sum")
        channel_attention = tf.reshape(channel_attention, shape=[feature_map_shape[0], 1, 1, feature_map_shape[3]])
        feature_map_with_channel_attention = tf.multiply(feature_map, channel_attention)
        return feature_map_with_channel_attention

    def __set_channel_wise_reshape(self,feature_map_shape,feature_map_with_channel_attention):
        channel_wise_avg_pooling = tf.reduce_mean(feature_map_with_channel_attention, axis=3)
        channel_wise_max_pooling = tf.reduce_max(feature_map_with_channel_attention, axis=3)
        channel_wise_avg_pooling = tf.reshape(channel_wise_avg_pooling,
                                              shape=[feature_map_shape[0], feature_map_shape[1], feature_map_shape[2],
                                                     1])
        channel_wise_max_pooling = tf.reshape(channel_wise_max_pooling,
                                              shape=[feature_map_shape[0], feature_map_shape[1], feature_map_shape[2],
                                                     1])
        channel_wise_pooling = tf.concat([channel_wise_avg_pooling, channel_wise_max_pooling], axis=3)
        return channel_wise_pooling

    def __set_convolution(self,channel_wise_pooling,feature_map_with_channel_attention):
        spatial_attention = slim.conv2d(
            channel_wise_pooling,
            1,
            [3, 3],
            padding='SAME',
            activation_fn=tf.nn.sigmoid,
            scope="spatial_attention_conv"
        )
        feature_map_with_attention = tf.multiply(feature_map_with_channel_attention, spatial_attention)
        return feature_map_with_attention

    def __set_cbam_(self,index=1,inner_units_ratio=0.5):
        feature_map=self.embedding_2
        with tf.variable_scope("cbam_%s" % (index)):
            feature_map_shape = self.__set_combined_static_and_dynamic_shape(feature_map)
            channel_weight_reshape=self.__set_channel_weight_reshape(feature_map,feature_map_shape)
            feature_map_with_channel_attention=self.__set_channel_attention(feature_map,feature_map_shape,channel_weight_reshape,inner_units_ratio)
            channel_wise_reshape=self.__set_channel_wise_reshape(feature_map_shape,feature_map_with_channel_attention)
            feature_map_with_attention=self.__set_convolution(channel_wise_reshape,feature_map_with_channel_attention)
            self.embedding_2_cbam=feature_map_with_attention
            return self.embedding_2_cbam

#
#instance_param
#cov_max_pool
#
    def __set_pool(self,filter_size):
        filter_shape = [filter_size, self.config['embedding_size'], 2, self.config['num_filters']]
        w = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="w")
        b = tf.Variable(tf.constant(0.1, shape=[self.config['num_filters']]), name="b")
        conv = tf.nn.conv2d(self.embedding_2_cbam, w, strides=[1, 1, 1, 1], padding='VALID', name='conv')
        h = tf.nn.relu(tf.nn.bias_add(conv, b), name='relu')
        pooled = tf.nn.max_pool(h, ksize=[1, self.config['seq_length'] - filter_size + 1, 1, 1],strides=[1, 1, 1, 1], padding='VALID', name='pool')
        return pooled

    def __set_h_pool(self,pooled_outputs):
        self.h_pool = tf.concat(pooled_outputs, 3)
        return self.h_pool

    def __set_h_pool_flat(self):
        self.num_filter_total = self.config['num_filters'] * len(self.config['filter_sizes'])
        self.h_pool_flat = tf.reshape(self.h_pool, [-1, self.num_filter_total])
        return self.h_pool_flat

    def __set_cov_max_pool(self):
        pooled_outputs = []
        for i, filter_size in enumerate(self.config['filter_sizes']):
                with tf.name_scope("conv-maxpool-%s" % filter_size):
                        pooled=self.__set_pool(filter_size)
                        pooled_outputs.append(pooled)
        h_pool = self.__set_h_pool(pooled_outputs)
        h_pool_flat=self.__set_h_pool_flat()

#
#instance_param
#dropout
#
    def __set_dropout(self):
        with tf.name_scope('dropout'):
            self.dropout = tf.nn.dropout(self.h_pool_flat, self.keep_prob)

#
#instance_param
#output
#
    def __set_output(self):
        with tf.name_scope('output'):
            w = tf.get_variable("w", shape=[self.num_filter_total, self.config['num_classes']],initializer=tf.contrib.layers.xavier_initializer())
            b = tf.Variable(tf.constant(0.1, shape=[self.config['num_classes']]), name='b')
            self.l2_loss += tf.nn.l2_loss(w)
            self.l2_loss += tf.nn.l2_loss(b)
            self.scores = tf.nn.xw_plus_b(self.dropout, w, b, name="scores")
            self.pro = tf.nn.softmax(self.scores)
            self.predictions = tf.argmax(self.pro, 1,name='predictions')

#
#instance_param
#loss
#
    def __set_loss(self):
        with tf.name_scope('loss'):
            losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.scores, labels=self.input_y)
            self.loss = tf.reduce_mean(losses) + self.config['l2_reg_lambda'] * self.l2_loss

#
#instance_param
#accuracy
#
    def __set_accuracy(self):
        with tf.name_scope('accuracy'):
            correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, 'float32'), name='accuracy')

#
#instance_param
#optimizer
#
    def __set_optimizer(self):
        with tf.name_scope('optimizer'):
            optimizer = tf.train.AdamOptimizer(self.config['lr'])
            gradients, variables = zip(*optimizer.compute_gradients(self.loss))
            gradients, _ = tf.clip_by_global_norm(gradients, self.config['clip'])
            self.optimizer = optimizer.apply_gradients(zip(gradients, variables), global_step=self.global_step)

#
#train
#
    def __init_tf_sum(self):
        tf.summary.scalar("loss", self.loss)
        tf.summary.scalar("accuracy", self.accuracy)
        self.merged_summary = tf.summary.merge_all()
        self.writer = tf.summary.FileWriter(self.tensorboard_dir)

    def __init_tf_sav(self,path=None):
        self.saver = tf.train.Saver()
        if path!=None:
            self.saver.restore(sess=self.session, save_path=path)

    def __init_tf_sess(self,train=True):
        self.session = tf.Session()
        self.session.run(tf.global_variables_initializer())
        if train:
            self.writer.add_graph(self.session.graph)

#
#train
#train_run
#
    def __init_mark(self):
        return 0,0,0,200,False

    def __evaluate(self,val_batch):
        batch_eval = val_batch
        total_loss = 0.0
        total_acc = 0.0
        total_len=0
        for x_batch, y_batch in batch_eval:
            batch_len = len(x_batch)
            feed_dict ={self.input_x:x_batch,self.input_y:y_batch,self.keep_prob:1.0}
            loss, acc = self.session.run([self.loss, self.accuracy], feed_dict=feed_dict)
            total_loss += loss * batch_len
            total_acc += acc * batch_len
            total_len += batch_len
        return total_loss / total_len, total_acc / total_len

    @staticmethod
    def get_time_dif(start_time):
        end_time = time.time()
        time_dif = end_time - start_time
        return timedelta(seconds=int(round(time_dif)))

    def __train_run(self,train_data,val_data):
        train_batch=train_data.batcher
        val_batch=val_data.batcher
        global_step,best_acc_val,last_improved,require_improvement,flag=self.__init_mark()
        start_time = time.time()
        for epoch in range(self.config['num_epochs']):
            for x_batch, y_batch in train_batch:
                feed_dict = {self.input_x:x_batch,self.input_y:y_batch,self.keep_prob:self.config['keep_prob']}
                if global_step - last_improved > require_improvement:
                    print("No optimization for a long time, auto-stopping...")
                    flag = True
                if global_step % self.config['save_per_batch'] == 0:
                    s = self.session.run(self.merged_summary, feed_dict=feed_dict)
                    self.writer.add_summary(s, global_step)
                if global_step % self.config["print_per_batch"] == 0:
                    feed_dict[self.keep_prob] = 1.0
                    loss_train, acc_train = self.session.run([self.loss, self.accuracy], feed_dict=feed_dict)
                    loss_val, acc_val = self.__evaluate(val_batch)
                    if acc_val > best_acc_val:
                        best_acc_val = acc_val
                        last_improved = global_step
                        self.saver.save(sess=self.session, save_path=self.trained_modelfile)
                        improved_str = '*'
                    else:
                        improved_str = ''
                    time_dif = tc_cnn.get_time_dif(start_time)
                    print("step: {},train loss: {:.3f}, train accuracy: {:.3f}, val loss: {:.3f}, val accuracy: {:.3f},time:{},training speed: {:.3f}sec/batch {}\n".format(
						global_step, loss_train, acc_train, loss_val, acc_val,time_dif,time_dif.seconds / self.config['print_per_batch'],improved_str))
                self.session.run(self.optimizer, feed_dict=feed_dict)
                global_step += 1
            if flag:
                break

#
#train
#test
#test_run
#
    def __test_run(self,test_data):
        start_time = time.time()

        test_batch=test_data.load()
        loss_test, acc_test =self.__evaluate(test_batch)
        print('Test Loss: {0:>6.2}, Test Acc: {1:>7.2%}'.format(loss_test, acc_test))

        data_len = len(test_data.x_pad)
        num_batch = int((data_len - 1) / batch_size) + 1

        y_test_cls = np.argmax(test_data.y_pad, 1)
        y_pred_cls = np.zeros(shape=len(test_data.x_pad), dtype=np.int32)
        for i in range(num_batch):
            start_id = i * self.config['batch_size']
            end_id = min((i + 1) * self.config['batch_size'], data_len)
            feed_dict = {
                self.input_x: test_data.x_pad[start_id:end_id],
                self.keep_prob: 1.0
            }
            y_pred_cls[start_id:end_id] = self.session.run(self.predictions, feed_dict=feed_dict)

        print(metrics.classification_report(y_test_cls, y_pred_cls, target_names=test_data.cat_iddic))
        print(metrics.confusion_matrix(y_test_cls, y_pred_cls))
        print("Time usage:", tc_cnn.get_time_dif(start_time))
#
#train
#test
#
    def test(self,test_data):
        self.__init_tf_sess(train=False)
        self.__init_tf_sav(path=self.trained_modelfile)
        self.__test_run(test_data)

    def train(self,train_data,val_data,restore=False):
        self.__init_tf_sum()
        self.__init_tf_sess()
        self.__init_tf_sav(path=self.trained_modelfile if restore else None)
        self.__train_run(train_data,val_data)
#
#predict
#
    def predict(self,content):
        self.__init_tf_sess(train=False)
        self.__init_tf_sav(path=self.trained_modelfile)
        feed_dict = {
            self.input_x: content,
            self.keep_prob: 1.0
        }

        y_pred_cls = self.session.run(self.predictions, feed_dict=feed_dict)
        return y_pred_cls[0]

if __name__ == "__main__":

    train_path='D:\DocuemtManagement\VSproject\Python_Proj\Py.Basic\Text_Categorization_JF\data_ori\\train1\\train_jf.txt'
    #train_path='D:\DocuemtManagement\VSproject\Python_Proj\Py.Basic\Text_Categorization_JF\data_n\\3\\train.txt'
    var_path=train_path
    stop_path='D:\DocuemtManagement\VSproject\Python_Proj\Py.Basic\Text_Categorization_JF\data_ori\\stopword.txt'
    train_data=tc_data(train_path,stop_path,batch_size=640,max_length=100,pos=False)
    train_batcher=train_data.load()

    tensorboard_dir='D:\DocuemtManagement\VSproject\Python_Proj\Py.Basic\Text_Categorization_JF\data_n\\'
    trained_model_file=os.path.join(tensorboard_dir, 'best_validation')
    cnn=tc_cnn(tensorboard_dir,trained_model_file,vocab_size=len(train_data.model.wv.vectors),pre_trianing=train_data.model.wv.vectors,batch_size=640,print_per_batch=100,save_per_batch=10)
    print(cnn.config)

    cnn.init_param()
    #ecnn.test(train_data)
    cnn.train(train_data,train_data,False)
    #data=train_data.content2id("泰达重启“魔鬼训练” 昆明气温回暖场地雪融")
    #print(train_data.categories[cnn.predict(data)])