text:
#!/usr/bin/python3

'''
Some common functions, more or less independent to the other submodules.
'''

import time

try:
    import numpy as np
except ModuleNotFoundError:
    print("\"numpy\" not found. This module requires numpy")
try:
    import matplotlib.pyplot as plt
except ModuleNotFoundError:
    print("\"matplotlib\" not found. This module requires matplotlib")

#######################################################################################################################################
#######################################################################################################################################
class MethodError(Exception):
    pass
#######################################################################################################################################
#######################################################################################################################################

def make_periodic(f,T,a=0,b=0):
    '''
    This make a "saw" function with the function f and a period T;
    makes the function repeats itself on intervals from b to b+T, with displacement a
    '''
    def p(x):
        return f((x-a)-T*np.floor((x-a)/T)+b)
    return p

def reduce_list(ls,it=1,method="random"):
    '''
    Delete elements froma a list. It is a good idea reduce pairs of lists with zip(l1,l2).
    If method is "random", each element will have a probability of 0.5 to be deleted.
    If as method you put an integer $n$, you will delete $n-1$ of each $n$ element.
    '''
    if type(method)!=int:
        if method!="random":
            raise MethodError("method must be \"random\" or an integer.")
    l=list(ls)
    rl=[]
    if method=="random":
        for i in range(len(l)):
            if np.random.choice([False, True]):
                rl.append(l[i])
        if it==1:
            return rl
        else:
            return reduce_list(rl,it-1)
    elif type(method)==int:
        for i in range(len(l)):
            if i%method==0:
                rl.append(l[i])
        if it==1:
            return rl
        else:
            return reduce_list(rl,it-1,method)

def boole(condition):
    '''
    I miss Mathematica's Boole. Returns 0 if condition is false, 1 otherwise.
    It is great for exclude terms from a sum.
    '''
    if condition:
        return 0
    else:
        return 1

def exclude(ex,n,strt=0,stp=1):
    '''
    Gives an Integer iterator [strt,ex)U(ex,n), with stp as steps.
    '''
    return [q for q in range(strt,n,stp) if q!=ex]

def generate_temperated(func,start,end,δ=0.03,σ=0.05,density=1):
    '''
    Generates an x-array of pseudo adaptative distributed points from start to end.
    Where the derivative of the function func is too high on the surroindings of a point,
    then let's put nore points there. It is quite slow and not precise.
    '''
    from .D import ffd
    adap=[start]
    while adap[-1]<=end:
        ap=min([1/(density*np.mean([abs(ffd(func,q))**2 for q in np.linspace(-σ+adap[-1],σ+adap[-1],100)])),δ])
        adap.append(adap[-1]+ap)
    return adap

def shapes_comparation(x,y):
    '''
    Compare the shapes of the arrays x and y, and raise and error if they have different shapes.
    '''
    if np.shape(x)[0] != np.shape(y)[0]:
        raise ValueError("x and y must have same length, but have  {} and {}".format(np.shape(x)[0],np.shape(y)[0]))

def real_interval(omega=4,end=1):
    '''
    Generates a psudo-temperated "real" interval from 0 to end, having more points on the zero surroundings.
    Begins on the numpy float128 machine epsilon, and then cummulatively sums up.
    I do not know why this works, or if it would work better on other way, but this works.
    '''
    if omega<3:
        print("\033[91m DANGER: \033[0m a factor less than 3 will result in memory error.")
        print("It is \033[01m NOT \033[0m recommended to put 3 or less as factor here.")
        print("You have 5 seconds to Interrupt this kernel with Ctrl+C")
        i=5
        while i>0:
            print(i)
            time.sleep(1)
            i-=1
    δ=np.finfo(np.longfloat).eps
    a=[δ]
    c=2
    while a[-1]<end:
        a.append(c**omega*δ)
        c+=1
    return a[:-1]


text:
# pylint: disable=E1101,R,C

import open3d as o3d

import torch.utils.data

import numpy as np

from utils import io as uio
from utils import geometry as ug
from features import det_uniform_sampling as kpus

class LRFBenchmarkLPCDDataset(torch.utils.data.Dataset):

    """
    PyTorch Dataset component to extract the local support of a given feature point.
    Initialized with a pair of clouds and a ground truth rotation matrix between them,
    the keypoints are uniformly sampled, local support extract and spherical voxelization computed.
    """

    def __init__(self,
                 cloud,
                 indices_kp,
                 support_radius,
                 input_transformation):
        """

        :param cloud: point cloud
        :param indices_kp: arrays of keypoint indices
        :param support_radius: radius for local support
        :param input_transformation: transformation to apply to local support
        """
        self.cloud = cloud
        self.indices_kp = indices_kp
        self.support_radius = support_radius
        self.input_transformation = input_transformation

        self.kdtree = o3d.geometry.KDTreeFlann(self.cloud)

    def __getitem__(self, index):
        index_kp = self.indices_kp[index]
        points, signal = self._transform(index_kp)

        return points, signal, index_kp

    def __len__(self):
        return len(self.indices_kp)

    def __repr__(self):
        return self.__class__.__name__ + 'Radius Support = {}'.format(self.support_radius)

    def _transform(self, point_index):
        feature_point = self.cloud.points[point_index]

        [_, idx, _] = self.kdtree.search_radius_vector_3d(feature_point, self.support_radius)

        if np.asarray(idx).shape[0] == 1:
            # raise e.EmptySupportException('The support for point {} is empty'.format(idx_keypoint))
            # signal = torch.zeros(self.input_transformation.num_radial_division,
            #                         self.input_transformation.bandwidth * 2,
            #                         self.input_transformation.bandwidth * 2)
            # pts_normed = torch.rand(self.input_transformation.num_points, 3)
            # ^ Why is it with torch?
            signal = torch.zeros(self.input_transformation.num_radial_division,
                                    self.input_transformation.bandwidth * 2,
                                    self.input_transformation.bandwidth * 2).cpu().numpy()
            pts_normed = torch.rand(self.input_transformation.num_points, 3).cpu().numpy()
        else:
            cloud_nn = np.asarray(np.asarray(self.cloud.points)[idx[1:], :]) - np.asarray(feature_point)
            signal, pts_normed = self.input_transformation(cloud_nn)

        return pts_normed, signal

ext: py

max_stars_repo_name: QiangZiBro/compass

max_issues_repo_name: CVLAB-Unibo/compass

max_forks_repo_name: QiangZiBro/compass

reason: import numpy


text:
# -------------------------------------------------------------------------------------
# Libraries
import copy
import logging
import os

import numpy as np
import pandas as pd
import xarray as xr

from src.hyde.dataset.nwp.wrf.lib_wrf_variables import computeRain
from src.hyde.algorithm.io.nwp.gfs.lib_gfs_io_generic import reshape_var3d
from src.hyde.algorithm.settings.model.rfarm.lib_rfarm_args import logger_name

log_stream = logging.getLogger(logger_name)

# Debug
# import matplotlib.pylab as plt
# -------------------------------------------------------------------------------------

# -------------------------------------------------------------------------------------
# Method to read data for wrf
def read_data_wrf(file_list,
                  var_name='RAINNC', var_geox='lon', var_geoy='lat', var_time='Time',
                  time_step_start=None, time_step_end=None):

    # Create list of filename
    if isinstance(file_list, str):
        file_list = [file_list]

    file_list_break = None
    file_list_n = file_list.__len__()

    if time_step_start is None:
        time_step_start = 0
    if time_step_end is None:
        time_step_end = file_list_n

    file_available = []
    for file_id, file_step in enumerate(file_list):
        if os.path.exists(file_step):
            file_available.append(file_step)
            file_list_break = False
        else:
            log_stream.warning(' ====> File not found [' + file_step + '] - Following files are not included!')
            file_list_break = True
            break

    file_available_n = file_available.__len__()
    if file_available_n != file_list_n:
        log_stream.warning(' ====> Available file(s) are ' + str(file_available_n) +
                           ' -- Expected file(s) are ' + str(file_list_n))
        log_stream.warning(' ====> Time steps are limited to ' + str(file_available_n))

        file_step_start = time_step_start
        file_step_end = file_available_n
    else:
        file_step_start = time_step_start
        file_step_end = time_step_end

    # Get data using a list of filename
    file_handle = xr.open_mfdataset(file_available)
    # Get data values
    var_data_tyx = file_handle[var_name][file_step_start:file_step_end, :, :]

    # Get time steps
    var_time_array = file_handle[var_time].values
    var_time_idx = pd.DatetimeIndex(var_time_array)[file_step_start:file_step_end]

    # Get geographical information
    var_geox_1d = file_handle[var_geox].values
    var_geoy_1d = file_handle[var_geoy].values
    [var_geox_2d, var_geoy_2d] = np.meshgrid(var_geox_1d, var_geoy_1d, sparse=False, indexing='xy')
    var_geoy_2d = np.flipud(var_geoy_2d)

    # Set correct south-north and west-east direction(s)
    var_data_xyt = np.zeros([var_data_tyx.shape[1], var_data_tyx.shape[2], file_list_n])
    var_data_xyt[:, :, :] = np.nan
    for i in range(0, var_data_tyx.shape[0]):
        var_data_xyt[:, :, i] = np.flipud(var_data_tyx[i, :, :])

    var_time_freq = var_time_idx.inferred_freq
    var_time_first = var_time_idx[0]
    var_time_last = var_time_idx[-1]
    if file_list_break:
        var_time_idx = pd.date_range(start=var_time_first, freq=var_time_freq, periods=file_list_n)

    time_issue = False
    if var_time_freq is None:
        var_time_exp = pd.date_range(start=var_time_first, end=var_time_last, periods=var_data_tyx.shape[0])
        for time_idx, time_exp in zip(var_time_idx, var_time_exp):
            time_idx = pd.DatetimeIndex([time_idx])
            time_exp = pd.DatetimeIndex([time_exp])
            if not time_idx.equals(time_exp):
                log_stream.warning(
                    ' ====> Time definition is not correct - Get: ' + str(time_idx[0]) + ' Exp: ' + str(time_exp[0]))
                time_issue = True
                break
        if time_issue:
            var_time_idx = var_time_exp
            log_stream.warning(
                ' ====> Time idx derived by file attribute have to be changed according with time exp')

    # DEBUG START
    # import matplotlib.pylab as plt
    # plt.figure()
    # plt.imshow(var_geox_2d)
    # plt.colorbar();
    # plt.figure()
    # plt.imshow(var_geoy_2d)
    # plt.colorbar()
    # plt.figure()
    # plt.imshow(var_data_tyx[2, :, :])
    # plt.clim(0, 20)
    # plt.figure()
    # plt.imshow(var_data_xyt[:, :, 2])
    # plt.clim(0, 20)
    # plt.show()
    # DEBUG END

    return var_data_xyt, var_time_idx, var_geox_2d, var_geoy_2d
# -------------------------------------------------------------------------------------

# -------------------------------------------------------------------------------------
# Method to convert data for wrf
def convert_data_wrf(var_data_raw, var_units, var_type_feat='accumulated'):
    var_data_cmp = computeRain(var_data_raw, oVarUnits=[var_units],  oVarType=[var_type_feat])

    var_data_n = var_data_cmp.shape[2]

    var_data_def = np.zeros([var_data_cmp.shape[0], var_data_cmp.shape[1], var_data_cmp.shape[2]])
    for i in range(0, var_data_n):
        var_data_step = var_data_cmp[:, :, i]

        if np.any(var_data_step < 0):
            log_stream.warning(' ====> Some values are less then 0.0 at step ' + str(i + 1) + ' -- Set to 0.0')
            var_data_step[var_data_step <= 0.0] = 0.0

        if np.all(np.isnan(var_data_step)):
            log_stream.warning(' ====> All values are NaNs at step ' + str(i + 1) + ' -- Set to 0.0')
            var_data_step[:, :] = 0.0
        elif np.any(np.isnan(var_data_step)):
            log_stream.warning(' ====> Some values are NaNs at step ' + str(i + 1) + ' -- Set to 0.0')
            var_data_step[np.isnan(var_data_step)] = 0.0

        var_data_def[:, :, i] = var_data_step

    return var_data_def
# -------------------------------------------------------------------------------------

# -------------------------------------------------------------------------------------
# Method to convert time for wrf
def convert_time_wrf(var_time_cmp, var_time_exp=None, var_type_feat='accumulated'):

    if var_type_feat == 'accumulated':
        var_time_cmp = var_time_cmp[1:]
    elif var_type_feat == 'istantaneous':
        var_time_cmp = var_time_cmp[0:]
    else:
        log_stream.error(' ====> Data type are not defined correctly')
        raise TypeError

    if var_time_exp.equals(var_time_cmp):
        var_time_series = var_time_exp
    else:
        var_freq_inferred = var_time_cmp.inferred_freq

        if var_freq_inferred is not None:
            var_time_series = pd.date_range(start=var_time_cmp[0],
                                            periods=list(var_time_cmp.values).__len__(),
                                            freq=var_freq_inferred)
        else:
            log_stream.error(' ====> Data frequency is not defined correctly')
            raise TypeError(' Data frequency is equal to ' + str(var_freq_inferred))

    return var_time_series
# -------------------------------------------------------------------------------------

# -------------------------------------------------------------------------------------
# Method to read gfs025 forecast

def read_data_gfs_025(file_name, tag_time='time', tag_geo_x='lon', tag_geo_y='lat',
                var_units='kg m**-2', var_step_type=None):

    # Parse args
    #var_name = list(var_name)[0]
    #var_units = var_units[0]
    #var_step_type = var_step_type[0]

    # Starting info
    log_stream.info(' --> Open file ' + file_name + ' ... ')

    # Open datasets
    dst_tmp = xr.open_dataset(file_name)

    if dst_tmp.dims.__len__() > 3:
        if 'height' in list(dst_tmp.dims):
            dst = dst_tmp.squeeze('height')
            dst = dst.drop('height')
        else:
            log_stream.warning(
                ' ==> Datasets has more then 3 dimensions. Add the name of extra-variable to remove it from datasets')
    else:
        dst = dst_tmp

    # Get variables ALL and DATA
    var_list_all = list(dst.variables)
    var_list_data = list(dst.data_vars)

    # Get time, geo x and geo y
    log_stream.info(' --->  Get time, geo_x and geo_y data ... ')
    if tag_time in var_list_all:
        da_time = dst[tag_time]
    else:
        log_stream.error(' ==> Time dimension name is not in the variables list of grib file')
        raise IOError(' ==> Check the time dimension!')
    if tag_geo_x in var_list_all:
        da_geo_x = dst[tag_geo_x]
    else:
        log_stream.error(' ==> GeoX dimension name is not in the variables list of grib file')
        raise IOError(' ==> Check the GeoX dimension!')
    if tag_geo_y in var_list_all:
        da_geo_y = dst[tag_geo_y]
    else:
        log_stream.error(' ==> GeoY dimension name is not in the variables list of grib file')
        raise IOError(' ==> Check the GeoY dimension!')
    log_stream.info(' --->  Get time, geo_x and geo_y data ... DONE')

    # Get data
    da_var = []
    for var_list_step in var_list_data:
        log_stream.info(' --->  Get ' + var_list_step + ' data ... ')
        da_step = dst[var_list_step]
        da_var.append(da_step)
        log_stream.info(' --->  Get ' + var_list_step + ' data ... DONE')

    # Ending info
    log_stream.info(' --> Open file ' + file_name + ' ... DONE')

    # Start Debug
    #mat = da_values[0].values
    #plt.figure()
    #plt.imshow(mat[0,:,:])
    #plt.colorbar()
    #plt.show()
    # End Debug

    if var_step_type is None:
        var_step_type = ['accum']
    if var_units is None:
        var_units = ['m']

    # Get values
    var_da_in = da_var[0]
    var_values_in = var_da_in.values
    var_dims_in = var_da_in.dims

    var_time = da_time
    var_geo_x = da_geo_x
    var_geo_y = da_geo_y

    if (var_units == 'kg m**-2') or (var_units == 'Kg m**-2'):
        var_units = 'mm'

    if var_units == 'm':
        var_scale_factor = 1000
    elif var_units == 'mm':
        var_scale_factor = 1
    else:
        log_stream.error(' ===> Rain components units are not allowed! Check your data!')
        raise IOError('Selected units are not allowed!')

    if (var_dims_in[0] == 'step') or (var_dims_in[0] == 'time'):
        var_values_in = reshape_var3d(var_values_in)
    var_shape_in = var_values_in.shape

    # Check attributes
    if not (var_units == 'mm') and not (var_units == 'm'):
        log_stream.error(' ===> Rain components units are not allowed! Check your data!')
        raise IOError('Data units is not allowed!')
    if (var_step_type == 'accum') or (var_step_type == 'accumulated'):

        var_values_step_start = None
        var_values_out = np.zeros([var_shape_in[0], var_shape_in[1], var_shape_in[2]])
        var_values_out[:, :, :] = np.nan

        for var_step in range(0, var_shape_in[2]):

            var_values_step_tmp = var_values_in[:, :, var_step]

            if var_values_step_start is None:
                var_values_step_end = var_values_step_tmp
                var_values_step = var_values_step_end
                var_values_step_start = var_values_step_end
            else:
                var_values_step_end = var_values_step_tmp
                var_values_step = var_values_step_end - var_values_step_start
                var_values_step_start = var_values_step_end

            var_values_step[var_values_step < 0.0] = 0.0
            var_values_out[:, :, var_step] = var_values_step / var_scale_factor

            [var_geox_2d, var_geoy_2d] = np.meshgrid(var_geo_x, var_geo_y, sparse=False, indexing='xy')

    elif (var_step_type == 'inst') or (var_step_type == 'instantaneous'):
        var_values_out = copy.deepcopy(var_values_in)
        var_values_in[var_values_in < 0.0] = 0.0
        [var_geox_2d, var_geoy_2d] = np.meshgrid(var_geo_x, var_geo_y, sparse=False, indexing='xy')

    # DEBUG START
    #import matplotlib.pylab as plt
    #plt.figure()
    #plt.imshow(var_geox_2d)
    #plt.colorbar();
    #plt.figure()
    #plt.imshow(var_geoy_2d)
    ##plt.colorbar()
    #plt.figure()
    #plt.imshow(var_values_out[:, :, 0])
    #plt.clim(0, 20)
    #plt.show(block=True)
    # DEBUG END

    return var_values_out, var_time, var_geox_2d, var_geoy_2d
# -------------------------------------------------------------------------------------

max_stars_repo_name: c-hydro/hyde

max_issues_repo_name: c-hydro/hyde

max_forks_repo_name: c-hydro/fp-hyde


text:
import math
import numpy as np
pi=math.pi
import sys

# CHOOSE YOUR STELLAR SAMPLE
sample_i = "A1"
# CHOOSE YOUR Z_LIM -- in the code, z_lim is given in kpc
z_lim = 0.2

### LOAD SAMPLE AND EFF. AREA
from LoadData import load_data, sample_name_2_cuts
R_cuts, l_cuts, absG_cuts = sample_name_2_cuts(sample_i)
sample = load_data(l_cuts, z_lim, R_cuts, absG_cuts)
sample.load_df()
df = sample.get_df()
eff_area, eff_area_z_vec = sample.get_eff_area()
sample_name = sample.sample_name

### IMPORT TENSORFLOW FUNCTIONS
from TfFunctions import tffunc
tff = tffunc(df, eff_area, eff_area_z_vec)
num_pop_params = tff.num_pop_params
# Initial minimization, 10 different starting points
minLogPost_MIN = np.inf
for i in range(10):
    minusLogPosterior, v_realisation = tff.minimize_posterior(fixed_stellar_params=True, neglect_vels=True, print_gap=1000, number_of_iterations=50000, learning_rate=1e-4)
    if not np.isnan(minusLogPosterior) and minusLogPosterior<minLogPost_MIN:
        minLogPost_MIN = minusLogPosterior
        vector = v_realisation
# HMC burn-in
step_size = tff.get_stepsize_guess()
# HMC burn-in
for ii in range(5):
    print("Burn-in round: "+str(ii+1))
    samples, log_prob, step_size, is_accepted = tff.run_HMC(p0=vector, steps=1e4, num_leapfrog_steps=3, step_size_start=step_size, burnin_steps=1e5, num_adaptation_steps=9e4)
    final_step_size = step_size[-1]
    step_size = tff.adjust_stepsize(samples)
    vector = samples[-1]
# Run chain
print("HMC, run chain")
samples, log_prob, step_size, is_accepted = tff.run_HMC(p0=vector, steps=1e4, num_steps_between_results=9, num_leapfrog_steps=3, step_size_start=step_size, burnin_steps=1e5, num_adaptation_steps=9e4)
final_step_size = step_size[-1]
step_size = tff.adjust_stepsize(samples)
vector = samples[-1]
# Save results
np.savez("../Results/chain_"+sample_name, pop_param_chain=np.array(samples, dtype=np.float32)[:,0:num_pop_params], final_step=np.array(samples, dtype=np.float32)[-1], log_prob=np.array(log_prob, dtype=np.float32), step_size=np.array(step_size, dtype=np.float32), final_step_size=np.array(final_step_size, dtype=np.float32))

print("\n\n\n", sample_name)
print("ALL DONE")

max_stars_repo_name: AxelWidmark/mosaik

max_issues_repo_name: AxelWidmark/mosaik

max_forks_repo_name: AxelWidmark/mosaik


text:
#!/usr/bin/python3
''' motion_detect.py '''
import cv2
import numpy as np

GAUSSIAN = (21, 21)

imageBG = None
gray = True

movement = []
AVG = 2
avgX = 0
avgY = 0
count = 0

def process_image(raw_image, control):
    ''' Process image with opencv '''
    global imageBG
    global count, avgX, avgY, movement, gray

    text = []
    images = []
    reset = False

    #Toggle Gray and reset background
    if control == ord("g"):
        if gray:
            gray = not gray
        reset = True
        print("Toggle Gray")
    #Reset the background image
    elif control == ord("r"):
        reset = True

    #Clear movement record and reset background
    if reset:
        print("Reset Background")
        imageBG = None
        movement = []

    #Keep a copy of the raw image
    text.append("Raw Image")
    images.append(raw_image)

    if gray:
        raw_image = cv2.cvtColor(raw_image, cv2.COLOR_BGR2GRAY)

    #Blur the raw image
    text.append("with Gaussian Blur...")
    images.append(cv2.GaussianBlur(raw_image, GAUSSIAN, 0))

    #Initialise background
    if imageBG is None:
        imageBG = images[-1]

    text.append("with image delta...")
    images.append(cv2.absdiff(imageBG, images[-1]))

    text.append("with threshold mask...")
    images.append(cv2.threshold(images[-1], 25, 255,
                  cv2.THRESH_BINARY)[1])

    text.append("with dilation...")
    images.append(cv2.dilate(images[-1], None, iterations=3))

    #Find contours
    if not gray:
        #Require gray image to find contours
        text.append("with dilation gray...")
        images.append(cv2.cvtColor(images[-1], cv2.COLOR_BGR2GRAY))
    text.append("with contours...")
    images.append(images[-1].copy())
    animage, contours, hierarchy = cv2.findContours(
        images[-1],
        cv2.RETR_LIST,
        cv2.CHAIN_APPROX_SIMPLE)

    #Determine the area of each of the contours
    largest_area = 0
    found_contour = []
    for cnt in contours:
        area = cv2.contourArea(cnt)
        #Find which one is largest
        if area > largest_area:
            largest_area = area
            found_contour = cnt

    if len(found_contour) > 0:
        #Find the centre of the contour
        M = cv2.moments(found_contour)
        cx, cy = int(M['m10']/M['m00']), int(M['m01']/M['m00'])
        #Calculate the average
        if count < AVG:
            avgX = (avgX+cx)/2
            avgY = (avgY+cy)/2
            count = count+1
        else:
            movement.append((int(avgX), int(avgY)))
            avgX = cx
            avgY = cy
            count = 0

    #Display
    if len(found_contour) > 0:
        cv2.circle(images[0], (cx, cy), 10, (255, 255, 255), -1)
    if len(movement) > 1:
        for i, j in enumerate(movement):
            if i > 1:
                cv2.line(images[0], movement[i-1], movement[i],
                         (255, 255, 255))
    #Display contour and hierarchy details
    if control == ord("i"):
        print("Contour: %s"%contours)
        print("Hierarchy: %s"%hierarchy)

    return(images, text)

print("r = Reset background")
print("g = Reset and toggle gray mode")
print("i = Display contour and hierarchy information")
#End


text:
"""
    Types
"""

from typing import Union
import numpy as np

class AgentType:
    config, policy, tbw = {}, None, None
    def initAgent(self, **kwargs):
        pass
    def currentState(self) -> dict:
        pass
    def loadState(self, state:dict):
        pass
    def getBatch(self, size: int,proportion: float = 1.0, random = False,) -> dict:
        pass
    def getEpisode(self, random, device = None) -> dict:
        pass
    def fullStep(self) -> dict:
        pass
    def envStep(self, action) -> dict:
        pass
    def test(self, iters:int) -> dict:
        pass
    def reset(self) -> dict:
        pass

class PolicyType:
    config, tbw = {}, None
    _test = True
    def initPolicy(self, **kwargs):
        pass
    def currentState(self) -> dict:
        pass
    def loadState(self, state:dict):
        pass
    def getAction(self, observation) -> Union[int, float, list, np.ndarray]:
        pass
    def getActions(self, observation) -> Union[list, np.ndarray]:
        pass
    def getValue(self, observation, action) -> float:
        pass
    def update(self, batchDict: dict):
        pass
    @property
    def test(self):
        return self._test
    @property
    def train(self):
        return not self._test

max_stars_repo_name: Bobobert/RoLas

max_issues_repo_name: Bobobert/RoLas

max_forks_repo_name: Bobobert/RoLas


text:
from gensim.models.keyedvectors import KeyedVectors
import numpy as np

#-----------------------------------------------------------------------------------------------------------------------

def getSemantiSimilarity(vectors_file, word_pairs):
    print("Start getSemantiSimilarity")

    words_ds = {wp[0]+"_"+wp[1]:0  for wp in word_pairs}

    # w2v vectors
    model = KeyedVectors.load_word2vec_format(vectors_file, binary=True)
    print("Vectors loaded")

    for wp0, wp1 in word_pairs:
        sim = 0
        if wp0 in model.vocab and wp1 in model.vocab:
            sim = model.similarity(wp0, wp1)
        words_ds[wp0+"_"+wp1] = sim

    print("getSemantiSimilarity finished")
    return words_ds

#-----------------------------------------------------------------------------------------------------------------------

def main():
  print("Start")
  vectors_file = "data/" + "ruwikiruscorpora_0_300_20.bin"

  l_word_pairs = [["работник_NOUN", "работник_NOUN"],["рабочий_ADJ", "работник_NOUN"],["рабочий_NOUN", "работник_NOUN"],["работодатель_NOUN", "работник_NOUN"],["работник_NOUN", "персонал_NOUN"],["персонал_NOUN", "работник_NOUN"]]
  # вывод нет смысла брать пары связанные меньше чем на 0.5 там уже мусор начинается

  l_res = getSemantiSimilarity(vectors_file, l_word_pairs)
  print(l_res)

  print("Done")

#-----------------------------------------------------------------------------------------------------------------------

if __name__ == "__main__":
    main()

max_stars_repo_name: StWishmaster/sentence-clustering

max_issues_repo_name: StWishmaster/sentence-clustering

max_forks_repo_name: StWishmaster/sentence-clustering


text:
from Tkinter import *
from PIL import Image, ImageTk
from scipy.ndimage import imread
import numpy as np
import sys
import math
import io

class Sobel:

    def __init__(self, path, root):
        self.imagePath = path
        self.image = self.rgb_conversion()
        self.w, self.h = self.image.size
        self.actualImage = self.image

        self.root = root

        self.display = ImageTk.PhotoImage(self.image)
        self.canvas = Canvas(self.root, width = self.w, height = self.h)
        self.label = Label(self.canvas, image = self.display)
        self.label.image = self.display
        self.label.pack()

        self.panelBotones = Canvas(self.root, width=150, height=self.h)
        self.boton = Button(self.panelBotones, text = 'Grey Scale', fg = 'black', command = self.black_and_white)
        self.panel = self.panelBotones.create_window(5,0, anchor = 'nw', window = self.boton)
        self.boton = Button(self.panelBotones, text = 'Sobel Convertion', fg = 'black', command = self.sobel_convertion)
        self.panel = self.panelBotones.create_window(5,30, anchor = 'nw', window = self.boton)
        self.panelBotones.pack(side=LEFT)
        self.canvas.pack()

    def update_image(self):
        display = ImageTk.PhotoImage(self.actualImage)
        self.label.config(image=display)
        self.label.image = display

    def rgb_conversion(self):
        image = Image.open(self.imagePath)
        image = image.convert('RGB')
        return image

    def black_and_white(self):
        greyImage = Image.new('L',(self.w, self.h))
        greyPixels = greyImage.load()
        pixels = self.image.load()

        for x in range(self.w):
            for y in range(self.h):
                average = sum(pixels[x,y])/3
                greyPixels[x,y] = average
        greyImage.save('prueba_gris.jpg')
        self.actualImage = greyImage
        self.update_image()

    def sobel_convertion(self):
        image = imread('prueba_gris.jpg')
        xMatrix = np.array([[1, 0, -1],
                            [2, 0, -2],
                            [1, 0, -1]])
        yMatrix = np.array([[-1, -2, -1],
                            [0, 0, 0],
                            [1, 2, 1]])

        sobelImage = np.zeros((image.shape[0]-2, image.shape[1]-2, 3), dtype='uint8')

        for y in range(1, image.shape[0]-1):
            for x in range(1, image.shape[1]-1):
                dx = np.sum(np.multiply(image[y-1:y+2, x-1:x+2], xMatrix))
                dy = np.sum(np.multiply(image[y-1:y+2, x-1:x+2], yMatrix))

                gradient = math.sqrt(dx * dx + dy * dy)
                if gradient > 0 and gradient < 255:
                    gradient = gradient
                elif gradient < 0:
                    gradient = 0
                else:
                    gradient = 0

                sobelImage[y-1][x-2] = gradient

        self.actualImage = Image.fromarray(sobelImage)
        self.update_image()

def main():
    try:
        imagePath = 'prueba.jpg'
    except:
        print "Selecciona una imagen"
        return
    root = Tk()
    conversion = Sobel(imagePath, root)
    root.title("Sobel Conversion")
    root.mainloop()

if __name__ == "__main__":
    main()

max_stars_repo_name: alfonsoirai/Image-Border-Analyzer

max_issues_repo_name: alfonsoirai/Image-Border-Analyzer

max_forks_repo_name: alfonsoirai/Image-Border-Analyzer

reason: import numpy,from scipy


text:
import torch
import numpy as np
from sty import fg, rs, Style, RgbFg

def get_ig_attributions(trunc_layer, model, tokenizer, query_tokens, para_tokens, label,
        batch_size, device, attr_segment, begin_num_reps,
        max_reps, max_allowed_error, max_query_length, max_seq_length, debug):
    integrated_gradients, input_ids, baseline_prediction, prediction, error_percentage, num_reps = \
        _compute_ig(attr_segment=attr_segment, trunc_layer=trunc_layer, model=model,
            tokenizer=tokenizer, query_tokens=query_tokens,
            para_tokens=para_tokens, label=label, begin_num_reps=begin_num_reps,
            batch_size=batch_size, device=device,
            max_allowed_error=max_allowed_error,
            max_reps=max_reps, max_query_length=max_query_length,
            max_seq_length=max_seq_length, debug=debug)

    integrated_gradients = _project_attributions(tokenizer,
        input_ids, integrated_gradients)

    return integrated_gradients, error_percentage, baseline_prediction, prediction, num_reps

def visualize_token_attrs(tokens, attrs):
    """
      Visualize attributions for given set of tokens.
      Args:
      - tokens: An array of tokens
      - attrs: An array of attributions, of same size as 'tokens',
        with attrs[i] being the attribution to tokens[i]

      Returns:
      - visualization: An IPython.core.display.HTML object showing
        tokens color-coded based on strength of their attribution.
    """
    def get_color(attr):
        if attr > 0:
            g = int(128*attr) + 127
            b = 128 - int(64*attr)
            r = 128 - int(64*attr)
        else:
            g = 128 + int(64*attr)
            b = 128 + int(64*attr)
            r = int(-128*attr) + 127
        fg.color = Style(RgbFg(r, g, b))
        return fg.color

    # normalize attributions for visualization.
    bound = max(abs(np.max(attrs)), abs(np.min(attrs)))
    attrs = attrs/bound
    color_text = ""
    for i, tok in enumerate(tokens):
        color = get_color(attrs[i])
        color_text += " " + color + tok + fg.rs
    return color_text

def transform_input(tokenizer, query_tokens, para_tokens, label, device, max_query_length, max_seq_length):
    query_tokens = query_tokens[:max_query_length]
    max_para_tokens_len = max_seq_length - 3 - len(query_tokens)
    para_tokens = para_tokens[:max_para_tokens_len]
    input_tokens = ['[CLS]'] + query_tokens + ['[SEP]'] + para_tokens + ['[SEP]']

    input_ids = tokenizer.convert_tokens_to_ids(input_tokens)
    token_type_ids = [0]*(2+len(query_tokens)) + [1]*(1+len(para_tokens))
    attention_mask = [1]*len(input_ids)
    assert len(input_ids) == len(token_type_ids) == len(attention_mask) <= max_seq_length
    '''
    if len(input_ids) < max_seq_length:
        pad_length = max_seq_length - len(input_ids)
        input_ids = input_ids + [0] * pad_length
        attention_mask = attention_mask + [0] * pad_length
        token_type_ids = token_type_ids + [0] * pad_length
    '''
    return (torch.LongTensor(input_ids).to(device),
            torch.LongTensor(attention_mask).to(device),
            torch.LongTensor(token_type_ids).to(device),
            torch.LongTensor([label]).to(device))

def generate_baseline(attr_segment, tokenizer, query_tokens, para_tokens, label, device, max_query_length, max_seq_length):
    if attr_segment == "all" or attr_segment == "query":
        query_tokens = ["[PAD]"] * len(query_tokens)
    if attr_segment == "all" or attr_segment == "para":
        para_tokens = ["[PAD]"] * len(para_tokens)
    return transform_input(tokenizer, query_tokens, para_tokens, label, device, max_query_length, max_seq_length)

def _project_attributions(tokenizer, input_ids, attributions):
    tokens = tokenizer.convert_ids_to_tokens(input_ids)
    return {'outputs': [tokens, attributions.astype(np.float).tolist()]}

def _get_ig_error(integrated_gradients, baseline_prediction, prediction,
                  debug=False):
    sum_attributions = 0
    sum_attributions += np.sum(integrated_gradients)

    delta_prediction = prediction - baseline_prediction

    error_percentage = \
        100 * (delta_prediction - sum_attributions) / delta_prediction
    if debug:
        print(f'prediction is {prediction}')
        print(f'baseline_prediction is {baseline_prediction}')
        print(f'delta_prediction is {delta_prediction}')
        print(f'sum_attributions are {sum_attributions}')
        print(f'Error percentage is {error_percentage}')

    return error_percentage

def _get_scaled_inputs(input_val, baseline_val, scale_arr, batch_size):
    list_scaled_embeddings = []
    scaled_embeddings = \
        [baseline_val + scale * (input_val - baseline_val) for scale in scale_arr]

    while scaled_embeddings:
        list_scaled_embeddings.append(
            np.array(scaled_embeddings[:batch_size]))
        scaled_embeddings = scaled_embeddings[batch_size:]

    return list_scaled_embeddings

def _calculate_integral(ig):
    # We use np.average here since the width of each
    # step rectangle is 1/number of steps and the height is the gradient,
    # so summing the areas is equivalent to averaging the gradient values.

    ig = (ig[:-1] + ig[1:]) / 2.0  # trapezoidal rule

    integral = np.average(ig, axis=0)

    return integral

def _compute_ig(attr_segment, trunc_layer, model, tokenizer, query_tokens, para_tokens, label, begin_num_reps, batch_size, device, max_allowed_error, max_reps, max_query_length, max_seq_length, debug):
    for name, param in model.named_parameters():
        param.requires_grad = False
    model.eval()

    origin_input = transform_input(tokenizer, query_tokens, para_tokens, label, device, max_query_length, max_seq_length)
    input_ids, attention_mask, token_type_ids, _ = origin_input
    attention_mask = attention_mask.unsqueeze(0)
    token_type_ids = token_type_ids.unsqueeze(0)

    baseline_input_ids = generate_baseline(attr_segment, tokenizer, query_tokens, para_tokens, label, device)[0]

    input_ids
    batch_pre_input_ids = torch.cat(
        [input_ids.unsqueeze(0), baseline_input_ids.unsqueeze(0)], dim=0)
    _, hidden_states = model(trunc_layer=-1,
                input_ids=batch_pre_input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids, position_ids=None, head_mask=None,
                inputs_embeds=None
    )
    # returned embeddings including word embeddings
    trunc_embeddings = hidden_states[trunc_layer].detach().cpu().numpy()
    origin_input_embds = trunc_embeddings[0]
    baseline_input_embds = trunc_embeddings[1]

    scale_arr = np.linspace(0, 1, begin_num_reps*batch_size)

    scaled_embeddings = _get_scaled_inputs(origin_input_embds, baseline_input_embds,
                                           scale_arr, batch_size)

    scores = []
    path_gradients = []
    baseline_prediction, prediction = None, None
    while True:
        for input_embds in scaled_embeddings:
            input_embds = torch.FloatTensor(input_embds).to(device)
            input_embds.requires_grad=True

            score_rep = model(trunc_layer=trunc_layer,
                input_ids=None, attention_mask=attention_mask,
                token_type_ids=token_type_ids, position_ids=None, head_mask=None,
                inputs_embeds=input_embds)[0][:, label]
            tp_sum = torch.sum(score_rep) # for backward
            tp_sum.backward()

            path_gradients_rep = input_embds.grad.detach().cpu().numpy()
            input_embds.requires_grad=False
            path_gradients.append(path_gradients_rep)
            scores.append(score_rep.detach().cpu().numpy())

        if baseline_prediction is None:
            baseline_prediction = scores[0][0]  # first score is the baseline prediction
        if prediction is None:
            prediction = scores[-1][-1]  # last score is the input prediction

        # integrating the gradients and multiplying with the difference of the
        # baseline and input.
        ig = np.concatenate(path_gradients, axis=0)

        arg_sort_idxes = np.argsort(scale_arr)
        ig = ig[arg_sort_idxes]

        integral = _calculate_integral(ig)
        integrated_gradients = (origin_input_embds - baseline_input_embds) * integral
        integrated_gradients = np.sum(integrated_gradients, axis=-1)
        error_percentage = \
            _get_ig_error(integrated_gradients, baseline_prediction, prediction, debug=debug)

        if abs(error_percentage) <= max_allowed_error or len(scaled_embeddings) >= max_reps:
            break
        sort_scale_arr = scale_arr[arg_sort_idxes]
        next_scale_arr = (sort_scale_arr[1:] + sort_scale_arr[:-1]) / 2.0
        scaled_embeddings = _get_scaled_inputs(origin_input_embds, baseline_input_embds,
                                           next_scale_arr, batch_size)
        scale_arr = np.concatenate((scale_arr, next_scale_arr), axis=0)

    input_ids = input_ids.detach().cpu().numpy()
    return (integrated_gradients, input_ids,
        float(baseline_prediction), float(prediction),
        float(error_percentage), len(scale_arr))

max_stars_repo_name: jingtaozhan/bert-ranking-analysis

max_issues_repo_name: jingtaozhan/bert-ranking-analysis

max_forks_repo_name: jingtaozhan/bert-ranking-analysis


text:
from itertools import product

import numpy as np
from numpy.testing import assert_allclose
import pandas as pd
import pytest

from linearmodels.iv import IV2SLS
from linearmodels.panel.data import PanelData
from linearmodels.panel.model import PooledOLS
from linearmodels.tests.panel._utility import (
    access_attributes,
    assert_frame_similar,
    assert_results_equal,
    datatypes,
    generate_data,
)

pytestmark = pytest.mark.filterwarnings(
    "ignore::linearmodels.shared.exceptions.MissingValueWarning"
)

missing = [0.0, 0.20]
has_const = [True, False]
perms = list(product(missing, datatypes, has_const))
ids = ["missing-" + "-".join([str(param) for param in perm]) for perm in perms]

@pytest.fixture(params=perms, ids=ids)
def data(request):
    missing, datatype, const = request.param
    return generate_data(missing, datatype, const=const, other_effects=1)

def test_pooled_ols(data):
    mod = PooledOLS(data.y, data.x)
    res = mod.fit(debiased=False)

    y = mod.dependent.dataframe.copy()
    x = mod.exog.dataframe.copy()
    y.index = np.arange(len(y))
    x.index = y.index

    res2 = IV2SLS(y, x, None, None).fit(cov_type="unadjusted")
    assert_results_equal(res, res2)

    res3 = mod.fit(cov_type="homoskedastic", debiased=False)
    assert_results_equal(res, res3)

def test_pooled_ols_weighted(data):
    mod = PooledOLS(data.y, data.x, weights=data.w)
    res = mod.fit(debiased=False)

    y = mod.dependent.dataframe
    x = mod.exog.dataframe
    w = mod.weights.dataframe
    y.index = np.arange(len(y))
    w.index = x.index = y.index

    res2 = IV2SLS(y, x, None, None, weights=w).fit(cov_type="unadjusted")
    assert_results_equal(res, res2)

def test_diff_data_size(data):
    if isinstance(data.x, pd.DataFrame):
        entities = data.x.index.levels[0]
        x = data.x.loc[pd.IndexSlice[entities[0] : entities[-2]]]
        y = data.y
    elif isinstance(data.x, np.ndarray):
        x = data.x
        y = data.y[:-1]
    else:  # xr.DataArray:
        x = data.x[:, :, :-1]
        y = data.y

    with pytest.raises(ValueError):
        PooledOLS(y, x)

def test_rank_deficient_array(data):
    x = data.x.copy()
    if isinstance(data.x, pd.DataFrame):
        x.iloc[:, 1] = x.iloc[:, 0]
    else:
        x[1] = x[0]
    with pytest.raises(ValueError):
        PooledOLS(data.y, x)

def test_results_access(data):
    mod = PooledOLS(data.y, data.x)
    res = mod.fit(debiased=False)
    access_attributes(res)

    mod = PooledOLS(data.y, data.x)
    res = mod.fit(debiased=True)
    access_attributes(res)

    if not isinstance(data.x, pd.DataFrame):
        return
    x = data.y.copy()
    x.iloc[:, :] = 1
    mod = PooledOLS(data.y, x)
    res = mod.fit(debiased=False)
    access_attributes(res)

def test_alt_rsquared(data):
    mod = PooledOLS(data.y, data.x)
    res = mod.fit(debiased=False)
    assert_allclose(res.rsquared, res.rsquared_overall)

def test_alt_rsquared_weighted(data):
    mod = PooledOLS(data.y, data.x, weights=data.w)
    res = mod.fit(debiased=False)
    assert_allclose(res.rsquared, res.rsquared_overall)

def test_cov_equiv(data):
    mod = PooledOLS(data.y, data.x)
    res = mod.fit(cov_type="robust", debiased=False)
    y = mod.dependent.dataframe.copy()
    x = mod.exog.dataframe.copy()
    y.index = np.arange(len(y))
    x.index = y.index
    res2 = IV2SLS(y, x, None, None).fit(cov_type="robust")
    assert_results_equal(res, res2)

    res3 = mod.fit(cov_type="heteroskedastic", debiased=False)
    assert_results_equal(res, res3)

def test_cov_equiv_weighted(data):
    mod = PooledOLS(data.y, data.x, weights=data.w)
    res = mod.fit(cov_type="robust", debiased=False)
    y = mod.dependent.dataframe.copy()
    x = mod.exog.dataframe.copy()
    w = mod.weights.dataframe.copy()
    y.index = np.arange(len(y))
    w.index = x.index = y.index

    res2 = IV2SLS(y, x, None, None, weights=w).fit(cov_type="robust")
    assert_results_equal(res, res2)

    res3 = mod.fit(cov_type="heteroskedastic", debiased=False)
    assert_results_equal(res, res3)

def test_cov_equiv_cluster(data):
    mod = PooledOLS(data.y, data.x)
    res = mod.fit(cov_type="clustered", cluster_entity=True, debiased=False)
    y = PanelData(data.y)
    clusters = pd.DataFrame(y.entity_ids, index=y.index)
    res2 = mod.fit(cov_type="clustered", clusters=clusters, debiased=False)
    assert_results_equal(res, res2)

    res = mod.fit(cov_type="clustered", cluster_time=True, debiased=False)
    clusters = pd.DataFrame(y.time_ids, index=y.index)
    res2 = mod.fit(cov_type="clustered", clusters=clusters, debiased=False)
    assert_results_equal(res, res2)

    res = mod.fit(cov_type="clustered", clusters=data.vc1, debiased=False)
    y = mod.dependent.dataframe.copy()
    x = mod.exog.dataframe.copy()
    y.index = np.arange(len(y))
    x.index = y.index
    clusters = mod.reformat_clusters(data.vc1)
    ols_mod = IV2SLS(y, x, None, None)
    res2 = ols_mod.fit(
        cov_type="clustered", clusters=clusters.dataframe, debiased=False
    )
    assert_results_equal(res, res2)

def test_cov_equiv_cluster_weighted(data):
    mod = PooledOLS(data.y, data.x, weights=data.w)
    res = mod.fit(cov_type="clustered", clusters=data.vc1, debiased=False)

    y = mod.dependent.dataframe.copy()
    x = mod.exog.dataframe.copy()
    w = mod.weights.dataframe
    y.index = np.arange(len(y))
    w.index = x.index = y.index
    clusters = mod.reformat_clusters(data.vc1)
    ols_mod = IV2SLS(y, x, None, None, weights=w)
    res2 = ols_mod.fit(cov_type="clustered", clusters=clusters.dataframe)
    assert_results_equal(res, res2)

def test_two_way_clustering(data):
    mod = PooledOLS(data.y, data.x)

    y = PanelData(data.y)
    entity_clusters = pd.DataFrame(y.entity_ids, index=y.index)
    vc1 = PanelData(data.vc1)
    clusters = vc1.copy()
    clusters.dataframe["var.cluster.entity"] = entity_clusters
    clusters._frame = clusters._frame.astype(np.int64)
    res = mod.fit(cov_type="clustered", clusters=clusters, debiased=False)

    y = mod.dependent.dataframe.copy()
    x = mod.exog.dataframe.copy()
    y.index = np.arange(len(y))
    x.index = y.index
    clusters = mod.reformat_clusters(clusters)

    ols_mod = IV2SLS(y, x, None, None)
    ols_res = ols_mod.fit(cov_type="clustered", clusters=clusters.dataframe)
    assert_results_equal(res, ols_res)

def test_fitted_effects_residuals(data):
    mod = PooledOLS(data.y, data.x)
    res = mod.fit()
    expected = pd.DataFrame(res.resids.copy())
    expected.columns = ["idiosyncratic"]
    assert_allclose(res.idiosyncratic, expected)
    assert_frame_similar(res.idiosyncratic, expected)

    expected = mod.dependent.values2d - res.resids.values[:, None]
    expected = pd.DataFrame(expected, index=res.resids.index, columns=["fitted_values"])
    assert_allclose(res.fitted_values, expected)
    assert_frame_similar(res.fitted_values, expected)

    expected.iloc[:, 0] = np.nan
    expected.columns = ["estimated_effects"]
    assert_allclose(res.estimated_effects, expected)
    assert_frame_similar(res.estimated_effects, expected)

def test_extra_df(data):
    mod = PooledOLS(data.y, data.x)
    res = mod.fit()
    res_extra = mod.fit(extra_df=10)
    assert np.all(np.diag(res_extra.cov) > np.diag(res.cov))

@pytest.mark.parametrize(
    "cov_type", ["dk", "driscoll-kraay", "kernel", "ac", "autocorrelated"]
)
def test_alt_cov(data, cov_type):
    mod = PooledOLS(data.y, data.x)
    res = mod.fit(cov_type=cov_type)
    res_def = mod.fit()
    assert not np.all(np.isclose(res.cov, res_def.cov))
    if cov_type.startswith("a"):
        cov_name = "Autocorrelation Rob. Cov."
    else:
        cov_name = "Driscoll-Kraay"
    assert cov_name in str(res.summary)

def test_uneuqal_samples():
    data = generate_data(False, "pandas")
    with pytest.raises(ValueError, match="dependent and exog must have"):
        PooledOLS(data.y.iloc[::2], data.x)

max_stars_repo_name: clarityai-eng/linearmodels

max_issues_repo_name: clarityai-eng/linearmodels

max_forks_repo_name: clarityai-eng/linearmodels

reason: import numpy,from numpy


text:
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

import numpy as np
import pytest

import mindspore.context as context
import mindspore.nn as nn
from mindspore import Tensor, Parameter
from mindspore.ops import operations as P
import mindspore.common.dtype as mstype

context.set_context(mode=context.GRAPH_MODE, device_target="GPU")

var_np = np.random.rand(3, 3).astype(np.float32)
accum_np = np.random.rand(3, 3).astype(np.float32)

class Net(nn.Cell):
    def __init__(self):
        super(Net, self).__init__()
        self.apply_adagrad = P.ApplyAdagrad()
        self.var = Parameter(Tensor(var_np), name="var")
        self.accum = Parameter(Tensor(accum_np), name="accum")

    def construct(self, lr, grad):
        self.apply_adagrad(self.var, self.accum, lr, grad)
        return self.var, self.accum

@pytest.mark.level0
@pytest.mark.platform_x86_gpu_training
@pytest.mark.env_onecard
def test_apply_adagrad():
    # numpy op
    grident_np = np.random.rand(3, 3).astype(np.float32)
    expect_accum_np = accum_np + grident_np * grident_np
    expect_var_np = var_np - (0.001 * grident_np * (1 / np.sqrt(expect_accum_np + 1e-6)))

    net = Net()
    lr = Tensor(0.001, mstype.float32)
    grad = Tensor(grident_np)
    out = net(lr, grad)
    res_var_mindspore = out[0].asnumpy()
    res_accum_mindspore = out[1].asnumpy()
    eps = np.array([1e-6 for i in range(9)]).reshape(3, 3)

    assert np.all(expect_var_np - res_var_mindspore < eps)
    assert np.all(expect_accum_np - res_accum_mindspore < eps)

max_stars_repo_name: GuoSuiming/mindspore


text:
import multiprocessing as mp
import pytest

from distributed import Client
from distributed.deploy.local import LocalCluster
from dask_cuda.explicit_comms import CommsContext, dataframe_merge

import pandas as pd
import dask.dataframe as dd

import numpy as np
import pytest
import cudf
import cupy

mp = mp.get_context("spawn")
ucp = pytest.importorskip("ucp")

# Notice, all of the following tests is executed in a new process such
# that UCX options of the different tests doesn't conflict.

async def my_rank(state):
    return state["rank"]

def _test_local_cluster(protocol):
    with LocalCluster(
        protocol=protocol,
        dashboard_address=None,
        n_workers=4,
        threads_per_worker=1,
        processes=True,
    ) as cluster:
        with Client(cluster) as client:
            comms = CommsContext(client)
            assert sum(comms.run(my_rank)) == sum(range(4))

@pytest.mark.parametrize("protocol", ["tcp", "ucx"])
def test_local_cluster(protocol):
    p = mp.Process(target=_test_local_cluster, args=(protocol,))
    p.start()
    p.join()
    assert not p.exitcode

def _test_dataframe_merge(backend, protocol, n_workers):
    with LocalCluster(
        protocol=protocol,
        dashboard_address=None,
        n_workers=n_workers,
        threads_per_worker=1,
        processes=True,
    ) as cluster:
        with Client(cluster) as client:
            comms = CommsContext(client)
            nrows = n_workers * 10

            # Let's make some dataframes that we can join on the "key" column
            df1 = pd.DataFrame({"key": np.arange(nrows), "payload1": np.arange(nrows)})
            key = np.arange(nrows)
            np.random.shuffle(key)
            df2 = pd.DataFrame(
                {"key": key[nrows // 3 :], "payload2": np.arange(nrows)[nrows // 3 :]}
            )
            expected = df1.merge(df2).set_index("key")

            if backend == "cudf":
                df1 = cudf.DataFrame.from_pandas(df1)
                df2 = cudf.DataFrame.from_pandas(df2)

            ddf1 = dd.from_pandas(df1, npartitions=n_workers + 1)
            ddf2 = dd.from_pandas(
                df2, npartitions=n_workers - 1 if n_workers > 1 else 1
            )
            ddf3 = dataframe_merge(ddf1, ddf2, on="key").set_index("key")
            got = ddf3.compute()

            if backend == "cudf":
                got = got.to_pandas()
                got.index.names = ["key"]  # TODO: this shouldn't be needed

            pd.testing.assert_frame_equal(got, expected)

@pytest.mark.parametrize("nworkers", [1, 2, 4])
@pytest.mark.parametrize("backend", ["pandas", "cudf"])
@pytest.mark.parametrize("protocol", ["tcp", "ucx"])
def test_dataframe_merge(backend, protocol, nworkers):
    p = mp.Process(target=_test_dataframe_merge, args=(backend, protocol, nworkers))
    p.start()
    p.join()
    assert not p.exitcode

max_stars_repo_name: efajardo-nv/dask-cuda

max_issues_repo_name: efajardo-nv/dask-cuda

max_forks_repo_name: efajardo-nv/dask-cuda

reason: import numpy,import cupy


text:
from pycocotools.coco import COCO
import numpy as np
import skimage.io as io
import matplotlib.pyplot as plt
import pylab
pylab.rcParams['figure.figsize'] = (8.0, 10.0)

import pycocotools.mask as maskUtils
from matplotlib.patches import Rectangle

from utils import *

dataDir='..'
dataType='train2017'
annFile='{}/annotations/instances_{}.json'.format(dataDir,dataType)
# initialize COCO api for instance annotations
coco_inst=COCO(annFile)

# initialize COCO api for person keypoints annotations
# Format: http://cocodataset.org/#format-data
annFile = '{}/annotations/person_keypoints_{}.json'.format(dataDir,dataType)
coco_kps=COCO(annFile)

# initialize COCO api for caption annotations
annFile = '{}/annotations/captions_{}.json'.format(dataDir,dataType)
coco_caps=COCO(annFile)

# initialize COCO STUFF api for caption annotations
annFile = '%s/annotations/stuff_%s.json' % (dataDir, dataType)
coco_stuff = COCO(annFile)

# display COCO **instance** categories and supercategories
cats = coco_inst.loadCats(coco_inst.getCatIds())
nms=[cat['name'] for cat in cats]
print('COCO categories: \n{}\n'.format(' '.join(nms)))
nms = set([cat['supercategory'] for cat in cats])
print('COCO supercategories: \n{}'.format(' '.join(nms)))

# Display COCO **stuff** categories and supercategories
categories = coco_stuff.loadCats(coco_stuff.getCatIds())
categoryNames = [cat['name'] for cat in categories]
print 'COCO Stuff leaf categories: \n', ' '.join(categoryNames)
superCategoryNames = sorted(set([cat['supercategory'] for cat in categories]))
print 'COCO Stuff super categories: \n', ' '.join(superCategoryNames)

if_vis = False
# split_name = 'imgs_with_morethan2_standing_persons'
split_name = 'imgs_with_morethan2_standing_persons_train2017'

import imageio
from tqdm import tqdm

# get all images containing given categories, select one at random
catIds = coco_inst.getCatIds(catNms=['person']);
imgIds = coco_inst.getImgIds(catIds=catIds );
# print(imgIds)
# imgIds = coco.getImgIds(imgIds = [324158])

count_total_obj = 0
size_list = []
valid_img_count = 0
for ImgIdx, imgID in tqdm(enumerate(imgIds)):
    # if valid_img_count%100 == 0:
    #     if_vis = True
    # else:
    #     if_vis = False
    img = coco_inst.loadImgs(imgID)[0]

    # load and display instance annotations
    annIds = coco_inst.getAnnIds(imgIds=img['id'], catIds=catIds, iscrowd=None)
    anns = coco_inst.loadAnns(annIds)
#     coco.showAnns(anns)
    anns_kps = coco_kps.loadAnns(annIds)
    anns_caps = coco_caps.loadAnns(coco_caps.getAnnIds(imgIds=img['id']))
    annIds_allCats = coco_inst.getAnnIds(imgIds=img['id'], iscrowd=None)
    anns_allCats = coco_inst.loadAnns(annIds_allCats)
    annIds_stuff = coco_stuff.getAnnIds(imgIds=img['id'])
    anns_stuff = coco_stuff.loadAnns(annIds_stuff)

    any_clear = False
    num_clear = 0
    num_valid = 0
    bbox_merged_list = []
    if_clear_list = []
    ratio_list = []
    for idx in range(len(anns)):
        ## Get kps
        ann_kps = anns_kps[idx]
        if_clear = check_clear(ann_kps) # check for head up and foot down person

        ## Get bbox
        ann = anns[idx]
        mask1 = ann['segmentation']
        # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/coco.py#L265
        rle = maskUtils.frPyObjects(mask1, img['height'], img['width'])
        area = maskUtils.area(rle)
        bboxes = maskUtils.toBbox(rle) # [x, y, w, h]
        if len(bboxes.shape)!=2:
#             print('Warning!! len(bboxes.shape)!=2')
            continue
        bbox_merged = bboxes[0] if len(bboxes)==1 else merge_bboxes(bboxes)

        if bbox_merged[2] == 0. or bbox_merged[3] == 0.:
            continue
        ratio = float(bbox_merged[3]) / float(bbox_merged[2])
#         if ratio <= 2.:
#             continue

        if if_clear:
            any_clear = True
            num_clear += 1
            if ratio > 2. and ratio < 8.:
                num_valid += 1
        bbox_merged_list.append(bbox_merged)
        if_clear_list.append(if_clear)
        ratio_list.append(ratio)

    if any_clear == False:
        continue
    if num_valid < 2:
        continue

    ## load and display image
    # I = io.imread('%s/images/%s/%s'%(dataDir,dataType,img['file_name']))
    # use url to load image
    I = io.imread(img['coco_url'])

    if if_vis:
    #     plt.figure(figsize=(10, 10))
        plt.figure(figsize=(30, 40))
        plt.subplot(121)
        plt.title('%s'%ImgIdx)
    # #     plt.imshow(I); plt.axis('off')
        plt.imshow(I);
    #     plt.axis('off')

        coco_kps.showAnns(anns_kps)
        coco_caps.showAnns(anns_caps)
        coco_inst.showAnns(anns_allCats)

        plt.subplot(122)
        plt.imshow(I); plt.axis('off')

    stuff_id_map = np.zeros((I.shape[0], I.shape[1]), dtype=np.int)
    for ann in anns_stuff:
        color = coco_stuff.showAnns([ann])
        cat = categories[ann['category_id']-92]['name']
        cat_sup = categories[ann['category_id']-92]['supercategory']
        if cat_sup != 'other':
            bbox = ann['bbox']
            plt.text(bbox[0]+bbox[2]/2., bbox[1]+bbox[3]/2., '%s/%s'%(cat_sup, cat),
                 fontsize=12, color='w', bbox=dict(facecolor=color[0], alpha=0.5, edgecolor='w'))

        mask = maskUtils.decode(ann['segmentation'])
        stuff_id_map += mask * (ann['category_id']-92)

    if if_vis:
        plt.subplot(121)
        ax = plt.gca()
    H, W = I.shape[0], I.shape[1]
    bbox_valid_list = []
    for bbox_merged, if_clear, ratio in zip(bbox_merged_list, if_clear_list, ratio_list):
        if not if_clear:
            continue
        if_valid = if_clear
        if_ratio = ratio > 2. and ratio < 8.
        if_valid = if_clear and if_ratio

        if if_vis:
            rect = Rectangle((bbox_merged[0], bbox_merged[1]), bbox_merged[2], bbox_merged[3], linewidth=2, edgecolor='lime' if if_valid else 'r', facecolor='none')
            # Add the patch to the Axes
            ax.add_patch(rect)
            plt.text(bbox_merged[0], bbox_merged[1]-12, '%.2f'%ratio,
                     fontsize=12, bbox=dict(facecolor='lime' if if_valid else 'red', alpha=0.5))

        lower_left = [np.clip(int(bbox_merged[0]), 0, W-1), np.clip(int(bbox_merged[1]+bbox_merged[3]), 0, H-1)]
        lower_right = [np.clip(int(bbox_merged[0]+bbox_merged[2]), 0, W-1), np.clip(int(bbox_merged[1]+bbox_merged[3]), 0, H-1)]
        lower_left_id = stuff_id_map[lower_left[1], lower_left[0]]
        lower_left_id_subs = [categories[lower_left_id]['supercategory'], categories[lower_left_id]['name']] if lower_left_id!=0 else ['null', 'null']
        lower_right_id = stuff_id_map[lower_right[1], lower_right[0]]
        lower_right_id_subs = [categories[lower_right_id]['supercategory'], categories[lower_right_id]['name']] if lower_right_id!=0 else ['null', 'null']
        if_valid_surface = check_valid_surface(lower_left_id_subs) or check_valid_surface(lower_right_id_subs)

        if if_vis:
            plt.text(bbox_merged[0], bbox_merged[1]+bbox_merged[3]+6, '%s/%s'%(lower_left_id_subs[0], lower_left_id_subs[1]),
                     fontsize=12, bbox=dict(facecolor='lime' if if_valid_surface else 'red', alpha=0.5))
            plt.text(bbox_merged[0], bbox_merged[1]+bbox_merged[3]+18, '%s/%s'%(lower_right_id_subs[0], lower_right_id_subs[1]),
                     fontsize=12, bbox=dict(facecolor='lime' if if_valid_surface else 'red', alpha=0.5))

        if if_valid:
            bbox_valid_list.append(bbox_merged) # [x, y, w, h]

    if if_vis:
        plt.savefig('/Users/ruzhu/Documents/Results/%s/%06d_%s.png'%(split_name, ImgIdx, imgID), bbox_inches='tight')
        plt.show()
        clear_output()

    bboxes_valid = np.vstack(bbox_valid_list)
    np.save('/Users/ruzhu/Documents/Results/%s/%06d_%s_bboxes_valid'%(split_name, ImgIdx, imgID), bboxes_valid)

    valid_img_count = valid_img_count + 1

max_stars_repo_name: Jerrypiglet/cocoapi

max_issues_repo_name: Jerrypiglet/cocoapi

max_forks_repo_name: Jerrypiglet/cocoapi


text:
import torch
import torch.nn as nn
from torch.autograd import Variable
import collections
import numpy as np

class strLabelConverter(object):
    """Convert between str and label.

    NOTE:
        Insert `blank` to the alphabet for CTC.

    Args:
        alphabet (str): set of the possible characters.
        ignore_case (bool, default=True): whether or not to ignore all of the case.
    """
    def __init__(self, alphabet, ignore_case=True):
        self._ignore_case = ignore_case
        if self._ignore_case:
            alphabet = alphabet.lower()
        self.alphabet = alphabet + '-'  # for `-1` index

        self.dict = {}
        for i, char in enumerate(alphabet):
            # NOTE: 0 is reserved for 'blank' required by wrap_ctc
            self.dict[char] = i + 1

    def encode(self, text):
        """Support batch or single str.

        Args:
            text (str or list of str): texts to convert.

        """
        if isinstance(text, str):
            text = [
                self.dict[char.lower() if self._ignore_case else char]
                for char in text
            ]
            length = [len(text)]

            return torch.Tensor(text), torch.Tensor(length)
        elif isinstance(text, collections.Iterable):
            temp = []
            length = []
            for s in text:
                st, le = self.encode(s)
                temp.append(st)
                length.append(float(le.item()))

            return nn.utils.rnn.pad_sequence([
                torch.Tensor(iter) for iter in temp
            ]).transpose(0, 1), torch.tensor(length)

    def decode(self, t, length, raw=False):
        """Decode encoded texts back into strs.

        Args:
            torch.IntTensor [length_0 , length_1 , ... length_{n - 1}]: encoded texts.
            torch.IntTensor [n]: length of each text.

        Raises:
            AssertionError: when the texts and its length does not match.

        Returns:
            text (str or list of str): texts to convert.
        """
        t.cpu()
        length.cpu()
        if length.numel() == 1:
            length = length.item()

            if raw:
                return ''.join([self.alphabet[i - 1] for i in t])
            else:
                char_list = []
                for i in range(length):
                    if t[i] != 0 and (not (i > 0 and t[i - 1] == t[i])):
                        char_list.append(self.alphabet[t[i] - 1])
                return ''.join(char_list)
        else:
            # batch mode
            texts = []
            for i in range(length.numel()):
                texts.append(
                    self.decode(t[i, :], length[i].clone().detach(), raw=raw))
            return texts


text:
import numpy as np
from scripts.helpers import batch_iter, sigmoid
from scripts.costs import compute_mse, compute_rmse, compute_loss_logistic, compute_loss_logistic_regularized
from scripts.gradients import compute_gradient_mse, compute_gradient_logistic, compute_gradient_logistic_regularized

# 1. Linear regression using gradient descent
def least_squares_GD(y, tx, initial_w, max_iters, gamma):
    """Function to perfom gradient descent algorithm.
    Args:
        y         (numpy array): Matrix output of size N x 1.
        tx        (numpy array): Matrix input of size N x D.
        initial_w (numpy array): Initial matrix weight (parameters of the model) of size D x 1.
        max_iters (int)        : Maximum iteration for gradient descent training.
        gamma     (float)      : Step-size/learning rate constant.
    Returns:
        w    (numpy array)  : Final weight matrix by gradient descent.
        loss (float, scalar): Final cost value by gradient descent.
    """

    w = initial_w
    for n_iter in range(max_iters):
        loss = compute_mse(y, tx, w)
        grad = compute_gradient_mse(y,tx,w)
        w    = w - gamma * grad

    return w, loss

# 2. Linear regression using stochastic gradient descent
def least_squares_SGD(y, tx, initial_w, max_iters, gamma):
    """Function to perfom stochastic gradient descent algorithm.
    Args:
        y         (numpy array): Matrix output of size N x 1.
        tx        (numpy array): Matrix input of size N x D.
        initial_w (numpy array): Initial matrix weight (parameters of the model) of size D x 1.
        max_iters (int)        : Maximum iteration for gradient descent training.
        gamma     (float)      : Step-size/learning rate constant.
    Returns:
        w    (numpy array)  : Final weight matrix by stochastic gradient descent.
        loss (float, scalar): Final cost value by stochastic gradient descent.
    """

    w = initial_w

    for n_iter in range(max_iters):
        for minibatch_y, minibatch_x in batch_iter(y, tx, 1):
            loss = compute_mse(minibatch_y, minibatch_x, w)
            grad = compute_gradient_mse(minibatch_y, minibatch_x, w)
            w    = w - gamma * grad

    return w, loss

# 3. Least squares regression using normal equations
def least_squares(y, tx):
    """Function to calculate the least squares solution using normal equations.
    Args:
        y         (numpy array): Matrix output of size N x 1.
        tx        (numpy array): Matrix input of size N x D.
    Returns:
        w    (numpy array)  : Weight matrix by least squares normal equations.
        loss (float, scalar): Cost value by least squares normal equations.
    """

    xt   = tx.transpose()
    w    = np.linalg.solve(xt.dot(tx), xt.dot(y))
    loss = compute_rmse(y, tx, w)

    return w, loss

# 4. Ridge regression using normal equations
def ridge_regression(y, tx, lambda_):
    """Function to calculate the least squares solution using normal equations.
    Args:
        y         (numpy array): Matrix output of size N x 1.
        tx        (numpy array): Matrix input of size N x D.
        lambda_   (float)      : Lifting constant for ridge regression.
    Returns:
        w    (numpy array)  : Weight matrix by least squares normal equations.
        loss (float, scalar): Cost value by least squares normal equations.
    """

    xt   = tx.transpose()
    w    = np.linalg.solve(xt.dot(tx) + lambda_ * (2 * y.shape[0]) * np.identity(xt.shape[0]), xt.dot(y))
    loss = compute_rmse(y, tx, w)

    return w, loss

# 5. Logistic regression using gradient descent
def logistic_regression(y, tx, initial_w, max_iters, gamma):
    """Function to perform logistic regression using gradient descent.
    Args:
        y         (numpy array): Matrix output of size N x 1.
        tx        (numpy array): Matrix input of size N x D.
        initial_w (numpy array): Initial matrix weight (parameters of the model) of size D x 1.
        max_iters (int)        : Maximum iteration for gradient descent training.
        gamma     (float)      : Step-size/learning rate constant.
    Returns:
        w    (numpy array)  : Final weight matrix by logistic regression using gradient descent.
        loss (float, scalar): Final cost value by logistic regression using gradient descent.
    """

    w = initial_w
    for n_iter in range(max_iters):
        loss = compute_loss_logistic(y, tx, w)
        grad = compute_gradient_logistic(y,tx,w)
        w    = w - gamma * grad

    return w, loss

# 6. Regularized logistic regression using gradient descent
def reg_logistic_regression(y, tx, initial_w, max_iters, gamma, lambda_):
    """Function to perform regularized logistic regression using gradient descent.
    Args:
        y         (numpy array): Matrix output of size N x 1.
        tx        (numpy array): Matrix input of size N x D.
        initial_w (numpy array): Initial matrix weight (parameters of the model) of size D x 1.
        max_iters (int)        : Maximum iteration for gradient descent training.
        gamma     (float)      : Step-size/learning rate constant.
        lambda    (float)      : Penalty constant.
    Returns:
        w    (numpy array)  : Final weight matrix by regularized logistic regression using gradient descent.
        loss (float, scalar): Final cost value by regularized logistic regression using gradient descent.
    """

    w = initial_w
    for n_iter in range(max_iters):
        loss = compute_loss_logistic_regularized(y, tx, w, lambda_)
        grad = compute_gradient_logistic_regularized(y,tx,w, lambda_)
        w    = w - gamma * grad

    return w, loss


text:
from numpy import array
from sympy import exp, sin, pi

def flux(u, q, w, v, x, t, mu, eta):
    f = mu[0]*q + u[0]*mu[1:];
    return f;

def source(u, q, w, v, x, t, mu, eta):
    x1 = x[0];
    x2 = x[1];
    v1 = mu[1];
    v2 = mu[2];
    s = array([(v1*x2*exp(v1)*exp(v2)*(exp(v2*(x2 - 1)) - 1)*(exp(v1*(x1 - 1)) + v1*x1*exp(v1*(x1 - 1)) - 1))/((exp(v1) - 1)*(exp(v2) - 1)) - (v2*x1*exp(v1 + v2*x2)*(exp(v1*(x1 - 1)) - 1)*(v2*x2 + 2))/((exp(v1) - 1)*(exp(v2) - 1)) - (v1*x2*exp(v2 + v1*x1)*(exp(v2*(x2 - 1)) - 1)*(v1*x1 + 2))/((exp(v1) - 1)*(exp(v2) - 1)) + (v2*x1*exp(v1)*exp(v2)*(exp(v1*(x1 - 1)) - 1)*(exp(v2*(x2 - 1)) + v2*x2*exp(v2*(x2 - 1)) - 1))/((exp(v1) - 1)*(exp(v2) - 1))]);
    return s;

def fbou(u, q, w, v, x, t, mu, eta, uhat, n, tau):
    f = flux(u, q, w, v, x, t, mu, eta);
    fb = array([f[0]*n[0] + f[1]*n[1] + tau[0]*(u[0]-uhat[0])]);
    return fb;

def ubou(u, q, w, v, x, t, mu, eta, uhat, n, tau):
    ub = array([0.0]);
    return ub;

def initu(x, mu, eta):
    u0 = array([0.0]);
    return u0;

max_stars_repo_name: glwagner/Exasim

max_issues_repo_name: glwagner/Exasim

max_forks_repo_name: glwagner/Exasim

reason: from numpy,from sympy


text:
"""
Title: Writing your own callbacks
Authors: Rick Chao, Francois Chollet
Date created: 2019/03/20
Last modified: 2020/07/12
Description: Complete guide to writing new Keras callbacks.
"""
"""
## Introduction

A callback is a powerful tool to customize the behavior of a Keras model during
training, evaluation, or inference. Examples include `tf.keras.callbacks.TensorBoard`
to visualize training progress and results with TensorBoard, or
`tf.keras.callbacks.ModelCheckpoint` to periodically save your model during training.

In this guide, you will learn what a Keras callback is, what it can do, and how you can
build your own. We provide a few demos of simple callback applications to get you
started.
"""

"""
## Setup
"""

import tensorflow as tf
from tensorflow import keras

"""
## Keras callbacks overview

All callbacks subclass the `keras.callbacks.Callback` class, and
override a set of methods called at various stages of training, testing, and
predicting. Callbacks are useful to get a view on internal states and statistics of
the model during training.

You can pass a list of callbacks (as the keyword argument `callbacks`) to the following
model methods:

- `keras.Model.fit()`
- `keras.Model.evaluate()`
- `keras.Model.predict()`
"""

"""
## An overview of callback methods

### Global methods

#### `on_(train|test|predict)_begin(self, logs=None)`

Called at the beginning of `fit`/`evaluate`/`predict`.

#### `on_(train|test|predict)_end(self, logs=None)`

Called at the end of `fit`/`evaluate`/`predict`.

### Batch-level methods for training/testing/predicting

#### `on_(train|test|predict)_batch_begin(self, batch, logs=None)`

Called right before processing a batch during training/testing/predicting.

#### `on_(train|test|predict)_batch_end(self, batch, logs=None)`

Called at the end of training/testing/predicting a batch. Within this method, `logs` is
a dict containing the metrics results.

### Epoch-level methods (training only)

#### `on_epoch_begin(self, epoch, logs=None)`

Called at the beginning of an epoch during training.

#### `on_epoch_end(self, epoch, logs=None)`

Called at the end of an epoch during training.
"""

"""
## A basic example

Let's take a look at a concrete example. To get started, let's import tensorflow and
define a simple Sequential Keras model:
"""

# Define the Keras model to add callbacks to
def get_model():
    model = keras.Sequential()
    model.add(keras.layers.Dense(1, input_dim=784))
    model.compile(
        optimizer=keras.optimizers.RMSprop(learning_rate=0.1),
        loss="mean_squared_error",
        metrics=["mean_absolute_error"],
    )
    return model

"""
Then, load the MNIST data for training and testing from Keras datasets API:
"""

# Load example MNIST data and pre-process it
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
x_train = x_train.reshape(-1, 784).astype("float32") / 255.0
x_test = x_test.reshape(-1, 784).astype("float32") / 255.0

# Limit the data to 1000 samples
x_train = x_train[:1000]
y_train = y_train[:1000]
x_test = x_test[:1000]
y_test = y_test[:1000]

"""
Now, define a simple custom callback that logs:

- When `fit`/`evaluate`/`predict` starts & ends
- When each epoch starts & ends
- When each training batch starts & ends
- When each evaluation (test) batch starts & ends
- When each inference (prediction) batch starts & ends
"""

class CustomCallback(keras.callbacks.Callback):
    def on_train_begin(self, logs=None):
        keys = list(logs.keys())
        print("Starting training; got log keys: {}".format(keys))

    def on_train_end(self, logs=None):
        keys = list(logs.keys())
        print("Stop training; got log keys: {}".format(keys))

    def on_epoch_begin(self, epoch, logs=None):
        keys = list(logs.keys())
        print("Start epoch {} of training; got log keys: {}".format(epoch, keys))

    def on_epoch_end(self, epoch, logs=None):
        keys = list(logs.keys())
        print("End epoch {} of training; got log keys: {}".format(epoch, keys))

    def on_test_begin(self, logs=None):
        keys = list(logs.keys())
        print("Start testing; got log keys: {}".format(keys))

    def on_test_end(self, logs=None):
        keys = list(logs.keys())
        print("Stop testing; got log keys: {}".format(keys))

    def on_predict_begin(self, logs=None):
        keys = list(logs.keys())
        print("Start predicting; got log keys: {}".format(keys))

    def on_predict_end(self, logs=None):
        keys = list(logs.keys())
        print("Stop predicting; got log keys: {}".format(keys))

    def on_train_batch_begin(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Training: start of batch {}; got log keys: {}".format(batch, keys))

    def on_train_batch_end(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Training: end of batch {}; got log keys: {}".format(batch, keys))

    def on_test_batch_begin(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Evaluating: start of batch {}; got log keys: {}".format(batch, keys))

    def on_test_batch_end(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Evaluating: end of batch {}; got log keys: {}".format(batch, keys))

    def on_predict_batch_begin(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Predicting: start of batch {}; got log keys: {}".format(batch, keys))

    def on_predict_batch_end(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Predicting: end of batch {}; got log keys: {}".format(batch, keys))

"""
Let's try it out:
"""

model = get_model()
model.fit(
    x_train,
    y_train,
    batch_size=128,
    epochs=1,
    verbose=0,
    validation_split=0.5,
    callbacks=[CustomCallback()],
)

res = model.evaluate(
    x_test, y_test, batch_size=128, verbose=0, callbacks=[CustomCallback()]
)

res = model.predict(x_test, batch_size=128, callbacks=[CustomCallback()])

"""
### Usage of `logs` dict
The `logs` dict contains the loss value, and all the metrics at the end of a batch or
epoch. Example includes the loss and mean absolute error.
"""

class LossAndErrorPrintingCallback(keras.callbacks.Callback):
    def on_train_batch_end(self, batch, logs=None):
        print(
            "Up to batch {}, the average loss is {:7.2f}.".format(batch, logs["loss"])
        )

    def on_test_batch_end(self, batch, logs=None):
        print(
            "Up to batch {}, the average loss is {:7.2f}.".format(batch, logs["loss"])
        )

    def on_epoch_end(self, epoch, logs=None):
        print(
            "The average loss for epoch {} is {:7.2f} "
            "and mean absolute error is {:7.2f}.".format(
                epoch, logs["loss"], logs["mean_absolute_error"]
            )
        )

model = get_model()
model.fit(
    x_train,
    y_train,
    batch_size=128,
    epochs=2,
    verbose=0,
    callbacks=[LossAndErrorPrintingCallback()],
)

res = model.evaluate(
    x_test,
    y_test,
    batch_size=128,
    verbose=0,
    callbacks=[LossAndErrorPrintingCallback()],
)

"""
## Usage of `self.model` attribute

In addition to receiving log information when one of their methods is called,
callbacks have access to the model associated with the current round of
training/evaluation/inference: `self.model`.

Here are of few of the things you can do with `self.model` in a callback:

- Set `self.model.stop_training = True` to immediately interrupt training.
- Mutate hyperparameters of the optimizer (available as `self.model.optimizer`),
such as `self.model.optimizer.learning_rate`.
- Save the model at period intervals.
- Record the output of `model.predict()` on a few test samples at the end of each
epoch, to use as a sanity check during training.
- Extract visualizations of intermediate features at the end of each epoch, to monitor
what the model is learning over time.
- etc.

Let's see this in action in a couple of examples.
"""

"""
## Examples of Keras callback applications
"""

"""
### Early stopping at minimum loss

This first example shows the creation of a `Callback` that stops training when the
minimum of loss has been reached, by setting the attribute `self.model.stop_training`
(boolean). Optionally, you can provide an argument `patience` to specify how many
epochs we should wait before stopping after having reached a local minimum.

`tf.keras.callbacks.EarlyStopping` provides a more complete and general implementation.
"""

import numpy as np

class EarlyStoppingAtMinLoss(keras.callbacks.Callback):
    """Stop training when the loss is at its min, i.e. the loss stops decreasing.

  Arguments:
      patience: Number of epochs to wait after min has been hit. After this
      number of no improvement, training stops.
  """

    def __init__(self, patience=0):
        super(EarlyStoppingAtMinLoss, self).__init__()
        self.patience = patience
        # best_weights to store the weights at which the minimum loss occurs.
        self.best_weights = None

    def on_train_begin(self, logs=None):
        # The number of epoch it has waited when loss is no longer minimum.
        self.wait = 0
        # The epoch the training stops at.
        self.stopped_epoch = 0
        # Initialize the best as infinity.
        self.best = np.Inf

    def on_epoch_end(self, epoch, logs=None):
        current = logs.get("loss")
        if np.less(current, self.best):
            self.best = current
            self.wait = 0
            # Record the best weights if current results is better (less).
            self.best_weights = self.model.get_weights()
        else:
            self.wait += 1
            if self.wait >= self.patience:
                self.stopped_epoch = epoch
                self.model.stop_training = True
                print("Restoring model weights from the end of the best epoch.")
                self.model.set_weights(self.best_weights)

    def on_train_end(self, logs=None):
        if self.stopped_epoch > 0:
            print("Epoch %05d: early stopping" % (self.stopped_epoch + 1))

model = get_model()
model.fit(
    x_train,
    y_train,
    batch_size=64,
    steps_per_epoch=5,
    epochs=30,
    verbose=0,
    callbacks=[LossAndErrorPrintingCallback(), EarlyStoppingAtMinLoss()],
)

"""
### Learning rate scheduling

In this example, we show how a custom Callback can be used to dynamically change the
learning rate of the optimizer during the course of training.

See `callbacks.LearningRateScheduler` for a more general implementations.
"""

class CustomLearningRateScheduler(keras.callbacks.Callback):
    """Learning rate scheduler which sets the learning rate according to schedule.

  Arguments:
      schedule: a function that takes an epoch index
          (integer, indexed from 0) and current learning rate
          as inputs and returns a new learning rate as output (float).
  """

    def __init__(self, schedule):
        super(CustomLearningRateScheduler, self).__init__()
        self.schedule = schedule

    def on_epoch_begin(self, epoch, logs=None):
        if not hasattr(self.model.optimizer, "lr"):
            raise ValueError('Optimizer must have a "lr" attribute.')
        # Get the current learning rate from model's optimizer.
        lr = float(tf.keras.backend.get_value(self.model.optimizer.learning_rate))
        # Call schedule function to get the scheduled learning rate.
        scheduled_lr = self.schedule(epoch, lr)
        # Set the value back to the optimizer before this epoch starts
        tf.keras.backend.set_value(self.model.optimizer.lr, scheduled_lr)
        print("\nEpoch %05d: Learning rate is %6.4f." % (epoch, scheduled_lr))

LR_SCHEDULE = [
    # (epoch to start, learning rate) tuples
    (3, 0.05),
    (6, 0.01),
    (9, 0.005),
    (12, 0.001),
]

def lr_schedule(epoch, lr):
    """Helper function to retrieve the scheduled learning rate based on epoch."""
    if epoch < LR_SCHEDULE[0][0] or epoch > LR_SCHEDULE[-1][0]:
        return lr
    for i in range(len(LR_SCHEDULE)):
        if epoch == LR_SCHEDULE[i][0]:
            return LR_SCHEDULE[i][1]
    return lr

model = get_model()
model.fit(
    x_train,
    y_train,
    batch_size=64,
    steps_per_epoch=5,
    epochs=15,
    verbose=0,
    callbacks=[
        LossAndErrorPrintingCallback(),
        CustomLearningRateScheduler(lr_schedule),
    ],
)

"""
### Built-in Keras callbacks
Be sure to check out the existing Keras callbacks by
reading the [API docs](https://keras.io/api/callbacks/).
Applications include logging to CSV, saving
the model, visualizing metrics in TensorBoard, and a lot more!
"""

max_stars_repo_name: rickiepark/keras-io

max_issues_repo_name: rickiepark/keras-io

max_forks_repo_name: rickiepark/keras-io


text:
# -*- coding: utf-8 -*-

# Sophie Dufour-Beauséjour
# s.dufour.beausejour@gmail.com
# Université INRS
# Québec, Canada

# Get image band values for pixels overlapping point features from a shapefile.
# The shapefile may include more than one point.
# Images expected in tiff format.
# The shapefile and image should have the same coordinate system, and no projection
# Absolute paths to image and shapefile should be put in pairs.csv (there can be more than one pair of image and shapefile)
# Requirements: fiona, rasterio
# Other files:
# - tiff_at_shp.py
# Input:
# - pairs_path (csv file with pairs of absolute paths image_path and shapefile_path (as column names in uncommented first row)
# - band_names (in the order that they appear in the TIFF files)
# - band_index_to_dB (indices of bands to convert from linear to dB before writing their values in result/)
# - overwrite (True or False to overwrite in results/)

from __future__ import print_function
__author__ = "Sophie Dufour-Beauséjour"

import os
import csv
import numpy as np
import pandas as pd
import platform
# from this project
import tiff_at_shp

# results_dir = "results/RS2_texture/"
# pairs_path = "pairs_RS2.csv"
# box = 5
# orbit = "orbit21"
# results_dir = "results/TSX_texture/"+orbit+"/"
# pairs_path = "pairs_TSX_"+orbit+".csv"
# results_dir = "results/no_window/RS2_texture/"
# pairs_path = "pairs_RS2.csv"
# box = 1
results_dir_parent = "results/win_3/"
results_dir = ""
linearTSX = 1
pairs_path = "pairs_TSX_VH_HH.csv"
box = 3
overwrite = 0 # overwrite result text files or not

## On PC ##
system_name = platform.system()
if system_name in "Windows":
    PC = 1
else:
    PC = 0

if ("RS2" in results_dir) & ("HAa" in results_dir):
    band_names = ["H", "A", "a"]
    band_index_to_dB = 0
    processing_done = "_sub_cal_C3_spk_HAa_TC2"
elif ("RS2" in results_dir) & ("ratios" in results_dir):
    band_names = ["HH", "HV", "VH", "VV", "VV/HH", "HV/HH", "VH/VV"]
    band_index_to_dB = [0, 1, 2, 3]
    processing_done = "_sub_cal_spk_rat2_TC2"
elif ("RS2" in results_dir) & ("texture" in results_dir):
    band_names = list()
    for band in ["HH","HV","VV"]:
        for tex in ["HOM","ASM","MEAN","VAR","COR"]:
            band_names.append(tex+"_"+band)
    band_index_to_dB = 0
    processing_done = "_tex2_TC2"
elif ("TSX" in results_dir) and ("ratios" in results_dir):
    if "13" in orbit:
        band_names = ["VVHHRatio"]
    elif ("21" in orbit) or ("89" in orbit):
        band_names = ["VHVVRatio"]
    band_index_to_dB = 0
    processing_done = "_sub_rat2"
elif ("TSX" in results_dir) and ("texture" in results_dir):
    if "13" in orbit:
        pols = ["VV","HH"]
    elif ("21" in orbit) or ("89" in orbit):
        pols = ["VH","VV"]
    band_names = list()
    for band in pols:
        for tex in ["HOM", "ASM", "MEAN", "VAR", "COR"]:
            band_names.append(tex + "_" + band)
    band_index_to_dB = 0
    processing_done = "_sub_tex2"
elif linearTSX:
    band_index_to_dB = 0

# batch over many pairs of files
with open(pairs_path, mode='r') as csv_file:
    csv_reader = csv.DictReader(csv_file)
    for i, row in enumerate(csv_reader):
        shapefile_path = row["shapefile_path"]
        if "#" in shapefile_path:
            continue
        if not linearTSX:
            image_name = os.path.basename(row["image_path"])[0:12]
            image_path = os.path.join(os.path.dirname(row["image_path"]), image_name+processing_done+".tif")
        elif linearTSX:
            image_path = row["image_path"]
            image_name = "TSX_"+os.path.basename(row["image_path"])[8:16]
            if "orbit13" in image_path:
                orbit = "orbit13"
            elif "orbit21" in image_path:
                orbit = "orbit21"
            elif "orbit89" in image_path:
                orbit = "orbit89"
            if "08K0001" in image_path:
                band_names = ["VV"]
            elif "01K0001" in image_path:
                band_names = ["HH"]
            elif "02K0001" in image_path:
                band_names = ["VH"]
            results_dir = results_dir_parent+"TSX_"+band_names[0]+"_"+orbit+"/"

        if PC:
            shapefile_path.replace("/Volumes/Carbe/Doctorat/BaieDeception/", "Z:"+os.sep)
            shapefile_path.replace("/", os.sep)
            image_path.replace("/Volumes/Carbe/Doctorat/BaieDeception/", "Z:"+os.sep)
            image_path.replace("/", os.sep)

        # Save name
        preffix = ""
        for s in ["_D_", "_S_", "_K_"]:
            if s in shapefile_path:
                preffix = s[1]+"_"
        suffix = ""
        for s in ["transect_noship", "notransect", "nopolynia", "no_33-34"]:
            if s in shapefile_path:
                suffix = "_" + s
        if "RS2" in image_path:
            save_name = os.path.basename(preffix + os.path.basename(image_path)[4:12] + suffix + ".csv")
        elif "K0001" in image_path:
            save_name = os.path.basename(preffix + image_name + suffix + ".csv")
        elif "TSX" in image_path:
            save_name = os.path.basename(preffix + os.path.basename(image_path)[4:12] + suffix + ".csv")

        # Get pixel values at each shapefile point feature
        # Check if files already there
        date = int(image_name[4:12])
        # date = int(os.path.basename(image_path)[8:16])
        if not overwrite and os.path.exists(results_dir+save_name):
            print("pixel values already written to text file: " + save_name)
            continue
        elif ("VH" in band_names) & ("orbit21" in image_path) & (date >= 20170502) & (date < 20170911):
            print("No TSX VH orbit21 image for those dates")
            print(save_name)
            print(date)
            continue

        if box:
            data = tiff_at_shp.mean_pixel_values(image_path, shapefile_path, results_dir,
                                             band_names, box, band_index_to_dB=band_index_to_dB)
        else:
            data = tiff_at_shp.pixel_values(image_path, shapefile_path, results_dir,
                                             band_names, band_index_to_dB=band_index_to_dB)

        ## Some data wrangling
        # Replace n/a by nan
        data = data.replace("n/a", np.nan)
        if "RS2_ratios" in results_dir:
            data = data[data["HH"] != "-inf"]
            data = data[data["HH"] != float("-inf")]
        elif "RS2_tex" in results_dir:
            data = data[data["MEAN_HH"] != "0.0"]
            data = data[data["MEAN_HH"] != 0]
        elif "RS2_HAa" in results_dir:
            data = data[data["H"] != "0.0"]
            data = data[data["H"] != 0]
        elif "TSX_VV" in results_dir:
            # Remove inf rows
            data = data[data["VV"] != "-inf"]
            data = data[data["VV"] != float("-inf")]
            data = data[data["VV"] != "nan"]
        elif "TSX_HH" in results_dir:
            # Remove inf rows
            data = data[data["HH"] != "-inf"]
            data = data[data["HH"] != float("-inf")]
            data = data[data["HH"] != "nan"]
        elif "TSX_VH" in results_dir:
            data = data[data["VH"] != "-inf"]
            data = data[data["VH"] != float("-inf")]
            data = data[data["VH"] != "nan"]
        elif "TSX_ratios/orbit13" in results_dir:
            data = data[data["VVHHRatio"] != "nan"]
        elif ("TSX_ratios/orbit21" in results_dir) or ("TSX_ratios/orbit89" in results_dir):
            data = data[data["VHVVRatio"] != "nan"]
            data = data[data["VHVVRatio"] != "0.0"]
            data = data[data["VHVVRatio"] != 0]
        elif "TSX_texture/orbit13" in results_dir:
            data = data[data["MEAN_VV"] != "nan"]
            data = data[data["MEAN_VV"] != "0.0"]
            data = data[data["MEAN_VV"] != 0]
            data = data[data["MEAN_VV"].notnull()]
        elif ("TSX_texture/orbit21" in results_dir) or ("TSX_texture/orbit89" in results_dir):
            data = data[data["MEAN_VV"] != "nan"]
            data = data[data["MEAN_VV"] != "0.0"]
            data = data[data["MEAN_VV"] != 0]
            data = data[data["MEAN_VV"].notnull()]

            # Replace ice == 0 by nan
        key_ice = [x for x in data.keys() if "Ice" in x][0]
        data = data.replace({key_ice: 0}, np.nan)
        data = data.replace({key_ice: "0.0"}, np.nan)
        # Remove track point in D_1704
        if "D_20170419_notransect" in save_name:
            data = data[data[key_ice] != "43.0"]
        # Remove 33-34 from K 1801
        if "K_20180129" in save_name:
            data = data[data["ID_Map"] != "K_33"]
            data = data[data["ID_Map"] != "K_34"]
        #
        # # Save to txt with a comment at the end
        # data.to_csv(results_dir+save_name)
        # with open(results_dir+save_name, 'a') as f:
        #     if box:
        #         f.write("# Image values are a mean over a "+str(box)+"x"+str(box)+" box\n")
        #     f.write("# Image: "+image_path+"\n")
        #     f.write("# Shp: "+shapefile_path+"\n")
        #
        # # Also combine the two DB files when transect; notransect comes first in rows
        # if "notransect" in shapefile_path:
        #     save_name = os.path.basename(preffix + image_name + "_both" + ".csv")
        #     data.to_csv(results_dir+save_name,header=1)
        # if "transect_noship" in shapefile_path:
        #     save_name = os.path.basename(preffix + image_name + "_both" + ".csv")
        #     data.to_csv(results_dir+save_name, mode='a', header=0)
        #     with open(results_dir+save_name, 'a') as f:
        #         if box:
        #             f.write("# Image values are a mean over a "+str(box)+"x"+str(box)+" box\n")
        #         f.write("# Image: "+image_path+"\n")
        #         f.write("# Shp: "+shapefile_path+"\n")
        #         f.write("# combined with notransect"+"\n")


text:
import numpy as np
import matplotlib.pyplot as plt
import sys

import tensorflow as tf
from tensorflow.keras.applications.vgg16 import preprocess_input
from tensorflow.keras.optimizers import Adadelta, RMSprop, SGD
from tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint, LearningRateScheduler

from models import *
from metrics import VoidPixelAccuracy, VoidPixelMeanIoU
from dalib import ImageAugmentator

TRAIN_LEN = 8498
VAL_LEN = 736
nb_classes = 21
void_pixel = 255
target_size = (480, 480)
dataset_dir = sys.argv[1]
log_dir = sys.argv[2]

strategy = tf.distribute.MirroredStrategy()
with strategy.scope():
    model = build_fcn16s(nb_classes)
    optimizer = Adadelta(lr=8e-3)
    loss = "sparse_categorical_crossentropy"
    metrics = [
        VoidPixelAccuracy(void_pixel=void_pixel),
        VoidPixelMeanIoU(num_classes=nb_classes, void_pixel=void_pixel),
    ]

    model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
    model.summary()

tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)
checkpoint_callback = ModelCheckpoint(filepath=f"{log_dir}/segmentation-fcn16s-voc-checkpoint-weights.h5", save_weights_only=True)
lr_schedule_callback = LearningRateScheduler(schedule=lambda ep, lr: lr * 0.95 if ep > 10 else lr, verbose=1)
callbacks = [checkpoint_callback, tensorboard_callback, lr_schedule_callback]

batch_size = 6

train_generator = ImageAugmentator(
    f"{dataset_dir}/train",
    batch_size=batch_size,
    target_size=target_size,
    num_classes=nb_classes,
    sampling="class",
    flip=True,
    scales=(480, 600),
    crop="class",
    preprocessing_function=preprocess_input,
    void_pixel=void_pixel,
    pad_to_multiple_of=32,
)

val_generator = ImageAugmentator(
    f"{dataset_dir}/val",
    batch_size=1,
    num_classes=nb_classes,
    sampling="random",
    flip=False,
    scales=None,
    crop=None,
    preprocessing_function=preprocess_input,
    void_pixel=void_pixel,
    pad_to_multiple_of=32,
)

model.fit(
    train_generator,
    steps_per_epoch=TRAIN_LEN // batch_size,
    epochs=50,
    validation_data=val_generator,
    validation_steps=VAL_LEN,
    callbacks=callbacks
)

model.save("segmentation-voc-fcn16s.h5")

max_stars_repo_name: tmkkk/fcn

max_issues_repo_name: tmkkk/fcn

max_forks_repo_name: tmkkk/fcn