text:
"""
"""
# TODO: marker alpha

import functools
import itertools

import numpy as np

from ..manage import auto_xyz_ds
from .core import (
    Plotter,
    AbstractLinePlot,
    AbstractScatter,
    AbstractHeatMap,
    calc_row_col_datasets,
    PLOTTER_DEFAULTS,
    intercept_call_arg,
    prettify,
)

@functools.lru_cache(1)
def _init_bokeh_nb():
    """Cache this so it doesn't happen over and over again.
    """
    from bokeh.plotting import output_notebook
    from bokeh.resources import INLINE
    output_notebook(resources=INLINE)

def bshow(figs, nb=True, interactive=False, **kwargs):
    """
    """
    from bokeh.plotting import show
    if nb:
        _init_bokeh_nb()
        show(figs, notebook_handle=interactive)
    else:
        show(figs)

# --------------------------------------------------------------------------- #
#                     Main lineplot interface for bokeh                       #
# --------------------------------------------------------------------------- #

class PlotterBokeh(Plotter):
    def __init__(self, ds, x, y, z=None, **kwargs):
        """
        """
        # bokeh custom options / defaults
        kwargs['return_fig'] = kwargs.pop('return_fig', False)
        self._interactive = kwargs.pop('interactive', False)

        super().__init__(ds, x, y, z, **kwargs, backend='BOKEH')

    def prepare_axes(self):
        """Make the bokeh plot figure and set options.
        """
        from bokeh.plotting import figure

        if self.add_to_axes is not None:
            self._plot = self.add_to_axes

        else:
            # Currently axes scale type must be set at figure creation?
            self._plot = figure(
                # convert figsize to roughly matplotlib dimensions
                width=int(self.figsize[0] * 80 +
                          (100 if self._use_legend else 0) +
                          (20 if self._ytitle else 0) +
                          (20 if not self.yticklabels_hide else 0)),
                height=int(self.figsize[1] * 80 +
                           (20 if self.title else 0) +
                           (20 if self._xtitle else 0) +
                           (20 if not self.xticklabels_hide else 0)),
                x_axis_type=('log' if self.xlog else 'linear'),
                y_axis_type=('log' if self.ylog else 'linear'),
                y_axis_location=('right' if self.ytitle_right else 'left'),
                title=self.title,
                toolbar_location="above",
                toolbar_sticky=False,
                active_scroll="wheel_zoom",
            )

    def set_axes_labels(self):
        """Set the labels on the axes.
        """
        if self._xtitle:
            self._plot.xaxis.axis_label = self._xtitle
        if self._ytitle:
            self._plot.yaxis.axis_label = self._ytitle

    def set_axes_range(self):
        """Set the plot ranges of the axes, and the panning limits.
        """
        from bokeh.models import DataRange1d

        self.calc_data_range()

        # plt_x_centre = (self._data_xmax + self._data_xmin) / 2
        # plt_x_range = self._data_xmax - self._data_xmin
        # xbounds = (plt_x_centre - plt_x_range, plt_x_centre + plt_x_range)
        xbounds = None
        self._plot.x_range = (DataRange1d(start=self._xlims[0],
                                          end=self._xlims[1],
                                          bounds=xbounds) if self._xlims else
                              DataRange1d(bounds=xbounds))

        # plt_y_centre = (self._data_ymax + self._data_ymin) / 2
        # plt_y_range = abs(self._data_ymax - self._data_ymin)
        # ybounds = (plt_y_centre - plt_y_range, plt_y_centre + plt_y_range)
        ybounds = None
        self._plot.y_range = (DataRange1d(start=self._ylims[0],
                                          end=self._ylims[1],
                                          bounds=ybounds) if self._ylims else
                              DataRange1d(bounds=ybounds))

    def set_spans(self):
        """Set custom horizontal and verical line spans.
        """
        from bokeh.models import Span

        span_opts = {
            'level': 'glyph',
            'line_dash': 'dashed',
            'line_color': (127, 127, 127),
            'line_width': self.span_width,
        }

        if self.hlines:
            for hl in self.hlines:
                self._plot.add_layout(Span(
                    location=hl, dimension='width', **span_opts))
        if self.vlines:
            for vl in self.vlines:
                self._plot.add_layout(Span(
                    location=vl, dimension='height', **span_opts))

    def set_gridlines(self):
        """Set whether to use gridlines or not.
        """
        if not self.gridlines:
            self._plot.xgrid.visible = False
            self._plot.ygrid.visible = False
        else:
            self._plot.xgrid.grid_line_dash = self.gridline_style
            self._plot.ygrid.grid_line_dash = self.gridline_style

    def set_tick_marks(self):
        """Set custom locations for the tick marks.
        """
        from bokeh.models import FixedTicker

        if self.xticks:
            self._plot.xaxis[0].ticker = FixedTicker(ticks=self.xticks)
        if self.yticks:
            self._plot.yaxis[0].ticker = FixedTicker(ticks=self.yticks)

        if self.xticklabels_hide:
            self._plot.xaxis.major_label_text_font_size = '0pt'
        if self.yticklabels_hide:
            self._plot.yaxis.major_label_text_font_size = '0pt'

    def set_sources_heatmap(self):
        from bokeh.plotting import ColumnDataSource

        # initialize empty source
        if not hasattr(self, '_source'):
            self._source = ColumnDataSource(data=dict())

        # remove mask from data -> not necessary soon? / convert to nan?
        var = np.ma.getdata(self._heatmap_var)

        self._source.add([var], 'image')
        self._source.add([self._data_xmin], 'x')
        self._source.add([self._data_ymin], 'y')
        self._source.add([self._data_xmax - self._data_xmin], 'dw')
        self._source.add([self._data_ymax - self._data_ymin], 'dh')

    def set_sources(self):
        """Set the source dictionaries to be used by the plotter functions.
        This is seperate to allow interactive updates of the data only.
        """
        from bokeh.plotting import ColumnDataSource

        # check if heatmap
        if hasattr(self, '_heatmap_var'):
            return self.set_sources_heatmap()

        # 'copy' the zlabels iterator into src_zlbs
        self._zlbls, src_zlbs = itertools.tee(self._zlbls)

        # Initialise with empty dicts
        if not hasattr(self, "_sources"):
            self._sources = [ColumnDataSource(dict())
                             for _ in range(len(self._z_vals))]

        # range through all data and update the sources
        for i, (zlabel, data) in enumerate(zip(src_zlbs, self._gen_xy())):
            self._sources[i].add(data['x'], 'x')
            self._sources[i].add(data['y'], 'y')
            self._sources[i].add([zlabel] * len(data['x']), 'z_coo')

            # check for color for scatter plot
            if 'c' in data:
                self._sources[i].add(data['c'], 'c')

            # check if should set y_err as well
            if 'ye' in data:
                y_err_p = data['y'] + data['ye']
                y_err_m = data['y'] - data['ye']
                self._sources[i].add(
                    list(zip(data['x'], data['x'])), 'y_err_xs')
                self._sources[i].add(list(zip(y_err_p, y_err_m)), 'y_err_ys')

            # check if should set x_err as well
            if 'xe' in data:
                x_err_p = data['x'] + data['xe']
                x_err_m = data['x'] - data['xe']
                self._sources[i].add(
                    list(zip(data['y'], data['y'])), 'x_err_ys')
                self._sources[i].add(list(zip(x_err_p, x_err_m)), 'x_err_xs')

    def plot_legend(self, legend_items=None):
        """Add a legend to the plot.
        """
        if self._use_legend:
            from bokeh.models import Legend

            loc = {'best': 'top_left'}.get(self.legend_loc, self.legend_loc)
            where = {None: 'right'}.get(self.legend_where, self.legend_where)

            # might be manually specified, e.g. from multiplot
            if legend_items is None:
                legend_items = self._lgnd_items

            lg = Legend(items=legend_items)
            lg.location = loc
            lg.click_policy = 'hide'
            self._plot.add_layout(lg, where)

            # Don't repeatedly redraw legend
            self._use_legend = False

    def set_mappable(self):
        from bokeh.models import LogColorMapper, LinearColorMapper
        import matplotlib as plt

        mappr_fn = (LogColorMapper if self.colormap_log else LinearColorMapper)
        bokehpalette = [plt.colors.rgb2hex(m) for m in self.cmap(range(256))]

        self.mappable = mappr_fn(palette=bokehpalette,
                                 low=self._zmin, high=self._zmax)

    def plot_colorbar(self):
        if self._use_colorbar:

            where = {None: 'right'}.get(self.legend_where, self.legend_where)

            from bokeh.models import ColorBar, LogTicker, BasicTicker
            ticker = LogTicker if self.colormap_log else BasicTicker
            color_bar = ColorBar(color_mapper=self.mappable, location=(0, 0),
                                 ticker=ticker(desired_num_ticks=6),
                                 title=self._ctitle)
            self._plot.add_layout(color_bar, where)

    def set_tools(self):
        """Set which tools appear for the plot.
        """
        from bokeh.models import HoverTool

        self._plot.add_tools(HoverTool(tooltips=[
            ("({}, {})".format(self.x_coo, self.y_coo
                               if isinstance(self.y_coo, str) else None),
             "(@x, @y)"), (self.z_coo, "@z_coo")]))

    def update(self):
        from bokeh.io import push_notebook
        self.set_sources()
        push_notebook()

    def show(self, **kwargs):
        """Show the produced figure.
        """
        if self.return_fig:
            return self._plot
        bshow(self._plot, **kwargs)
        return self

    def prepare_plot(self):
        self.prepare_axes()
        self.set_axes_labels()
        self.set_axes_range()
        self.set_spans()
        self.set_gridlines()
        self.set_tick_marks()
        self.set_sources()

def bokeh_multi_plot(fn):
    """Decorate a plotting function to plot a grid of values.
    """

    @functools.wraps(fn)
    def multi_plotter(ds, *args, row=None, col=None, link=False, **kwargs):

        if (row is None) and (col is None):
            return fn(ds, *args, **kwargs)

        # Set some global parameters
        p = fn(ds, *args, **kwargs, call=False)
        p.prepare_data_multi_grid()

        kwargs['xlims'] = kwargs.get('xlims', [p._data_xmin, p._data_xmax])
        kwargs['ylims'] = kwargs.get('ylims', [p._data_ymin, p._data_ymax])

        kwargs['vmin'] = kwargs.pop('vmin', p.vmin)
        kwargs['vmax'] = kwargs.pop('vmax', p.vmax)

        # split the dataset into its respective rows and columns
        ds_r_c, nrows, ncols = calc_row_col_datasets(ds, row=row, col=col)

        # intercept figsize as meaning *total* size for whole grid
        figsize = kwargs.pop('figsize', None)
        if figsize is None:
            av_n = (ncols + nrows) / 2
            figsize = (2 * (4 / av_n)**0.5, 2 * (4 / av_n)**0.5)
        else:
            figsize = (figsize[0] / ncols, figsize[1] / nrows)
        kwargs['figsize'] = figsize

        # intercept return_fig for the full grid and other options
        return_fig = kwargs.pop('return_fig', False)

        subplots = {}

        # range through rows and do subplots
        for i, ds_r in enumerate(ds_r_c):
            # range through columns
            for j, sub_ds in enumerate(ds_r):
                skws = {'legend': False, 'colorbar': False}

                # if not last row
                if i != nrows - 1:
                    skws['xticklabels_hide'] = True
                    skws['xtitle'] = ''

                # if not first column
                if j != 0:
                    skws['yticklabels_hide'] = True
                    skws['ytitle'] = ''

                # label each column
                if (i == 0) and (col is not None):
                    col_val = prettify(ds[col].values[j])
                    skws['title'] = "{} = {}".format(col, col_val)
                    fx = 'fontsize_xtitle'
                    skws['fontsize_title'] = kwargs.get(
                        fx, PLOTTER_DEFAULTS[fx])

                # label each row
                if (j == ncols - 1) and (row is not None):
                    skws['ytitle_right'] = True
                    row_val = prettify(ds[row].values[i])
                    skws['ytitle'] = "{} = {}".format(row, row_val)

                subplots[i, j] = fn(sub_ds, *args, return_fig=True, call=False,
                                    **{**kwargs, **skws})

        from bokeh.layouts import gridplot

        plts = [[subplots[i, j]() for j in range(ncols)] for i in range(nrows)]

        # link zooming and panning between all plots
        if link:
            x_range, y_range = plts[0][0].x_range, plts[0][0].y_range
            for i in range(nrows):
                for j in range(ncols):
                    plts[i][j].x_range = x_range
                    plts[i][j].y_range = y_range

        # the main grid
        p._plot = gridplot(plts)

        if p._use_legend or p._use_colorbar:
            from bokeh.models import Legend, GlyphRenderer, Range1d, ColorBar
            from bokeh.layouts import row

            # plot dummy using last sub_ds
            skws = {'title': "", 'legend_loc': 'center_left',
                    'legend_where': 'left'}
            lgren = fn(sub_ds, *args, return_fig=True, **{**kwargs, **skws})

            # remove all but legend, colorbar and glyph renderers
            lgren.renderers = [
                r for r in lgren.renderers
                if isinstance(r, (Legend, GlyphRenderer, ColorBar))
            ]
            lgren.toolbar_location = None
            lgren.outline_line_color = None

            # size it - this is pretty hacky at the moment
            lgren.width = 120
            lgren.height = int(80 * figsize[1] * nrows + 100)
            lgren.x_range = Range1d(0, 0)
            lgren.y_range = Range1d(0, 0)

            # append to the right of the gridplot
            p._plot = row([p._plot, lgren])

        if return_fig:
            return p._plot
        bshow(p._plot)

    return multi_plotter

class ILinePlot(PlotterBokeh, AbstractLinePlot):

    def __init__(self, ds, x, y, z=None, y_err=None, x_err=None, **kwargs):
        super().__init__(ds, x, y, z=z, y_err=y_err, x_err=x_err, **kwargs)

    def plot_lines(self):
        """Plot the data and a corresponding legend.
        """
        self._lgnd_items = []

        for src in self._sources:
            col = next(self._cols)
            zlabel = next(self._zlbls)
            legend_pics = []

            if self.lines:
                line = self._plot.line(
                    'x', 'y',
                    source=src,
                    color=col,
                    line_dash=next(self._lines),
                    line_width=next(self._lws) * 1.5,
                )
                legend_pics.append(line)

            if self.markers:
                marker = next(self._mrkrs)
                m = getattr(self._plot, marker)(
                    'x', 'y',
                    source=src,
                    name=zlabel,
                    color=col,
                    fill_alpha=0.5,
                    line_width=0.5,
                    size=self._marker_size,
                )
                legend_pics.append(m)

            # Check if errors specified as well
            if self.y_err:
                err = self._plot.multi_line(
                    xs='y_err_xs', ys='y_err_ys', source=src, color=col,
                    line_width=self.errorbar_linewidth)
                legend_pics.append(err)
            if self.x_err:
                err = self._plot.multi_line(
                    xs='x_err_xs', ys='x_err_ys', source=src, color=col,
                    line_width=self.errorbar_linewidth)
                legend_pics.append(err)

            # Add the names and styles of drawn lines for the legend
            self._lgnd_items.append((zlabel, legend_pics))

    def __call__(self):
        self.prepare_data_single()
        # Bokeh preparation
        self.prepare_plot()
        self.plot_lines()
        self.plot_legend()
        self.plot_colorbar()
        self.set_tools()
        return self.show(interactive=self._interactive)

@bokeh_multi_plot
@intercept_call_arg
def ilineplot(ds, x, y, z=None, y_err=None, x_err=None, **kwargs):
    """From  ``ds`` plot lines of ``y`` as a function of ``x``, optionally for
    varying ``z``. Interactive,

    Parameters
    ----------
    ds : xarray.Dataset
        Dataset to plot from.
    x : str
        Dimension to plot along the x-axis.
    y : str or tuple[str]
        Variable(s) to plot along the y-axis. If tuple, plot each of the
        variables - instead of ``z``.
    z : str, optional
        Dimension to plot into the page.
    y_err : str, optional
        Variable to plot as y-error.
    x_err : str, optional
        Variable to plot as x-error.
    row : str, optional
        Dimension to vary over as a function of rows.
    col : str, optional
        Dimension to vary over as a function of columns.
    plot_opts
        See ``xyzpy.plot.core.PLOTTER_DEFAULTS``.
    """
    return ILinePlot(ds, x, y, z, y_err=y_err, x_err=x_err, **kwargs)

class AutoILinePlot(ILinePlot):
    """Interactive raw data multi-line plot.
    """

    def __init__(self, x, y_z, **lineplot_opts):
        ds = auto_xyz_ds(x, y_z)
        super().__init__(ds, 'x', 'y', z='z', **lineplot_opts)

def auto_ilineplot(x, y_z, **lineplot_opts):
    """Auto version of :func:`~xyzpy.ilineplot` that accepts array arguments
    by converting them to a ``Dataset`` first.
    """
    return AutoILinePlot(x, y_z, **lineplot_opts)()

# --------------------------------------------------------------------------- #

class IScatter(PlotterBokeh, AbstractScatter):

    def __init__(self, ds, x, y, z=None, **kwargs):
        super().__init__(ds, x, y, z, **kwargs, markers=True)

    def plot_scatter(self):
        self._lgnd_items = []

        for src in self._sources:
            if 'c' in src.column_names:
                col = {'field': 'c', 'transform': self.mappable}
            else:
                col = next(self._cols)
            marker = next(self._mrkrs)
            zlabel = next(self._zlbls)
            legend_pics = []

            m = getattr(self._plot, marker)(
                'x', 'y',
                source=src,
                name=zlabel,
                color=col,
                fill_alpha=0.5,
                line_width=0.5,
                size=self._marker_size,
            )
            legend_pics.append(m)

            # Add the names and styles of drawn markers for the legend
            self._lgnd_items.append((zlabel, legend_pics))

    def __call__(self):
        self.prepare_data_single()
        # Bokeh preparation
        self.prepare_plot()
        self.plot_scatter()
        self.plot_legend()
        self.plot_colorbar()
        self.set_tools()
        return self.show(interactive=self._interactive)

@bokeh_multi_plot
@intercept_call_arg
def iscatter(ds, x, y, z=None, y_err=None, x_err=None, **kwargs):
    """From  ``ds`` plot a scatter of ``y`` against ``x``, optionally for
    varying ``z``. Interactive.

    Parameters
    ----------
    ds : xarray.Dataset
        Dataset to plot from.
    x : str
        Quantity to plot along the x-axis.
    y : str or tuple[str]
        Quantity(s) to plot along the y-axis. If tuple, plot each of the
        variables - instead of ``z``.
    z : str, optional
        Dimension to plot into the page.
    y_err : str, optional
        Variable to plot as y-error.
    x_err : str, optional
        Variable to plot as x-error.
    row : str, optional
        Dimension to vary over as a function of rows.
    col : str, optional
        Dimension to vary over as a function of columns.
    plot_opts
        See ``xyzpy.plot.core.PLOTTER_DEFAULTS``.
    """
    return IScatter(ds, x, y, z, y_err=y_err, x_err=x_err, **kwargs)

class AutoIScatter(IScatter):

    def __init__(self, x, y_z, **iscatter_opts):
        ds = auto_xyz_ds(x, y_z)
        super().__init__(ds, 'x', 'y', z='z', **iscatter_opts)

def auto_iscatter(x, y_z, **iscatter_opts):
    """Auto version of :func:`~xyzpy.iscatter` that accepts array arguments
    by converting them to a ``Dataset`` first.
    """
    return AutoIScatter(x, y_z, **iscatter_opts)()

# --------------------------------------------------------------------------- #

_HEATMAP_ALT_DEFAULTS = (
    ('legend', False),
    ('colorbar', True),
    ('colormap', 'inferno'),
    ('gridlines', False),
    ('padding', 0),
    ('figsize', (5, 5)),  # try to be square, maybe use aspect_ratio??
)

class IHeatMap(PlotterBokeh, AbstractHeatMap):

    def __init__(self, ds, x, y, z, **kwargs):
        # set some heatmap specific options
        for k, default in _HEATMAP_ALT_DEFAULTS:
            if k not in kwargs:
                kwargs[k] = default
        super().__init__(ds, x, y, z, **kwargs)

    def plot_heatmap(self):
        self.calc_color_norm()
        self._plot.image(image='image', x='x', y='y', dw='dw', dh='dh',
                         source=self._source, color_mapper=self.mappable)

    def __call__(self):
        # Core preparation
        self.prepare_data_single()
        # matplotlib preparation
        self.prepare_plot()
        self.plot_heatmap()
        self.plot_colorbar()
        self.set_tools()
        return self.show(interactive=self._interactive)

@bokeh_multi_plot
@intercept_call_arg
def iheatmap(ds, x, y, z, **kwargs):
    """From  ``ds`` plot variable ``z`` as a function of ``x`` and ``y`` using
    a 2D heatmap. Interactive,

    Parameters
    ----------
    ds : xarray.Dataset
        Dataset to plot from.
    x : str
        Dimension to plot along the x-axis.
    y : str
        Dimension to plot along the y-axis.
    z : str, optional
        Variable to plot as colormap.
    row : str, optional
        Dimension to vary over as a function of rows.
    col : str, optional
        Dimension to vary over as a function of columns.
    plot_opts
        See ``xyzpy.plot.core.PLOTTER_DEFAULTS``.
    """
    return IHeatMap(ds, x, y, z, **kwargs)

class AutoIHeatMap(IHeatMap):

    def __init__(self, x, **iheatmap_opts):
        ds = auto_xyz_ds(x)
        super().__init__(ds, 'y', 'z', 'x', **iheatmap_opts)

def auto_iheatmap(x, **iheatmap_opts):
    """Auto version of :func:`~xyzpy.iheatmap` that accepts array arguments
    by converting them to a ``Dataset`` first.
    """
    return AutoIHeatMap(x, **iheatmap_opts)()


text:
# Copyright (c) 2019-2020, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Copyright (c) 2019-2020, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import cudf
import cupy as cp
import dask_cudf
import pytest
import rmm

import numpy as np
import pandas as pd

from cuml.dask.ensemble import RandomForestClassifier as cuRFC_mg
from cuml.dask.ensemble import RandomForestRegressor as cuRFR_mg
from cuml.dask.common import utils as dask_utils

from dask.array import from_array
from sklearn.datasets import make_regression, make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, r2_score
from dask.distributed import Client

def _prep_training_data(c, X_train, y_train, partitions_per_worker):
    workers = c.has_what().keys()
    n_partitions = partitions_per_worker * len(workers)
    X_cudf = cudf.DataFrame.from_pandas(pd.DataFrame(X_train))
    X_train_df = dask_cudf.from_cudf(X_cudf, npartitions=n_partitions)

    y_cudf = np.array(pd.DataFrame(y_train).values)
    y_cudf = y_cudf[:, 0]
    y_cudf = cudf.Series(y_cudf)
    y_train_df = \
        dask_cudf.from_cudf(y_cudf, npartitions=n_partitions)

    X_train_df, \
        y_train_df = dask_utils.persist_across_workers(c,
                                                       [X_train_df,
                                                        y_train_df],
                                                       workers=workers)
    return X_train_df, y_train_df

@pytest.mark.parametrize('partitions_per_worker', [1, 3])
def test_rf_classification_dask(partitions_per_worker, cluster):

    # Use CUDA_VISIBLE_DEVICES to control the number of workers
    c = Client(cluster)

    try:

        X, y = make_classification(n_samples=10000, n_features=20,
                                   n_clusters_per_class=1, n_informative=10,
                                   random_state=123, n_classes=5)

        X = X.astype(np.float32)
        y = y.astype(np.int32)

        X_train, X_test, y_train, y_test = \
            train_test_split(X, y, test_size=1000)

        cu_rf_params = {
            'n_estimators': 40,
            'max_depth': 16,
            'n_streams': 1,
            'n_bins': 16,
        }

        X_train_df, y_train_df = _prep_training_data(c, X_train, y_train,
                                                     partitions_per_worker)
        cu_rf_mg = cuRFC_mg(**cu_rf_params)
        cu_rf_mg.fit(X_train_df, y_train_df)
        cu_rf_mg_predict = cu_rf_mg.predict(X_test)

        acc_score = accuracy_score(cu_rf_mg_predict, y_test, normalize=True)
        assert acc_score > 0.8

    finally:
        c.close()

@pytest.mark.parametrize('partitions_per_worker', [1, 3])
def test_rf_classification_dask_cudf(partitions_per_worker, cluster):

    # Use CUDA_VISIBLE_DEVICES to control the number of workers
    c = Client(cluster)

    try:

        X, y = make_classification(n_samples=10000, n_features=20,
                                   n_clusters_per_class=1, n_informative=10,
                                   random_state=123, n_classes=5)

        X = X.astype(np.float32)
        y = y.astype(np.int32)

        X_train, X_test, y_train, y_test = \
            train_test_split(X, y, test_size=1000)

        cu_rf_params = {
            'n_estimators': 40,
            'max_depth': 16,
            'n_bins': 16,
        }

        X_train_df, y_train_df = _prep_training_data(c, X_train, y_train,
                                                     partitions_per_worker)

        X_test_cudf = cudf.DataFrame.from_gpu_matrix(rmm.to_device(X_test))
        cu_rf_mg = cuRFC_mg(**cu_rf_params)
        cu_rf_mg.fit(X_train_df, y_train_df)
        cu_rf_mg_predict = cu_rf_mg.predict(X_test_cudf)

        acc_score = accuracy_score(cu_rf_mg_predict, y_test, normalize=True)

        assert acc_score > 0.8

    finally:
        c.close()

@pytest.mark.parametrize('partitions_per_worker', [1, 5])
def test_rf_regression_dask_fil(partitions_per_worker, cluster):

    # Use CUDA_VISIBLE_DEVICES to control the number of workers
    c = Client(cluster)

    try:

        X, y = make_regression(n_samples=10000, n_features=20,
                               n_informative=10, random_state=123)

        X = X.astype(np.float32)
        y = y.astype(np.float32)

        X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                            test_size=1000)

        cu_rf_params = {
            'n_estimators': 50,
            'max_depth': 16,
            'n_bins': 16,
        }

        workers = c.has_what().keys()
        n_partitions = partitions_per_worker * len(workers)

        X_cudf = cudf.DataFrame.from_pandas(pd.DataFrame(X_train))
        X_train_df = \
            dask_cudf.from_cudf(X_cudf, npartitions=n_partitions)

        y_cudf = np.array(pd.DataFrame(y_train).values)
        y_cudf = y_cudf[:, 0]
        y_cudf = cudf.Series(y_cudf)
        y_train_df = \
            dask_cudf.from_cudf(y_cudf, npartitions=n_partitions)
        X_cudf_test = cudf.DataFrame.from_pandas(pd.DataFrame(X_test))
        X_test_df = \
            dask_cudf.from_cudf(X_cudf_test, npartitions=n_partitions)

        X_train_df, y_train_df = dask_utils.persist_across_workers(
            c, [X_train_df, y_train_df], workers=workers)

        cu_rf_mg = cuRFR_mg(**cu_rf_params)
        cu_rf_mg.fit(X_train_df, y_train_df)

        cu_rf_mg_predict = cu_rf_mg.predict(X_test_df).compute()
        cu_rf_mg_predict = cp.asnumpy(cp.array(cu_rf_mg_predict))

        acc_score = r2_score(cu_rf_mg_predict, y_test)

        assert acc_score >= 0.67

    finally:
        c.close()

@pytest.mark.parametrize('partitions_per_worker', [1, 5])
@pytest.mark.parametrize('output_class', [True, False])
def test_rf_classification_dask_fil(partitions_per_worker, cluster,
                                    output_class):

    # Use CUDA_VISIBLE_DEVICES to control the number of workers
    c = Client(cluster)

    try:

        X, y = make_classification(n_samples=10000, n_features=30,
                                   n_clusters_per_class=1, n_informative=20,
                                   random_state=123, n_classes=2)

        X = X.astype(np.float32)
        y = y.astype(np.int32)

        X_train, X_test, y_train, y_test = \
            train_test_split(X, y, test_size=1000)

        cu_rf_params = {
            'n_estimators': 25,
            'max_depth': 13,
            'n_bins': 15,
        }

        X_train_df, y_train_df = _prep_training_data(c, X_train, y_train,
                                                     partitions_per_worker)
        X_test_df, _ = _prep_training_data(c, X_test, y_test,
                                           partitions_per_worker)
        cu_rf_mg = cuRFC_mg(**cu_rf_params)
        cu_rf_mg.fit(X_train_df, y_train_df)
        cu_rf_mg_predict = cu_rf_mg.predict(X_test_df, output_class).compute()
        cu_rf_mg_predict = cp.asnumpy(cp.array(cu_rf_mg_predict))
        if not output_class:
            cu_rf_mg_predict = np.round(cu_rf_mg_predict)

        acc_score = accuracy_score(cu_rf_mg_predict, y_test, normalize=True)

        assert acc_score > 0.8

    finally:
        c.close()

@pytest.mark.parametrize('partitions_per_worker', [1, 5])
@pytest.mark.parametrize('output_class', [True, False])
def test_rf_classification_dask_array(partitions_per_worker, cluster,
                                      output_class):

    # Use CUDA_VISIBLE_DEVICES to control the number of workers
    c = Client(cluster)

    try:

        X, y = make_classification(n_samples=10000, n_features=30,
                                   n_clusters_per_class=1, n_informative=20,
                                   random_state=123, n_classes=2)

        X = X.astype(np.float32)
        y = y.astype(np.int32)

        X_train, X_test, y_train, y_test = \
            train_test_split(X, y, test_size=1000)

        cu_rf_params = {
            'n_estimators': 25,
            'max_depth': 13,
            'n_bins': 15,
        }

        X_train_df, y_train_df = _prep_training_data(c, X_train, y_train,
                                                     partitions_per_worker)
        X_test_dask_array = from_array(X_test)
        cu_rf_mg = cuRFC_mg(**cu_rf_params)
        cu_rf_mg.fit(X_train_df, y_train_df)
        cu_rf_mg_predict = cu_rf_mg.predict(X_test_dask_array,
                                            output_class).compute()
        if not output_class:
            cu_rf_mg_predict = np.round(cu_rf_mg_predict)

        acc_score = accuracy_score(cu_rf_mg_predict, y_test, normalize=True)

        assert acc_score > 0.8

    finally:
        c.close()

ext: py

max_stars_repo_name: Pandinosaurus/cuml

max_issues_repo_name: mseneshen/cuml

max_forks_repo_name: mseneshen/cuml

reason: import numpy,import cupy


text:
import cv2
import numpy as np

LINES_HAND = [[0,1],[1,2],[2,3],[3,4],
            [0,5],[5,6],[6,7],[7,8],
            [5,9],[9,10],[10,11],[11,12],
            [9,13],[13,14],[14,15],[15,16],
            [13,17],[17,18],[18,19],[19,20],[0,17]]

# LINES_BODY to draw the body skeleton when Body Pre Focusing is used
LINES_BODY = [[4,2],[2,0],[0,1],[1,3],
            [10,8],[8,6],[6,5],[5,7],[7,9],
            [6,12],[12,11],[11,5],
            [12,14],[14,16],[11,13],[13,15]]

class HandTrackerRenderer:
    def __init__(self,
                tracker,
                output=None):

        self.tracker = tracker

        # Rendering flags
        if self.tracker.use_lm:
            self.show_pd_box = False
            self.show_pd_kps = False
            self.show_rot_rect = False
            self.show_handedness = False
            self.show_landmarks = True
            self.show_scores = False
            self.show_gesture = self.tracker.use_gesture
        else:
            self.show_pd_box = True
            self.show_pd_kps = False
            self.show_rot_rect = False
            self.show_scores = False

        self.show_xyz_zone = self.show_xyz = self.tracker.xyz
        self.show_fps = True
        self.show_body = False # self.tracker.body_pre_focusing is not None
        self.show_inferences_status = False

        if output is None:
            self.output = None
        else:
            fourcc = cv2.VideoWriter_fourcc(*"MJPG")
            self.output = cv2.VideoWriter(output,fourcc,self.tracker.video_fps,(self.tracker.img_w, self.tracker.img_h))

    def norm2abs(self, x_y):
        x = int(x_y[0] * self.tracker.frame_size - self.tracker.pad_w)
        y = int(x_y[1] * self.tracker.frame_size - self.tracker.pad_h)
        return (x, y)

    def draw_hand(self, hand):

        if self.tracker.use_lm:
            # (info_ref_x, info_ref_y): coords in the image of a reference point
            # relatively to which hands information (score, handedness, xyz,...) are drawn
            info_ref_x = hand.landmarks[0,0]
            info_ref_y = np.max(hand.landmarks[:,1])

            # thick_coef is used to adapt the size of the draw landmarks features according to the size of the hand.
            thick_coef = hand.rect_w_a / 400
            if hand.lm_score > self.tracker.lm_score_thresh:
                if self.show_rot_rect:
                    cv2.polylines(self.frame, [np.array(hand.rect_points)], True, (0,255,255), 2, cv2.LINE_AA)
                if self.show_landmarks:
                    lines = [np.array([hand.landmarks[point] for point in line]).astype(np.int) for line in LINES_HAND]
                    cv2.polylines(self.frame, lines, False, (255, 0, 0), int(1+thick_coef*3), cv2.LINE_AA)
                    radius = int(1+thick_coef*5)
                    if self.tracker.use_gesture:
                        # color depending on finger state (1=open, 0=close, -1=unknown)
                        color = { 1: (0,255,0), 0: (0,0,255), -1:(0,255,255)}
                        cv2.circle(self.frame, (hand.landmarks[0][0], hand.landmarks[0][1]), radius, color[-1], -1)
                        for i in range(1,5):
                            cv2.circle(self.frame, (hand.landmarks[i][0], hand.landmarks[i][1]), radius, color[hand.thumb_state], -1)
                        for i in range(5,9):
                            cv2.circle(self.frame, (hand.landmarks[i][0], hand.landmarks[i][1]), radius, color[hand.index_state], -1)
                        for i in range(9,13):
                            cv2.circle(self.frame, (hand.landmarks[i][0], hand.landmarks[i][1]), radius, color[hand.middle_state], -1)
                        for i in range(13,17):
                            cv2.circle(self.frame, (hand.landmarks[i][0], hand.landmarks[i][1]), radius, color[hand.ring_state], -1)
                        for i in range(17,21):
                            cv2.circle(self.frame, (hand.landmarks[i][0], hand.landmarks[i][1]), radius, color[hand.little_state], -1)
                    else:
                        for x,y in hand.landmarks[:,:2]:
                            cv2.circle(self.frame, (int(x), int(y)), radius, (0,128,255), -1)

                if self.show_handedness:
                    cv2.putText(self.frame, f"{hand.label.upper()} {hand.handedness:.2f}",
                            (info_ref_x-90, info_ref_y+40),
                            cv2.FONT_HERSHEY_PLAIN, 2, (0,255,0) if hand.handedness > 0.5 else (0,0,255), 2)
                if self.show_scores:
                    cv2.putText(self.frame, f"Landmark score: {hand.lm_score:.2f}",
                            (info_ref_x-90, info_ref_y+110),
                            cv2.FONT_HERSHEY_PLAIN, 2, (255,255,0), 2)
                if self.tracker.use_gesture and self.show_gesture:
                    cv2.putText(self.frame, hand.gesture, (info_ref_x-20, info_ref_y-50),
                            cv2.FONT_HERSHEY_PLAIN, 3, (255,255,255), 3)

        if hand.pd_box is not None:
            box = hand.pd_box
            box_tl = self.norm2abs((box[0], box[1]))
            box_br = self.norm2abs((box[0]+box[2], box[1]+box[3]))
            if self.show_pd_box:
                cv2.rectangle(self.frame, box_tl, box_br, (0,255,0), 2)
            if self.show_pd_kps:
                for i,kp in enumerate(hand.pd_kps):
                    x_y = self.norm2abs(kp)
                    cv2.circle(self.frame, x_y, 6, (0,0,255), -1)
                    cv2.putText(self.frame, str(i), (x_y[0], x_y[1]+12), cv2.FONT_HERSHEY_PLAIN, 1.5, (0,255,0), 2)
            if self.show_scores:
                if self.tracker.use_lm:
                    x, y = info_ref_x - 90, info_ref_y + 80
                else:
                    x, y = box_tl[0], box_br[1]+60
                cv2.putText(self.frame, f"Palm score: {hand.pd_score:.2f}",
                        (x, y),
                        cv2.FONT_HERSHEY_PLAIN, 2, (255,255,0), 2)

        if self.show_xyz:
            if self.tracker.use_lm:
                x0, y0 = info_ref_x - 40, info_ref_y + 40
            else:
                x0, y0 = box_tl[0], box_br[1]+20
            cv2.rectangle(self.frame, (x0,y0), (x0+100, y0+85), (220,220,240), -1)
            cv2.putText(self.frame, f"X:{hand.xyz[0]/10:3.0f} cm", (x0+10, y0+20), cv2.FONT_HERSHEY_PLAIN, 1, (20,180,0), 2)
            cv2.putText(self.frame, f"Y:{hand.xyz[1]/10:3.0f} cm", (x0+10, y0+45), cv2.FONT_HERSHEY_PLAIN, 1, (255,0,0), 2)
            cv2.putText(self.frame, f"Z:{hand.xyz[2]/10:3.0f} cm", (x0+10, y0+70), cv2.FONT_HERSHEY_PLAIN, 1, (0,0,255), 2)
        if self.show_xyz_zone:
            # Show zone on which the spatial data were calculated
            cv2.rectangle(self.frame, tuple(hand.xyz_zone[0:2]), tuple(hand.xyz_zone[2:4]), (180,0,180), 2)

    def draw_body(self, body):
        lines = [np.array([body.keypoints[point] for point in line]) for line in LINES_BODY if body.scores[line[0]] > self.tracker.body_score_thresh and body.scores[line[1]] > self.tracker.body_score_thresh]
        cv2.polylines(self.frame, lines, False, (255, 144, 30), 2, cv2.LINE_AA)

    def draw_bag(self, bag):

        if self.show_inferences_status:
            # Draw inferences status
            h = self.frame.shape[0]
            u = h // 10
            status=""
            if bag.get("bpf_inference", 0):
                cv2.rectangle(self.frame, (u, 8*u), (2*u, 9*u), (255,144,30), -1)
            if bag.get("pd_inference", 0):
                cv2.rectangle(self.frame, (2*u, 8*u), (3*u, 9*u), (0,255,0), -1)
            nb_lm_inferences = bag.get("lm_inference", 0)
            if nb_lm_inferences:
                cv2.rectangle(self.frame, (3*u, 8*u), ((3+nb_lm_inferences)*u, 9*u), (0,0,255), -1)

        body = bag.get("body", False)
        if body and self.show_body:
            # Draw skeleton
            self.draw_body(body)
            # Draw Movenet smart cropping rectangle
            cv2.rectangle(self.frame, (body.crop_region.xmin, body.crop_region.ymin), (body.crop_region.xmax, body.crop_region.ymax), (0,255,255), 2)
            # Draw focus zone
            focus_zone= bag.get("focus_zone", None)
            if focus_zone:
                cv2.rectangle(self.frame, focus_zone[0:2], focus_zone[2:4], (0,255,0),2)

    def draw(self, frame, hands, bag={}):
        self.frame = frame
        if bag:
            self.draw_bag(bag)
        for hand in hands:
            self.draw_hand(hand)
        return self.frame

    def exit(self):
        if self.output:
            self.output.release()

    def waitKey(self, delay=1):
        if self.show_fps:
                self.tracker.fps.draw(self.frame, orig=(50,50), size=1, color=(240,180,100))
        # cv2.imshow("Hand tracking", self.frame)
        if self.output:
            self.output.write(self.frame)
        key = cv2.waitKey(delay)
        if key == 32:
            # Pause on space bar
            key = cv2.waitKey(0)
            if key == ord('s'):
                print("Snapshot saved in snapshot.jpg")
                cv2.imwrite("snapshot.jpg", self.frame)
        elif key == ord('1'):
            self.show_pd_box = not self.show_pd_box
        elif key == ord('2'):
            self.show_pd_kps = not self.show_pd_kps
        elif key == ord('3'):
            self.show_rot_rect = not self.show_rot_rect
        elif key == ord('4') and self.tracker.use_lm:
            self.show_landmarks = not self.show_landmarks
        elif key == ord('5') and self.tracker.use_lm:
            self.show_handedness = not self.show_handedness
        elif key == ord('6'):
            self.show_scores = not self.show_scores
        elif key == ord('7') and self.tracker.use_lm:
            if self.tracker.use_gesture:
                self.show_gesture = not self.show_gesture
        elif key == ord('8'):
            if self.tracker.xyz:
                self.show_xyz = not self.show_xyz
        elif key == ord('9'):
            if self.tracker.xyz:
                self.show_xyz_zone = not self.show_xyz_zone
        elif key == ord('f'):
            self.show_fps = not self.show_fps
        elif key == ord('b'):
            if self.tracker.body_pre_focusing:
                self.show_body = not self.show_body
        elif key == ord('s'):
            self.show_inferences_status = not self.show_inferences_status
        return key

reason: import numpy


text:
#!/usr/bin/python2.7

# ROS
import rospy
import rospkg

import numpy as np
#import pylab as plt
import matplotlib.pyplot as plt

#keras
from keras.models import load_model, Sequential, Model
from keras.preprocessing.image import load_img, img_to_array, array_to_img
from keras import backend as K
from keras.utils import plot_model

#openCV
import cv2

#service
from path_prediction.srv import PathPrediction, PathPredictionResponse

#for directories and files manipulation
from os import listdir
from os.path import isfile, join

class Prediction(object):

  #--------------------------------------------------------------------------
  def __init__(self, norm, six):

    rospack = rospkg.RosPack()
    model_route = rospack.get_path('path_prediction') + '/model/my_model.h5'

    self.save_route = rospack.get_path('path_prediction') + '/saved_images/'
    self.save_count = 1
    self.save_img = False

    self.model_ = load_model(model_route)
    print "Loaded_model:"
    print self.model_.summary()
    #plot_model(model, to_file=(test_dir+'model.png'), show_shapes=True)
    self.norm_img_data_ = norm
    self.use_six_ = six

  #--------------------------------------------------------------------------
  def preprocess_image_array(self, img_array, rows, cols, norm=False, six=False):

    #print('input:')
    #print(img_array)
    # Transform 1D array to greyscale image numpy array (1, rows, cols)
    inputimg = np.asarray(img_array, np.int16) # dtype=K.floatx()
    #print('input numpy array shape:')
    #print(inputimg)
    #print(inputimg.shape)
    img = []
    for i in range(rows):
      if i==0:
        #print('\ni: %i' % i)
        #print('Subarray:')
        #print(inputimg[(i*cols):(i*cols+(cols))])
        img = np.array(inputimg[(i*cols):(i*cols+(cols))], dtype=K.floatx())
        img = np.expand_dims(img, axis=0)
        #print(img)
        #print(img.shape)
      else:
        #print('\ni: %i' % i)
        #print('Subarray:')
        #print(inputimg[(i*cols):(i*cols+(cols))])
        aux = np.array(inputimg[(i*cols):(i*cols+(cols))], dtype=K.floatx())
        aux = np.expand_dims(aux, axis=0)
        img = np.append(img, aux, axis=0)
        #print(img)

    #print('prepocess. numpyarray shape:')
    #print(img.shape)

    #img = np.asarray(img_array, dtype=np.float32)
    img = img.reshape((1, img.shape[0], img.shape[1]))
    #print('img.dtype: %s' % img.dtype)

    if(norm==True):
      if(six==True):
        img[img <=15] = 0     #free: 0
        img[img >=235] = 1    #path: 255
        img[img >=184] = 0.8  #obstacles: 204
        img[img >=133] = 0.4  #people front: 153
        img[img >=82] = 0.2   #people back: 102
        img[img >=43] = 0.6   # goal: 63
      else:
        img[img <=15] = 0     #free: 0
        img[img >=235] = 1    #path: 255
        img[img >=184] = 0.25  #obstacles: 204
        img[img >=82] = 0.5  #people front: 153, back: 102
        img[img >=43] = 0.75   # goal: 63
    else:
      if(six==True):
        img[img <=15] = 0    #free: 0
        img[img >=235] = 3   #path: 255
        img[img >=184] = 2   #obstacles: 204
        img[img >=133] = -2  #people front: 153
        img[img >=82] = -1   #people back: 102
        img[img >=43] = 1    # goal: 63
      else:
        img[img <=15] = 0    #free: 0
        img[img >=235] = 2   #path: 255
        img[img >=184] = -1   #obstacles: 204
        img[img >=82] = -2  #people front: 153, back: 102
        img[img >=43] = 1    # goal: 63

    img = np.expand_dims(img, axis=0)
    #print('Preprocess. Final Image shape:')
    #print(img.shape)
    return img

  #--------------------------------------------------------------------------
  def deprocess_image_pred(self, img):
    x = []
    # Util function to convert a tensor into a valid image.
    if K.image_data_format() == 'channels_first':
      #x = img.reshape((1, img.shape[2], img.shape[3]))
      #x = x.transpose((1, 2, 0))
      x = img.reshape(img.shape[2], img.shape[3])
    else:
      #x = img.reshape((img.shape[1], img.shape[2], 1))
      x = img.reshape(img.shape[1], img.shape[2])

    #print('Deprocess. Image shape:')
    #print(x.shape)

    #Transform numpy array to 1D array
    x = x.flatten()

    return x

  #--------------------------------------------------------------------------
  def deprocess_image_grey(self, x):

    img = x
    # Util function to convert a tensor into a valid image.
    if K.image_data_format() == 'channels_first':
      img = img.reshape((1, img.shape[2], img.shape[3]))
      img = img.transpose((1, 2, 0))
    else:
      img = img.reshape((img.shape[1], img.shape[2], 1))

    #x /= 2.
    #x += 0.5
    img *= 255.
    img = np.clip(img, 0, 255).astype('uint8')

    return img

  #--------------------------------------------------------------------------
  def predict_path(self, req):

    img = self.preprocess_image_array(req.input, req.input_rows, req.input_cols, self.norm_img_data_, self.use_six_)
    output = self.model_.predict(img)
    pred = self.deprocess_image_pred(output)
    if(self.save_img==True):
      ig = self.deprocess_image_grey(output)
      name = 'pred_{0}'.format(self.save_count)
      plt.imsave((self.save_route + name + ".jpeg"), ig[:,:,0])
      self.save_count = self.save_count + 1

    return PathPredictionResponse(pred, 200, 200)

#--------------------------------------------------------------------------
if __name__ == '__main__':

  rospy.init_node('prediction_node')
  print('--Starting prediction service--')

  normalize_values = rospy.get_param('~normalize_image_values', True)
  use_six_categories = rospy.get_param('~use_six_categories', True)

  print('Normalize values: %i ' % normalize_values)
  print('Use six categories: %i ' % use_six_categories)

  pred_node = Prediction(normalize_values, use_six_categories)

  # We call for prediction once for loading
  vec = np.ones(200*200, dtype=np.int16)
  img = pred_node.preprocess_image_array(vec, 200, 200, False, False)
  output = pred_node.model_.predict(img)

  # Declare service
  service = rospy.Service('path_prediction', PathPrediction, pred_node.predict_path)

  rospy.spin()


text:
import os, json
from PIL import Image
import numpy as np
from skimage import io

basePath = './log/log_joku/'
json_files = [pos_json for pos_json in os.listdir(basePath) if pos_json.endswith('.json')]

for file in json_files:
    if file != 'meta.json':
        #print(file)
        with open(basePath + file) as f:
            data = json.load(f)
            index = data.get('cam/image_array').split('_')[1].split('.')[0]
            if (int(index) % 1000) == 0:
                print(index)
            #print(index)
            imgPath = basePath + data.get('cam/image_array')
            img = Image.open(imgPath)
            imgFlipped = np.fliplr(img)
            flippedImgName = 'shotflipped_' + index + '.png'
            io.imsave(basePath + flippedImgName, imgFlipped)
            data['cam/image_array'] = flippedImgName
            data['user/angle'] = data.get('user/angle') * -1
            with open(basePath + 'recordflipped_' + index + '.json', 'w') as outfile:
                json.dump(data, outfile)

max_stars_repo_name: futurice/maximum-aittack

max_issues_repo_name: futurice/maximum-aittack

max_forks_repo_name: futurice/maximum-aittack


text:
# Copyright 2019 Kyoto University (Hirofumi Inaguma)
#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)

"""RNN sequence-to-sequence text translation model (pytorch)."""

import argparse
import logging
import math
import os

import chainer
from chainer import reporter
import nltk
import numpy as np
import torch

from espnet.nets.e2e_asr_common import label_smoothing_dist
from espnet.nets.mt_interface import MTInterface
from espnet.nets.pytorch_backend.initialization import uniform_init_parameters
from espnet.nets.pytorch_backend.nets_utils import get_subsample
from espnet.nets.pytorch_backend.nets_utils import pad_list
from espnet.nets.pytorch_backend.nets_utils import to_device
from espnet.nets.pytorch_backend.rnn.argument import (
    add_arguments_rnn_encoder_common,  # noqa: H301
    add_arguments_rnn_decoder_common,  # noqa: H301
    add_arguments_rnn_attention_common,  # noqa: H301
)
from espnet.nets.pytorch_backend.rnn.attentions import att_for
from espnet.nets.pytorch_backend.rnn.decoders import decoder_for
from espnet.nets.pytorch_backend.rnn.encoders import encoder_for
from espnet.utils.fill_missing_args import fill_missing_args

class Reporter(chainer.Chain):
    """A chainer reporter wrapper."""

    def report(self, loss, acc, ppl, bleu):
        """Report at every step."""
        reporter.report({"loss": loss}, self)
        reporter.report({"acc": acc}, self)
        reporter.report({"ppl": ppl}, self)
        reporter.report({"bleu": bleu}, self)

class E2E(MTInterface, torch.nn.Module):
    """E2E module.

    :param int idim: dimension of inputs
    :param int odim: dimension of outputs
    :param Namespace args: argument Namespace containing options

    """

    @staticmethod
    def add_arguments(parser):
        """Add arguments."""
        E2E.encoder_add_arguments(parser)
        E2E.attention_add_arguments(parser)
        E2E.decoder_add_arguments(parser)
        return parser

    @staticmethod
    def encoder_add_arguments(parser):
        """Add arguments for the encoder."""
        group = parser.add_argument_group("E2E encoder setting")
        group = add_arguments_rnn_encoder_common(group)
        return parser

    @staticmethod
    def attention_add_arguments(parser):
        """Add arguments for the attention."""
        group = parser.add_argument_group("E2E attention setting")
        group = add_arguments_rnn_attention_common(group)
        return parser

    @staticmethod
    def decoder_add_arguments(parser):
        """Add arguments for the decoder."""
        group = parser.add_argument_group("E2E decoder setting")
        group = add_arguments_rnn_decoder_common(group)
        return parser

    def __init__(self, idim, odim, args):
        """Construct an E2E object.

        :param int idim: dimension of inputs
        :param int odim: dimension of outputs
        :param Namespace args: argument Namespace containing options
        """
        super(E2E, self).__init__()
        torch.nn.Module.__init__(self)

        # fill missing arguments for compatibility
        args = fill_missing_args(args, self.add_arguments)

        self.etype = args.etype
        self.verbose = args.verbose
        # NOTE: for self.build method
        args.char_list = getattr(args, "char_list", None)
        self.char_list = args.char_list
        self.outdir = args.outdir
        self.space = args.sym_space
        self.blank = args.sym_blank
        self.reporter = Reporter()

        # below means the last number becomes eos/sos ID
        # note that sos/eos IDs are identical
        self.sos = odim - 1
        self.eos = odim - 1
        self.pad = 0
        # NOTE: we reserve index:0 for <pad> although this is reserved for a blank class
        # in ASR. However, blank labels are not used in MT.
        # To keep the vocabulary size,
        # we use index:0 for padding instead of adding one more class.

        # subsample info
        self.subsample = get_subsample(args, mode="mt", arch="rnn")

        # label smoothing info
        if args.lsm_type and os.path.isfile(args.train_json):
            logging.info("Use label smoothing with " + args.lsm_type)
            labeldist = label_smoothing_dist(
                odim, args.lsm_type, transcript=args.train_json
            )
        else:
            labeldist = None

        # multilingual related
        self.multilingual = getattr(args, "multilingual", False)
        self.replace_sos = getattr(args, "replace_sos", False)

        # encoder
        self.embed = torch.nn.Embedding(idim, args.eunits, padding_idx=self.pad)
        self.dropout = torch.nn.Dropout(p=args.dropout_rate)
        self.enc = encoder_for(args, args.eunits, self.subsample)
        # attention
        self.att = att_for(args)
        # decoder
        self.dec = decoder_for(args, odim, self.sos, self.eos, self.att, labeldist)

        # tie source and target emeddings
        if args.tie_src_tgt_embedding:
            if idim != odim:
                raise ValueError(
                    "When using tie_src_tgt_embedding, idim and odim must be equal."
                )
            if args.eunits != args.dunits:
                raise ValueError(
                    "When using tie_src_tgt_embedding, eunits and dunits must be equal."
                )
            self.embed.weight = self.dec.embed.weight

        # tie emeddings and the classfier
        if args.tie_classifier:
            if args.context_residual:
                raise ValueError(
                    "When using tie_classifier, context_residual must be turned off."
                )
            self.dec.output.weight = self.dec.embed.weight

        # weight initialization
        self.init_like_fairseq()

        # options for beam search
        if args.report_bleu:
            trans_args = {
                "beam_size": args.beam_size,
                "penalty": args.penalty,
                "ctc_weight": 0,
                "maxlenratio": args.maxlenratio,
                "minlenratio": args.minlenratio,
                "lm_weight": args.lm_weight,
                "rnnlm": args.rnnlm,
                "nbest": args.nbest,
                "space": args.sym_space,
                "blank": args.sym_blank,
                "tgt_lang": False,
            }

            self.trans_args = argparse.Namespace(**trans_args)
            self.report_bleu = args.report_bleu
        else:
            self.report_bleu = False
        self.rnnlm = None

        self.logzero = -10000000000.0
        self.loss = None
        self.acc = None

    def init_like_fairseq(self):
        """Initialize weight like Fairseq.

        Fairseq basically uses W, b, EmbedID.W ~ Uniform(-0.1, 0.1),
        """
        uniform_init_parameters(self)
        # exceptions
        # embed weight ~ Normal(-0.1, 0.1)
        torch.nn.init.uniform_(self.embed.weight, -0.1, 0.1)
        torch.nn.init.constant_(self.embed.weight[self.pad], 0)
        torch.nn.init.uniform_(self.dec.embed.weight, -0.1, 0.1)
        torch.nn.init.constant_(self.dec.embed.weight[self.pad], 0)

    def forward(self, xs_pad, ilens, ys_pad):
        """E2E forward.

        :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, idim)
        :param torch.Tensor ilens: batch of lengths of input sequences (B)
        :param torch.Tensor ys_pad: batch of padded token id sequence tensor (B, Lmax)
        :return: loss value
        :rtype: torch.Tensor
        """
        # 1. Encoder
        xs_pad, ys_pad = self.target_language_biasing(xs_pad, ilens, ys_pad)
        hs_pad, hlens, _ = self.enc(self.dropout(self.embed(xs_pad)), ilens)

        # 3. attention loss
        self.loss, self.acc, self.ppl = self.dec(hs_pad, hlens, ys_pad)

        # 4. compute bleu
        if self.training or not self.report_bleu:
            self.bleu = 0.0
        else:
            lpz = None

            nbest_hyps = self.dec.recognize_beam_batch(
                hs_pad,
                torch.tensor(hlens),
                lpz,
                self.trans_args,
                self.char_list,
                self.rnnlm,
            )
            # remove <sos> and <eos>
            list_of_refs = []
            hyps = []
            y_hats = [nbest_hyp[0]["yseq"][1:-1] for nbest_hyp in nbest_hyps]
            for i, y_hat in enumerate(y_hats):
                y_true = ys_pad[i]

                seq_hat = [self.char_list[int(idx)] for idx in y_hat if int(idx) != -1]
                seq_true = [
                    self.char_list[int(idx)] for idx in y_true if int(idx) != -1
                ]
                seq_hat_text = "".join(seq_hat).replace(self.trans_args.space, " ")
                seq_hat_text = seq_hat_text.replace(self.trans_args.blank, "")
                seq_true_text = "".join(seq_true).replace(self.trans_args.space, " ")

                hyps += [seq_hat_text.split(" ")]
                list_of_refs += [[seq_true_text.split(" ")]]

            self.bleu = nltk.bleu_score.corpus_bleu(list_of_refs, hyps) * 100

        loss_data = float(self.loss)
        if not math.isnan(loss_data):
            self.reporter.report(loss_data, self.acc, self.ppl, self.bleu)
        else:
            logging.warning("loss (=%f) is not correct", loss_data)
        return self.loss

    def target_language_biasing(self, xs_pad, ilens, ys_pad):
        """Prepend target language IDs to source sentences for multilingual MT.

        These tags are prepended in source/target sentences as pre-processing.

        :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, idim)
        :param torch.Tensor ilens: batch of lengths of input sequences (B)
        :return: source text without language IDs
        :rtype: torch.Tensor
        :return: target text without language IDs
        :rtype: torch.Tensor
        :return: target language IDs
        :rtype: torch.Tensor (B, 1)
        """
        if self.multilingual:
            # remove language ID in the beggining
            tgt_lang_ids = ys_pad[:, 0].unsqueeze(1)
            xs_pad = xs_pad[:, 1:]  # remove source language IDs here
            ys_pad = ys_pad[:, 1:]

            # prepend target language ID to source sentences
            xs_pad = torch.cat([tgt_lang_ids, xs_pad], dim=1)
        return xs_pad, ys_pad

    def translate(self, x, trans_args, char_list, rnnlm=None):
        """E2E beam search.

        :param ndarray x: input source text feature (B, T, D)
        :param Namespace trans_args: argument Namespace containing options
        :param list char_list: list of characters
        :param torch.nn.Module rnnlm: language model module
        :return: N-best decoding results
        :rtype: list
        """
        prev = self.training
        self.eval()

        # 1. encoder
        # make a utt list (1) to use the same interface for encoder
        if self.multilingual:
            ilen = [len(x[0][1:])]
            h = to_device(
                self, torch.from_numpy(np.fromiter(map(int, x[0][1:]), dtype=np.int64))
            )
        else:
            ilen = [len(x[0])]
            h = to_device(
                self, torch.from_numpy(np.fromiter(map(int, x[0]), dtype=np.int64))
            )
        hs, _, _ = self.enc(self.dropout(self.embed(h.unsqueeze(0))), ilen)

        # 2. decoder
        # decode the first utterance
        y = self.dec.recognize_beam(hs[0], None, trans_args, char_list, rnnlm)

        if prev:
            self.train()
        return y

    def translate_batch(self, xs, trans_args, char_list, rnnlm=None):
        """E2E batch beam search.

        :param list xs:
            list of input source text feature arrays [(T_1, D), (T_2, D), ...]
        :param Namespace trans_args: argument Namespace containing options
        :param list char_list: list of characters
        :param torch.nn.Module rnnlm: language model module
        :return: N-best decoding results
        :rtype: list
        """
        prev = self.training
        self.eval()

        # 1. Encoder
        if self.multilingual:
            ilens = np.fromiter((len(xx[1:]) for xx in xs), dtype=np.int64)
            hs = [to_device(self, torch.from_numpy(xx[1:])) for xx in xs]
        else:
            ilens = np.fromiter((len(xx) for xx in xs), dtype=np.int64)
            hs = [to_device(self, torch.from_numpy(xx)) for xx in xs]
        xpad = pad_list(hs, self.pad)
        hs_pad, hlens, _ = self.enc(self.dropout(self.embed(xpad)), ilens)

        # 2. Decoder
        hlens = torch.tensor(list(map(int, hlens)))  # make sure hlens is tensor
        y = self.dec.recognize_beam_batch(
            hs_pad, hlens, None, trans_args, char_list, rnnlm
        )

        if prev:
            self.train()
        return y

    def calculate_all_attentions(self, xs_pad, ilens, ys_pad):
        """E2E attention calculation.

        :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, idim)
        :param torch.Tensor ilens: batch of lengths of input sequences (B)
        :param torch.Tensor ys_pad: batch of padded token id sequence tensor (B, Lmax)
        :return: attention weights with the following shape,
            1) multi-head case => attention weights (B, H, Lmax, Tmax),
            2) other case => attention weights (B, Lmax, Tmax).
        :rtype: float ndarray
        """
        self.eval()
        with torch.no_grad():
            # 1. Encoder
            xs_pad, ys_pad = self.target_language_biasing(xs_pad, ilens, ys_pad)
            hpad, hlens, _ = self.enc(self.dropout(self.embed(xs_pad)), ilens)

            # 2. Decoder
            att_ws = self.dec.calculate_all_attentions(hpad, hlens, ys_pad)
        self.train()
        return att_ws

max_stars_repo_name: Hertin/espnet

max_issues_repo_name: Hertin/espnet

max_forks_repo_name: Hertin/espnet


text:
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rcParams
from matplotlib import cm
from matplotlib import colors
import preprocessing as prep
import barplot

"""
# Constants
"""
fontsize = 18
figsize=(8, 5)
prefix = 'performance/'
rcParams['font.family'] = 'serif'
rcParams.update({'font.size': fontsize})
plt.set_cmap('viridis')
norm = colors.Normalize(vmin=0, vmax=256.0)
m = cm.ScalarMappable(norm=norm, cmap=plt.get_cmap('viridis') )

"""
Update Throughput
"""
label = ['oKDE', 'SPDT', 'SPDTw', 'D-Sketch']
color = [m.to_rgba(210), m.to_rgba(180), m.to_rgba(120), m.to_rgba(50)]
stat_update_thrp = [69.0, 65.5, 0.590, 1081]
nonstat_update_thrp = [64.9, 65.3, 0.654, 374]
stat_update_thrp_k = [score / 1000 for score in stat_update_thrp]
nonstat_update_thrp_k = [score / 1000 for score in nonstat_update_thrp]
data = [stat_update_thrp, nonstat_update_thrp]

fig, ax = plt.subplots(figsize=figsize)
barplot.bar_plot(ax, data, label, color, (0, 1200), fontsize=fontsize)

ax.set_ylabel('Throughput (Kops/sec)')
ax.set_xticks(np.arange(len(data)) + barplot.bar_width(len(data[0])) * 3/2)
ax.set_xticklabels(['Stationary', 'Non-stationary'])
ax.tick_params(axis='x', which='both', bottom=False, top=False)
ax.tick_params(axis='y', labelsize=fontsize * 0.85)

fig.tight_layout()
update_name = 'update-throughput'
plt.savefig(prefix + update_name + '.pdf')
plt.savefig(prefix + update_name + '.png')

"""
Probability Throughput
"""
margin = 0.2
label = ['oKDE', 'SPDT', 'SPDTw', 'D-Sketch', 'D-Sketch + cache']
color = [m.to_rgba(210), m.to_rgba(180), m.to_rgba(120), m.to_rgba(50), m.to_rgba(30)]
prob_thrp = [3546, 448, 442, 466, 4131]
prob_thrp_k = [score / 1000 for score in prob_thrp]
data = [prob_thrp]

fig, ax = plt.subplots(figsize=figsize)
barplot.bar_plot(ax, data, label, color, (0, 5000), fontsize=fontsize)

bw = barplot.bar_width(len(data[0]))
ax.set_ylabel('Throughput (Kops/sec)')
ax.set_xticks(np.arange(len(data)) + bw * (len(data[0]) / 2 - 0.5))
ax.set_xticklabels([])
ax.tick_params(axis='x', which='both', bottom=False, top=False)
ax.tick_params(axis='y', labelsize=fontsize * 0.85)
ax.set_xlim((-bw / 2 - margin, bw * (len(data[0]) - 0.5) + margin))

fig.tight_layout()
prob_name = 'probability-throughput'
plt.savefig(prefix + prob_name + '.pdf')
plt.savefig(prefix + prob_name + '.png')

"""
Memory usage
"""
margin = 0.2
label = ['oKDE', 'SPDT', 'SPDTw', 'D-Sketch']
color = [m.to_rgba(210), m.to_rgba(180), m.to_rgba(120), m.to_rgba(50)]
mem = [4632, 4296, (4296 + 4816), 6192]
mem_k = [score / 1000 for score in mem]
data = [mem_k]

fig, ax = plt.subplots(figsize=figsize)
barplot.bar_plot(ax, data, label, color, (0, 10.0), fontsize=fontsize)

bw = barplot.bar_width(len(data[0]))
ax.set_ylabel('Memory (KB)')
ax.set_xticks(np.arange(len(data)) + bw * (len(data[0]) / 2 - 0.5))
ax.set_xticklabels([])
ax.tick_params(axis='x', which='both', bottom=False, top=False)
ax.tick_params(axis='y', labelsize=fontsize * 0.85)
ax.set_xlim((-bw / 2 - margin, bw * (len(data[0]) - 0.5) + margin))

fig.tight_layout()
memory_name = 'memory-usage'
plt.savefig(prefix + memory_name + '.pdf')
plt.savefig(prefix + memory_name + '.png')

max_stars_repo_name: taekyulee/flip

max_issues_repo_name: taekyulee/flip

max_forks_repo_name: taekyulee/flip


text:
"""Test sky projection module."""

from numpy.testing import assert_array_almost_equal as assert_array

from matplotlib.path import Path

from pyvims.projections import Sky
from pyvims.angles import DEC, RA

from pytest import fixture, raises

@fixture
def proj():
    """Sky projection."""
    return Sky()

def test_sky_attr():
    """Test sky projection attributes."""
    proj = Sky(ra=10, dec=25, twist=50)

    assert str(proj) == 'Sky'

    assert isinstance(proj.ra, RA)
    assert isinstance(proj.dec, DEC)

    assert proj.ra == 10
    assert proj.dec == 25
    assert proj.twist == 50

    assert_array(proj.pointing, (10, 25, 50))

def test_sky_matrix():
    """Test sky rotation matrix."""
    assert_array(Sky().m, [
        [1, 0, 0],
        [0, 1, 0],
        [0, 0, 1],
    ])

    assert_array(Sky(ra=90).m, [
        [0, 1, 0],
        [-1, 0, 0],
        [0, 0, 1],
    ])

    assert_array(Sky(ra=180).m, [
        [-1, 0, 0],
        [0, -1, 0],
        [0, 0, 1],
    ])

    assert_array(Sky(ra=-90).m, [
        [0, -1, 0],
        [1, 0, 0],
        [0, 0, 1],
    ])

    assert_array(Sky(dec=90).m, [
        [0, 0, 1],
        [0, 1, 0],
        [-1, 0, 0],
    ])

    assert_array(Sky(dec=-90).m, [
        [0, 0, -1],
        [0, 1, 0],
        [1, 0, 0],
    ])

    assert_array(Sky(twist=90).m, [
        [1, 0, 0],
        [0, 0, 1],
        [0, -1, 0],
    ])

    assert_array(Sky(twist=-90).m, [
        [1, 0, 0],
        [0, 0, -1],
        [0, 1, 0],
    ])

    assert_array(Sky(twist=180).m, [
        [1, 0, 0],
        [0, -1, 0],
        [0, 0, -1],
    ])

    assert_array(Sky(ra=90, dec=90).m, [
        [0, 0, 1],
        [-1, 0, 0],
        [0, -1, 0],
    ])

    assert_array(Sky(ra=90, dec=90, twist=90).m, [
        [0, 0, 1],
        [0, -1, 0],
        [1, 0, 0],
    ])

def test_sky_xy(proj):
    """Test Sky projection in map projection."""
    assert_array(proj(0, 0), (0, 0))
    assert_array(proj(10, 0), (.176, 0), decimal=3)
    assert_array(proj(-10, 0), (-.176, 0), decimal=3)
    assert_array(proj(0, 10), (0, .176), decimal=3)
    assert_array(proj(0, -10), (0, -.176), decimal=3)

    assert_array(proj([0, 10, -10], 0), [
        [0, .176, -.176],
        [0, 0, 0],
    ], decimal=3)

    assert_array(proj(0, [0, 10, -10]), [
        [0, 0, 0],
        [0, .176, -.176],
    ], decimal=3)

    assert_array(proj([10, -10, 0, 0], [0, 0, 10, -10]), [
        [.176, -.176, 0, 0],
        [0, 0, .176, -.176],
    ], decimal=3)

    assert_array(proj([[10, -10], [0, 0]], [[0, 0], [10, -10]]), [
        [[.176, -.176], [0, 0]],
        [[0, 0], [.176, -.176]],
    ], decimal=3)

    with raises(ValueError):
        _ = proj([0], [0, 0])  # Invalid array size

    proj = Sky(ra=10)
    assert_array(proj(0, 0), (-.176, 0), decimal=3)
    assert_array(proj(10, 0), (0, 0))
    assert_array(proj(20, 0), (.176, 0), decimal=3)

    proj = Sky(dec=10)
    assert_array(proj(0, 0), (0, -.176), decimal=3)
    assert_array(proj(0, 10), (0, 0))
    assert_array(proj(0, 20), (0, .176), decimal=3)

    proj = Sky(twist=90)
    assert_array(proj(0, 0), (0, 0))
    assert_array(proj(10, 0), (0, -.176), decimal=3)
    assert_array(proj(0, 10), (.176, 0), decimal=3)

def test_sky_radec(proj):
    """Test Sky reverse projection in map projection."""
    assert_array(proj(0, 0, invert=True), (0, 0))
    assert_array(proj(.176, 0, invert=True), (10, 0), decimal=1)
    assert_array(proj(-.176, 0, invert=True), (350, 0), decimal=1)
    assert_array(proj(0, .176, invert=True), (0, 10), decimal=1)
    assert_array(proj(0, -.176, invert=True), (0, -10), decimal=1)

    assert_array(proj([0, .176, -.176], 0, invert=True), [
        [0, 10, 350],
        [0, 0, 0],
    ], decimal=1)

    assert_array(proj(0, [0, .176, -.176], invert=True), [
        [0, 0, 0],
        [0, 10, -10],
    ], decimal=1)

    assert_array(
        proj(
            [0, .176, -.176, 0, 0],
            [0, 0, 0, .176, -.176],
            invert=True,
        ),
        [
            [0, 10, 350, 0, 0],
            [0, 0, 0, 10, -10],
        ], decimal=1)

    assert_array(
        proj(
            [[.176, -.176], [0, 0]],
            [[0, 0], [.176, -.176]],
            invert=True,
        ),
        [
            [[10, 350], [0, 0]],
            [[0, 0], [10, -10]],
        ], decimal=1)

    with raises(ValueError):
        _ = proj([0], [0, 0], invert=True)  # Invalid array size

    proj = Sky(ra=10)
    assert_array(proj(-.176, 0, invert=True), (0, 0), decimal=1)
    assert_array(proj(0, 0, invert=True), (10, 0), decimal=1)
    assert_array(proj(.176, 0, invert=True), (20, 0), decimal=1)

    proj = Sky(dec=10)
    assert_array(proj(0, -.176, invert=True), (0, 0), decimal=1)
    assert_array(proj(0, 0, invert=True), (0, 10), decimal=1)
    assert_array(proj(0, .176, invert=True), (0, 20), decimal=1)

    proj = Sky(twist=90)
    assert_array(proj(0, 0, invert=True), (0, 0), decimal=1)
    assert_array(proj(0, -.176, invert=True), (10, 0), decimal=1)
    assert_array(proj(.176, 0, invert=True), (0, 10), decimal=1)

def test_sky_path(proj):
    """Test sky projection on Path."""
    path = proj(Path([
        (10, 0),
        (0, 10),
        (-10, 0),
        (0, -10),
    ]))

    assert len(path.vertices) == 5

    assert_array(path.vertices, [
        [.176, 0],
        [0, .176],
        [-.176, 0],
        [0, -.176],
        [.176, 0],  # Added
    ], decimal=3)

    assert_array(
        path.codes,
        [Path.MOVETO] + 3 * [Path.LINETO] + [Path.CLOSEPOLY]
    )

max_stars_repo_name: seignovert/pyvims

max_issues_repo_name: seignovert/pyvims

max_forks_repo_name: seignovert/pyvims

reason: from numpy


text:
##
# \file joint_image_mask_builder.py
# \brief      Build common mask from multiple, individual ones
#
# \author     Michael Ebner (michael.ebner.14@ucl.ac.uk)
# \date       November 2017
#

import SimpleITK as sitk
import numpy as np

import pysitk.simple_itk_helper as sitkh
import niftymic.base.stack as st

class JointImageMaskBuilder(object):

    def __init__(self,
                 stacks,
                 target,
                 dilation_radius=1,
                 dilation_kernel="Ball",
                 max_distance=50):

        self._stacks = stacks
        self._target = target
        self._dilation_radius = dilation_radius
        self._dilation_kernel = dilation_kernel
        self._max_distance = max_distance

        self._joint_image_mask = None

    def run(self):

        recon_space = self._target.get_isotropically_resampled_stack(
            extra_frame=self._max_distance,
        )
        mask_sitk = 0 * recon_space.sitk_mask
        dim = mask_sitk.GetDimension()

        for stack in self._stacks:
            stack_mask_sitk = sitk.Resample(
                stack.sitk_mask,
                mask_sitk,
                eval("sitk.Euler%dDTransform()" % dim),
                sitk.sitkNearestNeighbor,
                0,
                mask_sitk.GetPixelIDValue())
            mask_sitk += stack_mask_sitk

        thresholder = sitk.BinaryThresholdImageFilter()
        mask_sitk = thresholder.Execute(mask_sitk, 0, 0.5, 0, 1)

        if self._dilation_radius > 0:
            dilater = sitk.BinaryDilateImageFilter()
            dilater.SetKernelType(eval("sitk.sitk" + self._dilation_kernel))
            dilater.SetKernelRadius(self._dilation_radius)
            mask_sitk = dilater.Execute(mask_sitk)

        self._joint_image_mask = st.Stack.from_sitk_image(
            image_sitk=recon_space.sitk,
            image_sitk_mask=mask_sitk,
            filename=self._target.get_filename(),
            slice_thickness=recon_space.get_slice_thickness(),
        )

    def get_stack(self):
        return st.Stack.from_stack(self._joint_image_mask)

max_stars_repo_name: martaranzini/NiftyMIC

max_issues_repo_name: martaranzini/NiftyMIC

max_forks_repo_name: martaranzini/NiftyMIC


text:
import os
import argparse
import cv2
import imageio
import numpy as np
from scipy import linalg as la

# Parse arguments.
parser = argparse.ArgumentParser(description = "Evaluate predicted disparity maps.")
parser.add_argument("--image_path", dest = "image_path", help = "Path to the images.", required = True)
parser.add_argument("--sparse_path", dest = "sparse_path", help = "Path to the sparse reconstruction.", required = True)
parser.add_argument("--output_path", dest = "output_path", help = "Path to store the predicted results.", required = True)
parser.add_argument("--gt_path", dest = "gt_path", help = "Path to store the ground truth results. The gt depth filenames should be <image_name>.depth.npy", required = True)
parser.add_argument("--load_bin", dest = "load_bin", type = bool, default = False, help = "Set if you want to load COLMAP .bin files")
# TODO: Support --gt_type = "colmap_bin"
parser.add_argument("--gt_type", dest = "gt_type", choices = ["depth", "disparity"], default = "disparity", help = "Specify whether the ground truth depth files store depth or disparity (=1/depth).")
parser.add_argument("--output_type", dest = "output_type", choices = ["depth", "disparity"], default = "disparity", help = "Specify whether the predicted depth files store depth value or disparity (=1/depth).")
parser.add_argument("--image_width", dest = "image_width", type = int, help = "Image width (>0).", required = True)
parser.add_argument("--image_height", dest = "image_height", type = int, help = "Image height (>0).", required = True)
parser.add_argument("--skip_rephoto", dest = "skip_rephoto", action = "store_true", default = False, help = "Skip rephoto error to speed up evaluation.")
parser.add_argument("--store_rephoto", dest = "store_rephoto", action = "store_true", default = False, help = "Store the rephotography result using the predicted depths to <output_path>/rephoto.")
# Set border = 6 since COLMAP predicts unuseful values for the borders.
parser.add_argument("--border", dest = "border", type = int, default = 6, help = "Width of the borders to ignore from evaluation.")
parser.add_argument("--size_mismatch", dest = "size_mismatch", choices = ["throw", "crop_pad", "resize"], default = "throw",
	help =
"""Specify what to do if the size of depth maps do not match the specified image_width and image_height.
"throw": throw an error.
"crop_pad": crop or pad the depth maps.
"resize": resample the depth maps using nearest neighbor sampling.
"""
	)

args = parser.parse_args()

image_path = args.image_path
sparse_path = args.sparse_path
output_path = args.output_path
gt_path = args.gt_path
load_bin = args.load_bin
gt_type = args.gt_type
output_type = args.output_type
image_width = args.image_width
image_height = args.image_height
skip_rephoto = args.skip_rephoto
store_rephoto = args.store_rephoto
border = args.border
size_mismatch = args.size_mismatch

if args.load_bin:
	from colmap_helpers_for_bin import ColmapSparse
else:
	from colmap_helpers import ColmapSparse

# Crop or pad the depth map to specific size.
def crop_pad(img, w, h):
	img_w = img.shape[1]
	img_h = img.shape[0]
	if img_w > w:
		padding = (img_w - w) / 2
		img = img[:,padding:padding+w]
	elif img_w < w:
		padding_pre = (w - img_w) / 2
		padding_post = w - img_w - padding_pre
		img = np.pad(img, ((0,0), (padding_pre,padding_post)), "edge")
	if img_h > h:
		padding = (img_h - h) / 2
		img = img[padding:padding+h,:]
	elif img_h < h:
		padding_pre = (h - img_h) / 2
		padding_post = h - img_h - padding_pre
		img = np.pad(img, ((padding_pre,padding_post), (0,0)), "edge")
	return img

# Resize the depth map to specific size.
def resize(img, w, h):
	img = cv2.resize(img, (w, h), interpolation = cv2.INTER_NEAREST)
	return img

# Compute rephoto error.
def get_rephoto_diff(rephoto_path, sparse_model, frame_idx, predict_depth, mask = None):

	# Get reference image and camera pose.
	target_image = sparse_model.image_list.images[frame_idx]
	target_camera = sparse_model.camera_list.get_by_id(target_image.camera_id)
	num_neighbors = len(target_image.neighbor_list)

	# Ignore borders.
	gt_img = target_image.rgb
	if border > 0:
		gt_img = gt_img[border:-border, border:-border, :]
	input_width = gt_img.shape[1]
	input_height = gt_img.shape[0]

	# Store color candidates for each pixel.
	rephoto_volume = -np.ones((num_neighbors, input_height, input_width, 3), dtype = np.float32)

	# Loop through all neighbors.
	for (idx, neighbor_idx) in enumerate(target_image.neighbor_list):
		print("  Warping neighbor No. {:d}/{:d}".format(idx, num_neighbors))
		# Get neighbor image and camera pose.
		n_image = sparse_model.image_list.images[neighbor_idx]
		n_camera = sparse_model.camera_list.get_by_id(n_image.camera_id)
		rgb_dst = n_image.rgb
		trans_matrix = n_image.extrinsic.dot(la.inv(target_image.extrinsic))
		# Warp the neighbor image to reference view.
		coord_buffer = np.zeros((input_height, input_width, 4), dtype = np.float32)
		coord_buffer[..., 0] = (border + np.mgrid[0:input_height, 0:input_width][1] - target_camera.cx) / target_camera.fx
		coord_buffer[..., 1] = (border + np.mgrid[0:input_height, 0:input_width][0] - target_camera.cy) / target_camera.fy
		coord_buffer[..., 2] = 1.0
		coord_buffer[..., 3] = np.where(predict_depth == 0.0, 0.0, 1.0)
		coord_buffer[..., 0:3] = np.where(predict_depth[..., np.newaxis] == 0.0, coord_buffer[..., 0:3], coord_buffer[..., 0:3] / predict_depth[..., np.newaxis])
		coord_buffer[...] = np.moveaxis(trans_matrix.dot(coord_buffer[..., np.newaxis])[..., 0], 0, -1)
		x_map = np.where(coord_buffer[..., 2] > 0.0,
			coord_buffer[..., 0] / coord_buffer[..., 2] * n_camera.fx + n_camera.cx, -1)
		y_map = np.where(coord_buffer[..., 2] > 0.0,
			coord_buffer[..., 1] / coord_buffer[..., 2] * n_camera.fy + n_camera.cy, -1)
		cv2.remap(rgb_dst, x_map, y_map, cv2.INTER_LINEAR, rephoto_volume[idx, ...], cv2.BORDER_CONSTANT, (-1.0, -1.0, -1.0))

	# Select the median color for each pixel.
	rephoto_volume = np.sort(rephoto_volume, axis = 0)[::-1, ...]
	valid_count = num_neighbors - np.sum(rephoto_volume < 0.0, axis = 0)
	valid_mask = valid_count != 0
	chosen_indices = (valid_count) / 2
	rephoto_img = np.zeros((input_height, input_width, 3), dtype = np.float32)
	row_grid = np.mgrid[0:input_height, 0:input_width, 0:3][0]
	col_grid = np.mgrid[0:input_height, 0:input_width, 0:3][1]
	channel_grid = np.mgrid[0:input_height, 0:input_width, 0:3][2]
	rephoto_img = rephoto_volume[chosen_indices, row_grid, col_grid, channel_grid]
	rephoto_img[np.logical_not(valid_mask)] = 0.5

	# Compute rephoto error.
	valid_mask = valid_mask[..., 0]
	diff = np.sum(np.abs(gt_img - rephoto_img), axis = -1)
	rephoto_img = np.where(valid_mask[..., np.newaxis], rephoto_img, np.array([0.0, 1.0, 0.0])[np.newaxis, np.newaxis, :])

	# Apply masks if needed.
	if not mask is None:
		rephoto_img = np.where(mask[..., np.newaxis], rephoto_img, np.array([1.0, 0.0, 0.0])[np.newaxis, np.newaxis, :])
		valid_mask = np.logical_and(valid_mask, mask)

	# Store rephoto images if needed.
	if store_rephoto:
		output_dir = os.path.dirname(os.path.join(rephoto_path, target_image.filename))
		if not os.path.exists(output_dir):
			os.makedirs(output_dir)
		imageio.imwrite("{:}/{:}.rephoto.png".format(rephoto_path, target_image.filename), rephoto_img.clip(0.0, 1.0))
		imageio.imwrite("{:}/{:}.gt_rgb.png".format(rephoto_path, target_image.filename), gt_img.clip(0.0, 1.0))

	return diff[valid_mask]

# Load COLMAP sparse model.
print("Loading the sparse model...")
sparse_model = ColmapSparse(sparse_path, image_path, image_width, image_height)
print("Successfully loaded the sparse model.")

# Loop through all reference images.
errors_L1 = []
errors_rephoto = []
for (frame_idx, frame) in enumerate(sparse_model.image_list.images):
	print("Processing reference image No. {:d}/{:d}".format(frame_idx, sparse_model.image_list.length))

	# Load ground truth depths.
	gt_depth = np.load("{:}/{:}.depth.npy".format(gt_path, frame.filename))
	gt_depth = np.pad(gt_depth, ((0,0), (15,15)), "edge")
	if gt_depth.shape[0] != image_height or gt_depth.shape[1] != image_width:
		if size_mismatch == "throw":
			raise RuntimeError("Invalid size of gt_depth. gt_depth has size = {:} but the specified image size = ({:d}, {:d}).".format(gt_depth.shape, image_height, image_width))
		elif size_mismatch == "crop_pad":
			gt_depth = crop_pad(gt_depth, image_width, image_height)
		elif size_mismatch == "resize":
			gt_depth = resize(gt_depth, image_width, image_height)
		else:
			raise ValueError("size_mismatch is not supported")
	if border > 0:
		gt_depth = gt_depth[border:-border, border:-border]
	if gt_type == "depth":
		gt_valid = gt_depth > 0.0
		gt_depth = np.where(gt_valid, 1.0 / gt_depth, 0.0)
	elif gt_type == "disparity":
		# In ground truth, disparity = 0 represents invalid values.
		gt_valid = gt_depth > 0.0
	else:
		raise ValueError("gt_type is not supported")

	# Load predicted depths.
	output_depth = np.load("{:}/{:}.output.npy".format(output_path, frame.filename))
	if output_depth.shape[0] != image_height or output_depth.shape[1] != image_width:
		if size_mismatch == "throw":
			raise RuntimeError("Invalid size of output_depth. output_depth has size = {:} but the specified image size = ({:d}, {:d}).".format(output_depth.shape, image_height, image_width))
		elif size_mismatch == "crop_pad":
			output_depth = crop_pad(output_depth, image_width, image_height)
		elif size_mismatch == "resize":
			output_depth = resize(output_depth, image_width, image_height)
		else:
			raise ValueError("size_mismatch is not supported")
	if border > 0:
		output_depth = output_depth[border:-border, border:-border]
	if output_type == "depth":
		output_valid = output_depth > 0.0
		output_depth = np.where(output_valid, 1.0 / output_depth, 0.0)
	elif output_type == "disparity":
		# In predicted results, disparity = 0 is a valid value.
		output_valid = output_depth >= 0.0
	else:
		raise ValueError("output_type is not supported")

	# Compute L1 error.
	valid_mask = np.logical_and(gt_valid, output_valid)
	error_L1 = np.abs(output_depth - gt_depth)[valid_mask]
	errors_L1.extend(error_L1.flatten().tolist())

	# Compute rephoto error.
	if not skip_rephoto:
		error_rephoto = get_rephoto_diff(os.path.join(output_path, "rephoto"), sparse_model, frame_idx, output_depth, output_valid)
		errors_rephoto.extend(error_rephoto.tolist())

# Report errors:
mean_L1 = np.mean(errors_L1)
print("Disparity L1 error = {:f}, number of valid pixels = {:d}".format(mean_L1, len(errors_L1)))
if not skip_rephoto:
	mean_rephoto = np.mean(errors_rephoto)
	print("Rephoto error = {:f}, number of valid pixels = {:d}".format(mean_rephoto, len(errors_rephoto)))

max_stars_repo_name: KhaledSharif/DeepMVS

max_issues_repo_name: KhaledSharif/DeepMVS

max_forks_repo_name: KhaledSharif/DeepMVS

reason: import numpy,from scipy


text:
__author__ = "Afiq Harith"
__email__ = "afiqharith05@gmail.com"
__date__ = "08 Oct 2020"
__status__ = "Development-OOP"

import cv2
import numpy as np
import math
import os
from setup.LoadModel import LoadModel
from setup.config import *

# Load video
VIDEOPATH = os.path.join(os.getcwd(), FOLDERNAME, VIDEONAME)

class SODV:
    def __init__(self, VIDEOPATH, DISTANCE, CAMERA, START = True):

        if CAMERA == True:
            self.video = cv2.VideoCapture(0)
        else:
            self.video = cv2.VideoCapture(VIDEOPATH)

        self.distance = DISTANCE

        if START == True:
            self.main()

    def calculateCentroid(self, xmin, ymin, xmax, ymax):
        xmid = ((xmax + xmin)/2)
        ymid = ((ymax + ymin)/2)
        centroid = (xmid, ymid)

        return xmid, ymid, centroid

    def get_distance(self, x1, x2, y1, y2):
        distance = math.sqrt((x1-x2)**2 + (y1-y2)**2)

        return distance

    def draw_detection_box(self, frame, x1, y1, x2, y2, color):
        cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2)

    def main(self):
        net, output_layers, classes = LoadModel.get()

        while (self.video.isOpened()):

            stat_H, stat_L = 0, 0
            centroids = []
            box_colors = []
            detectedBox = []

            self.ret, self.frame = self.video.read()

            if self.ret:
                frame_resized = cv2.resize(self.frame, (416, 416)) # resize frame for prediction
            else:
                break
            frame_rgb = cv2.cvtColor(self.frame, cv2.IMREAD_COLOR)
            height, width, channels = self.frame.shape

            # Detecting objects
            blob = cv2.dnn.blobFromImage(frame_resized, 0.00392, (416, 416), (0, 0, 0), True, crop=False)

            net.setInput(blob)
            outs = net.forward(output_layers)

            class_ids = []
            confidences = []
            boxes = []
            for out in outs:
                for detection in out:
                    scores = detection[5:]
                    class_id = np.argmax(scores)
                    confidence = scores[class_id]

                    #if prediction is 50% and class id is 0 which is 'person'
                    if confidence > 0.5 and class_id == 0:

                        # Object detected
                        center_x = int(detection[0] * width)
                        center_y = int(detection[1] * height)
                        w = int(detection[2] * width)
                        h = int(detection[3] * height)

                        # Rectangle coordinates
                        x = int(center_x - w / 2)
                        y = int(center_y - h / 2)

                        boxes.append([x, y, w, h])
                        confidences.append(float(confidence))
                        class_ids.append(class_id)

            # apply non-max suppression
            indexes = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)

            for i in range(len(boxes)):
                if i in indexes:
                    x, y, w, h = boxes[i]

                    xmin = x
                    ymin = y
                    xmax = (x + w)
                    ymax = (y + h)

                    #calculate centroid point for bounding boxes
                    xmid, ymid, centroid = self.calculateCentroid(xmin, ymin, xmax, ymax)
                    detectedBox.append([xmin, ymin, xmax, ymax, centroid])

                    my_color = 0
                    for k in range (len(centroids)):
                        c = centroids[k]

                        if self.get_distance(c[0], centroid[0], c[1], centroid[1]) <= self.distance:
                            box_colors[k] = 1
                            my_color = 1
                            cv2.line(self.frame, (int(c[0]), int(c[1])), (int(centroid[0]), int(centroid[1])), YELLOW, 1,cv2.LINE_AA)
                            cv2.circle(self.frame, (int(c[0]), int(c[1])), 3, ORANGE, -1,cv2.LINE_AA)
                            cv2.circle(self.frame, (int(centroid[0]), int(centroid[1])), 3, ORANGE, -1,cv2.LINE_AA)
                            break
                    centroids.append(centroid)
                    box_colors.append(my_color)

            for i in range (len(detectedBox)):
                x1 = detectedBox[i][0]
                y1 = detectedBox[i][1]
                x2 = detectedBox[i][2]
                y2 = detectedBox[i][3]

                #for ellipse output
                xc = ((x2+x1)/2)
                yc = y2-5
                centroide = (int(xc),int(yc))

                if box_colors[i] == 0:
                    self.draw_detection_box(self.frame, x1, y1, x2, y2, WHITE)
                    label = "Low-Risk"
                    labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)

                    y1label = max(y1, labelSize[1])
                    cv2.rectangle(self.frame, (x1, y1label - labelSize[1]), (x1 + labelSize[0], y1 + baseLine), WHITE, cv2.FILLED)
                    cv2.putText(self.frame, label, (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 0.5, GREEN, 1,cv2.LINE_AA)
                    stat_L += 1

                else:
                    self.draw_detection_box(self.frame, x1, y1, x2, y2, RED)
                    label = "High-Risk"
                    labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)

                    y1label = max(y1, labelSize[1])
                    cv2.rectangle(self.frame, (x1, y1label - labelSize[1]),(x1 + labelSize[0], y1 + baseLine), WHITE, cv2.FILLED)
                    cv2.putText(self.frame, label, (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 0.5, ORANGE, 1,cv2.LINE_AA)
                    stat_H += 1

            cv2.rectangle(self.frame, (13, 10),(250, 60), GREY, cv2.FILLED)
            LINE = "--"
            INDICATION_H = f'HIGH RISK: {str(stat_H)} people'
            cv2.putText(self.frame, LINE, (30,30), cv2.FONT_HERSHEY_SIMPLEX, 0.5, RED, 1,cv2.LINE_AA)
            cv2.putText(self.frame, INDICATION_H, (60,30), cv2.FONT_HERSHEY_SIMPLEX, 0.5, BLUE, 1,cv2.LINE_AA)

            INDICATION_L = f'LOW RISK : {str(stat_L)} people'
            cv2.putText(self.frame, LINE, (30,50), cv2.FONT_HERSHEY_SIMPLEX, 0.5, WHITE, 1,cv2.LINE_AA)
            cv2.putText(self.frame, INDICATION_L, (60,50), cv2.FONT_HERSHEY_SIMPLEX, 0.5, BLUE, 1,cv2.LINE_AA)

            cv2.imshow("SODV", self.frame)

            if cv2.waitKey(1) >= 0:
                break

        self.video.release()

if __name__ == '__main__':
    SODV(VIDEOPATH, DISTANCE, CAMERA)
    cv2.destroyAllWindows()


text:
import os
from PIL import Image
import numpy as np
import requests
from io import BytesIO

def map_url(bodyName, mapType):
    mapDirURL = "https://raw.githubusercontent.com/theastrogoth/KSP-Trajectory-Illustrator/master/assets/images/"

    return mapDirURL + str(bodyName) + str(mapType) + '.png'

def combine_tiles(path=None):

    if not path is None:
        os.chdir(path)

    for path, directories, files in os.walk(path):
        if 'Tile0000.png' in files and 'Tile0001.png' in files:

            images = [Image.open(x) for x in [os.path.join(path,'Tile0000.png'), os.path.join(path,'Tile0001.png')]]
            widths, heights = zip(*(i.size for i in images))

            total_width = sum(widths)
            max_height = max(heights)

            new_im = Image.new('RGB', (total_width, max_height))

            x_offset = 0
            for im in images:
              new_im.paste(im, (x_offset,0))
              x_offset += im.size[0]

            new_im.save(os.path.join(path,'combined.png'))

def make_small_image(path=None):

    if not path is None:
        os.chdir(path)

    for path, directories, files in os.walk(path):
        for file in files:
            if file[-4:] == '.png':
                image = Image.open(os.path.join(path, file))
                width, height = image.size

                if width==2048 and height==1024:
                    image = image.convert('RGB')
                    newImage = image.resize((512, 512))
                    pix = image_colormap(list(newImage.getdata()))[2]
                    newImage.putdata(pix)
                    newImage.save(os.path.join(path,file[:-4]+'Small.png'))

def round_colors(pix, roundVal):

    newPix = []
    for pick in pix:
        rgb = list(pick)
        for jj, cl in enumerate(pick):
            # rgb[jj] = cl - cl%roundVal + roundVal/2
            rgb[jj] = round(roundVal * round(cl/roundVal))
        newPix.append(tuple(rgb))

    return newPix

def get_pixel_values(imagePath, url=True):
    if url:
        response = requests.get(imagePath)
        image = Image.open(BytesIO(response.content))
    else:
        image = Image.open(imagePath)
    image = image.convert('RGB')
    pix = list(image.getdata())
    width, height = (image.size)
    return pix, width, height

def pixel_to_grayscale_value(r=None,g=None,b=None, tup=None):

    if not tup is None:
        r = tup[0]
        g = tup[1]
        b = tup[2]

    return (r+b+g)/3

def image_colormap(pix, roundNum=85, rounded=False):

    if not rounded:
        numUnique = 999
        # plotly seems to fail if the colormap has >~100 intervals
        while numUnique > 100:
            newPix = round_colors(pix, 255/roundNum)

            uniqueColors = np.unique(newPix, axis=0)
            numUnique = len(uniqueColors)
            if roundNum <= 5:
                roundNum = roundNum-1
            else:
                roundNum = roundNum - round(roundNum/10)
    else:
        newPix = pix
        uniqueColors = np.unique(pix, axis=0)
        numUnique = len(uniqueColors)

    breaks = np.linspace(0,1,numUnique+1)
    interval = breaks[1]
    mapVal = 0

    colorMap = [[0, 'rgb'+str(tuple(uniqueColors[0]))]]
    mapDict = {}
    mapDict[tuple(uniqueColors[0])] = interval/2
    for ii in range(numUnique):
        colorMap.append([mapVal, 'rgb'+str(tuple(uniqueColors[ii]))])
        mapDict[tuple(uniqueColors[ii])] = mapVal+interval/2
        mapVal = mapVal + interval
        if ii == numUnique-1:
            mapVal = 1
        colorMap.append([mapVal, 'rgb'+str(tuple(uniqueColors[ii]))])

    return colorMap, mapDict, newPix

max_stars_repo_name: theastrogoth/KSP-Trajectory-Illustrator

max_issues_repo_name: theastrogoth/KSP-Transfer-Illustrator

max_forks_repo_name: theastrogoth/KSP-Trajectory-Illustrator