text:
"""
Sage Runtime Environment

AUTHORS:

- \R. Andrew Ohana (2012): Initial version.

"""
########################################################################
#       Copyright (C) 2013 R. Andrew Ohana <andrew.ohana@gmail.com>
#
#  Distributed under the terms of the GNU General Public License (GPL)
#  as published by the Free Software Foundation; either version 2 of
#  the License, or (at your option) any later version.
#
#                  http://www.gnu.org/licenses/
########################################################################
from __future__ import absolute_import

import glob
import os
import socket
import site
from . import version

opj = os.path.join

# set default values for sage environment variables
# every variable can be overwritten by os.environ
SAGE_ENV = dict()

# Helper to build the SAGE_ENV dictionary
def _add_variable_or_fallback(key, fallback, force=False):
    """
    Set ``SAGE_ENV[key]``.

    If ``key`` is an environment variable, this is the
    value. Otherwise, the ``fallback`` is used.

    INPUT:

    - ``key`` -- string.

    - ``fallback`` -- anything.

    - ``force`` -- boolean (optional, default is ``False``). Whether
      to always use the fallback, regardless of environment variables.

    EXAMPLES::

        sage: import os, sage.env
        sage: sage.env.SAGE_ENV = dict()
        sage: os.environ['SAGE_FOO'] = 'foo'
        sage: sage.env._add_variable_or_fallback('SAGE_FOO', '---$SAGE_URL---')
        sage: sage.env.SAGE_FOO
        'foo'
        sage: sage.env.SAGE_ENV['SAGE_FOO']
        'foo'

    If the environment variable does not exist, the fallback is
    used. Previously-declared variables are replaced if they are
    prefixed with a dollar sign::

        sage: _ = os.environ.pop('SAGE_BAR', None)  # ensure that SAGE_BAR does not exist
        sage: sage.env._add_variable_or_fallback('SAGE_BAR', '---$SAGE_FOO---')
        sage: sage.env.SAGE_BAR
        '---foo---'
        sage: sage.env.SAGE_ENV['SAGE_BAR']
        '---foo---'

    Test that :trac:`23758` has been resolved::

        sage: sage.env._add_variable_or_fallback('SAGE_BA', '---hello---')
        sage: sage.env._add_variable_or_fallback('SAGE_QUX', '$SAGE_BAR')
        sage: sage.env.SAGE_ENV['SAGE_QUX']
        '---foo---'
    """
    global SAGE_ENV
    import six
    try:
        import os
        value = os.environ[key]
    except KeyError:
        value = fallback
    if force:
        value = fallback
    if isinstance(value, six.string_types):
        # Now do the variable replacement. First treat 'value' as if
        # it were a path and do the substitution on each of the
        # components. This is to avoid the sloppiness in the second
        # round of substitutions: if VAR and VAR_NEW are both in
        # SAGE_ENV, then when doing substitution on the string
        # "$VAR_NEW/a/b", we want to match VAR_NEW, not VAR, if
        # possible.
        for sep in set([os.path.sep, '/']):
            components = []
            for s in value.split(sep):
                if s.startswith('$'):
                    components.append(SAGE_ENV.get(s[1:], s))
                else:
                    components.append(s)
            value = sep.join(components)
        # Now deal with any remaining substitutions. The following is
        # sloppy, as mentioned above: if $VAR and $VAR_NEW are both in
        # SAGE_ENV, the substitution for "$VAR_NEw" depends on which
        # of the two appears first when iterating over
        # SAGE_ENV.items().
        for k,v in SAGE_ENV.items():
            if isinstance(v, six.string_types):
                value = value.replace('$'+k, v)
    SAGE_ENV[key] = value
    globals()[key] = value

# system info
_add_variable_or_fallback('UNAME',           os.uname()[0])
_add_variable_or_fallback('HOSTNAME',        socket.gethostname())
_add_variable_or_fallback('LOCAL_IDENTIFIER','$HOSTNAME.%s'%os.getpid())

# bunch of sage directories and files
_add_variable_or_fallback('SAGE_ROOT',       None)
_add_variable_or_fallback('SAGE_LOCAL',      None)
_add_variable_or_fallback('SAGE_ETC',        opj('$SAGE_LOCAL', 'etc'))
_add_variable_or_fallback('SAGE_INC',        opj('$SAGE_LOCAL', 'include'))
_add_variable_or_fallback('SAGE_SHARE',      opj('$SAGE_LOCAL', 'share'))

_add_variable_or_fallback('SAGE_SRC',        opj('$SAGE_ROOT', 'src'))

try:
    sitepackages_dirs = site.getsitepackages()
except AttributeError:  # in case of use inside virtualenv
    sitepackages_dirs = [os.path.join(os.path.dirname(site.__file__),
                                     'site-packages')]
_add_variable_or_fallback('SITE_PACKAGES',   sitepackages_dirs)

_add_variable_or_fallback('SAGE_LIB',        SITE_PACKAGES[0])

# Used by sage/misc/package.py.  Should be SAGE_SRC_ROOT in VPATH.
_add_variable_or_fallback('SAGE_PKGS', opj('$SAGE_ROOT', 'build', 'pkgs'))

_add_variable_or_fallback('SAGE_EXTCODE',    opj('$SAGE_SHARE', 'sage', 'ext'))
_add_variable_or_fallback('SAGE_LOGS',       opj('$SAGE_ROOT', 'logs', 'pkgs'))
_add_variable_or_fallback('SAGE_SPKG_INST',  opj('$SAGE_LOCAL', 'var', 'lib', 'sage', 'installed'))
_add_variable_or_fallback('SAGE_DOC_SRC',    opj('$SAGE_SRC', 'doc'))
_add_variable_or_fallback('SAGE_DOC',        opj('$SAGE_SHARE', 'doc', 'sage'))
_add_variable_or_fallback('DOT_SAGE',        opj(os.environ.get('HOME','$SAGE_ROOT'), '.sage'))
_add_variable_or_fallback('SAGE_DOT_GIT',    opj('$SAGE_ROOT', '.git'))
_add_variable_or_fallback('SAGE_DISTFILES',  opj('$SAGE_ROOT', 'upstream'))

# misc
_add_variable_or_fallback('SAGE_URL',                'http://sage.math.washington.edu/sage/')
_add_variable_or_fallback('REALM',                   'sage.math.washington.edu')
_add_variable_or_fallback('TRAC_SERVER_URI',         'https://trac.sagemath.org')
_add_variable_or_fallback('SAGE_REPO_AUTHENTICATED', 'ssh://git@trac.sagemath.org:2222/sage.git')
_add_variable_or_fallback('SAGE_REPO_ANONYMOUS',     'git://trac.sagemath.org/sage.git')
_add_variable_or_fallback('SAGE_VERSION',            version.version)
_add_variable_or_fallback('SAGE_DATE',               version.date)
_add_variable_or_fallback('SAGE_VERSION_BANNER',     version.banner)
_add_variable_or_fallback('SAGE_BANNER',             '')
_add_variable_or_fallback('SAGE_IMPORTALL',          'yes')

# additional packages locations
_add_variable_or_fallback('CONWAY_POLYNOMIALS_DATA_DIR',  opj('$SAGE_SHARE','conway_polynomials'))
_add_variable_or_fallback('GRAPHS_DATA_DIR',  opj('$SAGE_SHARE','graphs'))
_add_variable_or_fallback('ELLCURVE_DATA_DIR',opj('$SAGE_SHARE','ellcurves'))
_add_variable_or_fallback('POLYTOPE_DATA_DIR',opj('$SAGE_SHARE','reflexive_polytopes'))
_add_variable_or_fallback('GAP_ROOT_DIR',     opj('$SAGE_LOCAL','gap','latest'))
_add_variable_or_fallback('THEBE_DIR',        opj('$SAGE_SHARE','thebe'))

# locate singular shared object
if UNAME[:6] == "CYGWIN":
    SINGULAR_SO = ([None] + glob.glob(os.path.join(
        SAGE_LOCAL, "bin", "cygSingular-*.dll")))[-1]
else:
    if UNAME == "Darwin":
        extension = "dylib"
    else:
        extension = "so"
    # library name changed from libsingular to libSingular btw 3.x and 4.x
    SINGULAR_SO = SAGE_LOCAL+"/lib/libSingular."+extension

_add_variable_or_fallback('SINGULAR_SO', SINGULAR_SO)

if not SINGULAR_SO or not os.path.exists(SINGULAR_SO):
    raise RuntimeError(
        "libSingular not found--a working Singular install in $SAGE_LOCAL "
        "is required for Sage to work")

# post process
if ' ' in DOT_SAGE:
    if UNAME[:6] == 'CYGWIN':
        # on windows/cygwin it is typical for the home directory
        # to have a space in it.  Fortunately, users also have
        # write privileges to c:\cygwin\home, so we just put
        # .sage there.
        _add_variable_or_fallback('DOT_SAGE', "/home/.sage", force=True)
    else:
        print("Your home directory has a space in it.  This")
        print("will probably break some functionality of Sage.  E.g.,")
        print("the GAP interface will not work. A workaround")
        print("is to set the environment variable HOME to a")
        print("directory with no spaces that you have write")
        print("permissions to before you start sage.")

CYGWIN_VERSION = None
if UNAME[:6] == 'CYGWIN':
    import re
    _uname = os.uname()
    if len(_uname) >= 2:
        m = re.match(r'(\d+\.\d+\.\d+)\(.+\)', _uname[2])
        if m:
            CYGWIN_VERSION = tuple(map(int, m.group(1).split('.')))

        del m
    del _uname, re

# things that need DOT_SAGE
_add_variable_or_fallback('PYTHON_EGG_CACHE',   opj('$DOT_SAGE', '.python-eggs'))
_add_variable_or_fallback('SAGE_STARTUP_FILE',  opj('$DOT_SAGE', 'init.sage'))

# delete temporary variables used for setting up sage.env
del opj, os, socket, version, site

def sage_include_directories(use_sources=False):
    """
    Return the list of include directories for compiling Sage extension modules.

    INPUT:

    -  ``use_sources`` -- (default: False) a boolean

    OUTPUT:

    a list of include directories to be used to compile sage code
    1. while building sage (use_sources='True')
    2. while using sage (use_sources='False')

    EXAMPLES:

    Expected output while using sage

    ::

        sage: import sage.env
        sage: sage.env.sage_include_directories()
        ['.../include',
        '.../include/python...',
        '.../python.../numpy/core/include',
        '.../python.../site-packages',
        '.../python.../site-packages/sage/ext']
    """
    import os, numpy
    import distutils.sysconfig

    opj = os.path.join

    include_directories = [SAGE_INC,
                           distutils.sysconfig.get_python_inc(),
                           numpy.get_include()]

    if use_sources :
        include_directories.extend([SAGE_SRC,
                                    opj(SAGE_SRC, 'sage', 'ext')])
    else:
        include_directories.extend([SAGE_LIB,
                                    opj(SAGE_LIB, 'sage', 'ext')])

    return include_directories

def cython_aliases():
    """
    Return the aliases for compiling Cython code. These aliases are
    macros which can occur in ``# distutils`` headers.

    EXAMPLES::

        sage: from sage.env import cython_aliases
        sage: cython_aliases()
        {...}
        sage: sorted(cython_aliases().keys())
        ['FFLASFFPACK_CFLAGS',
         'FFLASFFPACK_INCDIR',
         'FFLASFFPACK_LIBDIR',
         'FFLASFFPACK_LIBRARIES',
         'GIVARO_CFLAGS',
         'GIVARO_INCDIR',
         'GIVARO_LIBDIR',
         'GIVARO_LIBRARIES',
         'GSL_CFLAGS',
         'GSL_INCDIR',
         'GSL_LIBDIR',
         'GSL_LIBRARIES',
         'LINBOX_CFLAGS',
         'LINBOX_INCDIR',
         'LINBOX_LIBDIR',
         'LINBOX_LIBRARIES',
         'SINGULAR_CFLAGS',
         'SINGULAR_INCDIR',
         'SINGULAR_LIBDIR',
         'SINGULAR_LIBRARIES']
    """
    import pkgconfig

    aliases = {}

    for lib in ['fflas-ffpack', 'givaro', 'gsl', 'linbox', 'Singular']:
        var = lib.upper().replace("-", "") + "_"
        aliases[var + "CFLAGS"] = pkgconfig.cflags(lib).split()
        pc = pkgconfig.parse(lib)
        # INCDIR should be redundant because the -I options are also
        # passed in CFLAGS
        aliases[var + "INCDIR"] = pc['include_dirs']
        aliases[var + "LIBDIR"] = pc['library_dirs']
        aliases[var + "LIBRARIES"] = pc['libraries']

    # LinBox needs special care because it actually requires C++11 with
    # GNU extensions: -std=c++11 does not work, you need -std=gnu++11
    # (this is true at least with GCC 7.2.0).
    #
    # Further, note that LinBox does not add any C++11 flag in its .pc
    # file (possibly because of confusion between CFLAGS and CXXFLAGS?).
    # This is not a problem in practice since LinBox depends on
    # fflas-ffpack and fflas-ffpack does add such a C++11 flag.
    aliases["LINBOX_CFLAGS"].append("-std=gnu++11")

    return aliases

max_stars_repo_name: rekhabiswal/sage

max_issues_repo_name: rekhabiswal/sage

max_forks_repo_name: rekhabiswal/sage

reason: import sage,from sage


text:
import numpy as np

pallet_mine = np.array([[[  0,  0,  0],
                    [255,255,255],
                    [255,200, 20],#2   [0,  0,  255]
                    [  0,255,  0],#3
                    [  255,  0,0],#4
                    [255,255,  0],
                    [  0,255,255],
                    [255,  0,255],
                    [128,  0,  0],
                    [  0,128,  0],
                    [  0,  0,128],
                    [128,128,  0],
                    [  0,128,128],
                    [128,  0,128],
                    [128,128,128],
                    [255,128,128],
                    [152, 251, 152],#10
                    [ 70, 130, 180],#11
                    [220,  20,  60],#12
                    [  0,   0, 142],#14
                    [  0,   0,  70],#15
                    [  0,  60, 100],#16
                    [  0,  80, 100],#17
                    [  0,   0, 230],#18
                    [119,  11,  32],#19
                    [128,  64, 128],#20
                    [255, 255, 255]]], np.uint8)/255
#%%
grassclover_class_names = np.array(['soil(BG)','clover','grass','weeds','white_clover','red_clover','dandelion','shepherds_purse','thistle',
                   'white_clover_flower','white_clover_leaf','red_clover_flower','red_clover_leaf','unknown_clover_leaf'
                   ,'unknown_clover_flower', 'Mean IoU']).reshape((-1,1))
#%%
lip_class_names = ['Background','Hat','Hair', 'Glove','Sunglasses','Upper_clothes','Dress','Coat','Socks','Pants',
                   'Jumpsuits','Scarf','Skirt', 'Face','Left_arm','Right_arm', 'Left_leg','Right_leg','Left_shoe','Right_shoe']
#%%
pallet_cityscape = np.array([[[128, 64, 128],
                            [244, 35, 232],
                            [70, 70, 70],
                            [102, 102, 156],
                            [190, 153, 153],
                            [153, 153, 153],
                            [250, 170, 30],
                            [220, 220, 0],
                            [107, 142, 35],
                            [152, 251, 152],
                            [70, 130, 180],
                            [220, 20, 60],
                            [255, 0, 0],
                            [0, 0, 142],
                            [0, 0, 70],
                            [0, 60, 100],
                            [0, 80, 100],
                            [0, 0, 230],
                            [119, 11, 32]]], np.uint8) / 255

cityscape_THING_LIST = [11, 12, 13, 14, 15, 16, 17, 18]

cityscape_class_names = ['road', 'sidewalk', 'building', 'wall', 'fence', 'pole',
                'traffic light', 'traffic sign',
                'vegetation', 'terrain', 'sky', 'person', 'rider', 'car',
                'truck', 'bus', 'train', 'motorcycle', 'bicycle']
#%%
pallet_VOC = np.array([[[0, 0, 0],
                        [0, 0, 128],
                        [0, 128, 0],
                        [0, 128, 128],
                        [128, 0, 0],
                        [128, 0, 128],
                        [128, 128, 0],
                        [128, 128, 128],
                        [0, 0, 64],
                        [0, 0, 192],
                        [0, 128, 64],
                        [0, 128, 192],
                        [128, 0, 64],
                        [128, 0, 192],
                        [128, 128, 64],
                        [128, 128, 192],
                        [0, 64, 0],
                        [0, 64, 128],
                        [0, 192, 0],
                        [0, 192, 128],
                        [128, 64, 0]]],np.uint8) / 255
VOC_class_names = ['background', 'aeroplane', 'bicycle', 'bird',
                'boat',
                'bottle', 'bus', 'car', 'cat', 'chair', 'cow',
                'diningtable',
                'dog', 'horse', 'motorbike', 'person',
                'pottedplant',
                'sheep', 'sofa', 'train', 'tv/monitor']
#%%
vistas_THING_LIST = [0,1,8,19,20,21,22,23,32,33,34,35,36,37,38,39,40,41,42,44,
                      45,46,47,48,49,50,51,52,53,54,55,56,57,59,60,61,62]

vistas_class_names = ["bird","ground-animal","curb","fence","guard-rail","other-barrier","wall",
                "bike-lane","crosswalk-plain","curb-cut","parking","pedestrian-area","rail-track","road",
                "service-lane","sidewalk","bridge","building","tunnel","person","bicyclist","motorcyclist",
                "other-rider","crosswalk-zebra","general","mountain","sand","sky","snow","terrain","vegetation",
                "water","banner","bench","bike-rack","billboard","catch-basin","cctv-camera","fire-hydrant",
                "junction-box","mailbox","manhole","phone-booth","pothole","street-light","pole","traffic-sign-frame",
                "utility-pole","traffic-light","back","front","trash-can","bicycle","boat","bus","car",
                "caravan","motorcycle","on-rails","other-vehicle","trailer","truck",
                "wheeled-slow","car-mount","ego-vehicle","unlabeled"]
pallet_vistas = np.array([[[165, 42, 42],
                  [0, 192, 0],
                  [196, 196, 196],
                  [190, 153, 153],
                  [180, 165, 180],
                  [102, 102, 156],
                  [102, 102, 156],
                  [128, 64, 255],
                  [140, 140, 200],
                  [170, 170, 170],
                  [250, 170, 160],
                  [96, 96, 96],
                  [230, 150, 140],
                  [128, 64, 128],
                  [110, 110, 110],
                  [244, 35, 232],
                  [150, 100, 100],
                  [70, 70, 70],
                  [150, 120, 90],
                  [220, 20, 60],
                  [255, 0, 0],
                  [255, 0, 0],
                  [255, 0, 0],
                  [200, 128, 128],
                  [255, 255, 255],
                  [64, 170, 64],
                  [128, 64, 64],
                  [70, 130, 180],
                  [255, 255, 255],
                  [152, 251, 152],
                  [107, 142, 35],
                  [0, 170, 30],
                  [255, 255, 128],
                  [250, 0, 30],
                  [0, 0, 0],
                  [220, 220, 220],
                  [170, 170, 170],
                  [222, 40, 40],
                  [100, 170, 30],
                  [40, 40, 40],
                  [33, 33, 33],
                  [170, 170, 170],
                  [0, 0, 142],
                  [170, 170, 170],
                  [210, 170, 100],
                  [153, 153, 153],
                  [128, 128, 128],
                  [0, 0, 142],
                  [250, 170, 30],
                  [192, 192, 192],
                  [220, 220, 0],
                  [180, 165, 180],
                  [119, 11, 32],
                  [0, 0, 142],
                  [0, 60, 100],
                  [0, 0, 142],
                  [0, 0, 90],
                  [0, 0, 230],
                  [0, 80, 100],
                  [128, 64, 64],
                  [0, 0, 110],
                  [0, 0, 70],
                  [0, 0, 192],
                  [32, 32, 32],
                  [0, 0, 0],
                  [0, 0, 0]]], np.uint8) / 255
#%%
pallet_ADE20K = np.array([[[0, 0, 0],
      [120, 120, 120],
      [180, 120, 120],
      [6, 230, 230],
      [80, 50, 50],
      [4, 200, 3],
      [120, 120, 80],
      [140, 140, 140],
      [204, 5, 255],
      [184, 255, 0],
      [4, 250, 7],
      [224, 5, 255],
      [235, 255, 7],
      [150, 5, 61],
      [120, 120, 70],
      [8, 255, 51],
      [255, 6, 82],
      [143, 255, 140],
      [204, 255, 4],
      [255, 51, 7],
      [204, 70, 3],
      [0, 102, 200],
      [61, 230, 250],
      [255, 6, 51],
      [11, 102, 255],
      [255, 7, 71],
      [255, 9, 224],
      [9, 7, 230],
      [220, 220, 220],
      [255, 9, 92],
      [112, 9, 255],
      [8, 255, 214],
      [7, 255, 224],
      [255, 184, 6],
      [10, 255, 71],
      [255, 41, 10],
      [7, 255, 255],
      [224, 255, 8],
      [102, 8, 255],
      [255, 61, 6],
      [255, 194, 7],
      [255, 122, 8],
      [0, 255, 20],
      [255, 8, 41],
      [255, 5, 153],
      [6, 51, 255],
      [235, 12, 255],
      [160, 150, 20],
      [0, 163, 255],
      [140, 140, 140],
      [250, 10, 15],
      [20, 255, 0],
      [31, 255, 0],
      [255, 31, 0],
      [255, 224, 0],
      [153, 255, 0],
      [0, 0, 255],
      [255, 71, 0],
      [0, 235, 255],
      [0, 173, 255],
      [31, 0, 255],
      [11, 200, 200],
      [255, 82, 0],
      [0, 255, 245],
      [0, 61, 255],
      [0, 255, 112],
      [0, 255, 133],
      [255, 0, 0],
      [255, 163, 0],
      [255, 102, 0],
      [194, 255, 0],
      [0, 143, 255],
      [51, 255, 0],
      [0, 82, 255],
      [0, 255, 41],
      [0, 255, 173],
      [10, 0, 255],
      [173, 255, 0],
      [0, 255, 153],
      [255, 92, 0],
      [255, 0, 255],
      [255, 0, 245],
      [255, 0, 102],
      [255, 173, 0],
      [255, 0, 20],
      [255, 184, 184],
      [0, 31, 255],
      [0, 255, 61],
      [0, 71, 255],
      [255, 0, 204],
      [0, 255, 194],
      [0, 255, 82],
      [0, 10, 255],
      [0, 112, 255],
      [51, 0, 255],
      [0, 194, 255],
      [0, 122, 255],
      [0, 255, 163],
      [255, 153, 0],
      [0, 255, 10],
      [255, 112, 0],
      [143, 255, 0],
      [82, 0, 255],
      [163, 255, 0],
      [255, 235, 0],
      [8, 184, 170],
      [133, 0, 255],
      [0, 255, 92],
      [184, 0, 255],
      [255, 0, 31],
      [0, 184, 255],
      [0, 214, 255],
      [255, 0, 112],
      [92, 255, 0],
      [0, 224, 255],
      [112, 224, 255],
      [70, 184, 160],
      [163, 0, 255],
      [153, 0, 255],
      [71, 255, 0],
      [255, 0, 163],
      [255, 204, 0],
      [255, 0, 143],
      [0, 255, 235],
      [133, 255, 0],
      [255, 0, 235],
      [245, 0, 255],
      [255, 0, 122],
      [255, 245, 0],
      [10, 190, 212],
      [214, 255, 0],
      [0, 204, 255],
      [20, 0, 255],
      [255, 255, 0],
      [0, 153, 255],
      [0, 41, 255],
      [0, 255, 204],
      [41, 0, 255],
      [41, 255, 0],
      [173, 0, 255],
      [0, 245, 255],
      [71, 0, 255],
      [122, 0, 255],
      [0, 255, 184],
      [0, 92, 255],
      [230, 230, 230],
      [0, 133, 255],
      [255, 214, 0],
      [25, 194, 194],
      [102, 255, 0],
      [92, 0, 255]]], np.uint8)/255
ADE20K_class_names  =  ['wall', 'building, edifice', 'sky',
                'floor, flooring', 'tree', 'ceiling', 'road, route',
                'bed ', 'windowpane, window ',
                'grass', 'cabinet', 'sidewalk, pavement',
                'person, individual, someone, somebody, mortal, soul',
                'earth, ground', 'door, double door', 'table',
                'mountain, mount', 'plant, flora, plant life',
                'curtain, drape, drapery, mantle, pall', 'chair',
                'car, auto, automobile, machine, motorcar', 'water',
                'painting, picture', 'sofa, couch, lounge', 'shelf',
                'house',
                'sea', 'mirror', 'rug, carpet, carpeting', 'field',
                'armchair', 'seat', 'fence, fencing', 'desk',
                'rock, stone',
                'wardrobe, closet, press', 'lamp',
                'bathtub, bathing tub, bath, tub', 'railing, rail',
                'cushion', 'base, pedestal, stand', 'box',
                'column, pillar',
                'signboard, sign',
                'chest of drawers, chest, bureau, dresser',
                'counter', 'sand', 'sink', 'skyscraper',
                'fireplace, hearth, open fireplace',
                'refrigerator, icebox', 'grandstand, covered stand',
                'path',
                'stairs, steps', 'runway',
                'case, display case, showcase, vitrine',
                'pool table, billiard table, snooker table',
                'pillow',
                'screen door, screen', 'stairway, staircase',
                'river',
                'bridge, span', 'bookcase', 'blind, screen',
                'coffee table, cocktail table',
                'toilet, can, commode, crapper, pot, potty, stool, throne',
                'flower', 'book', 'hill', 'bench', 'countertop',
                'stove, kitchen stove, range, kitchen range, cooking stove',
                'palm, palm tree', 'kitchen island',
                'computer, computing machine, computing device, data processor, electronic computer, information processing system',
                'swivel chair', 'boat', 'bar', 'arcade machine',
                'hovel, hut, hutch, shack, shanty',
                'bus, autobus, coach, charabanc, double-decker, jitney, motorbus, motorcoach, omnibus, passenger vehicle',
                'towel', 'light, light source', 'truck, motortruck',
                'tower',
                'chandelier, pendant, pendent',
                'awning, sunshade, sunblind',
                'streetlight, street lamp',
                'booth, cubicle, stall, kiosk',
                'television receiver, television, television set, tv, tv set, idiot box, boob tube, telly, goggle box',
                'airplane, aeroplane, plane', 'dirt track',
                'apparel, wearing apparel, dress, clothes', 'pole',
                'land, ground, soil',
                'bannister, banister, balustrade, balusters, handrail',
                'escalator, moving staircase, moving stairway',
                'ottoman, pouf, pouffe, puff, hassock',
                'bottle', 'buffet, counter, sideboard',
                'poster, posting, placard, notice, bill, card',
                'stage', 'van', 'ship', 'fountain',
                'conveyer belt, conveyor belt, conveyer, conveyor, transporter',
                'canopy',
                'washer, automatic washer, washing machine',
                'plaything, toy',
                'swimming pool, swimming bath, natatorium',
                'stool', 'barrel, cask', 'basket, handbasket',
                'waterfall, falls', 'tent, collapsible shelter',
                'bag',
                'minibike, motorbike', 'cradle', 'oven', 'ball',
                'food, solid food', 'step, stair',
                'tank, storage tank',
                'trade name, brand name, brand, marque',
                'microwave, microwave oven', 'pot, flowerpot',
                'animal, animate being, beast, brute, creature, fauna',
                'bicycle, bike, wheel, cycle ', 'lake',
                'dishwasher, dish washer, dishwashing machine',
                'screen, silver screen, projection screen',
                'blanket, cover', 'sculpture', 'hood, exhaust hood',
                'sconce',
                'vase', 'traffic light, traffic signal, stoplight',
                'tray',
                'ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin',
                'fan', 'pier, wharf, wharfage, dock', 'crt screen',
                'plate', 'monitor, monitoring device',
                'bulletin board, notice board', 'shower',
                'radiator',
                'glass, drinking glass', 'clock', 'flag']

ext: py

reason: import numpy


text:
#python 3.6.5
#matplotlib==3.0.3
#numpy==1.16.2
#pyserial==3.4

import time, threading
import numpy as np
from matplotlib.lines import Line2D
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import serial
import pylab
import struct
import string
import time
import queue

Baud = 115200                                                            #波特率设置
Port_name = "COM3"                                                       # 端口名
Datanum = 2                                                              #传入数据的数量
yMax =240                                                                #图中y轴上限
yMin = -240                                                              #图中y轴下限
x_span = 0.5                                                             #横轴跨度
xMax = 1                                                                 #图中x轴上限
xMin = 0                                                                 #图中y轴下限
#xMin = xMax - x_span                                                    #
x_name = "Time"                                                          #设置x轴名称
y_name = "multi_val"                                                     #设置y轴名称
Title =  "pic"                                                           #图像名称
line_name = ["X","Y"]                                                    #线名称
Color = ['red','orange','darkblue','darkslategray','deeppink','megenta'] #绘图线颜色

class Scope(object):
    def __init__(self, ax, maxt=2, dt=0.002):
        self.ax = ax
        self.dt = dt
        self.maxt = maxt
        self.tdata = [0]
        self.ydata = []
        self.line=[]
        for i in range(0,Datanum):
            self.ydata.append([0])         #创建参数个数的子列表
        for i in range(0,Datanum):
            self.ax.plot(self.tdata, self.ydata[i])
        for i in range(0,Datanum):
            self.line.append(Line2D(self.tdata, self.ydata[i],color = Color[i]))
            self.ax.add_line(self.line[i])
        self.ax.set_ylim(yMin, yMax)        #初始化y轴上下限
        self.ax.set_xlim(0,xMax)            #初始化x轴上下限

    def update(self, data):
        global yMax
        global yMin
        global xMax
        global xMin
        global x_span
        #lastt = self.tdata[-1]
        #if lastt > self.tdata[0] + self.maxt:  # reset the arrays
           #self.tdata = [self.tdata[-1]]
           #self.ydata = [self.ydata[-1]]
           #self.ax.set_xlim(self.tdata[0], self.tdata[0] + self.maxt)
           #self.ax.figure.canvas.draw()
        if data[0] == 0:
            return self.line;
        y = data[1]
        t = self.tdata[-1] + self.dt
        self.tdata.append(t)              #增加数组元素
        for i in range(0,Datanum):
            self.ydata[i].append(y[i])      #增加数组元素
            self.line[i].set_data(self.tdata, self.ydata[i])
            if (y[i]) > yMax:
                yMax = y[i] + 1
                self.ax.set_ylim(yMin, yMax)        #更新y轴上下限
                self.ax.figure.canvas.draw()
            elif (y[i]) < yMin:
                yMin = y[i] - 1
                self.ax.set_ylim(yMin, yMax)
                self.ax.figure.canvas.draw()
            if t > xMax:                            #更新x轴上下限
                xMax = t + 0.5
                #xMin = xMax - x_span
                self.ax.set_xlim(xMin, xMax)
                self.ax.figure.canvas.draw()

        return self.line

def emitter(p=0.03):
    if DataList.qsize() == 0:
        yield [0,0]
    else :
       yield [1, DataList.get()]

def SerialDataRead():
   while 1:
      SerData = ser.read_until()
      SerData = str(SerData)
      SerData = SerData[2:-3].split()
      try:
          data = list(map(float, SerData))
      except:
         print("Wrong Format")
      #print(data)
      DataList.put(data)

def SerialPlot():
   while 0:
      while DataList.qsize() != 0:
         a = DataList.get()
         print(type(a))
         print(a)
   fig, ax = plt.subplots(figsize = (7, 7))  #修改图像大小，这里是700x700像素
   scope = Scope(ax)
   ani = animation.FuncAnimation(fig, scope.update, emitter, interval=10, blit=True)

   #ax.xlabel(x_name)
   #ax.ylabel(y_name)
   #ax.title (Title)
   #ax.legend(line_name)
   plt.show()

if __name__ == "__main__":
   ser = serial.Serial(Port_name, Baud)#设置端口，波特率，响应时间
   DataList = queue.Queue()
   ReadData = threading.Thread(target=SerialDataRead, name='ReadData')
   Plot = threading.Thread(target=SerialPlot, name='Plot')
   ReadData.start()
   Plot.start()


text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

## Advanced Course in Machine Learning
## Week 2
## Exercise 1

import numpy as np
import scipy
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# In[2]:

# a)
def calcRHat (a, M):
    x = np.random.uniform(-2,2,M)
    noise = np.random.uniform(-1,1,M)
    y = 2*x+noise

    loss = (y - a*x)**2
    Rhat = sum(loss)/M
    return Rhat
    #return x, y, loss, Rhat

# In[3]:

a = 1
M = 100000

Rhats = list()
lowerA = -100
higherA = 100
axisX = range(lowerA, higherA+1)
for a in axisX:
    Rhats.append(calcRHat(a,M))
    #x, y, loss, RHat = calcRHat(a,M)

# In[4]:

sns.set_style("darkgrid")
plt.plot(axisX, Rhats)
#plt.xlim([-100, 100])
plt.xlabel('Alpha')
plt.ylabel('R_hat')
plt.title('Alpha vs. R_hat using Monte Carlo approximation')
plt.show()

# In[5]:

a = 2.5

Rhats = list()
lowerM = 1
higherM = 100000
axisX = range(lowerM, higherM+1)
for M in axisX:
    Rhats.append(calcRHat(a,M))

# In[6]:

plt.plot(axisX, Rhats)
#plt.xlim([-100, 100])
plt.xlabel('M')
plt.ylabel('R_hat')
plt.title('M vs. R_hat using Monte Carlo approximation')
plt.show()

max_stars_repo_name: mikkokotola/AdvancedMachineLearning

max_issues_repo_name: mikkokotola/AdvancedMachineLearning

max_forks_repo_name: mikkokotola/AdvancedMachineLearning

reason: import numpy,import scipy


text:
import cv2
import numpy as np
import tensorflow as tf
import torch
from tensorflow.math import (
    greater,
    add,
    subtract,
    multiply,
    divide,
    square,
    pow as tf_pow,
    reduce_mean as tf_mean,
    reduce_std as tf_std
)

from utils import (
    showImage
)
from utils.math import (
    log,
    multiOperation
)

class PyTorchLoss:
    # shape: [N, C, H, W]
    @staticmethod
    def ssim(x, y, is_normalized=False):
        k1 = 0.01
        k2 = 0.03
        L = 1.0 if is_normalized else 255.0

        c1 = np.power(k1 * L, 2.0)
        c2 = np.power(k2 * L, 2.0)
        c3 = c2 / 2.0

        ux = x.mean()
        uy = y.mean()

        std_x = x.std()
        std_y = y.std()

        xy = (x - ux) * (y - uy)
        std_xy = xy.mean()

        l_xy = (2.0 * ux * uy + c1) / (ux ** 2.0 + uy ** 2.0 + c1)
        c_xy = (2.0 * std_x * std_y + c2) / (std_x ** 2.0 + std_y ** 2.0 + c2)
        s_xy = (std_xy + c3) / (std_x * std_y + c3)

        ssim = l_xy * c_xy * s_xy
        ssim = torch.clamp(ssim, -1.0, 1.0)

        return ssim

    @staticmethod
    def ssim3(x, y, is_normalized=True):
        xr, xg, xb = torch.split(x, 1, dim=0)
        yr, yg, yb = torch.split(y, 1, dim=0)

        r = PyTorchLoss.ssim(xr, yr, is_normalized)
        g = PyTorchLoss.ssim(xg, yg, is_normalized)
        b = PyTorchLoss.ssim(xb, yb, is_normalized)

        result = (r + g + b) / 3.0

        return result

    @staticmethod
    def ssim4(x, y, is_normalized=True):
        ssim4_loss = 0
        n_image = 0
        for x_image, y_image in zip(x, y):
            ssim4_loss += PyTorchLoss.ssim3(x_image, y_image, is_normalized)
            n_image += 1

        ssim4_loss /= n_image

        return ssim4_loss

# psnr: 其值不能很好地反映人眼主觀感受
def psnr(y_label, y_pred):
    """
    PSNR is Peek Signal to Noise Ratio, which is similar to mean squared error.

    It can be calculated as
    PSNR = 20 * log10(MAXp) - 10 * log10(MSE)

    When providing an unscaled input, MAXp = 255. Therefore 20 * log10(255)== 48.1308036087.
    However, since we are scaling our input, MAXp = 1. Therefore 20 * log10(1) = 0.
    Thus we remove that component completely and only compute the remaining MSE component.
    """
    _result = subtract(y_label, y_pred)
    _result = square(_result)
    _result = tf_mean(_result)
    _result = multiply(-10., log(_result, 10.))
    return _result

def ssim(x, y, is_normalized=False):
    k1 = 0.01
    k2 = 0.03
    L = 1.0 if is_normalized else 255.0
    c1 = np.power(k1 * L, 2)
    c2 = np.power(k2 * L, 2)
    c3 = c2 / 2

    ux = x.mean()
    uy = y.mean()

    std_x = x.std()
    std_y = y.std()

    xy = (x - ux) * (y - uy)
    std_xy = xy.mean()

    l_xy = (2 * ux * uy + c1) / (np.power(ux, 2) + np.power(uy, 2) + c1)
    c_xy = (2 * std_x * std_y + c2) / (np.power(std_x, 2) + np.power(std_y, 2) + c2)
    s_xy = (std_xy + c3) / (std_x * std_y + c3)

    _ssim = l_xy * c_xy * s_xy
    _ssim = np.clip(_ssim, -1.0, 1.0)

    return _ssim

def tf_ssim(x, y, is_normalized=False):
    """
    k1 = 0.01
    k2 = 0.03
    L = 1.0 if is_normalized else 255.0
    c1 = np.power(k1 * L, 2)
    c2 = np.power(k2 * L, 2)
    c3 = c2 / 2
    """
    k1 = 0.01
    k2 = 0.03
    L = 1.0 if is_normalized else 255.0
    c1 = tf_pow(multiply(k1, L), 2.0)
    c2 = tf_pow(multiply(k2, L), 2.0)
    c3 = divide(c2, 2.0)

    # if type(x) is np.ndarray:
    #      x = tf.convert_to_tensor(x, dtype=tf.float32)
    # if type(y) is np.ndarray:
    #      y = tf.convert_to_tensor(y, dtype=tf.float32)

    """
    ux = x.mean()
    uy = y.mean()
    """
    ux = tf_mean(x)
    uy = tf_mean(y)

    """
    std_x = x.std()
    std_y = y.std()
    """
    std_x = tf_std(x)
    std_y = tf_std(y)

    """
    xy = (x - ux) * (y - uy)
    std_xy = xy.mean()
    """
    xy = multiply(subtract(x, ux), subtract(y, uy))
    std_xy = tf_mean(xy)

    """
    l_xy = (2 * ux * uy + c1) / (np.power(ux, 2) + np.power(uy, 2) + c1)
    """
    l_son = add(multiOperation(multiply, 2.0, ux, uy), c1)
    l_mom = multiOperation(add, tf_pow(ux, 2.0), tf_pow(uy, 2.0), c1)
    l_xy = divide(l_son, l_mom)

    """
    c_xy = (2 * std_x * std_y + c2) / (np.power(std_x, 2) + np.power(std_y, 2) + c2)
    """
    c_son = add(multiOperation(multiply, 2.0, std_x, std_y), c2)
    c_mom = multiOperation(add, tf_pow(std_x, 2.0), tf_pow(std_y, 2.0), c2)
    c_xy = divide(c_son, c_mom)

    """
    s_xy = (std_xy + c3) / (std_x * std_y + c3)
    """
    s_son = add(std_xy, c3)
    s_mom = add(multiply(std_x, std_y), c3)
    s_xy = divide(s_son, s_mom)

    one = tf.constant(1.0)
    _ssim = multiOperation(multiply, l_xy, c_xy, s_xy)
    _result = tf.cond(greater(_ssim, one), lambda: one, lambda: _ssim)

    return _result

def ssim3(x, y, is_normalized=True):
    x1, x2, x3 = np.split(x, 3, axis=2)
    y1, y2, y3 = np.split(y, 3, axis=2)

    s1 = ssim(x1, y1, is_normalized)
    s2 = ssim(x2, y2, is_normalized)
    s3 = ssim(x3, y3, is_normalized)

    result = (s1 + s2 + s3) / 3.0

    return result

def tf_ssim3(x, y, is_normalized=True):
    [x1, x2, x3] = tf.split(x, 3, axis=2)
    [y1, y2, y3] = tf.split(y, 3, axis=2)

    s1 = tf_ssim(x1, y1, is_normalized)
    s2 = tf_ssim(x2, y2, is_normalized)
    s3 = tf_ssim(x3, y3, is_normalized)

    three = tf.constant(3.0)
    result = divide(multiOperation(add, s1, s2, s3), three)

    return result

def tf_ssim3_(xy):
    x, y = tf.split(xy, 2, axis=3)
    x = tf.squeeze(x)
    print("[tf_ssim3_] x.shape:", x.shape)
    y = tf.squeeze(y)
    return tf_ssim3(x, y, is_normalized=False)

def tf_ssim3_norm(xy):
    x, y = tf.split(xy, 2, axis=3)
    x = tf.squeeze(x)
    print("[tf_ssim3_norm] x.shape:", x.shape)
    y = tf.squeeze(y)
    return tf_ssim3(x, y, is_normalized=True)

def ssim4(x, y, is_normalized=False):
    each_loss = []

    for _x, _y in zip(x, y):
        each_loss.append(ssim3(_x, _y, is_normalized))

    each_loss = np.array(each_loss)

    total_loss = each_loss.mean()

    return total_loss, each_loss

def tf_ssim4(x, y, is_normalized=False):
    print("x.shape: ", x.shape)
    print("y.shape: ", y.shape)
    stack = tf.stack([x, y], axis=4)
    print("stack.shape: ", stack.shape)

    # tf.map_fn: 接受參數為一個的函數，如 tf_ssim3_norm 等
    # 第二個參數為前一個函式的參數，若要傳遞多個參數，則須將他們合併成一個(元組不知是否可以)
    if is_normalized:
        each_loss = tf.map_fn(tf_ssim3_norm, stack)
    else:
        each_loss = tf.map_fn(tf_ssim3_, stack)

    total_loss = tf_mean(each_loss)

    return total_loss, each_loss

if __name__ == "__main__":
    def ssimTest(img_x, img_y):
        x = tf.placeholder(dtype=tf.float32,
                           shape=[None, None, None, 3])
        y = tf.placeholder(dtype=tf.float32,
                           shape=[None, None, None, 3])
        compute_ssim = tf_ssim4(x, y, True)

        # ssim(self.labels, self.pred, is_normalized=True)
        # labels[i].shape = (None, label_size, label_size, c_dim)
        # pred: (?, 32, 32, self.c_dim)
        tf_config = tf.ConfigProto(log_device_placement=True)
        tf_config.gpu_options.per_process_gpu_memory_fraction = 0.1
        with tf.Session(config=tf_config) as sess:
            # ssim_value = sess.run(compute_ssim,
            #                       feed_dict={x: img_x,
            #                                  y: img_y})
            # print("result:", ssim_value)

            total_loss, each_loss = sess.run(compute_ssim,
                                             feed_dict={x: img_x,
                                                        y: img_y})

            print("total_loss:", total_loss)  # total_loss: 0.9782068
            print("each_loss:", each_loss)    # each_loss: [0.9740695  0.97524875 0.98075956 0.98274946]

    # ================================================================================
    img1 = cv2.imread("data/splice1.png")
    img2 = cv2.imread("data/splice2.png")
    img3 = cv2.imread("data/splice3.png")
    img4 = cv2.imread("data/splice4.png")
    images = [img1, img2, img3, img4]
    showImage(img1, img2, img3, img4)

    dsts = []
    for i in range(len(images)):
        img = images[i] / 255.0
        # INTER_CUBIC = cv2.resize(_img, (_resize_cols, _resize_rows), interpolation=cv2.INTER_CUBIC)
        images[i] = cv2.resize(img, (256, 256), interpolation=cv2.INTER_CUBIC)
        dst = cv2.GaussianBlur(images[i].copy(), (5, 5), 0)
        dsts.append(dst)

    images = np.array(images)
    dsts = np.array(dsts)

    ssimTest(images, dsts)

    # ssim loss: 0.9724881069549962
    loss1 = ssim(images[0], dsts[0], is_normalized=True)
    loss2 = ssim(images[1], dsts[1], is_normalized=True)
    loss3 = ssim(images[2], dsts[2], is_normalized=True)
    loss4 = ssim(images[3], dsts[3], is_normalized=True)
    print("ssim loss1:", loss1)  # ssim loss1: 0.9748280551931702
    print("ssim loss2:", loss2)  # ssim loss2: 0.9754028104752155
    print("ssim loss3:", loss3)  # ssim loss3: 0.9837840687997413
    print("ssim loss4:", loss4)  # ssim loss4: 0.9857441547142608
    # ssim total loss: 0.979939772295597
    print("ssim total loss:", np.mean([loss1, loss2, loss3, loss4]))

    pt_images = torch.from_numpy(images)
    pt_images = pt_images.permute(0, 3, 1, 2)  # pt_images.shape = torch.Size([4, 3, 256, 256])
    pt_dsts = torch.from_numpy(dsts)
    pt_dsts = pt_dsts.permute(0, 3, 1, 2)  # pt_dsts.shape = torch.Size([4, 3, 256, 256])

    pt_image1 = pt_images[0]
    print("pt_image1.shape:", pt_image1.shape)  # torch.Size([3, 256, 256])
    result = torch.split(pt_image1, 1, dim=0)

    pt_loss1 = PyTorchLoss.ssim(pt_images[0], pt_dsts[0], is_normalized=True)
    print("pt_loss1:", pt_loss1)  # pt_loss1: tensor(0.9748, dtype=torch.float64)
    pt_loss2 = PyTorchLoss.ssim(pt_images[1], pt_dsts[1], is_normalized=True)
    print("pt_loss2:", pt_loss2)  # pt_loss2: tensor(0.9754, dtype=torch.float64)
    pt_loss3 = PyTorchLoss.ssim(pt_images[2], pt_dsts[2], is_normalized=True)
    print("pt_loss3:", pt_loss3)  # pt_loss3: tensor(0.9838, dtype=torch.float64)
    pt_loss4 = PyTorchLoss.ssim(pt_images[3], pt_dsts[3], is_normalized=True)
    print("pt_loss4:", pt_loss4)  # pt_loss4: tensor(0.9857, dtype=torch.float64)
    # pt_total_loss: tensor(0.9851, dtype=torch.float64)
    pt_total_loss = torch.mean(torch.tensor([pt_loss1, pt_loss2, pt_loss3, pt_loss4]))
    pt_loss = PyTorchLoss.ssim(torch.from_numpy(images), torch.from_numpy(dsts), is_normalized=True)
    print("pt_loss:", pt_loss)  # pt_loss: tensor(0.9851, dtype=torch.float64)

    """
    上面會將圖片的三個通道"共同"計算 ssim 值
    下面則將圖片的三個通道"分別"計算 ssim 值
    """

    pt_ssim3_loss = 0
    for img, dst in zip(pt_images, pt_dsts):
        pt_ssim3_loss += PyTorchLoss.ssim3(img, dst, is_normalized=True)
    pt_ssim3_loss /= pt_images.shape[0]
    # pt_ssim3_loss: 0.9782
    print("pt_ssim3_loss:", pt_ssim3_loss)

    ssim3_loss = 0
    for img, dst in zip(images, dsts):
        ssim3_loss += ssim3(img, dst, is_normalized=True)
    ssim3_loss /= len(images)

    # ssim3_loss: 0.9782067609078682
    print("ssim3_loss:", ssim3_loss)

    # ssim4: (0.9782067609078682, array([0.97406934, 0.97524874, 0.9807595 , 0.98274946]))
    print("ssim4:", ssim4(images, dsts, is_normalized=True))

    # PyTorch.ssim4: tensor(0.9782, dtype=torch.float64)
    pt_dsts = pt_dsts.requires_grad_(True)
    ssim4_loss = PyTorchLoss.ssim4(pt_images, pt_dsts, is_normalized=True)
    print("PyTorch.ssim4:", ssim4_loss)


text:
import numpy as np

from shfl.federated_aggregator.federated_aggregator import FederatedAggregator

class WeightedFedAvgAggregator(FederatedAggregator):
    """
    Implementation of Weighted Federated Avegaring Aggregator. The aggregation of the parameters is based in the number of data \
    in every node.

    It implements [Federated Aggregator](../federated_aggregator/#federatedaggregator-class)
    """

    def aggregate_weights(self, clients_params):
        """
        Implementation of abstract method of class [AggregateWeightsFunction](../federated_aggregator/#federatedaggregator-class)

        # Arguments:
            clients_params: list of multi-dimensional (numeric) arrays. Each entry in the list contains the model's parameters of one client.

        # Returns:
            aggregated_weights: aggregator weights representing the global learning model
        """
        clients_params_array = np.array(clients_params)

        num_clients = clients_params_array.shape[0]
        num_layers = clients_params_array.shape[1]

        ponderated_weights = np.array([self._percentage[client] * clients_params_array[client, :] for client in range(num_clients)])
        aggregated_weights = np.array([np.sum(ponderated_weights[:, layer], axis=0) for layer in range(num_layers)])

        return aggregated_weights


text:
"""Algorithms to select doublets and triplets
"""
from __future__ import print_function

import pandas as pd
import numpy as np

def create_segments(hits, layer_pair, gid_keys='layer',
                    only_true=False, cluster_info=True,
                    origin_pos=True,
                    verbose=False
                   ):
    """Return all segments from all hits in the layer pair.
    only_true --  option to return only true segments, otherwise return all segments.
    """
    if 'particle_id' not in hits.columns:
        print('particle_id should be in hits,'
              'please merge hits and particles')
        return None

    if 'eta' not in hits.columns:
        # calculate eta, phi for each hit
        x = hits.x.values
        y = hits.y.values
        z = hits.z.values
        r = np.sqrt(x**2 + y**2)
        r3 = np.sqrt(r**2 + z**2)
        phi = np.arctan2(y, x)
        theta = np.arccos(z/r3)
        eta = -np.log(np.tan(theta/2.))
        hits = hits.assign(eta=eta, phi=phi)

    try:
        l1, l2 = layer_pair
    except:
        print("layer_pair should be a tuple")
        return None

    def calc_dphi(phi1, phi2):
        """Computes phi2-phi1 given in range [-pi,pi]"""
        dphi = phi2 - phi1
        dphi[dphi > np.pi] -= 2*np.pi
        dphi[dphi < -np.pi] += 2*np.pi
        return dphi

    hits1 = hits[hits[gid_keys] == l1]
    hits2 = hits[hits[gid_keys] == l2]

    hit_pairs = pd.merge(
        hits1.reset_index(), hits2.reset_index(),
        how='inner', on='evtid', suffixes=('_in', '_out'))

    # Identify the true pairs
    true_edges = (hit_pairs.particle_id_in == hit_pairs.particle_id_out) \
                 & (hit_pairs.particle_id_in != 0)

    if only_true:
        hit_pairs = hit_pairs[true_edges]

    # Calculate coordinate differences
    dphi = calc_dphi(hit_pairs.phi_in, hit_pairs.phi_out)
    dz = hit_pairs.z_out - hit_pairs.z_in
    dr = hit_pairs.r_out - hit_pairs.r_in
    phi_slope = dphi / dr
    z0 = hit_pairs.z_in - hit_pairs.r_in * dz / dr
    deta = hit_pairs.eta_in - hit_pairs.eta_out
    rz_slope = np.arctan2(dr, dz)

    selected_features = [
        'evtid', 'index_in', 'index_out',
        'hit_id_in', 'hit_id_out',
        'particle_id_in',
        'pt_in',
    ]
    if origin_pos:
        selected_features += [
            'x_in', 'x_out',
            'y_in', 'y_out',
            'z_in', 'z_out',
            'r_in', 'r_out',
            'phi_in', 'phi_out',
            'layer_in', 'layer_out',
        ]
    if cluster_info and 'lx_in' in hit_pairs.columns:
        selected_features += [
            'lx_in', 'lx_out',
            'ly_in', 'ly_out',
            'lz_in', 'lz_out']

    # Put the results in a new dataframe
    df_pairs = hit_pairs[selected_features].assign(
        dphi=dphi, dz=dz, dr=dr, true=true_edges,
        phi_slope=phi_slope, z0=z0, deta=deta, rz_slope=rz_slope
    )

    df_pairs = df_pairs.rename(
        columns={
            'index_in': 'hit_idx_in',
            "index_out": 'hit_idx_out',
            'particle_id_in':'particle_id',
            'pt_in':"pt",
        }
    )
    if cluster_info:
        try:
            deta1 = hit_pairs.geta_out - hit_pairs.geta_in
            dphi1 = hit_pairs.gphi_out - hit_pairs.gphi_in
        except (KeyError, AttributeError):
            pass
        else:
            df_pairs = df_pairs.assign(deta1=deta1, dphi1=dphi1)
    if verbose:
        n_true = df_pairs[df_pairs['true']].shape[0]
        n_fake = df_pairs[~df_pairs['true']].shape[0]
        print('Layer:({}, {}), {:,} ({:.3f}% of total edges) True edges and {:,} fake edges,'.format(
            l1, l2, n_true, 100*n_true/(n_true+n_fake), n_fake))

    return df_pairs


text:
import cv2
import gym
from gym import spaces
import numpy as np

class DuckietownRewardWrapper(gym.RewardWrapper):

    def __init__(self, env):
        super().__init__(env)

    def reward(self, reward):
        if reward == -1000:
            reward = -10
        elif reward > 0:
            reward += 10
        else:
            reward += 4

        return reward

class DuckietownDiscreteWrapper(gym.ActionWrapper):
    """
    Duckietown environment with discrete actions (sharp left, sharp right, forward, sight left, slight right)
    instead of continuous control
    """

    def __init__(self, env):
        gym.ActionWrapper.__init__(self, env)
        self.action_space = spaces.Discrete(5)

    def action(self, action):
        # Sharp left
        if action == 0:
            vels = [0.3, +1.0]
        # Sharp right
        elif action == 1:
            vels = [0.3, -1.0]
        # Go forward
        elif action == 2:
            vels = [0.5, 0.0]
        # slight left
        elif action == 3:
            vels = [0.5, +1.0]
        # slight right
        elif action == 4:
            vels = [0.5, -1.0]

        else:
            assert False, "unknown action"
        return np.array(vels)

class WarpFrame(gym.ObservationWrapper):
    def __init__(self, env, color=False):
        """Warp frames to 84x84 as done in the Nature paper and later work."""
        gym.ObservationWrapper.__init__(self, env)
        self.width = 84
        self.height = 84
        self.color = color
        colors = 1
        if color:
            colors = 3
        self.observation_space = spaces.Box(low=0, high=255,
            shape=(self.height, self.width, colors), dtype=np.uint8)

    def observation(self, frame):
        frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_AREA)
        if not self.color:
            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
            return frame[:, :, None]
        else:
            return frame

class Normalize(gym.ObservationWrapper):
    def __init__(self, env=None):
        super().__init__(env)
        self.obs_lo = self.observation_space.low[0] #FIXME this is suuuuper shoddy
        self.obs_hi = self.observation_space.high[0]
        obs_shape = self.observation_space.shape
        self.observation_space = spaces.Box(0.0, 1.0, obs_shape, dtype=np.float32)

    def observation(self, obs):
        if self.obs_lo == 0.0 and self.obs_hi == 1.0:
            return obs
        else:
            return (obs - self.obs_lo) / (self.obs_hi - self.obs_lo)


text:
import os, sys
from pymongo import MongoClient
import datetime
from threading import Thread
from random import randint
import traceback

from requests import get
from re import sub
from bs4 import BeautifulSoup
from random import shuffle
import json
import numpy as np

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from mongo_helper import MongoHelper
from helpers import Parser
from config import getKeys

settings = getKeys(os)

parser_dependencies = {
    "get": get,
    "np": np,
    "BeautifulSoup": BeautifulSoup,
    "json": json,
    "shuffle": shuffle,
    "sub": sub
}
wiki_quote_scraper = Parser(**parser_dependencies)

mongo_helper_depenecies = {'datetime': datetime, 'MongoClient': MongoClient}
mongo_helper_depenecies['settings'] = settings
mongo_helper_depenecies['Thread'] = Thread
mongo_helper_depenecies['randint'] = randint
mongo_helper_depenecies['scraper'] = None
mongo_helper_depenecies["traceback"] = traceback

mongo_helper = MongoHelper(**mongo_helper_depenecies)

def init_svm_collection(mydb):
    collection = mydb[settings.get("SVM_DB")]
    collection.insert_one({"_id": "id_sequence", "value": 0})

def init_svm():
    mongo_helper.connect_to_db(init_svm_collection)

if __name__ == "__main__":

    methods_to_test = []

    # methods_to_test.append()

    for method in methods_to_test:
        method()

max_stars_repo_name: jzohdi/svmVisualizer

max_issues_repo_name: jzohdi/svmVisualizer

max_forks_repo_name: jzohdi/svmVisualizer


text:
#! usr/bin/python3
#%%
import config
from src.model.stldesc_model import define_desc_encoder, StyleNet, define_stl_encoder, stl_encoder
from src.support.loss_functions import pairWiseRankingLoss, MarginalAcc, triplet_loss

import os
import logging
import time
import math
from datetime import datetime
import pathlib
import pandas as pd
import numpy as np
import random
import tensorflow.keras.backend as K
import tensorflow as tf
import tensorflow.keras.layers.experimental.preprocessing as prep
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.initializers import RandomNormal
from tensorflow.keras import losses
from tensorflow.keras import metrics
from tensorflow.keras.callbacks import TensorBoard
from matplotlib import pyplot as plt
from livelossplot import PlotLosses
from livelossplot.outputs import MatplotlibPlot

#tf.executing_eagerly()
#%%
#tensorboard logger
logdir = config.LOG_DIR+ "/desc_pre_" + datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = TensorBoard(log_dir=logdir, histogram_freq=1, profile_batch=1)

# tf.profiler.experimental.server.start(6009)

# set logger
logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

def process_img(img):
  img = tf.image.decode_jpeg(img, channels=3)
  img = tf.image.convert_image_dtype(img, tf.float32)
  return tf.image.resize(img, config.IMAGE_SIZE)

def process_path(file_path):
    img = tf.io.read_file(file_path)
    img = tf.image.decode_jpeg(img, channels=3)
    #print(fp)
    img = tf.image.convert_image_dtype(img, tf.float32)
    img = tf.image.resize(img, size=(128, 128))
    return img

def train_gen():
    root_path, n ='./data/data/style datasetU/data', 2900
    idx = np.array(range(n))
    class_id = 1
    for i in idx:
        #i = random.randint(lower, higher)
        random_nums = np.random.choice(class_idx_list[class_id-1], 2, replace=False)
        # random_bool = random.randint(0,1)
        # if random_bool:
        #     if random_num == int(i):
        #         random_num = random.randint(lower, higher)
        # else:
        #     random_num = max(random.randint(1,10), int(i)-5)
        img1_det = stenc_df.loc[random_nums[0]-1, ['path', 'style_code']]
        img2_det = stenc_df.loc[random_nums[1]-1, ['path', 'style_code']]
        # label = 0
        # if img1_det['style_code'] == img2_det['style_code']:
        #     label = 1
        #print(os.path.join(root_path, img1_det['path']), os.path.join(root_path, img2_det['path']))
        try :
            img1 = process_path(os.path.join(root_path, img1_det['path']))
            img2 = process_path(os.path.join(root_path, img2_det['path']))
            if class_id < n_classes:
                class_id += 1
            else:
                class_id = 1
            yield img1, img2
        except:
            print(f"Error in file {img1_det['path']}")
            continue

def val_gen():
    root_path, n ='./data/data/style datasetU/data', 200
    idx = np.array(range(n))
    class_id = 1
    for i in idx:
        #i = random.randint(lower, higher)
        random_nums = np.random.choice(class_idx_list[class_id-1], 2, replace=False)
        # random_bool = random.randint(0,1)
        # if random_bool:
        #     if random_num == int(i):
        #         random_num = random.randint(lower, higher)
        # else:
        #     random_num = max(random.randint(1,10), int(i)-5)
        img1_det = stenc_df.loc[random_nums[0]-1, ['path', 'style_code']]
        img2_det = stenc_df.loc[random_nums[1]-1, ['path', 'style_code']]
        # label = 0
        # if img1_det['style_code'] == img2_det['style_code']:
        #     label = 1
        #print(os.path.join(root_path, img1_det['path']), os.path.join(root_path, img2_det['path']))
        try :
            img1 = process_path(os.path.join(root_path, img1_det['path']))
            img2 = process_path(os.path.join(root_path, img2_det['path']))
            if class_id < 12:
                class_id += 1
            else:
                class_id = 1
            yield img1, img2
        except:
            print(f"Error in file {img1_det['path']}")
            continue

# image resize and rescale pipeline
resize_and_rescale = tf.keras.Sequential([
    prep.Resizing(config.IMG_HEIGHT, config.IMG_WIDTH),
    prep.Normalization()
])

# image augmentation pipeline
data_augmentation = tf.keras.Sequential([
    prep.RandomContrast(0.2),
    prep.RandomFlip("horizontal"),
    prep.RandomCrop(config.IMG_HEIGHT, config.IMG_WIDTH),
    prep.RandomRotation(0.3, fill_mode='nearest', interpolation='bilinear'),
    prep.RandomZoom(height_factor=(-0.2, 0.2), width_factor=(-0.2, 0.2), fill_mode='nearest', interpolation='bilinear')
])

# data_augmentation = tf.keras.Sequential([
#   prep.RandomFlip("horizontal_and_vertical"),
#   prep.RandomRotation(0.2),
# ])

def prepare(ds, shuffle=False, augment=False):
    # ds = ds.map(lambda x: tf.py_function(process_path, [x], [tf.float32, tf.float32, tf.int32]),
    #                         num_parallel_calls=tf.data.AUTOTUNE)

    # ds = ds.map(lambda x1, x2, y: (process_path(x1), process_path(x2), y),
    #             num_parallel_calls=tf.data.AUTOTUNE)

    ds = ds.map(lambda x1, x2: (resize_and_rescale(x1), resize_and_rescale(x2)),
                num_parallel_calls=tf.data.AUTOTUNE)

    ds = ds.cache()

    ds = ds.batch(12, drop_remainder=True)

    if shuffle:
        ds = ds.shuffle(1000)

    if augment:
        ds = ds.map(lambda x1, x2: (data_augmentation(x1, training=True), data_augmentation(x2, training=True)),
                    num_parallel_calls=tf.data.AUTOTUNE)

    return ds.prefetch(buffer_size=tf.data.AUTOTUNE)

@tf.function
def train_step(ref_in, style_in):
    with tf.GradientTape() as tape:
        ref_emb, pos_emb = desc_pre_model([ref_in, style_in])

        similarity = tf.einsum(
            "ae,pe->ap", ref_emb, pos_emb
        )
        temp = 0.2
        similarity /= temp

        sparse_labels = tf.range(n_classes)

        loss = stlLoss(sparse_labels, similarity)
    grads = tape.gradient(loss, base_model.trainable_variables)
    opt.apply_gradients(zip(grads, base_model.trainable_variables))
    train_metrics.update_state(sparse_labels, similarity)
    return loss

@tf.function
def val_step(ref_in, style_in):
    with tf.GradientTape() as tape:
        ref_emb, pos_emb = desc_pre_model([ref_in, style_in])

        similarity = tf.einsum(
            "ae,pe->ap", ref_emb, pos_emb
        )
        temp = 0.2
        similarity /= temp

        sparse_labels = tf.range(n_classes)
        loss = stlLoss(sparse_labels, similarity)

    val_metrics.update_state(sparse_labels, similarity)
    return loss

def train(epochs=3):
    tensorboard_callback.set_model(desc_pre_model)
    plotlosses = PlotLosses(outputs=[MatplotlibPlot()], groups={'loss' : ['tr_loss', 'val_loss'], 'accuracy' : ['tr_acc', 'val_acc']})
    for epoch in range(epochs):
        start_time = time.time()

        # Iterate over the batches of the dataset.
        for step, (ref_batch_train, style_batch_train) in enumerate(train_dataset):
            train_loss = train_step(ref_batch_train, style_batch_train)

        # Run a validation loop at the end of each epoch.
        for ref_batch_val, style_batch_val in val_dataset:
            val_loss = val_step(ref_batch_val, style_batch_val)

        print(f'train_loss : {train_loss} | val_loss : {val_loss}')
        tr_acc = train_metrics.result()
        val_acc = val_metrics.result()
        plotlosses.update({
            'tr_loss' : train_loss,
            'tr_acc' : tr_acc,
            'val_loss' : val_loss,
            'val_acc' : val_acc,
        })
        plotlosses.send()

        train_metrics.reset_states()
        val_metrics.reset_states()
        # val_acc = val_metrics.result()
        # val_metrics.reset_states()
        # print("Validation acc: %.4f" % (float(val_acc),))
        print("Time taken: %.2fs" % (time.time() - start_time))

#%%
if __name__ == "__main__":
    #data importing
    stenc_df = pd.read_csv('./data/data/style datasetU/StyleEnc.csv')
    train_path = pathlib.Path(os.path.join(config.DESC_ROOT_DIR,'train'))
    val_path = pathlib.Path(os.path.join(config.DESC_ROOT_DIR,'validation'))
    n_classes = 12
    class_idx_list = [np.squeeze(stenc_df.loc[stenc_df['style_code']==i,['fname']].values) for i in range(1, n_classes+1)]
    #train_gen = gen(1, 2923, './data/data/StyleDataset', 2900)
    #val_gen = gen(2923, 3164, './data/data/StyleDataset', 240)
    train_ds = tf.data.Dataset.from_generator(
        train_gen,
        output_signature=(
            tf.TensorSpec(shape=(128,128, 3), dtype=tf.float32),
            tf.TensorSpec(shape=(128,128,3), dtype=tf.float32)
        )

    )
    val_ds = tf.data.Dataset.from_generator(
        val_gen,
        output_signature=(
            tf.TensorSpec(shape=(128,128, 3), dtype=tf.float32),
            tf.TensorSpec(shape=(128,128,3), dtype=tf.float32)
        )

    )
    #filtered_ds = list_ds.filter(lambda x: int(x.split(os.sep)[-1].strip('.jpg')) < config.DESC_TRAIN_SIZE)
    #sample_dt = sample_ds.shuffle(buffer_size=1000)   #config param
    train_dataset = prepare(train_ds, shuffle=True, augment=False)

    val_dataset = prepare(val_ds, shuffle=True, augment=False)
    # init model
    base_model = stl_encoder(config.DESCS_LATENT_SIZE, config.IMAGE_SHAPE)

    train_steps = 100
    lr_fn = tf.optimizers.schedules.PolynomialDecay(1e-4, train_steps, 1e-5, 2)
    opt = tf.optimizers.Adam(lr_fn)
    stlLoss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
    train_metrics = tf.keras.metrics.SparseCategoricalCrossentropy(from_logits=True)
    val_metrics = tf.keras.metrics.SparseCategoricalCrossentropy(from_logits=True)
    #desc_pre_model = define_descrminator((config.IMG_WIDTH, config.IMG_HEIGHT, 3))
    desc_pre_model = StyleNet(base_model)

    train(config.DESCS_EPOCHS)
    # tf.profiler.experimental.client.trace('grpc://localhost:6009',
    #
#%%                          config.LOG_DIR+'/profilers', 2000)
filename = 'descs_wgt7.h5'
base_model.save_weights(os.path.join(config.MODEL_DIR, filename))
logger.info(f">> Saved : {filename}  ")

# %%
import io

results = base_model.predict(val_dataset)
np.savetxt("./logs/gan/triplet/vecs4.tsv", results, delimiter='\t')
#%%
out_m = io.open('./logs/gan/triplet/meta4.tsv', 'w', encoding='utf-8')
t = True
labels = []
batchs = []
for img, label in val_dataset:
    # l = ' '.join(map(str, label))
    # if l not in batchs:
    labels.extend(range(12))
    #     batchs.append(l)
    # else:
    #     break
labels.sort()
[out_m.write(str(x) + "\n") for x in labels]
out_m.close()
# %%
near_neighbours_per_example = 12

embeddings = base_model.predict(val_dataset)
gram_matrix = np.einsum("ae,be->ab", embeddings, embeddings)
near_neighbours = np.argsort(gram_matrix.T)[:, -(near_neighbours_per_example + 1) :]
# %%
from sklearn.metrics import ConfusionMatrixDisplay
num_classes = 12
confusion_matrix = np.zeros((num_classes, num_classes))

# For each class.
for class_idx in range(num_classes):
    # Consider 10 examples.
    example_idxs = class_idx_list[class_idx][:12]
    for y_test_idx in example_idxs:
        # And count the classes of its near neighbours.
        for nn_idx in near_neighbours[y_test_idx][:-1]:
            nn_class_idx = labels[nn_idx]
            confusion_matrix[class_idx, nn_class_idx] += 1

# Display a confusion matrix.
labels = range(1, 13)
disp = ConfusionMatrixDisplay(confusion_matrix=confusion_matrix, display_labels=labels)
disp.plot(include_values=True, cmap="viridis", ax=None, xticks_rotation="vertical")
plt.show()
# %%

max_stars_repo_name: nipdep/STGAN

max_issues_repo_name: nipdep/STGAN

max_forks_repo_name: nipdep/STGAN


text:
# --------------
import numpy as np
# Not every data format will be in csv there are other file formats also.
# This exercise will help you deal with other file formats and how toa read it.
from numpy import genfromtxt
my_data = genfromtxt(path, delimiter=',',skip_header=1)
# Number of unique matches
unique_team = np.unique(my_data[:,0],axis=0)
print("Uniue no of mataches=", unique_team.shape[0])
print("Set of unique_team which played match=", unique_team[:-1])
print("Sum of all extras in all delivery=",np.sum(my_data[:,17].astype(int), axis = 0))
print("Get all deliveries which given player is out,tell wickettype=",my_data[my_data[:,22]!=np.nan][:,11])
toss_won_by_mum=len(my_data[my_data[:,5]=='Mumbai Indians'])
print("Toss won by Mumbai indians=",toss_won_by_mum)
print("Batsman who scored 6 runs",my_data[my_data[:,16].astype(int)>=6].shape[0])
 # How many matches were held in total we need to know so that we can analyze further       statistics keeping that in mind.

# Number of unique teams

 # this exercise deals with you getting to know that which are all those six teams that   played in the tournament.

# Sum of all extras

 # An exercise to make you familiar with indexing and slicing up within data.

# Delivery number when a given player got out

 # Get the array of all delivery numbers when a given player got out. Also mention the wicket type.

# Number of times Mumbai Indians won the toss

 # this exercise will help you get the statistics on one particular team

# Filter record where batsman scored six and player with most number of sixex

 # An exercise to know who is the most aggresive player or maybe the scoring player

reason: import numpy,from numpy


text:
import sys
import os
import time
import math
import numpy as np

import itertools
import struct  # get_image_size
import imghdr  # get_image_size

def sigmoid(x):
    return 1.0 / (np.exp(-x) + 1.)

def softmax(x):
    x = np.exp(x - np.expand_dims(np.max(x, axis=1), axis=1))
    x = x / np.expand_dims(x.sum(axis=1), axis=1)
    return x

def bbox_iou(box1, box2, x1y1x2y2=True):

    # print('iou box1:', box1)
    # print('iou box2:', box2)

    if x1y1x2y2:
        mx = min(box1[0], box2[0])
        Mx = max(box1[2], box2[2])
        my = min(box1[1], box2[1])
        My = max(box1[3], box2[3])
        w1 = box1[2] - box1[0]
        h1 = box1[3] - box1[1]
        w2 = box2[2] - box2[0]
        h2 = box2[3] - box2[1]
    else:
        w1 = box1[2]
        h1 = box1[3]
        w2 = box2[2]
        h2 = box2[3]

        mx = min(box1[0], box2[0])
        Mx = max(box1[0] + w1, box2[0] + w2)
        my = min(box1[1], box2[1])
        My = max(box1[1] + h1, box2[1] + h2)
    uw = Mx - mx
    uh = My - my
    cw = w1 + w2 - uw
    ch = h1 + h2 - uh
    carea = 0
    if cw <= 0 or ch <= 0:
        return 0.0

    area1 = w1 * h1
    area2 = w2 * h2
    carea = cw * ch
    uarea = area1 + area2 - carea
    return carea / uarea

def get_region_boxes(boxes, confs, conf_thresh):

    print('Getting boxes from boxes and confs ...')
    ########################################
    #   Figure out bboxes from slices     #
    ########################################

    # boxes: [batch, num_anchors * H * W, num_classes, 4]
    # confs: [batch, num_anchors * H * W, num_classes]

    # [batch, num_anchors * H * W, num_classes, 4] --> [batch, num_anchors * H * W, 4]
    boxes = boxes[:, :, 0, :]

    t1 = time.time()
    all_boxes = []
    for b in range(boxes.shape[0]):
        l_boxes = []

        # [num_anchors * H * W, num_classes] --> [num_anchors * H * W]
        max_conf = confs[b, :, :].max(axis=1)
        # [num_anchors * H * W, num_classes] --> [num_anchors * H * W]
        max_id = confs[b, :, :].argmax(axis=1)

        argwhere = np.argwhere(max_conf > conf_thresh)

        max_conf = max_conf[argwhere].flatten()
        max_id = max_id[argwhere].flatten()

        # print(max_conf)
        # print(max_id)

        bcx = boxes[b, argwhere, 0]
        bcy = boxes[b, argwhere, 1]
        bw = boxes[b, argwhere, 2]
        bh = boxes[b, argwhere, 3]

        # print(bcy)

        for i in range(bcx.shape[0]):
            # print(max_cls_conf[i])
            # print('bbox: [ {},\t{},\t{},\t{} ]'.format(bcx[i], bcy[i], bw[i], bh[i]))

            l_box = [bcx[i], bcy[i], bw[i], bh[i], max_conf[i], max_conf[i], max_id[i]]
            l_boxes.append(l_box)

        all_boxes.append(l_boxes)
    t2 = time.time()

    if False:
        print('---------------------------------')
        print('              boxes: %f' % (t2 - t1))
        print('---------------------------------')

    return all_boxes

def nms(boxes, nms_thresh):
    if len(boxes) == 0:
        return boxes

    det_confs = np.zeros(len(boxes))
    for i in range(len(boxes)):
        det_confs[i] = 1 - boxes[i][4]

    sortIds = np.argsort(det_confs)
    out_boxes = []

    for i in range(len(boxes)):
        box_i = boxes[sortIds[i]]
        if box_i[4] > 0:
            out_boxes.append(box_i)
            for j in range(i + 1, len(boxes)):
                box_j = boxes[sortIds[j]]
                if bbox_iou(box_i, box_j, x1y1x2y2=False) > nms_thresh:
                    # print(box_i, box_j, bbox_iou(box_i, box_j, x1y1x2y2=False))
                    box_j[4] = 0

    return out_boxes

def plot_boxes_cv2(img, boxes, savename=None, class_names=None, color=None):
    import cv2
    colors = np.array([[1, 0, 1], [0, 0, 1], [0, 1, 1], [0, 1, 0], [1, 1, 0], [1, 0, 0]], dtype=np.float32)

    def get_color(c, x, max_val):
        ratio = float(x) / max_val * 5
        i = int(math.floor(ratio))
        j = int(math.ceil(ratio))
        ratio = ratio - i
        r = (1 - ratio) * colors[i][c] + ratio * colors[j][c]
        return int(r * 255)

    width = img.shape[1]
    height = img.shape[0]
    for i in range(len(boxes)):
        box = boxes[i]
        x1 = int((box[0] - box[2] / 2.0) * width)
        y1 = int((box[1] - box[3] / 2.0) * height)
        x2 = int((box[0] + box[2] / 2.0) * width)
        y2 = int((box[1] + box[3] / 2.0) * height)

        print(img.shape)
        print(box)
        print(x1, y1, x2, y2)
        sys.exit(0)
        if color:
            rgb = color
        else:
            rgb = (255, 0, 0)
        if len(box) >= 7 and class_names:
            cls_conf = box[5]
            cls_id = box[6]
            print('%s: %f' % (class_names[cls_id], cls_conf))
            classes = len(class_names)
            offset = cls_id * 123457 % classes
            red = get_color(2, offset, classes)
            green = get_color(1, offset, classes)
            blue = get_color(0, offset, classes)
            if color is None:
                rgb = (red, green, blue)
            img = cv2.putText(img, class_names[cls_id], (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 1.2, rgb, 1)
        img = cv2.rectangle(img, (x1, y1), (x2, y2), rgb, 1)
    if savename:
        print("save plot results to %s" % savename)
        cv2.imwrite(savename, img)
    return img

def read_truths(lab_path):
    if not os.path.exists(lab_path):
        return np.array([])
    if os.path.getsize(lab_path):
        truths = np.loadtxt(lab_path)
        truths = truths.reshape(truths.size / 5, 5)  # to avoid single truth problem
        return truths
    else:
        return np.array([])

def load_class_names(namesfile):
    class_names = []
    with open(namesfile, 'r') as fp:
        lines = fp.readlines()
    for line in lines:
        line = line.rstrip()
        class_names.append(line)
    return class_names

def post_processing(img, conf_thresh, n_classes, nms_thresh, output):

    # anchors = [12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401]
    # num_anchors = 9
    # anchor_masks = [[0, 1, 2], [3, 4, 5], [6, 7, 8]]
    # strides = [8, 16, 32]
    # anchor_step = len(anchors) // num_anchors

    boxes = []
    t1 = time.time()
    for i in range(len(output)):
        boxes.append(get_region_boxes(output[i][0], output[i][1], conf_thresh))
    t2 = time.time()

    if img.shape[0] > 1:
        bboxs_for_imgs = [
            boxes[0][index] + boxes[1][index] + boxes[2][index]
            for index in range(img.shape[0])]
        # 分别对每一张图片的结果进行nms
        t3 = time.time()
        boxes = [nms(bboxs, nms_thresh) for bboxs in bboxs_for_imgs]
    else:
        boxes = boxes[0][0] + boxes[1][0] + boxes[2][0]
        t3 = time.time()
        boxes = nms(boxes, nms_thresh)
    t4 = time.time()

    print('-----------------------------------')
    print('     get_region_boxes : %f' % (t2 - t1))
    print('                  nms : %f' % (t4 - t3))
    print('   post process total : %f' % (t4 - t1))
    print('-----------------------------------')
    return boxes


text:
import json
import os

import numpy as np

from tomomibot.audio import pca
from tomomibot.const import GENERATED_FOLDER, ONSET_FILE
from tomomibot.utils import (make_wav_path,
                             encode_duration_class,
                             encode_dynamic_class)

def convert_positions(sequence):
    # We store frame indexes as strings since they are too large
    new_sequence = []
    for step in sequence:
        step['start'] = int(step['start'])
        step['end'] = int(step['end'])
        new_sequence.append(step)
    return new_sequence

class Voice:

    def __init__(self, name, **kwargs):
        self.name = name

        # Load data file from voice
        onset_path = os.path.join(GENERATED_FOLDER, name, ONSET_FILE)
        with open(onset_path) as file:
            data = json.load(file)

            if 'version' in data:
                self.version = data['version']
            else:
                self.version = 1  # Add version for legacy releases

            # Extract informations from data
            if self.version == 1:
                self.sequence = convert_positions(data)
                self.meta = {}
                self.rms_max = 1
            elif self.version == 2:
                self.sequence = convert_positions(data['sequence'])
                self.meta = data['meta']

                # Get RMS maximum for normalization
                self.rms_max = np.max([wav['rms'] for wav in self.sequence])

            # Prepare wav file informations for playback
            wavs = []
            for wav in self.sequence:
                wav_entry = {
                    'path': make_wav_path(name, wav['id']),
                }

                if self.version == 2:
                    sr = self.meta['samplerate']
                    duration = (wav['end'] - wav['start']) / sr * 1000

                    wav_entry['class_dynamic'] = encode_dynamic_class(
                        None, wav['rms'] / self.rms_max)
                    wav_entry['class_duration'] = encode_duration_class(
                        duration)

                wavs.append(wav_entry)

            self.wavs = np.array(wavs)

            self.fit()

    def fit(self, reference_voice=None):
        if reference_voice is None:
            reference_voice = self

        # Get MFCC data from voice sequence
        mfccs = [wav['mfcc'] for wav in reference_voice.sequence]

        # Calculate PCA
        n_components = 6 if self.version == 2 else 2
        _, pca_instance, pca_scaler = pca(mfccs, components=n_components)
        self.points = pca_instance.transform(mfccs)
        self._pca_instance = pca_instance
        self._pca_scaler = pca_scaler

    def project(self, vectors):
        """Project a new mfcc vector into given PCA space"""
        points = self._pca_instance.transform(vectors)
        return self._pca_scaler.transform(points)

    def find_wav(self, point_classes,
                 class_sound, class_dynamic, class_duration):
        """Return wav path depending on sound, duration and dynamic"""
        indices = point_classes[class_sound]
        possible_wavs = self.wavs[indices]

        if len(possible_wavs) == 0:
            return None

        if self.version == 2:
            # Filter by dynamic class
            if class_dynamic:
                possible_wavs = list(
                    filter(lambda x: x['class_dynamic'] == class_dynamic,
                           possible_wavs))

            # Filter by duration class
            if class_duration:
                possible_wavs = list(
                    filter(lambda x: x['class_duration'] == class_duration,
                           possible_wavs))

        # Fallback when we could not find anything
        if len(possible_wavs) == 0:
            possible_wavs = self.wavs[indices]

        # Pick a random sound from that group
        wav = np.random.choice(possible_wavs)

        return wav['path']

max_stars_repo_name: adzialocha/tomomibot

max_issues_repo_name: adzialocha/tomomibot

max_forks_repo_name: adzialocha/tomomibot


text:
"""
Evaluate classifiers created from the data generated based on the Panel A and B
Problem 1 - Part b
@author: raziei.z@husky.neu.edu

"""

import numpy as np
from sklearn.model_selection import KFold
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense
import keras
from sklearn.utils import resample
from keras.callbacks import EarlyStopping,ModelCheckpoint
from imblearn.over_sampling import RandomOverSampler
from sklearn.metrics import roc_auc_score
from keras.models import model_from_json
from tqdm import tqdm
import json

def generate_data(n, box, pos_boxes, pos_box_len, plot = False):
    """
        n = number of data points
        box = A list with format = [ [Top left corner of the box], [Bottom right corner of the box]]
        pos_boxes = A list with format = [ [Top left corner of the box1], [Top left corner of the box2], ...]
        pos_box_len = length of positive boxes
        plot = whether to plot the graph or not
    """
    x_points = (np.random.rand(n) * (box[1][0] - box[0][0]))+box[0][0]  # Generating n number of point in x range
    y_points = (np.random.rand(n) * (box[1][1] - box[0][1]))+box[0][1]  # Generating n number of point in y range
    points = np.column_stack((x_points,y_points)) # Column stacking these two dimention in 1 array
    all_pos = np.zeros(n) == 1
    for pos_box in pos_boxes:   # applying the condition for positive candidates as defined in the question
        pos_x = np.logical_and(x_points > pos_box[0],x_points < (pos_box[0]+pos_box_len)) # specifying x axis range of positive boxes
        pos_y = np.logical_and(y_points > (pos_box[1]-pos_box_len), y_points < pos_box[1]) # specifying y axis range of positive boxes
        pos = np.logical_and(pos_x,pos_y) # Taking AND of both the x and y axis
        all_pos = np.logical_or(all_pos,pos) # Now taking OR to join the currect rectangle with the previous ones
    if plot:   # plot if require
        plt.axis('equal')
        plt.scatter(x_points,y_points) # plotting all point default in blue color
        plt.scatter(x_points[all_pos],y_points[all_pos]) # plotting all the positive rectangles default in orange color
        plt.show()
    return points, (all_pos.reshape(-1,1)*2)-1 # reshaping y to single axis and normalizing between -1 and 1

def return_model(h1, h2, lear_rate=0.001):
    """
        h1 = Number of hidden units in layer 1
        h2 = Number of hidder units in layer 2
        lear_rate = learning rate
    """
    model = Sequential() # Specifying a sequential keras model
    model.add(Dense(units=h1, activation='tanh', input_dim=2)) # Adding 1st hidden layer
    if h2!=0:
        model.add(Dense(units=h2, activation='tanh')) # Adding 2nd hidden layer
    model.add(Dense(units=1, activation='tanh')) # output unit with tanh activation
    model.compile(loss=keras.losses.hinge, # since hinge loss caters loss for output value -1 to 1
              optimizer = keras.optimizers.Adam(lr = lear_rate,decay=1e-6,amsgrad =True), # chosing Adam Optimizer
              metrics=['accuracy']) # set metrics as accuracy for monitoring
    return model

# Generating data
X1,y1 = generate_data(1000,[[-6,4],[6,-4]],[[-4,3],[-2,-1],[2,1]],3,plot=True) # For the 1st dataset of Figure 1
X2,y2 = generate_data(1000,[[-6,4],[6,-4]],[[-4,3],[-1,-2],[2,0]],1,plot=True) # For the 2nd dataset of Figure 1
XT = [X1,X2] # Concatenating X of both the datasets
yT = [y1,y2] # Concatenating y of both the datasets

#Specifying hidden layers
h1 = [1,4,8] # 1st hidden layer
h2 = [0,3]   # 2nd hidder layer
total = len(XT)*len(h1)*len(h2)
pbar = tqdm(total=total, desc="Max Iterations: ")
acc={}
roc={}
for data_No,(X,y) in enumerate(zip(XT,yT)): # selecting data set (a) or (b)
    for fh in h1:                           # selecting number of neurons in 1st hidden layer
        for sh in h2:                       # selecting number of neurons in 2nd hidden layer
            keras.backend.clear_session()
            model = return_model(fh,sh)

            es = EarlyStopping(monitor='loss', min_delta=0.001,patience=20, verbose=0)

            training_No = './1b/'+str(data_No)+'_'+str(fh)+'_'+str(sh)
            history = model.fit(X, y, epochs=50, callbacks=[es],verbose=1)

            pbar.update(1)

            # serialize model to JSON
            model_json = model.to_json()
            with open(training_No+".json", "w") as json_file:
                json_file.write(model_json)

            # serialize weights to HDF5
            model.save_weights(training_No+".h5")

            # Overall score
            scores = model.evaluate(X, y, verbose=0)
            acc[training_No+'_acc']=scores[1] * 100
            try:
                roc[training_No+'_acc'] = roc_auc_score(y, model.predict(X))
            except:
                roc[training_No+'_acc'] = 0

            # Generating heatmap
            y1, x = np.meshgrid(np.linspace(-4, 4, 100), np.linspace(-6, 6, 100))
            z = np.zeros((np.shape(x)[0],np.shape(x)[1]))
            for i in range(len(x)):
                for j in range(len(y1)):
                    z[i][j]=model.predict(np.array([[x[i][i],y1[j][j]]]))

            z = z[:-1, :-1]
            z_min, z_max = -np.abs(z).max(), np.abs(z).max()

            fig, ax = plt.subplots(figsize=(18,8))

            c = ax.pcolormesh(x, y1, z, cmap='RdBu', vmin=z_min, vmax=z_max)
            ax.set_title('Heat Map')
            # set the limits of the plot to the limits of the data
            ax.axis([x.min(), x.max(), y1.min(), y1.max()])
            fig.colorbar(c, ax=ax)

            fig.savefig(training_No+'.png', dpi=fig.dpi)

with open("./1b/acc.json",'w') as f:
    json.dump(acc,f)
with open("./1b/roc.json",'w') as g:
    json.dump(roc,g)