text:
# mypy: ignore-errors

import copy
from distutils import ccompiler
from distutils import errors
from distutils import msvccompiler
from distutils import sysconfig
from distutils import unixccompiler
import os
from os import path
import shutil
import sys

import pkg_resources
import setuptools
from setuptools.command import build_ext

import cupy_builder
import cupy_builder.install_build as build
from cupy_builder._context import Context
from cupy_builder.install_build import PLATFORM_LINUX
from cupy_builder.install_build import PLATFORM_WIN32

try:
    # This is to avoid getting numpy imported inside other modules and
    # overwritting setuptools compilers (#5476)
    import numpy.distutils  # NOQA
except Exception:
    pass

use_hip = build.use_hip

def ensure_module_file(file):
    if isinstance(file, tuple):
        return file
    else:
        return file, []

def module_extension_name(file):
    return ensure_module_file(file)[0]

def module_extension_sources(file, use_cython, no_cuda):
    pyx, others = ensure_module_file(file)
    base = path.join(*pyx.split('.'))
    pyx = base + ('.pyx' if use_cython else '.cpp')

    # If CUDA SDK is not available, remove CUDA C files from extension sources
    # and use stubs defined in header files.
    if no_cuda:
        others1 = []
        for source in others:
            base, ext = os.path.splitext(source)
            if ext == '.cu':
                continue
            others1.append(source)
        others = others1

    return [pyx] + others

def get_required_modules(MODULES):
    return [m['name'] for m in MODULES if m.get('required', False)]

def check_library(compiler, includes=(), libraries=(),
                  include_dirs=(), library_dirs=(), define_macros=None,
                  extra_compile_args=()):

    source = ''.join(['#include <%s>\n' % header for header in includes])
    source += 'int main() {return 0;}'
    try:
        # We need to try to build a shared library because distutils
        # uses different option to build an executable and a shared library.
        # Especially when a user build an executable, distutils does not use
        # LDFLAGS environment variable.
        build.build_shlib(compiler, source, libraries,
                          include_dirs, library_dirs, define_macros,
                          extra_compile_args)
    except Exception as e:
        print(e)
        sys.stdout.flush()
        return False
    return True

def canonicalize_hip_libraries(hip_version, libraries):
    def ensure_tuple(x):
        return x if isinstance(x, tuple) else (x, None)
    new_libraries = []
    for library in libraries:
        lib_name, pred = ensure_tuple(library)
        if pred is None:
            new_libraries.append(lib_name)
        elif pred(hip_version):
            new_libraries.append(lib_name)
    libraries.clear()
    libraries.extend(new_libraries)

def preconfigure_modules(MODULES, compiler, settings):
    """Returns a list of modules buildable in given environment and settings.

    For each module in MODULES list, this function checks if the module
    can be built in the current environment and reports it.
    Returns a list of module names available.
    """

    nvcc_path = build.get_nvcc_path()
    hipcc_path = build.get_hipcc_path()
    summary = [
        '',
        '************************************************************',
        '* CuPy Configuration Summary                               *',
        '************************************************************',
        '',
        'Build Environment:',
        '  Include directories: {}'.format(str(settings['include_dirs'])),
        '  Library directories: {}'.format(str(settings['library_dirs'])),
        '  nvcc command       : {}'.format(
            nvcc_path if nvcc_path else '(not found)'),
        '  hipcc command      : {}'.format(
            hipcc_path if hipcc_path else '(not found)'),
        '',
        'Environment Variables:',
    ]

    for key in ['CFLAGS', 'LDFLAGS', 'LIBRARY_PATH',
                'CUDA_PATH', 'NVTOOLSEXT_PATH', 'NVCC', 'HIPCC',
                'ROCM_HOME']:
        summary += ['  {:<16}: {}'.format(key, os.environ.get(key, '(none)'))]

    summary += [
        '',
        'Modules:',
    ]

    ret = []
    for module in MODULES:
        installed = False
        status = 'No'
        errmsg = []

        if module['name'] == 'cutensor':
            cutensor_path = os.environ.get('CUTENSOR_PATH', '')
            inc_path = os.path.join(cutensor_path, 'include')
            if os.path.exists(inc_path):
                settings['include_dirs'].append(inc_path)
            cuda_version = build.get_cuda_version()
            cuda_major = str(cuda_version // 1000)
            cuda_major_minor = cuda_major + '.' + \
                str((cuda_version // 10) % 100)
            for cuda_ver in (cuda_major_minor, cuda_major):
                lib_path = os.path.join(cutensor_path, 'lib', cuda_ver)
                if os.path.exists(lib_path):
                    settings['library_dirs'].append(lib_path)
                    break

        # In ROCm 4.1 and later, we need to use the independent version of
        # hipfft as well as rocfft. We configure the lists of include
        # directories and libraries to link here depending on ROCm version
        # before the configuration process following.
        if use_hip and module['name'] == 'cuda':
            if module['check_method'](compiler, settings):
                hip_version = module['version_method']()
                if hip_version >= 401:
                    rocm_path = build.get_rocm_path()
                    inc_path = os.path.join(rocm_path, 'hipfft', 'include')
                    settings['include_dirs'].insert(0, inc_path)
                    lib_path = os.path.join(rocm_path, 'hipfft', 'lib')
                    settings['library_dirs'].insert(0, lib_path)
                # n.b., this modifieds MODULES['cuda']['libraries'] inplace
                canonicalize_hip_libraries(hip_version, module['libraries'])

        print('')
        print('-------- Configuring Module: {} --------'.format(
            module['name']))
        sys.stdout.flush()
        if not check_library(
                compiler,
                includes=module['include'],
                include_dirs=settings['include_dirs'],
                define_macros=settings['define_macros'],
                extra_compile_args=settings['extra_compile_args']):
            errmsg = ['Include files not found: %s' % module['include'],
                      'Check your CFLAGS environment variable.']
        elif not check_library(
                compiler,
                libraries=module['libraries'],
                library_dirs=settings['library_dirs'],
                define_macros=settings['define_macros'],
                extra_compile_args=settings['extra_compile_args']):
            errmsg = ['Cannot link libraries: %s' % module['libraries'],
                      'Check your LDFLAGS environment variable.']
        elif ('check_method' in module and
                not module['check_method'](compiler, settings)):
            # Fail on per-library condition check (version requirements etc.)
            installed = True
            errmsg = ['The library is installed but not supported.']
        elif (module['name'] in ('thrust', 'cub', 'random')
                and (nvcc_path is None and hipcc_path is None)):
            installed = True
            cmd = 'nvcc' if not use_hip else 'hipcc'
            errmsg = ['{} command could not be found in PATH.'.format(cmd),
                      'Check your PATH environment variable.']
        else:
            installed = True
            status = 'Yes'
            ret.append(module['name'])

        if installed and 'version_method' in module:
            status += ' (version {})'.format(module['version_method'](True))

        summary += [
            '  {:<10}: {}'.format(module['name'], status)
        ]

        # If error message exists...
        if len(errmsg) != 0:
            summary += ['    -> {}'.format(m) for m in errmsg]

            # Skip checking other modules when CUDA is unavailable.
            if module['name'] == 'cuda':
                break

    # Get a list of the CC of the devices connected to this node
    if not use_hip:
        build.check_compute_capabilities(compiler, settings)

    if len(ret) != len(MODULES):
        if 'cuda' in ret:
            lines = [
                'WARNING: Some modules could not be configured.',
                'CuPy will be installed without these modules.',
            ]
        else:
            lines = [
                'ERROR: CUDA could not be found on your system.',
                '',
                'HINT: You are trying to build CuPy from source, '
                'which is NOT recommended for general use.',
                '      Please consider using binary packages instead.',
                '',

            ]
        summary += [
            '',
        ] + lines + [
            'Please refer to the Installation Guide for details:',
            'https://docs.cupy.dev/en/stable/install.html',
            '',
        ]

    summary += [
        '************************************************************',
        '',
    ]

    print('\n'.join(summary))
    return ret, settings

def _rpath_base():
    if PLATFORM_LINUX:
        return '$ORIGIN'
    else:
        raise Exception('not supported on this platform')

def make_extensions(ctx: Context, compiler, use_cython):
    """Produce a list of Extension instances which passed to cythonize()."""

    MODULES = cupy_builder.get_modules(cupy_builder.get_context())

    no_cuda = ctx.use_stub
    use_hip = not no_cuda and ctx.use_hip
    settings = build.get_compiler_setting(use_hip)

    include_dirs = settings['include_dirs']

    settings['include_dirs'] = [
        x for x in include_dirs if path.exists(x)]
    settings['library_dirs'] = [
        x for x in settings['library_dirs'] if path.exists(x)]

    # Adjust rpath to use CUDA libraries in `cupy/.data/lib/*.so`) from CuPy.
    use_wheel_libs_rpath = (
        0 < len(ctx.wheel_libs) and not PLATFORM_WIN32)

    # In the environment with CUDA 7.5 on Ubuntu 16.04, gcc5.3 does not
    # automatically deal with memcpy because string.h header file has
    # been changed. This is a workaround for that environment.
    # See details in the below discussions:
    # https://github.com/BVLC/caffe/issues/4046
    # https://groups.google.com/forum/#!topic/theano-users/3ihQYiTRG4E
    settings['define_macros'].append(('_FORCE_INLINES', '1'))

    if ctx.linetrace:
        settings['define_macros'].append(('CYTHON_TRACE', '1'))
        settings['define_macros'].append(('CYTHON_TRACE_NOGIL', '1'))
    if no_cuda:
        settings['define_macros'].append(('CUPY_NO_CUDA', '1'))
    if use_hip:
        settings['define_macros'].append(('CUPY_USE_HIP', '1'))
        # introduced since ROCm 4.2.0
        settings['define_macros'].append(('__HIP_PLATFORM_AMD__', '1'))
        # deprecated since ROCm 4.2.0
        settings['define_macros'].append(('__HIP_PLATFORM_HCC__', '1'))

    available_modules = []
    if no_cuda:
        available_modules = [m['name'] for m in MODULES]
    else:
        available_modules, settings = preconfigure_modules(
            MODULES, compiler, settings)
        required_modules = get_required_modules(MODULES)
        if not (set(required_modules) <= set(available_modules)):
            raise Exception('Your CUDA environment is invalid. '
                            'Please check above error log.')

    ret = []
    for module in MODULES:
        if module['name'] not in available_modules:
            continue

        s = copy.deepcopy(settings)
        if not no_cuda:
            s['libraries'] = module['libraries']

        compile_args = s.setdefault('extra_compile_args', [])
        link_args = s.setdefault('extra_link_args', [])

        if module['name'] == 'cusolver':
            # cupy_backends/cupy_lapack.h has C++ template code
            compile_args.append('--std=c++11')
            # openmp is required for cusolver
            if use_hip:
                pass
            elif compiler.compiler_type == 'unix':
                compile_args.append('-fopenmp')
                link_args.append('-fopenmp')
            elif compiler.compiler_type == 'msvc':
                compile_args.append('/openmp')

        if module['name'] == 'random':
            if compiler.compiler_type == 'msvc':
                compile_args.append('-D_USE_MATH_DEFINES')

        if module['name'] == 'jitify':
            # this fixes RTD (no_cuda) builds...
            compile_args.append('--std=c++11')
            # if any change is made to the Jitify header, we force recompiling
            s['depends'] = ['./cupy/_core/include/cupy/jitify/jitify.hpp']

        if module['name'] == 'dlpack':
            # if any change is made to the DLPack header, we force recompiling
            s['depends'] = ['./cupy/_core/include/cupy/dlpack/dlpack.h']

        for f in module['file']:
            s_file = copy.deepcopy(s)
            name = module_extension_name(f)

            if name.endswith('fft._callback') and not PLATFORM_LINUX:
                continue

            rpath = []
            if not ctx.no_rpath:
                # Add library directories (e.g., `/usr/local/cuda/lib64`) to
                # RPATH.
                rpath += s_file['library_dirs']

            if use_wheel_libs_rpath:
                # Add `cupy/.data/lib` (where shared libraries included in
                # wheels reside) to RPATH.
                # The path is resolved relative to the module, e.g., use
                # `$ORIGIN/../cupy/.data/lib` for `cupy/cudnn.so` and
                # `$ORIGIN/../../../cupy/.data/lib` for
                # `cupy_backends/cuda/libs/cudnn.so`.
                depth = name.count('.')
                rpath.append(
                    '{}{}/cupy/.data/lib'.format(_rpath_base(), '/..' * depth))

            if not PLATFORM_WIN32 and not PLATFORM_LINUX:
                assert False, "macOS is no longer supported"
            if (PLATFORM_LINUX and len(rpath) != 0):
                ldflag = '-Wl,'
                if PLATFORM_LINUX:
                    ldflag += '--disable-new-dtags,'
                ldflag += ','.join('-rpath,' + p for p in rpath)
                args = s_file.setdefault('extra_link_args', [])
                args.append(ldflag)

            sources = module_extension_sources(f, use_cython, no_cuda)
            extension = setuptools.Extension(name, sources, **s_file)
            ret.append(extension)

    return ret

def prepare_wheel_libs(ctx: Context):
    """Prepare shared libraries and include files for wheels.

    Shared libraries are placed under `cupy/.data/lib` and
    RUNPATH will be set to this directory later (Linux only).
    Include files are placed under `cupy/.data/include`.

    Returns the list of files (path relative to `cupy` module) to add to
    the sdist/wheel distribution.
    """
    data_dir = os.path.abspath(os.path.join('cupy', '.data'))
    if os.path.exists(data_dir):
        print('Removing directory: {}'.format(data_dir))
        shutil.rmtree(data_dir)

    # Generate list files to copy
    # tuple of (src_path, dst_path)
    files_to_copy = []

    # Library files
    for srcpath in ctx.wheel_libs:
        relpath = os.path.basename(srcpath)
        dstpath = os.path.join(data_dir, 'lib', relpath)
        files_to_copy.append((srcpath, dstpath))

    # Include files
    for include_path_spec in ctx.wheel_includes:
        srcpath, relpath = include_path_spec.rsplit(':', 1)
        dstpath = os.path.join(data_dir, 'include', relpath)
        files_to_copy.append((srcpath, dstpath))

    # Wheel meta data
    if ctx.wheel_metadata_path:
        files_to_copy.append(
            (ctx.wheel_metadata_path, os.path.join(data_dir, '_wheel.json')))

    # Copy
    for srcpath, dstpath in files_to_copy:
        # Note: symlink is resolved by shutil.copy2.
        print('Copying file for wheel: {}'.format(srcpath))
        dirpath = os.path.dirname(dstpath)
        if not os.path.isdir(dirpath):
            os.makedirs(dirpath)
        shutil.copy2(srcpath, dstpath)

    return [os.path.relpath(x[1], 'cupy') for x in files_to_copy]

def cythonize(extensions, ctx: Context):
    # Delay importing Cython as it may be installed via setup_requires if
    # the user does not have Cython installed.
    import Cython
    import Cython.Build
    cython_version = pkg_resources.parse_version(Cython.__version__)

    directives = {
        'linetrace': ctx.linetrace,
        'profile': ctx.profile,
        # Embed signatures for Sphinx documentation.
        'embedsignature': True,
    }

    cythonize_options = {
        'annotate': ctx.annotate
    }

    # Compile-time constants to be used in Cython code
    compile_time_env = cythonize_options.get('compile_time_env')
    if compile_time_env is None:
        compile_time_env = {}
        cythonize_options['compile_time_env'] = compile_time_env

    # Enable CUDA Python.
    # TODO: add `cuda` to `setup_requires` only when this flag is set
    use_cuda_python = cupy_builder.get_context().use_cuda_python
    compile_time_env['CUPY_USE_CUDA_PYTHON'] = use_cuda_python
    if use_cuda_python:
        print('Using CUDA Python')

    compile_time_env['CUPY_CUFFT_STATIC'] = False
    compile_time_env['CUPY_CYTHON_VERSION'] = str(cython_version)
    if ctx.use_stub:  # on RTD
        compile_time_env['CUPY_CUDA_VERSION'] = 0
        compile_time_env['CUPY_HIP_VERSION'] = 0
    elif use_hip:  # on ROCm/HIP
        compile_time_env['CUPY_CUDA_VERSION'] = 0
        compile_time_env['CUPY_HIP_VERSION'] = build.get_hip_version()
    else:  # on CUDA
        compile_time_env['CUPY_CUDA_VERSION'] = build.get_cuda_version()
        compile_time_env['CUPY_HIP_VERSION'] = 0

    return Cython.Build.cythonize(
        extensions, verbose=True, language_level=3,
        compiler_directives=directives, **cythonize_options)

def check_extensions(extensions):
    for x in extensions:
        for f in x.sources:
            if not path.isfile(f):
                raise RuntimeError('''\
Missing file: {}
Please install Cython.
See https://docs.cupy.dev/en/stable/install.html for details.
'''.format(f))

def get_ext_modules(use_cython: bool, ctx: Context):
    # We need to call get_config_vars to initialize _config_vars in distutils
    # see #1849
    sysconfig.get_config_vars()
    compiler = ccompiler.new_compiler()
    sysconfig.customize_compiler(compiler)

    extensions = make_extensions(ctx, compiler, use_cython)

    return extensions

def _nvcc_gencode_options(cuda_version):
    """Returns NVCC GPU code generation options."""

    if sys.argv == ['setup.py', 'develop']:
        return []

    envcfg = os.getenv('CUPY_NVCC_GENERATE_CODE', None)
    if envcfg is not None and envcfg != 'current':
        return ['--generate-code={}'.format(arch)
                for arch in envcfg.split(';') if len(arch) > 0]
    if envcfg == 'current' and build.get_compute_capabilities() is not None:
        ccs = build.get_compute_capabilities()
        arch_list = [
            f'compute_{cc}' if cc < 60 else (f'compute_{cc}', f'sm_{cc}')
            for cc in ccs]
    else:
        # The arch_list specifies virtual architectures, such as 'compute_61',
        # and real architectures, such as 'sm_61', for which the CUDA
        # input files are to be compiled.
        #
        # The syntax of an entry of the list is
        #
        #     entry ::= virtual_arch | (virtual_arch, real_arch)
        #
        # where virtual_arch is a string which means a virtual architecture and
        # real_arch is a string which means a real architecture.
        #
        # If a virtual architecture is supplied, NVCC generates a PTX code
        # the virtual architecture. If a pair of a virtual architecture and a
        # real architecture is supplied, NVCC generates a PTX code for the
        # virtual architecture as well as a cubin code for the real one.
        #
        # For example, making NVCC generate a PTX code for 'compute_60' virtual
        # architecture, the arch_list has an entry of 'compute_60'.
        #
        #     arch_list = ['compute_60']
        #
        # For another, making NVCC generate a PTX code for 'compute_61' virtual
        # architecture and a cubin code for 'sm_61' real architecture, the
        # arch_list has an entry of ('compute_61', 'sm_61').
        #
        #     arch_list = [('compute_61', 'sm_61')]
        #
        # See the documentation of each CUDA version for the list of supported
        # architectures:
        #
        #   https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-steering-gpu-code-generation

        if cuda_version >= 11010:
            arch_list = ['compute_35',
                         'compute_50',
                         ('compute_60', 'sm_60'),
                         ('compute_61', 'sm_61'),
                         ('compute_70', 'sm_70'),
                         ('compute_75', 'sm_75'),
                         ('compute_80', 'sm_80'),
                         ('compute_86', 'sm_86'),
                         'compute_86']
        elif cuda_version >= 11000:
            arch_list = ['compute_35',
                         'compute_50',
                         ('compute_60', 'sm_60'),
                         ('compute_61', 'sm_61'),
                         ('compute_70', 'sm_70'),
                         ('compute_75', 'sm_75'),
                         ('compute_80', 'sm_80'),
                         'compute_80']
        elif cuda_version >= 10000:
            arch_list = ['compute_30',
                         'compute_50',
                         ('compute_60', 'sm_60'),
                         ('compute_61', 'sm_61'),
                         ('compute_70', 'sm_70'),
                         ('compute_75', 'sm_75'),
                         'compute_70']
        elif cuda_version >= 9020:
            arch_list = ['compute_30',
                         'compute_50',
                         ('compute_60', 'sm_60'),
                         ('compute_61', 'sm_61'),
                         ('compute_70', 'sm_70'),
                         'compute_70']
        else:
            # This should not happen.
            assert False

    options = []
    for arch in arch_list:
        if type(arch) is tuple:
            virtual_arch, real_arch = arch
            options.append('--generate-code=arch={},code={}'.format(
                virtual_arch, real_arch))
        else:
            options.append('--generate-code=arch={},code={}'.format(
                arch, arch))

    return options

class _UnixCCompiler(unixccompiler.UnixCCompiler):
    src_extensions = list(unixccompiler.UnixCCompiler.src_extensions)
    src_extensions.append('.cu')

    def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts):
        # For sources other than CUDA C ones, just call the super class method.
        if os.path.splitext(src)[1] != '.cu':
            return unixccompiler.UnixCCompiler._compile(
                self, obj, src, ext, cc_args, extra_postargs, pp_opts)

        if use_hip:
            return self._compile_unix_hipcc(
                obj, src, ext, cc_args, extra_postargs, pp_opts)
        else:
            return self._compile_unix_nvcc(
                obj, src, ext, cc_args, extra_postargs, pp_opts)

    def _compile_unix_nvcc(self,
                           obj, src, ext, cc_args, extra_postargs, pp_opts):
        # For CUDA C source files, compile them with NVCC.
        nvcc_path = build.get_nvcc_path()
        base_opts = build.get_compiler_base_options(nvcc_path)
        compiler_so = nvcc_path

        cuda_version = build.get_cuda_version()
        postargs = _nvcc_gencode_options(cuda_version) + [
            '-O2', '--compiler-options="-fPIC"']
        if cuda_version >= 11020:
            postargs += ['--std=c++14']
            num_threads = int(os.environ.get('CUPY_NUM_NVCC_THREADS', '2'))
            postargs += [f'-t{num_threads}']
        else:
            postargs += ['--std=c++11']
        postargs += ['-Xcompiler=-fno-gnu-unique']
        print('NVCC options:', postargs)
        try:
            self.spawn(compiler_so + base_opts + cc_args + [src, '-o', obj] +
                       postargs)
        except errors.DistutilsExecError as e:
            raise errors.CompileError(str(e))

    def _compile_unix_hipcc(self,
                            obj, src, ext, cc_args, extra_postargs, pp_opts):
        # For CUDA C source files, compile them with HIPCC.
        rocm_path = build.get_hipcc_path()
        base_opts = build.get_compiler_base_options(rocm_path)
        compiler_so = rocm_path

        hip_version = build.get_hip_version()
        postargs = ['-O2', '-fPIC', '--include', 'hip_runtime.h']
        if hip_version >= 402:
            postargs += ['--std=c++14']
        else:
            postargs += ['--std=c++11']
        print('HIPCC options:', postargs)
        try:
            self.spawn(compiler_so + base_opts + cc_args + [src, '-o', obj] +
                       postargs)
        except errors.DistutilsExecError as e:
            raise errors.CompileError(str(e))

    def link(self, target_desc, objects, output_filename, *args):
        use_hipcc = False
        if use_hip:
            for i in objects:
                if any(obj in i for obj in ('cupy_thrust.o', 'cupy_cub.o')):
                    use_hipcc = True
        if use_hipcc:
            _compiler_cxx = self.compiler_cxx
            try:
                rocm_path = build.get_hipcc_path()
                self.set_executable('compiler_cxx', rocm_path)

                return unixccompiler.UnixCCompiler.link(
                    self, target_desc, objects, output_filename, *args)
            finally:
                self.compiler_cxx = _compiler_cxx
        else:
            return unixccompiler.UnixCCompiler.link(
                self, target_desc, objects, output_filename, *args)

class _MSVCCompiler(msvccompiler.MSVCCompiler):
    _cu_extensions = ['.cu']

    src_extensions = list(unixccompiler.UnixCCompiler.src_extensions)
    src_extensions.extend(_cu_extensions)

    def _compile_cu(self, sources, output_dir=None, macros=None,
                    include_dirs=None, debug=0, extra_preargs=None,
                    extra_postargs=None, depends=None):
        # Compile CUDA C files, mainly derived from UnixCCompiler._compile().
        macros, objects, extra_postargs, pp_opts, _build = \
            self._setup_compile(output_dir, macros, include_dirs, sources,
                                depends, extra_postargs)

        compiler_so = build.get_nvcc_path()
        cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
        cuda_version = build.get_cuda_version()
        postargs = _nvcc_gencode_options(cuda_version) + ['-O2']
        if cuda_version >= 11020:
            # MSVC 14.0 (2015) is deprecated for CUDA 11.2 but we need it
            # to build CuPy because some Python versions were built using it.
            # REF: https://wiki.python.org/moin/WindowsCompilers
            postargs += ['-allow-unsupported-compiler']
        postargs += ['-Xcompiler', '/MD', '-D_USE_MATH_DEFINES']
        # This is to compile thrust with MSVC2015
        if cuda_version >= 11020:
            postargs += ['--std=c++14']
            num_threads = int(os.environ.get('CUPY_NUM_NVCC_THREADS', '2'))
            postargs += [f'-t{num_threads}']
        print('NVCC options:', postargs)

        for obj in objects:
            try:
                src, ext = _build[obj]
            except KeyError:
                continue
            try:
                self.spawn(compiler_so + cc_args + [src, '-o', obj] + postargs)
            except errors.DistutilsExecError as e:
                raise errors.CompileError(str(e))

        return objects

    def compile(self, sources, **kwargs):
        # Split CUDA C sources and others.
        if use_hip:
            raise RuntimeError('ROCm is not supported on Windows')

        cu_sources = []
        other_sources = []
        for source in sources:
            if os.path.splitext(source)[1] == '.cu':
                cu_sources.append(source)
            else:
                other_sources.append(source)

        # Compile source files other than CUDA C ones.
        other_objects = msvccompiler.MSVCCompiler.compile(
            self, other_sources, **kwargs)

        # Compile CUDA C sources.
        cu_objects = self._compile_cu(cu_sources, **kwargs)

        # Return compiled object filenames.
        return other_objects + cu_objects

class custom_build_ext(build_ext.build_ext):

    """Custom `build_ext` command to include CUDA C source files."""

    def run(self):
        if (build.get_nvcc_path() is not None
                or build.get_hipcc_path() is not None):
            def wrap_new_compiler(func):
                def _wrap_new_compiler(*args, **kwargs):
                    try:
                        return func(*args, **kwargs)
                    except errors.DistutilsPlatformError:
                        if not PLATFORM_WIN32:
                            CCompiler = _UnixCCompiler
                        else:
                            CCompiler = _MSVCCompiler
                        return CCompiler(
                            None, kwargs['dry_run'], kwargs['force'])
                return _wrap_new_compiler
            ccompiler.new_compiler = wrap_new_compiler(ccompiler.new_compiler)
            # Intentionally causes DistutilsPlatformError in
            # ccompiler.new_compiler() function to hook.
            self.compiler = 'nvidia'
        ctx = cupy_builder.get_context()
        ext_modules = get_ext_modules(True, ctx)  # get .pyx modules
        cythonize(ext_modules, ctx)
        check_extensions(self.extensions)
        build_ext.build_ext.run(self)

    def build_extensions(self):
        num_jobs = int(os.environ.get('CUPY_NUM_BUILD_JOBS', '4'))
        if num_jobs > 1:
            self.parallel = num_jobs
        super().build_extensions()

max_stars_repo_name: shwina/cupy

max_issues_repo_name: shwina/cupy

max_forks_repo_name: shwina/cupy

reason: import numpy,import cupy,from cupy


text:
__author__ = 'erik + jc + anna'
# TODO: Merge Quad and Shape
import numpy as np

class Shape(object):
    def __init__(self, id, vertices):
        self._id = id
        self._vertices = vertices

    def get_vertices(self):
        return self._vertices

class Quad(Shape):
    # _quadlist and _vertexlist have to be of type np.array!
    def __init__(self, id, vertex1, vertex2, vertex3, vertex4, edge1, edge2, edge3, edge4):

        super(Quad, self).__init__(id, [vertex1, vertex2, vertex3, vertex4])
        #self._id = id
       # self._vertices = [vertex1, vertex2, vertex3, vertex4]
        for vertex in self._vertices:
            vertex.add_quad(self)

        self._edges = [edge1, edge2, edge3, edge4]
        for edge in self._edges:
            edge.add_quad(self)

    def get_vertices(self):
        return self._vertices

    def get_edges(self):
        return self._edges

    def get_opposite_vertex(self, vertex):
        vertex_index = self._vertices.index(vertex)
        opposite_index = (vertex_index + 2) % 4
        return self._vertices[opposite_index]

    def get_edge_neighbouring_quads(self):
        quad_list = []
        for edge in self._edges:
            for quad in edge.get_quads:
                if quad != self:
                    quad_list.append(quad)

        return quad_list

    def get_vertex_neighbouring_quads(self):
        quad_list = []
        for vertex in self._vertices:
            for quad in vertex.get_quads:
                if quad != self:
                    quad_list.append(quad)

        return

class Shape_DooSabin(Shape):
    # _quadlist and _vertexlist have to be of type np.array!
    def __init__(self, id, vertices):

        super(Shape_DooSabin, self).__init__(id, vertices)
        self.vertex_ids = np.array([self._vertices[i].getId() for i in range(len(self._vertices))])
        self.centroid = self.compute_centroid()
        #types can be: vertex, edge, center
        self.type = ""
        self.parent_type = ""
        self.parents = []
        #only for the faces which were generated from some vertex children
        self.parent_vertex = None
        self.edges = self.getEdges()
        self.parent_edge = None
        self.edge_face_positioning = None
        self.ordered_refined_vertices = [None]*16

    def compute_centroid(self):
        return np.mean([self._vertices[i]._coordinates for i in range(len(self._vertices))], 0)

    def getEdges(self):

        edges = []
        n = len(self._vertices)

        for i in range(n):
            edges.append([self._vertices[i], self._vertices[(i + 1) % n]])

        return edges

    def isAdjacent(self, face):
        face_edges = face.getEdges()
        flag = 0
        shared_edge = None
        for edge in self.getEdges():
            if (edge in face_edges) or ([edge[1], edge[0]] in face_edges):
                #flag = 1
                shared_edge = edge

        return shared_edge

    def adjacentEdges(self, vert):
        adjacent_edges = []
        for edge in self.getEdges():
            if vert in edge:
                adjacent_edges.append(edge)
        return adjacent_edges

ext: py

max_stars_repo_name: mlaradji/CADO

max_issues_repo_name: mlaradji/CADO

max_forks_repo_name: mlaradji/CADO

reason: import numpy


text:
#
# Copyright 2021- IBM Inc. All rights reserved
# SPDX-License-Identifier: Apache2.0
#

from scipy.sparse import issparse
from hkmeans.c_package.c_hkmeans_optimizer import CHKMeansOptimizer as CHKMeansOptimizerExtension

class CHKMeansOptimizer:

    def __init__(self, n_clusters, n_features, n_samples, x, x_squared_norm):
        self.c_hkmeans_optimizer = CHKMeansOptimizerExtension(n_clusters, n_features)
        self.n_samples = n_samples
        self.x = x
        self.x_squared_norm = x_squared_norm
        self.sparse = issparse(x)
        if self.sparse:
            self.x_indices = self.x.indices
            self.x_indptr = self.x.indptr
            self.x_data = self.x.data
        else:
            self.x_indices = None
            self.x_indptr = None
            self.x_data = self.x.ravel()

    def init_partition(self, labels, t_size, t_centroid_sum, t_centroid_avg, t_squared_norm):
        return self.c_hkmeans_optimizer.init_partition(self.n_samples, self.x_indices, self.x_indptr,
                                                       self.x_data,  self.x_squared_norm, labels, t_size,
                                                       t_centroid_sum, t_centroid_avg, t_squared_norm)

    def optimize(self, x_permutation, t_size, t_centroid_sum, t_centroid_avg, t_squared_norm, labels, inertia):
        return self.c_hkmeans_optimizer.optimize(self.n_samples, self.x_indices, self.x_indptr, self.x_data,
                                                 self.x_squared_norm, x_permutation, t_size, t_centroid_sum,
                                                 t_centroid_avg, t_squared_norm, labels, inertia)

    def infer(self, n_samples, x, x_squared_norm, t_size, t_centroid_sum,
              t_centroid_avg, t_squared_norm, labels, costs):
        if self.sparse:
            x_indices = x.indices
            x_indptr = x.indptr
            x_data = x.data
        else:
            x_indices = None
            x_indptr = None
            x_data = x.ravel()
        return self.c_hkmeans_optimizer.infer(n_samples, x_indices, x_indptr, x_data, x_squared_norm,
                                              t_size, t_centroid_sum, t_centroid_avg, t_squared_norm,
                                              labels, costs)

max_stars_repo_name: IBM/hartigan-kmeans

max_issues_repo_name: IBM/hartigan-kmeans

max_forks_repo_name: IBM/hartigan-kmeans

reason: from scipy


text:
import numpy as np
list_k=[130,140,150,160,170,180,190,200]
for k in list_k:
    max_dist=[]
    if not k in [50,120,130,140,150,160,170,180,190,200]:
        data=np.load('./S1925_r_20.00_neighbor_%d_class_5.npz'%k)
    else:
        data=np.load('./S1925_r_50.00_neighbor_%d_class_5.npz'%k)
    x=data['x']
    for i in range(x.shape[0]):
        max_disti = max(x[i,:,0])
        max_dist.append(max_disti)
    print('此数据集中最大的距离为:',max(max_dist))
    print('最小距离为：',min(max_dist))
    print('平均距离为:',np.mean(max_dist))

max_stars_repo_name: ruiyangsong/mCNN

max_issues_repo_name: ruiyangsong/mCNN

max_forks_repo_name: ruiyangsong/mCNN


text:
import hdf5plugin # required to access LZ4-encoded HDF5 data sets
from diffractem import version, proc2d, pre_proc_opts, nexus, io
from diffractem.dataset import Dataset
from tifffile import imread
import numpy as np
import os
import dask.array as da
import h5py
from dask.distributed import Client, LocalCluster, Lock
import dask
import argparse
import subprocess
import pandas as pd
import random
from warnings import warn
from time import sleep

def _fast_correct(*args, data_key='/%/data/corrected',
                  shots_grp='/%/shots',
                  peaks_grp='/%/data', **kwargs):

    imgs, info = proc2d.analyze_and_correct(*args, **kwargs)
    store_dat = {shots_grp + '/' + k: v for k, v in info.items() if k != 'peak_data'}
    store_dat.update({peaks_grp + '/' + k: v for k, v in info['peak_data'].items()})
    store_dat[data_key] = imgs

    return store_dat

def quick_proc(ds, opts, label_raw, label, client, reference=None, pxmask=None):

    reference = imread(opts.reference) if reference is None else reference
    pxmask = imread(opts.pxmask) if pxmask is None else pxmask

    stack = ds.stacks[label_raw]
#     stk_del = ds.stacks['label_raw'].to_delayed().ravel()

    # get array names and shapes by correcting a single image (the last one)
    sample_res = _fast_correct(stack[-1:,...].compute(scheduler='threading'),
                               opts=opts,
                              data_key=ds.data_pattern + '/' + label,
                              shots_grp=ds.shots_pattern,
                              peaks_grp=ds.data_pattern)

#     print({k: v.dtype for k, v in sample_res.items()})

    # initialize file structure
    for (file, subset), grp in ds.shots.groupby(['file', 'subset']):
        with h5py.File(file) as fh:
            for pattern, data in sample_res.items():
                path = pattern.replace('%', subset)
#                 print('Initializing', file, path)
                fh.require_dataset(path,
                                    shape=(len(grp),) + data.shape[1:],
                                    dtype=data.dtype,
                                    chunks=(1,) + data.shape[1:],
                                    compression=opts.compression)
            fh[ds.data_pattern.replace('%', subset)].attrs['signal'] = label

    # array of integers corresponding to the chunk number
    chunk_label = np.concatenate([np.repeat(ii, cs)
                                  for ii, cs in enumerate(stack.chunks[0])])

    # delay objects returning the image and info dictionary
    cmp_del = [dask.delayed(_fast_correct)(raw_chk, opts)
               for raw_chk in ds.raw_counts.to_delayed().ravel()]

    # file lock objects
    locks = {fn: Lock() for fn in ds.files}

    # make delay objects for writing results to file (= maximum side effects!)
    dels = []
    for chks, (cl, sht) in zip(cmp_del, ds.shots.groupby(chunk_label)):
        assert len(sht.drop_duplicates(['file','subset'])) == 1
        ii_to = sht.shot_in_subset.values
        dels.append(dask.delayed(nexus._save_single_chunk_multi)(chks,
                                                                 file=sht.file.values[0],
                                                                 subset=sht.subset.values[0],
                                                                 idcs=ii_to,
                                                                 lock=locks[sht.file.values[0]]
                                                                ))

    # random.shuffle(dels) # shuffling tasks to minimize concurrent file access
    chunk_info = client.compute(dels, sync=True)
    return pd.DataFrame(chunk_info, columns=['file', 'subset', 'path', 'shot_in_subset'])

def main():

    parser = argparse.ArgumentParser(description='Quick and dirty pre-processing for Serial Electron Diffraction data',
                                     allow_abbrev=False, epilog='Any other options are passed on as modification to the option file')
    parser.add_argument('filename', type=str, nargs='*', help='List or HDF5 file or glob pattern')
    parser.add_argument('-s', '--settings', type=str, help='Option YAML file', default='preproc.yaml')
    parser.add_argument('-a', '--address', type=str, help='Address of dask.distributed cluster', default='127.0.0.1:8786')
    parser.add_argument('-c', '--chunksize', type=int, help='Chunk size of raw data stack. Should be integer multiple of movie stack frames!', default=100)
    parser.add_argument('-l', '--list-file', type=str, help='Name of output list file', default='processed.lst')
    parser.add_argument('-w', '--wait-for-files', help='Wait for files matching wildcard pattern', action='store_true')
    parser.add_argument('--include-existing', help='When using -w/--wait-for-file, also include existing files', action='store_true')
    parser.add_argument('--append', help='Append to list instead of overwrite', action='store_true')
    parser.add_argument('-d', '--data-path-old', type=str, help='Raw data field in HDF5 file(s)', default='/%/data/raw_counts')
    parser.add_argument('-n', '--data-path-new', type=str, help='Raw data field in HDF5 file(s)', default='/%/data/corrected')
    parser.add_argument('--no-bgcorr', help='Skip background correction', action='store_true')
    parser.add_argument('--no-validate', help='Do not validate files before attempting to process', action='store_true')
    # parser.add_argument('ppopt', nargs=argparse.REMAINDER, help='Preprocessing options to be overriden')

    args, extra = parser.parse_known_args()
    # print(args, extra)
    # raise RuntimeError('thus far!')
    opts = pre_proc_opts.PreProcOpts(args.settings)

    label_raw = args.data_path_old.rsplit('/', 1)[-1]
    label = args.data_path_new.rsplit('/', 1)[-1]

    if extra:
        # If extra arguments have been supplied, overwrite existing values
        opt_parser = argparse.ArgumentParser()
        for k, v in opts.__dict__.items():
            opt_parser.add_argument('--' + k, type=type(v), default=None)
        opts2 = opt_parser.parse_args(extra)

        for k, v in vars(opts2).items():
            if v is not None:
                if type(v) != type(opts.__dict__[k]):
                    warn('Mismatch of data types in overriden argument!', RuntimeWarning)
                print(f'Overriding option file setting {k} = {opts.__dict__[k]} ({type(opts.__dict__[k])}). ',
                    f'New value is {v} ({type(v)})')
                opts.__dict__[k] = v

    # raise RuntimeError('thus far!')
    print(f'Running on diffractem:', version())
    print(f'Current path is:', os.getcwd())

    print('Connecting to cluster scheduler at', args.address)
    client = Client(address=args.address)
    client.run(os.chdir, os.getcwd())

    if len(args.filename) == 1:
        args.filename = args.filename[0]

    # print(args.filename)
    seen_raw_files = [] if args.include_existing else io.expand_files(args.filename)

    while True:

        if args.wait_for_files:

            # slightly awkward sequence to only open finished files... (but believe me - it works!)

            fns = io.expand_files(args.filename)
            # print(fns)
            fns = [fn for fn in fns if fn not in seen_raw_files]
            # validation...
            try:
                fns = io.expand_files(fns, validate=not args.no_validate)
            except (OSError, IOError, RuntimeError) as err:
                print(f'Could not open file(s) {" ".join(fns)} because of', err)
                print('Possibly, it is still being written to. Waiting a bit...')
                sleep(5)
                continue

            if not fns:
                print('No new files, waiting a bit...')
                sleep(5)
                continue
            else:
                print(f'Found new files(s):\n', '\n'.join(fns))
                try:
                    ds_raw = Dataset.from_files(fns, chunking=args.chunksize)
                except Exception as err:
                    print(f'Could not open file(s) {" ".join(fns)} because of', err)
                    print('Possibly, it is still being written to. Waiting a bit...')
                    sleep(5)
                    continue

        else:
            fns = io.expand_files(args.filename, validate=not args.no_validate)
            ds_raw = Dataset.from_files(fns, chunking=args.chunksize)

        seen_raw_files.extend(ds_raw.files)

        print('---- Have dataset ----')
        print(ds_raw)

        # delete undesired stacks
        delstacks = [sn for sn in ds_raw.stacks.keys() if sn != args.data_path_old.rsplit('/', 1)[-1]]
        for sn in delstacks:
            ds_raw.delete_stack(sn)

        if opts.aggregate:
            print('---- Aggregating raw data ----')
            ds_compute = ds_raw.aggregate(query=opts.agg_query,
                                    by=['sample', 'region', 'run', 'crystal_id'],
                                    how='sum', new_folder=opts.proc_dir,
                                    file_suffix=opts.agg_file_suffix)
        else:
            ds_compute = ds_raw.get_selection(query=opts.select_query,
                                        file_suffix=opts.agg_file_suffix)

        print('Initializing data files...')
        os.makedirs(opts.proc_dir, exist_ok=True)
        ds_compute.init_files(overwrite=True)

        print('Storing meta tables...')
        ds_compute.store_tables(shots=True, features=True)

        print(f'Processing diffraction data... monitor progress at {client.dashboard_link} (or forward port if remote)')
        chunk_info = quick_proc(ds_compute, opts, label_raw, label, client)

        # make sure that the calculation went consistent with the data set
        for (sh, sh_grp), (ch, ch_grp) in zip(ds_compute.shots.groupby(['file', 'subset']), chunk_info.groupby(['file', 'subset'])):
            if any(sh_grp.shot_in_subset.values != np.sort(np.concatenate(ch_grp.shot_in_subset.values))):
                raise ValueError(f'Incosistency between calculated data and shot list in {sh[0]}: {sh[1]} found. Please investigate.')

        ds_compute.write_list(args.list_file, append = args.append)

        print(f'Computation done. Processed files are in {args.list_file}')

        if not args.wait_for_files:
            break

if __name__ == '__main__':
    main()


text:
try:
    from pysndfile import sndio
except:
    from scipy.io import wavfile
from scipy.signal import find_peaks
try:
    from fileio.sdif import mrk # ircam's library for our own use, not available from github
except:
    from file_utils.fileio import write_csv_file
import numpy as np
from matplotlib import pyplot as plt

def get_gci_triangle(targetPred, sr, thresh=0.5):
    '''
    Detect GCI positions using peak-picking on predicted triangle target curve
    :param targetPred: predicted triangle target curve
    :param sr: samling rate of predicted target curve (should be 16kHz)
    :return: positions of detected GCIs, in s
    '''
    fmax = 1000.
    minPeriodSamples = int(sr / fmax)

    prominence = 0.25 * thresh
    width = int(minPeriodSamples / 4)
    (peaks_pos, peaks_properties) = find_peaks(targetPred, height=thresh, distance=minPeriodSamples, prominence=prominence, width=width)
    GCI_times = peaks_pos / sr

    return GCI_times

def filter_peaks_for_no_sign_change_in_dgf_between_peaks(peaks, dgf, posDgfThresh = 0.005):
    '''

    :param peaks:
    :param dgf:
    :param posDgfThresh:
    :return:
    '''
    group = []
    peakGroups = []
    group.append(peaks[0])
    for i,p in enumerate(peaks[1:]):
        i += 1
        dgf_seg = dgf[peaks[i-1]:p]
        if(np.max(dgf_seg)<posDgfThresh):
            group.append(p)
        else:
            peakGroups.append(group)
            group = [p]

        if(i==len(peaks[1:])):
            peakGroups.append(group)

        filtered_peaks = []

    for g in peakGroups:
        filtered_peaks.append(int(np.round(np.mean(g))))

    return np.array(filtered_peaks)

def get_gci_GF(targetPred, sr, PLOT=False):
    '''
    Detect GCI positions using peak-picking on the negative derivative of the predicted glottal flow target curve
    :param targetPred: predicted glottal flow target curve
    :param sr: samling rate of predicted target curve (should be 16kHz)
    :param PLOT: plot glottal flow and derivative along with detected GCIs
    :return: positions of detected GCIs, in s
    '''

    maxF0 = 500.
    # margin_fact = 0.9
    margin_fact = 1.
    min_period_sec = margin_fact * (1 / maxF0)

    minGFHeightAtGCI = 0.2

    dgf = np.diff(targetPred)

    distance_samp = int(np.floor(min_period_sec * sr))
    minPeaksWidth = int(round((sr / maxF0) / 8.))
    peaks_res = find_peaks(-1 * dgf * (targetPred[:-1] > minGFHeightAtGCI), height=0.01, distance=distance_samp, width=minPeaksWidth)[0]

    peaks_res = filter_peaks_for_no_sign_change_in_dgf_between_peaks(peaks_res, dgf)

    if (PLOT):
        plt.figure()
        plt.plot(targetPred, 'g')
        plt.plot(dgf, 'r')
        plt.plot(dgf * (targetPred[:-1] > minGFHeightAtGCI), 'b')
        plt.plot(peaks_res, dgf[peaks_res], 'xk')
        plt.show()

    sndTimes = np.arange(len(dgf)) / sr
    GCI_times = sndTimes[peaks_res]

    return GCI_times

def get_gci_times(targetPred, sr, mode='GF', thresh=0.5):

    if(mode=='triangle'):
        GCI_times = get_gci_triangle(targetPred, sr, thresh=thresh)

    elif(mode=='GF'):
        GCI_times = get_gci_GF(targetPred, sr)

    return GCI_times

if __name__ == '__main__':

    from argparse import ArgumentParser
    parser = ArgumentParser(description="run analysis of a given model on a given database")

    global_args = parser.add_argument_group('global')
    global_args.add_argument('-i', "--input", default='/data/anasynth/ardaillon/pulse_pred/results/BDD_pulse_pred_16k/testModel_497-triangle_target-patience_64-use_zones-red_lr_patience_10-capacity_coeff_2/analysis/manual', help='input sound data to be analysed (either a directory or a file')
    global_args.add_argument('-o', "--output", default='~/deepF0/test', help='output (either a directory or a file with sdif extension)')
    global_args.add_argument('-t', "--thresh", default=0.5, type=float, help='peak heights threshold for determining GCIs positions from predicted target curve')
    global_args.add_argument('-m', "--mode", default='GF', help='mode for GCI detection from target curve, either triangle or GF (as the process is different depending on the type of target curve predicted by the model)')

    args = parser.parse_args()

    inputFile = args.input
    outputFile = args.output
    mode = args.mode
    thresh = args.thresh

    if(inputFile.endswith('.wav')):
        print("processing file "+inputFile)

        try:
            (targetPred, sr, enc) = sndio.read(inputFile)
        except:
            (sr, targetPred) = wavfile.read(inputFile)

        GCI_times = get_gci_times(targetPred, sr, mode=mode, thresh=thresh)

        try:
            mrk.store(outputFile, GCI_times, ['GCI'] * len(GCI_times))
        except:
            write_csv_file(outputFile, GCI_times, ['GCI'] * len(GCI_times))

reason: import numpy,from scipy


text:
from sympy import exp, Interval, oo, Symbol
from sympy.external import import_module
from sympy.stats import Beta, Chi, Normal, Gamma, Exponential, LogNormal, Pareto, ChiSquared, Uniform, sample, \
    BetaPrime, Cauchy, GammaInverse, GaussianInverse, StudentT, Weibull, density, ContinuousRV
from sympy.testing.pytest import skip, ignore_warnings, raises

def test_sample_numpy():
    distribs_numpy = [
        Beta("B", 1, 1),
        Normal("N", 0, 1),
        Gamma("G", 2, 7),
        Exponential("E", 2),
        LogNormal("LN", 0, 1),
        Pareto("P", 1, 1),
        ChiSquared("CS", 2),
        Uniform("U", 0, 1)
    ]
    size = 3
    numpy = import_module('numpy')
    if not numpy:
        skip('Numpy is not installed. Abort tests for _sample_numpy.')
    else:
        with ignore_warnings(UserWarning): ### TODO: Restore tests once warnings are removed
            for X in distribs_numpy:
                samps = next(sample(X, size=size, library='numpy'))
                for sam in samps:
                    assert sam in X.pspace.domain.set
            raises(NotImplementedError,
                   lambda: next(sample(Chi("C", 1), library='numpy')))
    raises(NotImplementedError,
           lambda: Chi("C", 1).pspace.distribution.sample(library='tensorflow'))

def test_sample_scipy():
    distribs_scipy = [
        Beta("B", 1, 1),
        BetaPrime("BP", 1, 1),
        Cauchy("C", 1, 1),
        Chi("C", 1),
        Normal("N", 0, 1),
        Gamma("G", 2, 7),
        GammaInverse("GI", 1, 1),
        GaussianInverse("GUI", 1, 1),
        Exponential("E", 2),
        LogNormal("LN", 0, 1),
        Pareto("P", 1, 1),
        StudentT("S", 2),
        ChiSquared("CS", 2),
        Uniform("U", 0, 1)
    ]
    size = 3
    numsamples = 5
    scipy = import_module('scipy')
    if not scipy:
        skip('Scipy is not installed. Abort tests for _sample_scipy.')
    else:
        with ignore_warnings(UserWarning): ### TODO: Restore tests once warnings are removed
            g_sample = list(sample(Gamma("G", 2, 7), size=size, numsamples=numsamples))
            assert len(g_sample) == numsamples
            for X in distribs_scipy:
                samps = next(sample(X, size=size, library='scipy'))
                samps2 = next(sample(X, size=(2, 2), library='scipy'))
                for sam in samps:
                    assert sam in X.pspace.domain.set
                for i in range(2):
                    for j in range(2):
                        assert samps2[i][j] in X.pspace.domain.set

def test_sample_pymc3():
    distribs_pymc3 = [
        Beta("B", 1, 1),
        Cauchy("C", 1, 1),
        Normal("N", 0, 1),
        Gamma("G", 2, 7),
        GaussianInverse("GI", 1, 1),
        Exponential("E", 2),
        LogNormal("LN", 0, 1),
        Pareto("P", 1, 1),
        ChiSquared("CS", 2),
        Uniform("U", 0, 1)
    ]
    size = 3
    pymc3 = import_module('pymc3')
    if not pymc3:
        skip('PyMC3 is not installed. Abort tests for _sample_pymc3.')
    else:
        with ignore_warnings(UserWarning): ### TODO: Restore tests once warnings are removed
            for X in distribs_pymc3:
                samps = next(sample(X, size=size, library='pymc3'))
                for sam in samps:
                    assert sam in X.pspace.domain.set
            raises(NotImplementedError,
                   lambda: next(sample(Chi("C", 1), library='pymc3')))

def test_sampling_gamma_inverse():
    scipy = import_module('scipy')
    if not scipy:
        skip('Scipy not installed. Abort tests for sampling of gamma inverse.')
    X = GammaInverse("x", 1, 1)
    with ignore_warnings(UserWarning): ### TODO: Restore tests once warnings are removed
        assert next(sample(X)) in X.pspace.domain.set

def test_lognormal_sampling():
    # Right now, only density function and sampling works
    scipy = import_module('scipy')
    if not scipy:
        skip('Scipy is not installed. Abort tests')
    with ignore_warnings(UserWarning): ### TODO: Restore tests once warnings are removed
        for i in range(3):
            X = LogNormal('x', i, 1)
            assert next(sample(X)) in X.pspace.domain.set

    size = 5
    with ignore_warnings(UserWarning): ### TODO: Restore tests once warnings are removed
        samps = next(sample(X, size=size))
        for samp in samps:
            assert samp in X.pspace.domain.set

def test_sampling_gaussian_inverse():
    scipy = import_module('scipy')
    if not scipy:
        skip('Scipy not installed. Abort tests for sampling of Gaussian inverse.')
    X = GaussianInverse("x", 1, 1)
    with ignore_warnings(UserWarning): ### TODO: Restore tests once warnings are removed
        assert next(sample(X, library='scipy')) in X.pspace.domain.set

def test_prefab_sampling():
    scipy = import_module('scipy')
    if not scipy:
        skip('Scipy is not installed. Abort tests')
    N = Normal('X', 0, 1)
    L = LogNormal('L', 0, 1)
    E = Exponential('Ex', 1)
    P = Pareto('P', 1, 3)
    W = Weibull('W', 1, 1)
    U = Uniform('U', 0, 1)
    B = Beta('B', 2, 5)
    G = Gamma('G', 1, 3)

    variables = [N, L, E, P, W, U, B, G]
    niter = 10
    size = 5
    with ignore_warnings(UserWarning): ### TODO: Restore tests once warnings are removed
        for var in variables:
            for _ in range(niter):
                assert next(sample(var)) in var.pspace.domain.set
                samps = next(sample(var, size=size))
                for samp in samps:
                    assert samp in var.pspace.domain.set

def test_sample_continuous():
    z = Symbol('z')
    Z = ContinuousRV(z, exp(-z), set=Interval(0, oo))
    assert density(Z)(-1) == 0

    scipy = import_module('scipy')
    if not scipy:
        skip('Scipy is not installed. Abort tests')
    with ignore_warnings(UserWarning): ### TODO: Restore tests once warnings are removed
        assert next(sample(Z)) in Z.pspace.domain.set
    sym, val = list(Z.pspace.sample().items())[0]
    assert sym == Z and val in Interval(0, oo)

    libraries = ['scipy', 'numpy', 'pymc3']
    for lib in libraries:
        try:
            imported_lib = import_module(lib)
            if imported_lib:
                s0, s1, s2 = [], [], []
                s0 = list(sample(Z, numsamples=10, library=lib, seed=0))
                s1 = list(sample(Z, numsamples=10, library=lib, seed=0))
                s2 = list(sample(Z, numsamples=10, library=lib, seed=1))
                assert s0 == s1
                assert s1 != s2
        except NotImplementedError:
            continue

max_stars_repo_name: bigfooted/sympy

max_issues_repo_name: bigfooted/sympy

max_forks_repo_name: bigfooted/sympy

reason: from sympy


text:
#!/usr/bin/python3
# coding: utf-8

# Import Python packages
import sys
import os
from cassandra.cluster import Cluster
import cassandra
import pandas as pd
import numpy as np
import re
import glob
import json
import csv

# ====================
# Part I. ETL Pipeline for Pre-Processing the Files
#

# Get current folder and subfolder event data
# Since the run.sh script is executed from the parent folder, consider
# the getcwd() as the parent dir.
filepath = os.getcwd() + '/data'

# Create a for loop to create a list of files and collect each filepath
for root, dirs, files in os.walk(filepath):

    # join the file path and roots with the subdirectories using glob
    file_path_list = glob.glob(os.path.join(root, '*'))
    # print(file_path_list)

# Process the files to create the single data file csv that will be used for Apache Casssandra tables
# initiating an empty list of rows that will be generated from each file
full_data_rows_list = []

# for every filepath in the file path list
for f in file_path_list:

    # reading csv file
    with open(f, 'r', encoding='utf8', newline='') as csvfile:
        # creating a csv reader object
        csvreader = csv.reader(csvfile)
        # remove the first row
        next(csvreader)

        # extracting each data row one by one and append it
        for line in csvreader:
            # print(line)
            full_data_rows_list.append(line)

# Creat a smaller event data csv file called event_datafile_full csv that will
# be used to insert data into the Apache Cassandra tables
csv.register_dialect('myDialect', quoting=csv.QUOTE_ALL, skipinitialspace=True)

with open('event_datafile_new.csv', 'w', encoding='utf8', newline='') as f:
    writer = csv.writer(f, dialect='myDialect')
    writer.writerow(['artist', 'firstName', 'gender', 'itemInSession', 'lastName',
                     'length', 'level', 'location', 'sessionId', 'song', 'userId'])
    for row in full_data_rows_list:
        if (row[0] == ''):
            continue
        writer.writerow((row[0], row[2], row[3], row[4], row[5],
                         row[6], row[7], row[8], row[12], row[13], row[16]))

# ====================
# Part II. Complete the Apache Cassandra coding portion of your project.
#

# Create connection to the database and a session
try:
    cluster = Cluster(['127.0.0.1'])
    session = cluster.connect()
except Exception as e:
    print(e)

# Create a keyspace named project_02
try:
    session.execute("""
    CREATE KEYSPACE IF NOT EXISTS project_02
    WITH REPLICATION =
    { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }"""
                    )
except Exception as e:
    print(e)

# Connect to keyspace
try:
    session.set_keyspace('project_02')
except Exception as e:
    print(e)

# ==========
# Query 1:  Give me the artist, song title and song's length in the
# music app history that was heard during sessionId = 338, and itemInSession = 4
#

print("\nQuery 1 results: \n")

# Ensure table is dropped prior to creation.
query = "DROP TABLE IF EXISTS music_library_0"
try:
    rows = session.execute(query)
except Exception as e:
    print(e)

# Create music library for query 1
query = "CREATE TABLE IF NOT EXISTS music_library_0 "
query = query + "(session_id int, item_in_session int, artist_name text, \
                song_title text, song_length float, PRIMARY KEY (session_id, \
                item_in_session))"

try:
    session.execute(query)
except Exception as e:
    print(e)

# We have provided part of the code to set up the CSV file. Please complete the
# Apache Cassandra code below
file = 'event_datafile_new.csv'

with open(file, encoding='utf8') as f:
    csvreader = csv.reader(f)
    next(csvreader)  # skip header
    for line in csvreader:
        # Create INSERT statement
        query = "INSERT INTO music_library_0 (session_id, item_in_session, artist_name, \
            song_title, song_length)"
        query = query + "VALUES (%s, %s, %s, %s, %s)"
        # Assign columns for INSERT query.
        session.execute(query, (int(line[8]), int(
            line[3]), line[0], line[9], float(line[5])))

# Verify the data was entered into the table by executing query
query = "select artist_name, song_title, song_length from music_library_0 \
    WHERE session_id=338 AND item_in_session=4"
try:
    rows = session.execute(query)
except Exception as e:
    print(e)

for row in rows:
    print(row.artist_name, row.song_title, row.song_length)

# ==========
# Query 2: Give me only the following: name of artist, song (sorted by itemInSession)
# and user (first and last name) for userid = 10, sessionid = 182
#

print("\nQuery 2 results: \n")

# Ensure table is dropped prior to creation.
query = "DROP TABLE IF EXISTS music_library_1"
try:
    rows = session.execute(query)
except Exception as e:
    print(e)

# Create music library for query 2. Add item_in_session to primary key to sort by it
query = "CREATE TABLE IF NOT EXISTS music_library_1 "
query = query + "(session_id int, user_id int, item_in_session int, artist_name text, song_title text, \
                user_first_name text, user_last_name text, PRIMARY KEY (session_id, user_id, \
                    item_in_session))"

try:
    session.execute(query)
except Exception as e:
    print(e)

# We have provided part of the code to set up the CSV file. Please complete the
# Apache Cassandra code below
file = 'event_datafile_new.csv'

with open(file, encoding='utf8') as f:
    csvreader = csv.reader(f)
    next(csvreader)  # skip header
    for line in csvreader:
        # Create INSERT statement
        query = "INSERT INTO music_library_1 (session_id, user_id, item_in_session, artist_name, song_title, \
                user_first_name, user_last_name)"
        query = query + "VALUES (%s, %s, %s, %s, %s, %s, %s)"
        # Assign columns for INSERT query.
        session.execute(query, (int(line[8]), int(line[10]), int(line[3]),
                                line[0], line[9], line[1], line[4]))

# Verify the data was entered into the table by executing query
query = "select artist_name, song_title, user_first_name, user_last_name from music_library_1 \
    WHERE session_id=182 AND user_id=10"
try:
    rows = session.execute(query)
except Exception as e:
    print(e)

for row in rows:
    print(row.artist_name, row.song_title,
          row.user_first_name, row.user_last_name)

# ==========
# Query 3: Give me every user name (first and last) in my music app history who
# listened to the song 'All Hands Against His Own'
#

print("\nQuery 3 Results: \n")

# Ensure table is dropped prior to creation.
query = "DROP TABLE IF EXISTS music_library_2"
try:
    rows = session.execute(query)
except Exception as e:
    print(e)

# Create music library for query 3
query = "CREATE TABLE IF NOT EXISTS music_library_2 "
query = query + "(song_title text, user_id int, user_first_name text, \
                    user_last_name text, PRIMARY KEY (song_title, user_id))"

try:
    session.execute(query)
except Exception as e:
    print(e)

# Data file for consumption
file = 'event_datafile_new.csv'

with open(file, encoding='utf8') as f:
    csvreader = csv.reader(f)
    next(csvreader)  # skip header
    for line in csvreader:
        # Create INSERT statement
        query = "INSERT INTO music_library_2 (song_title, user_id, \
                user_first_name, user_last_name)"
        query = query + "VALUES (%s, %s, %s, %s)"
        # Assign columns for INSERT query.
        session.execute(query, (line[9], int(line[10]), line[1], line[4]))

# Verify the data was entered into the table by executing query
query = "select user_first_name, user_last_name from music_library_2 \
    WHERE song_title='All Hands Against His Own'"
try:
    rows = session.execute(query)
except Exception as e:
    print(e)

for row in rows:
    print(row.user_first_name, row.user_last_name)

# Drop the tables
query = "DROP TABLE IF EXISTS music_library_0"
try:
    rows = session.execute(query)
except Exception as e:
    print(e)

query = "DROP TABLE IF EXISTS music_library_1"
try:
    rows = session.execute(query)
except Exception as e:
    print(e)

query = "DROP TABLE IF EXISTS music_library_2"
try:
    rows = session.execute(query)
except Exception as e:
    print(e)

# Close the session and cluster connections
session.shutdown()
cluster.shutdown()

max_stars_repo_name: ranstotz/data-eng-nanodegree

max_issues_repo_name: ranstotz/data-eng-nanodegree

max_forks_repo_name: ranstotz/data-eng-nanodegree


text:
#!/usr/bin/env python
import cluster
import sys
import csv
import math
import random
import scipy.optimize

# function to minimize
def g(l, returnObject=False):
        minError = sys.maxint
        for i in range(0,iters):
                k1 = cluster.dpmeans(res, l, xVal)
                err, xerr = k1.run()
                if xerr < minError:
                        minError = xerr
			kmin = k1
	if returnObject:
		return minError, k1
        return minError

#parameters
iters = 8        # iterations in search for min
maxClusters = 12 # used for setting minimum lambda
xValFrac = 0.2   # 20% of data for xVal

# Read data from standard in
res = []
for row in csv.reader(sys.stdin):
	res.append([float(x) for x in row])
nFeatures = len(res[0])
minx, maxx = [sys.maxint for i in range(0,nFeatures)], [-sys.maxint for i in range(0,nFeatures)]
for r in res:
	idx = 0
	for i in minx:
		if r[idx] < i:
			minx[idx] = r[idx]
		idx += 1
	idx = 0
	for i in maxx:
	        if r[idx] > i:
	                maxx[idx] = r[idx]
	        idx += 1
dataSpread = max([abs(x - y) for x, y in zip(maxx, minx)])
# quick and dirty min scale is average dist if data was along dataspread
dataGrain = dataSpread/maxClusters
# make sure data is in random order
random.shuffle(res)
# set aside for cross-validation
xVal = int(xValFrac*len(res))

optLambda = scipy.optimize.brent(g,
			brack=(1./dataSpread, 1./dataGrain),
			tol=1e-4,
			full_output=0,
			maxiter=100)

e,k = g(optLambda, returnObject=True)
wrtr = csv.writer(open("./output/opt_result.csv","wb"))
for x in k.getOutput():
	wrtr.writerow(x)
eWrtr = csv.writer(open("./output/opt_error.csv","wb"))
for x in k.getErrors():
	eWrtr.writerow(x)
print "lambda: %2.5f\n  with error: %2.5f\n"%(optLambda,e)

reason: import scipy


text:
from collections import Counter
from io import BufferedIOBase
from typing import Dict, Iterable, List, Optional

import numpy as np

from ps1_argonaut.BaseDataClasses import BaseDataClass
from ps1_argonaut.configuration import Configuration
from ps1_argonaut.wad_sections.SPSX.SoundContainers import LevelSFXContainer, LevelSFXGroupContainer
from ps1_argonaut.wad_sections.SPSX.Sounds import EffectSound

class LevelSFXMapping(BaseDataClass):
    def __init__(self, channel_group_mapping: Dict[int, int] = None, sounds_hashes: Iterable[int] = None,
                 mapping: np.ndarray = None):
        self.channel_group_mapping = channel_group_mapping if channel_group_mapping is not None else {}
        self.sounds_hashes = list(sounds_hashes) if sounds_hashes is not None else []
        if mapping is not None:
            self._mapping = mapping

    @property
    def n_unique_level_sfx(self):
        return self._mapping.shape[0] if hasattr(self, '_mapping') else len(self.sounds_hashes)

    @classmethod
    def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args, **kwargs):
        n_unique_level_sfx: int = kwargs['n_unique_level_sfx']

        mapping = np.array([[ord(data_in.read(1)) for _ in range(16)] for _ in range(n_unique_level_sfx)],
                           np.uint8)
        return cls(mapping=mapping)

    def serialize(self, data_out: BufferedIOBase, conf: Configuration, *args, **kwargs):
        level_sfx_groups: LevelSFXContainer = kwargs['level_sfx_groups']
        mapping = np.full((self.n_unique_level_sfx, 16), 255, np.uint8)
        for group_id, group in enumerate(level_sfx_groups):  # type: int, LevelSFXGroupContainer
            channel_id = self.channel_group_mapping[group_id]
            count = Counter()
            for sound_id, sound in enumerate(group):  # type: int, EffectSound
                sound_hash = hash(sound.vag.data)
                unique_sound_id = [i for i, v in enumerate(self.sounds_hashes) if v == sound_hash][count[sound_hash]]
                count[sound_hash] += 1
                mapping[unique_sound_id][channel_id] = sound_id
        data_out.write(bytes(mapping.flatten()))

    def parse_mapping(self, level_sfx_groups: LevelSFXContainer):
        channel_group_mapping = {}
        sounds_hashes: List[Optional[int]] = self.n_unique_level_sfx * [None]
        group_id = 0

        for channel_id in range(1, 16):
            empty_channel = True
            for unique_sound_id, sound_id in enumerate(self._mapping[:, channel_id]):  # type: int, int
                if sound_id != 255:
                    empty_channel = False
                    sounds_hashes[unique_sound_id] = hash(level_sfx_groups[group_id][sound_id].vag.data)
            if not empty_channel:
                channel_group_mapping[group_id] = channel_id
                group_id += 1
        del self._mapping
        self.channel_group_mapping = channel_group_mapping
        self.sounds_hashes = sounds_hashes


text:
from environmentv8 import Environment
from honestagentv8 import HonestAgent
import numpy as np
import progressbar as pb

alpha = 0.4
gamma = 0.5
T = 8
honest_agent = HonestAgent()

results = []
for mining_cost in np.arange(0,1.1, 0.1):
    env = Environment(alpha=alpha, gamma=gamma, T=T, mining_cost=mining_cost)

    simulated_block_count = int(1e6)
    env.reset()
    bar = pb.ProgressBar()
    cumulative_reward = 0
    for _ in bar(range(simulated_block_count)):
        current_state = env.current_state
        action = honest_agent.act(current_state)
        new_state, reward = env.takeAction(action)
        cumulative_reward += reward

    print(mining_cost, cumulative_reward, cumulative_reward / simulated_block_count)
    results.append(cumulative_reward / simulated_block_count)

print(results)


text:
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import torch
import glob

import os,sys,inspect

#Add these three lines of code to facilitate the import of the modules
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)

from pathlib import Path
import data_loader.data_loaders as module_data
from utils.util import prepare_device
import model.model_mult as model_mult
import model.mymodel as mymodel

import argparse
import collections
from parse_config import ConfigParser
from torchvision import  models
from figures import Plot
import numpy as np
from sklearn.metrics import confusion_matrix

@torch.no_grad() # turn off gradients during inference for memory effieciency
def get_all_preds(network, dataloader):
	"""function to return the number of correct predictions across data set"""
	all_preds = torch.tensor([])
	true_preds = torch.tensor([])
	model = network
	if dataloader.dataset.extended:
		for batch in dataloader:
			images, labels,data = batch
			#print(data.size())
			#print(data)
			preds = model(images,data) # get preds
			all_preds = torch.cat((all_preds, preds), dim=0) # join along existing axis
			true_preds = torch.cat((true_preds, labels), dim=0)
	else:
		for batch in dataloader:
			images, labels, = batch
			preds = model(images) # get preds
			all_preds = torch.cat((all_preds, preds), dim=0) # join along existing axis
			true_preds = torch.cat((true_preds, labels), dim=0)

	return all_preds, true_preds

def get_num_correct(preds, labels):
	return preds.argmax(dim=1).eq(labels).sum().item()

def afficher(config):
	# exemple  python plot.py saved/models/Garaham/\0414_170901/metrics.csv

	# exemple  python plot.py saved/models/Garaham/0415_130945/metrics.csv
	# exemple  python plot.py -c config_plot.json

	aff = Plot(config['affiche'])
	#displays the loss as a function of time
	aff.printLoss()
	#displays the Accuracy as a function of time
	aff.printAccuracy()

	device, device_ids = prepare_device(config['n_gpu'])

	PATH = config['model_path']
	print("Load model from: ",PATH)

	model = config.init_obj('model', model_mult)

	model = model.getModel()
	print(model)

	if len(device_ids) > 1:
		model = torch.nn.DataParallel(model, device_ids=device_ids)

	model.load_state_dict(torch.load(PATH))
	model.eval()

	data_loader = config.init_obj('data_loader', module_data)
	valid_data_loader = data_loader.split_validation()
	ds = data_loader.dataset

	train_Label = np.empty(len(data_loader.sampler))
	if not ds.extended:
		for i in range(len(data_loader.sampler)):
			imgSet,train_Label[i] = ds.getItem(i)
	else:
		for i in range(len(data_loader.sampler)):
			imgSet,train_Label[i],data = ds.getItem(i)

	pred_data_loader = torch.utils.data.DataLoader(batch_size=10000, dataset=train_Label, num_workers=1)
	valid_data_loader = data_loader.split_validation()
	all_preds, train_label1 = get_all_preds(network=model, dataloader=valid_data_loader) #data_loader)

	print(len(all_preds))
	print(len(train_label1))

	preds_correct = get_num_correct(all_preds, train_label1)
	print('total correct:', preds_correct,  'sur', len(train_label1) )
	print('accuracy:', preds_correct / len(train_label1))

	#displays confusion matrix
	aff.plot_confusion_matrix(cm=confusion_matrix(y_true=train_label1, y_pred=all_preds.argmax(1)), target_names = np.unique(train_label1), normalize=False)
	plt.savefig("testconf.png")

if __name__ == '__main__': #ne pas modifier cette fonction
	args = argparse.ArgumentParser(description='PyTorch Template')
	args.add_argument('-c', '--config', default=None, type=str,
					  help='config file path (default: None)')
	args.add_argument('-r', '--resume', default=None, type=str,
					  help='path to latest checkpoint (default: None)')
	args.add_argument('-d', '--device', default=None, type=str,
					  help='indices of GPUs to enable (default: all)')

	# custom cli options to modify configuration from default values given in json file.
	CustomArgs = collections.namedtuple('CustomArgs', 'flags type target')
	options = [
		CustomArgs(['--lr', '--learning_rate'], type=float, target='optimizer;args;lr'),
		CustomArgs(['--bs', '--batch_size'], type=int, target='data_loader;args;batch_size')
	]
	config = ConfigParser.from_args(args, options)
	#print(torch.cuda.is_available()) # affiche si cuda est dispo ou non
	afficher(config)

max_stars_repo_name: adblackx/ODR

max_issues_repo_name: adblackx/ODR

max_forks_repo_name: adblackx/ODR


text:
import pandas as pd
from sklearn.model_selection import train_test_split
from keras_text_summarization.library.utility.plot_utils import plot_and_save_history
from keras_text_summarization.library.seq2seq import Seq2SeqSummarizer
from keras_text_summarization.library.applications.fake_news_loader import fit_text
import numpy as np

data=pd.read_csv("/home/chuck/Downloads/fake_or_real_news.csv")

data.head()


text:
# Copyright (c) HP-NTU Digital Manufacturing Corporate Lab, Nanyang Technological University, Singapore.
#
# This source code is licensed under the Apache-2.0 license found in the
# LICENSE file in the root directory of this source tree.

import sys
import os
import numpy as np
import logging as log
from PIL import Image, ImageFile
from .preprocessing_factory import inception_preprocessing, vgg_preprocessing

ImageFile.LOAD_TRUNCATED_IMAGES = True

# class Images:
    # def __init__(self, input_dir, image_number, input_shape, preprocess):

    #     # self.image_number = image_number
    #     # self.input_shape = input_shape
    #     # self.image_ids = []
    #     # self.image_paths = self.get_image_path(input_dir)
    #     # self.image_orig_size = np.empty([len(self.image_paths),2],dtype=int)
    #     # self. images = self.read_images(preprocess)

    # def get_image_path(self, input_dir):
    #     image_paths = []
    #     image_names = os.listdir(input_dir)
    #     image_names.sort()
    #     if (len(image_names) < self.image_number):
    #         self.image_number = len(image_names) - (len(image_names) % self.input_shape[0])
    #         log.warning('Not enough images. Actual number of images: {}'.format(self.image_number))

    #     for i in range(self.image_number):
    #         image_path = os.path.join(input_dir, image_names[i])
    #         image_paths.append(image_path)
    #         self.image_ids.append(image_names[i][:12])

    #     return image_paths

    # def read_images(self, preprocess):
    #     n, c, h, w = self.input_shape
    #     images = []
    #     batch = []

    #     for i in range(self.image_number):
    #         image = Image.open(self.image_paths[i])
    #         self.image_orig_size[i] = image.size

    #         if image.mode == 'L':
    #             image = image.convert('RGB')

    #         if preprocess == 'vgg':
    #             resized_image = vgg_preprocessing(image, h, w)
    #         else:
    #             resized_image = inception_preprocessing(image, h, w)

    #         np_image = resized_image.transpose((2, 0, 1))
    #         batch.append(np_image)
    #         if ((i + 1) % n == 0):
    #             np_batch = np.array(batch).reshape((n, c, h, w))
    #             images.append(np_batch)
    #             batch.clear()

    #     return images

    # def read_idx_image(self, idx):
    #     return Image.open(self.image_paths[idx])

    # def normalize(self):
    #     for i in range(len(self.images)):
    #         self.images[i] =  self.images[i] / 255.0

def get_images(image_dir, num_images, shape, preprocess):
    images = []
    batch = []
    names = os.listdir(image_dir)
    names.sort()
    n, c, h, w = shape

    if len(names) < num_images:
        num_images = len(names)

    for i in range(num_images):
        image = Image.open(os.path.join(image_dir, names[i]))

        if image.mode != 'RGB':
            image = image.convert('RGB')

        if preprocess == 'vgg':
            image = vgg_preprocessing(image, h, w)
        else if preprocess == 'inception':
            image = inception_preprocessing(image, h, w)
        else:
            raise ValueError('No such pre-processing: ' + preprocess)

        image = image.transpose((2, 0, 1))
        batch.append(image)
        if (i + 1) % n == 0:
            batch = np.array(batch).reshape((n, c, h, w))
            images.append(batch)
            batch.clear()

        return images

def inception_preprocessing(image: Image, height: int, width: int, central_fraction=0.875):
    image = central_crop(image, central_fraction)

    if height and width:
        image = image.resize((width, height), Image.BILINEAR)

    image = np.array(image)
    image = (image / 255.0 - 0.5) * 2.0
    # -0.5
    # *2.0

    return image

def central_crop(image: Image, central_fraction: float):
    # image is PIL Image Format

    img_h = image.size[1]
    img_w = image.size[0]

    bbox_h_start = int((1.0 * img_h - img_h * central_fraction) / 2)
    bbox_w_start = int((1.0 * img_w - img_w * central_fraction) / 2)

    bbox_h_size = img_h - bbox_h_start * 2
    bbox_w_size = img_w - bbox_w_start * 2

    bbox = (bbox_w_start, bbox_h_start, bbox_w_start + bbox_w_size, bbox_h_start + bbox_h_size)
    return image.crop(bbox)


text:
from mphys.mphys_group import MphysGroup
import unittest
import numpy as np

import openmdao.api as om
from mpi4py import MPI

from mphys.mphys_group import MphysGroup
from common_methods import CommonMethods

class Comp(om.ExplicitComponent):
    def setup(self):
        self.add_input('dv_in', tags=['mphys_input'])
        self.add_output('dv_out', tags=['mphys_input'])

        self.add_input('result_in', tags=['mphys_result'])
        self.add_output('result_out', tags=['mphys_result'])

        self.add_input('coupling_in', tags=['mphys_coupling'])
        self.add_output('coupling_out', tags=['mphys_coupling'])

        self.add_input('coordinates_in', tags=['mphys_coordinates'])
        self.add_output('coordinates_out', tags=['mphys_coordinates'])

class TestMphysGroupAddingMphysSubsystem(unittest.TestCase):
    def setUp(self):
        self.common = CommonMethods()
        self.prob = om.Problem()
        group = self.prob.model.add_subsystem('mphys_group', MphysGroup())
        group.mphys_add_subsystem('comp1', Comp())

        self.prob.setup()

    def test_run_model(self):
        self.common.test_run_model(self, write_n2=False)

    def test_tagged_input_promotion(self):
        inputs = self.prob.model.mphys_group.get_io_metadata('input',
                                                             metadata_keys=['tags'])
        for key, val in inputs.items():
            if ('mphys_input' in val['tags'] or 'mphys_coupling' in val['tags'] or
                    'mphys_coordinates' in val['tags']):
                self.assertEqual(val['prom_name'], key.split('.')[-1])
            else:
                self.assertEqual(val['prom_name'], key)

    def test_tagged_output_promotion(self):
        inputs = self.prob.model.mphys_group.get_io_metadata('output',
                                                             metadata_keys=['tags'])
        for key, val in inputs.items():
            if ('mphys_result' in val['tags'] or 'mphys_coupling' in val['tags'] or
                    'mphys_coordinates' in val['tags']):
                self.assertEqual(val['prom_name'], key.split('.')[-1])
            else:
                self.assertEqual(val['prom_name'], key)

class TestMphysGroupNotAddingMphysSubsystem(unittest.TestCase):
    """
    Subsystem with inputs/outputs with mphys tags but not added with mphys_add_subsystem
    should not promote tagged variables
    """

    def setUp(self):
        self.common = CommonMethods()
        self.prob = om.Problem()
        group = self.prob.model.add_subsystem('mphys_group', MphysGroup())
        group.add_subsystem('comp1', Comp())

        self.prob.setup()

    def test_run_model(self):
        self.common.test_run_model(self, write_n2=False)

    def test_input_not_promoted(self):
        inputs = self.prob.model.mphys_group.get_io_metadata('input',
                                                             metadata_keys=['tags'])
        for key, val in inputs.items():
            self.assertEqual(val['prom_name'], key)

    def test_output_not_promoted(self):
        inputs = self.prob.model.mphys_group.get_io_metadata('output',
                                                             metadata_keys=['tags'])
        for key, val in inputs.items():
            self.assertEqual(val['prom_name'], key)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: timryanb/mphys

max_issues_repo_name: timryanb/mphys

max_forks_repo_name: timryanb/mphys


text:
import numpy as np
import scipy.stats as stats

from so3g.proj import Ranges, RangesMatrix

from .tod_ops import filters
from .tod_ops import fourier_filter

def get_turnaround_flags(tod, qlim=1, az=None, merge=True,
                         overwrite=False,name='turnarounds'):
    """Flag the scan turnaround times.

    Args:
        tod: AxisManager object
        qlim: percentile used to find turnaround
        az: The azimuth signal to look for turnarounds. If None it defaults to
            tod.boresight.az
        merge: If true, merge into tod.flags
        overwrite: If true and merge is True, write over existing flag
        name: name of flag when merged into tod.flags

    Returns:
        flag: Ranges object of turn-arounds

    """
    if az is None:
        az = tod.boresight.az
    lo, hi = np.percentile(az, [qlim,100-qlim])
    m = np.logical_or(az < lo, az > hi)

    flag = Ranges.from_bitmask(m)

    if merge:
        if name in tod.flags and not overwrite:
            raise ValueError('Flag name {} already exists in tod.flags'.format(name))
        elif name in tod.flags:
            tod.flags[name] = flag
        else:
            tod.flags.wrap(name, flag)
    return flag

def get_glitch_flags(tod, params={}, signal=None, merge=True,
                     overwrite=False, name='glitches'):
    """ Find glitches with fourier filtering
    Translation from moby2 as starting point

    Args:
        tod (AxisManager): the tod
        params (dictionary): Use to overwrite the default values
                n_sig: significance of detection
                t_glitch: Gaussian filter width
                hp_fc: high pass filter cutoff
                buffer: amount to buffer flags around found location
        signal (str): if None, defaults to tod.signal
        merge (bool): if true, add to tod.flags
        name (string): name of flag to add to tod.flags
        overwrite (bool): if true, write over flag. if false, don't

    Returns:
        flag: RangesMatrix object of glitches
    """
    gparams = {'n_sig':10,
               't_glitch':0.002,
               'hp_fc':5.0,
               'buffer':200,}
    gparams.update(params)
    params=gparams

    if signal is None:
        signal = 'signal'
    # f-space filtering
    filt = filters.high_pass_sine2(params['hp_fc']) * filters.gaussian_filter(params['t_glitch'])
    fvec = fourier_filter(tod, filt, detrend='linear',
                          signal_name=signal, resize='zero_pad')
    # get the threshods based on n_sig x nlev = n_sig x iqu x 0.741
    fvec = np.abs(fvec)
    thres = 0.741 * stats.iqr(fvec, axis=1) * params['n_sig']
    # get flags
    msk = fvec > thres[:,None]
    flag = RangesMatrix( [Ranges.from_bitmask(m) for m in msk])
    flag.buffer(params['buffer'])

    if merge:
        if name in tod.flags and not overwrite:
            raise ValueError('Flag name {} already exists in tod.flags'.format(name))
        elif name in tod.flags:
            tod.flags[name] = flag
        else:
            tod.flags.wrap(name, flag)

    return flag

def get_trending_flags(aman, max_trend=np.pi, n_pieces=1, signal=None,
                 merge=True, overwrite=True, name='trends'):
    """ Flag Detectors with trends larger than max_trend.

    Args:
        aman (AxisManager): the tod
        max_trend: maxmimum amount to always the detectors to change by.
            The default is pi for use in phase units.
        n_pieces: number of pieces to cut the timestream in to to look for
                    trends
        signal: Signal to use to generate flags, default is aman.signal.
        merge (bool): if true, merges the generated flag into aman
        overwrite (bool): if true, write over flag. if false, don't
        name (string): name of flag to add to aman.flags if merge is True

    Returns:
        flag: RangesMatrix object of glitches
    """
    if overwrite and name in aman.flags:
        aman.flags.move(name, None)

    if signal is None:
        signal = aman.signal

    signal = np.atleast_2d( signal )
    signal = signal[:,:aman.samps.count//n_pieces*n_pieces].reshape((signal.shape[0], n_pieces,-1))

    piece_size = signal.shape[2]
    slopes = signal[:,:,-1]-signal[:,:,0]

    bad_slopes = np.abs(slopes)>max_trend

    if n_pieces == 1:
        cut = bad_slopes[:,0]
    else:
        cut = aman.flags.get_zeros()
        dets = np.unique(np.where(bad_slopes)[0])
        for d in dets:
            clear = np.where(bad_slopes[d])[0]
            for c in clear:
                if c == n_pieces-1:
                    cut[d].add_interval(int(c*piece_size), cut.ranges[d].count-1)
                else:
                    cut[d].add_interval(int(c*piece_size), int((c+1)*piece_size))
    if merge:
        if name in aman.flags and not overwrite:
            raise ValueError('Flag name {} already exists in aman.flags'.format(name))
        elif name in aman.flags:
            aman.flags[name] = cut
        else:
            aman.flags.wrap(name, cut)

max_stars_repo_name: tskisner/sotodlib

max_issues_repo_name: tskisner/sotodlib

max_forks_repo_name: tskisner/sotodlib

reason: import numpy,import scipy


text:
import numpy as np
from ektelo.client.inference import LeastSquares
from ektelo.client.inference import NonNegativeLeastSquares
from ektelo.client.inference import WorkloadNonNegativeLeastSquares
from ektelo.client.inference import MultiplicativeWeights
from ektelo.client.inference import AHPThresholding
from ektelo.client.measurement import laplace_scale_factor
from ektelo.private.measurement import Laplace
from ektelo.matrix import EkteloMatrix
import unittest

class TestInference(unittest.TestCase):

    def setUp(self):
        self.n = 8
        self.eps_share = 0.1
        self.prng = np.random.RandomState(10)
        self.A = EkteloMatrix(np.eye(self.n))
        self.X = np.random.rand(self.n)

    def test_client_interaction_LS(self):
        laplace = Laplace(self.A, self.eps_share)
        ans = laplace.measure(self.X, self.prng)
        least_squares = LeastSquares()
        x_est = least_squares.infer(self.A, ans)

        self.assertEqual(self.X.shape, x_est.shape)

    def test_client_interaction_NLS(self):
        laplace = Laplace(self.A, self.eps_share)
        ans = laplace.measure(self.X, self.prng)

        non_neg_least_squares = NonNegativeLeastSquares()
        x_est = non_neg_least_squares.infer(self.A, ans)
        self.assertEqual(self.X.shape, x_est.shape)

    def test_client_interaction_WNLS(self):
        laplace = Laplace(self.A, self.eps_share)
        ans = laplace.measure(self.X, self.prng)

        engine = WorkloadNonNegativeLeastSquares(self.A)
        x_est = engine.infer(self.A, ans)
        self.assertEqual(self.X.shape, x_est.shape)

    def test_client_interaction_MW(self):
        laplace = Laplace(self.A, self.eps_share)
        ans = laplace.measure(self.X, self.prng)
        x_est_init = np.random.rand(self.n)

        mult_weight = MultiplicativeWeights()
        x_est = mult_weight.infer(self.A, ans, x_est_init)
        self.assertEqual(self.X.shape, x_est.shape)

    def test_client_interaction_HR(self):
        laplace = Laplace(self.A, self.eps_share)
        ans = laplace.measure(self.X, self.prng)
        eps_par = 0.1
        eta = 0.35
        ratio = 0.85

        AHP_threshold = AHPThresholding(eta, ratio)
        x_est = AHP_threshold.infer(self.A, ans, eps_par)
        self.assertEqual(self.X.shape, x_est.shape)

    def test_nnls(self):
        A = EkteloMatrix(np.random.rand(self.n, self.n))

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: dpcomp-org/ektelo

max_issues_repo_name: ektelo/ektelo

max_forks_repo_name: dpcomp-org/ektelo


text:
# -*- coding: utf-8 -*-
"""Example for wald_test for terms - `wald_anova`

Created on Mon Dec 15 11:19:23 2014

Author: Josef Perktold
License: BSD-3

"""

import numpy as np  # noqa:F401 --> needed for patsy

from statsmodels.formula.api import ols, glm, poisson
from statsmodels.discrete.discrete_model import Poisson

import statsmodels.stats.tests.test_anova as ttmod
from statsmodels.discrete.discrete_model import NegativeBinomial

test = ttmod.TestAnova3()
test.setup_class()

data = test.data.drop([0,1,2])
res_ols = ols("np.log(Days+1) ~ C(Duration, Sum)*C(Weight, Sum)", data).fit(use_t=False)

res_glm = glm("np.log(Days+1) ~ C(Duration, Sum)*C(Weight, Sum)",
                        data).fit()

res_poi = Poisson.from_formula("Days ~ C(Weight) * C(Duration)", data).fit(cov_type='HC0')
res_poi_2 = poisson("Days ~ C(Weight) + C(Duration)", data).fit(cov_type='HC0')

print('\nOLS')
print(res_ols.wald_test_terms())
print('\nGLM')
print(res_glm.wald_test_terms(skip_single=False, combine_terms=['Duration', 'Weight']))
print('\nPoisson 1')
print(res_poi.wald_test_terms(skip_single=False, combine_terms=['Duration', 'Weight']))
print('\nPoisson 2')
print(res_poi_2.wald_test_terms(skip_single=False))

res_nb2 = NegativeBinomial.from_formula("Days ~ C(Weight) * C(Duration)", data).fit()
print('\nNegative Binomial nb2')
print(res_nb2.wald_test_terms(skip_single=False))

res_nb1 = NegativeBinomial.from_formula("Days ~ C(Weight) * C(Duration)", data,
                                        loglike_method='nb1').fit(cov_type='HC0')
print('\nNegative Binomial nb2')
print(res_nb1.wald_test_terms(skip_single=False))

reason: import numpy,import statsmodels,from statsmodels